1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2016 Freescale Semiconductor, Inc. All rights reserved. 5 * Copyright 2017 NXP. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Freescale Semiconductor, Inc nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* System headers */ 35 #include <inttypes.h> 36 #include <unistd.h> 37 #include <stdio.h> 38 #include <limits.h> 39 #include <sched.h> 40 #include <pthread.h> 41 42 #include <rte_byteorder.h> 43 #include <rte_common.h> 44 #include <rte_interrupts.h> 45 #include <rte_log.h> 46 #include <rte_debug.h> 47 #include <rte_pci.h> 48 #include <rte_atomic.h> 49 #include <rte_branch_prediction.h> 50 #include <rte_memory.h> 51 #include <rte_tailq.h> 52 #include <rte_eal.h> 53 #include <rte_alarm.h> 54 #include <rte_ether.h> 55 #include <rte_ethdev.h> 56 #include <rte_malloc.h> 57 #include <rte_ring.h> 58 #include <rte_ip.h> 59 #include <rte_tcp.h> 60 #include <rte_udp.h> 61 #include <rte_net.h> 62 63 #include "dpaa_ethdev.h" 64 #include "dpaa_rxtx.h" 65 #include <rte_dpaa_bus.h> 66 #include <dpaa_mempool.h> 67 68 #include <fsl_usd.h> 69 #include <fsl_qman.h> 70 #include <fsl_bman.h> 71 #include <of.h> 72 #include <netcfg.h> 73 74 #define DPAA_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) \ 75 do { \ 76 (_fd)->cmd = 0; \ 77 (_fd)->opaque_addr = 0; \ 78 (_fd)->opaque = QM_FD_CONTIG << DPAA_FD_FORMAT_SHIFT; \ 79 (_fd)->opaque |= ((_mbuf)->data_off) << DPAA_FD_OFFSET_SHIFT; \ 80 (_fd)->opaque |= (_mbuf)->pkt_len; \ 81 (_fd)->addr = (_mbuf)->buf_iova; \ 82 (_fd)->bpid = _bpid; \ 83 } while (0) 84 85 #if (defined RTE_LIBRTE_DPAA_DEBUG_DRIVER) 86 void dpaa_display_frame(const struct qm_fd *fd) 87 { 88 int ii; 89 char *ptr; 90 91 printf("%s::bpid %x addr %08x%08x, format %d off %d, len %d stat %x\n", 92 __func__, fd->bpid, fd->addr_hi, fd->addr_lo, fd->format, 93 fd->offset, fd->length20, fd->status); 94 95 ptr = (char *)rte_dpaa_mem_ptov(fd->addr); 96 ptr += fd->offset; 97 printf("%02x ", *ptr); 98 for (ii = 1; ii < fd->length20; ii++) { 99 printf("%02x ", *ptr); 100 if ((ii % 16) == 0) 101 printf("\n"); 102 ptr++; 103 } 104 printf("\n"); 105 } 106 #else 107 #define dpaa_display_frame(a) 108 #endif 109 110 static inline void dpaa_slow_parsing(struct rte_mbuf *m __rte_unused, 111 uint64_t prs __rte_unused) 112 { 113 DPAA_DP_LOG(DEBUG, "Slow parsing"); 114 /*TBD:XXX: to be implemented*/ 115 } 116 117 static inline void dpaa_eth_packet_info(struct rte_mbuf *m, 118 uint64_t fd_virt_addr) 119 { 120 struct annotations_t *annot = GET_ANNOTATIONS(fd_virt_addr); 121 uint64_t prs = *((uint64_t *)(&annot->parse)) & DPAA_PARSE_MASK; 122 123 DPAA_DP_LOG(DEBUG, " Parsing mbuf: %p with annotations: %p", m, annot); 124 125 switch (prs) { 126 case DPAA_PKT_TYPE_NONE: 127 m->packet_type = 0; 128 break; 129 case DPAA_PKT_TYPE_ETHER: 130 m->packet_type = RTE_PTYPE_L2_ETHER; 131 break; 132 case DPAA_PKT_TYPE_IPV4: 133 m->packet_type = RTE_PTYPE_L2_ETHER | 134 RTE_PTYPE_L3_IPV4; 135 break; 136 case DPAA_PKT_TYPE_IPV6: 137 m->packet_type = RTE_PTYPE_L2_ETHER | 138 RTE_PTYPE_L3_IPV6; 139 break; 140 case DPAA_PKT_TYPE_IPV4_FRAG: 141 case DPAA_PKT_TYPE_IPV4_FRAG_UDP: 142 case DPAA_PKT_TYPE_IPV4_FRAG_TCP: 143 case DPAA_PKT_TYPE_IPV4_FRAG_SCTP: 144 m->packet_type = RTE_PTYPE_L2_ETHER | 145 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG; 146 break; 147 case DPAA_PKT_TYPE_IPV6_FRAG: 148 case DPAA_PKT_TYPE_IPV6_FRAG_UDP: 149 case DPAA_PKT_TYPE_IPV6_FRAG_TCP: 150 case DPAA_PKT_TYPE_IPV6_FRAG_SCTP: 151 m->packet_type = RTE_PTYPE_L2_ETHER | 152 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG; 153 break; 154 case DPAA_PKT_TYPE_IPV4_EXT: 155 m->packet_type = RTE_PTYPE_L2_ETHER | 156 RTE_PTYPE_L3_IPV4_EXT; 157 break; 158 case DPAA_PKT_TYPE_IPV6_EXT: 159 m->packet_type = RTE_PTYPE_L2_ETHER | 160 RTE_PTYPE_L3_IPV6_EXT; 161 break; 162 case DPAA_PKT_TYPE_IPV4_TCP: 163 m->packet_type = RTE_PTYPE_L2_ETHER | 164 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; 165 break; 166 case DPAA_PKT_TYPE_IPV6_TCP: 167 m->packet_type = RTE_PTYPE_L2_ETHER | 168 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; 169 break; 170 case DPAA_PKT_TYPE_IPV4_UDP: 171 m->packet_type = RTE_PTYPE_L2_ETHER | 172 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; 173 break; 174 case DPAA_PKT_TYPE_IPV6_UDP: 175 m->packet_type = RTE_PTYPE_L2_ETHER | 176 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; 177 break; 178 case DPAA_PKT_TYPE_IPV4_EXT_UDP: 179 m->packet_type = RTE_PTYPE_L2_ETHER | 180 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP; 181 break; 182 case DPAA_PKT_TYPE_IPV6_EXT_UDP: 183 m->packet_type = RTE_PTYPE_L2_ETHER | 184 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP; 185 break; 186 case DPAA_PKT_TYPE_IPV4_EXT_TCP: 187 m->packet_type = RTE_PTYPE_L2_ETHER | 188 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP; 189 break; 190 case DPAA_PKT_TYPE_IPV6_EXT_TCP: 191 m->packet_type = RTE_PTYPE_L2_ETHER | 192 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP; 193 break; 194 case DPAA_PKT_TYPE_IPV4_SCTP: 195 m->packet_type = RTE_PTYPE_L2_ETHER | 196 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; 197 break; 198 case DPAA_PKT_TYPE_IPV6_SCTP: 199 m->packet_type = RTE_PTYPE_L2_ETHER | 200 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; 201 break; 202 /* More switch cases can be added */ 203 default: 204 dpaa_slow_parsing(m, prs); 205 } 206 207 m->tx_offload = annot->parse.ip_off[0]; 208 m->tx_offload |= (annot->parse.l4_off - annot->parse.ip_off[0]) 209 << DPAA_PKT_L3_LEN_SHIFT; 210 211 /* Set the hash values */ 212 m->hash.rss = (uint32_t)(rte_be_to_cpu_64(annot->hash)); 213 m->ol_flags = PKT_RX_RSS_HASH; 214 /* All packets with Bad checksum are dropped by interface (and 215 * corresponding notification issued to RX error queues). 216 */ 217 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD; 218 219 /* Check if Vlan is present */ 220 if (prs & DPAA_PARSE_VLAN_MASK) 221 m->ol_flags |= PKT_RX_VLAN; 222 /* Packet received without stripping the vlan */ 223 } 224 225 static inline void dpaa_checksum(struct rte_mbuf *mbuf) 226 { 227 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); 228 char *l3_hdr = (char *)eth_hdr + mbuf->l2_len; 229 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)l3_hdr; 230 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)l3_hdr; 231 232 DPAA_DP_LOG(DEBUG, "Calculating checksum for mbuf: %p", mbuf); 233 234 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 235 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 236 RTE_PTYPE_L3_IPV4_EXT)) { 237 ipv4_hdr = (struct ipv4_hdr *)l3_hdr; 238 ipv4_hdr->hdr_checksum = 0; 239 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); 240 } else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 241 RTE_PTYPE_L3_IPV6) || 242 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 243 RTE_PTYPE_L3_IPV6_EXT)) 244 ipv6_hdr = (struct ipv6_hdr *)l3_hdr; 245 246 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { 247 struct tcp_hdr *tcp_hdr = (struct tcp_hdr *)(l3_hdr + 248 mbuf->l3_len); 249 tcp_hdr->cksum = 0; 250 if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) 251 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 252 tcp_hdr); 253 else /* assume ethertype == ETHER_TYPE_IPv6 */ 254 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 255 tcp_hdr); 256 } else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == 257 RTE_PTYPE_L4_UDP) { 258 struct udp_hdr *udp_hdr = (struct udp_hdr *)(l3_hdr + 259 mbuf->l3_len); 260 udp_hdr->dgram_cksum = 0; 261 if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) 262 udp_hdr->dgram_cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, 263 udp_hdr); 264 else /* assume ethertype == ETHER_TYPE_IPv6 */ 265 udp_hdr->dgram_cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, 266 udp_hdr); 267 } 268 } 269 270 static inline void dpaa_checksum_offload(struct rte_mbuf *mbuf, 271 struct qm_fd *fd, char *prs_buf) 272 { 273 struct dpaa_eth_parse_results_t *prs; 274 275 DPAA_DP_LOG(DEBUG, " Offloading checksum for mbuf: %p", mbuf); 276 277 prs = GET_TX_PRS(prs_buf); 278 prs->l3r = 0; 279 prs->l4r = 0; 280 if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || 281 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 282 RTE_PTYPE_L3_IPV4_EXT)) 283 prs->l3r = DPAA_L3_PARSE_RESULT_IPV4; 284 else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 285 RTE_PTYPE_L3_IPV6) || 286 ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == 287 RTE_PTYPE_L3_IPV6_EXT)) 288 prs->l3r = DPAA_L3_PARSE_RESULT_IPV6; 289 290 if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) 291 prs->l4r = DPAA_L4_PARSE_RESULT_TCP; 292 else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) 293 prs->l4r = DPAA_L4_PARSE_RESULT_UDP; 294 295 prs->ip_off[0] = mbuf->l2_len; 296 prs->l4_off = mbuf->l3_len + mbuf->l2_len; 297 /* Enable L3 (and L4, if TCP or UDP) HW checksum*/ 298 fd->cmd = DPAA_FD_CMD_RPD | DPAA_FD_CMD_DTC; 299 } 300 301 struct rte_mbuf * 302 dpaa_eth_sg_to_mbuf(struct qm_fd *fd, uint32_t ifid) 303 { 304 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 305 struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp; 306 struct qm_sg_entry *sgt, *sg_temp; 307 void *vaddr, *sg_vaddr; 308 int i = 0; 309 uint8_t fd_offset = fd->offset; 310 311 DPAA_DP_LOG(DEBUG, "Received an SG frame"); 312 313 vaddr = rte_dpaa_mem_ptov(qm_fd_addr(fd)); 314 if (!vaddr) { 315 DPAA_PMD_ERR("unable to convert physical address"); 316 return NULL; 317 } 318 sgt = vaddr + fd_offset; 319 sg_temp = &sgt[i++]; 320 hw_sg_to_cpu(sg_temp); 321 temp = (struct rte_mbuf *)((char *)vaddr - bp_info->meta_data_size); 322 sg_vaddr = rte_dpaa_mem_ptov(qm_sg_entry_get64(sg_temp)); 323 324 first_seg = (struct rte_mbuf *)((char *)sg_vaddr - 325 bp_info->meta_data_size); 326 first_seg->data_off = sg_temp->offset; 327 first_seg->data_len = sg_temp->length; 328 first_seg->pkt_len = sg_temp->length; 329 rte_mbuf_refcnt_set(first_seg, 1); 330 331 first_seg->port = ifid; 332 first_seg->nb_segs = 1; 333 first_seg->ol_flags = 0; 334 prev_seg = first_seg; 335 while (i < DPAA_SGT_MAX_ENTRIES) { 336 sg_temp = &sgt[i++]; 337 hw_sg_to_cpu(sg_temp); 338 sg_vaddr = rte_dpaa_mem_ptov(qm_sg_entry_get64(sg_temp)); 339 cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - 340 bp_info->meta_data_size); 341 cur_seg->data_off = sg_temp->offset; 342 cur_seg->data_len = sg_temp->length; 343 first_seg->pkt_len += sg_temp->length; 344 first_seg->nb_segs += 1; 345 rte_mbuf_refcnt_set(cur_seg, 1); 346 prev_seg->next = cur_seg; 347 if (sg_temp->final) { 348 cur_seg->next = NULL; 349 break; 350 } 351 prev_seg = cur_seg; 352 } 353 354 dpaa_eth_packet_info(first_seg, (uint64_t)vaddr); 355 rte_pktmbuf_free_seg(temp); 356 357 return first_seg; 358 } 359 360 static inline struct rte_mbuf *dpaa_eth_fd_to_mbuf(struct qm_fd *fd, 361 uint32_t ifid) 362 { 363 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); 364 struct rte_mbuf *mbuf; 365 void *ptr; 366 uint8_t format = 367 (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; 368 uint16_t offset = 369 (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT; 370 uint32_t length = fd->opaque & DPAA_FD_LENGTH_MASK; 371 372 DPAA_DP_LOG(DEBUG, " FD--->MBUF"); 373 374 if (unlikely(format == qm_fd_sg)) 375 return dpaa_eth_sg_to_mbuf(fd, ifid); 376 377 /* Ignoring case when format != qm_fd_contig */ 378 dpaa_display_frame(fd); 379 ptr = rte_dpaa_mem_ptov(fd->addr); 380 /* Ignoring case when ptr would be NULL. That is only possible incase 381 * of a corrupted packet 382 */ 383 384 mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); 385 /* Prefetch the Parse results and packet data to L1 */ 386 rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); 387 rte_prefetch0((void *)((uint8_t *)ptr + offset)); 388 389 mbuf->data_off = offset; 390 mbuf->data_len = length; 391 mbuf->pkt_len = length; 392 393 mbuf->port = ifid; 394 mbuf->nb_segs = 1; 395 mbuf->ol_flags = 0; 396 mbuf->next = NULL; 397 rte_mbuf_refcnt_set(mbuf, 1); 398 dpaa_eth_packet_info(mbuf, (uint64_t)mbuf->buf_addr); 399 400 return mbuf; 401 } 402 403 uint16_t dpaa_eth_queue_rx(void *q, 404 struct rte_mbuf **bufs, 405 uint16_t nb_bufs) 406 { 407 struct qman_fq *fq = q; 408 struct qm_dqrr_entry *dq; 409 uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; 410 int ret; 411 412 ret = rte_dpaa_portal_init((void *)0); 413 if (ret) { 414 DPAA_PMD_ERR("Failure in affining portal"); 415 return 0; 416 } 417 418 ret = qman_set_vdq(fq, (nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES) ? 419 DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_bufs); 420 if (ret) 421 return 0; 422 423 do { 424 dq = qman_dequeue(fq); 425 if (!dq) 426 continue; 427 bufs[num_rx++] = dpaa_eth_fd_to_mbuf(&dq->fd, ifid); 428 qman_dqrr_consume(fq, dq); 429 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 430 431 return num_rx; 432 } 433 434 static void *dpaa_get_pktbuf(struct dpaa_bp_info *bp_info) 435 { 436 int ret; 437 uint64_t buf = 0; 438 struct bm_buffer bufs; 439 440 ret = bman_acquire(bp_info->bp, &bufs, 1, 0); 441 if (ret <= 0) { 442 DPAA_PMD_WARN("Failed to allocate buffers %d", ret); 443 return (void *)buf; 444 } 445 446 DPAA_DP_LOG(DEBUG, "got buffer 0x%lx from pool %d", 447 (uint64_t)bufs.addr, bufs.bpid); 448 449 buf = (uint64_t)rte_dpaa_mem_ptov(bufs.addr) - bp_info->meta_data_size; 450 if (!buf) 451 goto out; 452 453 out: 454 return (void *)buf; 455 } 456 457 static struct rte_mbuf *dpaa_get_dmable_mbuf(struct rte_mbuf *mbuf, 458 struct dpaa_if *dpaa_intf) 459 { 460 struct rte_mbuf *dpaa_mbuf; 461 462 /* allocate pktbuffer on bpid for dpaa port */ 463 dpaa_mbuf = dpaa_get_pktbuf(dpaa_intf->bp_info); 464 if (!dpaa_mbuf) 465 return NULL; 466 467 memcpy((uint8_t *)(dpaa_mbuf->buf_addr) + mbuf->data_off, (void *) 468 ((uint8_t *)(mbuf->buf_addr) + mbuf->data_off), mbuf->pkt_len); 469 470 /* Copy only the required fields */ 471 dpaa_mbuf->data_off = mbuf->data_off; 472 dpaa_mbuf->pkt_len = mbuf->pkt_len; 473 dpaa_mbuf->ol_flags = mbuf->ol_flags; 474 dpaa_mbuf->packet_type = mbuf->packet_type; 475 dpaa_mbuf->tx_offload = mbuf->tx_offload; 476 rte_pktmbuf_free(mbuf); 477 return dpaa_mbuf; 478 } 479 480 int 481 dpaa_eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, 482 struct qm_fd *fd, 483 uint32_t bpid) 484 { 485 struct rte_mbuf *cur_seg = mbuf, *prev_seg = NULL; 486 struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(bpid); 487 struct rte_mbuf *temp, *mi; 488 struct qm_sg_entry *sg_temp, *sgt; 489 int i = 0; 490 491 DPAA_DP_LOG(DEBUG, "Creating SG FD to transmit"); 492 493 temp = rte_pktmbuf_alloc(bp_info->mp); 494 if (!temp) { 495 DPAA_PMD_ERR("Failure in allocation of mbuf"); 496 return -1; 497 } 498 if (temp->buf_len < ((mbuf->nb_segs * sizeof(struct qm_sg_entry)) 499 + temp->data_off)) { 500 DPAA_PMD_ERR("Insufficient space in mbuf for SG entries"); 501 return -1; 502 } 503 504 fd->cmd = 0; 505 fd->opaque_addr = 0; 506 507 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { 508 if (!mbuf->packet_type) { 509 struct rte_net_hdr_lens hdr_lens; 510 511 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 512 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 513 | RTE_PTYPE_L4_MASK); 514 mbuf->l2_len = hdr_lens.l2_len; 515 mbuf->l3_len = hdr_lens.l3_len; 516 } 517 if (temp->data_off < DEFAULT_TX_ICEOF 518 + sizeof(struct dpaa_eth_parse_results_t)) 519 temp->data_off = DEFAULT_TX_ICEOF 520 + sizeof(struct dpaa_eth_parse_results_t); 521 dcbz_64(temp->buf_addr); 522 dpaa_checksum_offload(mbuf, fd, temp->buf_addr); 523 } 524 525 sgt = temp->buf_addr + temp->data_off; 526 fd->format = QM_FD_SG; 527 fd->addr = temp->buf_iova; 528 fd->offset = temp->data_off; 529 fd->bpid = bpid; 530 fd->length20 = mbuf->pkt_len; 531 532 while (i < DPAA_SGT_MAX_ENTRIES) { 533 sg_temp = &sgt[i++]; 534 sg_temp->opaque = 0; 535 sg_temp->val = 0; 536 sg_temp->addr = cur_seg->buf_iova; 537 sg_temp->offset = cur_seg->data_off; 538 sg_temp->length = cur_seg->data_len; 539 if (RTE_MBUF_DIRECT(cur_seg)) { 540 if (rte_mbuf_refcnt_read(cur_seg) > 1) { 541 /*If refcnt > 1, invalid bpid is set to ensure 542 * buffer is not freed by HW. 543 */ 544 sg_temp->bpid = 0xff; 545 rte_mbuf_refcnt_update(cur_seg, -1); 546 } else { 547 sg_temp->bpid = 548 DPAA_MEMPOOL_TO_BPID(cur_seg->pool); 549 } 550 cur_seg = cur_seg->next; 551 } else { 552 /* Get owner MBUF from indirect buffer */ 553 mi = rte_mbuf_from_indirect(cur_seg); 554 if (rte_mbuf_refcnt_read(mi) > 1) { 555 /*If refcnt > 1, invalid bpid is set to ensure 556 * owner buffer is not freed by HW. 557 */ 558 sg_temp->bpid = 0xff; 559 } else { 560 sg_temp->bpid = DPAA_MEMPOOL_TO_BPID(mi->pool); 561 rte_mbuf_refcnt_update(mi, 1); 562 } 563 prev_seg = cur_seg; 564 cur_seg = cur_seg->next; 565 prev_seg->next = NULL; 566 rte_pktmbuf_free(prev_seg); 567 } 568 if (cur_seg == NULL) { 569 sg_temp->final = 1; 570 cpu_to_hw_sg(sg_temp); 571 break; 572 } 573 cpu_to_hw_sg(sg_temp); 574 } 575 return 0; 576 } 577 578 /* Handle mbufs which are not segmented (non SG) */ 579 static inline void 580 tx_on_dpaa_pool_unsegmented(struct rte_mbuf *mbuf, 581 struct dpaa_bp_info *bp_info, 582 struct qm_fd *fd_arr) 583 { 584 struct rte_mbuf *mi = NULL; 585 586 if (RTE_MBUF_DIRECT(mbuf)) { 587 if (rte_mbuf_refcnt_read(mbuf) > 1) { 588 /* In case of direct mbuf and mbuf being cloned, 589 * BMAN should _not_ release buffer. 590 */ 591 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 592 /* Buffer should be releasd by EAL */ 593 rte_mbuf_refcnt_update(mbuf, -1); 594 } else { 595 /* In case of direct mbuf and no cloning, mbuf can be 596 * released by BMAN. 597 */ 598 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 599 } 600 } else { 601 /* This is data-containing core mbuf: 'mi' */ 602 mi = rte_mbuf_from_indirect(mbuf); 603 if (rte_mbuf_refcnt_read(mi) > 1) { 604 /* In case of indirect mbuf, and mbuf being cloned, 605 * BMAN should _not_ release it and let EAL release 606 * it through pktmbuf_free below. 607 */ 608 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); 609 } else { 610 /* In case of indirect mbuf, and no cloning, core mbuf 611 * should be released by BMAN. 612 * Increate refcnt of core mbuf so that when 613 * pktmbuf_free is called and mbuf is released, EAL 614 * doesn't try to release core mbuf which would have 615 * been released by BMAN. 616 */ 617 rte_mbuf_refcnt_update(mi, 1); 618 DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); 619 } 620 rte_pktmbuf_free(mbuf); 621 } 622 623 if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { 624 if (!mbuf->packet_type) { 625 struct rte_net_hdr_lens hdr_lens; 626 627 mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, 628 RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK 629 | RTE_PTYPE_L4_MASK); 630 mbuf->l2_len = hdr_lens.l2_len; 631 mbuf->l3_len = hdr_lens.l3_len; 632 } 633 if (mbuf->data_off < (DEFAULT_TX_ICEOF + 634 sizeof(struct dpaa_eth_parse_results_t))) { 635 DPAA_DP_LOG(DEBUG, "Checksum offload Err: " 636 "Not enough Headroom " 637 "space for correct Checksum offload." 638 "So Calculating checksum in Software."); 639 dpaa_checksum(mbuf); 640 } else { 641 dpaa_checksum_offload(mbuf, fd_arr, mbuf->buf_addr); 642 } 643 } 644 } 645 646 /* Handle all mbufs on dpaa BMAN managed pool */ 647 static inline uint16_t 648 tx_on_dpaa_pool(struct rte_mbuf *mbuf, 649 struct dpaa_bp_info *bp_info, 650 struct qm_fd *fd_arr) 651 { 652 DPAA_DP_LOG(DEBUG, "BMAN offloaded buffer, mbuf: %p", mbuf); 653 654 if (mbuf->nb_segs == 1) { 655 /* Case for non-segmented buffers */ 656 tx_on_dpaa_pool_unsegmented(mbuf, bp_info, fd_arr); 657 } else if (mbuf->nb_segs > 1 && 658 mbuf->nb_segs <= DPAA_SGT_MAX_ENTRIES) { 659 if (dpaa_eth_mbuf_to_sg_fd(mbuf, fd_arr, bp_info->bpid)) { 660 DPAA_PMD_DEBUG("Unable to create Scatter Gather FD"); 661 return 1; 662 } 663 } else { 664 DPAA_PMD_DEBUG("Number of Segments not supported"); 665 return 1; 666 } 667 668 return 0; 669 } 670 671 /* Handle all mbufs on an external pool (non-dpaa) */ 672 static inline uint16_t 673 tx_on_external_pool(struct qman_fq *txq, struct rte_mbuf *mbuf, 674 struct qm_fd *fd_arr) 675 { 676 struct dpaa_if *dpaa_intf = txq->dpaa_intf; 677 struct rte_mbuf *dmable_mbuf; 678 679 DPAA_DP_LOG(DEBUG, "Non-BMAN offloaded buffer." 680 "Allocating an offloaded buffer"); 681 dmable_mbuf = dpaa_get_dmable_mbuf(mbuf, dpaa_intf); 682 if (!dmable_mbuf) { 683 DPAA_DP_LOG(DEBUG, "no dpaa buffers."); 684 return 1; 685 } 686 687 DPAA_MBUF_TO_CONTIG_FD(dmable_mbuf, fd_arr, dpaa_intf->bp_info->bpid); 688 689 return 0; 690 } 691 692 uint16_t 693 dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 694 { 695 struct rte_mbuf *mbuf, *mi = NULL; 696 struct rte_mempool *mp; 697 struct dpaa_bp_info *bp_info; 698 struct qm_fd fd_arr[MAX_TX_RING_SLOTS]; 699 uint32_t frames_to_send, loop, i = 0; 700 uint16_t state; 701 int ret; 702 703 ret = rte_dpaa_portal_init((void *)0); 704 if (ret) { 705 DPAA_PMD_ERR("Failure in affining portal"); 706 return 0; 707 } 708 709 DPAA_DP_LOG(DEBUG, "Transmitting %d buffers on queue: %p", nb_bufs, q); 710 711 while (nb_bufs) { 712 frames_to_send = (nb_bufs >> 3) ? MAX_TX_RING_SLOTS : nb_bufs; 713 for (loop = 0; loop < frames_to_send; loop++, i++) { 714 mbuf = bufs[i]; 715 if (RTE_MBUF_DIRECT(mbuf)) { 716 mp = mbuf->pool; 717 } else { 718 mi = rte_mbuf_from_indirect(mbuf); 719 mp = mi->pool; 720 } 721 722 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 723 if (likely(mp->ops_index == bp_info->dpaa_ops_index)) { 724 state = tx_on_dpaa_pool(mbuf, bp_info, 725 &fd_arr[loop]); 726 if (unlikely(state)) { 727 /* Set frames_to_send & nb_bufs so 728 * that packets are transmitted till 729 * previous frame. 730 */ 731 frames_to_send = loop; 732 nb_bufs = loop; 733 goto send_pkts; 734 } 735 } else { 736 state = tx_on_external_pool(q, mbuf, 737 &fd_arr[loop]); 738 if (unlikely(state)) { 739 /* Set frames_to_send & nb_bufs so 740 * that packets are transmitted till 741 * previous frame. 742 */ 743 frames_to_send = loop; 744 nb_bufs = loop; 745 goto send_pkts; 746 } 747 } 748 } 749 750 send_pkts: 751 loop = 0; 752 while (loop < frames_to_send) { 753 loop += qman_enqueue_multi(q, &fd_arr[loop], 754 frames_to_send - loop); 755 } 756 nb_bufs -= frames_to_send; 757 } 758 759 DPAA_DP_LOG(DEBUG, "Transmitted %d buffers on queue: %p", i, q); 760 761 return i; 762 } 763 764 uint16_t dpaa_eth_tx_drop_all(void *q __rte_unused, 765 struct rte_mbuf **bufs __rte_unused, 766 uint16_t nb_bufs __rte_unused) 767 { 768 DPAA_DP_LOG(DEBUG, "Drop all packets"); 769 770 /* Drop all incoming packets. No need to free packets here 771 * because the rte_eth f/w frees up the packets through tx_buffer 772 * callback in case this functions returns count less than nb_bufs 773 */ 774 return 0; 775 } 776