1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016-2018 Solarflare Communications Inc. 4 * All rights reserved. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 /* EF10 native datapath implementation */ 11 12 #include <stdbool.h> 13 14 #include <rte_byteorder.h> 15 #include <rte_mbuf_ptype.h> 16 #include <rte_mbuf.h> 17 #include <rte_io.h> 18 19 #include "efx.h" 20 #include "efx_types.h" 21 #include "efx_regs.h" 22 #include "efx_regs_ef10.h" 23 24 #include "sfc_tweak.h" 25 #include "sfc_dp_rx.h" 26 #include "sfc_kvargs.h" 27 #include "sfc_ef10.h" 28 29 #define SFC_EF10_RX_EV_ENCAP_SUPPORT 1 30 #include "sfc_ef10_rx_ev.h" 31 32 #define sfc_ef10_rx_err(dpq, ...) \ 33 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__) 34 35 /** 36 * Maximum number of descriptors/buffers in the Rx ring. 37 * It should guarantee that corresponding event queue never overfill. 38 * EF10 native datapath uses event queue of the same size as Rx queue. 39 * Maximum number of events on datapath can be estimated as number of 40 * Rx queue entries (one event per Rx buffer in the worst case) plus 41 * Rx error and flush events. 42 */ 43 #define SFC_EF10_RXQ_LIMIT(_ndesc) \ 44 ((_ndesc) - 1 /* head must not step on tail */ - \ 45 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \ 46 1 /* Rx error */ - 1 /* flush */) 47 48 struct sfc_ef10_rx_sw_desc { 49 struct rte_mbuf *mbuf; 50 }; 51 52 struct sfc_ef10_rxq { 53 /* Used on data path */ 54 unsigned int flags; 55 #define SFC_EF10_RXQ_STARTED 0x1 56 #define SFC_EF10_RXQ_NOT_RUNNING 0x2 57 #define SFC_EF10_RXQ_EXCEPTION 0x4 58 #define SFC_EF10_RXQ_RSS_HASH 0x8 59 unsigned int ptr_mask; 60 unsigned int pending; 61 unsigned int completed; 62 unsigned int evq_read_ptr; 63 efx_qword_t *evq_hw_ring; 64 struct sfc_ef10_rx_sw_desc *sw_ring; 65 uint64_t rearm_data; 66 struct rte_mbuf *scatter_pkt; 67 uint16_t prefix_size; 68 69 /* Used on refill */ 70 uint16_t buf_size; 71 unsigned int added; 72 unsigned int max_fill_level; 73 unsigned int refill_threshold; 74 struct rte_mempool *refill_mb_pool; 75 efx_qword_t *rxq_hw_ring; 76 volatile void *doorbell; 77 78 /* Datapath receive queue anchor */ 79 struct sfc_dp_rxq dp; 80 }; 81 82 static inline struct sfc_ef10_rxq * 83 sfc_ef10_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq) 84 { 85 return container_of(dp_rxq, struct sfc_ef10_rxq, dp); 86 } 87 88 static void 89 sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq) 90 { 91 const unsigned int ptr_mask = rxq->ptr_mask; 92 const uint32_t buf_size = rxq->buf_size; 93 unsigned int free_space; 94 unsigned int bulks; 95 void *objs[SFC_RX_REFILL_BULK]; 96 unsigned int added = rxq->added; 97 98 RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0); 99 100 free_space = rxq->max_fill_level - (added - rxq->completed); 101 102 if (free_space < rxq->refill_threshold) 103 return; 104 105 bulks = free_space / RTE_DIM(objs); 106 /* refill_threshold guarantees that bulks is positive */ 107 SFC_ASSERT(bulks > 0); 108 109 do { 110 unsigned int id; 111 unsigned int i; 112 113 if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, 114 RTE_DIM(objs)) < 0)) { 115 struct rte_eth_dev_data *dev_data = 116 rte_eth_devices[rxq->dp.dpq.port_id].data; 117 118 /* 119 * It is hardly a safe way to increment counter 120 * from different contexts, but all PMDs do it. 121 */ 122 dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs); 123 /* Return if we have posted nothing yet */ 124 if (added == rxq->added) 125 return; 126 /* Push posted */ 127 break; 128 } 129 130 for (i = 0, id = added & ptr_mask; 131 i < RTE_DIM(objs); 132 ++i, ++id) { 133 struct rte_mbuf *m = objs[i]; 134 struct sfc_ef10_rx_sw_desc *rxd; 135 rte_iova_t phys_addr; 136 137 MBUF_RAW_ALLOC_CHECK(m); 138 139 SFC_ASSERT((id & ~ptr_mask) == 0); 140 rxd = &rxq->sw_ring[id]; 141 rxd->mbuf = m; 142 143 /* 144 * Avoid writing to mbuf. It is cheaper to do it 145 * when we receive packet and fill in nearby 146 * structure members. 147 */ 148 149 phys_addr = rte_mbuf_data_iova_default(m); 150 EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id], 151 ESF_DZ_RX_KER_BYTE_CNT, buf_size, 152 ESF_DZ_RX_KER_BUF_ADDR, phys_addr); 153 } 154 155 added += RTE_DIM(objs); 156 } while (--bulks > 0); 157 158 SFC_ASSERT(rxq->added != added); 159 rxq->added = added; 160 sfc_ef10_rx_qpush(rxq->doorbell, added, ptr_mask); 161 } 162 163 static void 164 sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id) 165 { 166 struct rte_mbuf *next_mbuf; 167 168 /* Prefetch next bunch of software descriptors */ 169 if ((next_id % (RTE_CACHE_LINE_SIZE / sizeof(rxq->sw_ring[0]))) == 0) 170 rte_prefetch0(&rxq->sw_ring[next_id]); 171 172 /* 173 * It looks strange to prefetch depending on previous prefetch 174 * data, but measurements show that it is really efficient and 175 * increases packet rate. 176 */ 177 next_mbuf = rxq->sw_ring[next_id].mbuf; 178 if (likely(next_mbuf != NULL)) { 179 /* Prefetch the next mbuf structure */ 180 rte_mbuf_prefetch_part1(next_mbuf); 181 182 /* Prefetch pseudo header of the next packet */ 183 /* data_off is not filled in yet */ 184 /* Yes, data could be not ready yet, but we hope */ 185 rte_prefetch0((uint8_t *)next_mbuf->buf_addr + 186 RTE_PKTMBUF_HEADROOM); 187 } 188 } 189 190 static struct rte_mbuf ** 191 sfc_ef10_rx_pending(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts, 192 uint16_t nb_pkts) 193 { 194 uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending - rxq->completed); 195 196 SFC_ASSERT(rxq->pending == rxq->completed || rxq->scatter_pkt == NULL); 197 198 if (n_rx_pkts != 0) { 199 unsigned int completed = rxq->completed; 200 201 rxq->completed = completed + n_rx_pkts; 202 203 do { 204 *rx_pkts++ = 205 rxq->sw_ring[completed++ & rxq->ptr_mask].mbuf; 206 } while (completed != rxq->completed); 207 } 208 209 return rx_pkts; 210 } 211 212 static uint16_t 213 sfc_ef10_rx_pseudo_hdr_get_len(const uint8_t *pseudo_hdr) 214 { 215 return rte_le_to_cpu_16(*(const uint16_t *)&pseudo_hdr[8]); 216 } 217 218 static uint32_t 219 sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr) 220 { 221 return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr); 222 } 223 224 static struct rte_mbuf ** 225 sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev, 226 struct rte_mbuf **rx_pkts, 227 struct rte_mbuf ** const rx_pkts_end) 228 { 229 const unsigned int ptr_mask = rxq->ptr_mask; 230 unsigned int pending = rxq->pending; 231 unsigned int ready; 232 struct sfc_ef10_rx_sw_desc *rxd; 233 struct rte_mbuf *m; 234 struct rte_mbuf *m0; 235 const uint8_t *pseudo_hdr; 236 uint16_t seg_len; 237 238 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - pending) & 239 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); 240 241 if (ready == 0) { 242 /* Rx abort - it was no enough descriptors for Rx packet */ 243 rte_pktmbuf_free(rxq->scatter_pkt); 244 rxq->scatter_pkt = NULL; 245 return rx_pkts; 246 } 247 248 rxq->pending = pending + ready; 249 250 if (rx_ev.eq_u64[0] & 251 rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) | 252 (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) { 253 SFC_ASSERT(rxq->completed == pending); 254 do { 255 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 256 rte_mbuf_raw_free(rxd->mbuf); 257 } while (pending != rxq->pending); 258 rxq->completed = pending; 259 return rx_pkts; 260 } 261 262 /* If scattered packet is in progress */ 263 if (rxq->scatter_pkt != NULL) { 264 /* Events for scattered packet frags are not merged */ 265 SFC_ASSERT(ready == 1); 266 SFC_ASSERT(rxq->completed == pending); 267 268 /* There is no pseudo-header in scatter segments. */ 269 seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES); 270 271 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 272 m = rxd->mbuf; 273 274 MBUF_RAW_ALLOC_CHECK(m); 275 276 m->data_off = RTE_PKTMBUF_HEADROOM; 277 rte_pktmbuf_data_len(m) = seg_len; 278 rte_pktmbuf_pkt_len(m) = seg_len; 279 280 rxq->scatter_pkt->nb_segs++; 281 rte_pktmbuf_pkt_len(rxq->scatter_pkt) += seg_len; 282 rte_pktmbuf_lastseg(rxq->scatter_pkt)->next = m; 283 284 if (~rx_ev.eq_u64[0] & 285 rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) { 286 *rx_pkts++ = rxq->scatter_pkt; 287 rxq->scatter_pkt = NULL; 288 } 289 rxq->completed = pending; 290 return rx_pkts; 291 } 292 293 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 294 295 sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask); 296 297 m = rxd->mbuf; 298 299 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data)); 300 m->rearm_data[0] = rxq->rearm_data; 301 302 /* Classify packet based on Rx event */ 303 /* Mask RSS hash offload flag if RSS is not enabled */ 304 sfc_ef10_rx_ev_to_offloads(rx_ev, m, 305 (rxq->flags & SFC_EF10_RXQ_RSS_HASH) ? 306 ~0ull : ~PKT_RX_RSS_HASH); 307 308 /* data_off already moved past pseudo header */ 309 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 310 311 /* 312 * Always get RSS hash from pseudo header to avoid 313 * condition/branching. If it is valid or not depends on 314 * PKT_RX_RSS_HASH in m->ol_flags. 315 */ 316 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); 317 318 if (ready == 1) 319 seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) - 320 rxq->prefix_size; 321 else 322 seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); 323 SFC_ASSERT(seg_len > 0); 324 rte_pktmbuf_data_len(m) = seg_len; 325 rte_pktmbuf_pkt_len(m) = seg_len; 326 327 SFC_ASSERT(m->next == NULL); 328 329 if (~rx_ev.eq_u64[0] & rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) { 330 *rx_pkts++ = m; 331 rxq->completed = pending; 332 } else { 333 /* Events with CONT bit are not merged */ 334 SFC_ASSERT(ready == 1); 335 rxq->scatter_pkt = m; 336 rxq->completed = pending; 337 return rx_pkts; 338 } 339 340 /* Remember mbuf to copy offload flags and packet type from */ 341 m0 = m; 342 while (pending != rxq->pending) { 343 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 344 345 sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask); 346 347 m = rxd->mbuf; 348 349 if (rx_pkts != rx_pkts_end) { 350 *rx_pkts++ = m; 351 rxq->completed = pending; 352 } 353 354 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != 355 sizeof(rxq->rearm_data)); 356 m->rearm_data[0] = rxq->rearm_data; 357 358 /* Event-dependent information is the same */ 359 m->ol_flags = m0->ol_flags; 360 m->packet_type = m0->packet_type; 361 362 /* data_off already moved past pseudo header */ 363 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 364 365 /* 366 * Always get RSS hash from pseudo header to avoid 367 * condition/branching. If it is valid or not depends on 368 * PKT_RX_RSS_HASH in m->ol_flags. 369 */ 370 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); 371 372 seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); 373 SFC_ASSERT(seg_len > 0); 374 rte_pktmbuf_data_len(m) = seg_len; 375 rte_pktmbuf_pkt_len(m) = seg_len; 376 377 SFC_ASSERT(m->next == NULL); 378 } 379 380 return rx_pkts; 381 } 382 383 static bool 384 sfc_ef10_rx_get_event(struct sfc_ef10_rxq *rxq, efx_qword_t *rx_ev) 385 { 386 *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->ptr_mask]; 387 388 if (!sfc_ef10_ev_present(*rx_ev)) 389 return false; 390 391 if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) != 392 FSE_AZ_EV_CODE_RX_EV)) { 393 /* 394 * Do not move read_ptr to keep the event for exception 395 * handling by the control path. 396 */ 397 rxq->flags |= SFC_EF10_RXQ_EXCEPTION; 398 sfc_ef10_rx_err(&rxq->dp.dpq, 399 "RxQ exception at EvQ read ptr %#x", 400 rxq->evq_read_ptr); 401 return false; 402 } 403 404 rxq->evq_read_ptr++; 405 return true; 406 } 407 408 static uint16_t 409 sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 410 { 411 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue); 412 struct rte_mbuf ** const rx_pkts_end = &rx_pkts[nb_pkts]; 413 unsigned int evq_old_read_ptr; 414 efx_qword_t rx_ev; 415 416 rx_pkts = sfc_ef10_rx_pending(rxq, rx_pkts, nb_pkts); 417 418 if (unlikely(rxq->flags & 419 (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) 420 goto done; 421 422 evq_old_read_ptr = rxq->evq_read_ptr; 423 while (rx_pkts != rx_pkts_end && sfc_ef10_rx_get_event(rxq, &rx_ev)) { 424 /* 425 * DROP_EVENT is an internal to the NIC, software should 426 * never see it and, therefore, may ignore it. 427 */ 428 429 rx_pkts = sfc_ef10_rx_process_event(rxq, rx_ev, 430 rx_pkts, rx_pkts_end); 431 } 432 433 sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr, 434 rxq->evq_read_ptr); 435 436 /* It is not a problem if we refill in the case of exception */ 437 sfc_ef10_rx_qrefill(rxq); 438 439 done: 440 return nb_pkts - (rx_pkts_end - rx_pkts); 441 } 442 443 const uint32_t * 444 sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps) 445 { 446 static const uint32_t ef10_native_ptypes[] = { 447 RTE_PTYPE_L2_ETHER, 448 RTE_PTYPE_L2_ETHER_ARP, 449 RTE_PTYPE_L2_ETHER_VLAN, 450 RTE_PTYPE_L2_ETHER_QINQ, 451 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 452 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 453 RTE_PTYPE_L4_FRAG, 454 RTE_PTYPE_L4_TCP, 455 RTE_PTYPE_L4_UDP, 456 RTE_PTYPE_UNKNOWN 457 }; 458 static const uint32_t ef10_overlay_ptypes[] = { 459 RTE_PTYPE_L2_ETHER, 460 RTE_PTYPE_L2_ETHER_ARP, 461 RTE_PTYPE_L2_ETHER_VLAN, 462 RTE_PTYPE_L2_ETHER_QINQ, 463 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 464 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 465 RTE_PTYPE_L4_FRAG, 466 RTE_PTYPE_L4_TCP, 467 RTE_PTYPE_L4_UDP, 468 RTE_PTYPE_TUNNEL_VXLAN, 469 RTE_PTYPE_TUNNEL_NVGRE, 470 RTE_PTYPE_INNER_L2_ETHER, 471 RTE_PTYPE_INNER_L2_ETHER_VLAN, 472 RTE_PTYPE_INNER_L2_ETHER_QINQ, 473 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 474 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 475 RTE_PTYPE_INNER_L4_FRAG, 476 RTE_PTYPE_INNER_L4_TCP, 477 RTE_PTYPE_INNER_L4_UDP, 478 RTE_PTYPE_UNKNOWN 479 }; 480 481 /* 482 * The function returns static set of supported packet types, 483 * so we can't build it dynamically based on supported tunnel 484 * encapsulations and should limit to known sets. 485 */ 486 switch (tunnel_encaps) { 487 case (1u << EFX_TUNNEL_PROTOCOL_VXLAN | 488 1u << EFX_TUNNEL_PROTOCOL_GENEVE | 489 1u << EFX_TUNNEL_PROTOCOL_NVGRE): 490 return ef10_overlay_ptypes; 491 default: 492 SFC_GENERIC_LOG(ERR, 493 "Unexpected set of supported tunnel encapsulations: %#x", 494 tunnel_encaps); 495 /* FALLTHROUGH */ 496 case 0: 497 return ef10_native_ptypes; 498 } 499 } 500 501 static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending; 502 static unsigned int 503 sfc_ef10_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq) 504 { 505 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 506 efx_qword_t rx_ev; 507 const unsigned int evq_old_read_ptr = rxq->evq_read_ptr; 508 unsigned int pending = rxq->pending; 509 unsigned int ready; 510 511 if (unlikely(rxq->flags & 512 (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) 513 goto done; 514 515 while (sfc_ef10_rx_get_event(rxq, &rx_ev)) { 516 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - 517 pending) & 518 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); 519 pending += ready; 520 } 521 522 /* 523 * The function does not process events, so return event queue read 524 * pointer to the original position to allow the events that were 525 * read to be processed later 526 */ 527 rxq->evq_read_ptr = evq_old_read_ptr; 528 529 done: 530 return pending - rxq->completed; 531 } 532 533 static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status; 534 static int 535 sfc_ef10_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset) 536 { 537 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 538 unsigned int npending = sfc_ef10_rx_qdesc_npending(dp_rxq); 539 540 if (unlikely(offset > rxq->ptr_mask)) 541 return -EINVAL; 542 543 if (offset < npending) 544 return RTE_ETH_RX_DESC_DONE; 545 546 if (offset < (rxq->added - rxq->completed)) 547 return RTE_ETH_RX_DESC_AVAIL; 548 549 return RTE_ETH_RX_DESC_UNAVAIL; 550 } 551 552 553 static sfc_dp_rx_get_dev_info_t sfc_ef10_rx_get_dev_info; 554 static void 555 sfc_ef10_rx_get_dev_info(struct rte_eth_dev_info *dev_info) 556 { 557 /* 558 * Number of descriptors just defines maximum number of pushed 559 * descriptors (fill level). 560 */ 561 dev_info->rx_desc_lim.nb_min = SFC_RX_REFILL_BULK; 562 dev_info->rx_desc_lim.nb_align = SFC_RX_REFILL_BULK; 563 } 564 565 566 static sfc_dp_rx_qsize_up_rings_t sfc_ef10_rx_qsize_up_rings; 567 static int 568 sfc_ef10_rx_qsize_up_rings(uint16_t nb_rx_desc, 569 __rte_unused struct rte_mempool *mb_pool, 570 unsigned int *rxq_entries, 571 unsigned int *evq_entries, 572 unsigned int *rxq_max_fill_level) 573 { 574 /* 575 * rte_ethdev API guarantees that the number meets min, max and 576 * alignment requirements. 577 */ 578 if (nb_rx_desc <= EFX_RXQ_MINNDESCS) 579 *rxq_entries = EFX_RXQ_MINNDESCS; 580 else 581 *rxq_entries = rte_align32pow2(nb_rx_desc); 582 583 *evq_entries = *rxq_entries; 584 585 *rxq_max_fill_level = RTE_MIN(nb_rx_desc, 586 SFC_EF10_RXQ_LIMIT(*evq_entries)); 587 return 0; 588 } 589 590 591 static uint64_t 592 sfc_ef10_mk_mbuf_rearm_data(uint16_t port_id, uint16_t prefix_size) 593 { 594 struct rte_mbuf m; 595 596 memset(&m, 0, sizeof(m)); 597 598 rte_mbuf_refcnt_set(&m, 1); 599 m.data_off = RTE_PKTMBUF_HEADROOM + prefix_size; 600 m.nb_segs = 1; 601 m.port = port_id; 602 603 /* rearm_data covers structure members filled in above */ 604 rte_compiler_barrier(); 605 RTE_BUILD_BUG_ON(sizeof(m.rearm_data[0]) != sizeof(uint64_t)); 606 return m.rearm_data[0]; 607 } 608 609 static sfc_dp_rx_qcreate_t sfc_ef10_rx_qcreate; 610 static int 611 sfc_ef10_rx_qcreate(uint16_t port_id, uint16_t queue_id, 612 const struct rte_pci_addr *pci_addr, int socket_id, 613 const struct sfc_dp_rx_qcreate_info *info, 614 struct sfc_dp_rxq **dp_rxqp) 615 { 616 struct sfc_ef10_rxq *rxq; 617 int rc; 618 619 rc = EINVAL; 620 if (info->rxq_entries != info->evq_entries) 621 goto fail_rxq_args; 622 623 rc = ENOMEM; 624 rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq), 625 RTE_CACHE_LINE_SIZE, socket_id); 626 if (rxq == NULL) 627 goto fail_rxq_alloc; 628 629 sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); 630 631 rc = ENOMEM; 632 rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring", 633 info->rxq_entries, 634 sizeof(*rxq->sw_ring), 635 RTE_CACHE_LINE_SIZE, socket_id); 636 if (rxq->sw_ring == NULL) 637 goto fail_desc_alloc; 638 639 rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; 640 if (info->flags & SFC_RXQ_FLAG_RSS_HASH) 641 rxq->flags |= SFC_EF10_RXQ_RSS_HASH; 642 rxq->ptr_mask = info->rxq_entries - 1; 643 rxq->evq_hw_ring = info->evq_hw_ring; 644 rxq->max_fill_level = info->max_fill_level; 645 rxq->refill_threshold = info->refill_threshold; 646 rxq->rearm_data = 647 sfc_ef10_mk_mbuf_rearm_data(port_id, info->prefix_size); 648 rxq->prefix_size = info->prefix_size; 649 rxq->buf_size = info->buf_size; 650 rxq->refill_mb_pool = info->refill_mb_pool; 651 rxq->rxq_hw_ring = info->rxq_hw_ring; 652 rxq->doorbell = (volatile uint8_t *)info->mem_bar + 653 ER_DZ_RX_DESC_UPD_REG_OFST + 654 (info->hw_index << info->vi_window_shift); 655 656 *dp_rxqp = &rxq->dp; 657 return 0; 658 659 fail_desc_alloc: 660 rte_free(rxq); 661 662 fail_rxq_alloc: 663 fail_rxq_args: 664 return rc; 665 } 666 667 static sfc_dp_rx_qdestroy_t sfc_ef10_rx_qdestroy; 668 static void 669 sfc_ef10_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) 670 { 671 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 672 673 rte_free(rxq->sw_ring); 674 rte_free(rxq); 675 } 676 677 static sfc_dp_rx_qstart_t sfc_ef10_rx_qstart; 678 static int 679 sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr) 680 { 681 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 682 683 SFC_ASSERT(rxq->completed == 0); 684 SFC_ASSERT(rxq->pending == 0); 685 SFC_ASSERT(rxq->added == 0); 686 687 sfc_ef10_rx_qrefill(rxq); 688 689 rxq->evq_read_ptr = evq_read_ptr; 690 691 rxq->flags |= SFC_EF10_RXQ_STARTED; 692 rxq->flags &= ~(SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION); 693 694 return 0; 695 } 696 697 static sfc_dp_rx_qstop_t sfc_ef10_rx_qstop; 698 static void 699 sfc_ef10_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr) 700 { 701 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 702 703 rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; 704 705 *evq_read_ptr = rxq->evq_read_ptr; 706 } 707 708 static sfc_dp_rx_qrx_ev_t sfc_ef10_rx_qrx_ev; 709 static bool 710 sfc_ef10_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id) 711 { 712 __rte_unused struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 713 714 SFC_ASSERT(rxq->flags & SFC_EF10_RXQ_NOT_RUNNING); 715 716 /* 717 * It is safe to ignore Rx event since we free all mbufs on 718 * queue purge anyway. 719 */ 720 721 return false; 722 } 723 724 static sfc_dp_rx_qpurge_t sfc_ef10_rx_qpurge; 725 static void 726 sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq) 727 { 728 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 729 unsigned int i; 730 struct sfc_ef10_rx_sw_desc *rxd; 731 732 rte_pktmbuf_free(rxq->scatter_pkt); 733 rxq->scatter_pkt = NULL; 734 735 for (i = rxq->completed; i != rxq->added; ++i) { 736 rxd = &rxq->sw_ring[i & rxq->ptr_mask]; 737 rte_mbuf_raw_free(rxd->mbuf); 738 rxd->mbuf = NULL; 739 } 740 741 rxq->completed = rxq->pending = rxq->added = 0; 742 743 rxq->flags &= ~SFC_EF10_RXQ_STARTED; 744 } 745 746 struct sfc_dp_rx sfc_ef10_rx = { 747 .dp = { 748 .name = SFC_KVARG_DATAPATH_EF10, 749 .type = SFC_DP_RX, 750 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, 751 }, 752 .features = SFC_DP_RX_FEAT_SCATTER | 753 SFC_DP_RX_FEAT_MULTI_PROCESS | 754 SFC_DP_RX_FEAT_TUNNELS | 755 SFC_DP_RX_FEAT_CHECKSUM, 756 .get_dev_info = sfc_ef10_rx_get_dev_info, 757 .qsize_up_rings = sfc_ef10_rx_qsize_up_rings, 758 .qcreate = sfc_ef10_rx_qcreate, 759 .qdestroy = sfc_ef10_rx_qdestroy, 760 .qstart = sfc_ef10_rx_qstart, 761 .qstop = sfc_ef10_rx_qstop, 762 .qrx_ev = sfc_ef10_rx_qrx_ev, 763 .qpurge = sfc_ef10_rx_qpurge, 764 .supported_ptypes_get = sfc_ef10_supported_ptypes_get, 765 .qdesc_npending = sfc_ef10_rx_qdesc_npending, 766 .qdesc_status = sfc_ef10_rx_qdesc_status, 767 .pkt_burst = sfc_ef10_recv_pkts, 768 }; 769