1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was jointly developed between OKTET Labs (under contract 8 * for Solarflare) and Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* EF10 native datapath implementation */ 33 34 #include <stdbool.h> 35 36 #include <rte_byteorder.h> 37 #include <rte_mbuf_ptype.h> 38 #include <rte_mbuf.h> 39 #include <rte_io.h> 40 41 #include "efx.h" 42 #include "efx_types.h" 43 #include "efx_regs.h" 44 #include "efx_regs_ef10.h" 45 46 #include "sfc_tweak.h" 47 #include "sfc_dp_rx.h" 48 #include "sfc_kvargs.h" 49 #include "sfc_ef10.h" 50 51 #define sfc_ef10_rx_err(dpq, ...) \ 52 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__) 53 54 /** 55 * Alignment requirement for value written to RX WPTR: 56 * the WPTR must be aligned to an 8 descriptor boundary. 57 */ 58 #define SFC_EF10_RX_WPTR_ALIGN 8 59 60 /** 61 * Maximum number of descriptors/buffers in the Rx ring. 62 * It should guarantee that corresponding event queue never overfill. 63 * EF10 native datapath uses event queue of the same size as Rx queue. 64 * Maximum number of events on datapath can be estimated as number of 65 * Rx queue entries (one event per Rx buffer in the worst case) plus 66 * Rx error and flush events. 67 */ 68 #define SFC_EF10_RXQ_LIMIT(_ndesc) \ 69 ((_ndesc) - 1 /* head must not step on tail */ - \ 70 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \ 71 1 /* Rx error */ - 1 /* flush */) 72 73 struct sfc_ef10_rx_sw_desc { 74 struct rte_mbuf *mbuf; 75 }; 76 77 struct sfc_ef10_rxq { 78 /* Used on data path */ 79 unsigned int flags; 80 #define SFC_EF10_RXQ_STARTED 0x1 81 #define SFC_EF10_RXQ_NOT_RUNNING 0x2 82 #define SFC_EF10_RXQ_EXCEPTION 0x4 83 #define SFC_EF10_RXQ_RSS_HASH 0x8 84 unsigned int ptr_mask; 85 unsigned int prepared; 86 unsigned int completed; 87 unsigned int evq_read_ptr; 88 efx_qword_t *evq_hw_ring; 89 struct sfc_ef10_rx_sw_desc *sw_ring; 90 uint64_t rearm_data; 91 uint16_t prefix_size; 92 93 /* Used on refill */ 94 uint16_t buf_size; 95 unsigned int added; 96 unsigned int refill_threshold; 97 struct rte_mempool *refill_mb_pool; 98 efx_qword_t *rxq_hw_ring; 99 volatile void *doorbell; 100 101 /* Datapath receive queue anchor */ 102 struct sfc_dp_rxq dp; 103 }; 104 105 static inline struct sfc_ef10_rxq * 106 sfc_ef10_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq) 107 { 108 return container_of(dp_rxq, struct sfc_ef10_rxq, dp); 109 } 110 111 static void 112 sfc_ef10_rx_qpush(struct sfc_ef10_rxq *rxq) 113 { 114 efx_dword_t dword; 115 116 /* Hardware has alignment restriction for WPTR */ 117 RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0); 118 SFC_ASSERT(RTE_ALIGN(rxq->added, SFC_EF10_RX_WPTR_ALIGN) == rxq->added); 119 120 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, 121 rxq->added & rxq->ptr_mask); 122 123 /* DMA sync to device is not required */ 124 125 /* 126 * rte_write32() has rte_io_wmb() which guarantees that the STORE 127 * operations (i.e. Rx and event descriptor updates) that precede 128 * the rte_io_wmb() call are visible to NIC before the STORE 129 * operations that follow it (i.e. doorbell write). 130 */ 131 rte_write32(dword.ed_u32[0], rxq->doorbell); 132 } 133 134 static void 135 sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq) 136 { 137 const unsigned int ptr_mask = rxq->ptr_mask; 138 const uint32_t buf_size = rxq->buf_size; 139 unsigned int free_space; 140 unsigned int bulks; 141 void *objs[SFC_RX_REFILL_BULK]; 142 unsigned int added = rxq->added; 143 144 free_space = SFC_EF10_RXQ_LIMIT(ptr_mask + 1) - 145 (added - rxq->completed); 146 147 if (free_space < rxq->refill_threshold) 148 return; 149 150 bulks = free_space / RTE_DIM(objs); 151 /* refill_threshold guarantees that bulks is positive */ 152 SFC_ASSERT(bulks > 0); 153 154 do { 155 unsigned int id; 156 unsigned int i; 157 158 if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, 159 RTE_DIM(objs)) < 0)) { 160 struct rte_eth_dev_data *dev_data = 161 rte_eth_devices[rxq->dp.dpq.port_id].data; 162 163 /* 164 * It is hardly a safe way to increment counter 165 * from different contexts, but all PMDs do it. 166 */ 167 dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs); 168 /* Return if we have posted nothing yet */ 169 if (added == rxq->added) 170 return; 171 /* Push posted */ 172 break; 173 } 174 175 for (i = 0, id = added & ptr_mask; 176 i < RTE_DIM(objs); 177 ++i, ++id) { 178 struct rte_mbuf *m = objs[i]; 179 struct sfc_ef10_rx_sw_desc *rxd; 180 rte_iova_t phys_addr; 181 182 SFC_ASSERT((id & ~ptr_mask) == 0); 183 rxd = &rxq->sw_ring[id]; 184 rxd->mbuf = m; 185 186 /* 187 * Avoid writing to mbuf. It is cheaper to do it 188 * when we receive packet and fill in nearby 189 * structure members. 190 */ 191 192 phys_addr = rte_mbuf_data_iova_default(m); 193 EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id], 194 ESF_DZ_RX_KER_BYTE_CNT, buf_size, 195 ESF_DZ_RX_KER_BUF_ADDR, phys_addr); 196 } 197 198 added += RTE_DIM(objs); 199 } while (--bulks > 0); 200 201 SFC_ASSERT(rxq->added != added); 202 rxq->added = added; 203 sfc_ef10_rx_qpush(rxq); 204 } 205 206 static void 207 sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id) 208 { 209 struct rte_mbuf *next_mbuf; 210 211 /* Prefetch next bunch of software descriptors */ 212 if ((next_id % (RTE_CACHE_LINE_SIZE / sizeof(rxq->sw_ring[0]))) == 0) 213 rte_prefetch0(&rxq->sw_ring[next_id]); 214 215 /* 216 * It looks strange to prefetch depending on previous prefetch 217 * data, but measurements show that it is really efficient and 218 * increases packet rate. 219 */ 220 next_mbuf = rxq->sw_ring[next_id].mbuf; 221 if (likely(next_mbuf != NULL)) { 222 /* Prefetch the next mbuf structure */ 223 rte_mbuf_prefetch_part1(next_mbuf); 224 225 /* Prefetch pseudo header of the next packet */ 226 /* data_off is not filled in yet */ 227 /* Yes, data could be not ready yet, but we hope */ 228 rte_prefetch0((uint8_t *)next_mbuf->buf_addr + 229 RTE_PKTMBUF_HEADROOM); 230 } 231 } 232 233 static uint16_t 234 sfc_ef10_rx_prepared(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts, 235 uint16_t nb_pkts) 236 { 237 uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->prepared); 238 unsigned int completed = rxq->completed; 239 unsigned int i; 240 241 rxq->prepared -= n_rx_pkts; 242 rxq->completed = completed + n_rx_pkts; 243 244 for (i = 0; i < n_rx_pkts; ++i, ++completed) 245 rx_pkts[i] = rxq->sw_ring[completed & rxq->ptr_mask].mbuf; 246 247 return n_rx_pkts; 248 } 249 250 static void 251 sfc_ef10_rx_ev_to_offloads(struct sfc_ef10_rxq *rxq, const efx_qword_t rx_ev, 252 struct rte_mbuf *m) 253 { 254 uint32_t l2_ptype = 0; 255 uint32_t l3_ptype = 0; 256 uint32_t l4_ptype = 0; 257 uint64_t ol_flags = 0; 258 259 if (unlikely(EFX_TEST_QWORD_BIT(rx_ev, ESF_DZ_RX_PARSE_INCOMPLETE_LBN))) 260 goto done; 261 262 switch (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_ETH_TAG_CLASS)) { 263 case ESE_DZ_ETH_TAG_CLASS_NONE: 264 l2_ptype = RTE_PTYPE_L2_ETHER; 265 break; 266 case ESE_DZ_ETH_TAG_CLASS_VLAN1: 267 l2_ptype = RTE_PTYPE_L2_ETHER_VLAN; 268 break; 269 case ESE_DZ_ETH_TAG_CLASS_VLAN2: 270 l2_ptype = RTE_PTYPE_L2_ETHER_QINQ; 271 break; 272 default: 273 /* Unexpected Eth tag class */ 274 SFC_ASSERT(false); 275 } 276 277 switch (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_L3_CLASS)) { 278 case ESE_DZ_L3_CLASS_IP4_FRAG: 279 l4_ptype = RTE_PTYPE_L4_FRAG; 280 /* FALLTHROUGH */ 281 case ESE_DZ_L3_CLASS_IP4: 282 l3_ptype = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 283 ol_flags |= PKT_RX_RSS_HASH | 284 ((EFX_TEST_QWORD_BIT(rx_ev, 285 ESF_DZ_RX_IPCKSUM_ERR_LBN)) ? 286 PKT_RX_IP_CKSUM_BAD : PKT_RX_IP_CKSUM_GOOD); 287 break; 288 case ESE_DZ_L3_CLASS_IP6_FRAG: 289 l4_ptype = RTE_PTYPE_L4_FRAG; 290 /* FALLTHROUGH */ 291 case ESE_DZ_L3_CLASS_IP6: 292 l3_ptype = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; 293 ol_flags |= PKT_RX_RSS_HASH; 294 break; 295 case ESE_DZ_L3_CLASS_ARP: 296 /* Override Layer 2 packet type */ 297 l2_ptype = RTE_PTYPE_L2_ETHER_ARP; 298 break; 299 case ESE_DZ_L3_CLASS_UNKNOWN: 300 break; 301 default: 302 /* Unexpected Layer 3 class */ 303 SFC_ASSERT(false); 304 } 305 306 switch (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_L4_CLASS)) { 307 case ESE_DZ_L4_CLASS_TCP: 308 l4_ptype = RTE_PTYPE_L4_TCP; 309 ol_flags |= 310 (EFX_TEST_QWORD_BIT(rx_ev, 311 ESF_DZ_RX_TCPUDP_CKSUM_ERR_LBN)) ? 312 PKT_RX_L4_CKSUM_BAD : PKT_RX_L4_CKSUM_GOOD; 313 break; 314 case ESE_DZ_L4_CLASS_UDP: 315 l4_ptype = RTE_PTYPE_L4_UDP; 316 ol_flags |= 317 (EFX_TEST_QWORD_BIT(rx_ev, 318 ESF_DZ_RX_TCPUDP_CKSUM_ERR_LBN)) ? 319 PKT_RX_L4_CKSUM_BAD : PKT_RX_L4_CKSUM_GOOD; 320 break; 321 case ESE_DZ_L4_CLASS_UNKNOWN: 322 break; 323 default: 324 /* Unexpected Layer 4 class */ 325 SFC_ASSERT(false); 326 } 327 328 /* Remove RSS hash offload flag if RSS is not enabled */ 329 if (~rxq->flags & SFC_EF10_RXQ_RSS_HASH) 330 ol_flags &= ~PKT_RX_RSS_HASH; 331 332 done: 333 m->ol_flags = ol_flags; 334 m->packet_type = l2_ptype | l3_ptype | l4_ptype; 335 } 336 337 static uint16_t 338 sfc_ef10_rx_pseudo_hdr_get_len(const uint8_t *pseudo_hdr) 339 { 340 return rte_le_to_cpu_16(*(const uint16_t *)&pseudo_hdr[8]); 341 } 342 343 static uint32_t 344 sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr) 345 { 346 return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr); 347 } 348 349 static uint16_t 350 sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev, 351 struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 352 { 353 const unsigned int ptr_mask = rxq->ptr_mask; 354 unsigned int completed = rxq->completed; 355 unsigned int ready; 356 struct sfc_ef10_rx_sw_desc *rxd; 357 struct rte_mbuf *m; 358 struct rte_mbuf *m0; 359 uint16_t n_rx_pkts; 360 const uint8_t *pseudo_hdr; 361 uint16_t pkt_len; 362 363 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - completed) & 364 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); 365 SFC_ASSERT(ready > 0); 366 367 if (rx_ev.eq_u64[0] & 368 rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) | 369 (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) { 370 SFC_ASSERT(rxq->prepared == 0); 371 rxq->completed += ready; 372 while (ready-- > 0) { 373 rxd = &rxq->sw_ring[completed++ & ptr_mask]; 374 rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf); 375 } 376 return 0; 377 } 378 379 n_rx_pkts = RTE_MIN(ready, nb_pkts); 380 rxq->prepared = ready - n_rx_pkts; 381 rxq->completed += n_rx_pkts; 382 383 rxd = &rxq->sw_ring[completed++ & ptr_mask]; 384 385 sfc_ef10_rx_prefetch_next(rxq, completed & ptr_mask); 386 387 m = rxd->mbuf; 388 389 *rx_pkts++ = m; 390 391 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data)); 392 m->rearm_data[0] = rxq->rearm_data; 393 394 /* Classify packet based on Rx event */ 395 sfc_ef10_rx_ev_to_offloads(rxq, rx_ev, m); 396 397 /* data_off already moved past pseudo header */ 398 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 399 400 /* 401 * Always get RSS hash from pseudo header to avoid 402 * condition/branching. If it is valid or not depends on 403 * PKT_RX_RSS_HASH in m->ol_flags. 404 */ 405 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); 406 407 if (ready == 1) 408 pkt_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) - 409 rxq->prefix_size; 410 else 411 pkt_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); 412 SFC_ASSERT(pkt_len > 0); 413 rte_pktmbuf_data_len(m) = pkt_len; 414 rte_pktmbuf_pkt_len(m) = pkt_len; 415 416 SFC_ASSERT(m->next == NULL); 417 418 /* Remember mbuf to copy offload flags and packet type from */ 419 m0 = m; 420 for (--ready; ready > 0; --ready) { 421 rxd = &rxq->sw_ring[completed++ & ptr_mask]; 422 423 sfc_ef10_rx_prefetch_next(rxq, completed & ptr_mask); 424 425 m = rxd->mbuf; 426 427 if (ready > rxq->prepared) 428 *rx_pkts++ = m; 429 430 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != 431 sizeof(rxq->rearm_data)); 432 m->rearm_data[0] = rxq->rearm_data; 433 434 /* Event-dependent information is the same */ 435 m->ol_flags = m0->ol_flags; 436 m->packet_type = m0->packet_type; 437 438 /* data_off already moved past pseudo header */ 439 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 440 441 /* 442 * Always get RSS hash from pseudo header to avoid 443 * condition/branching. If it is valid or not depends on 444 * PKT_RX_RSS_HASH in m->ol_flags. 445 */ 446 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); 447 448 pkt_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); 449 SFC_ASSERT(pkt_len > 0); 450 rte_pktmbuf_data_len(m) = pkt_len; 451 rte_pktmbuf_pkt_len(m) = pkt_len; 452 453 SFC_ASSERT(m->next == NULL); 454 } 455 456 return n_rx_pkts; 457 } 458 459 static bool 460 sfc_ef10_rx_get_event(struct sfc_ef10_rxq *rxq, efx_qword_t *rx_ev) 461 { 462 *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->ptr_mask]; 463 464 if (!sfc_ef10_ev_present(*rx_ev)) 465 return false; 466 467 if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) != 468 FSE_AZ_EV_CODE_RX_EV)) { 469 /* 470 * Do not move read_ptr to keep the event for exception 471 * handling by the control path. 472 */ 473 rxq->flags |= SFC_EF10_RXQ_EXCEPTION; 474 sfc_ef10_rx_err(&rxq->dp.dpq, 475 "RxQ exception at EvQ read ptr %#x", 476 rxq->evq_read_ptr); 477 return false; 478 } 479 480 rxq->evq_read_ptr++; 481 return true; 482 } 483 484 static uint16_t 485 sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 486 { 487 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue); 488 unsigned int evq_old_read_ptr; 489 uint16_t n_rx_pkts; 490 efx_qword_t rx_ev; 491 492 if (unlikely(rxq->flags & 493 (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) 494 return 0; 495 496 n_rx_pkts = sfc_ef10_rx_prepared(rxq, rx_pkts, nb_pkts); 497 498 evq_old_read_ptr = rxq->evq_read_ptr; 499 while (n_rx_pkts != nb_pkts && sfc_ef10_rx_get_event(rxq, &rx_ev)) { 500 /* 501 * DROP_EVENT is an internal to the NIC, software should 502 * never see it and, therefore, may ignore it. 503 */ 504 505 n_rx_pkts += sfc_ef10_rx_process_event(rxq, rx_ev, 506 rx_pkts + n_rx_pkts, 507 nb_pkts - n_rx_pkts); 508 } 509 510 sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr, 511 rxq->evq_read_ptr); 512 513 /* It is not a problem if we refill in the case of exception */ 514 sfc_ef10_rx_qrefill(rxq); 515 516 return n_rx_pkts; 517 } 518 519 static const uint32_t * 520 sfc_ef10_supported_ptypes_get(void) 521 { 522 static const uint32_t ef10_native_ptypes[] = { 523 RTE_PTYPE_L2_ETHER, 524 RTE_PTYPE_L2_ETHER_ARP, 525 RTE_PTYPE_L2_ETHER_VLAN, 526 RTE_PTYPE_L2_ETHER_QINQ, 527 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 528 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 529 RTE_PTYPE_L4_FRAG, 530 RTE_PTYPE_L4_TCP, 531 RTE_PTYPE_L4_UDP, 532 RTE_PTYPE_UNKNOWN 533 }; 534 535 return ef10_native_ptypes; 536 } 537 538 static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending; 539 static unsigned int 540 sfc_ef10_rx_qdesc_npending(__rte_unused struct sfc_dp_rxq *dp_rxq) 541 { 542 /* 543 * Correct implementation requires EvQ polling and events 544 * processing (keeping all ready mbufs in prepared). 545 */ 546 return -ENOTSUP; 547 } 548 549 static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status; 550 static int 551 sfc_ef10_rx_qdesc_status(__rte_unused struct sfc_dp_rxq *dp_rxq, 552 __rte_unused uint16_t offset) 553 { 554 return -ENOTSUP; 555 } 556 557 558 static uint64_t 559 sfc_ef10_mk_mbuf_rearm_data(uint16_t port_id, uint16_t prefix_size) 560 { 561 struct rte_mbuf m; 562 563 memset(&m, 0, sizeof(m)); 564 565 rte_mbuf_refcnt_set(&m, 1); 566 m.data_off = RTE_PKTMBUF_HEADROOM + prefix_size; 567 m.nb_segs = 1; 568 m.port = port_id; 569 570 /* rearm_data covers structure members filled in above */ 571 rte_compiler_barrier(); 572 RTE_BUILD_BUG_ON(sizeof(m.rearm_data[0]) != sizeof(uint64_t)); 573 return m.rearm_data[0]; 574 } 575 576 static sfc_dp_rx_qcreate_t sfc_ef10_rx_qcreate; 577 static int 578 sfc_ef10_rx_qcreate(uint16_t port_id, uint16_t queue_id, 579 const struct rte_pci_addr *pci_addr, int socket_id, 580 const struct sfc_dp_rx_qcreate_info *info, 581 struct sfc_dp_rxq **dp_rxqp) 582 { 583 struct sfc_ef10_rxq *rxq; 584 int rc; 585 586 rc = EINVAL; 587 if (info->rxq_entries != info->evq_entries) 588 goto fail_rxq_args; 589 590 rc = ENOMEM; 591 rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq), 592 RTE_CACHE_LINE_SIZE, socket_id); 593 if (rxq == NULL) 594 goto fail_rxq_alloc; 595 596 sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); 597 598 rc = ENOMEM; 599 rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring", 600 info->rxq_entries, 601 sizeof(*rxq->sw_ring), 602 RTE_CACHE_LINE_SIZE, socket_id); 603 if (rxq->sw_ring == NULL) 604 goto fail_desc_alloc; 605 606 rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; 607 if (info->flags & SFC_RXQ_FLAG_RSS_HASH) 608 rxq->flags |= SFC_EF10_RXQ_RSS_HASH; 609 rxq->ptr_mask = info->rxq_entries - 1; 610 rxq->evq_hw_ring = info->evq_hw_ring; 611 rxq->refill_threshold = info->refill_threshold; 612 rxq->rearm_data = 613 sfc_ef10_mk_mbuf_rearm_data(port_id, info->prefix_size); 614 rxq->prefix_size = info->prefix_size; 615 rxq->buf_size = info->buf_size; 616 rxq->refill_mb_pool = info->refill_mb_pool; 617 rxq->rxq_hw_ring = info->rxq_hw_ring; 618 rxq->doorbell = (volatile uint8_t *)info->mem_bar + 619 ER_DZ_RX_DESC_UPD_REG_OFST + 620 info->hw_index * ER_DZ_RX_DESC_UPD_REG_STEP; 621 622 *dp_rxqp = &rxq->dp; 623 return 0; 624 625 fail_desc_alloc: 626 rte_free(rxq); 627 628 fail_rxq_alloc: 629 fail_rxq_args: 630 return rc; 631 } 632 633 static sfc_dp_rx_qdestroy_t sfc_ef10_rx_qdestroy; 634 static void 635 sfc_ef10_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) 636 { 637 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 638 639 rte_free(rxq->sw_ring); 640 rte_free(rxq); 641 } 642 643 static sfc_dp_rx_qstart_t sfc_ef10_rx_qstart; 644 static int 645 sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr) 646 { 647 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 648 649 rxq->prepared = 0; 650 rxq->completed = rxq->added = 0; 651 652 sfc_ef10_rx_qrefill(rxq); 653 654 rxq->evq_read_ptr = evq_read_ptr; 655 656 rxq->flags |= SFC_EF10_RXQ_STARTED; 657 rxq->flags &= ~(SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION); 658 659 return 0; 660 } 661 662 static sfc_dp_rx_qstop_t sfc_ef10_rx_qstop; 663 static void 664 sfc_ef10_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr) 665 { 666 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 667 668 rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; 669 670 *evq_read_ptr = rxq->evq_read_ptr; 671 } 672 673 static sfc_dp_rx_qrx_ev_t sfc_ef10_rx_qrx_ev; 674 static bool 675 sfc_ef10_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id) 676 { 677 __rte_unused struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 678 679 SFC_ASSERT(rxq->flags & SFC_EF10_RXQ_NOT_RUNNING); 680 681 /* 682 * It is safe to ignore Rx event since we free all mbufs on 683 * queue purge anyway. 684 */ 685 686 return false; 687 } 688 689 static sfc_dp_rx_qpurge_t sfc_ef10_rx_qpurge; 690 static void 691 sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq) 692 { 693 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 694 unsigned int i; 695 struct sfc_ef10_rx_sw_desc *rxd; 696 697 for (i = rxq->completed; i != rxq->added; ++i) { 698 rxd = &rxq->sw_ring[i & rxq->ptr_mask]; 699 rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf); 700 rxd->mbuf = NULL; 701 } 702 703 rxq->flags &= ~SFC_EF10_RXQ_STARTED; 704 } 705 706 struct sfc_dp_rx sfc_ef10_rx = { 707 .dp = { 708 .name = SFC_KVARG_DATAPATH_EF10, 709 .type = SFC_DP_RX, 710 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, 711 }, 712 .features = SFC_DP_RX_FEAT_MULTI_PROCESS, 713 .qcreate = sfc_ef10_rx_qcreate, 714 .qdestroy = sfc_ef10_rx_qdestroy, 715 .qstart = sfc_ef10_rx_qstart, 716 .qstop = sfc_ef10_rx_qstop, 717 .qrx_ev = sfc_ef10_rx_qrx_ev, 718 .qpurge = sfc_ef10_rx_qpurge, 719 .supported_ptypes_get = sfc_ef10_supported_ptypes_get, 720 .qdesc_npending = sfc_ef10_rx_qdesc_npending, 721 .qdesc_status = sfc_ef10_rx_qdesc_status, 722 .pkt_burst = sfc_ef10_recv_pkts, 723 }; 724