1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2018-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 /* EF100 native datapath implementation */ 11 12 #include <stdbool.h> 13 14 #include <rte_byteorder.h> 15 #include <rte_mbuf_ptype.h> 16 #include <rte_mbuf.h> 17 #include <rte_io.h> 18 19 #include "efx_types.h" 20 #include "efx_regs_ef100.h" 21 #include "efx.h" 22 23 #include "sfc.h" 24 #include "sfc_debug.h" 25 #include "sfc_flow_tunnel.h" 26 #include "sfc_tweak.h" 27 #include "sfc_dp_rx.h" 28 #include "sfc_kvargs.h" 29 #include "sfc_ef100.h" 30 31 32 #define sfc_ef100_rx_err(_rxq, ...) \ 33 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF100, ERR, &(_rxq)->dp.dpq, __VA_ARGS__) 34 35 #define sfc_ef100_rx_debug(_rxq, ...) \ 36 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF100, DEBUG, &(_rxq)->dp.dpq, \ 37 __VA_ARGS__) 38 39 /** 40 * Maximum number of descriptors/buffers in the Rx ring. 41 * It should guarantee that corresponding event queue never overfill. 42 * EF10 native datapath uses event queue of the same size as Rx queue. 43 * Maximum number of events on datapath can be estimated as number of 44 * Rx queue entries (one event per Rx buffer in the worst case) plus 45 * Rx error and flush events. 46 */ 47 #define SFC_EF100_RXQ_LIMIT(_ndesc) \ 48 ((_ndesc) - 1 /* head must not step on tail */ - \ 49 1 /* Rx error */ - 1 /* flush */) 50 51 /** Invalid user mark value when the mark should be treated as unset */ 52 #define SFC_EF100_USER_MARK_INVALID 0 53 54 struct sfc_ef100_rx_sw_desc { 55 struct rte_mbuf *mbuf; 56 }; 57 58 struct sfc_ef100_rxq { 59 /* Used on data path */ 60 unsigned int flags; 61 #define SFC_EF100_RXQ_STARTED 0x1 62 #define SFC_EF100_RXQ_NOT_RUNNING 0x2 63 #define SFC_EF100_RXQ_EXCEPTION 0x4 64 #define SFC_EF100_RXQ_RSS_HASH 0x10 65 #define SFC_EF100_RXQ_USER_MARK 0x20 66 #define SFC_EF100_RXQ_FLAG_INTR_EN 0x40 67 #define SFC_EF100_RXQ_INGRESS_MPORT 0x80 68 #define SFC_EF100_RXQ_USER_FLAG 0x100 69 unsigned int ptr_mask; 70 unsigned int evq_phase_bit_shift; 71 unsigned int ready_pkts; 72 unsigned int completed; 73 unsigned int evq_read_ptr; 74 unsigned int evq_read_ptr_primed; 75 volatile efx_qword_t *evq_hw_ring; 76 struct sfc_ef100_rx_sw_desc *sw_ring; 77 uint64_t rearm_data; 78 uint16_t buf_size; 79 uint16_t prefix_size; 80 uint32_t user_mark_mask; 81 82 unsigned int evq_hw_index; 83 volatile void *evq_prime; 84 85 /* Used on refill */ 86 unsigned int added; 87 unsigned int max_fill_level; 88 unsigned int refill_threshold; 89 struct rte_mempool *refill_mb_pool; 90 efx_qword_t *rxq_hw_ring; 91 volatile void *doorbell; 92 93 /* Datapath receive queue anchor */ 94 struct sfc_dp_rxq dp; 95 }; 96 97 static inline struct sfc_ef100_rxq * 98 sfc_ef100_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq) 99 { 100 return container_of(dp_rxq, struct sfc_ef100_rxq, dp); 101 } 102 103 static void 104 sfc_ef100_rx_qprime(struct sfc_ef100_rxq *rxq) 105 { 106 sfc_ef100_evq_prime(rxq->evq_prime, rxq->evq_hw_index, 107 rxq->evq_read_ptr & rxq->ptr_mask); 108 rxq->evq_read_ptr_primed = rxq->evq_read_ptr; 109 } 110 111 static inline void 112 sfc_ef100_rx_qpush(struct sfc_ef100_rxq *rxq, unsigned int added) 113 { 114 efx_dword_t dword; 115 116 EFX_POPULATE_DWORD_1(dword, ERF_GZ_RX_RING_PIDX, added & rxq->ptr_mask); 117 118 /* DMA sync to device is not required */ 119 120 /* 121 * rte_write32() has rte_io_wmb() which guarantees that the STORE 122 * operations (i.e. Rx and event descriptor updates) that precede 123 * the rte_io_wmb() call are visible to NIC before the STORE 124 * operations that follow it (i.e. doorbell write). 125 */ 126 rte_write32(dword.ed_u32[0], rxq->doorbell); 127 rxq->dp.dpq.dbells++; 128 129 sfc_ef100_rx_debug(rxq, "RxQ pushed doorbell at pidx %u (added=%u)", 130 EFX_DWORD_FIELD(dword, ERF_GZ_RX_RING_PIDX), 131 added); 132 } 133 134 static void 135 sfc_ef100_rx_qrefill(struct sfc_ef100_rxq *rxq) 136 { 137 const unsigned int ptr_mask = rxq->ptr_mask; 138 unsigned int free_space; 139 unsigned int bulks; 140 void *objs[SFC_RX_REFILL_BULK]; 141 unsigned int added = rxq->added; 142 143 free_space = rxq->max_fill_level - (added - rxq->completed); 144 145 if (free_space < rxq->refill_threshold) 146 return; 147 148 bulks = free_space / RTE_DIM(objs); 149 /* refill_threshold guarantees that bulks is positive */ 150 SFC_ASSERT(bulks > 0); 151 152 do { 153 unsigned int id; 154 unsigned int i; 155 156 if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, 157 RTE_DIM(objs)) < 0)) { 158 struct rte_eth_dev_data *dev_data = 159 rte_eth_devices[rxq->dp.dpq.port_id].data; 160 161 /* 162 * It is hardly a safe way to increment counter 163 * from different contexts, but all PMDs do it. 164 */ 165 dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs); 166 /* Return if we have posted nothing yet */ 167 if (added == rxq->added) 168 return; 169 /* Push posted */ 170 break; 171 } 172 173 for (i = 0, id = added & ptr_mask; 174 i < RTE_DIM(objs); 175 ++i, ++id) { 176 struct rte_mbuf *m = objs[i]; 177 struct sfc_ef100_rx_sw_desc *rxd; 178 rte_iova_t phys_addr; 179 180 __rte_mbuf_raw_sanity_check(m); 181 182 SFC_ASSERT((id & ~ptr_mask) == 0); 183 rxd = &rxq->sw_ring[id]; 184 rxd->mbuf = m; 185 186 /* 187 * Avoid writing to mbuf. It is cheaper to do it 188 * when we receive packet and fill in nearby 189 * structure members. 190 */ 191 192 phys_addr = rte_mbuf_data_iova_default(m); 193 EFX_POPULATE_QWORD_1(rxq->rxq_hw_ring[id], 194 ESF_GZ_RX_BUF_ADDR, phys_addr); 195 } 196 197 added += RTE_DIM(objs); 198 } while (--bulks > 0); 199 200 SFC_ASSERT(rxq->added != added); 201 rxq->added = added; 202 sfc_ef100_rx_qpush(rxq, added); 203 } 204 205 static inline uint64_t 206 sfc_ef100_rx_nt_or_inner_l4_csum(const efx_word_t class) 207 { 208 return EFX_WORD_FIELD(class, 209 ESF_GZ_RX_PREFIX_HCLASS_NT_OR_INNER_L4_CSUM) == 210 ESE_GZ_RH_HCLASS_L4_CSUM_GOOD ? 211 RTE_MBUF_F_RX_L4_CKSUM_GOOD : RTE_MBUF_F_RX_L4_CKSUM_BAD; 212 } 213 214 static inline uint64_t 215 sfc_ef100_rx_tun_outer_l4_csum(const efx_word_t class) 216 { 217 return EFX_WORD_FIELD(class, 218 ESF_GZ_RX_PREFIX_HCLASS_TUN_OUTER_L4_CSUM) == 219 ESE_GZ_RH_HCLASS_L4_CSUM_GOOD ? 220 RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD : RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD; 221 } 222 223 static uint32_t 224 sfc_ef100_rx_class_decode(const efx_word_t class, uint64_t *ol_flags) 225 { 226 uint32_t ptype; 227 bool no_tunnel = false; 228 229 if (unlikely(EFX_WORD_FIELD(class, ESF_GZ_RX_PREFIX_HCLASS_L2_CLASS) != 230 ESE_GZ_RH_HCLASS_L2_CLASS_E2_0123VLAN)) 231 return 0; 232 233 switch (EFX_WORD_FIELD(class, ESF_GZ_RX_PREFIX_HCLASS_L2_N_VLAN)) { 234 case 0: 235 ptype = RTE_PTYPE_L2_ETHER; 236 break; 237 case 1: 238 ptype = RTE_PTYPE_L2_ETHER_VLAN; 239 break; 240 default: 241 ptype = RTE_PTYPE_L2_ETHER_QINQ; 242 break; 243 } 244 245 switch (EFX_WORD_FIELD(class, ESF_GZ_RX_PREFIX_HCLASS_TUNNEL_CLASS)) { 246 case ESE_GZ_RH_HCLASS_TUNNEL_CLASS_NONE: 247 no_tunnel = true; 248 break; 249 case ESE_GZ_RH_HCLASS_TUNNEL_CLASS_VXLAN: 250 ptype |= RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP; 251 *ol_flags |= sfc_ef100_rx_tun_outer_l4_csum(class); 252 break; 253 case ESE_GZ_RH_HCLASS_TUNNEL_CLASS_NVGRE: 254 ptype |= RTE_PTYPE_TUNNEL_NVGRE; 255 break; 256 case ESE_GZ_RH_HCLASS_TUNNEL_CLASS_GENEVE: 257 ptype |= RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP; 258 *ol_flags |= sfc_ef100_rx_tun_outer_l4_csum(class); 259 break; 260 default: 261 /* 262 * Driver does not know the tunnel, but it is 263 * still a tunnel and NT_OR_INNER refer to inner 264 * frame. 265 */ 266 no_tunnel = false; 267 } 268 269 if (no_tunnel) { 270 bool l4_valid = true; 271 272 switch (EFX_WORD_FIELD(class, 273 ESF_GZ_RX_PREFIX_HCLASS_NT_OR_INNER_L3_CLASS)) { 274 case ESE_GZ_RH_HCLASS_L3_CLASS_IP4GOOD: 275 ptype |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 276 *ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 277 break; 278 case ESE_GZ_RH_HCLASS_L3_CLASS_IP4BAD: 279 ptype |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 280 *ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 281 break; 282 case ESE_GZ_RH_HCLASS_L3_CLASS_IP6: 283 ptype |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; 284 break; 285 default: 286 l4_valid = false; 287 } 288 289 if (l4_valid) { 290 switch (EFX_WORD_FIELD(class, 291 ESF_GZ_RX_PREFIX_HCLASS_NT_OR_INNER_L4_CLASS)) { 292 case ESE_GZ_RH_HCLASS_L4_CLASS_TCP: 293 ptype |= RTE_PTYPE_L4_TCP; 294 *ol_flags |= 295 sfc_ef100_rx_nt_or_inner_l4_csum(class); 296 break; 297 case ESE_GZ_RH_HCLASS_L4_CLASS_UDP: 298 ptype |= RTE_PTYPE_L4_UDP; 299 *ol_flags |= 300 sfc_ef100_rx_nt_or_inner_l4_csum(class); 301 break; 302 case ESE_GZ_RH_HCLASS_L4_CLASS_FRAG: 303 ptype |= RTE_PTYPE_L4_FRAG; 304 break; 305 } 306 } 307 } else { 308 bool l4_valid = true; 309 310 switch (EFX_WORD_FIELD(class, 311 ESF_GZ_RX_PREFIX_HCLASS_TUN_OUTER_L3_CLASS)) { 312 case ESE_GZ_RH_HCLASS_L3_CLASS_IP4GOOD: 313 ptype |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 314 break; 315 case ESE_GZ_RH_HCLASS_L3_CLASS_IP4BAD: 316 ptype |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 317 *ol_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 318 break; 319 case ESE_GZ_RH_HCLASS_L3_CLASS_IP6: 320 ptype |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; 321 break; 322 } 323 324 switch (EFX_WORD_FIELD(class, 325 ESF_GZ_RX_PREFIX_HCLASS_NT_OR_INNER_L3_CLASS)) { 326 case ESE_GZ_RH_HCLASS_L3_CLASS_IP4GOOD: 327 ptype |= RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; 328 *ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 329 break; 330 case ESE_GZ_RH_HCLASS_L3_CLASS_IP4BAD: 331 ptype |= RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; 332 *ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 333 break; 334 case ESE_GZ_RH_HCLASS_L3_CLASS_IP6: 335 ptype |= RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; 336 break; 337 default: 338 l4_valid = false; 339 break; 340 } 341 342 if (l4_valid) { 343 switch (EFX_WORD_FIELD(class, 344 ESF_GZ_RX_PREFIX_HCLASS_NT_OR_INNER_L4_CLASS)) { 345 case ESE_GZ_RH_HCLASS_L4_CLASS_TCP: 346 ptype |= RTE_PTYPE_INNER_L4_TCP; 347 *ol_flags |= 348 sfc_ef100_rx_nt_or_inner_l4_csum(class); 349 break; 350 case ESE_GZ_RH_HCLASS_L4_CLASS_UDP: 351 ptype |= RTE_PTYPE_INNER_L4_UDP; 352 *ol_flags |= 353 sfc_ef100_rx_nt_or_inner_l4_csum(class); 354 break; 355 case ESE_GZ_RH_HCLASS_L4_CLASS_FRAG: 356 ptype |= RTE_PTYPE_INNER_L4_FRAG; 357 break; 358 } 359 } 360 } 361 362 return ptype; 363 } 364 365 /* 366 * Below function relies on the following fields in Rx prefix. 367 * Some fields are mandatory, some fields are optional. 368 * See sfc_ef100_rx_qstart() below. 369 */ 370 static const efx_rx_prefix_layout_t sfc_ef100_rx_prefix_layout = { 371 .erpl_fields = { 372 #define SFC_EF100_RX_PREFIX_FIELD(_name, _big_endian) \ 373 EFX_RX_PREFIX_FIELD(_name, ESF_GZ_RX_PREFIX_ ## _name, _big_endian) 374 375 SFC_EF100_RX_PREFIX_FIELD(LENGTH, B_FALSE), 376 SFC_EF100_RX_PREFIX_FIELD(RSS_HASH_VALID, B_FALSE), 377 SFC_EF100_RX_PREFIX_FIELD(CLASS, B_FALSE), 378 EFX_RX_PREFIX_FIELD(INGRESS_MPORT, 379 ESF_GZ_RX_PREFIX_INGRESS_MPORT, B_FALSE), 380 SFC_EF100_RX_PREFIX_FIELD(RSS_HASH, B_FALSE), 381 SFC_EF100_RX_PREFIX_FIELD(USER_FLAG, B_FALSE), 382 SFC_EF100_RX_PREFIX_FIELD(USER_MARK, B_FALSE), 383 384 #undef SFC_EF100_RX_PREFIX_FIELD 385 } 386 }; 387 388 static bool 389 sfc_ef100_rx_prefix_to_offloads(const struct sfc_ef100_rxq *rxq, 390 const efx_xword_t *rx_prefix, 391 struct rte_mbuf *m) 392 { 393 const efx_word_t *class; 394 uint64_t ol_flags = 0; 395 396 RTE_BUILD_BUG_ON(EFX_LOW_BIT(ESF_GZ_RX_PREFIX_CLASS) % CHAR_BIT != 0); 397 RTE_BUILD_BUG_ON(EFX_WIDTH(ESF_GZ_RX_PREFIX_CLASS) % CHAR_BIT != 0); 398 RTE_BUILD_BUG_ON(EFX_WIDTH(ESF_GZ_RX_PREFIX_CLASS) / CHAR_BIT != 399 sizeof(*class)); 400 class = (const efx_word_t *)((const uint8_t *)rx_prefix + 401 EFX_LOW_BIT(ESF_GZ_RX_PREFIX_CLASS) / CHAR_BIT); 402 if (unlikely(EFX_WORD_FIELD(*class, 403 ESF_GZ_RX_PREFIX_HCLASS_L2_STATUS) != 404 ESE_GZ_RH_HCLASS_L2_STATUS_OK)) 405 return false; 406 407 m->packet_type = sfc_ef100_rx_class_decode(*class, &ol_flags); 408 409 if ((rxq->flags & SFC_EF100_RXQ_RSS_HASH) && 410 EFX_TEST_XWORD_BIT(rx_prefix[0], 411 ESF_GZ_RX_PREFIX_RSS_HASH_VALID_LBN)) { 412 ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 413 /* EFX_XWORD_FIELD converts little-endian to CPU */ 414 m->hash.rss = EFX_XWORD_FIELD(rx_prefix[0], 415 ESF_GZ_RX_PREFIX_RSS_HASH); 416 } 417 418 if (rxq->flags & SFC_EF100_RXQ_USER_FLAG) { 419 uint32_t user_flag; 420 421 user_flag = EFX_XWORD_FIELD(rx_prefix[0], 422 ESF_GZ_RX_PREFIX_USER_FLAG); 423 if (user_flag != 0) 424 ol_flags |= RTE_MBUF_F_RX_FDIR; 425 } 426 427 if (rxq->flags & SFC_EF100_RXQ_USER_MARK) { 428 uint8_t tunnel_mark; 429 uint32_t user_mark; 430 uint32_t mark; 431 432 /* EFX_XWORD_FIELD converts little-endian to CPU */ 433 mark = EFX_XWORD_FIELD(rx_prefix[0], 434 ESF_GZ_RX_PREFIX_USER_MARK); 435 436 user_mark = mark & rxq->user_mark_mask; 437 if (user_mark != SFC_EF100_USER_MARK_INVALID) { 438 ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 439 m->hash.fdir.hi = user_mark; 440 } 441 442 tunnel_mark = SFC_FT_GET_TUNNEL_MARK(mark); 443 if (tunnel_mark != SFC_FT_TUNNEL_MARK_INVALID) { 444 sfc_ft_id_t ft_id; 445 446 ft_id = SFC_FT_TUNNEL_MARK_TO_ID(tunnel_mark); 447 448 ol_flags |= sfc_dp_ft_id_valid; 449 *RTE_MBUF_DYNFIELD(m, sfc_dp_ft_id_offset, 450 sfc_ft_id_t *) = ft_id; 451 } 452 } 453 454 if (rxq->flags & SFC_EF100_RXQ_INGRESS_MPORT) { 455 ol_flags |= sfc_dp_mport_override; 456 *RTE_MBUF_DYNFIELD(m, 457 sfc_dp_mport_offset, 458 typeof(&((efx_mport_id_t *)0)->id)) = 459 EFX_XWORD_FIELD(rx_prefix[0], 460 ESF_GZ_RX_PREFIX_INGRESS_MPORT); 461 } 462 463 m->ol_flags = ol_flags; 464 return true; 465 } 466 467 static const uint8_t * 468 sfc_ef100_rx_pkt_prefix(const struct rte_mbuf *m) 469 { 470 return (const uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 471 } 472 473 static struct rte_mbuf * 474 sfc_ef100_rx_next_mbuf(struct sfc_ef100_rxq *rxq) 475 { 476 struct rte_mbuf *m; 477 unsigned int id; 478 479 /* mbuf associated with current Rx descriptor */ 480 m = rxq->sw_ring[rxq->completed++ & rxq->ptr_mask].mbuf; 481 482 /* completed is already moved to the next one */ 483 if (unlikely(rxq->completed == rxq->added)) 484 goto done; 485 486 /* 487 * Prefetch Rx prefix of the next packet. 488 * Current packet is scattered and the next mbuf is its fragment 489 * it simply prefetches some data - no harm since packet rate 490 * should not be high if scatter is used. 491 */ 492 id = rxq->completed & rxq->ptr_mask; 493 rte_prefetch0(sfc_ef100_rx_pkt_prefix(rxq->sw_ring[id].mbuf)); 494 495 if (unlikely(rxq->completed + 1 == rxq->added)) 496 goto done; 497 498 /* 499 * Prefetch mbuf control structure of the next after next Rx 500 * descriptor. 501 */ 502 id = (id == rxq->ptr_mask) ? 0 : (id + 1); 503 rte_mbuf_prefetch_part1(rxq->sw_ring[id].mbuf); 504 505 /* 506 * If the next time we'll need SW Rx descriptor from the next 507 * cache line, try to make sure that we have it in cache. 508 */ 509 if ((id & 0x7) == 0x7) 510 rte_prefetch0(&rxq->sw_ring[(id + 1) & rxq->ptr_mask]); 511 512 done: 513 return m; 514 } 515 516 static struct rte_mbuf ** 517 sfc_ef100_rx_process_ready_pkts(struct sfc_ef100_rxq *rxq, 518 struct rte_mbuf **rx_pkts, 519 struct rte_mbuf ** const rx_pkts_end) 520 { 521 while (rxq->ready_pkts > 0 && rx_pkts != rx_pkts_end) { 522 struct rte_mbuf *pkt; 523 struct rte_mbuf *lastseg; 524 const efx_xword_t *rx_prefix; 525 uint16_t pkt_len; 526 uint16_t seg_len; 527 bool deliver; 528 529 rxq->ready_pkts--; 530 531 pkt = sfc_ef100_rx_next_mbuf(rxq); 532 __rte_mbuf_raw_sanity_check(pkt); 533 534 RTE_BUILD_BUG_ON(sizeof(pkt->rearm_data[0]) != 535 sizeof(rxq->rearm_data)); 536 pkt->rearm_data[0] = rxq->rearm_data; 537 538 /* data_off already moved past Rx prefix */ 539 rx_prefix = (const efx_xword_t *)sfc_ef100_rx_pkt_prefix(pkt); 540 541 pkt_len = EFX_XWORD_FIELD(rx_prefix[0], 542 ESF_GZ_RX_PREFIX_LENGTH); 543 SFC_ASSERT(pkt_len > 0); 544 rte_pktmbuf_pkt_len(pkt) = pkt_len; 545 546 seg_len = RTE_MIN(pkt_len, rxq->buf_size - rxq->prefix_size); 547 rte_pktmbuf_data_len(pkt) = seg_len; 548 549 deliver = sfc_ef100_rx_prefix_to_offloads(rxq, rx_prefix, pkt); 550 551 lastseg = pkt; 552 while ((pkt_len -= seg_len) > 0) { 553 struct rte_mbuf *seg; 554 555 seg = sfc_ef100_rx_next_mbuf(rxq); 556 __rte_mbuf_raw_sanity_check(seg); 557 558 seg->data_off = RTE_PKTMBUF_HEADROOM; 559 560 seg_len = RTE_MIN(pkt_len, rxq->buf_size); 561 rte_pktmbuf_data_len(seg) = seg_len; 562 rte_pktmbuf_pkt_len(seg) = seg_len; 563 564 pkt->nb_segs++; 565 lastseg->next = seg; 566 lastseg = seg; 567 } 568 569 if (likely(deliver)) { 570 *rx_pkts++ = pkt; 571 sfc_pkts_bytes_add(&rxq->dp.dpq.stats, 1, 572 rte_pktmbuf_pkt_len(pkt)); 573 } else { 574 rte_pktmbuf_free(pkt); 575 } 576 } 577 578 return rx_pkts; 579 } 580 581 static bool 582 sfc_ef100_rx_get_event(struct sfc_ef100_rxq *rxq, efx_qword_t *ev) 583 { 584 *ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->ptr_mask]; 585 586 if (!sfc_ef100_ev_present(ev, 587 (rxq->evq_read_ptr >> rxq->evq_phase_bit_shift) & 1)) 588 return false; 589 590 if (unlikely(!sfc_ef100_ev_type_is(ev, ESE_GZ_EF100_EV_RX_PKTS))) { 591 /* 592 * Do not move read_ptr to keep the event for exception 593 * handling by the control path. 594 */ 595 rxq->flags |= SFC_EF100_RXQ_EXCEPTION; 596 sfc_ef100_rx_err(rxq, 597 "RxQ exception at EvQ ptr %u(%#x), event %08x:%08x", 598 rxq->evq_read_ptr, rxq->evq_read_ptr & rxq->ptr_mask, 599 EFX_QWORD_FIELD(*ev, EFX_DWORD_1), 600 EFX_QWORD_FIELD(*ev, EFX_DWORD_0)); 601 return false; 602 } 603 604 sfc_ef100_rx_debug(rxq, "RxQ got event %08x:%08x at %u (%#x)", 605 EFX_QWORD_FIELD(*ev, EFX_DWORD_1), 606 EFX_QWORD_FIELD(*ev, EFX_DWORD_0), 607 rxq->evq_read_ptr, 608 rxq->evq_read_ptr & rxq->ptr_mask); 609 610 rxq->evq_read_ptr++; 611 return true; 612 } 613 614 static uint16_t 615 sfc_ef100_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 616 { 617 struct sfc_ef100_rxq *rxq = sfc_ef100_rxq_by_dp_rxq(rx_queue); 618 struct rte_mbuf ** const rx_pkts_end = &rx_pkts[nb_pkts]; 619 efx_qword_t rx_ev; 620 621 rx_pkts = sfc_ef100_rx_process_ready_pkts(rxq, rx_pkts, rx_pkts_end); 622 623 if (unlikely(rxq->flags & 624 (SFC_EF100_RXQ_NOT_RUNNING | SFC_EF100_RXQ_EXCEPTION))) 625 goto done; 626 627 while (rx_pkts != rx_pkts_end && sfc_ef100_rx_get_event(rxq, &rx_ev)) { 628 rxq->ready_pkts = 629 EFX_QWORD_FIELD(rx_ev, ESF_GZ_EV_RXPKTS_NUM_PKT); 630 rx_pkts = sfc_ef100_rx_process_ready_pkts(rxq, rx_pkts, 631 rx_pkts_end); 632 } 633 634 /* It is not a problem if we refill in the case of exception */ 635 sfc_ef100_rx_qrefill(rxq); 636 637 if ((rxq->flags & SFC_EF100_RXQ_FLAG_INTR_EN) && 638 rxq->evq_read_ptr_primed != rxq->evq_read_ptr) 639 sfc_ef100_rx_qprime(rxq); 640 641 done: 642 return nb_pkts - (rx_pkts_end - rx_pkts); 643 } 644 645 static const uint32_t * 646 sfc_ef100_supported_ptypes_get(__rte_unused uint32_t tunnel_encaps) 647 { 648 static const uint32_t ef100_native_ptypes[] = { 649 RTE_PTYPE_L2_ETHER, 650 RTE_PTYPE_L2_ETHER_VLAN, 651 RTE_PTYPE_L2_ETHER_QINQ, 652 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 653 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 654 RTE_PTYPE_L4_TCP, 655 RTE_PTYPE_L4_UDP, 656 RTE_PTYPE_L4_FRAG, 657 RTE_PTYPE_TUNNEL_VXLAN, 658 RTE_PTYPE_TUNNEL_NVGRE, 659 RTE_PTYPE_TUNNEL_GENEVE, 660 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 661 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 662 RTE_PTYPE_INNER_L4_TCP, 663 RTE_PTYPE_INNER_L4_UDP, 664 RTE_PTYPE_INNER_L4_FRAG, 665 RTE_PTYPE_UNKNOWN 666 }; 667 668 return ef100_native_ptypes; 669 } 670 671 static sfc_dp_rx_qdesc_npending_t sfc_ef100_rx_qdesc_npending; 672 static unsigned int 673 sfc_ef100_rx_qdesc_npending(__rte_unused struct sfc_dp_rxq *dp_rxq) 674 { 675 return 0; 676 } 677 678 static sfc_dp_rx_qdesc_status_t sfc_ef100_rx_qdesc_status; 679 static int 680 sfc_ef100_rx_qdesc_status(__rte_unused struct sfc_dp_rxq *dp_rxq, 681 __rte_unused uint16_t offset) 682 { 683 return -ENOTSUP; 684 } 685 686 687 static sfc_dp_rx_get_dev_info_t sfc_ef100_rx_get_dev_info; 688 static void 689 sfc_ef100_rx_get_dev_info(struct rte_eth_dev_info *dev_info) 690 { 691 /* 692 * Number of descriptors just defines maximum number of pushed 693 * descriptors (fill level). 694 */ 695 dev_info->rx_desc_lim.nb_min = SFC_RX_REFILL_BULK; 696 dev_info->rx_desc_lim.nb_align = SFC_RX_REFILL_BULK; 697 } 698 699 700 static sfc_dp_rx_qsize_up_rings_t sfc_ef100_rx_qsize_up_rings; 701 static int 702 sfc_ef100_rx_qsize_up_rings(uint16_t nb_rx_desc, 703 struct sfc_dp_rx_hw_limits *limits, 704 __rte_unused struct rte_mempool *mb_pool, 705 unsigned int *rxq_entries, 706 unsigned int *evq_entries, 707 unsigned int *rxq_max_fill_level) 708 { 709 /* 710 * rte_ethdev API guarantees that the number meets min, max and 711 * alignment requirements. 712 */ 713 if (nb_rx_desc <= limits->rxq_min_entries) 714 *rxq_entries = limits->rxq_min_entries; 715 else 716 *rxq_entries = rte_align32pow2(nb_rx_desc); 717 718 *evq_entries = *rxq_entries; 719 720 *rxq_max_fill_level = RTE_MIN(nb_rx_desc, 721 SFC_EF100_RXQ_LIMIT(*evq_entries)); 722 return 0; 723 } 724 725 726 static uint64_t 727 sfc_ef100_mk_mbuf_rearm_data(uint16_t port_id, uint16_t prefix_size) 728 { 729 struct rte_mbuf m; 730 731 memset(&m, 0, sizeof(m)); 732 733 rte_mbuf_refcnt_set(&m, 1); 734 m.data_off = RTE_PKTMBUF_HEADROOM + prefix_size; 735 m.nb_segs = 1; 736 m.port = port_id; 737 738 /* rearm_data covers structure members filled in above */ 739 rte_compiler_barrier(); 740 RTE_BUILD_BUG_ON(sizeof(m.rearm_data[0]) != sizeof(uint64_t)); 741 return m.rearm_data[0]; 742 } 743 744 static sfc_dp_rx_qcreate_t sfc_ef100_rx_qcreate; 745 static int 746 sfc_ef100_rx_qcreate(uint16_t port_id, uint16_t queue_id, 747 const struct rte_pci_addr *pci_addr, int socket_id, 748 const struct sfc_dp_rx_qcreate_info *info, 749 struct sfc_dp_rxq **dp_rxqp) 750 { 751 struct sfc_ef100_rxq *rxq; 752 int rc; 753 754 rc = EINVAL; 755 if (info->rxq_entries != info->evq_entries) 756 goto fail_rxq_args; 757 758 rc = ENOMEM; 759 rxq = rte_zmalloc_socket("sfc-ef100-rxq", sizeof(*rxq), 760 RTE_CACHE_LINE_SIZE, socket_id); 761 if (rxq == NULL) 762 goto fail_rxq_alloc; 763 764 sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); 765 766 rc = ENOMEM; 767 rxq->sw_ring = rte_calloc_socket("sfc-ef100-rxq-sw_ring", 768 info->rxq_entries, 769 sizeof(*rxq->sw_ring), 770 RTE_CACHE_LINE_SIZE, socket_id); 771 if (rxq->sw_ring == NULL) 772 goto fail_desc_alloc; 773 774 rxq->flags |= SFC_EF100_RXQ_NOT_RUNNING; 775 rxq->ptr_mask = info->rxq_entries - 1; 776 rxq->evq_phase_bit_shift = rte_bsf32(info->evq_entries); 777 rxq->evq_hw_ring = info->evq_hw_ring; 778 rxq->max_fill_level = info->max_fill_level; 779 rxq->refill_threshold = info->refill_threshold; 780 rxq->prefix_size = info->prefix_size; 781 782 SFC_ASSERT(info->user_mark_mask != 0); 783 rxq->user_mark_mask = info->user_mark_mask; 784 785 rxq->buf_size = info->buf_size; 786 rxq->refill_mb_pool = info->refill_mb_pool; 787 rxq->rxq_hw_ring = info->rxq_hw_ring; 788 rxq->doorbell = (volatile uint8_t *)info->mem_bar + 789 ER_GZ_RX_RING_DOORBELL_OFST + 790 (info->hw_index << info->vi_window_shift); 791 792 rxq->evq_hw_index = info->evq_hw_index; 793 rxq->evq_prime = (volatile uint8_t *)info->mem_bar + 794 info->fcw_offset + 795 ER_GZ_EVQ_INT_PRIME_OFST; 796 797 sfc_ef100_rx_debug(rxq, "RxQ doorbell is %p", rxq->doorbell); 798 799 *dp_rxqp = &rxq->dp; 800 return 0; 801 802 fail_desc_alloc: 803 rte_free(rxq); 804 805 fail_rxq_alloc: 806 fail_rxq_args: 807 return rc; 808 } 809 810 static sfc_dp_rx_qdestroy_t sfc_ef100_rx_qdestroy; 811 static void 812 sfc_ef100_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) 813 { 814 struct sfc_ef100_rxq *rxq = sfc_ef100_rxq_by_dp_rxq(dp_rxq); 815 816 rte_free(rxq->sw_ring); 817 rte_free(rxq); 818 } 819 820 static sfc_dp_rx_qstart_t sfc_ef100_rx_qstart; 821 static int 822 sfc_ef100_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr, 823 const efx_rx_prefix_layout_t *pinfo) 824 { 825 struct sfc_ef100_rxq *rxq = sfc_ef100_rxq_by_dp_rxq(dp_rxq); 826 uint32_t unsup_rx_prefix_fields; 827 828 SFC_ASSERT(rxq->completed == 0); 829 SFC_ASSERT(rxq->added == 0); 830 831 /* Prefix must fit into reserved Rx buffer space */ 832 if (pinfo->erpl_length > rxq->prefix_size) 833 return ENOTSUP; 834 835 unsup_rx_prefix_fields = 836 efx_rx_prefix_layout_check(pinfo, &sfc_ef100_rx_prefix_layout); 837 838 /* LENGTH and CLASS filds must always be present */ 839 if ((unsup_rx_prefix_fields & 840 ((1U << EFX_RX_PREFIX_FIELD_LENGTH) | 841 (1U << EFX_RX_PREFIX_FIELD_CLASS))) != 0) 842 return ENOTSUP; 843 844 if ((unsup_rx_prefix_fields & 845 ((1U << EFX_RX_PREFIX_FIELD_RSS_HASH_VALID) | 846 (1U << EFX_RX_PREFIX_FIELD_RSS_HASH))) == 0) 847 rxq->flags |= SFC_EF100_RXQ_RSS_HASH; 848 else 849 rxq->flags &= ~SFC_EF100_RXQ_RSS_HASH; 850 851 if ((unsup_rx_prefix_fields & 852 (1U << EFX_RX_PREFIX_FIELD_USER_FLAG)) == 0) 853 rxq->flags |= SFC_EF100_RXQ_USER_FLAG; 854 else 855 rxq->flags &= ~SFC_EF100_RXQ_USER_FLAG; 856 857 if ((unsup_rx_prefix_fields & 858 (1U << EFX_RX_PREFIX_FIELD_USER_MARK)) == 0) 859 rxq->flags |= SFC_EF100_RXQ_USER_MARK; 860 else 861 rxq->flags &= ~SFC_EF100_RXQ_USER_MARK; 862 863 if ((unsup_rx_prefix_fields & 864 (1U << EFX_RX_PREFIX_FIELD_INGRESS_MPORT)) == 0) 865 rxq->flags |= SFC_EF100_RXQ_INGRESS_MPORT; 866 else 867 rxq->flags &= ~SFC_EF100_RXQ_INGRESS_MPORT; 868 869 rxq->prefix_size = pinfo->erpl_length; 870 rxq->rearm_data = sfc_ef100_mk_mbuf_rearm_data(rxq->dp.dpq.port_id, 871 rxq->prefix_size); 872 873 sfc_ef100_rx_qrefill(rxq); 874 875 rxq->evq_read_ptr = evq_read_ptr; 876 877 rxq->flags |= SFC_EF100_RXQ_STARTED; 878 rxq->flags &= ~(SFC_EF100_RXQ_NOT_RUNNING | SFC_EF100_RXQ_EXCEPTION); 879 880 if (rxq->flags & SFC_EF100_RXQ_FLAG_INTR_EN) 881 sfc_ef100_rx_qprime(rxq); 882 883 return 0; 884 } 885 886 static sfc_dp_rx_qstop_t sfc_ef100_rx_qstop; 887 static void 888 sfc_ef100_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr) 889 { 890 struct sfc_ef100_rxq *rxq = sfc_ef100_rxq_by_dp_rxq(dp_rxq); 891 892 rxq->flags |= SFC_EF100_RXQ_NOT_RUNNING; 893 894 *evq_read_ptr = rxq->evq_read_ptr; 895 } 896 897 static sfc_dp_rx_qrx_ev_t sfc_ef100_rx_qrx_ev; 898 static bool 899 sfc_ef100_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id) 900 { 901 __rte_unused struct sfc_ef100_rxq *rxq = sfc_ef100_rxq_by_dp_rxq(dp_rxq); 902 903 SFC_ASSERT(rxq->flags & SFC_EF100_RXQ_NOT_RUNNING); 904 905 /* 906 * It is safe to ignore Rx event since we free all mbufs on 907 * queue purge anyway. 908 */ 909 910 return false; 911 } 912 913 static sfc_dp_rx_qpurge_t sfc_ef100_rx_qpurge; 914 static void 915 sfc_ef100_rx_qpurge(struct sfc_dp_rxq *dp_rxq) 916 { 917 struct sfc_ef100_rxq *rxq = sfc_ef100_rxq_by_dp_rxq(dp_rxq); 918 unsigned int i; 919 struct sfc_ef100_rx_sw_desc *rxd; 920 921 for (i = rxq->completed; i != rxq->added; ++i) { 922 rxd = &rxq->sw_ring[i & rxq->ptr_mask]; 923 rte_mbuf_raw_free(rxd->mbuf); 924 rxd->mbuf = NULL; 925 } 926 927 rxq->completed = rxq->added = 0; 928 rxq->ready_pkts = 0; 929 930 rxq->flags &= ~SFC_EF100_RXQ_STARTED; 931 } 932 933 static sfc_dp_rx_intr_enable_t sfc_ef100_rx_intr_enable; 934 static int 935 sfc_ef100_rx_intr_enable(struct sfc_dp_rxq *dp_rxq) 936 { 937 struct sfc_ef100_rxq *rxq = sfc_ef100_rxq_by_dp_rxq(dp_rxq); 938 939 rxq->flags |= SFC_EF100_RXQ_FLAG_INTR_EN; 940 if (rxq->flags & SFC_EF100_RXQ_STARTED) 941 sfc_ef100_rx_qprime(rxq); 942 return 0; 943 } 944 945 static sfc_dp_rx_intr_disable_t sfc_ef100_rx_intr_disable; 946 static int 947 sfc_ef100_rx_intr_disable(struct sfc_dp_rxq *dp_rxq) 948 { 949 struct sfc_ef100_rxq *rxq = sfc_ef100_rxq_by_dp_rxq(dp_rxq); 950 951 /* Cannot disarm, just disable rearm */ 952 rxq->flags &= ~SFC_EF100_RXQ_FLAG_INTR_EN; 953 return 0; 954 } 955 956 static sfc_dp_rx_get_pushed_t sfc_ef100_rx_get_pushed; 957 static unsigned int 958 sfc_ef100_rx_get_pushed(struct sfc_dp_rxq *dp_rxq) 959 { 960 struct sfc_ef100_rxq *rxq = sfc_ef100_rxq_by_dp_rxq(dp_rxq); 961 962 /* 963 * The datapath keeps track only of added descriptors, since 964 * the number of pushed descriptors always equals the number 965 * of added descriptors due to enforced alignment. 966 */ 967 return rxq->added; 968 } 969 970 struct sfc_dp_rx sfc_ef100_rx = { 971 .dp = { 972 .name = SFC_KVARG_DATAPATH_EF100, 973 .type = SFC_DP_RX, 974 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF100, 975 }, 976 .features = SFC_DP_RX_FEAT_MULTI_PROCESS | 977 SFC_DP_RX_FEAT_FLOW_FLAG | 978 SFC_DP_RX_FEAT_FLOW_MARK | 979 SFC_DP_RX_FEAT_INTR | 980 SFC_DP_RX_FEAT_STATS, 981 .dev_offload_capa = 0, 982 .queue_offload_capa = RTE_ETH_RX_OFFLOAD_CHECKSUM | 983 RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM | 984 RTE_ETH_RX_OFFLOAD_OUTER_UDP_CKSUM | 985 RTE_ETH_RX_OFFLOAD_SCATTER | 986 RTE_ETH_RX_OFFLOAD_RSS_HASH, 987 .get_dev_info = sfc_ef100_rx_get_dev_info, 988 .qsize_up_rings = sfc_ef100_rx_qsize_up_rings, 989 .qcreate = sfc_ef100_rx_qcreate, 990 .qdestroy = sfc_ef100_rx_qdestroy, 991 .qstart = sfc_ef100_rx_qstart, 992 .qstop = sfc_ef100_rx_qstop, 993 .qrx_ev = sfc_ef100_rx_qrx_ev, 994 .qpurge = sfc_ef100_rx_qpurge, 995 .supported_ptypes_get = sfc_ef100_supported_ptypes_get, 996 .qdesc_npending = sfc_ef100_rx_qdesc_npending, 997 .qdesc_status = sfc_ef100_rx_qdesc_status, 998 .intr_enable = sfc_ef100_rx_intr_enable, 999 .intr_disable = sfc_ef100_rx_intr_disable, 1000 .get_pushed = sfc_ef100_rx_get_pushed, 1001 .pkt_burst = sfc_ef100_recv_pkts, 1002 }; 1003