1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
3 */
4
5 #include <rte_vect.h>
6
7 #include "otx2_ethdev.h"
8 #include "otx2_rx.h"
9
10 #define NIX_DESCS_PER_LOOP 4
11 #define CQE_CAST(x) ((struct nix_cqe_hdr_s *)(x))
12 #define CQE_SZ(x) ((x) * NIX_CQ_ENTRY_SZ)
13
14 static inline uint16_t
nix_rx_nb_pkts(struct otx2_eth_rxq * rxq,const uint64_t wdata,const uint16_t pkts,const uint32_t qmask)15 nix_rx_nb_pkts(struct otx2_eth_rxq *rxq, const uint64_t wdata,
16 const uint16_t pkts, const uint32_t qmask)
17 {
18 uint32_t available = rxq->available;
19
20 /* Update the available count if cached value is not enough */
21 if (unlikely(available < pkts)) {
22 uint64_t reg, head, tail;
23
24 /* Use LDADDA version to avoid reorder */
25 reg = otx2_atomic64_add_sync(wdata, rxq->cq_status);
26 /* CQ_OP_STATUS operation error */
27 if (reg & BIT_ULL(CQ_OP_STAT_OP_ERR) ||
28 reg & BIT_ULL(CQ_OP_STAT_CQ_ERR))
29 return 0;
30
31 tail = reg & 0xFFFFF;
32 head = (reg >> 20) & 0xFFFFF;
33 if (tail < head)
34 available = tail - head + qmask + 1;
35 else
36 available = tail - head;
37
38 rxq->available = available;
39 }
40
41 return RTE_MIN(pkts, available);
42 }
43
44 static __rte_always_inline uint16_t
nix_recv_pkts(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t pkts,const uint16_t flags)45 nix_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
46 uint16_t pkts, const uint16_t flags)
47 {
48 struct otx2_eth_rxq *rxq = rx_queue;
49 const uint64_t mbuf_init = rxq->mbuf_initializer;
50 const void *lookup_mem = rxq->lookup_mem;
51 const uint64_t data_off = rxq->data_off;
52 const uintptr_t desc = rxq->desc;
53 const uint64_t wdata = rxq->wdata;
54 const uint32_t qmask = rxq->qmask;
55 uint16_t packets = 0, nb_pkts;
56 uint32_t head = rxq->head;
57 struct nix_cqe_hdr_s *cq;
58 struct rte_mbuf *mbuf;
59
60 nb_pkts = nix_rx_nb_pkts(rxq, wdata, pkts, qmask);
61
62 while (packets < nb_pkts) {
63 /* Prefetch N desc ahead */
64 rte_prefetch_non_temporal((void *)(desc +
65 (CQE_SZ((head + 2) & qmask))));
66 cq = (struct nix_cqe_hdr_s *)(desc + CQE_SZ(head));
67
68 mbuf = nix_get_mbuf_from_cqe(cq, data_off);
69
70 otx2_nix_cqe_to_mbuf(cq, cq->tag, mbuf, lookup_mem, mbuf_init,
71 flags);
72 otx2_nix_mbuf_to_tstamp(mbuf, rxq->tstamp, flags,
73 (uint64_t *)((uint8_t *)mbuf + data_off));
74 rx_pkts[packets++] = mbuf;
75 otx2_prefetch_store_keep(mbuf);
76 head++;
77 head &= qmask;
78 }
79
80 rxq->head = head;
81 rxq->available -= nb_pkts;
82
83 /* Free all the CQs that we've processed */
84 otx2_write64((wdata | nb_pkts), rxq->cq_door);
85
86 return nb_pkts;
87 }
88
89 #if defined(RTE_ARCH_ARM64)
90
91 static __rte_always_inline uint64_t
nix_vlan_update(const uint64_t w2,uint64_t ol_flags,uint8x16_t * f)92 nix_vlan_update(const uint64_t w2, uint64_t ol_flags, uint8x16_t *f)
93 {
94 if (w2 & BIT_ULL(21) /* vtag0_gone */) {
95 ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
96 *f = vsetq_lane_u16((uint16_t)(w2 >> 32), *f, 5);
97 }
98
99 return ol_flags;
100 }
101
102 static __rte_always_inline uint64_t
nix_qinq_update(const uint64_t w2,uint64_t ol_flags,struct rte_mbuf * mbuf)103 nix_qinq_update(const uint64_t w2, uint64_t ol_flags, struct rte_mbuf *mbuf)
104 {
105 if (w2 & BIT_ULL(23) /* vtag1_gone */) {
106 ol_flags |= PKT_RX_QINQ | PKT_RX_QINQ_STRIPPED;
107 mbuf->vlan_tci_outer = (uint16_t)(w2 >> 48);
108 }
109
110 return ol_flags;
111 }
112
113 static __rte_always_inline uint16_t
nix_recv_pkts_vector(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t pkts,const uint16_t flags)114 nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts,
115 uint16_t pkts, const uint16_t flags)
116 {
117 struct otx2_eth_rxq *rxq = rx_queue; uint16_t packets = 0;
118 uint64x2_t cq0_w8, cq1_w8, cq2_w8, cq3_w8, mbuf01, mbuf23;
119 const uint64_t mbuf_initializer = rxq->mbuf_initializer;
120 const uint64x2_t data_off = vdupq_n_u64(rxq->data_off);
121 uint64_t ol_flags0, ol_flags1, ol_flags2, ol_flags3;
122 uint64x2_t rearm0 = vdupq_n_u64(mbuf_initializer);
123 uint64x2_t rearm1 = vdupq_n_u64(mbuf_initializer);
124 uint64x2_t rearm2 = vdupq_n_u64(mbuf_initializer);
125 uint64x2_t rearm3 = vdupq_n_u64(mbuf_initializer);
126 struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3;
127 const uint16_t *lookup_mem = rxq->lookup_mem;
128 const uint32_t qmask = rxq->qmask;
129 const uint64_t wdata = rxq->wdata;
130 const uintptr_t desc = rxq->desc;
131 uint8x16_t f0, f1, f2, f3;
132 uint32_t head = rxq->head;
133 uint16_t pkts_left;
134
135 pkts = nix_rx_nb_pkts(rxq, wdata, pkts, qmask);
136 pkts_left = pkts & (NIX_DESCS_PER_LOOP - 1);
137
138 /* Packets has to be floor-aligned to NIX_DESCS_PER_LOOP */
139 pkts = RTE_ALIGN_FLOOR(pkts, NIX_DESCS_PER_LOOP);
140
141 while (packets < pkts) {
142 /* Exit loop if head is about to wrap and become unaligned */
143 if (((head + NIX_DESCS_PER_LOOP - 1) & qmask) <
144 NIX_DESCS_PER_LOOP) {
145 pkts_left += (pkts - packets);
146 break;
147 }
148
149 const uintptr_t cq0 = desc + CQE_SZ(head);
150
151 /* Prefetch N desc ahead */
152 rte_prefetch_non_temporal((void *)(cq0 + CQE_SZ(8)));
153 rte_prefetch_non_temporal((void *)(cq0 + CQE_SZ(9)));
154 rte_prefetch_non_temporal((void *)(cq0 + CQE_SZ(10)));
155 rte_prefetch_non_temporal((void *)(cq0 + CQE_SZ(11)));
156
157 /* Get NIX_RX_SG_S for size and buffer pointer */
158 cq0_w8 = vld1q_u64((uint64_t *)(cq0 + CQE_SZ(0) + 64));
159 cq1_w8 = vld1q_u64((uint64_t *)(cq0 + CQE_SZ(1) + 64));
160 cq2_w8 = vld1q_u64((uint64_t *)(cq0 + CQE_SZ(2) + 64));
161 cq3_w8 = vld1q_u64((uint64_t *)(cq0 + CQE_SZ(3) + 64));
162
163 /* Extract mbuf from NIX_RX_SG_S */
164 mbuf01 = vzip2q_u64(cq0_w8, cq1_w8);
165 mbuf23 = vzip2q_u64(cq2_w8, cq3_w8);
166 mbuf01 = vqsubq_u64(mbuf01, data_off);
167 mbuf23 = vqsubq_u64(mbuf23, data_off);
168
169 /* Move mbufs to scalar registers for future use */
170 mbuf0 = (struct rte_mbuf *)vgetq_lane_u64(mbuf01, 0);
171 mbuf1 = (struct rte_mbuf *)vgetq_lane_u64(mbuf01, 1);
172 mbuf2 = (struct rte_mbuf *)vgetq_lane_u64(mbuf23, 0);
173 mbuf3 = (struct rte_mbuf *)vgetq_lane_u64(mbuf23, 1);
174
175 /* Mask to get packet len from NIX_RX_SG_S */
176 const uint8x16_t shuf_msk = {
177 0xFF, 0xFF, /* pkt_type set as unknown */
178 0xFF, 0xFF, /* pkt_type set as unknown */
179 0, 1, /* octet 1~0, low 16 bits pkt_len */
180 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
181 0, 1, /* octet 1~0, 16 bits data_len */
182 0xFF, 0xFF,
183 0xFF, 0xFF, 0xFF, 0xFF
184 };
185
186 /* Form the rx_descriptor_fields1 with pkt_len and data_len */
187 f0 = vqtbl1q_u8(cq0_w8, shuf_msk);
188 f1 = vqtbl1q_u8(cq1_w8, shuf_msk);
189 f2 = vqtbl1q_u8(cq2_w8, shuf_msk);
190 f3 = vqtbl1q_u8(cq3_w8, shuf_msk);
191
192 /* Load CQE word0 and word 1 */
193 uint64_t cq0_w0 = ((uint64_t *)(cq0 + CQE_SZ(0)))[0];
194 uint64_t cq0_w1 = ((uint64_t *)(cq0 + CQE_SZ(0)))[1];
195 uint64_t cq1_w0 = ((uint64_t *)(cq0 + CQE_SZ(1)))[0];
196 uint64_t cq1_w1 = ((uint64_t *)(cq0 + CQE_SZ(1)))[1];
197 uint64_t cq2_w0 = ((uint64_t *)(cq0 + CQE_SZ(2)))[0];
198 uint64_t cq2_w1 = ((uint64_t *)(cq0 + CQE_SZ(2)))[1];
199 uint64_t cq3_w0 = ((uint64_t *)(cq0 + CQE_SZ(3)))[0];
200 uint64_t cq3_w1 = ((uint64_t *)(cq0 + CQE_SZ(3)))[1];
201
202 if (flags & NIX_RX_OFFLOAD_RSS_F) {
203 /* Fill rss in the rx_descriptor_fields1 */
204 f0 = vsetq_lane_u32(cq0_w0, f0, 3);
205 f1 = vsetq_lane_u32(cq1_w0, f1, 3);
206 f2 = vsetq_lane_u32(cq2_w0, f2, 3);
207 f3 = vsetq_lane_u32(cq3_w0, f3, 3);
208 ol_flags0 = PKT_RX_RSS_HASH;
209 ol_flags1 = PKT_RX_RSS_HASH;
210 ol_flags2 = PKT_RX_RSS_HASH;
211 ol_flags3 = PKT_RX_RSS_HASH;
212 } else {
213 ol_flags0 = 0; ol_flags1 = 0;
214 ol_flags2 = 0; ol_flags3 = 0;
215 }
216
217 if (flags & NIX_RX_OFFLOAD_PTYPE_F) {
218 /* Fill packet_type in the rx_descriptor_fields1 */
219 f0 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq0_w1),
220 f0, 0);
221 f1 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq1_w1),
222 f1, 0);
223 f2 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq2_w1),
224 f2, 0);
225 f3 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq3_w1),
226 f3, 0);
227 }
228
229 if (flags & NIX_RX_OFFLOAD_CHECKSUM_F) {
230 ol_flags0 |= nix_rx_olflags_get(lookup_mem, cq0_w1);
231 ol_flags1 |= nix_rx_olflags_get(lookup_mem, cq1_w1);
232 ol_flags2 |= nix_rx_olflags_get(lookup_mem, cq2_w1);
233 ol_flags3 |= nix_rx_olflags_get(lookup_mem, cq3_w1);
234 }
235
236 if (flags & NIX_RX_OFFLOAD_VLAN_STRIP_F) {
237 uint64_t cq0_w2 = *(uint64_t *)(cq0 + CQE_SZ(0) + 16);
238 uint64_t cq1_w2 = *(uint64_t *)(cq0 + CQE_SZ(1) + 16);
239 uint64_t cq2_w2 = *(uint64_t *)(cq0 + CQE_SZ(2) + 16);
240 uint64_t cq3_w2 = *(uint64_t *)(cq0 + CQE_SZ(3) + 16);
241
242 ol_flags0 = nix_vlan_update(cq0_w2, ol_flags0, &f0);
243 ol_flags1 = nix_vlan_update(cq1_w2, ol_flags1, &f1);
244 ol_flags2 = nix_vlan_update(cq2_w2, ol_flags2, &f2);
245 ol_flags3 = nix_vlan_update(cq3_w2, ol_flags3, &f3);
246
247 ol_flags0 = nix_qinq_update(cq0_w2, ol_flags0, mbuf0);
248 ol_flags1 = nix_qinq_update(cq1_w2, ol_flags1, mbuf1);
249 ol_flags2 = nix_qinq_update(cq2_w2, ol_flags2, mbuf2);
250 ol_flags3 = nix_qinq_update(cq3_w2, ol_flags3, mbuf3);
251 }
252
253 if (flags & NIX_RX_OFFLOAD_MARK_UPDATE_F) {
254 ol_flags0 = nix_update_match_id(*(uint16_t *)
255 (cq0 + CQE_SZ(0) + 38), ol_flags0, mbuf0);
256 ol_flags1 = nix_update_match_id(*(uint16_t *)
257 (cq0 + CQE_SZ(1) + 38), ol_flags1, mbuf1);
258 ol_flags2 = nix_update_match_id(*(uint16_t *)
259 (cq0 + CQE_SZ(2) + 38), ol_flags2, mbuf2);
260 ol_flags3 = nix_update_match_id(*(uint16_t *)
261 (cq0 + CQE_SZ(3) + 38), ol_flags3, mbuf3);
262 }
263
264 /* Form rearm_data with ol_flags */
265 rearm0 = vsetq_lane_u64(ol_flags0, rearm0, 1);
266 rearm1 = vsetq_lane_u64(ol_flags1, rearm1, 1);
267 rearm2 = vsetq_lane_u64(ol_flags2, rearm2, 1);
268 rearm3 = vsetq_lane_u64(ol_flags3, rearm3, 1);
269
270 /* Update rx_descriptor_fields1 */
271 vst1q_u64((uint64_t *)mbuf0->rx_descriptor_fields1, f0);
272 vst1q_u64((uint64_t *)mbuf1->rx_descriptor_fields1, f1);
273 vst1q_u64((uint64_t *)mbuf2->rx_descriptor_fields1, f2);
274 vst1q_u64((uint64_t *)mbuf3->rx_descriptor_fields1, f3);
275
276 /* Update rearm_data */
277 vst1q_u64((uint64_t *)mbuf0->rearm_data, rearm0);
278 vst1q_u64((uint64_t *)mbuf1->rearm_data, rearm1);
279 vst1q_u64((uint64_t *)mbuf2->rearm_data, rearm2);
280 vst1q_u64((uint64_t *)mbuf3->rearm_data, rearm3);
281
282 /* Store the mbufs to rx_pkts */
283 vst1q_u64((uint64_t *)&rx_pkts[packets], mbuf01);
284 vst1q_u64((uint64_t *)&rx_pkts[packets + 2], mbuf23);
285
286 /* Prefetch mbufs */
287 otx2_prefetch_store_keep(mbuf0);
288 otx2_prefetch_store_keep(mbuf1);
289 otx2_prefetch_store_keep(mbuf2);
290 otx2_prefetch_store_keep(mbuf3);
291
292 /* Mark mempool obj as "get" as it is alloc'ed by NIX */
293 __mempool_check_cookies(mbuf0->pool, (void **)&mbuf0, 1, 1);
294 __mempool_check_cookies(mbuf1->pool, (void **)&mbuf1, 1, 1);
295 __mempool_check_cookies(mbuf2->pool, (void **)&mbuf2, 1, 1);
296 __mempool_check_cookies(mbuf3->pool, (void **)&mbuf3, 1, 1);
297
298 /* Advance head pointer and packets */
299 head += NIX_DESCS_PER_LOOP; head &= qmask;
300 packets += NIX_DESCS_PER_LOOP;
301 }
302
303 rxq->head = head;
304 rxq->available -= packets;
305
306 rte_io_wmb();
307 /* Free all the CQs that we've processed */
308 otx2_write64((rxq->wdata | packets), rxq->cq_door);
309
310 if (unlikely(pkts_left))
311 packets += nix_recv_pkts(rx_queue, &rx_pkts[packets],
312 pkts_left, flags);
313
314 return packets;
315 }
316
317 #else
318
319 static inline uint16_t
nix_recv_pkts_vector(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t pkts,const uint16_t flags)320 nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts,
321 uint16_t pkts, const uint16_t flags)
322 {
323 RTE_SET_USED(rx_queue);
324 RTE_SET_USED(rx_pkts);
325 RTE_SET_USED(pkts);
326 RTE_SET_USED(flags);
327
328 return 0;
329 }
330
331 #endif
332
333 #define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
334 static uint16_t __rte_noinline __rte_hot \
335 otx2_nix_recv_pkts_ ## name(void *rx_queue, \
336 struct rte_mbuf **rx_pkts, uint16_t pkts) \
337 { \
338 return nix_recv_pkts(rx_queue, rx_pkts, pkts, (flags)); \
339 } \
340 \
341 static uint16_t __rte_noinline __rte_hot \
342 otx2_nix_recv_pkts_mseg_ ## name(void *rx_queue, \
343 struct rte_mbuf **rx_pkts, uint16_t pkts) \
344 { \
345 return nix_recv_pkts(rx_queue, rx_pkts, pkts, \
346 (flags) | NIX_RX_MULTI_SEG_F); \
347 } \
348 \
349 static uint16_t __rte_noinline __rte_hot \
350 otx2_nix_recv_pkts_vec_ ## name(void *rx_queue, \
351 struct rte_mbuf **rx_pkts, uint16_t pkts) \
352 { \
353 /* TSTMP is not supported by vector */ \
354 if ((flags) & NIX_RX_OFFLOAD_TSTAMP_F) \
355 return 0; \
356 return nix_recv_pkts_vector(rx_queue, rx_pkts, pkts, (flags)); \
357 } \
358
359 NIX_RX_FASTPATH_MODES
360 #undef R
361
362 static inline void
pick_rx_func(struct rte_eth_dev * eth_dev,const eth_rx_burst_t rx_burst[2][2][2][2][2][2][2])363 pick_rx_func(struct rte_eth_dev *eth_dev,
364 const eth_rx_burst_t rx_burst[2][2][2][2][2][2][2])
365 {
366 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
367
368 /* [SEC] [TSTMP] [MARK] [VLAN] [CKSUM] [PTYPE] [RSS] */
369 eth_dev->rx_pkt_burst = rx_burst
370 [!!(dev->rx_offload_flags & NIX_RX_OFFLOAD_SECURITY_F)]
371 [!!(dev->rx_offload_flags & NIX_RX_OFFLOAD_TSTAMP_F)]
372 [!!(dev->rx_offload_flags & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
373 [!!(dev->rx_offload_flags & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
374 [!!(dev->rx_offload_flags & NIX_RX_OFFLOAD_CHECKSUM_F)]
375 [!!(dev->rx_offload_flags & NIX_RX_OFFLOAD_PTYPE_F)]
376 [!!(dev->rx_offload_flags & NIX_RX_OFFLOAD_RSS_F)];
377 }
378
379 void
otx2_eth_set_rx_function(struct rte_eth_dev * eth_dev)380 otx2_eth_set_rx_function(struct rte_eth_dev *eth_dev)
381 {
382 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
383
384 const eth_rx_burst_t nix_eth_rx_burst[2][2][2][2][2][2][2] = {
385 #define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
386 [f6][f5][f4][f3][f2][f1][f0] = otx2_nix_recv_pkts_ ## name,
387
388 NIX_RX_FASTPATH_MODES
389 #undef R
390 };
391
392 const eth_rx_burst_t nix_eth_rx_burst_mseg[2][2][2][2][2][2][2] = {
393 #define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
394 [f6][f5][f4][f3][f2][f1][f0] = otx2_nix_recv_pkts_mseg_ ## name,
395
396 NIX_RX_FASTPATH_MODES
397 #undef R
398 };
399
400 const eth_rx_burst_t nix_eth_rx_vec_burst[2][2][2][2][2][2][2] = {
401 #define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
402 [f6][f5][f4][f3][f2][f1][f0] = otx2_nix_recv_pkts_vec_ ## name,
403
404 NIX_RX_FASTPATH_MODES
405 #undef R
406 };
407
408 /* For PTP enabled, scalar rx function should be chosen as most of the
409 * PTP apps are implemented to rx burst 1 pkt.
410 */
411 if (dev->scalar_ena || dev->rx_offloads & DEV_RX_OFFLOAD_TIMESTAMP)
412 pick_rx_func(eth_dev, nix_eth_rx_burst);
413 else
414 pick_rx_func(eth_dev, nix_eth_rx_vec_burst);
415
416 if (dev->rx_offloads & DEV_RX_OFFLOAD_SCATTER)
417 pick_rx_func(eth_dev, nix_eth_rx_burst_mseg);
418
419 /* Copy multi seg version with no offload for tear down sequence */
420 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
421 dev->rx_pkt_burst_no_offload =
422 nix_eth_rx_burst_mseg[0][0][0][0][0][0][0];
423 rte_mb();
424 }
425