1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016 Intel Corporation.
3 * Copyright(c) 2017 IBM Corporation.
4 * All rights reserved.
5 */
6
7 #ifndef _L3FWD_ALTIVEC_H_
8 #define _L3FWD_ALTIVEC_H_
9
10 #include "l3fwd.h"
11 #include "l3fwd_common.h"
12
13 /*
14 * Update source and destination MAC addresses in the ethernet header.
15 * Perform RFC1812 checks and updates for IPV4 packets.
16 */
17 static inline void
processx4_step3(struct rte_mbuf * pkt[FWDSTEP],uint16_t dst_port[FWDSTEP])18 processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP])
19 {
20 vector unsigned int te[FWDSTEP];
21 vector unsigned int ve[FWDSTEP];
22 vector unsigned int *p[FWDSTEP];
23
24 p[0] = rte_pktmbuf_mtod(pkt[0], vector unsigned int *);
25 p[1] = rte_pktmbuf_mtod(pkt[1], vector unsigned int *);
26 p[2] = rte_pktmbuf_mtod(pkt[2], vector unsigned int *);
27 p[3] = rte_pktmbuf_mtod(pkt[3], vector unsigned int *);
28
29 ve[0] = (vector unsigned int)val_eth[dst_port[0]];
30 te[0] = *p[0];
31
32 ve[1] = (vector unsigned int)val_eth[dst_port[1]];
33 te[1] = *p[1];
34
35 ve[2] = (vector unsigned int)val_eth[dst_port[2]];
36 te[2] = *p[2];
37
38 ve[3] = (vector unsigned int)val_eth[dst_port[3]];
39 te[3] = *p[3];
40
41 /* Update first 12 bytes, keep rest bytes intact. */
42 te[0] = (vector unsigned int)vec_sel(
43 (vector unsigned short)ve[0],
44 (vector unsigned short)te[0],
45 (vector unsigned short) {0, 0, 0, 0,
46 0, 0, 0xffff, 0xffff});
47
48 te[1] = (vector unsigned int)vec_sel(
49 (vector unsigned short)ve[1],
50 (vector unsigned short)te[1],
51 (vector unsigned short) {0, 0, 0, 0,
52 0, 0, 0xffff, 0xffff});
53
54 te[2] = (vector unsigned int)vec_sel(
55 (vector unsigned short)ve[2],
56 (vector unsigned short)te[2],
57 (vector unsigned short) {0, 0, 0, 0, 0,
58 0, 0xffff, 0xffff});
59
60 te[3] = (vector unsigned int)vec_sel(
61 (vector unsigned short)ve[3],
62 (vector unsigned short)te[3],
63 (vector unsigned short) {0, 0, 0, 0,
64 0, 0, 0xffff, 0xffff});
65
66 *p[0] = te[0];
67 *p[1] = te[1];
68 *p[2] = te[2];
69 *p[3] = te[3];
70
71 rfc1812_process((struct rte_ipv4_hdr *)
72 ((struct rte_ether_hdr *)p[0] + 1),
73 &dst_port[0], pkt[0]->packet_type);
74 rfc1812_process((struct rte_ipv4_hdr *)
75 ((struct rte_ether_hdr *)p[1] + 1),
76 &dst_port[1], pkt[1]->packet_type);
77 rfc1812_process((struct rte_ipv4_hdr *)
78 ((struct rte_ether_hdr *)p[2] + 1),
79 &dst_port[2], pkt[2]->packet_type);
80 rfc1812_process((struct rte_ipv4_hdr *)
81 ((struct rte_ether_hdr *)p[3] + 1),
82 &dst_port[3], pkt[3]->packet_type);
83 }
84
85 /*
86 * Group consecutive packets with the same destination port in bursts of 4.
87 * Suppose we have array of destination ports:
88 * dst_port[] = {a, b, c, d,, e, ... }
89 * dp1 should contain: <a, b, c, d>, dp2: <b, c, d, e>.
90 * We doing 4 comparisons at once and the result is 4 bit mask.
91 * This mask is used as an index into prebuild array of pnum values.
92 */
93 static inline uint16_t *
port_groupx4(uint16_t pn[FWDSTEP+1],uint16_t * lp,vector unsigned short dp1,vector unsigned short dp2)94 port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, vector unsigned short dp1,
95 vector unsigned short dp2)
96 {
97 union {
98 uint16_t u16[FWDSTEP + 1];
99 uint64_t u64;
100 } *pnum = (void *)pn;
101
102 int32_t v;
103
104 v = vec_any_eq(dp1, dp2);
105
106
107 /* update last port counter. */
108 lp[0] += gptbl[v].lpv;
109
110 /* if dest port value has changed. */
111 if (v != GRPMSK) {
112 pnum->u64 = gptbl[v].pnum;
113 pnum->u16[FWDSTEP] = 1;
114 lp = pnum->u16 + gptbl[v].idx;
115 }
116
117 return lp;
118 }
119
120 /**
121 * Process one packet:
122 * Update source and destination MAC addresses in the ethernet header.
123 * Perform RFC1812 checks and updates for IPV4 packets.
124 */
125 static inline void
process_packet(struct rte_mbuf * pkt,uint16_t * dst_port)126 process_packet(struct rte_mbuf *pkt, uint16_t *dst_port)
127 {
128 struct rte_ether_hdr *eth_hdr;
129 vector unsigned int te, ve;
130
131 eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
132
133 te = *(vector unsigned int *)eth_hdr;
134 ve = (vector unsigned int)val_eth[dst_port[0]];
135
136 rfc1812_process((struct rte_ipv4_hdr *)(eth_hdr + 1), dst_port,
137 pkt->packet_type);
138
139 /* dynamically vec_sel te and ve for MASK_ETH (0x3f) */
140 te = (vector unsigned int)vec_sel(
141 (vector unsigned short)ve,
142 (vector unsigned short)te,
143 (vector unsigned short){0, 0, 0, 0,
144 0, 0, 0xffff, 0xffff});
145
146 *(vector unsigned int *)eth_hdr = te;
147 }
148
149 /**
150 * Send packets burst from pkts_burst to the ports in dst_port array
151 */
152 static __rte_always_inline void
send_packets_multi(struct lcore_conf * qconf,struct rte_mbuf ** pkts_burst,uint16_t dst_port[MAX_PKT_BURST],int nb_rx)153 send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst,
154 uint16_t dst_port[MAX_PKT_BURST], int nb_rx)
155 {
156 int32_t k;
157 int j = 0;
158 uint16_t dlp;
159 uint16_t *lp;
160 uint16_t pnum[MAX_PKT_BURST + 1];
161
162 /*
163 * Finish packet processing and group consecutive
164 * packets with the same destination port.
165 */
166 k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
167 if (k != 0) {
168 vector unsigned short dp1, dp2;
169
170 lp = pnum;
171 lp[0] = 1;
172
173 processx4_step3(pkts_burst, dst_port);
174
175 /* dp1: <d[0], d[1], d[2], d[3], ... > */
176 dp1 = *(vector unsigned short *)dst_port;
177
178 for (j = FWDSTEP; j != k; j += FWDSTEP) {
179 processx4_step3(&pkts_burst[j], &dst_port[j]);
180
181 /*
182 * dp2:
183 * <d[j-3], d[j-2], d[j-1], d[j], ... >
184 */
185 dp2 = *((vector unsigned short *)
186 &dst_port[j - FWDSTEP + 1]);
187 lp = port_groupx4(&pnum[j - FWDSTEP], lp, dp1, dp2);
188
189 /*
190 * dp1:
191 * <d[j], d[j+1], d[j+2], d[j+3], ... >
192 */
193 dp1 = vec_sro(dp2, (vector unsigned char) {
194 0, 0, 0, 0, 0, 0, 0, 0,
195 0, 0, 0, (FWDSTEP - 1) * sizeof(dst_port[0])});
196 }
197
198 /*
199 * dp2: <d[j-3], d[j-2], d[j-1], d[j-1], ... >
200 */
201 dp2 = vec_perm(dp1, (vector unsigned short){},
202 (vector unsigned char){0xf9});
203 lp = port_groupx4(&pnum[j - FWDSTEP], lp, dp1, dp2);
204
205 /*
206 * remove values added by the last repeated
207 * dst port.
208 */
209 lp[0]--;
210 dlp = dst_port[j - 1];
211 } else {
212 /* set dlp and lp to the never used values. */
213 dlp = BAD_PORT - 1;
214 lp = pnum + MAX_PKT_BURST;
215 }
216
217 /* Process up to last 3 packets one by one. */
218 switch (nb_rx % FWDSTEP) {
219 case 3:
220 process_packet(pkts_burst[j], dst_port + j);
221 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
222 j++;
223 /* fall-through */
224 case 2:
225 process_packet(pkts_burst[j], dst_port + j);
226 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
227 j++;
228 /* fall-through */
229 case 1:
230 process_packet(pkts_burst[j], dst_port + j);
231 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
232 j++;
233 }
234
235 /*
236 * Send packets out, through destination port.
237 * Consecutive packets with the same destination port
238 * are already grouped together.
239 * If destination port for the packet equals BAD_PORT,
240 * then free the packet without sending it out.
241 */
242 for (j = 0; j < nb_rx; j += k) {
243
244 int32_t m;
245 uint16_t pn;
246
247 pn = dst_port[j];
248 k = pnum[j];
249
250 if (likely(pn != BAD_PORT))
251 send_packetsx4(qconf, pn, pkts_burst + j, k);
252 else
253 for (m = j; m != j + k; m++)
254 rte_pktmbuf_free(pkts_burst[m]);
255
256 }
257 }
258
259 #endif /* _L3FWD_ALTIVEC_H_ */
260