1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2020 Marvell International Ltd.
3 */
4
5 #include <rte_debug.h>
6 #include <rte_ethdev.h>
7 #include <rte_ether.h>
8 #include <rte_graph.h>
9 #include <rte_graph_worker.h>
10 #include <rte_ip.h>
11 #include <rte_malloc.h>
12 #include <rte_mbuf.h>
13 #include <rte_tcp.h>
14 #include <rte_udp.h>
15 #include <rte_vect.h>
16
17 #include "rte_node_ip4_api.h"
18
19 #include "ip4_rewrite_priv.h"
20 #include "node_private.h"
21
22 struct ip4_rewrite_node_ctx {
23 /* Dynamic offset to mbuf priv1 */
24 int mbuf_priv1_off;
25 /* Cached next index */
26 uint16_t next_index;
27 };
28
29 static struct ip4_rewrite_node_main *ip4_rewrite_nm;
30
31 #define IP4_REWRITE_NODE_LAST_NEXT(ctx) \
32 (((struct ip4_rewrite_node_ctx *)ctx)->next_index)
33
34 #define IP4_REWRITE_NODE_PRIV1_OFF(ctx) \
35 (((struct ip4_rewrite_node_ctx *)ctx)->mbuf_priv1_off)
36
37 static uint16_t
ip4_rewrite_node_process(struct rte_graph * graph,struct rte_node * node,void ** objs,uint16_t nb_objs)38 ip4_rewrite_node_process(struct rte_graph *graph, struct rte_node *node,
39 void **objs, uint16_t nb_objs)
40 {
41 struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts;
42 struct ip4_rewrite_nh_header *nh = ip4_rewrite_nm->nh;
43 const int dyn = IP4_REWRITE_NODE_PRIV1_OFF(node->ctx);
44 uint16_t next0, next1, next2, next3, next_index;
45 struct rte_ipv4_hdr *ip0, *ip1, *ip2, *ip3;
46 uint16_t n_left_from, held = 0, last_spec = 0;
47 void *d0, *d1, *d2, *d3;
48 void **to_next, **from;
49 rte_xmm_t priv01;
50 rte_xmm_t priv23;
51 int i;
52
53 /* Speculative next as last next */
54 next_index = IP4_REWRITE_NODE_LAST_NEXT(node->ctx);
55 rte_prefetch0(nh);
56
57 pkts = (struct rte_mbuf **)objs;
58 from = objs;
59 n_left_from = nb_objs;
60
61 for (i = 0; i < 4 && i < n_left_from; i++)
62 rte_prefetch0(pkts[i]);
63
64 /* Get stream for the speculated next node */
65 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs);
66 /* Update Ethernet header of pkts */
67 while (n_left_from >= 4) {
68 if (likely(n_left_from > 7)) {
69 /* Prefetch only next-mbuf struct and priv area.
70 * Data need not be prefetched as we only write.
71 */
72 rte_prefetch0(pkts[4]);
73 rte_prefetch0(pkts[5]);
74 rte_prefetch0(pkts[6]);
75 rte_prefetch0(pkts[7]);
76 }
77
78 mbuf0 = pkts[0];
79 mbuf1 = pkts[1];
80 mbuf2 = pkts[2];
81 mbuf3 = pkts[3];
82
83 pkts += 4;
84 n_left_from -= 4;
85 priv01.u64[0] = node_mbuf_priv1(mbuf0, dyn)->u;
86 priv01.u64[1] = node_mbuf_priv1(mbuf1, dyn)->u;
87 priv23.u64[0] = node_mbuf_priv1(mbuf2, dyn)->u;
88 priv23.u64[1] = node_mbuf_priv1(mbuf3, dyn)->u;
89
90 /* Increment checksum by one. */
91 priv01.u32[1] += rte_cpu_to_be_16(0x0100);
92 priv01.u32[3] += rte_cpu_to_be_16(0x0100);
93 priv23.u32[1] += rte_cpu_to_be_16(0x0100);
94 priv23.u32[3] += rte_cpu_to_be_16(0x0100);
95
96 /* Update ttl,cksum rewrite ethernet hdr on mbuf0 */
97 d0 = rte_pktmbuf_mtod(mbuf0, void *);
98 rte_memcpy(d0, nh[priv01.u16[0]].rewrite_data,
99 nh[priv01.u16[0]].rewrite_len);
100
101 next0 = nh[priv01.u16[0]].tx_node;
102 ip0 = (struct rte_ipv4_hdr *)((uint8_t *)d0 +
103 sizeof(struct rte_ether_hdr));
104 ip0->time_to_live = priv01.u16[1] - 1;
105 ip0->hdr_checksum = priv01.u16[2] + priv01.u16[3];
106
107 /* Update ttl,cksum rewrite ethernet hdr on mbuf1 */
108 d1 = rte_pktmbuf_mtod(mbuf1, void *);
109 rte_memcpy(d1, nh[priv01.u16[4]].rewrite_data,
110 nh[priv01.u16[4]].rewrite_len);
111
112 next1 = nh[priv01.u16[4]].tx_node;
113 ip1 = (struct rte_ipv4_hdr *)((uint8_t *)d1 +
114 sizeof(struct rte_ether_hdr));
115 ip1->time_to_live = priv01.u16[5] - 1;
116 ip1->hdr_checksum = priv01.u16[6] + priv01.u16[7];
117
118 /* Update ttl,cksum rewrite ethernet hdr on mbuf2 */
119 d2 = rte_pktmbuf_mtod(mbuf2, void *);
120 rte_memcpy(d2, nh[priv23.u16[0]].rewrite_data,
121 nh[priv23.u16[0]].rewrite_len);
122 next2 = nh[priv23.u16[0]].tx_node;
123 ip2 = (struct rte_ipv4_hdr *)((uint8_t *)d2 +
124 sizeof(struct rte_ether_hdr));
125 ip2->time_to_live = priv23.u16[1] - 1;
126 ip2->hdr_checksum = priv23.u16[2] + priv23.u16[3];
127
128 /* Update ttl,cksum rewrite ethernet hdr on mbuf3 */
129 d3 = rte_pktmbuf_mtod(mbuf3, void *);
130 rte_memcpy(d3, nh[priv23.u16[4]].rewrite_data,
131 nh[priv23.u16[4]].rewrite_len);
132
133 next3 = nh[priv23.u16[4]].tx_node;
134 ip3 = (struct rte_ipv4_hdr *)((uint8_t *)d3 +
135 sizeof(struct rte_ether_hdr));
136 ip3->time_to_live = priv23.u16[5] - 1;
137 ip3->hdr_checksum = priv23.u16[6] + priv23.u16[7];
138
139 /* Enqueue four to next node */
140 rte_edge_t fix_spec =
141 ((next_index == next0) && (next0 == next1) &&
142 (next1 == next2) && (next2 == next3));
143
144 if (unlikely(fix_spec == 0)) {
145 /* Copy things successfully speculated till now */
146 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
147 from += last_spec;
148 to_next += last_spec;
149 held += last_spec;
150 last_spec = 0;
151
152 /* next0 */
153 if (next_index == next0) {
154 to_next[0] = from[0];
155 to_next++;
156 held++;
157 } else {
158 rte_node_enqueue_x1(graph, node, next0,
159 from[0]);
160 }
161
162 /* next1 */
163 if (next_index == next1) {
164 to_next[0] = from[1];
165 to_next++;
166 held++;
167 } else {
168 rte_node_enqueue_x1(graph, node, next1,
169 from[1]);
170 }
171
172 /* next2 */
173 if (next_index == next2) {
174 to_next[0] = from[2];
175 to_next++;
176 held++;
177 } else {
178 rte_node_enqueue_x1(graph, node, next2,
179 from[2]);
180 }
181
182 /* next3 */
183 if (next_index == next3) {
184 to_next[0] = from[3];
185 to_next++;
186 held++;
187 } else {
188 rte_node_enqueue_x1(graph, node, next3,
189 from[3]);
190 }
191
192 from += 4;
193
194 /* Change speculation if last two are same */
195 if ((next_index != next3) && (next2 == next3)) {
196 /* Put the current speculated node */
197 rte_node_next_stream_put(graph, node,
198 next_index, held);
199 held = 0;
200
201 /* Get next speculated stream */
202 next_index = next3;
203 to_next = rte_node_next_stream_get(
204 graph, node, next_index, nb_objs);
205 }
206 } else {
207 last_spec += 4;
208 }
209 }
210
211 while (n_left_from > 0) {
212 uint16_t chksum;
213
214 mbuf0 = pkts[0];
215
216 pkts += 1;
217 n_left_from -= 1;
218
219 d0 = rte_pktmbuf_mtod(mbuf0, void *);
220 rte_memcpy(d0, nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_data,
221 nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_len);
222
223 next0 = nh[node_mbuf_priv1(mbuf0, dyn)->nh].tx_node;
224 ip0 = (struct rte_ipv4_hdr *)((uint8_t *)d0 +
225 sizeof(struct rte_ether_hdr));
226 chksum = node_mbuf_priv1(mbuf0, dyn)->cksum +
227 rte_cpu_to_be_16(0x0100);
228 chksum += chksum >= 0xffff;
229 ip0->hdr_checksum = chksum;
230 ip0->time_to_live = node_mbuf_priv1(mbuf0, dyn)->ttl - 1;
231
232 if (unlikely(next_index ^ next0)) {
233 /* Copy things successfully speculated till now */
234 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
235 from += last_spec;
236 to_next += last_spec;
237 held += last_spec;
238 last_spec = 0;
239
240 rte_node_enqueue_x1(graph, node, next0, from[0]);
241 from += 1;
242 } else {
243 last_spec += 1;
244 }
245 }
246
247 /* !!! Home run !!! */
248 if (likely(last_spec == nb_objs)) {
249 rte_node_next_stream_move(graph, node, next_index);
250 return nb_objs;
251 }
252
253 held += last_spec;
254 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
255 rte_node_next_stream_put(graph, node, next_index, held);
256 /* Save the last next used */
257 IP4_REWRITE_NODE_LAST_NEXT(node->ctx) = next_index;
258
259 return nb_objs;
260 }
261
262 static int
ip4_rewrite_node_init(const struct rte_graph * graph,struct rte_node * node)263 ip4_rewrite_node_init(const struct rte_graph *graph, struct rte_node *node)
264 {
265 static bool init_once;
266
267 RTE_SET_USED(graph);
268 RTE_BUILD_BUG_ON(sizeof(struct ip4_rewrite_node_ctx) > RTE_NODE_CTX_SZ);
269
270 if (!init_once) {
271 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register(
272 &node_mbuf_priv1_dynfield_desc);
273 if (node_mbuf_priv1_dynfield_offset < 0)
274 return -rte_errno;
275 init_once = true;
276 }
277 IP4_REWRITE_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset;
278
279 node_dbg("ip4_rewrite", "Initialized ip4_rewrite node initialized");
280
281 return 0;
282 }
283
284 int
ip4_rewrite_set_next(uint16_t port_id,uint16_t next_index)285 ip4_rewrite_set_next(uint16_t port_id, uint16_t next_index)
286 {
287 if (ip4_rewrite_nm == NULL) {
288 ip4_rewrite_nm = rte_zmalloc(
289 "ip4_rewrite", sizeof(struct ip4_rewrite_node_main),
290 RTE_CACHE_LINE_SIZE);
291 if (ip4_rewrite_nm == NULL)
292 return -ENOMEM;
293 }
294 ip4_rewrite_nm->next_index[port_id] = next_index;
295
296 return 0;
297 }
298
299 int
rte_node_ip4_rewrite_add(uint16_t next_hop,uint8_t * rewrite_data,uint8_t rewrite_len,uint16_t dst_port)300 rte_node_ip4_rewrite_add(uint16_t next_hop, uint8_t *rewrite_data,
301 uint8_t rewrite_len, uint16_t dst_port)
302 {
303 struct ip4_rewrite_nh_header *nh;
304
305 if (next_hop >= RTE_GRAPH_IP4_REWRITE_MAX_NH)
306 return -EINVAL;
307
308 if (rewrite_len > RTE_GRAPH_IP4_REWRITE_MAX_LEN)
309 return -EINVAL;
310
311 if (ip4_rewrite_nm == NULL) {
312 ip4_rewrite_nm = rte_zmalloc(
313 "ip4_rewrite", sizeof(struct ip4_rewrite_node_main),
314 RTE_CACHE_LINE_SIZE);
315 if (ip4_rewrite_nm == NULL)
316 return -ENOMEM;
317 }
318
319 /* Check if dst port doesn't exist as edge */
320 if (!ip4_rewrite_nm->next_index[dst_port])
321 return -EINVAL;
322
323 /* Update next hop */
324 nh = &ip4_rewrite_nm->nh[next_hop];
325
326 memcpy(nh->rewrite_data, rewrite_data, rewrite_len);
327 nh->tx_node = ip4_rewrite_nm->next_index[dst_port];
328 nh->rewrite_len = rewrite_len;
329 nh->enabled = true;
330
331 return 0;
332 }
333
334 static struct rte_node_register ip4_rewrite_node = {
335 .process = ip4_rewrite_node_process,
336 .name = "ip4_rewrite",
337 /* Default edge i.e '0' is pkt drop */
338 .nb_edges = 1,
339 .next_nodes = {
340 [0] = "pkt_drop",
341 },
342 .init = ip4_rewrite_node_init,
343 };
344
345 struct rte_node_register *
ip4_rewrite_node_get(void)346 ip4_rewrite_node_get(void)
347 {
348 return &ip4_rewrite_node;
349 }
350
351 RTE_NODE_REGISTER(ip4_rewrite_node);
352