1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2020 Marvell International Ltd.
3 */
4
5 #include <arpa/inet.h>
6 #include <sys/socket.h>
7
8 #include <rte_debug.h>
9 #include <rte_ethdev.h>
10 #include <rte_ether.h>
11 #include <rte_graph.h>
12 #include <rte_graph_worker.h>
13 #include <rte_ip.h>
14 #include <rte_lpm.h>
15 #include <rte_mbuf.h>
16 #include <rte_tcp.h>
17 #include <rte_udp.h>
18 #include <rte_vect.h>
19
20 #include "rte_node_ip4_api.h"
21
22 #include "node_private.h"
23
24 #define IPV4_L3FWD_LPM_MAX_RULES 1024
25 #define IPV4_L3FWD_LPM_NUMBER_TBL8S (1 << 8)
26
27 /* IP4 Lookup global data struct */
28 struct ip4_lookup_node_main {
29 struct rte_lpm *lpm_tbl[RTE_MAX_NUMA_NODES];
30 };
31
32 struct ip4_lookup_node_ctx {
33 /* Socket's LPM table */
34 struct rte_lpm *lpm;
35 /* Dynamic offset to mbuf priv1 */
36 int mbuf_priv1_off;
37 };
38
39 int node_mbuf_priv1_dynfield_offset = -1;
40
41 static struct ip4_lookup_node_main ip4_lookup_nm;
42
43 #define IP4_LOOKUP_NODE_LPM(ctx) \
44 (((struct ip4_lookup_node_ctx *)ctx)->lpm)
45
46 #define IP4_LOOKUP_NODE_PRIV1_OFF(ctx) \
47 (((struct ip4_lookup_node_ctx *)ctx)->mbuf_priv1_off)
48
49 #if defined(__ARM_NEON)
50 #include "ip4_lookup_neon.h"
51 #elif defined(RTE_ARCH_X86)
52 #include "ip4_lookup_sse.h"
53 #endif
54
55 static uint16_t
ip4_lookup_node_process_scalar(struct rte_graph * graph,struct rte_node * node,void ** objs,uint16_t nb_objs)56 ip4_lookup_node_process_scalar(struct rte_graph *graph, struct rte_node *node,
57 void **objs, uint16_t nb_objs)
58 {
59 struct rte_lpm *lpm = IP4_LOOKUP_NODE_LPM(node->ctx);
60 const int dyn = IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx);
61 struct rte_ipv4_hdr *ipv4_hdr;
62 void **to_next, **from;
63 uint16_t last_spec = 0;
64 struct rte_mbuf *mbuf;
65 rte_edge_t next_index;
66 uint16_t held = 0;
67 uint32_t drop_nh;
68 int i, rc;
69
70 /* Speculative next */
71 next_index = RTE_NODE_IP4_LOOKUP_NEXT_REWRITE;
72 /* Drop node */
73 drop_nh = ((uint32_t)RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP) << 16;
74 from = objs;
75
76 /* Get stream for the speculated next node */
77 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs);
78 for (i = 0; i < nb_objs; i++) {
79 uint32_t next_hop;
80 uint16_t next;
81
82 mbuf = (struct rte_mbuf *)objs[i];
83
84 /* Extract DIP of mbuf0 */
85 ipv4_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv4_hdr *,
86 sizeof(struct rte_ether_hdr));
87 /* Extract cksum, ttl as ipv4 hdr is in cache */
88 node_mbuf_priv1(mbuf, dyn)->cksum = ipv4_hdr->hdr_checksum;
89 node_mbuf_priv1(mbuf, dyn)->ttl = ipv4_hdr->time_to_live;
90
91 rc = rte_lpm_lookup(lpm, rte_be_to_cpu_32(ipv4_hdr->dst_addr),
92 &next_hop);
93 next_hop = (rc == 0) ? next_hop : drop_nh;
94
95 node_mbuf_priv1(mbuf, dyn)->nh = (uint16_t)next_hop;
96 next_hop = next_hop >> 16;
97 next = (uint16_t)next_hop;
98
99 if (unlikely(next_index != next)) {
100 /* Copy things successfully speculated till now */
101 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
102 from += last_spec;
103 to_next += last_spec;
104 held += last_spec;
105 last_spec = 0;
106
107 rte_node_enqueue_x1(graph, node, next, from[0]);
108 from += 1;
109 } else {
110 last_spec += 1;
111 }
112 }
113
114 /* !!! Home run !!! */
115 if (likely(last_spec == nb_objs)) {
116 rte_node_next_stream_move(graph, node, next_index);
117 return nb_objs;
118 }
119 held += last_spec;
120 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
121 rte_node_next_stream_put(graph, node, next_index, held);
122
123 return nb_objs;
124 }
125
126 int
rte_node_ip4_route_add(uint32_t ip,uint8_t depth,uint16_t next_hop,enum rte_node_ip4_lookup_next next_node)127 rte_node_ip4_route_add(uint32_t ip, uint8_t depth, uint16_t next_hop,
128 enum rte_node_ip4_lookup_next next_node)
129 {
130 char abuf[INET6_ADDRSTRLEN];
131 struct in_addr in;
132 uint8_t socket;
133 uint32_t val;
134 int ret;
135
136 in.s_addr = htonl(ip);
137 inet_ntop(AF_INET, &in, abuf, sizeof(abuf));
138 /* Embedded next node id into 24 bit next hop */
139 val = ((next_node << 16) | next_hop) & ((1ull << 24) - 1);
140 node_dbg("ip4_lookup", "LPM: Adding route %s / %d nh (0x%x)", abuf,
141 depth, val);
142
143 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) {
144 if (!ip4_lookup_nm.lpm_tbl[socket])
145 continue;
146
147 ret = rte_lpm_add(ip4_lookup_nm.lpm_tbl[socket],
148 ip, depth, val);
149 if (ret < 0) {
150 node_err("ip4_lookup",
151 "Unable to add entry %s / %d nh (%x) to LPM table on sock %d, rc=%d\n",
152 abuf, depth, val, socket, ret);
153 return ret;
154 }
155 }
156
157 return 0;
158 }
159
160 static int
setup_lpm(struct ip4_lookup_node_main * nm,int socket)161 setup_lpm(struct ip4_lookup_node_main *nm, int socket)
162 {
163 struct rte_lpm_config config_ipv4;
164 char s[RTE_LPM_NAMESIZE];
165
166 /* One LPM table per socket */
167 if (nm->lpm_tbl[socket])
168 return 0;
169
170 /* create the LPM table */
171 config_ipv4.max_rules = IPV4_L3FWD_LPM_MAX_RULES;
172 config_ipv4.number_tbl8s = IPV4_L3FWD_LPM_NUMBER_TBL8S;
173 config_ipv4.flags = 0;
174 snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socket);
175 nm->lpm_tbl[socket] = rte_lpm_create(s, socket, &config_ipv4);
176 if (nm->lpm_tbl[socket] == NULL)
177 return -rte_errno;
178
179 return 0;
180 }
181
182 static int
ip4_lookup_node_init(const struct rte_graph * graph,struct rte_node * node)183 ip4_lookup_node_init(const struct rte_graph *graph, struct rte_node *node)
184 {
185 uint16_t socket, lcore_id;
186 static uint8_t init_once;
187 int rc;
188
189 RTE_SET_USED(graph);
190 RTE_BUILD_BUG_ON(sizeof(struct ip4_lookup_node_ctx) > RTE_NODE_CTX_SZ);
191
192 if (!init_once) {
193 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register(
194 &node_mbuf_priv1_dynfield_desc);
195 if (node_mbuf_priv1_dynfield_offset < 0)
196 return -rte_errno;
197
198 /* Setup LPM tables for all sockets */
199 RTE_LCORE_FOREACH(lcore_id)
200 {
201 socket = rte_lcore_to_socket_id(lcore_id);
202 rc = setup_lpm(&ip4_lookup_nm, socket);
203 if (rc) {
204 node_err("ip4_lookup",
205 "Failed to setup lpm tbl for sock %u, rc=%d",
206 socket, rc);
207 return rc;
208 }
209 }
210 init_once = 1;
211 }
212
213 /* Update socket's LPM and mbuf dyn priv1 offset in node ctx */
214 IP4_LOOKUP_NODE_LPM(node->ctx) = ip4_lookup_nm.lpm_tbl[graph->socket];
215 IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset;
216
217 #if defined(__ARM_NEON) || defined(RTE_ARCH_X86)
218 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
219 node->process = ip4_lookup_node_process_vec;
220 #endif
221
222 node_dbg("ip4_lookup", "Initialized ip4_lookup node");
223
224 return 0;
225 }
226
227 static struct rte_node_register ip4_lookup_node = {
228 .process = ip4_lookup_node_process_scalar,
229 .name = "ip4_lookup",
230
231 .init = ip4_lookup_node_init,
232
233 .nb_edges = RTE_NODE_IP4_LOOKUP_NEXT_MAX,
234 .next_nodes = {
235 [RTE_NODE_IP4_LOOKUP_NEXT_REWRITE] = "ip4_rewrite",
236 [RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP] = "pkt_drop",
237 },
238 };
239
240 RTE_NODE_REGISTER(ip4_lookup_node);
241