1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <arpa/inet.h> 6 #include <sys/socket.h> 7 8 #include <rte_debug.h> 9 #include <rte_ethdev.h> 10 #include <rte_ether.h> 11 #include <rte_graph.h> 12 #include <rte_graph_worker.h> 13 #include <rte_ip.h> 14 #include <rte_lpm.h> 15 #include <rte_mbuf.h> 16 #include <rte_tcp.h> 17 #include <rte_udp.h> 18 #include <rte_vect.h> 19 20 #include "rte_node_ip4_api.h" 21 22 #include "node_private.h" 23 24 #define IPV4_L3FWD_LPM_MAX_RULES 1024 25 #define IPV4_L3FWD_LPM_NUMBER_TBL8S (1 << 8) 26 27 /* IP4 Lookup global data struct */ 28 struct ip4_lookup_node_main { 29 struct rte_lpm *lpm_tbl[RTE_MAX_NUMA_NODES]; 30 }; 31 32 struct ip4_lookup_node_ctx { 33 /* Socket's LPM table */ 34 struct rte_lpm *lpm; 35 /* Dynamic offset to mbuf priv1 */ 36 int mbuf_priv1_off; 37 }; 38 39 int node_mbuf_priv1_dynfield_offset = -1; 40 41 static struct ip4_lookup_node_main ip4_lookup_nm; 42 43 #define IP4_LOOKUP_NODE_LPM(ctx) \ 44 (((struct ip4_lookup_node_ctx *)ctx)->lpm) 45 46 #define IP4_LOOKUP_NODE_PRIV1_OFF(ctx) \ 47 (((struct ip4_lookup_node_ctx *)ctx)->mbuf_priv1_off) 48 49 #if defined(__ARM_NEON) 50 #include "ip4_lookup_neon.h" 51 #elif defined(RTE_ARCH_X86) 52 #include "ip4_lookup_sse.h" 53 #endif 54 55 static uint16_t 56 ip4_lookup_node_process_scalar(struct rte_graph *graph, struct rte_node *node, 57 void **objs, uint16_t nb_objs) 58 { 59 struct rte_lpm *lpm = IP4_LOOKUP_NODE_LPM(node->ctx); 60 const int dyn = IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx); 61 struct rte_ipv4_hdr *ipv4_hdr; 62 void **to_next, **from; 63 uint16_t last_spec = 0; 64 struct rte_mbuf *mbuf; 65 rte_edge_t next_index; 66 uint16_t held = 0; 67 uint32_t drop_nh; 68 int i, rc; 69 70 /* Speculative next */ 71 next_index = RTE_NODE_IP4_LOOKUP_NEXT_REWRITE; 72 /* Drop node */ 73 drop_nh = ((uint32_t)RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP) << 16; 74 from = objs; 75 76 /* Get stream for the speculated next node */ 77 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs); 78 for (i = 0; i < nb_objs; i++) { 79 uint32_t next_hop; 80 uint16_t next; 81 82 mbuf = (struct rte_mbuf *)objs[i]; 83 84 /* Extract DIP of mbuf0 */ 85 ipv4_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv4_hdr *, 86 sizeof(struct rte_ether_hdr)); 87 /* Extract cksum, ttl as ipv4 hdr is in cache */ 88 node_mbuf_priv1(mbuf, dyn)->cksum = ipv4_hdr->hdr_checksum; 89 node_mbuf_priv1(mbuf, dyn)->ttl = ipv4_hdr->time_to_live; 90 91 rc = rte_lpm_lookup(lpm, rte_be_to_cpu_32(ipv4_hdr->dst_addr), 92 &next_hop); 93 next_hop = (rc == 0) ? next_hop : drop_nh; 94 95 node_mbuf_priv1(mbuf, dyn)->nh = (uint16_t)next_hop; 96 next_hop = next_hop >> 16; 97 next = (uint16_t)next_hop; 98 99 if (unlikely(next_index != next)) { 100 /* Copy things successfully speculated till now */ 101 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 102 from += last_spec; 103 to_next += last_spec; 104 held += last_spec; 105 last_spec = 0; 106 107 rte_node_enqueue_x1(graph, node, next, from[0]); 108 from += 1; 109 } else { 110 last_spec += 1; 111 } 112 } 113 114 /* !!! Home run !!! */ 115 if (likely(last_spec == nb_objs)) { 116 rte_node_next_stream_move(graph, node, next_index); 117 return nb_objs; 118 } 119 held += last_spec; 120 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 121 rte_node_next_stream_put(graph, node, next_index, held); 122 123 return nb_objs; 124 } 125 126 int 127 rte_node_ip4_route_add(uint32_t ip, uint8_t depth, uint16_t next_hop, 128 enum rte_node_ip4_lookup_next next_node) 129 { 130 char abuf[INET6_ADDRSTRLEN]; 131 struct in_addr in; 132 uint8_t socket; 133 uint32_t val; 134 int ret; 135 136 in.s_addr = htonl(ip); 137 inet_ntop(AF_INET, &in, abuf, sizeof(abuf)); 138 /* Embedded next node id into 24 bit next hop */ 139 val = ((next_node << 16) | next_hop) & ((1ull << 24) - 1); 140 node_dbg("ip4_lookup", "LPM: Adding route %s / %d nh (0x%x)", abuf, 141 depth, val); 142 143 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) { 144 if (!ip4_lookup_nm.lpm_tbl[socket]) 145 continue; 146 147 ret = rte_lpm_add(ip4_lookup_nm.lpm_tbl[socket], 148 ip, depth, val); 149 if (ret < 0) { 150 node_err("ip4_lookup", 151 "Unable to add entry %s / %d nh (%x) to LPM table on sock %d, rc=%d\n", 152 abuf, depth, val, socket, ret); 153 return ret; 154 } 155 } 156 157 return 0; 158 } 159 160 static int 161 setup_lpm(struct ip4_lookup_node_main *nm, int socket) 162 { 163 struct rte_lpm_config config_ipv4; 164 char s[RTE_LPM_NAMESIZE]; 165 166 /* One LPM table per socket */ 167 if (nm->lpm_tbl[socket]) 168 return 0; 169 170 /* create the LPM table */ 171 config_ipv4.max_rules = IPV4_L3FWD_LPM_MAX_RULES; 172 config_ipv4.number_tbl8s = IPV4_L3FWD_LPM_NUMBER_TBL8S; 173 config_ipv4.flags = 0; 174 snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socket); 175 nm->lpm_tbl[socket] = rte_lpm_create(s, socket, &config_ipv4); 176 if (nm->lpm_tbl[socket] == NULL) 177 return -rte_errno; 178 179 return 0; 180 } 181 182 static int 183 ip4_lookup_node_init(const struct rte_graph *graph, struct rte_node *node) 184 { 185 uint16_t socket, lcore_id; 186 static uint8_t init_once; 187 int rc; 188 189 RTE_SET_USED(graph); 190 RTE_BUILD_BUG_ON(sizeof(struct ip4_lookup_node_ctx) > RTE_NODE_CTX_SZ); 191 192 if (!init_once) { 193 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register( 194 &node_mbuf_priv1_dynfield_desc); 195 if (node_mbuf_priv1_dynfield_offset < 0) 196 return -rte_errno; 197 198 /* Setup LPM tables for all sockets */ 199 RTE_LCORE_FOREACH(lcore_id) 200 { 201 socket = rte_lcore_to_socket_id(lcore_id); 202 rc = setup_lpm(&ip4_lookup_nm, socket); 203 if (rc) { 204 node_err("ip4_lookup", 205 "Failed to setup lpm tbl for sock %u, rc=%d", 206 socket, rc); 207 return rc; 208 } 209 } 210 init_once = 1; 211 } 212 213 /* Update socket's LPM and mbuf dyn priv1 offset in node ctx */ 214 IP4_LOOKUP_NODE_LPM(node->ctx) = ip4_lookup_nm.lpm_tbl[graph->socket]; 215 IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset; 216 217 #if defined(__ARM_NEON) || defined(RTE_ARCH_X86) 218 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) 219 node->process = ip4_lookup_node_process_vec; 220 #endif 221 222 node_dbg("ip4_lookup", "Initialized ip4_lookup node"); 223 224 return 0; 225 } 226 227 static struct rte_node_register ip4_lookup_node = { 228 .process = ip4_lookup_node_process_scalar, 229 .name = "ip4_lookup", 230 231 .init = ip4_lookup_node_init, 232 233 .nb_edges = RTE_NODE_IP4_LOOKUP_NEXT_MAX, 234 .next_nodes = { 235 [RTE_NODE_IP4_LOOKUP_NEXT_REWRITE] = "ip4_rewrite", 236 [RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP] = "pkt_drop", 237 }, 238 }; 239 240 RTE_NODE_REGISTER(ip4_lookup_node); 241