1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 27 #include <stdlib.h> 28 #include <arpa/inet.h> 29 30 #include <rte_config.h> 31 #include <rte_ether.h> 32 #include <rte_ethdev.h> 33 #include <rte_kni.h> 34 #include <rte_malloc.h> 35 #include <rte_ring.h> 36 #include <rte_ip.h> 37 #include <rte_tcp.h> 38 #include <rte_udp.h> 39 40 #include "ff_dpdk_kni.h" 41 #include "ff_config.h" 42 43 #define KNI_QUEUE_SIZE 8192 44 45 /* Callback for request of changing MTU */ 46 /* Total octets in ethernet header */ 47 #define KNI_ENET_HEADER_SIZE 14 48 49 /* Total octets in the FCS */ 50 #define KNI_ENET_FCS_SIZE 4 51 52 #define set_bit(n, m) (n | magic_bits[m]) 53 #define clear_bit(n, m) (n & (~magic_bits[m])) 54 #define get_bit(n, m) (n & magic_bits[m]) 55 56 static const int magic_bits[8] = { 57 0x80, 0x40, 0x20, 0x10, 58 0x8, 0x4, 0x2, 0x1 59 }; 60 61 static unsigned char *udp_port_bitmap = NULL; 62 static unsigned char *tcp_port_bitmap = NULL; 63 64 /* Structure type for recording kni interface specific stats */ 65 struct kni_interface_stats { 66 struct rte_kni *kni; 67 68 /* number of pkts received from NIC, and sent to KNI */ 69 uint64_t rx_packets; 70 71 /* number of pkts received from NIC, but failed to send to KNI */ 72 uint64_t rx_dropped; 73 74 /* number of pkts received from KNI, and sent to NIC */ 75 uint64_t tx_packets; 76 77 /* number of pkts received from KNI, but failed to send to NIC */ 78 uint64_t tx_dropped; 79 }; 80 81 struct rte_ring **kni_rp; 82 struct kni_interface_stats **kni_stat; 83 84 static void 85 set_bitmap(uint16_t port, unsigned char *bitmap) 86 { 87 port = htons(port); 88 unsigned char *p = bitmap + port/8; 89 *p = set_bit(*p, port % 8); 90 } 91 92 static int 93 get_bitmap(uint16_t port, unsigned char *bitmap) 94 { 95 unsigned char *p = bitmap + port/8; 96 return get_bit(*p, port % 8) > 0 ? 1 : 0; 97 } 98 99 static void 100 kni_set_bitmap(const char *p, unsigned char *port_bitmap) 101 { 102 int i; 103 const char *head, *tail, *tail_num; 104 if(!p) 105 return; 106 107 head = p; 108 while (1) { 109 tail = strstr(head, ","); 110 tail_num = strstr(head, "-"); 111 if(tail_num && (!tail || tail_num < tail - 1)) { 112 for(i = atoi(head); i <= atoi(tail_num + 1); ++i) { 113 set_bitmap(i, port_bitmap); 114 } 115 } else { 116 set_bitmap(atoi(head), port_bitmap); 117 } 118 119 if(!tail) 120 break; 121 122 head = tail + 1; 123 } 124 } 125 126 /* Currently we don't support change mtu. */ 127 static int 128 kni_change_mtu(uint8_t port_id, unsigned new_mtu) 129 { 130 return 0; 131 } 132 133 static int 134 kni_config_network_interface(uint8_t port_id, uint8_t if_up) 135 { 136 int ret = 0; 137 138 if (port_id >= rte_eth_dev_count() || port_id >= RTE_MAX_ETHPORTS) { 139 printf("Invalid port id %d\n", port_id); 140 return -EINVAL; 141 } 142 143 printf("Configure network interface of %d %s\n", 144 port_id, if_up ? "up" : "down"); 145 146 ret = (if_up) ? 147 rte_eth_dev_set_link_up(port_id) : 148 rte_eth_dev_set_link_down(port_id); 149 150 if(-ENOTSUP == ret) { 151 if (if_up != 0) { 152 /* Configure network interface up */ 153 rte_eth_dev_stop(port_id); 154 ret = rte_eth_dev_start(port_id); 155 } else { 156 /* Configure network interface down */ 157 rte_eth_dev_stop(port_id); 158 ret = 0; 159 } 160 } 161 162 if (ret < 0) 163 printf("Failed to Configure network interface of %d %s\n", 164 port_id, if_up ? "up" : "down"); 165 166 return ret; 167 } 168 169 static int 170 kni_process_tx(uint8_t port_id, uint16_t queue_id, 171 struct rte_mbuf **pkts_burst, unsigned count) 172 { 173 /* read packet from kni ring(phy port) and transmit to kni */ 174 uint16_t nb_tx, nb_kni_tx; 175 nb_tx = rte_ring_dequeue_burst(kni_rp[port_id], (void **)pkts_burst, count); 176 177 /* NB. 178 * if nb_tx is 0,it must call rte_kni_tx_burst 179 * must Call regularly rte_kni_tx_burst(kni, NULL, 0). 180 * detail https://embedded.communities.intel.com/thread/6668 181 */ 182 nb_kni_tx = rte_kni_tx_burst(kni_stat[port_id]->kni, pkts_burst, nb_tx); 183 rte_kni_handle_request(kni_stat[port_id]->kni); 184 if(nb_kni_tx < nb_tx) { 185 uint16_t i; 186 for(i = nb_kni_tx; i < nb_tx; ++i) 187 rte_pktmbuf_free(pkts_burst[i]); 188 189 kni_stat[port_id]->rx_dropped += (nb_tx - nb_kni_tx); 190 } 191 192 kni_stat[port_id]->rx_packets += nb_kni_tx; 193 return 0; 194 } 195 196 static int 197 kni_process_rx(uint8_t port_id, uint16_t queue_id, 198 struct rte_mbuf **pkts_burst, unsigned count) 199 { 200 uint16_t nb_kni_rx, nb_rx; 201 202 /* read packet from kni, and transmit to phy port */ 203 nb_kni_rx = rte_kni_rx_burst(kni_stat[port_id]->kni, pkts_burst, count); 204 if (nb_kni_rx > 0) { 205 nb_rx = rte_eth_tx_burst(port_id, queue_id, pkts_burst, nb_kni_rx); 206 if (nb_rx < nb_kni_rx) { 207 uint16_t i; 208 for(i = nb_rx; i < nb_kni_rx; ++i) 209 rte_pktmbuf_free(pkts_burst[i]); 210 211 kni_stat[port_id]->tx_dropped += (nb_kni_rx - nb_rx); 212 } 213 214 kni_stat[port_id]->tx_packets += nb_rx; 215 } 216 return 0; 217 } 218 219 static enum FilterReturn 220 protocol_filter_l4(uint16_t port, unsigned char *bitmap) 221 { 222 if(get_bitmap(port, bitmap)) { 223 return FILTER_KNI; 224 } 225 226 return FILTER_UNKNOWN; 227 } 228 229 static enum FilterReturn 230 protocol_filter_tcp(const void *data, uint16_t len) 231 { 232 if (len < sizeof(struct tcp_hdr)) 233 return FILTER_UNKNOWN; 234 235 const struct tcp_hdr *hdr; 236 hdr = (const struct tcp_hdr *)data; 237 238 return protocol_filter_l4(hdr->dst_port, tcp_port_bitmap); 239 } 240 241 static enum FilterReturn 242 protocol_filter_udp(const void* data,uint16_t len) 243 { 244 if (len < sizeof(struct udp_hdr)) 245 return FILTER_UNKNOWN; 246 247 const struct udp_hdr *hdr; 248 hdr = (const struct udp_hdr *)data; 249 250 return protocol_filter_l4(hdr->dst_port, udp_port_bitmap); 251 } 252 253 static enum FilterReturn 254 protocol_filter_ip(const void *data, uint16_t len) 255 { 256 if(len < sizeof(struct ipv4_hdr)) 257 return FILTER_UNKNOWN; 258 259 const struct ipv4_hdr *hdr; 260 hdr = (const struct ipv4_hdr *)data; 261 262 void *next = (void *)data + sizeof(struct ipv4_hdr); 263 uint16_t next_len = len - sizeof(struct ipv4_hdr); 264 265 switch (hdr->next_proto_id) { 266 case IPPROTO_TCP: 267 return protocol_filter_tcp(next, next_len); 268 case IPPROTO_UDP: 269 return protocol_filter_udp(next, next_len); 270 case IPPROTO_IPIP: 271 return protocol_filter_ip(next, next_len); 272 } 273 274 return FILTER_UNKNOWN; 275 } 276 277 enum FilterReturn 278 ff_kni_proto_filter(const void *data, uint16_t len) 279 { 280 return protocol_filter_ip(data, len); 281 } 282 283 void 284 ff_kni_init(uint16_t nb_ports, const char *tcp_ports, const char *udp_ports) 285 { 286 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 287 kni_stat = rte_zmalloc("kni:stat", 288 sizeof(struct kni_interface_stats *) * nb_ports, 289 RTE_CACHE_LINE_SIZE); 290 if (kni_stat == NULL) 291 rte_exit(EXIT_FAILURE, "rte_zmalloc(1 (struct netio_kni_stat *)) " 292 "failed\n"); 293 294 rte_kni_init(nb_ports); 295 } 296 297 uint16_t lcoreid = rte_lcore_id(); 298 char name_buf[RTE_RING_NAMESIZE]; 299 snprintf(name_buf, RTE_RING_NAMESIZE, "kni::ring_%d", lcoreid); 300 kni_rp = rte_zmalloc(name_buf, 301 sizeof(struct rte_ring *) * nb_ports, 302 RTE_CACHE_LINE_SIZE); 303 if (kni_rp == NULL) { 304 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 305 "failed\n", name_buf); 306 } 307 308 snprintf(name_buf, RTE_RING_NAMESIZE, "kni:tcp_port_bitmap_%d", lcoreid); 309 tcp_port_bitmap = rte_zmalloc("kni:tcp_port_bitmap", 8192, 310 RTE_CACHE_LINE_SIZE); 311 if (tcp_port_bitmap == NULL) { 312 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (tcp_port_bitmap)) " 313 "failed\n", name_buf); 314 } 315 316 snprintf(name_buf, RTE_RING_NAMESIZE, "kni:udp_port_bitmap_%d", lcoreid); 317 udp_port_bitmap = rte_zmalloc("kni:udp_port_bitmap", 8192, 318 RTE_CACHE_LINE_SIZE); 319 if (udp_port_bitmap == NULL) { 320 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (udp_port_bitmap)) " 321 "failed\n",name_buf); 322 } 323 324 memset(tcp_port_bitmap, 0, 8192); 325 memset(udp_port_bitmap, 0, 8192); 326 327 kni_set_bitmap(tcp_ports, tcp_port_bitmap); 328 kni_set_bitmap(udp_ports, udp_port_bitmap); 329 } 330 331 void 332 ff_kni_alloc(uint8_t port_id, unsigned socket_id, 333 struct rte_mempool *mbuf_pool) 334 { 335 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 336 struct rte_kni_conf conf; 337 struct rte_kni_ops ops; 338 struct rte_eth_dev_info dev_info; 339 340 kni_stat[port_id] = (struct kni_interface_stats*)rte_zmalloc( 341 "kni:stat_lcore", 342 sizeof(struct kni_interface_stats), 343 RTE_CACHE_LINE_SIZE); 344 345 if (kni_stat[port_id] == NULL) 346 rte_panic("rte_zmalloc kni_interface_stats failed\n"); 347 348 /* only support one kni */ 349 memset(&conf, 0, sizeof(conf)); 350 snprintf(conf.name, RTE_KNI_NAMESIZE, "veth%u", port_id); 351 conf.core_id = rte_lcore_id(); 352 conf.force_bind = 1; 353 conf.group_id = port_id; 354 uint16_t mtu; 355 rte_eth_dev_get_mtu(port_id, &mtu); 356 conf.mbuf_size = mtu + KNI_ENET_HEADER_SIZE + KNI_ENET_FCS_SIZE; 357 358 memset(&dev_info, 0, sizeof(dev_info)); 359 rte_eth_dev_info_get(port_id, &dev_info); 360 conf.addr = dev_info.pci_dev->addr; 361 conf.id = dev_info.pci_dev->id; 362 363 memset(&ops, 0, sizeof(ops)); 364 ops.port_id = port_id; 365 ops.change_mtu = kni_change_mtu; 366 ops.config_network_if = kni_config_network_interface; 367 368 kni_stat[port_id]->kni = rte_kni_alloc(mbuf_pool, &conf, &ops); 369 if (kni_stat[port_id]->kni == NULL) 370 rte_panic("create kni on port %u failed!\n", port_id); 371 else 372 printf("create kni on port %u success!\n", port_id); 373 374 kni_stat[port_id]->rx_packets = 0; 375 kni_stat[port_id]->rx_dropped = 0; 376 kni_stat[port_id]->tx_packets = 0; 377 kni_stat[port_id]->tx_dropped = 0; 378 } 379 380 char ring_name[RTE_KNI_NAMESIZE]; 381 snprintf((char*)ring_name, RTE_KNI_NAMESIZE, "kni_ring_%u", port_id); 382 383 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 384 kni_rp[port_id] = rte_ring_create(ring_name, KNI_QUEUE_SIZE, 385 socket_id, RING_F_SC_DEQ); 386 } else { 387 kni_rp[port_id] = rte_ring_lookup(ring_name); 388 } 389 390 if (kni_rp[port_id] == NULL) 391 rte_panic("create kni ring failed!\n"); 392 393 if (rte_ring_lookup(ring_name) != kni_rp[port_id]) 394 rte_panic("lookup kni ring failed!\n"); 395 396 printf("create kni ring success, %u ring entries are now free!\n", 397 rte_ring_free_count(kni_rp[port_id])); 398 } 399 400 401 void 402 ff_kni_process(uint8_t port_id, uint16_t queue_id, 403 struct rte_mbuf **pkts_burst, unsigned count) 404 { 405 kni_process_tx(port_id, queue_id, pkts_burst, count); 406 kni_process_rx(port_id, queue_id, pkts_burst, count); 407 } 408 409 /* enqueue the packet, and own it */ 410 int 411 ff_kni_enqueue(uint8_t port_id, struct rte_mbuf *pkt) 412 { 413 int ret = rte_ring_enqueue(kni_rp[port_id], pkt); 414 if (ret < 0) 415 rte_pktmbuf_free(pkt); 416 417 return 0; 418 } 419 420