1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 27 #include <stdlib.h> 28 #include <arpa/inet.h> 29 30 #include <rte_config.h> 31 #include <rte_ether.h> 32 #include <rte_ethdev.h> 33 #include <rte_kni.h> 34 #include <rte_malloc.h> 35 #include <rte_ring.h> 36 #include <rte_ip.h> 37 #include <rte_tcp.h> 38 #include <rte_udp.h> 39 40 #include "ff_dpdk_kni.h" 41 #include "ff_config.h" 42 43 /* Callback for request of changing MTU */ 44 /* Total octets in ethernet header */ 45 #define KNI_ENET_HEADER_SIZE 14 46 47 /* Total octets in the FCS */ 48 #define KNI_ENET_FCS_SIZE 4 49 50 #define set_bit(n, m) (n | magic_bits[m]) 51 #define clear_bit(n, m) (n & (~magic_bits[m])) 52 #define get_bit(n, m) (n & magic_bits[m]) 53 54 static const int magic_bits[8] = { 55 0x80, 0x40, 0x20, 0x10, 56 0x8, 0x4, 0x2, 0x1 57 }; 58 59 static unsigned char *udp_port_bitmap = NULL; 60 static unsigned char *tcp_port_bitmap = NULL; 61 62 /* Structure type for recording kni interface specific stats */ 63 struct kni_interface_stats { 64 struct rte_kni *kni; 65 66 /* number of pkts received from NIC, and sent to KNI */ 67 uint64_t rx_packets; 68 69 /* number of pkts received from NIC, but failed to send to KNI */ 70 uint64_t rx_dropped; 71 72 /* number of pkts received from KNI, and sent to NIC */ 73 uint64_t tx_packets; 74 75 /* number of pkts received from KNI, but failed to send to NIC */ 76 uint64_t tx_dropped; 77 }; 78 79 struct rte_ring **kni_rp; 80 struct kni_interface_stats **kni_stat; 81 82 static void 83 set_bitmap(uint16_t port, unsigned char *bitmap) 84 { 85 port = htons(port); 86 unsigned char *p = bitmap + port/8; 87 *p = set_bit(*p, port % 8); 88 } 89 90 static int 91 get_bitmap(uint16_t port, unsigned char *bitmap) 92 { 93 unsigned char *p = bitmap + port/8; 94 return get_bit(*p, port % 8) > 0 ? 1 : 0; 95 } 96 97 static void 98 kni_set_bitmap(const char *p, unsigned char *port_bitmap) 99 { 100 int i; 101 const char *head, *tail, *tail_num; 102 if(!p) 103 return; 104 105 head = p; 106 while (1) { 107 tail = strstr(head, ","); 108 tail_num = strstr(head, "-"); 109 if(tail_num && (!tail || tail_num < tail - 1)) { 110 for(i = atoi(head); i <= atoi(tail_num + 1); ++i) { 111 set_bitmap(i, port_bitmap); 112 } 113 } else { 114 set_bitmap(atoi(head), port_bitmap); 115 } 116 117 if(!tail) 118 break; 119 120 head = tail + 1; 121 } 122 } 123 124 /* Currently we don't support change mtu. */ 125 static int 126 kni_change_mtu(uint8_t port_id, unsigned new_mtu) 127 { 128 return 0; 129 } 130 131 static int 132 kni_config_network_interface(uint8_t port_id, uint8_t if_up) 133 { 134 int ret = 0; 135 136 if (port_id >= rte_eth_dev_count() || port_id >= RTE_MAX_ETHPORTS) { 137 printf("Invalid port id %d\n", port_id); 138 return -EINVAL; 139 } 140 141 printf("Configure network interface of %d %s\n", 142 port_id, if_up ? "up" : "down"); 143 144 ret = (if_up) ? 145 rte_eth_dev_set_link_up(port_id) : 146 rte_eth_dev_set_link_down(port_id); 147 148 if(-ENOTSUP == ret) { 149 if (if_up != 0) { 150 /* Configure network interface up */ 151 rte_eth_dev_stop(port_id); 152 ret = rte_eth_dev_start(port_id); 153 } else { 154 /* Configure network interface down */ 155 rte_eth_dev_stop(port_id); 156 ret = 0; 157 } 158 } 159 160 if (ret < 0) 161 printf("Failed to Configure network interface of %d %s\n", 162 port_id, if_up ? "up" : "down"); 163 164 return ret; 165 } 166 167 static int 168 kni_process_tx(uint8_t port_id, uint16_t queue_id, 169 struct rte_mbuf **pkts_burst, unsigned count) 170 { 171 /* read packet from kni ring(phy port) and transmit to kni */ 172 uint16_t nb_tx, nb_kni_tx; 173 nb_tx = rte_ring_dequeue_burst(kni_rp[port_id], (void **)pkts_burst, count); 174 175 /* NB. 176 * if nb_tx is 0,it must call rte_kni_tx_burst 177 * must Call regularly rte_kni_tx_burst(kni, NULL, 0). 178 * detail https://embedded.communities.intel.com/thread/6668 179 */ 180 nb_kni_tx = rte_kni_tx_burst(kni_stat[port_id]->kni, pkts_burst, nb_tx); 181 rte_kni_handle_request(kni_stat[port_id]->kni); 182 if(nb_kni_tx < nb_tx) { 183 uint16_t i; 184 for(i = nb_kni_tx; i < nb_tx; ++i) 185 rte_pktmbuf_free(pkts_burst[i]); 186 187 kni_stat[port_id]->rx_dropped += (nb_tx - nb_kni_tx); 188 } 189 190 kni_stat[port_id]->rx_packets += nb_kni_tx; 191 return 0; 192 } 193 194 static int 195 kni_process_rx(uint8_t port_id, uint16_t queue_id, 196 struct rte_mbuf **pkts_burst, unsigned count) 197 { 198 uint16_t nb_kni_rx, nb_rx; 199 200 /* read packet from kni, and transmit to phy port */ 201 nb_kni_rx = rte_kni_rx_burst(kni_stat[port_id]->kni, pkts_burst, count); 202 if (nb_kni_rx > 0) { 203 nb_rx = rte_eth_tx_burst(port_id, queue_id, pkts_burst, nb_kni_rx); 204 if (nb_rx < nb_kni_rx) { 205 uint16_t i; 206 for(i = nb_rx; i < nb_kni_rx; ++i) 207 rte_pktmbuf_free(pkts_burst[i]); 208 209 kni_stat[port_id]->tx_dropped += (nb_kni_rx - nb_rx); 210 } 211 212 kni_stat[port_id]->tx_packets += nb_rx; 213 } 214 return 0; 215 } 216 217 static enum FilterReturn 218 protocol_filter_l4(uint16_t port, unsigned char *bitmap) 219 { 220 if(get_bitmap(port, bitmap)) { 221 return FILTER_KNI; 222 } 223 224 return FILTER_UNKNOWN; 225 } 226 227 static enum FilterReturn 228 protocol_filter_tcp(const void *data, uint16_t len) 229 { 230 if (len < sizeof(struct tcp_hdr)) 231 return FILTER_UNKNOWN; 232 233 const struct tcp_hdr *hdr; 234 hdr = (const struct tcp_hdr *)data; 235 236 return protocol_filter_l4(hdr->dst_port, tcp_port_bitmap); 237 } 238 239 static enum FilterReturn 240 protocol_filter_udp(const void* data,uint16_t len) 241 { 242 if (len < sizeof(struct udp_hdr)) 243 return FILTER_UNKNOWN; 244 245 const struct udp_hdr *hdr; 246 hdr = (const struct udp_hdr *)data; 247 248 return protocol_filter_l4(hdr->dst_port, udp_port_bitmap); 249 } 250 251 static enum FilterReturn 252 protocol_filter_ip(const void *data, uint16_t len) 253 { 254 if(len < sizeof(struct ipv4_hdr)) 255 return FILTER_UNKNOWN; 256 257 const struct ipv4_hdr *hdr; 258 hdr = (const struct ipv4_hdr *)data; 259 260 void *next = (void *)data + sizeof(struct ipv4_hdr); 261 uint16_t next_len = len - sizeof(struct ipv4_hdr); 262 263 switch (hdr->next_proto_id) { 264 case IPPROTO_TCP: 265 return protocol_filter_tcp(next, next_len); 266 case IPPROTO_UDP: 267 return protocol_filter_udp(next, next_len); 268 case IPPROTO_IPIP: 269 return protocol_filter_ip(next, next_len); 270 } 271 272 return FILTER_UNKNOWN; 273 } 274 275 enum FilterReturn 276 ff_kni_proto_filter(const void *data, uint16_t len) 277 { 278 return protocol_filter_ip(data, len); 279 } 280 281 void 282 ff_kni_init(uint16_t nb_ports, const char *tcp_ports, const char *udp_ports) 283 { 284 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 285 kni_stat = rte_zmalloc("kni:stat", 286 sizeof(struct kni_interface_stats *) * nb_ports, 287 RTE_CACHE_LINE_SIZE); 288 if (kni_stat == NULL) 289 rte_exit(EXIT_FAILURE, "rte_zmalloc(1 (struct netio_kni_stat *)) " 290 "failed\n"); 291 292 rte_kni_init(nb_ports); 293 } 294 295 uint16_t lcoreid = rte_lcore_id(); 296 char name_buf[RTE_RING_NAMESIZE]; 297 snprintf(name_buf, RTE_RING_NAMESIZE, "kni::ring_%d", lcoreid); 298 kni_rp = rte_zmalloc(name_buf, 299 sizeof(struct rte_ring *) * nb_ports, 300 RTE_CACHE_LINE_SIZE); 301 if (kni_rp == NULL) { 302 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 303 "failed\n", name_buf); 304 } 305 306 snprintf(name_buf, RTE_RING_NAMESIZE, "kni:tcp_port_bitmap_%d", lcoreid); 307 tcp_port_bitmap = rte_zmalloc("kni:tcp_port_bitmap", 8192, 308 RTE_CACHE_LINE_SIZE); 309 if (tcp_port_bitmap == NULL) { 310 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (tcp_port_bitmap)) " 311 "failed\n", name_buf); 312 } 313 314 snprintf(name_buf, RTE_RING_NAMESIZE, "kni:udp_port_bitmap_%d", lcoreid); 315 udp_port_bitmap = rte_zmalloc("kni:udp_port_bitmap", 8192, 316 RTE_CACHE_LINE_SIZE); 317 if (udp_port_bitmap == NULL) { 318 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (udp_port_bitmap)) " 319 "failed\n",name_buf); 320 } 321 322 memset(tcp_port_bitmap, 0, 8192); 323 memset(udp_port_bitmap, 0, 8192); 324 325 kni_set_bitmap(tcp_ports, tcp_port_bitmap); 326 kni_set_bitmap(udp_ports, udp_port_bitmap); 327 } 328 329 void 330 ff_kni_alloc(uint8_t port_id, unsigned socket_id, 331 struct rte_mempool *mbuf_pool, unsigned ring_queue_size) 332 { 333 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 334 struct rte_kni_conf conf; 335 struct rte_kni_ops ops; 336 struct rte_eth_dev_info dev_info; 337 338 kni_stat[port_id] = (struct kni_interface_stats*)rte_zmalloc( 339 "kni:stat_lcore", 340 sizeof(struct kni_interface_stats), 341 RTE_CACHE_LINE_SIZE); 342 343 if (kni_stat[port_id] == NULL) 344 rte_panic("rte_zmalloc kni_interface_stats failed\n"); 345 346 /* only support one kni */ 347 memset(&conf, 0, sizeof(conf)); 348 snprintf(conf.name, RTE_KNI_NAMESIZE, "veth%u", port_id); 349 conf.core_id = rte_lcore_id(); 350 conf.force_bind = 1; 351 conf.group_id = port_id; 352 uint16_t mtu; 353 rte_eth_dev_get_mtu(port_id, &mtu); 354 conf.mbuf_size = mtu + KNI_ENET_HEADER_SIZE + KNI_ENET_FCS_SIZE; 355 356 memset(&dev_info, 0, sizeof(dev_info)); 357 rte_eth_dev_info_get(port_id, &dev_info); 358 conf.addr = dev_info.pci_dev->addr; 359 conf.id = dev_info.pci_dev->id; 360 361 memset(&ops, 0, sizeof(ops)); 362 ops.port_id = port_id; 363 ops.change_mtu = kni_change_mtu; 364 ops.config_network_if = kni_config_network_interface; 365 366 kni_stat[port_id]->kni = rte_kni_alloc(mbuf_pool, &conf, &ops); 367 if (kni_stat[port_id]->kni == NULL) 368 rte_panic("create kni on port %u failed!\n", port_id); 369 else 370 printf("create kni on port %u success!\n", port_id); 371 372 kni_stat[port_id]->rx_packets = 0; 373 kni_stat[port_id]->rx_dropped = 0; 374 kni_stat[port_id]->tx_packets = 0; 375 kni_stat[port_id]->tx_dropped = 0; 376 } 377 378 char ring_name[RTE_KNI_NAMESIZE]; 379 snprintf((char*)ring_name, RTE_KNI_NAMESIZE, "kni_ring_%u", port_id); 380 381 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 382 kni_rp[port_id] = rte_ring_create(ring_name, ring_queue_size, 383 socket_id, RING_F_SC_DEQ); 384 385 if (rte_ring_lookup(ring_name) != kni_rp[port_id]) 386 rte_panic("lookup kni ring failed!\n"); 387 } else { 388 kni_rp[port_id] = rte_ring_lookup(ring_name); 389 } 390 391 if (kni_rp[port_id] == NULL) 392 rte_panic("create kni ring failed!\n"); 393 394 printf("create kni ring success, %u ring entries are now free!\n", 395 rte_ring_free_count(kni_rp[port_id])); 396 } 397 398 399 void 400 ff_kni_process(uint8_t port_id, uint16_t queue_id, 401 struct rte_mbuf **pkts_burst, unsigned count) 402 { 403 kni_process_tx(port_id, queue_id, pkts_burst, count); 404 kni_process_rx(port_id, queue_id, pkts_burst, count); 405 } 406 407 /* enqueue the packet, and own it */ 408 int 409 ff_kni_enqueue(uint8_t port_id, struct rte_mbuf *pkt) 410 { 411 int ret = rte_ring_enqueue(kni_rp[port_id], pkt); 412 if (ret < 0) 413 rte_pktmbuf_free(pkt); 414 415 return 0; 416 } 417 418