1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <netinet/in.h> 11 #include <netinet/ip.h> 12 #include <netinet/ip6.h> 13 #include <string.h> 14 #include <sys/queue.h> 15 #include <stdarg.h> 16 #include <errno.h> 17 #include <getopt.h> 18 19 #include <rte_common.h> 20 #include <rte_byteorder.h> 21 #include <rte_log.h> 22 #include <rte_eal.h> 23 #include <rte_launch.h> 24 #include <rte_atomic.h> 25 #include <rte_cycles.h> 26 #include <rte_prefetch.h> 27 #include <rte_lcore.h> 28 #include <rte_per_lcore.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_interrupts.h> 31 #include <rte_random.h> 32 #include <rte_debug.h> 33 #include <rte_ether.h> 34 #include <rte_ethdev.h> 35 #include <rte_mempool.h> 36 #include <rte_mbuf.h> 37 #include <rte_acl.h> 38 #include <rte_lpm.h> 39 #include <rte_lpm6.h> 40 #include <rte_hash.h> 41 #include <rte_jhash.h> 42 #include <rte_cryptodev.h> 43 #include <rte_security.h> 44 #include <rte_ip.h> 45 #include <rte_ip_frag.h> 46 47 #include "ipsec.h" 48 #include "parser.h" 49 50 #define RTE_LOGTYPE_IPSEC RTE_LOGTYPE_USER1 51 52 #define MAX_JUMBO_PKT_LEN 9600 53 54 #define MEMPOOL_CACHE_SIZE 256 55 56 #define NB_MBUF (32000) 57 58 #define CDEV_QUEUE_DESC 2048 59 #define CDEV_MAP_ENTRIES 16384 60 #define CDEV_MP_NB_OBJS 1024 61 #define CDEV_MP_CACHE_SZ 64 62 #define MAX_QUEUE_PAIRS 1 63 64 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 65 66 #define NB_SOCKETS 4 67 68 /* Configure how many packets ahead to prefetch, when reading packets */ 69 #define PREFETCH_OFFSET 3 70 71 #define MAX_RX_QUEUE_PER_LCORE 16 72 73 #define MAX_LCORE_PARAMS 1024 74 75 #define UNPROTECTED_PORT(port) (unprotected_port_mask & (1 << portid)) 76 77 /* 78 * Configurable number of RX/TX ring descriptors 79 */ 80 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024 81 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024 82 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 83 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 84 85 #if RTE_BYTE_ORDER != RTE_LITTLE_ENDIAN 86 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 87 (((uint64_t)((a) & 0xff) << 56) | \ 88 ((uint64_t)((b) & 0xff) << 48) | \ 89 ((uint64_t)((c) & 0xff) << 40) | \ 90 ((uint64_t)((d) & 0xff) << 32) | \ 91 ((uint64_t)((e) & 0xff) << 24) | \ 92 ((uint64_t)((f) & 0xff) << 16) | \ 93 ((uint64_t)((g) & 0xff) << 8) | \ 94 ((uint64_t)(h) & 0xff)) 95 #else 96 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 97 (((uint64_t)((h) & 0xff) << 56) | \ 98 ((uint64_t)((g) & 0xff) << 48) | \ 99 ((uint64_t)((f) & 0xff) << 40) | \ 100 ((uint64_t)((e) & 0xff) << 32) | \ 101 ((uint64_t)((d) & 0xff) << 24) | \ 102 ((uint64_t)((c) & 0xff) << 16) | \ 103 ((uint64_t)((b) & 0xff) << 8) | \ 104 ((uint64_t)(a) & 0xff)) 105 #endif 106 #define ETHADDR(a, b, c, d, e, f) (__BYTES_TO_UINT64(a, b, c, d, e, f, 0, 0)) 107 108 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 109 (addr)->addr_bytes[0], (addr)->addr_bytes[1], \ 110 (addr)->addr_bytes[2], (addr)->addr_bytes[3], \ 111 (addr)->addr_bytes[4], (addr)->addr_bytes[5], \ 112 0, 0) 113 114 #define FRAG_TBL_BUCKET_ENTRIES 4 115 #define FRAG_TTL_MS (10 * MS_PER_S) 116 117 #define MTU_TO_FRAMELEN(x) ((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN) 118 119 /* port/source ethernet addr and destination ethernet addr */ 120 struct ethaddr_info { 121 uint64_t src, dst; 122 }; 123 124 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 125 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 126 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 127 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 128 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 129 }; 130 131 #define CMD_LINE_OPT_CONFIG "config" 132 #define CMD_LINE_OPT_SINGLE_SA "single-sa" 133 #define CMD_LINE_OPT_CRYPTODEV_MASK "cryptodev_mask" 134 #define CMD_LINE_OPT_RX_OFFLOAD "rxoffload" 135 #define CMD_LINE_OPT_TX_OFFLOAD "txoffload" 136 #define CMD_LINE_OPT_REASSEMBLE "reassemble" 137 #define CMD_LINE_OPT_MTU "mtu" 138 139 enum { 140 /* long options mapped to a short option */ 141 142 /* first long only option value must be >= 256, so that we won't 143 * conflict with short options 144 */ 145 CMD_LINE_OPT_MIN_NUM = 256, 146 CMD_LINE_OPT_CONFIG_NUM, 147 CMD_LINE_OPT_SINGLE_SA_NUM, 148 CMD_LINE_OPT_CRYPTODEV_MASK_NUM, 149 CMD_LINE_OPT_RX_OFFLOAD_NUM, 150 CMD_LINE_OPT_TX_OFFLOAD_NUM, 151 CMD_LINE_OPT_REASSEMBLE_NUM, 152 CMD_LINE_OPT_MTU_NUM, 153 }; 154 155 static const struct option lgopts[] = { 156 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 157 {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM}, 158 {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM}, 159 {CMD_LINE_OPT_RX_OFFLOAD, 1, 0, CMD_LINE_OPT_RX_OFFLOAD_NUM}, 160 {CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM}, 161 {CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM}, 162 {CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM}, 163 {NULL, 0, 0, 0} 164 }; 165 166 /* mask of enabled ports */ 167 static uint32_t enabled_port_mask; 168 static uint64_t enabled_cryptodev_mask = UINT64_MAX; 169 static uint32_t unprotected_port_mask; 170 static int32_t promiscuous_on = 1; 171 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 172 static uint32_t nb_lcores; 173 static uint32_t single_sa; 174 static uint32_t single_sa_idx; 175 176 /* 177 * RX/TX HW offload capabilities to enable/use on ethernet ports. 178 * By default all capabilities are enabled. 179 */ 180 static uint64_t dev_rx_offload = UINT64_MAX; 181 static uint64_t dev_tx_offload = UINT64_MAX; 182 183 /* 184 * global values that determine multi-seg policy 185 */ 186 static uint32_t frag_tbl_sz; 187 static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE; 188 static uint32_t mtu_size = RTE_ETHER_MTU; 189 190 /* application wide librte_ipsec/SA parameters */ 191 struct app_sa_prm app_sa_prm = {.enable = 0}; 192 193 struct lcore_rx_queue { 194 uint16_t port_id; 195 uint8_t queue_id; 196 } __rte_cache_aligned; 197 198 struct lcore_params { 199 uint16_t port_id; 200 uint8_t queue_id; 201 uint8_t lcore_id; 202 } __rte_cache_aligned; 203 204 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 205 206 static struct lcore_params *lcore_params; 207 static uint16_t nb_lcore_params; 208 209 static struct rte_hash *cdev_map_in; 210 static struct rte_hash *cdev_map_out; 211 212 struct buffer { 213 uint16_t len; 214 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 215 }; 216 217 struct lcore_conf { 218 uint16_t nb_rx_queue; 219 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 220 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 221 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 222 struct ipsec_ctx inbound; 223 struct ipsec_ctx outbound; 224 struct rt_ctx *rt4_ctx; 225 struct rt_ctx *rt6_ctx; 226 struct { 227 struct rte_ip_frag_tbl *tbl; 228 struct rte_mempool *pool_dir; 229 struct rte_mempool *pool_indir; 230 struct rte_ip_frag_death_row dr; 231 } frag; 232 } __rte_cache_aligned; 233 234 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 235 236 static struct rte_eth_conf port_conf = { 237 .rxmode = { 238 .mq_mode = ETH_MQ_RX_RSS, 239 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 240 .split_hdr_size = 0, 241 .offloads = DEV_RX_OFFLOAD_CHECKSUM, 242 }, 243 .rx_adv_conf = { 244 .rss_conf = { 245 .rss_key = NULL, 246 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 247 ETH_RSS_TCP | ETH_RSS_SCTP, 248 }, 249 }, 250 .txmode = { 251 .mq_mode = ETH_MQ_TX_NONE, 252 }, 253 }; 254 255 static struct socket_ctx socket_ctx[NB_SOCKETS]; 256 257 /* 258 * Determine is multi-segment support required: 259 * - either frame buffer size is smaller then mtu 260 * - or reassmeble support is requested 261 */ 262 static int 263 multi_seg_required(void) 264 { 265 return (MTU_TO_FRAMELEN(mtu_size) + RTE_PKTMBUF_HEADROOM > 266 frame_buf_size || frag_tbl_sz != 0); 267 } 268 269 static inline void 270 adjust_ipv4_pktlen(struct rte_mbuf *m, const struct rte_ipv4_hdr *iph, 271 uint32_t l2_len) 272 { 273 uint32_t plen, trim; 274 275 plen = rte_be_to_cpu_16(iph->total_length) + l2_len; 276 if (plen < m->pkt_len) { 277 trim = m->pkt_len - plen; 278 rte_pktmbuf_trim(m, trim); 279 } 280 } 281 282 static inline void 283 adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, 284 uint32_t l2_len) 285 { 286 uint32_t plen, trim; 287 288 plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len; 289 if (plen < m->pkt_len) { 290 trim = m->pkt_len - plen; 291 rte_pktmbuf_trim(m, trim); 292 } 293 } 294 295 static inline void 296 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 297 { 298 const struct rte_ether_hdr *eth; 299 const struct rte_ipv4_hdr *iph4; 300 const struct rte_ipv6_hdr *iph6; 301 302 eth = rte_pktmbuf_mtod(pkt, const struct rte_ether_hdr *); 303 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 304 305 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt, 306 RTE_ETHER_HDR_LEN); 307 adjust_ipv4_pktlen(pkt, iph4, 0); 308 309 if (iph4->next_proto_id == IPPROTO_ESP) 310 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 311 else { 312 t->ip4.data[t->ip4.num] = &iph4->next_proto_id; 313 t->ip4.pkts[(t->ip4.num)++] = pkt; 314 } 315 pkt->l2_len = 0; 316 pkt->l3_len = sizeof(*iph4); 317 } else if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 318 int next_proto; 319 size_t l3len, ext_len; 320 uint8_t *p; 321 322 /* get protocol type */ 323 iph6 = (const struct rte_ipv6_hdr *)rte_pktmbuf_adj(pkt, 324 RTE_ETHER_HDR_LEN); 325 adjust_ipv6_pktlen(pkt, iph6, 0); 326 327 next_proto = iph6->proto; 328 329 /* determine l3 header size up to ESP extension */ 330 l3len = sizeof(struct ip6_hdr); 331 p = rte_pktmbuf_mtod(pkt, uint8_t *); 332 while (next_proto != IPPROTO_ESP && l3len < pkt->data_len && 333 (next_proto = rte_ipv6_get_next_ext(p + l3len, 334 next_proto, &ext_len)) >= 0) 335 l3len += ext_len; 336 337 /* drop packet when IPv6 header exceeds first segment length */ 338 if (unlikely(l3len > pkt->data_len)) { 339 rte_pktmbuf_free(pkt); 340 return; 341 } 342 343 if (next_proto == IPPROTO_ESP) 344 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 345 else { 346 t->ip6.data[t->ip6.num] = &iph6->proto; 347 t->ip6.pkts[(t->ip6.num)++] = pkt; 348 } 349 pkt->l2_len = 0; 350 pkt->l3_len = l3len; 351 } else { 352 /* Unknown/Unsupported type, drop the packet */ 353 RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n", 354 rte_be_to_cpu_16(eth->ether_type)); 355 rte_pktmbuf_free(pkt); 356 } 357 358 /* Check if the packet has been processed inline. For inline protocol 359 * processed packets, the metadata in the mbuf can be used to identify 360 * the security processing done on the packet. The metadata will be 361 * used to retrieve the application registered userdata associated 362 * with the security session. 363 */ 364 365 if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) { 366 struct ipsec_sa *sa; 367 struct ipsec_mbuf_metadata *priv; 368 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 369 rte_eth_dev_get_sec_ctx( 370 pkt->port); 371 372 /* Retrieve the userdata registered. Here, the userdata 373 * registered is the SA pointer. 374 */ 375 376 sa = (struct ipsec_sa *) 377 rte_security_get_userdata(ctx, pkt->udata64); 378 379 if (sa == NULL) { 380 /* userdata could not be retrieved */ 381 return; 382 } 383 384 /* Save SA as priv member in mbuf. This will be used in the 385 * IPsec selector(SP-SA) check. 386 */ 387 388 priv = get_priv(pkt); 389 priv->sa = sa; 390 } 391 } 392 393 static inline void 394 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 395 uint16_t nb_pkts) 396 { 397 int32_t i; 398 399 t->ipsec.num = 0; 400 t->ip4.num = 0; 401 t->ip6.num = 0; 402 403 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 404 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 405 void *)); 406 prepare_one_packet(pkts[i], t); 407 } 408 /* Process left packets */ 409 for (; i < nb_pkts; i++) 410 prepare_one_packet(pkts[i], t); 411 } 412 413 static inline void 414 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port, 415 const struct lcore_conf *qconf) 416 { 417 struct ip *ip; 418 struct rte_ether_hdr *ethhdr; 419 420 ip = rte_pktmbuf_mtod(pkt, struct ip *); 421 422 ethhdr = (struct rte_ether_hdr *) 423 rte_pktmbuf_prepend(pkt, RTE_ETHER_HDR_LEN); 424 425 if (ip->ip_v == IPVERSION) { 426 pkt->ol_flags |= qconf->outbound.ipv4_offloads; 427 pkt->l3_len = sizeof(struct ip); 428 pkt->l2_len = RTE_ETHER_HDR_LEN; 429 430 ip->ip_sum = 0; 431 432 /* calculate IPv4 cksum in SW */ 433 if ((pkt->ol_flags & PKT_TX_IP_CKSUM) == 0) 434 ip->ip_sum = rte_ipv4_cksum((struct rte_ipv4_hdr *)ip); 435 436 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 437 } else { 438 pkt->ol_flags |= qconf->outbound.ipv6_offloads; 439 pkt->l3_len = sizeof(struct ip6_hdr); 440 pkt->l2_len = RTE_ETHER_HDR_LEN; 441 442 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 443 } 444 445 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 446 sizeof(struct rte_ether_addr)); 447 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 448 sizeof(struct rte_ether_addr)); 449 } 450 451 static inline void 452 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port, 453 const struct lcore_conf *qconf) 454 { 455 int32_t i; 456 const int32_t prefetch_offset = 2; 457 458 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 459 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 460 prepare_tx_pkt(pkts[i], port, qconf); 461 } 462 /* Process left packets */ 463 for (; i < nb_pkts; i++) 464 prepare_tx_pkt(pkts[i], port, qconf); 465 } 466 467 /* Send burst of packets on an output interface */ 468 static inline int32_t 469 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 470 { 471 struct rte_mbuf **m_table; 472 int32_t ret; 473 uint16_t queueid; 474 475 queueid = qconf->tx_queue_id[port]; 476 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 477 478 prepare_tx_burst(m_table, n, port, qconf); 479 480 ret = rte_eth_tx_burst(port, queueid, m_table, n); 481 if (unlikely(ret < n)) { 482 do { 483 rte_pktmbuf_free(m_table[ret]); 484 } while (++ret < n); 485 } 486 487 return 0; 488 } 489 490 /* 491 * Helper function to fragment and queue for TX one packet. 492 */ 493 static inline uint32_t 494 send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m, 495 uint16_t port, uint8_t proto) 496 { 497 struct buffer *tbl; 498 uint32_t len, n; 499 int32_t rc; 500 501 tbl = qconf->tx_mbufs + port; 502 len = tbl->len; 503 504 /* free space for new fragments */ 505 if (len + RTE_LIBRTE_IP_FRAG_MAX_FRAG >= RTE_DIM(tbl->m_table)) { 506 send_burst(qconf, len, port); 507 len = 0; 508 } 509 510 n = RTE_DIM(tbl->m_table) - len; 511 512 if (proto == IPPROTO_IP) 513 rc = rte_ipv4_fragment_packet(m, tbl->m_table + len, 514 n, mtu_size, qconf->frag.pool_dir, 515 qconf->frag.pool_indir); 516 else 517 rc = rte_ipv6_fragment_packet(m, tbl->m_table + len, 518 n, mtu_size, qconf->frag.pool_dir, 519 qconf->frag.pool_indir); 520 521 if (rc >= 0) 522 len += rc; 523 else 524 RTE_LOG(ERR, IPSEC, 525 "%s: failed to fragment packet with size %u, " 526 "error code: %d\n", 527 __func__, m->pkt_len, rte_errno); 528 529 rte_pktmbuf_free(m); 530 return len; 531 } 532 533 /* Enqueue a single packet, and send burst if queue is filled */ 534 static inline int32_t 535 send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) 536 { 537 uint32_t lcore_id; 538 uint16_t len; 539 struct lcore_conf *qconf; 540 541 lcore_id = rte_lcore_id(); 542 543 qconf = &lcore_conf[lcore_id]; 544 len = qconf->tx_mbufs[port].len; 545 546 if (m->pkt_len <= mtu_size) { 547 qconf->tx_mbufs[port].m_table[len] = m; 548 len++; 549 550 /* need to fragment the packet */ 551 } else 552 len = send_fragment_packet(qconf, m, port, proto); 553 554 /* enough pkts to be sent */ 555 if (unlikely(len == MAX_PKT_BURST)) { 556 send_burst(qconf, MAX_PKT_BURST, port); 557 len = 0; 558 } 559 560 qconf->tx_mbufs[port].len = len; 561 return 0; 562 } 563 564 static inline void 565 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 566 uint16_t lim) 567 { 568 struct rte_mbuf *m; 569 uint32_t i, j, res, sa_idx; 570 571 if (ip->num == 0 || sp == NULL) 572 return; 573 574 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 575 ip->num, DEFAULT_MAX_CATEGORIES); 576 577 j = 0; 578 for (i = 0; i < ip->num; i++) { 579 m = ip->pkts[i]; 580 res = ip->res[i]; 581 if (res == BYPASS) { 582 ip->pkts[j++] = m; 583 continue; 584 } 585 if (res == DISCARD) { 586 rte_pktmbuf_free(m); 587 continue; 588 } 589 590 /* Only check SPI match for processed IPSec packets */ 591 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) { 592 rte_pktmbuf_free(m); 593 continue; 594 } 595 596 sa_idx = SPI2IDX(res); 597 if (!inbound_sa_check(sa, m, sa_idx)) { 598 rte_pktmbuf_free(m); 599 continue; 600 } 601 ip->pkts[j++] = m; 602 } 603 ip->num = j; 604 } 605 606 static void 607 split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) 608 { 609 uint32_t i, n4, n6; 610 struct ip *ip; 611 struct rte_mbuf *m; 612 613 n4 = trf->ip4.num; 614 n6 = trf->ip6.num; 615 616 for (i = 0; i < num; i++) { 617 618 m = mb[i]; 619 ip = rte_pktmbuf_mtod(m, struct ip *); 620 621 if (ip->ip_v == IPVERSION) { 622 trf->ip4.pkts[n4] = m; 623 trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m, 624 uint8_t *, offsetof(struct ip, ip_p)); 625 n4++; 626 } else if (ip->ip_v == IP6_VERSION) { 627 trf->ip6.pkts[n6] = m; 628 trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m, 629 uint8_t *, 630 offsetof(struct ip6_hdr, ip6_nxt)); 631 n6++; 632 } else 633 rte_pktmbuf_free(m); 634 } 635 636 trf->ip4.num = n4; 637 trf->ip6.num = n6; 638 } 639 640 641 static inline void 642 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 643 struct ipsec_traffic *traffic) 644 { 645 uint16_t nb_pkts_in, n_ip4, n_ip6; 646 647 n_ip4 = traffic->ip4.num; 648 n_ip6 = traffic->ip6.num; 649 650 if (app_sa_prm.enable == 0) { 651 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 652 traffic->ipsec.num, MAX_PKT_BURST); 653 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in); 654 } else { 655 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 656 traffic->ipsec.saptr, traffic->ipsec.num); 657 ipsec_process(ipsec_ctx, traffic); 658 } 659 660 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 661 n_ip4); 662 663 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 664 n_ip6); 665 } 666 667 static inline void 668 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 669 struct traffic_type *ipsec) 670 { 671 struct rte_mbuf *m; 672 uint32_t i, j, sa_idx; 673 674 if (ip->num == 0 || sp == NULL) 675 return; 676 677 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 678 ip->num, DEFAULT_MAX_CATEGORIES); 679 680 j = 0; 681 for (i = 0; i < ip->num; i++) { 682 m = ip->pkts[i]; 683 sa_idx = SPI2IDX(ip->res[i]); 684 if (ip->res[i] == DISCARD) 685 rte_pktmbuf_free(m); 686 else if (ip->res[i] == BYPASS) 687 ip->pkts[j++] = m; 688 else { 689 ipsec->res[ipsec->num] = sa_idx; 690 ipsec->pkts[ipsec->num++] = m; 691 } 692 } 693 ip->num = j; 694 } 695 696 static inline void 697 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 698 struct ipsec_traffic *traffic) 699 { 700 struct rte_mbuf *m; 701 uint16_t idx, nb_pkts_out, i; 702 703 /* Drop any IPsec traffic from protected ports */ 704 for (i = 0; i < traffic->ipsec.num; i++) 705 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 706 707 traffic->ipsec.num = 0; 708 709 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 710 711 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 712 713 if (app_sa_prm.enable == 0) { 714 715 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 716 traffic->ipsec.res, traffic->ipsec.num, 717 MAX_PKT_BURST); 718 719 for (i = 0; i < nb_pkts_out; i++) { 720 m = traffic->ipsec.pkts[i]; 721 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 722 if (ip->ip_v == IPVERSION) { 723 idx = traffic->ip4.num++; 724 traffic->ip4.pkts[idx] = m; 725 } else { 726 idx = traffic->ip6.num++; 727 traffic->ip6.pkts[idx] = m; 728 } 729 } 730 } else { 731 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 732 traffic->ipsec.saptr, traffic->ipsec.num); 733 ipsec_process(ipsec_ctx, traffic); 734 } 735 } 736 737 static inline void 738 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 739 struct ipsec_traffic *traffic) 740 { 741 struct rte_mbuf *m; 742 uint32_t nb_pkts_in, i, idx; 743 744 /* Drop any IPv4 traffic from unprotected ports */ 745 for (i = 0; i < traffic->ip4.num; i++) 746 rte_pktmbuf_free(traffic->ip4.pkts[i]); 747 748 traffic->ip4.num = 0; 749 750 /* Drop any IPv6 traffic from unprotected ports */ 751 for (i = 0; i < traffic->ip6.num; i++) 752 rte_pktmbuf_free(traffic->ip6.pkts[i]); 753 754 traffic->ip6.num = 0; 755 756 if (app_sa_prm.enable == 0) { 757 758 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 759 traffic->ipsec.num, MAX_PKT_BURST); 760 761 for (i = 0; i < nb_pkts_in; i++) { 762 m = traffic->ipsec.pkts[i]; 763 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 764 if (ip->ip_v == IPVERSION) { 765 idx = traffic->ip4.num++; 766 traffic->ip4.pkts[idx] = m; 767 } else { 768 idx = traffic->ip6.num++; 769 traffic->ip6.pkts[idx] = m; 770 } 771 } 772 } else { 773 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 774 traffic->ipsec.saptr, traffic->ipsec.num); 775 ipsec_process(ipsec_ctx, traffic); 776 } 777 } 778 779 static inline void 780 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 781 struct ipsec_traffic *traffic) 782 { 783 struct rte_mbuf *m; 784 uint32_t nb_pkts_out, i, n; 785 struct ip *ip; 786 787 /* Drop any IPsec traffic from protected ports */ 788 for (i = 0; i < traffic->ipsec.num; i++) 789 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 790 791 n = 0; 792 793 for (i = 0; i < traffic->ip4.num; i++) { 794 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i]; 795 traffic->ipsec.res[n++] = single_sa_idx; 796 } 797 798 for (i = 0; i < traffic->ip6.num; i++) { 799 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i]; 800 traffic->ipsec.res[n++] = single_sa_idx; 801 } 802 803 traffic->ip4.num = 0; 804 traffic->ip6.num = 0; 805 traffic->ipsec.num = n; 806 807 if (app_sa_prm.enable == 0) { 808 809 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 810 traffic->ipsec.res, traffic->ipsec.num, 811 MAX_PKT_BURST); 812 813 /* They all sue the same SA (ip4 or ip6 tunnel) */ 814 m = traffic->ipsec.pkts[0]; 815 ip = rte_pktmbuf_mtod(m, struct ip *); 816 if (ip->ip_v == IPVERSION) { 817 traffic->ip4.num = nb_pkts_out; 818 for (i = 0; i < nb_pkts_out; i++) 819 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i]; 820 } else { 821 traffic->ip6.num = nb_pkts_out; 822 for (i = 0; i < nb_pkts_out; i++) 823 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i]; 824 } 825 } else { 826 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 827 traffic->ipsec.saptr, traffic->ipsec.num); 828 ipsec_process(ipsec_ctx, traffic); 829 } 830 } 831 832 static inline int32_t 833 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) 834 { 835 struct ipsec_mbuf_metadata *priv; 836 struct ipsec_sa *sa; 837 838 priv = get_priv(pkt); 839 840 sa = priv->sa; 841 if (unlikely(sa == NULL)) { 842 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); 843 goto fail; 844 } 845 846 if (is_ipv6) 847 return sa->portid; 848 849 /* else */ 850 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); 851 852 fail: 853 if (is_ipv6) 854 return -1; 855 856 /* else */ 857 return 0; 858 } 859 860 static inline void 861 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 862 { 863 uint32_t hop[MAX_PKT_BURST * 2]; 864 uint32_t dst_ip[MAX_PKT_BURST * 2]; 865 int32_t pkt_hop = 0; 866 uint16_t i, offset; 867 uint16_t lpm_pkts = 0; 868 869 if (nb_pkts == 0) 870 return; 871 872 /* Need to do an LPM lookup for non-inline packets. Inline packets will 873 * have port ID in the SA 874 */ 875 876 for (i = 0; i < nb_pkts; i++) { 877 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 878 /* Security offload not enabled. So an LPM lookup is 879 * required to get the hop 880 */ 881 offset = offsetof(struct ip, ip_dst); 882 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], 883 uint32_t *, offset); 884 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); 885 lpm_pkts++; 886 } 887 } 888 889 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); 890 891 lpm_pkts = 0; 892 893 for (i = 0; i < nb_pkts; i++) { 894 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 895 /* Read hop from the SA */ 896 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); 897 } else { 898 /* Need to use hop returned by lookup */ 899 pkt_hop = hop[lpm_pkts++]; 900 } 901 902 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { 903 rte_pktmbuf_free(pkts[i]); 904 continue; 905 } 906 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP); 907 } 908 } 909 910 static inline void 911 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 912 { 913 int32_t hop[MAX_PKT_BURST * 2]; 914 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 915 uint8_t *ip6_dst; 916 int32_t pkt_hop = 0; 917 uint16_t i, offset; 918 uint16_t lpm_pkts = 0; 919 920 if (nb_pkts == 0) 921 return; 922 923 /* Need to do an LPM lookup for non-inline packets. Inline packets will 924 * have port ID in the SA 925 */ 926 927 for (i = 0; i < nb_pkts; i++) { 928 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 929 /* Security offload not enabled. So an LPM lookup is 930 * required to get the hop 931 */ 932 offset = offsetof(struct ip6_hdr, ip6_dst); 933 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, 934 offset); 935 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); 936 lpm_pkts++; 937 } 938 } 939 940 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, 941 lpm_pkts); 942 943 lpm_pkts = 0; 944 945 for (i = 0; i < nb_pkts; i++) { 946 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 947 /* Read hop from the SA */ 948 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); 949 } else { 950 /* Need to use hop returned by lookup */ 951 pkt_hop = hop[lpm_pkts++]; 952 } 953 954 if (pkt_hop == -1) { 955 rte_pktmbuf_free(pkts[i]); 956 continue; 957 } 958 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6); 959 } 960 } 961 962 static inline void 963 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 964 uint8_t nb_pkts, uint16_t portid) 965 { 966 struct ipsec_traffic traffic; 967 968 prepare_traffic(pkts, &traffic, nb_pkts); 969 970 if (unlikely(single_sa)) { 971 if (UNPROTECTED_PORT(portid)) 972 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 973 else 974 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 975 } else { 976 if (UNPROTECTED_PORT(portid)) 977 process_pkts_inbound(&qconf->inbound, &traffic); 978 else 979 process_pkts_outbound(&qconf->outbound, &traffic); 980 } 981 982 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 983 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 984 } 985 986 static inline void 987 drain_tx_buffers(struct lcore_conf *qconf) 988 { 989 struct buffer *buf; 990 uint32_t portid; 991 992 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 993 buf = &qconf->tx_mbufs[portid]; 994 if (buf->len == 0) 995 continue; 996 send_burst(qconf, buf->len, portid); 997 buf->len = 0; 998 } 999 } 1000 1001 static inline void 1002 drain_crypto_buffers(struct lcore_conf *qconf) 1003 { 1004 uint32_t i; 1005 struct ipsec_ctx *ctx; 1006 1007 /* drain inbound buffers*/ 1008 ctx = &qconf->inbound; 1009 for (i = 0; i != ctx->nb_qps; i++) { 1010 if (ctx->tbl[i].len != 0) 1011 enqueue_cop_burst(ctx->tbl + i); 1012 } 1013 1014 /* drain outbound buffers*/ 1015 ctx = &qconf->outbound; 1016 for (i = 0; i != ctx->nb_qps; i++) { 1017 if (ctx->tbl[i].len != 0) 1018 enqueue_cop_burst(ctx->tbl + i); 1019 } 1020 } 1021 1022 static void 1023 drain_inbound_crypto_queues(const struct lcore_conf *qconf, 1024 struct ipsec_ctx *ctx) 1025 { 1026 uint32_t n; 1027 struct ipsec_traffic trf; 1028 1029 if (app_sa_prm.enable == 0) { 1030 1031 /* dequeue packets from crypto-queue */ 1032 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts, 1033 RTE_DIM(trf.ipsec.pkts)); 1034 1035 trf.ip4.num = 0; 1036 trf.ip6.num = 0; 1037 1038 /* split traffic by ipv4-ipv6 */ 1039 split46_traffic(&trf, trf.ipsec.pkts, n); 1040 } else 1041 ipsec_cqp_process(ctx, &trf); 1042 1043 /* process ipv4 packets */ 1044 if (trf.ip4.num != 0) { 1045 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0); 1046 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 1047 } 1048 1049 /* process ipv6 packets */ 1050 if (trf.ip6.num != 0) { 1051 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0); 1052 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 1053 } 1054 } 1055 1056 static void 1057 drain_outbound_crypto_queues(const struct lcore_conf *qconf, 1058 struct ipsec_ctx *ctx) 1059 { 1060 uint32_t n; 1061 struct ipsec_traffic trf; 1062 1063 if (app_sa_prm.enable == 0) { 1064 1065 /* dequeue packets from crypto-queue */ 1066 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts, 1067 RTE_DIM(trf.ipsec.pkts)); 1068 1069 trf.ip4.num = 0; 1070 trf.ip6.num = 0; 1071 1072 /* split traffic by ipv4-ipv6 */ 1073 split46_traffic(&trf, trf.ipsec.pkts, n); 1074 } else 1075 ipsec_cqp_process(ctx, &trf); 1076 1077 /* process ipv4 packets */ 1078 if (trf.ip4.num != 0) 1079 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 1080 1081 /* process ipv6 packets */ 1082 if (trf.ip6.num != 0) 1083 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 1084 } 1085 1086 /* main processing loop */ 1087 static int32_t 1088 main_loop(__attribute__((unused)) void *dummy) 1089 { 1090 struct rte_mbuf *pkts[MAX_PKT_BURST]; 1091 uint32_t lcore_id; 1092 uint64_t prev_tsc, diff_tsc, cur_tsc; 1093 int32_t i, nb_rx; 1094 uint16_t portid; 1095 uint8_t queueid; 1096 struct lcore_conf *qconf; 1097 int32_t socket_id; 1098 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 1099 / US_PER_S * BURST_TX_DRAIN_US; 1100 struct lcore_rx_queue *rxql; 1101 1102 prev_tsc = 0; 1103 lcore_id = rte_lcore_id(); 1104 qconf = &lcore_conf[lcore_id]; 1105 rxql = qconf->rx_queue_list; 1106 socket_id = rte_lcore_to_socket_id(lcore_id); 1107 1108 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 1109 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 1110 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 1111 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 1112 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 1113 qconf->inbound.cdev_map = cdev_map_in; 1114 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 1115 qconf->inbound.session_priv_pool = 1116 socket_ctx[socket_id].session_priv_pool; 1117 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 1118 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 1119 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 1120 qconf->outbound.cdev_map = cdev_map_out; 1121 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 1122 qconf->outbound.session_priv_pool = 1123 socket_ctx[socket_id].session_priv_pool; 1124 qconf->frag.pool_dir = socket_ctx[socket_id].mbuf_pool; 1125 qconf->frag.pool_indir = socket_ctx[socket_id].mbuf_pool_indir; 1126 1127 if (qconf->nb_rx_queue == 0) { 1128 RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n", 1129 lcore_id); 1130 return 0; 1131 } 1132 1133 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 1134 1135 for (i = 0; i < qconf->nb_rx_queue; i++) { 1136 portid = rxql[i].port_id; 1137 queueid = rxql[i].queue_id; 1138 RTE_LOG(INFO, IPSEC, 1139 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 1140 lcore_id, portid, queueid); 1141 } 1142 1143 while (1) { 1144 cur_tsc = rte_rdtsc(); 1145 1146 /* TX queue buffer drain */ 1147 diff_tsc = cur_tsc - prev_tsc; 1148 1149 if (unlikely(diff_tsc > drain_tsc)) { 1150 drain_tx_buffers(qconf); 1151 drain_crypto_buffers(qconf); 1152 prev_tsc = cur_tsc; 1153 } 1154 1155 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1156 1157 /* Read packets from RX queues */ 1158 portid = rxql[i].port_id; 1159 queueid = rxql[i].queue_id; 1160 nb_rx = rte_eth_rx_burst(portid, queueid, 1161 pkts, MAX_PKT_BURST); 1162 1163 if (nb_rx > 0) 1164 process_pkts(qconf, pkts, nb_rx, portid); 1165 1166 /* dequeue and process completed crypto-ops */ 1167 if (UNPROTECTED_PORT(portid)) 1168 drain_inbound_crypto_queues(qconf, 1169 &qconf->inbound); 1170 else 1171 drain_outbound_crypto_queues(qconf, 1172 &qconf->outbound); 1173 } 1174 } 1175 } 1176 1177 static int32_t 1178 check_params(void) 1179 { 1180 uint8_t lcore; 1181 uint16_t portid; 1182 uint16_t i; 1183 int32_t socket_id; 1184 1185 if (lcore_params == NULL) { 1186 printf("Error: No port/queue/core mappings\n"); 1187 return -1; 1188 } 1189 1190 for (i = 0; i < nb_lcore_params; ++i) { 1191 lcore = lcore_params[i].lcore_id; 1192 if (!rte_lcore_is_enabled(lcore)) { 1193 printf("error: lcore %hhu is not enabled in " 1194 "lcore mask\n", lcore); 1195 return -1; 1196 } 1197 socket_id = rte_lcore_to_socket_id(lcore); 1198 if (socket_id != 0 && numa_on == 0) { 1199 printf("warning: lcore %hhu is on socket %d " 1200 "with numa off\n", 1201 lcore, socket_id); 1202 } 1203 portid = lcore_params[i].port_id; 1204 if ((enabled_port_mask & (1 << portid)) == 0) { 1205 printf("port %u is not enabled in port mask\n", portid); 1206 return -1; 1207 } 1208 if (!rte_eth_dev_is_valid_port(portid)) { 1209 printf("port %u is not present on the board\n", portid); 1210 return -1; 1211 } 1212 } 1213 return 0; 1214 } 1215 1216 static uint8_t 1217 get_port_nb_rx_queues(const uint16_t port) 1218 { 1219 int32_t queue = -1; 1220 uint16_t i; 1221 1222 for (i = 0; i < nb_lcore_params; ++i) { 1223 if (lcore_params[i].port_id == port && 1224 lcore_params[i].queue_id > queue) 1225 queue = lcore_params[i].queue_id; 1226 } 1227 return (uint8_t)(++queue); 1228 } 1229 1230 static int32_t 1231 init_lcore_rx_queues(void) 1232 { 1233 uint16_t i, nb_rx_queue; 1234 uint8_t lcore; 1235 1236 for (i = 0; i < nb_lcore_params; ++i) { 1237 lcore = lcore_params[i].lcore_id; 1238 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 1239 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1240 printf("error: too many queues (%u) for lcore: %u\n", 1241 nb_rx_queue + 1, lcore); 1242 return -1; 1243 } 1244 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1245 lcore_params[i].port_id; 1246 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1247 lcore_params[i].queue_id; 1248 lcore_conf[lcore].nb_rx_queue++; 1249 } 1250 return 0; 1251 } 1252 1253 /* display usage */ 1254 static void 1255 print_usage(const char *prgname) 1256 { 1257 fprintf(stderr, "%s [EAL options] --" 1258 " -p PORTMASK" 1259 " [-P]" 1260 " [-u PORTMASK]" 1261 " [-j FRAMESIZE]" 1262 " [-l]" 1263 " [-w REPLAY_WINDOW_SIZE]" 1264 " [-e]" 1265 " [-a]" 1266 " -f CONFIG_FILE" 1267 " --config (port,queue,lcore)[,(port,queue,lcore)]" 1268 " [--single-sa SAIDX]" 1269 " [--cryptodev_mask MASK]" 1270 " [--" CMD_LINE_OPT_RX_OFFLOAD " RX_OFFLOAD_MASK]" 1271 " [--" CMD_LINE_OPT_TX_OFFLOAD " TX_OFFLOAD_MASK]" 1272 " [--" CMD_LINE_OPT_REASSEMBLE " REASSEMBLE_TABLE_SIZE]" 1273 " [--" CMD_LINE_OPT_MTU " MTU]" 1274 "\n\n" 1275 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 1276 " -P : Enable promiscuous mode\n" 1277 " -u PORTMASK: Hexadecimal bitmask of unprotected ports\n" 1278 " -j FRAMESIZE: Data buffer size, minimum (and default)\n" 1279 " value: RTE_MBUF_DEFAULT_BUF_SIZE\n" 1280 " -l enables code-path that uses librte_ipsec\n" 1281 " -w REPLAY_WINDOW_SIZE specifies IPsec SQN replay window\n" 1282 " size for each SA\n" 1283 " -e enables ESN\n" 1284 " -a enables SA SQN atomic behaviour\n" 1285 " -f CONFIG_FILE: Configuration file\n" 1286 " --config (port,queue,lcore): Rx queue configuration\n" 1287 " --single-sa SAIDX: Use single SA index for outbound traffic,\n" 1288 " bypassing the SP\n" 1289 " --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n" 1290 " devices to configure\n" 1291 " --" CMD_LINE_OPT_RX_OFFLOAD 1292 ": bitmask of the RX HW offload capabilities to enable/use\n" 1293 " (DEV_RX_OFFLOAD_*)\n" 1294 " --" CMD_LINE_OPT_TX_OFFLOAD 1295 ": bitmask of the TX HW offload capabilities to enable/use\n" 1296 " (DEV_TX_OFFLOAD_*)\n" 1297 " --" CMD_LINE_OPT_REASSEMBLE " NUM" 1298 ": max number of entries in reassemble(fragment) table\n" 1299 " (zero (default value) disables reassembly)\n" 1300 " --" CMD_LINE_OPT_MTU " MTU" 1301 ": MTU value on all ports (default value: 1500)\n" 1302 " outgoing packets with bigger size will be fragmented\n" 1303 " incoming packets with bigger size will be discarded\n" 1304 "\n", 1305 prgname); 1306 } 1307 1308 static int 1309 parse_mask(const char *str, uint64_t *val) 1310 { 1311 char *end; 1312 unsigned long t; 1313 1314 errno = 0; 1315 t = strtoul(str, &end, 0); 1316 if (errno != 0 || end[0] != 0) 1317 return -EINVAL; 1318 1319 *val = t; 1320 return 0; 1321 } 1322 1323 static int32_t 1324 parse_portmask(const char *portmask) 1325 { 1326 char *end = NULL; 1327 unsigned long pm; 1328 1329 /* parse hexadecimal string */ 1330 pm = strtoul(portmask, &end, 16); 1331 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1332 return -1; 1333 1334 if ((pm == 0) && errno) 1335 return -1; 1336 1337 return pm; 1338 } 1339 1340 static int32_t 1341 parse_decimal(const char *str) 1342 { 1343 char *end = NULL; 1344 unsigned long num; 1345 1346 num = strtoul(str, &end, 10); 1347 if ((str[0] == '\0') || (end == NULL) || (*end != '\0')) 1348 return -1; 1349 1350 return num; 1351 } 1352 1353 static int32_t 1354 parse_config(const char *q_arg) 1355 { 1356 char s[256]; 1357 const char *p, *p0 = q_arg; 1358 char *end; 1359 enum fieldnames { 1360 FLD_PORT = 0, 1361 FLD_QUEUE, 1362 FLD_LCORE, 1363 _NUM_FLD 1364 }; 1365 unsigned long int_fld[_NUM_FLD]; 1366 char *str_fld[_NUM_FLD]; 1367 int32_t i; 1368 uint32_t size; 1369 1370 nb_lcore_params = 0; 1371 1372 while ((p = strchr(p0, '(')) != NULL) { 1373 ++p; 1374 p0 = strchr(p, ')'); 1375 if (p0 == NULL) 1376 return -1; 1377 1378 size = p0 - p; 1379 if (size >= sizeof(s)) 1380 return -1; 1381 1382 snprintf(s, sizeof(s), "%.*s", size, p); 1383 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1384 _NUM_FLD) 1385 return -1; 1386 for (i = 0; i < _NUM_FLD; i++) { 1387 errno = 0; 1388 int_fld[i] = strtoul(str_fld[i], &end, 0); 1389 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1390 return -1; 1391 } 1392 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1393 printf("exceeded max number of lcore params: %hu\n", 1394 nb_lcore_params); 1395 return -1; 1396 } 1397 lcore_params_array[nb_lcore_params].port_id = 1398 (uint8_t)int_fld[FLD_PORT]; 1399 lcore_params_array[nb_lcore_params].queue_id = 1400 (uint8_t)int_fld[FLD_QUEUE]; 1401 lcore_params_array[nb_lcore_params].lcore_id = 1402 (uint8_t)int_fld[FLD_LCORE]; 1403 ++nb_lcore_params; 1404 } 1405 lcore_params = lcore_params_array; 1406 return 0; 1407 } 1408 1409 static void 1410 print_app_sa_prm(const struct app_sa_prm *prm) 1411 { 1412 printf("librte_ipsec usage: %s\n", 1413 (prm->enable == 0) ? "disabled" : "enabled"); 1414 1415 if (prm->enable == 0) 1416 return; 1417 1418 printf("replay window size: %u\n", prm->window_size); 1419 printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled"); 1420 printf("SA flags: %#" PRIx64 "\n", prm->flags); 1421 } 1422 1423 static int32_t 1424 parse_args(int32_t argc, char **argv) 1425 { 1426 int32_t opt, ret; 1427 char **argvopt; 1428 int32_t option_index; 1429 char *prgname = argv[0]; 1430 int32_t f_present = 0; 1431 1432 argvopt = argv; 1433 1434 while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:", 1435 lgopts, &option_index)) != EOF) { 1436 1437 switch (opt) { 1438 case 'p': 1439 enabled_port_mask = parse_portmask(optarg); 1440 if (enabled_port_mask == 0) { 1441 printf("invalid portmask\n"); 1442 print_usage(prgname); 1443 return -1; 1444 } 1445 break; 1446 case 'P': 1447 printf("Promiscuous mode selected\n"); 1448 promiscuous_on = 1; 1449 break; 1450 case 'u': 1451 unprotected_port_mask = parse_portmask(optarg); 1452 if (unprotected_port_mask == 0) { 1453 printf("invalid unprotected portmask\n"); 1454 print_usage(prgname); 1455 return -1; 1456 } 1457 break; 1458 case 'f': 1459 if (f_present == 1) { 1460 printf("\"-f\" option present more than " 1461 "once!\n"); 1462 print_usage(prgname); 1463 return -1; 1464 } 1465 if (parse_cfg_file(optarg) < 0) { 1466 printf("parsing file \"%s\" failed\n", 1467 optarg); 1468 print_usage(prgname); 1469 return -1; 1470 } 1471 f_present = 1; 1472 break; 1473 case 'j': 1474 ret = parse_decimal(optarg); 1475 if (ret < RTE_MBUF_DEFAULT_BUF_SIZE || 1476 ret > UINT16_MAX) { 1477 printf("Invalid frame buffer size value: %s\n", 1478 optarg); 1479 print_usage(prgname); 1480 return -1; 1481 } 1482 frame_buf_size = ret; 1483 printf("Custom frame buffer size %u\n", frame_buf_size); 1484 break; 1485 case 'l': 1486 app_sa_prm.enable = 1; 1487 break; 1488 case 'w': 1489 app_sa_prm.enable = 1; 1490 app_sa_prm.window_size = parse_decimal(optarg); 1491 break; 1492 case 'e': 1493 app_sa_prm.enable = 1; 1494 app_sa_prm.enable_esn = 1; 1495 break; 1496 case 'a': 1497 app_sa_prm.enable = 1; 1498 app_sa_prm.flags |= RTE_IPSEC_SAFLAG_SQN_ATOM; 1499 break; 1500 case CMD_LINE_OPT_CONFIG_NUM: 1501 ret = parse_config(optarg); 1502 if (ret) { 1503 printf("Invalid config\n"); 1504 print_usage(prgname); 1505 return -1; 1506 } 1507 break; 1508 case CMD_LINE_OPT_SINGLE_SA_NUM: 1509 ret = parse_decimal(optarg); 1510 if (ret == -1) { 1511 printf("Invalid argument[sa_idx]\n"); 1512 print_usage(prgname); 1513 return -1; 1514 } 1515 1516 /* else */ 1517 single_sa = 1; 1518 single_sa_idx = ret; 1519 printf("Configured with single SA index %u\n", 1520 single_sa_idx); 1521 break; 1522 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM: 1523 ret = parse_portmask(optarg); 1524 if (ret == -1) { 1525 printf("Invalid argument[portmask]\n"); 1526 print_usage(prgname); 1527 return -1; 1528 } 1529 1530 /* else */ 1531 enabled_cryptodev_mask = ret; 1532 break; 1533 case CMD_LINE_OPT_RX_OFFLOAD_NUM: 1534 ret = parse_mask(optarg, &dev_rx_offload); 1535 if (ret != 0) { 1536 printf("Invalid argument for \'%s\': %s\n", 1537 CMD_LINE_OPT_RX_OFFLOAD, optarg); 1538 print_usage(prgname); 1539 return -1; 1540 } 1541 break; 1542 case CMD_LINE_OPT_TX_OFFLOAD_NUM: 1543 ret = parse_mask(optarg, &dev_tx_offload); 1544 if (ret != 0) { 1545 printf("Invalid argument for \'%s\': %s\n", 1546 CMD_LINE_OPT_TX_OFFLOAD, optarg); 1547 print_usage(prgname); 1548 return -1; 1549 } 1550 break; 1551 case CMD_LINE_OPT_REASSEMBLE_NUM: 1552 ret = parse_decimal(optarg); 1553 if (ret < 0) { 1554 printf("Invalid argument for \'%s\': %s\n", 1555 CMD_LINE_OPT_REASSEMBLE, optarg); 1556 print_usage(prgname); 1557 return -1; 1558 } 1559 frag_tbl_sz = ret; 1560 break; 1561 case CMD_LINE_OPT_MTU_NUM: 1562 ret = parse_decimal(optarg); 1563 if (ret < 0 || ret > RTE_IPV4_MAX_PKT_LEN) { 1564 printf("Invalid argument for \'%s\': %s\n", 1565 CMD_LINE_OPT_MTU, optarg); 1566 print_usage(prgname); 1567 return -1; 1568 } 1569 mtu_size = ret; 1570 break; 1571 default: 1572 print_usage(prgname); 1573 return -1; 1574 } 1575 } 1576 1577 if (f_present == 0) { 1578 printf("Mandatory option \"-f\" not present\n"); 1579 return -1; 1580 } 1581 1582 /* check do we need to enable multi-seg support */ 1583 if (multi_seg_required()) { 1584 /* legacy mode doesn't support multi-seg */ 1585 app_sa_prm.enable = 1; 1586 printf("frame buf size: %u, mtu: %u, " 1587 "number of reassemble entries: %u\n" 1588 "multi-segment support is required\n", 1589 frame_buf_size, mtu_size, frag_tbl_sz); 1590 } 1591 1592 print_app_sa_prm(&app_sa_prm); 1593 1594 if (optind >= 0) 1595 argv[optind-1] = prgname; 1596 1597 ret = optind-1; 1598 optind = 1; /* reset getopt lib */ 1599 return ret; 1600 } 1601 1602 static void 1603 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 1604 { 1605 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 1606 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 1607 printf("%s%s", name, buf); 1608 } 1609 1610 /* 1611 * Update destination ethaddr for the port. 1612 */ 1613 int 1614 add_dst_ethaddr(uint16_t port, const struct rte_ether_addr *addr) 1615 { 1616 if (port >= RTE_DIM(ethaddr_tbl)) 1617 return -EINVAL; 1618 1619 ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr); 1620 return 0; 1621 } 1622 1623 /* Check the link status of all ports in up to 9s, and print them finally */ 1624 static void 1625 check_all_ports_link_status(uint32_t port_mask) 1626 { 1627 #define CHECK_INTERVAL 100 /* 100ms */ 1628 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1629 uint16_t portid; 1630 uint8_t count, all_ports_up, print_flag = 0; 1631 struct rte_eth_link link; 1632 1633 printf("\nChecking link status"); 1634 fflush(stdout); 1635 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1636 all_ports_up = 1; 1637 RTE_ETH_FOREACH_DEV(portid) { 1638 if ((port_mask & (1 << portid)) == 0) 1639 continue; 1640 memset(&link, 0, sizeof(link)); 1641 rte_eth_link_get_nowait(portid, &link); 1642 /* print link status if flag set */ 1643 if (print_flag == 1) { 1644 if (link.link_status) 1645 printf( 1646 "Port%d Link Up - speed %u Mbps -%s\n", 1647 portid, link.link_speed, 1648 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1649 ("full-duplex") : ("half-duplex\n")); 1650 else 1651 printf("Port %d Link Down\n", portid); 1652 continue; 1653 } 1654 /* clear all_ports_up flag if any link down */ 1655 if (link.link_status == ETH_LINK_DOWN) { 1656 all_ports_up = 0; 1657 break; 1658 } 1659 } 1660 /* after finally printing all link status, get out */ 1661 if (print_flag == 1) 1662 break; 1663 1664 if (all_ports_up == 0) { 1665 printf("."); 1666 fflush(stdout); 1667 rte_delay_ms(CHECK_INTERVAL); 1668 } 1669 1670 /* set the print_flag if all ports up or timeout */ 1671 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1672 print_flag = 1; 1673 printf("done\n"); 1674 } 1675 } 1676 } 1677 1678 static int32_t 1679 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1680 uint16_t qp, struct lcore_params *params, 1681 struct ipsec_ctx *ipsec_ctx, 1682 const struct rte_cryptodev_capabilities *cipher, 1683 const struct rte_cryptodev_capabilities *auth, 1684 const struct rte_cryptodev_capabilities *aead) 1685 { 1686 int32_t ret = 0; 1687 unsigned long i; 1688 struct cdev_key key = { 0 }; 1689 1690 key.lcore_id = params->lcore_id; 1691 if (cipher) 1692 key.cipher_algo = cipher->sym.cipher.algo; 1693 if (auth) 1694 key.auth_algo = auth->sym.auth.algo; 1695 if (aead) 1696 key.aead_algo = aead->sym.aead.algo; 1697 1698 ret = rte_hash_lookup(map, &key); 1699 if (ret != -ENOENT) 1700 return 0; 1701 1702 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1703 if (ipsec_ctx->tbl[i].id == cdev_id) 1704 break; 1705 1706 if (i == ipsec_ctx->nb_qps) { 1707 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1708 printf("Maximum number of crypto devices assigned to " 1709 "a core, increase MAX_QP_PER_LCORE value\n"); 1710 return 0; 1711 } 1712 ipsec_ctx->tbl[i].id = cdev_id; 1713 ipsec_ctx->tbl[i].qp = qp; 1714 ipsec_ctx->nb_qps++; 1715 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1716 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1717 cdev_id, qp, i); 1718 } 1719 1720 ret = rte_hash_add_key_data(map, &key, (void *)i); 1721 if (ret < 0) { 1722 printf("Faled to insert cdev mapping for (lcore %u, " 1723 "cdev %u, qp %u), errno %d\n", 1724 key.lcore_id, ipsec_ctx->tbl[i].id, 1725 ipsec_ctx->tbl[i].qp, ret); 1726 return 0; 1727 } 1728 1729 return 1; 1730 } 1731 1732 static int32_t 1733 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1734 uint16_t qp, struct lcore_params *params) 1735 { 1736 int32_t ret = 0; 1737 const struct rte_cryptodev_capabilities *i, *j; 1738 struct rte_hash *map; 1739 struct lcore_conf *qconf; 1740 struct ipsec_ctx *ipsec_ctx; 1741 const char *str; 1742 1743 qconf = &lcore_conf[params->lcore_id]; 1744 1745 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 1746 map = cdev_map_out; 1747 ipsec_ctx = &qconf->outbound; 1748 str = "Outbound"; 1749 } else { 1750 map = cdev_map_in; 1751 ipsec_ctx = &qconf->inbound; 1752 str = "Inbound"; 1753 } 1754 1755 /* Required cryptodevs with operation chainning */ 1756 if (!(dev_info->feature_flags & 1757 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 1758 return ret; 1759 1760 for (i = dev_info->capabilities; 1761 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 1762 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1763 continue; 1764 1765 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1766 ret |= add_mapping(map, str, cdev_id, qp, params, 1767 ipsec_ctx, NULL, NULL, i); 1768 continue; 1769 } 1770 1771 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 1772 continue; 1773 1774 for (j = dev_info->capabilities; 1775 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 1776 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1777 continue; 1778 1779 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 1780 continue; 1781 1782 ret |= add_mapping(map, str, cdev_id, qp, params, 1783 ipsec_ctx, i, j, NULL); 1784 } 1785 } 1786 1787 return ret; 1788 } 1789 1790 /* Check if the device is enabled by cryptodev_mask */ 1791 static int 1792 check_cryptodev_mask(uint8_t cdev_id) 1793 { 1794 if (enabled_cryptodev_mask & (1 << cdev_id)) 1795 return 0; 1796 1797 return -1; 1798 } 1799 1800 static int32_t 1801 cryptodevs_init(void) 1802 { 1803 struct rte_cryptodev_config dev_conf; 1804 struct rte_cryptodev_qp_conf qp_conf; 1805 uint16_t idx, max_nb_qps, qp, i; 1806 int16_t cdev_id; 1807 struct rte_hash_parameters params = { 0 }; 1808 1809 const uint64_t mseg_flag = multi_seg_required() ? 1810 RTE_CRYPTODEV_FF_IN_PLACE_SGL : 0; 1811 1812 params.entries = CDEV_MAP_ENTRIES; 1813 params.key_len = sizeof(struct cdev_key); 1814 params.hash_func = rte_jhash; 1815 params.hash_func_init_val = 0; 1816 params.socket_id = rte_socket_id(); 1817 1818 params.name = "cdev_map_in"; 1819 cdev_map_in = rte_hash_create(¶ms); 1820 if (cdev_map_in == NULL) 1821 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1822 rte_errno); 1823 1824 params.name = "cdev_map_out"; 1825 cdev_map_out = rte_hash_create(¶ms); 1826 if (cdev_map_out == NULL) 1827 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1828 rte_errno); 1829 1830 printf("lcore/cryptodev/qp mappings:\n"); 1831 1832 idx = 0; 1833 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1834 struct rte_cryptodev_info cdev_info; 1835 1836 if (check_cryptodev_mask((uint8_t)cdev_id)) 1837 continue; 1838 1839 rte_cryptodev_info_get(cdev_id, &cdev_info); 1840 1841 if ((mseg_flag & cdev_info.feature_flags) != mseg_flag) 1842 rte_exit(EXIT_FAILURE, 1843 "Device %hd does not support \'%s\' feature\n", 1844 cdev_id, 1845 rte_cryptodev_get_feature_name(mseg_flag)); 1846 1847 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 1848 max_nb_qps = cdev_info.max_nb_queue_pairs; 1849 else 1850 max_nb_qps = nb_lcore_params; 1851 1852 qp = 0; 1853 i = 0; 1854 while (qp < max_nb_qps && i < nb_lcore_params) { 1855 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 1856 &lcore_params[idx])) 1857 qp++; 1858 idx++; 1859 idx = idx % nb_lcore_params; 1860 i++; 1861 } 1862 1863 if (qp == 0) 1864 continue; 1865 1866 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 1867 dev_conf.nb_queue_pairs = qp; 1868 dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO; 1869 1870 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions; 1871 if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS) 1872 rte_exit(EXIT_FAILURE, 1873 "Device does not support at least %u " 1874 "sessions", CDEV_MP_NB_OBJS); 1875 1876 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 1877 rte_panic("Failed to initialize cryptodev %u\n", 1878 cdev_id); 1879 1880 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 1881 qp_conf.mp_session = 1882 socket_ctx[dev_conf.socket_id].session_pool; 1883 qp_conf.mp_session_private = 1884 socket_ctx[dev_conf.socket_id].session_priv_pool; 1885 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 1886 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 1887 &qp_conf, dev_conf.socket_id)) 1888 rte_panic("Failed to setup queue %u for " 1889 "cdev_id %u\n", 0, cdev_id); 1890 1891 if (rte_cryptodev_start(cdev_id)) 1892 rte_panic("Failed to start cryptodev %u\n", 1893 cdev_id); 1894 } 1895 1896 printf("\n"); 1897 1898 return 0; 1899 } 1900 1901 static void 1902 port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads) 1903 { 1904 uint32_t frame_size; 1905 struct rte_eth_dev_info dev_info; 1906 struct rte_eth_txconf *txconf; 1907 uint16_t nb_tx_queue, nb_rx_queue; 1908 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 1909 int32_t ret, socket_id; 1910 struct lcore_conf *qconf; 1911 struct rte_ether_addr ethaddr; 1912 struct rte_eth_conf local_port_conf = port_conf; 1913 1914 rte_eth_dev_info_get(portid, &dev_info); 1915 1916 /* limit allowed HW offloafs, as user requested */ 1917 dev_info.rx_offload_capa &= dev_rx_offload; 1918 dev_info.tx_offload_capa &= dev_tx_offload; 1919 1920 printf("Configuring device port %u:\n", portid); 1921 1922 rte_eth_macaddr_get(portid, ðaddr); 1923 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ðaddr); 1924 print_ethaddr("Address: ", ðaddr); 1925 printf("\n"); 1926 1927 nb_rx_queue = get_port_nb_rx_queues(portid); 1928 nb_tx_queue = nb_lcores; 1929 1930 if (nb_rx_queue > dev_info.max_rx_queues) 1931 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1932 "(max rx queue is %u)\n", 1933 nb_rx_queue, dev_info.max_rx_queues); 1934 1935 if (nb_tx_queue > dev_info.max_tx_queues) 1936 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1937 "(max tx queue is %u)\n", 1938 nb_tx_queue, dev_info.max_tx_queues); 1939 1940 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 1941 nb_rx_queue, nb_tx_queue); 1942 1943 frame_size = MTU_TO_FRAMELEN(mtu_size); 1944 if (frame_size > local_port_conf.rxmode.max_rx_pkt_len) 1945 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1946 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 1947 1948 if (multi_seg_required()) { 1949 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SCATTER; 1950 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS; 1951 } 1952 1953 local_port_conf.rxmode.offloads |= req_rx_offloads; 1954 local_port_conf.txmode.offloads |= req_tx_offloads; 1955 1956 /* Check that all required capabilities are supported */ 1957 if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) != 1958 local_port_conf.rxmode.offloads) 1959 rte_exit(EXIT_FAILURE, 1960 "Error: port %u required RX offloads: 0x%" PRIx64 1961 ", avaialbe RX offloads: 0x%" PRIx64 "\n", 1962 portid, local_port_conf.rxmode.offloads, 1963 dev_info.rx_offload_capa); 1964 1965 if ((local_port_conf.txmode.offloads & dev_info.tx_offload_capa) != 1966 local_port_conf.txmode.offloads) 1967 rte_exit(EXIT_FAILURE, 1968 "Error: port %u required TX offloads: 0x%" PRIx64 1969 ", avaialbe TX offloads: 0x%" PRIx64 "\n", 1970 portid, local_port_conf.txmode.offloads, 1971 dev_info.tx_offload_capa); 1972 1973 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 1974 local_port_conf.txmode.offloads |= 1975 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1976 1977 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 1978 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 1979 1980 printf("port %u configurng rx_offloads=0x%" PRIx64 1981 ", tx_offloads=0x%" PRIx64 "\n", 1982 portid, local_port_conf.rxmode.offloads, 1983 local_port_conf.txmode.offloads); 1984 1985 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 1986 dev_info.flow_type_rss_offloads; 1987 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 1988 port_conf.rx_adv_conf.rss_conf.rss_hf) { 1989 printf("Port %u modified RSS hash function based on hardware support," 1990 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 1991 portid, 1992 port_conf.rx_adv_conf.rss_conf.rss_hf, 1993 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 1994 } 1995 1996 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 1997 &local_port_conf); 1998 if (ret < 0) 1999 rte_exit(EXIT_FAILURE, "Cannot configure device: " 2000 "err=%d, port=%d\n", ret, portid); 2001 2002 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 2003 if (ret < 0) 2004 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 2005 "err=%d, port=%d\n", ret, portid); 2006 2007 /* init one TX queue per lcore */ 2008 tx_queueid = 0; 2009 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2010 if (rte_lcore_is_enabled(lcore_id) == 0) 2011 continue; 2012 2013 if (numa_on) 2014 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2015 else 2016 socket_id = 0; 2017 2018 /* init TX queue */ 2019 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 2020 2021 txconf = &dev_info.default_txconf; 2022 txconf->offloads = local_port_conf.txmode.offloads; 2023 2024 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 2025 socket_id, txconf); 2026 if (ret < 0) 2027 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 2028 "err=%d, port=%d\n", ret, portid); 2029 2030 qconf = &lcore_conf[lcore_id]; 2031 qconf->tx_queue_id[portid] = tx_queueid; 2032 2033 /* Pre-populate pkt offloads based on capabilities */ 2034 qconf->outbound.ipv4_offloads = PKT_TX_IPV4; 2035 qconf->outbound.ipv6_offloads = PKT_TX_IPV6; 2036 if (local_port_conf.txmode.offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 2037 qconf->outbound.ipv4_offloads |= PKT_TX_IP_CKSUM; 2038 2039 tx_queueid++; 2040 2041 /* init RX queues */ 2042 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 2043 struct rte_eth_rxconf rxq_conf; 2044 2045 if (portid != qconf->rx_queue_list[queue].port_id) 2046 continue; 2047 2048 rx_queueid = qconf->rx_queue_list[queue].queue_id; 2049 2050 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 2051 socket_id); 2052 2053 rxq_conf = dev_info.default_rxconf; 2054 rxq_conf.offloads = local_port_conf.rxmode.offloads; 2055 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 2056 nb_rxd, socket_id, &rxq_conf, 2057 socket_ctx[socket_id].mbuf_pool); 2058 if (ret < 0) 2059 rte_exit(EXIT_FAILURE, 2060 "rte_eth_rx_queue_setup: err=%d, " 2061 "port=%d\n", ret, portid); 2062 } 2063 } 2064 printf("\n"); 2065 } 2066 2067 static size_t 2068 max_session_size(void) 2069 { 2070 size_t max_sz, sz; 2071 void *sec_ctx; 2072 int16_t cdev_id, port_id, n; 2073 2074 max_sz = 0; 2075 n = rte_cryptodev_count(); 2076 for (cdev_id = 0; cdev_id != n; cdev_id++) { 2077 sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 2078 if (sz > max_sz) 2079 max_sz = sz; 2080 /* 2081 * If crypto device is security capable, need to check the 2082 * size of security session as well. 2083 */ 2084 2085 /* Get security context of the crypto device */ 2086 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id); 2087 if (sec_ctx == NULL) 2088 continue; 2089 2090 /* Get size of security session */ 2091 sz = rte_security_session_get_size(sec_ctx); 2092 if (sz > max_sz) 2093 max_sz = sz; 2094 } 2095 2096 RTE_ETH_FOREACH_DEV(port_id) { 2097 if ((enabled_port_mask & (1 << port_id)) == 0) 2098 continue; 2099 2100 sec_ctx = rte_eth_dev_get_sec_ctx(port_id); 2101 if (sec_ctx == NULL) 2102 continue; 2103 2104 sz = rte_security_session_get_size(sec_ctx); 2105 if (sz > max_sz) 2106 max_sz = sz; 2107 } 2108 2109 return max_sz; 2110 } 2111 2112 static void 2113 session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz) 2114 { 2115 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2116 struct rte_mempool *sess_mp; 2117 2118 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2119 "sess_mp_%u", socket_id); 2120 sess_mp = rte_cryptodev_sym_session_pool_create( 2121 mp_name, CDEV_MP_NB_OBJS, 2122 sess_sz, CDEV_MP_CACHE_SZ, 0, 2123 socket_id); 2124 ctx->session_pool = sess_mp; 2125 2126 if (ctx->session_pool == NULL) 2127 rte_exit(EXIT_FAILURE, 2128 "Cannot init session pool on socket %d\n", socket_id); 2129 else 2130 printf("Allocated session pool on socket %d\n", socket_id); 2131 } 2132 2133 static void 2134 session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id, 2135 size_t sess_sz) 2136 { 2137 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2138 struct rte_mempool *sess_mp; 2139 2140 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2141 "sess_mp_priv_%u", socket_id); 2142 sess_mp = rte_mempool_create(mp_name, 2143 CDEV_MP_NB_OBJS, 2144 sess_sz, 2145 CDEV_MP_CACHE_SZ, 2146 0, NULL, NULL, NULL, 2147 NULL, socket_id, 2148 0); 2149 ctx->session_priv_pool = sess_mp; 2150 2151 if (ctx->session_priv_pool == NULL) 2152 rte_exit(EXIT_FAILURE, 2153 "Cannot init session priv pool on socket %d\n", 2154 socket_id); 2155 else 2156 printf("Allocated session priv pool on socket %d\n", 2157 socket_id); 2158 } 2159 2160 static void 2161 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 2162 { 2163 char s[64]; 2164 int32_t ms; 2165 2166 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 2167 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 2168 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 2169 frame_buf_size, socket_id); 2170 2171 /* 2172 * if multi-segment support is enabled, then create a pool 2173 * for indirect mbufs. 2174 */ 2175 ms = multi_seg_required(); 2176 if (ms != 0) { 2177 snprintf(s, sizeof(s), "mbuf_pool_indir_%d", socket_id); 2178 ctx->mbuf_pool_indir = rte_pktmbuf_pool_create(s, nb_mbuf, 2179 MEMPOOL_CACHE_SIZE, 0, 0, socket_id); 2180 } 2181 2182 if (ctx->mbuf_pool == NULL || (ms != 0 && ctx->mbuf_pool_indir == NULL)) 2183 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 2184 socket_id); 2185 else 2186 printf("Allocated mbuf pool on socket %d\n", socket_id); 2187 } 2188 2189 static inline int 2190 inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md) 2191 { 2192 struct ipsec_sa *sa; 2193 2194 /* For inline protocol processing, the metadata in the event will 2195 * uniquely identify the security session which raised the event. 2196 * Application would then need the userdata it had registered with the 2197 * security session to process the event. 2198 */ 2199 2200 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md); 2201 2202 if (sa == NULL) { 2203 /* userdata could not be retrieved */ 2204 return -1; 2205 } 2206 2207 /* Sequence number over flow. SA need to be re-established */ 2208 RTE_SET_USED(sa); 2209 return 0; 2210 } 2211 2212 static int 2213 inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type, 2214 void *param, void *ret_param) 2215 { 2216 uint64_t md; 2217 struct rte_eth_event_ipsec_desc *event_desc = NULL; 2218 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 2219 rte_eth_dev_get_sec_ctx(port_id); 2220 2221 RTE_SET_USED(param); 2222 2223 if (type != RTE_ETH_EVENT_IPSEC) 2224 return -1; 2225 2226 event_desc = ret_param; 2227 if (event_desc == NULL) { 2228 printf("Event descriptor not set\n"); 2229 return -1; 2230 } 2231 2232 md = event_desc->metadata; 2233 2234 if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW) 2235 return inline_ipsec_event_esn_overflow(ctx, md); 2236 else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) { 2237 printf("Invalid IPsec event reported\n"); 2238 return -1; 2239 } 2240 2241 return -1; 2242 } 2243 2244 static uint16_t 2245 rx_callback(__rte_unused uint16_t port, __rte_unused uint16_t queue, 2246 struct rte_mbuf *pkt[], uint16_t nb_pkts, 2247 __rte_unused uint16_t max_pkts, void *user_param) 2248 { 2249 uint64_t tm; 2250 uint32_t i, k; 2251 struct lcore_conf *lc; 2252 struct rte_mbuf *mb; 2253 struct rte_ether_hdr *eth; 2254 2255 lc = user_param; 2256 k = 0; 2257 tm = 0; 2258 2259 for (i = 0; i != nb_pkts; i++) { 2260 2261 mb = pkt[i]; 2262 eth = rte_pktmbuf_mtod(mb, struct rte_ether_hdr *); 2263 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 2264 2265 struct rte_ipv4_hdr *iph; 2266 2267 iph = (struct rte_ipv4_hdr *)(eth + 1); 2268 if (rte_ipv4_frag_pkt_is_fragmented(iph)) { 2269 2270 mb->l2_len = sizeof(*eth); 2271 mb->l3_len = sizeof(*iph); 2272 tm = (tm != 0) ? tm : rte_rdtsc(); 2273 mb = rte_ipv4_frag_reassemble_packet( 2274 lc->frag.tbl, &lc->frag.dr, 2275 mb, tm, iph); 2276 2277 if (mb != NULL) { 2278 /* fix ip cksum after reassemble. */ 2279 iph = rte_pktmbuf_mtod_offset(mb, 2280 struct rte_ipv4_hdr *, 2281 mb->l2_len); 2282 iph->hdr_checksum = 0; 2283 iph->hdr_checksum = rte_ipv4_cksum(iph); 2284 } 2285 } 2286 } else if (eth->ether_type == 2287 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 2288 2289 struct rte_ipv6_hdr *iph; 2290 struct ipv6_extension_fragment *fh; 2291 2292 iph = (struct rte_ipv6_hdr *)(eth + 1); 2293 fh = rte_ipv6_frag_get_ipv6_fragment_header(iph); 2294 if (fh != NULL) { 2295 mb->l2_len = sizeof(*eth); 2296 mb->l3_len = (uintptr_t)fh - (uintptr_t)iph + 2297 sizeof(*fh); 2298 tm = (tm != 0) ? tm : rte_rdtsc(); 2299 mb = rte_ipv6_frag_reassemble_packet( 2300 lc->frag.tbl, &lc->frag.dr, 2301 mb, tm, iph, fh); 2302 if (mb != NULL) 2303 /* fix l3_len after reassemble. */ 2304 mb->l3_len = mb->l3_len - sizeof(*fh); 2305 } 2306 } 2307 2308 pkt[k] = mb; 2309 k += (mb != NULL); 2310 } 2311 2312 /* some fragments were encountered, drain death row */ 2313 if (tm != 0) 2314 rte_ip_frag_free_death_row(&lc->frag.dr, 0); 2315 2316 return k; 2317 } 2318 2319 2320 static int 2321 reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid) 2322 { 2323 int32_t sid; 2324 uint32_t i; 2325 uint64_t frag_cycles; 2326 const struct lcore_rx_queue *rxq; 2327 const struct rte_eth_rxtx_callback *cb; 2328 2329 /* create fragment table */ 2330 sid = rte_lcore_to_socket_id(cid); 2331 frag_cycles = (rte_get_tsc_hz() + MS_PER_S - 1) / 2332 MS_PER_S * FRAG_TTL_MS; 2333 2334 lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz, 2335 FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid); 2336 if (lc->frag.tbl == NULL) { 2337 printf("%s(%u): failed to create fragment table of size: %u, " 2338 "error code: %d\n", 2339 __func__, cid, frag_tbl_sz, rte_errno); 2340 return -ENOMEM; 2341 } 2342 2343 /* setup reassemble RX callbacks for all queues */ 2344 for (i = 0; i != lc->nb_rx_queue; i++) { 2345 2346 rxq = lc->rx_queue_list + i; 2347 cb = rte_eth_add_rx_callback(rxq->port_id, rxq->queue_id, 2348 rx_callback, lc); 2349 if (cb == NULL) { 2350 printf("%s(%u): failed to install RX callback for " 2351 "portid=%u, queueid=%u, error code: %d\n", 2352 __func__, cid, 2353 rxq->port_id, rxq->queue_id, rte_errno); 2354 return -ENOMEM; 2355 } 2356 } 2357 2358 return 0; 2359 } 2360 2361 static int 2362 reassemble_init(void) 2363 { 2364 int32_t rc; 2365 uint32_t i, lc; 2366 2367 rc = 0; 2368 for (i = 0; i != nb_lcore_params; i++) { 2369 lc = lcore_params[i].lcore_id; 2370 rc = reassemble_lcore_init(lcore_conf + lc, lc); 2371 if (rc != 0) 2372 break; 2373 } 2374 2375 return rc; 2376 } 2377 2378 int32_t 2379 main(int32_t argc, char **argv) 2380 { 2381 int32_t ret; 2382 uint32_t lcore_id; 2383 uint32_t i; 2384 uint8_t socket_id; 2385 uint16_t portid; 2386 uint64_t req_rx_offloads, req_tx_offloads; 2387 size_t sess_sz; 2388 2389 /* init EAL */ 2390 ret = rte_eal_init(argc, argv); 2391 if (ret < 0) 2392 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 2393 argc -= ret; 2394 argv += ret; 2395 2396 /* parse application arguments (after the EAL ones) */ 2397 ret = parse_args(argc, argv); 2398 if (ret < 0) 2399 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 2400 2401 if ((unprotected_port_mask & enabled_port_mask) != 2402 unprotected_port_mask) 2403 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 2404 unprotected_port_mask); 2405 2406 if (check_params() < 0) 2407 rte_exit(EXIT_FAILURE, "check_params failed\n"); 2408 2409 ret = init_lcore_rx_queues(); 2410 if (ret < 0) 2411 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 2412 2413 nb_lcores = rte_lcore_count(); 2414 2415 sess_sz = max_session_size(); 2416 2417 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2418 if (rte_lcore_is_enabled(lcore_id) == 0) 2419 continue; 2420 2421 if (numa_on) 2422 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2423 else 2424 socket_id = 0; 2425 2426 /* mbuf_pool is initialised by the pool_init() function*/ 2427 if (socket_ctx[socket_id].mbuf_pool) 2428 continue; 2429 2430 pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF); 2431 session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz); 2432 session_priv_pool_init(&socket_ctx[socket_id], socket_id, 2433 sess_sz); 2434 } 2435 2436 RTE_ETH_FOREACH_DEV(portid) { 2437 if ((enabled_port_mask & (1 << portid)) == 0) 2438 continue; 2439 2440 sa_check_offloads(portid, &req_rx_offloads, &req_tx_offloads); 2441 port_init(portid, req_rx_offloads, req_tx_offloads); 2442 } 2443 2444 cryptodevs_init(); 2445 2446 /* start ports */ 2447 RTE_ETH_FOREACH_DEV(portid) { 2448 if ((enabled_port_mask & (1 << portid)) == 0) 2449 continue; 2450 2451 /* 2452 * Start device 2453 * note: device must be started before a flow rule 2454 * can be installed. 2455 */ 2456 ret = rte_eth_dev_start(portid); 2457 if (ret < 0) 2458 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 2459 "err=%d, port=%d\n", ret, portid); 2460 /* 2461 * If enabled, put device in promiscuous mode. 2462 * This allows IO forwarding mode to forward packets 2463 * to itself through 2 cross-connected ports of the 2464 * target machine. 2465 */ 2466 if (promiscuous_on) 2467 rte_eth_promiscuous_enable(portid); 2468 2469 rte_eth_dev_callback_register(portid, 2470 RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL); 2471 } 2472 2473 /* fragment reassemble is enabled */ 2474 if (frag_tbl_sz != 0) { 2475 ret = reassemble_init(); 2476 if (ret != 0) 2477 rte_exit(EXIT_FAILURE, "failed at reassemble init"); 2478 } 2479 2480 /* Replicate each context per socket */ 2481 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) { 2482 socket_id = rte_socket_id_by_idx(i); 2483 if ((socket_ctx[socket_id].mbuf_pool != NULL) && 2484 (socket_ctx[socket_id].sa_in == NULL) && 2485 (socket_ctx[socket_id].sa_out == NULL)) { 2486 sa_init(&socket_ctx[socket_id], socket_id); 2487 sp4_init(&socket_ctx[socket_id], socket_id); 2488 sp6_init(&socket_ctx[socket_id], socket_id); 2489 rt_init(&socket_ctx[socket_id], socket_id); 2490 } 2491 } 2492 2493 check_all_ports_link_status(enabled_port_mask); 2494 2495 /* launch per-lcore init on every lcore */ 2496 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 2497 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2498 if (rte_eal_wait_lcore(lcore_id) < 0) 2499 return -1; 2500 } 2501 2502 return 0; 2503 } 2504