1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <netinet/in.h> 11 #include <netinet/ip.h> 12 #include <netinet/ip6.h> 13 #include <string.h> 14 #include <sys/queue.h> 15 #include <stdarg.h> 16 #include <errno.h> 17 #include <getopt.h> 18 19 #include <rte_common.h> 20 #include <rte_byteorder.h> 21 #include <rte_log.h> 22 #include <rte_eal.h> 23 #include <rte_launch.h> 24 #include <rte_atomic.h> 25 #include <rte_cycles.h> 26 #include <rte_prefetch.h> 27 #include <rte_lcore.h> 28 #include <rte_per_lcore.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_interrupts.h> 31 #include <rte_random.h> 32 #include <rte_debug.h> 33 #include <rte_ether.h> 34 #include <rte_ethdev.h> 35 #include <rte_mempool.h> 36 #include <rte_mbuf.h> 37 #include <rte_acl.h> 38 #include <rte_lpm.h> 39 #include <rte_lpm6.h> 40 #include <rte_hash.h> 41 #include <rte_jhash.h> 42 #include <rte_cryptodev.h> 43 #include <rte_security.h> 44 45 #include "ipsec.h" 46 #include "parser.h" 47 48 #define RTE_LOGTYPE_IPSEC RTE_LOGTYPE_USER1 49 50 #define MAX_JUMBO_PKT_LEN 9600 51 52 #define MEMPOOL_CACHE_SIZE 256 53 54 #define NB_MBUF (32000) 55 56 #define CDEV_QUEUE_DESC 2048 57 #define CDEV_MAP_ENTRIES 16384 58 #define CDEV_MP_NB_OBJS 2048 59 #define CDEV_MP_CACHE_SZ 64 60 #define MAX_QUEUE_PAIRS 1 61 62 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 63 64 #define NB_SOCKETS 4 65 66 /* Configure how many packets ahead to prefetch, when reading packets */ 67 #define PREFETCH_OFFSET 3 68 69 #define MAX_RX_QUEUE_PER_LCORE 16 70 71 #define MAX_LCORE_PARAMS 1024 72 73 #define UNPROTECTED_PORT(port) (unprotected_port_mask & (1 << portid)) 74 75 /* 76 * Configurable number of RX/TX ring descriptors 77 */ 78 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024 79 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024 80 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 81 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 82 83 #if RTE_BYTE_ORDER != RTE_LITTLE_ENDIAN 84 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 85 (((uint64_t)((a) & 0xff) << 56) | \ 86 ((uint64_t)((b) & 0xff) << 48) | \ 87 ((uint64_t)((c) & 0xff) << 40) | \ 88 ((uint64_t)((d) & 0xff) << 32) | \ 89 ((uint64_t)((e) & 0xff) << 24) | \ 90 ((uint64_t)((f) & 0xff) << 16) | \ 91 ((uint64_t)((g) & 0xff) << 8) | \ 92 ((uint64_t)(h) & 0xff)) 93 #else 94 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 95 (((uint64_t)((h) & 0xff) << 56) | \ 96 ((uint64_t)((g) & 0xff) << 48) | \ 97 ((uint64_t)((f) & 0xff) << 40) | \ 98 ((uint64_t)((e) & 0xff) << 32) | \ 99 ((uint64_t)((d) & 0xff) << 24) | \ 100 ((uint64_t)((c) & 0xff) << 16) | \ 101 ((uint64_t)((b) & 0xff) << 8) | \ 102 ((uint64_t)(a) & 0xff)) 103 #endif 104 #define ETHADDR(a, b, c, d, e, f) (__BYTES_TO_UINT64(a, b, c, d, e, f, 0, 0)) 105 106 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 107 addr.addr_bytes[0], addr.addr_bytes[1], \ 108 addr.addr_bytes[2], addr.addr_bytes[3], \ 109 addr.addr_bytes[4], addr.addr_bytes[5], \ 110 0, 0) 111 112 /* port/source ethernet addr and destination ethernet addr */ 113 struct ethaddr_info { 114 uint64_t src, dst; 115 }; 116 117 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 118 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 119 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 120 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 121 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 122 }; 123 124 #define CMD_LINE_OPT_CONFIG "config" 125 #define CMD_LINE_OPT_SINGLE_SA "single-sa" 126 #define CMD_LINE_OPT_CRYPTODEV_MASK "cryptodev_mask" 127 128 enum { 129 /* long options mapped to a short option */ 130 131 /* first long only option value must be >= 256, so that we won't 132 * conflict with short options 133 */ 134 CMD_LINE_OPT_MIN_NUM = 256, 135 CMD_LINE_OPT_CONFIG_NUM, 136 CMD_LINE_OPT_SINGLE_SA_NUM, 137 CMD_LINE_OPT_CRYPTODEV_MASK_NUM, 138 }; 139 140 static const struct option lgopts[] = { 141 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 142 {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM}, 143 {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM}, 144 {NULL, 0, 0, 0} 145 }; 146 147 /* mask of enabled ports */ 148 static uint32_t enabled_port_mask; 149 static uint64_t enabled_cryptodev_mask = UINT64_MAX; 150 static uint32_t unprotected_port_mask; 151 static int32_t promiscuous_on = 1; 152 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 153 static uint32_t nb_lcores; 154 static uint32_t single_sa; 155 static uint32_t single_sa_idx; 156 static uint32_t frame_size; 157 158 struct lcore_rx_queue { 159 uint16_t port_id; 160 uint8_t queue_id; 161 } __rte_cache_aligned; 162 163 struct lcore_params { 164 uint16_t port_id; 165 uint8_t queue_id; 166 uint8_t lcore_id; 167 } __rte_cache_aligned; 168 169 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 170 171 static struct lcore_params *lcore_params; 172 static uint16_t nb_lcore_params; 173 174 static struct rte_hash *cdev_map_in; 175 static struct rte_hash *cdev_map_out; 176 177 struct buffer { 178 uint16_t len; 179 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 180 }; 181 182 struct lcore_conf { 183 uint16_t nb_rx_queue; 184 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 185 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 186 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 187 struct ipsec_ctx inbound; 188 struct ipsec_ctx outbound; 189 struct rt_ctx *rt4_ctx; 190 struct rt_ctx *rt6_ctx; 191 } __rte_cache_aligned; 192 193 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 194 195 static struct rte_eth_conf port_conf = { 196 .rxmode = { 197 .mq_mode = ETH_MQ_RX_RSS, 198 .max_rx_pkt_len = ETHER_MAX_LEN, 199 .split_hdr_size = 0, 200 .offloads = DEV_RX_OFFLOAD_CHECKSUM, 201 }, 202 .rx_adv_conf = { 203 .rss_conf = { 204 .rss_key = NULL, 205 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 206 ETH_RSS_TCP | ETH_RSS_SCTP, 207 }, 208 }, 209 .txmode = { 210 .mq_mode = ETH_MQ_TX_NONE, 211 .offloads = (DEV_TX_OFFLOAD_IPV4_CKSUM | 212 DEV_TX_OFFLOAD_MULTI_SEGS), 213 }, 214 }; 215 216 static struct socket_ctx socket_ctx[NB_SOCKETS]; 217 218 struct traffic_type { 219 const uint8_t *data[MAX_PKT_BURST * 2]; 220 struct rte_mbuf *pkts[MAX_PKT_BURST * 2]; 221 uint32_t res[MAX_PKT_BURST * 2]; 222 uint32_t num; 223 }; 224 225 struct ipsec_traffic { 226 struct traffic_type ipsec; 227 struct traffic_type ip4; 228 struct traffic_type ip6; 229 }; 230 231 static inline void 232 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 233 { 234 uint8_t *nlp; 235 struct ether_hdr *eth; 236 237 eth = rte_pktmbuf_mtod(pkt, struct ether_hdr *); 238 if (eth->ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) { 239 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, ETHER_HDR_LEN); 240 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip, ip_p)); 241 if (*nlp == IPPROTO_ESP) 242 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 243 else { 244 t->ip4.data[t->ip4.num] = nlp; 245 t->ip4.pkts[(t->ip4.num)++] = pkt; 246 } 247 } else if (eth->ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) { 248 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, ETHER_HDR_LEN); 249 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip6_hdr, ip6_nxt)); 250 if (*nlp == IPPROTO_ESP) 251 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 252 else { 253 t->ip6.data[t->ip6.num] = nlp; 254 t->ip6.pkts[(t->ip6.num)++] = pkt; 255 } 256 } else { 257 /* Unknown/Unsupported type, drop the packet */ 258 RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n", 259 rte_be_to_cpu_16(eth->ether_type)); 260 rte_pktmbuf_free(pkt); 261 } 262 263 /* Check if the packet has been processed inline. For inline protocol 264 * processed packets, the metadata in the mbuf can be used to identify 265 * the security processing done on the packet. The metadata will be 266 * used to retrieve the application registered userdata associated 267 * with the security session. 268 */ 269 270 if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) { 271 struct ipsec_sa *sa; 272 struct ipsec_mbuf_metadata *priv; 273 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 274 rte_eth_dev_get_sec_ctx( 275 pkt->port); 276 277 /* Retrieve the userdata registered. Here, the userdata 278 * registered is the SA pointer. 279 */ 280 281 sa = (struct ipsec_sa *) 282 rte_security_get_userdata(ctx, pkt->udata64); 283 284 if (sa == NULL) { 285 /* userdata could not be retrieved */ 286 return; 287 } 288 289 /* Save SA as priv member in mbuf. This will be used in the 290 * IPsec selector(SP-SA) check. 291 */ 292 293 priv = get_priv(pkt); 294 priv->sa = sa; 295 } 296 } 297 298 static inline void 299 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 300 uint16_t nb_pkts) 301 { 302 int32_t i; 303 304 t->ipsec.num = 0; 305 t->ip4.num = 0; 306 t->ip6.num = 0; 307 308 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 309 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 310 void *)); 311 prepare_one_packet(pkts[i], t); 312 } 313 /* Process left packets */ 314 for (; i < nb_pkts; i++) 315 prepare_one_packet(pkts[i], t); 316 } 317 318 static inline void 319 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port) 320 { 321 struct ip *ip; 322 struct ether_hdr *ethhdr; 323 324 ip = rte_pktmbuf_mtod(pkt, struct ip *); 325 326 ethhdr = (struct ether_hdr *)rte_pktmbuf_prepend(pkt, ETHER_HDR_LEN); 327 328 if (ip->ip_v == IPVERSION) { 329 pkt->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4; 330 pkt->l3_len = sizeof(struct ip); 331 pkt->l2_len = ETHER_HDR_LEN; 332 333 ip->ip_sum = 0; 334 ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); 335 } else { 336 pkt->ol_flags |= PKT_TX_IPV6; 337 pkt->l3_len = sizeof(struct ip6_hdr); 338 pkt->l2_len = ETHER_HDR_LEN; 339 340 ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6); 341 } 342 343 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 344 sizeof(struct ether_addr)); 345 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 346 sizeof(struct ether_addr)); 347 } 348 349 static inline void 350 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port) 351 { 352 int32_t i; 353 const int32_t prefetch_offset = 2; 354 355 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 356 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 357 prepare_tx_pkt(pkts[i], port); 358 } 359 /* Process left packets */ 360 for (; i < nb_pkts; i++) 361 prepare_tx_pkt(pkts[i], port); 362 } 363 364 /* Send burst of packets on an output interface */ 365 static inline int32_t 366 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 367 { 368 struct rte_mbuf **m_table; 369 int32_t ret; 370 uint16_t queueid; 371 372 queueid = qconf->tx_queue_id[port]; 373 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 374 375 prepare_tx_burst(m_table, n, port); 376 377 ret = rte_eth_tx_burst(port, queueid, m_table, n); 378 if (unlikely(ret < n)) { 379 do { 380 rte_pktmbuf_free(m_table[ret]); 381 } while (++ret < n); 382 } 383 384 return 0; 385 } 386 387 /* Enqueue a single packet, and send burst if queue is filled */ 388 static inline int32_t 389 send_single_packet(struct rte_mbuf *m, uint16_t port) 390 { 391 uint32_t lcore_id; 392 uint16_t len; 393 struct lcore_conf *qconf; 394 395 lcore_id = rte_lcore_id(); 396 397 qconf = &lcore_conf[lcore_id]; 398 len = qconf->tx_mbufs[port].len; 399 qconf->tx_mbufs[port].m_table[len] = m; 400 len++; 401 402 /* enough pkts to be sent */ 403 if (unlikely(len == MAX_PKT_BURST)) { 404 send_burst(qconf, MAX_PKT_BURST, port); 405 len = 0; 406 } 407 408 qconf->tx_mbufs[port].len = len; 409 return 0; 410 } 411 412 static inline void 413 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 414 uint16_t lim) 415 { 416 struct rte_mbuf *m; 417 uint32_t i, j, res, sa_idx; 418 419 if (ip->num == 0 || sp == NULL) 420 return; 421 422 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 423 ip->num, DEFAULT_MAX_CATEGORIES); 424 425 j = 0; 426 for (i = 0; i < ip->num; i++) { 427 m = ip->pkts[i]; 428 res = ip->res[i]; 429 if (res == BYPASS) { 430 ip->pkts[j++] = m; 431 continue; 432 } 433 if (res == DISCARD) { 434 rte_pktmbuf_free(m); 435 continue; 436 } 437 438 /* Only check SPI match for processed IPSec packets */ 439 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) { 440 rte_pktmbuf_free(m); 441 continue; 442 } 443 444 sa_idx = SPI2IDX(res); 445 if (!inbound_sa_check(sa, m, sa_idx)) { 446 rte_pktmbuf_free(m); 447 continue; 448 } 449 ip->pkts[j++] = m; 450 } 451 ip->num = j; 452 } 453 454 static void 455 split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) 456 { 457 uint32_t i, n4, n6; 458 struct ip *ip; 459 struct rte_mbuf *m; 460 461 n4 = trf->ip4.num; 462 n6 = trf->ip6.num; 463 464 for (i = 0; i < num; i++) { 465 466 m = mb[i]; 467 ip = rte_pktmbuf_mtod(m, struct ip *); 468 469 if (ip->ip_v == IPVERSION) { 470 trf->ip4.pkts[n4] = m; 471 trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m, 472 uint8_t *, offsetof(struct ip, ip_p)); 473 n4++; 474 } else if (ip->ip_v == IP6_VERSION) { 475 trf->ip6.pkts[n6] = m; 476 trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m, 477 uint8_t *, 478 offsetof(struct ip6_hdr, ip6_nxt)); 479 n6++; 480 } else 481 rte_pktmbuf_free(m); 482 } 483 484 trf->ip4.num = n4; 485 trf->ip6.num = n6; 486 } 487 488 489 static inline void 490 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 491 struct ipsec_traffic *traffic) 492 { 493 uint16_t nb_pkts_in, n_ip4, n_ip6; 494 495 n_ip4 = traffic->ip4.num; 496 n_ip6 = traffic->ip6.num; 497 498 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 499 traffic->ipsec.num, MAX_PKT_BURST); 500 501 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in); 502 503 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 504 n_ip4); 505 506 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 507 n_ip6); 508 } 509 510 static inline void 511 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 512 struct traffic_type *ipsec) 513 { 514 struct rte_mbuf *m; 515 uint32_t i, j, sa_idx; 516 517 if (ip->num == 0 || sp == NULL) 518 return; 519 520 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 521 ip->num, DEFAULT_MAX_CATEGORIES); 522 523 j = 0; 524 for (i = 0; i < ip->num; i++) { 525 m = ip->pkts[i]; 526 sa_idx = SPI2IDX(ip->res[i]); 527 if (ip->res[i] == DISCARD) 528 rte_pktmbuf_free(m); 529 else if (ip->res[i] == BYPASS) 530 ip->pkts[j++] = m; 531 else { 532 ipsec->res[ipsec->num] = sa_idx; 533 ipsec->pkts[ipsec->num++] = m; 534 } 535 } 536 ip->num = j; 537 } 538 539 static inline void 540 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 541 struct ipsec_traffic *traffic) 542 { 543 struct rte_mbuf *m; 544 uint16_t idx, nb_pkts_out, i; 545 546 /* Drop any IPsec traffic from protected ports */ 547 for (i = 0; i < traffic->ipsec.num; i++) 548 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 549 550 traffic->ipsec.num = 0; 551 552 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 553 554 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 555 556 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 557 traffic->ipsec.res, traffic->ipsec.num, 558 MAX_PKT_BURST); 559 560 for (i = 0; i < nb_pkts_out; i++) { 561 m = traffic->ipsec.pkts[i]; 562 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 563 if (ip->ip_v == IPVERSION) { 564 idx = traffic->ip4.num++; 565 traffic->ip4.pkts[idx] = m; 566 } else { 567 idx = traffic->ip6.num++; 568 traffic->ip6.pkts[idx] = m; 569 } 570 } 571 } 572 573 static inline void 574 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 575 struct ipsec_traffic *traffic) 576 { 577 struct rte_mbuf *m; 578 uint32_t nb_pkts_in, i, idx; 579 580 /* Drop any IPv4 traffic from unprotected ports */ 581 for (i = 0; i < traffic->ip4.num; i++) 582 rte_pktmbuf_free(traffic->ip4.pkts[i]); 583 584 traffic->ip4.num = 0; 585 586 /* Drop any IPv6 traffic from unprotected ports */ 587 for (i = 0; i < traffic->ip6.num; i++) 588 rte_pktmbuf_free(traffic->ip6.pkts[i]); 589 590 traffic->ip6.num = 0; 591 592 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 593 traffic->ipsec.num, MAX_PKT_BURST); 594 595 for (i = 0; i < nb_pkts_in; i++) { 596 m = traffic->ipsec.pkts[i]; 597 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 598 if (ip->ip_v == IPVERSION) { 599 idx = traffic->ip4.num++; 600 traffic->ip4.pkts[idx] = m; 601 } else { 602 idx = traffic->ip6.num++; 603 traffic->ip6.pkts[idx] = m; 604 } 605 } 606 } 607 608 static inline void 609 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 610 struct ipsec_traffic *traffic) 611 { 612 struct rte_mbuf *m; 613 uint32_t nb_pkts_out, i, n; 614 struct ip *ip; 615 616 /* Drop any IPsec traffic from protected ports */ 617 for (i = 0; i < traffic->ipsec.num; i++) 618 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 619 620 n = 0; 621 622 for (i = 0; i < traffic->ip4.num; i++) { 623 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i]; 624 traffic->ipsec.res[n++] = single_sa_idx; 625 } 626 627 for (i = 0; i < traffic->ip6.num; i++) { 628 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i]; 629 traffic->ipsec.res[n++] = single_sa_idx; 630 } 631 632 traffic->ip4.num = 0; 633 traffic->ip6.num = 0; 634 traffic->ipsec.num = n; 635 636 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 637 traffic->ipsec.res, traffic->ipsec.num, 638 MAX_PKT_BURST); 639 640 /* They all sue the same SA (ip4 or ip6 tunnel) */ 641 m = traffic->ipsec.pkts[i]; 642 ip = rte_pktmbuf_mtod(m, struct ip *); 643 if (ip->ip_v == IPVERSION) { 644 traffic->ip4.num = nb_pkts_out; 645 for (i = 0; i < nb_pkts_out; i++) 646 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i]; 647 } else { 648 traffic->ip6.num = nb_pkts_out; 649 for (i = 0; i < nb_pkts_out; i++) 650 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i]; 651 } 652 } 653 654 static inline int32_t 655 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) 656 { 657 struct ipsec_mbuf_metadata *priv; 658 struct ipsec_sa *sa; 659 660 priv = get_priv(pkt); 661 662 sa = priv->sa; 663 if (unlikely(sa == NULL)) { 664 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); 665 goto fail; 666 } 667 668 if (is_ipv6) 669 return sa->portid; 670 671 /* else */ 672 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); 673 674 fail: 675 if (is_ipv6) 676 return -1; 677 678 /* else */ 679 return 0; 680 } 681 682 static inline void 683 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 684 { 685 uint32_t hop[MAX_PKT_BURST * 2]; 686 uint32_t dst_ip[MAX_PKT_BURST * 2]; 687 int32_t pkt_hop = 0; 688 uint16_t i, offset; 689 uint16_t lpm_pkts = 0; 690 691 if (nb_pkts == 0) 692 return; 693 694 /* Need to do an LPM lookup for non-inline packets. Inline packets will 695 * have port ID in the SA 696 */ 697 698 for (i = 0; i < nb_pkts; i++) { 699 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 700 /* Security offload not enabled. So an LPM lookup is 701 * required to get the hop 702 */ 703 offset = offsetof(struct ip, ip_dst); 704 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], 705 uint32_t *, offset); 706 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); 707 lpm_pkts++; 708 } 709 } 710 711 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); 712 713 lpm_pkts = 0; 714 715 for (i = 0; i < nb_pkts; i++) { 716 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 717 /* Read hop from the SA */ 718 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); 719 } else { 720 /* Need to use hop returned by lookup */ 721 pkt_hop = hop[lpm_pkts++]; 722 } 723 724 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { 725 rte_pktmbuf_free(pkts[i]); 726 continue; 727 } 728 send_single_packet(pkts[i], pkt_hop & 0xff); 729 } 730 } 731 732 static inline void 733 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 734 { 735 int32_t hop[MAX_PKT_BURST * 2]; 736 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 737 uint8_t *ip6_dst; 738 int32_t pkt_hop = 0; 739 uint16_t i, offset; 740 uint16_t lpm_pkts = 0; 741 742 if (nb_pkts == 0) 743 return; 744 745 /* Need to do an LPM lookup for non-inline packets. Inline packets will 746 * have port ID in the SA 747 */ 748 749 for (i = 0; i < nb_pkts; i++) { 750 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 751 /* Security offload not enabled. So an LPM lookup is 752 * required to get the hop 753 */ 754 offset = offsetof(struct ip6_hdr, ip6_dst); 755 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, 756 offset); 757 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); 758 lpm_pkts++; 759 } 760 } 761 762 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, 763 lpm_pkts); 764 765 lpm_pkts = 0; 766 767 for (i = 0; i < nb_pkts; i++) { 768 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 769 /* Read hop from the SA */ 770 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); 771 } else { 772 /* Need to use hop returned by lookup */ 773 pkt_hop = hop[lpm_pkts++]; 774 } 775 776 if (pkt_hop == -1) { 777 rte_pktmbuf_free(pkts[i]); 778 continue; 779 } 780 send_single_packet(pkts[i], pkt_hop & 0xff); 781 } 782 } 783 784 static inline void 785 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 786 uint8_t nb_pkts, uint16_t portid) 787 { 788 struct ipsec_traffic traffic; 789 790 prepare_traffic(pkts, &traffic, nb_pkts); 791 792 if (unlikely(single_sa)) { 793 if (UNPROTECTED_PORT(portid)) 794 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 795 else 796 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 797 } else { 798 if (UNPROTECTED_PORT(portid)) 799 process_pkts_inbound(&qconf->inbound, &traffic); 800 else 801 process_pkts_outbound(&qconf->outbound, &traffic); 802 } 803 804 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 805 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 806 } 807 808 static inline void 809 drain_tx_buffers(struct lcore_conf *qconf) 810 { 811 struct buffer *buf; 812 uint32_t portid; 813 814 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 815 buf = &qconf->tx_mbufs[portid]; 816 if (buf->len == 0) 817 continue; 818 send_burst(qconf, buf->len, portid); 819 buf->len = 0; 820 } 821 } 822 823 static inline void 824 drain_crypto_buffers(struct lcore_conf *qconf) 825 { 826 uint32_t i; 827 struct ipsec_ctx *ctx; 828 829 /* drain inbound buffers*/ 830 ctx = &qconf->inbound; 831 for (i = 0; i != ctx->nb_qps; i++) { 832 if (ctx->tbl[i].len != 0) 833 enqueue_cop_burst(ctx->tbl + i); 834 } 835 836 /* drain outbound buffers*/ 837 ctx = &qconf->outbound; 838 for (i = 0; i != ctx->nb_qps; i++) { 839 if (ctx->tbl[i].len != 0) 840 enqueue_cop_burst(ctx->tbl + i); 841 } 842 } 843 844 static void 845 drain_inbound_crypto_queues(const struct lcore_conf *qconf, 846 struct ipsec_ctx *ctx) 847 { 848 uint32_t n; 849 struct ipsec_traffic trf; 850 851 /* dequeue packets from crypto-queue */ 852 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts, 853 RTE_DIM(trf.ipsec.pkts)); 854 if (n == 0) 855 return; 856 857 trf.ip4.num = 0; 858 trf.ip6.num = 0; 859 860 /* split traffic by ipv4-ipv6 */ 861 split46_traffic(&trf, trf.ipsec.pkts, n); 862 863 /* process ipv4 packets */ 864 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0); 865 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 866 867 /* process ipv6 packets */ 868 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0); 869 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 870 } 871 872 static void 873 drain_outbound_crypto_queues(const struct lcore_conf *qconf, 874 struct ipsec_ctx *ctx) 875 { 876 uint32_t n; 877 struct ipsec_traffic trf; 878 879 /* dequeue packets from crypto-queue */ 880 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts, 881 RTE_DIM(trf.ipsec.pkts)); 882 if (n == 0) 883 return; 884 885 trf.ip4.num = 0; 886 trf.ip6.num = 0; 887 888 /* split traffic by ipv4-ipv6 */ 889 split46_traffic(&trf, trf.ipsec.pkts, n); 890 891 /* process ipv4 packets */ 892 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 893 894 /* process ipv6 packets */ 895 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 896 } 897 898 /* main processing loop */ 899 static int32_t 900 main_loop(__attribute__((unused)) void *dummy) 901 { 902 struct rte_mbuf *pkts[MAX_PKT_BURST]; 903 uint32_t lcore_id; 904 uint64_t prev_tsc, diff_tsc, cur_tsc; 905 int32_t i, nb_rx; 906 uint16_t portid; 907 uint8_t queueid; 908 struct lcore_conf *qconf; 909 int32_t socket_id; 910 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 911 / US_PER_S * BURST_TX_DRAIN_US; 912 struct lcore_rx_queue *rxql; 913 914 prev_tsc = 0; 915 lcore_id = rte_lcore_id(); 916 qconf = &lcore_conf[lcore_id]; 917 rxql = qconf->rx_queue_list; 918 socket_id = rte_lcore_to_socket_id(lcore_id); 919 920 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 921 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 922 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 923 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 924 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 925 qconf->inbound.cdev_map = cdev_map_in; 926 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 927 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 928 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 929 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 930 qconf->outbound.cdev_map = cdev_map_out; 931 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 932 933 if (qconf->nb_rx_queue == 0) { 934 RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n", 935 lcore_id); 936 return 0; 937 } 938 939 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 940 941 for (i = 0; i < qconf->nb_rx_queue; i++) { 942 portid = rxql[i].port_id; 943 queueid = rxql[i].queue_id; 944 RTE_LOG(INFO, IPSEC, 945 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 946 lcore_id, portid, queueid); 947 } 948 949 while (1) { 950 cur_tsc = rte_rdtsc(); 951 952 /* TX queue buffer drain */ 953 diff_tsc = cur_tsc - prev_tsc; 954 955 if (unlikely(diff_tsc > drain_tsc)) { 956 drain_tx_buffers(qconf); 957 drain_crypto_buffers(qconf); 958 prev_tsc = cur_tsc; 959 } 960 961 for (i = 0; i < qconf->nb_rx_queue; ++i) { 962 963 /* Read packets from RX queues */ 964 portid = rxql[i].port_id; 965 queueid = rxql[i].queue_id; 966 nb_rx = rte_eth_rx_burst(portid, queueid, 967 pkts, MAX_PKT_BURST); 968 969 if (nb_rx > 0) 970 process_pkts(qconf, pkts, nb_rx, portid); 971 972 /* dequeue and process completed crypto-ops */ 973 if (UNPROTECTED_PORT(portid)) 974 drain_inbound_crypto_queues(qconf, 975 &qconf->inbound); 976 else 977 drain_outbound_crypto_queues(qconf, 978 &qconf->outbound); 979 } 980 } 981 } 982 983 static int32_t 984 check_params(void) 985 { 986 uint8_t lcore; 987 uint16_t portid; 988 uint16_t i; 989 int32_t socket_id; 990 991 if (lcore_params == NULL) { 992 printf("Error: No port/queue/core mappings\n"); 993 return -1; 994 } 995 996 for (i = 0; i < nb_lcore_params; ++i) { 997 lcore = lcore_params[i].lcore_id; 998 if (!rte_lcore_is_enabled(lcore)) { 999 printf("error: lcore %hhu is not enabled in " 1000 "lcore mask\n", lcore); 1001 return -1; 1002 } 1003 socket_id = rte_lcore_to_socket_id(lcore); 1004 if (socket_id != 0 && numa_on == 0) { 1005 printf("warning: lcore %hhu is on socket %d " 1006 "with numa off\n", 1007 lcore, socket_id); 1008 } 1009 portid = lcore_params[i].port_id; 1010 if ((enabled_port_mask & (1 << portid)) == 0) { 1011 printf("port %u is not enabled in port mask\n", portid); 1012 return -1; 1013 } 1014 if (!rte_eth_dev_is_valid_port(portid)) { 1015 printf("port %u is not present on the board\n", portid); 1016 return -1; 1017 } 1018 } 1019 return 0; 1020 } 1021 1022 static uint8_t 1023 get_port_nb_rx_queues(const uint16_t port) 1024 { 1025 int32_t queue = -1; 1026 uint16_t i; 1027 1028 for (i = 0; i < nb_lcore_params; ++i) { 1029 if (lcore_params[i].port_id == port && 1030 lcore_params[i].queue_id > queue) 1031 queue = lcore_params[i].queue_id; 1032 } 1033 return (uint8_t)(++queue); 1034 } 1035 1036 static int32_t 1037 init_lcore_rx_queues(void) 1038 { 1039 uint16_t i, nb_rx_queue; 1040 uint8_t lcore; 1041 1042 for (i = 0; i < nb_lcore_params; ++i) { 1043 lcore = lcore_params[i].lcore_id; 1044 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 1045 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1046 printf("error: too many queues (%u) for lcore: %u\n", 1047 nb_rx_queue + 1, lcore); 1048 return -1; 1049 } 1050 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1051 lcore_params[i].port_id; 1052 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1053 lcore_params[i].queue_id; 1054 lcore_conf[lcore].nb_rx_queue++; 1055 } 1056 return 0; 1057 } 1058 1059 /* display usage */ 1060 static void 1061 print_usage(const char *prgname) 1062 { 1063 fprintf(stderr, "%s [EAL options] --" 1064 " -p PORTMASK" 1065 " [-P]" 1066 " [-u PORTMASK]" 1067 " [-j FRAMESIZE]" 1068 " -f CONFIG_FILE" 1069 " --config (port,queue,lcore)[,(port,queue,lcore)]" 1070 " [--single-sa SAIDX]" 1071 " [--cryptodev_mask MASK]" 1072 "\n\n" 1073 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 1074 " -P : Enable promiscuous mode\n" 1075 " -u PORTMASK: Hexadecimal bitmask of unprotected ports\n" 1076 " -j FRAMESIZE: Enable jumbo frame with 'FRAMESIZE' as maximum\n" 1077 " packet size\n" 1078 " -f CONFIG_FILE: Configuration file\n" 1079 " --config (port,queue,lcore): Rx queue configuration\n" 1080 " --single-sa SAIDX: Use single SA index for outbound traffic,\n" 1081 " bypassing the SP\n" 1082 " --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n" 1083 " devices to configure\n" 1084 "\n", 1085 prgname); 1086 } 1087 1088 static int32_t 1089 parse_portmask(const char *portmask) 1090 { 1091 char *end = NULL; 1092 unsigned long pm; 1093 1094 /* parse hexadecimal string */ 1095 pm = strtoul(portmask, &end, 16); 1096 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1097 return -1; 1098 1099 if ((pm == 0) && errno) 1100 return -1; 1101 1102 return pm; 1103 } 1104 1105 static int32_t 1106 parse_decimal(const char *str) 1107 { 1108 char *end = NULL; 1109 unsigned long num; 1110 1111 num = strtoul(str, &end, 10); 1112 if ((str[0] == '\0') || (end == NULL) || (*end != '\0')) 1113 return -1; 1114 1115 return num; 1116 } 1117 1118 static int32_t 1119 parse_config(const char *q_arg) 1120 { 1121 char s[256]; 1122 const char *p, *p0 = q_arg; 1123 char *end; 1124 enum fieldnames { 1125 FLD_PORT = 0, 1126 FLD_QUEUE, 1127 FLD_LCORE, 1128 _NUM_FLD 1129 }; 1130 unsigned long int_fld[_NUM_FLD]; 1131 char *str_fld[_NUM_FLD]; 1132 int32_t i; 1133 uint32_t size; 1134 1135 nb_lcore_params = 0; 1136 1137 while ((p = strchr(p0, '(')) != NULL) { 1138 ++p; 1139 p0 = strchr(p, ')'); 1140 if (p0 == NULL) 1141 return -1; 1142 1143 size = p0 - p; 1144 if (size >= sizeof(s)) 1145 return -1; 1146 1147 snprintf(s, sizeof(s), "%.*s", size, p); 1148 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1149 _NUM_FLD) 1150 return -1; 1151 for (i = 0; i < _NUM_FLD; i++) { 1152 errno = 0; 1153 int_fld[i] = strtoul(str_fld[i], &end, 0); 1154 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1155 return -1; 1156 } 1157 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1158 printf("exceeded max number of lcore params: %hu\n", 1159 nb_lcore_params); 1160 return -1; 1161 } 1162 lcore_params_array[nb_lcore_params].port_id = 1163 (uint8_t)int_fld[FLD_PORT]; 1164 lcore_params_array[nb_lcore_params].queue_id = 1165 (uint8_t)int_fld[FLD_QUEUE]; 1166 lcore_params_array[nb_lcore_params].lcore_id = 1167 (uint8_t)int_fld[FLD_LCORE]; 1168 ++nb_lcore_params; 1169 } 1170 lcore_params = lcore_params_array; 1171 return 0; 1172 } 1173 1174 static int32_t 1175 parse_args(int32_t argc, char **argv) 1176 { 1177 int32_t opt, ret; 1178 char **argvopt; 1179 int32_t option_index; 1180 char *prgname = argv[0]; 1181 int32_t f_present = 0; 1182 1183 argvopt = argv; 1184 1185 while ((opt = getopt_long(argc, argvopt, "p:Pu:f:j:", 1186 lgopts, &option_index)) != EOF) { 1187 1188 switch (opt) { 1189 case 'p': 1190 enabled_port_mask = parse_portmask(optarg); 1191 if (enabled_port_mask == 0) { 1192 printf("invalid portmask\n"); 1193 print_usage(prgname); 1194 return -1; 1195 } 1196 break; 1197 case 'P': 1198 printf("Promiscuous mode selected\n"); 1199 promiscuous_on = 1; 1200 break; 1201 case 'u': 1202 unprotected_port_mask = parse_portmask(optarg); 1203 if (unprotected_port_mask == 0) { 1204 printf("invalid unprotected portmask\n"); 1205 print_usage(prgname); 1206 return -1; 1207 } 1208 break; 1209 case 'f': 1210 if (f_present == 1) { 1211 printf("\"-f\" option present more than " 1212 "once!\n"); 1213 print_usage(prgname); 1214 return -1; 1215 } 1216 if (parse_cfg_file(optarg) < 0) { 1217 printf("parsing file \"%s\" failed\n", 1218 optarg); 1219 print_usage(prgname); 1220 return -1; 1221 } 1222 f_present = 1; 1223 break; 1224 case 'j': 1225 { 1226 int32_t size = parse_decimal(optarg); 1227 if (size <= 1518) { 1228 printf("Invalid jumbo frame size\n"); 1229 if (size < 0) { 1230 print_usage(prgname); 1231 return -1; 1232 } 1233 printf("Using default value 9000\n"); 1234 frame_size = 9000; 1235 } else { 1236 frame_size = size; 1237 } 1238 } 1239 printf("Enabled jumbo frames size %u\n", frame_size); 1240 break; 1241 case CMD_LINE_OPT_CONFIG_NUM: 1242 ret = parse_config(optarg); 1243 if (ret) { 1244 printf("Invalid config\n"); 1245 print_usage(prgname); 1246 return -1; 1247 } 1248 break; 1249 case CMD_LINE_OPT_SINGLE_SA_NUM: 1250 ret = parse_decimal(optarg); 1251 if (ret == -1) { 1252 printf("Invalid argument[sa_idx]\n"); 1253 print_usage(prgname); 1254 return -1; 1255 } 1256 1257 /* else */ 1258 single_sa = 1; 1259 single_sa_idx = ret; 1260 printf("Configured with single SA index %u\n", 1261 single_sa_idx); 1262 break; 1263 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM: 1264 ret = parse_portmask(optarg); 1265 if (ret == -1) { 1266 printf("Invalid argument[portmask]\n"); 1267 print_usage(prgname); 1268 return -1; 1269 } 1270 1271 /* else */ 1272 enabled_cryptodev_mask = ret; 1273 break; 1274 default: 1275 print_usage(prgname); 1276 return -1; 1277 } 1278 } 1279 1280 if (f_present == 0) { 1281 printf("Mandatory option \"-f\" not present\n"); 1282 return -1; 1283 } 1284 1285 if (optind >= 0) 1286 argv[optind-1] = prgname; 1287 1288 ret = optind-1; 1289 optind = 1; /* reset getopt lib */ 1290 return ret; 1291 } 1292 1293 static void 1294 print_ethaddr(const char *name, const struct ether_addr *eth_addr) 1295 { 1296 char buf[ETHER_ADDR_FMT_SIZE]; 1297 ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); 1298 printf("%s%s", name, buf); 1299 } 1300 1301 /* Check the link status of all ports in up to 9s, and print them finally */ 1302 static void 1303 check_all_ports_link_status(uint32_t port_mask) 1304 { 1305 #define CHECK_INTERVAL 100 /* 100ms */ 1306 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1307 uint16_t portid; 1308 uint8_t count, all_ports_up, print_flag = 0; 1309 struct rte_eth_link link; 1310 1311 printf("\nChecking link status"); 1312 fflush(stdout); 1313 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1314 all_ports_up = 1; 1315 RTE_ETH_FOREACH_DEV(portid) { 1316 if ((port_mask & (1 << portid)) == 0) 1317 continue; 1318 memset(&link, 0, sizeof(link)); 1319 rte_eth_link_get_nowait(portid, &link); 1320 /* print link status if flag set */ 1321 if (print_flag == 1) { 1322 if (link.link_status) 1323 printf( 1324 "Port%d Link Up - speed %u Mbps -%s\n", 1325 portid, link.link_speed, 1326 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1327 ("full-duplex") : ("half-duplex\n")); 1328 else 1329 printf("Port %d Link Down\n", portid); 1330 continue; 1331 } 1332 /* clear all_ports_up flag if any link down */ 1333 if (link.link_status == ETH_LINK_DOWN) { 1334 all_ports_up = 0; 1335 break; 1336 } 1337 } 1338 /* after finally printing all link status, get out */ 1339 if (print_flag == 1) 1340 break; 1341 1342 if (all_ports_up == 0) { 1343 printf("."); 1344 fflush(stdout); 1345 rte_delay_ms(CHECK_INTERVAL); 1346 } 1347 1348 /* set the print_flag if all ports up or timeout */ 1349 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1350 print_flag = 1; 1351 printf("done\n"); 1352 } 1353 } 1354 } 1355 1356 static int32_t 1357 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1358 uint16_t qp, struct lcore_params *params, 1359 struct ipsec_ctx *ipsec_ctx, 1360 const struct rte_cryptodev_capabilities *cipher, 1361 const struct rte_cryptodev_capabilities *auth, 1362 const struct rte_cryptodev_capabilities *aead) 1363 { 1364 int32_t ret = 0; 1365 unsigned long i; 1366 struct cdev_key key = { 0 }; 1367 1368 key.lcore_id = params->lcore_id; 1369 if (cipher) 1370 key.cipher_algo = cipher->sym.cipher.algo; 1371 if (auth) 1372 key.auth_algo = auth->sym.auth.algo; 1373 if (aead) 1374 key.aead_algo = aead->sym.aead.algo; 1375 1376 ret = rte_hash_lookup(map, &key); 1377 if (ret != -ENOENT) 1378 return 0; 1379 1380 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1381 if (ipsec_ctx->tbl[i].id == cdev_id) 1382 break; 1383 1384 if (i == ipsec_ctx->nb_qps) { 1385 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1386 printf("Maximum number of crypto devices assigned to " 1387 "a core, increase MAX_QP_PER_LCORE value\n"); 1388 return 0; 1389 } 1390 ipsec_ctx->tbl[i].id = cdev_id; 1391 ipsec_ctx->tbl[i].qp = qp; 1392 ipsec_ctx->nb_qps++; 1393 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1394 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1395 cdev_id, qp, i); 1396 } 1397 1398 ret = rte_hash_add_key_data(map, &key, (void *)i); 1399 if (ret < 0) { 1400 printf("Faled to insert cdev mapping for (lcore %u, " 1401 "cdev %u, qp %u), errno %d\n", 1402 key.lcore_id, ipsec_ctx->tbl[i].id, 1403 ipsec_ctx->tbl[i].qp, ret); 1404 return 0; 1405 } 1406 1407 return 1; 1408 } 1409 1410 static int32_t 1411 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1412 uint16_t qp, struct lcore_params *params) 1413 { 1414 int32_t ret = 0; 1415 const struct rte_cryptodev_capabilities *i, *j; 1416 struct rte_hash *map; 1417 struct lcore_conf *qconf; 1418 struct ipsec_ctx *ipsec_ctx; 1419 const char *str; 1420 1421 qconf = &lcore_conf[params->lcore_id]; 1422 1423 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 1424 map = cdev_map_out; 1425 ipsec_ctx = &qconf->outbound; 1426 str = "Outbound"; 1427 } else { 1428 map = cdev_map_in; 1429 ipsec_ctx = &qconf->inbound; 1430 str = "Inbound"; 1431 } 1432 1433 /* Required cryptodevs with operation chainning */ 1434 if (!(dev_info->feature_flags & 1435 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 1436 return ret; 1437 1438 for (i = dev_info->capabilities; 1439 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 1440 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1441 continue; 1442 1443 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1444 ret |= add_mapping(map, str, cdev_id, qp, params, 1445 ipsec_ctx, NULL, NULL, i); 1446 continue; 1447 } 1448 1449 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 1450 continue; 1451 1452 for (j = dev_info->capabilities; 1453 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 1454 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1455 continue; 1456 1457 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 1458 continue; 1459 1460 ret |= add_mapping(map, str, cdev_id, qp, params, 1461 ipsec_ctx, i, j, NULL); 1462 } 1463 } 1464 1465 return ret; 1466 } 1467 1468 /* Check if the device is enabled by cryptodev_mask */ 1469 static int 1470 check_cryptodev_mask(uint8_t cdev_id) 1471 { 1472 if (enabled_cryptodev_mask & (1 << cdev_id)) 1473 return 0; 1474 1475 return -1; 1476 } 1477 1478 static int32_t 1479 cryptodevs_init(void) 1480 { 1481 struct rte_cryptodev_config dev_conf; 1482 struct rte_cryptodev_qp_conf qp_conf; 1483 uint16_t idx, max_nb_qps, qp, i; 1484 int16_t cdev_id, port_id; 1485 struct rte_hash_parameters params = { 0 }; 1486 1487 params.entries = CDEV_MAP_ENTRIES; 1488 params.key_len = sizeof(struct cdev_key); 1489 params.hash_func = rte_jhash; 1490 params.hash_func_init_val = 0; 1491 params.socket_id = rte_socket_id(); 1492 1493 params.name = "cdev_map_in"; 1494 cdev_map_in = rte_hash_create(¶ms); 1495 if (cdev_map_in == NULL) 1496 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1497 rte_errno); 1498 1499 params.name = "cdev_map_out"; 1500 cdev_map_out = rte_hash_create(¶ms); 1501 if (cdev_map_out == NULL) 1502 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1503 rte_errno); 1504 1505 printf("lcore/cryptodev/qp mappings:\n"); 1506 1507 uint32_t max_sess_sz = 0, sess_sz; 1508 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1509 void *sec_ctx; 1510 1511 /* Get crypto priv session size */ 1512 sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 1513 if (sess_sz > max_sess_sz) 1514 max_sess_sz = sess_sz; 1515 1516 /* 1517 * If crypto device is security capable, need to check the 1518 * size of security session as well. 1519 */ 1520 1521 /* Get security context of the crypto device */ 1522 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id); 1523 if (sec_ctx == NULL) 1524 continue; 1525 1526 /* Get size of security session */ 1527 sess_sz = rte_security_session_get_size(sec_ctx); 1528 if (sess_sz > max_sess_sz) 1529 max_sess_sz = sess_sz; 1530 } 1531 RTE_ETH_FOREACH_DEV(port_id) { 1532 void *sec_ctx; 1533 1534 if ((enabled_port_mask & (1 << port_id)) == 0) 1535 continue; 1536 1537 sec_ctx = rte_eth_dev_get_sec_ctx(port_id); 1538 if (sec_ctx == NULL) 1539 continue; 1540 1541 sess_sz = rte_security_session_get_size(sec_ctx); 1542 if (sess_sz > max_sess_sz) 1543 max_sess_sz = sess_sz; 1544 } 1545 1546 idx = 0; 1547 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1548 struct rte_cryptodev_info cdev_info; 1549 1550 if (check_cryptodev_mask((uint8_t)cdev_id)) 1551 continue; 1552 1553 rte_cryptodev_info_get(cdev_id, &cdev_info); 1554 1555 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 1556 max_nb_qps = cdev_info.max_nb_queue_pairs; 1557 else 1558 max_nb_qps = nb_lcore_params; 1559 1560 qp = 0; 1561 i = 0; 1562 while (qp < max_nb_qps && i < nb_lcore_params) { 1563 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 1564 &lcore_params[idx])) 1565 qp++; 1566 idx++; 1567 idx = idx % nb_lcore_params; 1568 i++; 1569 } 1570 1571 if (qp == 0) 1572 continue; 1573 1574 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 1575 dev_conf.nb_queue_pairs = qp; 1576 1577 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions; 1578 if (dev_max_sess != 0 && dev_max_sess < (CDEV_MP_NB_OBJS / 2)) 1579 rte_exit(EXIT_FAILURE, 1580 "Device does not support at least %u " 1581 "sessions", CDEV_MP_NB_OBJS / 2); 1582 1583 if (!socket_ctx[dev_conf.socket_id].session_pool) { 1584 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1585 struct rte_mempool *sess_mp; 1586 1587 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1588 "sess_mp_%u", dev_conf.socket_id); 1589 sess_mp = rte_mempool_create(mp_name, 1590 CDEV_MP_NB_OBJS, 1591 max_sess_sz, 1592 CDEV_MP_CACHE_SZ, 1593 0, NULL, NULL, NULL, 1594 NULL, dev_conf.socket_id, 1595 0); 1596 if (sess_mp == NULL) 1597 rte_exit(EXIT_FAILURE, 1598 "Cannot create session pool on socket %d\n", 1599 dev_conf.socket_id); 1600 else 1601 printf("Allocated session pool on socket %d\n", 1602 dev_conf.socket_id); 1603 socket_ctx[dev_conf.socket_id].session_pool = sess_mp; 1604 } 1605 1606 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 1607 rte_panic("Failed to initialize cryptodev %u\n", 1608 cdev_id); 1609 1610 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 1611 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 1612 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 1613 &qp_conf, dev_conf.socket_id, 1614 socket_ctx[dev_conf.socket_id].session_pool)) 1615 rte_panic("Failed to setup queue %u for " 1616 "cdev_id %u\n", 0, cdev_id); 1617 1618 if (rte_cryptodev_start(cdev_id)) 1619 rte_panic("Failed to start cryptodev %u\n", 1620 cdev_id); 1621 } 1622 1623 /* create session pools for eth devices that implement security */ 1624 RTE_ETH_FOREACH_DEV(port_id) { 1625 if ((enabled_port_mask & (1 << port_id)) && 1626 rte_eth_dev_get_sec_ctx(port_id)) { 1627 int socket_id = rte_eth_dev_socket_id(port_id); 1628 1629 if (!socket_ctx[socket_id].session_pool) { 1630 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1631 struct rte_mempool *sess_mp; 1632 1633 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1634 "sess_mp_%u", socket_id); 1635 sess_mp = rte_mempool_create(mp_name, 1636 CDEV_MP_NB_OBJS, 1637 max_sess_sz, 1638 CDEV_MP_CACHE_SZ, 1639 0, NULL, NULL, NULL, 1640 NULL, socket_id, 1641 0); 1642 if (sess_mp == NULL) 1643 rte_exit(EXIT_FAILURE, 1644 "Cannot create session pool " 1645 "on socket %d\n", socket_id); 1646 else 1647 printf("Allocated session pool " 1648 "on socket %d\n", socket_id); 1649 socket_ctx[socket_id].session_pool = sess_mp; 1650 } 1651 } 1652 } 1653 1654 1655 printf("\n"); 1656 1657 return 0; 1658 } 1659 1660 static void 1661 port_init(uint16_t portid) 1662 { 1663 struct rte_eth_dev_info dev_info; 1664 struct rte_eth_txconf *txconf; 1665 uint16_t nb_tx_queue, nb_rx_queue; 1666 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 1667 int32_t ret, socket_id; 1668 struct lcore_conf *qconf; 1669 struct ether_addr ethaddr; 1670 struct rte_eth_conf local_port_conf = port_conf; 1671 1672 rte_eth_dev_info_get(portid, &dev_info); 1673 1674 printf("Configuring device port %u:\n", portid); 1675 1676 rte_eth_macaddr_get(portid, ðaddr); 1677 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ethaddr); 1678 print_ethaddr("Address: ", ðaddr); 1679 printf("\n"); 1680 1681 nb_rx_queue = get_port_nb_rx_queues(portid); 1682 nb_tx_queue = nb_lcores; 1683 1684 if (nb_rx_queue > dev_info.max_rx_queues) 1685 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1686 "(max rx queue is %u)\n", 1687 nb_rx_queue, dev_info.max_rx_queues); 1688 1689 if (nb_tx_queue > dev_info.max_tx_queues) 1690 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1691 "(max tx queue is %u)\n", 1692 nb_tx_queue, dev_info.max_tx_queues); 1693 1694 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 1695 nb_rx_queue, nb_tx_queue); 1696 1697 if (frame_size) { 1698 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 1699 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1700 } 1701 1702 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_SECURITY) 1703 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SECURITY; 1704 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SECURITY) 1705 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SECURITY; 1706 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 1707 local_port_conf.txmode.offloads |= 1708 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1709 1710 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 1711 dev_info.flow_type_rss_offloads; 1712 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 1713 port_conf.rx_adv_conf.rss_conf.rss_hf) { 1714 printf("Port %u modified RSS hash function based on hardware support," 1715 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 1716 portid, 1717 port_conf.rx_adv_conf.rss_conf.rss_hf, 1718 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 1719 } 1720 1721 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 1722 &local_port_conf); 1723 if (ret < 0) 1724 rte_exit(EXIT_FAILURE, "Cannot configure device: " 1725 "err=%d, port=%d\n", ret, portid); 1726 1727 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 1728 if (ret < 0) 1729 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 1730 "err=%d, port=%d\n", ret, portid); 1731 1732 /* init one TX queue per lcore */ 1733 tx_queueid = 0; 1734 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1735 if (rte_lcore_is_enabled(lcore_id) == 0) 1736 continue; 1737 1738 if (numa_on) 1739 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1740 else 1741 socket_id = 0; 1742 1743 /* init TX queue */ 1744 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 1745 1746 txconf = &dev_info.default_txconf; 1747 txconf->offloads = local_port_conf.txmode.offloads; 1748 1749 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 1750 socket_id, txconf); 1751 if (ret < 0) 1752 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 1753 "err=%d, port=%d\n", ret, portid); 1754 1755 qconf = &lcore_conf[lcore_id]; 1756 qconf->tx_queue_id[portid] = tx_queueid; 1757 tx_queueid++; 1758 1759 /* init RX queues */ 1760 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 1761 struct rte_eth_rxconf rxq_conf; 1762 1763 if (portid != qconf->rx_queue_list[queue].port_id) 1764 continue; 1765 1766 rx_queueid = qconf->rx_queue_list[queue].queue_id; 1767 1768 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 1769 socket_id); 1770 1771 rxq_conf = dev_info.default_rxconf; 1772 rxq_conf.offloads = local_port_conf.rxmode.offloads; 1773 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 1774 nb_rxd, socket_id, &rxq_conf, 1775 socket_ctx[socket_id].mbuf_pool); 1776 if (ret < 0) 1777 rte_exit(EXIT_FAILURE, 1778 "rte_eth_rx_queue_setup: err=%d, " 1779 "port=%d\n", ret, portid); 1780 } 1781 } 1782 printf("\n"); 1783 } 1784 1785 static void 1786 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 1787 { 1788 char s[64]; 1789 uint32_t buff_size = frame_size ? (frame_size + RTE_PKTMBUF_HEADROOM) : 1790 RTE_MBUF_DEFAULT_BUF_SIZE; 1791 1792 1793 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 1794 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 1795 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 1796 buff_size, 1797 socket_id); 1798 if (ctx->mbuf_pool == NULL) 1799 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 1800 socket_id); 1801 else 1802 printf("Allocated mbuf pool on socket %d\n", socket_id); 1803 } 1804 1805 static inline int 1806 inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md) 1807 { 1808 struct ipsec_sa *sa; 1809 1810 /* For inline protocol processing, the metadata in the event will 1811 * uniquely identify the security session which raised the event. 1812 * Application would then need the userdata it had registered with the 1813 * security session to process the event. 1814 */ 1815 1816 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md); 1817 1818 if (sa == NULL) { 1819 /* userdata could not be retrieved */ 1820 return -1; 1821 } 1822 1823 /* Sequence number over flow. SA need to be re-established */ 1824 RTE_SET_USED(sa); 1825 return 0; 1826 } 1827 1828 static int 1829 inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type, 1830 void *param, void *ret_param) 1831 { 1832 uint64_t md; 1833 struct rte_eth_event_ipsec_desc *event_desc = NULL; 1834 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 1835 rte_eth_dev_get_sec_ctx(port_id); 1836 1837 RTE_SET_USED(param); 1838 1839 if (type != RTE_ETH_EVENT_IPSEC) 1840 return -1; 1841 1842 event_desc = ret_param; 1843 if (event_desc == NULL) { 1844 printf("Event descriptor not set\n"); 1845 return -1; 1846 } 1847 1848 md = event_desc->metadata; 1849 1850 if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW) 1851 return inline_ipsec_event_esn_overflow(ctx, md); 1852 else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) { 1853 printf("Invalid IPsec event reported\n"); 1854 return -1; 1855 } 1856 1857 return -1; 1858 } 1859 1860 int32_t 1861 main(int32_t argc, char **argv) 1862 { 1863 int32_t ret; 1864 uint32_t lcore_id; 1865 uint8_t socket_id; 1866 uint16_t portid; 1867 1868 /* init EAL */ 1869 ret = rte_eal_init(argc, argv); 1870 if (ret < 0) 1871 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 1872 argc -= ret; 1873 argv += ret; 1874 1875 /* parse application arguments (after the EAL ones) */ 1876 ret = parse_args(argc, argv); 1877 if (ret < 0) 1878 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 1879 1880 if ((unprotected_port_mask & enabled_port_mask) != 1881 unprotected_port_mask) 1882 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 1883 unprotected_port_mask); 1884 1885 if (check_params() < 0) 1886 rte_exit(EXIT_FAILURE, "check_params failed\n"); 1887 1888 ret = init_lcore_rx_queues(); 1889 if (ret < 0) 1890 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 1891 1892 nb_lcores = rte_lcore_count(); 1893 1894 /* Replicate each context per socket */ 1895 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1896 if (rte_lcore_is_enabled(lcore_id) == 0) 1897 continue; 1898 1899 if (numa_on) 1900 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1901 else 1902 socket_id = 0; 1903 1904 if (socket_ctx[socket_id].mbuf_pool) 1905 continue; 1906 1907 sa_init(&socket_ctx[socket_id], socket_id); 1908 1909 sp4_init(&socket_ctx[socket_id], socket_id); 1910 1911 sp6_init(&socket_ctx[socket_id], socket_id); 1912 1913 rt_init(&socket_ctx[socket_id], socket_id); 1914 1915 pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF); 1916 } 1917 1918 RTE_ETH_FOREACH_DEV(portid) { 1919 if ((enabled_port_mask & (1 << portid)) == 0) 1920 continue; 1921 1922 port_init(portid); 1923 } 1924 1925 cryptodevs_init(); 1926 1927 /* start ports */ 1928 RTE_ETH_FOREACH_DEV(portid) { 1929 if ((enabled_port_mask & (1 << portid)) == 0) 1930 continue; 1931 1932 /* Start device */ 1933 ret = rte_eth_dev_start(portid); 1934 if (ret < 0) 1935 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 1936 "err=%d, port=%d\n", ret, portid); 1937 /* 1938 * If enabled, put device in promiscuous mode. 1939 * This allows IO forwarding mode to forward packets 1940 * to itself through 2 cross-connected ports of the 1941 * target machine. 1942 */ 1943 if (promiscuous_on) 1944 rte_eth_promiscuous_enable(portid); 1945 1946 rte_eth_dev_callback_register(portid, 1947 RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL); 1948 } 1949 1950 check_all_ports_link_status(enabled_port_mask); 1951 1952 /* launch per-lcore init on every lcore */ 1953 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 1954 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1955 if (rte_eal_wait_lcore(lcore_id) < 0) 1956 return -1; 1957 } 1958 1959 return 0; 1960 } 1961