1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <netinet/in.h> 11 #include <netinet/ip.h> 12 #include <netinet/ip6.h> 13 #include <string.h> 14 #include <sys/queue.h> 15 #include <stdarg.h> 16 #include <errno.h> 17 #include <getopt.h> 18 19 #include <rte_common.h> 20 #include <rte_byteorder.h> 21 #include <rte_log.h> 22 #include <rte_eal.h> 23 #include <rte_launch.h> 24 #include <rte_atomic.h> 25 #include <rte_cycles.h> 26 #include <rte_prefetch.h> 27 #include <rte_lcore.h> 28 #include <rte_per_lcore.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_interrupts.h> 31 #include <rte_random.h> 32 #include <rte_debug.h> 33 #include <rte_ether.h> 34 #include <rte_ethdev.h> 35 #include <rte_mempool.h> 36 #include <rte_mbuf.h> 37 #include <rte_acl.h> 38 #include <rte_lpm.h> 39 #include <rte_lpm6.h> 40 #include <rte_hash.h> 41 #include <rte_jhash.h> 42 #include <rte_cryptodev.h> 43 44 #include "ipsec.h" 45 #include "parser.h" 46 47 #define RTE_LOGTYPE_IPSEC RTE_LOGTYPE_USER1 48 49 #define MAX_JUMBO_PKT_LEN 9600 50 51 #define MEMPOOL_CACHE_SIZE 256 52 53 #define NB_MBUF (32000) 54 55 #define CDEV_QUEUE_DESC 2048 56 #define CDEV_MAP_ENTRIES 1024 57 #define CDEV_MP_NB_OBJS 2048 58 #define CDEV_MP_CACHE_SZ 64 59 #define MAX_QUEUE_PAIRS 1 60 61 #define OPTION_CONFIG "config" 62 #define OPTION_SINGLE_SA "single-sa" 63 64 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 65 66 #define NB_SOCKETS 4 67 68 /* Configure how many packets ahead to prefetch, when reading packets */ 69 #define PREFETCH_OFFSET 3 70 71 #define MAX_RX_QUEUE_PER_LCORE 16 72 73 #define MAX_LCORE_PARAMS 1024 74 75 #define UNPROTECTED_PORT(port) (unprotected_port_mask & (1 << portid)) 76 77 /* 78 * Configurable number of RX/TX ring descriptors 79 */ 80 #define IPSEC_SECGW_RX_DESC_DEFAULT 128 81 #define IPSEC_SECGW_TX_DESC_DEFAULT 512 82 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 83 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 84 85 #if RTE_BYTE_ORDER != RTE_LITTLE_ENDIAN 86 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 87 (((uint64_t)((a) & 0xff) << 56) | \ 88 ((uint64_t)((b) & 0xff) << 48) | \ 89 ((uint64_t)((c) & 0xff) << 40) | \ 90 ((uint64_t)((d) & 0xff) << 32) | \ 91 ((uint64_t)((e) & 0xff) << 24) | \ 92 ((uint64_t)((f) & 0xff) << 16) | \ 93 ((uint64_t)((g) & 0xff) << 8) | \ 94 ((uint64_t)(h) & 0xff)) 95 #else 96 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 97 (((uint64_t)((h) & 0xff) << 56) | \ 98 ((uint64_t)((g) & 0xff) << 48) | \ 99 ((uint64_t)((f) & 0xff) << 40) | \ 100 ((uint64_t)((e) & 0xff) << 32) | \ 101 ((uint64_t)((d) & 0xff) << 24) | \ 102 ((uint64_t)((c) & 0xff) << 16) | \ 103 ((uint64_t)((b) & 0xff) << 8) | \ 104 ((uint64_t)(a) & 0xff)) 105 #endif 106 #define ETHADDR(a, b, c, d, e, f) (__BYTES_TO_UINT64(a, b, c, d, e, f, 0, 0)) 107 108 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 109 addr.addr_bytes[0], addr.addr_bytes[1], \ 110 addr.addr_bytes[2], addr.addr_bytes[3], \ 111 addr.addr_bytes[4], addr.addr_bytes[5], \ 112 0, 0) 113 114 /* port/source ethernet addr and destination ethernet addr */ 115 struct ethaddr_info { 116 uint64_t src, dst; 117 }; 118 119 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 120 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 121 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 122 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 123 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 124 }; 125 126 /* mask of enabled ports */ 127 static uint32_t enabled_port_mask; 128 static uint32_t unprotected_port_mask; 129 static int32_t promiscuous_on = 1; 130 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 131 static uint32_t nb_lcores; 132 static uint32_t single_sa; 133 static uint32_t single_sa_idx; 134 static uint32_t frame_size; 135 136 struct lcore_rx_queue { 137 uint16_t port_id; 138 uint8_t queue_id; 139 } __rte_cache_aligned; 140 141 struct lcore_params { 142 uint16_t port_id; 143 uint8_t queue_id; 144 uint8_t lcore_id; 145 } __rte_cache_aligned; 146 147 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 148 149 static struct lcore_params *lcore_params; 150 static uint16_t nb_lcore_params; 151 152 static struct rte_hash *cdev_map_in; 153 static struct rte_hash *cdev_map_out; 154 155 struct buffer { 156 uint16_t len; 157 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 158 }; 159 160 struct lcore_conf { 161 uint16_t nb_rx_queue; 162 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 163 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 164 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 165 struct ipsec_ctx inbound; 166 struct ipsec_ctx outbound; 167 struct rt_ctx *rt4_ctx; 168 struct rt_ctx *rt6_ctx; 169 } __rte_cache_aligned; 170 171 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 172 173 static struct rte_eth_conf port_conf = { 174 .rxmode = { 175 .mq_mode = ETH_MQ_RX_RSS, 176 .max_rx_pkt_len = ETHER_MAX_LEN, 177 .split_hdr_size = 0, 178 .offloads = DEV_RX_OFFLOAD_CHECKSUM | 179 DEV_RX_OFFLOAD_CRC_STRIP, 180 .ignore_offload_bitfield = 1, 181 }, 182 .rx_adv_conf = { 183 .rss_conf = { 184 .rss_key = NULL, 185 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 186 ETH_RSS_TCP | ETH_RSS_SCTP, 187 }, 188 }, 189 .txmode = { 190 .mq_mode = ETH_MQ_TX_NONE, 191 .offloads = (DEV_TX_OFFLOAD_IPV4_CKSUM | 192 DEV_TX_OFFLOAD_MULTI_SEGS), 193 }, 194 }; 195 196 static struct socket_ctx socket_ctx[NB_SOCKETS]; 197 198 struct traffic_type { 199 const uint8_t *data[MAX_PKT_BURST * 2]; 200 struct rte_mbuf *pkts[MAX_PKT_BURST * 2]; 201 uint32_t res[MAX_PKT_BURST * 2]; 202 uint32_t num; 203 }; 204 205 struct ipsec_traffic { 206 struct traffic_type ipsec; 207 struct traffic_type ip4; 208 struct traffic_type ip6; 209 }; 210 211 static inline void 212 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 213 { 214 uint8_t *nlp; 215 struct ether_hdr *eth; 216 217 eth = rte_pktmbuf_mtod(pkt, struct ether_hdr *); 218 if (eth->ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) { 219 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, ETHER_HDR_LEN); 220 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip, ip_p)); 221 if (*nlp == IPPROTO_ESP) 222 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 223 else { 224 t->ip4.data[t->ip4.num] = nlp; 225 t->ip4.pkts[(t->ip4.num)++] = pkt; 226 } 227 } else if (eth->ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) { 228 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, ETHER_HDR_LEN); 229 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip6_hdr, ip6_nxt)); 230 if (*nlp == IPPROTO_ESP) 231 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 232 else { 233 t->ip6.data[t->ip6.num] = nlp; 234 t->ip6.pkts[(t->ip6.num)++] = pkt; 235 } 236 } else { 237 /* Unknown/Unsupported type, drop the packet */ 238 RTE_LOG(ERR, IPSEC, "Unsupported packet type\n"); 239 rte_pktmbuf_free(pkt); 240 } 241 } 242 243 static inline void 244 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 245 uint16_t nb_pkts) 246 { 247 int32_t i; 248 249 t->ipsec.num = 0; 250 t->ip4.num = 0; 251 t->ip6.num = 0; 252 253 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 254 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 255 void *)); 256 prepare_one_packet(pkts[i], t); 257 } 258 /* Process left packets */ 259 for (; i < nb_pkts; i++) 260 prepare_one_packet(pkts[i], t); 261 } 262 263 static inline void 264 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port) 265 { 266 struct ip *ip; 267 struct ether_hdr *ethhdr; 268 269 ip = rte_pktmbuf_mtod(pkt, struct ip *); 270 271 ethhdr = (struct ether_hdr *)rte_pktmbuf_prepend(pkt, ETHER_HDR_LEN); 272 273 if (ip->ip_v == IPVERSION) { 274 pkt->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4; 275 pkt->l3_len = sizeof(struct ip); 276 pkt->l2_len = ETHER_HDR_LEN; 277 278 ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); 279 } else { 280 pkt->ol_flags |= PKT_TX_IPV6; 281 pkt->l3_len = sizeof(struct ip6_hdr); 282 pkt->l2_len = ETHER_HDR_LEN; 283 284 ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6); 285 } 286 287 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 288 sizeof(struct ether_addr)); 289 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 290 sizeof(struct ether_addr)); 291 } 292 293 static inline void 294 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port) 295 { 296 int32_t i; 297 const int32_t prefetch_offset = 2; 298 299 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 300 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 301 prepare_tx_pkt(pkts[i], port); 302 } 303 /* Process left packets */ 304 for (; i < nb_pkts; i++) 305 prepare_tx_pkt(pkts[i], port); 306 } 307 308 /* Send burst of packets on an output interface */ 309 static inline int32_t 310 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 311 { 312 struct rte_mbuf **m_table; 313 int32_t ret; 314 uint16_t queueid; 315 316 queueid = qconf->tx_queue_id[port]; 317 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 318 319 prepare_tx_burst(m_table, n, port); 320 321 ret = rte_eth_tx_burst(port, queueid, m_table, n); 322 if (unlikely(ret < n)) { 323 do { 324 rte_pktmbuf_free(m_table[ret]); 325 } while (++ret < n); 326 } 327 328 return 0; 329 } 330 331 /* Enqueue a single packet, and send burst if queue is filled */ 332 static inline int32_t 333 send_single_packet(struct rte_mbuf *m, uint16_t port) 334 { 335 uint32_t lcore_id; 336 uint16_t len; 337 struct lcore_conf *qconf; 338 339 lcore_id = rte_lcore_id(); 340 341 qconf = &lcore_conf[lcore_id]; 342 len = qconf->tx_mbufs[port].len; 343 qconf->tx_mbufs[port].m_table[len] = m; 344 len++; 345 346 /* enough pkts to be sent */ 347 if (unlikely(len == MAX_PKT_BURST)) { 348 send_burst(qconf, MAX_PKT_BURST, port); 349 len = 0; 350 } 351 352 qconf->tx_mbufs[port].len = len; 353 return 0; 354 } 355 356 static inline void 357 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 358 uint16_t lim) 359 { 360 struct rte_mbuf *m; 361 uint32_t i, j, res, sa_idx; 362 363 if (ip->num == 0 || sp == NULL) 364 return; 365 366 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 367 ip->num, DEFAULT_MAX_CATEGORIES); 368 369 j = 0; 370 for (i = 0; i < ip->num; i++) { 371 m = ip->pkts[i]; 372 res = ip->res[i]; 373 if (res & BYPASS) { 374 ip->pkts[j++] = m; 375 continue; 376 } 377 if (res & DISCARD || i < lim) { 378 rte_pktmbuf_free(m); 379 continue; 380 } 381 /* Only check SPI match for processed IPSec packets */ 382 sa_idx = ip->res[i] & PROTECT_MASK; 383 if (sa_idx == 0 || !inbound_sa_check(sa, m, sa_idx)) { 384 rte_pktmbuf_free(m); 385 continue; 386 } 387 ip->pkts[j++] = m; 388 } 389 ip->num = j; 390 } 391 392 static inline void 393 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 394 struct ipsec_traffic *traffic) 395 { 396 struct rte_mbuf *m; 397 uint16_t idx, nb_pkts_in, i, n_ip4, n_ip6; 398 399 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 400 traffic->ipsec.num, MAX_PKT_BURST); 401 402 n_ip4 = traffic->ip4.num; 403 n_ip6 = traffic->ip6.num; 404 405 /* SP/ACL Inbound check ipsec and ip4 */ 406 for (i = 0; i < nb_pkts_in; i++) { 407 m = traffic->ipsec.pkts[i]; 408 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 409 if (ip->ip_v == IPVERSION) { 410 idx = traffic->ip4.num++; 411 traffic->ip4.pkts[idx] = m; 412 traffic->ip4.data[idx] = rte_pktmbuf_mtod_offset(m, 413 uint8_t *, offsetof(struct ip, ip_p)); 414 } else if (ip->ip_v == IP6_VERSION) { 415 idx = traffic->ip6.num++; 416 traffic->ip6.pkts[idx] = m; 417 traffic->ip6.data[idx] = rte_pktmbuf_mtod_offset(m, 418 uint8_t *, 419 offsetof(struct ip6_hdr, ip6_nxt)); 420 } else 421 rte_pktmbuf_free(m); 422 } 423 424 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 425 n_ip4); 426 427 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 428 n_ip6); 429 } 430 431 static inline void 432 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 433 struct traffic_type *ipsec) 434 { 435 struct rte_mbuf *m; 436 uint32_t i, j, sa_idx; 437 438 if (ip->num == 0 || sp == NULL) 439 return; 440 441 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 442 ip->num, DEFAULT_MAX_CATEGORIES); 443 444 j = 0; 445 for (i = 0; i < ip->num; i++) { 446 m = ip->pkts[i]; 447 sa_idx = ip->res[i] & PROTECT_MASK; 448 if ((ip->res[i] == 0) || (ip->res[i] & DISCARD)) 449 rte_pktmbuf_free(m); 450 else if (sa_idx != 0) { 451 ipsec->res[ipsec->num] = sa_idx; 452 ipsec->pkts[ipsec->num++] = m; 453 } else /* BYPASS */ 454 ip->pkts[j++] = m; 455 } 456 ip->num = j; 457 } 458 459 static inline void 460 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 461 struct ipsec_traffic *traffic) 462 { 463 struct rte_mbuf *m; 464 uint16_t idx, nb_pkts_out, i; 465 466 /* Drop any IPsec traffic from protected ports */ 467 for (i = 0; i < traffic->ipsec.num; i++) 468 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 469 470 traffic->ipsec.num = 0; 471 472 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 473 474 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 475 476 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 477 traffic->ipsec.res, traffic->ipsec.num, 478 MAX_PKT_BURST); 479 480 for (i = 0; i < nb_pkts_out; i++) { 481 m = traffic->ipsec.pkts[i]; 482 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 483 if (ip->ip_v == IPVERSION) { 484 idx = traffic->ip4.num++; 485 traffic->ip4.pkts[idx] = m; 486 } else { 487 idx = traffic->ip6.num++; 488 traffic->ip6.pkts[idx] = m; 489 } 490 } 491 } 492 493 static inline void 494 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 495 struct ipsec_traffic *traffic) 496 { 497 struct rte_mbuf *m; 498 uint32_t nb_pkts_in, i, idx; 499 500 /* Drop any IPv4 traffic from unprotected ports */ 501 for (i = 0; i < traffic->ip4.num; i++) 502 rte_pktmbuf_free(traffic->ip4.pkts[i]); 503 504 traffic->ip4.num = 0; 505 506 /* Drop any IPv6 traffic from unprotected ports */ 507 for (i = 0; i < traffic->ip6.num; i++) 508 rte_pktmbuf_free(traffic->ip6.pkts[i]); 509 510 traffic->ip6.num = 0; 511 512 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 513 traffic->ipsec.num, MAX_PKT_BURST); 514 515 for (i = 0; i < nb_pkts_in; i++) { 516 m = traffic->ipsec.pkts[i]; 517 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 518 if (ip->ip_v == IPVERSION) { 519 idx = traffic->ip4.num++; 520 traffic->ip4.pkts[idx] = m; 521 } else { 522 idx = traffic->ip6.num++; 523 traffic->ip6.pkts[idx] = m; 524 } 525 } 526 } 527 528 static inline void 529 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 530 struct ipsec_traffic *traffic) 531 { 532 struct rte_mbuf *m; 533 uint32_t nb_pkts_out, i; 534 struct ip *ip; 535 536 /* Drop any IPsec traffic from protected ports */ 537 for (i = 0; i < traffic->ipsec.num; i++) 538 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 539 540 traffic->ipsec.num = 0; 541 542 for (i = 0; i < traffic->ip4.num; i++) 543 traffic->ip4.res[i] = single_sa_idx; 544 545 for (i = 0; i < traffic->ip6.num; i++) 546 traffic->ip6.res[i] = single_sa_idx; 547 548 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ip4.pkts, 549 traffic->ip4.res, traffic->ip4.num, 550 MAX_PKT_BURST); 551 552 /* They all sue the same SA (ip4 or ip6 tunnel) */ 553 m = traffic->ipsec.pkts[i]; 554 ip = rte_pktmbuf_mtod(m, struct ip *); 555 if (ip->ip_v == IPVERSION) 556 traffic->ip4.num = nb_pkts_out; 557 else 558 traffic->ip6.num = nb_pkts_out; 559 } 560 561 static inline void 562 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 563 { 564 uint32_t hop[MAX_PKT_BURST * 2]; 565 uint32_t dst_ip[MAX_PKT_BURST * 2]; 566 uint16_t i, offset; 567 568 if (nb_pkts == 0) 569 return; 570 571 for (i = 0; i < nb_pkts; i++) { 572 offset = offsetof(struct ip, ip_dst); 573 dst_ip[i] = *rte_pktmbuf_mtod_offset(pkts[i], 574 uint32_t *, offset); 575 dst_ip[i] = rte_be_to_cpu_32(dst_ip[i]); 576 } 577 578 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, nb_pkts); 579 580 for (i = 0; i < nb_pkts; i++) { 581 if ((hop[i] & RTE_LPM_LOOKUP_SUCCESS) == 0) { 582 rte_pktmbuf_free(pkts[i]); 583 continue; 584 } 585 send_single_packet(pkts[i], hop[i] & 0xff); 586 } 587 } 588 589 static inline void 590 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 591 { 592 int32_t hop[MAX_PKT_BURST * 2]; 593 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 594 uint8_t *ip6_dst; 595 uint16_t i, offset; 596 597 if (nb_pkts == 0) 598 return; 599 600 for (i = 0; i < nb_pkts; i++) { 601 offset = offsetof(struct ip6_hdr, ip6_dst); 602 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, offset); 603 memcpy(&dst_ip[i][0], ip6_dst, 16); 604 } 605 606 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, 607 hop, nb_pkts); 608 609 for (i = 0; i < nb_pkts; i++) { 610 if (hop[i] == -1) { 611 rte_pktmbuf_free(pkts[i]); 612 continue; 613 } 614 send_single_packet(pkts[i], hop[i] & 0xff); 615 } 616 } 617 618 static inline void 619 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 620 uint8_t nb_pkts, uint16_t portid) 621 { 622 struct ipsec_traffic traffic; 623 624 prepare_traffic(pkts, &traffic, nb_pkts); 625 626 if (unlikely(single_sa)) { 627 if (UNPROTECTED_PORT(portid)) 628 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 629 else 630 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 631 } else { 632 if (UNPROTECTED_PORT(portid)) 633 process_pkts_inbound(&qconf->inbound, &traffic); 634 else 635 process_pkts_outbound(&qconf->outbound, &traffic); 636 } 637 638 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 639 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 640 } 641 642 static inline void 643 drain_buffers(struct lcore_conf *qconf) 644 { 645 struct buffer *buf; 646 uint32_t portid; 647 648 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 649 buf = &qconf->tx_mbufs[portid]; 650 if (buf->len == 0) 651 continue; 652 send_burst(qconf, buf->len, portid); 653 buf->len = 0; 654 } 655 } 656 657 /* main processing loop */ 658 static int32_t 659 main_loop(__attribute__((unused)) void *dummy) 660 { 661 struct rte_mbuf *pkts[MAX_PKT_BURST]; 662 uint32_t lcore_id; 663 uint64_t prev_tsc, diff_tsc, cur_tsc; 664 int32_t i, nb_rx; 665 uint16_t portid; 666 uint8_t queueid; 667 struct lcore_conf *qconf; 668 int32_t socket_id; 669 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 670 / US_PER_S * BURST_TX_DRAIN_US; 671 struct lcore_rx_queue *rxql; 672 673 prev_tsc = 0; 674 lcore_id = rte_lcore_id(); 675 qconf = &lcore_conf[lcore_id]; 676 rxql = qconf->rx_queue_list; 677 socket_id = rte_lcore_to_socket_id(lcore_id); 678 679 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 680 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 681 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 682 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 683 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 684 qconf->inbound.cdev_map = cdev_map_in; 685 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 686 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 687 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 688 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 689 qconf->outbound.cdev_map = cdev_map_out; 690 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 691 692 if (qconf->nb_rx_queue == 0) { 693 RTE_LOG(INFO, IPSEC, "lcore %u has nothing to do\n", lcore_id); 694 return 0; 695 } 696 697 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 698 699 for (i = 0; i < qconf->nb_rx_queue; i++) { 700 portid = rxql[i].port_id; 701 queueid = rxql[i].queue_id; 702 RTE_LOG(INFO, IPSEC, 703 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 704 lcore_id, portid, queueid); 705 } 706 707 while (1) { 708 cur_tsc = rte_rdtsc(); 709 710 /* TX queue buffer drain */ 711 diff_tsc = cur_tsc - prev_tsc; 712 713 if (unlikely(diff_tsc > drain_tsc)) { 714 drain_buffers(qconf); 715 prev_tsc = cur_tsc; 716 } 717 718 /* Read packet from RX queues */ 719 for (i = 0; i < qconf->nb_rx_queue; ++i) { 720 portid = rxql[i].port_id; 721 queueid = rxql[i].queue_id; 722 nb_rx = rte_eth_rx_burst(portid, queueid, 723 pkts, MAX_PKT_BURST); 724 725 if (nb_rx > 0) 726 process_pkts(qconf, pkts, nb_rx, portid); 727 } 728 } 729 } 730 731 static int32_t 732 check_params(void) 733 { 734 uint8_t lcore; 735 uint16_t portid, nb_ports; 736 uint16_t i; 737 int32_t socket_id; 738 739 if (lcore_params == NULL) { 740 printf("Error: No port/queue/core mappings\n"); 741 return -1; 742 } 743 744 nb_ports = rte_eth_dev_count(); 745 746 for (i = 0; i < nb_lcore_params; ++i) { 747 lcore = lcore_params[i].lcore_id; 748 if (!rte_lcore_is_enabled(lcore)) { 749 printf("error: lcore %hhu is not enabled in " 750 "lcore mask\n", lcore); 751 return -1; 752 } 753 socket_id = rte_lcore_to_socket_id(lcore); 754 if (socket_id != 0 && numa_on == 0) { 755 printf("warning: lcore %hhu is on socket %d " 756 "with numa off\n", 757 lcore, socket_id); 758 } 759 portid = lcore_params[i].port_id; 760 if ((enabled_port_mask & (1 << portid)) == 0) { 761 printf("port %u is not enabled in port mask\n", portid); 762 return -1; 763 } 764 if (portid >= nb_ports) { 765 printf("port %u is not present on the board\n", portid); 766 return -1; 767 } 768 } 769 return 0; 770 } 771 772 static uint8_t 773 get_port_nb_rx_queues(const uint16_t port) 774 { 775 int32_t queue = -1; 776 uint16_t i; 777 778 for (i = 0; i < nb_lcore_params; ++i) { 779 if (lcore_params[i].port_id == port && 780 lcore_params[i].queue_id > queue) 781 queue = lcore_params[i].queue_id; 782 } 783 return (uint8_t)(++queue); 784 } 785 786 static int32_t 787 init_lcore_rx_queues(void) 788 { 789 uint16_t i, nb_rx_queue; 790 uint8_t lcore; 791 792 for (i = 0; i < nb_lcore_params; ++i) { 793 lcore = lcore_params[i].lcore_id; 794 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 795 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 796 printf("error: too many queues (%u) for lcore: %u\n", 797 nb_rx_queue + 1, lcore); 798 return -1; 799 } 800 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 801 lcore_params[i].port_id; 802 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 803 lcore_params[i].queue_id; 804 lcore_conf[lcore].nb_rx_queue++; 805 } 806 return 0; 807 } 808 809 /* display usage */ 810 static void 811 print_usage(const char *prgname) 812 { 813 printf("%s [EAL options] -- -p PORTMASK -P -u PORTMASK" 814 " --"OPTION_CONFIG" (port,queue,lcore)[,(port,queue,lcore]" 815 " --single-sa SAIDX -f CONFIG_FILE\n" 816 " -p PORTMASK: hexadecimal bitmask of ports to configure\n" 817 " -P : enable promiscuous mode\n" 818 " -u PORTMASK: hexadecimal bitmask of unprotected ports\n" 819 " -j FRAMESIZE: jumbo frame maximum size\n" 820 " --"OPTION_CONFIG": (port,queue,lcore): " 821 "rx queues configuration\n" 822 " --single-sa SAIDX: use single SA index for outbound, " 823 "bypassing the SP\n" 824 " -f CONFIG_FILE: Configuration file path\n", 825 prgname); 826 } 827 828 static int32_t 829 parse_portmask(const char *portmask) 830 { 831 char *end = NULL; 832 unsigned long pm; 833 834 /* parse hexadecimal string */ 835 pm = strtoul(portmask, &end, 16); 836 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 837 return -1; 838 839 if ((pm == 0) && errno) 840 return -1; 841 842 return pm; 843 } 844 845 static int32_t 846 parse_decimal(const char *str) 847 { 848 char *end = NULL; 849 unsigned long num; 850 851 num = strtoul(str, &end, 10); 852 if ((str[0] == '\0') || (end == NULL) || (*end != '\0')) 853 return -1; 854 855 return num; 856 } 857 858 static int32_t 859 parse_config(const char *q_arg) 860 { 861 char s[256]; 862 const char *p, *p0 = q_arg; 863 char *end; 864 enum fieldnames { 865 FLD_PORT = 0, 866 FLD_QUEUE, 867 FLD_LCORE, 868 _NUM_FLD 869 }; 870 unsigned long int_fld[_NUM_FLD]; 871 char *str_fld[_NUM_FLD]; 872 int32_t i; 873 uint32_t size; 874 875 nb_lcore_params = 0; 876 877 while ((p = strchr(p0, '(')) != NULL) { 878 ++p; 879 p0 = strchr(p, ')'); 880 if (p0 == NULL) 881 return -1; 882 883 size = p0 - p; 884 if (size >= sizeof(s)) 885 return -1; 886 887 snprintf(s, sizeof(s), "%.*s", size, p); 888 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 889 _NUM_FLD) 890 return -1; 891 for (i = 0; i < _NUM_FLD; i++) { 892 errno = 0; 893 int_fld[i] = strtoul(str_fld[i], &end, 0); 894 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 895 return -1; 896 } 897 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 898 printf("exceeded max number of lcore params: %hu\n", 899 nb_lcore_params); 900 return -1; 901 } 902 lcore_params_array[nb_lcore_params].port_id = 903 (uint8_t)int_fld[FLD_PORT]; 904 lcore_params_array[nb_lcore_params].queue_id = 905 (uint8_t)int_fld[FLD_QUEUE]; 906 lcore_params_array[nb_lcore_params].lcore_id = 907 (uint8_t)int_fld[FLD_LCORE]; 908 ++nb_lcore_params; 909 } 910 lcore_params = lcore_params_array; 911 return 0; 912 } 913 914 #define __STRNCMP(name, opt) (!strncmp(name, opt, sizeof(opt))) 915 static int32_t 916 parse_args_long_options(struct option *lgopts, int32_t option_index) 917 { 918 int32_t ret = -1; 919 const char *optname = lgopts[option_index].name; 920 921 if (__STRNCMP(optname, OPTION_CONFIG)) { 922 ret = parse_config(optarg); 923 if (ret) 924 printf("invalid config\n"); 925 } 926 927 if (__STRNCMP(optname, OPTION_SINGLE_SA)) { 928 ret = parse_decimal(optarg); 929 if (ret != -1) { 930 single_sa = 1; 931 single_sa_idx = ret; 932 printf("Configured with single SA index %u\n", 933 single_sa_idx); 934 ret = 0; 935 } 936 } 937 938 return ret; 939 } 940 #undef __STRNCMP 941 942 static int32_t 943 parse_args(int32_t argc, char **argv) 944 { 945 int32_t opt, ret; 946 char **argvopt; 947 int32_t option_index; 948 char *prgname = argv[0]; 949 static struct option lgopts[] = { 950 {OPTION_CONFIG, 1, 0, 0}, 951 {OPTION_SINGLE_SA, 1, 0, 0}, 952 {NULL, 0, 0, 0} 953 }; 954 int32_t f_present = 0; 955 956 argvopt = argv; 957 958 while ((opt = getopt_long(argc, argvopt, "p:Pu:f:j:", 959 lgopts, &option_index)) != EOF) { 960 961 switch (opt) { 962 case 'p': 963 enabled_port_mask = parse_portmask(optarg); 964 if (enabled_port_mask == 0) { 965 printf("invalid portmask\n"); 966 print_usage(prgname); 967 return -1; 968 } 969 break; 970 case 'P': 971 printf("Promiscuous mode selected\n"); 972 promiscuous_on = 1; 973 break; 974 case 'u': 975 unprotected_port_mask = parse_portmask(optarg); 976 if (unprotected_port_mask == 0) { 977 printf("invalid unprotected portmask\n"); 978 print_usage(prgname); 979 return -1; 980 } 981 break; 982 case 'f': 983 if (f_present == 1) { 984 printf("\"-f\" option present more than " 985 "once!\n"); 986 print_usage(prgname); 987 return -1; 988 } 989 if (parse_cfg_file(optarg) < 0) { 990 printf("parsing file \"%s\" failed\n", 991 optarg); 992 print_usage(prgname); 993 return -1; 994 } 995 f_present = 1; 996 break; 997 case 'j': 998 { 999 int32_t size = parse_decimal(optarg); 1000 if (size <= 1518) { 1001 printf("Invalid jumbo frame size\n"); 1002 if (size < 0) { 1003 print_usage(prgname); 1004 return -1; 1005 } 1006 printf("Using default value 9000\n"); 1007 frame_size = 9000; 1008 } else { 1009 frame_size = size; 1010 } 1011 } 1012 printf("Enabled jumbo frames size %u\n", frame_size); 1013 break; 1014 case 0: 1015 if (parse_args_long_options(lgopts, option_index)) { 1016 print_usage(prgname); 1017 return -1; 1018 } 1019 break; 1020 default: 1021 print_usage(prgname); 1022 return -1; 1023 } 1024 } 1025 1026 if (f_present == 0) { 1027 printf("Mandatory option \"-f\" not present\n"); 1028 return -1; 1029 } 1030 1031 if (optind >= 0) 1032 argv[optind-1] = prgname; 1033 1034 ret = optind-1; 1035 optind = 1; /* reset getopt lib */ 1036 return ret; 1037 } 1038 1039 static void 1040 print_ethaddr(const char *name, const struct ether_addr *eth_addr) 1041 { 1042 char buf[ETHER_ADDR_FMT_SIZE]; 1043 ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); 1044 printf("%s%s", name, buf); 1045 } 1046 1047 /* Check the link status of all ports in up to 9s, and print them finally */ 1048 static void 1049 check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) 1050 { 1051 #define CHECK_INTERVAL 100 /* 100ms */ 1052 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1053 uint16_t portid; 1054 uint8_t count, all_ports_up, print_flag = 0; 1055 struct rte_eth_link link; 1056 1057 printf("\nChecking link status"); 1058 fflush(stdout); 1059 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1060 all_ports_up = 1; 1061 for (portid = 0; portid < port_num; portid++) { 1062 if ((port_mask & (1 << portid)) == 0) 1063 continue; 1064 memset(&link, 0, sizeof(link)); 1065 rte_eth_link_get_nowait(portid, &link); 1066 /* print link status if flag set */ 1067 if (print_flag == 1) { 1068 if (link.link_status) 1069 printf( 1070 "Port%d Link Up - speed %u Mbps -%s\n", 1071 portid, link.link_speed, 1072 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1073 ("full-duplex") : ("half-duplex\n")); 1074 else 1075 printf("Port %d Link Down\n", portid); 1076 continue; 1077 } 1078 /* clear all_ports_up flag if any link down */ 1079 if (link.link_status == ETH_LINK_DOWN) { 1080 all_ports_up = 0; 1081 break; 1082 } 1083 } 1084 /* after finally printing all link status, get out */ 1085 if (print_flag == 1) 1086 break; 1087 1088 if (all_ports_up == 0) { 1089 printf("."); 1090 fflush(stdout); 1091 rte_delay_ms(CHECK_INTERVAL); 1092 } 1093 1094 /* set the print_flag if all ports up or timeout */ 1095 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1096 print_flag = 1; 1097 printf("done\n"); 1098 } 1099 } 1100 } 1101 1102 static int32_t 1103 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1104 uint16_t qp, struct lcore_params *params, 1105 struct ipsec_ctx *ipsec_ctx, 1106 const struct rte_cryptodev_capabilities *cipher, 1107 const struct rte_cryptodev_capabilities *auth, 1108 const struct rte_cryptodev_capabilities *aead) 1109 { 1110 int32_t ret = 0; 1111 unsigned long i; 1112 struct cdev_key key = { 0 }; 1113 1114 key.lcore_id = params->lcore_id; 1115 if (cipher) 1116 key.cipher_algo = cipher->sym.cipher.algo; 1117 if (auth) 1118 key.auth_algo = auth->sym.auth.algo; 1119 if (aead) 1120 key.aead_algo = aead->sym.aead.algo; 1121 1122 ret = rte_hash_lookup(map, &key); 1123 if (ret != -ENOENT) 1124 return 0; 1125 1126 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1127 if (ipsec_ctx->tbl[i].id == cdev_id) 1128 break; 1129 1130 if (i == ipsec_ctx->nb_qps) { 1131 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1132 printf("Maximum number of crypto devices assigned to " 1133 "a core, increase MAX_QP_PER_LCORE value\n"); 1134 return 0; 1135 } 1136 ipsec_ctx->tbl[i].id = cdev_id; 1137 ipsec_ctx->tbl[i].qp = qp; 1138 ipsec_ctx->nb_qps++; 1139 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1140 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1141 cdev_id, qp, i); 1142 } 1143 1144 ret = rte_hash_add_key_data(map, &key, (void *)i); 1145 if (ret < 0) { 1146 printf("Faled to insert cdev mapping for (lcore %u, " 1147 "cdev %u, qp %u), errno %d\n", 1148 key.lcore_id, ipsec_ctx->tbl[i].id, 1149 ipsec_ctx->tbl[i].qp, ret); 1150 return 0; 1151 } 1152 1153 return 1; 1154 } 1155 1156 static int32_t 1157 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1158 uint16_t qp, struct lcore_params *params) 1159 { 1160 int32_t ret = 0; 1161 const struct rte_cryptodev_capabilities *i, *j; 1162 struct rte_hash *map; 1163 struct lcore_conf *qconf; 1164 struct ipsec_ctx *ipsec_ctx; 1165 const char *str; 1166 1167 qconf = &lcore_conf[params->lcore_id]; 1168 1169 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 1170 map = cdev_map_out; 1171 ipsec_ctx = &qconf->outbound; 1172 str = "Outbound"; 1173 } else { 1174 map = cdev_map_in; 1175 ipsec_ctx = &qconf->inbound; 1176 str = "Inbound"; 1177 } 1178 1179 /* Required cryptodevs with operation chainning */ 1180 if (!(dev_info->feature_flags & 1181 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 1182 return ret; 1183 1184 for (i = dev_info->capabilities; 1185 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 1186 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1187 continue; 1188 1189 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1190 ret |= add_mapping(map, str, cdev_id, qp, params, 1191 ipsec_ctx, NULL, NULL, i); 1192 continue; 1193 } 1194 1195 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 1196 continue; 1197 1198 for (j = dev_info->capabilities; 1199 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 1200 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1201 continue; 1202 1203 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 1204 continue; 1205 1206 ret |= add_mapping(map, str, cdev_id, qp, params, 1207 ipsec_ctx, i, j, NULL); 1208 } 1209 } 1210 1211 return ret; 1212 } 1213 1214 static int32_t 1215 cryptodevs_init(void) 1216 { 1217 struct rte_cryptodev_config dev_conf; 1218 struct rte_cryptodev_qp_conf qp_conf; 1219 uint16_t idx, max_nb_qps, qp, i; 1220 int16_t cdev_id; 1221 struct rte_hash_parameters params = { 0 }; 1222 1223 params.entries = CDEV_MAP_ENTRIES; 1224 params.key_len = sizeof(struct cdev_key); 1225 params.hash_func = rte_jhash; 1226 params.hash_func_init_val = 0; 1227 params.socket_id = rte_socket_id(); 1228 1229 params.name = "cdev_map_in"; 1230 cdev_map_in = rte_hash_create(¶ms); 1231 if (cdev_map_in == NULL) 1232 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1233 rte_errno); 1234 1235 params.name = "cdev_map_out"; 1236 cdev_map_out = rte_hash_create(¶ms); 1237 if (cdev_map_out == NULL) 1238 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1239 rte_errno); 1240 1241 printf("lcore/cryptodev/qp mappings:\n"); 1242 1243 uint32_t max_sess_sz = 0, sess_sz; 1244 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1245 sess_sz = rte_cryptodev_get_private_session_size(cdev_id); 1246 if (sess_sz > max_sess_sz) 1247 max_sess_sz = sess_sz; 1248 } 1249 1250 idx = 0; 1251 /* Start from last cdev id to give HW priority */ 1252 for (cdev_id = rte_cryptodev_count() - 1; cdev_id >= 0; cdev_id--) { 1253 struct rte_cryptodev_info cdev_info; 1254 1255 rte_cryptodev_info_get(cdev_id, &cdev_info); 1256 1257 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 1258 max_nb_qps = cdev_info.max_nb_queue_pairs; 1259 else 1260 max_nb_qps = nb_lcore_params; 1261 1262 qp = 0; 1263 i = 0; 1264 while (qp < max_nb_qps && i < nb_lcore_params) { 1265 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 1266 &lcore_params[idx])) 1267 qp++; 1268 idx++; 1269 idx = idx % nb_lcore_params; 1270 i++; 1271 } 1272 1273 if (qp == 0) 1274 continue; 1275 1276 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 1277 dev_conf.nb_queue_pairs = qp; 1278 1279 if (!socket_ctx[dev_conf.socket_id].session_pool) { 1280 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1281 struct rte_mempool *sess_mp; 1282 1283 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1284 "sess_mp_%u", dev_conf.socket_id); 1285 sess_mp = rte_mempool_create(mp_name, 1286 CDEV_MP_NB_OBJS, 1287 max_sess_sz, 1288 CDEV_MP_CACHE_SZ, 1289 0, NULL, NULL, NULL, 1290 NULL, dev_conf.socket_id, 1291 0); 1292 if (sess_mp == NULL) 1293 rte_exit(EXIT_FAILURE, 1294 "Cannot create session pool on socket %d\n", 1295 dev_conf.socket_id); 1296 else 1297 printf("Allocated session pool on socket %d\n", 1298 dev_conf.socket_id); 1299 socket_ctx[dev_conf.socket_id].session_pool = sess_mp; 1300 } 1301 1302 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 1303 rte_panic("Failed to initialize cryptodev %u\n", 1304 cdev_id); 1305 1306 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 1307 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 1308 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 1309 &qp_conf, dev_conf.socket_id, 1310 socket_ctx[dev_conf.socket_id].session_pool)) 1311 rte_panic("Failed to setup queue %u for " 1312 "cdev_id %u\n", 0, cdev_id); 1313 1314 if (rte_cryptodev_start(cdev_id)) 1315 rte_panic("Failed to start cryptodev %u\n", 1316 cdev_id); 1317 } 1318 1319 printf("\n"); 1320 1321 return 0; 1322 } 1323 1324 static void 1325 port_init(uint16_t portid) 1326 { 1327 struct rte_eth_dev_info dev_info; 1328 struct rte_eth_txconf *txconf; 1329 uint16_t nb_tx_queue, nb_rx_queue; 1330 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 1331 int32_t ret, socket_id; 1332 struct lcore_conf *qconf; 1333 struct ether_addr ethaddr; 1334 struct rte_eth_conf local_port_conf = port_conf; 1335 1336 rte_eth_dev_info_get(portid, &dev_info); 1337 1338 printf("Configuring device port %u:\n", portid); 1339 1340 rte_eth_macaddr_get(portid, ðaddr); 1341 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ethaddr); 1342 print_ethaddr("Address: ", ðaddr); 1343 printf("\n"); 1344 1345 nb_rx_queue = get_port_nb_rx_queues(portid); 1346 nb_tx_queue = nb_lcores; 1347 1348 if (nb_rx_queue > dev_info.max_rx_queues) 1349 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1350 "(max rx queue is %u)\n", 1351 nb_rx_queue, dev_info.max_rx_queues); 1352 1353 if (nb_tx_queue > dev_info.max_tx_queues) 1354 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1355 "(max tx queue is %u)\n", 1356 nb_tx_queue, dev_info.max_tx_queues); 1357 1358 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 1359 nb_rx_queue, nb_tx_queue); 1360 1361 if (frame_size) { 1362 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 1363 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1364 } 1365 1366 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_SECURITY) 1367 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SECURITY; 1368 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SECURITY) 1369 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SECURITY; 1370 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 1371 local_port_conf.txmode.offloads |= 1372 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1373 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 1374 &local_port_conf); 1375 if (ret < 0) 1376 rte_exit(EXIT_FAILURE, "Cannot configure device: " 1377 "err=%d, port=%d\n", ret, portid); 1378 1379 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 1380 if (ret < 0) 1381 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 1382 "err=%d, port=%d\n", ret, portid); 1383 1384 /* init one TX queue per lcore */ 1385 tx_queueid = 0; 1386 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1387 if (rte_lcore_is_enabled(lcore_id) == 0) 1388 continue; 1389 1390 if (numa_on) 1391 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1392 else 1393 socket_id = 0; 1394 1395 /* init TX queue */ 1396 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 1397 1398 txconf = &dev_info.default_txconf; 1399 txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE; 1400 txconf->offloads = local_port_conf.txmode.offloads; 1401 1402 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 1403 socket_id, txconf); 1404 if (ret < 0) 1405 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 1406 "err=%d, port=%d\n", ret, portid); 1407 1408 qconf = &lcore_conf[lcore_id]; 1409 qconf->tx_queue_id[portid] = tx_queueid; 1410 tx_queueid++; 1411 1412 /* init RX queues */ 1413 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 1414 struct rte_eth_rxconf rxq_conf; 1415 1416 if (portid != qconf->rx_queue_list[queue].port_id) 1417 continue; 1418 1419 rx_queueid = qconf->rx_queue_list[queue].queue_id; 1420 1421 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 1422 socket_id); 1423 1424 rxq_conf = dev_info.default_rxconf; 1425 rxq_conf.offloads = local_port_conf.rxmode.offloads; 1426 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 1427 nb_rxd, socket_id, &rxq_conf, 1428 socket_ctx[socket_id].mbuf_pool); 1429 if (ret < 0) 1430 rte_exit(EXIT_FAILURE, 1431 "rte_eth_rx_queue_setup: err=%d, " 1432 "port=%d\n", ret, portid); 1433 } 1434 } 1435 printf("\n"); 1436 } 1437 1438 static void 1439 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 1440 { 1441 char s[64]; 1442 uint32_t buff_size = frame_size ? (frame_size + RTE_PKTMBUF_HEADROOM) : 1443 RTE_MBUF_DEFAULT_BUF_SIZE; 1444 1445 1446 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 1447 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 1448 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 1449 buff_size, 1450 socket_id); 1451 if (ctx->mbuf_pool == NULL) 1452 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 1453 socket_id); 1454 else 1455 printf("Allocated mbuf pool on socket %d\n", socket_id); 1456 } 1457 1458 int32_t 1459 main(int32_t argc, char **argv) 1460 { 1461 int32_t ret; 1462 uint32_t lcore_id; 1463 uint8_t socket_id; 1464 uint16_t portid, nb_ports; 1465 1466 /* init EAL */ 1467 ret = rte_eal_init(argc, argv); 1468 if (ret < 0) 1469 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 1470 argc -= ret; 1471 argv += ret; 1472 1473 /* parse application arguments (after the EAL ones) */ 1474 ret = parse_args(argc, argv); 1475 if (ret < 0) 1476 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 1477 1478 if ((unprotected_port_mask & enabled_port_mask) != 1479 unprotected_port_mask) 1480 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 1481 unprotected_port_mask); 1482 1483 nb_ports = rte_eth_dev_count(); 1484 1485 if (check_params() < 0) 1486 rte_exit(EXIT_FAILURE, "check_params failed\n"); 1487 1488 ret = init_lcore_rx_queues(); 1489 if (ret < 0) 1490 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 1491 1492 nb_lcores = rte_lcore_count(); 1493 1494 /* Replicate each context per socket */ 1495 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1496 if (rte_lcore_is_enabled(lcore_id) == 0) 1497 continue; 1498 1499 if (numa_on) 1500 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1501 else 1502 socket_id = 0; 1503 1504 if (socket_ctx[socket_id].mbuf_pool) 1505 continue; 1506 1507 sa_init(&socket_ctx[socket_id], socket_id); 1508 1509 sp4_init(&socket_ctx[socket_id], socket_id); 1510 1511 sp6_init(&socket_ctx[socket_id], socket_id); 1512 1513 rt_init(&socket_ctx[socket_id], socket_id); 1514 1515 pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF); 1516 } 1517 1518 for (portid = 0; portid < nb_ports; portid++) { 1519 if ((enabled_port_mask & (1 << portid)) == 0) 1520 continue; 1521 1522 port_init(portid); 1523 } 1524 1525 cryptodevs_init(); 1526 1527 /* start ports */ 1528 for (portid = 0; portid < nb_ports; portid++) { 1529 if ((enabled_port_mask & (1 << portid)) == 0) 1530 continue; 1531 1532 /* Start device */ 1533 ret = rte_eth_dev_start(portid); 1534 if (ret < 0) 1535 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 1536 "err=%d, port=%d\n", ret, portid); 1537 /* 1538 * If enabled, put device in promiscuous mode. 1539 * This allows IO forwarding mode to forward packets 1540 * to itself through 2 cross-connected ports of the 1541 * target machine. 1542 */ 1543 if (promiscuous_on) 1544 rte_eth_promiscuous_enable(portid); 1545 } 1546 1547 check_all_ports_link_status(nb_ports, enabled_port_mask); 1548 1549 /* launch per-lcore init on every lcore */ 1550 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 1551 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1552 if (rte_eal_wait_lcore(lcore_id) < 0) 1553 return -1; 1554 } 1555 1556 return 0; 1557 } 1558