1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <netinet/in.h> 11 #include <netinet/ip.h> 12 #include <netinet/ip6.h> 13 #include <string.h> 14 #include <sys/queue.h> 15 #include <stdarg.h> 16 #include <errno.h> 17 #include <getopt.h> 18 19 #include <rte_common.h> 20 #include <rte_byteorder.h> 21 #include <rte_log.h> 22 #include <rte_eal.h> 23 #include <rte_launch.h> 24 #include <rte_atomic.h> 25 #include <rte_cycles.h> 26 #include <rte_prefetch.h> 27 #include <rte_lcore.h> 28 #include <rte_per_lcore.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_interrupts.h> 31 #include <rte_random.h> 32 #include <rte_debug.h> 33 #include <rte_ether.h> 34 #include <rte_ethdev.h> 35 #include <rte_mempool.h> 36 #include <rte_mbuf.h> 37 #include <rte_acl.h> 38 #include <rte_lpm.h> 39 #include <rte_lpm6.h> 40 #include <rte_hash.h> 41 #include <rte_jhash.h> 42 #include <rte_cryptodev.h> 43 #include <rte_security.h> 44 45 #include "ipsec.h" 46 #include "parser.h" 47 48 #define RTE_LOGTYPE_IPSEC RTE_LOGTYPE_USER1 49 50 #define MAX_JUMBO_PKT_LEN 9600 51 52 #define MEMPOOL_CACHE_SIZE 256 53 54 #define NB_MBUF (32000) 55 56 #define CDEV_QUEUE_DESC 2048 57 #define CDEV_MAP_ENTRIES 16384 58 #define CDEV_MP_NB_OBJS 1024 59 #define CDEV_MP_CACHE_SZ 64 60 #define MAX_QUEUE_PAIRS 1 61 62 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 63 64 #define NB_SOCKETS 4 65 66 /* Configure how many packets ahead to prefetch, when reading packets */ 67 #define PREFETCH_OFFSET 3 68 69 #define MAX_RX_QUEUE_PER_LCORE 16 70 71 #define MAX_LCORE_PARAMS 1024 72 73 #define UNPROTECTED_PORT(port) (unprotected_port_mask & (1 << portid)) 74 75 /* 76 * Configurable number of RX/TX ring descriptors 77 */ 78 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024 79 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024 80 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 81 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 82 83 #if RTE_BYTE_ORDER != RTE_LITTLE_ENDIAN 84 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 85 (((uint64_t)((a) & 0xff) << 56) | \ 86 ((uint64_t)((b) & 0xff) << 48) | \ 87 ((uint64_t)((c) & 0xff) << 40) | \ 88 ((uint64_t)((d) & 0xff) << 32) | \ 89 ((uint64_t)((e) & 0xff) << 24) | \ 90 ((uint64_t)((f) & 0xff) << 16) | \ 91 ((uint64_t)((g) & 0xff) << 8) | \ 92 ((uint64_t)(h) & 0xff)) 93 #else 94 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 95 (((uint64_t)((h) & 0xff) << 56) | \ 96 ((uint64_t)((g) & 0xff) << 48) | \ 97 ((uint64_t)((f) & 0xff) << 40) | \ 98 ((uint64_t)((e) & 0xff) << 32) | \ 99 ((uint64_t)((d) & 0xff) << 24) | \ 100 ((uint64_t)((c) & 0xff) << 16) | \ 101 ((uint64_t)((b) & 0xff) << 8) | \ 102 ((uint64_t)(a) & 0xff)) 103 #endif 104 #define ETHADDR(a, b, c, d, e, f) (__BYTES_TO_UINT64(a, b, c, d, e, f, 0, 0)) 105 106 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 107 (addr)->addr_bytes[0], (addr)->addr_bytes[1], \ 108 (addr)->addr_bytes[2], (addr)->addr_bytes[3], \ 109 (addr)->addr_bytes[4], (addr)->addr_bytes[5], \ 110 0, 0) 111 112 /* port/source ethernet addr and destination ethernet addr */ 113 struct ethaddr_info { 114 uint64_t src, dst; 115 }; 116 117 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 118 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 119 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 120 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 121 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 122 }; 123 124 #define CMD_LINE_OPT_CONFIG "config" 125 #define CMD_LINE_OPT_SINGLE_SA "single-sa" 126 #define CMD_LINE_OPT_CRYPTODEV_MASK "cryptodev_mask" 127 #define CMD_LINE_OPT_RX_OFFLOAD "rxoffload" 128 #define CMD_LINE_OPT_TX_OFFLOAD "txoffload" 129 130 enum { 131 /* long options mapped to a short option */ 132 133 /* first long only option value must be >= 256, so that we won't 134 * conflict with short options 135 */ 136 CMD_LINE_OPT_MIN_NUM = 256, 137 CMD_LINE_OPT_CONFIG_NUM, 138 CMD_LINE_OPT_SINGLE_SA_NUM, 139 CMD_LINE_OPT_CRYPTODEV_MASK_NUM, 140 CMD_LINE_OPT_RX_OFFLOAD_NUM, 141 CMD_LINE_OPT_TX_OFFLOAD_NUM, 142 }; 143 144 static const struct option lgopts[] = { 145 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 146 {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM}, 147 {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM}, 148 {CMD_LINE_OPT_RX_OFFLOAD, 1, 0, CMD_LINE_OPT_RX_OFFLOAD_NUM}, 149 {CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM}, 150 {NULL, 0, 0, 0} 151 }; 152 153 /* mask of enabled ports */ 154 static uint32_t enabled_port_mask; 155 static uint64_t enabled_cryptodev_mask = UINT64_MAX; 156 static uint32_t unprotected_port_mask; 157 static int32_t promiscuous_on = 1; 158 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 159 static uint32_t nb_lcores; 160 static uint32_t single_sa; 161 static uint32_t single_sa_idx; 162 static uint32_t frame_size; 163 164 /* 165 * RX/TX HW offload capabilities to enable/use on ethernet ports. 166 * By default all capabilities are enabled. 167 */ 168 static uint64_t dev_rx_offload = UINT64_MAX; 169 static uint64_t dev_tx_offload = UINT64_MAX; 170 171 /* application wide librte_ipsec/SA parameters */ 172 struct app_sa_prm app_sa_prm = {.enable = 0}; 173 174 struct lcore_rx_queue { 175 uint16_t port_id; 176 uint8_t queue_id; 177 } __rte_cache_aligned; 178 179 struct lcore_params { 180 uint16_t port_id; 181 uint8_t queue_id; 182 uint8_t lcore_id; 183 } __rte_cache_aligned; 184 185 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 186 187 static struct lcore_params *lcore_params; 188 static uint16_t nb_lcore_params; 189 190 static struct rte_hash *cdev_map_in; 191 static struct rte_hash *cdev_map_out; 192 193 struct buffer { 194 uint16_t len; 195 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 196 }; 197 198 struct lcore_conf { 199 uint16_t nb_rx_queue; 200 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 201 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 202 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 203 struct ipsec_ctx inbound; 204 struct ipsec_ctx outbound; 205 struct rt_ctx *rt4_ctx; 206 struct rt_ctx *rt6_ctx; 207 } __rte_cache_aligned; 208 209 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 210 211 static struct rte_eth_conf port_conf = { 212 .rxmode = { 213 .mq_mode = ETH_MQ_RX_RSS, 214 .max_rx_pkt_len = ETHER_MAX_LEN, 215 .split_hdr_size = 0, 216 .offloads = DEV_RX_OFFLOAD_CHECKSUM, 217 }, 218 .rx_adv_conf = { 219 .rss_conf = { 220 .rss_key = NULL, 221 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 222 ETH_RSS_TCP | ETH_RSS_SCTP, 223 }, 224 }, 225 .txmode = { 226 .mq_mode = ETH_MQ_TX_NONE, 227 }, 228 }; 229 230 static struct socket_ctx socket_ctx[NB_SOCKETS]; 231 232 static inline void 233 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 234 { 235 uint8_t *nlp; 236 struct ether_hdr *eth; 237 238 eth = rte_pktmbuf_mtod(pkt, struct ether_hdr *); 239 if (eth->ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) { 240 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, ETHER_HDR_LEN); 241 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip, ip_p)); 242 if (*nlp == IPPROTO_ESP) 243 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 244 else { 245 t->ip4.data[t->ip4.num] = nlp; 246 t->ip4.pkts[(t->ip4.num)++] = pkt; 247 } 248 pkt->l2_len = 0; 249 pkt->l3_len = sizeof(struct ip); 250 } else if (eth->ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) { 251 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, ETHER_HDR_LEN); 252 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip6_hdr, ip6_nxt)); 253 if (*nlp == IPPROTO_ESP) 254 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 255 else { 256 t->ip6.data[t->ip6.num] = nlp; 257 t->ip6.pkts[(t->ip6.num)++] = pkt; 258 } 259 pkt->l2_len = 0; 260 pkt->l3_len = sizeof(struct ip6_hdr); 261 } else { 262 /* Unknown/Unsupported type, drop the packet */ 263 RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n", 264 rte_be_to_cpu_16(eth->ether_type)); 265 rte_pktmbuf_free(pkt); 266 } 267 268 /* Check if the packet has been processed inline. For inline protocol 269 * processed packets, the metadata in the mbuf can be used to identify 270 * the security processing done on the packet. The metadata will be 271 * used to retrieve the application registered userdata associated 272 * with the security session. 273 */ 274 275 if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) { 276 struct ipsec_sa *sa; 277 struct ipsec_mbuf_metadata *priv; 278 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 279 rte_eth_dev_get_sec_ctx( 280 pkt->port); 281 282 /* Retrieve the userdata registered. Here, the userdata 283 * registered is the SA pointer. 284 */ 285 286 sa = (struct ipsec_sa *) 287 rte_security_get_userdata(ctx, pkt->udata64); 288 289 if (sa == NULL) { 290 /* userdata could not be retrieved */ 291 return; 292 } 293 294 /* Save SA as priv member in mbuf. This will be used in the 295 * IPsec selector(SP-SA) check. 296 */ 297 298 priv = get_priv(pkt); 299 priv->sa = sa; 300 } 301 } 302 303 static inline void 304 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 305 uint16_t nb_pkts) 306 { 307 int32_t i; 308 309 t->ipsec.num = 0; 310 t->ip4.num = 0; 311 t->ip6.num = 0; 312 313 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 314 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 315 void *)); 316 prepare_one_packet(pkts[i], t); 317 } 318 /* Process left packets */ 319 for (; i < nb_pkts; i++) 320 prepare_one_packet(pkts[i], t); 321 } 322 323 static inline void 324 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port, 325 const struct lcore_conf *qconf) 326 { 327 struct ip *ip; 328 struct ether_hdr *ethhdr; 329 330 ip = rte_pktmbuf_mtod(pkt, struct ip *); 331 332 ethhdr = (struct ether_hdr *)rte_pktmbuf_prepend(pkt, ETHER_HDR_LEN); 333 334 if (ip->ip_v == IPVERSION) { 335 pkt->ol_flags |= qconf->outbound.ipv4_offloads; 336 pkt->l3_len = sizeof(struct ip); 337 pkt->l2_len = ETHER_HDR_LEN; 338 339 ip->ip_sum = 0; 340 341 /* calculate IPv4 cksum in SW */ 342 if ((pkt->ol_flags & PKT_TX_IP_CKSUM) == 0) 343 ip->ip_sum = rte_ipv4_cksum((struct ipv4_hdr *)ip); 344 345 ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); 346 } else { 347 pkt->ol_flags |= qconf->outbound.ipv6_offloads; 348 pkt->l3_len = sizeof(struct ip6_hdr); 349 pkt->l2_len = ETHER_HDR_LEN; 350 351 ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6); 352 } 353 354 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 355 sizeof(struct ether_addr)); 356 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 357 sizeof(struct ether_addr)); 358 } 359 360 static inline void 361 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port, 362 const struct lcore_conf *qconf) 363 { 364 int32_t i; 365 const int32_t prefetch_offset = 2; 366 367 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 368 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 369 prepare_tx_pkt(pkts[i], port, qconf); 370 } 371 /* Process left packets */ 372 for (; i < nb_pkts; i++) 373 prepare_tx_pkt(pkts[i], port, qconf); 374 } 375 376 /* Send burst of packets on an output interface */ 377 static inline int32_t 378 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 379 { 380 struct rte_mbuf **m_table; 381 int32_t ret; 382 uint16_t queueid; 383 384 queueid = qconf->tx_queue_id[port]; 385 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 386 387 prepare_tx_burst(m_table, n, port, qconf); 388 389 ret = rte_eth_tx_burst(port, queueid, m_table, n); 390 if (unlikely(ret < n)) { 391 do { 392 rte_pktmbuf_free(m_table[ret]); 393 } while (++ret < n); 394 } 395 396 return 0; 397 } 398 399 /* Enqueue a single packet, and send burst if queue is filled */ 400 static inline int32_t 401 send_single_packet(struct rte_mbuf *m, uint16_t port) 402 { 403 uint32_t lcore_id; 404 uint16_t len; 405 struct lcore_conf *qconf; 406 407 lcore_id = rte_lcore_id(); 408 409 qconf = &lcore_conf[lcore_id]; 410 len = qconf->tx_mbufs[port].len; 411 qconf->tx_mbufs[port].m_table[len] = m; 412 len++; 413 414 /* enough pkts to be sent */ 415 if (unlikely(len == MAX_PKT_BURST)) { 416 send_burst(qconf, MAX_PKT_BURST, port); 417 len = 0; 418 } 419 420 qconf->tx_mbufs[port].len = len; 421 return 0; 422 } 423 424 static inline void 425 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 426 uint16_t lim) 427 { 428 struct rte_mbuf *m; 429 uint32_t i, j, res, sa_idx; 430 431 if (ip->num == 0 || sp == NULL) 432 return; 433 434 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 435 ip->num, DEFAULT_MAX_CATEGORIES); 436 437 j = 0; 438 for (i = 0; i < ip->num; i++) { 439 m = ip->pkts[i]; 440 res = ip->res[i]; 441 if (res == BYPASS) { 442 ip->pkts[j++] = m; 443 continue; 444 } 445 if (res == DISCARD) { 446 rte_pktmbuf_free(m); 447 continue; 448 } 449 450 /* Only check SPI match for processed IPSec packets */ 451 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) { 452 rte_pktmbuf_free(m); 453 continue; 454 } 455 456 sa_idx = SPI2IDX(res); 457 if (!inbound_sa_check(sa, m, sa_idx)) { 458 rte_pktmbuf_free(m); 459 continue; 460 } 461 ip->pkts[j++] = m; 462 } 463 ip->num = j; 464 } 465 466 static void 467 split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) 468 { 469 uint32_t i, n4, n6; 470 struct ip *ip; 471 struct rte_mbuf *m; 472 473 n4 = trf->ip4.num; 474 n6 = trf->ip6.num; 475 476 for (i = 0; i < num; i++) { 477 478 m = mb[i]; 479 ip = rte_pktmbuf_mtod(m, struct ip *); 480 481 if (ip->ip_v == IPVERSION) { 482 trf->ip4.pkts[n4] = m; 483 trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m, 484 uint8_t *, offsetof(struct ip, ip_p)); 485 n4++; 486 } else if (ip->ip_v == IP6_VERSION) { 487 trf->ip6.pkts[n6] = m; 488 trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m, 489 uint8_t *, 490 offsetof(struct ip6_hdr, ip6_nxt)); 491 n6++; 492 } else 493 rte_pktmbuf_free(m); 494 } 495 496 trf->ip4.num = n4; 497 trf->ip6.num = n6; 498 } 499 500 501 static inline void 502 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 503 struct ipsec_traffic *traffic) 504 { 505 uint16_t nb_pkts_in, n_ip4, n_ip6; 506 507 n_ip4 = traffic->ip4.num; 508 n_ip6 = traffic->ip6.num; 509 510 if (app_sa_prm.enable == 0) { 511 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 512 traffic->ipsec.num, MAX_PKT_BURST); 513 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in); 514 } else { 515 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 516 traffic->ipsec.saptr, traffic->ipsec.num); 517 ipsec_process(ipsec_ctx, traffic); 518 } 519 520 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 521 n_ip4); 522 523 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 524 n_ip6); 525 } 526 527 static inline void 528 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 529 struct traffic_type *ipsec) 530 { 531 struct rte_mbuf *m; 532 uint32_t i, j, sa_idx; 533 534 if (ip->num == 0 || sp == NULL) 535 return; 536 537 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 538 ip->num, DEFAULT_MAX_CATEGORIES); 539 540 j = 0; 541 for (i = 0; i < ip->num; i++) { 542 m = ip->pkts[i]; 543 sa_idx = SPI2IDX(ip->res[i]); 544 if (ip->res[i] == DISCARD) 545 rte_pktmbuf_free(m); 546 else if (ip->res[i] == BYPASS) 547 ip->pkts[j++] = m; 548 else { 549 ipsec->res[ipsec->num] = sa_idx; 550 ipsec->pkts[ipsec->num++] = m; 551 } 552 } 553 ip->num = j; 554 } 555 556 static inline void 557 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 558 struct ipsec_traffic *traffic) 559 { 560 struct rte_mbuf *m; 561 uint16_t idx, nb_pkts_out, i; 562 563 /* Drop any IPsec traffic from protected ports */ 564 for (i = 0; i < traffic->ipsec.num; i++) 565 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 566 567 traffic->ipsec.num = 0; 568 569 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 570 571 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 572 573 if (app_sa_prm.enable == 0) { 574 575 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 576 traffic->ipsec.res, traffic->ipsec.num, 577 MAX_PKT_BURST); 578 579 for (i = 0; i < nb_pkts_out; i++) { 580 m = traffic->ipsec.pkts[i]; 581 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 582 if (ip->ip_v == IPVERSION) { 583 idx = traffic->ip4.num++; 584 traffic->ip4.pkts[idx] = m; 585 } else { 586 idx = traffic->ip6.num++; 587 traffic->ip6.pkts[idx] = m; 588 } 589 } 590 } else { 591 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 592 traffic->ipsec.saptr, traffic->ipsec.num); 593 ipsec_process(ipsec_ctx, traffic); 594 } 595 } 596 597 static inline void 598 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 599 struct ipsec_traffic *traffic) 600 { 601 struct rte_mbuf *m; 602 uint32_t nb_pkts_in, i, idx; 603 604 /* Drop any IPv4 traffic from unprotected ports */ 605 for (i = 0; i < traffic->ip4.num; i++) 606 rte_pktmbuf_free(traffic->ip4.pkts[i]); 607 608 traffic->ip4.num = 0; 609 610 /* Drop any IPv6 traffic from unprotected ports */ 611 for (i = 0; i < traffic->ip6.num; i++) 612 rte_pktmbuf_free(traffic->ip6.pkts[i]); 613 614 traffic->ip6.num = 0; 615 616 if (app_sa_prm.enable == 0) { 617 618 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 619 traffic->ipsec.num, MAX_PKT_BURST); 620 621 for (i = 0; i < nb_pkts_in; i++) { 622 m = traffic->ipsec.pkts[i]; 623 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 624 if (ip->ip_v == IPVERSION) { 625 idx = traffic->ip4.num++; 626 traffic->ip4.pkts[idx] = m; 627 } else { 628 idx = traffic->ip6.num++; 629 traffic->ip6.pkts[idx] = m; 630 } 631 } 632 } else { 633 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 634 traffic->ipsec.saptr, traffic->ipsec.num); 635 ipsec_process(ipsec_ctx, traffic); 636 } 637 } 638 639 static inline void 640 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 641 struct ipsec_traffic *traffic) 642 { 643 struct rte_mbuf *m; 644 uint32_t nb_pkts_out, i, n; 645 struct ip *ip; 646 647 /* Drop any IPsec traffic from protected ports */ 648 for (i = 0; i < traffic->ipsec.num; i++) 649 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 650 651 n = 0; 652 653 for (i = 0; i < traffic->ip4.num; i++) { 654 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i]; 655 traffic->ipsec.res[n++] = single_sa_idx; 656 } 657 658 for (i = 0; i < traffic->ip6.num; i++) { 659 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i]; 660 traffic->ipsec.res[n++] = single_sa_idx; 661 } 662 663 traffic->ip4.num = 0; 664 traffic->ip6.num = 0; 665 traffic->ipsec.num = n; 666 667 if (app_sa_prm.enable == 0) { 668 669 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 670 traffic->ipsec.res, traffic->ipsec.num, 671 MAX_PKT_BURST); 672 673 /* They all sue the same SA (ip4 or ip6 tunnel) */ 674 m = traffic->ipsec.pkts[0]; 675 ip = rte_pktmbuf_mtod(m, struct ip *); 676 if (ip->ip_v == IPVERSION) { 677 traffic->ip4.num = nb_pkts_out; 678 for (i = 0; i < nb_pkts_out; i++) 679 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i]; 680 } else { 681 traffic->ip6.num = nb_pkts_out; 682 for (i = 0; i < nb_pkts_out; i++) 683 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i]; 684 } 685 } else { 686 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 687 traffic->ipsec.saptr, traffic->ipsec.num); 688 ipsec_process(ipsec_ctx, traffic); 689 } 690 } 691 692 static inline int32_t 693 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) 694 { 695 struct ipsec_mbuf_metadata *priv; 696 struct ipsec_sa *sa; 697 698 priv = get_priv(pkt); 699 700 sa = priv->sa; 701 if (unlikely(sa == NULL)) { 702 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); 703 goto fail; 704 } 705 706 if (is_ipv6) 707 return sa->portid; 708 709 /* else */ 710 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); 711 712 fail: 713 if (is_ipv6) 714 return -1; 715 716 /* else */ 717 return 0; 718 } 719 720 static inline void 721 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 722 { 723 uint32_t hop[MAX_PKT_BURST * 2]; 724 uint32_t dst_ip[MAX_PKT_BURST * 2]; 725 int32_t pkt_hop = 0; 726 uint16_t i, offset; 727 uint16_t lpm_pkts = 0; 728 729 if (nb_pkts == 0) 730 return; 731 732 /* Need to do an LPM lookup for non-inline packets. Inline packets will 733 * have port ID in the SA 734 */ 735 736 for (i = 0; i < nb_pkts; i++) { 737 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 738 /* Security offload not enabled. So an LPM lookup is 739 * required to get the hop 740 */ 741 offset = offsetof(struct ip, ip_dst); 742 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], 743 uint32_t *, offset); 744 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); 745 lpm_pkts++; 746 } 747 } 748 749 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); 750 751 lpm_pkts = 0; 752 753 for (i = 0; i < nb_pkts; i++) { 754 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 755 /* Read hop from the SA */ 756 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); 757 } else { 758 /* Need to use hop returned by lookup */ 759 pkt_hop = hop[lpm_pkts++]; 760 } 761 762 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { 763 rte_pktmbuf_free(pkts[i]); 764 continue; 765 } 766 send_single_packet(pkts[i], pkt_hop & 0xff); 767 } 768 } 769 770 static inline void 771 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 772 { 773 int32_t hop[MAX_PKT_BURST * 2]; 774 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 775 uint8_t *ip6_dst; 776 int32_t pkt_hop = 0; 777 uint16_t i, offset; 778 uint16_t lpm_pkts = 0; 779 780 if (nb_pkts == 0) 781 return; 782 783 /* Need to do an LPM lookup for non-inline packets. Inline packets will 784 * have port ID in the SA 785 */ 786 787 for (i = 0; i < nb_pkts; i++) { 788 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 789 /* Security offload not enabled. So an LPM lookup is 790 * required to get the hop 791 */ 792 offset = offsetof(struct ip6_hdr, ip6_dst); 793 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, 794 offset); 795 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); 796 lpm_pkts++; 797 } 798 } 799 800 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, 801 lpm_pkts); 802 803 lpm_pkts = 0; 804 805 for (i = 0; i < nb_pkts; i++) { 806 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 807 /* Read hop from the SA */ 808 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); 809 } else { 810 /* Need to use hop returned by lookup */ 811 pkt_hop = hop[lpm_pkts++]; 812 } 813 814 if (pkt_hop == -1) { 815 rte_pktmbuf_free(pkts[i]); 816 continue; 817 } 818 send_single_packet(pkts[i], pkt_hop & 0xff); 819 } 820 } 821 822 static inline void 823 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 824 uint8_t nb_pkts, uint16_t portid) 825 { 826 struct ipsec_traffic traffic; 827 828 prepare_traffic(pkts, &traffic, nb_pkts); 829 830 if (unlikely(single_sa)) { 831 if (UNPROTECTED_PORT(portid)) 832 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 833 else 834 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 835 } else { 836 if (UNPROTECTED_PORT(portid)) 837 process_pkts_inbound(&qconf->inbound, &traffic); 838 else 839 process_pkts_outbound(&qconf->outbound, &traffic); 840 } 841 842 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 843 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 844 } 845 846 static inline void 847 drain_tx_buffers(struct lcore_conf *qconf) 848 { 849 struct buffer *buf; 850 uint32_t portid; 851 852 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 853 buf = &qconf->tx_mbufs[portid]; 854 if (buf->len == 0) 855 continue; 856 send_burst(qconf, buf->len, portid); 857 buf->len = 0; 858 } 859 } 860 861 static inline void 862 drain_crypto_buffers(struct lcore_conf *qconf) 863 { 864 uint32_t i; 865 struct ipsec_ctx *ctx; 866 867 /* drain inbound buffers*/ 868 ctx = &qconf->inbound; 869 for (i = 0; i != ctx->nb_qps; i++) { 870 if (ctx->tbl[i].len != 0) 871 enqueue_cop_burst(ctx->tbl + i); 872 } 873 874 /* drain outbound buffers*/ 875 ctx = &qconf->outbound; 876 for (i = 0; i != ctx->nb_qps; i++) { 877 if (ctx->tbl[i].len != 0) 878 enqueue_cop_burst(ctx->tbl + i); 879 } 880 } 881 882 static void 883 drain_inbound_crypto_queues(const struct lcore_conf *qconf, 884 struct ipsec_ctx *ctx) 885 { 886 uint32_t n; 887 struct ipsec_traffic trf; 888 889 if (app_sa_prm.enable == 0) { 890 891 /* dequeue packets from crypto-queue */ 892 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts, 893 RTE_DIM(trf.ipsec.pkts)); 894 895 trf.ip4.num = 0; 896 trf.ip6.num = 0; 897 898 /* split traffic by ipv4-ipv6 */ 899 split46_traffic(&trf, trf.ipsec.pkts, n); 900 } else 901 ipsec_cqp_process(ctx, &trf); 902 903 /* process ipv4 packets */ 904 if (trf.ip4.num != 0) { 905 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0); 906 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 907 } 908 909 /* process ipv6 packets */ 910 if (trf.ip6.num != 0) { 911 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0); 912 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 913 } 914 } 915 916 static void 917 drain_outbound_crypto_queues(const struct lcore_conf *qconf, 918 struct ipsec_ctx *ctx) 919 { 920 uint32_t n; 921 struct ipsec_traffic trf; 922 923 if (app_sa_prm.enable == 0) { 924 925 /* dequeue packets from crypto-queue */ 926 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts, 927 RTE_DIM(trf.ipsec.pkts)); 928 929 trf.ip4.num = 0; 930 trf.ip6.num = 0; 931 932 /* split traffic by ipv4-ipv6 */ 933 split46_traffic(&trf, trf.ipsec.pkts, n); 934 } else 935 ipsec_cqp_process(ctx, &trf); 936 937 /* process ipv4 packets */ 938 if (trf.ip4.num != 0) 939 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 940 941 /* process ipv6 packets */ 942 if (trf.ip6.num != 0) 943 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 944 } 945 946 /* main processing loop */ 947 static int32_t 948 main_loop(__attribute__((unused)) void *dummy) 949 { 950 struct rte_mbuf *pkts[MAX_PKT_BURST]; 951 uint32_t lcore_id; 952 uint64_t prev_tsc, diff_tsc, cur_tsc; 953 int32_t i, nb_rx; 954 uint16_t portid; 955 uint8_t queueid; 956 struct lcore_conf *qconf; 957 int32_t socket_id; 958 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 959 / US_PER_S * BURST_TX_DRAIN_US; 960 struct lcore_rx_queue *rxql; 961 962 prev_tsc = 0; 963 lcore_id = rte_lcore_id(); 964 qconf = &lcore_conf[lcore_id]; 965 rxql = qconf->rx_queue_list; 966 socket_id = rte_lcore_to_socket_id(lcore_id); 967 968 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 969 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 970 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 971 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 972 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 973 qconf->inbound.cdev_map = cdev_map_in; 974 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 975 qconf->inbound.session_priv_pool = 976 socket_ctx[socket_id].session_priv_pool; 977 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 978 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 979 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 980 qconf->outbound.cdev_map = cdev_map_out; 981 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 982 qconf->outbound.session_priv_pool = 983 socket_ctx[socket_id].session_priv_pool; 984 985 if (qconf->nb_rx_queue == 0) { 986 RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n", 987 lcore_id); 988 return 0; 989 } 990 991 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 992 993 for (i = 0; i < qconf->nb_rx_queue; i++) { 994 portid = rxql[i].port_id; 995 queueid = rxql[i].queue_id; 996 RTE_LOG(INFO, IPSEC, 997 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 998 lcore_id, portid, queueid); 999 } 1000 1001 while (1) { 1002 cur_tsc = rte_rdtsc(); 1003 1004 /* TX queue buffer drain */ 1005 diff_tsc = cur_tsc - prev_tsc; 1006 1007 if (unlikely(diff_tsc > drain_tsc)) { 1008 drain_tx_buffers(qconf); 1009 drain_crypto_buffers(qconf); 1010 prev_tsc = cur_tsc; 1011 } 1012 1013 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1014 1015 /* Read packets from RX queues */ 1016 portid = rxql[i].port_id; 1017 queueid = rxql[i].queue_id; 1018 nb_rx = rte_eth_rx_burst(portid, queueid, 1019 pkts, MAX_PKT_BURST); 1020 1021 if (nb_rx > 0) 1022 process_pkts(qconf, pkts, nb_rx, portid); 1023 1024 /* dequeue and process completed crypto-ops */ 1025 if (UNPROTECTED_PORT(portid)) 1026 drain_inbound_crypto_queues(qconf, 1027 &qconf->inbound); 1028 else 1029 drain_outbound_crypto_queues(qconf, 1030 &qconf->outbound); 1031 } 1032 } 1033 } 1034 1035 static int32_t 1036 check_params(void) 1037 { 1038 uint8_t lcore; 1039 uint16_t portid; 1040 uint16_t i; 1041 int32_t socket_id; 1042 1043 if (lcore_params == NULL) { 1044 printf("Error: No port/queue/core mappings\n"); 1045 return -1; 1046 } 1047 1048 for (i = 0; i < nb_lcore_params; ++i) { 1049 lcore = lcore_params[i].lcore_id; 1050 if (!rte_lcore_is_enabled(lcore)) { 1051 printf("error: lcore %hhu is not enabled in " 1052 "lcore mask\n", lcore); 1053 return -1; 1054 } 1055 socket_id = rte_lcore_to_socket_id(lcore); 1056 if (socket_id != 0 && numa_on == 0) { 1057 printf("warning: lcore %hhu is on socket %d " 1058 "with numa off\n", 1059 lcore, socket_id); 1060 } 1061 portid = lcore_params[i].port_id; 1062 if ((enabled_port_mask & (1 << portid)) == 0) { 1063 printf("port %u is not enabled in port mask\n", portid); 1064 return -1; 1065 } 1066 if (!rte_eth_dev_is_valid_port(portid)) { 1067 printf("port %u is not present on the board\n", portid); 1068 return -1; 1069 } 1070 } 1071 return 0; 1072 } 1073 1074 static uint8_t 1075 get_port_nb_rx_queues(const uint16_t port) 1076 { 1077 int32_t queue = -1; 1078 uint16_t i; 1079 1080 for (i = 0; i < nb_lcore_params; ++i) { 1081 if (lcore_params[i].port_id == port && 1082 lcore_params[i].queue_id > queue) 1083 queue = lcore_params[i].queue_id; 1084 } 1085 return (uint8_t)(++queue); 1086 } 1087 1088 static int32_t 1089 init_lcore_rx_queues(void) 1090 { 1091 uint16_t i, nb_rx_queue; 1092 uint8_t lcore; 1093 1094 for (i = 0; i < nb_lcore_params; ++i) { 1095 lcore = lcore_params[i].lcore_id; 1096 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 1097 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1098 printf("error: too many queues (%u) for lcore: %u\n", 1099 nb_rx_queue + 1, lcore); 1100 return -1; 1101 } 1102 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1103 lcore_params[i].port_id; 1104 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1105 lcore_params[i].queue_id; 1106 lcore_conf[lcore].nb_rx_queue++; 1107 } 1108 return 0; 1109 } 1110 1111 /* display usage */ 1112 static void 1113 print_usage(const char *prgname) 1114 { 1115 fprintf(stderr, "%s [EAL options] --" 1116 " -p PORTMASK" 1117 " [-P]" 1118 " [-u PORTMASK]" 1119 " [-j FRAMESIZE]" 1120 " [-l]" 1121 " [-w REPLAY_WINDOW_SIZE]" 1122 " [-e]" 1123 " [-a]" 1124 " -f CONFIG_FILE" 1125 " --config (port,queue,lcore)[,(port,queue,lcore)]" 1126 " [--single-sa SAIDX]" 1127 " [--cryptodev_mask MASK]" 1128 " [--" CMD_LINE_OPT_RX_OFFLOAD " RX_OFFLOAD_MASK]" 1129 " [--" CMD_LINE_OPT_TX_OFFLOAD " TX_OFFLOAD_MASK]" 1130 "\n\n" 1131 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 1132 " -P : Enable promiscuous mode\n" 1133 " -u PORTMASK: Hexadecimal bitmask of unprotected ports\n" 1134 " -j FRAMESIZE: Enable jumbo frame with 'FRAMESIZE' as maximum\n" 1135 " packet size\n" 1136 " -l enables code-path that uses librte_ipsec\n" 1137 " -w REPLAY_WINDOW_SIZE specifies IPsec SQN replay window\n" 1138 " size for each SA\n" 1139 " -e enables ESN\n" 1140 " -a enables SA SQN atomic behaviour\n" 1141 " -f CONFIG_FILE: Configuration file\n" 1142 " --config (port,queue,lcore): Rx queue configuration\n" 1143 " --single-sa SAIDX: Use single SA index for outbound traffic,\n" 1144 " bypassing the SP\n" 1145 " --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n" 1146 " devices to configure\n" 1147 " --" CMD_LINE_OPT_RX_OFFLOAD 1148 ": bitmask of the RX HW offload capabilities to enable/use\n" 1149 " (DEV_RX_OFFLOAD_*)\n" 1150 " --" CMD_LINE_OPT_TX_OFFLOAD 1151 ": bitmask of the TX HW offload capabilities to enable/use\n" 1152 " (DEV_TX_OFFLOAD_*)\n" 1153 "\n", 1154 prgname); 1155 } 1156 1157 static int 1158 parse_mask(const char *str, uint64_t *val) 1159 { 1160 char *end; 1161 unsigned long t; 1162 1163 errno = 0; 1164 t = strtoul(str, &end, 0); 1165 if (errno != 0 || end[0] != 0) 1166 return -EINVAL; 1167 1168 *val = t; 1169 return 0; 1170 } 1171 1172 static int32_t 1173 parse_portmask(const char *portmask) 1174 { 1175 char *end = NULL; 1176 unsigned long pm; 1177 1178 /* parse hexadecimal string */ 1179 pm = strtoul(portmask, &end, 16); 1180 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1181 return -1; 1182 1183 if ((pm == 0) && errno) 1184 return -1; 1185 1186 return pm; 1187 } 1188 1189 static int32_t 1190 parse_decimal(const char *str) 1191 { 1192 char *end = NULL; 1193 unsigned long num; 1194 1195 num = strtoul(str, &end, 10); 1196 if ((str[0] == '\0') || (end == NULL) || (*end != '\0')) 1197 return -1; 1198 1199 return num; 1200 } 1201 1202 static int32_t 1203 parse_config(const char *q_arg) 1204 { 1205 char s[256]; 1206 const char *p, *p0 = q_arg; 1207 char *end; 1208 enum fieldnames { 1209 FLD_PORT = 0, 1210 FLD_QUEUE, 1211 FLD_LCORE, 1212 _NUM_FLD 1213 }; 1214 unsigned long int_fld[_NUM_FLD]; 1215 char *str_fld[_NUM_FLD]; 1216 int32_t i; 1217 uint32_t size; 1218 1219 nb_lcore_params = 0; 1220 1221 while ((p = strchr(p0, '(')) != NULL) { 1222 ++p; 1223 p0 = strchr(p, ')'); 1224 if (p0 == NULL) 1225 return -1; 1226 1227 size = p0 - p; 1228 if (size >= sizeof(s)) 1229 return -1; 1230 1231 snprintf(s, sizeof(s), "%.*s", size, p); 1232 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1233 _NUM_FLD) 1234 return -1; 1235 for (i = 0; i < _NUM_FLD; i++) { 1236 errno = 0; 1237 int_fld[i] = strtoul(str_fld[i], &end, 0); 1238 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1239 return -1; 1240 } 1241 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1242 printf("exceeded max number of lcore params: %hu\n", 1243 nb_lcore_params); 1244 return -1; 1245 } 1246 lcore_params_array[nb_lcore_params].port_id = 1247 (uint8_t)int_fld[FLD_PORT]; 1248 lcore_params_array[nb_lcore_params].queue_id = 1249 (uint8_t)int_fld[FLD_QUEUE]; 1250 lcore_params_array[nb_lcore_params].lcore_id = 1251 (uint8_t)int_fld[FLD_LCORE]; 1252 ++nb_lcore_params; 1253 } 1254 lcore_params = lcore_params_array; 1255 return 0; 1256 } 1257 1258 static void 1259 print_app_sa_prm(const struct app_sa_prm *prm) 1260 { 1261 printf("librte_ipsec usage: %s\n", 1262 (prm->enable == 0) ? "disabled" : "enabled"); 1263 1264 if (prm->enable == 0) 1265 return; 1266 1267 printf("replay window size: %u\n", prm->window_size); 1268 printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled"); 1269 printf("SA flags: %#" PRIx64 "\n", prm->flags); 1270 } 1271 1272 static int32_t 1273 parse_args(int32_t argc, char **argv) 1274 { 1275 int32_t opt, ret; 1276 char **argvopt; 1277 int32_t option_index; 1278 char *prgname = argv[0]; 1279 int32_t f_present = 0; 1280 1281 argvopt = argv; 1282 1283 while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:", 1284 lgopts, &option_index)) != EOF) { 1285 1286 switch (opt) { 1287 case 'p': 1288 enabled_port_mask = parse_portmask(optarg); 1289 if (enabled_port_mask == 0) { 1290 printf("invalid portmask\n"); 1291 print_usage(prgname); 1292 return -1; 1293 } 1294 break; 1295 case 'P': 1296 printf("Promiscuous mode selected\n"); 1297 promiscuous_on = 1; 1298 break; 1299 case 'u': 1300 unprotected_port_mask = parse_portmask(optarg); 1301 if (unprotected_port_mask == 0) { 1302 printf("invalid unprotected portmask\n"); 1303 print_usage(prgname); 1304 return -1; 1305 } 1306 break; 1307 case 'f': 1308 if (f_present == 1) { 1309 printf("\"-f\" option present more than " 1310 "once!\n"); 1311 print_usage(prgname); 1312 return -1; 1313 } 1314 if (parse_cfg_file(optarg) < 0) { 1315 printf("parsing file \"%s\" failed\n", 1316 optarg); 1317 print_usage(prgname); 1318 return -1; 1319 } 1320 f_present = 1; 1321 break; 1322 case 'j': 1323 { 1324 int32_t size = parse_decimal(optarg); 1325 if (size <= 1518) { 1326 printf("Invalid jumbo frame size\n"); 1327 if (size < 0) { 1328 print_usage(prgname); 1329 return -1; 1330 } 1331 printf("Using default value 9000\n"); 1332 frame_size = 9000; 1333 } else { 1334 frame_size = size; 1335 } 1336 } 1337 printf("Enabled jumbo frames size %u\n", frame_size); 1338 break; 1339 case 'l': 1340 app_sa_prm.enable = 1; 1341 break; 1342 case 'w': 1343 app_sa_prm.enable = 1; 1344 app_sa_prm.window_size = parse_decimal(optarg); 1345 break; 1346 case 'e': 1347 app_sa_prm.enable = 1; 1348 app_sa_prm.enable_esn = 1; 1349 break; 1350 case 'a': 1351 app_sa_prm.enable = 1; 1352 app_sa_prm.flags |= RTE_IPSEC_SAFLAG_SQN_ATOM; 1353 break; 1354 case CMD_LINE_OPT_CONFIG_NUM: 1355 ret = parse_config(optarg); 1356 if (ret) { 1357 printf("Invalid config\n"); 1358 print_usage(prgname); 1359 return -1; 1360 } 1361 break; 1362 case CMD_LINE_OPT_SINGLE_SA_NUM: 1363 ret = parse_decimal(optarg); 1364 if (ret == -1) { 1365 printf("Invalid argument[sa_idx]\n"); 1366 print_usage(prgname); 1367 return -1; 1368 } 1369 1370 /* else */ 1371 single_sa = 1; 1372 single_sa_idx = ret; 1373 printf("Configured with single SA index %u\n", 1374 single_sa_idx); 1375 break; 1376 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM: 1377 ret = parse_portmask(optarg); 1378 if (ret == -1) { 1379 printf("Invalid argument[portmask]\n"); 1380 print_usage(prgname); 1381 return -1; 1382 } 1383 1384 /* else */ 1385 enabled_cryptodev_mask = ret; 1386 break; 1387 case CMD_LINE_OPT_RX_OFFLOAD_NUM: 1388 ret = parse_mask(optarg, &dev_rx_offload); 1389 if (ret != 0) { 1390 printf("Invalid argument for \'%s\': %s\n", 1391 CMD_LINE_OPT_RX_OFFLOAD, optarg); 1392 print_usage(prgname); 1393 return -1; 1394 } 1395 break; 1396 case CMD_LINE_OPT_TX_OFFLOAD_NUM: 1397 ret = parse_mask(optarg, &dev_tx_offload); 1398 if (ret != 0) { 1399 printf("Invalid argument for \'%s\': %s\n", 1400 CMD_LINE_OPT_TX_OFFLOAD, optarg); 1401 print_usage(prgname); 1402 return -1; 1403 } 1404 break; 1405 default: 1406 print_usage(prgname); 1407 return -1; 1408 } 1409 } 1410 1411 if (f_present == 0) { 1412 printf("Mandatory option \"-f\" not present\n"); 1413 return -1; 1414 } 1415 1416 print_app_sa_prm(&app_sa_prm); 1417 1418 if (optind >= 0) 1419 argv[optind-1] = prgname; 1420 1421 ret = optind-1; 1422 optind = 1; /* reset getopt lib */ 1423 return ret; 1424 } 1425 1426 static void 1427 print_ethaddr(const char *name, const struct ether_addr *eth_addr) 1428 { 1429 char buf[ETHER_ADDR_FMT_SIZE]; 1430 ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); 1431 printf("%s%s", name, buf); 1432 } 1433 1434 /* 1435 * Update destination ethaddr for the port. 1436 */ 1437 int 1438 add_dst_ethaddr(uint16_t port, const struct ether_addr *addr) 1439 { 1440 if (port >= RTE_DIM(ethaddr_tbl)) 1441 return -EINVAL; 1442 1443 ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr); 1444 return 0; 1445 } 1446 1447 /* Check the link status of all ports in up to 9s, and print them finally */ 1448 static void 1449 check_all_ports_link_status(uint32_t port_mask) 1450 { 1451 #define CHECK_INTERVAL 100 /* 100ms */ 1452 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1453 uint16_t portid; 1454 uint8_t count, all_ports_up, print_flag = 0; 1455 struct rte_eth_link link; 1456 1457 printf("\nChecking link status"); 1458 fflush(stdout); 1459 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1460 all_ports_up = 1; 1461 RTE_ETH_FOREACH_DEV(portid) { 1462 if ((port_mask & (1 << portid)) == 0) 1463 continue; 1464 memset(&link, 0, sizeof(link)); 1465 rte_eth_link_get_nowait(portid, &link); 1466 /* print link status if flag set */ 1467 if (print_flag == 1) { 1468 if (link.link_status) 1469 printf( 1470 "Port%d Link Up - speed %u Mbps -%s\n", 1471 portid, link.link_speed, 1472 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1473 ("full-duplex") : ("half-duplex\n")); 1474 else 1475 printf("Port %d Link Down\n", portid); 1476 continue; 1477 } 1478 /* clear all_ports_up flag if any link down */ 1479 if (link.link_status == ETH_LINK_DOWN) { 1480 all_ports_up = 0; 1481 break; 1482 } 1483 } 1484 /* after finally printing all link status, get out */ 1485 if (print_flag == 1) 1486 break; 1487 1488 if (all_ports_up == 0) { 1489 printf("."); 1490 fflush(stdout); 1491 rte_delay_ms(CHECK_INTERVAL); 1492 } 1493 1494 /* set the print_flag if all ports up or timeout */ 1495 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1496 print_flag = 1; 1497 printf("done\n"); 1498 } 1499 } 1500 } 1501 1502 static int32_t 1503 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1504 uint16_t qp, struct lcore_params *params, 1505 struct ipsec_ctx *ipsec_ctx, 1506 const struct rte_cryptodev_capabilities *cipher, 1507 const struct rte_cryptodev_capabilities *auth, 1508 const struct rte_cryptodev_capabilities *aead) 1509 { 1510 int32_t ret = 0; 1511 unsigned long i; 1512 struct cdev_key key = { 0 }; 1513 1514 key.lcore_id = params->lcore_id; 1515 if (cipher) 1516 key.cipher_algo = cipher->sym.cipher.algo; 1517 if (auth) 1518 key.auth_algo = auth->sym.auth.algo; 1519 if (aead) 1520 key.aead_algo = aead->sym.aead.algo; 1521 1522 ret = rte_hash_lookup(map, &key); 1523 if (ret != -ENOENT) 1524 return 0; 1525 1526 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1527 if (ipsec_ctx->tbl[i].id == cdev_id) 1528 break; 1529 1530 if (i == ipsec_ctx->nb_qps) { 1531 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1532 printf("Maximum number of crypto devices assigned to " 1533 "a core, increase MAX_QP_PER_LCORE value\n"); 1534 return 0; 1535 } 1536 ipsec_ctx->tbl[i].id = cdev_id; 1537 ipsec_ctx->tbl[i].qp = qp; 1538 ipsec_ctx->nb_qps++; 1539 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1540 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1541 cdev_id, qp, i); 1542 } 1543 1544 ret = rte_hash_add_key_data(map, &key, (void *)i); 1545 if (ret < 0) { 1546 printf("Faled to insert cdev mapping for (lcore %u, " 1547 "cdev %u, qp %u), errno %d\n", 1548 key.lcore_id, ipsec_ctx->tbl[i].id, 1549 ipsec_ctx->tbl[i].qp, ret); 1550 return 0; 1551 } 1552 1553 return 1; 1554 } 1555 1556 static int32_t 1557 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1558 uint16_t qp, struct lcore_params *params) 1559 { 1560 int32_t ret = 0; 1561 const struct rte_cryptodev_capabilities *i, *j; 1562 struct rte_hash *map; 1563 struct lcore_conf *qconf; 1564 struct ipsec_ctx *ipsec_ctx; 1565 const char *str; 1566 1567 qconf = &lcore_conf[params->lcore_id]; 1568 1569 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 1570 map = cdev_map_out; 1571 ipsec_ctx = &qconf->outbound; 1572 str = "Outbound"; 1573 } else { 1574 map = cdev_map_in; 1575 ipsec_ctx = &qconf->inbound; 1576 str = "Inbound"; 1577 } 1578 1579 /* Required cryptodevs with operation chainning */ 1580 if (!(dev_info->feature_flags & 1581 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 1582 return ret; 1583 1584 for (i = dev_info->capabilities; 1585 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 1586 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1587 continue; 1588 1589 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1590 ret |= add_mapping(map, str, cdev_id, qp, params, 1591 ipsec_ctx, NULL, NULL, i); 1592 continue; 1593 } 1594 1595 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 1596 continue; 1597 1598 for (j = dev_info->capabilities; 1599 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 1600 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1601 continue; 1602 1603 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 1604 continue; 1605 1606 ret |= add_mapping(map, str, cdev_id, qp, params, 1607 ipsec_ctx, i, j, NULL); 1608 } 1609 } 1610 1611 return ret; 1612 } 1613 1614 /* Check if the device is enabled by cryptodev_mask */ 1615 static int 1616 check_cryptodev_mask(uint8_t cdev_id) 1617 { 1618 if (enabled_cryptodev_mask & (1 << cdev_id)) 1619 return 0; 1620 1621 return -1; 1622 } 1623 1624 static int32_t 1625 cryptodevs_init(void) 1626 { 1627 struct rte_cryptodev_config dev_conf; 1628 struct rte_cryptodev_qp_conf qp_conf; 1629 uint16_t idx, max_nb_qps, qp, i; 1630 int16_t cdev_id, port_id; 1631 struct rte_hash_parameters params = { 0 }; 1632 1633 params.entries = CDEV_MAP_ENTRIES; 1634 params.key_len = sizeof(struct cdev_key); 1635 params.hash_func = rte_jhash; 1636 params.hash_func_init_val = 0; 1637 params.socket_id = rte_socket_id(); 1638 1639 params.name = "cdev_map_in"; 1640 cdev_map_in = rte_hash_create(¶ms); 1641 if (cdev_map_in == NULL) 1642 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1643 rte_errno); 1644 1645 params.name = "cdev_map_out"; 1646 cdev_map_out = rte_hash_create(¶ms); 1647 if (cdev_map_out == NULL) 1648 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1649 rte_errno); 1650 1651 printf("lcore/cryptodev/qp mappings:\n"); 1652 1653 uint32_t max_sess_sz = 0, sess_sz; 1654 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1655 void *sec_ctx; 1656 1657 /* Get crypto priv session size */ 1658 sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 1659 if (sess_sz > max_sess_sz) 1660 max_sess_sz = sess_sz; 1661 1662 /* 1663 * If crypto device is security capable, need to check the 1664 * size of security session as well. 1665 */ 1666 1667 /* Get security context of the crypto device */ 1668 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id); 1669 if (sec_ctx == NULL) 1670 continue; 1671 1672 /* Get size of security session */ 1673 sess_sz = rte_security_session_get_size(sec_ctx); 1674 if (sess_sz > max_sess_sz) 1675 max_sess_sz = sess_sz; 1676 } 1677 RTE_ETH_FOREACH_DEV(port_id) { 1678 void *sec_ctx; 1679 1680 if ((enabled_port_mask & (1 << port_id)) == 0) 1681 continue; 1682 1683 sec_ctx = rte_eth_dev_get_sec_ctx(port_id); 1684 if (sec_ctx == NULL) 1685 continue; 1686 1687 sess_sz = rte_security_session_get_size(sec_ctx); 1688 if (sess_sz > max_sess_sz) 1689 max_sess_sz = sess_sz; 1690 } 1691 1692 idx = 0; 1693 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1694 struct rte_cryptodev_info cdev_info; 1695 1696 if (check_cryptodev_mask((uint8_t)cdev_id)) 1697 continue; 1698 1699 rte_cryptodev_info_get(cdev_id, &cdev_info); 1700 1701 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 1702 max_nb_qps = cdev_info.max_nb_queue_pairs; 1703 else 1704 max_nb_qps = nb_lcore_params; 1705 1706 qp = 0; 1707 i = 0; 1708 while (qp < max_nb_qps && i < nb_lcore_params) { 1709 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 1710 &lcore_params[idx])) 1711 qp++; 1712 idx++; 1713 idx = idx % nb_lcore_params; 1714 i++; 1715 } 1716 1717 if (qp == 0) 1718 continue; 1719 1720 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 1721 dev_conf.nb_queue_pairs = qp; 1722 1723 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions; 1724 if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS) 1725 rte_exit(EXIT_FAILURE, 1726 "Device does not support at least %u " 1727 "sessions", CDEV_MP_NB_OBJS); 1728 1729 if (!socket_ctx[dev_conf.socket_id].session_pool) { 1730 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1731 struct rte_mempool *sess_mp; 1732 1733 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1734 "sess_mp_%u", dev_conf.socket_id); 1735 sess_mp = rte_cryptodev_sym_session_pool_create( 1736 mp_name, CDEV_MP_NB_OBJS, 1737 0, CDEV_MP_CACHE_SZ, 0, 1738 dev_conf.socket_id); 1739 socket_ctx[dev_conf.socket_id].session_pool = sess_mp; 1740 } 1741 1742 if (!socket_ctx[dev_conf.socket_id].session_priv_pool) { 1743 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1744 struct rte_mempool *sess_mp; 1745 1746 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1747 "sess_mp_priv_%u", dev_conf.socket_id); 1748 sess_mp = rte_mempool_create(mp_name, 1749 CDEV_MP_NB_OBJS, 1750 max_sess_sz, 1751 CDEV_MP_CACHE_SZ, 1752 0, NULL, NULL, NULL, 1753 NULL, dev_conf.socket_id, 1754 0); 1755 socket_ctx[dev_conf.socket_id].session_priv_pool = 1756 sess_mp; 1757 } 1758 1759 if (!socket_ctx[dev_conf.socket_id].session_priv_pool || 1760 !socket_ctx[dev_conf.socket_id].session_pool) 1761 rte_exit(EXIT_FAILURE, 1762 "Cannot create session pool on socket %d\n", 1763 dev_conf.socket_id); 1764 else 1765 printf("Allocated session pool on socket %d\n", 1766 dev_conf.socket_id); 1767 1768 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 1769 rte_panic("Failed to initialize cryptodev %u\n", 1770 cdev_id); 1771 1772 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 1773 qp_conf.mp_session = 1774 socket_ctx[dev_conf.socket_id].session_pool; 1775 qp_conf.mp_session_private = 1776 socket_ctx[dev_conf.socket_id].session_priv_pool; 1777 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 1778 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 1779 &qp_conf, dev_conf.socket_id)) 1780 rte_panic("Failed to setup queue %u for " 1781 "cdev_id %u\n", 0, cdev_id); 1782 1783 if (rte_cryptodev_start(cdev_id)) 1784 rte_panic("Failed to start cryptodev %u\n", 1785 cdev_id); 1786 } 1787 1788 /* create session pools for eth devices that implement security */ 1789 RTE_ETH_FOREACH_DEV(port_id) { 1790 if ((enabled_port_mask & (1 << port_id)) && 1791 rte_eth_dev_get_sec_ctx(port_id)) { 1792 int socket_id = rte_eth_dev_socket_id(port_id); 1793 1794 if (!socket_ctx[socket_id].session_priv_pool) { 1795 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1796 struct rte_mempool *sess_mp; 1797 1798 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1799 "sess_mp_%u", socket_id); 1800 sess_mp = rte_mempool_create(mp_name, 1801 (CDEV_MP_NB_OBJS * 2), 1802 max_sess_sz, 1803 CDEV_MP_CACHE_SZ, 1804 0, NULL, NULL, NULL, 1805 NULL, socket_id, 1806 0); 1807 if (sess_mp == NULL) 1808 rte_exit(EXIT_FAILURE, 1809 "Cannot create session pool " 1810 "on socket %d\n", socket_id); 1811 else 1812 printf("Allocated session pool " 1813 "on socket %d\n", socket_id); 1814 socket_ctx[socket_id].session_priv_pool = 1815 sess_mp; 1816 } 1817 } 1818 } 1819 1820 1821 printf("\n"); 1822 1823 return 0; 1824 } 1825 1826 static void 1827 port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads) 1828 { 1829 struct rte_eth_dev_info dev_info; 1830 struct rte_eth_txconf *txconf; 1831 uint16_t nb_tx_queue, nb_rx_queue; 1832 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 1833 int32_t ret, socket_id; 1834 struct lcore_conf *qconf; 1835 struct ether_addr ethaddr; 1836 struct rte_eth_conf local_port_conf = port_conf; 1837 1838 rte_eth_dev_info_get(portid, &dev_info); 1839 1840 /* limit allowed HW offloafs, as user requested */ 1841 dev_info.rx_offload_capa &= dev_rx_offload; 1842 dev_info.tx_offload_capa &= dev_tx_offload; 1843 1844 printf("Configuring device port %u:\n", portid); 1845 1846 rte_eth_macaddr_get(portid, ðaddr); 1847 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ðaddr); 1848 print_ethaddr("Address: ", ðaddr); 1849 printf("\n"); 1850 1851 nb_rx_queue = get_port_nb_rx_queues(portid); 1852 nb_tx_queue = nb_lcores; 1853 1854 if (nb_rx_queue > dev_info.max_rx_queues) 1855 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1856 "(max rx queue is %u)\n", 1857 nb_rx_queue, dev_info.max_rx_queues); 1858 1859 if (nb_tx_queue > dev_info.max_tx_queues) 1860 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1861 "(max tx queue is %u)\n", 1862 nb_tx_queue, dev_info.max_tx_queues); 1863 1864 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 1865 nb_rx_queue, nb_tx_queue); 1866 1867 if (frame_size) { 1868 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 1869 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1870 } 1871 1872 local_port_conf.rxmode.offloads |= req_rx_offloads; 1873 local_port_conf.txmode.offloads |= req_tx_offloads; 1874 1875 /* Check that all required capabilities are supported */ 1876 if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) != 1877 local_port_conf.rxmode.offloads) 1878 rte_exit(EXIT_FAILURE, 1879 "Error: port %u required RX offloads: 0x%" PRIx64 1880 ", avaialbe RX offloads: 0x%" PRIx64 "\n", 1881 portid, local_port_conf.rxmode.offloads, 1882 dev_info.rx_offload_capa); 1883 1884 if ((local_port_conf.txmode.offloads & dev_info.tx_offload_capa) != 1885 local_port_conf.txmode.offloads) 1886 rte_exit(EXIT_FAILURE, 1887 "Error: port %u required TX offloads: 0x%" PRIx64 1888 ", avaialbe TX offloads: 0x%" PRIx64 "\n", 1889 portid, local_port_conf.txmode.offloads, 1890 dev_info.tx_offload_capa); 1891 1892 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 1893 local_port_conf.txmode.offloads |= 1894 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1895 1896 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 1897 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 1898 1899 printf("port %u configurng rx_offloads=0x%" PRIx64 1900 ", tx_offloads=0x%" PRIx64 "\n", 1901 portid, local_port_conf.rxmode.offloads, 1902 local_port_conf.txmode.offloads); 1903 1904 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 1905 dev_info.flow_type_rss_offloads; 1906 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 1907 port_conf.rx_adv_conf.rss_conf.rss_hf) { 1908 printf("Port %u modified RSS hash function based on hardware support," 1909 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 1910 portid, 1911 port_conf.rx_adv_conf.rss_conf.rss_hf, 1912 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 1913 } 1914 1915 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 1916 &local_port_conf); 1917 if (ret < 0) 1918 rte_exit(EXIT_FAILURE, "Cannot configure device: " 1919 "err=%d, port=%d\n", ret, portid); 1920 1921 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 1922 if (ret < 0) 1923 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 1924 "err=%d, port=%d\n", ret, portid); 1925 1926 /* init one TX queue per lcore */ 1927 tx_queueid = 0; 1928 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1929 if (rte_lcore_is_enabled(lcore_id) == 0) 1930 continue; 1931 1932 if (numa_on) 1933 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1934 else 1935 socket_id = 0; 1936 1937 /* init TX queue */ 1938 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 1939 1940 txconf = &dev_info.default_txconf; 1941 txconf->offloads = local_port_conf.txmode.offloads; 1942 1943 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 1944 socket_id, txconf); 1945 if (ret < 0) 1946 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 1947 "err=%d, port=%d\n", ret, portid); 1948 1949 qconf = &lcore_conf[lcore_id]; 1950 qconf->tx_queue_id[portid] = tx_queueid; 1951 1952 /* Pre-populate pkt offloads based on capabilities */ 1953 qconf->outbound.ipv4_offloads = PKT_TX_IPV4; 1954 qconf->outbound.ipv6_offloads = PKT_TX_IPV6; 1955 if (local_port_conf.txmode.offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 1956 qconf->outbound.ipv4_offloads |= PKT_TX_IP_CKSUM; 1957 1958 tx_queueid++; 1959 1960 /* init RX queues */ 1961 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 1962 struct rte_eth_rxconf rxq_conf; 1963 1964 if (portid != qconf->rx_queue_list[queue].port_id) 1965 continue; 1966 1967 rx_queueid = qconf->rx_queue_list[queue].queue_id; 1968 1969 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 1970 socket_id); 1971 1972 rxq_conf = dev_info.default_rxconf; 1973 rxq_conf.offloads = local_port_conf.rxmode.offloads; 1974 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 1975 nb_rxd, socket_id, &rxq_conf, 1976 socket_ctx[socket_id].mbuf_pool); 1977 if (ret < 0) 1978 rte_exit(EXIT_FAILURE, 1979 "rte_eth_rx_queue_setup: err=%d, " 1980 "port=%d\n", ret, portid); 1981 } 1982 } 1983 printf("\n"); 1984 } 1985 1986 static void 1987 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 1988 { 1989 char s[64]; 1990 uint32_t buff_size = frame_size ? (frame_size + RTE_PKTMBUF_HEADROOM) : 1991 RTE_MBUF_DEFAULT_BUF_SIZE; 1992 1993 1994 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 1995 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 1996 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 1997 buff_size, 1998 socket_id); 1999 if (ctx->mbuf_pool == NULL) 2000 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 2001 socket_id); 2002 else 2003 printf("Allocated mbuf pool on socket %d\n", socket_id); 2004 } 2005 2006 static inline int 2007 inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md) 2008 { 2009 struct ipsec_sa *sa; 2010 2011 /* For inline protocol processing, the metadata in the event will 2012 * uniquely identify the security session which raised the event. 2013 * Application would then need the userdata it had registered with the 2014 * security session to process the event. 2015 */ 2016 2017 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md); 2018 2019 if (sa == NULL) { 2020 /* userdata could not be retrieved */ 2021 return -1; 2022 } 2023 2024 /* Sequence number over flow. SA need to be re-established */ 2025 RTE_SET_USED(sa); 2026 return 0; 2027 } 2028 2029 static int 2030 inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type, 2031 void *param, void *ret_param) 2032 { 2033 uint64_t md; 2034 struct rte_eth_event_ipsec_desc *event_desc = NULL; 2035 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 2036 rte_eth_dev_get_sec_ctx(port_id); 2037 2038 RTE_SET_USED(param); 2039 2040 if (type != RTE_ETH_EVENT_IPSEC) 2041 return -1; 2042 2043 event_desc = ret_param; 2044 if (event_desc == NULL) { 2045 printf("Event descriptor not set\n"); 2046 return -1; 2047 } 2048 2049 md = event_desc->metadata; 2050 2051 if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW) 2052 return inline_ipsec_event_esn_overflow(ctx, md); 2053 else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) { 2054 printf("Invalid IPsec event reported\n"); 2055 return -1; 2056 } 2057 2058 return -1; 2059 } 2060 2061 int32_t 2062 main(int32_t argc, char **argv) 2063 { 2064 int32_t ret; 2065 uint32_t lcore_id; 2066 uint8_t socket_id; 2067 uint16_t portid; 2068 uint64_t req_rx_offloads, req_tx_offloads; 2069 2070 /* init EAL */ 2071 ret = rte_eal_init(argc, argv); 2072 if (ret < 0) 2073 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 2074 argc -= ret; 2075 argv += ret; 2076 2077 /* parse application arguments (after the EAL ones) */ 2078 ret = parse_args(argc, argv); 2079 if (ret < 0) 2080 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 2081 2082 if ((unprotected_port_mask & enabled_port_mask) != 2083 unprotected_port_mask) 2084 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 2085 unprotected_port_mask); 2086 2087 if (check_params() < 0) 2088 rte_exit(EXIT_FAILURE, "check_params failed\n"); 2089 2090 ret = init_lcore_rx_queues(); 2091 if (ret < 0) 2092 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 2093 2094 nb_lcores = rte_lcore_count(); 2095 2096 /* Replicate each context per socket */ 2097 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2098 if (rte_lcore_is_enabled(lcore_id) == 0) 2099 continue; 2100 2101 if (numa_on) 2102 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2103 else 2104 socket_id = 0; 2105 2106 if (socket_ctx[socket_id].mbuf_pool) 2107 continue; 2108 2109 /* initilaze SPD */ 2110 sp4_init(&socket_ctx[socket_id], socket_id); 2111 2112 sp6_init(&socket_ctx[socket_id], socket_id); 2113 2114 /* initilaze SAD */ 2115 sa_init(&socket_ctx[socket_id], socket_id); 2116 2117 rt_init(&socket_ctx[socket_id], socket_id); 2118 2119 pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF); 2120 } 2121 2122 RTE_ETH_FOREACH_DEV(portid) { 2123 if ((enabled_port_mask & (1 << portid)) == 0) 2124 continue; 2125 2126 sa_check_offloads(portid, &req_rx_offloads, &req_tx_offloads); 2127 port_init(portid, req_rx_offloads, req_tx_offloads); 2128 } 2129 2130 cryptodevs_init(); 2131 2132 /* start ports */ 2133 RTE_ETH_FOREACH_DEV(portid) { 2134 if ((enabled_port_mask & (1 << portid)) == 0) 2135 continue; 2136 2137 /* Start device */ 2138 ret = rte_eth_dev_start(portid); 2139 if (ret < 0) 2140 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 2141 "err=%d, port=%d\n", ret, portid); 2142 /* 2143 * If enabled, put device in promiscuous mode. 2144 * This allows IO forwarding mode to forward packets 2145 * to itself through 2 cross-connected ports of the 2146 * target machine. 2147 */ 2148 if (promiscuous_on) 2149 rte_eth_promiscuous_enable(portid); 2150 2151 rte_eth_dev_callback_register(portid, 2152 RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL); 2153 } 2154 2155 check_all_ports_link_status(enabled_port_mask); 2156 2157 /* launch per-lcore init on every lcore */ 2158 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 2159 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2160 if (rte_eal_wait_lcore(lcore_id) < 0) 2161 return -1; 2162 } 2163 2164 return 0; 2165 } 2166