1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <netinet/in.h> 11 #include <netinet/ip.h> 12 #include <netinet/ip6.h> 13 #include <string.h> 14 #include <sys/queue.h> 15 #include <stdarg.h> 16 #include <errno.h> 17 #include <getopt.h> 18 19 #include <rte_common.h> 20 #include <rte_byteorder.h> 21 #include <rte_log.h> 22 #include <rte_eal.h> 23 #include <rte_launch.h> 24 #include <rte_atomic.h> 25 #include <rte_cycles.h> 26 #include <rte_prefetch.h> 27 #include <rte_lcore.h> 28 #include <rte_per_lcore.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_interrupts.h> 31 #include <rte_random.h> 32 #include <rte_debug.h> 33 #include <rte_ether.h> 34 #include <rte_ethdev.h> 35 #include <rte_mempool.h> 36 #include <rte_mbuf.h> 37 #include <rte_acl.h> 38 #include <rte_lpm.h> 39 #include <rte_lpm6.h> 40 #include <rte_hash.h> 41 #include <rte_jhash.h> 42 #include <rte_cryptodev.h> 43 #include <rte_security.h> 44 45 #include "ipsec.h" 46 #include "parser.h" 47 48 #define RTE_LOGTYPE_IPSEC RTE_LOGTYPE_USER1 49 50 #define MAX_JUMBO_PKT_LEN 9600 51 52 #define MEMPOOL_CACHE_SIZE 256 53 54 #define NB_MBUF (32000) 55 56 #define CDEV_QUEUE_DESC 2048 57 #define CDEV_MAP_ENTRIES 16384 58 #define CDEV_MP_NB_OBJS 1024 59 #define CDEV_MP_CACHE_SZ 64 60 #define MAX_QUEUE_PAIRS 1 61 62 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 63 64 #define NB_SOCKETS 4 65 66 /* Configure how many packets ahead to prefetch, when reading packets */ 67 #define PREFETCH_OFFSET 3 68 69 #define MAX_RX_QUEUE_PER_LCORE 16 70 71 #define MAX_LCORE_PARAMS 1024 72 73 #define UNPROTECTED_PORT(port) (unprotected_port_mask & (1 << portid)) 74 75 /* 76 * Configurable number of RX/TX ring descriptors 77 */ 78 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024 79 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024 80 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 81 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 82 83 #if RTE_BYTE_ORDER != RTE_LITTLE_ENDIAN 84 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 85 (((uint64_t)((a) & 0xff) << 56) | \ 86 ((uint64_t)((b) & 0xff) << 48) | \ 87 ((uint64_t)((c) & 0xff) << 40) | \ 88 ((uint64_t)((d) & 0xff) << 32) | \ 89 ((uint64_t)((e) & 0xff) << 24) | \ 90 ((uint64_t)((f) & 0xff) << 16) | \ 91 ((uint64_t)((g) & 0xff) << 8) | \ 92 ((uint64_t)(h) & 0xff)) 93 #else 94 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 95 (((uint64_t)((h) & 0xff) << 56) | \ 96 ((uint64_t)((g) & 0xff) << 48) | \ 97 ((uint64_t)((f) & 0xff) << 40) | \ 98 ((uint64_t)((e) & 0xff) << 32) | \ 99 ((uint64_t)((d) & 0xff) << 24) | \ 100 ((uint64_t)((c) & 0xff) << 16) | \ 101 ((uint64_t)((b) & 0xff) << 8) | \ 102 ((uint64_t)(a) & 0xff)) 103 #endif 104 #define ETHADDR(a, b, c, d, e, f) (__BYTES_TO_UINT64(a, b, c, d, e, f, 0, 0)) 105 106 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 107 (addr)->addr_bytes[0], (addr)->addr_bytes[1], \ 108 (addr)->addr_bytes[2], (addr)->addr_bytes[3], \ 109 (addr)->addr_bytes[4], (addr)->addr_bytes[5], \ 110 0, 0) 111 112 /* port/source ethernet addr and destination ethernet addr */ 113 struct ethaddr_info { 114 uint64_t src, dst; 115 }; 116 117 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 118 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 119 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 120 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 121 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 122 }; 123 124 #define CMD_LINE_OPT_CONFIG "config" 125 #define CMD_LINE_OPT_SINGLE_SA "single-sa" 126 #define CMD_LINE_OPT_CRYPTODEV_MASK "cryptodev_mask" 127 #define CMD_LINE_OPT_RX_OFFLOAD "rxoffload" 128 #define CMD_LINE_OPT_TX_OFFLOAD "txoffload" 129 130 enum { 131 /* long options mapped to a short option */ 132 133 /* first long only option value must be >= 256, so that we won't 134 * conflict with short options 135 */ 136 CMD_LINE_OPT_MIN_NUM = 256, 137 CMD_LINE_OPT_CONFIG_NUM, 138 CMD_LINE_OPT_SINGLE_SA_NUM, 139 CMD_LINE_OPT_CRYPTODEV_MASK_NUM, 140 CMD_LINE_OPT_RX_OFFLOAD_NUM, 141 CMD_LINE_OPT_TX_OFFLOAD_NUM, 142 }; 143 144 static const struct option lgopts[] = { 145 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 146 {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM}, 147 {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM}, 148 {CMD_LINE_OPT_RX_OFFLOAD, 1, 0, CMD_LINE_OPT_RX_OFFLOAD_NUM}, 149 {CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM}, 150 {NULL, 0, 0, 0} 151 }; 152 153 /* mask of enabled ports */ 154 static uint32_t enabled_port_mask; 155 static uint64_t enabled_cryptodev_mask = UINT64_MAX; 156 static uint32_t unprotected_port_mask; 157 static int32_t promiscuous_on = 1; 158 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 159 static uint32_t nb_lcores; 160 static uint32_t single_sa; 161 static uint32_t single_sa_idx; 162 static uint32_t frame_size; 163 164 /* 165 * RX/TX HW offload capabilities to enable/use on ethernet ports. 166 * By default all capabilities are enabled. 167 */ 168 static uint64_t dev_rx_offload = UINT64_MAX; 169 static uint64_t dev_tx_offload = UINT64_MAX; 170 171 /* application wide librte_ipsec/SA parameters */ 172 struct app_sa_prm app_sa_prm = {.enable = 0}; 173 174 struct lcore_rx_queue { 175 uint16_t port_id; 176 uint8_t queue_id; 177 } __rte_cache_aligned; 178 179 struct lcore_params { 180 uint16_t port_id; 181 uint8_t queue_id; 182 uint8_t lcore_id; 183 } __rte_cache_aligned; 184 185 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 186 187 static struct lcore_params *lcore_params; 188 static uint16_t nb_lcore_params; 189 190 static struct rte_hash *cdev_map_in; 191 static struct rte_hash *cdev_map_out; 192 193 struct buffer { 194 uint16_t len; 195 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 196 }; 197 198 struct lcore_conf { 199 uint16_t nb_rx_queue; 200 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 201 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 202 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 203 struct ipsec_ctx inbound; 204 struct ipsec_ctx outbound; 205 struct rt_ctx *rt4_ctx; 206 struct rt_ctx *rt6_ctx; 207 } __rte_cache_aligned; 208 209 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 210 211 static struct rte_eth_conf port_conf = { 212 .rxmode = { 213 .mq_mode = ETH_MQ_RX_RSS, 214 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 215 .split_hdr_size = 0, 216 .offloads = DEV_RX_OFFLOAD_CHECKSUM, 217 }, 218 .rx_adv_conf = { 219 .rss_conf = { 220 .rss_key = NULL, 221 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 222 ETH_RSS_TCP | ETH_RSS_SCTP, 223 }, 224 }, 225 .txmode = { 226 .mq_mode = ETH_MQ_TX_NONE, 227 }, 228 }; 229 230 static struct socket_ctx socket_ctx[NB_SOCKETS]; 231 232 static inline void 233 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 234 { 235 uint8_t *nlp; 236 struct rte_ether_hdr *eth; 237 238 eth = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *); 239 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 240 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, RTE_ETHER_HDR_LEN); 241 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip, ip_p)); 242 if (*nlp == IPPROTO_ESP) 243 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 244 else { 245 t->ip4.data[t->ip4.num] = nlp; 246 t->ip4.pkts[(t->ip4.num)++] = pkt; 247 } 248 pkt->l2_len = 0; 249 pkt->l3_len = sizeof(struct ip); 250 } else if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 251 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, RTE_ETHER_HDR_LEN); 252 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip6_hdr, ip6_nxt)); 253 if (*nlp == IPPROTO_ESP) 254 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 255 else { 256 t->ip6.data[t->ip6.num] = nlp; 257 t->ip6.pkts[(t->ip6.num)++] = pkt; 258 } 259 pkt->l2_len = 0; 260 pkt->l3_len = sizeof(struct ip6_hdr); 261 } else { 262 /* Unknown/Unsupported type, drop the packet */ 263 RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n", 264 rte_be_to_cpu_16(eth->ether_type)); 265 rte_pktmbuf_free(pkt); 266 } 267 268 /* Check if the packet has been processed inline. For inline protocol 269 * processed packets, the metadata in the mbuf can be used to identify 270 * the security processing done on the packet. The metadata will be 271 * used to retrieve the application registered userdata associated 272 * with the security session. 273 */ 274 275 if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) { 276 struct ipsec_sa *sa; 277 struct ipsec_mbuf_metadata *priv; 278 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 279 rte_eth_dev_get_sec_ctx( 280 pkt->port); 281 282 /* Retrieve the userdata registered. Here, the userdata 283 * registered is the SA pointer. 284 */ 285 286 sa = (struct ipsec_sa *) 287 rte_security_get_userdata(ctx, pkt->udata64); 288 289 if (sa == NULL) { 290 /* userdata could not be retrieved */ 291 return; 292 } 293 294 /* Save SA as priv member in mbuf. This will be used in the 295 * IPsec selector(SP-SA) check. 296 */ 297 298 priv = get_priv(pkt); 299 priv->sa = sa; 300 } 301 } 302 303 static inline void 304 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 305 uint16_t nb_pkts) 306 { 307 int32_t i; 308 309 t->ipsec.num = 0; 310 t->ip4.num = 0; 311 t->ip6.num = 0; 312 313 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 314 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 315 void *)); 316 prepare_one_packet(pkts[i], t); 317 } 318 /* Process left packets */ 319 for (; i < nb_pkts; i++) 320 prepare_one_packet(pkts[i], t); 321 } 322 323 static inline void 324 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port, 325 const struct lcore_conf *qconf) 326 { 327 struct ip *ip; 328 struct rte_ether_hdr *ethhdr; 329 330 ip = rte_pktmbuf_mtod(pkt, struct ip *); 331 332 ethhdr = (struct rte_ether_hdr *) 333 rte_pktmbuf_prepend(pkt, RTE_ETHER_HDR_LEN); 334 335 if (ip->ip_v == IPVERSION) { 336 pkt->ol_flags |= qconf->outbound.ipv4_offloads; 337 pkt->l3_len = sizeof(struct ip); 338 pkt->l2_len = RTE_ETHER_HDR_LEN; 339 340 ip->ip_sum = 0; 341 342 /* calculate IPv4 cksum in SW */ 343 if ((pkt->ol_flags & PKT_TX_IP_CKSUM) == 0) 344 ip->ip_sum = rte_ipv4_cksum((struct rte_ipv4_hdr *)ip); 345 346 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 347 } else { 348 pkt->ol_flags |= qconf->outbound.ipv6_offloads; 349 pkt->l3_len = sizeof(struct ip6_hdr); 350 pkt->l2_len = RTE_ETHER_HDR_LEN; 351 352 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 353 } 354 355 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 356 sizeof(struct rte_ether_addr)); 357 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 358 sizeof(struct rte_ether_addr)); 359 } 360 361 static inline void 362 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port, 363 const struct lcore_conf *qconf) 364 { 365 int32_t i; 366 const int32_t prefetch_offset = 2; 367 368 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 369 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 370 prepare_tx_pkt(pkts[i], port, qconf); 371 } 372 /* Process left packets */ 373 for (; i < nb_pkts; i++) 374 prepare_tx_pkt(pkts[i], port, qconf); 375 } 376 377 /* Send burst of packets on an output interface */ 378 static inline int32_t 379 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 380 { 381 struct rte_mbuf **m_table; 382 int32_t ret; 383 uint16_t queueid; 384 385 queueid = qconf->tx_queue_id[port]; 386 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 387 388 prepare_tx_burst(m_table, n, port, qconf); 389 390 ret = rte_eth_tx_burst(port, queueid, m_table, n); 391 if (unlikely(ret < n)) { 392 do { 393 rte_pktmbuf_free(m_table[ret]); 394 } while (++ret < n); 395 } 396 397 return 0; 398 } 399 400 /* Enqueue a single packet, and send burst if queue is filled */ 401 static inline int32_t 402 send_single_packet(struct rte_mbuf *m, uint16_t port) 403 { 404 uint32_t lcore_id; 405 uint16_t len; 406 struct lcore_conf *qconf; 407 408 lcore_id = rte_lcore_id(); 409 410 qconf = &lcore_conf[lcore_id]; 411 len = qconf->tx_mbufs[port].len; 412 qconf->tx_mbufs[port].m_table[len] = m; 413 len++; 414 415 /* enough pkts to be sent */ 416 if (unlikely(len == MAX_PKT_BURST)) { 417 send_burst(qconf, MAX_PKT_BURST, port); 418 len = 0; 419 } 420 421 qconf->tx_mbufs[port].len = len; 422 return 0; 423 } 424 425 static inline void 426 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 427 uint16_t lim) 428 { 429 struct rte_mbuf *m; 430 uint32_t i, j, res, sa_idx; 431 432 if (ip->num == 0 || sp == NULL) 433 return; 434 435 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 436 ip->num, DEFAULT_MAX_CATEGORIES); 437 438 j = 0; 439 for (i = 0; i < ip->num; i++) { 440 m = ip->pkts[i]; 441 res = ip->res[i]; 442 if (res == BYPASS) { 443 ip->pkts[j++] = m; 444 continue; 445 } 446 if (res == DISCARD) { 447 rte_pktmbuf_free(m); 448 continue; 449 } 450 451 /* Only check SPI match for processed IPSec packets */ 452 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) { 453 rte_pktmbuf_free(m); 454 continue; 455 } 456 457 sa_idx = SPI2IDX(res); 458 if (!inbound_sa_check(sa, m, sa_idx)) { 459 rte_pktmbuf_free(m); 460 continue; 461 } 462 ip->pkts[j++] = m; 463 } 464 ip->num = j; 465 } 466 467 static void 468 split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) 469 { 470 uint32_t i, n4, n6; 471 struct ip *ip; 472 struct rte_mbuf *m; 473 474 n4 = trf->ip4.num; 475 n6 = trf->ip6.num; 476 477 for (i = 0; i < num; i++) { 478 479 m = mb[i]; 480 ip = rte_pktmbuf_mtod(m, struct ip *); 481 482 if (ip->ip_v == IPVERSION) { 483 trf->ip4.pkts[n4] = m; 484 trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m, 485 uint8_t *, offsetof(struct ip, ip_p)); 486 n4++; 487 } else if (ip->ip_v == IP6_VERSION) { 488 trf->ip6.pkts[n6] = m; 489 trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m, 490 uint8_t *, 491 offsetof(struct ip6_hdr, ip6_nxt)); 492 n6++; 493 } else 494 rte_pktmbuf_free(m); 495 } 496 497 trf->ip4.num = n4; 498 trf->ip6.num = n6; 499 } 500 501 502 static inline void 503 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 504 struct ipsec_traffic *traffic) 505 { 506 uint16_t nb_pkts_in, n_ip4, n_ip6; 507 508 n_ip4 = traffic->ip4.num; 509 n_ip6 = traffic->ip6.num; 510 511 if (app_sa_prm.enable == 0) { 512 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 513 traffic->ipsec.num, MAX_PKT_BURST); 514 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in); 515 } else { 516 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 517 traffic->ipsec.saptr, traffic->ipsec.num); 518 ipsec_process(ipsec_ctx, traffic); 519 } 520 521 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 522 n_ip4); 523 524 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 525 n_ip6); 526 } 527 528 static inline void 529 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 530 struct traffic_type *ipsec) 531 { 532 struct rte_mbuf *m; 533 uint32_t i, j, sa_idx; 534 535 if (ip->num == 0 || sp == NULL) 536 return; 537 538 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 539 ip->num, DEFAULT_MAX_CATEGORIES); 540 541 j = 0; 542 for (i = 0; i < ip->num; i++) { 543 m = ip->pkts[i]; 544 sa_idx = SPI2IDX(ip->res[i]); 545 if (ip->res[i] == DISCARD) 546 rte_pktmbuf_free(m); 547 else if (ip->res[i] == BYPASS) 548 ip->pkts[j++] = m; 549 else { 550 ipsec->res[ipsec->num] = sa_idx; 551 ipsec->pkts[ipsec->num++] = m; 552 } 553 } 554 ip->num = j; 555 } 556 557 static inline void 558 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 559 struct ipsec_traffic *traffic) 560 { 561 struct rte_mbuf *m; 562 uint16_t idx, nb_pkts_out, i; 563 564 /* Drop any IPsec traffic from protected ports */ 565 for (i = 0; i < traffic->ipsec.num; i++) 566 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 567 568 traffic->ipsec.num = 0; 569 570 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 571 572 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 573 574 if (app_sa_prm.enable == 0) { 575 576 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 577 traffic->ipsec.res, traffic->ipsec.num, 578 MAX_PKT_BURST); 579 580 for (i = 0; i < nb_pkts_out; i++) { 581 m = traffic->ipsec.pkts[i]; 582 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 583 if (ip->ip_v == IPVERSION) { 584 idx = traffic->ip4.num++; 585 traffic->ip4.pkts[idx] = m; 586 } else { 587 idx = traffic->ip6.num++; 588 traffic->ip6.pkts[idx] = m; 589 } 590 } 591 } else { 592 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 593 traffic->ipsec.saptr, traffic->ipsec.num); 594 ipsec_process(ipsec_ctx, traffic); 595 } 596 } 597 598 static inline void 599 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 600 struct ipsec_traffic *traffic) 601 { 602 struct rte_mbuf *m; 603 uint32_t nb_pkts_in, i, idx; 604 605 /* Drop any IPv4 traffic from unprotected ports */ 606 for (i = 0; i < traffic->ip4.num; i++) 607 rte_pktmbuf_free(traffic->ip4.pkts[i]); 608 609 traffic->ip4.num = 0; 610 611 /* Drop any IPv6 traffic from unprotected ports */ 612 for (i = 0; i < traffic->ip6.num; i++) 613 rte_pktmbuf_free(traffic->ip6.pkts[i]); 614 615 traffic->ip6.num = 0; 616 617 if (app_sa_prm.enable == 0) { 618 619 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 620 traffic->ipsec.num, MAX_PKT_BURST); 621 622 for (i = 0; i < nb_pkts_in; i++) { 623 m = traffic->ipsec.pkts[i]; 624 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 625 if (ip->ip_v == IPVERSION) { 626 idx = traffic->ip4.num++; 627 traffic->ip4.pkts[idx] = m; 628 } else { 629 idx = traffic->ip6.num++; 630 traffic->ip6.pkts[idx] = m; 631 } 632 } 633 } else { 634 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 635 traffic->ipsec.saptr, traffic->ipsec.num); 636 ipsec_process(ipsec_ctx, traffic); 637 } 638 } 639 640 static inline void 641 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 642 struct ipsec_traffic *traffic) 643 { 644 struct rte_mbuf *m; 645 uint32_t nb_pkts_out, i, n; 646 struct ip *ip; 647 648 /* Drop any IPsec traffic from protected ports */ 649 for (i = 0; i < traffic->ipsec.num; i++) 650 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 651 652 n = 0; 653 654 for (i = 0; i < traffic->ip4.num; i++) { 655 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i]; 656 traffic->ipsec.res[n++] = single_sa_idx; 657 } 658 659 for (i = 0; i < traffic->ip6.num; i++) { 660 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i]; 661 traffic->ipsec.res[n++] = single_sa_idx; 662 } 663 664 traffic->ip4.num = 0; 665 traffic->ip6.num = 0; 666 traffic->ipsec.num = n; 667 668 if (app_sa_prm.enable == 0) { 669 670 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 671 traffic->ipsec.res, traffic->ipsec.num, 672 MAX_PKT_BURST); 673 674 /* They all sue the same SA (ip4 or ip6 tunnel) */ 675 m = traffic->ipsec.pkts[0]; 676 ip = rte_pktmbuf_mtod(m, struct ip *); 677 if (ip->ip_v == IPVERSION) { 678 traffic->ip4.num = nb_pkts_out; 679 for (i = 0; i < nb_pkts_out; i++) 680 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i]; 681 } else { 682 traffic->ip6.num = nb_pkts_out; 683 for (i = 0; i < nb_pkts_out; i++) 684 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i]; 685 } 686 } else { 687 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 688 traffic->ipsec.saptr, traffic->ipsec.num); 689 ipsec_process(ipsec_ctx, traffic); 690 } 691 } 692 693 static inline int32_t 694 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) 695 { 696 struct ipsec_mbuf_metadata *priv; 697 struct ipsec_sa *sa; 698 699 priv = get_priv(pkt); 700 701 sa = priv->sa; 702 if (unlikely(sa == NULL)) { 703 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); 704 goto fail; 705 } 706 707 if (is_ipv6) 708 return sa->portid; 709 710 /* else */ 711 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); 712 713 fail: 714 if (is_ipv6) 715 return -1; 716 717 /* else */ 718 return 0; 719 } 720 721 static inline void 722 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 723 { 724 uint32_t hop[MAX_PKT_BURST * 2]; 725 uint32_t dst_ip[MAX_PKT_BURST * 2]; 726 int32_t pkt_hop = 0; 727 uint16_t i, offset; 728 uint16_t lpm_pkts = 0; 729 730 if (nb_pkts == 0) 731 return; 732 733 /* Need to do an LPM lookup for non-inline packets. Inline packets will 734 * have port ID in the SA 735 */ 736 737 for (i = 0; i < nb_pkts; i++) { 738 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 739 /* Security offload not enabled. So an LPM lookup is 740 * required to get the hop 741 */ 742 offset = offsetof(struct ip, ip_dst); 743 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], 744 uint32_t *, offset); 745 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); 746 lpm_pkts++; 747 } 748 } 749 750 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); 751 752 lpm_pkts = 0; 753 754 for (i = 0; i < nb_pkts; i++) { 755 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 756 /* Read hop from the SA */ 757 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); 758 } else { 759 /* Need to use hop returned by lookup */ 760 pkt_hop = hop[lpm_pkts++]; 761 } 762 763 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { 764 rte_pktmbuf_free(pkts[i]); 765 continue; 766 } 767 send_single_packet(pkts[i], pkt_hop & 0xff); 768 } 769 } 770 771 static inline void 772 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 773 { 774 int32_t hop[MAX_PKT_BURST * 2]; 775 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 776 uint8_t *ip6_dst; 777 int32_t pkt_hop = 0; 778 uint16_t i, offset; 779 uint16_t lpm_pkts = 0; 780 781 if (nb_pkts == 0) 782 return; 783 784 /* Need to do an LPM lookup for non-inline packets. Inline packets will 785 * have port ID in the SA 786 */ 787 788 for (i = 0; i < nb_pkts; i++) { 789 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 790 /* Security offload not enabled. So an LPM lookup is 791 * required to get the hop 792 */ 793 offset = offsetof(struct ip6_hdr, ip6_dst); 794 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, 795 offset); 796 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); 797 lpm_pkts++; 798 } 799 } 800 801 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, 802 lpm_pkts); 803 804 lpm_pkts = 0; 805 806 for (i = 0; i < nb_pkts; i++) { 807 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 808 /* Read hop from the SA */ 809 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); 810 } else { 811 /* Need to use hop returned by lookup */ 812 pkt_hop = hop[lpm_pkts++]; 813 } 814 815 if (pkt_hop == -1) { 816 rte_pktmbuf_free(pkts[i]); 817 continue; 818 } 819 send_single_packet(pkts[i], pkt_hop & 0xff); 820 } 821 } 822 823 static inline void 824 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 825 uint8_t nb_pkts, uint16_t portid) 826 { 827 struct ipsec_traffic traffic; 828 829 prepare_traffic(pkts, &traffic, nb_pkts); 830 831 if (unlikely(single_sa)) { 832 if (UNPROTECTED_PORT(portid)) 833 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 834 else 835 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 836 } else { 837 if (UNPROTECTED_PORT(portid)) 838 process_pkts_inbound(&qconf->inbound, &traffic); 839 else 840 process_pkts_outbound(&qconf->outbound, &traffic); 841 } 842 843 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 844 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 845 } 846 847 static inline void 848 drain_tx_buffers(struct lcore_conf *qconf) 849 { 850 struct buffer *buf; 851 uint32_t portid; 852 853 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 854 buf = &qconf->tx_mbufs[portid]; 855 if (buf->len == 0) 856 continue; 857 send_burst(qconf, buf->len, portid); 858 buf->len = 0; 859 } 860 } 861 862 static inline void 863 drain_crypto_buffers(struct lcore_conf *qconf) 864 { 865 uint32_t i; 866 struct ipsec_ctx *ctx; 867 868 /* drain inbound buffers*/ 869 ctx = &qconf->inbound; 870 for (i = 0; i != ctx->nb_qps; i++) { 871 if (ctx->tbl[i].len != 0) 872 enqueue_cop_burst(ctx->tbl + i); 873 } 874 875 /* drain outbound buffers*/ 876 ctx = &qconf->outbound; 877 for (i = 0; i != ctx->nb_qps; i++) { 878 if (ctx->tbl[i].len != 0) 879 enqueue_cop_burst(ctx->tbl + i); 880 } 881 } 882 883 static void 884 drain_inbound_crypto_queues(const struct lcore_conf *qconf, 885 struct ipsec_ctx *ctx) 886 { 887 uint32_t n; 888 struct ipsec_traffic trf; 889 890 if (app_sa_prm.enable == 0) { 891 892 /* dequeue packets from crypto-queue */ 893 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts, 894 RTE_DIM(trf.ipsec.pkts)); 895 896 trf.ip4.num = 0; 897 trf.ip6.num = 0; 898 899 /* split traffic by ipv4-ipv6 */ 900 split46_traffic(&trf, trf.ipsec.pkts, n); 901 } else 902 ipsec_cqp_process(ctx, &trf); 903 904 /* process ipv4 packets */ 905 if (trf.ip4.num != 0) { 906 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0); 907 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 908 } 909 910 /* process ipv6 packets */ 911 if (trf.ip6.num != 0) { 912 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0); 913 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 914 } 915 } 916 917 static void 918 drain_outbound_crypto_queues(const struct lcore_conf *qconf, 919 struct ipsec_ctx *ctx) 920 { 921 uint32_t n; 922 struct ipsec_traffic trf; 923 924 if (app_sa_prm.enable == 0) { 925 926 /* dequeue packets from crypto-queue */ 927 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts, 928 RTE_DIM(trf.ipsec.pkts)); 929 930 trf.ip4.num = 0; 931 trf.ip6.num = 0; 932 933 /* split traffic by ipv4-ipv6 */ 934 split46_traffic(&trf, trf.ipsec.pkts, n); 935 } else 936 ipsec_cqp_process(ctx, &trf); 937 938 /* process ipv4 packets */ 939 if (trf.ip4.num != 0) 940 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 941 942 /* process ipv6 packets */ 943 if (trf.ip6.num != 0) 944 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 945 } 946 947 /* main processing loop */ 948 static int32_t 949 main_loop(__attribute__((unused)) void *dummy) 950 { 951 struct rte_mbuf *pkts[MAX_PKT_BURST]; 952 uint32_t lcore_id; 953 uint64_t prev_tsc, diff_tsc, cur_tsc; 954 int32_t i, nb_rx; 955 uint16_t portid; 956 uint8_t queueid; 957 struct lcore_conf *qconf; 958 int32_t socket_id; 959 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 960 / US_PER_S * BURST_TX_DRAIN_US; 961 struct lcore_rx_queue *rxql; 962 963 prev_tsc = 0; 964 lcore_id = rte_lcore_id(); 965 qconf = &lcore_conf[lcore_id]; 966 rxql = qconf->rx_queue_list; 967 socket_id = rte_lcore_to_socket_id(lcore_id); 968 969 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 970 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 971 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 972 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 973 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 974 qconf->inbound.cdev_map = cdev_map_in; 975 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 976 qconf->inbound.session_priv_pool = 977 socket_ctx[socket_id].session_priv_pool; 978 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 979 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 980 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 981 qconf->outbound.cdev_map = cdev_map_out; 982 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 983 qconf->outbound.session_priv_pool = 984 socket_ctx[socket_id].session_priv_pool; 985 986 if (qconf->nb_rx_queue == 0) { 987 RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n", 988 lcore_id); 989 return 0; 990 } 991 992 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 993 994 for (i = 0; i < qconf->nb_rx_queue; i++) { 995 portid = rxql[i].port_id; 996 queueid = rxql[i].queue_id; 997 RTE_LOG(INFO, IPSEC, 998 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 999 lcore_id, portid, queueid); 1000 } 1001 1002 while (1) { 1003 cur_tsc = rte_rdtsc(); 1004 1005 /* TX queue buffer drain */ 1006 diff_tsc = cur_tsc - prev_tsc; 1007 1008 if (unlikely(diff_tsc > drain_tsc)) { 1009 drain_tx_buffers(qconf); 1010 drain_crypto_buffers(qconf); 1011 prev_tsc = cur_tsc; 1012 } 1013 1014 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1015 1016 /* Read packets from RX queues */ 1017 portid = rxql[i].port_id; 1018 queueid = rxql[i].queue_id; 1019 nb_rx = rte_eth_rx_burst(portid, queueid, 1020 pkts, MAX_PKT_BURST); 1021 1022 if (nb_rx > 0) 1023 process_pkts(qconf, pkts, nb_rx, portid); 1024 1025 /* dequeue and process completed crypto-ops */ 1026 if (UNPROTECTED_PORT(portid)) 1027 drain_inbound_crypto_queues(qconf, 1028 &qconf->inbound); 1029 else 1030 drain_outbound_crypto_queues(qconf, 1031 &qconf->outbound); 1032 } 1033 } 1034 } 1035 1036 static int32_t 1037 check_params(void) 1038 { 1039 uint8_t lcore; 1040 uint16_t portid; 1041 uint16_t i; 1042 int32_t socket_id; 1043 1044 if (lcore_params == NULL) { 1045 printf("Error: No port/queue/core mappings\n"); 1046 return -1; 1047 } 1048 1049 for (i = 0; i < nb_lcore_params; ++i) { 1050 lcore = lcore_params[i].lcore_id; 1051 if (!rte_lcore_is_enabled(lcore)) { 1052 printf("error: lcore %hhu is not enabled in " 1053 "lcore mask\n", lcore); 1054 return -1; 1055 } 1056 socket_id = rte_lcore_to_socket_id(lcore); 1057 if (socket_id != 0 && numa_on == 0) { 1058 printf("warning: lcore %hhu is on socket %d " 1059 "with numa off\n", 1060 lcore, socket_id); 1061 } 1062 portid = lcore_params[i].port_id; 1063 if ((enabled_port_mask & (1 << portid)) == 0) { 1064 printf("port %u is not enabled in port mask\n", portid); 1065 return -1; 1066 } 1067 if (!rte_eth_dev_is_valid_port(portid)) { 1068 printf("port %u is not present on the board\n", portid); 1069 return -1; 1070 } 1071 } 1072 return 0; 1073 } 1074 1075 static uint8_t 1076 get_port_nb_rx_queues(const uint16_t port) 1077 { 1078 int32_t queue = -1; 1079 uint16_t i; 1080 1081 for (i = 0; i < nb_lcore_params; ++i) { 1082 if (lcore_params[i].port_id == port && 1083 lcore_params[i].queue_id > queue) 1084 queue = lcore_params[i].queue_id; 1085 } 1086 return (uint8_t)(++queue); 1087 } 1088 1089 static int32_t 1090 init_lcore_rx_queues(void) 1091 { 1092 uint16_t i, nb_rx_queue; 1093 uint8_t lcore; 1094 1095 for (i = 0; i < nb_lcore_params; ++i) { 1096 lcore = lcore_params[i].lcore_id; 1097 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 1098 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1099 printf("error: too many queues (%u) for lcore: %u\n", 1100 nb_rx_queue + 1, lcore); 1101 return -1; 1102 } 1103 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1104 lcore_params[i].port_id; 1105 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1106 lcore_params[i].queue_id; 1107 lcore_conf[lcore].nb_rx_queue++; 1108 } 1109 return 0; 1110 } 1111 1112 /* display usage */ 1113 static void 1114 print_usage(const char *prgname) 1115 { 1116 fprintf(stderr, "%s [EAL options] --" 1117 " -p PORTMASK" 1118 " [-P]" 1119 " [-u PORTMASK]" 1120 " [-j FRAMESIZE]" 1121 " [-l]" 1122 " [-w REPLAY_WINDOW_SIZE]" 1123 " [-e]" 1124 " [-a]" 1125 " -f CONFIG_FILE" 1126 " --config (port,queue,lcore)[,(port,queue,lcore)]" 1127 " [--single-sa SAIDX]" 1128 " [--cryptodev_mask MASK]" 1129 " [--" CMD_LINE_OPT_RX_OFFLOAD " RX_OFFLOAD_MASK]" 1130 " [--" CMD_LINE_OPT_TX_OFFLOAD " TX_OFFLOAD_MASK]" 1131 "\n\n" 1132 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 1133 " -P : Enable promiscuous mode\n" 1134 " -u PORTMASK: Hexadecimal bitmask of unprotected ports\n" 1135 " -j FRAMESIZE: Enable jumbo frame with 'FRAMESIZE' as maximum\n" 1136 " packet size\n" 1137 " -l enables code-path that uses librte_ipsec\n" 1138 " -w REPLAY_WINDOW_SIZE specifies IPsec SQN replay window\n" 1139 " size for each SA\n" 1140 " -e enables ESN\n" 1141 " -a enables SA SQN atomic behaviour\n" 1142 " -f CONFIG_FILE: Configuration file\n" 1143 " --config (port,queue,lcore): Rx queue configuration\n" 1144 " --single-sa SAIDX: Use single SA index for outbound traffic,\n" 1145 " bypassing the SP\n" 1146 " --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n" 1147 " devices to configure\n" 1148 " --" CMD_LINE_OPT_RX_OFFLOAD 1149 ": bitmask of the RX HW offload capabilities to enable/use\n" 1150 " (DEV_RX_OFFLOAD_*)\n" 1151 " --" CMD_LINE_OPT_TX_OFFLOAD 1152 ": bitmask of the TX HW offload capabilities to enable/use\n" 1153 " (DEV_TX_OFFLOAD_*)\n" 1154 "\n", 1155 prgname); 1156 } 1157 1158 static int 1159 parse_mask(const char *str, uint64_t *val) 1160 { 1161 char *end; 1162 unsigned long t; 1163 1164 errno = 0; 1165 t = strtoul(str, &end, 0); 1166 if (errno != 0 || end[0] != 0) 1167 return -EINVAL; 1168 1169 *val = t; 1170 return 0; 1171 } 1172 1173 static int32_t 1174 parse_portmask(const char *portmask) 1175 { 1176 char *end = NULL; 1177 unsigned long pm; 1178 1179 /* parse hexadecimal string */ 1180 pm = strtoul(portmask, &end, 16); 1181 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1182 return -1; 1183 1184 if ((pm == 0) && errno) 1185 return -1; 1186 1187 return pm; 1188 } 1189 1190 static int32_t 1191 parse_decimal(const char *str) 1192 { 1193 char *end = NULL; 1194 unsigned long num; 1195 1196 num = strtoul(str, &end, 10); 1197 if ((str[0] == '\0') || (end == NULL) || (*end != '\0')) 1198 return -1; 1199 1200 return num; 1201 } 1202 1203 static int32_t 1204 parse_config(const char *q_arg) 1205 { 1206 char s[256]; 1207 const char *p, *p0 = q_arg; 1208 char *end; 1209 enum fieldnames { 1210 FLD_PORT = 0, 1211 FLD_QUEUE, 1212 FLD_LCORE, 1213 _NUM_FLD 1214 }; 1215 unsigned long int_fld[_NUM_FLD]; 1216 char *str_fld[_NUM_FLD]; 1217 int32_t i; 1218 uint32_t size; 1219 1220 nb_lcore_params = 0; 1221 1222 while ((p = strchr(p0, '(')) != NULL) { 1223 ++p; 1224 p0 = strchr(p, ')'); 1225 if (p0 == NULL) 1226 return -1; 1227 1228 size = p0 - p; 1229 if (size >= sizeof(s)) 1230 return -1; 1231 1232 snprintf(s, sizeof(s), "%.*s", size, p); 1233 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1234 _NUM_FLD) 1235 return -1; 1236 for (i = 0; i < _NUM_FLD; i++) { 1237 errno = 0; 1238 int_fld[i] = strtoul(str_fld[i], &end, 0); 1239 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1240 return -1; 1241 } 1242 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1243 printf("exceeded max number of lcore params: %hu\n", 1244 nb_lcore_params); 1245 return -1; 1246 } 1247 lcore_params_array[nb_lcore_params].port_id = 1248 (uint8_t)int_fld[FLD_PORT]; 1249 lcore_params_array[nb_lcore_params].queue_id = 1250 (uint8_t)int_fld[FLD_QUEUE]; 1251 lcore_params_array[nb_lcore_params].lcore_id = 1252 (uint8_t)int_fld[FLD_LCORE]; 1253 ++nb_lcore_params; 1254 } 1255 lcore_params = lcore_params_array; 1256 return 0; 1257 } 1258 1259 static void 1260 print_app_sa_prm(const struct app_sa_prm *prm) 1261 { 1262 printf("librte_ipsec usage: %s\n", 1263 (prm->enable == 0) ? "disabled" : "enabled"); 1264 1265 if (prm->enable == 0) 1266 return; 1267 1268 printf("replay window size: %u\n", prm->window_size); 1269 printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled"); 1270 printf("SA flags: %#" PRIx64 "\n", prm->flags); 1271 } 1272 1273 static int32_t 1274 parse_args(int32_t argc, char **argv) 1275 { 1276 int32_t opt, ret; 1277 char **argvopt; 1278 int32_t option_index; 1279 char *prgname = argv[0]; 1280 int32_t f_present = 0; 1281 1282 argvopt = argv; 1283 1284 while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:", 1285 lgopts, &option_index)) != EOF) { 1286 1287 switch (opt) { 1288 case 'p': 1289 enabled_port_mask = parse_portmask(optarg); 1290 if (enabled_port_mask == 0) { 1291 printf("invalid portmask\n"); 1292 print_usage(prgname); 1293 return -1; 1294 } 1295 break; 1296 case 'P': 1297 printf("Promiscuous mode selected\n"); 1298 promiscuous_on = 1; 1299 break; 1300 case 'u': 1301 unprotected_port_mask = parse_portmask(optarg); 1302 if (unprotected_port_mask == 0) { 1303 printf("invalid unprotected portmask\n"); 1304 print_usage(prgname); 1305 return -1; 1306 } 1307 break; 1308 case 'f': 1309 if (f_present == 1) { 1310 printf("\"-f\" option present more than " 1311 "once!\n"); 1312 print_usage(prgname); 1313 return -1; 1314 } 1315 if (parse_cfg_file(optarg) < 0) { 1316 printf("parsing file \"%s\" failed\n", 1317 optarg); 1318 print_usage(prgname); 1319 return -1; 1320 } 1321 f_present = 1; 1322 break; 1323 case 'j': 1324 { 1325 int32_t size = parse_decimal(optarg); 1326 if (size <= 1518) { 1327 printf("Invalid jumbo frame size\n"); 1328 if (size < 0) { 1329 print_usage(prgname); 1330 return -1; 1331 } 1332 printf("Using default value 9000\n"); 1333 frame_size = 9000; 1334 } else { 1335 frame_size = size; 1336 } 1337 } 1338 printf("Enabled jumbo frames size %u\n", frame_size); 1339 break; 1340 case 'l': 1341 app_sa_prm.enable = 1; 1342 break; 1343 case 'w': 1344 app_sa_prm.enable = 1; 1345 app_sa_prm.window_size = parse_decimal(optarg); 1346 break; 1347 case 'e': 1348 app_sa_prm.enable = 1; 1349 app_sa_prm.enable_esn = 1; 1350 break; 1351 case 'a': 1352 app_sa_prm.enable = 1; 1353 app_sa_prm.flags |= RTE_IPSEC_SAFLAG_SQN_ATOM; 1354 break; 1355 case CMD_LINE_OPT_CONFIG_NUM: 1356 ret = parse_config(optarg); 1357 if (ret) { 1358 printf("Invalid config\n"); 1359 print_usage(prgname); 1360 return -1; 1361 } 1362 break; 1363 case CMD_LINE_OPT_SINGLE_SA_NUM: 1364 ret = parse_decimal(optarg); 1365 if (ret == -1) { 1366 printf("Invalid argument[sa_idx]\n"); 1367 print_usage(prgname); 1368 return -1; 1369 } 1370 1371 /* else */ 1372 single_sa = 1; 1373 single_sa_idx = ret; 1374 printf("Configured with single SA index %u\n", 1375 single_sa_idx); 1376 break; 1377 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM: 1378 ret = parse_portmask(optarg); 1379 if (ret == -1) { 1380 printf("Invalid argument[portmask]\n"); 1381 print_usage(prgname); 1382 return -1; 1383 } 1384 1385 /* else */ 1386 enabled_cryptodev_mask = ret; 1387 break; 1388 case CMD_LINE_OPT_RX_OFFLOAD_NUM: 1389 ret = parse_mask(optarg, &dev_rx_offload); 1390 if (ret != 0) { 1391 printf("Invalid argument for \'%s\': %s\n", 1392 CMD_LINE_OPT_RX_OFFLOAD, optarg); 1393 print_usage(prgname); 1394 return -1; 1395 } 1396 break; 1397 case CMD_LINE_OPT_TX_OFFLOAD_NUM: 1398 ret = parse_mask(optarg, &dev_tx_offload); 1399 if (ret != 0) { 1400 printf("Invalid argument for \'%s\': %s\n", 1401 CMD_LINE_OPT_TX_OFFLOAD, optarg); 1402 print_usage(prgname); 1403 return -1; 1404 } 1405 break; 1406 default: 1407 print_usage(prgname); 1408 return -1; 1409 } 1410 } 1411 1412 if (f_present == 0) { 1413 printf("Mandatory option \"-f\" not present\n"); 1414 return -1; 1415 } 1416 1417 print_app_sa_prm(&app_sa_prm); 1418 1419 if (optind >= 0) 1420 argv[optind-1] = prgname; 1421 1422 ret = optind-1; 1423 optind = 1; /* reset getopt lib */ 1424 return ret; 1425 } 1426 1427 static void 1428 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 1429 { 1430 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 1431 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 1432 printf("%s%s", name, buf); 1433 } 1434 1435 /* 1436 * Update destination ethaddr for the port. 1437 */ 1438 int 1439 add_dst_ethaddr(uint16_t port, const struct rte_ether_addr *addr) 1440 { 1441 if (port >= RTE_DIM(ethaddr_tbl)) 1442 return -EINVAL; 1443 1444 ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr); 1445 return 0; 1446 } 1447 1448 /* Check the link status of all ports in up to 9s, and print them finally */ 1449 static void 1450 check_all_ports_link_status(uint32_t port_mask) 1451 { 1452 #define CHECK_INTERVAL 100 /* 100ms */ 1453 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1454 uint16_t portid; 1455 uint8_t count, all_ports_up, print_flag = 0; 1456 struct rte_eth_link link; 1457 1458 printf("\nChecking link status"); 1459 fflush(stdout); 1460 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1461 all_ports_up = 1; 1462 RTE_ETH_FOREACH_DEV(portid) { 1463 if ((port_mask & (1 << portid)) == 0) 1464 continue; 1465 memset(&link, 0, sizeof(link)); 1466 rte_eth_link_get_nowait(portid, &link); 1467 /* print link status if flag set */ 1468 if (print_flag == 1) { 1469 if (link.link_status) 1470 printf( 1471 "Port%d Link Up - speed %u Mbps -%s\n", 1472 portid, link.link_speed, 1473 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1474 ("full-duplex") : ("half-duplex\n")); 1475 else 1476 printf("Port %d Link Down\n", portid); 1477 continue; 1478 } 1479 /* clear all_ports_up flag if any link down */ 1480 if (link.link_status == ETH_LINK_DOWN) { 1481 all_ports_up = 0; 1482 break; 1483 } 1484 } 1485 /* after finally printing all link status, get out */ 1486 if (print_flag == 1) 1487 break; 1488 1489 if (all_ports_up == 0) { 1490 printf("."); 1491 fflush(stdout); 1492 rte_delay_ms(CHECK_INTERVAL); 1493 } 1494 1495 /* set the print_flag if all ports up or timeout */ 1496 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1497 print_flag = 1; 1498 printf("done\n"); 1499 } 1500 } 1501 } 1502 1503 static int32_t 1504 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1505 uint16_t qp, struct lcore_params *params, 1506 struct ipsec_ctx *ipsec_ctx, 1507 const struct rte_cryptodev_capabilities *cipher, 1508 const struct rte_cryptodev_capabilities *auth, 1509 const struct rte_cryptodev_capabilities *aead) 1510 { 1511 int32_t ret = 0; 1512 unsigned long i; 1513 struct cdev_key key = { 0 }; 1514 1515 key.lcore_id = params->lcore_id; 1516 if (cipher) 1517 key.cipher_algo = cipher->sym.cipher.algo; 1518 if (auth) 1519 key.auth_algo = auth->sym.auth.algo; 1520 if (aead) 1521 key.aead_algo = aead->sym.aead.algo; 1522 1523 ret = rte_hash_lookup(map, &key); 1524 if (ret != -ENOENT) 1525 return 0; 1526 1527 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1528 if (ipsec_ctx->tbl[i].id == cdev_id) 1529 break; 1530 1531 if (i == ipsec_ctx->nb_qps) { 1532 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1533 printf("Maximum number of crypto devices assigned to " 1534 "a core, increase MAX_QP_PER_LCORE value\n"); 1535 return 0; 1536 } 1537 ipsec_ctx->tbl[i].id = cdev_id; 1538 ipsec_ctx->tbl[i].qp = qp; 1539 ipsec_ctx->nb_qps++; 1540 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1541 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1542 cdev_id, qp, i); 1543 } 1544 1545 ret = rte_hash_add_key_data(map, &key, (void *)i); 1546 if (ret < 0) { 1547 printf("Faled to insert cdev mapping for (lcore %u, " 1548 "cdev %u, qp %u), errno %d\n", 1549 key.lcore_id, ipsec_ctx->tbl[i].id, 1550 ipsec_ctx->tbl[i].qp, ret); 1551 return 0; 1552 } 1553 1554 return 1; 1555 } 1556 1557 static int32_t 1558 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1559 uint16_t qp, struct lcore_params *params) 1560 { 1561 int32_t ret = 0; 1562 const struct rte_cryptodev_capabilities *i, *j; 1563 struct rte_hash *map; 1564 struct lcore_conf *qconf; 1565 struct ipsec_ctx *ipsec_ctx; 1566 const char *str; 1567 1568 qconf = &lcore_conf[params->lcore_id]; 1569 1570 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 1571 map = cdev_map_out; 1572 ipsec_ctx = &qconf->outbound; 1573 str = "Outbound"; 1574 } else { 1575 map = cdev_map_in; 1576 ipsec_ctx = &qconf->inbound; 1577 str = "Inbound"; 1578 } 1579 1580 /* Required cryptodevs with operation chainning */ 1581 if (!(dev_info->feature_flags & 1582 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 1583 return ret; 1584 1585 for (i = dev_info->capabilities; 1586 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 1587 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1588 continue; 1589 1590 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1591 ret |= add_mapping(map, str, cdev_id, qp, params, 1592 ipsec_ctx, NULL, NULL, i); 1593 continue; 1594 } 1595 1596 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 1597 continue; 1598 1599 for (j = dev_info->capabilities; 1600 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 1601 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1602 continue; 1603 1604 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 1605 continue; 1606 1607 ret |= add_mapping(map, str, cdev_id, qp, params, 1608 ipsec_ctx, i, j, NULL); 1609 } 1610 } 1611 1612 return ret; 1613 } 1614 1615 /* Check if the device is enabled by cryptodev_mask */ 1616 static int 1617 check_cryptodev_mask(uint8_t cdev_id) 1618 { 1619 if (enabled_cryptodev_mask & (1 << cdev_id)) 1620 return 0; 1621 1622 return -1; 1623 } 1624 1625 static int32_t 1626 cryptodevs_init(void) 1627 { 1628 struct rte_cryptodev_config dev_conf; 1629 struct rte_cryptodev_qp_conf qp_conf; 1630 uint16_t idx, max_nb_qps, qp, i; 1631 int16_t cdev_id, port_id; 1632 struct rte_hash_parameters params = { 0 }; 1633 1634 params.entries = CDEV_MAP_ENTRIES; 1635 params.key_len = sizeof(struct cdev_key); 1636 params.hash_func = rte_jhash; 1637 params.hash_func_init_val = 0; 1638 params.socket_id = rte_socket_id(); 1639 1640 params.name = "cdev_map_in"; 1641 cdev_map_in = rte_hash_create(¶ms); 1642 if (cdev_map_in == NULL) 1643 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1644 rte_errno); 1645 1646 params.name = "cdev_map_out"; 1647 cdev_map_out = rte_hash_create(¶ms); 1648 if (cdev_map_out == NULL) 1649 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1650 rte_errno); 1651 1652 printf("lcore/cryptodev/qp mappings:\n"); 1653 1654 uint32_t max_sess_sz = 0, sess_sz; 1655 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1656 void *sec_ctx; 1657 1658 /* Get crypto priv session size */ 1659 sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 1660 if (sess_sz > max_sess_sz) 1661 max_sess_sz = sess_sz; 1662 1663 /* 1664 * If crypto device is security capable, need to check the 1665 * size of security session as well. 1666 */ 1667 1668 /* Get security context of the crypto device */ 1669 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id); 1670 if (sec_ctx == NULL) 1671 continue; 1672 1673 /* Get size of security session */ 1674 sess_sz = rte_security_session_get_size(sec_ctx); 1675 if (sess_sz > max_sess_sz) 1676 max_sess_sz = sess_sz; 1677 } 1678 RTE_ETH_FOREACH_DEV(port_id) { 1679 void *sec_ctx; 1680 1681 if ((enabled_port_mask & (1 << port_id)) == 0) 1682 continue; 1683 1684 sec_ctx = rte_eth_dev_get_sec_ctx(port_id); 1685 if (sec_ctx == NULL) 1686 continue; 1687 1688 sess_sz = rte_security_session_get_size(sec_ctx); 1689 if (sess_sz > max_sess_sz) 1690 max_sess_sz = sess_sz; 1691 } 1692 1693 idx = 0; 1694 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1695 struct rte_cryptodev_info cdev_info; 1696 1697 if (check_cryptodev_mask((uint8_t)cdev_id)) 1698 continue; 1699 1700 rte_cryptodev_info_get(cdev_id, &cdev_info); 1701 1702 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 1703 max_nb_qps = cdev_info.max_nb_queue_pairs; 1704 else 1705 max_nb_qps = nb_lcore_params; 1706 1707 qp = 0; 1708 i = 0; 1709 while (qp < max_nb_qps && i < nb_lcore_params) { 1710 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 1711 &lcore_params[idx])) 1712 qp++; 1713 idx++; 1714 idx = idx % nb_lcore_params; 1715 i++; 1716 } 1717 1718 if (qp == 0) 1719 continue; 1720 1721 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 1722 dev_conf.nb_queue_pairs = qp; 1723 1724 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions; 1725 if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS) 1726 rte_exit(EXIT_FAILURE, 1727 "Device does not support at least %u " 1728 "sessions", CDEV_MP_NB_OBJS); 1729 1730 if (!socket_ctx[dev_conf.socket_id].session_pool) { 1731 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1732 struct rte_mempool *sess_mp; 1733 1734 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1735 "sess_mp_%u", dev_conf.socket_id); 1736 sess_mp = rte_cryptodev_sym_session_pool_create( 1737 mp_name, CDEV_MP_NB_OBJS, 1738 0, CDEV_MP_CACHE_SZ, 0, 1739 dev_conf.socket_id); 1740 socket_ctx[dev_conf.socket_id].session_pool = sess_mp; 1741 } 1742 1743 if (!socket_ctx[dev_conf.socket_id].session_priv_pool) { 1744 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1745 struct rte_mempool *sess_mp; 1746 1747 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1748 "sess_mp_priv_%u", dev_conf.socket_id); 1749 sess_mp = rte_mempool_create(mp_name, 1750 CDEV_MP_NB_OBJS, 1751 max_sess_sz, 1752 CDEV_MP_CACHE_SZ, 1753 0, NULL, NULL, NULL, 1754 NULL, dev_conf.socket_id, 1755 0); 1756 socket_ctx[dev_conf.socket_id].session_priv_pool = 1757 sess_mp; 1758 } 1759 1760 if (!socket_ctx[dev_conf.socket_id].session_priv_pool || 1761 !socket_ctx[dev_conf.socket_id].session_pool) 1762 rte_exit(EXIT_FAILURE, 1763 "Cannot create session pool on socket %d\n", 1764 dev_conf.socket_id); 1765 else 1766 printf("Allocated session pool on socket %d\n", 1767 dev_conf.socket_id); 1768 1769 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 1770 rte_panic("Failed to initialize cryptodev %u\n", 1771 cdev_id); 1772 1773 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 1774 qp_conf.mp_session = 1775 socket_ctx[dev_conf.socket_id].session_pool; 1776 qp_conf.mp_session_private = 1777 socket_ctx[dev_conf.socket_id].session_priv_pool; 1778 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 1779 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 1780 &qp_conf, dev_conf.socket_id)) 1781 rte_panic("Failed to setup queue %u for " 1782 "cdev_id %u\n", 0, cdev_id); 1783 1784 if (rte_cryptodev_start(cdev_id)) 1785 rte_panic("Failed to start cryptodev %u\n", 1786 cdev_id); 1787 } 1788 1789 /* create session pools for eth devices that implement security */ 1790 RTE_ETH_FOREACH_DEV(port_id) { 1791 if ((enabled_port_mask & (1 << port_id)) && 1792 rte_eth_dev_get_sec_ctx(port_id)) { 1793 int socket_id = rte_eth_dev_socket_id(port_id); 1794 1795 if (!socket_ctx[socket_id].session_priv_pool) { 1796 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1797 struct rte_mempool *sess_mp; 1798 1799 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1800 "sess_mp_%u", socket_id); 1801 sess_mp = rte_mempool_create(mp_name, 1802 (CDEV_MP_NB_OBJS * 2), 1803 max_sess_sz, 1804 CDEV_MP_CACHE_SZ, 1805 0, NULL, NULL, NULL, 1806 NULL, socket_id, 1807 0); 1808 if (sess_mp == NULL) 1809 rte_exit(EXIT_FAILURE, 1810 "Cannot create session pool " 1811 "on socket %d\n", socket_id); 1812 else 1813 printf("Allocated session pool " 1814 "on socket %d\n", socket_id); 1815 socket_ctx[socket_id].session_priv_pool = 1816 sess_mp; 1817 } 1818 } 1819 } 1820 1821 1822 printf("\n"); 1823 1824 return 0; 1825 } 1826 1827 static void 1828 port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads) 1829 { 1830 struct rte_eth_dev_info dev_info; 1831 struct rte_eth_txconf *txconf; 1832 uint16_t nb_tx_queue, nb_rx_queue; 1833 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 1834 int32_t ret, socket_id; 1835 struct lcore_conf *qconf; 1836 struct rte_ether_addr ethaddr; 1837 struct rte_eth_conf local_port_conf = port_conf; 1838 1839 rte_eth_dev_info_get(portid, &dev_info); 1840 1841 /* limit allowed HW offloafs, as user requested */ 1842 dev_info.rx_offload_capa &= dev_rx_offload; 1843 dev_info.tx_offload_capa &= dev_tx_offload; 1844 1845 printf("Configuring device port %u:\n", portid); 1846 1847 rte_eth_macaddr_get(portid, ðaddr); 1848 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ðaddr); 1849 print_ethaddr("Address: ", ðaddr); 1850 printf("\n"); 1851 1852 nb_rx_queue = get_port_nb_rx_queues(portid); 1853 nb_tx_queue = nb_lcores; 1854 1855 if (nb_rx_queue > dev_info.max_rx_queues) 1856 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1857 "(max rx queue is %u)\n", 1858 nb_rx_queue, dev_info.max_rx_queues); 1859 1860 if (nb_tx_queue > dev_info.max_tx_queues) 1861 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1862 "(max tx queue is %u)\n", 1863 nb_tx_queue, dev_info.max_tx_queues); 1864 1865 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 1866 nb_rx_queue, nb_tx_queue); 1867 1868 if (frame_size) { 1869 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 1870 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1871 } 1872 1873 local_port_conf.rxmode.offloads |= req_rx_offloads; 1874 local_port_conf.txmode.offloads |= req_tx_offloads; 1875 1876 /* Check that all required capabilities are supported */ 1877 if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) != 1878 local_port_conf.rxmode.offloads) 1879 rte_exit(EXIT_FAILURE, 1880 "Error: port %u required RX offloads: 0x%" PRIx64 1881 ", avaialbe RX offloads: 0x%" PRIx64 "\n", 1882 portid, local_port_conf.rxmode.offloads, 1883 dev_info.rx_offload_capa); 1884 1885 if ((local_port_conf.txmode.offloads & dev_info.tx_offload_capa) != 1886 local_port_conf.txmode.offloads) 1887 rte_exit(EXIT_FAILURE, 1888 "Error: port %u required TX offloads: 0x%" PRIx64 1889 ", avaialbe TX offloads: 0x%" PRIx64 "\n", 1890 portid, local_port_conf.txmode.offloads, 1891 dev_info.tx_offload_capa); 1892 1893 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 1894 local_port_conf.txmode.offloads |= 1895 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1896 1897 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 1898 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 1899 1900 printf("port %u configurng rx_offloads=0x%" PRIx64 1901 ", tx_offloads=0x%" PRIx64 "\n", 1902 portid, local_port_conf.rxmode.offloads, 1903 local_port_conf.txmode.offloads); 1904 1905 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 1906 dev_info.flow_type_rss_offloads; 1907 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 1908 port_conf.rx_adv_conf.rss_conf.rss_hf) { 1909 printf("Port %u modified RSS hash function based on hardware support," 1910 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 1911 portid, 1912 port_conf.rx_adv_conf.rss_conf.rss_hf, 1913 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 1914 } 1915 1916 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 1917 &local_port_conf); 1918 if (ret < 0) 1919 rte_exit(EXIT_FAILURE, "Cannot configure device: " 1920 "err=%d, port=%d\n", ret, portid); 1921 1922 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 1923 if (ret < 0) 1924 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 1925 "err=%d, port=%d\n", ret, portid); 1926 1927 /* init one TX queue per lcore */ 1928 tx_queueid = 0; 1929 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1930 if (rte_lcore_is_enabled(lcore_id) == 0) 1931 continue; 1932 1933 if (numa_on) 1934 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1935 else 1936 socket_id = 0; 1937 1938 /* init TX queue */ 1939 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 1940 1941 txconf = &dev_info.default_txconf; 1942 txconf->offloads = local_port_conf.txmode.offloads; 1943 1944 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 1945 socket_id, txconf); 1946 if (ret < 0) 1947 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 1948 "err=%d, port=%d\n", ret, portid); 1949 1950 qconf = &lcore_conf[lcore_id]; 1951 qconf->tx_queue_id[portid] = tx_queueid; 1952 1953 /* Pre-populate pkt offloads based on capabilities */ 1954 qconf->outbound.ipv4_offloads = PKT_TX_IPV4; 1955 qconf->outbound.ipv6_offloads = PKT_TX_IPV6; 1956 if (local_port_conf.txmode.offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 1957 qconf->outbound.ipv4_offloads |= PKT_TX_IP_CKSUM; 1958 1959 tx_queueid++; 1960 1961 /* init RX queues */ 1962 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 1963 struct rte_eth_rxconf rxq_conf; 1964 1965 if (portid != qconf->rx_queue_list[queue].port_id) 1966 continue; 1967 1968 rx_queueid = qconf->rx_queue_list[queue].queue_id; 1969 1970 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 1971 socket_id); 1972 1973 rxq_conf = dev_info.default_rxconf; 1974 rxq_conf.offloads = local_port_conf.rxmode.offloads; 1975 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 1976 nb_rxd, socket_id, &rxq_conf, 1977 socket_ctx[socket_id].mbuf_pool); 1978 if (ret < 0) 1979 rte_exit(EXIT_FAILURE, 1980 "rte_eth_rx_queue_setup: err=%d, " 1981 "port=%d\n", ret, portid); 1982 } 1983 } 1984 printf("\n"); 1985 } 1986 1987 static void 1988 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 1989 { 1990 char s[64]; 1991 uint32_t buff_size = frame_size ? (frame_size + RTE_PKTMBUF_HEADROOM) : 1992 RTE_MBUF_DEFAULT_BUF_SIZE; 1993 1994 1995 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 1996 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 1997 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 1998 buff_size, 1999 socket_id); 2000 if (ctx->mbuf_pool == NULL) 2001 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 2002 socket_id); 2003 else 2004 printf("Allocated mbuf pool on socket %d\n", socket_id); 2005 } 2006 2007 static inline int 2008 inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md) 2009 { 2010 struct ipsec_sa *sa; 2011 2012 /* For inline protocol processing, the metadata in the event will 2013 * uniquely identify the security session which raised the event. 2014 * Application would then need the userdata it had registered with the 2015 * security session to process the event. 2016 */ 2017 2018 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md); 2019 2020 if (sa == NULL) { 2021 /* userdata could not be retrieved */ 2022 return -1; 2023 } 2024 2025 /* Sequence number over flow. SA need to be re-established */ 2026 RTE_SET_USED(sa); 2027 return 0; 2028 } 2029 2030 static int 2031 inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type, 2032 void *param, void *ret_param) 2033 { 2034 uint64_t md; 2035 struct rte_eth_event_ipsec_desc *event_desc = NULL; 2036 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 2037 rte_eth_dev_get_sec_ctx(port_id); 2038 2039 RTE_SET_USED(param); 2040 2041 if (type != RTE_ETH_EVENT_IPSEC) 2042 return -1; 2043 2044 event_desc = ret_param; 2045 if (event_desc == NULL) { 2046 printf("Event descriptor not set\n"); 2047 return -1; 2048 } 2049 2050 md = event_desc->metadata; 2051 2052 if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW) 2053 return inline_ipsec_event_esn_overflow(ctx, md); 2054 else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) { 2055 printf("Invalid IPsec event reported\n"); 2056 return -1; 2057 } 2058 2059 return -1; 2060 } 2061 2062 int32_t 2063 main(int32_t argc, char **argv) 2064 { 2065 int32_t ret; 2066 uint32_t lcore_id; 2067 uint8_t socket_id; 2068 uint16_t portid; 2069 uint64_t req_rx_offloads, req_tx_offloads; 2070 2071 /* init EAL */ 2072 ret = rte_eal_init(argc, argv); 2073 if (ret < 0) 2074 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 2075 argc -= ret; 2076 argv += ret; 2077 2078 /* parse application arguments (after the EAL ones) */ 2079 ret = parse_args(argc, argv); 2080 if (ret < 0) 2081 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 2082 2083 if ((unprotected_port_mask & enabled_port_mask) != 2084 unprotected_port_mask) 2085 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 2086 unprotected_port_mask); 2087 2088 if (check_params() < 0) 2089 rte_exit(EXIT_FAILURE, "check_params failed\n"); 2090 2091 ret = init_lcore_rx_queues(); 2092 if (ret < 0) 2093 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 2094 2095 nb_lcores = rte_lcore_count(); 2096 2097 /* Replicate each context per socket */ 2098 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2099 if (rte_lcore_is_enabled(lcore_id) == 0) 2100 continue; 2101 2102 if (numa_on) 2103 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2104 else 2105 socket_id = 0; 2106 2107 if (socket_ctx[socket_id].mbuf_pool) 2108 continue; 2109 2110 /* initilaze SPD */ 2111 sp4_init(&socket_ctx[socket_id], socket_id); 2112 2113 sp6_init(&socket_ctx[socket_id], socket_id); 2114 2115 /* initilaze SAD */ 2116 sa_init(&socket_ctx[socket_id], socket_id); 2117 2118 rt_init(&socket_ctx[socket_id], socket_id); 2119 2120 pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF); 2121 } 2122 2123 RTE_ETH_FOREACH_DEV(portid) { 2124 if ((enabled_port_mask & (1 << portid)) == 0) 2125 continue; 2126 2127 sa_check_offloads(portid, &req_rx_offloads, &req_tx_offloads); 2128 port_init(portid, req_rx_offloads, req_tx_offloads); 2129 } 2130 2131 cryptodevs_init(); 2132 2133 /* start ports */ 2134 RTE_ETH_FOREACH_DEV(portid) { 2135 if ((enabled_port_mask & (1 << portid)) == 0) 2136 continue; 2137 2138 /* Start device */ 2139 ret = rte_eth_dev_start(portid); 2140 if (ret < 0) 2141 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 2142 "err=%d, port=%d\n", ret, portid); 2143 /* 2144 * If enabled, put device in promiscuous mode. 2145 * This allows IO forwarding mode to forward packets 2146 * to itself through 2 cross-connected ports of the 2147 * target machine. 2148 */ 2149 if (promiscuous_on) 2150 rte_eth_promiscuous_enable(portid); 2151 2152 rte_eth_dev_callback_register(portid, 2153 RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL); 2154 } 2155 2156 check_all_ports_link_status(enabled_port_mask); 2157 2158 /* launch per-lcore init on every lcore */ 2159 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 2160 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2161 if (rte_eal_wait_lcore(lcore_id) < 0) 2162 return -1; 2163 } 2164 2165 return 0; 2166 } 2167