1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdbool.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <stdint.h> 9 #include <inttypes.h> 10 #include <sys/types.h> 11 #include <netinet/in.h> 12 #include <netinet/ip.h> 13 #include <netinet/ip6.h> 14 #include <string.h> 15 #include <sys/queue.h> 16 #include <stdarg.h> 17 #include <errno.h> 18 #include <signal.h> 19 #include <getopt.h> 20 21 #include <rte_common.h> 22 #include <rte_bitmap.h> 23 #include <rte_byteorder.h> 24 #include <rte_log.h> 25 #include <rte_eal.h> 26 #include <rte_launch.h> 27 #include <rte_atomic.h> 28 #include <rte_cycles.h> 29 #include <rte_prefetch.h> 30 #include <rte_lcore.h> 31 #include <rte_per_lcore.h> 32 #include <rte_branch_prediction.h> 33 #include <rte_interrupts.h> 34 #include <rte_random.h> 35 #include <rte_debug.h> 36 #include <rte_ether.h> 37 #include <rte_ethdev.h> 38 #include <rte_mempool.h> 39 #include <rte_mbuf.h> 40 #include <rte_acl.h> 41 #include <rte_lpm.h> 42 #include <rte_lpm6.h> 43 #include <rte_hash.h> 44 #include <rte_jhash.h> 45 #include <rte_cryptodev.h> 46 #include <rte_security.h> 47 #include <rte_eventdev.h> 48 #include <rte_ip.h> 49 #include <rte_ip_frag.h> 50 #include <rte_alarm.h> 51 52 #include "event_helper.h" 53 #include "ipsec.h" 54 #include "ipsec_worker.h" 55 #include "parser.h" 56 #include "sad.h" 57 58 volatile bool force_quit; 59 60 #define MAX_JUMBO_PKT_LEN 9600 61 62 #define MEMPOOL_CACHE_SIZE 256 63 64 #define CDEV_QUEUE_DESC 2048 65 #define CDEV_MAP_ENTRIES 16384 66 #define CDEV_MP_CACHE_SZ 64 67 #define MAX_QUEUE_PAIRS 1 68 69 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 70 71 /* Configure how many packets ahead to prefetch, when reading packets */ 72 #define PREFETCH_OFFSET 3 73 74 #define MAX_RX_QUEUE_PER_LCORE 16 75 76 #define MAX_LCORE_PARAMS 1024 77 78 /* 79 * Configurable number of RX/TX ring descriptors 80 */ 81 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024 82 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024 83 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 84 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 85 86 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 87 (addr)->addr_bytes[0], (addr)->addr_bytes[1], \ 88 (addr)->addr_bytes[2], (addr)->addr_bytes[3], \ 89 (addr)->addr_bytes[4], (addr)->addr_bytes[5], \ 90 0, 0) 91 92 #define FRAG_TBL_BUCKET_ENTRIES 4 93 #define MAX_FRAG_TTL_NS (10LL * NS_PER_S) 94 95 #define MTU_TO_FRAMELEN(x) ((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN) 96 97 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 98 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 99 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 100 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 101 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 102 }; 103 104 struct flow_info flow_info_tbl[RTE_MAX_ETHPORTS]; 105 106 #define CMD_LINE_OPT_CONFIG "config" 107 #define CMD_LINE_OPT_SINGLE_SA "single-sa" 108 #define CMD_LINE_OPT_CRYPTODEV_MASK "cryptodev_mask" 109 #define CMD_LINE_OPT_TRANSFER_MODE "transfer-mode" 110 #define CMD_LINE_OPT_SCHEDULE_TYPE "event-schedule-type" 111 #define CMD_LINE_OPT_RX_OFFLOAD "rxoffload" 112 #define CMD_LINE_OPT_TX_OFFLOAD "txoffload" 113 #define CMD_LINE_OPT_REASSEMBLE "reassemble" 114 #define CMD_LINE_OPT_MTU "mtu" 115 #define CMD_LINE_OPT_FRAG_TTL "frag-ttl" 116 117 #define CMD_LINE_ARG_EVENT "event" 118 #define CMD_LINE_ARG_POLL "poll" 119 #define CMD_LINE_ARG_ORDERED "ordered" 120 #define CMD_LINE_ARG_ATOMIC "atomic" 121 #define CMD_LINE_ARG_PARALLEL "parallel" 122 123 enum { 124 /* long options mapped to a short option */ 125 126 /* first long only option value must be >= 256, so that we won't 127 * conflict with short options 128 */ 129 CMD_LINE_OPT_MIN_NUM = 256, 130 CMD_LINE_OPT_CONFIG_NUM, 131 CMD_LINE_OPT_SINGLE_SA_NUM, 132 CMD_LINE_OPT_CRYPTODEV_MASK_NUM, 133 CMD_LINE_OPT_TRANSFER_MODE_NUM, 134 CMD_LINE_OPT_SCHEDULE_TYPE_NUM, 135 CMD_LINE_OPT_RX_OFFLOAD_NUM, 136 CMD_LINE_OPT_TX_OFFLOAD_NUM, 137 CMD_LINE_OPT_REASSEMBLE_NUM, 138 CMD_LINE_OPT_MTU_NUM, 139 CMD_LINE_OPT_FRAG_TTL_NUM, 140 }; 141 142 static const struct option lgopts[] = { 143 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, 144 {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM}, 145 {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM}, 146 {CMD_LINE_OPT_TRANSFER_MODE, 1, 0, CMD_LINE_OPT_TRANSFER_MODE_NUM}, 147 {CMD_LINE_OPT_SCHEDULE_TYPE, 1, 0, CMD_LINE_OPT_SCHEDULE_TYPE_NUM}, 148 {CMD_LINE_OPT_RX_OFFLOAD, 1, 0, CMD_LINE_OPT_RX_OFFLOAD_NUM}, 149 {CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM}, 150 {CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM}, 151 {CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM}, 152 {CMD_LINE_OPT_FRAG_TTL, 1, 0, CMD_LINE_OPT_FRAG_TTL_NUM}, 153 {NULL, 0, 0, 0} 154 }; 155 156 uint32_t unprotected_port_mask; 157 uint32_t single_sa_idx; 158 /* mask of enabled ports */ 159 static uint32_t enabled_port_mask; 160 static uint64_t enabled_cryptodev_mask = UINT64_MAX; 161 static int32_t promiscuous_on = 1; 162 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 163 static uint32_t nb_lcores; 164 static uint32_t single_sa; 165 static uint32_t nb_bufs_in_pool; 166 167 /* 168 * RX/TX HW offload capabilities to enable/use on ethernet ports. 169 * By default all capabilities are enabled. 170 */ 171 static uint64_t dev_rx_offload = UINT64_MAX; 172 static uint64_t dev_tx_offload = UINT64_MAX; 173 174 /* 175 * global values that determine multi-seg policy 176 */ 177 static uint32_t frag_tbl_sz; 178 static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE; 179 static uint32_t mtu_size = RTE_ETHER_MTU; 180 static uint64_t frag_ttl_ns = MAX_FRAG_TTL_NS; 181 182 /* application wide librte_ipsec/SA parameters */ 183 struct app_sa_prm app_sa_prm = { 184 .enable = 0, 185 .cache_sz = SA_CACHE_SZ 186 }; 187 static const char *cfgfile; 188 189 struct lcore_rx_queue { 190 uint16_t port_id; 191 uint8_t queue_id; 192 } __rte_cache_aligned; 193 194 struct lcore_params { 195 uint16_t port_id; 196 uint8_t queue_id; 197 uint8_t lcore_id; 198 } __rte_cache_aligned; 199 200 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 201 202 static struct lcore_params *lcore_params; 203 static uint16_t nb_lcore_params; 204 205 static struct rte_hash *cdev_map_in; 206 static struct rte_hash *cdev_map_out; 207 208 struct buffer { 209 uint16_t len; 210 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 211 }; 212 213 struct lcore_conf { 214 uint16_t nb_rx_queue; 215 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 216 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 217 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 218 struct ipsec_ctx inbound; 219 struct ipsec_ctx outbound; 220 struct rt_ctx *rt4_ctx; 221 struct rt_ctx *rt6_ctx; 222 struct { 223 struct rte_ip_frag_tbl *tbl; 224 struct rte_mempool *pool_dir; 225 struct rte_mempool *pool_indir; 226 struct rte_ip_frag_death_row dr; 227 } frag; 228 } __rte_cache_aligned; 229 230 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 231 232 static struct rte_eth_conf port_conf = { 233 .rxmode = { 234 .mq_mode = ETH_MQ_RX_RSS, 235 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 236 .split_hdr_size = 0, 237 .offloads = DEV_RX_OFFLOAD_CHECKSUM, 238 }, 239 .rx_adv_conf = { 240 .rss_conf = { 241 .rss_key = NULL, 242 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 243 ETH_RSS_TCP | ETH_RSS_SCTP, 244 }, 245 }, 246 .txmode = { 247 .mq_mode = ETH_MQ_TX_NONE, 248 }, 249 }; 250 251 struct socket_ctx socket_ctx[NB_SOCKETS]; 252 253 /* 254 * Determine is multi-segment support required: 255 * - either frame buffer size is smaller then mtu 256 * - or reassmeble support is requested 257 */ 258 static int 259 multi_seg_required(void) 260 { 261 return (MTU_TO_FRAMELEN(mtu_size) + RTE_PKTMBUF_HEADROOM > 262 frame_buf_size || frag_tbl_sz != 0); 263 } 264 265 static inline void 266 adjust_ipv4_pktlen(struct rte_mbuf *m, const struct rte_ipv4_hdr *iph, 267 uint32_t l2_len) 268 { 269 uint32_t plen, trim; 270 271 plen = rte_be_to_cpu_16(iph->total_length) + l2_len; 272 if (plen < m->pkt_len) { 273 trim = m->pkt_len - plen; 274 rte_pktmbuf_trim(m, trim); 275 } 276 } 277 278 static inline void 279 adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph, 280 uint32_t l2_len) 281 { 282 uint32_t plen, trim; 283 284 plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len; 285 if (plen < m->pkt_len) { 286 trim = m->pkt_len - plen; 287 rte_pktmbuf_trim(m, trim); 288 } 289 } 290 291 #if (STATS_INTERVAL > 0) 292 293 /* Print out statistics on packet distribution */ 294 static void 295 print_stats_cb(__rte_unused void *param) 296 { 297 uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; 298 float burst_percent, rx_per_call, tx_per_call; 299 unsigned int coreid; 300 301 total_packets_dropped = 0; 302 total_packets_tx = 0; 303 total_packets_rx = 0; 304 305 const char clr[] = { 27, '[', '2', 'J', '\0' }; 306 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 307 308 /* Clear screen and move to top left */ 309 printf("%s%s", clr, topLeft); 310 311 printf("\nCore statistics ===================================="); 312 313 for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) { 314 /* skip disabled cores */ 315 if (rte_lcore_is_enabled(coreid) == 0) 316 continue; 317 burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/ 318 core_statistics[coreid].rx; 319 rx_per_call = (float)(core_statistics[coreid].rx)/ 320 core_statistics[coreid].rx_call; 321 tx_per_call = (float)(core_statistics[coreid].tx)/ 322 core_statistics[coreid].tx_call; 323 printf("\nStatistics for core %u ------------------------------" 324 "\nPackets received: %20"PRIu64 325 "\nPackets sent: %24"PRIu64 326 "\nPackets dropped: %21"PRIu64 327 "\nBurst percent: %23.2f" 328 "\nPackets per Rx call: %17.2f" 329 "\nPackets per Tx call: %17.2f", 330 coreid, 331 core_statistics[coreid].rx, 332 core_statistics[coreid].tx, 333 core_statistics[coreid].dropped, 334 burst_percent, 335 rx_per_call, 336 tx_per_call); 337 338 total_packets_dropped += core_statistics[coreid].dropped; 339 total_packets_tx += core_statistics[coreid].tx; 340 total_packets_rx += core_statistics[coreid].rx; 341 } 342 printf("\nAggregate statistics ===============================" 343 "\nTotal packets received: %14"PRIu64 344 "\nTotal packets sent: %18"PRIu64 345 "\nTotal packets dropped: %15"PRIu64, 346 total_packets_rx, 347 total_packets_tx, 348 total_packets_dropped); 349 printf("\n====================================================\n"); 350 351 rte_eal_alarm_set(STATS_INTERVAL * US_PER_S, print_stats_cb, NULL); 352 } 353 #endif /* STATS_INTERVAL */ 354 355 static inline void 356 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 357 { 358 const struct rte_ether_hdr *eth; 359 const struct rte_ipv4_hdr *iph4; 360 const struct rte_ipv6_hdr *iph6; 361 362 eth = rte_pktmbuf_mtod(pkt, const struct rte_ether_hdr *); 363 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 364 365 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt, 366 RTE_ETHER_HDR_LEN); 367 adjust_ipv4_pktlen(pkt, iph4, 0); 368 369 if (iph4->next_proto_id == IPPROTO_ESP) 370 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 371 else { 372 t->ip4.data[t->ip4.num] = &iph4->next_proto_id; 373 t->ip4.pkts[(t->ip4.num)++] = pkt; 374 } 375 pkt->l2_len = 0; 376 pkt->l3_len = sizeof(*iph4); 377 pkt->packet_type |= RTE_PTYPE_L3_IPV4; 378 } else if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 379 int next_proto; 380 size_t l3len, ext_len; 381 uint8_t *p; 382 383 /* get protocol type */ 384 iph6 = (const struct rte_ipv6_hdr *)rte_pktmbuf_adj(pkt, 385 RTE_ETHER_HDR_LEN); 386 adjust_ipv6_pktlen(pkt, iph6, 0); 387 388 next_proto = iph6->proto; 389 390 /* determine l3 header size up to ESP extension */ 391 l3len = sizeof(struct ip6_hdr); 392 p = rte_pktmbuf_mtod(pkt, uint8_t *); 393 while (next_proto != IPPROTO_ESP && l3len < pkt->data_len && 394 (next_proto = rte_ipv6_get_next_ext(p + l3len, 395 next_proto, &ext_len)) >= 0) 396 l3len += ext_len; 397 398 /* drop packet when IPv6 header exceeds first segment length */ 399 if (unlikely(l3len > pkt->data_len)) { 400 free_pkts(&pkt, 1); 401 return; 402 } 403 404 if (next_proto == IPPROTO_ESP) 405 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 406 else { 407 t->ip6.data[t->ip6.num] = &iph6->proto; 408 t->ip6.pkts[(t->ip6.num)++] = pkt; 409 } 410 pkt->l2_len = 0; 411 pkt->l3_len = l3len; 412 pkt->packet_type |= RTE_PTYPE_L3_IPV6; 413 } else { 414 /* Unknown/Unsupported type, drop the packet */ 415 RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n", 416 rte_be_to_cpu_16(eth->ether_type)); 417 free_pkts(&pkt, 1); 418 return; 419 } 420 421 /* Check if the packet has been processed inline. For inline protocol 422 * processed packets, the metadata in the mbuf can be used to identify 423 * the security processing done on the packet. The metadata will be 424 * used to retrieve the application registered userdata associated 425 * with the security session. 426 */ 427 428 if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) { 429 struct ipsec_sa *sa; 430 struct ipsec_mbuf_metadata *priv; 431 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 432 rte_eth_dev_get_sec_ctx( 433 pkt->port); 434 435 /* Retrieve the userdata registered. Here, the userdata 436 * registered is the SA pointer. 437 */ 438 439 sa = (struct ipsec_sa *) 440 rte_security_get_userdata(ctx, pkt->udata64); 441 442 if (sa == NULL) { 443 /* userdata could not be retrieved */ 444 return; 445 } 446 447 /* Save SA as priv member in mbuf. This will be used in the 448 * IPsec selector(SP-SA) check. 449 */ 450 451 priv = get_priv(pkt); 452 priv->sa = sa; 453 } 454 } 455 456 static inline void 457 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 458 uint16_t nb_pkts) 459 { 460 int32_t i; 461 462 t->ipsec.num = 0; 463 t->ip4.num = 0; 464 t->ip6.num = 0; 465 466 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 467 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 468 void *)); 469 prepare_one_packet(pkts[i], t); 470 } 471 /* Process left packets */ 472 for (; i < nb_pkts; i++) 473 prepare_one_packet(pkts[i], t); 474 } 475 476 static inline void 477 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port, 478 const struct lcore_conf *qconf) 479 { 480 struct ip *ip; 481 struct rte_ether_hdr *ethhdr; 482 483 ip = rte_pktmbuf_mtod(pkt, struct ip *); 484 485 ethhdr = (struct rte_ether_hdr *) 486 rte_pktmbuf_prepend(pkt, RTE_ETHER_HDR_LEN); 487 488 if (ip->ip_v == IPVERSION) { 489 pkt->ol_flags |= qconf->outbound.ipv4_offloads; 490 pkt->l3_len = sizeof(struct ip); 491 pkt->l2_len = RTE_ETHER_HDR_LEN; 492 493 ip->ip_sum = 0; 494 495 /* calculate IPv4 cksum in SW */ 496 if ((pkt->ol_flags & PKT_TX_IP_CKSUM) == 0) 497 ip->ip_sum = rte_ipv4_cksum((struct rte_ipv4_hdr *)ip); 498 499 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 500 } else { 501 pkt->ol_flags |= qconf->outbound.ipv6_offloads; 502 pkt->l3_len = sizeof(struct ip6_hdr); 503 pkt->l2_len = RTE_ETHER_HDR_LEN; 504 505 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 506 } 507 508 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 509 sizeof(struct rte_ether_addr)); 510 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 511 sizeof(struct rte_ether_addr)); 512 } 513 514 static inline void 515 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port, 516 const struct lcore_conf *qconf) 517 { 518 int32_t i; 519 const int32_t prefetch_offset = 2; 520 521 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 522 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 523 prepare_tx_pkt(pkts[i], port, qconf); 524 } 525 /* Process left packets */ 526 for (; i < nb_pkts; i++) 527 prepare_tx_pkt(pkts[i], port, qconf); 528 } 529 530 /* Send burst of packets on an output interface */ 531 static inline int32_t 532 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 533 { 534 struct rte_mbuf **m_table; 535 int32_t ret; 536 uint16_t queueid; 537 538 queueid = qconf->tx_queue_id[port]; 539 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 540 541 prepare_tx_burst(m_table, n, port, qconf); 542 543 ret = rte_eth_tx_burst(port, queueid, m_table, n); 544 545 core_stats_update_tx(ret); 546 547 if (unlikely(ret < n)) { 548 do { 549 free_pkts(&m_table[ret], 1); 550 } while (++ret < n); 551 } 552 553 return 0; 554 } 555 556 /* 557 * Helper function to fragment and queue for TX one packet. 558 */ 559 static inline uint32_t 560 send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m, 561 uint16_t port, uint8_t proto) 562 { 563 struct buffer *tbl; 564 uint32_t len, n; 565 int32_t rc; 566 567 tbl = qconf->tx_mbufs + port; 568 len = tbl->len; 569 570 /* free space for new fragments */ 571 if (len + RTE_LIBRTE_IP_FRAG_MAX_FRAG >= RTE_DIM(tbl->m_table)) { 572 send_burst(qconf, len, port); 573 len = 0; 574 } 575 576 n = RTE_DIM(tbl->m_table) - len; 577 578 if (proto == IPPROTO_IP) 579 rc = rte_ipv4_fragment_packet(m, tbl->m_table + len, 580 n, mtu_size, qconf->frag.pool_dir, 581 qconf->frag.pool_indir); 582 else 583 rc = rte_ipv6_fragment_packet(m, tbl->m_table + len, 584 n, mtu_size, qconf->frag.pool_dir, 585 qconf->frag.pool_indir); 586 587 if (rc >= 0) 588 len += rc; 589 else 590 RTE_LOG(ERR, IPSEC, 591 "%s: failed to fragment packet with size %u, " 592 "error code: %d\n", 593 __func__, m->pkt_len, rte_errno); 594 595 free_pkts(&m, 1); 596 return len; 597 } 598 599 /* Enqueue a single packet, and send burst if queue is filled */ 600 static inline int32_t 601 send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto) 602 { 603 uint32_t lcore_id; 604 uint16_t len; 605 struct lcore_conf *qconf; 606 607 lcore_id = rte_lcore_id(); 608 609 qconf = &lcore_conf[lcore_id]; 610 len = qconf->tx_mbufs[port].len; 611 612 if (m->pkt_len <= mtu_size) { 613 qconf->tx_mbufs[port].m_table[len] = m; 614 len++; 615 616 /* need to fragment the packet */ 617 } else if (frag_tbl_sz > 0) 618 len = send_fragment_packet(qconf, m, port, proto); 619 else 620 free_pkts(&m, 1); 621 622 /* enough pkts to be sent */ 623 if (unlikely(len == MAX_PKT_BURST)) { 624 send_burst(qconf, MAX_PKT_BURST, port); 625 len = 0; 626 } 627 628 qconf->tx_mbufs[port].len = len; 629 return 0; 630 } 631 632 static inline void 633 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 634 uint16_t lim) 635 { 636 struct rte_mbuf *m; 637 uint32_t i, j, res, sa_idx; 638 639 if (ip->num == 0 || sp == NULL) 640 return; 641 642 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 643 ip->num, DEFAULT_MAX_CATEGORIES); 644 645 j = 0; 646 for (i = 0; i < ip->num; i++) { 647 m = ip->pkts[i]; 648 res = ip->res[i]; 649 if (res == BYPASS) { 650 ip->pkts[j++] = m; 651 continue; 652 } 653 if (res == DISCARD) { 654 free_pkts(&m, 1); 655 continue; 656 } 657 658 /* Only check SPI match for processed IPSec packets */ 659 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) { 660 free_pkts(&m, 1); 661 continue; 662 } 663 664 sa_idx = res - 1; 665 if (!inbound_sa_check(sa, m, sa_idx)) { 666 free_pkts(&m, 1); 667 continue; 668 } 669 ip->pkts[j++] = m; 670 } 671 ip->num = j; 672 } 673 674 static void 675 split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num) 676 { 677 uint32_t i, n4, n6; 678 struct ip *ip; 679 struct rte_mbuf *m; 680 681 n4 = trf->ip4.num; 682 n6 = trf->ip6.num; 683 684 for (i = 0; i < num; i++) { 685 686 m = mb[i]; 687 ip = rte_pktmbuf_mtod(m, struct ip *); 688 689 if (ip->ip_v == IPVERSION) { 690 trf->ip4.pkts[n4] = m; 691 trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m, 692 uint8_t *, offsetof(struct ip, ip_p)); 693 n4++; 694 } else if (ip->ip_v == IP6_VERSION) { 695 trf->ip6.pkts[n6] = m; 696 trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m, 697 uint8_t *, 698 offsetof(struct ip6_hdr, ip6_nxt)); 699 n6++; 700 } else 701 free_pkts(&m, 1); 702 } 703 704 trf->ip4.num = n4; 705 trf->ip6.num = n6; 706 } 707 708 709 static inline void 710 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 711 struct ipsec_traffic *traffic) 712 { 713 uint16_t nb_pkts_in, n_ip4, n_ip6; 714 715 n_ip4 = traffic->ip4.num; 716 n_ip6 = traffic->ip6.num; 717 718 if (app_sa_prm.enable == 0) { 719 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 720 traffic->ipsec.num, MAX_PKT_BURST); 721 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in); 722 } else { 723 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 724 traffic->ipsec.saptr, traffic->ipsec.num); 725 ipsec_process(ipsec_ctx, traffic); 726 } 727 728 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 729 n_ip4); 730 731 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 732 n_ip6); 733 } 734 735 static inline void 736 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 737 struct traffic_type *ipsec) 738 { 739 struct rte_mbuf *m; 740 uint32_t i, j, sa_idx; 741 742 if (ip->num == 0 || sp == NULL) 743 return; 744 745 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 746 ip->num, DEFAULT_MAX_CATEGORIES); 747 748 j = 0; 749 for (i = 0; i < ip->num; i++) { 750 m = ip->pkts[i]; 751 sa_idx = ip->res[i] - 1; 752 if (ip->res[i] == DISCARD) 753 free_pkts(&m, 1); 754 else if (ip->res[i] == BYPASS) 755 ip->pkts[j++] = m; 756 else { 757 ipsec->res[ipsec->num] = sa_idx; 758 ipsec->pkts[ipsec->num++] = m; 759 } 760 } 761 ip->num = j; 762 } 763 764 static inline void 765 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 766 struct ipsec_traffic *traffic) 767 { 768 struct rte_mbuf *m; 769 uint16_t idx, nb_pkts_out, i; 770 771 /* Drop any IPsec traffic from protected ports */ 772 free_pkts(traffic->ipsec.pkts, traffic->ipsec.num); 773 774 traffic->ipsec.num = 0; 775 776 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 777 778 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 779 780 if (app_sa_prm.enable == 0) { 781 782 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 783 traffic->ipsec.res, traffic->ipsec.num, 784 MAX_PKT_BURST); 785 786 for (i = 0; i < nb_pkts_out; i++) { 787 m = traffic->ipsec.pkts[i]; 788 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 789 if (ip->ip_v == IPVERSION) { 790 idx = traffic->ip4.num++; 791 traffic->ip4.pkts[idx] = m; 792 } else { 793 idx = traffic->ip6.num++; 794 traffic->ip6.pkts[idx] = m; 795 } 796 } 797 } else { 798 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 799 traffic->ipsec.saptr, traffic->ipsec.num); 800 ipsec_process(ipsec_ctx, traffic); 801 } 802 } 803 804 static inline void 805 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 806 struct ipsec_traffic *traffic) 807 { 808 struct rte_mbuf *m; 809 uint32_t nb_pkts_in, i, idx; 810 811 /* Drop any IPv4 traffic from unprotected ports */ 812 free_pkts(traffic->ip4.pkts, traffic->ip4.num); 813 814 traffic->ip4.num = 0; 815 816 /* Drop any IPv6 traffic from unprotected ports */ 817 free_pkts(traffic->ip6.pkts, traffic->ip6.num); 818 819 traffic->ip6.num = 0; 820 821 if (app_sa_prm.enable == 0) { 822 823 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 824 traffic->ipsec.num, MAX_PKT_BURST); 825 826 for (i = 0; i < nb_pkts_in; i++) { 827 m = traffic->ipsec.pkts[i]; 828 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 829 if (ip->ip_v == IPVERSION) { 830 idx = traffic->ip4.num++; 831 traffic->ip4.pkts[idx] = m; 832 } else { 833 idx = traffic->ip6.num++; 834 traffic->ip6.pkts[idx] = m; 835 } 836 } 837 } else { 838 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts, 839 traffic->ipsec.saptr, traffic->ipsec.num); 840 ipsec_process(ipsec_ctx, traffic); 841 } 842 } 843 844 static inline void 845 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 846 struct ipsec_traffic *traffic) 847 { 848 struct rte_mbuf *m; 849 uint32_t nb_pkts_out, i, n; 850 struct ip *ip; 851 852 /* Drop any IPsec traffic from protected ports */ 853 free_pkts(traffic->ipsec.pkts, traffic->ipsec.num); 854 855 n = 0; 856 857 for (i = 0; i < traffic->ip4.num; i++) { 858 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i]; 859 traffic->ipsec.res[n++] = single_sa_idx; 860 } 861 862 for (i = 0; i < traffic->ip6.num; i++) { 863 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i]; 864 traffic->ipsec.res[n++] = single_sa_idx; 865 } 866 867 traffic->ip4.num = 0; 868 traffic->ip6.num = 0; 869 traffic->ipsec.num = n; 870 871 if (app_sa_prm.enable == 0) { 872 873 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 874 traffic->ipsec.res, traffic->ipsec.num, 875 MAX_PKT_BURST); 876 877 /* They all sue the same SA (ip4 or ip6 tunnel) */ 878 m = traffic->ipsec.pkts[0]; 879 ip = rte_pktmbuf_mtod(m, struct ip *); 880 if (ip->ip_v == IPVERSION) { 881 traffic->ip4.num = nb_pkts_out; 882 for (i = 0; i < nb_pkts_out; i++) 883 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i]; 884 } else { 885 traffic->ip6.num = nb_pkts_out; 886 for (i = 0; i < nb_pkts_out; i++) 887 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i]; 888 } 889 } else { 890 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res, 891 traffic->ipsec.saptr, traffic->ipsec.num); 892 ipsec_process(ipsec_ctx, traffic); 893 } 894 } 895 896 static inline int32_t 897 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) 898 { 899 struct ipsec_mbuf_metadata *priv; 900 struct ipsec_sa *sa; 901 902 priv = get_priv(pkt); 903 904 sa = priv->sa; 905 if (unlikely(sa == NULL)) { 906 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); 907 goto fail; 908 } 909 910 if (is_ipv6) 911 return sa->portid; 912 913 /* else */ 914 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); 915 916 fail: 917 if (is_ipv6) 918 return -1; 919 920 /* else */ 921 return 0; 922 } 923 924 static inline void 925 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 926 { 927 uint32_t hop[MAX_PKT_BURST * 2]; 928 uint32_t dst_ip[MAX_PKT_BURST * 2]; 929 int32_t pkt_hop = 0; 930 uint16_t i, offset; 931 uint16_t lpm_pkts = 0; 932 933 if (nb_pkts == 0) 934 return; 935 936 /* Need to do an LPM lookup for non-inline packets. Inline packets will 937 * have port ID in the SA 938 */ 939 940 for (i = 0; i < nb_pkts; i++) { 941 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 942 /* Security offload not enabled. So an LPM lookup is 943 * required to get the hop 944 */ 945 offset = offsetof(struct ip, ip_dst); 946 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], 947 uint32_t *, offset); 948 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); 949 lpm_pkts++; 950 } 951 } 952 953 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); 954 955 lpm_pkts = 0; 956 957 for (i = 0; i < nb_pkts; i++) { 958 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 959 /* Read hop from the SA */ 960 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); 961 } else { 962 /* Need to use hop returned by lookup */ 963 pkt_hop = hop[lpm_pkts++]; 964 } 965 966 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { 967 free_pkts(&pkts[i], 1); 968 continue; 969 } 970 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP); 971 } 972 } 973 974 static inline void 975 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 976 { 977 int32_t hop[MAX_PKT_BURST * 2]; 978 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 979 uint8_t *ip6_dst; 980 int32_t pkt_hop = 0; 981 uint16_t i, offset; 982 uint16_t lpm_pkts = 0; 983 984 if (nb_pkts == 0) 985 return; 986 987 /* Need to do an LPM lookup for non-inline packets. Inline packets will 988 * have port ID in the SA 989 */ 990 991 for (i = 0; i < nb_pkts; i++) { 992 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 993 /* Security offload not enabled. So an LPM lookup is 994 * required to get the hop 995 */ 996 offset = offsetof(struct ip6_hdr, ip6_dst); 997 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, 998 offset); 999 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); 1000 lpm_pkts++; 1001 } 1002 } 1003 1004 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, 1005 lpm_pkts); 1006 1007 lpm_pkts = 0; 1008 1009 for (i = 0; i < nb_pkts; i++) { 1010 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 1011 /* Read hop from the SA */ 1012 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); 1013 } else { 1014 /* Need to use hop returned by lookup */ 1015 pkt_hop = hop[lpm_pkts++]; 1016 } 1017 1018 if (pkt_hop == -1) { 1019 free_pkts(&pkts[i], 1); 1020 continue; 1021 } 1022 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6); 1023 } 1024 } 1025 1026 static inline void 1027 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 1028 uint8_t nb_pkts, uint16_t portid) 1029 { 1030 struct ipsec_traffic traffic; 1031 1032 prepare_traffic(pkts, &traffic, nb_pkts); 1033 1034 if (unlikely(single_sa)) { 1035 if (is_unprotected_port(portid)) 1036 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 1037 else 1038 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 1039 } else { 1040 if (is_unprotected_port(portid)) 1041 process_pkts_inbound(&qconf->inbound, &traffic); 1042 else 1043 process_pkts_outbound(&qconf->outbound, &traffic); 1044 } 1045 1046 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 1047 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 1048 } 1049 1050 static inline void 1051 drain_tx_buffers(struct lcore_conf *qconf) 1052 { 1053 struct buffer *buf; 1054 uint32_t portid; 1055 1056 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 1057 buf = &qconf->tx_mbufs[portid]; 1058 if (buf->len == 0) 1059 continue; 1060 send_burst(qconf, buf->len, portid); 1061 buf->len = 0; 1062 } 1063 } 1064 1065 static inline void 1066 drain_crypto_buffers(struct lcore_conf *qconf) 1067 { 1068 uint32_t i; 1069 struct ipsec_ctx *ctx; 1070 1071 /* drain inbound buffers*/ 1072 ctx = &qconf->inbound; 1073 for (i = 0; i != ctx->nb_qps; i++) { 1074 if (ctx->tbl[i].len != 0) 1075 enqueue_cop_burst(ctx->tbl + i); 1076 } 1077 1078 /* drain outbound buffers*/ 1079 ctx = &qconf->outbound; 1080 for (i = 0; i != ctx->nb_qps; i++) { 1081 if (ctx->tbl[i].len != 0) 1082 enqueue_cop_burst(ctx->tbl + i); 1083 } 1084 } 1085 1086 static void 1087 drain_inbound_crypto_queues(const struct lcore_conf *qconf, 1088 struct ipsec_ctx *ctx) 1089 { 1090 uint32_t n; 1091 struct ipsec_traffic trf; 1092 1093 if (app_sa_prm.enable == 0) { 1094 1095 /* dequeue packets from crypto-queue */ 1096 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts, 1097 RTE_DIM(trf.ipsec.pkts)); 1098 1099 trf.ip4.num = 0; 1100 trf.ip6.num = 0; 1101 1102 /* split traffic by ipv4-ipv6 */ 1103 split46_traffic(&trf, trf.ipsec.pkts, n); 1104 } else 1105 ipsec_cqp_process(ctx, &trf); 1106 1107 /* process ipv4 packets */ 1108 if (trf.ip4.num != 0) { 1109 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0); 1110 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 1111 } 1112 1113 /* process ipv6 packets */ 1114 if (trf.ip6.num != 0) { 1115 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0); 1116 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 1117 } 1118 } 1119 1120 static void 1121 drain_outbound_crypto_queues(const struct lcore_conf *qconf, 1122 struct ipsec_ctx *ctx) 1123 { 1124 uint32_t n; 1125 struct ipsec_traffic trf; 1126 1127 if (app_sa_prm.enable == 0) { 1128 1129 /* dequeue packets from crypto-queue */ 1130 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts, 1131 RTE_DIM(trf.ipsec.pkts)); 1132 1133 trf.ip4.num = 0; 1134 trf.ip6.num = 0; 1135 1136 /* split traffic by ipv4-ipv6 */ 1137 split46_traffic(&trf, trf.ipsec.pkts, n); 1138 } else 1139 ipsec_cqp_process(ctx, &trf); 1140 1141 /* process ipv4 packets */ 1142 if (trf.ip4.num != 0) 1143 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num); 1144 1145 /* process ipv6 packets */ 1146 if (trf.ip6.num != 0) 1147 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num); 1148 } 1149 1150 /* main processing loop */ 1151 void 1152 ipsec_poll_mode_worker(void) 1153 { 1154 struct rte_mbuf *pkts[MAX_PKT_BURST]; 1155 uint32_t lcore_id; 1156 uint64_t prev_tsc, diff_tsc, cur_tsc; 1157 int32_t i, nb_rx; 1158 uint16_t portid; 1159 uint8_t queueid; 1160 struct lcore_conf *qconf; 1161 int32_t rc, socket_id; 1162 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 1163 / US_PER_S * BURST_TX_DRAIN_US; 1164 struct lcore_rx_queue *rxql; 1165 1166 prev_tsc = 0; 1167 lcore_id = rte_lcore_id(); 1168 qconf = &lcore_conf[lcore_id]; 1169 rxql = qconf->rx_queue_list; 1170 socket_id = rte_lcore_to_socket_id(lcore_id); 1171 1172 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 1173 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 1174 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 1175 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 1176 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 1177 qconf->inbound.cdev_map = cdev_map_in; 1178 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 1179 qconf->inbound.session_priv_pool = 1180 socket_ctx[socket_id].session_priv_pool; 1181 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 1182 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 1183 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 1184 qconf->outbound.cdev_map = cdev_map_out; 1185 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 1186 qconf->outbound.session_priv_pool = 1187 socket_ctx[socket_id].session_priv_pool; 1188 qconf->frag.pool_dir = socket_ctx[socket_id].mbuf_pool; 1189 qconf->frag.pool_indir = socket_ctx[socket_id].mbuf_pool_indir; 1190 1191 rc = ipsec_sad_lcore_cache_init(app_sa_prm.cache_sz); 1192 if (rc != 0) { 1193 RTE_LOG(ERR, IPSEC, 1194 "SAD cache init on lcore %u, failed with code: %d\n", 1195 lcore_id, rc); 1196 return; 1197 } 1198 1199 if (qconf->nb_rx_queue == 0) { 1200 RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n", 1201 lcore_id); 1202 return; 1203 } 1204 1205 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 1206 1207 for (i = 0; i < qconf->nb_rx_queue; i++) { 1208 portid = rxql[i].port_id; 1209 queueid = rxql[i].queue_id; 1210 RTE_LOG(INFO, IPSEC, 1211 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 1212 lcore_id, portid, queueid); 1213 } 1214 1215 while (!force_quit) { 1216 cur_tsc = rte_rdtsc(); 1217 1218 /* TX queue buffer drain */ 1219 diff_tsc = cur_tsc - prev_tsc; 1220 1221 if (unlikely(diff_tsc > drain_tsc)) { 1222 drain_tx_buffers(qconf); 1223 drain_crypto_buffers(qconf); 1224 prev_tsc = cur_tsc; 1225 } 1226 1227 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1228 1229 /* Read packets from RX queues */ 1230 portid = rxql[i].port_id; 1231 queueid = rxql[i].queue_id; 1232 nb_rx = rte_eth_rx_burst(portid, queueid, 1233 pkts, MAX_PKT_BURST); 1234 1235 if (nb_rx > 0) { 1236 core_stats_update_rx(nb_rx); 1237 process_pkts(qconf, pkts, nb_rx, portid); 1238 } 1239 1240 /* dequeue and process completed crypto-ops */ 1241 if (is_unprotected_port(portid)) 1242 drain_inbound_crypto_queues(qconf, 1243 &qconf->inbound); 1244 else 1245 drain_outbound_crypto_queues(qconf, 1246 &qconf->outbound); 1247 } 1248 } 1249 } 1250 1251 int 1252 check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid) 1253 { 1254 uint16_t i; 1255 uint16_t portid; 1256 uint8_t queueid; 1257 1258 for (i = 0; i < nb_lcore_params; ++i) { 1259 portid = lcore_params_array[i].port_id; 1260 if (portid == fdir_portid) { 1261 queueid = lcore_params_array[i].queue_id; 1262 if (queueid == fdir_qid) 1263 break; 1264 } 1265 1266 if (i == nb_lcore_params - 1) 1267 return -1; 1268 } 1269 1270 return 1; 1271 } 1272 1273 static int32_t 1274 check_poll_mode_params(struct eh_conf *eh_conf) 1275 { 1276 uint8_t lcore; 1277 uint16_t portid; 1278 uint16_t i; 1279 int32_t socket_id; 1280 1281 if (!eh_conf) 1282 return -EINVAL; 1283 1284 if (eh_conf->mode != EH_PKT_TRANSFER_MODE_POLL) 1285 return 0; 1286 1287 if (lcore_params == NULL) { 1288 printf("Error: No port/queue/core mappings\n"); 1289 return -1; 1290 } 1291 1292 for (i = 0; i < nb_lcore_params; ++i) { 1293 lcore = lcore_params[i].lcore_id; 1294 if (!rte_lcore_is_enabled(lcore)) { 1295 printf("error: lcore %hhu is not enabled in " 1296 "lcore mask\n", lcore); 1297 return -1; 1298 } 1299 socket_id = rte_lcore_to_socket_id(lcore); 1300 if (socket_id != 0 && numa_on == 0) { 1301 printf("warning: lcore %hhu is on socket %d " 1302 "with numa off\n", 1303 lcore, socket_id); 1304 } 1305 portid = lcore_params[i].port_id; 1306 if ((enabled_port_mask & (1 << portid)) == 0) { 1307 printf("port %u is not enabled in port mask\n", portid); 1308 return -1; 1309 } 1310 if (!rte_eth_dev_is_valid_port(portid)) { 1311 printf("port %u is not present on the board\n", portid); 1312 return -1; 1313 } 1314 } 1315 return 0; 1316 } 1317 1318 static uint8_t 1319 get_port_nb_rx_queues(const uint16_t port) 1320 { 1321 int32_t queue = -1; 1322 uint16_t i; 1323 1324 for (i = 0; i < nb_lcore_params; ++i) { 1325 if (lcore_params[i].port_id == port && 1326 lcore_params[i].queue_id > queue) 1327 queue = lcore_params[i].queue_id; 1328 } 1329 return (uint8_t)(++queue); 1330 } 1331 1332 static int32_t 1333 init_lcore_rx_queues(void) 1334 { 1335 uint16_t i, nb_rx_queue; 1336 uint8_t lcore; 1337 1338 for (i = 0; i < nb_lcore_params; ++i) { 1339 lcore = lcore_params[i].lcore_id; 1340 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 1341 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 1342 printf("error: too many queues (%u) for lcore: %u\n", 1343 nb_rx_queue + 1, lcore); 1344 return -1; 1345 } 1346 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 1347 lcore_params[i].port_id; 1348 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 1349 lcore_params[i].queue_id; 1350 lcore_conf[lcore].nb_rx_queue++; 1351 } 1352 return 0; 1353 } 1354 1355 /* display usage */ 1356 static void 1357 print_usage(const char *prgname) 1358 { 1359 fprintf(stderr, "%s [EAL options] --" 1360 " -p PORTMASK" 1361 " [-P]" 1362 " [-u PORTMASK]" 1363 " [-j FRAMESIZE]" 1364 " [-l]" 1365 " [-w REPLAY_WINDOW_SIZE]" 1366 " [-e]" 1367 " [-a]" 1368 " [-c]" 1369 " [-s NUMBER_OF_MBUFS_IN_PKT_POOL]" 1370 " -f CONFIG_FILE" 1371 " --config (port,queue,lcore)[,(port,queue,lcore)]" 1372 " [--single-sa SAIDX]" 1373 " [--cryptodev_mask MASK]" 1374 " [--transfer-mode MODE]" 1375 " [--event-schedule-type TYPE]" 1376 " [--" CMD_LINE_OPT_RX_OFFLOAD " RX_OFFLOAD_MASK]" 1377 " [--" CMD_LINE_OPT_TX_OFFLOAD " TX_OFFLOAD_MASK]" 1378 " [--" CMD_LINE_OPT_REASSEMBLE " REASSEMBLE_TABLE_SIZE]" 1379 " [--" CMD_LINE_OPT_MTU " MTU]" 1380 "\n\n" 1381 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n" 1382 " -P : Enable promiscuous mode\n" 1383 " -u PORTMASK: Hexadecimal bitmask of unprotected ports\n" 1384 " -j FRAMESIZE: Data buffer size, minimum (and default)\n" 1385 " value: RTE_MBUF_DEFAULT_BUF_SIZE\n" 1386 " -l enables code-path that uses librte_ipsec\n" 1387 " -w REPLAY_WINDOW_SIZE specifies IPsec SQN replay window\n" 1388 " size for each SA\n" 1389 " -e enables ESN\n" 1390 " -a enables SA SQN atomic behaviour\n" 1391 " -c specifies inbound SAD cache size,\n" 1392 " zero value disables the cache (default value: 128)\n" 1393 " -s number of mbufs in packet pool, if not specified number\n" 1394 " of mbufs will be calculated based on number of cores,\n" 1395 " ports and crypto queues\n" 1396 " -f CONFIG_FILE: Configuration file\n" 1397 " --config (port,queue,lcore): Rx queue configuration. In poll\n" 1398 " mode determines which queues from\n" 1399 " which ports are mapped to which cores.\n" 1400 " In event mode this option is not used\n" 1401 " as packets are dynamically scheduled\n" 1402 " to cores by HW.\n" 1403 " --single-sa SAIDX: In poll mode use single SA index for\n" 1404 " outbound traffic, bypassing the SP\n" 1405 " In event mode selects driver submode,\n" 1406 " SA index value is ignored\n" 1407 " --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n" 1408 " devices to configure\n" 1409 " --transfer-mode MODE\n" 1410 " \"poll\" : Packet transfer via polling (default)\n" 1411 " \"event\" : Packet transfer via event device\n" 1412 " --event-schedule-type TYPE queue schedule type, used only when\n" 1413 " transfer mode is set to event\n" 1414 " \"ordered\" : Ordered (default)\n" 1415 " \"atomic\" : Atomic\n" 1416 " \"parallel\" : Parallel\n" 1417 " --" CMD_LINE_OPT_RX_OFFLOAD 1418 ": bitmask of the RX HW offload capabilities to enable/use\n" 1419 " (DEV_RX_OFFLOAD_*)\n" 1420 " --" CMD_LINE_OPT_TX_OFFLOAD 1421 ": bitmask of the TX HW offload capabilities to enable/use\n" 1422 " (DEV_TX_OFFLOAD_*)\n" 1423 " --" CMD_LINE_OPT_REASSEMBLE " NUM" 1424 ": max number of entries in reassemble(fragment) table\n" 1425 " (zero (default value) disables reassembly)\n" 1426 " --" CMD_LINE_OPT_MTU " MTU" 1427 ": MTU value on all ports (default value: 1500)\n" 1428 " outgoing packets with bigger size will be fragmented\n" 1429 " incoming packets with bigger size will be discarded\n" 1430 " --" CMD_LINE_OPT_FRAG_TTL " FRAG_TTL_NS" 1431 ": fragments lifetime in nanoseconds, default\n" 1432 " and maximum value is 10.000.000.000 ns (10 s)\n" 1433 "\n", 1434 prgname); 1435 } 1436 1437 static int 1438 parse_mask(const char *str, uint64_t *val) 1439 { 1440 char *end; 1441 unsigned long t; 1442 1443 errno = 0; 1444 t = strtoul(str, &end, 0); 1445 if (errno != 0 || end[0] != 0) 1446 return -EINVAL; 1447 1448 *val = t; 1449 return 0; 1450 } 1451 1452 static int32_t 1453 parse_portmask(const char *portmask) 1454 { 1455 char *end = NULL; 1456 unsigned long pm; 1457 1458 /* parse hexadecimal string */ 1459 pm = strtoul(portmask, &end, 16); 1460 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 1461 return -1; 1462 1463 if ((pm == 0) && errno) 1464 return -1; 1465 1466 return pm; 1467 } 1468 1469 static int64_t 1470 parse_decimal(const char *str) 1471 { 1472 char *end = NULL; 1473 uint64_t num; 1474 1475 num = strtoull(str, &end, 10); 1476 if ((str[0] == '\0') || (end == NULL) || (*end != '\0') 1477 || num > INT64_MAX) 1478 return -1; 1479 1480 return num; 1481 } 1482 1483 static int32_t 1484 parse_config(const char *q_arg) 1485 { 1486 char s[256]; 1487 const char *p, *p0 = q_arg; 1488 char *end; 1489 enum fieldnames { 1490 FLD_PORT = 0, 1491 FLD_QUEUE, 1492 FLD_LCORE, 1493 _NUM_FLD 1494 }; 1495 unsigned long int_fld[_NUM_FLD]; 1496 char *str_fld[_NUM_FLD]; 1497 int32_t i; 1498 uint32_t size; 1499 1500 nb_lcore_params = 0; 1501 1502 while ((p = strchr(p0, '(')) != NULL) { 1503 ++p; 1504 p0 = strchr(p, ')'); 1505 if (p0 == NULL) 1506 return -1; 1507 1508 size = p0 - p; 1509 if (size >= sizeof(s)) 1510 return -1; 1511 1512 snprintf(s, sizeof(s), "%.*s", size, p); 1513 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1514 _NUM_FLD) 1515 return -1; 1516 for (i = 0; i < _NUM_FLD; i++) { 1517 errno = 0; 1518 int_fld[i] = strtoul(str_fld[i], &end, 0); 1519 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1520 return -1; 1521 } 1522 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1523 printf("exceeded max number of lcore params: %hu\n", 1524 nb_lcore_params); 1525 return -1; 1526 } 1527 lcore_params_array[nb_lcore_params].port_id = 1528 (uint8_t)int_fld[FLD_PORT]; 1529 lcore_params_array[nb_lcore_params].queue_id = 1530 (uint8_t)int_fld[FLD_QUEUE]; 1531 lcore_params_array[nb_lcore_params].lcore_id = 1532 (uint8_t)int_fld[FLD_LCORE]; 1533 ++nb_lcore_params; 1534 } 1535 lcore_params = lcore_params_array; 1536 return 0; 1537 } 1538 1539 static void 1540 print_app_sa_prm(const struct app_sa_prm *prm) 1541 { 1542 printf("librte_ipsec usage: %s\n", 1543 (prm->enable == 0) ? "disabled" : "enabled"); 1544 1545 printf("replay window size: %u\n", prm->window_size); 1546 printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled"); 1547 printf("SA flags: %#" PRIx64 "\n", prm->flags); 1548 printf("Frag TTL: %" PRIu64 " ns\n", frag_ttl_ns); 1549 } 1550 1551 static int 1552 parse_transfer_mode(struct eh_conf *conf, const char *optarg) 1553 { 1554 if (!strcmp(CMD_LINE_ARG_POLL, optarg)) 1555 conf->mode = EH_PKT_TRANSFER_MODE_POLL; 1556 else if (!strcmp(CMD_LINE_ARG_EVENT, optarg)) 1557 conf->mode = EH_PKT_TRANSFER_MODE_EVENT; 1558 else { 1559 printf("Unsupported packet transfer mode\n"); 1560 return -EINVAL; 1561 } 1562 1563 return 0; 1564 } 1565 1566 static int 1567 parse_schedule_type(struct eh_conf *conf, const char *optarg) 1568 { 1569 struct eventmode_conf *em_conf = NULL; 1570 1571 /* Get eventmode conf */ 1572 em_conf = conf->mode_params; 1573 1574 if (!strcmp(CMD_LINE_ARG_ORDERED, optarg)) 1575 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED; 1576 else if (!strcmp(CMD_LINE_ARG_ATOMIC, optarg)) 1577 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ATOMIC; 1578 else if (!strcmp(CMD_LINE_ARG_PARALLEL, optarg)) 1579 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_PARALLEL; 1580 else { 1581 printf("Unsupported queue schedule type\n"); 1582 return -EINVAL; 1583 } 1584 1585 return 0; 1586 } 1587 1588 static int32_t 1589 parse_args(int32_t argc, char **argv, struct eh_conf *eh_conf) 1590 { 1591 int opt; 1592 int64_t ret; 1593 char **argvopt; 1594 int32_t option_index; 1595 char *prgname = argv[0]; 1596 int32_t f_present = 0; 1597 1598 argvopt = argv; 1599 1600 while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:c:s:", 1601 lgopts, &option_index)) != EOF) { 1602 1603 switch (opt) { 1604 case 'p': 1605 enabled_port_mask = parse_portmask(optarg); 1606 if (enabled_port_mask == 0) { 1607 printf("invalid portmask\n"); 1608 print_usage(prgname); 1609 return -1; 1610 } 1611 break; 1612 case 'P': 1613 printf("Promiscuous mode selected\n"); 1614 promiscuous_on = 1; 1615 break; 1616 case 'u': 1617 unprotected_port_mask = parse_portmask(optarg); 1618 if (unprotected_port_mask == 0) { 1619 printf("invalid unprotected portmask\n"); 1620 print_usage(prgname); 1621 return -1; 1622 } 1623 break; 1624 case 'f': 1625 if (f_present == 1) { 1626 printf("\"-f\" option present more than " 1627 "once!\n"); 1628 print_usage(prgname); 1629 return -1; 1630 } 1631 cfgfile = optarg; 1632 f_present = 1; 1633 break; 1634 1635 case 's': 1636 ret = parse_decimal(optarg); 1637 if (ret < 0) { 1638 printf("Invalid number of buffers in a pool: " 1639 "%s\n", optarg); 1640 print_usage(prgname); 1641 return -1; 1642 } 1643 1644 nb_bufs_in_pool = ret; 1645 break; 1646 1647 case 'j': 1648 ret = parse_decimal(optarg); 1649 if (ret < RTE_MBUF_DEFAULT_BUF_SIZE || 1650 ret > UINT16_MAX) { 1651 printf("Invalid frame buffer size value: %s\n", 1652 optarg); 1653 print_usage(prgname); 1654 return -1; 1655 } 1656 frame_buf_size = ret; 1657 printf("Custom frame buffer size %u\n", frame_buf_size); 1658 break; 1659 case 'l': 1660 app_sa_prm.enable = 1; 1661 break; 1662 case 'w': 1663 app_sa_prm.window_size = parse_decimal(optarg); 1664 break; 1665 case 'e': 1666 app_sa_prm.enable_esn = 1; 1667 break; 1668 case 'a': 1669 app_sa_prm.enable = 1; 1670 app_sa_prm.flags |= RTE_IPSEC_SAFLAG_SQN_ATOM; 1671 break; 1672 case 'c': 1673 ret = parse_decimal(optarg); 1674 if (ret < 0) { 1675 printf("Invalid SA cache size: %s\n", optarg); 1676 print_usage(prgname); 1677 return -1; 1678 } 1679 app_sa_prm.cache_sz = ret; 1680 break; 1681 case CMD_LINE_OPT_CONFIG_NUM: 1682 ret = parse_config(optarg); 1683 if (ret) { 1684 printf("Invalid config\n"); 1685 print_usage(prgname); 1686 return -1; 1687 } 1688 break; 1689 case CMD_LINE_OPT_SINGLE_SA_NUM: 1690 ret = parse_decimal(optarg); 1691 if (ret == -1 || ret > UINT32_MAX) { 1692 printf("Invalid argument[sa_idx]\n"); 1693 print_usage(prgname); 1694 return -1; 1695 } 1696 1697 /* else */ 1698 single_sa = 1; 1699 single_sa_idx = ret; 1700 eh_conf->ipsec_mode = EH_IPSEC_MODE_TYPE_DRIVER; 1701 printf("Configured with single SA index %u\n", 1702 single_sa_idx); 1703 break; 1704 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM: 1705 ret = parse_portmask(optarg); 1706 if (ret == -1) { 1707 printf("Invalid argument[portmask]\n"); 1708 print_usage(prgname); 1709 return -1; 1710 } 1711 1712 /* else */ 1713 enabled_cryptodev_mask = ret; 1714 break; 1715 1716 case CMD_LINE_OPT_TRANSFER_MODE_NUM: 1717 ret = parse_transfer_mode(eh_conf, optarg); 1718 if (ret < 0) { 1719 printf("Invalid packet transfer mode\n"); 1720 print_usage(prgname); 1721 return -1; 1722 } 1723 break; 1724 1725 case CMD_LINE_OPT_SCHEDULE_TYPE_NUM: 1726 ret = parse_schedule_type(eh_conf, optarg); 1727 if (ret < 0) { 1728 printf("Invalid queue schedule type\n"); 1729 print_usage(prgname); 1730 return -1; 1731 } 1732 break; 1733 1734 case CMD_LINE_OPT_RX_OFFLOAD_NUM: 1735 ret = parse_mask(optarg, &dev_rx_offload); 1736 if (ret != 0) { 1737 printf("Invalid argument for \'%s\': %s\n", 1738 CMD_LINE_OPT_RX_OFFLOAD, optarg); 1739 print_usage(prgname); 1740 return -1; 1741 } 1742 break; 1743 case CMD_LINE_OPT_TX_OFFLOAD_NUM: 1744 ret = parse_mask(optarg, &dev_tx_offload); 1745 if (ret != 0) { 1746 printf("Invalid argument for \'%s\': %s\n", 1747 CMD_LINE_OPT_TX_OFFLOAD, optarg); 1748 print_usage(prgname); 1749 return -1; 1750 } 1751 break; 1752 case CMD_LINE_OPT_REASSEMBLE_NUM: 1753 ret = parse_decimal(optarg); 1754 if (ret < 0 || ret > UINT32_MAX) { 1755 printf("Invalid argument for \'%s\': %s\n", 1756 CMD_LINE_OPT_REASSEMBLE, optarg); 1757 print_usage(prgname); 1758 return -1; 1759 } 1760 frag_tbl_sz = ret; 1761 break; 1762 case CMD_LINE_OPT_MTU_NUM: 1763 ret = parse_decimal(optarg); 1764 if (ret < 0 || ret > RTE_IPV4_MAX_PKT_LEN) { 1765 printf("Invalid argument for \'%s\': %s\n", 1766 CMD_LINE_OPT_MTU, optarg); 1767 print_usage(prgname); 1768 return -1; 1769 } 1770 mtu_size = ret; 1771 break; 1772 case CMD_LINE_OPT_FRAG_TTL_NUM: 1773 ret = parse_decimal(optarg); 1774 if (ret < 0 || ret > MAX_FRAG_TTL_NS) { 1775 printf("Invalid argument for \'%s\': %s\n", 1776 CMD_LINE_OPT_MTU, optarg); 1777 print_usage(prgname); 1778 return -1; 1779 } 1780 frag_ttl_ns = ret; 1781 break; 1782 default: 1783 print_usage(prgname); 1784 return -1; 1785 } 1786 } 1787 1788 if (f_present == 0) { 1789 printf("Mandatory option \"-f\" not present\n"); 1790 return -1; 1791 } 1792 1793 /* check do we need to enable multi-seg support */ 1794 if (multi_seg_required()) { 1795 /* legacy mode doesn't support multi-seg */ 1796 app_sa_prm.enable = 1; 1797 printf("frame buf size: %u, mtu: %u, " 1798 "number of reassemble entries: %u\n" 1799 "multi-segment support is required\n", 1800 frame_buf_size, mtu_size, frag_tbl_sz); 1801 } 1802 1803 print_app_sa_prm(&app_sa_prm); 1804 1805 if (optind >= 0) 1806 argv[optind-1] = prgname; 1807 1808 ret = optind-1; 1809 optind = 1; /* reset getopt lib */ 1810 return ret; 1811 } 1812 1813 static void 1814 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) 1815 { 1816 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 1817 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); 1818 printf("%s%s", name, buf); 1819 } 1820 1821 /* 1822 * Update destination ethaddr for the port. 1823 */ 1824 int 1825 add_dst_ethaddr(uint16_t port, const struct rte_ether_addr *addr) 1826 { 1827 if (port >= RTE_DIM(ethaddr_tbl)) 1828 return -EINVAL; 1829 1830 ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr); 1831 return 0; 1832 } 1833 1834 /* Check the link status of all ports in up to 9s, and print them finally */ 1835 static void 1836 check_all_ports_link_status(uint32_t port_mask) 1837 { 1838 #define CHECK_INTERVAL 100 /* 100ms */ 1839 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1840 uint16_t portid; 1841 uint8_t count, all_ports_up, print_flag = 0; 1842 struct rte_eth_link link; 1843 int ret; 1844 1845 printf("\nChecking link status"); 1846 fflush(stdout); 1847 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1848 all_ports_up = 1; 1849 RTE_ETH_FOREACH_DEV(portid) { 1850 if ((port_mask & (1 << portid)) == 0) 1851 continue; 1852 memset(&link, 0, sizeof(link)); 1853 ret = rte_eth_link_get_nowait(portid, &link); 1854 if (ret < 0) { 1855 all_ports_up = 0; 1856 if (print_flag == 1) 1857 printf("Port %u link get failed: %s\n", 1858 portid, rte_strerror(-ret)); 1859 continue; 1860 } 1861 /* print link status if flag set */ 1862 if (print_flag == 1) { 1863 if (link.link_status) 1864 printf( 1865 "Port%d Link Up - speed %u Mbps -%s\n", 1866 portid, link.link_speed, 1867 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1868 ("full-duplex") : ("half-duplex")); 1869 else 1870 printf("Port %d Link Down\n", portid); 1871 continue; 1872 } 1873 /* clear all_ports_up flag if any link down */ 1874 if (link.link_status == ETH_LINK_DOWN) { 1875 all_ports_up = 0; 1876 break; 1877 } 1878 } 1879 /* after finally printing all link status, get out */ 1880 if (print_flag == 1) 1881 break; 1882 1883 if (all_ports_up == 0) { 1884 printf("."); 1885 fflush(stdout); 1886 rte_delay_ms(CHECK_INTERVAL); 1887 } 1888 1889 /* set the print_flag if all ports up or timeout */ 1890 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1891 print_flag = 1; 1892 printf("done\n"); 1893 } 1894 } 1895 } 1896 1897 static int32_t 1898 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1899 uint16_t qp, struct lcore_params *params, 1900 struct ipsec_ctx *ipsec_ctx, 1901 const struct rte_cryptodev_capabilities *cipher, 1902 const struct rte_cryptodev_capabilities *auth, 1903 const struct rte_cryptodev_capabilities *aead) 1904 { 1905 int32_t ret = 0; 1906 unsigned long i; 1907 struct cdev_key key = { 0 }; 1908 1909 key.lcore_id = params->lcore_id; 1910 if (cipher) 1911 key.cipher_algo = cipher->sym.cipher.algo; 1912 if (auth) 1913 key.auth_algo = auth->sym.auth.algo; 1914 if (aead) 1915 key.aead_algo = aead->sym.aead.algo; 1916 1917 ret = rte_hash_lookup(map, &key); 1918 if (ret != -ENOENT) 1919 return 0; 1920 1921 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1922 if (ipsec_ctx->tbl[i].id == cdev_id) 1923 break; 1924 1925 if (i == ipsec_ctx->nb_qps) { 1926 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1927 printf("Maximum number of crypto devices assigned to " 1928 "a core, increase MAX_QP_PER_LCORE value\n"); 1929 return 0; 1930 } 1931 ipsec_ctx->tbl[i].id = cdev_id; 1932 ipsec_ctx->tbl[i].qp = qp; 1933 ipsec_ctx->nb_qps++; 1934 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1935 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1936 cdev_id, qp, i); 1937 } 1938 1939 ret = rte_hash_add_key_data(map, &key, (void *)i); 1940 if (ret < 0) { 1941 printf("Faled to insert cdev mapping for (lcore %u, " 1942 "cdev %u, qp %u), errno %d\n", 1943 key.lcore_id, ipsec_ctx->tbl[i].id, 1944 ipsec_ctx->tbl[i].qp, ret); 1945 return 0; 1946 } 1947 1948 return 1; 1949 } 1950 1951 static int32_t 1952 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1953 uint16_t qp, struct lcore_params *params) 1954 { 1955 int32_t ret = 0; 1956 const struct rte_cryptodev_capabilities *i, *j; 1957 struct rte_hash *map; 1958 struct lcore_conf *qconf; 1959 struct ipsec_ctx *ipsec_ctx; 1960 const char *str; 1961 1962 qconf = &lcore_conf[params->lcore_id]; 1963 1964 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 1965 map = cdev_map_out; 1966 ipsec_ctx = &qconf->outbound; 1967 str = "Outbound"; 1968 } else { 1969 map = cdev_map_in; 1970 ipsec_ctx = &qconf->inbound; 1971 str = "Inbound"; 1972 } 1973 1974 /* Required cryptodevs with operation chainning */ 1975 if (!(dev_info->feature_flags & 1976 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 1977 return ret; 1978 1979 for (i = dev_info->capabilities; 1980 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 1981 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1982 continue; 1983 1984 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1985 ret |= add_mapping(map, str, cdev_id, qp, params, 1986 ipsec_ctx, NULL, NULL, i); 1987 continue; 1988 } 1989 1990 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 1991 continue; 1992 1993 for (j = dev_info->capabilities; 1994 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 1995 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1996 continue; 1997 1998 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 1999 continue; 2000 2001 ret |= add_mapping(map, str, cdev_id, qp, params, 2002 ipsec_ctx, i, j, NULL); 2003 } 2004 } 2005 2006 return ret; 2007 } 2008 2009 /* Check if the device is enabled by cryptodev_mask */ 2010 static int 2011 check_cryptodev_mask(uint8_t cdev_id) 2012 { 2013 if (enabled_cryptodev_mask & (1 << cdev_id)) 2014 return 0; 2015 2016 return -1; 2017 } 2018 2019 static uint16_t 2020 cryptodevs_init(uint16_t req_queue_num) 2021 { 2022 struct rte_cryptodev_config dev_conf; 2023 struct rte_cryptodev_qp_conf qp_conf; 2024 uint16_t idx, max_nb_qps, qp, total_nb_qps, i; 2025 int16_t cdev_id; 2026 struct rte_hash_parameters params = { 0 }; 2027 2028 const uint64_t mseg_flag = multi_seg_required() ? 2029 RTE_CRYPTODEV_FF_IN_PLACE_SGL : 0; 2030 2031 params.entries = CDEV_MAP_ENTRIES; 2032 params.key_len = sizeof(struct cdev_key); 2033 params.hash_func = rte_jhash; 2034 params.hash_func_init_val = 0; 2035 params.socket_id = rte_socket_id(); 2036 2037 params.name = "cdev_map_in"; 2038 cdev_map_in = rte_hash_create(¶ms); 2039 if (cdev_map_in == NULL) 2040 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 2041 rte_errno); 2042 2043 params.name = "cdev_map_out"; 2044 cdev_map_out = rte_hash_create(¶ms); 2045 if (cdev_map_out == NULL) 2046 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 2047 rte_errno); 2048 2049 printf("lcore/cryptodev/qp mappings:\n"); 2050 2051 idx = 0; 2052 total_nb_qps = 0; 2053 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 2054 struct rte_cryptodev_info cdev_info; 2055 2056 if (check_cryptodev_mask((uint8_t)cdev_id)) 2057 continue; 2058 2059 rte_cryptodev_info_get(cdev_id, &cdev_info); 2060 2061 if ((mseg_flag & cdev_info.feature_flags) != mseg_flag) 2062 rte_exit(EXIT_FAILURE, 2063 "Device %hd does not support \'%s\' feature\n", 2064 cdev_id, 2065 rte_cryptodev_get_feature_name(mseg_flag)); 2066 2067 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 2068 max_nb_qps = cdev_info.max_nb_queue_pairs; 2069 else 2070 max_nb_qps = nb_lcore_params; 2071 2072 qp = 0; 2073 i = 0; 2074 while (qp < max_nb_qps && i < nb_lcore_params) { 2075 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 2076 &lcore_params[idx])) 2077 qp++; 2078 idx++; 2079 idx = idx % nb_lcore_params; 2080 i++; 2081 } 2082 2083 qp = RTE_MIN(max_nb_qps, RTE_MAX(req_queue_num, qp)); 2084 if (qp == 0) 2085 continue; 2086 2087 total_nb_qps += qp; 2088 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 2089 dev_conf.nb_queue_pairs = qp; 2090 dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO; 2091 2092 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions; 2093 if (dev_max_sess != 0 && 2094 dev_max_sess < get_nb_crypto_sessions()) 2095 rte_exit(EXIT_FAILURE, 2096 "Device does not support at least %u " 2097 "sessions", get_nb_crypto_sessions()); 2098 2099 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 2100 rte_panic("Failed to initialize cryptodev %u\n", 2101 cdev_id); 2102 2103 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 2104 qp_conf.mp_session = 2105 socket_ctx[dev_conf.socket_id].session_pool; 2106 qp_conf.mp_session_private = 2107 socket_ctx[dev_conf.socket_id].session_priv_pool; 2108 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 2109 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 2110 &qp_conf, dev_conf.socket_id)) 2111 rte_panic("Failed to setup queue %u for " 2112 "cdev_id %u\n", 0, cdev_id); 2113 2114 if (rte_cryptodev_start(cdev_id)) 2115 rte_panic("Failed to start cryptodev %u\n", 2116 cdev_id); 2117 } 2118 2119 printf("\n"); 2120 2121 return total_nb_qps; 2122 } 2123 2124 static void 2125 port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads) 2126 { 2127 uint32_t frame_size; 2128 struct rte_eth_dev_info dev_info; 2129 struct rte_eth_txconf *txconf; 2130 uint16_t nb_tx_queue, nb_rx_queue; 2131 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 2132 int32_t ret, socket_id; 2133 struct lcore_conf *qconf; 2134 struct rte_ether_addr ethaddr; 2135 struct rte_eth_conf local_port_conf = port_conf; 2136 2137 ret = rte_eth_dev_info_get(portid, &dev_info); 2138 if (ret != 0) 2139 rte_exit(EXIT_FAILURE, 2140 "Error during getting device (port %u) info: %s\n", 2141 portid, strerror(-ret)); 2142 2143 /* limit allowed HW offloafs, as user requested */ 2144 dev_info.rx_offload_capa &= dev_rx_offload; 2145 dev_info.tx_offload_capa &= dev_tx_offload; 2146 2147 printf("Configuring device port %u:\n", portid); 2148 2149 ret = rte_eth_macaddr_get(portid, ðaddr); 2150 if (ret != 0) 2151 rte_exit(EXIT_FAILURE, 2152 "Error getting MAC address (port %u): %s\n", 2153 portid, rte_strerror(-ret)); 2154 2155 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ðaddr); 2156 print_ethaddr("Address: ", ðaddr); 2157 printf("\n"); 2158 2159 nb_rx_queue = get_port_nb_rx_queues(portid); 2160 nb_tx_queue = nb_lcores; 2161 2162 if (nb_rx_queue > dev_info.max_rx_queues) 2163 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 2164 "(max rx queue is %u)\n", 2165 nb_rx_queue, dev_info.max_rx_queues); 2166 2167 if (nb_tx_queue > dev_info.max_tx_queues) 2168 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 2169 "(max tx queue is %u)\n", 2170 nb_tx_queue, dev_info.max_tx_queues); 2171 2172 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 2173 nb_rx_queue, nb_tx_queue); 2174 2175 frame_size = MTU_TO_FRAMELEN(mtu_size); 2176 if (frame_size > local_port_conf.rxmode.max_rx_pkt_len) 2177 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 2178 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 2179 2180 if (multi_seg_required()) { 2181 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SCATTER; 2182 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS; 2183 } 2184 2185 local_port_conf.rxmode.offloads |= req_rx_offloads; 2186 local_port_conf.txmode.offloads |= req_tx_offloads; 2187 2188 /* Check that all required capabilities are supported */ 2189 if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) != 2190 local_port_conf.rxmode.offloads) 2191 rte_exit(EXIT_FAILURE, 2192 "Error: port %u required RX offloads: 0x%" PRIx64 2193 ", avaialbe RX offloads: 0x%" PRIx64 "\n", 2194 portid, local_port_conf.rxmode.offloads, 2195 dev_info.rx_offload_capa); 2196 2197 if ((local_port_conf.txmode.offloads & dev_info.tx_offload_capa) != 2198 local_port_conf.txmode.offloads) 2199 rte_exit(EXIT_FAILURE, 2200 "Error: port %u required TX offloads: 0x%" PRIx64 2201 ", avaialbe TX offloads: 0x%" PRIx64 "\n", 2202 portid, local_port_conf.txmode.offloads, 2203 dev_info.tx_offload_capa); 2204 2205 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 2206 local_port_conf.txmode.offloads |= 2207 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 2208 2209 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 2210 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 2211 2212 printf("port %u configurng rx_offloads=0x%" PRIx64 2213 ", tx_offloads=0x%" PRIx64 "\n", 2214 portid, local_port_conf.rxmode.offloads, 2215 local_port_conf.txmode.offloads); 2216 2217 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 2218 dev_info.flow_type_rss_offloads; 2219 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 2220 port_conf.rx_adv_conf.rss_conf.rss_hf) { 2221 printf("Port %u modified RSS hash function based on hardware support," 2222 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 2223 portid, 2224 port_conf.rx_adv_conf.rss_conf.rss_hf, 2225 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 2226 } 2227 2228 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 2229 &local_port_conf); 2230 if (ret < 0) 2231 rte_exit(EXIT_FAILURE, "Cannot configure device: " 2232 "err=%d, port=%d\n", ret, portid); 2233 2234 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 2235 if (ret < 0) 2236 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 2237 "err=%d, port=%d\n", ret, portid); 2238 2239 /* init one TX queue per lcore */ 2240 tx_queueid = 0; 2241 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2242 if (rte_lcore_is_enabled(lcore_id) == 0) 2243 continue; 2244 2245 if (numa_on) 2246 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2247 else 2248 socket_id = 0; 2249 2250 /* init TX queue */ 2251 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 2252 2253 txconf = &dev_info.default_txconf; 2254 txconf->offloads = local_port_conf.txmode.offloads; 2255 2256 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 2257 socket_id, txconf); 2258 if (ret < 0) 2259 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 2260 "err=%d, port=%d\n", ret, portid); 2261 2262 qconf = &lcore_conf[lcore_id]; 2263 qconf->tx_queue_id[portid] = tx_queueid; 2264 2265 /* Pre-populate pkt offloads based on capabilities */ 2266 qconf->outbound.ipv4_offloads = PKT_TX_IPV4; 2267 qconf->outbound.ipv6_offloads = PKT_TX_IPV6; 2268 if (local_port_conf.txmode.offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 2269 qconf->outbound.ipv4_offloads |= PKT_TX_IP_CKSUM; 2270 2271 tx_queueid++; 2272 2273 /* init RX queues */ 2274 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 2275 struct rte_eth_rxconf rxq_conf; 2276 2277 if (portid != qconf->rx_queue_list[queue].port_id) 2278 continue; 2279 2280 rx_queueid = qconf->rx_queue_list[queue].queue_id; 2281 2282 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 2283 socket_id); 2284 2285 rxq_conf = dev_info.default_rxconf; 2286 rxq_conf.offloads = local_port_conf.rxmode.offloads; 2287 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 2288 nb_rxd, socket_id, &rxq_conf, 2289 socket_ctx[socket_id].mbuf_pool); 2290 if (ret < 0) 2291 rte_exit(EXIT_FAILURE, 2292 "rte_eth_rx_queue_setup: err=%d, " 2293 "port=%d\n", ret, portid); 2294 } 2295 } 2296 printf("\n"); 2297 } 2298 2299 static size_t 2300 max_session_size(void) 2301 { 2302 size_t max_sz, sz; 2303 void *sec_ctx; 2304 int16_t cdev_id, port_id, n; 2305 2306 max_sz = 0; 2307 n = rte_cryptodev_count(); 2308 for (cdev_id = 0; cdev_id != n; cdev_id++) { 2309 sz = rte_cryptodev_sym_get_private_session_size(cdev_id); 2310 if (sz > max_sz) 2311 max_sz = sz; 2312 /* 2313 * If crypto device is security capable, need to check the 2314 * size of security session as well. 2315 */ 2316 2317 /* Get security context of the crypto device */ 2318 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id); 2319 if (sec_ctx == NULL) 2320 continue; 2321 2322 /* Get size of security session */ 2323 sz = rte_security_session_get_size(sec_ctx); 2324 if (sz > max_sz) 2325 max_sz = sz; 2326 } 2327 2328 RTE_ETH_FOREACH_DEV(port_id) { 2329 if ((enabled_port_mask & (1 << port_id)) == 0) 2330 continue; 2331 2332 sec_ctx = rte_eth_dev_get_sec_ctx(port_id); 2333 if (sec_ctx == NULL) 2334 continue; 2335 2336 sz = rte_security_session_get_size(sec_ctx); 2337 if (sz > max_sz) 2338 max_sz = sz; 2339 } 2340 2341 return max_sz; 2342 } 2343 2344 static void 2345 session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz) 2346 { 2347 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2348 struct rte_mempool *sess_mp; 2349 uint32_t nb_sess; 2350 2351 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2352 "sess_mp_%u", socket_id); 2353 /* 2354 * Doubled due to rte_security_session_create() uses one mempool for 2355 * session and for session private data. 2356 */ 2357 nb_sess = (get_nb_crypto_sessions() + CDEV_MP_CACHE_SZ * 2358 rte_lcore_count()) * 2; 2359 sess_mp = rte_cryptodev_sym_session_pool_create( 2360 mp_name, nb_sess, sess_sz, CDEV_MP_CACHE_SZ, 0, 2361 socket_id); 2362 ctx->session_pool = sess_mp; 2363 2364 if (ctx->session_pool == NULL) 2365 rte_exit(EXIT_FAILURE, 2366 "Cannot init session pool on socket %d\n", socket_id); 2367 else 2368 printf("Allocated session pool on socket %d\n", socket_id); 2369 } 2370 2371 static void 2372 session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id, 2373 size_t sess_sz) 2374 { 2375 char mp_name[RTE_MEMPOOL_NAMESIZE]; 2376 struct rte_mempool *sess_mp; 2377 uint32_t nb_sess; 2378 2379 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 2380 "sess_mp_priv_%u", socket_id); 2381 /* 2382 * Doubled due to rte_security_session_create() uses one mempool for 2383 * session and for session private data. 2384 */ 2385 nb_sess = (get_nb_crypto_sessions() + CDEV_MP_CACHE_SZ * 2386 rte_lcore_count()) * 2; 2387 sess_mp = rte_mempool_create(mp_name, 2388 nb_sess, 2389 sess_sz, 2390 CDEV_MP_CACHE_SZ, 2391 0, NULL, NULL, NULL, 2392 NULL, socket_id, 2393 0); 2394 ctx->session_priv_pool = sess_mp; 2395 2396 if (ctx->session_priv_pool == NULL) 2397 rte_exit(EXIT_FAILURE, 2398 "Cannot init session priv pool on socket %d\n", 2399 socket_id); 2400 else 2401 printf("Allocated session priv pool on socket %d\n", 2402 socket_id); 2403 } 2404 2405 static void 2406 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 2407 { 2408 char s[64]; 2409 int32_t ms; 2410 2411 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 2412 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 2413 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 2414 frame_buf_size, socket_id); 2415 2416 /* 2417 * if multi-segment support is enabled, then create a pool 2418 * for indirect mbufs. 2419 */ 2420 ms = multi_seg_required(); 2421 if (ms != 0) { 2422 snprintf(s, sizeof(s), "mbuf_pool_indir_%d", socket_id); 2423 ctx->mbuf_pool_indir = rte_pktmbuf_pool_create(s, nb_mbuf, 2424 MEMPOOL_CACHE_SIZE, 0, 0, socket_id); 2425 } 2426 2427 if (ctx->mbuf_pool == NULL || (ms != 0 && ctx->mbuf_pool_indir == NULL)) 2428 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 2429 socket_id); 2430 else 2431 printf("Allocated mbuf pool on socket %d\n", socket_id); 2432 } 2433 2434 static inline int 2435 inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md) 2436 { 2437 struct ipsec_sa *sa; 2438 2439 /* For inline protocol processing, the metadata in the event will 2440 * uniquely identify the security session which raised the event. 2441 * Application would then need the userdata it had registered with the 2442 * security session to process the event. 2443 */ 2444 2445 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md); 2446 2447 if (sa == NULL) { 2448 /* userdata could not be retrieved */ 2449 return -1; 2450 } 2451 2452 /* Sequence number over flow. SA need to be re-established */ 2453 RTE_SET_USED(sa); 2454 return 0; 2455 } 2456 2457 static int 2458 inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type, 2459 void *param, void *ret_param) 2460 { 2461 uint64_t md; 2462 struct rte_eth_event_ipsec_desc *event_desc = NULL; 2463 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 2464 rte_eth_dev_get_sec_ctx(port_id); 2465 2466 RTE_SET_USED(param); 2467 2468 if (type != RTE_ETH_EVENT_IPSEC) 2469 return -1; 2470 2471 event_desc = ret_param; 2472 if (event_desc == NULL) { 2473 printf("Event descriptor not set\n"); 2474 return -1; 2475 } 2476 2477 md = event_desc->metadata; 2478 2479 if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW) 2480 return inline_ipsec_event_esn_overflow(ctx, md); 2481 else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) { 2482 printf("Invalid IPsec event reported\n"); 2483 return -1; 2484 } 2485 2486 return -1; 2487 } 2488 2489 static uint16_t 2490 rx_callback(__rte_unused uint16_t port, __rte_unused uint16_t queue, 2491 struct rte_mbuf *pkt[], uint16_t nb_pkts, 2492 __rte_unused uint16_t max_pkts, void *user_param) 2493 { 2494 uint64_t tm; 2495 uint32_t i, k; 2496 struct lcore_conf *lc; 2497 struct rte_mbuf *mb; 2498 struct rte_ether_hdr *eth; 2499 2500 lc = user_param; 2501 k = 0; 2502 tm = 0; 2503 2504 for (i = 0; i != nb_pkts; i++) { 2505 2506 mb = pkt[i]; 2507 eth = rte_pktmbuf_mtod(mb, struct rte_ether_hdr *); 2508 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 2509 2510 struct rte_ipv4_hdr *iph; 2511 2512 iph = (struct rte_ipv4_hdr *)(eth + 1); 2513 if (rte_ipv4_frag_pkt_is_fragmented(iph)) { 2514 2515 mb->l2_len = sizeof(*eth); 2516 mb->l3_len = sizeof(*iph); 2517 tm = (tm != 0) ? tm : rte_rdtsc(); 2518 mb = rte_ipv4_frag_reassemble_packet( 2519 lc->frag.tbl, &lc->frag.dr, 2520 mb, tm, iph); 2521 2522 if (mb != NULL) { 2523 /* fix ip cksum after reassemble. */ 2524 iph = rte_pktmbuf_mtod_offset(mb, 2525 struct rte_ipv4_hdr *, 2526 mb->l2_len); 2527 iph->hdr_checksum = 0; 2528 iph->hdr_checksum = rte_ipv4_cksum(iph); 2529 } 2530 } 2531 } else if (eth->ether_type == 2532 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 2533 2534 struct rte_ipv6_hdr *iph; 2535 struct ipv6_extension_fragment *fh; 2536 2537 iph = (struct rte_ipv6_hdr *)(eth + 1); 2538 fh = rte_ipv6_frag_get_ipv6_fragment_header(iph); 2539 if (fh != NULL) { 2540 mb->l2_len = sizeof(*eth); 2541 mb->l3_len = (uintptr_t)fh - (uintptr_t)iph + 2542 sizeof(*fh); 2543 tm = (tm != 0) ? tm : rte_rdtsc(); 2544 mb = rte_ipv6_frag_reassemble_packet( 2545 lc->frag.tbl, &lc->frag.dr, 2546 mb, tm, iph, fh); 2547 if (mb != NULL) 2548 /* fix l3_len after reassemble. */ 2549 mb->l3_len = mb->l3_len - sizeof(*fh); 2550 } 2551 } 2552 2553 pkt[k] = mb; 2554 k += (mb != NULL); 2555 } 2556 2557 /* some fragments were encountered, drain death row */ 2558 if (tm != 0) 2559 rte_ip_frag_free_death_row(&lc->frag.dr, 0); 2560 2561 return k; 2562 } 2563 2564 2565 static int 2566 reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid) 2567 { 2568 int32_t sid; 2569 uint32_t i; 2570 uint64_t frag_cycles; 2571 const struct lcore_rx_queue *rxq; 2572 const struct rte_eth_rxtx_callback *cb; 2573 2574 /* create fragment table */ 2575 sid = rte_lcore_to_socket_id(cid); 2576 frag_cycles = (rte_get_tsc_hz() + NS_PER_S - 1) / 2577 NS_PER_S * frag_ttl_ns; 2578 2579 lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz, 2580 FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid); 2581 if (lc->frag.tbl == NULL) { 2582 printf("%s(%u): failed to create fragment table of size: %u, " 2583 "error code: %d\n", 2584 __func__, cid, frag_tbl_sz, rte_errno); 2585 return -ENOMEM; 2586 } 2587 2588 /* setup reassemble RX callbacks for all queues */ 2589 for (i = 0; i != lc->nb_rx_queue; i++) { 2590 2591 rxq = lc->rx_queue_list + i; 2592 cb = rte_eth_add_rx_callback(rxq->port_id, rxq->queue_id, 2593 rx_callback, lc); 2594 if (cb == NULL) { 2595 printf("%s(%u): failed to install RX callback for " 2596 "portid=%u, queueid=%u, error code: %d\n", 2597 __func__, cid, 2598 rxq->port_id, rxq->queue_id, rte_errno); 2599 return -ENOMEM; 2600 } 2601 } 2602 2603 return 0; 2604 } 2605 2606 static int 2607 reassemble_init(void) 2608 { 2609 int32_t rc; 2610 uint32_t i, lc; 2611 2612 rc = 0; 2613 for (i = 0; i != nb_lcore_params; i++) { 2614 lc = lcore_params[i].lcore_id; 2615 rc = reassemble_lcore_init(lcore_conf + lc, lc); 2616 if (rc != 0) 2617 break; 2618 } 2619 2620 return rc; 2621 } 2622 2623 static void 2624 create_default_ipsec_flow(uint16_t port_id, uint64_t rx_offloads) 2625 { 2626 struct rte_flow_action action[2]; 2627 struct rte_flow_item pattern[2]; 2628 struct rte_flow_attr attr = {0}; 2629 struct rte_flow_error err; 2630 struct rte_flow *flow; 2631 int ret; 2632 2633 if (!(rx_offloads & DEV_RX_OFFLOAD_SECURITY)) 2634 return; 2635 2636 /* Add the default rte_flow to enable SECURITY for all ESP packets */ 2637 2638 pattern[0].type = RTE_FLOW_ITEM_TYPE_ESP; 2639 pattern[0].spec = NULL; 2640 pattern[0].mask = NULL; 2641 pattern[0].last = NULL; 2642 pattern[1].type = RTE_FLOW_ITEM_TYPE_END; 2643 2644 action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY; 2645 action[0].conf = NULL; 2646 action[1].type = RTE_FLOW_ACTION_TYPE_END; 2647 action[1].conf = NULL; 2648 2649 attr.ingress = 1; 2650 2651 ret = rte_flow_validate(port_id, &attr, pattern, action, &err); 2652 if (ret) 2653 return; 2654 2655 flow = rte_flow_create(port_id, &attr, pattern, action, &err); 2656 if (flow == NULL) 2657 return; 2658 2659 flow_info_tbl[port_id].rx_def_flow = flow; 2660 RTE_LOG(INFO, IPSEC, 2661 "Created default flow enabling SECURITY for all ESP traffic on port %d\n", 2662 port_id); 2663 } 2664 2665 static void 2666 signal_handler(int signum) 2667 { 2668 if (signum == SIGINT || signum == SIGTERM) { 2669 printf("\n\nSignal %d received, preparing to exit...\n", 2670 signum); 2671 force_quit = true; 2672 } 2673 } 2674 2675 static void 2676 ev_mode_sess_verify(struct ipsec_sa *sa, int nb_sa) 2677 { 2678 struct rte_ipsec_session *ips; 2679 int32_t i; 2680 2681 if (!sa || !nb_sa) 2682 return; 2683 2684 for (i = 0; i < nb_sa; i++) { 2685 ips = ipsec_get_primary_session(&sa[i]); 2686 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) 2687 rte_exit(EXIT_FAILURE, "Event mode supports only " 2688 "inline protocol sessions\n"); 2689 } 2690 2691 } 2692 2693 static int32_t 2694 check_event_mode_params(struct eh_conf *eh_conf) 2695 { 2696 struct eventmode_conf *em_conf = NULL; 2697 struct lcore_params *params; 2698 uint16_t portid; 2699 2700 if (!eh_conf || !eh_conf->mode_params) 2701 return -EINVAL; 2702 2703 /* Get eventmode conf */ 2704 em_conf = eh_conf->mode_params; 2705 2706 if (eh_conf->mode == EH_PKT_TRANSFER_MODE_POLL && 2707 em_conf->ext_params.sched_type != SCHED_TYPE_NOT_SET) { 2708 printf("error: option --event-schedule-type applies only to " 2709 "event mode\n"); 2710 return -EINVAL; 2711 } 2712 2713 if (eh_conf->mode != EH_PKT_TRANSFER_MODE_EVENT) 2714 return 0; 2715 2716 /* Set schedule type to ORDERED if it wasn't explicitly set by user */ 2717 if (em_conf->ext_params.sched_type == SCHED_TYPE_NOT_SET) 2718 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED; 2719 2720 /* 2721 * Event mode currently supports only inline protocol sessions. 2722 * If there are other types of sessions configured then exit with 2723 * error. 2724 */ 2725 ev_mode_sess_verify(sa_in, nb_sa_in); 2726 ev_mode_sess_verify(sa_out, nb_sa_out); 2727 2728 2729 /* Option --config does not apply to event mode */ 2730 if (nb_lcore_params > 0) { 2731 printf("error: option --config applies only to poll mode\n"); 2732 return -EINVAL; 2733 } 2734 2735 /* 2736 * In order to use the same port_init routine for both poll and event 2737 * modes initialize lcore_params with one queue for each eth port 2738 */ 2739 lcore_params = lcore_params_array; 2740 RTE_ETH_FOREACH_DEV(portid) { 2741 if ((enabled_port_mask & (1 << portid)) == 0) 2742 continue; 2743 2744 params = &lcore_params[nb_lcore_params++]; 2745 params->port_id = portid; 2746 params->queue_id = 0; 2747 params->lcore_id = rte_get_next_lcore(0, 0, 1); 2748 } 2749 2750 return 0; 2751 } 2752 2753 static void 2754 inline_sessions_free(struct sa_ctx *sa_ctx) 2755 { 2756 struct rte_ipsec_session *ips; 2757 struct ipsec_sa *sa; 2758 int32_t ret; 2759 uint32_t i; 2760 2761 if (!sa_ctx) 2762 return; 2763 2764 for (i = 0; i < sa_ctx->nb_sa; i++) { 2765 2766 sa = &sa_ctx->sa[i]; 2767 if (!sa->spi) 2768 continue; 2769 2770 ips = ipsec_get_primary_session(sa); 2771 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL && 2772 ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) 2773 continue; 2774 2775 if (!rte_eth_dev_is_valid_port(sa->portid)) 2776 continue; 2777 2778 ret = rte_security_session_destroy( 2779 rte_eth_dev_get_sec_ctx(sa->portid), 2780 ips->security.ses); 2781 if (ret) 2782 RTE_LOG(ERR, IPSEC, "Failed to destroy security " 2783 "session type %d, spi %d\n", 2784 ips->type, sa->spi); 2785 } 2786 } 2787 2788 static uint32_t 2789 calculate_nb_mbufs(uint16_t nb_ports, uint16_t nb_crypto_qp, uint32_t nb_rxq, 2790 uint32_t nb_txq) 2791 { 2792 return RTE_MAX((nb_rxq * nb_rxd + 2793 nb_ports * nb_lcores * MAX_PKT_BURST + 2794 nb_ports * nb_txq * nb_txd + 2795 nb_lcores * MEMPOOL_CACHE_SIZE + 2796 nb_crypto_qp * CDEV_QUEUE_DESC + 2797 nb_lcores * frag_tbl_sz * 2798 FRAG_TBL_BUCKET_ENTRIES), 2799 8192U); 2800 } 2801 2802 int32_t 2803 main(int32_t argc, char **argv) 2804 { 2805 int32_t ret; 2806 uint32_t lcore_id, nb_txq, nb_rxq = 0; 2807 uint32_t cdev_id; 2808 uint32_t i; 2809 uint8_t socket_id; 2810 uint16_t portid, nb_crypto_qp, nb_ports = 0; 2811 uint64_t req_rx_offloads[RTE_MAX_ETHPORTS]; 2812 uint64_t req_tx_offloads[RTE_MAX_ETHPORTS]; 2813 struct eh_conf *eh_conf = NULL; 2814 size_t sess_sz; 2815 2816 nb_bufs_in_pool = 0; 2817 2818 /* init EAL */ 2819 ret = rte_eal_init(argc, argv); 2820 if (ret < 0) 2821 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 2822 argc -= ret; 2823 argv += ret; 2824 2825 force_quit = false; 2826 signal(SIGINT, signal_handler); 2827 signal(SIGTERM, signal_handler); 2828 2829 /* initialize event helper configuration */ 2830 eh_conf = eh_conf_init(); 2831 if (eh_conf == NULL) 2832 rte_exit(EXIT_FAILURE, "Failed to init event helper config"); 2833 2834 /* parse application arguments (after the EAL ones) */ 2835 ret = parse_args(argc, argv, eh_conf); 2836 if (ret < 0) 2837 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 2838 2839 /* parse configuration file */ 2840 if (parse_cfg_file(cfgfile) < 0) { 2841 printf("parsing file \"%s\" failed\n", 2842 optarg); 2843 print_usage(argv[0]); 2844 return -1; 2845 } 2846 2847 if ((unprotected_port_mask & enabled_port_mask) != 2848 unprotected_port_mask) 2849 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 2850 unprotected_port_mask); 2851 2852 if (check_poll_mode_params(eh_conf) < 0) 2853 rte_exit(EXIT_FAILURE, "check_poll_mode_params failed\n"); 2854 2855 if (check_event_mode_params(eh_conf) < 0) 2856 rte_exit(EXIT_FAILURE, "check_event_mode_params failed\n"); 2857 2858 ret = init_lcore_rx_queues(); 2859 if (ret < 0) 2860 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 2861 2862 nb_lcores = rte_lcore_count(); 2863 2864 sess_sz = max_session_size(); 2865 2866 /* 2867 * In event mode request minimum number of crypto queues 2868 * to be reserved equal to number of ports. 2869 */ 2870 if (eh_conf->mode == EH_PKT_TRANSFER_MODE_EVENT) 2871 nb_crypto_qp = rte_eth_dev_count_avail(); 2872 else 2873 nb_crypto_qp = 0; 2874 2875 nb_crypto_qp = cryptodevs_init(nb_crypto_qp); 2876 2877 if (nb_bufs_in_pool == 0) { 2878 RTE_ETH_FOREACH_DEV(portid) { 2879 if ((enabled_port_mask & (1 << portid)) == 0) 2880 continue; 2881 nb_ports++; 2882 nb_rxq += get_port_nb_rx_queues(portid); 2883 } 2884 2885 nb_txq = nb_lcores; 2886 2887 nb_bufs_in_pool = calculate_nb_mbufs(nb_ports, nb_crypto_qp, 2888 nb_rxq, nb_txq); 2889 } 2890 2891 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 2892 if (rte_lcore_is_enabled(lcore_id) == 0) 2893 continue; 2894 2895 if (numa_on) 2896 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 2897 else 2898 socket_id = 0; 2899 2900 /* mbuf_pool is initialised by the pool_init() function*/ 2901 if (socket_ctx[socket_id].mbuf_pool) 2902 continue; 2903 2904 pool_init(&socket_ctx[socket_id], socket_id, nb_bufs_in_pool); 2905 session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz); 2906 session_priv_pool_init(&socket_ctx[socket_id], socket_id, 2907 sess_sz); 2908 } 2909 printf("Number of mbufs in packet pool %d\n", nb_bufs_in_pool); 2910 2911 RTE_ETH_FOREACH_DEV(portid) { 2912 if ((enabled_port_mask & (1 << portid)) == 0) 2913 continue; 2914 2915 sa_check_offloads(portid, &req_rx_offloads[portid], 2916 &req_tx_offloads[portid]); 2917 port_init(portid, req_rx_offloads[portid], 2918 req_tx_offloads[portid]); 2919 } 2920 2921 /* 2922 * Set the enabled port mask in helper config for use by helper 2923 * sub-system. This will be used while initializing devices using 2924 * helper sub-system. 2925 */ 2926 eh_conf->eth_portmask = enabled_port_mask; 2927 2928 /* Initialize eventmode components */ 2929 ret = eh_devs_init(eh_conf); 2930 if (ret < 0) 2931 rte_exit(EXIT_FAILURE, "eh_devs_init failed, err=%d\n", ret); 2932 2933 /* start ports */ 2934 RTE_ETH_FOREACH_DEV(portid) { 2935 if ((enabled_port_mask & (1 << portid)) == 0) 2936 continue; 2937 2938 /* Create flow before starting the device */ 2939 create_default_ipsec_flow(portid, req_rx_offloads[portid]); 2940 2941 ret = rte_eth_dev_start(portid); 2942 if (ret < 0) 2943 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 2944 "err=%d, port=%d\n", ret, portid); 2945 /* 2946 * If enabled, put device in promiscuous mode. 2947 * This allows IO forwarding mode to forward packets 2948 * to itself through 2 cross-connected ports of the 2949 * target machine. 2950 */ 2951 if (promiscuous_on) { 2952 ret = rte_eth_promiscuous_enable(portid); 2953 if (ret != 0) 2954 rte_exit(EXIT_FAILURE, 2955 "rte_eth_promiscuous_enable: err=%s, port=%d\n", 2956 rte_strerror(-ret), portid); 2957 } 2958 2959 rte_eth_dev_callback_register(portid, 2960 RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL); 2961 } 2962 2963 /* fragment reassemble is enabled */ 2964 if (frag_tbl_sz != 0) { 2965 ret = reassemble_init(); 2966 if (ret != 0) 2967 rte_exit(EXIT_FAILURE, "failed at reassemble init"); 2968 } 2969 2970 /* Replicate each context per socket */ 2971 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) { 2972 socket_id = rte_socket_id_by_idx(i); 2973 if ((socket_ctx[socket_id].mbuf_pool != NULL) && 2974 (socket_ctx[socket_id].sa_in == NULL) && 2975 (socket_ctx[socket_id].sa_out == NULL)) { 2976 sa_init(&socket_ctx[socket_id], socket_id); 2977 sp4_init(&socket_ctx[socket_id], socket_id); 2978 sp6_init(&socket_ctx[socket_id], socket_id); 2979 rt_init(&socket_ctx[socket_id], socket_id); 2980 } 2981 } 2982 2983 check_all_ports_link_status(enabled_port_mask); 2984 2985 #if (STATS_INTERVAL > 0) 2986 rte_eal_alarm_set(STATS_INTERVAL * US_PER_S, print_stats_cb, NULL); 2987 #else 2988 RTE_LOG(INFO, IPSEC, "Stats display disabled\n"); 2989 #endif /* STATS_INTERVAL */ 2990 2991 /* launch per-lcore init on every lcore */ 2992 rte_eal_mp_remote_launch(ipsec_launch_one_lcore, eh_conf, CALL_MASTER); 2993 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2994 if (rte_eal_wait_lcore(lcore_id) < 0) 2995 return -1; 2996 } 2997 2998 /* Uninitialize eventmode components */ 2999 ret = eh_devs_uninit(eh_conf); 3000 if (ret < 0) 3001 rte_exit(EXIT_FAILURE, "eh_devs_uninit failed, err=%d\n", ret); 3002 3003 /* Free eventmode configuration memory */ 3004 eh_conf_uninit(eh_conf); 3005 3006 /* Destroy inline inbound and outbound sessions */ 3007 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) { 3008 socket_id = rte_socket_id_by_idx(i); 3009 inline_sessions_free(socket_ctx[socket_id].sa_in); 3010 inline_sessions_free(socket_ctx[socket_id].sa_out); 3011 } 3012 3013 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 3014 printf("Closing cryptodev %d...", cdev_id); 3015 rte_cryptodev_stop(cdev_id); 3016 rte_cryptodev_close(cdev_id); 3017 printf(" Done\n"); 3018 } 3019 3020 RTE_ETH_FOREACH_DEV(portid) { 3021 if ((enabled_port_mask & (1 << portid)) == 0) 3022 continue; 3023 3024 printf("Closing port %d...", portid); 3025 if (flow_info_tbl[portid].rx_def_flow) { 3026 struct rte_flow_error err; 3027 3028 ret = rte_flow_destroy(portid, 3029 flow_info_tbl[portid].rx_def_flow, &err); 3030 if (ret) 3031 RTE_LOG(ERR, IPSEC, "Failed to destroy flow " 3032 " for port %u, err msg: %s\n", portid, 3033 err.message); 3034 } 3035 rte_eth_dev_stop(portid); 3036 rte_eth_dev_close(portid); 3037 printf(" Done\n"); 3038 } 3039 printf("Bye...\n"); 3040 3041 return 0; 3042 } 3043