1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <netinet/in.h> 11 #include <netinet/ip.h> 12 #include <netinet/ip6.h> 13 #include <string.h> 14 #include <sys/queue.h> 15 #include <stdarg.h> 16 #include <errno.h> 17 #include <getopt.h> 18 19 #include <rte_common.h> 20 #include <rte_byteorder.h> 21 #include <rte_log.h> 22 #include <rte_eal.h> 23 #include <rte_launch.h> 24 #include <rte_atomic.h> 25 #include <rte_cycles.h> 26 #include <rte_prefetch.h> 27 #include <rte_lcore.h> 28 #include <rte_per_lcore.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_interrupts.h> 31 #include <rte_random.h> 32 #include <rte_debug.h> 33 #include <rte_ether.h> 34 #include <rte_ethdev.h> 35 #include <rte_mempool.h> 36 #include <rte_mbuf.h> 37 #include <rte_acl.h> 38 #include <rte_lpm.h> 39 #include <rte_lpm6.h> 40 #include <rte_hash.h> 41 #include <rte_jhash.h> 42 #include <rte_cryptodev.h> 43 44 #include "ipsec.h" 45 #include "parser.h" 46 47 #define RTE_LOGTYPE_IPSEC RTE_LOGTYPE_USER1 48 49 #define MAX_JUMBO_PKT_LEN 9600 50 51 #define MEMPOOL_CACHE_SIZE 256 52 53 #define NB_MBUF (32000) 54 55 #define CDEV_QUEUE_DESC 2048 56 #define CDEV_MAP_ENTRIES 1024 57 #define CDEV_MP_NB_OBJS 2048 58 #define CDEV_MP_CACHE_SZ 64 59 #define MAX_QUEUE_PAIRS 1 60 61 #define OPTION_CONFIG "config" 62 #define OPTION_SINGLE_SA "single-sa" 63 #define OPTION_CRYPTODEV_MASK "cryptodev_mask" 64 65 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 66 67 #define NB_SOCKETS 4 68 69 /* Configure how many packets ahead to prefetch, when reading packets */ 70 #define PREFETCH_OFFSET 3 71 72 #define MAX_RX_QUEUE_PER_LCORE 16 73 74 #define MAX_LCORE_PARAMS 1024 75 76 #define UNPROTECTED_PORT(port) (unprotected_port_mask & (1 << portid)) 77 78 /* 79 * Configurable number of RX/TX ring descriptors 80 */ 81 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024 82 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024 83 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT; 84 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT; 85 86 #if RTE_BYTE_ORDER != RTE_LITTLE_ENDIAN 87 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 88 (((uint64_t)((a) & 0xff) << 56) | \ 89 ((uint64_t)((b) & 0xff) << 48) | \ 90 ((uint64_t)((c) & 0xff) << 40) | \ 91 ((uint64_t)((d) & 0xff) << 32) | \ 92 ((uint64_t)((e) & 0xff) << 24) | \ 93 ((uint64_t)((f) & 0xff) << 16) | \ 94 ((uint64_t)((g) & 0xff) << 8) | \ 95 ((uint64_t)(h) & 0xff)) 96 #else 97 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \ 98 (((uint64_t)((h) & 0xff) << 56) | \ 99 ((uint64_t)((g) & 0xff) << 48) | \ 100 ((uint64_t)((f) & 0xff) << 40) | \ 101 ((uint64_t)((e) & 0xff) << 32) | \ 102 ((uint64_t)((d) & 0xff) << 24) | \ 103 ((uint64_t)((c) & 0xff) << 16) | \ 104 ((uint64_t)((b) & 0xff) << 8) | \ 105 ((uint64_t)(a) & 0xff)) 106 #endif 107 #define ETHADDR(a, b, c, d, e, f) (__BYTES_TO_UINT64(a, b, c, d, e, f, 0, 0)) 108 109 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \ 110 addr.addr_bytes[0], addr.addr_bytes[1], \ 111 addr.addr_bytes[2], addr.addr_bytes[3], \ 112 addr.addr_bytes[4], addr.addr_bytes[5], \ 113 0, 0) 114 115 /* port/source ethernet addr and destination ethernet addr */ 116 struct ethaddr_info { 117 uint64_t src, dst; 118 }; 119 120 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = { 121 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) }, 122 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) }, 123 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) }, 124 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) } 125 }; 126 127 /* mask of enabled ports */ 128 static uint32_t enabled_port_mask; 129 static uint64_t enabled_cryptodev_mask = UINT64_MAX; 130 static uint32_t unprotected_port_mask; 131 static int32_t promiscuous_on = 1; 132 static int32_t numa_on = 1; /**< NUMA is enabled by default. */ 133 static uint32_t nb_lcores; 134 static uint32_t single_sa; 135 static uint32_t single_sa_idx; 136 static uint32_t frame_size; 137 138 struct lcore_rx_queue { 139 uint16_t port_id; 140 uint8_t queue_id; 141 } __rte_cache_aligned; 142 143 struct lcore_params { 144 uint16_t port_id; 145 uint8_t queue_id; 146 uint8_t lcore_id; 147 } __rte_cache_aligned; 148 149 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; 150 151 static struct lcore_params *lcore_params; 152 static uint16_t nb_lcore_params; 153 154 static struct rte_hash *cdev_map_in; 155 static struct rte_hash *cdev_map_out; 156 157 struct buffer { 158 uint16_t len; 159 struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); 160 }; 161 162 struct lcore_conf { 163 uint16_t nb_rx_queue; 164 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 165 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 166 struct buffer tx_mbufs[RTE_MAX_ETHPORTS]; 167 struct ipsec_ctx inbound; 168 struct ipsec_ctx outbound; 169 struct rt_ctx *rt4_ctx; 170 struct rt_ctx *rt6_ctx; 171 } __rte_cache_aligned; 172 173 static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; 174 175 static struct rte_eth_conf port_conf = { 176 .rxmode = { 177 .mq_mode = ETH_MQ_RX_RSS, 178 .max_rx_pkt_len = ETHER_MAX_LEN, 179 .split_hdr_size = 0, 180 .offloads = DEV_RX_OFFLOAD_CHECKSUM | 181 DEV_RX_OFFLOAD_CRC_STRIP, 182 .ignore_offload_bitfield = 1, 183 }, 184 .rx_adv_conf = { 185 .rss_conf = { 186 .rss_key = NULL, 187 .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | 188 ETH_RSS_TCP | ETH_RSS_SCTP, 189 }, 190 }, 191 .txmode = { 192 .mq_mode = ETH_MQ_TX_NONE, 193 .offloads = (DEV_TX_OFFLOAD_IPV4_CKSUM | 194 DEV_TX_OFFLOAD_MULTI_SEGS), 195 }, 196 }; 197 198 static struct socket_ctx socket_ctx[NB_SOCKETS]; 199 200 struct traffic_type { 201 const uint8_t *data[MAX_PKT_BURST * 2]; 202 struct rte_mbuf *pkts[MAX_PKT_BURST * 2]; 203 uint32_t res[MAX_PKT_BURST * 2]; 204 uint32_t num; 205 }; 206 207 struct ipsec_traffic { 208 struct traffic_type ipsec; 209 struct traffic_type ip4; 210 struct traffic_type ip6; 211 }; 212 213 static inline void 214 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t) 215 { 216 uint8_t *nlp; 217 struct ether_hdr *eth; 218 219 eth = rte_pktmbuf_mtod(pkt, struct ether_hdr *); 220 if (eth->ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) { 221 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, ETHER_HDR_LEN); 222 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip, ip_p)); 223 if (*nlp == IPPROTO_ESP) 224 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 225 else { 226 t->ip4.data[t->ip4.num] = nlp; 227 t->ip4.pkts[(t->ip4.num)++] = pkt; 228 } 229 } else if (eth->ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) { 230 nlp = (uint8_t *)rte_pktmbuf_adj(pkt, ETHER_HDR_LEN); 231 nlp = RTE_PTR_ADD(nlp, offsetof(struct ip6_hdr, ip6_nxt)); 232 if (*nlp == IPPROTO_ESP) 233 t->ipsec.pkts[(t->ipsec.num)++] = pkt; 234 else { 235 t->ip6.data[t->ip6.num] = nlp; 236 t->ip6.pkts[(t->ip6.num)++] = pkt; 237 } 238 } else { 239 /* Unknown/Unsupported type, drop the packet */ 240 RTE_LOG(ERR, IPSEC, "Unsupported packet type\n"); 241 rte_pktmbuf_free(pkt); 242 } 243 244 /* Check if the packet has been processed inline. For inline protocol 245 * processed packets, the metadata in the mbuf can be used to identify 246 * the security processing done on the packet. The metadata will be 247 * used to retrieve the application registered userdata associated 248 * with the security session. 249 */ 250 251 if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) { 252 struct ipsec_sa *sa; 253 struct ipsec_mbuf_metadata *priv; 254 struct rte_security_ctx *ctx = (struct rte_security_ctx *) 255 rte_eth_dev_get_sec_ctx( 256 pkt->port); 257 258 /* Retrieve the userdata registered. Here, the userdata 259 * registered is the SA pointer. 260 */ 261 262 sa = (struct ipsec_sa *) 263 rte_security_get_userdata(ctx, pkt->udata64); 264 265 if (sa == NULL) { 266 /* userdata could not be retrieved */ 267 return; 268 } 269 270 /* Save SA as priv member in mbuf. This will be used in the 271 * IPsec selector(SP-SA) check. 272 */ 273 274 priv = get_priv(pkt); 275 priv->sa = sa; 276 } 277 } 278 279 static inline void 280 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, 281 uint16_t nb_pkts) 282 { 283 int32_t i; 284 285 t->ipsec.num = 0; 286 t->ip4.num = 0; 287 t->ip6.num = 0; 288 289 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) { 290 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET], 291 void *)); 292 prepare_one_packet(pkts[i], t); 293 } 294 /* Process left packets */ 295 for (; i < nb_pkts; i++) 296 prepare_one_packet(pkts[i], t); 297 } 298 299 static inline void 300 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port) 301 { 302 struct ip *ip; 303 struct ether_hdr *ethhdr; 304 305 ip = rte_pktmbuf_mtod(pkt, struct ip *); 306 307 ethhdr = (struct ether_hdr *)rte_pktmbuf_prepend(pkt, ETHER_HDR_LEN); 308 309 if (ip->ip_v == IPVERSION) { 310 pkt->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4; 311 pkt->l3_len = sizeof(struct ip); 312 pkt->l2_len = ETHER_HDR_LEN; 313 314 ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); 315 } else { 316 pkt->ol_flags |= PKT_TX_IPV6; 317 pkt->l3_len = sizeof(struct ip6_hdr); 318 pkt->l2_len = ETHER_HDR_LEN; 319 320 ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6); 321 } 322 323 memcpy(ðhdr->s_addr, ðaddr_tbl[port].src, 324 sizeof(struct ether_addr)); 325 memcpy(ðhdr->d_addr, ðaddr_tbl[port].dst, 326 sizeof(struct ether_addr)); 327 } 328 329 static inline void 330 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port) 331 { 332 int32_t i; 333 const int32_t prefetch_offset = 2; 334 335 for (i = 0; i < (nb_pkts - prefetch_offset); i++) { 336 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]); 337 prepare_tx_pkt(pkts[i], port); 338 } 339 /* Process left packets */ 340 for (; i < nb_pkts; i++) 341 prepare_tx_pkt(pkts[i], port); 342 } 343 344 /* Send burst of packets on an output interface */ 345 static inline int32_t 346 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) 347 { 348 struct rte_mbuf **m_table; 349 int32_t ret; 350 uint16_t queueid; 351 352 queueid = qconf->tx_queue_id[port]; 353 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 354 355 prepare_tx_burst(m_table, n, port); 356 357 ret = rte_eth_tx_burst(port, queueid, m_table, n); 358 if (unlikely(ret < n)) { 359 do { 360 rte_pktmbuf_free(m_table[ret]); 361 } while (++ret < n); 362 } 363 364 return 0; 365 } 366 367 /* Enqueue a single packet, and send burst if queue is filled */ 368 static inline int32_t 369 send_single_packet(struct rte_mbuf *m, uint16_t port) 370 { 371 uint32_t lcore_id; 372 uint16_t len; 373 struct lcore_conf *qconf; 374 375 lcore_id = rte_lcore_id(); 376 377 qconf = &lcore_conf[lcore_id]; 378 len = qconf->tx_mbufs[port].len; 379 qconf->tx_mbufs[port].m_table[len] = m; 380 len++; 381 382 /* enough pkts to be sent */ 383 if (unlikely(len == MAX_PKT_BURST)) { 384 send_burst(qconf, MAX_PKT_BURST, port); 385 len = 0; 386 } 387 388 qconf->tx_mbufs[port].len = len; 389 return 0; 390 } 391 392 static inline void 393 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, 394 uint16_t lim) 395 { 396 struct rte_mbuf *m; 397 uint32_t i, j, res, sa_idx; 398 399 if (ip->num == 0 || sp == NULL) 400 return; 401 402 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 403 ip->num, DEFAULT_MAX_CATEGORIES); 404 405 j = 0; 406 for (i = 0; i < ip->num; i++) { 407 m = ip->pkts[i]; 408 res = ip->res[i]; 409 if (res & BYPASS) { 410 ip->pkts[j++] = m; 411 continue; 412 } 413 if (res & DISCARD) { 414 rte_pktmbuf_free(m); 415 continue; 416 } 417 418 /* Only check SPI match for processed IPSec packets */ 419 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) { 420 rte_pktmbuf_free(m); 421 continue; 422 } 423 424 sa_idx = ip->res[i] & PROTECT_MASK; 425 if (sa_idx >= IPSEC_SA_MAX_ENTRIES || 426 !inbound_sa_check(sa, m, sa_idx)) { 427 rte_pktmbuf_free(m); 428 continue; 429 } 430 ip->pkts[j++] = m; 431 } 432 ip->num = j; 433 } 434 435 static inline void 436 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx, 437 struct ipsec_traffic *traffic) 438 { 439 struct rte_mbuf *m; 440 uint16_t idx, nb_pkts_in, i, n_ip4, n_ip6; 441 442 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 443 traffic->ipsec.num, MAX_PKT_BURST); 444 445 n_ip4 = traffic->ip4.num; 446 n_ip6 = traffic->ip6.num; 447 448 /* SP/ACL Inbound check ipsec and ip4 */ 449 for (i = 0; i < nb_pkts_in; i++) { 450 m = traffic->ipsec.pkts[i]; 451 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 452 if (ip->ip_v == IPVERSION) { 453 idx = traffic->ip4.num++; 454 traffic->ip4.pkts[idx] = m; 455 traffic->ip4.data[idx] = rte_pktmbuf_mtod_offset(m, 456 uint8_t *, offsetof(struct ip, ip_p)); 457 } else if (ip->ip_v == IP6_VERSION) { 458 idx = traffic->ip6.num++; 459 traffic->ip6.pkts[idx] = m; 460 traffic->ip6.data[idx] = rte_pktmbuf_mtod_offset(m, 461 uint8_t *, 462 offsetof(struct ip6_hdr, ip6_nxt)); 463 } else 464 rte_pktmbuf_free(m); 465 } 466 467 inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4, 468 n_ip4); 469 470 inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6, 471 n_ip6); 472 } 473 474 static inline void 475 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, 476 struct traffic_type *ipsec) 477 { 478 struct rte_mbuf *m; 479 uint32_t i, j, sa_idx; 480 481 if (ip->num == 0 || sp == NULL) 482 return; 483 484 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res, 485 ip->num, DEFAULT_MAX_CATEGORIES); 486 487 j = 0; 488 for (i = 0; i < ip->num; i++) { 489 m = ip->pkts[i]; 490 sa_idx = ip->res[i] & PROTECT_MASK; 491 if (ip->res[i] & DISCARD) 492 rte_pktmbuf_free(m); 493 else if (sa_idx < IPSEC_SA_MAX_ENTRIES) { 494 ipsec->res[ipsec->num] = sa_idx; 495 ipsec->pkts[ipsec->num++] = m; 496 } else /* BYPASS */ 497 ip->pkts[j++] = m; 498 } 499 ip->num = j; 500 } 501 502 static inline void 503 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx, 504 struct ipsec_traffic *traffic) 505 { 506 struct rte_mbuf *m; 507 uint16_t idx, nb_pkts_out, i; 508 509 /* Drop any IPsec traffic from protected ports */ 510 for (i = 0; i < traffic->ipsec.num; i++) 511 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 512 513 traffic->ipsec.num = 0; 514 515 outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec); 516 517 outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec); 518 519 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts, 520 traffic->ipsec.res, traffic->ipsec.num, 521 MAX_PKT_BURST); 522 523 for (i = 0; i < nb_pkts_out; i++) { 524 m = traffic->ipsec.pkts[i]; 525 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 526 if (ip->ip_v == IPVERSION) { 527 idx = traffic->ip4.num++; 528 traffic->ip4.pkts[idx] = m; 529 } else { 530 idx = traffic->ip6.num++; 531 traffic->ip6.pkts[idx] = m; 532 } 533 } 534 } 535 536 static inline void 537 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx, 538 struct ipsec_traffic *traffic) 539 { 540 struct rte_mbuf *m; 541 uint32_t nb_pkts_in, i, idx; 542 543 /* Drop any IPv4 traffic from unprotected ports */ 544 for (i = 0; i < traffic->ip4.num; i++) 545 rte_pktmbuf_free(traffic->ip4.pkts[i]); 546 547 traffic->ip4.num = 0; 548 549 /* Drop any IPv6 traffic from unprotected ports */ 550 for (i = 0; i < traffic->ip6.num; i++) 551 rte_pktmbuf_free(traffic->ip6.pkts[i]); 552 553 traffic->ip6.num = 0; 554 555 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts, 556 traffic->ipsec.num, MAX_PKT_BURST); 557 558 for (i = 0; i < nb_pkts_in; i++) { 559 m = traffic->ipsec.pkts[i]; 560 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *); 561 if (ip->ip_v == IPVERSION) { 562 idx = traffic->ip4.num++; 563 traffic->ip4.pkts[idx] = m; 564 } else { 565 idx = traffic->ip6.num++; 566 traffic->ip6.pkts[idx] = m; 567 } 568 } 569 } 570 571 static inline void 572 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, 573 struct ipsec_traffic *traffic) 574 { 575 struct rte_mbuf *m; 576 uint32_t nb_pkts_out, i; 577 struct ip *ip; 578 579 /* Drop any IPsec traffic from protected ports */ 580 for (i = 0; i < traffic->ipsec.num; i++) 581 rte_pktmbuf_free(traffic->ipsec.pkts[i]); 582 583 traffic->ipsec.num = 0; 584 585 for (i = 0; i < traffic->ip4.num; i++) 586 traffic->ip4.res[i] = single_sa_idx; 587 588 for (i = 0; i < traffic->ip6.num; i++) 589 traffic->ip6.res[i] = single_sa_idx; 590 591 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ip4.pkts, 592 traffic->ip4.res, traffic->ip4.num, 593 MAX_PKT_BURST); 594 595 /* They all sue the same SA (ip4 or ip6 tunnel) */ 596 m = traffic->ipsec.pkts[i]; 597 ip = rte_pktmbuf_mtod(m, struct ip *); 598 if (ip->ip_v == IPVERSION) 599 traffic->ip4.num = nb_pkts_out; 600 else 601 traffic->ip6.num = nb_pkts_out; 602 } 603 604 static inline int32_t 605 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) 606 { 607 struct ipsec_mbuf_metadata *priv; 608 struct ipsec_sa *sa; 609 610 priv = get_priv(pkt); 611 612 sa = priv->sa; 613 if (unlikely(sa == NULL)) { 614 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); 615 goto fail; 616 } 617 618 if (is_ipv6) 619 return sa->portid; 620 621 /* else */ 622 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); 623 624 fail: 625 if (is_ipv6) 626 return -1; 627 628 /* else */ 629 return 0; 630 } 631 632 static inline void 633 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 634 { 635 uint32_t hop[MAX_PKT_BURST * 2]; 636 uint32_t dst_ip[MAX_PKT_BURST * 2]; 637 int32_t pkt_hop = 0; 638 uint16_t i, offset; 639 uint16_t lpm_pkts = 0; 640 641 if (nb_pkts == 0) 642 return; 643 644 /* Need to do an LPM lookup for non-inline packets. Inline packets will 645 * have port ID in the SA 646 */ 647 648 for (i = 0; i < nb_pkts; i++) { 649 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 650 /* Security offload not enabled. So an LPM lookup is 651 * required to get the hop 652 */ 653 offset = offsetof(struct ip, ip_dst); 654 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], 655 uint32_t *, offset); 656 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); 657 lpm_pkts++; 658 } 659 } 660 661 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); 662 663 lpm_pkts = 0; 664 665 for (i = 0; i < nb_pkts; i++) { 666 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 667 /* Read hop from the SA */ 668 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); 669 } else { 670 /* Need to use hop returned by lookup */ 671 pkt_hop = hop[lpm_pkts++]; 672 } 673 674 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { 675 rte_pktmbuf_free(pkts[i]); 676 continue; 677 } 678 send_single_packet(pkts[i], pkt_hop & 0xff); 679 } 680 } 681 682 static inline void 683 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) 684 { 685 int32_t hop[MAX_PKT_BURST * 2]; 686 uint8_t dst_ip[MAX_PKT_BURST * 2][16]; 687 uint8_t *ip6_dst; 688 int32_t pkt_hop = 0; 689 uint16_t i, offset; 690 uint16_t lpm_pkts = 0; 691 692 if (nb_pkts == 0) 693 return; 694 695 /* Need to do an LPM lookup for non-inline packets. Inline packets will 696 * have port ID in the SA 697 */ 698 699 for (i = 0; i < nb_pkts; i++) { 700 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { 701 /* Security offload not enabled. So an LPM lookup is 702 * required to get the hop 703 */ 704 offset = offsetof(struct ip6_hdr, ip6_dst); 705 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, 706 offset); 707 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); 708 lpm_pkts++; 709 } 710 } 711 712 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, 713 lpm_pkts); 714 715 lpm_pkts = 0; 716 717 for (i = 0; i < nb_pkts; i++) { 718 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { 719 /* Read hop from the SA */ 720 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); 721 } else { 722 /* Need to use hop returned by lookup */ 723 pkt_hop = hop[lpm_pkts++]; 724 } 725 726 if (pkt_hop == -1) { 727 rte_pktmbuf_free(pkts[i]); 728 continue; 729 } 730 send_single_packet(pkts[i], pkt_hop & 0xff); 731 } 732 } 733 734 static inline void 735 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, 736 uint8_t nb_pkts, uint16_t portid) 737 { 738 struct ipsec_traffic traffic; 739 740 prepare_traffic(pkts, &traffic, nb_pkts); 741 742 if (unlikely(single_sa)) { 743 if (UNPROTECTED_PORT(portid)) 744 process_pkts_inbound_nosp(&qconf->inbound, &traffic); 745 else 746 process_pkts_outbound_nosp(&qconf->outbound, &traffic); 747 } else { 748 if (UNPROTECTED_PORT(portid)) 749 process_pkts_inbound(&qconf->inbound, &traffic); 750 else 751 process_pkts_outbound(&qconf->outbound, &traffic); 752 } 753 754 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num); 755 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num); 756 } 757 758 static inline void 759 drain_buffers(struct lcore_conf *qconf) 760 { 761 struct buffer *buf; 762 uint32_t portid; 763 764 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { 765 buf = &qconf->tx_mbufs[portid]; 766 if (buf->len == 0) 767 continue; 768 send_burst(qconf, buf->len, portid); 769 buf->len = 0; 770 } 771 } 772 773 /* main processing loop */ 774 static int32_t 775 main_loop(__attribute__((unused)) void *dummy) 776 { 777 struct rte_mbuf *pkts[MAX_PKT_BURST]; 778 uint32_t lcore_id; 779 uint64_t prev_tsc, diff_tsc, cur_tsc; 780 int32_t i, nb_rx; 781 uint16_t portid; 782 uint8_t queueid; 783 struct lcore_conf *qconf; 784 int32_t socket_id; 785 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) 786 / US_PER_S * BURST_TX_DRAIN_US; 787 struct lcore_rx_queue *rxql; 788 789 prev_tsc = 0; 790 lcore_id = rte_lcore_id(); 791 qconf = &lcore_conf[lcore_id]; 792 rxql = qconf->rx_queue_list; 793 socket_id = rte_lcore_to_socket_id(lcore_id); 794 795 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4; 796 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6; 797 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in; 798 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; 799 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; 800 qconf->inbound.cdev_map = cdev_map_in; 801 qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; 802 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; 803 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; 804 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; 805 qconf->outbound.cdev_map = cdev_map_out; 806 qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; 807 808 if (qconf->nb_rx_queue == 0) { 809 RTE_LOG(INFO, IPSEC, "lcore %u has nothing to do\n", lcore_id); 810 return 0; 811 } 812 813 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id); 814 815 for (i = 0; i < qconf->nb_rx_queue; i++) { 816 portid = rxql[i].port_id; 817 queueid = rxql[i].queue_id; 818 RTE_LOG(INFO, IPSEC, 819 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", 820 lcore_id, portid, queueid); 821 } 822 823 while (1) { 824 cur_tsc = rte_rdtsc(); 825 826 /* TX queue buffer drain */ 827 diff_tsc = cur_tsc - prev_tsc; 828 829 if (unlikely(diff_tsc > drain_tsc)) { 830 drain_buffers(qconf); 831 prev_tsc = cur_tsc; 832 } 833 834 /* Read packet from RX queues */ 835 for (i = 0; i < qconf->nb_rx_queue; ++i) { 836 portid = rxql[i].port_id; 837 queueid = rxql[i].queue_id; 838 nb_rx = rte_eth_rx_burst(portid, queueid, 839 pkts, MAX_PKT_BURST); 840 841 if (nb_rx > 0) 842 process_pkts(qconf, pkts, nb_rx, portid); 843 } 844 } 845 } 846 847 static int32_t 848 check_params(void) 849 { 850 uint8_t lcore; 851 uint16_t portid, nb_ports; 852 uint16_t i; 853 int32_t socket_id; 854 855 if (lcore_params == NULL) { 856 printf("Error: No port/queue/core mappings\n"); 857 return -1; 858 } 859 860 nb_ports = rte_eth_dev_count(); 861 862 for (i = 0; i < nb_lcore_params; ++i) { 863 lcore = lcore_params[i].lcore_id; 864 if (!rte_lcore_is_enabled(lcore)) { 865 printf("error: lcore %hhu is not enabled in " 866 "lcore mask\n", lcore); 867 return -1; 868 } 869 socket_id = rte_lcore_to_socket_id(lcore); 870 if (socket_id != 0 && numa_on == 0) { 871 printf("warning: lcore %hhu is on socket %d " 872 "with numa off\n", 873 lcore, socket_id); 874 } 875 portid = lcore_params[i].port_id; 876 if ((enabled_port_mask & (1 << portid)) == 0) { 877 printf("port %u is not enabled in port mask\n", portid); 878 return -1; 879 } 880 if (portid >= nb_ports) { 881 printf("port %u is not present on the board\n", portid); 882 return -1; 883 } 884 } 885 return 0; 886 } 887 888 static uint8_t 889 get_port_nb_rx_queues(const uint16_t port) 890 { 891 int32_t queue = -1; 892 uint16_t i; 893 894 for (i = 0; i < nb_lcore_params; ++i) { 895 if (lcore_params[i].port_id == port && 896 lcore_params[i].queue_id > queue) 897 queue = lcore_params[i].queue_id; 898 } 899 return (uint8_t)(++queue); 900 } 901 902 static int32_t 903 init_lcore_rx_queues(void) 904 { 905 uint16_t i, nb_rx_queue; 906 uint8_t lcore; 907 908 for (i = 0; i < nb_lcore_params; ++i) { 909 lcore = lcore_params[i].lcore_id; 910 nb_rx_queue = lcore_conf[lcore].nb_rx_queue; 911 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { 912 printf("error: too many queues (%u) for lcore: %u\n", 913 nb_rx_queue + 1, lcore); 914 return -1; 915 } 916 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = 917 lcore_params[i].port_id; 918 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = 919 lcore_params[i].queue_id; 920 lcore_conf[lcore].nb_rx_queue++; 921 } 922 return 0; 923 } 924 925 /* display usage */ 926 static void 927 print_usage(const char *prgname) 928 { 929 printf("%s [EAL options] -- -p PORTMASK -P -u PORTMASK" 930 " --"OPTION_CONFIG" (port,queue,lcore)[,(port,queue,lcore]" 931 " --single-sa SAIDX -f CONFIG_FILE\n" 932 " -p PORTMASK: hexadecimal bitmask of ports to configure\n" 933 " -P : enable promiscuous mode\n" 934 " -u PORTMASK: hexadecimal bitmask of unprotected ports\n" 935 " -j FRAMESIZE: jumbo frame maximum size\n" 936 " --"OPTION_CONFIG": (port,queue,lcore): " 937 "rx queues configuration\n" 938 " --single-sa SAIDX: use single SA index for outbound, " 939 "bypassing the SP\n" 940 " --cryptodev_mask MASK: hexadecimal bitmask of the " 941 "crypto devices to configure\n" 942 " -f CONFIG_FILE: Configuration file path\n", 943 prgname); 944 } 945 946 static int32_t 947 parse_portmask(const char *portmask) 948 { 949 char *end = NULL; 950 unsigned long pm; 951 952 /* parse hexadecimal string */ 953 pm = strtoul(portmask, &end, 16); 954 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) 955 return -1; 956 957 if ((pm == 0) && errno) 958 return -1; 959 960 return pm; 961 } 962 963 static int32_t 964 parse_decimal(const char *str) 965 { 966 char *end = NULL; 967 unsigned long num; 968 969 num = strtoul(str, &end, 10); 970 if ((str[0] == '\0') || (end == NULL) || (*end != '\0')) 971 return -1; 972 973 return num; 974 } 975 976 static int32_t 977 parse_config(const char *q_arg) 978 { 979 char s[256]; 980 const char *p, *p0 = q_arg; 981 char *end; 982 enum fieldnames { 983 FLD_PORT = 0, 984 FLD_QUEUE, 985 FLD_LCORE, 986 _NUM_FLD 987 }; 988 unsigned long int_fld[_NUM_FLD]; 989 char *str_fld[_NUM_FLD]; 990 int32_t i; 991 uint32_t size; 992 993 nb_lcore_params = 0; 994 995 while ((p = strchr(p0, '(')) != NULL) { 996 ++p; 997 p0 = strchr(p, ')'); 998 if (p0 == NULL) 999 return -1; 1000 1001 size = p0 - p; 1002 if (size >= sizeof(s)) 1003 return -1; 1004 1005 snprintf(s, sizeof(s), "%.*s", size, p); 1006 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 1007 _NUM_FLD) 1008 return -1; 1009 for (i = 0; i < _NUM_FLD; i++) { 1010 errno = 0; 1011 int_fld[i] = strtoul(str_fld[i], &end, 0); 1012 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) 1013 return -1; 1014 } 1015 if (nb_lcore_params >= MAX_LCORE_PARAMS) { 1016 printf("exceeded max number of lcore params: %hu\n", 1017 nb_lcore_params); 1018 return -1; 1019 } 1020 lcore_params_array[nb_lcore_params].port_id = 1021 (uint8_t)int_fld[FLD_PORT]; 1022 lcore_params_array[nb_lcore_params].queue_id = 1023 (uint8_t)int_fld[FLD_QUEUE]; 1024 lcore_params_array[nb_lcore_params].lcore_id = 1025 (uint8_t)int_fld[FLD_LCORE]; 1026 ++nb_lcore_params; 1027 } 1028 lcore_params = lcore_params_array; 1029 return 0; 1030 } 1031 1032 #define __STRNCMP(name, opt) (!strncmp(name, opt, sizeof(opt))) 1033 static int32_t 1034 parse_args_long_options(struct option *lgopts, int32_t option_index) 1035 { 1036 int32_t ret = -1; 1037 const char *optname = lgopts[option_index].name; 1038 1039 if (__STRNCMP(optname, OPTION_CONFIG)) { 1040 ret = parse_config(optarg); 1041 if (ret) 1042 printf("invalid config\n"); 1043 } 1044 1045 if (__STRNCMP(optname, OPTION_SINGLE_SA)) { 1046 ret = parse_decimal(optarg); 1047 if (ret != -1) { 1048 single_sa = 1; 1049 single_sa_idx = ret; 1050 printf("Configured with single SA index %u\n", 1051 single_sa_idx); 1052 ret = 0; 1053 } 1054 } 1055 1056 if (__STRNCMP(optname, OPTION_CRYPTODEV_MASK)) { 1057 ret = parse_portmask(optarg); 1058 if (ret != -1) { 1059 enabled_cryptodev_mask = ret; 1060 ret = 0; 1061 } 1062 } 1063 1064 return ret; 1065 } 1066 #undef __STRNCMP 1067 1068 static int32_t 1069 parse_args(int32_t argc, char **argv) 1070 { 1071 int32_t opt, ret; 1072 char **argvopt; 1073 int32_t option_index; 1074 char *prgname = argv[0]; 1075 static struct option lgopts[] = { 1076 {OPTION_CONFIG, 1, 0, 0}, 1077 {OPTION_SINGLE_SA, 1, 0, 0}, 1078 {OPTION_CRYPTODEV_MASK, 1, 0, 0}, 1079 {NULL, 0, 0, 0} 1080 }; 1081 int32_t f_present = 0; 1082 1083 argvopt = argv; 1084 1085 while ((opt = getopt_long(argc, argvopt, "p:Pu:f:j:", 1086 lgopts, &option_index)) != EOF) { 1087 1088 switch (opt) { 1089 case 'p': 1090 enabled_port_mask = parse_portmask(optarg); 1091 if (enabled_port_mask == 0) { 1092 printf("invalid portmask\n"); 1093 print_usage(prgname); 1094 return -1; 1095 } 1096 break; 1097 case 'P': 1098 printf("Promiscuous mode selected\n"); 1099 promiscuous_on = 1; 1100 break; 1101 case 'u': 1102 unprotected_port_mask = parse_portmask(optarg); 1103 if (unprotected_port_mask == 0) { 1104 printf("invalid unprotected portmask\n"); 1105 print_usage(prgname); 1106 return -1; 1107 } 1108 break; 1109 case 'f': 1110 if (f_present == 1) { 1111 printf("\"-f\" option present more than " 1112 "once!\n"); 1113 print_usage(prgname); 1114 return -1; 1115 } 1116 if (parse_cfg_file(optarg) < 0) { 1117 printf("parsing file \"%s\" failed\n", 1118 optarg); 1119 print_usage(prgname); 1120 return -1; 1121 } 1122 f_present = 1; 1123 break; 1124 case 'j': 1125 { 1126 int32_t size = parse_decimal(optarg); 1127 if (size <= 1518) { 1128 printf("Invalid jumbo frame size\n"); 1129 if (size < 0) { 1130 print_usage(prgname); 1131 return -1; 1132 } 1133 printf("Using default value 9000\n"); 1134 frame_size = 9000; 1135 } else { 1136 frame_size = size; 1137 } 1138 } 1139 printf("Enabled jumbo frames size %u\n", frame_size); 1140 break; 1141 case 0: 1142 if (parse_args_long_options(lgopts, option_index)) { 1143 print_usage(prgname); 1144 return -1; 1145 } 1146 break; 1147 default: 1148 print_usage(prgname); 1149 return -1; 1150 } 1151 } 1152 1153 if (f_present == 0) { 1154 printf("Mandatory option \"-f\" not present\n"); 1155 return -1; 1156 } 1157 1158 if (optind >= 0) 1159 argv[optind-1] = prgname; 1160 1161 ret = optind-1; 1162 optind = 1; /* reset getopt lib */ 1163 return ret; 1164 } 1165 1166 static void 1167 print_ethaddr(const char *name, const struct ether_addr *eth_addr) 1168 { 1169 char buf[ETHER_ADDR_FMT_SIZE]; 1170 ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); 1171 printf("%s%s", name, buf); 1172 } 1173 1174 /* Check the link status of all ports in up to 9s, and print them finally */ 1175 static void 1176 check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) 1177 { 1178 #define CHECK_INTERVAL 100 /* 100ms */ 1179 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1180 uint16_t portid; 1181 uint8_t count, all_ports_up, print_flag = 0; 1182 struct rte_eth_link link; 1183 1184 printf("\nChecking link status"); 1185 fflush(stdout); 1186 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1187 all_ports_up = 1; 1188 for (portid = 0; portid < port_num; portid++) { 1189 if ((port_mask & (1 << portid)) == 0) 1190 continue; 1191 memset(&link, 0, sizeof(link)); 1192 rte_eth_link_get_nowait(portid, &link); 1193 /* print link status if flag set */ 1194 if (print_flag == 1) { 1195 if (link.link_status) 1196 printf( 1197 "Port%d Link Up - speed %u Mbps -%s\n", 1198 portid, link.link_speed, 1199 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1200 ("full-duplex") : ("half-duplex\n")); 1201 else 1202 printf("Port %d Link Down\n", portid); 1203 continue; 1204 } 1205 /* clear all_ports_up flag if any link down */ 1206 if (link.link_status == ETH_LINK_DOWN) { 1207 all_ports_up = 0; 1208 break; 1209 } 1210 } 1211 /* after finally printing all link status, get out */ 1212 if (print_flag == 1) 1213 break; 1214 1215 if (all_ports_up == 0) { 1216 printf("."); 1217 fflush(stdout); 1218 rte_delay_ms(CHECK_INTERVAL); 1219 } 1220 1221 /* set the print_flag if all ports up or timeout */ 1222 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1223 print_flag = 1; 1224 printf("done\n"); 1225 } 1226 } 1227 } 1228 1229 static int32_t 1230 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, 1231 uint16_t qp, struct lcore_params *params, 1232 struct ipsec_ctx *ipsec_ctx, 1233 const struct rte_cryptodev_capabilities *cipher, 1234 const struct rte_cryptodev_capabilities *auth, 1235 const struct rte_cryptodev_capabilities *aead) 1236 { 1237 int32_t ret = 0; 1238 unsigned long i; 1239 struct cdev_key key = { 0 }; 1240 1241 key.lcore_id = params->lcore_id; 1242 if (cipher) 1243 key.cipher_algo = cipher->sym.cipher.algo; 1244 if (auth) 1245 key.auth_algo = auth->sym.auth.algo; 1246 if (aead) 1247 key.aead_algo = aead->sym.aead.algo; 1248 1249 ret = rte_hash_lookup(map, &key); 1250 if (ret != -ENOENT) 1251 return 0; 1252 1253 for (i = 0; i < ipsec_ctx->nb_qps; i++) 1254 if (ipsec_ctx->tbl[i].id == cdev_id) 1255 break; 1256 1257 if (i == ipsec_ctx->nb_qps) { 1258 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) { 1259 printf("Maximum number of crypto devices assigned to " 1260 "a core, increase MAX_QP_PER_LCORE value\n"); 1261 return 0; 1262 } 1263 ipsec_ctx->tbl[i].id = cdev_id; 1264 ipsec_ctx->tbl[i].qp = qp; 1265 ipsec_ctx->nb_qps++; 1266 printf("%s cdev mapping: lcore %u using cdev %u qp %u " 1267 "(cdev_id_qp %lu)\n", str, key.lcore_id, 1268 cdev_id, qp, i); 1269 } 1270 1271 ret = rte_hash_add_key_data(map, &key, (void *)i); 1272 if (ret < 0) { 1273 printf("Faled to insert cdev mapping for (lcore %u, " 1274 "cdev %u, qp %u), errno %d\n", 1275 key.lcore_id, ipsec_ctx->tbl[i].id, 1276 ipsec_ctx->tbl[i].qp, ret); 1277 return 0; 1278 } 1279 1280 return 1; 1281 } 1282 1283 static int32_t 1284 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, 1285 uint16_t qp, struct lcore_params *params) 1286 { 1287 int32_t ret = 0; 1288 const struct rte_cryptodev_capabilities *i, *j; 1289 struct rte_hash *map; 1290 struct lcore_conf *qconf; 1291 struct ipsec_ctx *ipsec_ctx; 1292 const char *str; 1293 1294 qconf = &lcore_conf[params->lcore_id]; 1295 1296 if ((unprotected_port_mask & (1 << params->port_id)) == 0) { 1297 map = cdev_map_out; 1298 ipsec_ctx = &qconf->outbound; 1299 str = "Outbound"; 1300 } else { 1301 map = cdev_map_in; 1302 ipsec_ctx = &qconf->inbound; 1303 str = "Inbound"; 1304 } 1305 1306 /* Required cryptodevs with operation chainning */ 1307 if (!(dev_info->feature_flags & 1308 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) 1309 return ret; 1310 1311 for (i = dev_info->capabilities; 1312 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) { 1313 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1314 continue; 1315 1316 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1317 ret |= add_mapping(map, str, cdev_id, qp, params, 1318 ipsec_ctx, NULL, NULL, i); 1319 continue; 1320 } 1321 1322 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) 1323 continue; 1324 1325 for (j = dev_info->capabilities; 1326 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) { 1327 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) 1328 continue; 1329 1330 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) 1331 continue; 1332 1333 ret |= add_mapping(map, str, cdev_id, qp, params, 1334 ipsec_ctx, i, j, NULL); 1335 } 1336 } 1337 1338 return ret; 1339 } 1340 1341 /* Check if the device is enabled by cryptodev_mask */ 1342 static int 1343 check_cryptodev_mask(uint8_t cdev_id) 1344 { 1345 if (enabled_cryptodev_mask & (1 << cdev_id)) 1346 return 0; 1347 1348 return -1; 1349 } 1350 1351 static int32_t 1352 cryptodevs_init(void) 1353 { 1354 struct rte_cryptodev_config dev_conf; 1355 struct rte_cryptodev_qp_conf qp_conf; 1356 uint16_t idx, max_nb_qps, qp, i; 1357 int16_t cdev_id, port_id; 1358 struct rte_hash_parameters params = { 0 }; 1359 1360 params.entries = CDEV_MAP_ENTRIES; 1361 params.key_len = sizeof(struct cdev_key); 1362 params.hash_func = rte_jhash; 1363 params.hash_func_init_val = 0; 1364 params.socket_id = rte_socket_id(); 1365 1366 params.name = "cdev_map_in"; 1367 cdev_map_in = rte_hash_create(¶ms); 1368 if (cdev_map_in == NULL) 1369 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1370 rte_errno); 1371 1372 params.name = "cdev_map_out"; 1373 cdev_map_out = rte_hash_create(¶ms); 1374 if (cdev_map_out == NULL) 1375 rte_panic("Failed to create cdev_map hash table, errno = %d\n", 1376 rte_errno); 1377 1378 printf("lcore/cryptodev/qp mappings:\n"); 1379 1380 uint32_t max_sess_sz = 0, sess_sz; 1381 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1382 sess_sz = rte_cryptodev_get_private_session_size(cdev_id); 1383 if (sess_sz > max_sess_sz) 1384 max_sess_sz = sess_sz; 1385 } 1386 for (port_id = 0; port_id < rte_eth_dev_count(); port_id++) { 1387 void *sec_ctx; 1388 1389 if ((enabled_port_mask & (1 << port_id)) == 0) 1390 continue; 1391 1392 sec_ctx = rte_eth_dev_get_sec_ctx(port_id); 1393 if (sec_ctx == NULL) 1394 continue; 1395 1396 sess_sz = rte_security_session_get_size(sec_ctx); 1397 if (sess_sz > max_sess_sz) 1398 max_sess_sz = sess_sz; 1399 } 1400 1401 idx = 0; 1402 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { 1403 struct rte_cryptodev_info cdev_info; 1404 1405 if (check_cryptodev_mask((uint8_t)cdev_id)) 1406 continue; 1407 1408 rte_cryptodev_info_get(cdev_id, &cdev_info); 1409 1410 if (nb_lcore_params > cdev_info.max_nb_queue_pairs) 1411 max_nb_qps = cdev_info.max_nb_queue_pairs; 1412 else 1413 max_nb_qps = nb_lcore_params; 1414 1415 qp = 0; 1416 i = 0; 1417 while (qp < max_nb_qps && i < nb_lcore_params) { 1418 if (add_cdev_mapping(&cdev_info, cdev_id, qp, 1419 &lcore_params[idx])) 1420 qp++; 1421 idx++; 1422 idx = idx % nb_lcore_params; 1423 i++; 1424 } 1425 1426 if (qp == 0) 1427 continue; 1428 1429 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); 1430 dev_conf.nb_queue_pairs = qp; 1431 1432 if (!socket_ctx[dev_conf.socket_id].session_pool) { 1433 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1434 struct rte_mempool *sess_mp; 1435 1436 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1437 "sess_mp_%u", dev_conf.socket_id); 1438 sess_mp = rte_mempool_create(mp_name, 1439 CDEV_MP_NB_OBJS, 1440 max_sess_sz, 1441 CDEV_MP_CACHE_SZ, 1442 0, NULL, NULL, NULL, 1443 NULL, dev_conf.socket_id, 1444 0); 1445 if (sess_mp == NULL) 1446 rte_exit(EXIT_FAILURE, 1447 "Cannot create session pool on socket %d\n", 1448 dev_conf.socket_id); 1449 else 1450 printf("Allocated session pool on socket %d\n", 1451 dev_conf.socket_id); 1452 socket_ctx[dev_conf.socket_id].session_pool = sess_mp; 1453 } 1454 1455 if (rte_cryptodev_configure(cdev_id, &dev_conf)) 1456 rte_panic("Failed to initialize cryptodev %u\n", 1457 cdev_id); 1458 1459 qp_conf.nb_descriptors = CDEV_QUEUE_DESC; 1460 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) 1461 if (rte_cryptodev_queue_pair_setup(cdev_id, qp, 1462 &qp_conf, dev_conf.socket_id, 1463 socket_ctx[dev_conf.socket_id].session_pool)) 1464 rte_panic("Failed to setup queue %u for " 1465 "cdev_id %u\n", 0, cdev_id); 1466 1467 if (rte_cryptodev_start(cdev_id)) 1468 rte_panic("Failed to start cryptodev %u\n", 1469 cdev_id); 1470 } 1471 1472 /* create session pools for eth devices that implement security */ 1473 for (port_id = 0; port_id < rte_eth_dev_count(); port_id++) { 1474 if ((enabled_port_mask & (1 << port_id)) && 1475 rte_eth_dev_get_sec_ctx(port_id)) { 1476 int socket_id = rte_eth_dev_socket_id(port_id); 1477 1478 if (!socket_ctx[socket_id].session_pool) { 1479 char mp_name[RTE_MEMPOOL_NAMESIZE]; 1480 struct rte_mempool *sess_mp; 1481 1482 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, 1483 "sess_mp_%u", socket_id); 1484 sess_mp = rte_mempool_create(mp_name, 1485 CDEV_MP_NB_OBJS, 1486 max_sess_sz, 1487 CDEV_MP_CACHE_SZ, 1488 0, NULL, NULL, NULL, 1489 NULL, socket_id, 1490 0); 1491 if (sess_mp == NULL) 1492 rte_exit(EXIT_FAILURE, 1493 "Cannot create session pool " 1494 "on socket %d\n", socket_id); 1495 else 1496 printf("Allocated session pool " 1497 "on socket %d\n", socket_id); 1498 socket_ctx[socket_id].session_pool = sess_mp; 1499 } 1500 } 1501 } 1502 1503 1504 printf("\n"); 1505 1506 return 0; 1507 } 1508 1509 static void 1510 port_init(uint16_t portid) 1511 { 1512 struct rte_eth_dev_info dev_info; 1513 struct rte_eth_txconf *txconf; 1514 uint16_t nb_tx_queue, nb_rx_queue; 1515 uint16_t tx_queueid, rx_queueid, queue, lcore_id; 1516 int32_t ret, socket_id; 1517 struct lcore_conf *qconf; 1518 struct ether_addr ethaddr; 1519 struct rte_eth_conf local_port_conf = port_conf; 1520 1521 rte_eth_dev_info_get(portid, &dev_info); 1522 1523 printf("Configuring device port %u:\n", portid); 1524 1525 rte_eth_macaddr_get(portid, ðaddr); 1526 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ethaddr); 1527 print_ethaddr("Address: ", ðaddr); 1528 printf("\n"); 1529 1530 nb_rx_queue = get_port_nb_rx_queues(portid); 1531 nb_tx_queue = nb_lcores; 1532 1533 if (nb_rx_queue > dev_info.max_rx_queues) 1534 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1535 "(max rx queue is %u)\n", 1536 nb_rx_queue, dev_info.max_rx_queues); 1537 1538 if (nb_tx_queue > dev_info.max_tx_queues) 1539 rte_exit(EXIT_FAILURE, "Error: queue %u not available " 1540 "(max tx queue is %u)\n", 1541 nb_tx_queue, dev_info.max_tx_queues); 1542 1543 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", 1544 nb_rx_queue, nb_tx_queue); 1545 1546 if (frame_size) { 1547 local_port_conf.rxmode.max_rx_pkt_len = frame_size; 1548 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1549 } 1550 1551 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_SECURITY) 1552 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SECURITY; 1553 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SECURITY) 1554 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SECURITY; 1555 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 1556 local_port_conf.txmode.offloads |= 1557 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1558 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, 1559 &local_port_conf); 1560 if (ret < 0) 1561 rte_exit(EXIT_FAILURE, "Cannot configure device: " 1562 "err=%d, port=%d\n", ret, portid); 1563 1564 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); 1565 if (ret < 0) 1566 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " 1567 "err=%d, port=%d\n", ret, portid); 1568 1569 /* init one TX queue per lcore */ 1570 tx_queueid = 0; 1571 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1572 if (rte_lcore_is_enabled(lcore_id) == 0) 1573 continue; 1574 1575 if (numa_on) 1576 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1577 else 1578 socket_id = 0; 1579 1580 /* init TX queue */ 1581 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id); 1582 1583 txconf = &dev_info.default_txconf; 1584 txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE; 1585 txconf->offloads = local_port_conf.txmode.offloads; 1586 1587 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd, 1588 socket_id, txconf); 1589 if (ret < 0) 1590 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " 1591 "err=%d, port=%d\n", ret, portid); 1592 1593 qconf = &lcore_conf[lcore_id]; 1594 qconf->tx_queue_id[portid] = tx_queueid; 1595 tx_queueid++; 1596 1597 /* init RX queues */ 1598 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) { 1599 struct rte_eth_rxconf rxq_conf; 1600 1601 if (portid != qconf->rx_queue_list[queue].port_id) 1602 continue; 1603 1604 rx_queueid = qconf->rx_queue_list[queue].queue_id; 1605 1606 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid, 1607 socket_id); 1608 1609 rxq_conf = dev_info.default_rxconf; 1610 rxq_conf.offloads = local_port_conf.rxmode.offloads; 1611 ret = rte_eth_rx_queue_setup(portid, rx_queueid, 1612 nb_rxd, socket_id, &rxq_conf, 1613 socket_ctx[socket_id].mbuf_pool); 1614 if (ret < 0) 1615 rte_exit(EXIT_FAILURE, 1616 "rte_eth_rx_queue_setup: err=%d, " 1617 "port=%d\n", ret, portid); 1618 } 1619 } 1620 printf("\n"); 1621 } 1622 1623 static void 1624 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) 1625 { 1626 char s[64]; 1627 uint32_t buff_size = frame_size ? (frame_size + RTE_PKTMBUF_HEADROOM) : 1628 RTE_MBUF_DEFAULT_BUF_SIZE; 1629 1630 1631 snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); 1632 ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, 1633 MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), 1634 buff_size, 1635 socket_id); 1636 if (ctx->mbuf_pool == NULL) 1637 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", 1638 socket_id); 1639 else 1640 printf("Allocated mbuf pool on socket %d\n", socket_id); 1641 } 1642 1643 int32_t 1644 main(int32_t argc, char **argv) 1645 { 1646 int32_t ret; 1647 uint32_t lcore_id; 1648 uint8_t socket_id; 1649 uint16_t portid, nb_ports; 1650 1651 /* init EAL */ 1652 ret = rte_eal_init(argc, argv); 1653 if (ret < 0) 1654 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 1655 argc -= ret; 1656 argv += ret; 1657 1658 /* parse application arguments (after the EAL ones) */ 1659 ret = parse_args(argc, argv); 1660 if (ret < 0) 1661 rte_exit(EXIT_FAILURE, "Invalid parameters\n"); 1662 1663 if ((unprotected_port_mask & enabled_port_mask) != 1664 unprotected_port_mask) 1665 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", 1666 unprotected_port_mask); 1667 1668 nb_ports = rte_eth_dev_count(); 1669 1670 if (check_params() < 0) 1671 rte_exit(EXIT_FAILURE, "check_params failed\n"); 1672 1673 ret = init_lcore_rx_queues(); 1674 if (ret < 0) 1675 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); 1676 1677 nb_lcores = rte_lcore_count(); 1678 1679 /* Replicate each context per socket */ 1680 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1681 if (rte_lcore_is_enabled(lcore_id) == 0) 1682 continue; 1683 1684 if (numa_on) 1685 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id); 1686 else 1687 socket_id = 0; 1688 1689 if (socket_ctx[socket_id].mbuf_pool) 1690 continue; 1691 1692 sa_init(&socket_ctx[socket_id], socket_id); 1693 1694 sp4_init(&socket_ctx[socket_id], socket_id); 1695 1696 sp6_init(&socket_ctx[socket_id], socket_id); 1697 1698 rt_init(&socket_ctx[socket_id], socket_id); 1699 1700 pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF); 1701 } 1702 1703 for (portid = 0; portid < nb_ports; portid++) { 1704 if ((enabled_port_mask & (1 << portid)) == 0) 1705 continue; 1706 1707 port_init(portid); 1708 } 1709 1710 cryptodevs_init(); 1711 1712 /* start ports */ 1713 for (portid = 0; portid < nb_ports; portid++) { 1714 if ((enabled_port_mask & (1 << portid)) == 0) 1715 continue; 1716 1717 /* Start device */ 1718 ret = rte_eth_dev_start(portid); 1719 if (ret < 0) 1720 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: " 1721 "err=%d, port=%d\n", ret, portid); 1722 /* 1723 * If enabled, put device in promiscuous mode. 1724 * This allows IO forwarding mode to forward packets 1725 * to itself through 2 cross-connected ports of the 1726 * target machine. 1727 */ 1728 if (promiscuous_on) 1729 rte_eth_promiscuous_enable(portid); 1730 } 1731 1732 check_all_ports_link_status(nb_ports, enabled_port_mask); 1733 1734 /* launch per-lcore init on every lcore */ 1735 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); 1736 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1737 if (rte_eal_wait_lcore(lcore_id) < 0) 1738 return -1; 1739 } 1740 1741 return 0; 1742 } 1743