18 	struct rte_mbuf *mb[], uint32_t sqn[], uint32_t dr[], uint16_t num,
60 	const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,  in inb_cop_prepare()  argument
72 	ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *,  in inb_cop_prepare()
136 inb_cpu_crypto_prepare(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,  in inb_cpu_crypto_prepare()  argument
146 	ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *,  in inb_cpu_crypto_prepare()
254 	struct rte_mbuf *mb, uint32_t hlen, rte_be64_t *sqc)  in inb_get_sqn()  argument
260 	esph = rte_pktmbuf_mtod_offset(mb, struct rte_esp_hdr *, hlen);  in inb_get_sqn()
279 inb_prepare(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,  in inb_prepare()  argument
286 	plen = mb->pkt_len;  in inb_prepare()
296 	icv_ofs = mb->pkt_len - icv_len;  in inb_prepare()
298 	ml = mbuf_get_seg_ofs(mb, &icv_ofs);  in inb_prepare()
335 	mb->pkt_len += sa->sqh_len;  in inb_prepare()
343 	struct rte_mbuf *mb, uint32_t hlen, union sym_op_data *icv)  in inb_pkt_prepare()  argument
348 	rc = inb_get_sqn(sa, rsn, mb, hlen, &sqn);  in inb_pkt_prepare()
352 	rc = inb_prepare(sa, mb, hlen, icv);  in inb_pkt_prepare()
364 esp_inb_pkt_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],  in esp_inb_pkt_prepare()  argument
382 		hl = mb[i]->l2_len + mb[i]->l3_len;  in esp_inb_pkt_prepare()
383 		rc = inb_pkt_prepare(sa, rsn, mb[i], hl, &icv);  in esp_inb_pkt_prepare()
385 			lksd_none_cop_prepare(cop[k], cs, mb[i]);  in esp_inb_pkt_prepare()
386 			inb_cop_prepare(cop[k], sa, mb[i], &icv, hl, rc);  in esp_inb_pkt_prepare()
398 		move_bad_mbufs(mb, dr, num, num - k);  in esp_inb_pkt_prepare()
413 process_step1(struct rte_mbuf *mb, uint32_t tlen, struct rte_mbuf **ml,  in process_step1()  argument
419 	ofs = mb->pkt_len - tlen;  in process_step1()
420 	hlen[0] = mb->l2_len + mb->l3_len;  in process_step1()
421 	ml[0] = mbuf_get_seg_ofs(mb, &ofs);  in process_step1()
432 check_pad_bytes(struct rte_mbuf *mb, uint32_t ofs, uint32_t len)  in check_pad_bytes()  argument
437 	for (n = 0; n != len; n += k, mb = mb->next) {  in check_pad_bytes()
438 		k = mb->data_len - ofs;  in check_pad_bytes()
440 		pd = rte_pktmbuf_mtod_offset(mb, const uint8_t *, ofs);  in check_pad_bytes()
458 trs_process_check(struct rte_mbuf *mb, struct rte_mbuf **ml,  in trs_process_check()  argument
461 	if ((mb->ol_flags & RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED) != 0 ||  in trs_process_check()
462 			tlen + hlen > mb->pkt_len)  in trs_process_check()
467 		tofs[0] = mb->pkt_len - tlen;  in trs_process_check()
468 		ml[0] = mbuf_get_seg_ofs(mb, tofs);  in trs_process_check()
481 tun_process_check(struct rte_mbuf *mb, struct rte_mbuf **ml,  in tun_process_check()  argument
485 	return (trs_process_check(mb, ml, tofs, espt, hlen, tlen) ||  in tun_process_check()
497 tun_process_step2(struct rte_mbuf *mb, struct rte_mbuf *ml, uint32_t hlen,  in tun_process_step2()  argument
503 	ph = rte_pktmbuf_mtod_offset(mb, const struct rte_esp_hdr *, hlen);  in tun_process_step2()
507 	mbuf_cut_seg_ofs(mb, ml, tofs, tlen);  in tun_process_step2()
510 	return rte_pktmbuf_adj(mb, adj);  in tun_process_step2()
521 trs_process_step2(struct rte_mbuf *mb, struct rte_mbuf *ml, uint32_t hlen,  in trs_process_step2()  argument
527 	op = rte_pktmbuf_mtod(mb, char *);  in trs_process_step2()
530 	np = tun_process_step2(mb, ml, hlen, adj, tofs, tlen, sqn);  in trs_process_step2()
544 trs_process_step3(struct rte_mbuf *mb)  in trs_process_step3()  argument
547 	mb->packet_type &= (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);  in trs_process_step3()
550 	mb->ol_flags &= ~RTE_MBUF_F_RX_SEC_OFFLOAD;  in trs_process_step3()
561 tun_process_step3(struct rte_mbuf *mb, uint64_t txof_msk, uint64_t txof_val)  in tun_process_step3()  argument
564 	mb->packet_type = RTE_PTYPE_UNKNOWN;  in tun_process_step3()
565 	mb->tx_offload = (mb->tx_offload & txof_msk) | txof_val;  in tun_process_step3()
568 	mb->ol_flags &= ~RTE_MBUF_F_RX_SEC_OFFLOAD;  in tun_process_step3()
575 tun_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],  in tun_process()  argument
597 		process_step1(mb[i], tlen, &ml[i], &espt[i], &hl[i], &to[i]);  in tun_process()
607 		if (tun_process_check(mb[i], &ml[i], &to[i], espt[i], adj, tl,  in tun_process()
610 			outh = rte_pktmbuf_mtod_offset(mb[i], uint8_t *,  in tun_process()
611 					mb[i]->l2_len);  in tun_process()
614 			inh = tun_process_step2(mb[i], ml[i], hl[i], adj,  in tun_process()
621 			tun_process_step3(mb[i], sa->tx_offload.msk,  in tun_process()
624 			bytes += mb[i]->pkt_len;  in tun_process()
638 trs_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],  in trs_process()  argument
659 		process_step1(mb[i], tlen, &ml[i], &espt[i], &hl[i], &to[i]);  in trs_process()
666 		l2 = mb[i]->l2_len;  in trs_process()
669 		if (trs_process_check(mb[i], &ml[i], &to[i], espt[i],  in trs_process()
673 			np = trs_process_step2(mb[i], ml[i], hl[i], cofs,  in trs_process()
675 			update_trs_l3hdr(sa, np + l2, mb[i]->pkt_len,  in trs_process()
679 			trs_process_step3(mb[i]);  in trs_process()
681 			bytes += mb[i]->pkt_len;  in trs_process()
723 esp_inb_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],  in esp_inb_pkt_process()  argument
731 	k = process(sa, mb, sqn, dr, num, sqh_len);  in esp_inb_pkt_process()
735 		move_bad_mbufs(mb, dr, num, num - k);  in esp_inb_pkt_process()
742 		move_bad_mbufs(mb, dr, k, k - n);  in esp_inb_pkt_process()
756 	struct rte_mbuf *mb[], uint16_t num)  in cpu_inb_pkt_prepare()  argument
780 		l4ofs[k] = mb[i]->l2_len + mb[i]->l3_len;  in cpu_inb_pkt_prepare()
783 		rc = inb_pkt_prepare(sa, rsn, mb[i], l4ofs[k], &icv);  in cpu_inb_pkt_prepare()
786 			clen[k] = inb_cpu_crypto_prepare(sa, mb[i],  in cpu_inb_pkt_prepare()
804 		move_bad_mbufs(mb, dr, num, num - k);  in cpu_inb_pkt_prepare()
808 		cpu_crypto_bulk(ss, sa->cofs, mb, iv, aad, dgst,  in cpu_inb_pkt_prepare()
818 	struct rte_mbuf *mb[], uint16_t num)  in esp_inb_tun_pkt_process()  argument
822 	return esp_inb_pkt_process(sa, mb, num, sa->sqh_len, tun_process);  in esp_inb_tun_pkt_process()
827 	struct rte_mbuf *mb[], uint16_t num)  in inline_inb_tun_pkt_process()  argument
829 	return esp_inb_pkt_process(ss->sa, mb, num, 0, tun_process);  in inline_inb_tun_pkt_process()
837 	struct rte_mbuf *mb[], uint16_t num)  in esp_inb_trs_pkt_process()  argument
841 	return esp_inb_pkt_process(sa, mb, num, sa->sqh_len, trs_process);  in esp_inb_trs_pkt_process()
846 	struct rte_mbuf *mb[], uint16_t num)  in inline_inb_trs_pkt_process()  argument
848 	return esp_inb_pkt_process(ss->sa, mb, num, 0, trs_process);  in inline_inb_trs_pkt_process()