18 typedef int32_t (*esp_outb_prepare_t)(struct rte_ipsec_sa *sa, rte_be64_t sqc,
28 	const struct rte_ipsec_sa *sa, const union sym_op_data *icv,  in sop_ciph_auth_prepare()  argument
31 	sop->cipher.data.offset = sa->ctp.cipher.offset + pofs;  in sop_ciph_auth_prepare()
32 	sop->cipher.data.length = sa->ctp.cipher.length + plen;  in sop_ciph_auth_prepare()
33 	sop->auth.data.offset = sa->ctp.auth.offset + pofs;  in sop_ciph_auth_prepare()
34 	sop->auth.data.length = sa->ctp.auth.length + plen;  in sop_ciph_auth_prepare()
45 	const struct rte_ipsec_sa *sa, const union sym_op_data *icv,  in sop_aead_prepare()  argument
48 	sop->aead.data.offset = sa->ctp.cipher.offset + pofs;  in sop_aead_prepare()
49 	sop->aead.data.length = sa->ctp.cipher.length + plen;  in sop_aead_prepare()
52 	sop->aead.aad.data = icv->va + sa->icv_len;  in sop_aead_prepare()
53 	sop->aead.aad.phys_addr = icv->pa + sa->icv_len;  in sop_aead_prepare()
61 	const struct rte_ipsec_sa *sa, const uint64_t ivp[IPSEC_MAX_IV_QWORD],  in outb_cop_prepare()  argument
71 	algo = sa->algo_type;  in outb_cop_prepare()
83 		sop_ciph_auth_prepare(sop, sa, icv, hlen, plen);  in outb_cop_prepare()
87 		sop_ciph_auth_prepare(sop, sa, icv, hlen, plen);  in outb_cop_prepare()
91 			sa->iv_ofs);  in outb_cop_prepare()
92 		aead_gcm_iv_fill(gcm, ivp[0], sa->salt);  in outb_cop_prepare()
96 		sop_aead_prepare(sop, sa, icv, hlen, plen);  in outb_cop_prepare()
100 			sa->iv_ofs);  in outb_cop_prepare()
101 		aead_gcm_iv_fill(gcm, ivp[0], sa->salt);  in outb_cop_prepare()
105 		sop_aead_prepare(sop, sa, icv, hlen, plen);  in outb_cop_prepare()
109 			sa->iv_ofs);  in outb_cop_prepare()
110 		aead_ccm_iv_fill(ccm, ivp[0], sa->salt);  in outb_cop_prepare()
114 		sop_aead_prepare(sop, sa, icv, hlen, plen);  in outb_cop_prepare()
119 			sa->iv_ofs);  in outb_cop_prepare()
121 					       ivp[0], sa->salt);  in outb_cop_prepare()
125 		sop_ciph_auth_prepare(sop, sa, icv, hlen, plen);  in outb_cop_prepare()
129 			sa->iv_ofs);  in outb_cop_prepare()
130 		aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);  in outb_cop_prepare()
139 outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,  in outb_tun_pkt_prepare()  argument
151 	hlen = sa->hdr_len + sa->iv_len + sizeof(*esph);  in outb_tun_pkt_prepare()
161 		clen = RTE_ALIGN_CEIL(clen, sa->pad_align);  in outb_tun_pkt_prepare()
164 		tlen = pdlen + sa->icv_len + sqh_len;  in outb_tun_pkt_prepare()
175 	if (tlen + sa->aad_len > rte_pktmbuf_tailroom(ml))  in outb_tun_pkt_prepare()
190 	mb->tx_offload = (mb->tx_offload & sa->tx_offload.msk) |  in outb_tun_pkt_prepare()
191 		sa->tx_offload.val;  in outb_tun_pkt_prepare()
194 	rte_memcpy(ph, sa->hdr, sa->hdr_len);  in outb_tun_pkt_prepare()
197 	if (sa->type & RTE_IPSEC_SATP_NATT_ENABLE) {  in outb_tun_pkt_prepare()
201 				sa->hdr_l3_off - sa->hdr_len);  in outb_tun_pkt_prepare()
205 	update_tun_outb_l3hdr(sa, ph + sa->hdr_l3_off, ph + hlen,  in outb_tun_pkt_prepare()
206 			mb->pkt_len - sqh_len, sa->hdr_l3_off, sqn_low16(sqc));  in outb_tun_pkt_prepare()
209 	esph = (struct rte_esp_hdr *)(ph + sa->hdr_len);  in outb_tun_pkt_prepare()
211 	copy_iv(iv, ivp, sa->iv_len);  in outb_tun_pkt_prepare()
213 	esph->spi = sa->spi;  in outb_tun_pkt_prepare()
217 	pdofs += pdlen + sa->sqh_len;  in outb_tun_pkt_prepare()
228 	espt->next_proto = sa->proto;  in outb_tun_pkt_prepare()
242 outb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,  in outb_pkt_xprepare()  argument
251 	if (sa->sqh_len != 0) {  in outb_pkt_xprepare()
252 		psqh = (uint32_t *)(icv->va - sa->sqh_len);  in outb_pkt_xprepare()
260 	switch (sa->algo_type) {  in outb_pkt_xprepare()
262 	if (sa->aad_len != 0) {  in outb_pkt_xprepare()
263 		gaad = (struct aead_gcm_aad *)(icv->va + sa->icv_len);  in outb_pkt_xprepare()
264 		aead_gcm_aad_fill(gaad, sa->spi, sqc, IS_ESN(sa));  in outb_pkt_xprepare()
268 	if (sa->aad_len != 0) {  in outb_pkt_xprepare()
269 		caad = (struct aead_ccm_aad *)(icv->va + sa->icv_len);  in outb_pkt_xprepare()
270 		aead_ccm_aad_fill(caad, sa->spi, sqc, IS_ESN(sa));  in outb_pkt_xprepare()
274 	if (sa->aad_len != 0) {  in outb_pkt_xprepare()
276 			(icv->va + sa->icv_len);  in outb_pkt_xprepare()
278 			sa->spi, sqc, IS_ESN(sa));  in outb_pkt_xprepare()
297 	struct rte_ipsec_sa *sa;  in esp_outb_tun_prepare()  local
303 	sa = ss->sa;  in esp_outb_tun_prepare()
307 	sqn = esn_outb_update_sqn(sa, &n);  in esp_outb_tun_prepare()
318 		rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv,  in esp_outb_tun_prepare()
319 					  sa->sqh_len, 0);  in esp_outb_tun_prepare()
322 			outb_pkt_xprepare(sa, sqc, &icv);  in esp_outb_tun_prepare()
324 			outb_cop_prepare(cop[k], sa, iv, &icv, 0, rc);  in esp_outb_tun_prepare()
344 outb_trs_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,  in outb_trs_pkt_prepare()  argument
364 	hlen = sa->iv_len + sizeof(*esph);  in outb_trs_pkt_prepare()
370 		clen = RTE_ALIGN_CEIL(clen, sa->pad_align);  in outb_trs_pkt_prepare()
373 		tlen = pdlen + sa->icv_len + sqh_len;  in outb_trs_pkt_prepare()
384 	if (tlen + sa->aad_len > rte_pktmbuf_tailroom(ml))  in outb_trs_pkt_prepare()
402 	np = update_trs_l3hdr(sa, ph + l2len, mb->pkt_len - sqh_len, l2len,  in outb_trs_pkt_prepare()
408 	copy_iv(iv, ivp, sa->iv_len);  in outb_trs_pkt_prepare()
410 	esph->spi = sa->spi;  in outb_trs_pkt_prepare()
414 	pdofs += pdlen + sa->sqh_len;  in outb_trs_pkt_prepare()
445 	struct rte_ipsec_sa *sa;  in esp_outb_trs_prepare()  local
451 	sa = ss->sa;  in esp_outb_trs_prepare()
455 	sqn = esn_outb_update_sqn(sa, &n);  in esp_outb_trs_prepare()
469 		rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], &icv,  in esp_outb_trs_prepare()
470 				  sa->sqh_len, 0);  in esp_outb_trs_prepare()
473 			outb_pkt_xprepare(sa, sqc, &icv);  in esp_outb_trs_prepare()
475 			outb_cop_prepare(cop[k], sa, iv, &icv, l2 + l3, rc);  in esp_outb_trs_prepare()
493 outb_cpu_crypto_prepare(const struct rte_ipsec_sa *sa, uint32_t *pofs,  in outb_cpu_crypto_prepare()  argument
503 	switch (sa->algo_type) {  in outb_cpu_crypto_prepare()
506 		aead_gcm_iv_fill(gcm, ivp[0], sa->salt);  in outb_cpu_crypto_prepare()
510 		aead_ccm_iv_fill(ccm, ivp[0], sa->salt);  in outb_cpu_crypto_prepare()
515 					       ivp[0], sa->salt);  in outb_cpu_crypto_prepare()
519 		aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);  in outb_cpu_crypto_prepare()
523 	*pofs += sa->ctp.auth.offset;  in outb_cpu_crypto_prepare()
524 	clen = plen + sa->ctp.auth.length;  in outb_cpu_crypto_prepare()
536 	struct rte_ipsec_sa *sa;  in cpu_outb_pkt_prepare()  local
548 	sa = ss->sa;  in cpu_outb_pkt_prepare()
551 	sqn = esn_outb_update_sqn(sa, &n);  in cpu_outb_pkt_prepare()
567 		rc = prepare(sa, sqc, ivbuf[k], mb[i], &icv, sa->sqh_len, 0);  in cpu_outb_pkt_prepare()
572 			outb_pkt_xprepare(sa, sqc, &icv);  in cpu_outb_pkt_prepare()
575 			clen[k] = outb_cpu_crypto_prepare(sa, l4ofs + k, rc,  in cpu_outb_pkt_prepare()
580 			aad[k].va = icv.va + sa->icv_len;  in cpu_outb_pkt_prepare()
594 		cpu_crypto_bulk(ss, sa->cofs, mb, iv, aad, dgst,  in cpu_outb_pkt_prepare()
626 	struct rte_ipsec_sa *sa;  in esp_outb_sqh_process()  local
629 	sa = ss->sa;  in esp_outb_sqh_process()
632 	icv_len = sa->icv_len;  in esp_outb_sqh_process()
639 			mb[i]->pkt_len -= sa->sqh_len;  in esp_outb_sqh_process()
640 			ml->data_len -= sa->sqh_len;  in esp_outb_sqh_process()
649 	sa->statistics.count += k;  in esp_outb_sqh_process()
650 	sa->statistics.bytes += bytes;  in esp_outb_sqh_process()
682 	ss->sa->statistics.count += num;  in inline_outb_mbuf_prepare()
683 	ss->sa->statistics.bytes += bytes;  in inline_outb_mbuf_prepare()
723 	struct rte_ipsec_sa *sa;  in inline_outb_tun_pkt_process()  local
729 	sa = ss->sa;  in inline_outb_tun_pkt_process()
738 	sqn = esn_outb_update_sqn(sa, &n_sqn);  in inline_outb_tun_pkt_process()
758 		rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv, 0,  in inline_outb_tun_pkt_process()
791 	struct rte_ipsec_sa *sa;  in inline_outb_trs_pkt_process()  local
797 	sa = ss->sa;  in inline_outb_trs_pkt_process()
806 	sqn = esn_outb_update_sqn(sa, &n_sqn);  in inline_outb_trs_pkt_process()
826 		rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], &icv, 0,  in inline_outb_trs_pkt_process()