17 typedef uint16_t (*esp_inb_process_t)(struct rte_ipsec_sa *sa,
27 	const struct rte_ipsec_sa *sa, const union sym_op_data *icv,  in sop_ciph_auth_prepare()  argument
30 	sop->cipher.data.offset = pofs + sa->ctp.cipher.offset;  in sop_ciph_auth_prepare()
31 	sop->cipher.data.length = plen - sa->ctp.cipher.length;  in sop_ciph_auth_prepare()
32 	sop->auth.data.offset = pofs + sa->ctp.auth.offset;  in sop_ciph_auth_prepare()
33 	sop->auth.data.length = plen - sa->ctp.auth.length;  in sop_ciph_auth_prepare()
44 	const struct rte_ipsec_sa *sa, const union sym_op_data *icv,  in sop_aead_prepare()  argument
47 	sop->aead.data.offset = pofs + sa->ctp.cipher.offset;  in sop_aead_prepare()
48 	sop->aead.data.length = plen - sa->ctp.cipher.length;  in sop_aead_prepare()
51 	sop->aead.aad.data = icv->va + sa->icv_len;  in sop_aead_prepare()
52 	sop->aead.aad.phys_addr = icv->pa + sa->icv_len;  in sop_aead_prepare()
60 	const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,  in inb_cop_prepare()  argument
71 	algo = sa->algo_type;  in inb_cop_prepare()
80 		sop_aead_prepare(sop, sa, icv, pofs, plen);  in inb_cop_prepare()
84 			sa->iv_ofs);  in inb_cop_prepare()
85 		aead_gcm_iv_fill(gcm, ivp[0], sa->salt);  in inb_cop_prepare()
88 		sop_aead_prepare(sop, sa, icv, pofs, plen);  in inb_cop_prepare()
92 			sa->iv_ofs);  in inb_cop_prepare()
93 		aead_ccm_iv_fill(ccm, ivp[0], sa->salt);  in inb_cop_prepare()
96 		sop_aead_prepare(sop, sa, icv, pofs, plen);  in inb_cop_prepare()
101 				sa->iv_ofs);  in inb_cop_prepare()
103 					       ivp[0], sa->salt);  in inb_cop_prepare()
107 		sop_ciph_auth_prepare(sop, sa, icv, pofs, plen);  in inb_cop_prepare()
110 		ivc = rte_crypto_op_ctod_offset(cop, uint64_t *, sa->iv_ofs);  in inb_cop_prepare()
111 		copy_iv(ivc, ivp, sa->iv_len);  in inb_cop_prepare()
114 		sop_ciph_auth_prepare(sop, sa, icv, pofs, plen);  in inb_cop_prepare()
118 			sa->iv_ofs);  in inb_cop_prepare()
119 		aead_gcm_iv_fill(gcm, ivp[0], sa->salt);  in inb_cop_prepare()
122 		sop_ciph_auth_prepare(sop, sa, icv, pofs, plen);  in inb_cop_prepare()
126 			sa->iv_ofs);  in inb_cop_prepare()
127 		aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);  in inb_cop_prepare()
130 		sop_ciph_auth_prepare(sop, sa, icv, pofs, plen);  in inb_cop_prepare()
136 inb_cpu_crypto_prepare(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,  in inb_cpu_crypto_prepare()  argument
150 	switch (sa->algo_type) {  in inb_cpu_crypto_prepare()
154 		aead_gcm_iv_fill(gcm, ivp[0], sa->salt);  in inb_cpu_crypto_prepare()
158 		aead_ccm_iv_fill(ccm, ivp[0], sa->salt);  in inb_cpu_crypto_prepare()
163 					       ivp[0], sa->salt);  in inb_cpu_crypto_prepare()
167 		copy_iv(iv, ivp, sa->iv_len);  in inb_cpu_crypto_prepare()
171 		aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);  in inb_cpu_crypto_prepare()
175 	*pofs += sa->ctp.auth.offset;  in inb_cpu_crypto_prepare()
176 	clen = plen - sa->ctp.auth.length;  in inb_cpu_crypto_prepare()
213 inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,  in inb_pkt_xprepare()  argument
221 	if (sa->sqh_len != 0)  in inb_pkt_xprepare()
222 		insert_sqh(sqn_hi32(sqc), icv->va, sa->icv_len);  in inb_pkt_xprepare()
228 	switch (sa->algo_type) {  in inb_pkt_xprepare()
230 		if (sa->aad_len != 0) {  in inb_pkt_xprepare()
231 			aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len);  in inb_pkt_xprepare()
232 			aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa));  in inb_pkt_xprepare()
236 		if (sa->aad_len != 0) {  in inb_pkt_xprepare()
237 			caad = (struct aead_ccm_aad *)(icv->va + sa->icv_len);  in inb_pkt_xprepare()
238 			aead_ccm_aad_fill(caad, sa->spi, sqc, IS_ESN(sa));  in inb_pkt_xprepare()
242 		if (sa->aad_len != 0) {  in inb_pkt_xprepare()
244 			    (icv->va + sa->icv_len);  in inb_pkt_xprepare()
246 						sa->spi, sqc, IS_ESN(sa));  in inb_pkt_xprepare()
253 inb_get_sqn(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,  in inb_get_sqn()  argument
267 	if (IS_ESN(sa))  in inb_get_sqn()
268 		sqn = reconstruct_esn(rsn->sqn, sqn, sa->replay.win_sz);  in inb_get_sqn()
272 	rc = esn_inb_check_sqn(rsn, sa, sqn);  in inb_get_sqn()
279 inb_prepare(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,  in inb_prepare()  argument
290 	clen = plen - sa->ctp.cipher.length;  in inb_prepare()
291 	if ((int32_t)clen < 0 || (clen & (sa->pad_align - 1)) != 0)  in inb_prepare()
295 	icv_len = sa->icv_len;  in inb_prepare()
314 	icv_ofs += sa->sqh_len;  in inb_prepare()
323 	if (sa->aad_len + sa->sqh_len > rte_pktmbuf_tailroom(ml))  in inb_prepare()
335 	mb->pkt_len += sa->sqh_len;  in inb_prepare()
336 	ml->data_len += sa->sqh_len;  in inb_prepare()
342 inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,  in inb_pkt_prepare()  argument
348 	rc = inb_get_sqn(sa, rsn, mb, hlen, &sqn);  in inb_pkt_prepare()
352 	rc = inb_prepare(sa, mb, hlen, icv);  in inb_pkt_prepare()
356 	inb_pkt_xprepare(sa, sqn, icv);  in inb_pkt_prepare()
369 	struct rte_ipsec_sa *sa;  in esp_inb_pkt_prepare()  local
375 	sa = ss->sa;  in esp_inb_pkt_prepare()
377 	rsn = rsn_acquire(sa);  in esp_inb_pkt_prepare()
383 		rc = inb_pkt_prepare(sa, rsn, mb[i], hl, &icv);  in esp_inb_pkt_prepare()
386 			inb_cop_prepare(cop[k], sa, mb[i], &icv, hl, rc);  in esp_inb_pkt_prepare()
394 	rsn_release(sa, rsn);  in esp_inb_pkt_prepare()
575 tun_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],  in tun_process()  argument
589 	const uint32_t tlen = sa->icv_len + sizeof(espt[0]) + sqh_len;  in tun_process()
590 	const uint32_t cofs = sa->ctp.cipher.offset;  in tun_process()
608 					sa->proto) == 0) {  in tun_process()
618 			update_tun_inb_l3hdr(sa, outh, inh);  in tun_process()
621 			tun_process_step3(mb[i], sa->tx_offload.msk,  in tun_process()
622 				sa->tx_offload.val);  in tun_process()
629 	sa->statistics.count += k;  in tun_process()
630 	sa->statistics.bytes += bytes;  in tun_process()
638 trs_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],  in trs_process()  argument
651 	const uint32_t tlen = sa->icv_len + sizeof(espt[0]) + sqh_len;  in trs_process()
652 	const uint32_t cofs = sa->ctp.cipher.offset;  in trs_process()
675 			update_trs_l3hdr(sa, np + l2, mb[i]->pkt_len,  in trs_process()
686 	sa->statistics.count += k;  in trs_process()
687 	sa->statistics.bytes += bytes;  in trs_process()
695 esp_inb_rsn_update(struct rte_ipsec_sa *sa, const uint32_t sqn[],  in esp_inb_rsn_update()  argument
702 	if (sa->replay.win_sz == 0)  in esp_inb_rsn_update()
705 	rsn = rsn_update_start(sa);  in esp_inb_rsn_update()
709 		if (esn_inb_update_sqn(rsn, sa, rte_be_to_cpu_32(sqn[i])) == 0)  in esp_inb_rsn_update()
715 	rsn_update_finish(sa, rsn);  in esp_inb_rsn_update()
723 esp_inb_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],  in esp_inb_pkt_process()  argument
731 	k = process(sa, mb, sqn, dr, num, sqh_len);  in esp_inb_pkt_process()
738 	n = esp_inb_rsn_update(sa, sqn, dr, k);  in esp_inb_pkt_process()
760 	struct rte_ipsec_sa *sa;  in cpu_inb_pkt_prepare()  local
771 	sa = ss->sa;  in cpu_inb_pkt_prepare()
774 	rsn = rsn_acquire(sa);  in cpu_inb_pkt_prepare()
783 		rc = inb_pkt_prepare(sa, rsn, mb[i], l4ofs[k], &icv);  in cpu_inb_pkt_prepare()
786 			clen[k] = inb_cpu_crypto_prepare(sa, mb[i],  in cpu_inb_pkt_prepare()
791 			aad[k].va = icv.va + sa->icv_len;  in cpu_inb_pkt_prepare()
800 	rsn_release(sa, rsn);  in cpu_inb_pkt_prepare()
808 		cpu_crypto_bulk(ss, sa->cofs, mb, iv, aad, dgst,  in cpu_inb_pkt_prepare()
820 	struct rte_ipsec_sa *sa = ss->sa;  in esp_inb_tun_pkt_process()  local
822 	return esp_inb_pkt_process(sa, mb, num, sa->sqh_len, tun_process);  in esp_inb_tun_pkt_process()
829 	return esp_inb_pkt_process(ss->sa, mb, num, 0, tun_process);  in inline_inb_tun_pkt_process()
839 	struct rte_ipsec_sa *sa = ss->sa;  in esp_inb_trs_pkt_process()  local
841 	return esp_inb_pkt_process(sa, mb, num, sa->sqh_len, trs_process);  in esp_inb_trs_pkt_process()
848 	return esp_inb_pkt_process(ss->sa, mb, num, 0, trs_process);  in inline_inb_trs_pkt_process()