490 find_write_desc(struct vhost_crypto_desc *head, struct vhost_crypto_desc *desc,  in find_write_desc()  argument
493 	if (desc < head)  in find_write_desc()
496 	while (desc - head < (int)max_n_descs) {  in find_write_desc()
497 		if (desc->flags & VRING_DESC_F_WRITE)  in find_write_desc()
498 			return desc;  in find_write_desc()
499 		desc++;  in find_write_desc()
530 	struct vhost_crypto_desc *desc = *cur_desc;  in move_desc()  local
531 	int left = size - desc->len;  in move_desc()
533 	while (desc->flags & VRING_DESC_F_NEXT && left > 0 &&  in move_desc()
534 			desc >= head &&  in move_desc()
535 			desc - head < (int)max_n_descs) {  in move_desc()
536 		desc++;  in move_desc()
537 		left -= desc->len;  in move_desc()
543 	if (unlikely(head - desc == (int)max_n_descs))  in move_desc()
546 		*cur_desc = desc + 1;  in move_desc()
574 	struct vhost_crypto_desc *desc = *cur_desc;  in copy_data()  local
581 	to_copy = RTE_MIN(desc->len, (uint32_t)left);  in copy_data()
583 	src = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen,  in copy_data()
593 		addr = desc->addr + dlen;  in copy_data()
613 	while (desc >= head && desc - head < (int)max_n_descs && left) {  in copy_data()
614 		desc++;  in copy_data()
615 		to_copy = RTE_MIN(desc->len, (uint32_t)left);  in copy_data()
617 		src = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen,  in copy_data()
629 			addr = desc->addr + dlen;  in copy_data()
655 	if (unlikely(desc - head == (int)max_n_descs))  in copy_data()
658 		*cur_desc = desc + 1;  in copy_data()
723 	struct vhost_crypto_desc *desc = *cur_desc;  in prepare_write_back_data()  local
736 	if (likely(desc->len > offset)) {  in prepare_write_back_data()
738 		dlen = desc->len;  in prepare_write_back_data()
739 		dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr,  in prepare_write_back_data()
741 		if (unlikely(!dst || dlen != desc->len)) {  in prepare_write_back_data()
764 		offset -= desc->len;  in prepare_write_back_data()
767 			desc >= head_desc &&  in prepare_write_back_data()
768 			desc - head_desc < (int)max_n_descs) {  in prepare_write_back_data()
769 		desc++;  in prepare_write_back_data()
770 		if (unlikely(!(desc->flags & VRING_DESC_F_WRITE))) {  in prepare_write_back_data()
775 		if (desc->len <= offset) {  in prepare_write_back_data()
776 			offset -= desc->len;  in prepare_write_back_data()
780 		dlen = desc->len;  in prepare_write_back_data()
781 		dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen,  in prepare_write_back_data()
783 		if (unlikely(dst == NULL || dlen != desc->len)) {  in prepare_write_back_data()
790 		wb_data->len = RTE_MIN(desc->len - offset, write_back_len);  in prepare_write_back_data()
808 	if (unlikely(desc - head_desc == (int)max_n_descs))  in prepare_write_back_data()
811 		*cur_desc = desc + 1;  in prepare_write_back_data()
842 	struct vhost_crypto_desc *desc = head;  in prepare_sym_cipher_op()  local
853 	if (unlikely(copy_data(iv_data, vc_req, head, &desc,  in prepare_sym_cipher_op()
862 		m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,  in prepare_sym_cipher_op()
864 		m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO);  in prepare_sym_cipher_op()
872 		if (unlikely(move_desc(head, &desc, cipher->para.src_data_len,  in prepare_sym_cipher_op()
884 				vc_req, head, &desc, cipher->para.src_data_len,  in prepare_sym_cipher_op()
896 	desc = find_write_desc(head, desc, max_n_descs);  in prepare_sym_cipher_op()
897 	if (unlikely(!desc)) {  in prepare_sym_cipher_op()
906 				desc->addr, cipher->para.dst_data_len);  in prepare_sym_cipher_op()
907 		m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW);  in prepare_sym_cipher_op()
914 		if (unlikely(move_desc(head, &desc, cipher->para.dst_data_len,  in prepare_sym_cipher_op()
924 		vc_req->wb = prepare_write_back_data(vc_req, head, &desc, &ewb,  in prepare_sym_cipher_op()
945 	vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO);  in prepare_sym_cipher_op()
994 	struct vhost_crypto_desc *desc = head, *digest_desc;  in prepare_sym_chain_op()  local
1007 	if (unlikely(copy_data(iv_data, vc_req, head, &desc,  in prepare_sym_chain_op()
1018 		m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,  in prepare_sym_chain_op()
1020 		m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO);  in prepare_sym_chain_op()
1027 		if (unlikely(move_desc(head, &desc, chain->para.src_data_len,  in prepare_sym_chain_op()
1038 				vc_req, head, &desc, chain->para.src_data_len,  in prepare_sym_chain_op()
1051 	desc = find_write_desc(head, desc, max_n_descs);  in prepare_sym_chain_op()
1052 	if (unlikely(!desc)) {  in prepare_sym_chain_op()
1061 				desc->addr, chain->para.dst_data_len);  in prepare_sym_chain_op()
1062 		m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW);  in prepare_sym_chain_op()
1069 		if (unlikely(move_desc(vc_req->head, &desc,  in prepare_sym_chain_op()
1077 				desc->addr, chain->para.hash_result_len);  in prepare_sym_chain_op()
1078 		op->sym->auth.digest.data = get_data_ptr(vc_req, desc,  in prepare_sym_chain_op()
1086 		if (unlikely(move_desc(head, &desc,  in prepare_sym_chain_op()
1096 		vc_req->wb = prepare_write_back_data(vc_req, head, &desc, &ewb,  in prepare_sym_chain_op()
1107 		digest_desc = desc;  in prepare_sym_chain_op()
1113 		ewb->next = prepare_write_back_data(vc_req, head, &desc,  in prepare_sym_chain_op()
1138 	vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO);  in prepare_sym_chain_op()
1180 	struct vhost_crypto_desc *desc = descs;  in vhost_crypto_process_one_req()  local
1230 		desc->addr = src_desc->addr;  in vhost_crypto_process_one_req()
1231 		desc->len = src_desc->len;  in vhost_crypto_process_one_req()
1232 		desc->flags = src_desc->flags;  in vhost_crypto_process_one_req()
1233 		desc++;  in vhost_crypto_process_one_req()
1247 	nb_descs = desc - descs;  in vhost_crypto_process_one_req()
1248 	desc = descs;  in vhost_crypto_process_one_req()
1250 	if (unlikely(desc->len < sizeof(req))) {  in vhost_crypto_process_one_req()
1256 	if (unlikely(copy_data(&req, vc_req, descs, &desc, sizeof(req),  in vhost_crypto_process_one_req()
1301 					&req.u.sym_req.u.cipher, desc,  in vhost_crypto_process_one_req()
1306 					&req.u.sym_req.u.chain, desc,  in vhost_crypto_process_one_req()
1648 			struct vring_desc *head = &vq->desc[desc_idx];  in rte_vhost_crypto_fetch_requests()
1678 			struct vring_desc *head = &vq->desc[desc_idx];  in rte_vhost_crypto_fetch_requests()