1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017-2018 Intel Corporation
3  */
4 #include <rte_malloc.h>
5 #include <rte_hash.h>
6 #include <rte_jhash.h>
7 #include <rte_mbuf.h>
8 #include <rte_cryptodev.h>
9 
10 #include "rte_vhost_crypto.h"
11 #include "vhost.h"
12 #include "vhost_user.h"
13 #include "virtio_crypto.h"
14 
15 #define INHDR_LEN		(sizeof(struct virtio_crypto_inhdr))
16 #define IV_OFFSET		(sizeof(struct rte_crypto_op) + \
17 				sizeof(struct rte_crypto_sym_op))
18 
19 #ifdef RTE_LIBRTE_VHOST_DEBUG
20 #define VC_LOG_ERR(fmt, args...)				\
21 	RTE_LOG(ERR, USER1, "[%s] %s() line %u: " fmt "\n",	\
22 		"Vhost-Crypto",	__func__, __LINE__, ## args)
23 #define VC_LOG_INFO(fmt, args...)				\
24 	RTE_LOG(INFO, USER1, "[%s] %s() line %u: " fmt "\n",	\
25 		"Vhost-Crypto",	__func__, __LINE__, ## args)
26 
27 #define VC_LOG_DBG(fmt, args...)				\
28 	RTE_LOG(DEBUG, USER1, "[%s] %s() line %u: " fmt "\n",	\
29 		"Vhost-Crypto",	__func__, __LINE__, ## args)
30 #else
31 #define VC_LOG_ERR(fmt, args...)				\
32 	RTE_LOG(ERR, USER1, "[VHOST-Crypto]: " fmt "\n", ## args)
33 #define VC_LOG_INFO(fmt, args...)				\
34 	RTE_LOG(INFO, USER1, "[VHOST-Crypto]: " fmt "\n", ## args)
35 #define VC_LOG_DBG(fmt, args...)
36 #endif
37 
38 #define VIRTIO_CRYPTO_FEATURES ((1 << VIRTIO_F_NOTIFY_ON_EMPTY) |	\
39 		(1 << VIRTIO_RING_F_INDIRECT_DESC) |			\
40 		(1 << VIRTIO_RING_F_EVENT_IDX) |			\
41 		(1 << VIRTIO_CRYPTO_SERVICE_CIPHER) |			\
42 		(1 << VIRTIO_CRYPTO_SERVICE_MAC) |			\
43 		(1 << VIRTIO_NET_F_CTRL_VQ))
44 
45 #define IOVA_TO_VVA(t, r, a, l, p)					\
46 	((t)(uintptr_t)vhost_iova_to_vva(r->dev, r->vq, a, l, p))
47 
48 static int
49 cipher_algo_transform(uint32_t virtio_cipher_algo,
50 		enum rte_crypto_cipher_algorithm *algo)
51 {
52 	switch (virtio_cipher_algo) {
53 	case VIRTIO_CRYPTO_CIPHER_AES_CBC:
54 		*algo = RTE_CRYPTO_CIPHER_AES_CBC;
55 		break;
56 	case VIRTIO_CRYPTO_CIPHER_AES_CTR:
57 		*algo = RTE_CRYPTO_CIPHER_AES_CTR;
58 		break;
59 	case VIRTIO_CRYPTO_CIPHER_DES_ECB:
60 		*algo = -VIRTIO_CRYPTO_NOTSUPP;
61 		break;
62 	case VIRTIO_CRYPTO_CIPHER_DES_CBC:
63 		*algo = RTE_CRYPTO_CIPHER_DES_CBC;
64 		break;
65 	case VIRTIO_CRYPTO_CIPHER_3DES_ECB:
66 		*algo = RTE_CRYPTO_CIPHER_3DES_ECB;
67 		break;
68 	case VIRTIO_CRYPTO_CIPHER_3DES_CBC:
69 		*algo = RTE_CRYPTO_CIPHER_3DES_CBC;
70 		break;
71 	case VIRTIO_CRYPTO_CIPHER_3DES_CTR:
72 		*algo = RTE_CRYPTO_CIPHER_3DES_CTR;
73 		break;
74 	case VIRTIO_CRYPTO_CIPHER_KASUMI_F8:
75 		*algo = RTE_CRYPTO_CIPHER_KASUMI_F8;
76 		break;
77 	case VIRTIO_CRYPTO_CIPHER_SNOW3G_UEA2:
78 		*algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2;
79 		break;
80 	case VIRTIO_CRYPTO_CIPHER_AES_F8:
81 		*algo = RTE_CRYPTO_CIPHER_AES_F8;
82 		break;
83 	case VIRTIO_CRYPTO_CIPHER_AES_XTS:
84 		*algo = RTE_CRYPTO_CIPHER_AES_XTS;
85 		break;
86 	case VIRTIO_CRYPTO_CIPHER_ZUC_EEA3:
87 		*algo = RTE_CRYPTO_CIPHER_ZUC_EEA3;
88 		break;
89 	default:
90 		return -VIRTIO_CRYPTO_BADMSG;
91 		break;
92 	}
93 
94 	return 0;
95 }
96 
97 static int
98 auth_algo_transform(uint32_t virtio_auth_algo,
99 		enum rte_crypto_auth_algorithm *algo)
100 {
101 	switch (virtio_auth_algo) {
102 	case VIRTIO_CRYPTO_NO_MAC:
103 		*algo = RTE_CRYPTO_AUTH_NULL;
104 		break;
105 	case VIRTIO_CRYPTO_MAC_HMAC_MD5:
106 		*algo = RTE_CRYPTO_AUTH_MD5_HMAC;
107 		break;
108 	case VIRTIO_CRYPTO_MAC_HMAC_SHA1:
109 		*algo = RTE_CRYPTO_AUTH_SHA1_HMAC;
110 		break;
111 	case VIRTIO_CRYPTO_MAC_HMAC_SHA_224:
112 		*algo = RTE_CRYPTO_AUTH_SHA224_HMAC;
113 		break;
114 	case VIRTIO_CRYPTO_MAC_HMAC_SHA_256:
115 		*algo = RTE_CRYPTO_AUTH_SHA256_HMAC;
116 		break;
117 	case VIRTIO_CRYPTO_MAC_HMAC_SHA_384:
118 		*algo = RTE_CRYPTO_AUTH_SHA384_HMAC;
119 		break;
120 	case VIRTIO_CRYPTO_MAC_HMAC_SHA_512:
121 		*algo = RTE_CRYPTO_AUTH_SHA512_HMAC;
122 		break;
123 	case VIRTIO_CRYPTO_MAC_CMAC_AES:
124 		*algo = RTE_CRYPTO_AUTH_AES_CMAC;
125 		break;
126 	case VIRTIO_CRYPTO_MAC_KASUMI_F9:
127 		*algo = RTE_CRYPTO_AUTH_KASUMI_F9;
128 		break;
129 	case VIRTIO_CRYPTO_MAC_SNOW3G_UIA2:
130 		*algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2;
131 		break;
132 	case VIRTIO_CRYPTO_MAC_GMAC_AES:
133 		*algo = RTE_CRYPTO_AUTH_AES_GMAC;
134 		break;
135 	case VIRTIO_CRYPTO_MAC_CBCMAC_AES:
136 		*algo = RTE_CRYPTO_AUTH_AES_CBC_MAC;
137 		break;
138 	case VIRTIO_CRYPTO_MAC_XCBC_AES:
139 		*algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC;
140 		break;
141 	case VIRTIO_CRYPTO_MAC_CMAC_3DES:
142 	case VIRTIO_CRYPTO_MAC_GMAC_TWOFISH:
143 	case VIRTIO_CRYPTO_MAC_CBCMAC_KASUMI_F9:
144 		return -VIRTIO_CRYPTO_NOTSUPP;
145 	default:
146 		return -VIRTIO_CRYPTO_BADMSG;
147 	}
148 
149 	return 0;
150 }
151 
152 static int get_iv_len(enum rte_crypto_cipher_algorithm algo)
153 {
154 	int len;
155 
156 	switch (algo) {
157 	case RTE_CRYPTO_CIPHER_3DES_CBC:
158 		len = 8;
159 		break;
160 	case RTE_CRYPTO_CIPHER_3DES_CTR:
161 		len = 8;
162 		break;
163 	case RTE_CRYPTO_CIPHER_3DES_ECB:
164 		len = 8;
165 		break;
166 	case RTE_CRYPTO_CIPHER_AES_CBC:
167 		len = 16;
168 		break;
169 
170 	/* TODO: add common algos */
171 
172 	default:
173 		len = -1;
174 		break;
175 	}
176 
177 	return len;
178 }
179 
180 /**
181  * vhost_crypto struct is used to maintain a number of virtio_cryptos and
182  * one DPDK crypto device that deals with all crypto workloads. It is declared
183  * here and defined in vhost_crypto.c
184  */
185 struct vhost_crypto {
186 	/** Used to lookup DPDK Cryptodev Session based on VIRTIO crypto
187 	 *  session ID.
188 	 */
189 	struct rte_hash *session_map;
190 	struct rte_mempool *mbuf_pool;
191 	struct rte_mempool *sess_pool;
192 	struct rte_mempool *wb_pool;
193 
194 	/** DPDK cryptodev ID */
195 	uint8_t cid;
196 	uint16_t nb_qps;
197 
198 	uint64_t last_session_id;
199 
200 	uint64_t cache_session_id;
201 	struct rte_cryptodev_sym_session *cache_session;
202 	/** socket id for the device */
203 	int socket_id;
204 
205 	struct virtio_net *dev;
206 
207 	uint8_t option;
208 } __rte_cache_aligned;
209 
210 struct vhost_crypto_writeback_data {
211 	uint8_t *src;
212 	uint8_t *dst;
213 	uint64_t len;
214 	struct vhost_crypto_writeback_data *next;
215 };
216 
217 struct vhost_crypto_data_req {
218 	struct vring_desc *head;
219 	struct virtio_net *dev;
220 	struct virtio_crypto_inhdr *inhdr;
221 	struct vhost_virtqueue *vq;
222 	struct vhost_crypto_writeback_data *wb;
223 	struct rte_mempool *wb_pool;
224 	uint16_t desc_idx;
225 	uint16_t len;
226 	uint16_t zero_copy;
227 };
228 
229 static int
230 transform_cipher_param(struct rte_crypto_sym_xform *xform,
231 		VhostUserCryptoSessionParam *param)
232 {
233 	int ret;
234 
235 	ret = cipher_algo_transform(param->cipher_algo, &xform->cipher.algo);
236 	if (unlikely(ret < 0))
237 		return ret;
238 
239 	xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
240 	xform->cipher.key.length = param->cipher_key_len;
241 	if (xform->cipher.key.length > 0)
242 		xform->cipher.key.data = param->cipher_key_buf;
243 	if (param->dir == VIRTIO_CRYPTO_OP_ENCRYPT)
244 		xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
245 	else if (param->dir == VIRTIO_CRYPTO_OP_DECRYPT)
246 		xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
247 	else {
248 		VC_LOG_DBG("Bad operation type");
249 		return -VIRTIO_CRYPTO_BADMSG;
250 	}
251 
252 	ret = get_iv_len(xform->cipher.algo);
253 	if (unlikely(ret < 0))
254 		return ret;
255 	xform->cipher.iv.length = (uint16_t)ret;
256 	xform->cipher.iv.offset = IV_OFFSET;
257 	return 0;
258 }
259 
260 static int
261 transform_chain_param(struct rte_crypto_sym_xform *xforms,
262 		VhostUserCryptoSessionParam *param)
263 {
264 	struct rte_crypto_sym_xform *xform_cipher, *xform_auth;
265 	int ret;
266 
267 	switch (param->chaining_dir) {
268 	case VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER:
269 		xform_auth = xforms;
270 		xform_cipher = xforms->next;
271 		xform_cipher->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
272 		xform_auth->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
273 		break;
274 	case VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH:
275 		xform_cipher = xforms;
276 		xform_auth = xforms->next;
277 		xform_cipher->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
278 		xform_auth->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
279 		break;
280 	default:
281 		return -VIRTIO_CRYPTO_BADMSG;
282 	}
283 
284 	/* cipher */
285 	ret = cipher_algo_transform(param->cipher_algo,
286 			&xform_cipher->cipher.algo);
287 	if (unlikely(ret < 0))
288 		return ret;
289 	xform_cipher->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
290 	xform_cipher->cipher.key.length = param->cipher_key_len;
291 	xform_cipher->cipher.key.data = param->cipher_key_buf;
292 	ret = get_iv_len(xform_cipher->cipher.algo);
293 	if (unlikely(ret < 0))
294 		return ret;
295 	xform_cipher->cipher.iv.length = (uint16_t)ret;
296 	xform_cipher->cipher.iv.offset = IV_OFFSET;
297 
298 	/* auth */
299 	xform_auth->type = RTE_CRYPTO_SYM_XFORM_AUTH;
300 	ret = auth_algo_transform(param->hash_algo, &xform_auth->auth.algo);
301 	if (unlikely(ret < 0))
302 		return ret;
303 	xform_auth->auth.digest_length = param->digest_len;
304 	xform_auth->auth.key.length = param->auth_key_len;
305 	xform_auth->auth.key.data = param->auth_key_buf;
306 
307 	return 0;
308 }
309 
310 static void
311 vhost_crypto_create_sess(struct vhost_crypto *vcrypto,
312 		VhostUserCryptoSessionParam *sess_param)
313 {
314 	struct rte_crypto_sym_xform xform1 = {0}, xform2 = {0};
315 	struct rte_cryptodev_sym_session *session;
316 	int ret;
317 
318 	switch (sess_param->op_type) {
319 	case VIRTIO_CRYPTO_SYM_OP_NONE:
320 	case VIRTIO_CRYPTO_SYM_OP_CIPHER:
321 		ret = transform_cipher_param(&xform1, sess_param);
322 		if (unlikely(ret)) {
323 			VC_LOG_ERR("Error transform session msg (%i)", ret);
324 			sess_param->session_id = ret;
325 			return;
326 		}
327 		break;
328 	case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
329 		if (unlikely(sess_param->hash_mode !=
330 				VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH)) {
331 			sess_param->session_id = -VIRTIO_CRYPTO_NOTSUPP;
332 			VC_LOG_ERR("Error transform session message (%i)",
333 					-VIRTIO_CRYPTO_NOTSUPP);
334 			return;
335 		}
336 
337 		xform1.next = &xform2;
338 
339 		ret = transform_chain_param(&xform1, sess_param);
340 		if (unlikely(ret)) {
341 			VC_LOG_ERR("Error transform session message (%i)", ret);
342 			sess_param->session_id = ret;
343 			return;
344 		}
345 
346 		break;
347 	default:
348 		VC_LOG_ERR("Algorithm not yet supported");
349 		sess_param->session_id = -VIRTIO_CRYPTO_NOTSUPP;
350 		return;
351 	}
352 
353 	session = rte_cryptodev_sym_session_create(vcrypto->sess_pool);
354 	if (!session) {
355 		VC_LOG_ERR("Failed to create session");
356 		sess_param->session_id = -VIRTIO_CRYPTO_ERR;
357 		return;
358 	}
359 
360 	if (rte_cryptodev_sym_session_init(vcrypto->cid, session, &xform1,
361 			vcrypto->sess_pool) < 0) {
362 		VC_LOG_ERR("Failed to initialize session");
363 		sess_param->session_id = -VIRTIO_CRYPTO_ERR;
364 		return;
365 	}
366 
367 	/* insert hash to map */
368 	if (rte_hash_add_key_data(vcrypto->session_map,
369 			&vcrypto->last_session_id, session) < 0) {
370 		VC_LOG_ERR("Failed to insert session to hash table");
371 
372 		if (rte_cryptodev_sym_session_clear(vcrypto->cid, session) < 0)
373 			VC_LOG_ERR("Failed to clear session");
374 		else {
375 			if (rte_cryptodev_sym_session_free(session) < 0)
376 				VC_LOG_ERR("Failed to free session");
377 		}
378 		sess_param->session_id = -VIRTIO_CRYPTO_ERR;
379 		return;
380 	}
381 
382 	VC_LOG_INFO("Session %"PRIu64" created for vdev %i.",
383 			vcrypto->last_session_id, vcrypto->dev->vid);
384 
385 	sess_param->session_id = vcrypto->last_session_id;
386 	vcrypto->last_session_id++;
387 }
388 
389 static int
390 vhost_crypto_close_sess(struct vhost_crypto *vcrypto, uint64_t session_id)
391 {
392 	struct rte_cryptodev_sym_session *session;
393 	uint64_t sess_id = session_id;
394 	int ret;
395 
396 	ret = rte_hash_lookup_data(vcrypto->session_map, &sess_id,
397 			(void **)&session);
398 
399 	if (unlikely(ret < 0)) {
400 		VC_LOG_ERR("Failed to delete session %"PRIu64".", session_id);
401 		return -VIRTIO_CRYPTO_INVSESS;
402 	}
403 
404 	if (rte_cryptodev_sym_session_clear(vcrypto->cid, session) < 0) {
405 		VC_LOG_DBG("Failed to clear session");
406 		return -VIRTIO_CRYPTO_ERR;
407 	}
408 
409 	if (rte_cryptodev_sym_session_free(session) < 0) {
410 		VC_LOG_DBG("Failed to free session");
411 		return -VIRTIO_CRYPTO_ERR;
412 	}
413 
414 	if (rte_hash_del_key(vcrypto->session_map, &sess_id) < 0) {
415 		VC_LOG_DBG("Failed to delete session from hash table.");
416 		return -VIRTIO_CRYPTO_ERR;
417 	}
418 
419 	VC_LOG_INFO("Session %"PRIu64" deleted for vdev %i.", sess_id,
420 			vcrypto->dev->vid);
421 
422 	return 0;
423 }
424 
425 static enum vh_result
426 vhost_crypto_msg_post_handler(int vid, void *msg)
427 {
428 	struct virtio_net *dev = get_device(vid);
429 	struct vhost_crypto *vcrypto;
430 	VhostUserMsg *vmsg = msg;
431 	enum vh_result ret = VH_RESULT_OK;
432 
433 	if (dev == NULL) {
434 		VC_LOG_ERR("Invalid vid %i", vid);
435 		return VH_RESULT_ERR;
436 	}
437 
438 	vcrypto = dev->extern_data;
439 	if (vcrypto == NULL) {
440 		VC_LOG_ERR("Cannot find required data, is it initialized?");
441 		return VH_RESULT_ERR;
442 	}
443 
444 	if (vmsg->request.master == VHOST_USER_CRYPTO_CREATE_SESS) {
445 		vhost_crypto_create_sess(vcrypto,
446 				&vmsg->payload.crypto_session);
447 		vmsg->fd_num = 0;
448 		ret = VH_RESULT_REPLY;
449 	} else if (vmsg->request.master == VHOST_USER_CRYPTO_CLOSE_SESS) {
450 		if (vhost_crypto_close_sess(vcrypto, vmsg->payload.u64))
451 			ret = VH_RESULT_ERR;
452 	}
453 
454 	return ret;
455 }
456 
457 static __rte_always_inline struct vring_desc *
458 find_write_desc(struct vring_desc *head, struct vring_desc *desc,
459 		uint32_t *nb_descs, uint32_t vq_size)
460 {
461 	if (desc->flags & VRING_DESC_F_WRITE)
462 		return desc;
463 
464 	while (desc->flags & VRING_DESC_F_NEXT) {
465 		if (unlikely(*nb_descs == 0 || desc->next >= vq_size))
466 			return NULL;
467 		(*nb_descs)--;
468 
469 		desc = &head[desc->next];
470 		if (desc->flags & VRING_DESC_F_WRITE)
471 			return desc;
472 	}
473 
474 	return NULL;
475 }
476 
477 static struct virtio_crypto_inhdr *
478 reach_inhdr(struct vhost_crypto_data_req *vc_req, struct vring_desc *desc,
479 		uint32_t *nb_descs, uint32_t vq_size)
480 {
481 	uint64_t dlen;
482 	struct virtio_crypto_inhdr *inhdr;
483 
484 	while (desc->flags & VRING_DESC_F_NEXT) {
485 		if (unlikely(*nb_descs == 0 || desc->next >= vq_size))
486 			return NULL;
487 		(*nb_descs)--;
488 		desc = &vc_req->head[desc->next];
489 	}
490 
491 	dlen = desc->len;
492 	inhdr = IOVA_TO_VVA(struct virtio_crypto_inhdr *, vc_req, desc->addr,
493 			&dlen, VHOST_ACCESS_WO);
494 	if (unlikely(!inhdr || dlen != desc->len))
495 		return NULL;
496 
497 	return inhdr;
498 }
499 
500 static __rte_always_inline int
501 move_desc(struct vring_desc *head, struct vring_desc **cur_desc,
502 		uint32_t size, uint32_t *nb_descs, uint32_t vq_size)
503 {
504 	struct vring_desc *desc = *cur_desc;
505 	int left = size - desc->len;
506 
507 	while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) {
508 		(*nb_descs)--;
509 		if (unlikely(*nb_descs == 0 || desc->next >= vq_size))
510 			return -1;
511 
512 		desc = &head[desc->next];
513 		rte_prefetch0(&head[desc->next]);
514 		left -= desc->len;
515 	}
516 
517 	if (unlikely(left > 0))
518 		return -1;
519 
520 	if (unlikely(*nb_descs == 0))
521 		*cur_desc = NULL;
522 	else {
523 		if (unlikely(desc->next >= vq_size))
524 			return -1;
525 		*cur_desc = &head[desc->next];
526 	}
527 
528 	return 0;
529 }
530 
531 static __rte_always_inline void *
532 get_data_ptr(struct vhost_crypto_data_req *vc_req, struct vring_desc *cur_desc,
533 		uint8_t perm)
534 {
535 	void *data;
536 	uint64_t dlen = cur_desc->len;
537 
538 	data = IOVA_TO_VVA(void *, vc_req, cur_desc->addr, &dlen, perm);
539 	if (unlikely(!data || dlen != cur_desc->len)) {
540 		VC_LOG_ERR("Failed to map object");
541 		return NULL;
542 	}
543 
544 	return data;
545 }
546 
547 static int
548 copy_data(void *dst_data, struct vhost_crypto_data_req *vc_req,
549 		struct vring_desc **cur_desc, uint32_t size,
550 		uint32_t *nb_descs, uint32_t vq_size)
551 {
552 	struct vring_desc *desc = *cur_desc;
553 	uint64_t remain, addr, dlen, len;
554 	uint32_t to_copy;
555 	uint8_t *data = dst_data;
556 	uint8_t *src;
557 	int left = size;
558 
559 	to_copy = RTE_MIN(desc->len, (uint32_t)left);
560 	dlen = to_copy;
561 	src = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen,
562 			VHOST_ACCESS_RO);
563 	if (unlikely(!src || !dlen))
564 		return -1;
565 
566 	rte_memcpy((uint8_t *)data, src, dlen);
567 	data += dlen;
568 
569 	if (unlikely(dlen < to_copy)) {
570 		remain = to_copy - dlen;
571 		addr = desc->addr + dlen;
572 
573 		while (remain) {
574 			len = remain;
575 			src = IOVA_TO_VVA(uint8_t *, vc_req, addr, &len,
576 					VHOST_ACCESS_RO);
577 			if (unlikely(!src || !len)) {
578 				VC_LOG_ERR("Failed to map descriptor");
579 				return -1;
580 			}
581 
582 			rte_memcpy(data, src, len);
583 			addr += len;
584 			remain -= len;
585 			data += len;
586 		}
587 	}
588 
589 	left -= to_copy;
590 
591 	while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) {
592 		if (unlikely(*nb_descs == 0 || desc->next >= vq_size)) {
593 			VC_LOG_ERR("Invalid descriptors");
594 			return -1;
595 		}
596 		(*nb_descs)--;
597 
598 		desc = &vc_req->head[desc->next];
599 		rte_prefetch0(&vc_req->head[desc->next]);
600 		to_copy = RTE_MIN(desc->len, (uint32_t)left);
601 		dlen = desc->len;
602 		src = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen,
603 				VHOST_ACCESS_RO);
604 		if (unlikely(!src || !dlen)) {
605 			VC_LOG_ERR("Failed to map descriptor");
606 			return -1;
607 		}
608 
609 		rte_memcpy(data, src, dlen);
610 		data += dlen;
611 
612 		if (unlikely(dlen < to_copy)) {
613 			remain = to_copy - dlen;
614 			addr = desc->addr + dlen;
615 
616 			while (remain) {
617 				len = remain;
618 				src = IOVA_TO_VVA(uint8_t *, vc_req, addr, &len,
619 						VHOST_ACCESS_RO);
620 				if (unlikely(!src || !len)) {
621 					VC_LOG_ERR("Failed to map descriptor");
622 					return -1;
623 				}
624 
625 				rte_memcpy(data, src, len);
626 				addr += len;
627 				remain -= len;
628 				data += len;
629 			}
630 		}
631 
632 		left -= to_copy;
633 	}
634 
635 	if (unlikely(left > 0)) {
636 		VC_LOG_ERR("Incorrect virtio descriptor");
637 		return -1;
638 	}
639 
640 	if (unlikely(*nb_descs == 0))
641 		*cur_desc = NULL;
642 	else {
643 		if (unlikely(desc->next >= vq_size))
644 			return -1;
645 		*cur_desc = &vc_req->head[desc->next];
646 	}
647 
648 	return 0;
649 }
650 
651 static void
652 write_back_data(struct vhost_crypto_data_req *vc_req)
653 {
654 	struct vhost_crypto_writeback_data *wb_data = vc_req->wb, *wb_last;
655 
656 	while (wb_data) {
657 		rte_memcpy(wb_data->dst, wb_data->src, wb_data->len);
658 		wb_last = wb_data;
659 		wb_data = wb_data->next;
660 		rte_mempool_put(vc_req->wb_pool, wb_last);
661 	}
662 }
663 
664 static void
665 free_wb_data(struct vhost_crypto_writeback_data *wb_data,
666 		struct rte_mempool *mp)
667 {
668 	while (wb_data->next != NULL)
669 		free_wb_data(wb_data->next, mp);
670 
671 	rte_mempool_put(mp, wb_data);
672 }
673 
674 /**
675  * The function will allocate a vhost_crypto_writeback_data linked list
676  * containing the source and destination data pointers for the write back
677  * operation after dequeued from Cryptodev PMD queues.
678  *
679  * @param vc_req
680  *   The vhost crypto data request pointer
681  * @param cur_desc
682  *   The pointer of the current in use descriptor pointer. The content of
683  *   cur_desc is expected to be updated after the function execution.
684  * @param end_wb_data
685  *   The last write back data element to be returned. It is used only in cipher
686  *   and hash chain operations.
687  * @param src
688  *   The source data pointer
689  * @param offset
690  *   The offset to both source and destination data. For source data the offset
691  *   is the number of bytes between src and start point of cipher operation. For
692  *   destination data the offset is the number of bytes from *cur_desc->addr
693  *   to the point where the src will be written to.
694  * @param write_back_len
695  *   The size of the write back length.
696  * @return
697  *   The pointer to the start of the write back data linked list.
698  */
699 static struct vhost_crypto_writeback_data *
700 prepare_write_back_data(struct vhost_crypto_data_req *vc_req,
701 		struct vring_desc **cur_desc,
702 		struct vhost_crypto_writeback_data **end_wb_data,
703 		uint8_t *src,
704 		uint32_t offset,
705 		uint64_t write_back_len,
706 		uint32_t *nb_descs, uint32_t vq_size)
707 {
708 	struct vhost_crypto_writeback_data *wb_data, *head;
709 	struct vring_desc *desc = *cur_desc;
710 	uint64_t dlen;
711 	uint8_t *dst;
712 	int ret;
713 
714 	ret = rte_mempool_get(vc_req->wb_pool, (void **)&head);
715 	if (unlikely(ret < 0)) {
716 		VC_LOG_ERR("no memory");
717 		goto error_exit;
718 	}
719 
720 	wb_data = head;
721 
722 	if (likely(desc->len > offset)) {
723 		wb_data->src = src + offset;
724 		dlen = desc->len;
725 		dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr,
726 			&dlen, VHOST_ACCESS_RW) + offset;
727 		if (unlikely(!dst || dlen != desc->len)) {
728 			VC_LOG_ERR("Failed to map descriptor");
729 			goto error_exit;
730 		}
731 
732 		wb_data->dst = dst;
733 		wb_data->len = desc->len - offset;
734 		write_back_len -= wb_data->len;
735 		src += offset + wb_data->len;
736 		offset = 0;
737 
738 		if (unlikely(write_back_len)) {
739 			ret = rte_mempool_get(vc_req->wb_pool,
740 					(void **)&(wb_data->next));
741 			if (unlikely(ret < 0)) {
742 				VC_LOG_ERR("no memory");
743 				goto error_exit;
744 			}
745 
746 			wb_data = wb_data->next;
747 		} else
748 			wb_data->next = NULL;
749 	} else
750 		offset -= desc->len;
751 
752 	while (write_back_len) {
753 		if (unlikely(*nb_descs == 0 || desc->next >= vq_size)) {
754 			VC_LOG_ERR("Invalid descriptors");
755 			goto error_exit;
756 		}
757 		(*nb_descs)--;
758 
759 		desc = &vc_req->head[desc->next];
760 		if (unlikely(!(desc->flags & VRING_DESC_F_WRITE))) {
761 			VC_LOG_ERR("incorrect descriptor");
762 			goto error_exit;
763 		}
764 
765 		if (desc->len <= offset) {
766 			offset -= desc->len;
767 			continue;
768 		}
769 
770 		dlen = desc->len;
771 		dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen,
772 				VHOST_ACCESS_RW) + offset;
773 		if (unlikely(dst == NULL || dlen != desc->len)) {
774 			VC_LOG_ERR("Failed to map descriptor");
775 			goto error_exit;
776 		}
777 
778 		wb_data->src = src;
779 		wb_data->dst = dst;
780 		wb_data->len = RTE_MIN(desc->len - offset, write_back_len);
781 		write_back_len -= wb_data->len;
782 		src += wb_data->len;
783 		offset = 0;
784 
785 		if (write_back_len) {
786 			ret = rte_mempool_get(vc_req->wb_pool,
787 					(void **)&(wb_data->next));
788 			if (unlikely(ret < 0)) {
789 				VC_LOG_ERR("no memory");
790 				goto error_exit;
791 			}
792 
793 			wb_data = wb_data->next;
794 		} else
795 			wb_data->next = NULL;
796 	}
797 
798 	if (unlikely(*nb_descs == 0))
799 		*cur_desc = NULL;
800 	else {
801 		if (unlikely(desc->next >= vq_size))
802 			goto error_exit;
803 		*cur_desc = &vc_req->head[desc->next];
804 	}
805 
806 	*end_wb_data = wb_data;
807 
808 	return head;
809 
810 error_exit:
811 	if (head)
812 		free_wb_data(head, vc_req->wb_pool);
813 
814 	return NULL;
815 }
816 
817 static uint8_t
818 prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op,
819 		struct vhost_crypto_data_req *vc_req,
820 		struct virtio_crypto_cipher_data_req *cipher,
821 		struct vring_desc *cur_desc,
822 		uint32_t *nb_descs, uint32_t vq_size)
823 {
824 	struct vring_desc *desc = cur_desc;
825 	struct vhost_crypto_writeback_data *ewb = NULL;
826 	struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst;
827 	uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
828 	uint8_t ret = 0;
829 
830 	/* prepare */
831 	/* iv */
832 	if (unlikely(copy_data(iv_data, vc_req, &desc, cipher->para.iv_len,
833 			nb_descs, vq_size) < 0)) {
834 		ret = VIRTIO_CRYPTO_BADMSG;
835 		goto error_exit;
836 	}
837 
838 	m_src->data_len = cipher->para.src_data_len;
839 
840 	switch (vcrypto->option) {
841 	case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
842 		m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,
843 				cipher->para.src_data_len);
844 		m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO);
845 		if (unlikely(m_src->buf_iova == 0 ||
846 				m_src->buf_addr == NULL)) {
847 			VC_LOG_ERR("zero_copy may fail due to cross page data");
848 			ret = VIRTIO_CRYPTO_ERR;
849 			goto error_exit;
850 		}
851 
852 		if (unlikely(move_desc(vc_req->head, &desc,
853 				cipher->para.src_data_len, nb_descs,
854 				vq_size) < 0)) {
855 			VC_LOG_ERR("Incorrect descriptor");
856 			ret = VIRTIO_CRYPTO_ERR;
857 			goto error_exit;
858 		}
859 
860 		break;
861 	case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
862 		vc_req->wb_pool = vcrypto->wb_pool;
863 
864 		if (unlikely(cipher->para.src_data_len >
865 				RTE_MBUF_DEFAULT_BUF_SIZE)) {
866 			VC_LOG_ERR("Not enough space to do data copy");
867 			ret = VIRTIO_CRYPTO_ERR;
868 			goto error_exit;
869 		}
870 		if (unlikely(copy_data(rte_pktmbuf_mtod(m_src, uint8_t *),
871 				vc_req, &desc, cipher->para.src_data_len,
872 				nb_descs, vq_size) < 0)) {
873 			ret = VIRTIO_CRYPTO_BADMSG;
874 			goto error_exit;
875 		}
876 		break;
877 	default:
878 		ret = VIRTIO_CRYPTO_BADMSG;
879 		goto error_exit;
880 	}
881 
882 	/* dst */
883 	desc = find_write_desc(vc_req->head, desc, nb_descs, vq_size);
884 	if (unlikely(!desc)) {
885 		VC_LOG_ERR("Cannot find write location");
886 		ret = VIRTIO_CRYPTO_BADMSG;
887 		goto error_exit;
888 	}
889 
890 	switch (vcrypto->option) {
891 	case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
892 		m_dst->buf_iova = gpa_to_hpa(vcrypto->dev,
893 				desc->addr, cipher->para.dst_data_len);
894 		m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW);
895 		if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) {
896 			VC_LOG_ERR("zero_copy may fail due to cross page data");
897 			ret = VIRTIO_CRYPTO_ERR;
898 			goto error_exit;
899 		}
900 
901 		if (unlikely(move_desc(vc_req->head, &desc,
902 				cipher->para.dst_data_len,
903 				nb_descs, vq_size) < 0)) {
904 			VC_LOG_ERR("Incorrect descriptor");
905 			ret = VIRTIO_CRYPTO_ERR;
906 			goto error_exit;
907 		}
908 
909 		m_dst->data_len = cipher->para.dst_data_len;
910 		break;
911 	case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
912 		vc_req->wb = prepare_write_back_data(vc_req, &desc, &ewb,
913 				rte_pktmbuf_mtod(m_src, uint8_t *), 0,
914 				cipher->para.dst_data_len, nb_descs, vq_size);
915 		if (unlikely(vc_req->wb == NULL)) {
916 			ret = VIRTIO_CRYPTO_ERR;
917 			goto error_exit;
918 		}
919 
920 		break;
921 	default:
922 		ret = VIRTIO_CRYPTO_BADMSG;
923 		goto error_exit;
924 	}
925 
926 	/* src data */
927 	op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
928 	op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
929 
930 	op->sym->cipher.data.offset = 0;
931 	op->sym->cipher.data.length = cipher->para.src_data_len;
932 
933 	vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO);
934 	if (unlikely(vc_req->inhdr == NULL)) {
935 		ret = VIRTIO_CRYPTO_BADMSG;
936 		goto error_exit;
937 	}
938 
939 	vc_req->inhdr->status = VIRTIO_CRYPTO_OK;
940 	vc_req->len = cipher->para.dst_data_len + INHDR_LEN;
941 
942 	return 0;
943 
944 error_exit:
945 	if (vc_req->wb)
946 		free_wb_data(vc_req->wb, vc_req->wb_pool);
947 
948 	vc_req->len = INHDR_LEN;
949 	return ret;
950 }
951 
952 static uint8_t
953 prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op,
954 		struct vhost_crypto_data_req *vc_req,
955 		struct virtio_crypto_alg_chain_data_req *chain,
956 		struct vring_desc *cur_desc,
957 		uint32_t *nb_descs, uint32_t vq_size)
958 {
959 	struct vring_desc *desc = cur_desc, *digest_desc;
960 	struct vhost_crypto_writeback_data *ewb = NULL, *ewb2 = NULL;
961 	struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst;
962 	uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
963 	uint32_t digest_offset;
964 	void *digest_addr;
965 	uint8_t ret = 0;
966 
967 	/* prepare */
968 	/* iv */
969 	if (unlikely(copy_data(iv_data, vc_req, &desc,
970 			chain->para.iv_len, nb_descs, vq_size) < 0)) {
971 		ret = VIRTIO_CRYPTO_BADMSG;
972 		goto error_exit;
973 	}
974 
975 	m_src->data_len = chain->para.src_data_len;
976 
977 	switch (vcrypto->option) {
978 	case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
979 		m_dst->data_len = chain->para.dst_data_len;
980 
981 		m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,
982 				chain->para.src_data_len);
983 		m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO);
984 		if (unlikely(m_src->buf_iova == 0 || m_src->buf_addr == NULL)) {
985 			VC_LOG_ERR("zero_copy may fail due to cross page data");
986 			ret = VIRTIO_CRYPTO_ERR;
987 			goto error_exit;
988 		}
989 
990 		if (unlikely(move_desc(vc_req->head, &desc,
991 				chain->para.src_data_len,
992 				nb_descs, vq_size) < 0)) {
993 			VC_LOG_ERR("Incorrect descriptor");
994 			ret = VIRTIO_CRYPTO_ERR;
995 			goto error_exit;
996 		}
997 		break;
998 	case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
999 		vc_req->wb_pool = vcrypto->wb_pool;
1000 
1001 		if (unlikely(chain->para.src_data_len >
1002 				RTE_MBUF_DEFAULT_BUF_SIZE)) {
1003 			VC_LOG_ERR("Not enough space to do data copy");
1004 			ret = VIRTIO_CRYPTO_ERR;
1005 			goto error_exit;
1006 		}
1007 		if (unlikely(copy_data(rte_pktmbuf_mtod(m_src, uint8_t *),
1008 				vc_req, &desc, chain->para.src_data_len,
1009 				nb_descs, vq_size) < 0)) {
1010 			ret = VIRTIO_CRYPTO_BADMSG;
1011 			goto error_exit;
1012 		}
1013 
1014 		break;
1015 	default:
1016 		ret = VIRTIO_CRYPTO_BADMSG;
1017 		goto error_exit;
1018 	}
1019 
1020 	/* dst */
1021 	desc = find_write_desc(vc_req->head, desc, nb_descs, vq_size);
1022 	if (unlikely(!desc)) {
1023 		VC_LOG_ERR("Cannot find write location");
1024 		ret = VIRTIO_CRYPTO_BADMSG;
1025 		goto error_exit;
1026 	}
1027 
1028 	switch (vcrypto->option) {
1029 	case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
1030 		m_dst->buf_iova = gpa_to_hpa(vcrypto->dev,
1031 				desc->addr, chain->para.dst_data_len);
1032 		m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW);
1033 		if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) {
1034 			VC_LOG_ERR("zero_copy may fail due to cross page data");
1035 			ret = VIRTIO_CRYPTO_ERR;
1036 			goto error_exit;
1037 		}
1038 
1039 		if (unlikely(move_desc(vc_req->head, &desc,
1040 				chain->para.dst_data_len,
1041 				nb_descs, vq_size) < 0)) {
1042 			VC_LOG_ERR("Incorrect descriptor");
1043 			ret = VIRTIO_CRYPTO_ERR;
1044 			goto error_exit;
1045 		}
1046 
1047 		op->sym->auth.digest.phys_addr = gpa_to_hpa(vcrypto->dev,
1048 				desc->addr, chain->para.hash_result_len);
1049 		op->sym->auth.digest.data = get_data_ptr(vc_req, desc,
1050 				VHOST_ACCESS_RW);
1051 		if (unlikely(op->sym->auth.digest.phys_addr == 0)) {
1052 			VC_LOG_ERR("zero_copy may fail due to cross page data");
1053 			ret = VIRTIO_CRYPTO_ERR;
1054 			goto error_exit;
1055 		}
1056 
1057 		if (unlikely(move_desc(vc_req->head, &desc,
1058 				chain->para.hash_result_len,
1059 				nb_descs, vq_size) < 0)) {
1060 			VC_LOG_ERR("Incorrect descriptor");
1061 			ret = VIRTIO_CRYPTO_ERR;
1062 			goto error_exit;
1063 		}
1064 
1065 		break;
1066 	case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
1067 		vc_req->wb = prepare_write_back_data(vc_req, &desc, &ewb,
1068 				rte_pktmbuf_mtod(m_src, uint8_t *),
1069 				chain->para.cipher_start_src_offset,
1070 				chain->para.dst_data_len -
1071 				chain->para.cipher_start_src_offset,
1072 				nb_descs, vq_size);
1073 		if (unlikely(vc_req->wb == NULL)) {
1074 			ret = VIRTIO_CRYPTO_ERR;
1075 			goto error_exit;
1076 		}
1077 
1078 		digest_offset = m_src->data_len;
1079 		digest_addr = rte_pktmbuf_mtod_offset(m_src, void *,
1080 				digest_offset);
1081 		digest_desc = desc;
1082 
1083 		/** create a wb_data for digest */
1084 		ewb->next = prepare_write_back_data(vc_req, &desc, &ewb2,
1085 				digest_addr, 0, chain->para.hash_result_len,
1086 				nb_descs, vq_size);
1087 		if (unlikely(ewb->next == NULL)) {
1088 			ret = VIRTIO_CRYPTO_ERR;
1089 			goto error_exit;
1090 		}
1091 
1092 		if (unlikely(copy_data(digest_addr, vc_req, &digest_desc,
1093 				chain->para.hash_result_len,
1094 				nb_descs, vq_size) < 0)) {
1095 			ret = VIRTIO_CRYPTO_BADMSG;
1096 			goto error_exit;
1097 		}
1098 
1099 		op->sym->auth.digest.data = digest_addr;
1100 		op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m_src,
1101 				digest_offset);
1102 		break;
1103 	default:
1104 		ret = VIRTIO_CRYPTO_BADMSG;
1105 		goto error_exit;
1106 	}
1107 
1108 	/* record inhdr */
1109 	vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO);
1110 	if (unlikely(vc_req->inhdr == NULL)) {
1111 		ret = VIRTIO_CRYPTO_BADMSG;
1112 		goto error_exit;
1113 	}
1114 
1115 	vc_req->inhdr->status = VIRTIO_CRYPTO_OK;
1116 
1117 	op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
1118 	op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
1119 
1120 	op->sym->cipher.data.offset = chain->para.cipher_start_src_offset;
1121 	op->sym->cipher.data.length = chain->para.src_data_len -
1122 			chain->para.cipher_start_src_offset;
1123 
1124 	op->sym->auth.data.offset = chain->para.hash_start_src_offset;
1125 	op->sym->auth.data.length = chain->para.len_to_hash;
1126 
1127 	vc_req->len = chain->para.dst_data_len + chain->para.hash_result_len +
1128 			INHDR_LEN;
1129 	return 0;
1130 
1131 error_exit:
1132 	if (vc_req->wb)
1133 		free_wb_data(vc_req->wb, vc_req->wb_pool);
1134 	vc_req->len = INHDR_LEN;
1135 	return ret;
1136 }
1137 
1138 /**
1139  * Process on descriptor
1140  */
1141 static __rte_always_inline int
1142 vhost_crypto_process_one_req(struct vhost_crypto *vcrypto,
1143 		struct vhost_virtqueue *vq, struct rte_crypto_op *op,
1144 		struct vring_desc *head, uint16_t desc_idx)
1145 {
1146 	struct vhost_crypto_data_req *vc_req = rte_mbuf_to_priv(op->sym->m_src);
1147 	struct rte_cryptodev_sym_session *session;
1148 	struct virtio_crypto_op_data_req *req, tmp_req;
1149 	struct virtio_crypto_inhdr *inhdr;
1150 	struct vring_desc *desc = NULL;
1151 	uint64_t session_id;
1152 	uint64_t dlen;
1153 	uint32_t nb_descs = vq->size;
1154 	int err = 0;
1155 
1156 	vc_req->desc_idx = desc_idx;
1157 	vc_req->dev = vcrypto->dev;
1158 	vc_req->vq = vq;
1159 
1160 	if (likely(head->flags & VRING_DESC_F_INDIRECT)) {
1161 		dlen = head->len;
1162 		nb_descs = dlen / sizeof(struct vring_desc);
1163 		/* drop invalid descriptors */
1164 		if (unlikely(nb_descs > vq->size))
1165 			return -1;
1166 		desc = IOVA_TO_VVA(struct vring_desc *, vc_req, head->addr,
1167 				&dlen, VHOST_ACCESS_RO);
1168 		if (unlikely(!desc || dlen != head->len))
1169 			return -1;
1170 		desc_idx = 0;
1171 		head = desc;
1172 	} else {
1173 		desc = head;
1174 	}
1175 
1176 	vc_req->head = head;
1177 	vc_req->zero_copy = vcrypto->option;
1178 
1179 	req = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO);
1180 	if (unlikely(req == NULL)) {
1181 		switch (vcrypto->option) {
1182 		case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
1183 			err = VIRTIO_CRYPTO_BADMSG;
1184 			VC_LOG_ERR("Invalid descriptor");
1185 			goto error_exit;
1186 		case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
1187 			req = &tmp_req;
1188 			if (unlikely(copy_data(req, vc_req, &desc, sizeof(*req),
1189 					&nb_descs, vq->size) < 0)) {
1190 				err = VIRTIO_CRYPTO_BADMSG;
1191 				VC_LOG_ERR("Invalid descriptor");
1192 				goto error_exit;
1193 			}
1194 			break;
1195 		default:
1196 			err = VIRTIO_CRYPTO_ERR;
1197 			VC_LOG_ERR("Invalid option");
1198 			goto error_exit;
1199 		}
1200 	} else {
1201 		if (unlikely(move_desc(vc_req->head, &desc,
1202 				sizeof(*req), &nb_descs, vq->size) < 0)) {
1203 			VC_LOG_ERR("Incorrect descriptor");
1204 			goto error_exit;
1205 		}
1206 	}
1207 
1208 	switch (req->header.opcode) {
1209 	case VIRTIO_CRYPTO_CIPHER_ENCRYPT:
1210 	case VIRTIO_CRYPTO_CIPHER_DECRYPT:
1211 		session_id = req->header.session_id;
1212 
1213 		/* one branch to avoid unnecessary table lookup */
1214 		if (vcrypto->cache_session_id != session_id) {
1215 			err = rte_hash_lookup_data(vcrypto->session_map,
1216 					&session_id, (void **)&session);
1217 			if (unlikely(err < 0)) {
1218 				err = VIRTIO_CRYPTO_ERR;
1219 				VC_LOG_ERR("Failed to find session %"PRIu64,
1220 						session_id);
1221 				goto error_exit;
1222 			}
1223 
1224 			vcrypto->cache_session = session;
1225 			vcrypto->cache_session_id = session_id;
1226 		}
1227 
1228 		session = vcrypto->cache_session;
1229 
1230 		err = rte_crypto_op_attach_sym_session(op, session);
1231 		if (unlikely(err < 0)) {
1232 			err = VIRTIO_CRYPTO_ERR;
1233 			VC_LOG_ERR("Failed to attach session to op");
1234 			goto error_exit;
1235 		}
1236 
1237 		switch (req->u.sym_req.op_type) {
1238 		case VIRTIO_CRYPTO_SYM_OP_NONE:
1239 			err = VIRTIO_CRYPTO_NOTSUPP;
1240 			break;
1241 		case VIRTIO_CRYPTO_SYM_OP_CIPHER:
1242 			err = prepare_sym_cipher_op(vcrypto, op, vc_req,
1243 					&req->u.sym_req.u.cipher, desc,
1244 					&nb_descs, vq->size);
1245 			break;
1246 		case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
1247 			err = prepare_sym_chain_op(vcrypto, op, vc_req,
1248 					&req->u.sym_req.u.chain, desc,
1249 					&nb_descs, vq->size);
1250 			break;
1251 		}
1252 		if (unlikely(err != 0)) {
1253 			VC_LOG_ERR("Failed to process sym request");
1254 			goto error_exit;
1255 		}
1256 		break;
1257 	default:
1258 		VC_LOG_ERR("Unsupported symmetric crypto request type %u",
1259 				req->header.opcode);
1260 		goto error_exit;
1261 	}
1262 
1263 	return 0;
1264 
1265 error_exit:
1266 
1267 	inhdr = reach_inhdr(vc_req, desc, &nb_descs, vq->size);
1268 	if (likely(inhdr != NULL))
1269 		inhdr->status = (uint8_t)err;
1270 
1271 	return -1;
1272 }
1273 
1274 static __rte_always_inline struct vhost_virtqueue *
1275 vhost_crypto_finalize_one_request(struct rte_crypto_op *op,
1276 		struct vhost_virtqueue *old_vq)
1277 {
1278 	struct rte_mbuf *m_src = op->sym->m_src;
1279 	struct rte_mbuf *m_dst = op->sym->m_dst;
1280 	struct vhost_crypto_data_req *vc_req = rte_mbuf_to_priv(m_src);
1281 	uint16_t desc_idx;
1282 
1283 	if (unlikely(!vc_req)) {
1284 		VC_LOG_ERR("Failed to retrieve vc_req");
1285 		return NULL;
1286 	}
1287 
1288 	if (old_vq && (vc_req->vq != old_vq))
1289 		return vc_req->vq;
1290 
1291 	desc_idx = vc_req->desc_idx;
1292 
1293 	if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS))
1294 		vc_req->inhdr->status = VIRTIO_CRYPTO_ERR;
1295 	else {
1296 		if (vc_req->zero_copy == 0)
1297 			write_back_data(vc_req);
1298 	}
1299 
1300 	vc_req->vq->used->ring[desc_idx].id = desc_idx;
1301 	vc_req->vq->used->ring[desc_idx].len = vc_req->len;
1302 
1303 	rte_mempool_put(m_src->pool, (void *)m_src);
1304 
1305 	if (m_dst)
1306 		rte_mempool_put(m_dst->pool, (void *)m_dst);
1307 
1308 	return vc_req->vq;
1309 }
1310 
1311 static __rte_always_inline uint16_t
1312 vhost_crypto_complete_one_vm_requests(struct rte_crypto_op **ops,
1313 		uint16_t nb_ops, int *callfd)
1314 {
1315 	uint16_t processed = 1;
1316 	struct vhost_virtqueue *vq, *tmp_vq;
1317 
1318 	if (unlikely(nb_ops == 0))
1319 		return 0;
1320 
1321 	vq = vhost_crypto_finalize_one_request(ops[0], NULL);
1322 	if (unlikely(vq == NULL))
1323 		return 0;
1324 	tmp_vq = vq;
1325 
1326 	while ((processed < nb_ops)) {
1327 		tmp_vq = vhost_crypto_finalize_one_request(ops[processed],
1328 				tmp_vq);
1329 
1330 		if (unlikely(vq != tmp_vq))
1331 			break;
1332 
1333 		processed++;
1334 	}
1335 
1336 	*callfd = vq->callfd;
1337 
1338 	*(volatile uint16_t *)&vq->used->idx += processed;
1339 
1340 	return processed;
1341 }
1342 
1343 int __rte_experimental
1344 rte_vhost_crypto_create(int vid, uint8_t cryptodev_id,
1345 		struct rte_mempool *sess_pool, int socket_id)
1346 {
1347 	struct virtio_net *dev = get_device(vid);
1348 	struct rte_hash_parameters params = {0};
1349 	struct vhost_crypto *vcrypto;
1350 	char name[128];
1351 	int ret;
1352 
1353 	if (!dev) {
1354 		VC_LOG_ERR("Invalid vid %i", vid);
1355 		return -EINVAL;
1356 	}
1357 
1358 	ret = rte_vhost_driver_set_features(dev->ifname,
1359 			VIRTIO_CRYPTO_FEATURES);
1360 	if (ret < 0) {
1361 		VC_LOG_ERR("Error setting features");
1362 		return -1;
1363 	}
1364 
1365 	vcrypto = rte_zmalloc_socket(NULL, sizeof(*vcrypto),
1366 			RTE_CACHE_LINE_SIZE, socket_id);
1367 	if (!vcrypto) {
1368 		VC_LOG_ERR("Insufficient memory");
1369 		return -ENOMEM;
1370 	}
1371 
1372 	vcrypto->sess_pool = sess_pool;
1373 	vcrypto->cid = cryptodev_id;
1374 	vcrypto->cache_session_id = UINT64_MAX;
1375 	vcrypto->last_session_id = 1;
1376 	vcrypto->dev = dev;
1377 	vcrypto->option = RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE;
1378 
1379 	snprintf(name, 127, "HASH_VHOST_CRYPT_%u", (uint32_t)vid);
1380 	params.name = name;
1381 	params.entries = VHOST_CRYPTO_SESSION_MAP_ENTRIES;
1382 	params.hash_func = rte_jhash;
1383 	params.key_len = sizeof(uint64_t);
1384 	params.socket_id = socket_id;
1385 	vcrypto->session_map = rte_hash_create(&params);
1386 	if (!vcrypto->session_map) {
1387 		VC_LOG_ERR("Failed to creath session map");
1388 		ret = -ENOMEM;
1389 		goto error_exit;
1390 	}
1391 
1392 	snprintf(name, 127, "MBUF_POOL_VM_%u", (uint32_t)vid);
1393 	vcrypto->mbuf_pool = rte_pktmbuf_pool_create(name,
1394 			VHOST_CRYPTO_MBUF_POOL_SIZE, 512,
1395 			sizeof(struct vhost_crypto_data_req),
1396 			RTE_MBUF_DEFAULT_DATAROOM * 2 + RTE_PKTMBUF_HEADROOM,
1397 			rte_socket_id());
1398 	if (!vcrypto->mbuf_pool) {
1399 		VC_LOG_ERR("Failed to creath mbuf pool");
1400 		ret = -ENOMEM;
1401 		goto error_exit;
1402 	}
1403 
1404 	snprintf(name, 127, "WB_POOL_VM_%u", (uint32_t)vid);
1405 	vcrypto->wb_pool = rte_mempool_create(name,
1406 			VHOST_CRYPTO_MBUF_POOL_SIZE,
1407 			sizeof(struct vhost_crypto_writeback_data),
1408 			128, 0, NULL, NULL, NULL, NULL,
1409 			rte_socket_id(), 0);
1410 	if (!vcrypto->wb_pool) {
1411 		VC_LOG_ERR("Failed to creath mempool");
1412 		ret = -ENOMEM;
1413 		goto error_exit;
1414 	}
1415 
1416 	dev->extern_data = vcrypto;
1417 	dev->extern_ops.pre_msg_handle = NULL;
1418 	dev->extern_ops.post_msg_handle = vhost_crypto_msg_post_handler;
1419 
1420 	return 0;
1421 
1422 error_exit:
1423 	if (vcrypto->session_map)
1424 		rte_hash_free(vcrypto->session_map);
1425 	if (vcrypto->mbuf_pool)
1426 		rte_mempool_free(vcrypto->mbuf_pool);
1427 
1428 	rte_free(vcrypto);
1429 
1430 	return ret;
1431 }
1432 
1433 int __rte_experimental
1434 rte_vhost_crypto_free(int vid)
1435 {
1436 	struct virtio_net *dev = get_device(vid);
1437 	struct vhost_crypto *vcrypto;
1438 
1439 	if (unlikely(dev == NULL)) {
1440 		VC_LOG_ERR("Invalid vid %i", vid);
1441 		return -EINVAL;
1442 	}
1443 
1444 	vcrypto = dev->extern_data;
1445 	if (unlikely(vcrypto == NULL)) {
1446 		VC_LOG_ERR("Cannot find required data, is it initialized?");
1447 		return -ENOENT;
1448 	}
1449 
1450 	rte_hash_free(vcrypto->session_map);
1451 	rte_mempool_free(vcrypto->mbuf_pool);
1452 	rte_mempool_free(vcrypto->wb_pool);
1453 	rte_free(vcrypto);
1454 
1455 	dev->extern_data = NULL;
1456 	dev->extern_ops.pre_msg_handle = NULL;
1457 	dev->extern_ops.post_msg_handle = NULL;
1458 
1459 	return 0;
1460 }
1461 
1462 int __rte_experimental
1463 rte_vhost_crypto_set_zero_copy(int vid, enum rte_vhost_crypto_zero_copy option)
1464 {
1465 	struct virtio_net *dev = get_device(vid);
1466 	struct vhost_crypto *vcrypto;
1467 
1468 	if (unlikely(dev == NULL)) {
1469 		VC_LOG_ERR("Invalid vid %i", vid);
1470 		return -EINVAL;
1471 	}
1472 
1473 	if (unlikely((uint32_t)option >=
1474 				RTE_VHOST_CRYPTO_MAX_ZERO_COPY_OPTIONS)) {
1475 		VC_LOG_ERR("Invalid option %i", option);
1476 		return -EINVAL;
1477 	}
1478 
1479 	vcrypto = (struct vhost_crypto *)dev->extern_data;
1480 	if (unlikely(vcrypto == NULL)) {
1481 		VC_LOG_ERR("Cannot find required data, is it initialized?");
1482 		return -ENOENT;
1483 	}
1484 
1485 	if (vcrypto->option == (uint8_t)option)
1486 		return 0;
1487 
1488 	if (!(rte_mempool_full(vcrypto->mbuf_pool)) ||
1489 			!(rte_mempool_full(vcrypto->wb_pool))) {
1490 		VC_LOG_ERR("Cannot update zero copy as mempool is not full");
1491 		return -EINVAL;
1492 	}
1493 
1494 	if (option == RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE) {
1495 		char name[128];
1496 
1497 		snprintf(name, 127, "WB_POOL_VM_%u", (uint32_t)vid);
1498 		vcrypto->wb_pool = rte_mempool_create(name,
1499 				VHOST_CRYPTO_MBUF_POOL_SIZE,
1500 				sizeof(struct vhost_crypto_writeback_data),
1501 				128, 0, NULL, NULL, NULL, NULL,
1502 				rte_socket_id(), 0);
1503 		if (!vcrypto->wb_pool) {
1504 			VC_LOG_ERR("Failed to creath mbuf pool");
1505 			return -ENOMEM;
1506 		}
1507 	} else {
1508 		rte_mempool_free(vcrypto->wb_pool);
1509 		vcrypto->wb_pool = NULL;
1510 	}
1511 
1512 	vcrypto->option = (uint8_t)option;
1513 
1514 	return 0;
1515 }
1516 
1517 uint16_t __rte_experimental
1518 rte_vhost_crypto_fetch_requests(int vid, uint32_t qid,
1519 		struct rte_crypto_op **ops, uint16_t nb_ops)
1520 {
1521 	struct rte_mbuf *mbufs[VHOST_CRYPTO_MAX_BURST_SIZE * 2];
1522 	struct virtio_net *dev = get_device(vid);
1523 	struct vhost_crypto *vcrypto;
1524 	struct vhost_virtqueue *vq;
1525 	uint16_t avail_idx;
1526 	uint16_t start_idx;
1527 	uint16_t count;
1528 	uint16_t i = 0;
1529 
1530 	if (unlikely(dev == NULL)) {
1531 		VC_LOG_ERR("Invalid vid %i", vid);
1532 		return -EINVAL;
1533 	}
1534 
1535 	if (unlikely(qid >= VHOST_MAX_QUEUE_PAIRS)) {
1536 		VC_LOG_ERR("Invalid qid %u", qid);
1537 		return -EINVAL;
1538 	}
1539 
1540 	vcrypto = (struct vhost_crypto *)dev->extern_data;
1541 	if (unlikely(vcrypto == NULL)) {
1542 		VC_LOG_ERR("Cannot find required data, is it initialized?");
1543 		return -ENOENT;
1544 	}
1545 
1546 	vq = dev->virtqueue[qid];
1547 
1548 	avail_idx = *((volatile uint16_t *)&vq->avail->idx);
1549 	start_idx = vq->last_used_idx;
1550 	count = avail_idx - start_idx;
1551 	count = RTE_MIN(count, VHOST_CRYPTO_MAX_BURST_SIZE);
1552 	count = RTE_MIN(count, nb_ops);
1553 
1554 	if (unlikely(count == 0))
1555 		return 0;
1556 
1557 	/* for zero copy, we need 2 empty mbufs for src and dst, otherwise
1558 	 * we need only 1 mbuf as src and dst
1559 	 */
1560 	switch (vcrypto->option) {
1561 	case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
1562 		if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool,
1563 				(void **)mbufs, count * 2) < 0)) {
1564 			VC_LOG_ERR("Insufficient memory");
1565 			return -ENOMEM;
1566 		}
1567 
1568 		for (i = 0; i < count; i++) {
1569 			uint16_t used_idx = (start_idx + i) & (vq->size - 1);
1570 			uint16_t desc_idx = vq->avail->ring[used_idx];
1571 			struct vring_desc *head = &vq->desc[desc_idx];
1572 			struct rte_crypto_op *op = ops[i];
1573 
1574 			op->sym->m_src = mbufs[i * 2];
1575 			op->sym->m_dst = mbufs[i * 2 + 1];
1576 			op->sym->m_src->data_off = 0;
1577 			op->sym->m_dst->data_off = 0;
1578 
1579 			if (unlikely(vhost_crypto_process_one_req(vcrypto, vq,
1580 					op, head, desc_idx) < 0))
1581 				break;
1582 		}
1583 
1584 		if (unlikely(i < count))
1585 			rte_mempool_put_bulk(vcrypto->mbuf_pool,
1586 					(void **)&mbufs[i * 2],
1587 					(count - i) * 2);
1588 
1589 		break;
1590 
1591 	case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
1592 		if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool,
1593 				(void **)mbufs, count) < 0)) {
1594 			VC_LOG_ERR("Insufficient memory");
1595 			return -ENOMEM;
1596 		}
1597 
1598 		for (i = 0; i < count; i++) {
1599 			uint16_t used_idx = (start_idx + i) & (vq->size - 1);
1600 			uint16_t desc_idx = vq->avail->ring[used_idx];
1601 			struct vring_desc *head = &vq->desc[desc_idx];
1602 			struct rte_crypto_op *op = ops[i];
1603 
1604 			op->sym->m_src = mbufs[i];
1605 			op->sym->m_dst = NULL;
1606 			op->sym->m_src->data_off = 0;
1607 
1608 			if (unlikely(vhost_crypto_process_one_req(vcrypto, vq,
1609 					op, head, desc_idx) < 0))
1610 				break;
1611 		}
1612 
1613 		if (unlikely(i < count))
1614 			rte_mempool_put_bulk(vcrypto->mbuf_pool,
1615 					(void **)&mbufs[i],
1616 					count - i);
1617 
1618 		break;
1619 
1620 	}
1621 
1622 	vq->last_used_idx += i;
1623 
1624 	return i;
1625 }
1626 
1627 uint16_t __rte_experimental
1628 rte_vhost_crypto_finalize_requests(struct rte_crypto_op **ops,
1629 		uint16_t nb_ops, int *callfds, uint16_t *nb_callfds)
1630 {
1631 	struct rte_crypto_op **tmp_ops = ops;
1632 	uint16_t count = 0, left = nb_ops;
1633 	int callfd;
1634 	uint16_t idx = 0;
1635 
1636 	while (left) {
1637 		count = vhost_crypto_complete_one_vm_requests(tmp_ops, left,
1638 				&callfd);
1639 		if (unlikely(count == 0))
1640 			break;
1641 
1642 		tmp_ops = &tmp_ops[count];
1643 		left -= count;
1644 
1645 		callfds[idx++] = callfd;
1646 
1647 		if (unlikely(idx >= VIRTIO_CRYPTO_MAX_NUM_BURST_VQS)) {
1648 			VC_LOG_ERR("Too many vqs");
1649 			break;
1650 		}
1651 	}
1652 
1653 	*nb_callfds = idx;
1654 
1655 	return nb_ops - left;
1656 }
1657