1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2022 Intel Corporation
3 */
4
5 #ifndef _QAT_CRYPTO_PMD_GENS_H_
6 #define _QAT_CRYPTO_PMD_GENS_H_
7
8 #include <rte_cryptodev.h>
9 #include "qat_crypto.h"
10 #include "qat_sym_session.h"
11 #include "qat_sym.h"
12
13 #define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
14 RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
15
16 #define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
17 (ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
18 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
19
20 static __rte_always_inline int
op_bpi_cipher_decrypt(uint8_t * src,uint8_t * dst,uint8_t * iv,int ivlen,int srclen,void * bpi_ctx)21 op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
22 uint8_t *iv, int ivlen, int srclen,
23 void *bpi_ctx)
24 {
25 EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
26 int encrypted_ivlen;
27 uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
28 uint8_t *encr = encrypted_iv;
29
30 /* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
31 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
32 <= 0)
33 goto cipher_decrypt_err;
34
35 for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
36 *dst = *src ^ *encr;
37
38 return 0;
39
40 cipher_decrypt_err:
41 QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
42 return -EINVAL;
43 }
44
45 static __rte_always_inline uint32_t
qat_bpicipher_preprocess(struct qat_sym_session * ctx,struct rte_crypto_op * op)46 qat_bpicipher_preprocess(struct qat_sym_session *ctx,
47 struct rte_crypto_op *op)
48 {
49 int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
50 struct rte_crypto_sym_op *sym_op = op->sym;
51 uint8_t last_block_len = block_len > 0 ?
52 sym_op->cipher.data.length % block_len : 0;
53
54 if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
55 /* Decrypt last block */
56 uint8_t *last_block, *dst, *iv;
57 uint32_t last_block_offset = sym_op->cipher.data.offset +
58 sym_op->cipher.data.length - last_block_len;
59 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
60 uint8_t *, last_block_offset);
61
62 if (unlikely((sym_op->m_dst != NULL)
63 && (sym_op->m_dst != sym_op->m_src)))
64 /* out-of-place operation (OOP) */
65 dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
66 uint8_t *, last_block_offset);
67 else
68 dst = last_block;
69
70 if (last_block_len < sym_op->cipher.data.length)
71 /* use previous block ciphertext as IV */
72 iv = last_block - block_len;
73 else
74 /* runt block, i.e. less than one full block */
75 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
76 ctx->cipher_iv.offset);
77
78 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
79 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
80 last_block, last_block_len);
81 if (sym_op->m_dst != NULL)
82 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
83 dst, last_block_len);
84 #endif
85 op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
86 last_block_len, ctx->bpi_ctx);
87 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
88 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
89 last_block, last_block_len);
90 if (sym_op->m_dst != NULL)
91 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
92 dst, last_block_len);
93 #endif
94 }
95
96 return sym_op->cipher.data.length - last_block_len;
97 }
98
99 static __rte_always_inline int
qat_auth_is_len_in_bits(struct qat_sym_session * ctx,struct rte_crypto_op * op)100 qat_auth_is_len_in_bits(struct qat_sym_session *ctx,
101 struct rte_crypto_op *op)
102 {
103 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
104 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
105 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
106 if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
107 (op->sym->auth.data.length % BYTE_LENGTH != 0)))
108 return -EINVAL;
109 return 1;
110 }
111 return 0;
112 }
113
114 static __rte_always_inline int
qat_cipher_is_len_in_bits(struct qat_sym_session * ctx,struct rte_crypto_op * op)115 qat_cipher_is_len_in_bits(struct qat_sym_session *ctx,
116 struct rte_crypto_op *op)
117 {
118 if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
119 ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
120 ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
121 if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
122 ((op->sym->cipher.data.offset %
123 BYTE_LENGTH) != 0)))
124 return -EINVAL;
125 return 1;
126 }
127 return 0;
128 }
129
130 static __rte_always_inline int32_t
qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req * req,void * opaque,struct qat_sym_op_cookie * cookie,struct rte_crypto_vec * src_vec,uint16_t n_src,struct rte_crypto_vec * dst_vec,uint16_t n_dst)131 qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
132 void *opaque, struct qat_sym_op_cookie *cookie,
133 struct rte_crypto_vec *src_vec, uint16_t n_src,
134 struct rte_crypto_vec *dst_vec, uint16_t n_dst)
135 {
136 struct qat_sgl *list;
137 uint32_t i;
138 uint32_t tl_src = 0, total_len_src, total_len_dst;
139 uint64_t src_data_start = 0, dst_data_start = 0;
140 int is_sgl = n_src > 1 || n_dst > 1;
141
142 if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
143 n_dst > QAT_SYM_SGL_MAX_NUMBER))
144 return -1;
145
146 if (likely(!is_sgl)) {
147 src_data_start = src_vec[0].iova;
148 tl_src = total_len_src =
149 src_vec[0].len;
150 if (unlikely(n_dst)) { /* oop */
151 total_len_dst = dst_vec[0].len;
152
153 dst_data_start = dst_vec[0].iova;
154 if (unlikely(total_len_src != total_len_dst))
155 return -EINVAL;
156 } else {
157 dst_data_start = src_data_start;
158 total_len_dst = tl_src;
159 }
160 } else { /* sgl */
161 total_len_dst = total_len_src = 0;
162
163 ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
164 QAT_COMN_PTR_TYPE_SGL);
165
166 list = (struct qat_sgl *)&cookie->qat_sgl_src;
167 for (i = 0; i < n_src; i++) {
168 list->buffers[i].len = src_vec[i].len;
169 list->buffers[i].resrvd = 0;
170 list->buffers[i].addr = src_vec[i].iova;
171 if (tl_src + src_vec[i].len > UINT32_MAX) {
172 QAT_DP_LOG(ERR, "Message too long");
173 return -1;
174 }
175 tl_src += src_vec[i].len;
176 }
177
178 list->num_bufs = i;
179 src_data_start = cookie->qat_sgl_src_phys_addr;
180
181 if (unlikely(n_dst > 0)) { /* oop sgl */
182 uint32_t tl_dst = 0;
183
184 list = (struct qat_sgl *)&cookie->qat_sgl_dst;
185
186 for (i = 0; i < n_dst; i++) {
187 list->buffers[i].len = dst_vec[i].len;
188 list->buffers[i].resrvd = 0;
189 list->buffers[i].addr = dst_vec[i].iova;
190 if (tl_dst + dst_vec[i].len > UINT32_MAX) {
191 QAT_DP_LOG(ERR, "Message too long");
192 return -ENOTSUP;
193 }
194
195 tl_dst += dst_vec[i].len;
196 }
197
198 if (tl_src != tl_dst)
199 return -EINVAL;
200 list->num_bufs = i;
201 dst_data_start = cookie->qat_sgl_dst_phys_addr;
202 } else
203 dst_data_start = src_data_start;
204 }
205
206 req->comn_mid.src_data_addr = src_data_start;
207 req->comn_mid.dest_data_addr = dst_data_start;
208 req->comn_mid.src_length = total_len_src;
209 req->comn_mid.dst_length = total_len_dst;
210 req->comn_mid.opaque_data = (uintptr_t)opaque;
211
212 return tl_src;
213 }
214
215 static __rte_always_inline uint64_t
qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op * op,struct qat_sym_session * ctx,struct rte_crypto_sgl * in_sgl,struct rte_crypto_sgl * out_sgl,struct rte_crypto_va_iova_ptr * cipher_iv,struct rte_crypto_va_iova_ptr * auth_iv_or_aad __rte_unused,struct rte_crypto_va_iova_ptr * digest __rte_unused)216 qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
217 struct qat_sym_session *ctx,
218 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
219 struct rte_crypto_va_iova_ptr *cipher_iv,
220 struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
221 struct rte_crypto_va_iova_ptr *digest __rte_unused)
222 {
223 uint32_t cipher_len = 0, cipher_ofs = 0;
224 int n_src = 0;
225 int ret;
226
227 ret = qat_cipher_is_len_in_bits(ctx, op);
228 switch (ret) {
229 case 1:
230 cipher_len = op->sym->cipher.data.length >> 3;
231 cipher_ofs = op->sym->cipher.data.offset >> 3;
232 break;
233 case 0:
234 if (ctx->bpi_ctx) {
235 /* DOCSIS - only send complete blocks to device.
236 * Process any partial block using CFB mode.
237 * Even if 0 complete blocks, still send this to device
238 * to get into rx queue for post-process and dequeuing
239 */
240 cipher_len = qat_bpicipher_preprocess(ctx, op);
241 cipher_ofs = op->sym->cipher.data.offset;
242 } else {
243 cipher_len = op->sym->cipher.data.length;
244 cipher_ofs = op->sym->cipher.data.offset;
245 }
246 break;
247 default:
248 QAT_DP_LOG(ERR,
249 "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
250 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
251 return UINT64_MAX;
252 }
253
254 cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
255 ctx->cipher_iv.offset);
256 cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
257 ctx->cipher_iv.offset);
258
259 n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
260 cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
261 if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
262 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
263 return UINT64_MAX;
264 }
265
266 in_sgl->num = n_src;
267
268 /* Out-Of-Place operation */
269 if (unlikely((op->sym->m_dst != NULL) &&
270 (op->sym->m_dst != op->sym->m_src))) {
271 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
272 cipher_len, out_sgl->vec,
273 QAT_SYM_SGL_MAX_NUMBER);
274
275 if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
276 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
277 return UINT64_MAX;
278 }
279
280 out_sgl->num = n_dst;
281 } else
282 out_sgl->num = 0;
283
284 return 0;
285 }
286
287 static __rte_always_inline uint64_t
qat_sym_convert_op_to_vec_auth(struct rte_crypto_op * op,struct qat_sym_session * ctx,struct rte_crypto_sgl * in_sgl,struct rte_crypto_sgl * out_sgl,struct rte_crypto_va_iova_ptr * cipher_iv __rte_unused,struct rte_crypto_va_iova_ptr * auth_iv,struct rte_crypto_va_iova_ptr * digest)288 qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
289 struct qat_sym_session *ctx,
290 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
291 struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
292 struct rte_crypto_va_iova_ptr *auth_iv,
293 struct rte_crypto_va_iova_ptr *digest)
294 {
295 uint32_t auth_ofs = 0, auth_len = 0;
296 int n_src, ret;
297
298 ret = qat_auth_is_len_in_bits(ctx, op);
299 switch (ret) {
300 case 1:
301 auth_ofs = op->sym->auth.data.offset >> 3;
302 auth_len = op->sym->auth.data.length >> 3;
303 auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
304 ctx->auth_iv.offset);
305 auth_iv->iova = rte_crypto_op_ctophys_offset(op,
306 ctx->auth_iv.offset);
307 break;
308 case 0:
309 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
310 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
311 /* AES-GMAC */
312 auth_ofs = op->sym->auth.data.offset;
313 auth_len = op->sym->auth.data.length;
314 auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
315 ctx->auth_iv.offset);
316 auth_iv->iova = rte_crypto_op_ctophys_offset(op,
317 ctx->auth_iv.offset);
318 } else {
319 auth_ofs = op->sym->auth.data.offset;
320 auth_len = op->sym->auth.data.length;
321 auth_iv->va = NULL;
322 auth_iv->iova = 0;
323 }
324 break;
325 default:
326 QAT_DP_LOG(ERR,
327 "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
328 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
329 return UINT64_MAX;
330 }
331
332 n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
333 auth_ofs + auth_len, in_sgl->vec,
334 QAT_SYM_SGL_MAX_NUMBER);
335 if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
336 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
337 return UINT64_MAX;
338 }
339
340 in_sgl->num = n_src;
341
342 /* Out-Of-Place operation */
343 if (unlikely((op->sym->m_dst != NULL) &&
344 (op->sym->m_dst != op->sym->m_src))) {
345 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
346 auth_ofs + auth_len, out_sgl->vec,
347 QAT_SYM_SGL_MAX_NUMBER);
348
349 if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
350 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
351 return UINT64_MAX;
352 }
353 out_sgl->num = n_dst;
354 } else
355 out_sgl->num = 0;
356
357 digest->va = (void *)op->sym->auth.digest.data;
358 digest->iova = op->sym->auth.digest.phys_addr;
359
360 return 0;
361 }
362
363 static __rte_always_inline uint64_t
qat_sym_convert_op_to_vec_chain(struct rte_crypto_op * op,struct qat_sym_session * ctx,struct rte_crypto_sgl * in_sgl,struct rte_crypto_sgl * out_sgl,struct rte_crypto_va_iova_ptr * cipher_iv,struct rte_crypto_va_iova_ptr * auth_iv_or_aad,struct rte_crypto_va_iova_ptr * digest)364 qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
365 struct qat_sym_session *ctx,
366 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
367 struct rte_crypto_va_iova_ptr *cipher_iv,
368 struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
369 struct rte_crypto_va_iova_ptr *digest)
370 {
371 union rte_crypto_sym_ofs ofs;
372 uint32_t min_ofs = 0, max_len = 0;
373 uint32_t cipher_len = 0, cipher_ofs = 0;
374 uint32_t auth_len = 0, auth_ofs = 0;
375 int is_oop = (op->sym->m_dst != NULL) &&
376 (op->sym->m_dst != op->sym->m_src);
377 int is_sgl = op->sym->m_src->nb_segs > 1;
378 int n_src;
379 int ret;
380
381 if (unlikely(is_oop))
382 is_sgl |= op->sym->m_dst->nb_segs > 1;
383
384 cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
385 ctx->cipher_iv.offset);
386 cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
387 ctx->cipher_iv.offset);
388 auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
389 ctx->auth_iv.offset);
390 auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
391 ctx->auth_iv.offset);
392 digest->va = (void *)op->sym->auth.digest.data;
393 digest->iova = op->sym->auth.digest.phys_addr;
394
395 ret = qat_cipher_is_len_in_bits(ctx, op);
396 switch (ret) {
397 case 1:
398 cipher_len = op->sym->aead.data.length >> 3;
399 cipher_ofs = op->sym->aead.data.offset >> 3;
400 break;
401 case 0:
402 cipher_len = op->sym->aead.data.length;
403 cipher_ofs = op->sym->aead.data.offset;
404 break;
405 default:
406 QAT_DP_LOG(ERR,
407 "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
408 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
409 return -EINVAL;
410 }
411
412 ret = qat_auth_is_len_in_bits(ctx, op);
413 switch (ret) {
414 case 1:
415 auth_len = op->sym->auth.data.length >> 3;
416 auth_ofs = op->sym->auth.data.offset >> 3;
417 break;
418 case 0:
419 auth_len = op->sym->auth.data.length;
420 auth_ofs = op->sym->auth.data.offset;
421 break;
422 default:
423 QAT_DP_LOG(ERR,
424 "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
425 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
426 return -EINVAL;
427 }
428
429 min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
430 max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
431
432 /* digest in buffer check. Needed only for wireless algos */
433 if (ret == 1) {
434 /* Handle digest-encrypted cases, i.e.
435 * auth-gen-then-cipher-encrypt and
436 * cipher-decrypt-then-auth-verify
437 */
438 uint64_t auth_end_iova;
439
440 if (unlikely(is_sgl)) {
441 uint32_t remaining_off = auth_ofs + auth_len;
442 struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
443 op->sym->m_src);
444
445 while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
446 && sgl_buf->next != NULL) {
447 remaining_off -= rte_pktmbuf_data_len(sgl_buf);
448 sgl_buf = sgl_buf->next;
449 }
450
451 auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
452 sgl_buf, remaining_off);
453 } else
454 auth_end_iova = (is_oop ?
455 rte_pktmbuf_iova(op->sym->m_dst) :
456 rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
457 auth_len;
458
459 /* Then check if digest-encrypted conditions are met */
460 if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
461 (digest->iova == auth_end_iova))
462 max_len = RTE_MAX(max_len, auth_ofs + auth_len +
463 ctx->digest_length);
464 }
465
466 n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, min_ofs, max_len,
467 in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
468 if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
469 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
470 return -1;
471 }
472 in_sgl->num = n_src;
473
474 if (unlikely((op->sym->m_dst != NULL) &&
475 (op->sym->m_dst != op->sym->m_src))) {
476 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, min_ofs,
477 max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
478
479 if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
480 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
481 return -1;
482 }
483 out_sgl->num = n_dst;
484 } else
485 out_sgl->num = 0;
486
487 ofs.ofs.cipher.head = cipher_ofs;
488 ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
489 ofs.ofs.auth.head = auth_ofs;
490 ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
491
492 return ofs.raw;
493 }
494
495 static __rte_always_inline uint64_t
qat_sym_convert_op_to_vec_aead(struct rte_crypto_op * op,struct qat_sym_session * ctx,struct rte_crypto_sgl * in_sgl,struct rte_crypto_sgl * out_sgl,struct rte_crypto_va_iova_ptr * cipher_iv,struct rte_crypto_va_iova_ptr * auth_iv_or_aad,struct rte_crypto_va_iova_ptr * digest)496 qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
497 struct qat_sym_session *ctx,
498 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
499 struct rte_crypto_va_iova_ptr *cipher_iv,
500 struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
501 struct rte_crypto_va_iova_ptr *digest)
502 {
503 uint32_t cipher_len = 0, cipher_ofs = 0;
504 int32_t n_src = 0;
505
506 cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
507 ctx->cipher_iv.offset);
508 cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
509 ctx->cipher_iv.offset);
510 auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
511 auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
512 digest->va = (void *)op->sym->aead.digest.data;
513 digest->iova = op->sym->aead.digest.phys_addr;
514
515 cipher_len = op->sym->aead.data.length;
516 cipher_ofs = op->sym->aead.data.offset;
517
518 n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
519 in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
520 if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
521 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
522 return UINT64_MAX;
523 }
524 in_sgl->num = n_src;
525
526 /* Out-Of-Place operation */
527 if (unlikely((op->sym->m_dst != NULL) &&
528 (op->sym->m_dst != op->sym->m_src))) {
529 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
530 cipher_len, out_sgl->vec,
531 QAT_SYM_SGL_MAX_NUMBER);
532 if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
533 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
534 return UINT64_MAX;
535 }
536
537 out_sgl->num = n_dst;
538 } else
539 out_sgl->num = 0;
540
541 return 0;
542 }
543
544 static __rte_always_inline void
qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params * cipher_param,struct rte_crypto_va_iova_ptr * iv_ptr,uint32_t iv_len,struct icp_qat_fw_la_bulk_req * qat_req)545 qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
546 struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
547 struct icp_qat_fw_la_bulk_req *qat_req)
548 {
549 /* copy IV into request if it fits */
550 if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
551 rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
552 iv_len);
553 else {
554 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
555 qat_req->comn_hdr.serv_specif_flags,
556 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
557 cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
558 }
559 }
560
561 static __rte_always_inline void
qat_sym_dp_fill_vec_status(int32_t * sta,int status,uint32_t n)562 qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
563 {
564 uint32_t i;
565
566 for (i = 0; i < n; i++)
567 sta[i] = status;
568 }
569
570 static __rte_always_inline void
enqueue_one_cipher_job_gen1(struct qat_sym_session * ctx,struct icp_qat_fw_la_bulk_req * req,struct rte_crypto_va_iova_ptr * iv,union rte_crypto_sym_ofs ofs,uint32_t data_len)571 enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
572 struct icp_qat_fw_la_bulk_req *req,
573 struct rte_crypto_va_iova_ptr *iv,
574 union rte_crypto_sym_ofs ofs, uint32_t data_len)
575 {
576 struct icp_qat_fw_la_cipher_req_params *cipher_param;
577
578 cipher_param = (void *)&req->serv_specif_rqpars;
579
580 /* cipher IV */
581 qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
582 cipher_param->cipher_offset = ofs.ofs.cipher.head;
583 cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
584 ofs.ofs.cipher.tail;
585 }
586
587 static __rte_always_inline void
enqueue_one_auth_job_gen1(struct qat_sym_session * ctx,struct icp_qat_fw_la_bulk_req * req,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * auth_iv,union rte_crypto_sym_ofs ofs,uint32_t data_len)588 enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
589 struct icp_qat_fw_la_bulk_req *req,
590 struct rte_crypto_va_iova_ptr *digest,
591 struct rte_crypto_va_iova_ptr *auth_iv,
592 union rte_crypto_sym_ofs ofs, uint32_t data_len)
593 {
594 struct icp_qat_fw_la_cipher_req_params *cipher_param;
595 struct icp_qat_fw_la_auth_req_params *auth_param;
596
597 cipher_param = (void *)&req->serv_specif_rqpars;
598 auth_param = (void *)((uint8_t *)cipher_param +
599 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
600
601 auth_param->auth_off = ofs.ofs.auth.head;
602 auth_param->auth_len = data_len - ofs.ofs.auth.head -
603 ofs.ofs.auth.tail;
604 auth_param->auth_res_addr = digest->iova;
605
606 switch (ctx->qat_hash_alg) {
607 case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
608 case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
609 case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
610 auth_param->u1.aad_adr = auth_iv->iova;
611 break;
612 case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
613 case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
614 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
615 req->comn_hdr.serv_specif_flags,
616 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
617 rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
618 ctx->auth_iv.length);
619 break;
620 default:
621 break;
622 }
623 }
624
625 static __rte_always_inline int
enqueue_one_chain_job_gen1(struct qat_sym_session * ctx,struct icp_qat_fw_la_bulk_req * req,struct rte_crypto_vec * src_vec,uint16_t n_src_vecs,struct rte_crypto_vec * dst_vec,uint16_t n_dst_vecs,struct rte_crypto_va_iova_ptr * cipher_iv,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * auth_iv,union rte_crypto_sym_ofs ofs,uint32_t data_len)626 enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
627 struct icp_qat_fw_la_bulk_req *req,
628 struct rte_crypto_vec *src_vec,
629 uint16_t n_src_vecs,
630 struct rte_crypto_vec *dst_vec,
631 uint16_t n_dst_vecs,
632 struct rte_crypto_va_iova_ptr *cipher_iv,
633 struct rte_crypto_va_iova_ptr *digest,
634 struct rte_crypto_va_iova_ptr *auth_iv,
635 union rte_crypto_sym_ofs ofs, uint32_t data_len)
636 {
637 struct icp_qat_fw_la_cipher_req_params *cipher_param;
638 struct icp_qat_fw_la_auth_req_params *auth_param;
639 struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
640 dst_vec : src_vec;
641 rte_iova_t auth_iova_end;
642 int cipher_len, auth_len;
643 int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
644
645 cipher_param = (void *)&req->serv_specif_rqpars;
646 auth_param = (void *)((uint8_t *)cipher_param +
647 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
648
649 cipher_len = data_len - ofs.ofs.cipher.head -
650 ofs.ofs.cipher.tail;
651 auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
652
653 if (unlikely(cipher_len < 0 || auth_len < 0))
654 return -1;
655
656 cipher_param->cipher_offset = ofs.ofs.cipher.head;
657 cipher_param->cipher_length = cipher_len;
658 qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
659
660 auth_param->auth_off = ofs.ofs.auth.head;
661 auth_param->auth_len = auth_len;
662 auth_param->auth_res_addr = digest->iova;
663
664 switch (ctx->qat_hash_alg) {
665 case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
666 case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
667 case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
668 auth_param->u1.aad_adr = auth_iv->iova;
669 break;
670 case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
671 case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
672 break;
673 default:
674 break;
675 }
676
677 if (unlikely(is_sgl)) {
678 /* sgl */
679 int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
680 uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
681
682 while (remaining_off >= cvec->len && i >= 1) {
683 i--;
684 remaining_off -= cvec->len;
685 cvec++;
686 }
687
688 auth_iova_end = cvec->iova + remaining_off;
689 } else
690 auth_iova_end = cvec[0].iova + auth_param->auth_off +
691 auth_param->auth_len;
692
693 /* Then check if digest-encrypted conditions are met */
694 if ((auth_param->auth_off + auth_param->auth_len <
695 cipher_param->cipher_offset + cipher_param->cipher_length) &&
696 (digest->iova == auth_iova_end)) {
697 /* Handle partial digest encryption */
698 if (cipher_param->cipher_offset + cipher_param->cipher_length <
699 auth_param->auth_off + auth_param->auth_len +
700 ctx->digest_length && !is_sgl)
701 req->comn_mid.dst_length = req->comn_mid.src_length =
702 auth_param->auth_off + auth_param->auth_len +
703 ctx->digest_length;
704 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
705 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
706 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
707 }
708
709 return 0;
710 }
711
712 static __rte_always_inline void
enqueue_one_aead_job_gen1(struct qat_sym_session * ctx,struct icp_qat_fw_la_bulk_req * req,struct rte_crypto_va_iova_ptr * iv,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * aad,union rte_crypto_sym_ofs ofs,uint32_t data_len)713 enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
714 struct icp_qat_fw_la_bulk_req *req,
715 struct rte_crypto_va_iova_ptr *iv,
716 struct rte_crypto_va_iova_ptr *digest,
717 struct rte_crypto_va_iova_ptr *aad,
718 union rte_crypto_sym_ofs ofs, uint32_t data_len)
719 {
720 struct icp_qat_fw_la_cipher_req_params *cipher_param =
721 (void *)&req->serv_specif_rqpars;
722 struct icp_qat_fw_la_auth_req_params *auth_param =
723 (void *)((uint8_t *)&req->serv_specif_rqpars +
724 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
725 uint8_t *aad_data;
726 uint8_t aad_ccm_real_len;
727 uint8_t aad_len_field_sz;
728 uint32_t msg_len_be;
729 rte_iova_t aad_iova = 0;
730 uint8_t q;
731
732 switch (ctx->qat_hash_alg) {
733 case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
734 case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
735 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
736 req->comn_hdr.serv_specif_flags,
737 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
738 rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
739 ctx->cipher_iv.length);
740 aad_iova = aad->iova;
741 break;
742 case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
743 aad_data = aad->va;
744 aad_iova = aad->iova;
745 aad_ccm_real_len = 0;
746 aad_len_field_sz = 0;
747 msg_len_be = rte_bswap32((uint32_t)data_len -
748 ofs.ofs.cipher.head);
749
750 if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
751 aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
752 aad_ccm_real_len = ctx->aad_len -
753 ICP_QAT_HW_CCM_AAD_B0_LEN -
754 ICP_QAT_HW_CCM_AAD_LEN_INFO;
755 } else {
756 aad_data = iv->va;
757 aad_iova = iv->iova;
758 }
759
760 q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
761 aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
762 aad_len_field_sz, ctx->digest_length, q);
763 if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
764 memcpy(aad_data + ctx->cipher_iv.length +
765 ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
766 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
767 (uint8_t *)&msg_len_be,
768 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
769 } else {
770 memcpy(aad_data + ctx->cipher_iv.length +
771 ICP_QAT_HW_CCM_NONCE_OFFSET,
772 (uint8_t *)&msg_len_be +
773 (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
774 - q), q);
775 }
776
777 if (aad_len_field_sz > 0) {
778 *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
779 rte_bswap16(aad_ccm_real_len);
780
781 if ((aad_ccm_real_len + aad_len_field_sz)
782 % ICP_QAT_HW_CCM_AAD_B0_LEN) {
783 uint8_t pad_len = 0;
784 uint8_t pad_idx = 0;
785
786 pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
787 ((aad_ccm_real_len +
788 aad_len_field_sz) %
789 ICP_QAT_HW_CCM_AAD_B0_LEN);
790 pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
791 aad_ccm_real_len +
792 aad_len_field_sz;
793 memset(&aad_data[pad_idx], 0, pad_len);
794 }
795 }
796
797 rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
798 + ICP_QAT_HW_CCM_NONCE_OFFSET,
799 (uint8_t *)iv->va +
800 ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
801 *(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
802 q - ICP_QAT_HW_CCM_NONCE_OFFSET;
803
804 rte_memcpy((uint8_t *)aad->va +
805 ICP_QAT_HW_CCM_NONCE_OFFSET,
806 (uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
807 ctx->cipher_iv.length);
808 break;
809 default:
810 break;
811 }
812
813 cipher_param->cipher_offset = ofs.ofs.cipher.head;
814 cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
815 ofs.ofs.cipher.tail;
816 auth_param->auth_off = ofs.ofs.cipher.head;
817 auth_param->auth_len = cipher_param->cipher_length;
818 auth_param->auth_res_addr = digest->iova;
819 auth_param->u1.aad_adr = aad_iova;
820 }
821
822 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
823 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
824
825 /* -----------------GEN 1 sym crypto op data path APIs ---------------- */
826 int
827 qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
828 uint8_t *out_msg, void *op_cookie);
829
830 int
831 qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
832 uint8_t *out_msg, void *op_cookie);
833
834 int
835 qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
836 uint8_t *out_msg, void *op_cookie);
837
838 int
839 qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
840 uint8_t *out_msg, void *op_cookie);
841
842 /* -----------------GEN 1 sym crypto raw data path APIs ---------------- */
843 int
844 qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,
845 struct rte_crypto_vec *data, uint16_t n_data_vecs,
846 union rte_crypto_sym_ofs ofs,
847 struct rte_crypto_va_iova_ptr *iv,
848 struct rte_crypto_va_iova_ptr *digest __rte_unused,
849 struct rte_crypto_va_iova_ptr *aad __rte_unused,
850 void *user_data);
851
852 uint32_t
853 qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
854 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
855 void *user_data[], int *status);
856
857 int
858 qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,
859 struct rte_crypto_vec *data, uint16_t n_data_vecs,
860 union rte_crypto_sym_ofs ofs,
861 struct rte_crypto_va_iova_ptr *iv __rte_unused,
862 struct rte_crypto_va_iova_ptr *digest,
863 struct rte_crypto_va_iova_ptr *auth_iv,
864 void *user_data);
865
866 uint32_t
867 qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
868 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
869 void *user_data[], int *status);
870
871 int
872 qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,
873 struct rte_crypto_vec *data, uint16_t n_data_vecs,
874 union rte_crypto_sym_ofs ofs,
875 struct rte_crypto_va_iova_ptr *cipher_iv,
876 struct rte_crypto_va_iova_ptr *digest,
877 struct rte_crypto_va_iova_ptr *auth_iv,
878 void *user_data);
879
880 uint32_t
881 qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
882 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
883 void *user_data[], int *status);
884
885 int
886 qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,
887 struct rte_crypto_vec *data, uint16_t n_data_vecs,
888 union rte_crypto_sym_ofs ofs,
889 struct rte_crypto_va_iova_ptr *iv,
890 struct rte_crypto_va_iova_ptr *digest,
891 struct rte_crypto_va_iova_ptr *aad,
892 void *user_data);
893
894 uint32_t
895 qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
896 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
897 void *user_data[], int *status);
898
899 void *
900 qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,
901 int *dequeue_status, enum rte_crypto_op_status *op_status);
902
903 uint32_t
904 qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,
905 rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
906 uint32_t max_nb_to_dequeue,
907 rte_cryptodev_raw_post_dequeue_t post_dequeue,
908 void **out_user_data, uint8_t is_user_data_array,
909 uint32_t *n_success_jobs, int *return_status);
910
911 int
912 qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
913
914 int
915 qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
916
917 int
918 qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx);
919
920 /* -----------------GENx control path APIs ---------------- */
921 uint64_t
922 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
923
924 int
925 qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
926
927 void
928 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
929 uint8_t hash_flag);
930
931 struct qat_capabilities_info
932 qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
933
934 uint64_t
935 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
936
937 int
938 qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
939
940 #ifdef RTE_LIB_SECURITY
941 extern struct rte_security_ops security_qat_ops_gen1;
942
943 void *
944 qat_sym_create_security_gen1(void *cryptodev);
945 #endif
946
947 #endif
948