1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2022 Intel Corporation
3 */
4
5 #ifndef _QAT_SYM_H_
6 #define _QAT_SYM_H_
7
8 #include <cryptodev_pmd.h>
9 #ifdef RTE_LIB_SECURITY
10 #include <rte_net_crc.h>
11 #endif
12
13 #ifdef BUILD_QAT_SYM
14 #include <openssl/evp.h>
15
16 #include "qat_common.h"
17 #include "qat_sym_session.h"
18 #include "qat_crypto.h"
19 #include "qat_logs.h"
20
21 #define BYTE_LENGTH 8
22 /* bpi is only used for partial blocks of DES and AES
23 * so AES block len can be assumed as max len for iv, src and dst
24 */
25 #define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
26
27 /** Intel(R) QAT Symmetric Crypto PMD name */
28 #define CRYPTODEV_NAME_QAT_SYM_PMD crypto_qat
29
30 /* Internal capabilities */
31 #define QAT_SYM_CAP_MIXED_CRYPTO (1 << 0)
32 #define QAT_SYM_CAP_VALID (1 << 31)
33
34 /**
35 * Macro to add a sym capability
36 * helper function to add an sym capability
37 * <n: name> <b: block size> <k: key size> <d: digest size>
38 * <a: aad_size> <i: iv_size>
39 **/
40 #define QAT_SYM_PLAIN_AUTH_CAP(n, b, d) \
41 { \
42 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \
43 {.sym = { \
44 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \
45 {.auth = { \
46 .algo = RTE_CRYPTO_AUTH_##n, \
47 b, d \
48 }, } \
49 }, } \
50 }
51
52 #define QAT_SYM_AUTH_CAP(n, b, k, d, a, i) \
53 { \
54 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \
55 {.sym = { \
56 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \
57 {.auth = { \
58 .algo = RTE_CRYPTO_AUTH_##n, \
59 b, k, d, a, i \
60 }, } \
61 }, } \
62 }
63
64 #define QAT_SYM_AEAD_CAP(n, b, k, d, a, i) \
65 { \
66 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \
67 {.sym = { \
68 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, \
69 {.aead = { \
70 .algo = RTE_CRYPTO_AEAD_##n, \
71 b, k, d, a, i \
72 }, } \
73 }, } \
74 }
75
76 #define QAT_SYM_CIPHER_CAP(n, b, k, i) \
77 { \
78 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \
79 {.sym = { \
80 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \
81 {.cipher = { \
82 .algo = RTE_CRYPTO_CIPHER_##n, \
83 b, k, i \
84 }, } \
85 }, } \
86 }
87
88 /*
89 * Maximum number of SGL entries
90 */
91 #define QAT_SYM_SGL_MAX_NUMBER 16
92
93 /* Maximum data length for single pass GMAC: 2^14-1 */
94 #define QAT_AES_GMAC_SPC_MAX_SIZE 16383
95
96 struct qat_sym_session;
97
98 struct qat_sym_sgl {
99 qat_sgl_hdr;
100 struct qat_flat_buf buffers[QAT_SYM_SGL_MAX_NUMBER];
101 } __rte_packed __rte_cache_aligned;
102
103 struct qat_sym_op_cookie {
104 struct qat_sym_sgl qat_sgl_src;
105 struct qat_sym_sgl qat_sgl_dst;
106 phys_addr_t qat_sgl_src_phys_addr;
107 phys_addr_t qat_sgl_dst_phys_addr;
108 union {
109 /* Used for Single-Pass AES-GMAC only */
110 struct {
111 struct icp_qat_hw_cipher_algo_blk cd_cipher
112 __rte_packed __rte_cache_aligned;
113 phys_addr_t cd_phys_addr;
114 } spc_gmac;
115 } opt;
116 };
117
118 struct qat_sym_dp_ctx {
119 struct qat_sym_session *session;
120 uint32_t tail;
121 uint32_t head;
122 uint16_t cached_enqueue;
123 uint16_t cached_dequeue;
124 };
125
126 uint16_t
127 qat_sym_enqueue_burst(void *qp, struct rte_crypto_op **ops,
128 uint16_t nb_ops);
129
130 uint16_t
131 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
132 uint16_t nb_ops);
133
134 /** Encrypt a single partial block
135 * Depends on openssl libcrypto
136 * Uses ECB+XOR to do CFB encryption, same result, more performant
137 */
138 static inline int
bpi_cipher_encrypt(uint8_t * src,uint8_t * dst,uint8_t * iv,int ivlen,int srclen,void * bpi_ctx)139 bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
140 uint8_t *iv, int ivlen, int srclen,
141 void *bpi_ctx)
142 {
143 EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
144 int encrypted_ivlen;
145 uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
146 uint8_t *encr = encrypted_iv;
147
148 /* ECB method: encrypt the IV, then XOR this with plaintext */
149 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
150 <= 0)
151 goto cipher_encrypt_err;
152
153 for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
154 *dst = *src ^ *encr;
155
156 return 0;
157
158 cipher_encrypt_err:
159 QAT_DP_LOG(ERR, "libcrypto ECB cipher encrypt failed");
160 return -EINVAL;
161 }
162
163 static inline uint32_t
qat_bpicipher_postprocess(struct qat_sym_session * ctx,struct rte_crypto_op * op)164 qat_bpicipher_postprocess(struct qat_sym_session *ctx,
165 struct rte_crypto_op *op)
166 {
167 int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
168 struct rte_crypto_sym_op *sym_op = op->sym;
169 uint8_t last_block_len = block_len > 0 ?
170 sym_op->cipher.data.length % block_len : 0;
171
172 if (last_block_len > 0 &&
173 ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
174
175 /* Encrypt last block */
176 uint8_t *last_block, *dst, *iv;
177 uint32_t last_block_offset;
178
179 last_block_offset = sym_op->cipher.data.offset +
180 sym_op->cipher.data.length - last_block_len;
181 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
182 uint8_t *, last_block_offset);
183
184 if (unlikely(sym_op->m_dst != NULL))
185 /* out-of-place operation (OOP) */
186 dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
187 uint8_t *, last_block_offset);
188 else
189 dst = last_block;
190
191 if (last_block_len < sym_op->cipher.data.length)
192 /* use previous block ciphertext as IV */
193 iv = dst - block_len;
194 else
195 /* runt block, i.e. less than one full block */
196 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
197 ctx->cipher_iv.offset);
198
199 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
200 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before post-process:",
201 last_block, last_block_len);
202 if (sym_op->m_dst != NULL)
203 QAT_DP_HEXDUMP_LOG(DEBUG,
204 "BPI: dst before post-process:",
205 dst, last_block_len);
206 #endif
207 bpi_cipher_encrypt(last_block, dst, iv, block_len,
208 last_block_len, ctx->bpi_ctx);
209 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
210 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after post-process:",
211 last_block, last_block_len);
212 if (sym_op->m_dst != NULL)
213 QAT_DP_HEXDUMP_LOG(DEBUG,
214 "BPI: dst after post-process:",
215 dst, last_block_len);
216 #endif
217 }
218 return sym_op->cipher.data.length - last_block_len;
219 }
220
221 #ifdef RTE_LIB_SECURITY
222 static inline void
qat_crc_verify(struct qat_sym_session * ctx,struct rte_crypto_op * op)223 qat_crc_verify(struct qat_sym_session *ctx, struct rte_crypto_op *op)
224 {
225 struct rte_crypto_sym_op *sym_op = op->sym;
226 uint32_t crc_data_ofs, crc_data_len, crc;
227 uint8_t *crc_data;
228
229 if (ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT &&
230 sym_op->auth.data.length != 0) {
231
232 crc_data_ofs = sym_op->auth.data.offset;
233 crc_data_len = sym_op->auth.data.length;
234 crc_data = rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *,
235 crc_data_ofs);
236
237 crc = rte_net_crc_calc(crc_data, crc_data_len,
238 RTE_NET_CRC32_ETH);
239
240 if (crc != *(uint32_t *)(crc_data + crc_data_len))
241 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
242 }
243 }
244
245 static inline void
qat_crc_generate(struct qat_sym_session * ctx,struct rte_crypto_op * op)246 qat_crc_generate(struct qat_sym_session *ctx,
247 struct rte_crypto_op *op)
248 {
249 struct rte_crypto_sym_op *sym_op = op->sym;
250 uint32_t *crc, crc_data_len;
251 uint8_t *crc_data;
252
253 if (ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT &&
254 sym_op->auth.data.length != 0 &&
255 sym_op->m_src->nb_segs == 1) {
256
257 crc_data_len = sym_op->auth.data.length;
258 crc_data = rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *,
259 sym_op->auth.data.offset);
260 crc = (uint32_t *)(crc_data + crc_data_len);
261 *crc = rte_net_crc_calc(crc_data, crc_data_len,
262 RTE_NET_CRC32_ETH);
263 }
264 }
265
266 static inline void
qat_sym_preprocess_requests(void ** ops,uint16_t nb_ops)267 qat_sym_preprocess_requests(void **ops, uint16_t nb_ops)
268 {
269 struct rte_crypto_op *op;
270 struct qat_sym_session *ctx;
271 uint16_t i;
272
273 for (i = 0; i < nb_ops; i++) {
274 op = (struct rte_crypto_op *)ops[i];
275
276 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
277 ctx = (struct qat_sym_session *)
278 get_sec_session_private_data(
279 op->sym->sec_session);
280
281 if (ctx == NULL || ctx->bpi_ctx == NULL)
282 continue;
283
284 qat_crc_generate(ctx, op);
285 }
286 }
287 }
288 #endif
289
290 static __rte_always_inline int
qat_sym_process_response(void ** op,uint8_t * resp,void * op_cookie,uint64_t * dequeue_err_count __rte_unused)291 qat_sym_process_response(void **op, uint8_t *resp, void *op_cookie,
292 uint64_t *dequeue_err_count __rte_unused)
293 {
294 struct icp_qat_fw_comn_resp *resp_msg =
295 (struct icp_qat_fw_comn_resp *)resp;
296 struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
297 (resp_msg->opaque_data);
298 struct qat_sym_session *sess;
299 uint8_t is_docsis_sec;
300
301 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
302 QAT_DP_HEXDUMP_LOG(DEBUG, "qat_response:", (uint8_t *)resp_msg,
303 sizeof(struct icp_qat_fw_comn_resp));
304 #endif
305
306 #ifdef RTE_LIB_SECURITY
307 if (rx_op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
308 /*
309 * Assuming at this point that if it's a security
310 * op, that this is for DOCSIS
311 */
312 sess = (struct qat_sym_session *)
313 get_sec_session_private_data(
314 rx_op->sym->sec_session);
315 is_docsis_sec = 1;
316 } else
317 #endif
318 {
319 sess = (struct qat_sym_session *)
320 get_sym_session_private_data(
321 rx_op->sym->session,
322 qat_sym_driver_id);
323 is_docsis_sec = 0;
324 }
325
326 if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
327 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
328 resp_msg->comn_hdr.comn_status)) {
329
330 rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
331 } else {
332 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
333
334 if (sess->bpi_ctx) {
335 qat_bpicipher_postprocess(sess, rx_op);
336 #ifdef RTE_LIB_SECURITY
337 if (is_docsis_sec)
338 qat_crc_verify(sess, rx_op);
339 #endif
340 }
341 }
342
343 if (sess->is_single_pass_gmac) {
344 struct qat_sym_op_cookie *cookie =
345 (struct qat_sym_op_cookie *) op_cookie;
346 memset(cookie->opt.spc_gmac.cd_cipher.key, 0,
347 sess->auth_key_length);
348 }
349
350 *op = (void *)rx_op;
351
352 /*
353 * return 1 as dequeue op only move on to the next op
354 * if one was ready to return to API
355 */
356 return 1;
357 }
358
359 int
360 qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
361 struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
362 enum rte_crypto_op_sess_type sess_type,
363 union rte_cryptodev_session_ctx session_ctx, uint8_t is_update);
364
365 int
366 qat_sym_get_dp_ctx_size(struct rte_cryptodev *dev);
367
368 void
369 qat_sym_init_op_cookie(void *cookie);
370
371 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
372 static __rte_always_inline void
qat_sym_debug_log_dump(struct icp_qat_fw_la_bulk_req * qat_req,struct qat_sym_session * ctx,struct rte_crypto_vec * vec,uint32_t vec_len,struct rte_crypto_va_iova_ptr * cipher_iv,struct rte_crypto_va_iova_ptr * auth_iv,struct rte_crypto_va_iova_ptr * aad,struct rte_crypto_va_iova_ptr * digest)373 qat_sym_debug_log_dump(struct icp_qat_fw_la_bulk_req *qat_req,
374 struct qat_sym_session *ctx,
375 struct rte_crypto_vec *vec, uint32_t vec_len,
376 struct rte_crypto_va_iova_ptr *cipher_iv,
377 struct rte_crypto_va_iova_ptr *auth_iv,
378 struct rte_crypto_va_iova_ptr *aad,
379 struct rte_crypto_va_iova_ptr *digest)
380 {
381 uint32_t i;
382
383 QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
384 sizeof(struct icp_qat_fw_la_bulk_req));
385 for (i = 0; i < vec_len; i++)
386 QAT_DP_HEXDUMP_LOG(DEBUG, "src_data:", vec[i].base, vec[i].len);
387 if (cipher_iv && ctx->cipher_iv.length > 0)
388 QAT_DP_HEXDUMP_LOG(DEBUG, "cipher iv:", cipher_iv->va,
389 ctx->cipher_iv.length);
390 if (auth_iv && ctx->auth_iv.length > 0)
391 QAT_DP_HEXDUMP_LOG(DEBUG, "auth iv:", auth_iv->va,
392 ctx->auth_iv.length);
393 if (aad && ctx->aad_len > 0)
394 QAT_DP_HEXDUMP_LOG(DEBUG, "aad:", aad->va,
395 ctx->aad_len);
396 if (digest && ctx->digest_length > 0)
397 QAT_DP_HEXDUMP_LOG(DEBUG, "digest:", digest->va,
398 ctx->digest_length);
399 }
400 #else
401 static __rte_always_inline void
qat_sym_debug_log_dump(struct icp_qat_fw_la_bulk_req * qat_req __rte_unused,struct qat_sym_session * ctx __rte_unused,struct rte_crypto_vec * vec __rte_unused,uint32_t vec_len __rte_unused,struct rte_crypto_va_iova_ptr * cipher_iv __rte_unused,struct rte_crypto_va_iova_ptr * auth_iv __rte_unused,struct rte_crypto_va_iova_ptr * aad __rte_unused,struct rte_crypto_va_iova_ptr * digest __rte_unused)402 qat_sym_debug_log_dump(struct icp_qat_fw_la_bulk_req *qat_req __rte_unused,
403 struct qat_sym_session *ctx __rte_unused,
404 struct rte_crypto_vec *vec __rte_unused,
405 uint32_t vec_len __rte_unused,
406 struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
407 struct rte_crypto_va_iova_ptr *auth_iv __rte_unused,
408 struct rte_crypto_va_iova_ptr *aad __rte_unused,
409 struct rte_crypto_va_iova_ptr *digest __rte_unused)
410 {
411 }
412 #endif
413
414 #else
415
416 static inline void
qat_sym_preprocess_requests(void ** ops __rte_unused,uint16_t nb_ops __rte_unused)417 qat_sym_preprocess_requests(void **ops __rte_unused,
418 uint16_t nb_ops __rte_unused)
419 {
420 }
421
422 static inline void
qat_sym_process_response(void ** op __rte_unused,uint8_t * resp __rte_unused,void * op_cookie __rte_unused)423 qat_sym_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
424 void *op_cookie __rte_unused)
425 {
426 }
427
428 #endif /* BUILD_QAT_SYM */
429 #endif /* _QAT_SYM_H_ */
430