1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2022 Intel Corporation
3 */
4
5 #include <rte_cryptodev.h>
6 #include <cryptodev_pmd.h>
7 #include "qat_sym_session.h"
8 #include "qat_sym.h"
9 #include "qat_asym.h"
10 #include "qat_crypto.h"
11 #include "qat_crypto_pmd_gens.h"
12
13 static struct rte_cryptodev_capabilities qat_sym_crypto_caps_gen4[] = {
14 QAT_SYM_CIPHER_CAP(AES_CBC,
15 CAP_SET(block_size, 16),
16 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)),
17 QAT_SYM_AUTH_CAP(SHA1_HMAC,
18 CAP_SET(block_size, 64),
19 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 20, 1),
20 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
21 QAT_SYM_AUTH_CAP(SHA224_HMAC,
22 CAP_SET(block_size, 64),
23 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 28, 1),
24 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
25 QAT_SYM_AUTH_CAP(SHA256_HMAC,
26 CAP_SET(block_size, 64),
27 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 32, 1),
28 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
29 QAT_SYM_AUTH_CAP(SHA384_HMAC,
30 CAP_SET(block_size, 128),
31 CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 48, 1),
32 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
33 QAT_SYM_AUTH_CAP(SHA512_HMAC,
34 CAP_SET(block_size, 128),
35 CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 64, 1),
36 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
37 QAT_SYM_AUTH_CAP(AES_XCBC_MAC,
38 CAP_SET(block_size, 16),
39 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 12, 12, 0),
40 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
41 QAT_SYM_AUTH_CAP(AES_CMAC,
42 CAP_SET(block_size, 16),
43 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 4),
44 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
45 QAT_SYM_CIPHER_CAP(AES_DOCSISBPI,
46 CAP_SET(block_size, 16),
47 CAP_RNG(key_size, 16, 32, 16), CAP_RNG(iv_size, 16, 16, 0)),
48 QAT_SYM_AUTH_CAP(NULL,
49 CAP_SET(block_size, 1),
50 CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(digest_size),
51 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
52 QAT_SYM_CIPHER_CAP(NULL,
53 CAP_SET(block_size, 1),
54 CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(iv_size)),
55 QAT_SYM_PLAIN_AUTH_CAP(SHA1,
56 CAP_SET(block_size, 64),
57 CAP_RNG(digest_size, 1, 20, 1)),
58 QAT_SYM_AUTH_CAP(SHA224,
59 CAP_SET(block_size, 64),
60 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 28, 1),
61 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
62 QAT_SYM_AUTH_CAP(SHA256,
63 CAP_SET(block_size, 64),
64 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 32, 1),
65 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
66 QAT_SYM_AUTH_CAP(SHA384,
67 CAP_SET(block_size, 128),
68 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 48, 1),
69 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
70 QAT_SYM_AUTH_CAP(SHA512,
71 CAP_SET(block_size, 128),
72 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 64, 1),
73 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
74 QAT_SYM_CIPHER_CAP(AES_CTR,
75 CAP_SET(block_size, 16),
76 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)),
77 QAT_SYM_AEAD_CAP(AES_GCM,
78 CAP_SET(block_size, 16),
79 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4),
80 CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 0, 12, 12)),
81 QAT_SYM_AEAD_CAP(AES_CCM,
82 CAP_SET(block_size, 16),
83 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 2),
84 CAP_RNG(aad_size, 0, 224, 1), CAP_RNG(iv_size, 7, 13, 1)),
85 QAT_SYM_AUTH_CAP(AES_GMAC,
86 CAP_SET(block_size, 16),
87 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4),
88 CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 0, 12, 12)),
89 QAT_SYM_AEAD_CAP(CHACHA20_POLY1305,
90 CAP_SET(block_size, 64),
91 CAP_RNG(key_size, 32, 32, 0),
92 CAP_RNG(digest_size, 16, 16, 0),
93 CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 12, 12, 0)),
94 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
95 };
96
97 static struct qat_capabilities_info
qat_sym_crypto_cap_get_gen4(struct qat_pci_device * qat_dev __rte_unused)98 qat_sym_crypto_cap_get_gen4(struct qat_pci_device *qat_dev __rte_unused)
99 {
100 struct qat_capabilities_info capa_info;
101 capa_info.data = qat_sym_crypto_caps_gen4;
102 capa_info.size = sizeof(qat_sym_crypto_caps_gen4);
103 return capa_info;
104 }
105
106 static __rte_always_inline void
enqueue_one_aead_job_gen4(struct qat_sym_session * ctx,struct icp_qat_fw_la_bulk_req * req,struct rte_crypto_va_iova_ptr * iv,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * aad,union rte_crypto_sym_ofs ofs,uint32_t data_len)107 enqueue_one_aead_job_gen4(struct qat_sym_session *ctx,
108 struct icp_qat_fw_la_bulk_req *req,
109 struct rte_crypto_va_iova_ptr *iv,
110 struct rte_crypto_va_iova_ptr *digest,
111 struct rte_crypto_va_iova_ptr *aad,
112 union rte_crypto_sym_ofs ofs, uint32_t data_len)
113 {
114 if (ctx->is_single_pass && ctx->is_ucs) {
115 struct icp_qat_fw_la_cipher_20_req_params *cipher_param_20 =
116 (void *)&req->serv_specif_rqpars;
117 struct icp_qat_fw_la_cipher_req_params *cipher_param =
118 (void *)&req->serv_specif_rqpars;
119
120 /* QAT GEN4 uses single pass to treat AEAD as cipher
121 * operation
122 */
123 qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length,
124 req);
125 cipher_param->cipher_offset = ofs.ofs.cipher.head;
126 cipher_param->cipher_length = data_len -
127 ofs.ofs.cipher.head - ofs.ofs.cipher.tail;
128
129 cipher_param_20->spc_aad_addr = aad->iova;
130 cipher_param_20->spc_auth_res_addr = digest->iova;
131
132 return;
133 }
134
135 enqueue_one_aead_job_gen1(ctx, req, iv, digest, aad, ofs, data_len);
136 }
137
138 static int
qat_sym_build_op_aead_gen4(void * in_op,struct qat_sym_session * ctx,uint8_t * out_msg,void * op_cookie)139 qat_sym_build_op_aead_gen4(void *in_op, struct qat_sym_session *ctx,
140 uint8_t *out_msg, void *op_cookie)
141 {
142 register struct icp_qat_fw_la_bulk_req *qat_req;
143 struct rte_crypto_op *op = in_op;
144 struct qat_sym_op_cookie *cookie = op_cookie;
145 struct rte_crypto_sgl in_sgl, out_sgl;
146 struct rte_crypto_vec in_vec[QAT_SYM_SGL_MAX_NUMBER],
147 out_vec[QAT_SYM_SGL_MAX_NUMBER];
148 struct rte_crypto_va_iova_ptr cipher_iv;
149 struct rte_crypto_va_iova_ptr aad;
150 struct rte_crypto_va_iova_ptr digest;
151 union rte_crypto_sym_ofs ofs;
152 int32_t total_len;
153
154 in_sgl.vec = in_vec;
155 out_sgl.vec = out_vec;
156
157 qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
158 rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
159
160 ofs.raw = qat_sym_convert_op_to_vec_aead(op, ctx, &in_sgl, &out_sgl,
161 &cipher_iv, &aad, &digest);
162 if (unlikely(ofs.raw == UINT64_MAX)) {
163 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
164 return -EINVAL;
165 }
166
167 total_len = qat_sym_build_req_set_data(qat_req, in_op, cookie,
168 in_sgl.vec, in_sgl.num, out_sgl.vec, out_sgl.num);
169 if (unlikely(total_len < 0)) {
170 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
171 return -EINVAL;
172 }
173
174 enqueue_one_aead_job_gen4(ctx, qat_req, &cipher_iv, &digest, &aad, ofs,
175 total_len);
176
177 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
178 qat_sym_debug_log_dump(qat_req, ctx, in_sgl.vec, in_sgl.num, &cipher_iv,
179 NULL, &aad, &digest);
180 #endif
181
182 return 0;
183 }
184
185 static int
qat_sym_crypto_set_session_gen4(void * cdev,void * session)186 qat_sym_crypto_set_session_gen4(void *cdev, void *session)
187 {
188 struct qat_sym_session *ctx = session;
189 enum rte_proc_type_t proc_type = rte_eal_process_type();
190 int ret;
191
192 if (proc_type == RTE_PROC_AUTO || proc_type == RTE_PROC_INVALID)
193 return -EINVAL;
194
195 ret = qat_sym_crypto_set_session_gen1(cdev, session);
196 /* special single pass build request for GEN4 */
197 if (ctx->is_single_pass && ctx->is_ucs)
198 ctx->build_request[proc_type] = qat_sym_build_op_aead_gen4;
199
200 if (ret == -ENOTSUP) {
201 /* GEN1 returning -ENOTSUP as it cannot handle some mixed algo,
202 * this is addressed by GEN4
203 */
204 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 &&
205 ctx->qat_cipher_alg !=
206 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
207 qat_sym_session_set_ext_hash_flags_gen2(ctx,
208 1 << ICP_QAT_FW_AUTH_HDR_FLAG_ZUC_EIA3_BITPOS);
209 } else if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 &&
210 ctx->qat_cipher_alg !=
211 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) {
212 qat_sym_session_set_ext_hash_flags_gen2(ctx,
213 1 << ICP_QAT_FW_AUTH_HDR_FLAG_SNOW3G_UIA2_BITPOS);
214 } else if ((ctx->aes_cmac ||
215 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) &&
216 (ctx->qat_cipher_alg ==
217 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
218 ctx->qat_cipher_alg ==
219 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3)) {
220 qat_sym_session_set_ext_hash_flags_gen2(ctx, 0);
221 }
222
223 ret = 0;
224 }
225
226 return ret;
227 }
228
229 static int
qat_sym_dp_enqueue_single_aead_gen4(void * qp_data,uint8_t * drv_ctx,struct rte_crypto_vec * data,uint16_t n_data_vecs,union rte_crypto_sym_ofs ofs,struct rte_crypto_va_iova_ptr * iv,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * aad,void * user_data)230 qat_sym_dp_enqueue_single_aead_gen4(void *qp_data, uint8_t *drv_ctx,
231 struct rte_crypto_vec *data, uint16_t n_data_vecs,
232 union rte_crypto_sym_ofs ofs,
233 struct rte_crypto_va_iova_ptr *iv,
234 struct rte_crypto_va_iova_ptr *digest,
235 struct rte_crypto_va_iova_ptr *aad,
236 void *user_data)
237 {
238 struct qat_qp *qp = qp_data;
239 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
240 struct qat_queue *tx_queue = &qp->tx_q;
241 struct qat_sym_op_cookie *cookie;
242 struct qat_sym_session *ctx = dp_ctx->session;
243 struct icp_qat_fw_la_bulk_req *req;
244
245 int32_t data_len;
246 uint32_t tail = dp_ctx->tail;
247
248 req = (struct icp_qat_fw_la_bulk_req *)(
249 (uint8_t *)tx_queue->base_addr + tail);
250 cookie = qp->op_cookies[tail >> tx_queue->trailz];
251 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
252 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
253 rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
254 data_len = qat_sym_build_req_set_data(req, user_data, cookie,
255 data, n_data_vecs, NULL, 0);
256 if (unlikely(data_len < 0))
257 return -1;
258
259 enqueue_one_aead_job_gen4(ctx, req, iv, digest, aad, ofs,
260 (uint32_t)data_len);
261
262 dp_ctx->tail = tail;
263 dp_ctx->cached_enqueue++;
264
265 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
266 qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,
267 NULL, aad, digest);
268 #endif
269 return 0;
270 }
271
272 static uint32_t
qat_sym_dp_enqueue_aead_jobs_gen4(void * qp_data,uint8_t * drv_ctx,struct rte_crypto_sym_vec * vec,union rte_crypto_sym_ofs ofs,void * user_data[],int * status)273 qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,
274 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
275 void *user_data[], int *status)
276 {
277 struct qat_qp *qp = qp_data;
278 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
279 struct qat_queue *tx_queue = &qp->tx_q;
280 struct qat_sym_session *ctx = dp_ctx->session;
281 uint32_t i, n;
282 uint32_t tail;
283 struct icp_qat_fw_la_bulk_req *req;
284 int32_t data_len;
285
286 n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
287 if (unlikely(n == 0)) {
288 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
289 *status = 0;
290 return 0;
291 }
292
293 tail = dp_ctx->tail;
294
295 for (i = 0; i < n; i++) {
296 struct qat_sym_op_cookie *cookie =
297 qp->op_cookies[tail >> tx_queue->trailz];
298
299 req = (struct icp_qat_fw_la_bulk_req *)(
300 (uint8_t *)tx_queue->base_addr + tail);
301 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
302
303 if (vec->dest_sgl) {
304 data_len = qat_sym_build_req_set_data(req,
305 user_data[i], cookie,
306 vec->src_sgl[i].vec, vec->src_sgl[i].num,
307 vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
308 } else {
309 data_len = qat_sym_build_req_set_data(req,
310 user_data[i], cookie,
311 vec->src_sgl[i].vec,
312 vec->src_sgl[i].num, NULL, 0);
313 }
314
315 if (unlikely(data_len < 0))
316 break;
317
318 enqueue_one_aead_job_gen4(ctx, req, &vec->iv[i],
319 &vec->digest[i], &vec->aad[i], ofs,
320 (uint32_t)data_len);
321
322 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
323
324 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
325 qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
326 vec->src_sgl[i].num, &vec->iv[i], NULL,
327 &vec->aad[i], &vec->digest[i]);
328 #endif
329 }
330
331 if (unlikely(i < n))
332 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
333
334 dp_ctx->tail = tail;
335 dp_ctx->cached_enqueue += i;
336 *status = 0;
337 return i;
338 }
339
340 static int
qat_sym_configure_raw_dp_ctx_gen4(void * _raw_dp_ctx,void * _ctx)341 qat_sym_configure_raw_dp_ctx_gen4(void *_raw_dp_ctx, void *_ctx)
342 {
343 struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;
344 struct qat_sym_session *ctx = _ctx;
345 int ret;
346
347 ret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx);
348 if (ret < 0)
349 return ret;
350
351 if (ctx->is_single_pass && ctx->is_ucs) {
352 raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen4;
353 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead_gen4;
354 }
355
356 return 0;
357 }
358
RTE_INIT(qat_sym_crypto_gen4_init)359 RTE_INIT(qat_sym_crypto_gen4_init)
360 {
361 qat_sym_gen_dev_ops[QAT_GEN4].cryptodev_ops = &qat_sym_crypto_ops_gen1;
362 qat_sym_gen_dev_ops[QAT_GEN4].get_capabilities =
363 qat_sym_crypto_cap_get_gen4;
364 qat_sym_gen_dev_ops[QAT_GEN4].set_session =
365 qat_sym_crypto_set_session_gen4;
366 qat_sym_gen_dev_ops[QAT_GEN4].set_raw_dp_ctx =
367 qat_sym_configure_raw_dp_ctx_gen4;
368 qat_sym_gen_dev_ops[QAT_GEN4].get_feature_flags =
369 qat_sym_crypto_feature_flags_get_gen1;
370 #ifdef RTE_LIB_SECURITY
371 qat_sym_gen_dev_ops[QAT_GEN4].create_security_ctx =
372 qat_sym_create_security_gen1;
373 #endif
374 }
375
RTE_INIT(qat_asym_crypto_gen4_init)376 RTE_INIT(qat_asym_crypto_gen4_init)
377 {
378 qat_asym_gen_dev_ops[QAT_GEN4].cryptodev_ops = NULL;
379 qat_asym_gen_dev_ops[QAT_GEN4].get_capabilities = NULL;
380 qat_asym_gen_dev_ops[QAT_GEN4].get_feature_flags = NULL;
381 qat_asym_gen_dev_ops[QAT_GEN4].set_session = NULL;
382 }
383