1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017-2019 NXP
3 */
4
5 #include <fcntl.h>
6 #include <unistd.h>
7 #include <sched.h>
8 #include <net/if.h>
9
10 #include <rte_byteorder.h>
11 #include <rte_common.h>
12 #include <rte_cryptodev_pmd.h>
13 #include <rte_crypto.h>
14 #include <rte_cryptodev.h>
15 #include <rte_bus_vdev.h>
16 #include <rte_malloc.h>
17 #include <rte_security_driver.h>
18 #include <rte_hexdump.h>
19
20 #include <caam_jr_capabilities.h>
21 #include <caam_jr_config.h>
22 #include <caam_jr_hw_specific.h>
23 #include <caam_jr_pvt.h>
24 #include <caam_jr_desc.h>
25 #include <caam_jr_log.h>
26
27 /* RTA header files */
28 #include <desc/common.h>
29 #include <desc/algo.h>
30 #include <dpaa_of.h>
31 #ifdef RTE_LIBRTE_PMD_CAAM_JR_DEBUG
32 #define CAAM_JR_DBG 1
33 #else
34 #define CAAM_JR_DBG 0
35 #endif
36 #define CRYPTODEV_NAME_CAAM_JR_PMD crypto_caam_jr
37 static uint8_t cryptodev_driver_id;
38
39 /* Lists the states possible for the SEC user space driver. */
40 enum sec_driver_state_e {
41 SEC_DRIVER_STATE_IDLE, /* Driver not initialized */
42 SEC_DRIVER_STATE_STARTED, /* Driver initialized and can be used*/
43 SEC_DRIVER_STATE_RELEASE, /* Driver release is in progress */
44 };
45
46 /* Job rings used for communication with SEC HW */
47 static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
48
49 /* The current state of SEC user space driver */
50 static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
51
52 /* The number of job rings used by SEC user space driver */
53 static int g_job_rings_no;
54 static int g_job_rings_max;
55
56 struct sec_outring_entry {
57 phys_addr_t desc; /* Pointer to completed descriptor */
58 uint32_t status; /* Status for completed descriptor */
59 } __rte_packed;
60
61 /* virtual address conversin when mempool support is available for ctx */
62 static inline phys_addr_t
caam_jr_vtop_ctx(struct caam_jr_op_ctx * ctx,void * vaddr)63 caam_jr_vtop_ctx(struct caam_jr_op_ctx *ctx, void *vaddr)
64 {
65 return (size_t)vaddr - ctx->vtop_offset;
66 }
67
68 static inline void
caam_jr_op_ending(struct caam_jr_op_ctx * ctx)69 caam_jr_op_ending(struct caam_jr_op_ctx *ctx)
70 {
71 /* report op status to sym->op and then free the ctx memory */
72 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
73 }
74
75 static inline struct caam_jr_op_ctx *
caam_jr_alloc_ctx(struct caam_jr_session * ses)76 caam_jr_alloc_ctx(struct caam_jr_session *ses)
77 {
78 struct caam_jr_op_ctx *ctx;
79 int ret;
80
81 ret = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
82 if (!ctx || ret) {
83 CAAM_JR_DP_WARN("Alloc sec descriptor failed!");
84 return NULL;
85 }
86 /*
87 * Clear SG memory. There are 16 SG entries of 16 Bytes each.
88 * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
89 * to clear all the SG entries. caam_jr_alloc_ctx() is called for
90 * each packet, memset is costlier than dcbz_64().
91 */
92 dcbz_64(&ctx->sg[SG_CACHELINE_0]);
93 dcbz_64(&ctx->sg[SG_CACHELINE_1]);
94 dcbz_64(&ctx->sg[SG_CACHELINE_2]);
95 dcbz_64(&ctx->sg[SG_CACHELINE_3]);
96
97 ctx->ctx_pool = ses->ctx_pool;
98 ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
99
100 return ctx;
101 }
102
103 static
caam_jr_stats_get(struct rte_cryptodev * dev,struct rte_cryptodev_stats * stats)104 void caam_jr_stats_get(struct rte_cryptodev *dev,
105 struct rte_cryptodev_stats *stats)
106 {
107 struct caam_jr_qp **qp = (struct caam_jr_qp **)
108 dev->data->queue_pairs;
109 int i;
110
111 PMD_INIT_FUNC_TRACE();
112 if (stats == NULL) {
113 CAAM_JR_ERR("Invalid stats ptr NULL");
114 return;
115 }
116 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
117 if (qp[i] == NULL) {
118 CAAM_JR_WARN("Uninitialised queue pair");
119 continue;
120 }
121
122 stats->enqueued_count += qp[i]->tx_pkts;
123 stats->dequeued_count += qp[i]->rx_pkts;
124 stats->enqueue_err_count += qp[i]->tx_errs;
125 stats->dequeue_err_count += qp[i]->rx_errs;
126 CAAM_JR_INFO("extra stats:\n\tRX Poll ERR = %" PRIu64
127 "\n\tTX Ring Full = %" PRIu64,
128 qp[i]->rx_poll_err,
129 qp[i]->tx_ring_full);
130 }
131 }
132
133 static
caam_jr_stats_reset(struct rte_cryptodev * dev)134 void caam_jr_stats_reset(struct rte_cryptodev *dev)
135 {
136 int i;
137 struct caam_jr_qp **qp = (struct caam_jr_qp **)
138 (dev->data->queue_pairs);
139
140 PMD_INIT_FUNC_TRACE();
141 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
142 if (qp[i] == NULL) {
143 CAAM_JR_WARN("Uninitialised queue pair");
144 continue;
145 }
146 qp[i]->rx_pkts = 0;
147 qp[i]->rx_errs = 0;
148 qp[i]->rx_poll_err = 0;
149 qp[i]->tx_pkts = 0;
150 qp[i]->tx_errs = 0;
151 qp[i]->tx_ring_full = 0;
152 }
153 }
154
155 static inline int
is_cipher_only(struct caam_jr_session * ses)156 is_cipher_only(struct caam_jr_session *ses)
157 {
158 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
159 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
160 }
161
162 static inline int
is_auth_only(struct caam_jr_session * ses)163 is_auth_only(struct caam_jr_session *ses)
164 {
165 return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
166 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
167 }
168
169 static inline int
is_aead(struct caam_jr_session * ses)170 is_aead(struct caam_jr_session *ses)
171 {
172 return ((ses->cipher_alg == 0) &&
173 (ses->auth_alg == 0) &&
174 (ses->aead_alg != 0));
175 }
176
177 static inline int
is_auth_cipher(struct caam_jr_session * ses)178 is_auth_cipher(struct caam_jr_session *ses)
179 {
180 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
181 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
182 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
183 }
184
185 static inline int
is_proto_ipsec(struct caam_jr_session * ses)186 is_proto_ipsec(struct caam_jr_session *ses)
187 {
188 return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
189 }
190
191 static inline int
is_encode(struct caam_jr_session * ses)192 is_encode(struct caam_jr_session *ses)
193 {
194 return ses->dir == DIR_ENC;
195 }
196
197 static inline int
is_decode(struct caam_jr_session * ses)198 is_decode(struct caam_jr_session *ses)
199 {
200 return ses->dir == DIR_DEC;
201 }
202
203 static inline void
caam_auth_alg(struct caam_jr_session * ses,struct alginfo * alginfo_a)204 caam_auth_alg(struct caam_jr_session *ses, struct alginfo *alginfo_a)
205 {
206 switch (ses->auth_alg) {
207 case RTE_CRYPTO_AUTH_NULL:
208 ses->digest_length = 0;
209 break;
210 case RTE_CRYPTO_AUTH_MD5_HMAC:
211 alginfo_a->algtype =
212 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
213 OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
214 alginfo_a->algmode = OP_ALG_AAI_HMAC;
215 break;
216 case RTE_CRYPTO_AUTH_SHA1_HMAC:
217 alginfo_a->algtype =
218 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
219 OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
220 alginfo_a->algmode = OP_ALG_AAI_HMAC;
221 break;
222 case RTE_CRYPTO_AUTH_SHA224_HMAC:
223 alginfo_a->algtype =
224 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
225 OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
226 alginfo_a->algmode = OP_ALG_AAI_HMAC;
227 break;
228 case RTE_CRYPTO_AUTH_SHA256_HMAC:
229 alginfo_a->algtype =
230 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
231 OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
232 alginfo_a->algmode = OP_ALG_AAI_HMAC;
233 break;
234 case RTE_CRYPTO_AUTH_SHA384_HMAC:
235 alginfo_a->algtype =
236 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
237 OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
238 alginfo_a->algmode = OP_ALG_AAI_HMAC;
239 break;
240 case RTE_CRYPTO_AUTH_SHA512_HMAC:
241 alginfo_a->algtype =
242 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
243 OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
244 alginfo_a->algmode = OP_ALG_AAI_HMAC;
245 break;
246 default:
247 CAAM_JR_DEBUG("unsupported auth alg %u", ses->auth_alg);
248 }
249 }
250
251 static inline void
caam_cipher_alg(struct caam_jr_session * ses,struct alginfo * alginfo_c)252 caam_cipher_alg(struct caam_jr_session *ses, struct alginfo *alginfo_c)
253 {
254 switch (ses->cipher_alg) {
255 case RTE_CRYPTO_CIPHER_NULL:
256 break;
257 case RTE_CRYPTO_CIPHER_AES_CBC:
258 alginfo_c->algtype =
259 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
260 OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
261 alginfo_c->algmode = OP_ALG_AAI_CBC;
262 break;
263 case RTE_CRYPTO_CIPHER_3DES_CBC:
264 alginfo_c->algtype =
265 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
266 OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
267 alginfo_c->algmode = OP_ALG_AAI_CBC;
268 break;
269 case RTE_CRYPTO_CIPHER_AES_CTR:
270 alginfo_c->algtype =
271 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
272 OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
273 alginfo_c->algmode = OP_ALG_AAI_CTR;
274 break;
275 default:
276 CAAM_JR_DEBUG("unsupported cipher alg %d", ses->cipher_alg);
277 }
278 }
279
280 static inline void
caam_aead_alg(struct caam_jr_session * ses,struct alginfo * alginfo)281 caam_aead_alg(struct caam_jr_session *ses, struct alginfo *alginfo)
282 {
283 switch (ses->aead_alg) {
284 case RTE_CRYPTO_AEAD_AES_GCM:
285 alginfo->algtype = OP_ALG_ALGSEL_AES;
286 alginfo->algmode = OP_ALG_AAI_GCM;
287 break;
288 default:
289 CAAM_JR_DEBUG("unsupported AEAD alg %d", ses->aead_alg);
290 }
291 }
292
293 /* prepare command block of the session */
294 static int
caam_jr_prep_cdb(struct caam_jr_session * ses)295 caam_jr_prep_cdb(struct caam_jr_session *ses)
296 {
297 struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
298 int32_t shared_desc_len = 0;
299 struct sec_cdb *cdb;
300 int err;
301 #if CAAM_BYTE_ORDER == CORE_BYTE_ORDER
302 int swap = false;
303 #else
304 int swap = true;
305 #endif
306
307 if (ses->cdb)
308 caam_jr_dma_free(ses->cdb);
309
310 cdb = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, sizeof(struct sec_cdb));
311 if (!cdb) {
312 CAAM_JR_ERR("failed to allocate memory for cdb\n");
313 return -1;
314 }
315
316 ses->cdb = cdb;
317
318 memset(cdb, 0, sizeof(struct sec_cdb));
319
320 if (is_cipher_only(ses)) {
321 caam_cipher_alg(ses, &alginfo_c);
322 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
323 CAAM_JR_ERR("not supported cipher alg");
324 rte_free(cdb);
325 return -ENOTSUP;
326 }
327
328 alginfo_c.key = (size_t)ses->cipher_key.data;
329 alginfo_c.keylen = ses->cipher_key.length;
330 alginfo_c.key_enc_flags = 0;
331 alginfo_c.key_type = RTA_DATA_IMM;
332
333 shared_desc_len = cnstr_shdsc_blkcipher(
334 cdb->sh_desc, true,
335 swap, SHR_NEVER, &alginfo_c,
336 ses->iv.length,
337 ses->dir);
338 } else if (is_auth_only(ses)) {
339 caam_auth_alg(ses, &alginfo_a);
340 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
341 CAAM_JR_ERR("not supported auth alg");
342 rte_free(cdb);
343 return -ENOTSUP;
344 }
345
346 alginfo_a.key = (size_t)ses->auth_key.data;
347 alginfo_a.keylen = ses->auth_key.length;
348 alginfo_a.key_enc_flags = 0;
349 alginfo_a.key_type = RTA_DATA_IMM;
350
351 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
352 swap, SHR_NEVER, &alginfo_a,
353 !ses->dir,
354 ses->digest_length);
355 } else if (is_aead(ses)) {
356 caam_aead_alg(ses, &alginfo);
357 if (alginfo.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
358 CAAM_JR_ERR("not supported aead alg");
359 rte_free(cdb);
360 return -ENOTSUP;
361 }
362 alginfo.key = (size_t)ses->aead_key.data;
363 alginfo.keylen = ses->aead_key.length;
364 alginfo.key_enc_flags = 0;
365 alginfo.key_type = RTA_DATA_IMM;
366
367 if (ses->dir == DIR_ENC)
368 shared_desc_len = cnstr_shdsc_gcm_encap(
369 cdb->sh_desc, true, swap,
370 SHR_NEVER, &alginfo,
371 ses->iv.length,
372 ses->digest_length);
373 else
374 shared_desc_len = cnstr_shdsc_gcm_decap(
375 cdb->sh_desc, true, swap,
376 SHR_NEVER, &alginfo,
377 ses->iv.length,
378 ses->digest_length);
379 } else {
380 caam_cipher_alg(ses, &alginfo_c);
381 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
382 CAAM_JR_ERR("not supported cipher alg");
383 rte_free(cdb);
384 return -ENOTSUP;
385 }
386
387 alginfo_c.key = (size_t)ses->cipher_key.data;
388 alginfo_c.keylen = ses->cipher_key.length;
389 alginfo_c.key_enc_flags = 0;
390 alginfo_c.key_type = RTA_DATA_IMM;
391
392 caam_auth_alg(ses, &alginfo_a);
393 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
394 CAAM_JR_ERR("not supported auth alg");
395 rte_free(cdb);
396 return -ENOTSUP;
397 }
398
399 alginfo_a.key = (size_t)ses->auth_key.data;
400 alginfo_a.keylen = ses->auth_key.length;
401 alginfo_a.key_enc_flags = 0;
402 alginfo_a.key_type = RTA_DATA_IMM;
403
404 cdb->sh_desc[0] = alginfo_c.keylen;
405 cdb->sh_desc[1] = alginfo_a.keylen;
406 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
407 MIN_JOB_DESC_SIZE,
408 (unsigned int *)cdb->sh_desc,
409 &cdb->sh_desc[2], 2);
410
411 if (err < 0) {
412 CAAM_JR_ERR("Crypto: Incorrect key lengths");
413 rte_free(cdb);
414 return err;
415 }
416 if (cdb->sh_desc[2] & 1)
417 alginfo_c.key_type = RTA_DATA_IMM;
418 else {
419 alginfo_c.key = (size_t)caam_jr_mem_vtop(
420 (void *)(size_t)alginfo_c.key);
421 alginfo_c.key_type = RTA_DATA_PTR;
422 }
423 if (cdb->sh_desc[2] & (1<<1))
424 alginfo_a.key_type = RTA_DATA_IMM;
425 else {
426 alginfo_a.key = (size_t)caam_jr_mem_vtop(
427 (void *)(size_t)alginfo_a.key);
428 alginfo_a.key_type = RTA_DATA_PTR;
429 }
430 cdb->sh_desc[0] = 0;
431 cdb->sh_desc[1] = 0;
432 cdb->sh_desc[2] = 0;
433 if (is_proto_ipsec(ses)) {
434 if (ses->dir == DIR_ENC) {
435 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
436 cdb->sh_desc,
437 true, swap, SHR_SERIAL,
438 &ses->encap_pdb,
439 (uint8_t *)&ses->ip4_hdr,
440 &alginfo_c, &alginfo_a);
441 } else if (ses->dir == DIR_DEC) {
442 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
443 cdb->sh_desc,
444 true, swap, SHR_SERIAL,
445 &ses->decap_pdb,
446 &alginfo_c, &alginfo_a);
447 }
448 } else {
449 /* Auth_only_len is overwritten in fd for each job */
450 shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
451 true, swap, SHR_SERIAL,
452 &alginfo_c, &alginfo_a,
453 ses->iv.length,
454 ses->digest_length, ses->dir);
455 }
456 }
457
458 if (shared_desc_len < 0) {
459 CAAM_JR_ERR("error in preparing command block");
460 return shared_desc_len;
461 }
462
463 #if CAAM_JR_DBG
464 SEC_DUMP_DESC(cdb->sh_desc);
465 #endif
466
467 cdb->sh_hdr.hi.field.idlen = shared_desc_len;
468
469 return 0;
470 }
471
472 /* @brief Poll the HW for already processed jobs in the JR
473 * and silently discard the available jobs or notify them to UA
474 * with indicated error code.
475 *
476 * @param [in,out] job_ring The job ring to poll.
477 * @param [in] do_notify Can be #TRUE or #FALSE. Indicates if
478 * descriptors are to be discarded
479 * or notified to UA with given error_code.
480 * @param [out] notified_descs Number of notified descriptors. Can be NULL
481 * if do_notify is #FALSE
482 */
483 static void
hw_flush_job_ring(struct sec_job_ring_t * job_ring,uint32_t do_notify,uint32_t * notified_descs)484 hw_flush_job_ring(struct sec_job_ring_t *job_ring,
485 uint32_t do_notify,
486 uint32_t *notified_descs)
487 {
488 int32_t jobs_no_to_discard = 0;
489 int32_t discarded_descs_no = 0;
490
491 CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
492 job_ring, job_ring->pidx, job_ring->cidx, do_notify);
493
494 jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
495
496 /* Discard all jobs */
497 CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
498 job_ring, job_ring->pidx, job_ring->cidx,
499 jobs_no_to_discard);
500
501 while (jobs_no_to_discard > discarded_descs_no) {
502 discarded_descs_no++;
503 /* Now increment the consumer index for the current job ring,
504 * AFTER saving job in temporary location!
505 * Increment the consumer index for the current job ring
506 */
507 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
508 SEC_JOB_RING_SIZE);
509
510 hw_remove_entries(job_ring, 1);
511 }
512
513 if (do_notify == true) {
514 ASSERT(notified_descs != NULL);
515 *notified_descs = discarded_descs_no;
516 }
517 }
518
519 /* @brief Poll the HW for already processed jobs in the JR
520 * and notify the available jobs to UA.
521 *
522 * @param [in] job_ring The job ring to poll.
523 * @param [in] limit The maximum number of jobs to notify.
524 * If set to negative value, all available jobs are
525 * notified.
526 *
527 * @retval >=0 for No of jobs notified to UA.
528 * @retval -1 for error
529 */
530 static int
hw_poll_job_ring(struct sec_job_ring_t * job_ring,struct rte_crypto_op ** ops,int32_t limit,struct caam_jr_qp * jr_qp)531 hw_poll_job_ring(struct sec_job_ring_t *job_ring,
532 struct rte_crypto_op **ops, int32_t limit,
533 struct caam_jr_qp *jr_qp)
534 {
535 int32_t jobs_no_to_notify = 0; /* the number of done jobs to notify*/
536 int32_t number_of_jobs_available = 0;
537 int32_t notified_descs_no = 0;
538 uint32_t sec_error_code = 0;
539 struct job_descriptor *current_desc;
540 phys_addr_t current_desc_addr;
541 phys_addr_t *temp_addr;
542 struct caam_jr_op_ctx *ctx;
543
544 /* TODO check for ops have memory*/
545 /* check here if any JR error that cannot be written
546 * in the output status word has occurred
547 */
548 if (JR_REG_JRINT_JRE_EXTRACT(GET_JR_REG(JRINT, job_ring))) {
549 CAAM_JR_INFO("err received");
550 sec_error_code = JR_REG_JRINT_ERR_TYPE_EXTRACT(
551 GET_JR_REG(JRINT, job_ring));
552 if (unlikely(sec_error_code)) {
553 hw_job_ring_error_print(job_ring, sec_error_code);
554 return -1;
555 }
556 }
557 /* compute the number of jobs available in the job ring based on the
558 * producer and consumer index values.
559 */
560 number_of_jobs_available = hw_get_no_finished_jobs(job_ring);
561 /* Compute the number of notifications that need to be raised to UA
562 * If limit > total number of done jobs -> notify all done jobs
563 * If limit = 0 -> error
564 * If limit < total number of done jobs -> notify a number
565 * of done jobs equal with limit
566 */
567 jobs_no_to_notify = (limit > number_of_jobs_available) ?
568 number_of_jobs_available : limit;
569 CAAM_JR_DP_DEBUG(
570 "Jr[%p] pi[%d] ci[%d].limit =%d Available=%d.Jobs to notify=%d",
571 job_ring, job_ring->pidx, job_ring->cidx,
572 limit, number_of_jobs_available, jobs_no_to_notify);
573
574 rte_smp_rmb();
575
576 while (jobs_no_to_notify > notified_descs_no) {
577 static uint64_t false_alarm;
578 static uint64_t real_poll;
579
580 /* Get job status here */
581 sec_error_code = job_ring->output_ring[job_ring->cidx].status;
582 /* Get completed descriptor */
583 temp_addr = &(job_ring->output_ring[job_ring->cidx].desc);
584 current_desc_addr = (phys_addr_t)sec_read_addr(temp_addr);
585
586 real_poll++;
587 /* todo check if it is false alarm no desc present */
588 if (!current_desc_addr) {
589 false_alarm++;
590 printf("false alarm %" PRIu64 "real %" PRIu64
591 " sec_err =0x%x cidx Index =0%d\n",
592 false_alarm, real_poll,
593 sec_error_code, job_ring->cidx);
594 rte_panic("CAAM JR descriptor NULL");
595 return notified_descs_no;
596 }
597 current_desc = (struct job_descriptor *)
598 caam_jr_dma_ptov(current_desc_addr);
599 /* now increment the consumer index for the current job ring,
600 * AFTER saving job in temporary location!
601 */
602 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
603 SEC_JOB_RING_SIZE);
604 /* Signal that the job has been processed and the slot is free*/
605 hw_remove_entries(job_ring, 1);
606 /*TODO for multiple ops, packets*/
607 ctx = container_of(current_desc, struct caam_jr_op_ctx, jobdes);
608 if (unlikely(sec_error_code)) {
609 CAAM_JR_ERR("desc at cidx %d generated error 0x%x\n",
610 job_ring->cidx, sec_error_code);
611 hw_handle_job_ring_error(job_ring, sec_error_code);
612 //todo improve with exact errors
613 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
614 jr_qp->rx_errs++;
615 } else {
616 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
617 #if CAAM_JR_DBG
618 if (ctx->op->sym->m_dst) {
619 rte_hexdump(stdout, "PROCESSED",
620 rte_pktmbuf_mtod(ctx->op->sym->m_dst, void *),
621 rte_pktmbuf_data_len(ctx->op->sym->m_dst));
622 } else {
623 rte_hexdump(stdout, "PROCESSED",
624 rte_pktmbuf_mtod(ctx->op->sym->m_src, void *),
625 rte_pktmbuf_data_len(ctx->op->sym->m_src));
626 }
627 #endif
628 }
629 if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
630 struct ip *ip4_hdr;
631
632 if (ctx->op->sym->m_dst) {
633 /*TODO check for ip header or other*/
634 ip4_hdr = (struct ip *)
635 rte_pktmbuf_mtod(ctx->op->sym->m_dst, char*);
636 ctx->op->sym->m_dst->pkt_len =
637 rte_be_to_cpu_16(ip4_hdr->ip_len);
638 ctx->op->sym->m_dst->data_len =
639 rte_be_to_cpu_16(ip4_hdr->ip_len);
640 } else {
641 ip4_hdr = (struct ip *)
642 rte_pktmbuf_mtod(ctx->op->sym->m_src, char*);
643 ctx->op->sym->m_src->pkt_len =
644 rte_be_to_cpu_16(ip4_hdr->ip_len);
645 ctx->op->sym->m_src->data_len =
646 rte_be_to_cpu_16(ip4_hdr->ip_len);
647 }
648 }
649 *ops = ctx->op;
650 caam_jr_op_ending(ctx);
651 ops++;
652 notified_descs_no++;
653 }
654 return notified_descs_no;
655 }
656
657 static uint16_t
caam_jr_dequeue_burst(void * qp,struct rte_crypto_op ** ops,uint16_t nb_ops)658 caam_jr_dequeue_burst(void *qp, struct rte_crypto_op **ops,
659 uint16_t nb_ops)
660 {
661 struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
662 struct sec_job_ring_t *ring = jr_qp->ring;
663 int num_rx;
664 int ret;
665
666 CAAM_JR_DP_DEBUG("Jr[%p]Polling. limit[%d]", ring, nb_ops);
667
668 /* Poll job ring
669 * If nb_ops < 0 -> poll JR until no more notifications are available.
670 * If nb_ops > 0 -> poll JR until limit is reached.
671 */
672
673 /* Run hw poll job ring */
674 num_rx = hw_poll_job_ring(ring, ops, nb_ops, jr_qp);
675 if (num_rx < 0) {
676 CAAM_JR_ERR("Error polling SEC engine (%d)", num_rx);
677 return 0;
678 }
679
680 CAAM_JR_DP_DEBUG("Jr[%p].Jobs notified[%d]. ", ring, num_rx);
681
682 if (ring->jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
683 if (num_rx < nb_ops) {
684 ret = caam_jr_enable_irqs(ring->irq_fd);
685 SEC_ASSERT(ret == 0, ret,
686 "Failed to enable irqs for job ring %p", ring);
687 }
688 } else if (ring->jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
689
690 /* Always enable IRQ generation when in pure IRQ mode */
691 ret = caam_jr_enable_irqs(ring->irq_fd);
692 SEC_ASSERT(ret == 0, ret,
693 "Failed to enable irqs for job ring %p", ring);
694 }
695
696 jr_qp->rx_pkts += num_rx;
697
698 return num_rx;
699 }
700
701 /**
702 * packet looks like:
703 * |<----data_len------->|
704 * |ip_header|ah_header|icv|payload|
705 * ^
706 * |
707 * mbuf->pkt.data
708 */
709 static inline struct caam_jr_op_ctx *
build_auth_only_sg(struct rte_crypto_op * op,struct caam_jr_session * ses)710 build_auth_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
711 {
712 struct rte_crypto_sym_op *sym = op->sym;
713 struct rte_mbuf *mbuf = sym->m_src;
714 struct caam_jr_op_ctx *ctx;
715 struct sec4_sg_entry *sg;
716 int length;
717 struct sec_cdb *cdb;
718 uint64_t sdesc_offset;
719 struct sec_job_descriptor_t *jobdescr;
720 uint8_t extra_segs;
721
722 if (is_decode(ses))
723 extra_segs = 2;
724 else
725 extra_segs = 1;
726
727 if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
728 CAAM_JR_DP_ERR("Auth: Max sec segs supported is %d",
729 MAX_SG_ENTRIES);
730 return NULL;
731 }
732
733 ctx = caam_jr_alloc_ctx(ses);
734 if (!ctx)
735 return NULL;
736
737 ctx->op = op;
738
739 cdb = ses->cdb;
740 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
741
742 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
743
744 SEC_JD_INIT(jobdescr);
745 SEC_JD_SET_SD(jobdescr,
746 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
747 cdb->sh_hdr.hi.field.idlen);
748
749 /* output */
750 SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
751 0, ses->digest_length);
752
753 /*input */
754 sg = &ctx->sg[0];
755 length = sym->auth.data.length;
756 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + sym->auth.data.offset);
757 sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
758
759 /* Successive segs */
760 mbuf = mbuf->next;
761 while (mbuf) {
762 sg++;
763 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
764 sg->len = cpu_to_caam32(mbuf->data_len);
765 mbuf = mbuf->next;
766 }
767
768 if (is_decode(ses)) {
769 /* digest verification case */
770 sg++;
771 /* hash result or digest, save digest first */
772 rte_memcpy(ctx->digest, sym->auth.digest.data,
773 ses->digest_length);
774 #if CAAM_JR_DBG
775 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
776 #endif
777 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
778 sg->len = cpu_to_caam32(ses->digest_length);
779 length += ses->digest_length;
780 } else {
781 sg->len -= ses->digest_length;
782 }
783
784 /* last element*/
785 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
786
787 SEC_JD_SET_IN_PTR(jobdescr,
788 (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, length);
789 /* enabling sg list */
790 (jobdescr)->seq_in.command.word |= 0x01000000;
791
792 return ctx;
793 }
794
795 static inline struct caam_jr_op_ctx *
build_auth_only(struct rte_crypto_op * op,struct caam_jr_session * ses)796 build_auth_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
797 {
798 struct rte_crypto_sym_op *sym = op->sym;
799 struct caam_jr_op_ctx *ctx;
800 struct sec4_sg_entry *sg;
801 rte_iova_t start_addr;
802 struct sec_cdb *cdb;
803 uint64_t sdesc_offset;
804 struct sec_job_descriptor_t *jobdescr;
805
806 ctx = caam_jr_alloc_ctx(ses);
807 if (!ctx)
808 return NULL;
809
810 ctx->op = op;
811
812 cdb = ses->cdb;
813 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
814
815 start_addr = rte_pktmbuf_iova(sym->m_src);
816
817 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
818
819 SEC_JD_INIT(jobdescr);
820 SEC_JD_SET_SD(jobdescr,
821 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
822 cdb->sh_hdr.hi.field.idlen);
823
824 /* output */
825 SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
826 0, ses->digest_length);
827
828 /*input */
829 if (is_decode(ses)) {
830 sg = &ctx->sg[0];
831 SEC_JD_SET_IN_PTR(jobdescr,
832 (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
833 (sym->auth.data.length + ses->digest_length));
834 /* enabling sg list */
835 (jobdescr)->seq_in.command.word |= 0x01000000;
836
837 /* hash result or digest, save digest first */
838 rte_memcpy(ctx->digest, sym->auth.digest.data,
839 ses->digest_length);
840 sg->ptr = cpu_to_caam64(start_addr + sym->auth.data.offset);
841 sg->len = cpu_to_caam32(sym->auth.data.length);
842
843 #if CAAM_JR_DBG
844 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
845 #endif
846 /* let's check digest by hw */
847 sg++;
848 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
849 sg->len = cpu_to_caam32(ses->digest_length);
850 /* last element*/
851 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
852 } else {
853 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)start_addr,
854 sym->auth.data.offset, sym->auth.data.length);
855 }
856 return ctx;
857 }
858
859 static inline struct caam_jr_op_ctx *
build_cipher_only_sg(struct rte_crypto_op * op,struct caam_jr_session * ses)860 build_cipher_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
861 {
862 struct rte_crypto_sym_op *sym = op->sym;
863 struct rte_mbuf *mbuf = sym->m_src;
864 struct caam_jr_op_ctx *ctx;
865 struct sec4_sg_entry *sg, *in_sg;
866 int length;
867 struct sec_cdb *cdb;
868 uint64_t sdesc_offset;
869 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
870 ses->iv.offset);
871 struct sec_job_descriptor_t *jobdescr;
872 uint8_t reg_segs;
873
874 if (sym->m_dst) {
875 mbuf = sym->m_dst;
876 reg_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
877 } else {
878 mbuf = sym->m_src;
879 reg_segs = mbuf->nb_segs * 2 + 2;
880 }
881
882 if (reg_segs > MAX_SG_ENTRIES) {
883 CAAM_JR_DP_ERR("Cipher: Max sec segs supported is %d",
884 MAX_SG_ENTRIES);
885 return NULL;
886 }
887
888 ctx = caam_jr_alloc_ctx(ses);
889 if (!ctx)
890 return NULL;
891
892 ctx->op = op;
893 cdb = ses->cdb;
894 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
895
896 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
897
898 SEC_JD_INIT(jobdescr);
899 SEC_JD_SET_SD(jobdescr,
900 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
901 cdb->sh_hdr.hi.field.idlen);
902
903 #if CAAM_JR_DBG
904 CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
905 sym->m_src->data_off, sym->cipher.data.offset,
906 sym->cipher.data.length, ses->iv.length);
907 #endif
908 /* output */
909 if (sym->m_dst)
910 mbuf = sym->m_dst;
911 else
912 mbuf = sym->m_src;
913
914 sg = &ctx->sg[0];
915 length = sym->cipher.data.length;
916
917 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
918 + sym->cipher.data.offset);
919 sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
920
921 /* Successive segs */
922 mbuf = mbuf->next;
923 while (mbuf) {
924 sg++;
925 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
926 sg->len = cpu_to_caam32(mbuf->data_len);
927 mbuf = mbuf->next;
928 }
929 /* last element*/
930 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
931
932 SEC_JD_SET_OUT_PTR(jobdescr,
933 (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0,
934 length);
935 /*enabling sg bit */
936 (jobdescr)->seq_out.command.word |= 0x01000000;
937
938 /*input */
939 sg++;
940 mbuf = sym->m_src;
941 in_sg = sg;
942
943 length = sym->cipher.data.length + ses->iv.length;
944
945 /* IV */
946 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
947 sg->len = cpu_to_caam32(ses->iv.length);
948
949 /* 1st seg */
950 sg++;
951 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
952 + sym->cipher.data.offset);
953 sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
954
955 /* Successive segs */
956 mbuf = mbuf->next;
957 while (mbuf) {
958 sg++;
959 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
960 sg->len = cpu_to_caam32(mbuf->data_len);
961 mbuf = mbuf->next;
962 }
963 /* last element*/
964 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
965
966
967 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, in_sg), 0,
968 length);
969 /*enabling sg bit */
970 (jobdescr)->seq_in.command.word |= 0x01000000;
971
972 return ctx;
973 }
974
975 static inline struct caam_jr_op_ctx *
build_cipher_only(struct rte_crypto_op * op,struct caam_jr_session * ses)976 build_cipher_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
977 {
978 struct rte_crypto_sym_op *sym = op->sym;
979 struct caam_jr_op_ctx *ctx;
980 struct sec4_sg_entry *sg;
981 rte_iova_t src_start_addr, dst_start_addr;
982 struct sec_cdb *cdb;
983 uint64_t sdesc_offset;
984 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
985 ses->iv.offset);
986 struct sec_job_descriptor_t *jobdescr;
987
988 ctx = caam_jr_alloc_ctx(ses);
989 if (!ctx)
990 return NULL;
991
992 ctx->op = op;
993 cdb = ses->cdb;
994 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
995
996 src_start_addr = rte_pktmbuf_iova(sym->m_src);
997 if (sym->m_dst)
998 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
999 else
1000 dst_start_addr = src_start_addr;
1001
1002 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1003
1004 SEC_JD_INIT(jobdescr);
1005 SEC_JD_SET_SD(jobdescr,
1006 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1007 cdb->sh_hdr.hi.field.idlen);
1008
1009 #if CAAM_JR_DBG
1010 CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
1011 sym->m_src->data_off, sym->cipher.data.offset,
1012 sym->cipher.data.length, ses->iv.length);
1013 #endif
1014 /* output */
1015 SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr,
1016 sym->cipher.data.offset,
1017 sym->cipher.data.length + ses->iv.length);
1018
1019 /*input */
1020 sg = &ctx->sg[0];
1021 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
1022 sym->cipher.data.length + ses->iv.length);
1023 /*enabling sg bit */
1024 (jobdescr)->seq_in.command.word |= 0x01000000;
1025
1026 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1027 sg->len = cpu_to_caam32(ses->iv.length);
1028
1029 sg = &ctx->sg[1];
1030 sg->ptr = cpu_to_caam64(src_start_addr + sym->cipher.data.offset);
1031 sg->len = cpu_to_caam32(sym->cipher.data.length);
1032 /* last element*/
1033 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1034
1035 return ctx;
1036 }
1037
1038 /* For decapsulation:
1039 * Input:
1040 * +----+----------------+--------------------------------+-----+
1041 * | IV | Auth-only data | Authenticated & Encrypted data | ICV |
1042 * +----+----------------+--------------------------------+-----+
1043 * Output:
1044 * +----+--------------------------+
1045 * | Decrypted & authenticated data |
1046 * +----+--------------------------+
1047 */
1048
1049 static inline struct caam_jr_op_ctx *
build_cipher_auth_sg(struct rte_crypto_op * op,struct caam_jr_session * ses)1050 build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
1051 {
1052 struct rte_crypto_sym_op *sym = op->sym;
1053 struct caam_jr_op_ctx *ctx;
1054 struct sec4_sg_entry *sg, *out_sg, *in_sg;
1055 struct rte_mbuf *mbuf;
1056 uint32_t length = 0;
1057 struct sec_cdb *cdb;
1058 uint64_t sdesc_offset;
1059 uint8_t req_segs;
1060 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1061 ses->iv.offset);
1062 struct sec_job_descriptor_t *jobdescr;
1063 uint16_t auth_hdr_len = sym->cipher.data.offset -
1064 sym->auth.data.offset;
1065 uint16_t auth_tail_len = sym->auth.data.length -
1066 sym->cipher.data.length - auth_hdr_len;
1067 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
1068
1069 if (sym->m_dst) {
1070 mbuf = sym->m_dst;
1071 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
1072 } else {
1073 mbuf = sym->m_src;
1074 req_segs = mbuf->nb_segs * 2 + 3;
1075 }
1076
1077 if (req_segs > MAX_SG_ENTRIES) {
1078 CAAM_JR_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1079 MAX_SG_ENTRIES);
1080 return NULL;
1081 }
1082
1083 ctx = caam_jr_alloc_ctx(ses);
1084 if (!ctx)
1085 return NULL;
1086
1087 ctx->op = op;
1088 cdb = ses->cdb;
1089 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1090
1091 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1092
1093 SEC_JD_INIT(jobdescr);
1094 SEC_JD_SET_SD(jobdescr,
1095 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1096 cdb->sh_hdr.hi.field.idlen);
1097
1098 /* output */
1099 if (sym->m_dst)
1100 mbuf = sym->m_dst;
1101 else
1102 mbuf = sym->m_src;
1103
1104 out_sg = &ctx->sg[0];
1105 if (is_encode(ses))
1106 length = sym->auth.data.length + ses->digest_length;
1107 else
1108 length = sym->auth.data.length;
1109
1110 sg = &ctx->sg[0];
1111
1112 /* 1st seg */
1113 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
1114 + sym->auth.data.offset);
1115 sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
1116
1117 /* Successive segs */
1118 mbuf = mbuf->next;
1119 while (mbuf) {
1120 sg++;
1121 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
1122 sg->len = cpu_to_caam32(mbuf->data_len);
1123 mbuf = mbuf->next;
1124 }
1125
1126 if (is_encode(ses)) {
1127 /* set auth output */
1128 sg++;
1129 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
1130 sg->len = cpu_to_caam32(ses->digest_length);
1131 }
1132 /* last element*/
1133 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1134
1135 SEC_JD_SET_OUT_PTR(jobdescr,
1136 (uint64_t)caam_jr_dma_vtop(out_sg), 0, length);
1137 /* set sg bit */
1138 (jobdescr)->seq_out.command.word |= 0x01000000;
1139
1140 /* input */
1141 sg++;
1142 mbuf = sym->m_src;
1143 in_sg = sg;
1144 if (is_encode(ses))
1145 length = ses->iv.length + sym->auth.data.length;
1146 else
1147 length = ses->iv.length + sym->auth.data.length
1148 + ses->digest_length;
1149
1150 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1151 sg->len = cpu_to_caam32(ses->iv.length);
1152
1153 sg++;
1154 /* 1st seg */
1155 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
1156 + sym->auth.data.offset);
1157 sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
1158
1159 /* Successive segs */
1160 mbuf = mbuf->next;
1161 while (mbuf) {
1162 sg++;
1163 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
1164 sg->len = cpu_to_caam32(mbuf->data_len);
1165 mbuf = mbuf->next;
1166 }
1167
1168 if (is_decode(ses)) {
1169 sg++;
1170 rte_memcpy(ctx->digest, sym->auth.digest.data,
1171 ses->digest_length);
1172 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
1173 sg->len = cpu_to_caam32(ses->digest_length);
1174 }
1175 /* last element*/
1176 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1177
1178 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(in_sg), 0,
1179 length);
1180 /* set sg bit */
1181 (jobdescr)->seq_in.command.word |= 0x01000000;
1182 /* Auth_only_len is set as 0 in descriptor and it is
1183 * overwritten here in the jd which will update
1184 * the DPOVRD reg.
1185 */
1186 if (auth_only_len)
1187 /* set sg bit */
1188 (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
1189
1190 return ctx;
1191 }
1192
1193 static inline struct caam_jr_op_ctx *
build_cipher_auth(struct rte_crypto_op * op,struct caam_jr_session * ses)1194 build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses)
1195 {
1196 struct rte_crypto_sym_op *sym = op->sym;
1197 struct caam_jr_op_ctx *ctx;
1198 struct sec4_sg_entry *sg;
1199 rte_iova_t src_start_addr, dst_start_addr;
1200 uint32_t length = 0;
1201 struct sec_cdb *cdb;
1202 uint64_t sdesc_offset;
1203 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1204 ses->iv.offset);
1205 struct sec_job_descriptor_t *jobdescr;
1206 uint16_t auth_hdr_len = sym->cipher.data.offset -
1207 sym->auth.data.offset;
1208 uint16_t auth_tail_len = sym->auth.data.length -
1209 sym->cipher.data.length - auth_hdr_len;
1210 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
1211
1212 src_start_addr = rte_pktmbuf_iova(sym->m_src);
1213 if (sym->m_dst)
1214 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1215 else
1216 dst_start_addr = src_start_addr;
1217
1218 ctx = caam_jr_alloc_ctx(ses);
1219 if (!ctx)
1220 return NULL;
1221
1222 ctx->op = op;
1223 cdb = ses->cdb;
1224 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1225
1226 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1227
1228 SEC_JD_INIT(jobdescr);
1229 SEC_JD_SET_SD(jobdescr,
1230 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1231 cdb->sh_hdr.hi.field.idlen);
1232
1233 /* input */
1234 sg = &ctx->sg[0];
1235 if (is_encode(ses)) {
1236 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1237 sg->len = cpu_to_caam32(ses->iv.length);
1238 length += ses->iv.length;
1239
1240 sg++;
1241 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
1242 sg->len = cpu_to_caam32(sym->auth.data.length);
1243 length += sym->auth.data.length;
1244 /* last element*/
1245 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1246 } else {
1247 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1248 sg->len = cpu_to_caam32(ses->iv.length);
1249 length += ses->iv.length;
1250
1251 sg++;
1252 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
1253 sg->len = cpu_to_caam32(sym->auth.data.length);
1254 length += sym->auth.data.length;
1255
1256 rte_memcpy(ctx->digest, sym->auth.digest.data,
1257 ses->digest_length);
1258 sg++;
1259 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
1260 sg->len = cpu_to_caam32(ses->digest_length);
1261 length += ses->digest_length;
1262 /* last element*/
1263 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1264 }
1265
1266 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(&ctx->sg[0]), 0,
1267 length);
1268 /* set sg bit */
1269 (jobdescr)->seq_in.command.word |= 0x01000000;
1270
1271 /* output */
1272 sg = &ctx->sg[6];
1273
1274 sg->ptr = cpu_to_caam64(dst_start_addr + sym->cipher.data.offset);
1275 sg->len = cpu_to_caam32(sym->cipher.data.length);
1276 length = sym->cipher.data.length;
1277
1278 if (is_encode(ses)) {
1279 /* set auth output */
1280 sg++;
1281 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
1282 sg->len = cpu_to_caam32(ses->digest_length);
1283 length += ses->digest_length;
1284 }
1285 /* last element*/
1286 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1287
1288 SEC_JD_SET_OUT_PTR(jobdescr,
1289 (uint64_t)caam_jr_dma_vtop(&ctx->sg[6]), 0, length);
1290 /* set sg bit */
1291 (jobdescr)->seq_out.command.word |= 0x01000000;
1292
1293 /* Auth_only_len is set as 0 in descriptor and it is
1294 * overwritten here in the jd which will update
1295 * the DPOVRD reg.
1296 */
1297 if (auth_only_len)
1298 /* set sg bit */
1299 (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
1300
1301 return ctx;
1302 }
1303
1304 static inline struct caam_jr_op_ctx *
build_proto(struct rte_crypto_op * op,struct caam_jr_session * ses)1305 build_proto(struct rte_crypto_op *op, struct caam_jr_session *ses)
1306 {
1307 struct rte_crypto_sym_op *sym = op->sym;
1308 struct caam_jr_op_ctx *ctx = NULL;
1309 phys_addr_t src_start_addr, dst_start_addr;
1310 struct sec_cdb *cdb;
1311 uint64_t sdesc_offset;
1312 struct sec_job_descriptor_t *jobdescr;
1313
1314 ctx = caam_jr_alloc_ctx(ses);
1315 if (!ctx)
1316 return NULL;
1317 ctx->op = op;
1318
1319 src_start_addr = rte_pktmbuf_iova(sym->m_src);
1320 if (sym->m_dst)
1321 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1322 else
1323 dst_start_addr = src_start_addr;
1324
1325 cdb = ses->cdb;
1326 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1327
1328 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1329
1330 SEC_JD_INIT(jobdescr);
1331 SEC_JD_SET_SD(jobdescr,
1332 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1333 cdb->sh_hdr.hi.field.idlen);
1334
1335 /* output */
1336 SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 0,
1337 sym->m_src->buf_len - sym->m_src->data_off);
1338 /* input */
1339 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)src_start_addr, 0,
1340 sym->m_src->pkt_len);
1341 sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1342
1343 return ctx;
1344 }
1345
1346 static int
caam_jr_enqueue_op(struct rte_crypto_op * op,struct caam_jr_qp * qp)1347 caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp)
1348 {
1349 struct sec_job_ring_t *ring = qp->ring;
1350 struct caam_jr_session *ses;
1351 struct caam_jr_op_ctx *ctx = NULL;
1352 struct sec_job_descriptor_t *jobdescr __rte_unused;
1353 #if CAAM_JR_DBG
1354 int i;
1355 #endif
1356
1357 switch (op->sess_type) {
1358 case RTE_CRYPTO_OP_WITH_SESSION:
1359 ses = (struct caam_jr_session *)
1360 get_sym_session_private_data(op->sym->session,
1361 cryptodev_driver_id);
1362 break;
1363 case RTE_CRYPTO_OP_SECURITY_SESSION:
1364 ses = (struct caam_jr_session *)
1365 get_sec_session_private_data(
1366 op->sym->sec_session);
1367 break;
1368 default:
1369 CAAM_JR_DP_ERR("sessionless crypto op not supported");
1370 qp->tx_errs++;
1371 return -1;
1372 }
1373
1374 if (unlikely(!ses->qp || ses->qp != qp)) {
1375 CAAM_JR_DP_DEBUG("Old:sess->qp=%p New qp = %p\n", ses->qp, qp);
1376 ses->qp = qp;
1377 caam_jr_prep_cdb(ses);
1378 }
1379
1380 if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1381 if (is_auth_cipher(ses))
1382 ctx = build_cipher_auth(op, ses);
1383 else if (is_aead(ses))
1384 goto err1;
1385 else if (is_auth_only(ses))
1386 ctx = build_auth_only(op, ses);
1387 else if (is_cipher_only(ses))
1388 ctx = build_cipher_only(op, ses);
1389 else if (is_proto_ipsec(ses))
1390 ctx = build_proto(op, ses);
1391 } else {
1392 if (is_auth_cipher(ses))
1393 ctx = build_cipher_auth_sg(op, ses);
1394 else if (is_aead(ses))
1395 goto err1;
1396 else if (is_auth_only(ses))
1397 ctx = build_auth_only_sg(op, ses);
1398 else if (is_cipher_only(ses))
1399 ctx = build_cipher_only_sg(op, ses);
1400 }
1401 err1:
1402 if (unlikely(!ctx)) {
1403 qp->tx_errs++;
1404 CAAM_JR_ERR("not supported sec op");
1405 return -1;
1406 }
1407 #if CAAM_JR_DBG
1408 if (is_decode(ses))
1409 rte_hexdump(stdout, "DECODE",
1410 rte_pktmbuf_mtod(op->sym->m_src, void *),
1411 rte_pktmbuf_data_len(op->sym->m_src));
1412 else
1413 rte_hexdump(stdout, "ENCODE",
1414 rte_pktmbuf_mtod(op->sym->m_src, void *),
1415 rte_pktmbuf_data_len(op->sym->m_src));
1416
1417 printf("\n JD before conversion\n");
1418 for (i = 0; i < 12; i++)
1419 printf("\n 0x%08x", ctx->jobdes.desc[i]);
1420 #endif
1421
1422 CAAM_JR_DP_DEBUG("Jr[%p] pi[%d] ci[%d].Before sending desc",
1423 ring, ring->pidx, ring->cidx);
1424
1425 /* todo - do we want to retry */
1426 if (SEC_JOB_RING_IS_FULL(ring->pidx, ring->cidx,
1427 SEC_JOB_RING_SIZE, SEC_JOB_RING_SIZE)) {
1428 CAAM_JR_DP_DEBUG("Ring FULL Jr[%p] pi[%d] ci[%d].Size = %d",
1429 ring, ring->pidx, ring->cidx, SEC_JOB_RING_SIZE);
1430 caam_jr_op_ending(ctx);
1431 qp->tx_ring_full++;
1432 return -EBUSY;
1433 }
1434
1435 #if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
1436 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1437
1438 jobdescr->deschdr.command.word =
1439 cpu_to_caam32(jobdescr->deschdr.command.word);
1440 jobdescr->sd_ptr = cpu_to_caam64(jobdescr->sd_ptr);
1441 jobdescr->seq_out.command.word =
1442 cpu_to_caam32(jobdescr->seq_out.command.word);
1443 jobdescr->seq_out_ptr = cpu_to_caam64(jobdescr->seq_out_ptr);
1444 jobdescr->out_ext_length = cpu_to_caam32(jobdescr->out_ext_length);
1445 jobdescr->seq_in.command.word =
1446 cpu_to_caam32(jobdescr->seq_in.command.word);
1447 jobdescr->seq_in_ptr = cpu_to_caam64(jobdescr->seq_in_ptr);
1448 jobdescr->in_ext_length = cpu_to_caam32(jobdescr->in_ext_length);
1449 jobdescr->load_dpovrd.command.word =
1450 cpu_to_caam32(jobdescr->load_dpovrd.command.word);
1451 jobdescr->dpovrd = cpu_to_caam32(jobdescr->dpovrd);
1452 #endif
1453
1454 /* Set ptr in input ring to current descriptor */
1455 sec_write_addr(&ring->input_ring[ring->pidx],
1456 (phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc));
1457 rte_smp_wmb();
1458
1459 /* Notify HW that a new job is enqueued */
1460 hw_enqueue_desc_on_job_ring(ring);
1461
1462 /* increment the producer index for the current job ring */
1463 ring->pidx = SEC_CIRCULAR_COUNTER(ring->pidx, SEC_JOB_RING_SIZE);
1464
1465 return 0;
1466 }
1467
1468 static uint16_t
caam_jr_enqueue_burst(void * qp,struct rte_crypto_op ** ops,uint16_t nb_ops)1469 caam_jr_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1470 uint16_t nb_ops)
1471 {
1472 /* Function to transmit the frames to given device and queuepair */
1473 uint32_t loop;
1474 int32_t ret;
1475 struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
1476 uint16_t num_tx = 0;
1477 /*Prepare each packet which is to be sent*/
1478 for (loop = 0; loop < nb_ops; loop++) {
1479 ret = caam_jr_enqueue_op(ops[loop], jr_qp);
1480 if (!ret)
1481 num_tx++;
1482 }
1483
1484 jr_qp->tx_pkts += num_tx;
1485
1486 return num_tx;
1487 }
1488
1489 /* Release queue pair */
1490 static int
caam_jr_queue_pair_release(struct rte_cryptodev * dev,uint16_t qp_id)1491 caam_jr_queue_pair_release(struct rte_cryptodev *dev,
1492 uint16_t qp_id)
1493 {
1494 struct sec_job_ring_t *internals;
1495 struct caam_jr_qp *qp = NULL;
1496
1497 PMD_INIT_FUNC_TRACE();
1498 CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id);
1499
1500 internals = dev->data->dev_private;
1501 if (qp_id >= internals->max_nb_queue_pairs) {
1502 CAAM_JR_ERR("Max supported qpid %d",
1503 internals->max_nb_queue_pairs);
1504 return -EINVAL;
1505 }
1506
1507 qp = &internals->qps[qp_id];
1508 qp->ring = NULL;
1509 dev->data->queue_pairs[qp_id] = NULL;
1510
1511 return 0;
1512 }
1513
1514 /* Setup a queue pair */
1515 static int
caam_jr_queue_pair_setup(struct rte_cryptodev * dev,uint16_t qp_id,__rte_unused const struct rte_cryptodev_qp_conf * qp_conf,__rte_unused int socket_id)1516 caam_jr_queue_pair_setup(
1517 struct rte_cryptodev *dev, uint16_t qp_id,
1518 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1519 __rte_unused int socket_id)
1520 {
1521 struct sec_job_ring_t *internals;
1522 struct caam_jr_qp *qp = NULL;
1523
1524 PMD_INIT_FUNC_TRACE();
1525 CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
1526
1527 internals = dev->data->dev_private;
1528 if (qp_id >= internals->max_nb_queue_pairs) {
1529 CAAM_JR_ERR("Max supported qpid %d",
1530 internals->max_nb_queue_pairs);
1531 return -EINVAL;
1532 }
1533
1534 qp = &internals->qps[qp_id];
1535 qp->ring = internals;
1536 dev->data->queue_pairs[qp_id] = qp;
1537
1538 return 0;
1539 }
1540
1541 /* Returns the size of the aesni gcm session structure */
1542 static unsigned int
caam_jr_sym_session_get_size(struct rte_cryptodev * dev __rte_unused)1543 caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1544 {
1545 PMD_INIT_FUNC_TRACE();
1546
1547 return sizeof(struct caam_jr_session);
1548 }
1549
1550 static int
caam_jr_cipher_init(struct rte_cryptodev * dev __rte_unused,struct rte_crypto_sym_xform * xform,struct caam_jr_session * session)1551 caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused,
1552 struct rte_crypto_sym_xform *xform,
1553 struct caam_jr_session *session)
1554 {
1555 session->cipher_alg = xform->cipher.algo;
1556 session->iv.length = xform->cipher.iv.length;
1557 session->iv.offset = xform->cipher.iv.offset;
1558 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1559 RTE_CACHE_LINE_SIZE);
1560 if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1561 CAAM_JR_ERR("No Memory for cipher key\n");
1562 return -ENOMEM;
1563 }
1564 session->cipher_key.length = xform->cipher.key.length;
1565
1566 memcpy(session->cipher_key.data, xform->cipher.key.data,
1567 xform->cipher.key.length);
1568 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1569 DIR_ENC : DIR_DEC;
1570
1571 return 0;
1572 }
1573
1574 static int
caam_jr_auth_init(struct rte_cryptodev * dev __rte_unused,struct rte_crypto_sym_xform * xform,struct caam_jr_session * session)1575 caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused,
1576 struct rte_crypto_sym_xform *xform,
1577 struct caam_jr_session *session)
1578 {
1579 session->auth_alg = xform->auth.algo;
1580 session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1581 RTE_CACHE_LINE_SIZE);
1582 if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1583 CAAM_JR_ERR("No Memory for auth key\n");
1584 return -ENOMEM;
1585 }
1586 session->auth_key.length = xform->auth.key.length;
1587 session->digest_length = xform->auth.digest_length;
1588
1589 memcpy(session->auth_key.data, xform->auth.key.data,
1590 xform->auth.key.length);
1591 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1592 DIR_ENC : DIR_DEC;
1593
1594 return 0;
1595 }
1596
1597 static int
caam_jr_aead_init(struct rte_cryptodev * dev __rte_unused,struct rte_crypto_sym_xform * xform,struct caam_jr_session * session)1598 caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused,
1599 struct rte_crypto_sym_xform *xform,
1600 struct caam_jr_session *session)
1601 {
1602 session->aead_alg = xform->aead.algo;
1603 session->iv.length = xform->aead.iv.length;
1604 session->iv.offset = xform->aead.iv.offset;
1605 session->auth_only_len = xform->aead.aad_length;
1606 session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1607 RTE_CACHE_LINE_SIZE);
1608 if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1609 CAAM_JR_ERR("No Memory for aead key\n");
1610 return -ENOMEM;
1611 }
1612 session->aead_key.length = xform->aead.key.length;
1613 session->digest_length = xform->aead.digest_length;
1614
1615 memcpy(session->aead_key.data, xform->aead.key.data,
1616 xform->aead.key.length);
1617 session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1618 DIR_ENC : DIR_DEC;
1619
1620 return 0;
1621 }
1622
1623 static int
caam_jr_set_session_parameters(struct rte_cryptodev * dev,struct rte_crypto_sym_xform * xform,void * sess)1624 caam_jr_set_session_parameters(struct rte_cryptodev *dev,
1625 struct rte_crypto_sym_xform *xform, void *sess)
1626 {
1627 struct sec_job_ring_t *internals = dev->data->dev_private;
1628 struct caam_jr_session *session = sess;
1629
1630 PMD_INIT_FUNC_TRACE();
1631
1632 if (unlikely(sess == NULL)) {
1633 CAAM_JR_ERR("invalid session struct");
1634 return -EINVAL;
1635 }
1636
1637 /* Default IV length = 0 */
1638 session->iv.length = 0;
1639
1640 /* Cipher Only */
1641 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1642 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1643 caam_jr_cipher_init(dev, xform, session);
1644
1645 /* Authentication Only */
1646 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1647 xform->next == NULL) {
1648 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1649 caam_jr_auth_init(dev, xform, session);
1650
1651 /* Cipher then Authenticate */
1652 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1653 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1654 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1655 caam_jr_cipher_init(dev, xform, session);
1656 caam_jr_auth_init(dev, xform->next, session);
1657 } else {
1658 CAAM_JR_ERR("Not supported: Auth then Cipher");
1659 goto err1;
1660 }
1661
1662 /* Authenticate then Cipher */
1663 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1664 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1665 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1666 caam_jr_auth_init(dev, xform, session);
1667 caam_jr_cipher_init(dev, xform->next, session);
1668 } else {
1669 CAAM_JR_ERR("Not supported: Auth then Cipher");
1670 goto err1;
1671 }
1672
1673 /* AEAD operation for AES-GCM kind of Algorithms */
1674 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1675 xform->next == NULL) {
1676 caam_jr_aead_init(dev, xform, session);
1677
1678 } else {
1679 CAAM_JR_ERR("Invalid crypto type");
1680 return -EINVAL;
1681 }
1682 session->ctx_pool = internals->ctx_pool;
1683
1684 return 0;
1685
1686 err1:
1687 rte_free(session->cipher_key.data);
1688 rte_free(session->auth_key.data);
1689 memset(session, 0, sizeof(struct caam_jr_session));
1690
1691 return -EINVAL;
1692 }
1693
1694 static int
caam_jr_sym_session_configure(struct rte_cryptodev * dev,struct rte_crypto_sym_xform * xform,struct rte_cryptodev_sym_session * sess,struct rte_mempool * mempool)1695 caam_jr_sym_session_configure(struct rte_cryptodev *dev,
1696 struct rte_crypto_sym_xform *xform,
1697 struct rte_cryptodev_sym_session *sess,
1698 struct rte_mempool *mempool)
1699 {
1700 void *sess_private_data;
1701 int ret;
1702
1703 PMD_INIT_FUNC_TRACE();
1704
1705 if (rte_mempool_get(mempool, &sess_private_data)) {
1706 CAAM_JR_ERR("Couldn't get object from session mempool");
1707 return -ENOMEM;
1708 }
1709
1710 memset(sess_private_data, 0, sizeof(struct caam_jr_session));
1711 ret = caam_jr_set_session_parameters(dev, xform, sess_private_data);
1712 if (ret != 0) {
1713 CAAM_JR_ERR("failed to configure session parameters");
1714 /* Return session to mempool */
1715 rte_mempool_put(mempool, sess_private_data);
1716 return ret;
1717 }
1718
1719 set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
1720
1721 return 0;
1722 }
1723
1724 /* Clear the memory of session so it doesn't leave key material behind */
1725 static void
caam_jr_sym_session_clear(struct rte_cryptodev * dev,struct rte_cryptodev_sym_session * sess)1726 caam_jr_sym_session_clear(struct rte_cryptodev *dev,
1727 struct rte_cryptodev_sym_session *sess)
1728 {
1729 uint8_t index = dev->driver_id;
1730 void *sess_priv = get_sym_session_private_data(sess, index);
1731 struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
1732
1733 PMD_INIT_FUNC_TRACE();
1734
1735 if (sess_priv) {
1736 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1737
1738 rte_free(s->cipher_key.data);
1739 rte_free(s->auth_key.data);
1740 memset(s, 0, sizeof(struct caam_jr_session));
1741 set_sym_session_private_data(sess, index, NULL);
1742 rte_mempool_put(sess_mp, sess_priv);
1743 }
1744 }
1745
1746 static int
caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev * dev,struct rte_security_session_conf * conf,void * sess)1747 caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1748 struct rte_security_session_conf *conf,
1749 void *sess)
1750 {
1751 struct sec_job_ring_t *internals = dev->data->dev_private;
1752 struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1753 struct rte_crypto_auth_xform *auth_xform;
1754 struct rte_crypto_cipher_xform *cipher_xform;
1755 struct caam_jr_session *session = (struct caam_jr_session *)sess;
1756
1757 PMD_INIT_FUNC_TRACE();
1758
1759 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1760 cipher_xform = &conf->crypto_xform->cipher;
1761 auth_xform = &conf->crypto_xform->next->auth;
1762 } else {
1763 auth_xform = &conf->crypto_xform->auth;
1764 cipher_xform = &conf->crypto_xform->next->cipher;
1765 }
1766 session->proto_alg = conf->protocol;
1767 session->cipher_key.data = rte_zmalloc(NULL,
1768 cipher_xform->key.length,
1769 RTE_CACHE_LINE_SIZE);
1770 if (session->cipher_key.data == NULL &&
1771 cipher_xform->key.length > 0) {
1772 CAAM_JR_ERR("No Memory for cipher key\n");
1773 return -ENOMEM;
1774 }
1775
1776 session->cipher_key.length = cipher_xform->key.length;
1777 session->auth_key.data = rte_zmalloc(NULL,
1778 auth_xform->key.length,
1779 RTE_CACHE_LINE_SIZE);
1780 if (session->auth_key.data == NULL &&
1781 auth_xform->key.length > 0) {
1782 CAAM_JR_ERR("No Memory for auth key\n");
1783 rte_free(session->cipher_key.data);
1784 return -ENOMEM;
1785 }
1786 session->auth_key.length = auth_xform->key.length;
1787 memcpy(session->cipher_key.data, cipher_xform->key.data,
1788 cipher_xform->key.length);
1789 memcpy(session->auth_key.data, auth_xform->key.data,
1790 auth_xform->key.length);
1791
1792 switch (auth_xform->algo) {
1793 case RTE_CRYPTO_AUTH_SHA1_HMAC:
1794 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
1795 break;
1796 case RTE_CRYPTO_AUTH_MD5_HMAC:
1797 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
1798 break;
1799 case RTE_CRYPTO_AUTH_SHA256_HMAC:
1800 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
1801 break;
1802 case RTE_CRYPTO_AUTH_SHA384_HMAC:
1803 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
1804 break;
1805 case RTE_CRYPTO_AUTH_SHA512_HMAC:
1806 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
1807 break;
1808 case RTE_CRYPTO_AUTH_AES_CMAC:
1809 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
1810 break;
1811 case RTE_CRYPTO_AUTH_NULL:
1812 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1813 break;
1814 case RTE_CRYPTO_AUTH_SHA224_HMAC:
1815 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
1816 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
1817 case RTE_CRYPTO_AUTH_SHA1:
1818 case RTE_CRYPTO_AUTH_SHA256:
1819 case RTE_CRYPTO_AUTH_SHA512:
1820 case RTE_CRYPTO_AUTH_SHA224:
1821 case RTE_CRYPTO_AUTH_SHA384:
1822 case RTE_CRYPTO_AUTH_MD5:
1823 case RTE_CRYPTO_AUTH_AES_GMAC:
1824 case RTE_CRYPTO_AUTH_KASUMI_F9:
1825 case RTE_CRYPTO_AUTH_AES_CBC_MAC:
1826 case RTE_CRYPTO_AUTH_ZUC_EIA3:
1827 CAAM_JR_ERR("Crypto: Unsupported auth alg %u\n",
1828 auth_xform->algo);
1829 goto out;
1830 default:
1831 CAAM_JR_ERR("Crypto: Undefined Auth specified %u\n",
1832 auth_xform->algo);
1833 goto out;
1834 }
1835
1836 switch (cipher_xform->algo) {
1837 case RTE_CRYPTO_CIPHER_AES_CBC:
1838 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1839 break;
1840 case RTE_CRYPTO_CIPHER_3DES_CBC:
1841 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1842 break;
1843 case RTE_CRYPTO_CIPHER_AES_CTR:
1844 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1845 break;
1846 case RTE_CRYPTO_CIPHER_NULL:
1847 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1848 case RTE_CRYPTO_CIPHER_3DES_ECB:
1849 case RTE_CRYPTO_CIPHER_AES_ECB:
1850 case RTE_CRYPTO_CIPHER_KASUMI_F8:
1851 CAAM_JR_ERR("Crypto: Unsupported Cipher alg %u\n",
1852 cipher_xform->algo);
1853 goto out;
1854 default:
1855 CAAM_JR_ERR("Crypto: Undefined Cipher specified %u\n",
1856 cipher_xform->algo);
1857 goto out;
1858 }
1859
1860 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1861 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
1862 sizeof(session->ip4_hdr));
1863 session->ip4_hdr.ip_v = IPVERSION;
1864 session->ip4_hdr.ip_hl = 5;
1865 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
1866 sizeof(session->ip4_hdr));
1867 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
1868 session->ip4_hdr.ip_id = 0;
1869 session->ip4_hdr.ip_off = 0;
1870 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
1871 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
1872 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
1873 : IPPROTO_AH;
1874 session->ip4_hdr.ip_sum = 0;
1875 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
1876 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
1877 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
1878 (void *)&session->ip4_hdr,
1879 sizeof(struct ip));
1880
1881 session->encap_pdb.options =
1882 (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
1883 PDBOPTS_ESP_OIHI_PDB_INL |
1884 PDBOPTS_ESP_IVSRC |
1885 PDBHMO_ESP_ENCAP_DTTL;
1886 if (ipsec_xform->options.esn)
1887 session->encap_pdb.options |= PDBOPTS_ESP_ESN;
1888 session->encap_pdb.spi = ipsec_xform->spi;
1889 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
1890
1891 session->dir = DIR_ENC;
1892 } else if (ipsec_xform->direction ==
1893 RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
1894 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
1895 session->decap_pdb.options = sizeof(struct ip) << 16;
1896 if (ipsec_xform->options.esn)
1897 session->decap_pdb.options |= PDBOPTS_ESP_ESN;
1898 session->dir = DIR_DEC;
1899 } else
1900 goto out;
1901 session->ctx_pool = internals->ctx_pool;
1902
1903 return 0;
1904 out:
1905 rte_free(session->auth_key.data);
1906 rte_free(session->cipher_key.data);
1907 memset(session, 0, sizeof(struct caam_jr_session));
1908 return -1;
1909 }
1910
1911 static int
caam_jr_security_session_create(void * dev,struct rte_security_session_conf * conf,struct rte_security_session * sess,struct rte_mempool * mempool)1912 caam_jr_security_session_create(void *dev,
1913 struct rte_security_session_conf *conf,
1914 struct rte_security_session *sess,
1915 struct rte_mempool *mempool)
1916 {
1917 void *sess_private_data;
1918 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
1919 int ret;
1920
1921 if (rte_mempool_get(mempool, &sess_private_data)) {
1922 CAAM_JR_ERR("Couldn't get object from session mempool");
1923 return -ENOMEM;
1924 }
1925
1926 switch (conf->protocol) {
1927 case RTE_SECURITY_PROTOCOL_IPSEC:
1928 ret = caam_jr_set_ipsec_session(cdev, conf,
1929 sess_private_data);
1930 break;
1931 case RTE_SECURITY_PROTOCOL_MACSEC:
1932 return -ENOTSUP;
1933 default:
1934 return -EINVAL;
1935 }
1936 if (ret != 0) {
1937 CAAM_JR_ERR("failed to configure session parameters");
1938 /* Return session to mempool */
1939 rte_mempool_put(mempool, sess_private_data);
1940 return ret;
1941 }
1942
1943 set_sec_session_private_data(sess, sess_private_data);
1944
1945 return ret;
1946 }
1947
1948 /* Clear the memory of session so it doesn't leave key material behind */
1949 static int
caam_jr_security_session_destroy(void * dev __rte_unused,struct rte_security_session * sess)1950 caam_jr_security_session_destroy(void *dev __rte_unused,
1951 struct rte_security_session *sess)
1952 {
1953 PMD_INIT_FUNC_TRACE();
1954 void *sess_priv = get_sec_session_private_data(sess);
1955
1956 struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
1957
1958 if (sess_priv) {
1959 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1960
1961 rte_free(s->cipher_key.data);
1962 rte_free(s->auth_key.data);
1963 memset(sess, 0, sizeof(struct caam_jr_session));
1964 set_sec_session_private_data(sess, NULL);
1965 rte_mempool_put(sess_mp, sess_priv);
1966 }
1967 return 0;
1968 }
1969
1970
1971 static int
caam_jr_dev_configure(struct rte_cryptodev * dev,struct rte_cryptodev_config * config __rte_unused)1972 caam_jr_dev_configure(struct rte_cryptodev *dev,
1973 struct rte_cryptodev_config *config __rte_unused)
1974 {
1975 char str[20];
1976 struct sec_job_ring_t *internals;
1977
1978 PMD_INIT_FUNC_TRACE();
1979
1980 internals = dev->data->dev_private;
1981 snprintf(str, sizeof(str), "ctx_pool_%d", dev->data->dev_id);
1982 if (!internals->ctx_pool) {
1983 internals->ctx_pool = rte_mempool_create((const char *)str,
1984 CTX_POOL_NUM_BUFS,
1985 sizeof(struct caam_jr_op_ctx),
1986 CTX_POOL_CACHE_SIZE, 0,
1987 NULL, NULL, NULL, NULL,
1988 SOCKET_ID_ANY, 0);
1989 if (!internals->ctx_pool) {
1990 CAAM_JR_ERR("%s create failed\n", str);
1991 return -ENOMEM;
1992 }
1993 } else
1994 CAAM_JR_INFO("mempool already created for dev_id : %d",
1995 dev->data->dev_id);
1996
1997 return 0;
1998 }
1999
2000 static int
caam_jr_dev_start(struct rte_cryptodev * dev __rte_unused)2001 caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused)
2002 {
2003 PMD_INIT_FUNC_TRACE();
2004 return 0;
2005 }
2006
2007 static void
caam_jr_dev_stop(struct rte_cryptodev * dev __rte_unused)2008 caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused)
2009 {
2010 PMD_INIT_FUNC_TRACE();
2011 }
2012
2013 static int
caam_jr_dev_close(struct rte_cryptodev * dev)2014 caam_jr_dev_close(struct rte_cryptodev *dev)
2015 {
2016 struct sec_job_ring_t *internals;
2017
2018 PMD_INIT_FUNC_TRACE();
2019
2020 if (dev == NULL)
2021 return -ENOMEM;
2022
2023 internals = dev->data->dev_private;
2024 rte_mempool_free(internals->ctx_pool);
2025 internals->ctx_pool = NULL;
2026
2027 return 0;
2028 }
2029
2030 static void
caam_jr_dev_infos_get(struct rte_cryptodev * dev,struct rte_cryptodev_info * info)2031 caam_jr_dev_infos_get(struct rte_cryptodev *dev,
2032 struct rte_cryptodev_info *info)
2033 {
2034 struct sec_job_ring_t *internals = dev->data->dev_private;
2035
2036 PMD_INIT_FUNC_TRACE();
2037 if (info != NULL) {
2038 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2039 info->feature_flags = dev->feature_flags;
2040 info->capabilities = caam_jr_get_cryptodev_capabilities();
2041 info->sym.max_nb_sessions = internals->max_nb_sessions;
2042 info->driver_id = cryptodev_driver_id;
2043 }
2044 }
2045
2046 static struct rte_cryptodev_ops caam_jr_ops = {
2047 .dev_configure = caam_jr_dev_configure,
2048 .dev_start = caam_jr_dev_start,
2049 .dev_stop = caam_jr_dev_stop,
2050 .dev_close = caam_jr_dev_close,
2051 .dev_infos_get = caam_jr_dev_infos_get,
2052 .stats_get = caam_jr_stats_get,
2053 .stats_reset = caam_jr_stats_reset,
2054 .queue_pair_setup = caam_jr_queue_pair_setup,
2055 .queue_pair_release = caam_jr_queue_pair_release,
2056 .sym_session_get_size = caam_jr_sym_session_get_size,
2057 .sym_session_configure = caam_jr_sym_session_configure,
2058 .sym_session_clear = caam_jr_sym_session_clear
2059 };
2060
2061 static struct rte_security_ops caam_jr_security_ops = {
2062 .session_create = caam_jr_security_session_create,
2063 .session_update = NULL,
2064 .session_stats_get = NULL,
2065 .session_destroy = caam_jr_security_session_destroy,
2066 .set_pkt_metadata = NULL,
2067 .capabilities_get = caam_jr_get_security_capabilities
2068 };
2069
2070 /* @brief Flush job rings of any processed descs.
2071 * The processed descs are silently dropped,
2072 * WITHOUT being notified to UA.
2073 */
2074 static void
close_job_ring(struct sec_job_ring_t * job_ring)2075 close_job_ring(struct sec_job_ring_t *job_ring)
2076 {
2077 if (job_ring->irq_fd != -1) {
2078 /* Producer index is frozen. If consumer index is not equal
2079 * with producer index, then we have descs to flush.
2080 */
2081 while (job_ring->pidx != job_ring->cidx)
2082 hw_flush_job_ring(job_ring, false, NULL);
2083
2084 /* free the uio job ring */
2085 free_job_ring(job_ring->irq_fd);
2086 job_ring->irq_fd = -1;
2087 caam_jr_dma_free(job_ring->input_ring);
2088 caam_jr_dma_free(job_ring->output_ring);
2089 g_job_rings_no--;
2090 }
2091 }
2092
2093 /** @brief Release the software and hardware resources tied to a job ring.
2094 * @param [in] job_ring The job ring
2095 *
2096 * @retval 0 for success
2097 * @retval -1 for error
2098 */
2099 static int
shutdown_job_ring(struct sec_job_ring_t * job_ring)2100 shutdown_job_ring(struct sec_job_ring_t *job_ring)
2101 {
2102 int ret = 0;
2103
2104 PMD_INIT_FUNC_TRACE();
2105 ASSERT(job_ring != NULL);
2106 ret = hw_shutdown_job_ring(job_ring);
2107 SEC_ASSERT(ret == 0, ret,
2108 "Failed to shutdown hardware job ring %p",
2109 job_ring);
2110
2111 if (job_ring->coalescing_en)
2112 hw_job_ring_disable_coalescing(job_ring);
2113
2114 if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
2115 ret = caam_jr_disable_irqs(job_ring->irq_fd);
2116 SEC_ASSERT(ret == 0, ret,
2117 "Failed to disable irqs for job ring %p",
2118 job_ring);
2119 }
2120
2121 return ret;
2122 }
2123
2124 /*
2125 * @brief Release the resources used by the SEC user space driver.
2126 *
2127 * Reset and release SEC's job rings indicated by the User Application at
2128 * init_job_ring() and free any memory allocated internally.
2129 * Call once during application tear down.
2130 *
2131 * @note In case there are any descriptors in-flight (descriptors received by
2132 * SEC driver for processing and for which no response was yet provided to UA),
2133 * the descriptors are discarded without any notifications to User Application.
2134 *
2135 * @retval ::0 is returned for a successful execution
2136 * @retval ::-1 is returned if SEC driver release is in progress
2137 */
2138 static int
caam_jr_dev_uninit(struct rte_cryptodev * dev)2139 caam_jr_dev_uninit(struct rte_cryptodev *dev)
2140 {
2141 struct sec_job_ring_t *internals;
2142
2143 PMD_INIT_FUNC_TRACE();
2144 if (dev == NULL)
2145 return -ENODEV;
2146
2147 internals = dev->data->dev_private;
2148 rte_free(dev->security_ctx);
2149
2150 /* If any descriptors in flight , poll and wait
2151 * until all descriptors are received and silently discarded.
2152 */
2153 if (internals) {
2154 shutdown_job_ring(internals);
2155 close_job_ring(internals);
2156 rte_mempool_free(internals->ctx_pool);
2157 }
2158
2159 CAAM_JR_INFO("Closing crypto device %s", dev->data->name);
2160
2161 /* last caam jr instance) */
2162 if (g_job_rings_no == 0)
2163 g_driver_state = SEC_DRIVER_STATE_IDLE;
2164
2165 return SEC_SUCCESS;
2166 }
2167
2168 /* @brief Initialize the software and hardware resources tied to a job ring.
2169 * @param [in] jr_mode; Model to be used by SEC Driver to receive
2170 * notifications from SEC. Can be either
2171 * of the three: #SEC_NOTIFICATION_TYPE_NAPI
2172 * #SEC_NOTIFICATION_TYPE_IRQ or
2173 * #SEC_NOTIFICATION_TYPE_POLL
2174 * @param [in] NAPI_mode The NAPI work mode to configure a job ring at
2175 * startup. Used only when #SEC_NOTIFICATION_TYPE
2176 * is set to #SEC_NOTIFICATION_TYPE_NAPI.
2177 * @param [in] irq_coalescing_timer This value determines the maximum
2178 * amount of time after processing a
2179 * descriptor before raising an interrupt.
2180 * @param [in] irq_coalescing_count This value determines how many
2181 * descriptors are completed before
2182 * raising an interrupt.
2183 * @param [in] reg_base_addr, The job ring base address register
2184 * @param [in] irq_id The job ring interrupt identification number.
2185 * @retval job_ring_handle for successful job ring configuration
2186 * @retval NULL on error
2187 *
2188 */
2189 static void *
init_job_ring(void * reg_base_addr,int irq_id)2190 init_job_ring(void *reg_base_addr, int irq_id)
2191 {
2192 struct sec_job_ring_t *job_ring = NULL;
2193 int i, ret = 0;
2194 int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
2195 int napi_mode = 0;
2196 int irq_coalescing_timer = 0;
2197 int irq_coalescing_count = 0;
2198
2199 for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
2200 if (g_job_rings[i].irq_fd == -1) {
2201 job_ring = &g_job_rings[i];
2202 g_job_rings_no++;
2203 break;
2204 }
2205 }
2206 if (job_ring == NULL) {
2207 CAAM_JR_ERR("No free job ring\n");
2208 return NULL;
2209 }
2210
2211 job_ring->register_base_addr = reg_base_addr;
2212 job_ring->jr_mode = jr_mode;
2213 job_ring->napi_mode = 0;
2214 job_ring->irq_fd = irq_id;
2215
2216 /* Allocate mem for input and output ring */
2217
2218 /* Allocate memory for input ring */
2219 job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
2220 SEC_DMA_MEM_INPUT_RING_SIZE);
2221 memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
2222
2223 /* Allocate memory for output ring */
2224 job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
2225 SEC_DMA_MEM_OUTPUT_RING_SIZE);
2226 memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
2227
2228 /* Reset job ring in SEC hw and configure job ring registers */
2229 ret = hw_reset_job_ring(job_ring);
2230 if (ret != 0) {
2231 CAAM_JR_ERR("Failed to reset hardware job ring");
2232 goto cleanup;
2233 }
2234
2235 if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
2236 /* When SEC US driver works in NAPI mode, the UA can select
2237 * if the driver starts with IRQs on or off.
2238 */
2239 if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
2240 CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
2241 job_ring);
2242 ret = caam_jr_enable_irqs(job_ring->irq_fd);
2243 if (ret != 0) {
2244 CAAM_JR_ERR("Failed to enable irqs for job ring");
2245 goto cleanup;
2246 }
2247 }
2248 } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
2249 /* When SEC US driver works in pure interrupt mode,
2250 * IRQ's are always enabled.
2251 */
2252 CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
2253 job_ring);
2254 ret = caam_jr_enable_irqs(job_ring->irq_fd);
2255 if (ret != 0) {
2256 CAAM_JR_ERR("Failed to enable irqs for job ring");
2257 goto cleanup;
2258 }
2259 }
2260 if (irq_coalescing_timer || irq_coalescing_count) {
2261 hw_job_ring_set_coalescing_param(job_ring,
2262 irq_coalescing_timer,
2263 irq_coalescing_count);
2264
2265 hw_job_ring_enable_coalescing(job_ring);
2266 job_ring->coalescing_en = 1;
2267 }
2268
2269 job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
2270 job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
2271 job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
2272
2273 return job_ring;
2274 cleanup:
2275 caam_jr_dma_free(job_ring->output_ring);
2276 caam_jr_dma_free(job_ring->input_ring);
2277 return NULL;
2278 }
2279
2280
2281 static int
caam_jr_dev_init(const char * name,struct rte_vdev_device * vdev,struct rte_cryptodev_pmd_init_params * init_params)2282 caam_jr_dev_init(const char *name,
2283 struct rte_vdev_device *vdev,
2284 struct rte_cryptodev_pmd_init_params *init_params)
2285 {
2286 struct rte_cryptodev *dev;
2287 struct rte_security_ctx *security_instance;
2288 struct uio_job_ring *job_ring;
2289 char str[RTE_CRYPTODEV_NAME_MAX_LEN];
2290
2291 PMD_INIT_FUNC_TRACE();
2292
2293 /* Validate driver state */
2294 if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
2295 g_job_rings_max = sec_configure();
2296 if (!g_job_rings_max) {
2297 CAAM_JR_ERR("No job ring detected on UIO !!!!");
2298 return -1;
2299 }
2300 /* Update driver state */
2301 g_driver_state = SEC_DRIVER_STATE_STARTED;
2302 }
2303
2304 if (g_job_rings_no >= g_job_rings_max) {
2305 CAAM_JR_ERR("No more job rings available max=%d!!!!",
2306 g_job_rings_max);
2307 return -1;
2308 }
2309
2310 job_ring = config_job_ring();
2311 if (job_ring == NULL) {
2312 CAAM_JR_ERR("failed to create job ring");
2313 goto init_error;
2314 }
2315
2316 snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
2317
2318 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2319 if (dev == NULL) {
2320 CAAM_JR_ERR("failed to create cryptodev vdev");
2321 goto cleanup;
2322 }
2323 /*TODO free it during teardown*/
2324 dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
2325 job_ring->uio_fd);
2326
2327 if (!dev->data->dev_private) {
2328 CAAM_JR_ERR("Ring memory allocation failed\n");
2329 goto cleanup2;
2330 }
2331
2332 dev->driver_id = cryptodev_driver_id;
2333 dev->dev_ops = &caam_jr_ops;
2334
2335 /* register rx/tx burst functions for data path */
2336 dev->dequeue_burst = caam_jr_dequeue_burst;
2337 dev->enqueue_burst = caam_jr_enqueue_burst;
2338 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2339 RTE_CRYPTODEV_FF_HW_ACCELERATED |
2340 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2341 RTE_CRYPTODEV_FF_SECURITY |
2342 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2343 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
2344 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2345 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
2346 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
2347
2348 /* For secondary processes, we don't initialise any further as primary
2349 * has already done this work. Only check we don't need a different
2350 * RX function
2351 */
2352 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2353 CAAM_JR_WARN("Device already init by primary process");
2354 return 0;
2355 }
2356
2357 /*TODO free it during teardown*/
2358 security_instance = rte_malloc("caam_jr",
2359 sizeof(struct rte_security_ctx), 0);
2360 if (security_instance == NULL) {
2361 CAAM_JR_ERR("memory allocation failed\n");
2362 //todo error handling.
2363 goto cleanup2;
2364 }
2365
2366 security_instance->device = (void *)dev;
2367 security_instance->ops = &caam_jr_security_ops;
2368 security_instance->sess_cnt = 0;
2369 dev->security_ctx = security_instance;
2370
2371 RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);
2372
2373 return 0;
2374
2375 cleanup2:
2376 caam_jr_dev_uninit(dev);
2377 rte_cryptodev_pmd_release_device(dev);
2378 cleanup:
2379 free_job_ring(job_ring->uio_fd);
2380 init_error:
2381 CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
2382 init_params->name);
2383
2384 return -ENXIO;
2385 }
2386
2387 /** Initialise CAAM JR crypto device */
2388 static int
cryptodev_caam_jr_probe(struct rte_vdev_device * vdev)2389 cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
2390 {
2391 int ret;
2392
2393 struct rte_cryptodev_pmd_init_params init_params = {
2394 "",
2395 sizeof(struct sec_job_ring_t),
2396 rte_socket_id(),
2397 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2398 };
2399 const char *name;
2400 const char *input_args;
2401
2402 name = rte_vdev_device_name(vdev);
2403 if (name == NULL)
2404 return -EINVAL;
2405
2406 input_args = rte_vdev_device_args(vdev);
2407 rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
2408
2409 ret = of_init();
2410 if (ret) {
2411 RTE_LOG(ERR, PMD,
2412 "of_init failed\n");
2413 return -EINVAL;
2414 }
2415 /* if sec device version is not configured */
2416 if (!rta_get_sec_era()) {
2417 const struct device_node *caam_node;
2418
2419 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2420 const uint32_t *prop = of_get_property(caam_node,
2421 "fsl,sec-era",
2422 NULL);
2423 if (prop) {
2424 rta_set_sec_era(
2425 INTL_SEC_ERA(rte_be_to_cpu_32(*prop)));
2426 break;
2427 }
2428 }
2429 }
2430 #ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
2431 if (rta_get_sec_era() > RTA_SEC_ERA_8) {
2432 RTE_LOG(ERR, PMD,
2433 "CAAM is compiled in BE mode for device with sec era > 8???\n");
2434 return -EINVAL;
2435 }
2436 #endif
2437
2438 return caam_jr_dev_init(name, vdev, &init_params);
2439 }
2440
2441 /** Uninitialise CAAM JR crypto device */
2442 static int
cryptodev_caam_jr_remove(struct rte_vdev_device * vdev)2443 cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
2444 {
2445 struct rte_cryptodev *cryptodev;
2446 const char *name;
2447
2448 name = rte_vdev_device_name(vdev);
2449 if (name == NULL)
2450 return -EINVAL;
2451
2452 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2453 if (cryptodev == NULL)
2454 return -ENODEV;
2455
2456 caam_jr_dev_uninit(cryptodev);
2457
2458 return rte_cryptodev_pmd_destroy(cryptodev);
2459 }
2460
2461 static void
sec_job_rings_init(void)2462 sec_job_rings_init(void)
2463 {
2464 int i;
2465
2466 for (i = 0; i < MAX_SEC_JOB_RINGS; i++)
2467 g_job_rings[i].irq_fd = -1;
2468 }
2469
2470 static struct rte_vdev_driver cryptodev_caam_jr_drv = {
2471 .probe = cryptodev_caam_jr_probe,
2472 .remove = cryptodev_caam_jr_remove
2473 };
2474
2475 static struct cryptodev_driver caam_jr_crypto_drv;
2476
2477 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
2478 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
2479 "max_nb_queue_pairs=<int>"
2480 "socket_id=<int>");
2481 RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
2482 cryptodev_driver_id);
2483
RTE_INIT(caam_jr_init)2484 RTE_INIT(caam_jr_init)
2485 {
2486 sec_uio_job_rings_init();
2487 sec_job_rings_init();
2488 }
2489
2490 RTE_LOG_REGISTER(caam_jr_logtype, pmd.crypto.caam, NOTICE);
2491