1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2016-2018 NXP 5 * 6 */ 7 8 #include <time.h> 9 #include <net/if.h> 10 11 #include <rte_mbuf.h> 12 #include <rte_cryptodev.h> 13 #include <rte_malloc.h> 14 #include <rte_memcpy.h> 15 #include <rte_string_fns.h> 16 #include <rte_cycles.h> 17 #include <rte_kvargs.h> 18 #include <rte_dev.h> 19 #include <rte_cryptodev_pmd.h> 20 #include <rte_common.h> 21 #include <rte_fslmc.h> 22 #include <fslmc_vfio.h> 23 #include <dpaa2_hw_pvt.h> 24 #include <dpaa2_hw_dpio.h> 25 #include <dpaa2_hw_mempool.h> 26 #include <fsl_dpopr.h> 27 #include <fsl_dpseci.h> 28 #include <fsl_mc_sys.h> 29 30 #include "dpaa2_sec_priv.h" 31 #include "dpaa2_sec_event.h" 32 #include "dpaa2_sec_logs.h" 33 34 /* Required types */ 35 typedef uint64_t dma_addr_t; 36 37 /* RTA header files */ 38 #include <hw/desc/ipsec.h> 39 #include <hw/desc/pdcp.h> 40 #include <hw/desc/algo.h> 41 42 /* Minimum job descriptor consists of a oneword job descriptor HEADER and 43 * a pointer to the shared descriptor 44 */ 45 #define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ) 46 #define FSL_VENDOR_ID 0x1957 47 #define FSL_DEVICE_ID 0x410 48 #define FSL_SUBSYSTEM_SEC 1 49 #define FSL_MC_DPSECI_DEVID 3 50 51 #define NO_PREFETCH 0 52 /* FLE_POOL_NUM_BUFS is set as per the ipsec-secgw application */ 53 #define FLE_POOL_NUM_BUFS 32000 54 #define FLE_POOL_BUF_SIZE 256 55 #define FLE_POOL_CACHE_SIZE 512 56 #define FLE_SG_MEM_SIZE 2048 57 #define SEC_FLC_DHR_OUTBOUND -114 58 #define SEC_FLC_DHR_INBOUND 0 59 60 enum rta_sec_era rta_sec_era = RTA_SEC_ERA_8; 61 62 static uint8_t cryptodev_driver_id; 63 64 int dpaa2_logtype_sec; 65 66 static inline int 67 build_proto_compound_fd(dpaa2_sec_session *sess, 68 struct rte_crypto_op *op, 69 struct qbman_fd *fd, uint16_t bpid) 70 { 71 struct rte_crypto_sym_op *sym_op = op->sym; 72 struct ctxt_priv *priv = sess->ctxt; 73 struct qbman_fle *fle, *ip_fle, *op_fle; 74 struct sec_flow_context *flc; 75 struct rte_mbuf *src_mbuf = sym_op->m_src; 76 struct rte_mbuf *dst_mbuf = sym_op->m_dst; 77 int retval; 78 79 if (!dst_mbuf) 80 dst_mbuf = src_mbuf; 81 82 /* Save the shared descriptor */ 83 flc = &priv->flc_desc[0].flc; 84 85 /* we are using the first FLE entry to store Mbuf */ 86 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); 87 if (retval) { 88 DPAA2_SEC_ERR("Memory alloc failed"); 89 return -1; 90 } 91 memset(fle, 0, FLE_POOL_BUF_SIZE); 92 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 93 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 94 95 op_fle = fle + 1; 96 ip_fle = fle + 2; 97 98 if (likely(bpid < MAX_BPID)) { 99 DPAA2_SET_FD_BPID(fd, bpid); 100 DPAA2_SET_FLE_BPID(op_fle, bpid); 101 DPAA2_SET_FLE_BPID(ip_fle, bpid); 102 } else { 103 DPAA2_SET_FD_IVP(fd); 104 DPAA2_SET_FLE_IVP(op_fle); 105 DPAA2_SET_FLE_IVP(ip_fle); 106 } 107 108 /* Configure FD as a FRAME LIST */ 109 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 110 DPAA2_SET_FD_COMPOUND_FMT(fd); 111 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 112 113 /* Configure Output FLE with dst mbuf data */ 114 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_MBUF_VADDR_TO_IOVA(dst_mbuf)); 115 DPAA2_SET_FLE_OFFSET(op_fle, dst_mbuf->data_off); 116 DPAA2_SET_FLE_LEN(op_fle, dst_mbuf->buf_len); 117 118 /* Configure Input FLE with src mbuf data */ 119 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_MBUF_VADDR_TO_IOVA(src_mbuf)); 120 DPAA2_SET_FLE_OFFSET(ip_fle, src_mbuf->data_off); 121 DPAA2_SET_FLE_LEN(ip_fle, src_mbuf->pkt_len); 122 123 DPAA2_SET_FD_LEN(fd, ip_fle->length); 124 DPAA2_SET_FLE_FIN(ip_fle); 125 126 #ifdef ENABLE_HFN_OVERRIDE 127 if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) { 128 /*enable HFN override override */ 129 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, sess->pdcp.hfn_ovd); 130 DPAA2_SET_FLE_INTERNAL_JD(op_fle, sess->pdcp.hfn_ovd); 131 DPAA2_SET_FD_INTERNAL_JD(fd, sess->pdcp.hfn_ovd); 132 } 133 #endif 134 135 return 0; 136 137 } 138 139 static inline int 140 build_proto_fd(dpaa2_sec_session *sess, 141 struct rte_crypto_op *op, 142 struct qbman_fd *fd, uint16_t bpid) 143 { 144 struct rte_crypto_sym_op *sym_op = op->sym; 145 if (sym_op->m_dst) 146 return build_proto_compound_fd(sess, op, fd, bpid); 147 148 struct ctxt_priv *priv = sess->ctxt; 149 struct sec_flow_context *flc; 150 struct rte_mbuf *mbuf = sym_op->m_src; 151 152 if (likely(bpid < MAX_BPID)) 153 DPAA2_SET_FD_BPID(fd, bpid); 154 else 155 DPAA2_SET_FD_IVP(fd); 156 157 /* Save the shared descriptor */ 158 flc = &priv->flc_desc[0].flc; 159 160 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 161 DPAA2_SET_FD_OFFSET(fd, sym_op->m_src->data_off); 162 DPAA2_SET_FD_LEN(fd, sym_op->m_src->pkt_len); 163 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 164 165 /* save physical address of mbuf */ 166 op->sym->aead.digest.phys_addr = mbuf->buf_iova; 167 mbuf->buf_iova = (size_t)op; 168 169 return 0; 170 } 171 172 static inline int 173 build_authenc_gcm_sg_fd(dpaa2_sec_session *sess, 174 struct rte_crypto_op *op, 175 struct qbman_fd *fd, __rte_unused uint16_t bpid) 176 { 177 struct rte_crypto_sym_op *sym_op = op->sym; 178 struct ctxt_priv *priv = sess->ctxt; 179 struct qbman_fle *fle, *sge, *ip_fle, *op_fle; 180 struct sec_flow_context *flc; 181 uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len; 182 int icv_len = sess->digest_length; 183 uint8_t *old_icv; 184 struct rte_mbuf *mbuf; 185 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 186 sess->iv.offset); 187 188 PMD_INIT_FUNC_TRACE(); 189 190 if (sym_op->m_dst) 191 mbuf = sym_op->m_dst; 192 else 193 mbuf = sym_op->m_src; 194 195 /* first FLE entry used to store mbuf and session ctxt */ 196 fle = (struct qbman_fle *)rte_malloc(NULL, FLE_SG_MEM_SIZE, 197 RTE_CACHE_LINE_SIZE); 198 if (unlikely(!fle)) { 199 DPAA2_SEC_ERR("GCM SG: Memory alloc failed for SGE"); 200 return -1; 201 } 202 memset(fle, 0, FLE_SG_MEM_SIZE); 203 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 204 DPAA2_FLE_SAVE_CTXT(fle, (size_t)priv); 205 206 op_fle = fle + 1; 207 ip_fle = fle + 2; 208 sge = fle + 3; 209 210 /* Save the shared descriptor */ 211 flc = &priv->flc_desc[0].flc; 212 213 /* Configure FD as a FRAME LIST */ 214 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 215 DPAA2_SET_FD_COMPOUND_FMT(fd); 216 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 217 218 DPAA2_SEC_DP_DEBUG("GCM SG: auth_off: 0x%x/length %d, digest-len=%d\n" 219 "iv-len=%d data_off: 0x%x\n", 220 sym_op->aead.data.offset, 221 sym_op->aead.data.length, 222 sess->digest_length, 223 sess->iv.length, 224 sym_op->m_src->data_off); 225 226 /* Configure Output FLE with Scatter/Gather Entry */ 227 DPAA2_SET_FLE_SG_EXT(op_fle); 228 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge)); 229 230 if (auth_only_len) 231 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len); 232 233 op_fle->length = (sess->dir == DIR_ENC) ? 234 (sym_op->aead.data.length + icv_len + auth_only_len) : 235 sym_op->aead.data.length + auth_only_len; 236 237 /* Configure Output SGE for Encap/Decap */ 238 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 239 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + 240 RTE_ALIGN_CEIL(auth_only_len, 16) - auth_only_len); 241 sge->length = mbuf->data_len - sym_op->aead.data.offset + auth_only_len; 242 243 mbuf = mbuf->next; 244 /* o/p segs */ 245 while (mbuf) { 246 sge++; 247 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 248 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 249 sge->length = mbuf->data_len; 250 mbuf = mbuf->next; 251 } 252 sge->length -= icv_len; 253 254 if (sess->dir == DIR_ENC) { 255 sge++; 256 DPAA2_SET_FLE_ADDR(sge, 257 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data)); 258 sge->length = icv_len; 259 } 260 DPAA2_SET_FLE_FIN(sge); 261 262 sge++; 263 mbuf = sym_op->m_src; 264 265 /* Configure Input FLE with Scatter/Gather Entry */ 266 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge)); 267 DPAA2_SET_FLE_SG_EXT(ip_fle); 268 DPAA2_SET_FLE_FIN(ip_fle); 269 ip_fle->length = (sess->dir == DIR_ENC) ? 270 (sym_op->aead.data.length + sess->iv.length + auth_only_len) : 271 (sym_op->aead.data.length + sess->iv.length + auth_only_len + 272 icv_len); 273 274 /* Configure Input SGE for Encap/Decap */ 275 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr)); 276 sge->length = sess->iv.length; 277 278 sge++; 279 if (auth_only_len) { 280 DPAA2_SET_FLE_ADDR(sge, 281 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data)); 282 sge->length = auth_only_len; 283 sge++; 284 } 285 286 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 287 DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + 288 mbuf->data_off); 289 sge->length = mbuf->data_len - sym_op->aead.data.offset; 290 291 mbuf = mbuf->next; 292 /* i/p segs */ 293 while (mbuf) { 294 sge++; 295 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 296 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 297 sge->length = mbuf->data_len; 298 mbuf = mbuf->next; 299 } 300 301 if (sess->dir == DIR_DEC) { 302 sge++; 303 old_icv = (uint8_t *)(sge + 1); 304 memcpy(old_icv, sym_op->aead.digest.data, icv_len); 305 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); 306 sge->length = icv_len; 307 } 308 309 DPAA2_SET_FLE_FIN(sge); 310 if (auth_only_len) { 311 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len); 312 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); 313 } 314 DPAA2_SET_FD_LEN(fd, ip_fle->length); 315 316 return 0; 317 } 318 319 static inline int 320 build_authenc_gcm_fd(dpaa2_sec_session *sess, 321 struct rte_crypto_op *op, 322 struct qbman_fd *fd, uint16_t bpid) 323 { 324 struct rte_crypto_sym_op *sym_op = op->sym; 325 struct ctxt_priv *priv = sess->ctxt; 326 struct qbman_fle *fle, *sge; 327 struct sec_flow_context *flc; 328 uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len; 329 int icv_len = sess->digest_length, retval; 330 uint8_t *old_icv; 331 struct rte_mbuf *dst; 332 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 333 sess->iv.offset); 334 335 PMD_INIT_FUNC_TRACE(); 336 337 if (sym_op->m_dst) 338 dst = sym_op->m_dst; 339 else 340 dst = sym_op->m_src; 341 342 /* TODO we are using the first FLE entry to store Mbuf and session ctxt. 343 * Currently we donot know which FLE has the mbuf stored. 344 * So while retreiving we can go back 1 FLE from the FD -ADDR 345 * to get the MBUF Addr from the previous FLE. 346 * We can have a better approach to use the inline Mbuf 347 */ 348 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); 349 if (retval) { 350 DPAA2_SEC_ERR("GCM: Memory alloc failed for SGE"); 351 return -1; 352 } 353 memset(fle, 0, FLE_POOL_BUF_SIZE); 354 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 355 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 356 fle = fle + 1; 357 sge = fle + 2; 358 if (likely(bpid < MAX_BPID)) { 359 DPAA2_SET_FD_BPID(fd, bpid); 360 DPAA2_SET_FLE_BPID(fle, bpid); 361 DPAA2_SET_FLE_BPID(fle + 1, bpid); 362 DPAA2_SET_FLE_BPID(sge, bpid); 363 DPAA2_SET_FLE_BPID(sge + 1, bpid); 364 DPAA2_SET_FLE_BPID(sge + 2, bpid); 365 DPAA2_SET_FLE_BPID(sge + 3, bpid); 366 } else { 367 DPAA2_SET_FD_IVP(fd); 368 DPAA2_SET_FLE_IVP(fle); 369 DPAA2_SET_FLE_IVP((fle + 1)); 370 DPAA2_SET_FLE_IVP(sge); 371 DPAA2_SET_FLE_IVP((sge + 1)); 372 DPAA2_SET_FLE_IVP((sge + 2)); 373 DPAA2_SET_FLE_IVP((sge + 3)); 374 } 375 376 /* Save the shared descriptor */ 377 flc = &priv->flc_desc[0].flc; 378 /* Configure FD as a FRAME LIST */ 379 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); 380 DPAA2_SET_FD_COMPOUND_FMT(fd); 381 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 382 383 DPAA2_SEC_DP_DEBUG("GCM: auth_off: 0x%x/length %d, digest-len=%d\n" 384 "iv-len=%d data_off: 0x%x\n", 385 sym_op->aead.data.offset, 386 sym_op->aead.data.length, 387 sess->digest_length, 388 sess->iv.length, 389 sym_op->m_src->data_off); 390 391 /* Configure Output FLE with Scatter/Gather Entry */ 392 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 393 if (auth_only_len) 394 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); 395 fle->length = (sess->dir == DIR_ENC) ? 396 (sym_op->aead.data.length + icv_len + auth_only_len) : 397 sym_op->aead.data.length + auth_only_len; 398 399 DPAA2_SET_FLE_SG_EXT(fle); 400 401 /* Configure Output SGE for Encap/Decap */ 402 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst)); 403 DPAA2_SET_FLE_OFFSET(sge, dst->data_off + 404 RTE_ALIGN_CEIL(auth_only_len, 16) - auth_only_len); 405 sge->length = sym_op->aead.data.length + auth_only_len; 406 407 if (sess->dir == DIR_ENC) { 408 sge++; 409 DPAA2_SET_FLE_ADDR(sge, 410 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data)); 411 sge->length = sess->digest_length; 412 DPAA2_SET_FD_LEN(fd, (sym_op->aead.data.length + 413 sess->iv.length + auth_only_len)); 414 } 415 DPAA2_SET_FLE_FIN(sge); 416 417 sge++; 418 fle++; 419 420 /* Configure Input FLE with Scatter/Gather Entry */ 421 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 422 DPAA2_SET_FLE_SG_EXT(fle); 423 DPAA2_SET_FLE_FIN(fle); 424 fle->length = (sess->dir == DIR_ENC) ? 425 (sym_op->aead.data.length + sess->iv.length + auth_only_len) : 426 (sym_op->aead.data.length + sess->iv.length + auth_only_len + 427 sess->digest_length); 428 429 /* Configure Input SGE for Encap/Decap */ 430 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr)); 431 sge->length = sess->iv.length; 432 sge++; 433 if (auth_only_len) { 434 DPAA2_SET_FLE_ADDR(sge, 435 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data)); 436 sge->length = auth_only_len; 437 DPAA2_SET_FLE_BPID(sge, bpid); 438 sge++; 439 } 440 441 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 442 DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + 443 sym_op->m_src->data_off); 444 sge->length = sym_op->aead.data.length; 445 if (sess->dir == DIR_DEC) { 446 sge++; 447 old_icv = (uint8_t *)(sge + 1); 448 memcpy(old_icv, sym_op->aead.digest.data, 449 sess->digest_length); 450 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); 451 sge->length = sess->digest_length; 452 DPAA2_SET_FD_LEN(fd, (sym_op->aead.data.length + 453 sess->digest_length + 454 sess->iv.length + 455 auth_only_len)); 456 } 457 DPAA2_SET_FLE_FIN(sge); 458 459 if (auth_only_len) { 460 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); 461 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); 462 } 463 464 return 0; 465 } 466 467 static inline int 468 build_authenc_sg_fd(dpaa2_sec_session *sess, 469 struct rte_crypto_op *op, 470 struct qbman_fd *fd, __rte_unused uint16_t bpid) 471 { 472 struct rte_crypto_sym_op *sym_op = op->sym; 473 struct ctxt_priv *priv = sess->ctxt; 474 struct qbman_fle *fle, *sge, *ip_fle, *op_fle; 475 struct sec_flow_context *flc; 476 uint32_t auth_only_len = sym_op->auth.data.length - 477 sym_op->cipher.data.length; 478 int icv_len = sess->digest_length; 479 uint8_t *old_icv; 480 struct rte_mbuf *mbuf; 481 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 482 sess->iv.offset); 483 484 PMD_INIT_FUNC_TRACE(); 485 486 if (sym_op->m_dst) 487 mbuf = sym_op->m_dst; 488 else 489 mbuf = sym_op->m_src; 490 491 /* first FLE entry used to store mbuf and session ctxt */ 492 fle = (struct qbman_fle *)rte_malloc(NULL, FLE_SG_MEM_SIZE, 493 RTE_CACHE_LINE_SIZE); 494 if (unlikely(!fle)) { 495 DPAA2_SEC_ERR("AUTHENC SG: Memory alloc failed for SGE"); 496 return -1; 497 } 498 memset(fle, 0, FLE_SG_MEM_SIZE); 499 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 500 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 501 502 op_fle = fle + 1; 503 ip_fle = fle + 2; 504 sge = fle + 3; 505 506 /* Save the shared descriptor */ 507 flc = &priv->flc_desc[0].flc; 508 509 /* Configure FD as a FRAME LIST */ 510 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 511 DPAA2_SET_FD_COMPOUND_FMT(fd); 512 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 513 514 DPAA2_SEC_DP_DEBUG( 515 "AUTHENC SG: auth_off: 0x%x/length %d, digest-len=%d\n" 516 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n", 517 sym_op->auth.data.offset, 518 sym_op->auth.data.length, 519 sess->digest_length, 520 sym_op->cipher.data.offset, 521 sym_op->cipher.data.length, 522 sess->iv.length, 523 sym_op->m_src->data_off); 524 525 /* Configure Output FLE with Scatter/Gather Entry */ 526 DPAA2_SET_FLE_SG_EXT(op_fle); 527 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge)); 528 529 if (auth_only_len) 530 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len); 531 532 op_fle->length = (sess->dir == DIR_ENC) ? 533 (sym_op->cipher.data.length + icv_len) : 534 sym_op->cipher.data.length; 535 536 /* Configure Output SGE for Encap/Decap */ 537 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 538 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->auth.data.offset); 539 sge->length = mbuf->data_len - sym_op->auth.data.offset; 540 541 mbuf = mbuf->next; 542 /* o/p segs */ 543 while (mbuf) { 544 sge++; 545 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 546 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 547 sge->length = mbuf->data_len; 548 mbuf = mbuf->next; 549 } 550 sge->length -= icv_len; 551 552 if (sess->dir == DIR_ENC) { 553 sge++; 554 DPAA2_SET_FLE_ADDR(sge, 555 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); 556 sge->length = icv_len; 557 } 558 DPAA2_SET_FLE_FIN(sge); 559 560 sge++; 561 mbuf = sym_op->m_src; 562 563 /* Configure Input FLE with Scatter/Gather Entry */ 564 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge)); 565 DPAA2_SET_FLE_SG_EXT(ip_fle); 566 DPAA2_SET_FLE_FIN(ip_fle); 567 ip_fle->length = (sess->dir == DIR_ENC) ? 568 (sym_op->auth.data.length + sess->iv.length) : 569 (sym_op->auth.data.length + sess->iv.length + 570 icv_len); 571 572 /* Configure Input SGE for Encap/Decap */ 573 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); 574 sge->length = sess->iv.length; 575 576 sge++; 577 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 578 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset + 579 mbuf->data_off); 580 sge->length = mbuf->data_len - sym_op->auth.data.offset; 581 582 mbuf = mbuf->next; 583 /* i/p segs */ 584 while (mbuf) { 585 sge++; 586 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 587 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 588 sge->length = mbuf->data_len; 589 mbuf = mbuf->next; 590 } 591 sge->length -= icv_len; 592 593 if (sess->dir == DIR_DEC) { 594 sge++; 595 old_icv = (uint8_t *)(sge + 1); 596 memcpy(old_icv, sym_op->auth.digest.data, 597 icv_len); 598 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); 599 sge->length = icv_len; 600 } 601 602 DPAA2_SET_FLE_FIN(sge); 603 if (auth_only_len) { 604 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len); 605 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); 606 } 607 DPAA2_SET_FD_LEN(fd, ip_fle->length); 608 609 return 0; 610 } 611 612 static inline int 613 build_authenc_fd(dpaa2_sec_session *sess, 614 struct rte_crypto_op *op, 615 struct qbman_fd *fd, uint16_t bpid) 616 { 617 struct rte_crypto_sym_op *sym_op = op->sym; 618 struct ctxt_priv *priv = sess->ctxt; 619 struct qbman_fle *fle, *sge; 620 struct sec_flow_context *flc; 621 uint32_t auth_only_len = sym_op->auth.data.length - 622 sym_op->cipher.data.length; 623 int icv_len = sess->digest_length, retval; 624 uint8_t *old_icv; 625 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 626 sess->iv.offset); 627 struct rte_mbuf *dst; 628 629 PMD_INIT_FUNC_TRACE(); 630 631 if (sym_op->m_dst) 632 dst = sym_op->m_dst; 633 else 634 dst = sym_op->m_src; 635 636 /* we are using the first FLE entry to store Mbuf. 637 * Currently we donot know which FLE has the mbuf stored. 638 * So while retreiving we can go back 1 FLE from the FD -ADDR 639 * to get the MBUF Addr from the previous FLE. 640 * We can have a better approach to use the inline Mbuf 641 */ 642 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); 643 if (retval) { 644 DPAA2_SEC_ERR("Memory alloc failed for SGE"); 645 return -1; 646 } 647 memset(fle, 0, FLE_POOL_BUF_SIZE); 648 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 649 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 650 fle = fle + 1; 651 sge = fle + 2; 652 if (likely(bpid < MAX_BPID)) { 653 DPAA2_SET_FD_BPID(fd, bpid); 654 DPAA2_SET_FLE_BPID(fle, bpid); 655 DPAA2_SET_FLE_BPID(fle + 1, bpid); 656 DPAA2_SET_FLE_BPID(sge, bpid); 657 DPAA2_SET_FLE_BPID(sge + 1, bpid); 658 DPAA2_SET_FLE_BPID(sge + 2, bpid); 659 DPAA2_SET_FLE_BPID(sge + 3, bpid); 660 } else { 661 DPAA2_SET_FD_IVP(fd); 662 DPAA2_SET_FLE_IVP(fle); 663 DPAA2_SET_FLE_IVP((fle + 1)); 664 DPAA2_SET_FLE_IVP(sge); 665 DPAA2_SET_FLE_IVP((sge + 1)); 666 DPAA2_SET_FLE_IVP((sge + 2)); 667 DPAA2_SET_FLE_IVP((sge + 3)); 668 } 669 670 /* Save the shared descriptor */ 671 flc = &priv->flc_desc[0].flc; 672 /* Configure FD as a FRAME LIST */ 673 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); 674 DPAA2_SET_FD_COMPOUND_FMT(fd); 675 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 676 677 DPAA2_SEC_DP_DEBUG( 678 "AUTHENC: auth_off: 0x%x/length %d, digest-len=%d\n" 679 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n", 680 sym_op->auth.data.offset, 681 sym_op->auth.data.length, 682 sess->digest_length, 683 sym_op->cipher.data.offset, 684 sym_op->cipher.data.length, 685 sess->iv.length, 686 sym_op->m_src->data_off); 687 688 /* Configure Output FLE with Scatter/Gather Entry */ 689 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 690 if (auth_only_len) 691 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); 692 fle->length = (sess->dir == DIR_ENC) ? 693 (sym_op->cipher.data.length + icv_len) : 694 sym_op->cipher.data.length; 695 696 DPAA2_SET_FLE_SG_EXT(fle); 697 698 /* Configure Output SGE for Encap/Decap */ 699 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst)); 700 DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset + 701 dst->data_off); 702 sge->length = sym_op->cipher.data.length; 703 704 if (sess->dir == DIR_ENC) { 705 sge++; 706 DPAA2_SET_FLE_ADDR(sge, 707 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); 708 sge->length = sess->digest_length; 709 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + 710 sess->iv.length)); 711 } 712 DPAA2_SET_FLE_FIN(sge); 713 714 sge++; 715 fle++; 716 717 /* Configure Input FLE with Scatter/Gather Entry */ 718 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 719 DPAA2_SET_FLE_SG_EXT(fle); 720 DPAA2_SET_FLE_FIN(fle); 721 fle->length = (sess->dir == DIR_ENC) ? 722 (sym_op->auth.data.length + sess->iv.length) : 723 (sym_op->auth.data.length + sess->iv.length + 724 sess->digest_length); 725 726 /* Configure Input SGE for Encap/Decap */ 727 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); 728 sge->length = sess->iv.length; 729 sge++; 730 731 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 732 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset + 733 sym_op->m_src->data_off); 734 sge->length = sym_op->auth.data.length; 735 if (sess->dir == DIR_DEC) { 736 sge++; 737 old_icv = (uint8_t *)(sge + 1); 738 memcpy(old_icv, sym_op->auth.digest.data, 739 sess->digest_length); 740 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); 741 sge->length = sess->digest_length; 742 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + 743 sess->digest_length + 744 sess->iv.length)); 745 } 746 DPAA2_SET_FLE_FIN(sge); 747 if (auth_only_len) { 748 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); 749 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); 750 } 751 return 0; 752 } 753 754 static inline int build_auth_sg_fd( 755 dpaa2_sec_session *sess, 756 struct rte_crypto_op *op, 757 struct qbman_fd *fd, 758 __rte_unused uint16_t bpid) 759 { 760 struct rte_crypto_sym_op *sym_op = op->sym; 761 struct qbman_fle *fle, *sge, *ip_fle, *op_fle; 762 struct sec_flow_context *flc; 763 struct ctxt_priv *priv = sess->ctxt; 764 uint8_t *old_digest; 765 struct rte_mbuf *mbuf; 766 767 PMD_INIT_FUNC_TRACE(); 768 769 mbuf = sym_op->m_src; 770 fle = (struct qbman_fle *)rte_malloc(NULL, FLE_SG_MEM_SIZE, 771 RTE_CACHE_LINE_SIZE); 772 if (unlikely(!fle)) { 773 DPAA2_SEC_ERR("AUTH SG: Memory alloc failed for SGE"); 774 return -1; 775 } 776 memset(fle, 0, FLE_SG_MEM_SIZE); 777 /* first FLE entry used to store mbuf and session ctxt */ 778 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 779 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 780 op_fle = fle + 1; 781 ip_fle = fle + 2; 782 sge = fle + 3; 783 784 flc = &priv->flc_desc[DESC_INITFINAL].flc; 785 /* sg FD */ 786 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 787 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 788 DPAA2_SET_FD_COMPOUND_FMT(fd); 789 790 /* o/p fle */ 791 DPAA2_SET_FLE_ADDR(op_fle, 792 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); 793 op_fle->length = sess->digest_length; 794 795 /* i/p fle */ 796 DPAA2_SET_FLE_SG_EXT(ip_fle); 797 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge)); 798 /* i/p 1st seg */ 799 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 800 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset + mbuf->data_off); 801 sge->length = mbuf->data_len - sym_op->auth.data.offset; 802 803 /* i/p segs */ 804 mbuf = mbuf->next; 805 while (mbuf) { 806 sge++; 807 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 808 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 809 sge->length = mbuf->data_len; 810 mbuf = mbuf->next; 811 } 812 if (sess->dir == DIR_ENC) { 813 /* Digest calculation case */ 814 sge->length -= sess->digest_length; 815 ip_fle->length = sym_op->auth.data.length; 816 } else { 817 /* Digest verification case */ 818 sge++; 819 old_digest = (uint8_t *)(sge + 1); 820 rte_memcpy(old_digest, sym_op->auth.digest.data, 821 sess->digest_length); 822 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest)); 823 sge->length = sess->digest_length; 824 ip_fle->length = sym_op->auth.data.length + 825 sess->digest_length; 826 } 827 DPAA2_SET_FLE_FIN(sge); 828 DPAA2_SET_FLE_FIN(ip_fle); 829 DPAA2_SET_FD_LEN(fd, ip_fle->length); 830 831 return 0; 832 } 833 834 static inline int 835 build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, 836 struct qbman_fd *fd, uint16_t bpid) 837 { 838 struct rte_crypto_sym_op *sym_op = op->sym; 839 struct qbman_fle *fle, *sge; 840 struct sec_flow_context *flc; 841 struct ctxt_priv *priv = sess->ctxt; 842 uint8_t *old_digest; 843 int retval; 844 845 PMD_INIT_FUNC_TRACE(); 846 847 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); 848 if (retval) { 849 DPAA2_SEC_ERR("AUTH Memory alloc failed for SGE"); 850 return -1; 851 } 852 memset(fle, 0, FLE_POOL_BUF_SIZE); 853 /* TODO we are using the first FLE entry to store Mbuf. 854 * Currently we donot know which FLE has the mbuf stored. 855 * So while retreiving we can go back 1 FLE from the FD -ADDR 856 * to get the MBUF Addr from the previous FLE. 857 * We can have a better approach to use the inline Mbuf 858 */ 859 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 860 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 861 fle = fle + 1; 862 863 if (likely(bpid < MAX_BPID)) { 864 DPAA2_SET_FD_BPID(fd, bpid); 865 DPAA2_SET_FLE_BPID(fle, bpid); 866 DPAA2_SET_FLE_BPID(fle + 1, bpid); 867 } else { 868 DPAA2_SET_FD_IVP(fd); 869 DPAA2_SET_FLE_IVP(fle); 870 DPAA2_SET_FLE_IVP((fle + 1)); 871 } 872 flc = &priv->flc_desc[DESC_INITFINAL].flc; 873 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 874 875 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); 876 fle->length = sess->digest_length; 877 878 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); 879 DPAA2_SET_FD_COMPOUND_FMT(fd); 880 fle++; 881 882 if (sess->dir == DIR_ENC) { 883 DPAA2_SET_FLE_ADDR(fle, 884 DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 885 DPAA2_SET_FLE_OFFSET(fle, sym_op->auth.data.offset + 886 sym_op->m_src->data_off); 887 DPAA2_SET_FD_LEN(fd, sym_op->auth.data.length); 888 fle->length = sym_op->auth.data.length; 889 } else { 890 sge = fle + 2; 891 DPAA2_SET_FLE_SG_EXT(fle); 892 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 893 894 if (likely(bpid < MAX_BPID)) { 895 DPAA2_SET_FLE_BPID(sge, bpid); 896 DPAA2_SET_FLE_BPID(sge + 1, bpid); 897 } else { 898 DPAA2_SET_FLE_IVP(sge); 899 DPAA2_SET_FLE_IVP((sge + 1)); 900 } 901 DPAA2_SET_FLE_ADDR(sge, 902 DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 903 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset + 904 sym_op->m_src->data_off); 905 906 DPAA2_SET_FD_LEN(fd, sym_op->auth.data.length + 907 sess->digest_length); 908 sge->length = sym_op->auth.data.length; 909 sge++; 910 old_digest = (uint8_t *)(sge + 1); 911 rte_memcpy(old_digest, sym_op->auth.digest.data, 912 sess->digest_length); 913 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest)); 914 sge->length = sess->digest_length; 915 fle->length = sym_op->auth.data.length + 916 sess->digest_length; 917 DPAA2_SET_FLE_FIN(sge); 918 } 919 DPAA2_SET_FLE_FIN(fle); 920 921 return 0; 922 } 923 924 static int 925 build_cipher_sg_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, 926 struct qbman_fd *fd, __rte_unused uint16_t bpid) 927 { 928 struct rte_crypto_sym_op *sym_op = op->sym; 929 struct qbman_fle *ip_fle, *op_fle, *sge, *fle; 930 struct sec_flow_context *flc; 931 struct ctxt_priv *priv = sess->ctxt; 932 struct rte_mbuf *mbuf; 933 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 934 sess->iv.offset); 935 936 PMD_INIT_FUNC_TRACE(); 937 938 if (sym_op->m_dst) 939 mbuf = sym_op->m_dst; 940 else 941 mbuf = sym_op->m_src; 942 943 fle = (struct qbman_fle *)rte_malloc(NULL, FLE_SG_MEM_SIZE, 944 RTE_CACHE_LINE_SIZE); 945 if (!fle) { 946 DPAA2_SEC_ERR("CIPHER SG: Memory alloc failed for SGE"); 947 return -1; 948 } 949 memset(fle, 0, FLE_SG_MEM_SIZE); 950 /* first FLE entry used to store mbuf and session ctxt */ 951 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 952 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 953 954 op_fle = fle + 1; 955 ip_fle = fle + 2; 956 sge = fle + 3; 957 958 flc = &priv->flc_desc[0].flc; 959 960 DPAA2_SEC_DP_DEBUG( 961 "CIPHER SG: cipher_off: 0x%x/length %d, ivlen=%d" 962 " data_off: 0x%x\n", 963 sym_op->cipher.data.offset, 964 sym_op->cipher.data.length, 965 sess->iv.length, 966 sym_op->m_src->data_off); 967 968 /* o/p fle */ 969 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge)); 970 op_fle->length = sym_op->cipher.data.length; 971 DPAA2_SET_FLE_SG_EXT(op_fle); 972 973 /* o/p 1st seg */ 974 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 975 DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset + mbuf->data_off); 976 sge->length = mbuf->data_len - sym_op->cipher.data.offset; 977 978 mbuf = mbuf->next; 979 /* o/p segs */ 980 while (mbuf) { 981 sge++; 982 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 983 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 984 sge->length = mbuf->data_len; 985 mbuf = mbuf->next; 986 } 987 DPAA2_SET_FLE_FIN(sge); 988 989 DPAA2_SEC_DP_DEBUG( 990 "CIPHER SG: 1 - flc = %p, fle = %p FLEaddr = %x-%x, len %d\n", 991 flc, fle, fle->addr_hi, fle->addr_lo, 992 fle->length); 993 994 /* i/p fle */ 995 mbuf = sym_op->m_src; 996 sge++; 997 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge)); 998 ip_fle->length = sess->iv.length + sym_op->cipher.data.length; 999 DPAA2_SET_FLE_SG_EXT(ip_fle); 1000 1001 /* i/p IV */ 1002 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); 1003 DPAA2_SET_FLE_OFFSET(sge, 0); 1004 sge->length = sess->iv.length; 1005 1006 sge++; 1007 1008 /* i/p 1st seg */ 1009 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 1010 DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset + 1011 mbuf->data_off); 1012 sge->length = mbuf->data_len - sym_op->cipher.data.offset; 1013 1014 mbuf = mbuf->next; 1015 /* i/p segs */ 1016 while (mbuf) { 1017 sge++; 1018 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 1019 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 1020 sge->length = mbuf->data_len; 1021 mbuf = mbuf->next; 1022 } 1023 DPAA2_SET_FLE_FIN(sge); 1024 DPAA2_SET_FLE_FIN(ip_fle); 1025 1026 /* sg fd */ 1027 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 1028 DPAA2_SET_FD_LEN(fd, ip_fle->length); 1029 DPAA2_SET_FD_COMPOUND_FMT(fd); 1030 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 1031 1032 DPAA2_SEC_DP_DEBUG( 1033 "CIPHER SG: fdaddr =%" PRIx64 " bpid =%d meta =%d" 1034 " off =%d, len =%d\n", 1035 DPAA2_GET_FD_ADDR(fd), 1036 DPAA2_GET_FD_BPID(fd), 1037 rte_dpaa2_bpid_info[bpid].meta_data_size, 1038 DPAA2_GET_FD_OFFSET(fd), 1039 DPAA2_GET_FD_LEN(fd)); 1040 return 0; 1041 } 1042 1043 static int 1044 build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, 1045 struct qbman_fd *fd, uint16_t bpid) 1046 { 1047 struct rte_crypto_sym_op *sym_op = op->sym; 1048 struct qbman_fle *fle, *sge; 1049 int retval; 1050 struct sec_flow_context *flc; 1051 struct ctxt_priv *priv = sess->ctxt; 1052 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1053 sess->iv.offset); 1054 struct rte_mbuf *dst; 1055 1056 PMD_INIT_FUNC_TRACE(); 1057 1058 if (sym_op->m_dst) 1059 dst = sym_op->m_dst; 1060 else 1061 dst = sym_op->m_src; 1062 1063 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); 1064 if (retval) { 1065 DPAA2_SEC_ERR("CIPHER: Memory alloc failed for SGE"); 1066 return -1; 1067 } 1068 memset(fle, 0, FLE_POOL_BUF_SIZE); 1069 /* TODO we are using the first FLE entry to store Mbuf. 1070 * Currently we donot know which FLE has the mbuf stored. 1071 * So while retreiving we can go back 1 FLE from the FD -ADDR 1072 * to get the MBUF Addr from the previous FLE. 1073 * We can have a better approach to use the inline Mbuf 1074 */ 1075 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 1076 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 1077 fle = fle + 1; 1078 sge = fle + 2; 1079 1080 if (likely(bpid < MAX_BPID)) { 1081 DPAA2_SET_FD_BPID(fd, bpid); 1082 DPAA2_SET_FLE_BPID(fle, bpid); 1083 DPAA2_SET_FLE_BPID(fle + 1, bpid); 1084 DPAA2_SET_FLE_BPID(sge, bpid); 1085 DPAA2_SET_FLE_BPID(sge + 1, bpid); 1086 } else { 1087 DPAA2_SET_FD_IVP(fd); 1088 DPAA2_SET_FLE_IVP(fle); 1089 DPAA2_SET_FLE_IVP((fle + 1)); 1090 DPAA2_SET_FLE_IVP(sge); 1091 DPAA2_SET_FLE_IVP((sge + 1)); 1092 } 1093 1094 flc = &priv->flc_desc[0].flc; 1095 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); 1096 DPAA2_SET_FD_LEN(fd, sym_op->cipher.data.length + 1097 sess->iv.length); 1098 DPAA2_SET_FD_COMPOUND_FMT(fd); 1099 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 1100 1101 DPAA2_SEC_DP_DEBUG( 1102 "CIPHER: cipher_off: 0x%x/length %d, ivlen=%d," 1103 " data_off: 0x%x\n", 1104 sym_op->cipher.data.offset, 1105 sym_op->cipher.data.length, 1106 sess->iv.length, 1107 sym_op->m_src->data_off); 1108 1109 DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(dst)); 1110 DPAA2_SET_FLE_OFFSET(fle, sym_op->cipher.data.offset + 1111 dst->data_off); 1112 1113 fle->length = sym_op->cipher.data.length + sess->iv.length; 1114 1115 DPAA2_SEC_DP_DEBUG( 1116 "CIPHER: 1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d\n", 1117 flc, fle, fle->addr_hi, fle->addr_lo, 1118 fle->length); 1119 1120 fle++; 1121 1122 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 1123 fle->length = sym_op->cipher.data.length + sess->iv.length; 1124 1125 DPAA2_SET_FLE_SG_EXT(fle); 1126 1127 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); 1128 sge->length = sess->iv.length; 1129 1130 sge++; 1131 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 1132 DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset + 1133 sym_op->m_src->data_off); 1134 1135 sge->length = sym_op->cipher.data.length; 1136 DPAA2_SET_FLE_FIN(sge); 1137 DPAA2_SET_FLE_FIN(fle); 1138 1139 DPAA2_SEC_DP_DEBUG( 1140 "CIPHER: fdaddr =%" PRIx64 " bpid =%d meta =%d" 1141 " off =%d, len =%d\n", 1142 DPAA2_GET_FD_ADDR(fd), 1143 DPAA2_GET_FD_BPID(fd), 1144 rte_dpaa2_bpid_info[bpid].meta_data_size, 1145 DPAA2_GET_FD_OFFSET(fd), 1146 DPAA2_GET_FD_LEN(fd)); 1147 1148 return 0; 1149 } 1150 1151 static inline int 1152 build_sec_fd(struct rte_crypto_op *op, 1153 struct qbman_fd *fd, uint16_t bpid) 1154 { 1155 int ret = -1; 1156 dpaa2_sec_session *sess; 1157 1158 PMD_INIT_FUNC_TRACE(); 1159 1160 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) 1161 sess = (dpaa2_sec_session *)get_sym_session_private_data( 1162 op->sym->session, cryptodev_driver_id); 1163 else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) 1164 sess = (dpaa2_sec_session *)get_sec_session_private_data( 1165 op->sym->sec_session); 1166 else 1167 return -1; 1168 1169 /* Segmented buffer */ 1170 if (unlikely(!rte_pktmbuf_is_contiguous(op->sym->m_src))) { 1171 switch (sess->ctxt_type) { 1172 case DPAA2_SEC_CIPHER: 1173 ret = build_cipher_sg_fd(sess, op, fd, bpid); 1174 break; 1175 case DPAA2_SEC_AUTH: 1176 ret = build_auth_sg_fd(sess, op, fd, bpid); 1177 break; 1178 case DPAA2_SEC_AEAD: 1179 ret = build_authenc_gcm_sg_fd(sess, op, fd, bpid); 1180 break; 1181 case DPAA2_SEC_CIPHER_HASH: 1182 ret = build_authenc_sg_fd(sess, op, fd, bpid); 1183 break; 1184 case DPAA2_SEC_HASH_CIPHER: 1185 default: 1186 DPAA2_SEC_ERR("error: Unsupported session"); 1187 } 1188 } else { 1189 switch (sess->ctxt_type) { 1190 case DPAA2_SEC_CIPHER: 1191 ret = build_cipher_fd(sess, op, fd, bpid); 1192 break; 1193 case DPAA2_SEC_AUTH: 1194 ret = build_auth_fd(sess, op, fd, bpid); 1195 break; 1196 case DPAA2_SEC_AEAD: 1197 ret = build_authenc_gcm_fd(sess, op, fd, bpid); 1198 break; 1199 case DPAA2_SEC_CIPHER_HASH: 1200 ret = build_authenc_fd(sess, op, fd, bpid); 1201 break; 1202 case DPAA2_SEC_IPSEC: 1203 ret = build_proto_fd(sess, op, fd, bpid); 1204 break; 1205 case DPAA2_SEC_PDCP: 1206 ret = build_proto_compound_fd(sess, op, fd, bpid); 1207 break; 1208 case DPAA2_SEC_HASH_CIPHER: 1209 default: 1210 DPAA2_SEC_ERR("error: Unsupported session"); 1211 } 1212 } 1213 return ret; 1214 } 1215 1216 static uint16_t 1217 dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, 1218 uint16_t nb_ops) 1219 { 1220 /* Function to transmit the frames to given device and VQ*/ 1221 uint32_t loop; 1222 int32_t ret; 1223 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1224 uint32_t frames_to_send; 1225 struct qbman_eq_desc eqdesc; 1226 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp; 1227 struct qbman_swp *swp; 1228 uint16_t num_tx = 0; 1229 uint32_t flags[MAX_TX_RING_SLOTS] = {0}; 1230 /*todo - need to support multiple buffer pools */ 1231 uint16_t bpid; 1232 struct rte_mempool *mb_pool; 1233 1234 if (unlikely(nb_ops == 0)) 1235 return 0; 1236 1237 if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 1238 DPAA2_SEC_ERR("sessionless crypto op not supported"); 1239 return 0; 1240 } 1241 /*Prepare enqueue descriptor*/ 1242 qbman_eq_desc_clear(&eqdesc); 1243 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); 1244 qbman_eq_desc_set_response(&eqdesc, 0, 0); 1245 qbman_eq_desc_set_fq(&eqdesc, dpaa2_qp->tx_vq.fqid); 1246 1247 if (!DPAA2_PER_LCORE_DPIO) { 1248 ret = dpaa2_affine_qbman_swp(); 1249 if (ret) { 1250 DPAA2_SEC_ERR("Failure in affining portal"); 1251 return 0; 1252 } 1253 } 1254 swp = DPAA2_PER_LCORE_PORTAL; 1255 1256 while (nb_ops) { 1257 frames_to_send = (nb_ops > dpaa2_eqcr_size) ? 1258 dpaa2_eqcr_size : nb_ops; 1259 1260 for (loop = 0; loop < frames_to_send; loop++) { 1261 if ((*ops)->sym->m_src->seqn) { 1262 uint8_t dqrr_index = (*ops)->sym->m_src->seqn - 1; 1263 1264 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA | dqrr_index; 1265 DPAA2_PER_LCORE_DQRR_SIZE--; 1266 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index); 1267 (*ops)->sym->m_src->seqn = DPAA2_INVALID_MBUF_SEQN; 1268 } 1269 1270 /*Clear the unused FD fields before sending*/ 1271 memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); 1272 mb_pool = (*ops)->sym->m_src->pool; 1273 bpid = mempool_to_bpid(mb_pool); 1274 ret = build_sec_fd(*ops, &fd_arr[loop], bpid); 1275 if (ret) { 1276 DPAA2_SEC_ERR("error: Improper packet contents" 1277 " for crypto operation"); 1278 goto skip_tx; 1279 } 1280 ops++; 1281 } 1282 loop = 0; 1283 while (loop < frames_to_send) { 1284 loop += qbman_swp_enqueue_multiple(swp, &eqdesc, 1285 &fd_arr[loop], 1286 &flags[loop], 1287 frames_to_send - loop); 1288 } 1289 1290 num_tx += frames_to_send; 1291 nb_ops -= frames_to_send; 1292 } 1293 skip_tx: 1294 dpaa2_qp->tx_vq.tx_pkts += num_tx; 1295 dpaa2_qp->tx_vq.err_pkts += nb_ops; 1296 return num_tx; 1297 } 1298 1299 static inline struct rte_crypto_op * 1300 sec_simple_fd_to_mbuf(const struct qbman_fd *fd, __rte_unused uint8_t id) 1301 { 1302 struct rte_crypto_op *op; 1303 uint16_t len = DPAA2_GET_FD_LEN(fd); 1304 uint16_t diff = 0; 1305 dpaa2_sec_session *sess_priv; 1306 1307 struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF( 1308 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)), 1309 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 1310 1311 diff = len - mbuf->pkt_len; 1312 mbuf->pkt_len += diff; 1313 mbuf->data_len += diff; 1314 op = (struct rte_crypto_op *)(size_t)mbuf->buf_iova; 1315 mbuf->buf_iova = op->sym->aead.digest.phys_addr; 1316 op->sym->aead.digest.phys_addr = 0L; 1317 1318 sess_priv = (dpaa2_sec_session *)get_sec_session_private_data( 1319 op->sym->sec_session); 1320 if (sess_priv->dir == DIR_ENC) 1321 mbuf->data_off += SEC_FLC_DHR_OUTBOUND; 1322 else 1323 mbuf->data_off += SEC_FLC_DHR_INBOUND; 1324 1325 return op; 1326 } 1327 1328 static inline struct rte_crypto_op * 1329 sec_fd_to_mbuf(const struct qbman_fd *fd, uint8_t driver_id) 1330 { 1331 struct qbman_fle *fle; 1332 struct rte_crypto_op *op; 1333 struct ctxt_priv *priv; 1334 struct rte_mbuf *dst, *src; 1335 1336 if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single) 1337 return sec_simple_fd_to_mbuf(fd, driver_id); 1338 1339 fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 1340 1341 DPAA2_SEC_DP_DEBUG("FLE addr = %x - %x, offset = %x\n", 1342 fle->addr_hi, fle->addr_lo, fle->fin_bpid_offset); 1343 1344 /* we are using the first FLE entry to store Mbuf. 1345 * Currently we donot know which FLE has the mbuf stored. 1346 * So while retreiving we can go back 1 FLE from the FD -ADDR 1347 * to get the MBUF Addr from the previous FLE. 1348 * We can have a better approach to use the inline Mbuf 1349 */ 1350 1351 if (unlikely(DPAA2_GET_FD_IVP(fd))) { 1352 /* TODO complete it. */ 1353 DPAA2_SEC_ERR("error: non inline buffer"); 1354 return NULL; 1355 } 1356 op = (struct rte_crypto_op *)DPAA2_GET_FLE_ADDR((fle - 1)); 1357 1358 /* Prefeth op */ 1359 src = op->sym->m_src; 1360 rte_prefetch0(src); 1361 1362 if (op->sym->m_dst) { 1363 dst = op->sym->m_dst; 1364 rte_prefetch0(dst); 1365 } else 1366 dst = src; 1367 1368 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 1369 dpaa2_sec_session *sess = (dpaa2_sec_session *) 1370 get_sec_session_private_data(op->sym->sec_session); 1371 if (sess->ctxt_type == DPAA2_SEC_IPSEC) { 1372 uint16_t len = DPAA2_GET_FD_LEN(fd); 1373 dst->pkt_len = len; 1374 dst->data_len = len; 1375 } 1376 } 1377 1378 DPAA2_SEC_DP_DEBUG("mbuf %p BMAN buf addr %p," 1379 " fdaddr =%" PRIx64 " bpid =%d meta =%d off =%d, len =%d\n", 1380 (void *)dst, 1381 dst->buf_addr, 1382 DPAA2_GET_FD_ADDR(fd), 1383 DPAA2_GET_FD_BPID(fd), 1384 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 1385 DPAA2_GET_FD_OFFSET(fd), 1386 DPAA2_GET_FD_LEN(fd)); 1387 1388 /* free the fle memory */ 1389 if (likely(rte_pktmbuf_is_contiguous(src))) { 1390 priv = (struct ctxt_priv *)(size_t)DPAA2_GET_FLE_CTXT(fle - 1); 1391 rte_mempool_put(priv->fle_pool, (void *)(fle-1)); 1392 } else 1393 rte_free((void *)(fle-1)); 1394 1395 return op; 1396 } 1397 1398 static uint16_t 1399 dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, 1400 uint16_t nb_ops) 1401 { 1402 /* Function is responsible to receive frames for a given device and VQ*/ 1403 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp; 1404 struct rte_cryptodev *dev = 1405 (struct rte_cryptodev *)(dpaa2_qp->rx_vq.dev); 1406 struct qbman_result *dq_storage; 1407 uint32_t fqid = dpaa2_qp->rx_vq.fqid; 1408 int ret, num_rx = 0; 1409 uint8_t is_last = 0, status; 1410 struct qbman_swp *swp; 1411 const struct qbman_fd *fd; 1412 struct qbman_pull_desc pulldesc; 1413 1414 if (!DPAA2_PER_LCORE_DPIO) { 1415 ret = dpaa2_affine_qbman_swp(); 1416 if (ret) { 1417 DPAA2_SEC_ERR("Failure in affining portal"); 1418 return 0; 1419 } 1420 } 1421 swp = DPAA2_PER_LCORE_PORTAL; 1422 dq_storage = dpaa2_qp->rx_vq.q_storage->dq_storage[0]; 1423 1424 qbman_pull_desc_clear(&pulldesc); 1425 qbman_pull_desc_set_numframes(&pulldesc, 1426 (nb_ops > dpaa2_dqrr_size) ? 1427 dpaa2_dqrr_size : nb_ops); 1428 qbman_pull_desc_set_fq(&pulldesc, fqid); 1429 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 1430 (dma_addr_t)DPAA2_VADDR_TO_IOVA(dq_storage), 1431 1); 1432 1433 /*Issue a volatile dequeue command. */ 1434 while (1) { 1435 if (qbman_swp_pull(swp, &pulldesc)) { 1436 DPAA2_SEC_WARN( 1437 "SEC VDQ command is not issued : QBMAN busy"); 1438 /* Portal was busy, try again */ 1439 continue; 1440 } 1441 break; 1442 }; 1443 1444 /* Receive the packets till Last Dequeue entry is found with 1445 * respect to the above issues PULL command. 1446 */ 1447 while (!is_last) { 1448 /* Check if the previous issued command is completed. 1449 * Also seems like the SWP is shared between the Ethernet Driver 1450 * and the SEC driver. 1451 */ 1452 while (!qbman_check_command_complete(dq_storage)) 1453 ; 1454 1455 /* Loop until the dq_storage is updated with 1456 * new token by QBMAN 1457 */ 1458 while (!qbman_check_new_result(dq_storage)) 1459 ; 1460 /* Check whether Last Pull command is Expired and 1461 * setting Condition for Loop termination 1462 */ 1463 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 1464 is_last = 1; 1465 /* Check for valid frame. */ 1466 status = (uint8_t)qbman_result_DQ_flags(dq_storage); 1467 if (unlikely( 1468 (status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) { 1469 DPAA2_SEC_DP_DEBUG("No frame is delivered\n"); 1470 continue; 1471 } 1472 } 1473 1474 fd = qbman_result_DQ_fd(dq_storage); 1475 ops[num_rx] = sec_fd_to_mbuf(fd, dev->driver_id); 1476 1477 if (unlikely(fd->simple.frc)) { 1478 /* TODO Parse SEC errors */ 1479 DPAA2_SEC_ERR("SEC returned Error - %x", 1480 fd->simple.frc); 1481 ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_ERROR; 1482 } else { 1483 ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1484 } 1485 1486 num_rx++; 1487 dq_storage++; 1488 } /* End of Packet Rx loop */ 1489 1490 dpaa2_qp->rx_vq.rx_pkts += num_rx; 1491 1492 DPAA2_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx); 1493 /*Return the total number of packets received to DPAA2 app*/ 1494 return num_rx; 1495 } 1496 1497 /** Release queue pair */ 1498 static int 1499 dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) 1500 { 1501 struct dpaa2_sec_qp *qp = 1502 (struct dpaa2_sec_qp *)dev->data->queue_pairs[queue_pair_id]; 1503 1504 PMD_INIT_FUNC_TRACE(); 1505 1506 if (qp->rx_vq.q_storage) { 1507 dpaa2_free_dq_storage(qp->rx_vq.q_storage); 1508 rte_free(qp->rx_vq.q_storage); 1509 } 1510 rte_free(qp); 1511 1512 dev->data->queue_pairs[queue_pair_id] = NULL; 1513 1514 return 0; 1515 } 1516 1517 /** Setup a queue pair */ 1518 static int 1519 dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 1520 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, 1521 __rte_unused int socket_id, 1522 __rte_unused struct rte_mempool *session_pool) 1523 { 1524 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 1525 struct dpaa2_sec_qp *qp; 1526 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 1527 struct dpseci_rx_queue_cfg cfg; 1528 int32_t retcode; 1529 1530 PMD_INIT_FUNC_TRACE(); 1531 1532 /* If qp is already in use free ring memory and qp metadata. */ 1533 if (dev->data->queue_pairs[qp_id] != NULL) { 1534 DPAA2_SEC_INFO("QP already setup"); 1535 return 0; 1536 } 1537 1538 DPAA2_SEC_DEBUG("dev =%p, queue =%d, conf =%p", 1539 dev, qp_id, qp_conf); 1540 1541 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg)); 1542 1543 qp = rte_malloc(NULL, sizeof(struct dpaa2_sec_qp), 1544 RTE_CACHE_LINE_SIZE); 1545 if (!qp) { 1546 DPAA2_SEC_ERR("malloc failed for rx/tx queues"); 1547 return -1; 1548 } 1549 1550 qp->rx_vq.dev = dev; 1551 qp->tx_vq.dev = dev; 1552 qp->rx_vq.q_storage = rte_malloc("sec dq storage", 1553 sizeof(struct queue_storage_info_t), 1554 RTE_CACHE_LINE_SIZE); 1555 if (!qp->rx_vq.q_storage) { 1556 DPAA2_SEC_ERR("malloc failed for q_storage"); 1557 return -1; 1558 } 1559 memset(qp->rx_vq.q_storage, 0, sizeof(struct queue_storage_info_t)); 1560 1561 if (dpaa2_alloc_dq_storage(qp->rx_vq.q_storage)) { 1562 DPAA2_SEC_ERR("Unable to allocate dequeue storage"); 1563 return -1; 1564 } 1565 1566 dev->data->queue_pairs[qp_id] = qp; 1567 1568 cfg.options = cfg.options | DPSECI_QUEUE_OPT_USER_CTX; 1569 cfg.user_ctx = (size_t)(&qp->rx_vq); 1570 retcode = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token, 1571 qp_id, &cfg); 1572 return retcode; 1573 } 1574 1575 /** Return the number of allocated queue pairs */ 1576 static uint32_t 1577 dpaa2_sec_queue_pair_count(struct rte_cryptodev *dev) 1578 { 1579 PMD_INIT_FUNC_TRACE(); 1580 1581 return dev->data->nb_queue_pairs; 1582 } 1583 1584 /** Returns the size of the aesni gcm session structure */ 1585 static unsigned int 1586 dpaa2_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 1587 { 1588 PMD_INIT_FUNC_TRACE(); 1589 1590 return sizeof(dpaa2_sec_session); 1591 } 1592 1593 static int 1594 dpaa2_sec_cipher_init(struct rte_cryptodev *dev, 1595 struct rte_crypto_sym_xform *xform, 1596 dpaa2_sec_session *session) 1597 { 1598 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 1599 struct alginfo cipherdata; 1600 int bufsize, i; 1601 struct ctxt_priv *priv; 1602 struct sec_flow_context *flc; 1603 1604 PMD_INIT_FUNC_TRACE(); 1605 1606 /* For SEC CIPHER only one descriptor is required. */ 1607 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 1608 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), 1609 RTE_CACHE_LINE_SIZE); 1610 if (priv == NULL) { 1611 DPAA2_SEC_ERR("No Memory for priv CTXT"); 1612 return -1; 1613 } 1614 1615 priv->fle_pool = dev_priv->fle_pool; 1616 1617 flc = &priv->flc_desc[0].flc; 1618 1619 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, 1620 RTE_CACHE_LINE_SIZE); 1621 if (session->cipher_key.data == NULL) { 1622 DPAA2_SEC_ERR("No Memory for cipher key"); 1623 rte_free(priv); 1624 return -1; 1625 } 1626 session->cipher_key.length = xform->cipher.key.length; 1627 1628 memcpy(session->cipher_key.data, xform->cipher.key.data, 1629 xform->cipher.key.length); 1630 cipherdata.key = (size_t)session->cipher_key.data; 1631 cipherdata.keylen = session->cipher_key.length; 1632 cipherdata.key_enc_flags = 0; 1633 cipherdata.key_type = RTA_DATA_IMM; 1634 1635 /* Set IV parameters */ 1636 session->iv.offset = xform->cipher.iv.offset; 1637 session->iv.length = xform->cipher.iv.length; 1638 1639 switch (xform->cipher.algo) { 1640 case RTE_CRYPTO_CIPHER_AES_CBC: 1641 cipherdata.algtype = OP_ALG_ALGSEL_AES; 1642 cipherdata.algmode = OP_ALG_AAI_CBC; 1643 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; 1644 break; 1645 case RTE_CRYPTO_CIPHER_3DES_CBC: 1646 cipherdata.algtype = OP_ALG_ALGSEL_3DES; 1647 cipherdata.algmode = OP_ALG_AAI_CBC; 1648 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; 1649 break; 1650 case RTE_CRYPTO_CIPHER_AES_CTR: 1651 cipherdata.algtype = OP_ALG_ALGSEL_AES; 1652 cipherdata.algmode = OP_ALG_AAI_CTR; 1653 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; 1654 break; 1655 case RTE_CRYPTO_CIPHER_3DES_CTR: 1656 case RTE_CRYPTO_CIPHER_AES_ECB: 1657 case RTE_CRYPTO_CIPHER_3DES_ECB: 1658 case RTE_CRYPTO_CIPHER_AES_XTS: 1659 case RTE_CRYPTO_CIPHER_AES_F8: 1660 case RTE_CRYPTO_CIPHER_ARC4: 1661 case RTE_CRYPTO_CIPHER_KASUMI_F8: 1662 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 1663 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 1664 case RTE_CRYPTO_CIPHER_NULL: 1665 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u", 1666 xform->cipher.algo); 1667 goto error_out; 1668 default: 1669 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u", 1670 xform->cipher.algo); 1671 goto error_out; 1672 } 1673 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 1674 DIR_ENC : DIR_DEC; 1675 1676 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0, 1677 &cipherdata, NULL, session->iv.length, 1678 session->dir); 1679 if (bufsize < 0) { 1680 DPAA2_SEC_ERR("Crypto: Descriptor build failed"); 1681 goto error_out; 1682 } 1683 flc->dhr = 0; 1684 flc->bpv0 = 0x1; 1685 flc->mode_bits = 0x8000; 1686 1687 flc->word1_sdl = (uint8_t)bufsize; 1688 flc->word2_rflc_31_0 = lower_32_bits( 1689 (size_t)&(((struct dpaa2_sec_qp *) 1690 dev->data->queue_pairs[0])->rx_vq)); 1691 flc->word3_rflc_63_32 = upper_32_bits( 1692 (size_t)&(((struct dpaa2_sec_qp *) 1693 dev->data->queue_pairs[0])->rx_vq)); 1694 session->ctxt = priv; 1695 1696 for (i = 0; i < bufsize; i++) 1697 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", i, priv->flc_desc[0].desc[i]); 1698 1699 return 0; 1700 1701 error_out: 1702 rte_free(session->cipher_key.data); 1703 rte_free(priv); 1704 return -1; 1705 } 1706 1707 static int 1708 dpaa2_sec_auth_init(struct rte_cryptodev *dev, 1709 struct rte_crypto_sym_xform *xform, 1710 dpaa2_sec_session *session) 1711 { 1712 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 1713 struct alginfo authdata; 1714 int bufsize, i; 1715 struct ctxt_priv *priv; 1716 struct sec_flow_context *flc; 1717 1718 PMD_INIT_FUNC_TRACE(); 1719 1720 /* For SEC AUTH three descriptors are required for various stages */ 1721 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 1722 sizeof(struct ctxt_priv) + 3 * 1723 sizeof(struct sec_flc_desc), 1724 RTE_CACHE_LINE_SIZE); 1725 if (priv == NULL) { 1726 DPAA2_SEC_ERR("No Memory for priv CTXT"); 1727 return -1; 1728 } 1729 1730 priv->fle_pool = dev_priv->fle_pool; 1731 flc = &priv->flc_desc[DESC_INITFINAL].flc; 1732 1733 session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, 1734 RTE_CACHE_LINE_SIZE); 1735 if (session->auth_key.data == NULL) { 1736 DPAA2_SEC_ERR("Unable to allocate memory for auth key"); 1737 rte_free(priv); 1738 return -1; 1739 } 1740 session->auth_key.length = xform->auth.key.length; 1741 1742 memcpy(session->auth_key.data, xform->auth.key.data, 1743 xform->auth.key.length); 1744 authdata.key = (size_t)session->auth_key.data; 1745 authdata.keylen = session->auth_key.length; 1746 authdata.key_enc_flags = 0; 1747 authdata.key_type = RTA_DATA_IMM; 1748 1749 session->digest_length = xform->auth.digest_length; 1750 1751 switch (xform->auth.algo) { 1752 case RTE_CRYPTO_AUTH_SHA1_HMAC: 1753 authdata.algtype = OP_ALG_ALGSEL_SHA1; 1754 authdata.algmode = OP_ALG_AAI_HMAC; 1755 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; 1756 break; 1757 case RTE_CRYPTO_AUTH_MD5_HMAC: 1758 authdata.algtype = OP_ALG_ALGSEL_MD5; 1759 authdata.algmode = OP_ALG_AAI_HMAC; 1760 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; 1761 break; 1762 case RTE_CRYPTO_AUTH_SHA256_HMAC: 1763 authdata.algtype = OP_ALG_ALGSEL_SHA256; 1764 authdata.algmode = OP_ALG_AAI_HMAC; 1765 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; 1766 break; 1767 case RTE_CRYPTO_AUTH_SHA384_HMAC: 1768 authdata.algtype = OP_ALG_ALGSEL_SHA384; 1769 authdata.algmode = OP_ALG_AAI_HMAC; 1770 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; 1771 break; 1772 case RTE_CRYPTO_AUTH_SHA512_HMAC: 1773 authdata.algtype = OP_ALG_ALGSEL_SHA512; 1774 authdata.algmode = OP_ALG_AAI_HMAC; 1775 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; 1776 break; 1777 case RTE_CRYPTO_AUTH_SHA224_HMAC: 1778 authdata.algtype = OP_ALG_ALGSEL_SHA224; 1779 authdata.algmode = OP_ALG_AAI_HMAC; 1780 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC; 1781 break; 1782 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 1783 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 1784 case RTE_CRYPTO_AUTH_NULL: 1785 case RTE_CRYPTO_AUTH_SHA1: 1786 case RTE_CRYPTO_AUTH_SHA256: 1787 case RTE_CRYPTO_AUTH_SHA512: 1788 case RTE_CRYPTO_AUTH_SHA224: 1789 case RTE_CRYPTO_AUTH_SHA384: 1790 case RTE_CRYPTO_AUTH_MD5: 1791 case RTE_CRYPTO_AUTH_AES_GMAC: 1792 case RTE_CRYPTO_AUTH_KASUMI_F9: 1793 case RTE_CRYPTO_AUTH_AES_CMAC: 1794 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 1795 case RTE_CRYPTO_AUTH_ZUC_EIA3: 1796 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %un", 1797 xform->auth.algo); 1798 goto error_out; 1799 default: 1800 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u", 1801 xform->auth.algo); 1802 goto error_out; 1803 } 1804 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? 1805 DIR_ENC : DIR_DEC; 1806 1807 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc, 1808 1, 0, &authdata, !session->dir, 1809 session->digest_length); 1810 if (bufsize < 0) { 1811 DPAA2_SEC_ERR("Crypto: Invalid buffer length"); 1812 goto error_out; 1813 } 1814 1815 flc->word1_sdl = (uint8_t)bufsize; 1816 flc->word2_rflc_31_0 = lower_32_bits( 1817 (size_t)&(((struct dpaa2_sec_qp *) 1818 dev->data->queue_pairs[0])->rx_vq)); 1819 flc->word3_rflc_63_32 = upper_32_bits( 1820 (size_t)&(((struct dpaa2_sec_qp *) 1821 dev->data->queue_pairs[0])->rx_vq)); 1822 session->ctxt = priv; 1823 for (i = 0; i < bufsize; i++) 1824 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", 1825 i, priv->flc_desc[DESC_INITFINAL].desc[i]); 1826 1827 1828 return 0; 1829 1830 error_out: 1831 rte_free(session->auth_key.data); 1832 rte_free(priv); 1833 return -1; 1834 } 1835 1836 static int 1837 dpaa2_sec_aead_init(struct rte_cryptodev *dev, 1838 struct rte_crypto_sym_xform *xform, 1839 dpaa2_sec_session *session) 1840 { 1841 struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; 1842 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 1843 struct alginfo aeaddata; 1844 int bufsize, i; 1845 struct ctxt_priv *priv; 1846 struct sec_flow_context *flc; 1847 struct rte_crypto_aead_xform *aead_xform = &xform->aead; 1848 int err; 1849 1850 PMD_INIT_FUNC_TRACE(); 1851 1852 /* Set IV parameters */ 1853 session->iv.offset = aead_xform->iv.offset; 1854 session->iv.length = aead_xform->iv.length; 1855 session->ctxt_type = DPAA2_SEC_AEAD; 1856 1857 /* For SEC AEAD only one descriptor is required */ 1858 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 1859 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), 1860 RTE_CACHE_LINE_SIZE); 1861 if (priv == NULL) { 1862 DPAA2_SEC_ERR("No Memory for priv CTXT"); 1863 return -1; 1864 } 1865 1866 priv->fle_pool = dev_priv->fle_pool; 1867 flc = &priv->flc_desc[0].flc; 1868 1869 session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length, 1870 RTE_CACHE_LINE_SIZE); 1871 if (session->aead_key.data == NULL && aead_xform->key.length > 0) { 1872 DPAA2_SEC_ERR("No Memory for aead key"); 1873 rte_free(priv); 1874 return -1; 1875 } 1876 memcpy(session->aead_key.data, aead_xform->key.data, 1877 aead_xform->key.length); 1878 1879 session->digest_length = aead_xform->digest_length; 1880 session->aead_key.length = aead_xform->key.length; 1881 ctxt->auth_only_len = aead_xform->aad_length; 1882 1883 aeaddata.key = (size_t)session->aead_key.data; 1884 aeaddata.keylen = session->aead_key.length; 1885 aeaddata.key_enc_flags = 0; 1886 aeaddata.key_type = RTA_DATA_IMM; 1887 1888 switch (aead_xform->algo) { 1889 case RTE_CRYPTO_AEAD_AES_GCM: 1890 aeaddata.algtype = OP_ALG_ALGSEL_AES; 1891 aeaddata.algmode = OP_ALG_AAI_GCM; 1892 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM; 1893 break; 1894 case RTE_CRYPTO_AEAD_AES_CCM: 1895 DPAA2_SEC_ERR("Crypto: Unsupported AEAD alg %u", 1896 aead_xform->algo); 1897 goto error_out; 1898 default: 1899 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u", 1900 aead_xform->algo); 1901 goto error_out; 1902 } 1903 session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 1904 DIR_ENC : DIR_DEC; 1905 1906 priv->flc_desc[0].desc[0] = aeaddata.keylen; 1907 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, 1908 MIN_JOB_DESC_SIZE, 1909 (unsigned int *)priv->flc_desc[0].desc, 1910 &priv->flc_desc[0].desc[1], 1); 1911 1912 if (err < 0) { 1913 DPAA2_SEC_ERR("Crypto: Incorrect key lengths"); 1914 goto error_out; 1915 } 1916 if (priv->flc_desc[0].desc[1] & 1) { 1917 aeaddata.key_type = RTA_DATA_IMM; 1918 } else { 1919 aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key); 1920 aeaddata.key_type = RTA_DATA_PTR; 1921 } 1922 priv->flc_desc[0].desc[0] = 0; 1923 priv->flc_desc[0].desc[1] = 0; 1924 1925 if (session->dir == DIR_ENC) 1926 bufsize = cnstr_shdsc_gcm_encap( 1927 priv->flc_desc[0].desc, 1, 0, 1928 &aeaddata, session->iv.length, 1929 session->digest_length); 1930 else 1931 bufsize = cnstr_shdsc_gcm_decap( 1932 priv->flc_desc[0].desc, 1, 0, 1933 &aeaddata, session->iv.length, 1934 session->digest_length); 1935 if (bufsize < 0) { 1936 DPAA2_SEC_ERR("Crypto: Invalid buffer length"); 1937 goto error_out; 1938 } 1939 1940 flc->word1_sdl = (uint8_t)bufsize; 1941 flc->word2_rflc_31_0 = lower_32_bits( 1942 (size_t)&(((struct dpaa2_sec_qp *) 1943 dev->data->queue_pairs[0])->rx_vq)); 1944 flc->word3_rflc_63_32 = upper_32_bits( 1945 (size_t)&(((struct dpaa2_sec_qp *) 1946 dev->data->queue_pairs[0])->rx_vq)); 1947 session->ctxt = priv; 1948 for (i = 0; i < bufsize; i++) 1949 DPAA2_SEC_DEBUG("DESC[%d]:0x%x\n", 1950 i, priv->flc_desc[0].desc[i]); 1951 1952 return 0; 1953 1954 error_out: 1955 rte_free(session->aead_key.data); 1956 rte_free(priv); 1957 return -1; 1958 } 1959 1960 1961 static int 1962 dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev, 1963 struct rte_crypto_sym_xform *xform, 1964 dpaa2_sec_session *session) 1965 { 1966 struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; 1967 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 1968 struct alginfo authdata, cipherdata; 1969 int bufsize, i; 1970 struct ctxt_priv *priv; 1971 struct sec_flow_context *flc; 1972 struct rte_crypto_cipher_xform *cipher_xform; 1973 struct rte_crypto_auth_xform *auth_xform; 1974 int err; 1975 1976 PMD_INIT_FUNC_TRACE(); 1977 1978 if (session->ext_params.aead_ctxt.auth_cipher_text) { 1979 cipher_xform = &xform->cipher; 1980 auth_xform = &xform->next->auth; 1981 session->ctxt_type = 1982 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 1983 DPAA2_SEC_CIPHER_HASH : DPAA2_SEC_HASH_CIPHER; 1984 } else { 1985 cipher_xform = &xform->next->cipher; 1986 auth_xform = &xform->auth; 1987 session->ctxt_type = 1988 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 1989 DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH; 1990 } 1991 1992 /* Set IV parameters */ 1993 session->iv.offset = cipher_xform->iv.offset; 1994 session->iv.length = cipher_xform->iv.length; 1995 1996 /* For SEC AEAD only one descriptor is required */ 1997 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 1998 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), 1999 RTE_CACHE_LINE_SIZE); 2000 if (priv == NULL) { 2001 DPAA2_SEC_ERR("No Memory for priv CTXT"); 2002 return -1; 2003 } 2004 2005 priv->fle_pool = dev_priv->fle_pool; 2006 flc = &priv->flc_desc[0].flc; 2007 2008 session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length, 2009 RTE_CACHE_LINE_SIZE); 2010 if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) { 2011 DPAA2_SEC_ERR("No Memory for cipher key"); 2012 rte_free(priv); 2013 return -1; 2014 } 2015 session->cipher_key.length = cipher_xform->key.length; 2016 session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length, 2017 RTE_CACHE_LINE_SIZE); 2018 if (session->auth_key.data == NULL && auth_xform->key.length > 0) { 2019 DPAA2_SEC_ERR("No Memory for auth key"); 2020 rte_free(session->cipher_key.data); 2021 rte_free(priv); 2022 return -1; 2023 } 2024 session->auth_key.length = auth_xform->key.length; 2025 memcpy(session->cipher_key.data, cipher_xform->key.data, 2026 cipher_xform->key.length); 2027 memcpy(session->auth_key.data, auth_xform->key.data, 2028 auth_xform->key.length); 2029 2030 authdata.key = (size_t)session->auth_key.data; 2031 authdata.keylen = session->auth_key.length; 2032 authdata.key_enc_flags = 0; 2033 authdata.key_type = RTA_DATA_IMM; 2034 2035 session->digest_length = auth_xform->digest_length; 2036 2037 switch (auth_xform->algo) { 2038 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2039 authdata.algtype = OP_ALG_ALGSEL_SHA1; 2040 authdata.algmode = OP_ALG_AAI_HMAC; 2041 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; 2042 break; 2043 case RTE_CRYPTO_AUTH_MD5_HMAC: 2044 authdata.algtype = OP_ALG_ALGSEL_MD5; 2045 authdata.algmode = OP_ALG_AAI_HMAC; 2046 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; 2047 break; 2048 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2049 authdata.algtype = OP_ALG_ALGSEL_SHA224; 2050 authdata.algmode = OP_ALG_AAI_HMAC; 2051 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC; 2052 break; 2053 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2054 authdata.algtype = OP_ALG_ALGSEL_SHA256; 2055 authdata.algmode = OP_ALG_AAI_HMAC; 2056 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; 2057 break; 2058 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2059 authdata.algtype = OP_ALG_ALGSEL_SHA384; 2060 authdata.algmode = OP_ALG_AAI_HMAC; 2061 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; 2062 break; 2063 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2064 authdata.algtype = OP_ALG_ALGSEL_SHA512; 2065 authdata.algmode = OP_ALG_AAI_HMAC; 2066 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; 2067 break; 2068 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2069 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2070 case RTE_CRYPTO_AUTH_NULL: 2071 case RTE_CRYPTO_AUTH_SHA1: 2072 case RTE_CRYPTO_AUTH_SHA256: 2073 case RTE_CRYPTO_AUTH_SHA512: 2074 case RTE_CRYPTO_AUTH_SHA224: 2075 case RTE_CRYPTO_AUTH_SHA384: 2076 case RTE_CRYPTO_AUTH_MD5: 2077 case RTE_CRYPTO_AUTH_AES_GMAC: 2078 case RTE_CRYPTO_AUTH_KASUMI_F9: 2079 case RTE_CRYPTO_AUTH_AES_CMAC: 2080 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2081 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2082 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u", 2083 auth_xform->algo); 2084 goto error_out; 2085 default: 2086 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u", 2087 auth_xform->algo); 2088 goto error_out; 2089 } 2090 cipherdata.key = (size_t)session->cipher_key.data; 2091 cipherdata.keylen = session->cipher_key.length; 2092 cipherdata.key_enc_flags = 0; 2093 cipherdata.key_type = RTA_DATA_IMM; 2094 2095 switch (cipher_xform->algo) { 2096 case RTE_CRYPTO_CIPHER_AES_CBC: 2097 cipherdata.algtype = OP_ALG_ALGSEL_AES; 2098 cipherdata.algmode = OP_ALG_AAI_CBC; 2099 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; 2100 break; 2101 case RTE_CRYPTO_CIPHER_3DES_CBC: 2102 cipherdata.algtype = OP_ALG_ALGSEL_3DES; 2103 cipherdata.algmode = OP_ALG_AAI_CBC; 2104 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; 2105 break; 2106 case RTE_CRYPTO_CIPHER_AES_CTR: 2107 cipherdata.algtype = OP_ALG_ALGSEL_AES; 2108 cipherdata.algmode = OP_ALG_AAI_CTR; 2109 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; 2110 break; 2111 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2112 case RTE_CRYPTO_CIPHER_NULL: 2113 case RTE_CRYPTO_CIPHER_3DES_ECB: 2114 case RTE_CRYPTO_CIPHER_AES_ECB: 2115 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2116 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u", 2117 cipher_xform->algo); 2118 goto error_out; 2119 default: 2120 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u", 2121 cipher_xform->algo); 2122 goto error_out; 2123 } 2124 session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 2125 DIR_ENC : DIR_DEC; 2126 2127 priv->flc_desc[0].desc[0] = cipherdata.keylen; 2128 priv->flc_desc[0].desc[1] = authdata.keylen; 2129 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, 2130 MIN_JOB_DESC_SIZE, 2131 (unsigned int *)priv->flc_desc[0].desc, 2132 &priv->flc_desc[0].desc[2], 2); 2133 2134 if (err < 0) { 2135 DPAA2_SEC_ERR("Crypto: Incorrect key lengths"); 2136 goto error_out; 2137 } 2138 if (priv->flc_desc[0].desc[2] & 1) { 2139 cipherdata.key_type = RTA_DATA_IMM; 2140 } else { 2141 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key); 2142 cipherdata.key_type = RTA_DATA_PTR; 2143 } 2144 if (priv->flc_desc[0].desc[2] & (1 << 1)) { 2145 authdata.key_type = RTA_DATA_IMM; 2146 } else { 2147 authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key); 2148 authdata.key_type = RTA_DATA_PTR; 2149 } 2150 priv->flc_desc[0].desc[0] = 0; 2151 priv->flc_desc[0].desc[1] = 0; 2152 priv->flc_desc[0].desc[2] = 0; 2153 2154 if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) { 2155 bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1, 2156 0, &cipherdata, &authdata, 2157 session->iv.length, 2158 ctxt->auth_only_len, 2159 session->digest_length, 2160 session->dir); 2161 if (bufsize < 0) { 2162 DPAA2_SEC_ERR("Crypto: Invalid buffer length"); 2163 goto error_out; 2164 } 2165 } else { 2166 DPAA2_SEC_ERR("Hash before cipher not supported"); 2167 goto error_out; 2168 } 2169 2170 flc->word1_sdl = (uint8_t)bufsize; 2171 flc->word2_rflc_31_0 = lower_32_bits( 2172 (size_t)&(((struct dpaa2_sec_qp *) 2173 dev->data->queue_pairs[0])->rx_vq)); 2174 flc->word3_rflc_63_32 = upper_32_bits( 2175 (size_t)&(((struct dpaa2_sec_qp *) 2176 dev->data->queue_pairs[0])->rx_vq)); 2177 session->ctxt = priv; 2178 for (i = 0; i < bufsize; i++) 2179 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", 2180 i, priv->flc_desc[0].desc[i]); 2181 2182 return 0; 2183 2184 error_out: 2185 rte_free(session->cipher_key.data); 2186 rte_free(session->auth_key.data); 2187 rte_free(priv); 2188 return -1; 2189 } 2190 2191 static int 2192 dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev, 2193 struct rte_crypto_sym_xform *xform, void *sess) 2194 { 2195 dpaa2_sec_session *session = sess; 2196 int ret; 2197 2198 PMD_INIT_FUNC_TRACE(); 2199 2200 if (unlikely(sess == NULL)) { 2201 DPAA2_SEC_ERR("Invalid session struct"); 2202 return -1; 2203 } 2204 2205 memset(session, 0, sizeof(dpaa2_sec_session)); 2206 /* Default IV length = 0 */ 2207 session->iv.length = 0; 2208 2209 /* Cipher Only */ 2210 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { 2211 session->ctxt_type = DPAA2_SEC_CIPHER; 2212 ret = dpaa2_sec_cipher_init(dev, xform, session); 2213 2214 /* Authentication Only */ 2215 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 2216 xform->next == NULL) { 2217 session->ctxt_type = DPAA2_SEC_AUTH; 2218 ret = dpaa2_sec_auth_init(dev, xform, session); 2219 2220 /* Cipher then Authenticate */ 2221 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 2222 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 2223 session->ext_params.aead_ctxt.auth_cipher_text = true; 2224 ret = dpaa2_sec_aead_chain_init(dev, xform, session); 2225 2226 /* Authenticate then Cipher */ 2227 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 2228 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 2229 session->ext_params.aead_ctxt.auth_cipher_text = false; 2230 ret = dpaa2_sec_aead_chain_init(dev, xform, session); 2231 2232 /* AEAD operation for AES-GCM kind of Algorithms */ 2233 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && 2234 xform->next == NULL) { 2235 ret = dpaa2_sec_aead_init(dev, xform, session); 2236 2237 } else { 2238 DPAA2_SEC_ERR("Invalid crypto type"); 2239 return -EINVAL; 2240 } 2241 2242 return ret; 2243 } 2244 2245 static int 2246 dpaa2_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform, 2247 dpaa2_sec_session *session, 2248 struct alginfo *aeaddata) 2249 { 2250 PMD_INIT_FUNC_TRACE(); 2251 2252 session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length, 2253 RTE_CACHE_LINE_SIZE); 2254 if (session->aead_key.data == NULL && aead_xform->key.length > 0) { 2255 DPAA2_SEC_ERR("No Memory for aead key"); 2256 return -1; 2257 } 2258 memcpy(session->aead_key.data, aead_xform->key.data, 2259 aead_xform->key.length); 2260 2261 session->digest_length = aead_xform->digest_length; 2262 session->aead_key.length = aead_xform->key.length; 2263 2264 aeaddata->key = (size_t)session->aead_key.data; 2265 aeaddata->keylen = session->aead_key.length; 2266 aeaddata->key_enc_flags = 0; 2267 aeaddata->key_type = RTA_DATA_IMM; 2268 2269 switch (aead_xform->algo) { 2270 case RTE_CRYPTO_AEAD_AES_GCM: 2271 aeaddata->algtype = OP_ALG_ALGSEL_AES; 2272 aeaddata->algmode = OP_ALG_AAI_GCM; 2273 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM; 2274 break; 2275 case RTE_CRYPTO_AEAD_AES_CCM: 2276 aeaddata->algtype = OP_ALG_ALGSEL_AES; 2277 aeaddata->algmode = OP_ALG_AAI_CCM; 2278 session->aead_alg = RTE_CRYPTO_AEAD_AES_CCM; 2279 break; 2280 default: 2281 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u", 2282 aead_xform->algo); 2283 return -1; 2284 } 2285 session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 2286 DIR_ENC : DIR_DEC; 2287 2288 return 0; 2289 } 2290 2291 static int 2292 dpaa2_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform, 2293 struct rte_crypto_auth_xform *auth_xform, 2294 dpaa2_sec_session *session, 2295 struct alginfo *cipherdata, 2296 struct alginfo *authdata) 2297 { 2298 if (cipher_xform) { 2299 session->cipher_key.data = rte_zmalloc(NULL, 2300 cipher_xform->key.length, 2301 RTE_CACHE_LINE_SIZE); 2302 if (session->cipher_key.data == NULL && 2303 cipher_xform->key.length > 0) { 2304 DPAA2_SEC_ERR("No Memory for cipher key"); 2305 return -ENOMEM; 2306 } 2307 2308 session->cipher_key.length = cipher_xform->key.length; 2309 memcpy(session->cipher_key.data, cipher_xform->key.data, 2310 cipher_xform->key.length); 2311 session->cipher_alg = cipher_xform->algo; 2312 } else { 2313 session->cipher_key.data = NULL; 2314 session->cipher_key.length = 0; 2315 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; 2316 } 2317 2318 if (auth_xform) { 2319 session->auth_key.data = rte_zmalloc(NULL, 2320 auth_xform->key.length, 2321 RTE_CACHE_LINE_SIZE); 2322 if (session->auth_key.data == NULL && 2323 auth_xform->key.length > 0) { 2324 DPAA2_SEC_ERR("No Memory for auth key"); 2325 return -ENOMEM; 2326 } 2327 session->auth_key.length = auth_xform->key.length; 2328 memcpy(session->auth_key.data, auth_xform->key.data, 2329 auth_xform->key.length); 2330 session->auth_alg = auth_xform->algo; 2331 } else { 2332 session->auth_key.data = NULL; 2333 session->auth_key.length = 0; 2334 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 2335 } 2336 2337 authdata->key = (size_t)session->auth_key.data; 2338 authdata->keylen = session->auth_key.length; 2339 authdata->key_enc_flags = 0; 2340 authdata->key_type = RTA_DATA_IMM; 2341 switch (session->auth_alg) { 2342 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2343 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA1_96; 2344 authdata->algmode = OP_ALG_AAI_HMAC; 2345 break; 2346 case RTE_CRYPTO_AUTH_MD5_HMAC: 2347 authdata->algtype = OP_PCL_IPSEC_HMAC_MD5_96; 2348 authdata->algmode = OP_ALG_AAI_HMAC; 2349 break; 2350 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2351 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128; 2352 authdata->algmode = OP_ALG_AAI_HMAC; 2353 break; 2354 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2355 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192; 2356 authdata->algmode = OP_ALG_AAI_HMAC; 2357 break; 2358 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2359 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256; 2360 authdata->algmode = OP_ALG_AAI_HMAC; 2361 break; 2362 case RTE_CRYPTO_AUTH_AES_CMAC: 2363 authdata->algtype = OP_PCL_IPSEC_AES_CMAC_96; 2364 break; 2365 case RTE_CRYPTO_AUTH_NULL: 2366 authdata->algtype = OP_PCL_IPSEC_HMAC_NULL; 2367 break; 2368 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2369 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2370 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2371 case RTE_CRYPTO_AUTH_SHA1: 2372 case RTE_CRYPTO_AUTH_SHA256: 2373 case RTE_CRYPTO_AUTH_SHA512: 2374 case RTE_CRYPTO_AUTH_SHA224: 2375 case RTE_CRYPTO_AUTH_SHA384: 2376 case RTE_CRYPTO_AUTH_MD5: 2377 case RTE_CRYPTO_AUTH_AES_GMAC: 2378 case RTE_CRYPTO_AUTH_KASUMI_F9: 2379 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2380 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2381 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u", 2382 session->auth_alg); 2383 return -1; 2384 default: 2385 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u", 2386 session->auth_alg); 2387 return -1; 2388 } 2389 cipherdata->key = (size_t)session->cipher_key.data; 2390 cipherdata->keylen = session->cipher_key.length; 2391 cipherdata->key_enc_flags = 0; 2392 cipherdata->key_type = RTA_DATA_IMM; 2393 2394 switch (session->cipher_alg) { 2395 case RTE_CRYPTO_CIPHER_AES_CBC: 2396 cipherdata->algtype = OP_PCL_IPSEC_AES_CBC; 2397 cipherdata->algmode = OP_ALG_AAI_CBC; 2398 break; 2399 case RTE_CRYPTO_CIPHER_3DES_CBC: 2400 cipherdata->algtype = OP_PCL_IPSEC_3DES; 2401 cipherdata->algmode = OP_ALG_AAI_CBC; 2402 break; 2403 case RTE_CRYPTO_CIPHER_AES_CTR: 2404 cipherdata->algtype = OP_PCL_IPSEC_AES_CTR; 2405 cipherdata->algmode = OP_ALG_AAI_CTR; 2406 break; 2407 case RTE_CRYPTO_CIPHER_NULL: 2408 cipherdata->algtype = OP_PCL_IPSEC_NULL; 2409 break; 2410 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2411 case RTE_CRYPTO_CIPHER_3DES_ECB: 2412 case RTE_CRYPTO_CIPHER_AES_ECB: 2413 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2414 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u", 2415 session->cipher_alg); 2416 return -1; 2417 default: 2418 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u", 2419 session->cipher_alg); 2420 return -1; 2421 } 2422 2423 return 0; 2424 } 2425 2426 #ifdef RTE_LIBRTE_SECURITY_TEST 2427 static uint8_t aes_cbc_iv[] = { 2428 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 2429 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }; 2430 #endif 2431 2432 static int 2433 dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev, 2434 struct rte_security_session_conf *conf, 2435 void *sess) 2436 { 2437 struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; 2438 struct rte_crypto_cipher_xform *cipher_xform = NULL; 2439 struct rte_crypto_auth_xform *auth_xform = NULL; 2440 struct rte_crypto_aead_xform *aead_xform = NULL; 2441 dpaa2_sec_session *session = (dpaa2_sec_session *)sess; 2442 struct ctxt_priv *priv; 2443 struct ipsec_encap_pdb encap_pdb; 2444 struct ipsec_decap_pdb decap_pdb; 2445 struct alginfo authdata, cipherdata; 2446 int bufsize; 2447 struct sec_flow_context *flc; 2448 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 2449 int ret = -1; 2450 2451 PMD_INIT_FUNC_TRACE(); 2452 2453 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 2454 sizeof(struct ctxt_priv) + 2455 sizeof(struct sec_flc_desc), 2456 RTE_CACHE_LINE_SIZE); 2457 2458 if (priv == NULL) { 2459 DPAA2_SEC_ERR("No memory for priv CTXT"); 2460 return -ENOMEM; 2461 } 2462 2463 priv->fle_pool = dev_priv->fle_pool; 2464 flc = &priv->flc_desc[0].flc; 2465 2466 memset(session, 0, sizeof(dpaa2_sec_session)); 2467 2468 if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 2469 cipher_xform = &conf->crypto_xform->cipher; 2470 if (conf->crypto_xform->next) 2471 auth_xform = &conf->crypto_xform->next->auth; 2472 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform, 2473 session, &cipherdata, &authdata); 2474 } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 2475 auth_xform = &conf->crypto_xform->auth; 2476 if (conf->crypto_xform->next) 2477 cipher_xform = &conf->crypto_xform->next->cipher; 2478 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform, 2479 session, &cipherdata, &authdata); 2480 } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { 2481 aead_xform = &conf->crypto_xform->aead; 2482 ret = dpaa2_sec_ipsec_aead_init(aead_xform, 2483 session, &cipherdata); 2484 } else { 2485 DPAA2_SEC_ERR("XFORM not specified"); 2486 ret = -EINVAL; 2487 goto out; 2488 } 2489 if (ret) { 2490 DPAA2_SEC_ERR("Failed to process xform"); 2491 goto out; 2492 } 2493 2494 session->ctxt_type = DPAA2_SEC_IPSEC; 2495 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 2496 struct ip ip4_hdr; 2497 2498 flc->dhr = SEC_FLC_DHR_OUTBOUND; 2499 ip4_hdr.ip_v = IPVERSION; 2500 ip4_hdr.ip_hl = 5; 2501 ip4_hdr.ip_len = rte_cpu_to_be_16(sizeof(ip4_hdr)); 2502 ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; 2503 ip4_hdr.ip_id = 0; 2504 ip4_hdr.ip_off = 0; 2505 ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; 2506 ip4_hdr.ip_p = IPPROTO_ESP; 2507 ip4_hdr.ip_sum = 0; 2508 ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip; 2509 ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip; 2510 ip4_hdr.ip_sum = calc_chksum((uint16_t *)(void *)&ip4_hdr, 2511 sizeof(struct ip)); 2512 2513 /* For Sec Proto only one descriptor is required. */ 2514 memset(&encap_pdb, 0, sizeof(struct ipsec_encap_pdb)); 2515 encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | 2516 PDBOPTS_ESP_OIHI_PDB_INL | 2517 PDBOPTS_ESP_IVSRC | 2518 PDBHMO_ESP_ENCAP_DTTL | 2519 PDBHMO_ESP_SNR; 2520 encap_pdb.spi = ipsec_xform->spi; 2521 encap_pdb.ip_hdr_len = sizeof(struct ip); 2522 2523 session->dir = DIR_ENC; 2524 bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc, 2525 1, 0, SHR_SERIAL, &encap_pdb, 2526 (uint8_t *)&ip4_hdr, 2527 &cipherdata, &authdata); 2528 } else if (ipsec_xform->direction == 2529 RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { 2530 flc->dhr = SEC_FLC_DHR_INBOUND; 2531 memset(&decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); 2532 decap_pdb.options = sizeof(struct ip) << 16; 2533 session->dir = DIR_DEC; 2534 bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc, 2535 1, 0, SHR_SERIAL, 2536 &decap_pdb, &cipherdata, &authdata); 2537 } else 2538 goto out; 2539 2540 if (bufsize < 0) { 2541 DPAA2_SEC_ERR("Crypto: Invalid buffer length"); 2542 goto out; 2543 } 2544 2545 flc->word1_sdl = (uint8_t)bufsize; 2546 2547 /* Enable the stashing control bit */ 2548 DPAA2_SET_FLC_RSC(flc); 2549 flc->word2_rflc_31_0 = lower_32_bits( 2550 (size_t)&(((struct dpaa2_sec_qp *) 2551 dev->data->queue_pairs[0])->rx_vq) | 0x14); 2552 flc->word3_rflc_63_32 = upper_32_bits( 2553 (size_t)&(((struct dpaa2_sec_qp *) 2554 dev->data->queue_pairs[0])->rx_vq)); 2555 2556 /* Set EWS bit i.e. enable write-safe */ 2557 DPAA2_SET_FLC_EWS(flc); 2558 /* Set BS = 1 i.e reuse input buffers as output buffers */ 2559 DPAA2_SET_FLC_REUSE_BS(flc); 2560 /* Set FF = 10; reuse input buffers if they provide sufficient space */ 2561 DPAA2_SET_FLC_REUSE_FF(flc); 2562 2563 session->ctxt = priv; 2564 2565 return 0; 2566 out: 2567 rte_free(session->auth_key.data); 2568 rte_free(session->cipher_key.data); 2569 rte_free(priv); 2570 return ret; 2571 } 2572 2573 static int 2574 dpaa2_sec_set_pdcp_session(struct rte_cryptodev *dev, 2575 struct rte_security_session_conf *conf, 2576 void *sess) 2577 { 2578 struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp; 2579 struct rte_crypto_sym_xform *xform = conf->crypto_xform; 2580 struct rte_crypto_auth_xform *auth_xform = NULL; 2581 struct rte_crypto_cipher_xform *cipher_xform; 2582 dpaa2_sec_session *session = (dpaa2_sec_session *)sess; 2583 struct ctxt_priv *priv; 2584 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 2585 struct alginfo authdata, cipherdata; 2586 int bufsize = -1; 2587 struct sec_flow_context *flc; 2588 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 2589 int swap = true; 2590 #else 2591 int swap = false; 2592 #endif 2593 2594 PMD_INIT_FUNC_TRACE(); 2595 2596 memset(session, 0, sizeof(dpaa2_sec_session)); 2597 2598 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 2599 sizeof(struct ctxt_priv) + 2600 sizeof(struct sec_flc_desc), 2601 RTE_CACHE_LINE_SIZE); 2602 2603 if (priv == NULL) { 2604 DPAA2_SEC_ERR("No memory for priv CTXT"); 2605 return -ENOMEM; 2606 } 2607 2608 priv->fle_pool = dev_priv->fle_pool; 2609 flc = &priv->flc_desc[0].flc; 2610 2611 /* find xfrm types */ 2612 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { 2613 cipher_xform = &xform->cipher; 2614 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 2615 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 2616 session->ext_params.aead_ctxt.auth_cipher_text = true; 2617 cipher_xform = &xform->cipher; 2618 auth_xform = &xform->next->auth; 2619 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 2620 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 2621 session->ext_params.aead_ctxt.auth_cipher_text = false; 2622 cipher_xform = &xform->next->cipher; 2623 auth_xform = &xform->auth; 2624 } else { 2625 DPAA2_SEC_ERR("Invalid crypto type"); 2626 return -EINVAL; 2627 } 2628 2629 session->ctxt_type = DPAA2_SEC_PDCP; 2630 if (cipher_xform) { 2631 session->cipher_key.data = rte_zmalloc(NULL, 2632 cipher_xform->key.length, 2633 RTE_CACHE_LINE_SIZE); 2634 if (session->cipher_key.data == NULL && 2635 cipher_xform->key.length > 0) { 2636 DPAA2_SEC_ERR("No Memory for cipher key"); 2637 rte_free(priv); 2638 return -ENOMEM; 2639 } 2640 session->cipher_key.length = cipher_xform->key.length; 2641 memcpy(session->cipher_key.data, cipher_xform->key.data, 2642 cipher_xform->key.length); 2643 session->dir = 2644 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 2645 DIR_ENC : DIR_DEC; 2646 session->cipher_alg = cipher_xform->algo; 2647 } else { 2648 session->cipher_key.data = NULL; 2649 session->cipher_key.length = 0; 2650 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; 2651 session->dir = DIR_ENC; 2652 } 2653 2654 session->pdcp.domain = pdcp_xform->domain; 2655 session->pdcp.bearer = pdcp_xform->bearer; 2656 session->pdcp.pkt_dir = pdcp_xform->pkt_dir; 2657 session->pdcp.sn_size = pdcp_xform->sn_size; 2658 #ifdef ENABLE_HFN_OVERRIDE 2659 session->pdcp.hfn_ovd = pdcp_xform->hfn_ovd; 2660 #endif 2661 session->pdcp.hfn = pdcp_xform->hfn; 2662 session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold; 2663 2664 cipherdata.key = (size_t)session->cipher_key.data; 2665 cipherdata.keylen = session->cipher_key.length; 2666 cipherdata.key_enc_flags = 0; 2667 cipherdata.key_type = RTA_DATA_IMM; 2668 2669 switch (session->cipher_alg) { 2670 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2671 cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW; 2672 break; 2673 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2674 cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC; 2675 break; 2676 case RTE_CRYPTO_CIPHER_AES_CTR: 2677 cipherdata.algtype = PDCP_CIPHER_TYPE_AES; 2678 break; 2679 case RTE_CRYPTO_CIPHER_NULL: 2680 cipherdata.algtype = PDCP_CIPHER_TYPE_NULL; 2681 break; 2682 default: 2683 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u", 2684 session->cipher_alg); 2685 goto out; 2686 } 2687 2688 /* Auth is only applicable for control mode operation. */ 2689 if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) { 2690 if (pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_5) { 2691 DPAA2_SEC_ERR( 2692 "PDCP Seq Num size should be 5 bits for cmode"); 2693 goto out; 2694 } 2695 if (auth_xform) { 2696 session->auth_key.data = rte_zmalloc(NULL, 2697 auth_xform->key.length, 2698 RTE_CACHE_LINE_SIZE); 2699 if (session->auth_key.data == NULL && 2700 auth_xform->key.length > 0) { 2701 DPAA2_SEC_ERR("No Memory for auth key"); 2702 rte_free(session->cipher_key.data); 2703 rte_free(priv); 2704 return -ENOMEM; 2705 } 2706 session->auth_key.length = auth_xform->key.length; 2707 memcpy(session->auth_key.data, auth_xform->key.data, 2708 auth_xform->key.length); 2709 session->auth_alg = auth_xform->algo; 2710 } else { 2711 session->auth_key.data = NULL; 2712 session->auth_key.length = 0; 2713 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 2714 } 2715 authdata.key = (size_t)session->auth_key.data; 2716 authdata.keylen = session->auth_key.length; 2717 authdata.key_enc_flags = 0; 2718 authdata.key_type = RTA_DATA_IMM; 2719 2720 switch (session->auth_alg) { 2721 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2722 authdata.algtype = PDCP_AUTH_TYPE_SNOW; 2723 break; 2724 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2725 authdata.algtype = PDCP_AUTH_TYPE_ZUC; 2726 break; 2727 case RTE_CRYPTO_AUTH_AES_CMAC: 2728 authdata.algtype = PDCP_AUTH_TYPE_AES; 2729 break; 2730 case RTE_CRYPTO_AUTH_NULL: 2731 authdata.algtype = PDCP_AUTH_TYPE_NULL; 2732 break; 2733 default: 2734 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u", 2735 session->auth_alg); 2736 goto out; 2737 } 2738 2739 if (session->dir == DIR_ENC) 2740 bufsize = cnstr_shdsc_pdcp_c_plane_encap( 2741 priv->flc_desc[0].desc, 1, swap, 2742 pdcp_xform->hfn, 2743 pdcp_xform->bearer, 2744 pdcp_xform->pkt_dir, 2745 pdcp_xform->hfn_threshold, 2746 &cipherdata, &authdata, 2747 0); 2748 else if (session->dir == DIR_DEC) 2749 bufsize = cnstr_shdsc_pdcp_c_plane_decap( 2750 priv->flc_desc[0].desc, 1, swap, 2751 pdcp_xform->hfn, 2752 pdcp_xform->bearer, 2753 pdcp_xform->pkt_dir, 2754 pdcp_xform->hfn_threshold, 2755 &cipherdata, &authdata, 2756 0); 2757 } else { 2758 if (session->dir == DIR_ENC) 2759 bufsize = cnstr_shdsc_pdcp_u_plane_encap( 2760 priv->flc_desc[0].desc, 1, swap, 2761 (enum pdcp_sn_size)pdcp_xform->sn_size, 2762 pdcp_xform->hfn, 2763 pdcp_xform->bearer, 2764 pdcp_xform->pkt_dir, 2765 pdcp_xform->hfn_threshold, 2766 &cipherdata, 0); 2767 else if (session->dir == DIR_DEC) 2768 bufsize = cnstr_shdsc_pdcp_u_plane_decap( 2769 priv->flc_desc[0].desc, 1, swap, 2770 (enum pdcp_sn_size)pdcp_xform->sn_size, 2771 pdcp_xform->hfn, 2772 pdcp_xform->bearer, 2773 pdcp_xform->pkt_dir, 2774 pdcp_xform->hfn_threshold, 2775 &cipherdata, 0); 2776 } 2777 2778 if (bufsize < 0) { 2779 DPAA2_SEC_ERR("Crypto: Invalid buffer length"); 2780 goto out; 2781 } 2782 2783 /* Enable the stashing control bit */ 2784 DPAA2_SET_FLC_RSC(flc); 2785 flc->word2_rflc_31_0 = lower_32_bits( 2786 (size_t)&(((struct dpaa2_sec_qp *) 2787 dev->data->queue_pairs[0])->rx_vq) | 0x14); 2788 flc->word3_rflc_63_32 = upper_32_bits( 2789 (size_t)&(((struct dpaa2_sec_qp *) 2790 dev->data->queue_pairs[0])->rx_vq)); 2791 2792 flc->word1_sdl = (uint8_t)bufsize; 2793 2794 /* Set EWS bit i.e. enable write-safe */ 2795 DPAA2_SET_FLC_EWS(flc); 2796 /* Set BS = 1 i.e reuse input buffers as output buffers */ 2797 DPAA2_SET_FLC_REUSE_BS(flc); 2798 /* Set FF = 10; reuse input buffers if they provide sufficient space */ 2799 DPAA2_SET_FLC_REUSE_FF(flc); 2800 2801 session->ctxt = priv; 2802 2803 return 0; 2804 out: 2805 rte_free(session->auth_key.data); 2806 rte_free(session->cipher_key.data); 2807 rte_free(priv); 2808 return -1; 2809 } 2810 2811 static int 2812 dpaa2_sec_security_session_create(void *dev, 2813 struct rte_security_session_conf *conf, 2814 struct rte_security_session *sess, 2815 struct rte_mempool *mempool) 2816 { 2817 void *sess_private_data; 2818 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; 2819 int ret; 2820 2821 if (rte_mempool_get(mempool, &sess_private_data)) { 2822 DPAA2_SEC_ERR("Couldn't get object from session mempool"); 2823 return -ENOMEM; 2824 } 2825 2826 switch (conf->protocol) { 2827 case RTE_SECURITY_PROTOCOL_IPSEC: 2828 ret = dpaa2_sec_set_ipsec_session(cdev, conf, 2829 sess_private_data); 2830 break; 2831 case RTE_SECURITY_PROTOCOL_MACSEC: 2832 return -ENOTSUP; 2833 case RTE_SECURITY_PROTOCOL_PDCP: 2834 ret = dpaa2_sec_set_pdcp_session(cdev, conf, 2835 sess_private_data); 2836 break; 2837 default: 2838 return -EINVAL; 2839 } 2840 if (ret != 0) { 2841 DPAA2_SEC_ERR("Failed to configure session parameters"); 2842 /* Return session to mempool */ 2843 rte_mempool_put(mempool, sess_private_data); 2844 return ret; 2845 } 2846 2847 set_sec_session_private_data(sess, sess_private_data); 2848 2849 return ret; 2850 } 2851 2852 /** Clear the memory of session so it doesn't leave key material behind */ 2853 static int 2854 dpaa2_sec_security_session_destroy(void *dev __rte_unused, 2855 struct rte_security_session *sess) 2856 { 2857 PMD_INIT_FUNC_TRACE(); 2858 void *sess_priv = get_sec_session_private_data(sess); 2859 2860 dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; 2861 2862 if (sess_priv) { 2863 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 2864 2865 rte_free(s->ctxt); 2866 rte_free(s->cipher_key.data); 2867 rte_free(s->auth_key.data); 2868 memset(s, 0, sizeof(dpaa2_sec_session)); 2869 set_sec_session_private_data(sess, NULL); 2870 rte_mempool_put(sess_mp, sess_priv); 2871 } 2872 return 0; 2873 } 2874 2875 static int 2876 dpaa2_sec_sym_session_configure(struct rte_cryptodev *dev, 2877 struct rte_crypto_sym_xform *xform, 2878 struct rte_cryptodev_sym_session *sess, 2879 struct rte_mempool *mempool) 2880 { 2881 void *sess_private_data; 2882 int ret; 2883 2884 if (rte_mempool_get(mempool, &sess_private_data)) { 2885 DPAA2_SEC_ERR("Couldn't get object from session mempool"); 2886 return -ENOMEM; 2887 } 2888 2889 ret = dpaa2_sec_set_session_parameters(dev, xform, sess_private_data); 2890 if (ret != 0) { 2891 DPAA2_SEC_ERR("Failed to configure session parameters"); 2892 /* Return session to mempool */ 2893 rte_mempool_put(mempool, sess_private_data); 2894 return ret; 2895 } 2896 2897 set_sym_session_private_data(sess, dev->driver_id, 2898 sess_private_data); 2899 2900 return 0; 2901 } 2902 2903 /** Clear the memory of session so it doesn't leave key material behind */ 2904 static void 2905 dpaa2_sec_sym_session_clear(struct rte_cryptodev *dev, 2906 struct rte_cryptodev_sym_session *sess) 2907 { 2908 PMD_INIT_FUNC_TRACE(); 2909 uint8_t index = dev->driver_id; 2910 void *sess_priv = get_sym_session_private_data(sess, index); 2911 dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; 2912 2913 if (sess_priv) { 2914 rte_free(s->ctxt); 2915 rte_free(s->cipher_key.data); 2916 rte_free(s->auth_key.data); 2917 memset(s, 0, sizeof(dpaa2_sec_session)); 2918 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 2919 set_sym_session_private_data(sess, index, NULL); 2920 rte_mempool_put(sess_mp, sess_priv); 2921 } 2922 } 2923 2924 static int 2925 dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, 2926 struct rte_cryptodev_config *config __rte_unused) 2927 { 2928 PMD_INIT_FUNC_TRACE(); 2929 2930 return 0; 2931 } 2932 2933 static int 2934 dpaa2_sec_dev_start(struct rte_cryptodev *dev) 2935 { 2936 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 2937 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 2938 struct dpseci_attr attr; 2939 struct dpaa2_queue *dpaa2_q; 2940 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **) 2941 dev->data->queue_pairs; 2942 struct dpseci_rx_queue_attr rx_attr; 2943 struct dpseci_tx_queue_attr tx_attr; 2944 int ret, i; 2945 2946 PMD_INIT_FUNC_TRACE(); 2947 2948 memset(&attr, 0, sizeof(struct dpseci_attr)); 2949 2950 ret = dpseci_enable(dpseci, CMD_PRI_LOW, priv->token); 2951 if (ret) { 2952 DPAA2_SEC_ERR("DPSECI with HW_ID = %d ENABLE FAILED", 2953 priv->hw_id); 2954 goto get_attr_failure; 2955 } 2956 ret = dpseci_get_attributes(dpseci, CMD_PRI_LOW, priv->token, &attr); 2957 if (ret) { 2958 DPAA2_SEC_ERR("DPSEC ATTRIBUTE READ FAILED, disabling DPSEC"); 2959 goto get_attr_failure; 2960 } 2961 for (i = 0; i < attr.num_rx_queues && qp[i]; i++) { 2962 dpaa2_q = &qp[i]->rx_vq; 2963 dpseci_get_rx_queue(dpseci, CMD_PRI_LOW, priv->token, i, 2964 &rx_attr); 2965 dpaa2_q->fqid = rx_attr.fqid; 2966 DPAA2_SEC_DEBUG("rx_fqid: %d", dpaa2_q->fqid); 2967 } 2968 for (i = 0; i < attr.num_tx_queues && qp[i]; i++) { 2969 dpaa2_q = &qp[i]->tx_vq; 2970 dpseci_get_tx_queue(dpseci, CMD_PRI_LOW, priv->token, i, 2971 &tx_attr); 2972 dpaa2_q->fqid = tx_attr.fqid; 2973 DPAA2_SEC_DEBUG("tx_fqid: %d", dpaa2_q->fqid); 2974 } 2975 2976 return 0; 2977 get_attr_failure: 2978 dpseci_disable(dpseci, CMD_PRI_LOW, priv->token); 2979 return -1; 2980 } 2981 2982 static void 2983 dpaa2_sec_dev_stop(struct rte_cryptodev *dev) 2984 { 2985 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 2986 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 2987 int ret; 2988 2989 PMD_INIT_FUNC_TRACE(); 2990 2991 ret = dpseci_disable(dpseci, CMD_PRI_LOW, priv->token); 2992 if (ret) { 2993 DPAA2_SEC_ERR("Failure in disabling dpseci %d device", 2994 priv->hw_id); 2995 return; 2996 } 2997 2998 ret = dpseci_reset(dpseci, CMD_PRI_LOW, priv->token); 2999 if (ret < 0) { 3000 DPAA2_SEC_ERR("SEC Device cannot be reset:Error = %0x", ret); 3001 return; 3002 } 3003 } 3004 3005 static int 3006 dpaa2_sec_dev_close(struct rte_cryptodev *dev) 3007 { 3008 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 3009 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 3010 int ret; 3011 3012 PMD_INIT_FUNC_TRACE(); 3013 3014 /* Function is reverse of dpaa2_sec_dev_init. 3015 * It does the following: 3016 * 1. Detach a DPSECI from attached resources i.e. buffer pools, dpbp_id 3017 * 2. Close the DPSECI device 3018 * 3. Free the allocated resources. 3019 */ 3020 3021 /*Close the device at underlying layer*/ 3022 ret = dpseci_close(dpseci, CMD_PRI_LOW, priv->token); 3023 if (ret) { 3024 DPAA2_SEC_ERR("Failure closing dpseci device: err(%d)", ret); 3025 return -1; 3026 } 3027 3028 /*Free the allocated memory for ethernet private data and dpseci*/ 3029 priv->hw = NULL; 3030 rte_free(dpseci); 3031 3032 return 0; 3033 } 3034 3035 static void 3036 dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev, 3037 struct rte_cryptodev_info *info) 3038 { 3039 struct dpaa2_sec_dev_private *internals = dev->data->dev_private; 3040 3041 PMD_INIT_FUNC_TRACE(); 3042 if (info != NULL) { 3043 info->max_nb_queue_pairs = internals->max_nb_queue_pairs; 3044 info->feature_flags = dev->feature_flags; 3045 info->capabilities = dpaa2_sec_capabilities; 3046 /* No limit of number of sessions */ 3047 info->sym.max_nb_sessions = 0; 3048 info->driver_id = cryptodev_driver_id; 3049 } 3050 } 3051 3052 static 3053 void dpaa2_sec_stats_get(struct rte_cryptodev *dev, 3054 struct rte_cryptodev_stats *stats) 3055 { 3056 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 3057 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 3058 struct dpseci_sec_counters counters = {0}; 3059 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **) 3060 dev->data->queue_pairs; 3061 int ret, i; 3062 3063 PMD_INIT_FUNC_TRACE(); 3064 if (stats == NULL) { 3065 DPAA2_SEC_ERR("Invalid stats ptr NULL"); 3066 return; 3067 } 3068 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 3069 if (qp[i] == NULL) { 3070 DPAA2_SEC_DEBUG("Uninitialised queue pair"); 3071 continue; 3072 } 3073 3074 stats->enqueued_count += qp[i]->tx_vq.tx_pkts; 3075 stats->dequeued_count += qp[i]->rx_vq.rx_pkts; 3076 stats->enqueue_err_count += qp[i]->tx_vq.err_pkts; 3077 stats->dequeue_err_count += qp[i]->rx_vq.err_pkts; 3078 } 3079 3080 ret = dpseci_get_sec_counters(dpseci, CMD_PRI_LOW, priv->token, 3081 &counters); 3082 if (ret) { 3083 DPAA2_SEC_ERR("SEC counters failed"); 3084 } else { 3085 DPAA2_SEC_INFO("dpseci hardware stats:" 3086 "\n\tNum of Requests Dequeued = %" PRIu64 3087 "\n\tNum of Outbound Encrypt Requests = %" PRIu64 3088 "\n\tNum of Inbound Decrypt Requests = %" PRIu64 3089 "\n\tNum of Outbound Bytes Encrypted = %" PRIu64 3090 "\n\tNum of Outbound Bytes Protected = %" PRIu64 3091 "\n\tNum of Inbound Bytes Decrypted = %" PRIu64 3092 "\n\tNum of Inbound Bytes Validated = %" PRIu64, 3093 counters.dequeued_requests, 3094 counters.ob_enc_requests, 3095 counters.ib_dec_requests, 3096 counters.ob_enc_bytes, 3097 counters.ob_prot_bytes, 3098 counters.ib_dec_bytes, 3099 counters.ib_valid_bytes); 3100 } 3101 } 3102 3103 static 3104 void dpaa2_sec_stats_reset(struct rte_cryptodev *dev) 3105 { 3106 int i; 3107 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **) 3108 (dev->data->queue_pairs); 3109 3110 PMD_INIT_FUNC_TRACE(); 3111 3112 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 3113 if (qp[i] == NULL) { 3114 DPAA2_SEC_DEBUG("Uninitialised queue pair"); 3115 continue; 3116 } 3117 qp[i]->tx_vq.rx_pkts = 0; 3118 qp[i]->tx_vq.tx_pkts = 0; 3119 qp[i]->tx_vq.err_pkts = 0; 3120 qp[i]->rx_vq.rx_pkts = 0; 3121 qp[i]->rx_vq.tx_pkts = 0; 3122 qp[i]->rx_vq.err_pkts = 0; 3123 } 3124 } 3125 3126 static void __attribute__((hot)) 3127 dpaa2_sec_process_parallel_event(struct qbman_swp *swp, 3128 const struct qbman_fd *fd, 3129 const struct qbman_result *dq, 3130 struct dpaa2_queue *rxq, 3131 struct rte_event *ev) 3132 { 3133 /* Prefetching mbuf */ 3134 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)- 3135 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size)); 3136 3137 /* Prefetching ipsec crypto_op stored in priv data of mbuf */ 3138 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64)); 3139 3140 ev->flow_id = rxq->ev.flow_id; 3141 ev->sub_event_type = rxq->ev.sub_event_type; 3142 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV; 3143 ev->op = RTE_EVENT_OP_NEW; 3144 ev->sched_type = rxq->ev.sched_type; 3145 ev->queue_id = rxq->ev.queue_id; 3146 ev->priority = rxq->ev.priority; 3147 ev->event_ptr = sec_fd_to_mbuf(fd, ((struct rte_cryptodev *) 3148 (rxq->dev))->driver_id); 3149 3150 qbman_swp_dqrr_consume(swp, dq); 3151 } 3152 static void 3153 dpaa2_sec_process_atomic_event(struct qbman_swp *swp __attribute__((unused)), 3154 const struct qbman_fd *fd, 3155 const struct qbman_result *dq, 3156 struct dpaa2_queue *rxq, 3157 struct rte_event *ev) 3158 { 3159 uint8_t dqrr_index; 3160 struct rte_crypto_op *crypto_op = (struct rte_crypto_op *)ev->event_ptr; 3161 /* Prefetching mbuf */ 3162 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)- 3163 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size)); 3164 3165 /* Prefetching ipsec crypto_op stored in priv data of mbuf */ 3166 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64)); 3167 3168 ev->flow_id = rxq->ev.flow_id; 3169 ev->sub_event_type = rxq->ev.sub_event_type; 3170 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV; 3171 ev->op = RTE_EVENT_OP_NEW; 3172 ev->sched_type = rxq->ev.sched_type; 3173 ev->queue_id = rxq->ev.queue_id; 3174 ev->priority = rxq->ev.priority; 3175 3176 ev->event_ptr = sec_fd_to_mbuf(fd, ((struct rte_cryptodev *) 3177 (rxq->dev))->driver_id); 3178 dqrr_index = qbman_get_dqrr_idx(dq); 3179 crypto_op->sym->m_src->seqn = dqrr_index + 1; 3180 DPAA2_PER_LCORE_DQRR_SIZE++; 3181 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index; 3182 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = crypto_op->sym->m_src; 3183 } 3184 3185 int 3186 dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev, 3187 int qp_id, 3188 uint16_t dpcon_id, 3189 const struct rte_event *event) 3190 { 3191 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 3192 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 3193 struct dpaa2_sec_qp *qp = dev->data->queue_pairs[qp_id]; 3194 struct dpseci_rx_queue_cfg cfg; 3195 int ret; 3196 3197 if (event->sched_type == RTE_SCHED_TYPE_PARALLEL) 3198 qp->rx_vq.cb = dpaa2_sec_process_parallel_event; 3199 else if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) 3200 qp->rx_vq.cb = dpaa2_sec_process_atomic_event; 3201 else 3202 return -EINVAL; 3203 3204 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg)); 3205 cfg.options = DPSECI_QUEUE_OPT_DEST; 3206 cfg.dest_cfg.dest_type = DPSECI_DEST_DPCON; 3207 cfg.dest_cfg.dest_id = dpcon_id; 3208 cfg.dest_cfg.priority = event->priority; 3209 3210 cfg.options |= DPSECI_QUEUE_OPT_USER_CTX; 3211 cfg.user_ctx = (size_t)(qp); 3212 if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) { 3213 cfg.options |= DPSECI_QUEUE_OPT_ORDER_PRESERVATION; 3214 cfg.order_preservation_en = 1; 3215 } 3216 ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token, 3217 qp_id, &cfg); 3218 if (ret) { 3219 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret); 3220 return ret; 3221 } 3222 3223 memcpy(&qp->rx_vq.ev, event, sizeof(struct rte_event)); 3224 3225 return 0; 3226 } 3227 3228 int 3229 dpaa2_sec_eventq_detach(const struct rte_cryptodev *dev, 3230 int qp_id) 3231 { 3232 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 3233 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 3234 struct dpseci_rx_queue_cfg cfg; 3235 int ret; 3236 3237 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg)); 3238 cfg.options = DPSECI_QUEUE_OPT_DEST; 3239 cfg.dest_cfg.dest_type = DPSECI_DEST_NONE; 3240 3241 ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token, 3242 qp_id, &cfg); 3243 if (ret) 3244 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret); 3245 3246 return ret; 3247 } 3248 3249 static struct rte_cryptodev_ops crypto_ops = { 3250 .dev_configure = dpaa2_sec_dev_configure, 3251 .dev_start = dpaa2_sec_dev_start, 3252 .dev_stop = dpaa2_sec_dev_stop, 3253 .dev_close = dpaa2_sec_dev_close, 3254 .dev_infos_get = dpaa2_sec_dev_infos_get, 3255 .stats_get = dpaa2_sec_stats_get, 3256 .stats_reset = dpaa2_sec_stats_reset, 3257 .queue_pair_setup = dpaa2_sec_queue_pair_setup, 3258 .queue_pair_release = dpaa2_sec_queue_pair_release, 3259 .queue_pair_count = dpaa2_sec_queue_pair_count, 3260 .sym_session_get_size = dpaa2_sec_sym_session_get_size, 3261 .sym_session_configure = dpaa2_sec_sym_session_configure, 3262 .sym_session_clear = dpaa2_sec_sym_session_clear, 3263 }; 3264 3265 static const struct rte_security_capability * 3266 dpaa2_sec_capabilities_get(void *device __rte_unused) 3267 { 3268 return dpaa2_sec_security_cap; 3269 } 3270 3271 static const struct rte_security_ops dpaa2_sec_security_ops = { 3272 .session_create = dpaa2_sec_security_session_create, 3273 .session_update = NULL, 3274 .session_stats_get = NULL, 3275 .session_destroy = dpaa2_sec_security_session_destroy, 3276 .set_pkt_metadata = NULL, 3277 .capabilities_get = dpaa2_sec_capabilities_get 3278 }; 3279 3280 static int 3281 dpaa2_sec_uninit(const struct rte_cryptodev *dev) 3282 { 3283 struct dpaa2_sec_dev_private *internals = dev->data->dev_private; 3284 3285 rte_free(dev->security_ctx); 3286 3287 rte_mempool_free(internals->fle_pool); 3288 3289 DPAA2_SEC_INFO("Closing DPAA2_SEC device %s on numa socket %u", 3290 dev->data->name, rte_socket_id()); 3291 3292 return 0; 3293 } 3294 3295 static int 3296 dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) 3297 { 3298 struct dpaa2_sec_dev_private *internals; 3299 struct rte_device *dev = cryptodev->device; 3300 struct rte_dpaa2_device *dpaa2_dev; 3301 struct rte_security_ctx *security_instance; 3302 struct fsl_mc_io *dpseci; 3303 uint16_t token; 3304 struct dpseci_attr attr; 3305 int retcode, hw_id; 3306 char str[20]; 3307 3308 PMD_INIT_FUNC_TRACE(); 3309 dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device); 3310 if (dpaa2_dev == NULL) { 3311 DPAA2_SEC_ERR("DPAA2 SEC device not found"); 3312 return -1; 3313 } 3314 hw_id = dpaa2_dev->object_id; 3315 3316 cryptodev->driver_id = cryptodev_driver_id; 3317 cryptodev->dev_ops = &crypto_ops; 3318 3319 cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst; 3320 cryptodev->dequeue_burst = dpaa2_sec_dequeue_burst; 3321 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 3322 RTE_CRYPTODEV_FF_HW_ACCELERATED | 3323 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 3324 RTE_CRYPTODEV_FF_SECURITY | 3325 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 3326 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | 3327 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 3328 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | 3329 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; 3330 3331 internals = cryptodev->data->dev_private; 3332 3333 /* 3334 * For secondary processes, we don't initialise any further as primary 3335 * has already done this work. Only check we don't need a different 3336 * RX function 3337 */ 3338 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 3339 DPAA2_SEC_DEBUG("Device already init by primary process"); 3340 return 0; 3341 } 3342 3343 /* Initialize security_ctx only for primary process*/ 3344 security_instance = rte_malloc("rte_security_instances_ops", 3345 sizeof(struct rte_security_ctx), 0); 3346 if (security_instance == NULL) 3347 return -ENOMEM; 3348 security_instance->device = (void *)cryptodev; 3349 security_instance->ops = &dpaa2_sec_security_ops; 3350 security_instance->sess_cnt = 0; 3351 cryptodev->security_ctx = security_instance; 3352 3353 /*Open the rte device via MC and save the handle for further use*/ 3354 dpseci = (struct fsl_mc_io *)rte_calloc(NULL, 1, 3355 sizeof(struct fsl_mc_io), 0); 3356 if (!dpseci) { 3357 DPAA2_SEC_ERR( 3358 "Error in allocating the memory for dpsec object"); 3359 return -1; 3360 } 3361 dpseci->regs = rte_mcp_ptr_list[0]; 3362 3363 retcode = dpseci_open(dpseci, CMD_PRI_LOW, hw_id, &token); 3364 if (retcode != 0) { 3365 DPAA2_SEC_ERR("Cannot open the dpsec device: Error = %x", 3366 retcode); 3367 goto init_error; 3368 } 3369 retcode = dpseci_get_attributes(dpseci, CMD_PRI_LOW, token, &attr); 3370 if (retcode != 0) { 3371 DPAA2_SEC_ERR( 3372 "Cannot get dpsec device attributed: Error = %x", 3373 retcode); 3374 goto init_error; 3375 } 3376 snprintf(cryptodev->data->name, sizeof(cryptodev->data->name), 3377 "dpsec-%u", hw_id); 3378 3379 internals->max_nb_queue_pairs = attr.num_tx_queues; 3380 cryptodev->data->nb_queue_pairs = internals->max_nb_queue_pairs; 3381 internals->hw = dpseci; 3382 internals->token = token; 3383 3384 snprintf(str, sizeof(str), "fle_pool_%d", cryptodev->data->dev_id); 3385 internals->fle_pool = rte_mempool_create((const char *)str, 3386 FLE_POOL_NUM_BUFS, 3387 FLE_POOL_BUF_SIZE, 3388 FLE_POOL_CACHE_SIZE, 0, 3389 NULL, NULL, NULL, NULL, 3390 SOCKET_ID_ANY, 0); 3391 if (!internals->fle_pool) { 3392 DPAA2_SEC_ERR("Mempool (%s) creation failed", str); 3393 goto init_error; 3394 } 3395 3396 DPAA2_SEC_INFO("driver %s: created", cryptodev->data->name); 3397 return 0; 3398 3399 init_error: 3400 DPAA2_SEC_ERR("driver %s: create failed", cryptodev->data->name); 3401 3402 /* dpaa2_sec_uninit(crypto_dev_name); */ 3403 return -EFAULT; 3404 } 3405 3406 static int 3407 cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused, 3408 struct rte_dpaa2_device *dpaa2_dev) 3409 { 3410 struct rte_cryptodev *cryptodev; 3411 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; 3412 3413 int retval; 3414 3415 snprintf(cryptodev_name, sizeof(cryptodev_name), "dpsec-%d", 3416 dpaa2_dev->object_id); 3417 3418 cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); 3419 if (cryptodev == NULL) 3420 return -ENOMEM; 3421 3422 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 3423 cryptodev->data->dev_private = rte_zmalloc_socket( 3424 "cryptodev private structure", 3425 sizeof(struct dpaa2_sec_dev_private), 3426 RTE_CACHE_LINE_SIZE, 3427 rte_socket_id()); 3428 3429 if (cryptodev->data->dev_private == NULL) 3430 rte_panic("Cannot allocate memzone for private " 3431 "device data"); 3432 } 3433 3434 dpaa2_dev->cryptodev = cryptodev; 3435 cryptodev->device = &dpaa2_dev->device; 3436 3437 /* init user callbacks */ 3438 TAILQ_INIT(&(cryptodev->link_intr_cbs)); 3439 3440 /* Invoke PMD device initialization function */ 3441 retval = dpaa2_sec_dev_init(cryptodev); 3442 if (retval == 0) 3443 return 0; 3444 3445 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 3446 rte_free(cryptodev->data->dev_private); 3447 3448 cryptodev->attached = RTE_CRYPTODEV_DETACHED; 3449 3450 return -ENXIO; 3451 } 3452 3453 static int 3454 cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev) 3455 { 3456 struct rte_cryptodev *cryptodev; 3457 int ret; 3458 3459 cryptodev = dpaa2_dev->cryptodev; 3460 if (cryptodev == NULL) 3461 return -ENODEV; 3462 3463 ret = dpaa2_sec_uninit(cryptodev); 3464 if (ret) 3465 return ret; 3466 3467 return rte_cryptodev_pmd_destroy(cryptodev); 3468 } 3469 3470 static struct rte_dpaa2_driver rte_dpaa2_sec_driver = { 3471 .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA, 3472 .drv_type = DPAA2_CRYPTO, 3473 .driver = { 3474 .name = "DPAA2 SEC PMD" 3475 }, 3476 .probe = cryptodev_dpaa2_sec_probe, 3477 .remove = cryptodev_dpaa2_sec_remove, 3478 }; 3479 3480 static struct cryptodev_driver dpaa2_sec_crypto_drv; 3481 3482 RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver); 3483 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv, 3484 rte_dpaa2_sec_driver.driver, cryptodev_driver_id); 3485 3486 RTE_INIT(dpaa2_sec_init_log) 3487 { 3488 /* Bus level logs */ 3489 dpaa2_logtype_sec = rte_log_register("pmd.crypto.dpaa2"); 3490 if (dpaa2_logtype_sec >= 0) 3491 rte_log_set_level(dpaa2_logtype_sec, RTE_LOG_NOTICE); 3492 } 3493