1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <rte_cryptodev.h> 6 #include <cryptodev_pmd.h> 7 #include <rte_event_crypto_adapter.h> 8 #include <rte_ip.h> 9 #include <rte_vect.h> 10 11 #include "cn9k_cryptodev.h" 12 #include "cn9k_cryptodev_ops.h" 13 #include "cn9k_ipsec.h" 14 #include "cn9k_ipsec_la_ops.h" 15 #include "cnxk_ae.h" 16 #include "cnxk_cryptodev.h" 17 #include "cnxk_cryptodev_ops.h" 18 #include "cnxk_se.h" 19 20 static __rte_always_inline int __rte_hot 21 cn9k_cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op, 22 struct cnxk_se_sess *sess, 23 struct cpt_inflight_req *infl_req, 24 struct cpt_inst_s *inst) 25 { 26 uint64_t cpt_op; 27 int ret; 28 29 cpt_op = sess->cpt_op; 30 31 if (cpt_op & ROC_SE_OP_CIPHER_MASK) 32 ret = fill_fc_params(op, sess, &qp->meta_info, infl_req, inst); 33 else 34 ret = fill_digest_params(op, sess, &qp->meta_info, infl_req, 35 inst); 36 37 return ret; 38 } 39 40 static __rte_always_inline int __rte_hot 41 cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op, 42 struct cpt_inflight_req *infl_req, 43 struct cpt_inst_s *inst) 44 { 45 struct rte_crypto_sym_op *sym_op = op->sym; 46 struct cn9k_sec_session *priv; 47 struct cn9k_ipsec_sa *sa; 48 49 if (unlikely(sym_op->m_dst && sym_op->m_dst != sym_op->m_src)) { 50 plt_dp_err("Out of place is not supported"); 51 return -ENOTSUP; 52 } 53 54 if (unlikely(!rte_pktmbuf_is_contiguous(sym_op->m_src))) { 55 plt_dp_err("Scatter Gather mode is not supported"); 56 return -ENOTSUP; 57 } 58 59 priv = get_sec_session_private_data(op->sym->sec_session); 60 sa = &priv->sa; 61 62 if (sa->dir == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) 63 return process_outb_sa(op, sa, inst); 64 65 infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND; 66 67 return process_inb_sa(op, sa, inst); 68 } 69 70 static inline struct cnxk_se_sess * 71 cn9k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op) 72 { 73 const int driver_id = cn9k_cryptodev_driver_id; 74 struct rte_crypto_sym_op *sym_op = op->sym; 75 struct rte_cryptodev_sym_session *sess; 76 struct cnxk_se_sess *priv; 77 int ret; 78 79 /* Create temporary session */ 80 sess = rte_cryptodev_sym_session_create(qp->sess_mp); 81 if (sess == NULL) 82 return NULL; 83 84 ret = sym_session_configure(qp->lf.roc_cpt, driver_id, sym_op->xform, 85 sess, qp->sess_mp_priv); 86 if (ret) 87 goto sess_put; 88 89 priv = get_sym_session_private_data(sess, driver_id); 90 91 sym_op->session = sess; 92 93 return priv; 94 95 sess_put: 96 rte_mempool_put(qp->sess_mp, sess); 97 return NULL; 98 } 99 100 static inline int 101 cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op, 102 struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst) 103 { 104 int ret; 105 106 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 107 struct rte_crypto_sym_op *sym_op; 108 struct cnxk_se_sess *sess; 109 110 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 111 sym_op = op->sym; 112 sess = get_sym_session_private_data( 113 sym_op->session, cn9k_cryptodev_driver_id); 114 ret = cn9k_cpt_sym_inst_fill(qp, op, sess, infl_req, 115 inst); 116 inst->w7.u64 = sess->cpt_inst_w7; 117 } else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) 118 ret = cn9k_cpt_sec_inst_fill(op, infl_req, inst); 119 else { 120 sess = cn9k_cpt_sym_temp_sess_create(qp, op); 121 if (unlikely(sess == NULL)) { 122 plt_dp_err("Could not create temp session"); 123 return -1; 124 } 125 126 ret = cn9k_cpt_sym_inst_fill(qp, op, sess, infl_req, 127 inst); 128 if (unlikely(ret)) { 129 sym_session_clear(cn9k_cryptodev_driver_id, 130 op->sym->session); 131 rte_mempool_put(qp->sess_mp, op->sym->session); 132 } 133 inst->w7.u64 = sess->cpt_inst_w7; 134 } 135 } else if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) { 136 struct rte_crypto_asym_op *asym_op; 137 struct cnxk_ae_sess *sess; 138 139 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 140 asym_op = op->asym; 141 sess = get_asym_session_private_data( 142 asym_op->session, cn9k_cryptodev_driver_id); 143 ret = cnxk_ae_enqueue(qp, op, infl_req, inst, sess); 144 inst->w7.u64 = sess->cpt_inst_w7; 145 } else { 146 ret = -EINVAL; 147 } 148 } else { 149 ret = -EINVAL; 150 plt_dp_err("Unsupported op type"); 151 } 152 153 return ret; 154 } 155 156 static inline void 157 cn9k_cpt_inst_submit(struct cpt_inst_s *inst, uint64_t lmtline, 158 uint64_t io_addr) 159 { 160 uint64_t lmt_status; 161 162 do { 163 /* Copy CPT command to LMTLINE */ 164 roc_lmt_mov((void *)lmtline, inst, 2); 165 166 /* 167 * Make sure compiler does not reorder memcpy and ldeor. 168 * LMTST transactions are always flushed from the write 169 * buffer immediately, a DMB is not required to push out 170 * LMTSTs. 171 */ 172 rte_io_wmb(); 173 lmt_status = roc_lmt_submit_ldeor(io_addr); 174 } while (lmt_status == 0); 175 } 176 177 static __plt_always_inline void 178 cn9k_cpt_inst_submit_dual(struct cpt_inst_s *inst, uint64_t lmtline, 179 uint64_t io_addr) 180 { 181 uint64_t lmt_status; 182 183 do { 184 /* Copy 2 CPT inst_s to LMTLINE */ 185 #if defined(RTE_ARCH_ARM64) 186 uint64_t *s = (uint64_t *)inst; 187 uint64_t *d = (uint64_t *)lmtline; 188 189 vst1q_u64(&d[0], vld1q_u64(&s[0])); 190 vst1q_u64(&d[2], vld1q_u64(&s[2])); 191 vst1q_u64(&d[4], vld1q_u64(&s[4])); 192 vst1q_u64(&d[6], vld1q_u64(&s[6])); 193 vst1q_u64(&d[8], vld1q_u64(&s[8])); 194 vst1q_u64(&d[10], vld1q_u64(&s[10])); 195 vst1q_u64(&d[12], vld1q_u64(&s[12])); 196 vst1q_u64(&d[14], vld1q_u64(&s[14])); 197 #else 198 roc_lmt_mov_seg((void *)lmtline, inst, 8); 199 #endif 200 201 /* 202 * Make sure compiler does not reorder memcpy and ldeor. 203 * LMTST transactions are always flushed from the write 204 * buffer immediately, a DMB is not required to push out 205 * LMTSTs. 206 */ 207 rte_io_wmb(); 208 lmt_status = roc_lmt_submit_ldeor(io_addr); 209 } while (lmt_status == 0); 210 } 211 212 static uint16_t 213 cn9k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops) 214 { 215 struct cpt_inflight_req *infl_req_1, *infl_req_2; 216 struct cpt_inst_s inst[2] __rte_cache_aligned; 217 struct rte_crypto_op *op_1, *op_2; 218 uint16_t nb_allowed, count = 0; 219 struct cnxk_cpt_qp *qp = qptr; 220 struct pending_queue *pend_q; 221 uint64_t enq_tail; 222 int ret; 223 224 const uint32_t nb_desc = qp->lf.nb_desc; 225 const uint64_t lmt_base = qp->lf.lmt_base; 226 const uint64_t io_addr = qp->lf.io_addr; 227 228 pend_q = &qp->pend_q; 229 230 /* Clear w0, w2, w3 of both inst */ 231 232 inst[0].w0.u64 = 0; 233 inst[0].w2.u64 = 0; 234 inst[0].w3.u64 = 0; 235 inst[1].w0.u64 = 0; 236 inst[1].w2.u64 = 0; 237 inst[1].w3.u64 = 0; 238 239 nb_allowed = qp->lf.nb_desc - pend_q->pending_count; 240 nb_ops = RTE_MIN(nb_ops, nb_allowed); 241 242 enq_tail = pend_q->enq_tail; 243 244 if (unlikely(nb_ops & 1)) { 245 op_1 = ops[0]; 246 infl_req_1 = &pend_q->req_queue[enq_tail]; 247 infl_req_1->op_flags = 0; 248 249 ret = cn9k_cpt_inst_prep(qp, op_1, infl_req_1, &inst[0]); 250 if (unlikely(ret)) { 251 plt_dp_err("Could not process op: %p", op_1); 252 return 0; 253 } 254 255 infl_req_1->cop = op_1; 256 infl_req_1->res.cn9k.compcode = CPT_COMP_NOT_DONE; 257 inst[0].res_addr = (uint64_t)&infl_req_1->res; 258 259 cn9k_cpt_inst_submit(&inst[0], lmt_base, io_addr); 260 MOD_INC(enq_tail, nb_desc); 261 count++; 262 } 263 264 while (count < nb_ops) { 265 op_1 = ops[count]; 266 op_2 = ops[count + 1]; 267 268 infl_req_1 = &pend_q->req_queue[enq_tail]; 269 MOD_INC(enq_tail, nb_desc); 270 infl_req_2 = &pend_q->req_queue[enq_tail]; 271 MOD_INC(enq_tail, nb_desc); 272 273 infl_req_1->cop = op_1; 274 infl_req_2->cop = op_2; 275 infl_req_1->op_flags = 0; 276 infl_req_2->op_flags = 0; 277 278 infl_req_1->res.cn9k.compcode = CPT_COMP_NOT_DONE; 279 inst[0].res_addr = (uint64_t)&infl_req_1->res; 280 281 infl_req_2->res.cn9k.compcode = CPT_COMP_NOT_DONE; 282 inst[1].res_addr = (uint64_t)&infl_req_2->res; 283 284 ret = cn9k_cpt_inst_prep(qp, op_1, infl_req_1, &inst[0]); 285 if (unlikely(ret)) { 286 plt_dp_err("Could not process op: %p", op_1); 287 if (enq_tail == 0) 288 enq_tail = nb_desc - 2; 289 else if (enq_tail == 1) 290 enq_tail = nb_desc - 1; 291 else 292 enq_tail--; 293 break; 294 } 295 296 ret = cn9k_cpt_inst_prep(qp, op_2, infl_req_2, &inst[1]); 297 if (unlikely(ret)) { 298 plt_dp_err("Could not process op: %p", op_2); 299 if (enq_tail == 0) 300 enq_tail = nb_desc - 1; 301 else 302 enq_tail--; 303 304 cn9k_cpt_inst_submit(&inst[0], lmt_base, io_addr); 305 count++; 306 break; 307 } 308 309 cn9k_cpt_inst_submit_dual(&inst[0], lmt_base, io_addr); 310 311 count += 2; 312 } 313 314 pend_q->enq_tail = enq_tail; 315 pend_q->pending_count += count; 316 pend_q->time_out = rte_get_timer_cycles() + 317 DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz(); 318 319 return count; 320 } 321 322 uint16_t 323 cn9k_cpt_crypto_adapter_enqueue(uintptr_t tag_op, struct rte_crypto_op *op) 324 { 325 union rte_event_crypto_metadata *ec_mdata; 326 struct cpt_inflight_req *infl_req; 327 struct rte_event *rsp_info; 328 struct cnxk_cpt_qp *qp; 329 struct cpt_inst_s inst; 330 uint8_t cdev_id; 331 uint16_t qp_id; 332 int ret; 333 334 ec_mdata = cnxk_event_crypto_mdata_get(op); 335 if (!ec_mdata) { 336 rte_errno = EINVAL; 337 return 0; 338 } 339 340 cdev_id = ec_mdata->request_info.cdev_id; 341 qp_id = ec_mdata->request_info.queue_pair_id; 342 qp = rte_cryptodevs[cdev_id].data->queue_pairs[qp_id]; 343 rsp_info = &ec_mdata->response_info; 344 345 if (unlikely(!qp->ca.enabled)) { 346 rte_errno = EINVAL; 347 return 0; 348 } 349 350 if (unlikely(rte_mempool_get(qp->ca.req_mp, (void **)&infl_req))) { 351 rte_errno = ENOMEM; 352 return 0; 353 } 354 infl_req->op_flags = 0; 355 356 ret = cn9k_cpt_inst_prep(qp, op, infl_req, &inst); 357 if (unlikely(ret)) { 358 plt_dp_err("Could not process op: %p", op); 359 rte_mempool_put(qp->ca.req_mp, infl_req); 360 return 0; 361 } 362 363 infl_req->cop = op; 364 infl_req->res.cn9k.compcode = CPT_COMP_NOT_DONE; 365 infl_req->qp = qp; 366 inst.w0.u64 = 0; 367 inst.res_addr = (uint64_t)&infl_req->res; 368 inst.w2.u64 = CNXK_CPT_INST_W2( 369 (RTE_EVENT_TYPE_CRYPTODEV << 28) | rsp_info->flow_id, 370 rsp_info->sched_type, rsp_info->queue_id, 0); 371 inst.w3.u64 = CNXK_CPT_INST_W3(1, infl_req); 372 373 if (roc_cpt_is_iq_full(&qp->lf)) { 374 rte_mempool_put(qp->ca.req_mp, infl_req); 375 rte_errno = EAGAIN; 376 return 0; 377 } 378 379 if (!rsp_info->sched_type) 380 roc_sso_hws_head_wait(tag_op); 381 382 cn9k_cpt_inst_submit(&inst, qp->lmtline.lmt_base, qp->lmtline.io_addr); 383 384 return 1; 385 } 386 387 static inline void 388 cn9k_cpt_sec_post_process(struct rte_crypto_op *cop, 389 struct cpt_inflight_req *infl_req) 390 { 391 struct rte_crypto_sym_op *sym_op = cop->sym; 392 struct rte_mbuf *m = sym_op->m_src; 393 struct rte_ipv6_hdr *ip6; 394 struct rte_ipv4_hdr *ip; 395 uint16_t m_len = 0; 396 char *data; 397 398 if (infl_req->op_flags & CPT_OP_FLAGS_IPSEC_DIR_INBOUND) { 399 data = rte_pktmbuf_mtod(m, char *); 400 401 ip = (struct rte_ipv4_hdr *)(data + ROC_IE_ON_INB_RPTR_HDR); 402 403 if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == 404 IPVERSION) { 405 m_len = rte_be_to_cpu_16(ip->total_length); 406 } else { 407 PLT_ASSERT(((ip->version_ihl & 0xf0) >> 408 RTE_IPV4_IHL_MULTIPLIER) == 6); 409 ip6 = (struct rte_ipv6_hdr *)ip; 410 m_len = rte_be_to_cpu_16(ip6->payload_len) + 411 sizeof(struct rte_ipv6_hdr); 412 } 413 414 m->data_len = m_len; 415 m->pkt_len = m_len; 416 m->data_off += ROC_IE_ON_INB_RPTR_HDR; 417 } 418 } 419 420 static inline void 421 cn9k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, struct rte_crypto_op *cop, 422 struct cpt_inflight_req *infl_req) 423 { 424 struct cpt_cn9k_res_s *res = (struct cpt_cn9k_res_s *)&infl_req->res; 425 unsigned int sz; 426 427 if (likely(res->compcode == CPT_COMP_GOOD)) { 428 if (unlikely(res->uc_compcode)) { 429 if (res->uc_compcode == ROC_SE_ERR_GC_ICV_MISCOMPARE) 430 cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 431 else 432 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 433 434 plt_dp_info("Request failed with microcode error"); 435 plt_dp_info("MC completion code 0x%x", 436 res->uc_compcode); 437 goto temp_sess_free; 438 } 439 440 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 441 if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 442 if (cop->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 443 cn9k_cpt_sec_post_process(cop, infl_req); 444 return; 445 } 446 447 /* Verify authentication data if required */ 448 if (unlikely(infl_req->op_flags & 449 CPT_OP_FLAGS_AUTH_VERIFY)) { 450 uintptr_t *rsp = infl_req->mdata; 451 compl_auth_verify(cop, (uint8_t *)rsp[0], 452 rsp[1]); 453 } 454 } else if (cop->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) { 455 struct rte_crypto_asym_op *op = cop->asym; 456 uintptr_t *mdata = infl_req->mdata; 457 struct cnxk_ae_sess *sess; 458 459 sess = get_asym_session_private_data( 460 op->session, cn9k_cryptodev_driver_id); 461 462 cnxk_ae_post_process(cop, sess, (uint8_t *)mdata[0]); 463 } 464 } else { 465 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 466 plt_dp_info("HW completion code 0x%x", res->compcode); 467 468 switch (res->compcode) { 469 case CPT_COMP_INSTERR: 470 plt_dp_err("Request failed with instruction error"); 471 break; 472 case CPT_COMP_FAULT: 473 plt_dp_err("Request failed with DMA fault"); 474 break; 475 case CPT_COMP_HWERR: 476 plt_dp_err("Request failed with hardware error"); 477 break; 478 default: 479 plt_dp_err( 480 "Request failed with unknown completion code"); 481 } 482 } 483 484 temp_sess_free: 485 if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) { 486 if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 487 sym_session_clear(cn9k_cryptodev_driver_id, 488 cop->sym->session); 489 sz = rte_cryptodev_sym_get_existing_header_session_size( 490 cop->sym->session); 491 memset(cop->sym->session, 0, sz); 492 rte_mempool_put(qp->sess_mp, cop->sym->session); 493 cop->sym->session = NULL; 494 } 495 } 496 } 497 498 uintptr_t 499 cn9k_cpt_crypto_adapter_dequeue(uintptr_t get_work1) 500 { 501 struct cpt_inflight_req *infl_req; 502 struct rte_crypto_op *cop; 503 struct cnxk_cpt_qp *qp; 504 505 infl_req = (struct cpt_inflight_req *)(get_work1); 506 cop = infl_req->cop; 507 qp = infl_req->qp; 508 509 cn9k_cpt_dequeue_post_process(qp, infl_req->cop, infl_req); 510 511 if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF)) 512 rte_mempool_put(qp->meta_info.pool, infl_req->mdata); 513 514 rte_mempool_put(qp->ca.req_mp, infl_req); 515 return (uintptr_t)cop; 516 } 517 518 static uint16_t 519 cn9k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops) 520 { 521 struct cpt_inflight_req *infl_req; 522 struct cnxk_cpt_qp *qp = qptr; 523 struct pending_queue *pend_q; 524 struct cpt_cn9k_res_s *res; 525 struct rte_crypto_op *cop; 526 uint32_t pq_deq_head; 527 int i; 528 529 const uint32_t nb_desc = qp->lf.nb_desc; 530 531 pend_q = &qp->pend_q; 532 533 nb_ops = RTE_MIN(nb_ops, pend_q->pending_count); 534 535 pq_deq_head = pend_q->deq_head; 536 537 for (i = 0; i < nb_ops; i++) { 538 infl_req = &pend_q->req_queue[pq_deq_head]; 539 540 res = (struct cpt_cn9k_res_s *)&infl_req->res; 541 542 if (unlikely(res->compcode == CPT_COMP_NOT_DONE)) { 543 if (unlikely(rte_get_timer_cycles() > 544 pend_q->time_out)) { 545 plt_err("Request timed out"); 546 pend_q->time_out = rte_get_timer_cycles() + 547 DEFAULT_COMMAND_TIMEOUT * 548 rte_get_timer_hz(); 549 } 550 break; 551 } 552 553 MOD_INC(pq_deq_head, nb_desc); 554 555 cop = infl_req->cop; 556 557 ops[i] = cop; 558 559 cn9k_cpt_dequeue_post_process(qp, cop, infl_req); 560 561 if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF)) 562 rte_mempool_put(qp->meta_info.pool, infl_req->mdata); 563 } 564 565 pend_q->pending_count -= i; 566 pend_q->deq_head = pq_deq_head; 567 568 return i; 569 } 570 void 571 cn9k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev) 572 { 573 dev->enqueue_burst = cn9k_cpt_enqueue_burst; 574 dev->dequeue_burst = cn9k_cpt_dequeue_burst; 575 576 rte_mb(); 577 } 578 579 static void 580 cn9k_cpt_dev_info_get(struct rte_cryptodev *dev, 581 struct rte_cryptodev_info *info) 582 { 583 if (info != NULL) { 584 cnxk_cpt_dev_info_get(dev, info); 585 info->driver_id = cn9k_cryptodev_driver_id; 586 } 587 } 588 589 struct rte_cryptodev_ops cn9k_cpt_ops = { 590 /* Device control ops */ 591 .dev_configure = cnxk_cpt_dev_config, 592 .dev_start = cnxk_cpt_dev_start, 593 .dev_stop = cnxk_cpt_dev_stop, 594 .dev_close = cnxk_cpt_dev_close, 595 .dev_infos_get = cn9k_cpt_dev_info_get, 596 597 .stats_get = NULL, 598 .stats_reset = NULL, 599 .queue_pair_setup = cnxk_cpt_queue_pair_setup, 600 .queue_pair_release = cnxk_cpt_queue_pair_release, 601 602 /* Symmetric crypto ops */ 603 .sym_session_get_size = cnxk_cpt_sym_session_get_size, 604 .sym_session_configure = cnxk_cpt_sym_session_configure, 605 .sym_session_clear = cnxk_cpt_sym_session_clear, 606 607 /* Asymmetric crypto ops */ 608 .asym_session_get_size = cnxk_ae_session_size_get, 609 .asym_session_configure = cnxk_ae_session_cfg, 610 .asym_session_clear = cnxk_ae_session_clear, 611 612 }; 613