1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2021 Intel Corporation 3 */ 4 5 #ifndef _IPSEC_MB_PRIVATE_H_ 6 #define _IPSEC_MB_PRIVATE_H_ 7 8 #include <intel-ipsec-mb.h> 9 #include <cryptodev_pmd.h> 10 #include <rte_bus_vdev.h> 11 12 #if defined(RTE_LIB_SECURITY) 13 #define IPSEC_MB_DOCSIS_SEC_ENABLED 1 14 #include <rte_security.h> 15 #include <rte_security_driver.h> 16 #endif 17 18 /* Maximum length for digest */ 19 #define DIGEST_LENGTH_MAX 64 20 21 /* Maximum length for memzone name */ 22 #define IPSEC_MB_MAX_MZ_NAME 32 23 24 enum ipsec_mb_vector_mode { 25 IPSEC_MB_NOT_SUPPORTED = 0, 26 IPSEC_MB_SSE, 27 IPSEC_MB_AVX, 28 IPSEC_MB_AVX2, 29 IPSEC_MB_AVX512 30 }; 31 32 extern enum ipsec_mb_vector_mode vector_mode; 33 34 /** IMB_MGR instances, one per thread */ 35 extern RTE_DEFINE_PER_LCORE(IMB_MGR *, mb_mgr); 36 37 #define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb 38 /**< IPSEC Multi buffer aesni_mb PMD device name */ 39 40 #define CRYPTODEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm 41 /**< IPSEC Multi buffer PMD aesni_gcm device name */ 42 43 #define CRYPTODEV_NAME_KASUMI_PMD crypto_kasumi 44 /**< IPSEC Multi buffer PMD kasumi device name */ 45 46 #define CRYPTODEV_NAME_SNOW3G_PMD crypto_snow3g 47 /**< IPSEC Multi buffer PMD snow3g device name */ 48 49 #define CRYPTODEV_NAME_ZUC_PMD crypto_zuc 50 /**< IPSEC Multi buffer PMD zuc device name */ 51 52 #define CRYPTODEV_NAME_CHACHA20_POLY1305_PMD crypto_chacha20_poly1305 53 /**< IPSEC Multi buffer PMD chacha20_poly1305 device name */ 54 55 /** PMD LOGTYPE DRIVER, common to all PMDs */ 56 extern int ipsec_mb_logtype_driver; 57 #define IPSEC_MB_LOG(level, fmt, ...) \ 58 rte_log(RTE_LOG_##level, ipsec_mb_logtype_driver, \ 59 "%s() line %u: " fmt "\n", __func__, __LINE__, ##__VA_ARGS__) 60 61 /** All supported device types */ 62 enum ipsec_mb_pmd_types { 63 IPSEC_MB_PMD_TYPE_AESNI_MB = 0, 64 IPSEC_MB_PMD_TYPE_AESNI_GCM, 65 IPSEC_MB_PMD_TYPE_KASUMI, 66 IPSEC_MB_PMD_TYPE_SNOW3G, 67 IPSEC_MB_PMD_TYPE_ZUC, 68 IPSEC_MB_PMD_TYPE_CHACHA20_POLY1305, 69 IPSEC_MB_N_PMD_TYPES 70 }; 71 72 /** Crypto operations */ 73 enum ipsec_mb_operation { 74 IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN = 0, 75 IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT, 76 IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT, 77 IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY, 78 IPSEC_MB_OP_ENCRYPT_ONLY, 79 IPSEC_MB_OP_DECRYPT_ONLY, 80 IPSEC_MB_OP_HASH_GEN_ONLY, 81 IPSEC_MB_OP_HASH_VERIFY_ONLY, 82 IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT, 83 IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT, 84 IPSEC_MB_OP_NOT_SUPPORTED 85 }; 86 87 extern uint8_t pmd_driver_id_aesni_mb; 88 extern uint8_t pmd_driver_id_aesni_gcm; 89 extern uint8_t pmd_driver_id_kasumi; 90 extern uint8_t pmd_driver_id_snow3g; 91 extern uint8_t pmd_driver_id_zuc; 92 extern uint8_t pmd_driver_id_chacha20_poly1305; 93 94 /** Helper function. Gets driver ID based on PMD type */ 95 static __rte_always_inline uint8_t 96 ipsec_mb_get_driver_id(enum ipsec_mb_pmd_types pmd_type) 97 { 98 switch (pmd_type) { 99 case IPSEC_MB_PMD_TYPE_AESNI_MB: 100 return pmd_driver_id_aesni_mb; 101 case IPSEC_MB_PMD_TYPE_AESNI_GCM: 102 return pmd_driver_id_aesni_gcm; 103 case IPSEC_MB_PMD_TYPE_KASUMI: 104 return pmd_driver_id_kasumi; 105 case IPSEC_MB_PMD_TYPE_SNOW3G: 106 return pmd_driver_id_snow3g; 107 case IPSEC_MB_PMD_TYPE_ZUC: 108 return pmd_driver_id_zuc; 109 case IPSEC_MB_PMD_TYPE_CHACHA20_POLY1305: 110 return pmd_driver_id_chacha20_poly1305; 111 default: 112 break; 113 } 114 return UINT8_MAX; 115 } 116 117 /** Common private data structure for each PMD */ 118 struct ipsec_mb_dev_private { 119 enum ipsec_mb_pmd_types pmd_type; 120 /**< PMD type */ 121 uint32_t max_nb_queue_pairs; 122 /**< Max number of queue pairs supported by device */ 123 __extension__ uint8_t priv[0]; 124 }; 125 126 /** IPSEC Multi buffer queue pair common queue pair data for all PMDs */ 127 struct ipsec_mb_qp { 128 uint16_t id; 129 /**< Queue Pair Identifier */ 130 char name[RTE_CRYPTODEV_NAME_MAX_LEN]; 131 struct rte_ring *ingress_queue; 132 /**< Ring for placing operations ready for processing */ 133 struct rte_mempool *sess_mp; 134 /**< Session Mempool */ 135 struct rte_mempool *sess_mp_priv; 136 /**< Session Private Data Mempool */ 137 struct rte_cryptodev_stats stats; 138 /**< Queue pair statistics */ 139 enum ipsec_mb_pmd_types pmd_type; 140 /**< pmd type */ 141 uint8_t digest_idx; 142 /**< Index of the next 143 * slot to be used in temp_digests, 144 * to store the digest for a given operation 145 */ 146 IMB_MGR *mb_mgr; 147 /* Multi buffer manager */ 148 const struct rte_memzone *mb_mgr_mz; 149 /* Shared memzone for storing mb_mgr */ 150 __extension__ uint8_t additional_data[0]; 151 /**< Storing PMD specific additional data */ 152 }; 153 154 static __rte_always_inline void * 155 ipsec_mb_get_qp_private_data(struct ipsec_mb_qp *qp) 156 { 157 return (void *)qp->additional_data; 158 } 159 160 /** Helper function. Allocates job manager */ 161 static __rte_always_inline IMB_MGR * 162 alloc_init_mb_mgr(void) 163 { 164 IMB_MGR *mb_mgr = alloc_mb_mgr(0); 165 166 if (unlikely(mb_mgr == NULL)) { 167 IPSEC_MB_LOG(ERR, "Failed to allocate IMB_MGR data\n"); 168 return NULL; 169 } 170 171 init_mb_mgr_auto(mb_mgr, NULL); 172 173 return mb_mgr; 174 } 175 176 /** Helper function. Gets per thread job manager */ 177 static __rte_always_inline IMB_MGR * 178 get_per_thread_mb_mgr(void) 179 { 180 if (unlikely(RTE_PER_LCORE(mb_mgr) == NULL)) 181 RTE_PER_LCORE(mb_mgr) = alloc_init_mb_mgr(); 182 183 return RTE_PER_LCORE(mb_mgr); 184 } 185 186 /** Helper function. Gets mode and chained xforms from the xform */ 187 static __rte_always_inline int 188 ipsec_mb_parse_xform(const struct rte_crypto_sym_xform *xform, 189 enum ipsec_mb_operation *mode, 190 const struct rte_crypto_sym_xform **auth_xform, 191 const struct rte_crypto_sym_xform **cipher_xform, 192 const struct rte_crypto_sym_xform **aead_xform) 193 { 194 if (xform == NULL) { 195 *mode = IPSEC_MB_OP_NOT_SUPPORTED; 196 return -ENOTSUP; 197 } 198 199 const struct rte_crypto_sym_xform *next = xform->next; 200 201 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 202 if (next == NULL) { 203 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 204 *mode = IPSEC_MB_OP_ENCRYPT_ONLY; 205 *cipher_xform = xform; 206 *auth_xform = NULL; 207 return 0; 208 } 209 *mode = IPSEC_MB_OP_DECRYPT_ONLY; 210 *cipher_xform = xform; 211 *auth_xform = NULL; 212 return 0; 213 } 214 215 if (next->type != RTE_CRYPTO_SYM_XFORM_AUTH) { 216 *mode = IPSEC_MB_OP_NOT_SUPPORTED; 217 return -ENOTSUP; 218 } 219 220 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 221 if (next->auth.op != RTE_CRYPTO_AUTH_OP_GENERATE) { 222 *mode = IPSEC_MB_OP_NOT_SUPPORTED; 223 return -ENOTSUP; 224 } 225 226 *mode = IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN; 227 *cipher_xform = xform; 228 *auth_xform = xform->next; 229 return 0; 230 } 231 if (next->auth.op != RTE_CRYPTO_AUTH_OP_VERIFY) { 232 *mode = IPSEC_MB_OP_NOT_SUPPORTED; 233 return -ENOTSUP; 234 } 235 236 *mode = IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY; 237 *cipher_xform = xform; 238 *auth_xform = xform->next; 239 return 0; 240 } 241 242 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 243 if (next == NULL) { 244 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) { 245 *mode = IPSEC_MB_OP_HASH_GEN_ONLY; 246 *auth_xform = xform; 247 *cipher_xform = NULL; 248 return 0; 249 } 250 *mode = IPSEC_MB_OP_HASH_VERIFY_ONLY; 251 *auth_xform = xform; 252 *cipher_xform = NULL; 253 return 0; 254 } 255 256 if (next->type != RTE_CRYPTO_SYM_XFORM_CIPHER) { 257 *mode = IPSEC_MB_OP_NOT_SUPPORTED; 258 return -ENOTSUP; 259 } 260 261 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) { 262 if (next->cipher.op != RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 263 *mode = IPSEC_MB_OP_NOT_SUPPORTED; 264 return -ENOTSUP; 265 } 266 267 *mode = IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT; 268 *auth_xform = xform; 269 *cipher_xform = xform->next; 270 return 0; 271 } 272 if (next->cipher.op != RTE_CRYPTO_CIPHER_OP_DECRYPT) { 273 *mode = IPSEC_MB_OP_NOT_SUPPORTED; 274 return -ENOTSUP; 275 } 276 277 *mode = IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT; 278 *auth_xform = xform; 279 *cipher_xform = xform->next; 280 return 0; 281 } 282 283 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { 284 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { 285 /* 286 * CCM requires to hash first and cipher later 287 * when encrypting 288 */ 289 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) { 290 *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT; 291 *aead_xform = xform; 292 return 0; 293 } else { 294 *mode = 295 IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT; 296 *aead_xform = xform; 297 return 0; 298 } 299 } else { 300 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) { 301 *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT; 302 *aead_xform = xform; 303 return 0; 304 } 305 *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT; 306 *aead_xform = xform; 307 return 0; 308 } 309 } 310 311 *mode = IPSEC_MB_OP_NOT_SUPPORTED; 312 return -ENOTSUP; 313 } 314 315 /** Device creation function */ 316 int 317 ipsec_mb_create(struct rte_vdev_device *vdev, 318 enum ipsec_mb_pmd_types pmd_type); 319 320 /** Device remove function */ 321 int 322 ipsec_mb_remove(struct rte_vdev_device *vdev); 323 324 /** Configure queue pair PMD type specific data */ 325 typedef int (*ipsec_mb_queue_pair_configure_t)(struct ipsec_mb_qp *qp); 326 327 /** Configure session PMD type specific data */ 328 typedef int (*ipsec_mb_session_configure_t)(IMB_MGR *mbr_mgr, 329 void *session_private, 330 const struct rte_crypto_sym_xform *xform); 331 332 /** Configure internals PMD type specific data */ 333 typedef int (*ipsec_mb_dev_configure_t)(struct rte_cryptodev *dev); 334 335 /** Per PMD type operation and data */ 336 struct ipsec_mb_internals { 337 uint8_t is_configured; 338 dequeue_pkt_burst_t dequeue_burst; 339 ipsec_mb_dev_configure_t dev_config; 340 ipsec_mb_queue_pair_configure_t queue_pair_configure; 341 ipsec_mb_session_configure_t session_configure; 342 const struct rte_cryptodev_capabilities *caps; 343 struct rte_cryptodev_ops *ops; 344 struct rte_security_ops *security_ops; 345 uint64_t feature_flags; 346 uint32_t session_priv_size; 347 uint32_t qp_priv_size; 348 uint32_t internals_priv_size; 349 }; 350 351 /** Global PMD type specific data */ 352 extern struct ipsec_mb_internals ipsec_mb_pmds[IPSEC_MB_N_PMD_TYPES]; 353 354 int 355 ipsec_mb_config(struct rte_cryptodev *dev, 356 struct rte_cryptodev_config *config); 357 358 int 359 ipsec_mb_start(struct rte_cryptodev *dev); 360 361 void 362 ipsec_mb_stop(struct rte_cryptodev *dev); 363 364 int 365 ipsec_mb_close(struct rte_cryptodev *dev); 366 367 void 368 ipsec_mb_stats_get(struct rte_cryptodev *dev, 369 struct rte_cryptodev_stats *stats); 370 371 void 372 ipsec_mb_stats_reset(struct rte_cryptodev *dev); 373 374 void 375 ipsec_mb_info_get(struct rte_cryptodev *dev, 376 struct rte_cryptodev_info *dev_info); 377 378 int 379 ipsec_mb_qp_release(struct rte_cryptodev *dev, uint16_t qp_id); 380 381 int 382 ipsec_mb_qp_set_unique_name(struct rte_cryptodev *dev, struct ipsec_mb_qp *qp); 383 384 int 385 ipsec_mb_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, 386 const struct rte_cryptodev_qp_conf *qp_conf, 387 int socket_id); 388 389 /** Returns the size of the aesni multi-buffer session structure */ 390 unsigned 391 ipsec_mb_sym_session_get_size(struct rte_cryptodev *dev); 392 393 /** Configure an aesni multi-buffer session from a crypto xform chain */ 394 int ipsec_mb_sym_session_configure( 395 struct rte_cryptodev *dev, 396 struct rte_crypto_sym_xform *xform, 397 struct rte_cryptodev_sym_session *sess, 398 struct rte_mempool *mempool); 399 400 /** Clear the memory of session so it does not leave key material behind */ 401 void 402 ipsec_mb_sym_session_clear(struct rte_cryptodev *dev, 403 struct rte_cryptodev_sym_session *sess); 404 405 /** Get session from op. If sessionless create a session */ 406 static __rte_always_inline void * 407 ipsec_mb_get_session_private(struct ipsec_mb_qp *qp, struct rte_crypto_op *op) 408 { 409 void *sess = NULL; 410 uint32_t driver_id = ipsec_mb_get_driver_id(qp->pmd_type); 411 struct rte_crypto_sym_op *sym_op = op->sym; 412 uint8_t sess_type = op->sess_type; 413 void *_sess; 414 void *_sess_private_data = NULL; 415 struct ipsec_mb_internals *pmd_data = &ipsec_mb_pmds[qp->pmd_type]; 416 417 switch (sess_type) { 418 case RTE_CRYPTO_OP_WITH_SESSION: 419 if (likely(sym_op->session != NULL)) 420 sess = get_sym_session_private_data(sym_op->session, 421 driver_id); 422 break; 423 case RTE_CRYPTO_OP_SESSIONLESS: 424 if (!qp->sess_mp || 425 rte_mempool_get(qp->sess_mp, (void **)&_sess)) 426 return NULL; 427 428 if (!qp->sess_mp_priv || 429 rte_mempool_get(qp->sess_mp_priv, 430 (void **)&_sess_private_data)) 431 return NULL; 432 433 sess = _sess_private_data; 434 if (unlikely(pmd_data->session_configure(qp->mb_mgr, 435 sess, sym_op->xform) != 0)) { 436 rte_mempool_put(qp->sess_mp, _sess); 437 rte_mempool_put(qp->sess_mp_priv, _sess_private_data); 438 sess = NULL; 439 } 440 441 sym_op->session = (struct rte_cryptodev_sym_session *)_sess; 442 set_sym_session_private_data(sym_op->session, driver_id, 443 _sess_private_data); 444 break; 445 default: 446 IPSEC_MB_LOG(ERR, "Unrecognized session type %u", sess_type); 447 } 448 449 if (unlikely(sess == NULL)) 450 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 451 452 return sess; 453 } 454 455 #endif /* _IPSEC_MB_PRIVATE_H_ */ 456