1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_cycles.h> 7 #include <rte_crypto.h> 8 #include <rte_cryptodev.h> 9 10 #include "cperf_test_verify.h" 11 #include "cperf_ops.h" 12 #include "cperf_test_common.h" 13 14 struct cperf_verify_ctx { 15 uint8_t dev_id; 16 uint16_t qp_id; 17 uint8_t lcore_id; 18 19 struct rte_mempool *pool; 20 21 struct rte_cryptodev_sym_session *sess; 22 23 cperf_populate_ops_t populate_ops; 24 25 uint32_t src_buf_offset; 26 uint32_t dst_buf_offset; 27 28 const struct cperf_options *options; 29 const struct cperf_test_vector *test_vector; 30 }; 31 32 struct cperf_op_result { 33 enum rte_crypto_op_status status; 34 }; 35 36 static void 37 cperf_verify_test_free(struct cperf_verify_ctx *ctx) 38 { 39 if (ctx) { 40 if (ctx->sess) { 41 rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); 42 rte_cryptodev_sym_session_free(ctx->sess); 43 } 44 45 rte_mempool_free(ctx->pool); 46 47 rte_free(ctx); 48 } 49 } 50 51 void * 52 cperf_verify_test_constructor(struct rte_mempool *sess_mp, 53 struct rte_mempool *sess_priv_mp, 54 uint8_t dev_id, uint16_t qp_id, 55 const struct cperf_options *options, 56 const struct cperf_test_vector *test_vector, 57 const struct cperf_op_fns *op_fns) 58 { 59 struct cperf_verify_ctx *ctx = NULL; 60 61 ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0); 62 if (ctx == NULL) 63 goto err; 64 65 ctx->dev_id = dev_id; 66 ctx->qp_id = qp_id; 67 68 ctx->populate_ops = op_fns->populate_ops; 69 ctx->options = options; 70 ctx->test_vector = test_vector; 71 72 /* IV goes at the end of the crypto operation */ 73 uint16_t iv_offset = sizeof(struct rte_crypto_op) + 74 sizeof(struct rte_crypto_sym_op); 75 76 ctx->sess = op_fns->sess_create(sess_mp, sess_priv_mp, dev_id, options, 77 test_vector, iv_offset); 78 if (ctx->sess == NULL) 79 goto err; 80 81 if (cperf_alloc_common_memory(options, test_vector, dev_id, qp_id, 0, 82 &ctx->src_buf_offset, &ctx->dst_buf_offset, 83 &ctx->pool) < 0) 84 goto err; 85 86 return ctx; 87 err: 88 cperf_verify_test_free(ctx); 89 90 return NULL; 91 } 92 93 static int 94 cperf_verify_op(struct rte_crypto_op *op, 95 const struct cperf_options *options, 96 const struct cperf_test_vector *vector) 97 { 98 const struct rte_mbuf *m; 99 uint32_t len; 100 uint16_t nb_segs; 101 uint8_t *data; 102 uint32_t cipher_offset, auth_offset; 103 uint8_t cipher, auth; 104 int res = 0; 105 106 if (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS) 107 return 1; 108 109 if (op->sym->m_dst) 110 m = op->sym->m_dst; 111 else 112 m = op->sym->m_src; 113 nb_segs = m->nb_segs; 114 len = 0; 115 while (m && nb_segs != 0) { 116 len += m->data_len; 117 m = m->next; 118 nb_segs--; 119 } 120 121 data = rte_malloc(NULL, len, 0); 122 if (data == NULL) 123 return 1; 124 125 if (op->sym->m_dst) 126 m = op->sym->m_dst; 127 else 128 m = op->sym->m_src; 129 nb_segs = m->nb_segs; 130 len = 0; 131 while (m && nb_segs != 0) { 132 memcpy(data + len, rte_pktmbuf_mtod(m, uint8_t *), 133 m->data_len); 134 len += m->data_len; 135 m = m->next; 136 nb_segs--; 137 } 138 139 switch (options->op_type) { 140 case CPERF_CIPHER_ONLY: 141 cipher = 1; 142 cipher_offset = 0; 143 auth = 0; 144 auth_offset = 0; 145 break; 146 case CPERF_CIPHER_THEN_AUTH: 147 cipher = 1; 148 cipher_offset = 0; 149 auth = 1; 150 auth_offset = options->test_buffer_size; 151 break; 152 case CPERF_AUTH_ONLY: 153 cipher = 0; 154 cipher_offset = 0; 155 auth = 1; 156 auth_offset = options->test_buffer_size; 157 break; 158 case CPERF_AUTH_THEN_CIPHER: 159 cipher = 1; 160 cipher_offset = 0; 161 auth = 1; 162 auth_offset = options->test_buffer_size; 163 break; 164 case CPERF_AEAD: 165 cipher = 1; 166 cipher_offset = 0; 167 auth = 1; 168 auth_offset = options->test_buffer_size; 169 break; 170 default: 171 res = 1; 172 goto out; 173 } 174 175 if (cipher == 1) { 176 if (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 177 res += memcmp(data + cipher_offset, 178 vector->ciphertext.data, 179 options->test_buffer_size); 180 else 181 res += memcmp(data + cipher_offset, 182 vector->plaintext.data, 183 options->test_buffer_size); 184 } 185 186 if (auth == 1) { 187 if (options->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE) 188 res += memcmp(data + auth_offset, 189 vector->digest.data, 190 options->digest_sz); 191 } 192 193 out: 194 rte_free(data); 195 return !!res; 196 } 197 198 static void 199 cperf_mbuf_set(struct rte_mbuf *mbuf, 200 const struct cperf_options *options, 201 const struct cperf_test_vector *test_vector) 202 { 203 uint32_t segment_sz = options->segment_sz; 204 uint8_t *mbuf_data; 205 uint8_t *test_data; 206 uint32_t remaining_bytes = options->max_buffer_size; 207 208 if (options->op_type == CPERF_AEAD) { 209 test_data = (options->aead_op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 210 test_vector->plaintext.data : 211 test_vector->ciphertext.data; 212 } else { 213 test_data = 214 (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 215 test_vector->plaintext.data : 216 test_vector->ciphertext.data; 217 } 218 219 while (remaining_bytes) { 220 mbuf_data = rte_pktmbuf_mtod(mbuf, uint8_t *); 221 222 if (remaining_bytes <= segment_sz) { 223 memcpy(mbuf_data, test_data, remaining_bytes); 224 return; 225 } 226 227 memcpy(mbuf_data, test_data, segment_sz); 228 remaining_bytes -= segment_sz; 229 test_data += segment_sz; 230 mbuf = mbuf->next; 231 } 232 } 233 234 int 235 cperf_verify_test_runner(void *test_ctx) 236 { 237 struct cperf_verify_ctx *ctx = test_ctx; 238 239 uint64_t ops_enqd = 0, ops_enqd_total = 0, ops_enqd_failed = 0; 240 uint64_t ops_deqd = 0, ops_deqd_total = 0, ops_deqd_failed = 0; 241 uint64_t ops_failed = 0; 242 243 static uint16_t display_once; 244 245 uint64_t i; 246 uint16_t ops_unused = 0; 247 uint32_t imix_idx = 0; 248 249 struct rte_crypto_op *ops[ctx->options->max_burst_size]; 250 struct rte_crypto_op *ops_processed[ctx->options->max_burst_size]; 251 252 uint32_t lcore = rte_lcore_id(); 253 254 #ifdef CPERF_LINEARIZATION_ENABLE 255 struct rte_cryptodev_info dev_info; 256 int linearize = 0; 257 258 /* Check if source mbufs require coalescing */ 259 if (ctx->options->segment_sz < ctx->options->max_buffer_size) { 260 rte_cryptodev_info_get(ctx->dev_id, &dev_info); 261 if ((dev_info.feature_flags & 262 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER) == 0) 263 linearize = 1; 264 } 265 #endif /* CPERF_LINEARIZATION_ENABLE */ 266 267 ctx->lcore_id = lcore; 268 269 if (!ctx->options->csv) 270 printf("\n# Running verify test on device: %u, lcore: %u\n", 271 ctx->dev_id, lcore); 272 273 uint16_t iv_offset = sizeof(struct rte_crypto_op) + 274 sizeof(struct rte_crypto_sym_op); 275 276 while (ops_enqd_total < ctx->options->total_ops) { 277 278 uint16_t burst_size = ((ops_enqd_total + ctx->options->max_burst_size) 279 <= ctx->options->total_ops) ? 280 ctx->options->max_burst_size : 281 ctx->options->total_ops - 282 ops_enqd_total; 283 284 uint16_t ops_needed = burst_size - ops_unused; 285 286 /* Allocate objects containing crypto operations and mbufs */ 287 if (rte_mempool_get_bulk(ctx->pool, (void **)ops, 288 ops_needed) != 0) { 289 RTE_LOG(ERR, USER1, 290 "Failed to allocate more crypto operations " 291 "from the crypto operation pool.\n" 292 "Consider increasing the pool size " 293 "with --pool-sz\n"); 294 return -1; 295 } 296 297 /* Setup crypto op, attach mbuf etc */ 298 (ctx->populate_ops)(ops, ctx->src_buf_offset, 299 ctx->dst_buf_offset, 300 ops_needed, ctx->sess, ctx->options, 301 ctx->test_vector, iv_offset, &imix_idx, NULL); 302 303 304 /* Populate the mbuf with the test vector, for verification */ 305 for (i = 0; i < ops_needed; i++) 306 cperf_mbuf_set(ops[i]->sym->m_src, 307 ctx->options, 308 ctx->test_vector); 309 310 #ifdef CPERF_LINEARIZATION_ENABLE 311 if (linearize) { 312 /* PMD doesn't support scatter-gather and source buffer 313 * is segmented. 314 * We need to linearize it before enqueuing. 315 */ 316 for (i = 0; i < burst_size; i++) 317 rte_pktmbuf_linearize(ops[i]->sym->m_src); 318 } 319 #endif /* CPERF_LINEARIZATION_ENABLE */ 320 321 /* Enqueue burst of ops on crypto device */ 322 ops_enqd = rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, 323 ops, burst_size); 324 if (ops_enqd < burst_size) 325 ops_enqd_failed++; 326 327 /** 328 * Calculate number of ops not enqueued (mainly for hw 329 * accelerators whose ingress queue can fill up). 330 */ 331 ops_unused = burst_size - ops_enqd; 332 ops_enqd_total += ops_enqd; 333 334 335 /* Dequeue processed burst of ops from crypto device */ 336 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, 337 ops_processed, ctx->options->max_burst_size); 338 339 if (ops_deqd == 0) { 340 /** 341 * Count dequeue polls which didn't return any 342 * processed operations. This statistic is mainly 343 * relevant to hw accelerators. 344 */ 345 ops_deqd_failed++; 346 continue; 347 } 348 349 for (i = 0; i < ops_deqd; i++) { 350 if (cperf_verify_op(ops_processed[i], ctx->options, 351 ctx->test_vector)) 352 ops_failed++; 353 } 354 /* Free crypto ops so they can be reused. */ 355 rte_mempool_put_bulk(ctx->pool, 356 (void **)ops_processed, ops_deqd); 357 ops_deqd_total += ops_deqd; 358 } 359 360 /* Dequeue any operations still in the crypto device */ 361 362 while (ops_deqd_total < ctx->options->total_ops) { 363 /* Sending 0 length burst to flush sw crypto device */ 364 rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0); 365 366 /* dequeue burst */ 367 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, 368 ops_processed, ctx->options->max_burst_size); 369 if (ops_deqd == 0) { 370 ops_deqd_failed++; 371 continue; 372 } 373 374 for (i = 0; i < ops_deqd; i++) { 375 if (cperf_verify_op(ops_processed[i], ctx->options, 376 ctx->test_vector)) 377 ops_failed++; 378 } 379 /* Free crypto ops so they can be reused. */ 380 rte_mempool_put_bulk(ctx->pool, 381 (void **)ops_processed, ops_deqd); 382 ops_deqd_total += ops_deqd; 383 } 384 385 uint16_t exp = 0; 386 if (!ctx->options->csv) { 387 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0, 388 __ATOMIC_RELAXED, __ATOMIC_RELAXED)) 389 printf("%12s%12s%12s%12s%12s%12s%12s%12s\n\n", 390 "lcore id", "Buf Size", "Burst size", 391 "Enqueued", "Dequeued", "Failed Enq", 392 "Failed Deq", "Failed Ops"); 393 394 printf("%12u%12u%12u%12"PRIu64"%12"PRIu64"%12"PRIu64 395 "%12"PRIu64"%12"PRIu64"\n", 396 ctx->lcore_id, 397 ctx->options->max_buffer_size, 398 ctx->options->max_burst_size, 399 ops_enqd_total, 400 ops_deqd_total, 401 ops_enqd_failed, 402 ops_deqd_failed, 403 ops_failed); 404 } else { 405 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0, 406 __ATOMIC_RELAXED, __ATOMIC_RELAXED)) 407 printf("\n# lcore id, Buffer Size(B), " 408 "Burst Size,Enqueued,Dequeued,Failed Enq," 409 "Failed Deq,Failed Ops\n"); 410 411 printf("%10u,%10u,%u,%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64"," 412 "%"PRIu64"\n", 413 ctx->lcore_id, 414 ctx->options->max_buffer_size, 415 ctx->options->max_burst_size, 416 ops_enqd_total, 417 ops_deqd_total, 418 ops_enqd_failed, 419 ops_deqd_failed, 420 ops_failed); 421 } 422 423 return 0; 424 } 425 426 427 428 void 429 cperf_verify_test_destructor(void *arg) 430 { 431 struct cperf_verify_ctx *ctx = arg; 432 433 if (ctx == NULL) 434 return; 435 436 cperf_verify_test_free(ctx); 437 } 438