1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <string.h> 35 #include <stdio.h> 36 #include <stdlib.h> 37 #include <stdint.h> 38 #include <inttypes.h> 39 #include <stdarg.h> 40 #include <errno.h> 41 #include <sys/queue.h> 42 43 #include <rte_common.h> 44 #include <rte_log.h> 45 #include <rte_debug.h> 46 #include <rte_memory.h> 47 #include <rte_memzone.h> 48 #include <rte_launch.h> 49 #include <rte_cycles.h> 50 #include <rte_eal.h> 51 #include <rte_per_lcore.h> 52 #include <rte_lcore.h> 53 #include <rte_atomic.h> 54 #include <rte_branch_prediction.h> 55 #include <rte_mempool.h> 56 #include <rte_spinlock.h> 57 #include <rte_malloc.h> 58 59 #include "test.h" 60 61 /* 62 * Mempool 63 * ======= 64 * 65 * Basic tests: done on one core with and without cache: 66 * 67 * - Get one object, put one object 68 * - Get two objects, put two objects 69 * - Get all objects, test that their content is not modified and 70 * put them back in the pool. 71 */ 72 73 #define MEMPOOL_ELT_SIZE 2048 74 #define MAX_KEEP 16 75 #define MEMPOOL_SIZE ((rte_lcore_count()*(MAX_KEEP+RTE_MEMPOOL_CACHE_MAX_SIZE))-1) 76 77 #define LOG_ERR() printf("test failed at %s():%d\n", __func__, __LINE__) 78 #define RET_ERR() do { \ 79 LOG_ERR(); \ 80 return -1; \ 81 } while (0) 82 #define GOTO_ERR(var, label) do { \ 83 LOG_ERR(); \ 84 var = -1; \ 85 goto label; \ 86 } while (0) 87 88 static rte_atomic32_t synchro; 89 90 /* 91 * save the object number in the first 4 bytes of object data. All 92 * other bytes are set to 0. 93 */ 94 static void 95 my_obj_init(struct rte_mempool *mp, __attribute__((unused)) void *arg, 96 void *obj, unsigned i) 97 { 98 uint32_t *objnum = obj; 99 100 memset(obj, 0, mp->elt_size); 101 *objnum = i; 102 } 103 104 /* basic tests (done on one core) */ 105 static int 106 test_mempool_basic(struct rte_mempool *mp, int use_external_cache) 107 { 108 uint32_t *objnum; 109 void **objtable; 110 void *obj, *obj2; 111 char *obj_data; 112 int ret = 0; 113 unsigned i, j; 114 int offset; 115 struct rte_mempool_cache *cache; 116 117 if (use_external_cache) { 118 /* Create a user-owned mempool cache. */ 119 cache = rte_mempool_cache_create(RTE_MEMPOOL_CACHE_MAX_SIZE, 120 SOCKET_ID_ANY); 121 if (cache == NULL) 122 RET_ERR(); 123 } else { 124 /* May be NULL if cache is disabled. */ 125 cache = rte_mempool_default_cache(mp, rte_lcore_id()); 126 } 127 128 /* dump the mempool status */ 129 rte_mempool_dump(stdout, mp); 130 131 printf("get an object\n"); 132 if (rte_mempool_generic_get(mp, &obj, 1, cache, 0) < 0) 133 GOTO_ERR(ret, out); 134 rte_mempool_dump(stdout, mp); 135 136 /* tests that improve coverage */ 137 printf("get object count\n"); 138 /* We have to count the extra caches, one in this case. */ 139 offset = use_external_cache ? 1 * cache->len : 0; 140 if (rte_mempool_avail_count(mp) + offset != MEMPOOL_SIZE - 1) 141 GOTO_ERR(ret, out); 142 143 printf("get private data\n"); 144 if (rte_mempool_get_priv(mp) != (char *)mp + 145 MEMPOOL_HEADER_SIZE(mp, mp->cache_size)) 146 GOTO_ERR(ret, out); 147 148 #ifndef RTE_EXEC_ENV_BSDAPP /* rte_mem_virt2phy() not supported on bsd */ 149 printf("get physical address of an object\n"); 150 if (rte_mempool_virt2phy(mp, obj) != rte_mem_virt2phy(obj)) 151 GOTO_ERR(ret, out); 152 #endif 153 154 printf("put the object back\n"); 155 rte_mempool_generic_put(mp, &obj, 1, cache, 0); 156 rte_mempool_dump(stdout, mp); 157 158 printf("get 2 objects\n"); 159 if (rte_mempool_generic_get(mp, &obj, 1, cache, 0) < 0) 160 GOTO_ERR(ret, out); 161 if (rte_mempool_generic_get(mp, &obj2, 1, cache, 0) < 0) { 162 rte_mempool_generic_put(mp, &obj, 1, cache, 0); 163 GOTO_ERR(ret, out); 164 } 165 rte_mempool_dump(stdout, mp); 166 167 printf("put the objects back\n"); 168 rte_mempool_generic_put(mp, &obj, 1, cache, 0); 169 rte_mempool_generic_put(mp, &obj2, 1, cache, 0); 170 rte_mempool_dump(stdout, mp); 171 172 /* 173 * get many objects: we cannot get them all because the cache 174 * on other cores may not be empty. 175 */ 176 objtable = malloc(MEMPOOL_SIZE * sizeof(void *)); 177 if (objtable == NULL) 178 GOTO_ERR(ret, out); 179 180 for (i = 0; i < MEMPOOL_SIZE; i++) { 181 if (rte_mempool_generic_get(mp, &objtable[i], 1, cache, 0) < 0) 182 break; 183 } 184 185 /* 186 * for each object, check that its content was not modified, 187 * and put objects back in pool 188 */ 189 while (i--) { 190 obj = objtable[i]; 191 obj_data = obj; 192 objnum = obj; 193 if (*objnum > MEMPOOL_SIZE) { 194 printf("bad object number(%d)\n", *objnum); 195 ret = -1; 196 break; 197 } 198 for (j = sizeof(*objnum); j < mp->elt_size; j++) { 199 if (obj_data[j] != 0) 200 ret = -1; 201 } 202 203 rte_mempool_generic_put(mp, &objtable[i], 1, cache, 0); 204 } 205 206 free(objtable); 207 if (ret == -1) 208 printf("objects were modified!\n"); 209 210 out: 211 if (use_external_cache) { 212 rte_mempool_cache_flush(cache, mp); 213 rte_mempool_cache_free(cache); 214 } 215 216 return ret; 217 } 218 219 static int test_mempool_creation_with_exceeded_cache_size(void) 220 { 221 struct rte_mempool *mp_cov; 222 223 mp_cov = rte_mempool_create("test_mempool_cache_too_big", 224 MEMPOOL_SIZE, 225 MEMPOOL_ELT_SIZE, 226 RTE_MEMPOOL_CACHE_MAX_SIZE + 32, 0, 227 NULL, NULL, 228 my_obj_init, NULL, 229 SOCKET_ID_ANY, 0); 230 231 if (mp_cov != NULL) { 232 rte_mempool_free(mp_cov); 233 RET_ERR(); 234 } 235 236 return 0; 237 } 238 239 static struct rte_mempool *mp_spsc; 240 static rte_spinlock_t scsp_spinlock; 241 static void *scsp_obj_table[MAX_KEEP]; 242 243 /* 244 * single producer function 245 */ 246 static int test_mempool_single_producer(void) 247 { 248 unsigned int i; 249 void *obj = NULL; 250 uint64_t start_cycles, end_cycles; 251 uint64_t duration = rte_get_timer_hz() / 4; 252 253 start_cycles = rte_get_timer_cycles(); 254 while (1) { 255 end_cycles = rte_get_timer_cycles(); 256 /* duration uses up, stop producing */ 257 if (start_cycles + duration < end_cycles) 258 break; 259 rte_spinlock_lock(&scsp_spinlock); 260 for (i = 0; i < MAX_KEEP; i ++) { 261 if (NULL != scsp_obj_table[i]) { 262 obj = scsp_obj_table[i]; 263 break; 264 } 265 } 266 rte_spinlock_unlock(&scsp_spinlock); 267 if (i >= MAX_KEEP) { 268 continue; 269 } 270 if (rte_mempool_from_obj(obj) != mp_spsc) { 271 printf("obj not owned by this mempool\n"); 272 RET_ERR(); 273 } 274 rte_mempool_put(mp_spsc, obj); 275 rte_spinlock_lock(&scsp_spinlock); 276 scsp_obj_table[i] = NULL; 277 rte_spinlock_unlock(&scsp_spinlock); 278 } 279 280 return 0; 281 } 282 283 /* 284 * single consumer function 285 */ 286 static int test_mempool_single_consumer(void) 287 { 288 unsigned int i; 289 void * obj; 290 uint64_t start_cycles, end_cycles; 291 uint64_t duration = rte_get_timer_hz() / 8; 292 293 start_cycles = rte_get_timer_cycles(); 294 while (1) { 295 end_cycles = rte_get_timer_cycles(); 296 /* duration uses up, stop consuming */ 297 if (start_cycles + duration < end_cycles) 298 break; 299 rte_spinlock_lock(&scsp_spinlock); 300 for (i = 0; i < MAX_KEEP; i ++) { 301 if (NULL == scsp_obj_table[i]) 302 break; 303 } 304 rte_spinlock_unlock(&scsp_spinlock); 305 if (i >= MAX_KEEP) 306 continue; 307 if (rte_mempool_get(mp_spsc, &obj) < 0) 308 break; 309 rte_spinlock_lock(&scsp_spinlock); 310 scsp_obj_table[i] = obj; 311 rte_spinlock_unlock(&scsp_spinlock); 312 } 313 314 return 0; 315 } 316 317 /* 318 * test function for mempool test based on singple consumer and single producer, 319 * can run on one lcore only 320 */ 321 static int 322 test_mempool_launch_single_consumer(__attribute__((unused)) void *arg) 323 { 324 return test_mempool_single_consumer(); 325 } 326 327 static void 328 my_mp_init(struct rte_mempool *mp, __attribute__((unused)) void *arg) 329 { 330 printf("mempool name is %s\n", mp->name); 331 /* nothing to be implemented here*/ 332 return ; 333 } 334 335 /* 336 * it tests the mempool operations based on singple producer and single consumer 337 */ 338 static int 339 test_mempool_sp_sc(void) 340 { 341 int ret = 0; 342 unsigned lcore_id = rte_lcore_id(); 343 unsigned lcore_next; 344 345 /* create a mempool with single producer/consumer ring */ 346 if (mp_spsc == NULL) { 347 mp_spsc = rte_mempool_create("test_mempool_sp_sc", MEMPOOL_SIZE, 348 MEMPOOL_ELT_SIZE, 0, 0, 349 my_mp_init, NULL, 350 my_obj_init, NULL, 351 SOCKET_ID_ANY, 352 MEMPOOL_F_NO_CACHE_ALIGN | MEMPOOL_F_SP_PUT | 353 MEMPOOL_F_SC_GET); 354 if (mp_spsc == NULL) 355 RET_ERR(); 356 } 357 if (rte_mempool_lookup("test_mempool_sp_sc") != mp_spsc) { 358 printf("Cannot lookup mempool from its name\n"); 359 rte_mempool_free(mp_spsc); 360 RET_ERR(); 361 } 362 lcore_next = rte_get_next_lcore(lcore_id, 0, 1); 363 if (lcore_next >= RTE_MAX_LCORE) { 364 rte_mempool_free(mp_spsc); 365 RET_ERR(); 366 } 367 if (rte_eal_lcore_role(lcore_next) != ROLE_RTE) { 368 rte_mempool_free(mp_spsc); 369 RET_ERR(); 370 } 371 rte_spinlock_init(&scsp_spinlock); 372 memset(scsp_obj_table, 0, sizeof(scsp_obj_table)); 373 rte_eal_remote_launch(test_mempool_launch_single_consumer, NULL, 374 lcore_next); 375 if (test_mempool_single_producer() < 0) 376 ret = -1; 377 378 if (rte_eal_wait_lcore(lcore_next) < 0) 379 ret = -1; 380 rte_mempool_free(mp_spsc); 381 382 return ret; 383 } 384 385 /* 386 * it tests some more basic of mempool 387 */ 388 static int 389 test_mempool_basic_ex(struct rte_mempool *mp) 390 { 391 unsigned i; 392 void **obj; 393 void *err_obj; 394 int ret = -1; 395 396 if (mp == NULL) 397 return ret; 398 399 obj = rte_calloc("test_mempool_basic_ex", MEMPOOL_SIZE, 400 sizeof(void *), 0); 401 if (obj == NULL) { 402 printf("test_mempool_basic_ex fail to rte_malloc\n"); 403 return ret; 404 } 405 printf("test_mempool_basic_ex now mempool (%s) has %u free entries\n", 406 mp->name, rte_mempool_in_use_count(mp)); 407 if (rte_mempool_full(mp) != 1) { 408 printf("test_mempool_basic_ex the mempool should be full\n"); 409 goto fail_mp_basic_ex; 410 } 411 412 for (i = 0; i < MEMPOOL_SIZE; i ++) { 413 if (rte_mempool_get(mp, &obj[i]) < 0) { 414 printf("test_mp_basic_ex fail to get object for [%u]\n", 415 i); 416 goto fail_mp_basic_ex; 417 } 418 } 419 if (rte_mempool_get(mp, &err_obj) == 0) { 420 printf("test_mempool_basic_ex get an impossible obj\n"); 421 goto fail_mp_basic_ex; 422 } 423 printf("number: %u\n", i); 424 if (rte_mempool_empty(mp) != 1) { 425 printf("test_mempool_basic_ex the mempool should be empty\n"); 426 goto fail_mp_basic_ex; 427 } 428 429 for (i = 0; i < MEMPOOL_SIZE; i++) 430 rte_mempool_put(mp, obj[i]); 431 432 if (rte_mempool_full(mp) != 1) { 433 printf("test_mempool_basic_ex the mempool should be full\n"); 434 goto fail_mp_basic_ex; 435 } 436 437 ret = 0; 438 439 fail_mp_basic_ex: 440 if (obj != NULL) 441 rte_free((void *)obj); 442 443 return ret; 444 } 445 446 static int 447 test_mempool_same_name_twice_creation(void) 448 { 449 struct rte_mempool *mp_tc, *mp_tc2; 450 451 mp_tc = rte_mempool_create("test_mempool_same_name", MEMPOOL_SIZE, 452 MEMPOOL_ELT_SIZE, 0, 0, 453 NULL, NULL, 454 NULL, NULL, 455 SOCKET_ID_ANY, 0); 456 457 if (mp_tc == NULL) 458 RET_ERR(); 459 460 mp_tc2 = rte_mempool_create("test_mempool_same_name", MEMPOOL_SIZE, 461 MEMPOOL_ELT_SIZE, 0, 0, 462 NULL, NULL, 463 NULL, NULL, 464 SOCKET_ID_ANY, 0); 465 466 if (mp_tc2 != NULL) { 467 rte_mempool_free(mp_tc); 468 rte_mempool_free(mp_tc2); 469 RET_ERR(); 470 } 471 472 rte_mempool_free(mp_tc); 473 return 0; 474 } 475 476 /* 477 * BAsic test for mempool_xmem functions. 478 */ 479 static int 480 test_mempool_xmem_misc(void) 481 { 482 uint32_t elt_num, total_size; 483 size_t sz; 484 ssize_t usz; 485 486 elt_num = MAX_KEEP; 487 total_size = rte_mempool_calc_obj_size(MEMPOOL_ELT_SIZE, 0, NULL); 488 sz = rte_mempool_xmem_size(elt_num, total_size, MEMPOOL_PG_SHIFT_MAX); 489 490 usz = rte_mempool_xmem_usage(NULL, elt_num, total_size, 0, 1, 491 MEMPOOL_PG_SHIFT_MAX); 492 493 if (sz != (size_t)usz) { 494 printf("failure @ %s: rte_mempool_xmem_usage(%u, %u) " 495 "returns: %#zx, while expected: %#zx;\n", 496 __func__, elt_num, total_size, sz, (size_t)usz); 497 return -1; 498 } 499 500 return 0; 501 } 502 503 static int 504 test_mempool(void) 505 { 506 struct rte_mempool *mp_cache = NULL; 507 struct rte_mempool *mp_nocache = NULL; 508 struct rte_mempool *mp_stack = NULL; 509 510 rte_atomic32_init(&synchro); 511 512 /* create a mempool (without cache) */ 513 mp_nocache = rte_mempool_create("test_nocache", MEMPOOL_SIZE, 514 MEMPOOL_ELT_SIZE, 0, 0, 515 NULL, NULL, 516 my_obj_init, NULL, 517 SOCKET_ID_ANY, 0); 518 519 if (mp_nocache == NULL) { 520 printf("cannot allocate mp_nocache mempool\n"); 521 goto err; 522 } 523 524 /* create a mempool (with cache) */ 525 mp_cache = rte_mempool_create("test_cache", MEMPOOL_SIZE, 526 MEMPOOL_ELT_SIZE, 527 RTE_MEMPOOL_CACHE_MAX_SIZE, 0, 528 NULL, NULL, 529 my_obj_init, NULL, 530 SOCKET_ID_ANY, 0); 531 532 if (mp_cache == NULL) { 533 printf("cannot allocate mp_cache mempool\n"); 534 goto err; 535 } 536 537 /* create a mempool with an external handler */ 538 mp_stack = rte_mempool_create_empty("test_stack", 539 MEMPOOL_SIZE, 540 MEMPOOL_ELT_SIZE, 541 RTE_MEMPOOL_CACHE_MAX_SIZE, 0, 542 SOCKET_ID_ANY, 0); 543 544 if (mp_stack == NULL) { 545 printf("cannot allocate mp_stack mempool\n"); 546 goto err; 547 } 548 if (rte_mempool_set_ops_byname(mp_stack, "stack", NULL) < 0) { 549 printf("cannot set stack handler\n"); 550 goto err; 551 } 552 if (rte_mempool_populate_default(mp_stack) < 0) { 553 printf("cannot populate mp_stack mempool\n"); 554 goto err; 555 } 556 rte_mempool_obj_iter(mp_stack, my_obj_init, NULL); 557 558 /* retrieve the mempool from its name */ 559 if (rte_mempool_lookup("test_nocache") != mp_nocache) { 560 printf("Cannot lookup mempool from its name\n"); 561 goto err; 562 } 563 564 rte_mempool_list_dump(stdout); 565 566 /* basic tests without cache */ 567 if (test_mempool_basic(mp_nocache, 0) < 0) 568 goto err; 569 570 /* basic tests with cache */ 571 if (test_mempool_basic(mp_cache, 0) < 0) 572 goto err; 573 574 /* basic tests with user-owned cache */ 575 if (test_mempool_basic(mp_nocache, 1) < 0) 576 goto err; 577 578 /* more basic tests without cache */ 579 if (test_mempool_basic_ex(mp_nocache) < 0) 580 goto err; 581 582 /* mempool operation test based on single producer and single comsumer */ 583 if (test_mempool_sp_sc() < 0) 584 goto err; 585 586 if (test_mempool_creation_with_exceeded_cache_size() < 0) 587 goto err; 588 589 if (test_mempool_same_name_twice_creation() < 0) 590 goto err; 591 592 if (test_mempool_xmem_misc() < 0) 593 goto err; 594 595 /* test the stack handler */ 596 if (test_mempool_basic(mp_stack, 1) < 0) 597 goto err; 598 599 rte_mempool_list_dump(stdout); 600 601 return 0; 602 603 err: 604 rte_mempool_free(mp_nocache); 605 rte_mempool_free(mp_cache); 606 rte_mempool_free(mp_stack); 607 return -1; 608 } 609 610 REGISTER_TEST_COMMAND(mempool_autotest, test_mempool); 611