1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
3 * Copyright(c) 2022 SmartShare Systems
4 */
5
6 #include <string.h>
7 #include <stdio.h>
8 #include <stdlib.h>
9 #include <stdint.h>
10 #include <inttypes.h>
11 #include <stdarg.h>
12 #include <errno.h>
13 #include <sys/queue.h>
14
15 #include <rte_common.h>
16 #include <rte_log.h>
17 #include <rte_debug.h>
18 #include <rte_memory.h>
19 #include <rte_launch.h>
20 #include <rte_cycles.h>
21 #include <rte_eal.h>
22 #include <rte_per_lcore.h>
23 #include <rte_lcore.h>
24 #include <rte_branch_prediction.h>
25 #include <rte_mempool.h>
26 #include <rte_spinlock.h>
27 #include <rte_malloc.h>
28 #include <rte_mbuf_pool_ops.h>
29
30 #include "test.h"
31
32 /*
33 * Mempool performance
34 * =======
35 *
36 * Each core get *n_keep* objects per bulk of *n_get_bulk*. Then,
37 * objects are put back in the pool per bulk of *n_put_bulk*.
38 *
39 * This sequence is done during TIME_S seconds.
40 *
41 * This test is done on the following configurations:
42 *
43 * - Cores configuration (*cores*)
44 *
45 * - One core with cache
46 * - Two cores with cache
47 * - Max. cores with cache
48 * - One core without cache
49 * - Two cores without cache
50 * - Max. cores without cache
51 * - One core with user-owned cache
52 * - Two cores with user-owned cache
53 * - Max. cores with user-owned cache
54 *
55 * - Bulk size (*n_get_bulk*, *n_put_bulk*)
56 *
57 * - Bulk get from 1 to 32
58 * - Bulk put from 1 to 32
59 * - Bulk get and put from 1 to 32, compile time constant
60 *
61 * - Number of kept objects (*n_keep*)
62 *
63 * - 32
64 * - 128
65 * - 512
66 */
67
68 #define N 65536
69 #define TIME_S 5
70 #define MEMPOOL_ELT_SIZE 2048
71 #define MAX_KEEP 512
72 #define MEMPOOL_SIZE ((rte_lcore_count()*(MAX_KEEP+RTE_MEMPOOL_CACHE_MAX_SIZE))-1)
73
74 /* Number of pointers fitting into one cache line. */
75 #define CACHE_LINE_BURST (RTE_CACHE_LINE_SIZE / sizeof(uintptr_t))
76
77 #define LOG_ERR() printf("test failed at %s():%d\n", __func__, __LINE__)
78 #define RET_ERR() do { \
79 LOG_ERR(); \
80 return -1; \
81 } while (0)
82 #define GOTO_ERR(var, label) do { \
83 LOG_ERR(); \
84 var = -1; \
85 goto label; \
86 } while (0)
87
88 static int use_external_cache;
89 static unsigned external_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE;
90
91 static uint32_t synchro;
92
93 /* number of objects in one bulk operation (get or put) */
94 static unsigned n_get_bulk;
95 static unsigned n_put_bulk;
96
97 /* number of objects retrieved from mempool before putting them back */
98 static unsigned n_keep;
99
100 /* true if we want to test with constant n_get_bulk and n_put_bulk */
101 static int use_constant_values;
102
103 /* number of enqueues / dequeues */
104 struct mempool_test_stats {
105 uint64_t enq_count;
106 } __rte_cache_aligned;
107
108 static struct mempool_test_stats stats[RTE_MAX_LCORE];
109
110 /*
111 * save the object number in the first 4 bytes of object data. All
112 * other bytes are set to 0.
113 */
114 static void
my_obj_init(struct rte_mempool * mp,__rte_unused void * arg,void * obj,unsigned i)115 my_obj_init(struct rte_mempool *mp, __rte_unused void *arg,
116 void *obj, unsigned i)
117 {
118 uint32_t *objnum = obj;
119 memset(obj, 0, mp->elt_size);
120 *objnum = i;
121 }
122
123 static __rte_always_inline int
test_loop(struct rte_mempool * mp,struct rte_mempool_cache * cache,unsigned int x_keep,unsigned int x_get_bulk,unsigned int x_put_bulk)124 test_loop(struct rte_mempool *mp, struct rte_mempool_cache *cache,
125 unsigned int x_keep, unsigned int x_get_bulk, unsigned int x_put_bulk)
126 {
127 void *obj_table[MAX_KEEP] __rte_cache_aligned;
128 unsigned int idx;
129 unsigned int i;
130 int ret;
131
132 for (i = 0; likely(i < (N / x_keep)); i++) {
133 /* get x_keep objects by bulk of x_get_bulk */
134 for (idx = 0; idx < x_keep; idx += x_get_bulk) {
135 ret = rte_mempool_generic_get(mp,
136 &obj_table[idx],
137 x_get_bulk,
138 cache);
139 if (unlikely(ret < 0)) {
140 rte_mempool_dump(stdout, mp);
141 return ret;
142 }
143 }
144
145 /* put the objects back by bulk of x_put_bulk */
146 for (idx = 0; idx < x_keep; idx += x_put_bulk) {
147 rte_mempool_generic_put(mp,
148 &obj_table[idx],
149 x_put_bulk,
150 cache);
151 }
152 }
153
154 return 0;
155 }
156
157 static int
per_lcore_mempool_test(void * arg)158 per_lcore_mempool_test(void *arg)
159 {
160 struct rte_mempool *mp = arg;
161 unsigned lcore_id = rte_lcore_id();
162 int ret = 0;
163 uint64_t start_cycles, end_cycles;
164 uint64_t time_diff = 0, hz = rte_get_timer_hz();
165 struct rte_mempool_cache *cache;
166
167 if (use_external_cache) {
168 /* Create a user-owned mempool cache. */
169 cache = rte_mempool_cache_create(external_cache_size,
170 SOCKET_ID_ANY);
171 if (cache == NULL)
172 RET_ERR();
173 } else {
174 /* May be NULL if cache is disabled. */
175 cache = rte_mempool_default_cache(mp, lcore_id);
176 }
177
178 /* n_get_bulk and n_put_bulk must be divisors of n_keep */
179 if (((n_keep / n_get_bulk) * n_get_bulk) != n_keep)
180 GOTO_ERR(ret, out);
181 if (((n_keep / n_put_bulk) * n_put_bulk) != n_keep)
182 GOTO_ERR(ret, out);
183 /* for constant n, n_get_bulk and n_put_bulk must be the same */
184 if (use_constant_values && n_put_bulk != n_get_bulk)
185 GOTO_ERR(ret, out);
186
187 stats[lcore_id].enq_count = 0;
188
189 /* wait synchro for workers */
190 if (lcore_id != rte_get_main_lcore())
191 rte_wait_until_equal_32(&synchro, 1, __ATOMIC_RELAXED);
192
193 start_cycles = rte_get_timer_cycles();
194
195 while (time_diff/hz < TIME_S) {
196 if (!use_constant_values)
197 ret = test_loop(mp, cache, n_keep, n_get_bulk, n_put_bulk);
198 else if (n_get_bulk == 1)
199 ret = test_loop(mp, cache, n_keep, 1, 1);
200 else if (n_get_bulk == 4)
201 ret = test_loop(mp, cache, n_keep, 4, 4);
202 else if (n_get_bulk == CACHE_LINE_BURST)
203 ret = test_loop(mp, cache, n_keep,
204 CACHE_LINE_BURST, CACHE_LINE_BURST);
205 else if (n_get_bulk == 32)
206 ret = test_loop(mp, cache, n_keep, 32, 32);
207 else
208 ret = -1;
209
210 if (ret < 0)
211 GOTO_ERR(ret, out);
212
213 end_cycles = rte_get_timer_cycles();
214 time_diff = end_cycles - start_cycles;
215 stats[lcore_id].enq_count += N;
216 }
217
218 out:
219 if (use_external_cache) {
220 rte_mempool_cache_flush(cache, mp);
221 rte_mempool_cache_free(cache);
222 }
223
224 return ret;
225 }
226
227 /* launch all the per-lcore test, and display the result */
228 static int
launch_cores(struct rte_mempool * mp,unsigned int cores)229 launch_cores(struct rte_mempool *mp, unsigned int cores)
230 {
231 unsigned lcore_id;
232 uint64_t rate;
233 int ret;
234 unsigned cores_save = cores;
235
236 __atomic_store_n(&synchro, 0, __ATOMIC_RELAXED);
237
238 /* reset stats */
239 memset(stats, 0, sizeof(stats));
240
241 printf("mempool_autotest cache=%u cores=%u n_get_bulk=%u "
242 "n_put_bulk=%u n_keep=%u constant_n=%u ",
243 use_external_cache ?
244 external_cache_size : (unsigned) mp->cache_size,
245 cores, n_get_bulk, n_put_bulk, n_keep, use_constant_values);
246
247 if (rte_mempool_avail_count(mp) != MEMPOOL_SIZE) {
248 printf("mempool is not full\n");
249 return -1;
250 }
251
252 RTE_LCORE_FOREACH_WORKER(lcore_id) {
253 if (cores == 1)
254 break;
255 cores--;
256 rte_eal_remote_launch(per_lcore_mempool_test,
257 mp, lcore_id);
258 }
259
260 /* start synchro and launch test on main */
261 __atomic_store_n(&synchro, 1, __ATOMIC_RELAXED);
262
263 ret = per_lcore_mempool_test(mp);
264
265 cores = cores_save;
266 RTE_LCORE_FOREACH_WORKER(lcore_id) {
267 if (cores == 1)
268 break;
269 cores--;
270 if (rte_eal_wait_lcore(lcore_id) < 0)
271 ret = -1;
272 }
273
274 if (ret < 0) {
275 printf("per-lcore test returned -1\n");
276 return -1;
277 }
278
279 rate = 0;
280 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++)
281 rate += (stats[lcore_id].enq_count / TIME_S);
282
283 printf("rate_persec=%" PRIu64 "\n", rate);
284
285 return 0;
286 }
287
288 /* for a given number of core, launch all test cases */
289 static int
do_one_mempool_test(struct rte_mempool * mp,unsigned int cores)290 do_one_mempool_test(struct rte_mempool *mp, unsigned int cores)
291 {
292 unsigned int bulk_tab_get[] = { 1, 4, CACHE_LINE_BURST, 32, 0 };
293 unsigned int bulk_tab_put[] = { 1, 4, CACHE_LINE_BURST, 32, 0 };
294 unsigned int keep_tab[] = { 32, 128, 512, 0 };
295 unsigned *get_bulk_ptr;
296 unsigned *put_bulk_ptr;
297 unsigned *keep_ptr;
298 int ret;
299
300 for (get_bulk_ptr = bulk_tab_get; *get_bulk_ptr; get_bulk_ptr++) {
301 for (put_bulk_ptr = bulk_tab_put; *put_bulk_ptr; put_bulk_ptr++) {
302 for (keep_ptr = keep_tab; *keep_ptr; keep_ptr++) {
303
304 use_constant_values = 0;
305 n_get_bulk = *get_bulk_ptr;
306 n_put_bulk = *put_bulk_ptr;
307 n_keep = *keep_ptr;
308 ret = launch_cores(mp, cores);
309 if (ret < 0)
310 return -1;
311
312 /* replay test with constant values */
313 if (n_get_bulk == n_put_bulk) {
314 use_constant_values = 1;
315 ret = launch_cores(mp, cores);
316 if (ret < 0)
317 return -1;
318 }
319 }
320 }
321 }
322 return 0;
323 }
324
325 static int
test_mempool_perf(void)326 test_mempool_perf(void)
327 {
328 struct rte_mempool *mp_cache = NULL;
329 struct rte_mempool *mp_nocache = NULL;
330 struct rte_mempool *default_pool = NULL;
331 const char *default_pool_ops;
332 int ret = -1;
333
334 /* create a mempool (without cache) */
335 mp_nocache = rte_mempool_create("perf_test_nocache", MEMPOOL_SIZE,
336 MEMPOOL_ELT_SIZE, 0, 0,
337 NULL, NULL,
338 my_obj_init, NULL,
339 SOCKET_ID_ANY, 0);
340 if (mp_nocache == NULL)
341 goto err;
342
343 /* create a mempool (with cache) */
344 mp_cache = rte_mempool_create("perf_test_cache", MEMPOOL_SIZE,
345 MEMPOOL_ELT_SIZE,
346 RTE_MEMPOOL_CACHE_MAX_SIZE, 0,
347 NULL, NULL,
348 my_obj_init, NULL,
349 SOCKET_ID_ANY, 0);
350 if (mp_cache == NULL)
351 goto err;
352
353 default_pool_ops = rte_mbuf_best_mempool_ops();
354 /* Create a mempool based on Default handler */
355 default_pool = rte_mempool_create_empty("default_pool",
356 MEMPOOL_SIZE,
357 MEMPOOL_ELT_SIZE,
358 0, 0,
359 SOCKET_ID_ANY, 0);
360
361 if (default_pool == NULL) {
362 printf("cannot allocate %s mempool\n", default_pool_ops);
363 goto err;
364 }
365
366 if (rte_mempool_set_ops_byname(default_pool, default_pool_ops, NULL)
367 < 0) {
368 printf("cannot set %s handler\n", default_pool_ops);
369 goto err;
370 }
371
372 if (rte_mempool_populate_default(default_pool) < 0) {
373 printf("cannot populate %s mempool\n", default_pool_ops);
374 goto err;
375 }
376
377 rte_mempool_obj_iter(default_pool, my_obj_init, NULL);
378
379 /* performance test with 1, 2 and max cores */
380 printf("start performance test (without cache)\n");
381
382 if (do_one_mempool_test(mp_nocache, 1) < 0)
383 goto err;
384
385 if (do_one_mempool_test(mp_nocache, 2) < 0)
386 goto err;
387
388 if (do_one_mempool_test(mp_nocache, rte_lcore_count()) < 0)
389 goto err;
390
391 /* performance test with 1, 2 and max cores */
392 printf("start performance test for %s (without cache)\n",
393 default_pool_ops);
394
395 if (do_one_mempool_test(default_pool, 1) < 0)
396 goto err;
397
398 if (do_one_mempool_test(default_pool, 2) < 0)
399 goto err;
400
401 if (do_one_mempool_test(default_pool, rte_lcore_count()) < 0)
402 goto err;
403
404 /* performance test with 1, 2 and max cores */
405 printf("start performance test (with cache)\n");
406
407 if (do_one_mempool_test(mp_cache, 1) < 0)
408 goto err;
409
410 if (do_one_mempool_test(mp_cache, 2) < 0)
411 goto err;
412
413 if (do_one_mempool_test(mp_cache, rte_lcore_count()) < 0)
414 goto err;
415
416 /* performance test with 1, 2 and max cores */
417 printf("start performance test (with user-owned cache)\n");
418 use_external_cache = 1;
419
420 if (do_one_mempool_test(mp_nocache, 1) < 0)
421 goto err;
422
423 if (do_one_mempool_test(mp_nocache, 2) < 0)
424 goto err;
425
426 if (do_one_mempool_test(mp_nocache, rte_lcore_count()) < 0)
427 goto err;
428
429 rte_mempool_list_dump(stdout);
430
431 ret = 0;
432
433 err:
434 rte_mempool_free(mp_cache);
435 rte_mempool_free(mp_nocache);
436 rte_mempool_free(default_pool);
437 return ret;
438 }
439
440 REGISTER_TEST_COMMAND(mempool_perf_autotest, test_mempool_perf);
441