1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4
5 #include <errno.h>
6 #include <stdio.h>
7 #include <stdlib.h>
8 #include <string.h>
9 #include <fcntl.h>
10 #include <sys/mman.h>
11 #include <sys/wait.h>
12
13 #include <rte_common.h>
14 #include <rte_debug.h>
15 #include <rte_eal.h>
16 #include <rte_errno.h>
17 #include <rte_malloc.h>
18 #include <rte_ring.h>
19 #include <rte_string_fns.h>
20
21 #include "test.h"
22
23 #define EXTERNAL_MEM_SZ (RTE_PGSIZE_4K << 10) /* 4M of data */
24
25 static int
check_mem(void * addr,rte_iova_t * iova,size_t pgsz,int n_pages)26 check_mem(void *addr, rte_iova_t *iova, size_t pgsz, int n_pages)
27 {
28 int i;
29
30 /* check that we can get this memory from EAL now */
31 for (i = 0; i < n_pages; i++) {
32 const struct rte_memseg_list *msl;
33 const struct rte_memseg *ms;
34 void *cur = RTE_PTR_ADD(addr, pgsz * i);
35 rte_iova_t expected_iova;
36
37 msl = rte_mem_virt2memseg_list(cur);
38 if (!msl->external) {
39 printf("%s():%i: Memseg list is not marked as external\n",
40 __func__, __LINE__);
41 return -1;
42 }
43
44 ms = rte_mem_virt2memseg(cur, msl);
45 if (ms == NULL) {
46 printf("%s():%i: Failed to retrieve memseg for external mem\n",
47 __func__, __LINE__);
48 return -1;
49 }
50 if (ms->addr != cur) {
51 printf("%s():%i: VA mismatch\n", __func__, __LINE__);
52 return -1;
53 }
54 expected_iova = (iova == NULL) ? RTE_BAD_IOVA : iova[i];
55 if (ms->iova != expected_iova) {
56 printf("%s():%i: IOVA mismatch\n", __func__, __LINE__);
57 return -1;
58 }
59 }
60 return 0;
61 }
62
63 static int
test_malloc_invalid_param(void * addr,size_t len,size_t pgsz,rte_iova_t * iova,int n_pages)64 test_malloc_invalid_param(void *addr, size_t len, size_t pgsz, rte_iova_t *iova,
65 int n_pages)
66 {
67 static const char * const names[] = {
68 NULL, /* NULL name */
69 "", /* empty name */
70 "this heap name is definitely way too long to be valid"
71 };
72 const char *valid_name = "valid heap name";
73 unsigned int i;
74
75 /* check invalid name handling */
76 for (i = 0; i < RTE_DIM(names); i++) {
77 const char *name = names[i];
78
79 /* these calls may fail for other reasons, so check errno */
80 if (rte_malloc_heap_create(name) >= 0 || rte_errno != EINVAL) {
81 printf("%s():%i: Created heap with invalid name\n",
82 __func__, __LINE__);
83 goto fail;
84 }
85
86 if (rte_malloc_heap_destroy(name) >= 0 || rte_errno != EINVAL) {
87 printf("%s():%i: Destroyed heap with invalid name\n",
88 __func__, __LINE__);
89 goto fail;
90 }
91
92 if (rte_malloc_heap_get_socket(name) >= 0 ||
93 rte_errno != EINVAL) {
94 printf("%s():%i: Found socket for heap with invalid name\n",
95 __func__, __LINE__);
96 goto fail;
97 }
98
99 if (rte_malloc_heap_memory_add(name, addr, len,
100 NULL, 0, pgsz) >= 0 || rte_errno != EINVAL) {
101 printf("%s():%i: Added memory to heap with invalid name\n",
102 __func__, __LINE__);
103 goto fail;
104 }
105 if (rte_malloc_heap_memory_remove(name, addr, len) >= 0 ||
106 rte_errno != EINVAL) {
107 printf("%s():%i: Removed memory from heap with invalid name\n",
108 __func__, __LINE__);
109 goto fail;
110 }
111
112 if (rte_malloc_heap_memory_attach(name, addr, len) >= 0 ||
113 rte_errno != EINVAL) {
114 printf("%s():%i: Attached memory to heap with invalid name\n",
115 __func__, __LINE__);
116 goto fail;
117 }
118 if (rte_malloc_heap_memory_detach(name, addr, len) >= 0 ||
119 rte_errno != EINVAL) {
120 printf("%s():%i: Detached memory from heap with invalid name\n",
121 __func__, __LINE__);
122 goto fail;
123 }
124 }
125
126 /* do same as above, but with a valid heap name */
127
128 /* skip create call */
129 if (rte_malloc_heap_destroy(valid_name) >= 0 || rte_errno != ENOENT) {
130 printf("%s():%i: Destroyed heap with invalid name\n",
131 __func__, __LINE__);
132 goto fail;
133 }
134 if (rte_malloc_heap_get_socket(valid_name) >= 0 ||
135 rte_errno != ENOENT) {
136 printf("%s():%i: Found socket for heap with invalid name\n",
137 __func__, __LINE__);
138 goto fail;
139 }
140
141 /* these calls may fail for other reasons, so check errno */
142 if (rte_malloc_heap_memory_add(valid_name, addr, len,
143 NULL, 0, pgsz) >= 0 || rte_errno != ENOENT) {
144 printf("%s():%i: Added memory to non-existent heap\n",
145 __func__, __LINE__);
146 goto fail;
147 }
148 if (rte_malloc_heap_memory_remove(valid_name, addr, len) >= 0 ||
149 rte_errno != ENOENT) {
150 printf("%s():%i: Removed memory from non-existent heap\n",
151 __func__, __LINE__);
152 goto fail;
153 }
154
155 if (rte_malloc_heap_memory_attach(valid_name, addr, len) >= 0 ||
156 rte_errno != ENOENT) {
157 printf("%s():%i: Attached memory to non-existent heap\n",
158 __func__, __LINE__);
159 goto fail;
160 }
161 if (rte_malloc_heap_memory_detach(valid_name, addr, len) >= 0 ||
162 rte_errno != ENOENT) {
163 printf("%s():%i: Detached memory from non-existent heap\n",
164 __func__, __LINE__);
165 goto fail;
166 }
167
168 /* create a valid heap but test other invalid parameters */
169 if (rte_malloc_heap_create(valid_name) != 0) {
170 printf("%s():%i: Failed to create valid heap\n",
171 __func__, __LINE__);
172 goto fail;
173 }
174
175 /* zero length */
176 if (rte_malloc_heap_memory_add(valid_name, addr, 0,
177 NULL, 0, pgsz) >= 0 || rte_errno != EINVAL) {
178 printf("%s():%i: Added memory with invalid parameters\n",
179 __func__, __LINE__);
180 goto fail;
181 }
182
183 if (rte_malloc_heap_memory_remove(valid_name, addr, 0) >= 0 ||
184 rte_errno != EINVAL) {
185 printf("%s():%i: Removed memory with invalid parameters\n",
186 __func__, __LINE__);
187 goto fail;
188 }
189
190 if (rte_malloc_heap_memory_attach(valid_name, addr, 0) >= 0 ||
191 rte_errno != EINVAL) {
192 printf("%s():%i: Attached memory with invalid parameters\n",
193 __func__, __LINE__);
194 goto fail;
195 }
196 if (rte_malloc_heap_memory_detach(valid_name, addr, 0) >= 0 ||
197 rte_errno != EINVAL) {
198 printf("%s():%i: Detached memory with invalid parameters\n",
199 __func__, __LINE__);
200 goto fail;
201 }
202
203 /* zero address */
204 if (rte_malloc_heap_memory_add(valid_name, NULL, len,
205 NULL, 0, pgsz) >= 0 || rte_errno != EINVAL) {
206 printf("%s():%i: Added memory with invalid parameters\n",
207 __func__, __LINE__);
208 goto fail;
209 }
210
211 if (rte_malloc_heap_memory_remove(valid_name, NULL, len) >= 0 ||
212 rte_errno != EINVAL) {
213 printf("%s():%i: Removed memory with invalid parameters\n",
214 __func__, __LINE__);
215 goto fail;
216 }
217
218 if (rte_malloc_heap_memory_attach(valid_name, NULL, len) >= 0 ||
219 rte_errno != EINVAL) {
220 printf("%s():%i: Attached memory with invalid parameters\n",
221 __func__, __LINE__);
222 goto fail;
223 }
224 if (rte_malloc_heap_memory_detach(valid_name, NULL, len) >= 0 ||
225 rte_errno != EINVAL) {
226 printf("%s():%i: Detached memory with invalid parameters\n",
227 __func__, __LINE__);
228 goto fail;
229 }
230
231 /* the following tests are only valid if IOVA table is not NULL */
232 if (iova != NULL) {
233 /* wrong page count */
234 if (rte_malloc_heap_memory_add(valid_name, addr, len,
235 iova, 0, pgsz) >= 0 || rte_errno != EINVAL) {
236 printf("%s():%i: Added memory with invalid parameters\n",
237 __func__, __LINE__);
238 goto fail;
239 }
240 if (rte_malloc_heap_memory_add(valid_name, addr, len,
241 iova, n_pages - 1, pgsz) >= 0 ||
242 rte_errno != EINVAL) {
243 printf("%s():%i: Added memory with invalid parameters\n",
244 __func__, __LINE__);
245 goto fail;
246 }
247 if (rte_malloc_heap_memory_add(valid_name, addr, len,
248 iova, n_pages + 1, pgsz) >= 0 ||
249 rte_errno != EINVAL) {
250 printf("%s():%i: Added memory with invalid parameters\n",
251 __func__, __LINE__);
252 goto fail;
253 }
254 }
255
256 /* tests passed, destroy heap */
257 if (rte_malloc_heap_destroy(valid_name) != 0) {
258 printf("%s():%i: Failed to destroy valid heap\n",
259 __func__, __LINE__);
260 goto fail;
261 }
262 return 0;
263 fail:
264 rte_malloc_heap_destroy(valid_name);
265 return -1;
266 }
267
268 static int
test_malloc_basic(void * addr,size_t len,size_t pgsz,rte_iova_t * iova,int n_pages)269 test_malloc_basic(void *addr, size_t len, size_t pgsz, rte_iova_t *iova,
270 int n_pages)
271 {
272 const char *heap_name = "heap";
273 void *ptr = NULL;
274 int socket_id;
275 const struct rte_memzone *mz = NULL, *contig_mz = NULL;
276
277 /* create heap */
278 if (rte_malloc_heap_create(heap_name) != 0) {
279 printf("%s():%i: Failed to create malloc heap\n",
280 __func__, __LINE__);
281 goto fail;
282 }
283
284 /* get socket ID corresponding to this heap */
285 socket_id = rte_malloc_heap_get_socket(heap_name);
286 if (socket_id < 0) {
287 printf("%s():%i: cannot find socket for external heap\n",
288 __func__, __LINE__);
289 goto fail;
290 }
291
292 /* heap is empty, so any allocation should fail */
293 ptr = rte_malloc_socket("EXTMEM", 64, 0, socket_id);
294 if (ptr != NULL) {
295 printf("%s():%i: Allocated from empty heap\n", __func__,
296 __LINE__);
297 goto fail;
298 }
299
300 /* add memory to heap */
301 if (rte_malloc_heap_memory_add(heap_name, addr, len,
302 iova, n_pages, pgsz) != 0) {
303 printf("%s():%i: Failed to add memory to heap\n",
304 __func__, __LINE__);
305 goto fail;
306 }
307
308 /* check if memory is accessible from EAL */
309 if (check_mem(addr, iova, pgsz, n_pages) < 0)
310 goto fail;
311
312 /* allocate - this now should succeed */
313 ptr = rte_malloc_socket("EXTMEM", 64, 0, socket_id);
314 if (ptr == NULL) {
315 printf("%s():%i: Failed to allocate from external heap\n",
316 __func__, __LINE__);
317 goto fail;
318 }
319
320 /* check if address is in expected range */
321 if (ptr < addr || ptr >= RTE_PTR_ADD(addr, len)) {
322 printf("%s():%i: Allocated from unexpected address space\n",
323 __func__, __LINE__);
324 goto fail;
325 }
326
327 /* we've allocated something - removing memory should fail */
328 if (rte_malloc_heap_memory_remove(heap_name, addr, len) >= 0 ||
329 rte_errno != EBUSY) {
330 printf("%s():%i: Removing memory succeeded when memory is not free\n",
331 __func__, __LINE__);
332 goto fail;
333 }
334 if (rte_malloc_heap_destroy(heap_name) >= 0 || rte_errno != EBUSY) {
335 printf("%s():%i: Destroying heap succeeded when memory is not free\n",
336 __func__, __LINE__);
337 goto fail;
338 }
339
340 /* try allocating a memzone */
341 mz = rte_memzone_reserve("heap_test", pgsz * 2, socket_id, 0);
342 if (mz == NULL) {
343 printf("%s():%i: Failed to reserve memzone\n",
344 __func__, __LINE__);
345 goto fail;
346 }
347 /* try allocating an IOVA-contiguous memzone - this should succeed
348 * if we've set up a contiguous IOVA table, and fail if we haven't.
349 */
350 contig_mz = rte_memzone_reserve("heap_test_contig", pgsz * 2, socket_id,
351 RTE_MEMZONE_IOVA_CONTIG);
352 if ((iova == NULL) != (contig_mz == NULL)) {
353 printf("%s():%i: Failed to reserve memzone\n",
354 __func__, __LINE__);
355 goto fail;
356 }
357
358 rte_malloc_dump_stats(stdout, NULL);
359 rte_malloc_dump_heaps(stdout);
360
361 /* free memory - removing it should now succeed */
362 rte_free(ptr);
363 ptr = NULL;
364
365 rte_memzone_free(mz);
366 mz = NULL;
367 rte_memzone_free(contig_mz);
368 contig_mz = NULL;
369
370 if (rte_malloc_heap_memory_remove(heap_name, addr, len) != 0) {
371 printf("%s():%i: Removing memory from heap failed\n",
372 __func__, __LINE__);
373 goto fail;
374 }
375 if (rte_malloc_heap_destroy(heap_name) != 0) {
376 printf("%s():%i: Destroying heap failed\n",
377 __func__, __LINE__);
378 goto fail;
379 }
380
381 return 0;
382 fail:
383 rte_memzone_free(contig_mz);
384 rte_memzone_free(mz);
385 rte_free(ptr);
386 /* even if something failed, attempt to clean up */
387 rte_malloc_heap_memory_remove(heap_name, addr, len);
388 rte_malloc_heap_destroy(heap_name);
389
390 return -1;
391 }
392
393 static int
test_extmem_invalid_param(void * addr,size_t len,size_t pgsz,rte_iova_t * iova,int n_pages)394 test_extmem_invalid_param(void *addr, size_t len, size_t pgsz, rte_iova_t *iova,
395 int n_pages)
396 {
397 /* these calls may fail for other reasons, so check errno */
398 if (rte_extmem_unregister(addr, len) >= 0 ||
399 rte_errno != ENOENT) {
400 printf("%s():%i: Unregistered non-existent memory\n",
401 __func__, __LINE__);
402 return -1;
403 }
404
405 if (rte_extmem_attach(addr, len) >= 0 ||
406 rte_errno != ENOENT) {
407 printf("%s():%i: Attached to non-existent memory\n",
408 __func__, __LINE__);
409 return -1;
410 }
411 if (rte_extmem_attach(addr, len) >= 0 ||
412 rte_errno != ENOENT) {
413 printf("%s():%i: Detached from non-existent memory\n",
414 __func__, __LINE__);
415 return -1;
416 }
417
418 /* zero length */
419 if (rte_extmem_register(addr, 0, NULL, 0, pgsz) >= 0 ||
420 rte_errno != EINVAL) {
421 printf("%s():%i: Registered memory with invalid parameters\n",
422 __func__, __LINE__);
423 return -1;
424 }
425
426 if (rte_extmem_unregister(addr, 0) >= 0 ||
427 rte_errno != EINVAL) {
428 printf("%s():%i: Unregistered memory with invalid parameters\n",
429 __func__, __LINE__);
430 return -1;
431 }
432
433 if (rte_extmem_attach(addr, 0) >= 0 ||
434 rte_errno != EINVAL) {
435 printf("%s():%i: Attached memory with invalid parameters\n",
436 __func__, __LINE__);
437 return -1;
438 }
439 if (rte_extmem_attach(addr, 0) >= 0 ||
440 rte_errno != EINVAL) {
441 printf("%s():%i: Detached memory with invalid parameters\n",
442 __func__, __LINE__);
443 return -1;
444 }
445
446 /* zero address */
447 if (rte_extmem_register(NULL, len, NULL, 0, pgsz) >= 0 ||
448 rte_errno != EINVAL) {
449 printf("%s():%i: Registered memory with invalid parameters\n",
450 __func__, __LINE__);
451 return -1;
452 }
453
454 if (rte_extmem_unregister(NULL, len) >= 0 ||
455 rte_errno != EINVAL) {
456 printf("%s():%i: Unregistered memory with invalid parameters\n",
457 __func__, __LINE__);
458 return -1;
459 }
460
461 if (rte_extmem_attach(NULL, len) >= 0 ||
462 rte_errno != EINVAL) {
463 printf("%s():%i: Attached memory with invalid parameters\n",
464 __func__, __LINE__);
465 return -1;
466 }
467 if (rte_extmem_attach(NULL, len) >= 0 ||
468 rte_errno != EINVAL) {
469 printf("%s():%i: Detached memory with invalid parameters\n",
470 __func__, __LINE__);
471 return -1;
472 }
473
474 /* the following tests are only valid if IOVA table is not NULL */
475 if (iova != NULL) {
476 /* wrong page count */
477 if (rte_extmem_register(addr, len,
478 iova, 0, pgsz) >= 0 || rte_errno != EINVAL) {
479 printf("%s():%i: Registered memory with invalid parameters\n",
480 __func__, __LINE__);
481 return -1;
482 }
483 if (rte_extmem_register(addr, len,
484 iova, n_pages - 1, pgsz) >= 0 ||
485 rte_errno != EINVAL) {
486 printf("%s():%i: Registered memory with invalid parameters\n",
487 __func__, __LINE__);
488 return -1;
489 }
490 if (rte_extmem_register(addr, len,
491 iova, n_pages + 1, pgsz) >= 0 ||
492 rte_errno != EINVAL) {
493 printf("%s():%i: Registered memory with invalid parameters\n",
494 __func__, __LINE__);
495 return -1;
496 }
497 }
498
499 return 0;
500 }
501
502 static int
test_extmem_basic(void * addr,size_t len,size_t pgsz,rte_iova_t * iova,int n_pages)503 test_extmem_basic(void *addr, size_t len, size_t pgsz, rte_iova_t *iova,
504 int n_pages)
505 {
506 /* register memory */
507 if (rte_extmem_register(addr, len, iova, n_pages, pgsz) != 0) {
508 printf("%s():%i: Failed to register memory\n",
509 __func__, __LINE__);
510 goto fail;
511 }
512
513 /* check if memory is accessible from EAL */
514 if (check_mem(addr, iova, pgsz, n_pages) < 0)
515 goto fail;
516
517 if (rte_extmem_unregister(addr, len) != 0) {
518 printf("%s():%i: Removing memory from heap failed\n",
519 __func__, __LINE__);
520 goto fail;
521 }
522
523 return 0;
524 fail:
525 /* even if something failed, attempt to clean up */
526 rte_extmem_unregister(addr, len);
527
528 return -1;
529 }
530
531 /* we need to test attach/detach in secondary processes. */
532 static int
test_external_mem(void)533 test_external_mem(void)
534 {
535 size_t len = EXTERNAL_MEM_SZ;
536 size_t pgsz = RTE_PGSIZE_4K;
537 rte_iova_t iova[len / pgsz];
538 void *addr;
539 int ret, n_pages;
540 int i;
541
542 /* create external memory area */
543 n_pages = RTE_DIM(iova);
544 addr = mmap(NULL, len, PROT_WRITE | PROT_READ,
545 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
546 if (addr == MAP_FAILED) {
547 printf("%s():%i: Failed to create dummy memory area\n",
548 __func__, __LINE__);
549 return -1;
550 }
551 for (i = 0; i < n_pages; i++) {
552 /* arbitrary IOVA */
553 rte_iova_t tmp = 0x100000000 + i * pgsz;
554 iova[i] = tmp;
555 }
556
557 /* test external heap memory */
558 ret = test_malloc_invalid_param(addr, len, pgsz, iova, n_pages);
559 ret |= test_malloc_basic(addr, len, pgsz, iova, n_pages);
560 /* when iova table is NULL, everything should still work */
561 ret |= test_malloc_invalid_param(addr, len, pgsz, NULL, n_pages);
562 ret |= test_malloc_basic(addr, len, pgsz, NULL, n_pages);
563
564 /* test non-heap memory */
565 ret |= test_extmem_invalid_param(addr, len, pgsz, iova, n_pages);
566 ret |= test_extmem_basic(addr, len, pgsz, iova, n_pages);
567 /* when iova table is NULL, everything should still work */
568 ret |= test_extmem_invalid_param(addr, len, pgsz, NULL, n_pages);
569 ret |= test_extmem_basic(addr, len, pgsz, NULL, n_pages);
570
571 munmap(addr, len);
572
573 return ret;
574 }
575
576 REGISTER_TEST_COMMAND(external_mem_autotest, test_external_mem);
577