1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stddef.h> 7 #include <stdio.h> 8 #include <string.h> 9 #include <sys/queue.h> 10 11 #include <rte_errno.h> 12 #include <rte_memcpy.h> 13 #include <rte_memory.h> 14 #include <rte_eal.h> 15 #include <rte_eal_memconfig.h> 16 #include <rte_branch_prediction.h> 17 #include <rte_debug.h> 18 #include <rte_launch.h> 19 #include <rte_per_lcore.h> 20 #include <rte_lcore.h> 21 #include <rte_common.h> 22 #include <rte_spinlock.h> 23 24 #include <rte_malloc.h> 25 #include "malloc_elem.h" 26 #include "malloc_heap.h" 27 #include "eal_memalloc.h" 28 29 30 /* Free the memory space back to heap */ 31 void rte_free(void *addr) 32 { 33 if (addr == NULL) return; 34 if (malloc_heap_free(malloc_elem_from_data(addr)) < 0) 35 RTE_LOG(ERR, EAL, "Error: Invalid memory\n"); 36 } 37 38 /* 39 * Allocate memory on specified heap. 40 */ 41 void * 42 rte_malloc_socket(const char *type, size_t size, unsigned int align, 43 int socket_arg) 44 { 45 /* return NULL if size is 0 or alignment is not power-of-2 */ 46 if (size == 0 || (align && !rte_is_power_of_2(align))) 47 return NULL; 48 49 /* if there are no hugepages and if we are not allocating from an 50 * external heap, use memory from any socket available. checking for 51 * socket being external may return -1 in case of invalid socket, but 52 * that's OK - if there are no hugepages, it doesn't matter. 53 */ 54 if (rte_malloc_heap_socket_is_external(socket_arg) != 1 && 55 !rte_eal_has_hugepages()) 56 socket_arg = SOCKET_ID_ANY; 57 58 return malloc_heap_alloc(type, size, socket_arg, 0, 59 align == 0 ? 1 : align, 0, false); 60 } 61 62 /* 63 * Allocate memory on default heap. 64 */ 65 void * 66 rte_malloc(const char *type, size_t size, unsigned align) 67 { 68 return rte_malloc_socket(type, size, align, SOCKET_ID_ANY); 69 } 70 71 /* 72 * Allocate zero'd memory on specified heap. 73 */ 74 void * 75 rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket) 76 { 77 return rte_malloc_socket(type, size, align, socket); 78 } 79 80 /* 81 * Allocate zero'd memory on default heap. 82 */ 83 void * 84 rte_zmalloc(const char *type, size_t size, unsigned align) 85 { 86 return rte_zmalloc_socket(type, size, align, SOCKET_ID_ANY); 87 } 88 89 /* 90 * Allocate zero'd memory on specified heap. 91 */ 92 void * 93 rte_calloc_socket(const char *type, size_t num, size_t size, unsigned align, int socket) 94 { 95 return rte_zmalloc_socket(type, num * size, align, socket); 96 } 97 98 /* 99 * Allocate zero'd memory on default heap. 100 */ 101 void * 102 rte_calloc(const char *type, size_t num, size_t size, unsigned align) 103 { 104 return rte_zmalloc(type, num * size, align); 105 } 106 107 /* 108 * Resize allocated memory. 109 */ 110 void * 111 rte_realloc(void *ptr, size_t size, unsigned align) 112 { 113 if (ptr == NULL) 114 return rte_malloc(NULL, size, align); 115 116 struct malloc_elem *elem = malloc_elem_from_data(ptr); 117 if (elem == NULL) { 118 RTE_LOG(ERR, EAL, "Error: memory corruption detected\n"); 119 return NULL; 120 } 121 122 size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align); 123 /* check alignment matches first, and if ok, see if we can resize block */ 124 if (RTE_PTR_ALIGN(ptr,align) == ptr && 125 malloc_heap_resize(elem, size) == 0) 126 return ptr; 127 128 /* either alignment is off, or we have no room to expand, 129 * so move data. */ 130 void *new_ptr = rte_malloc(NULL, size, align); 131 if (new_ptr == NULL) 132 return NULL; 133 const unsigned old_size = elem->size - MALLOC_ELEM_OVERHEAD; 134 rte_memcpy(new_ptr, ptr, old_size < size ? old_size : size); 135 rte_free(ptr); 136 137 return new_ptr; 138 } 139 140 int 141 rte_malloc_validate(const void *ptr, size_t *size) 142 { 143 const struct malloc_elem *elem = malloc_elem_from_data(ptr); 144 if (!malloc_elem_cookies_ok(elem)) 145 return -1; 146 if (size != NULL) 147 *size = elem->size - elem->pad - MALLOC_ELEM_OVERHEAD; 148 return 0; 149 } 150 151 /* 152 * Function to retrieve data for heap on given socket 153 */ 154 int 155 rte_malloc_get_socket_stats(int socket, 156 struct rte_malloc_socket_stats *socket_stats) 157 { 158 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 159 int heap_idx; 160 161 heap_idx = malloc_socket_to_heap_id(socket); 162 if (heap_idx < 0) 163 return -1; 164 165 return malloc_heap_get_stats(&mcfg->malloc_heaps[heap_idx], 166 socket_stats); 167 } 168 169 /* 170 * Function to dump contents of all heaps 171 */ 172 void __rte_experimental 173 rte_malloc_dump_heaps(FILE *f) 174 { 175 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 176 unsigned int idx; 177 178 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 179 fprintf(f, "Heap id: %u\n", idx); 180 malloc_heap_dump(&mcfg->malloc_heaps[idx], f); 181 } 182 } 183 184 int 185 rte_malloc_heap_get_socket(const char *name) 186 { 187 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 188 struct malloc_heap *heap = NULL; 189 unsigned int idx; 190 int ret; 191 192 if (name == NULL || 193 strnlen(name, RTE_HEAP_NAME_MAX_LEN) == 0 || 194 strnlen(name, RTE_HEAP_NAME_MAX_LEN) == 195 RTE_HEAP_NAME_MAX_LEN) { 196 rte_errno = EINVAL; 197 return -1; 198 } 199 rte_rwlock_read_lock(&mcfg->memory_hotplug_lock); 200 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 201 struct malloc_heap *tmp = &mcfg->malloc_heaps[idx]; 202 203 if (!strncmp(name, tmp->name, RTE_HEAP_NAME_MAX_LEN)) { 204 heap = tmp; 205 break; 206 } 207 } 208 209 if (heap != NULL) { 210 ret = heap->socket_id; 211 } else { 212 rte_errno = ENOENT; 213 ret = -1; 214 } 215 rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock); 216 217 return ret; 218 } 219 220 int 221 rte_malloc_heap_socket_is_external(int socket_id) 222 { 223 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 224 unsigned int idx; 225 int ret = -1; 226 227 if (socket_id == SOCKET_ID_ANY) 228 return 0; 229 230 rte_rwlock_read_lock(&mcfg->memory_hotplug_lock); 231 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 232 struct malloc_heap *tmp = &mcfg->malloc_heaps[idx]; 233 234 if ((int)tmp->socket_id == socket_id) { 235 /* external memory always has large socket ID's */ 236 ret = tmp->socket_id >= RTE_MAX_NUMA_NODES; 237 break; 238 } 239 } 240 rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock); 241 242 return ret; 243 } 244 245 /* 246 * Print stats on memory type. If type is NULL, info on all types is printed 247 */ 248 void 249 rte_malloc_dump_stats(FILE *f, __rte_unused const char *type) 250 { 251 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 252 unsigned int heap_id; 253 struct rte_malloc_socket_stats sock_stats; 254 255 /* Iterate through all initialised heaps */ 256 for (heap_id = 0; heap_id < RTE_MAX_HEAPS; heap_id++) { 257 struct malloc_heap *heap = &mcfg->malloc_heaps[heap_id]; 258 259 malloc_heap_get_stats(heap, &sock_stats); 260 261 fprintf(f, "Heap id:%u\n", heap_id); 262 fprintf(f, "\tHeap name:%s\n", heap->name); 263 fprintf(f, "\tHeap_size:%zu,\n", sock_stats.heap_totalsz_bytes); 264 fprintf(f, "\tFree_size:%zu,\n", sock_stats.heap_freesz_bytes); 265 fprintf(f, "\tAlloc_size:%zu,\n", sock_stats.heap_allocsz_bytes); 266 fprintf(f, "\tGreatest_free_size:%zu,\n", 267 sock_stats.greatest_free_size); 268 fprintf(f, "\tAlloc_count:%u,\n",sock_stats.alloc_count); 269 fprintf(f, "\tFree_count:%u,\n", sock_stats.free_count); 270 } 271 return; 272 } 273 274 /* 275 * TODO: Set limit to memory that can be allocated to memory type 276 */ 277 int 278 rte_malloc_set_limit(__rte_unused const char *type, 279 __rte_unused size_t max) 280 { 281 return 0; 282 } 283 284 /* 285 * Return the IO address of a virtual address obtained through rte_malloc 286 */ 287 rte_iova_t 288 rte_malloc_virt2iova(const void *addr) 289 { 290 const struct rte_memseg *ms; 291 struct malloc_elem *elem = malloc_elem_from_data(addr); 292 293 if (elem == NULL) 294 return RTE_BAD_IOVA; 295 296 if (!elem->msl->external && rte_eal_iova_mode() == RTE_IOVA_VA) 297 return (uintptr_t) addr; 298 299 ms = rte_mem_virt2memseg(addr, elem->msl); 300 if (ms == NULL) 301 return RTE_BAD_IOVA; 302 303 if (ms->iova == RTE_BAD_IOVA) 304 return RTE_BAD_IOVA; 305 306 return ms->iova + RTE_PTR_DIFF(addr, ms->addr); 307 } 308 309 static struct malloc_heap * 310 find_named_heap(const char *name) 311 { 312 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 313 unsigned int i; 314 315 for (i = 0; i < RTE_MAX_HEAPS; i++) { 316 struct malloc_heap *heap = &mcfg->malloc_heaps[i]; 317 318 if (!strncmp(name, heap->name, RTE_HEAP_NAME_MAX_LEN)) 319 return heap; 320 } 321 return NULL; 322 } 323 324 int 325 rte_malloc_heap_memory_add(const char *heap_name, void *va_addr, size_t len, 326 rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz) 327 { 328 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 329 struct malloc_heap *heap = NULL; 330 unsigned int n; 331 int ret; 332 333 if (heap_name == NULL || va_addr == NULL || 334 page_sz == 0 || !rte_is_power_of_2(page_sz) || 335 RTE_ALIGN(len, page_sz) != len || 336 !rte_is_aligned(va_addr, page_sz) || 337 ((len / page_sz) != n_pages && iova_addrs != NULL) || 338 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 339 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 340 RTE_HEAP_NAME_MAX_LEN) { 341 rte_errno = EINVAL; 342 return -1; 343 } 344 rte_rwlock_write_lock(&mcfg->memory_hotplug_lock); 345 346 /* find our heap */ 347 heap = find_named_heap(heap_name); 348 if (heap == NULL) { 349 rte_errno = ENOENT; 350 ret = -1; 351 goto unlock; 352 } 353 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 354 /* cannot add memory to internal heaps */ 355 rte_errno = EPERM; 356 ret = -1; 357 goto unlock; 358 } 359 n = len / page_sz; 360 361 rte_spinlock_lock(&heap->lock); 362 ret = malloc_heap_add_external_memory(heap, va_addr, iova_addrs, n, 363 page_sz); 364 rte_spinlock_unlock(&heap->lock); 365 366 unlock: 367 rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock); 368 369 return ret; 370 } 371 372 int 373 rte_malloc_heap_memory_remove(const char *heap_name, void *va_addr, size_t len) 374 { 375 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 376 struct malloc_heap *heap = NULL; 377 int ret; 378 379 if (heap_name == NULL || va_addr == NULL || len == 0 || 380 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 381 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 382 RTE_HEAP_NAME_MAX_LEN) { 383 rte_errno = EINVAL; 384 return -1; 385 } 386 rte_rwlock_write_lock(&mcfg->memory_hotplug_lock); 387 /* find our heap */ 388 heap = find_named_heap(heap_name); 389 if (heap == NULL) { 390 rte_errno = ENOENT; 391 ret = -1; 392 goto unlock; 393 } 394 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 395 /* cannot remove memory from internal heaps */ 396 rte_errno = EPERM; 397 ret = -1; 398 goto unlock; 399 } 400 401 rte_spinlock_lock(&heap->lock); 402 ret = malloc_heap_remove_external_memory(heap, va_addr, len); 403 rte_spinlock_unlock(&heap->lock); 404 405 unlock: 406 rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock); 407 408 return ret; 409 } 410 411 struct sync_mem_walk_arg { 412 void *va_addr; 413 size_t len; 414 int result; 415 bool attach; 416 }; 417 418 static int 419 sync_mem_walk(const struct rte_memseg_list *msl, void *arg) 420 { 421 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 422 struct sync_mem_walk_arg *wa = arg; 423 size_t len = msl->page_sz * msl->memseg_arr.len; 424 425 if (msl->base_va == wa->va_addr && 426 len == wa->len) { 427 struct rte_memseg_list *found_msl; 428 int msl_idx, ret; 429 430 /* msl is const */ 431 msl_idx = msl - mcfg->memsegs; 432 found_msl = &mcfg->memsegs[msl_idx]; 433 434 if (wa->attach) { 435 ret = rte_fbarray_attach(&found_msl->memseg_arr); 436 } else { 437 /* notify all subscribers that a memory area is about to 438 * be removed 439 */ 440 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_FREE, 441 msl->base_va, msl->len); 442 ret = rte_fbarray_detach(&found_msl->memseg_arr); 443 } 444 445 if (ret < 0) { 446 wa->result = -rte_errno; 447 } else { 448 /* notify all subscribers that a new memory area was 449 * added 450 */ 451 if (wa->attach) 452 eal_memalloc_mem_event_notify( 453 RTE_MEM_EVENT_ALLOC, 454 msl->base_va, msl->len); 455 wa->result = 0; 456 } 457 return 1; 458 } 459 return 0; 460 } 461 462 static int 463 sync_memory(const char *heap_name, void *va_addr, size_t len, bool attach) 464 { 465 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 466 struct malloc_heap *heap = NULL; 467 struct sync_mem_walk_arg wa; 468 int ret; 469 470 if (heap_name == NULL || va_addr == NULL || len == 0 || 471 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 472 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 473 RTE_HEAP_NAME_MAX_LEN) { 474 rte_errno = EINVAL; 475 return -1; 476 } 477 rte_rwlock_read_lock(&mcfg->memory_hotplug_lock); 478 479 /* find our heap */ 480 heap = find_named_heap(heap_name); 481 if (heap == NULL) { 482 rte_errno = ENOENT; 483 ret = -1; 484 goto unlock; 485 } 486 /* we shouldn't be able to sync to internal heaps */ 487 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 488 rte_errno = EPERM; 489 ret = -1; 490 goto unlock; 491 } 492 493 /* find corresponding memseg list to sync to */ 494 wa.va_addr = va_addr; 495 wa.len = len; 496 wa.result = -ENOENT; /* fail unless explicitly told to succeed */ 497 wa.attach = attach; 498 499 /* we're already holding a read lock */ 500 rte_memseg_list_walk_thread_unsafe(sync_mem_walk, &wa); 501 502 if (wa.result < 0) { 503 rte_errno = -wa.result; 504 ret = -1; 505 } else 506 ret = 0; 507 unlock: 508 rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock); 509 return ret; 510 } 511 512 int 513 rte_malloc_heap_memory_attach(const char *heap_name, void *va_addr, size_t len) 514 { 515 return sync_memory(heap_name, va_addr, len, true); 516 } 517 518 int 519 rte_malloc_heap_memory_detach(const char *heap_name, void *va_addr, size_t len) 520 { 521 return sync_memory(heap_name, va_addr, len, false); 522 } 523 524 int 525 rte_malloc_heap_create(const char *heap_name) 526 { 527 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 528 struct malloc_heap *heap = NULL; 529 int i, ret; 530 531 if (heap_name == NULL || 532 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 533 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 534 RTE_HEAP_NAME_MAX_LEN) { 535 rte_errno = EINVAL; 536 return -1; 537 } 538 /* check if there is space in the heap list, or if heap with this name 539 * already exists. 540 */ 541 rte_rwlock_write_lock(&mcfg->memory_hotplug_lock); 542 543 for (i = 0; i < RTE_MAX_HEAPS; i++) { 544 struct malloc_heap *tmp = &mcfg->malloc_heaps[i]; 545 /* existing heap */ 546 if (strncmp(heap_name, tmp->name, 547 RTE_HEAP_NAME_MAX_LEN) == 0) { 548 RTE_LOG(ERR, EAL, "Heap %s already exists\n", 549 heap_name); 550 rte_errno = EEXIST; 551 ret = -1; 552 goto unlock; 553 } 554 /* empty heap */ 555 if (strnlen(tmp->name, RTE_HEAP_NAME_MAX_LEN) == 0) { 556 heap = tmp; 557 break; 558 } 559 } 560 if (heap == NULL) { 561 RTE_LOG(ERR, EAL, "Cannot create new heap: no space\n"); 562 rte_errno = ENOSPC; 563 ret = -1; 564 goto unlock; 565 } 566 567 /* we're sure that we can create a new heap, so do it */ 568 ret = malloc_heap_create(heap, heap_name); 569 unlock: 570 rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock); 571 572 return ret; 573 } 574 575 int 576 rte_malloc_heap_destroy(const char *heap_name) 577 { 578 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 579 struct malloc_heap *heap = NULL; 580 int ret; 581 582 if (heap_name == NULL || 583 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 584 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 585 RTE_HEAP_NAME_MAX_LEN) { 586 rte_errno = EINVAL; 587 return -1; 588 } 589 rte_rwlock_write_lock(&mcfg->memory_hotplug_lock); 590 591 /* start from non-socket heaps */ 592 heap = find_named_heap(heap_name); 593 if (heap == NULL) { 594 RTE_LOG(ERR, EAL, "Heap %s not found\n", heap_name); 595 rte_errno = ENOENT; 596 ret = -1; 597 goto unlock; 598 } 599 /* we shouldn't be able to destroy internal heaps */ 600 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 601 rte_errno = EPERM; 602 ret = -1; 603 goto unlock; 604 } 605 /* sanity checks done, now we can destroy the heap */ 606 rte_spinlock_lock(&heap->lock); 607 ret = malloc_heap_destroy(heap); 608 609 /* if we failed, lock is still active */ 610 if (ret < 0) 611 rte_spinlock_unlock(&heap->lock); 612 unlock: 613 rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock); 614 615 return ret; 616 } 617