1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 malloc_mutex_t arenas_lock; 8 arena_t **arenas; 9 unsigned narenas; 10 11 pthread_key_t arenas_tsd; 12 #ifndef NO_TLS 13 __thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); 14 #endif 15 16 #ifdef JEMALLOC_STATS 17 # ifndef NO_TLS 18 __thread thread_allocated_t thread_allocated_tls; 19 # else 20 pthread_key_t thread_allocated_tsd; 21 # endif 22 #endif 23 24 /* Set to true once the allocator has been initialized. */ 25 static bool malloc_initialized = false; 26 27 /* Used to let the initializing thread recursively allocate. */ 28 static pthread_t malloc_initializer = (unsigned long)0; 29 30 /* Used to avoid initialization races. */ 31 static malloc_mutex_t init_lock = 32 #ifdef JEMALLOC_OSSPIN 33 0 34 #else 35 MALLOC_MUTEX_INITIALIZER 36 #endif 37 ; 38 39 #ifdef DYNAMIC_PAGE_SHIFT 40 size_t pagesize; 41 size_t pagesize_mask; 42 size_t lg_pagesize; 43 #endif 44 45 unsigned ncpus; 46 47 /* Runtime configuration options. */ 48 const char *JEMALLOC_P(malloc_conf) JEMALLOC_ATTR(visibility("default")); 49 #ifdef JEMALLOC_DEBUG 50 bool opt_abort = true; 51 # ifdef JEMALLOC_FILL 52 bool opt_junk = true; 53 # endif 54 #else 55 bool opt_abort = false; 56 # ifdef JEMALLOC_FILL 57 bool opt_junk = false; 58 # endif 59 #endif 60 #ifdef JEMALLOC_SYSV 61 bool opt_sysv = false; 62 #endif 63 #ifdef JEMALLOC_XMALLOC 64 bool opt_xmalloc = false; 65 #endif 66 #ifdef JEMALLOC_FILL 67 bool opt_zero = false; 68 #endif 69 size_t opt_narenas = 0; 70 71 /******************************************************************************/ 72 /* Function prototypes for non-inline static functions. */ 73 74 static void wrtmessage(void *cbopaque, const char *s); 75 static void stats_print_atexit(void); 76 static unsigned malloc_ncpus(void); 77 static void arenas_cleanup(void *arg); 78 #if (defined(JEMALLOC_STATS) && defined(NO_TLS)) 79 static void thread_allocated_cleanup(void *arg); 80 #endif 81 static bool malloc_conf_next(char const **opts_p, char const **k_p, 82 size_t *klen_p, char const **v_p, size_t *vlen_p); 83 static void malloc_conf_error(const char *msg, const char *k, size_t klen, 84 const char *v, size_t vlen); 85 static void malloc_conf_init(void); 86 static bool malloc_init_hard(void); 87 88 /******************************************************************************/ 89 /* malloc_message() setup. */ 90 91 #ifdef JEMALLOC_HAVE_ATTR 92 JEMALLOC_ATTR(visibility("hidden")) 93 #else 94 static 95 #endif 96 void 97 wrtmessage(void *cbopaque, const char *s) 98 { 99 #ifdef JEMALLOC_CC_SILENCE 100 int result = 101 #endif 102 write(STDERR_FILENO, s, strlen(s)); 103 #ifdef JEMALLOC_CC_SILENCE 104 if (result < 0) 105 result = errno; 106 #endif 107 } 108 109 void (*JEMALLOC_P(malloc_message))(void *, const char *s) 110 JEMALLOC_ATTR(visibility("default")) = wrtmessage; 111 112 /******************************************************************************/ 113 /* 114 * Begin miscellaneous support functions. 115 */ 116 117 /* Create a new arena and insert it into the arenas array at index ind. */ 118 arena_t * 119 arenas_extend(unsigned ind) 120 { 121 arena_t *ret; 122 123 /* Allocate enough space for trailing bins. */ 124 ret = (arena_t *)base_alloc(offsetof(arena_t, bins) 125 + (sizeof(arena_bin_t) * nbins)); 126 if (ret != NULL && arena_new(ret, ind) == false) { 127 arenas[ind] = ret; 128 return (ret); 129 } 130 /* Only reached if there is an OOM error. */ 131 132 /* 133 * OOM here is quite inconvenient to propagate, since dealing with it 134 * would require a check for failure in the fast path. Instead, punt 135 * by using arenas[0]. In practice, this is an extremely unlikely 136 * failure. 137 */ 138 malloc_write("<jemalloc>: Error initializing arena\n"); 139 if (opt_abort) 140 abort(); 141 142 return (arenas[0]); 143 } 144 145 /* 146 * Choose an arena based on a per-thread value (slow-path code only, called 147 * only by choose_arena()). 148 */ 149 arena_t * 150 choose_arena_hard(void) 151 { 152 arena_t *ret; 153 154 if (narenas > 1) { 155 unsigned i, choose, first_null; 156 157 choose = 0; 158 first_null = narenas; 159 malloc_mutex_lock(&arenas_lock); 160 assert(arenas[0] != NULL); 161 for (i = 1; i < narenas; i++) { 162 if (arenas[i] != NULL) { 163 /* 164 * Choose the first arena that has the lowest 165 * number of threads assigned to it. 166 */ 167 if (arenas[i]->nthreads < 168 arenas[choose]->nthreads) 169 choose = i; 170 } else if (first_null == narenas) { 171 /* 172 * Record the index of the first uninitialized 173 * arena, in case all extant arenas are in use. 174 * 175 * NB: It is possible for there to be 176 * discontinuities in terms of initialized 177 * versus uninitialized arenas, due to the 178 * "thread.arena" mallctl. 179 */ 180 first_null = i; 181 } 182 } 183 184 if (arenas[choose] == 0 || first_null == narenas) { 185 /* 186 * Use an unloaded arena, or the least loaded arena if 187 * all arenas are already initialized. 188 */ 189 ret = arenas[choose]; 190 } else { 191 /* Initialize a new arena. */ 192 ret = arenas_extend(first_null); 193 } 194 ret->nthreads++; 195 malloc_mutex_unlock(&arenas_lock); 196 } else { 197 ret = arenas[0]; 198 malloc_mutex_lock(&arenas_lock); 199 ret->nthreads++; 200 malloc_mutex_unlock(&arenas_lock); 201 } 202 203 ARENA_SET(ret); 204 205 return (ret); 206 } 207 208 /* 209 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so 210 * provide a wrapper. 211 */ 212 int 213 buferror(int errnum, char *buf, size_t buflen) 214 { 215 #ifdef _GNU_SOURCE 216 char *b = strerror_r(errno, buf, buflen); 217 if (b != buf) { 218 strncpy(buf, b, buflen); 219 buf[buflen-1] = '\0'; 220 } 221 return (0); 222 #else 223 return (strerror_r(errno, buf, buflen)); 224 #endif 225 } 226 227 static void 228 stats_print_atexit(void) 229 { 230 231 #if (defined(JEMALLOC_TCACHE) && defined(JEMALLOC_STATS)) 232 unsigned i; 233 234 /* 235 * Merge stats from extant threads. This is racy, since individual 236 * threads do not lock when recording tcache stats events. As a 237 * consequence, the final stats may be slightly out of date by the time 238 * they are reported, if other threads continue to allocate. 239 */ 240 for (i = 0; i < narenas; i++) { 241 arena_t *arena = arenas[i]; 242 if (arena != NULL) { 243 tcache_t *tcache; 244 245 /* 246 * tcache_stats_merge() locks bins, so if any code is 247 * introduced that acquires both arena and bin locks in 248 * the opposite order, deadlocks may result. 249 */ 250 malloc_mutex_lock(&arena->lock); 251 ql_foreach(tcache, &arena->tcache_ql, link) { 252 tcache_stats_merge(tcache, arena); 253 } 254 malloc_mutex_unlock(&arena->lock); 255 } 256 } 257 #endif 258 JEMALLOC_P(malloc_stats_print)(NULL, NULL, NULL); 259 } 260 261 #if (defined(JEMALLOC_STATS) && defined(NO_TLS)) 262 thread_allocated_t * 263 thread_allocated_get_hard(void) 264 { 265 thread_allocated_t *thread_allocated = (thread_allocated_t *) 266 imalloc(sizeof(thread_allocated_t)); 267 if (thread_allocated == NULL) { 268 static thread_allocated_t static_thread_allocated = {0, 0}; 269 malloc_write("<jemalloc>: Error allocating TSD;" 270 " mallctl(\"thread.{de,}allocated[p]\", ...)" 271 " will be inaccurate\n"); 272 if (opt_abort) 273 abort(); 274 return (&static_thread_allocated); 275 } 276 pthread_setspecific(thread_allocated_tsd, thread_allocated); 277 thread_allocated->allocated = 0; 278 thread_allocated->deallocated = 0; 279 return (thread_allocated); 280 } 281 #endif 282 283 /* 284 * End miscellaneous support functions. 285 */ 286 /******************************************************************************/ 287 /* 288 * Begin initialization functions. 289 */ 290 291 static unsigned 292 malloc_ncpus(void) 293 { 294 unsigned ret; 295 long result; 296 297 result = sysconf(_SC_NPROCESSORS_ONLN); 298 if (result == -1) { 299 /* Error. */ 300 ret = 1; 301 } 302 ret = (unsigned)result; 303 304 return (ret); 305 } 306 307 static void 308 arenas_cleanup(void *arg) 309 { 310 arena_t *arena = (arena_t *)arg; 311 312 malloc_mutex_lock(&arenas_lock); 313 arena->nthreads--; 314 malloc_mutex_unlock(&arenas_lock); 315 } 316 317 #if (defined(JEMALLOC_STATS) && defined(NO_TLS)) 318 static void 319 thread_allocated_cleanup(void *arg) 320 { 321 uint64_t *allocated = (uint64_t *)arg; 322 323 if (allocated != NULL) 324 idalloc(allocated); 325 } 326 #endif 327 328 /* 329 * FreeBSD's pthreads implementation calls malloc(3), so the malloc 330 * implementation has to take pains to avoid infinite recursion during 331 * initialization. 332 */ 333 static inline bool 334 malloc_init(void) 335 { 336 337 if (malloc_initialized == false) 338 return (malloc_init_hard()); 339 340 return (false); 341 } 342 343 static bool 344 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 345 char const **v_p, size_t *vlen_p) 346 { 347 bool accept; 348 const char *opts = *opts_p; 349 350 *k_p = opts; 351 352 for (accept = false; accept == false;) { 353 switch (*opts) { 354 case 'A': case 'B': case 'C': case 'D': case 'E': 355 case 'F': case 'G': case 'H': case 'I': case 'J': 356 case 'K': case 'L': case 'M': case 'N': case 'O': 357 case 'P': case 'Q': case 'R': case 'S': case 'T': 358 case 'U': case 'V': case 'W': case 'X': case 'Y': 359 case 'Z': 360 case 'a': case 'b': case 'c': case 'd': case 'e': 361 case 'f': case 'g': case 'h': case 'i': case 'j': 362 case 'k': case 'l': case 'm': case 'n': case 'o': 363 case 'p': case 'q': case 'r': case 's': case 't': 364 case 'u': case 'v': case 'w': case 'x': case 'y': 365 case 'z': 366 case '0': case '1': case '2': case '3': case '4': 367 case '5': case '6': case '7': case '8': case '9': 368 case '_': 369 opts++; 370 break; 371 case ':': 372 opts++; 373 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 374 *v_p = opts; 375 accept = true; 376 break; 377 case '\0': 378 if (opts != *opts_p) { 379 malloc_write("<jemalloc>: Conf string " 380 "ends with key\n"); 381 } 382 return (true); 383 default: 384 malloc_write("<jemalloc>: Malformed conf " 385 "string\n"); 386 return (true); 387 } 388 } 389 390 for (accept = false; accept == false;) { 391 switch (*opts) { 392 case ',': 393 opts++; 394 /* 395 * Look ahead one character here, because the 396 * next time this function is called, it will 397 * assume that end of input has been cleanly 398 * reached if no input remains, but we have 399 * optimistically already consumed the comma if 400 * one exists. 401 */ 402 if (*opts == '\0') { 403 malloc_write("<jemalloc>: Conf string " 404 "ends with comma\n"); 405 } 406 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 407 accept = true; 408 break; 409 case '\0': 410 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 411 accept = true; 412 break; 413 default: 414 opts++; 415 break; 416 } 417 } 418 419 *opts_p = opts; 420 return (false); 421 } 422 423 static void 424 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 425 size_t vlen) 426 { 427 char buf[PATH_MAX + 1]; 428 429 malloc_write("<jemalloc>: "); 430 malloc_write(msg); 431 malloc_write(": "); 432 memcpy(buf, k, klen); 433 memcpy(&buf[klen], ":", 1); 434 memcpy(&buf[klen+1], v, vlen); 435 buf[klen+1+vlen] = '\0'; 436 malloc_write(buf); 437 malloc_write("\n"); 438 } 439 440 static void 441 malloc_conf_init(void) 442 { 443 unsigned i; 444 char buf[PATH_MAX + 1]; 445 const char *opts, *k, *v; 446 size_t klen, vlen; 447 448 for (i = 0; i < 3; i++) { 449 /* Get runtime configuration. */ 450 switch (i) { 451 case 0: 452 if (JEMALLOC_P(malloc_conf) != NULL) { 453 /* 454 * Use options that were compiled into the 455 * program. 456 */ 457 opts = JEMALLOC_P(malloc_conf); 458 } else { 459 /* No configuration specified. */ 460 buf[0] = '\0'; 461 opts = buf; 462 } 463 break; 464 case 1: { 465 int linklen; 466 const char *linkname = 467 #ifdef JEMALLOC_PREFIX 468 "/etc/"JEMALLOC_PREFIX"malloc.conf" 469 #else 470 "/etc/malloc.conf" 471 #endif 472 ; 473 474 if ((linklen = readlink(linkname, buf, 475 sizeof(buf) - 1)) != -1) { 476 /* 477 * Use the contents of the "/etc/malloc.conf" 478 * symbolic link's name. 479 */ 480 buf[linklen] = '\0'; 481 opts = buf; 482 } else { 483 /* No configuration specified. */ 484 buf[0] = '\0'; 485 opts = buf; 486 } 487 break; 488 } 489 case 2: { 490 const char *envname = 491 #ifdef JEMALLOC_PREFIX 492 JEMALLOC_CPREFIX"MALLOC_CONF" 493 #else 494 "MALLOC_CONF" 495 #endif 496 ; 497 498 if ((opts = getenv(envname)) != NULL) { 499 /* 500 * Do nothing; opts is already initialized to 501 * the value of the MALLOC_CONF environment 502 * variable. 503 */ 504 } else { 505 /* No configuration specified. */ 506 buf[0] = '\0'; 507 opts = buf; 508 } 509 break; 510 } 511 default: 512 /* NOTREACHED */ 513 assert(false); 514 buf[0] = '\0'; 515 opts = buf; 516 } 517 518 while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, 519 &vlen) == false) { 520 #define CONF_HANDLE_BOOL(n) \ 521 if (sizeof(#n)-1 == klen && strncmp(#n, k, \ 522 klen) == 0) { \ 523 if (strncmp("true", v, vlen) == 0 && \ 524 vlen == sizeof("true")-1) \ 525 opt_##n = true; \ 526 else if (strncmp("false", v, vlen) == \ 527 0 && vlen == sizeof("false")-1) \ 528 opt_##n = false; \ 529 else { \ 530 malloc_conf_error( \ 531 "Invalid conf value", \ 532 k, klen, v, vlen); \ 533 } \ 534 continue; \ 535 } 536 #define CONF_HANDLE_SIZE_T(n, min, max) \ 537 if (sizeof(#n)-1 == klen && strncmp(#n, k, \ 538 klen) == 0) { \ 539 unsigned long ul; \ 540 char *end; \ 541 \ 542 errno = 0; \ 543 ul = strtoul(v, &end, 0); \ 544 if (errno != 0 || (uintptr_t)end - \ 545 (uintptr_t)v != vlen) { \ 546 malloc_conf_error( \ 547 "Invalid conf value", \ 548 k, klen, v, vlen); \ 549 } else if (ul < min || ul > max) { \ 550 malloc_conf_error( \ 551 "Out-of-range conf value", \ 552 k, klen, v, vlen); \ 553 } else \ 554 opt_##n = ul; \ 555 continue; \ 556 } 557 #define CONF_HANDLE_SSIZE_T(n, min, max) \ 558 if (sizeof(#n)-1 == klen && strncmp(#n, k, \ 559 klen) == 0) { \ 560 long l; \ 561 char *end; \ 562 \ 563 errno = 0; \ 564 l = strtol(v, &end, 0); \ 565 if (errno != 0 || (uintptr_t)end - \ 566 (uintptr_t)v != vlen) { \ 567 malloc_conf_error( \ 568 "Invalid conf value", \ 569 k, klen, v, vlen); \ 570 } else if (l < (ssize_t)min || l > \ 571 (ssize_t)max) { \ 572 malloc_conf_error( \ 573 "Out-of-range conf value", \ 574 k, klen, v, vlen); \ 575 } else \ 576 opt_##n = l; \ 577 continue; \ 578 } 579 #define CONF_HANDLE_CHAR_P(n, d) \ 580 if (sizeof(#n)-1 == klen && strncmp(#n, k, \ 581 klen) == 0) { \ 582 size_t cpylen = (vlen <= \ 583 sizeof(opt_##n)-1) ? vlen : \ 584 sizeof(opt_##n)-1; \ 585 strncpy(opt_##n, v, cpylen); \ 586 opt_##n[cpylen] = '\0'; \ 587 continue; \ 588 } 589 590 CONF_HANDLE_BOOL(abort) 591 CONF_HANDLE_SIZE_T(lg_qspace_max, LG_QUANTUM, 592 PAGE_SHIFT-1) 593 CONF_HANDLE_SIZE_T(lg_cspace_max, LG_QUANTUM, 594 PAGE_SHIFT-1) 595 /* 596 * Chunks always require at least one * header page, 597 * plus one data page. 598 */ 599 CONF_HANDLE_SIZE_T(lg_chunk, PAGE_SHIFT+1, 600 (sizeof(size_t) << 3) - 1) 601 CONF_HANDLE_SIZE_T(narenas, 1, SIZE_T_MAX) 602 CONF_HANDLE_SSIZE_T(lg_dirty_mult, -1, 603 (sizeof(size_t) << 3) - 1) 604 CONF_HANDLE_BOOL(stats_print) 605 #ifdef JEMALLOC_FILL 606 CONF_HANDLE_BOOL(junk) 607 CONF_HANDLE_BOOL(zero) 608 #endif 609 #ifdef JEMALLOC_SYSV 610 CONF_HANDLE_BOOL(sysv) 611 #endif 612 #ifdef JEMALLOC_XMALLOC 613 CONF_HANDLE_BOOL(xmalloc) 614 #endif 615 #ifdef JEMALLOC_TCACHE 616 CONF_HANDLE_BOOL(tcache) 617 CONF_HANDLE_SSIZE_T(lg_tcache_gc_sweep, -1, 618 (sizeof(size_t) << 3) - 1) 619 CONF_HANDLE_SSIZE_T(lg_tcache_max, -1, 620 (sizeof(size_t) << 3) - 1) 621 #endif 622 #ifdef JEMALLOC_PROF 623 CONF_HANDLE_BOOL(prof) 624 CONF_HANDLE_CHAR_P(prof_prefix, "jeprof") 625 CONF_HANDLE_SIZE_T(lg_prof_bt_max, 0, LG_PROF_BT_MAX) 626 CONF_HANDLE_BOOL(prof_active) 627 CONF_HANDLE_SSIZE_T(lg_prof_sample, 0, 628 (sizeof(uint64_t) << 3) - 1) 629 CONF_HANDLE_BOOL(prof_accum) 630 CONF_HANDLE_SSIZE_T(lg_prof_tcmax, -1, 631 (sizeof(size_t) << 3) - 1) 632 CONF_HANDLE_SSIZE_T(lg_prof_interval, -1, 633 (sizeof(uint64_t) << 3) - 1) 634 CONF_HANDLE_BOOL(prof_gdump) 635 CONF_HANDLE_BOOL(prof_leak) 636 #endif 637 #ifdef JEMALLOC_SWAP 638 CONF_HANDLE_BOOL(overcommit) 639 #endif 640 malloc_conf_error("Invalid conf pair", k, klen, v, 641 vlen); 642 #undef CONF_HANDLE_BOOL 643 #undef CONF_HANDLE_SIZE_T 644 #undef CONF_HANDLE_SSIZE_T 645 #undef CONF_HANDLE_CHAR_P 646 } 647 648 /* Validate configuration of options that are inter-related. */ 649 if (opt_lg_qspace_max+1 >= opt_lg_cspace_max) { 650 malloc_write("<jemalloc>: Invalid lg_[qc]space_max " 651 "relationship; restoring defaults\n"); 652 opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; 653 opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; 654 } 655 } 656 } 657 658 static bool 659 malloc_init_hard(void) 660 { 661 arena_t *init_arenas[1]; 662 663 malloc_mutex_lock(&init_lock); 664 if (malloc_initialized || malloc_initializer == pthread_self()) { 665 /* 666 * Another thread initialized the allocator before this one 667 * acquired init_lock, or this thread is the initializing 668 * thread, and it is recursively allocating. 669 */ 670 malloc_mutex_unlock(&init_lock); 671 return (false); 672 } 673 if (malloc_initializer != (unsigned long)0) { 674 /* Busy-wait until the initializing thread completes. */ 675 do { 676 malloc_mutex_unlock(&init_lock); 677 CPU_SPINWAIT; 678 malloc_mutex_lock(&init_lock); 679 } while (malloc_initialized == false); 680 malloc_mutex_unlock(&init_lock); 681 return (false); 682 } 683 684 #ifdef DYNAMIC_PAGE_SHIFT 685 /* Get page size. */ 686 { 687 long result; 688 689 result = sysconf(_SC_PAGESIZE); 690 assert(result != -1); 691 pagesize = (unsigned)result; 692 693 /* 694 * We assume that pagesize is a power of 2 when calculating 695 * pagesize_mask and lg_pagesize. 696 */ 697 assert(((result - 1) & result) == 0); 698 pagesize_mask = result - 1; 699 lg_pagesize = ffs((int)result) - 1; 700 } 701 #endif 702 703 #ifdef JEMALLOC_PROF 704 prof_boot0(); 705 #endif 706 707 malloc_conf_init(); 708 709 /* Register fork handlers. */ 710 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork, 711 jemalloc_postfork) != 0) { 712 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 713 if (opt_abort) 714 abort(); 715 } 716 717 if (ctl_boot()) { 718 malloc_mutex_unlock(&init_lock); 719 return (true); 720 } 721 722 if (opt_stats_print) { 723 /* Print statistics at exit. */ 724 if (atexit(stats_print_atexit) != 0) { 725 malloc_write("<jemalloc>: Error in atexit()\n"); 726 if (opt_abort) 727 abort(); 728 } 729 } 730 731 if (chunk_boot()) { 732 malloc_mutex_unlock(&init_lock); 733 return (true); 734 } 735 736 if (base_boot()) { 737 malloc_mutex_unlock(&init_lock); 738 return (true); 739 } 740 741 #ifdef JEMALLOC_PROF 742 prof_boot1(); 743 #endif 744 745 if (arena_boot()) { 746 malloc_mutex_unlock(&init_lock); 747 return (true); 748 } 749 750 #ifdef JEMALLOC_TCACHE 751 if (tcache_boot()) { 752 malloc_mutex_unlock(&init_lock); 753 return (true); 754 } 755 #endif 756 757 if (huge_boot()) { 758 malloc_mutex_unlock(&init_lock); 759 return (true); 760 } 761 762 #if (defined(JEMALLOC_STATS) && defined(NO_TLS)) 763 /* Initialize allocation counters before any allocations can occur. */ 764 if (pthread_key_create(&thread_allocated_tsd, thread_allocated_cleanup) 765 != 0) { 766 malloc_mutex_unlock(&init_lock); 767 return (true); 768 } 769 #endif 770 771 /* 772 * Create enough scaffolding to allow recursive allocation in 773 * malloc_ncpus(). 774 */ 775 narenas = 1; 776 arenas = init_arenas; 777 memset(arenas, 0, sizeof(arena_t *) * narenas); 778 779 /* 780 * Initialize one arena here. The rest are lazily created in 781 * choose_arena_hard(). 782 */ 783 arenas_extend(0); 784 if (arenas[0] == NULL) { 785 malloc_mutex_unlock(&init_lock); 786 return (true); 787 } 788 789 /* 790 * Assign the initial arena to the initial thread, in order to avoid 791 * spurious creation of an extra arena if the application switches to 792 * threaded mode. 793 */ 794 ARENA_SET(arenas[0]); 795 arenas[0]->nthreads++; 796 797 if (malloc_mutex_init(&arenas_lock)) 798 return (true); 799 800 if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) { 801 malloc_mutex_unlock(&init_lock); 802 return (true); 803 } 804 805 #ifdef JEMALLOC_PROF 806 if (prof_boot2()) { 807 malloc_mutex_unlock(&init_lock); 808 return (true); 809 } 810 #endif 811 812 /* Get number of CPUs. */ 813 malloc_initializer = pthread_self(); 814 malloc_mutex_unlock(&init_lock); 815 ncpus = malloc_ncpus(); 816 malloc_mutex_lock(&init_lock); 817 818 if (opt_narenas == 0) { 819 /* 820 * For SMP systems, create more than one arena per CPU by 821 * default. 822 */ 823 if (ncpus > 1) 824 opt_narenas = ncpus << 2; 825 else 826 opt_narenas = 1; 827 } 828 narenas = opt_narenas; 829 /* 830 * Make sure that the arenas array can be allocated. In practice, this 831 * limit is enough to allow the allocator to function, but the ctl 832 * machinery will fail to allocate memory at far lower limits. 833 */ 834 if (narenas > chunksize / sizeof(arena_t *)) { 835 char buf[UMAX2S_BUFSIZE]; 836 837 narenas = chunksize / sizeof(arena_t *); 838 malloc_write("<jemalloc>: Reducing narenas to limit ("); 839 malloc_write(u2s(narenas, 10, buf)); 840 malloc_write(")\n"); 841 } 842 843 /* Allocate and initialize arenas. */ 844 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas); 845 if (arenas == NULL) { 846 malloc_mutex_unlock(&init_lock); 847 return (true); 848 } 849 /* 850 * Zero the array. In practice, this should always be pre-zeroed, 851 * since it was just mmap()ed, but let's be sure. 852 */ 853 memset(arenas, 0, sizeof(arena_t *) * narenas); 854 /* Copy the pointer to the one arena that was already initialized. */ 855 arenas[0] = init_arenas[0]; 856 857 #ifdef JEMALLOC_ZONE 858 /* Register the custom zone. */ 859 malloc_zone_register(create_zone()); 860 861 /* 862 * Convert the default szone to an "overlay zone" that is capable of 863 * deallocating szone-allocated objects, but allocating new objects 864 * from jemalloc. 865 */ 866 szone2ozone(malloc_default_zone()); 867 #endif 868 869 malloc_initialized = true; 870 malloc_mutex_unlock(&init_lock); 871 return (false); 872 } 873 874 #ifdef JEMALLOC_ZONE 875 JEMALLOC_ATTR(constructor) 876 void 877 jemalloc_darwin_init(void) 878 { 879 880 if (malloc_init_hard()) 881 abort(); 882 } 883 #endif 884 885 /* 886 * End initialization functions. 887 */ 888 /******************************************************************************/ 889 /* 890 * Begin malloc(3)-compatible functions. 891 */ 892 893 JEMALLOC_ATTR(malloc) 894 JEMALLOC_ATTR(visibility("default")) 895 void * 896 JEMALLOC_P(malloc)(size_t size) 897 { 898 void *ret; 899 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 900 size_t usize 901 # ifdef JEMALLOC_CC_SILENCE 902 = 0 903 # endif 904 ; 905 #endif 906 #ifdef JEMALLOC_PROF 907 prof_thr_cnt_t *cnt 908 # ifdef JEMALLOC_CC_SILENCE 909 = NULL 910 # endif 911 ; 912 #endif 913 914 if (malloc_init()) { 915 ret = NULL; 916 goto OOM; 917 } 918 919 if (size == 0) { 920 #ifdef JEMALLOC_SYSV 921 if (opt_sysv == false) 922 #endif 923 size = 1; 924 #ifdef JEMALLOC_SYSV 925 else { 926 # ifdef JEMALLOC_XMALLOC 927 if (opt_xmalloc) { 928 malloc_write("<jemalloc>: Error in malloc(): " 929 "invalid size 0\n"); 930 abort(); 931 } 932 # endif 933 ret = NULL; 934 goto RETURN; 935 } 936 #endif 937 } 938 939 #ifdef JEMALLOC_PROF 940 if (opt_prof) { 941 usize = s2u(size); 942 if ((cnt = prof_alloc_prep(usize)) == NULL) { 943 ret = NULL; 944 goto OOM; 945 } 946 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= 947 small_maxclass) { 948 ret = imalloc(small_maxclass+1); 949 if (ret != NULL) 950 arena_prof_promoted(ret, usize); 951 } else 952 ret = imalloc(size); 953 } else 954 #endif 955 { 956 #ifdef JEMALLOC_STATS 957 usize = s2u(size); 958 #endif 959 ret = imalloc(size); 960 } 961 962 OOM: 963 if (ret == NULL) { 964 #ifdef JEMALLOC_XMALLOC 965 if (opt_xmalloc) { 966 malloc_write("<jemalloc>: Error in malloc(): " 967 "out of memory\n"); 968 abort(); 969 } 970 #endif 971 errno = ENOMEM; 972 } 973 974 #ifdef JEMALLOC_SYSV 975 RETURN: 976 #endif 977 #ifdef JEMALLOC_PROF 978 if (opt_prof && ret != NULL) 979 prof_malloc(ret, usize, cnt); 980 #endif 981 #ifdef JEMALLOC_STATS 982 if (ret != NULL) { 983 assert(usize == isalloc(ret)); 984 ALLOCATED_ADD(usize, 0); 985 } 986 #endif 987 return (ret); 988 } 989 990 JEMALLOC_ATTR(nonnull(1)) 991 JEMALLOC_ATTR(visibility("default")) 992 int 993 JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size) 994 { 995 int ret; 996 size_t usize 997 #ifdef JEMALLOC_CC_SILENCE 998 = 0 999 #endif 1000 ; 1001 void *result; 1002 #ifdef JEMALLOC_PROF 1003 prof_thr_cnt_t *cnt 1004 # ifdef JEMALLOC_CC_SILENCE 1005 = NULL 1006 # endif 1007 ; 1008 #endif 1009 1010 if (malloc_init()) 1011 result = NULL; 1012 else { 1013 if (size == 0) { 1014 #ifdef JEMALLOC_SYSV 1015 if (opt_sysv == false) 1016 #endif 1017 size = 1; 1018 #ifdef JEMALLOC_SYSV 1019 else { 1020 # ifdef JEMALLOC_XMALLOC 1021 if (opt_xmalloc) { 1022 malloc_write("<jemalloc>: Error in " 1023 "posix_memalign(): invalid size " 1024 "0\n"); 1025 abort(); 1026 } 1027 # endif 1028 result = NULL; 1029 *memptr = NULL; 1030 ret = 0; 1031 goto RETURN; 1032 } 1033 #endif 1034 } 1035 1036 /* Make sure that alignment is a large enough power of 2. */ 1037 if (((alignment - 1) & alignment) != 0 1038 || alignment < sizeof(void *)) { 1039 #ifdef JEMALLOC_XMALLOC 1040 if (opt_xmalloc) { 1041 malloc_write("<jemalloc>: Error in " 1042 "posix_memalign(): invalid alignment\n"); 1043 abort(); 1044 } 1045 #endif 1046 result = NULL; 1047 ret = EINVAL; 1048 goto RETURN; 1049 } 1050 1051 usize = sa2u(size, alignment, NULL); 1052 if (usize == 0) { 1053 result = NULL; 1054 ret = ENOMEM; 1055 goto RETURN; 1056 } 1057 1058 #ifdef JEMALLOC_PROF 1059 if (opt_prof) { 1060 if ((cnt = prof_alloc_prep(usize)) == NULL) { 1061 result = NULL; 1062 ret = EINVAL; 1063 } else { 1064 if (prof_promote && (uintptr_t)cnt != 1065 (uintptr_t)1U && usize <= small_maxclass) { 1066 assert(sa2u(small_maxclass+1, 1067 alignment, NULL) != 0); 1068 result = ipalloc(sa2u(small_maxclass+1, 1069 alignment, NULL), alignment, false); 1070 if (result != NULL) { 1071 arena_prof_promoted(result, 1072 usize); 1073 } 1074 } else { 1075 result = ipalloc(usize, alignment, 1076 false); 1077 } 1078 } 1079 } else 1080 #endif 1081 result = ipalloc(usize, alignment, false); 1082 } 1083 1084 if (result == NULL) { 1085 #ifdef JEMALLOC_XMALLOC 1086 if (opt_xmalloc) { 1087 malloc_write("<jemalloc>: Error in posix_memalign(): " 1088 "out of memory\n"); 1089 abort(); 1090 } 1091 #endif 1092 ret = ENOMEM; 1093 goto RETURN; 1094 } 1095 1096 *memptr = result; 1097 ret = 0; 1098 1099 RETURN: 1100 #ifdef JEMALLOC_STATS 1101 if (result != NULL) { 1102 assert(usize == isalloc(result)); 1103 ALLOCATED_ADD(usize, 0); 1104 } 1105 #endif 1106 #ifdef JEMALLOC_PROF 1107 if (opt_prof && result != NULL) 1108 prof_malloc(result, usize, cnt); 1109 #endif 1110 return (ret); 1111 } 1112 1113 JEMALLOC_ATTR(malloc) 1114 JEMALLOC_ATTR(visibility("default")) 1115 void * 1116 JEMALLOC_P(calloc)(size_t num, size_t size) 1117 { 1118 void *ret; 1119 size_t num_size; 1120 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1121 size_t usize 1122 # ifdef JEMALLOC_CC_SILENCE 1123 = 0 1124 # endif 1125 ; 1126 #endif 1127 #ifdef JEMALLOC_PROF 1128 prof_thr_cnt_t *cnt 1129 # ifdef JEMALLOC_CC_SILENCE 1130 = NULL 1131 # endif 1132 ; 1133 #endif 1134 1135 if (malloc_init()) { 1136 num_size = 0; 1137 ret = NULL; 1138 goto RETURN; 1139 } 1140 1141 num_size = num * size; 1142 if (num_size == 0) { 1143 #ifdef JEMALLOC_SYSV 1144 if ((opt_sysv == false) && ((num == 0) || (size == 0))) 1145 #endif 1146 num_size = 1; 1147 #ifdef JEMALLOC_SYSV 1148 else { 1149 ret = NULL; 1150 goto RETURN; 1151 } 1152 #endif 1153 /* 1154 * Try to avoid division here. We know that it isn't possible to 1155 * overflow during multiplication if neither operand uses any of the 1156 * most significant half of the bits in a size_t. 1157 */ 1158 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) 1159 && (num_size / size != num)) { 1160 /* size_t overflow. */ 1161 ret = NULL; 1162 goto RETURN; 1163 } 1164 1165 #ifdef JEMALLOC_PROF 1166 if (opt_prof) { 1167 usize = s2u(num_size); 1168 if ((cnt = prof_alloc_prep(usize)) == NULL) { 1169 ret = NULL; 1170 goto RETURN; 1171 } 1172 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize 1173 <= small_maxclass) { 1174 ret = icalloc(small_maxclass+1); 1175 if (ret != NULL) 1176 arena_prof_promoted(ret, usize); 1177 } else 1178 ret = icalloc(num_size); 1179 } else 1180 #endif 1181 { 1182 #ifdef JEMALLOC_STATS 1183 usize = s2u(num_size); 1184 #endif 1185 ret = icalloc(num_size); 1186 } 1187 1188 RETURN: 1189 if (ret == NULL) { 1190 #ifdef JEMALLOC_XMALLOC 1191 if (opt_xmalloc) { 1192 malloc_write("<jemalloc>: Error in calloc(): out of " 1193 "memory\n"); 1194 abort(); 1195 } 1196 #endif 1197 errno = ENOMEM; 1198 } 1199 1200 #ifdef JEMALLOC_PROF 1201 if (opt_prof && ret != NULL) 1202 prof_malloc(ret, usize, cnt); 1203 #endif 1204 #ifdef JEMALLOC_STATS 1205 if (ret != NULL) { 1206 assert(usize == isalloc(ret)); 1207 ALLOCATED_ADD(usize, 0); 1208 } 1209 #endif 1210 return (ret); 1211 } 1212 1213 JEMALLOC_ATTR(visibility("default")) 1214 void * 1215 JEMALLOC_P(realloc)(void *ptr, size_t size) 1216 { 1217 void *ret; 1218 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1219 size_t usize 1220 # ifdef JEMALLOC_CC_SILENCE 1221 = 0 1222 # endif 1223 ; 1224 size_t old_size = 0; 1225 #endif 1226 #ifdef JEMALLOC_PROF 1227 prof_thr_cnt_t *cnt 1228 # ifdef JEMALLOC_CC_SILENCE 1229 = NULL 1230 # endif 1231 ; 1232 prof_ctx_t *old_ctx 1233 # ifdef JEMALLOC_CC_SILENCE 1234 = NULL 1235 # endif 1236 ; 1237 #endif 1238 1239 if (size == 0) { 1240 #ifdef JEMALLOC_SYSV 1241 if (opt_sysv == false) 1242 #endif 1243 size = 1; 1244 #ifdef JEMALLOC_SYSV 1245 else { 1246 if (ptr != NULL) { 1247 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1248 old_size = isalloc(ptr); 1249 #endif 1250 #ifdef JEMALLOC_PROF 1251 if (opt_prof) { 1252 old_ctx = prof_ctx_get(ptr); 1253 cnt = NULL; 1254 } 1255 #endif 1256 idalloc(ptr); 1257 } 1258 #ifdef JEMALLOC_PROF 1259 else if (opt_prof) { 1260 old_ctx = NULL; 1261 cnt = NULL; 1262 } 1263 #endif 1264 ret = NULL; 1265 goto RETURN; 1266 } 1267 #endif 1268 } 1269 1270 if (ptr != NULL) { 1271 assert(malloc_initialized || malloc_initializer == 1272 pthread_self()); 1273 1274 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1275 old_size = isalloc(ptr); 1276 #endif 1277 #ifdef JEMALLOC_PROF 1278 if (opt_prof) { 1279 usize = s2u(size); 1280 old_ctx = prof_ctx_get(ptr); 1281 if ((cnt = prof_alloc_prep(usize)) == NULL) { 1282 ret = NULL; 1283 goto OOM; 1284 } 1285 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && 1286 usize <= small_maxclass) { 1287 ret = iralloc(ptr, small_maxclass+1, 0, 0, 1288 false, false); 1289 if (ret != NULL) 1290 arena_prof_promoted(ret, usize); 1291 } else 1292 ret = iralloc(ptr, size, 0, 0, false, false); 1293 } else 1294 #endif 1295 { 1296 #ifdef JEMALLOC_STATS 1297 usize = s2u(size); 1298 #endif 1299 ret = iralloc(ptr, size, 0, 0, false, false); 1300 } 1301 1302 #ifdef JEMALLOC_PROF 1303 OOM: 1304 #endif 1305 if (ret == NULL) { 1306 #ifdef JEMALLOC_XMALLOC 1307 if (opt_xmalloc) { 1308 malloc_write("<jemalloc>: Error in realloc(): " 1309 "out of memory\n"); 1310 abort(); 1311 } 1312 #endif 1313 errno = ENOMEM; 1314 } 1315 } else { 1316 #ifdef JEMALLOC_PROF 1317 if (opt_prof) 1318 old_ctx = NULL; 1319 #endif 1320 if (malloc_init()) { 1321 #ifdef JEMALLOC_PROF 1322 if (opt_prof) 1323 cnt = NULL; 1324 #endif 1325 ret = NULL; 1326 } else { 1327 #ifdef JEMALLOC_PROF 1328 if (opt_prof) { 1329 usize = s2u(size); 1330 if ((cnt = prof_alloc_prep(usize)) == NULL) 1331 ret = NULL; 1332 else { 1333 if (prof_promote && (uintptr_t)cnt != 1334 (uintptr_t)1U && usize <= 1335 small_maxclass) { 1336 ret = imalloc(small_maxclass+1); 1337 if (ret != NULL) { 1338 arena_prof_promoted(ret, 1339 usize); 1340 } 1341 } else 1342 ret = imalloc(size); 1343 } 1344 } else 1345 #endif 1346 { 1347 #ifdef JEMALLOC_STATS 1348 usize = s2u(size); 1349 #endif 1350 ret = imalloc(size); 1351 } 1352 } 1353 1354 if (ret == NULL) { 1355 #ifdef JEMALLOC_XMALLOC 1356 if (opt_xmalloc) { 1357 malloc_write("<jemalloc>: Error in realloc(): " 1358 "out of memory\n"); 1359 abort(); 1360 } 1361 #endif 1362 errno = ENOMEM; 1363 } 1364 } 1365 1366 #ifdef JEMALLOC_SYSV 1367 RETURN: 1368 #endif 1369 #ifdef JEMALLOC_PROF 1370 if (opt_prof) 1371 prof_realloc(ret, usize, cnt, old_size, old_ctx); 1372 #endif 1373 #ifdef JEMALLOC_STATS 1374 if (ret != NULL) { 1375 assert(usize == isalloc(ret)); 1376 ALLOCATED_ADD(usize, old_size); 1377 } 1378 #endif 1379 return (ret); 1380 } 1381 1382 JEMALLOC_ATTR(visibility("default")) 1383 void 1384 JEMALLOC_P(free)(void *ptr) 1385 { 1386 1387 if (ptr != NULL) { 1388 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1389 size_t usize; 1390 #endif 1391 1392 assert(malloc_initialized || malloc_initializer == 1393 pthread_self()); 1394 1395 #ifdef JEMALLOC_STATS 1396 usize = isalloc(ptr); 1397 #endif 1398 #ifdef JEMALLOC_PROF 1399 if (opt_prof) { 1400 # ifndef JEMALLOC_STATS 1401 usize = isalloc(ptr); 1402 # endif 1403 prof_free(ptr, usize); 1404 } 1405 #endif 1406 #ifdef JEMALLOC_STATS 1407 ALLOCATED_ADD(0, usize); 1408 #endif 1409 idalloc(ptr); 1410 } 1411 } 1412 1413 /* 1414 * End malloc(3)-compatible functions. 1415 */ 1416 /******************************************************************************/ 1417 /* 1418 * Begin non-standard override functions. 1419 * 1420 * These overrides are omitted if the JEMALLOC_PREFIX is defined, since the 1421 * entire point is to avoid accidental mixed allocator usage. 1422 */ 1423 #ifndef JEMALLOC_PREFIX 1424 1425 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 1426 JEMALLOC_ATTR(malloc) 1427 JEMALLOC_ATTR(visibility("default")) 1428 void * 1429 JEMALLOC_P(memalign)(size_t alignment, size_t size) 1430 { 1431 void *ret; 1432 #ifdef JEMALLOC_CC_SILENCE 1433 int result = 1434 #endif 1435 JEMALLOC_P(posix_memalign)(&ret, alignment, size); 1436 #ifdef JEMALLOC_CC_SILENCE 1437 if (result != 0) 1438 return (NULL); 1439 #endif 1440 return (ret); 1441 } 1442 #endif 1443 1444 #ifdef JEMALLOC_OVERRIDE_VALLOC 1445 JEMALLOC_ATTR(malloc) 1446 JEMALLOC_ATTR(visibility("default")) 1447 void * 1448 JEMALLOC_P(valloc)(size_t size) 1449 { 1450 void *ret; 1451 #ifdef JEMALLOC_CC_SILENCE 1452 int result = 1453 #endif 1454 JEMALLOC_P(posix_memalign)(&ret, PAGE_SIZE, size); 1455 #ifdef JEMALLOC_CC_SILENCE 1456 if (result != 0) 1457 return (NULL); 1458 #endif 1459 return (ret); 1460 } 1461 #endif 1462 1463 #endif /* JEMALLOC_PREFIX */ 1464 /* 1465 * End non-standard override functions. 1466 */ 1467 /******************************************************************************/ 1468 /* 1469 * Begin non-standard functions. 1470 */ 1471 1472 JEMALLOC_ATTR(visibility("default")) 1473 size_t 1474 JEMALLOC_P(malloc_usable_size)(const void *ptr) 1475 { 1476 size_t ret; 1477 1478 assert(malloc_initialized || malloc_initializer == pthread_self()); 1479 1480 #ifdef JEMALLOC_IVSALLOC 1481 ret = ivsalloc(ptr); 1482 #else 1483 assert(ptr != NULL); 1484 ret = isalloc(ptr); 1485 #endif 1486 1487 return (ret); 1488 } 1489 1490 JEMALLOC_ATTR(visibility("default")) 1491 void 1492 JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *), 1493 void *cbopaque, const char *opts) 1494 { 1495 1496 stats_print(write_cb, cbopaque, opts); 1497 } 1498 1499 JEMALLOC_ATTR(visibility("default")) 1500 int 1501 JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, void *newp, 1502 size_t newlen) 1503 { 1504 1505 if (malloc_init()) 1506 return (EAGAIN); 1507 1508 return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 1509 } 1510 1511 JEMALLOC_ATTR(visibility("default")) 1512 int 1513 JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, size_t *miblenp) 1514 { 1515 1516 if (malloc_init()) 1517 return (EAGAIN); 1518 1519 return (ctl_nametomib(name, mibp, miblenp)); 1520 } 1521 1522 JEMALLOC_ATTR(visibility("default")) 1523 int 1524 JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, 1525 size_t *oldlenp, void *newp, size_t newlen) 1526 { 1527 1528 if (malloc_init()) 1529 return (EAGAIN); 1530 1531 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 1532 } 1533 1534 JEMALLOC_INLINE void * 1535 iallocm(size_t usize, size_t alignment, bool zero) 1536 { 1537 1538 assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, alignment, 1539 NULL))); 1540 1541 if (alignment != 0) 1542 return (ipalloc(usize, alignment, zero)); 1543 else if (zero) 1544 return (icalloc(usize)); 1545 else 1546 return (imalloc(usize)); 1547 } 1548 1549 JEMALLOC_ATTR(nonnull(1)) 1550 JEMALLOC_ATTR(visibility("default")) 1551 int 1552 JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags) 1553 { 1554 void *p; 1555 size_t usize; 1556 size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1557 & (SIZE_T_MAX-1)); 1558 bool zero = flags & ALLOCM_ZERO; 1559 #ifdef JEMALLOC_PROF 1560 prof_thr_cnt_t *cnt; 1561 #endif 1562 1563 assert(ptr != NULL); 1564 assert(size != 0); 1565 1566 if (malloc_init()) 1567 goto OOM; 1568 1569 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment, 1570 NULL); 1571 if (usize == 0) 1572 goto OOM; 1573 1574 #ifdef JEMALLOC_PROF 1575 if (opt_prof) { 1576 if ((cnt = prof_alloc_prep(usize)) == NULL) 1577 goto OOM; 1578 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= 1579 small_maxclass) { 1580 size_t usize_promoted = (alignment == 0) ? 1581 s2u(small_maxclass+1) : sa2u(small_maxclass+1, 1582 alignment, NULL); 1583 assert(usize_promoted != 0); 1584 p = iallocm(usize_promoted, alignment, zero); 1585 if (p == NULL) 1586 goto OOM; 1587 arena_prof_promoted(p, usize); 1588 } else { 1589 p = iallocm(usize, alignment, zero); 1590 if (p == NULL) 1591 goto OOM; 1592 } 1593 1594 if (rsize != NULL) 1595 *rsize = usize; 1596 } else 1597 #endif 1598 { 1599 p = iallocm(usize, alignment, zero); 1600 if (p == NULL) 1601 goto OOM; 1602 #ifndef JEMALLOC_STATS 1603 if (rsize != NULL) 1604 #endif 1605 { 1606 #ifdef JEMALLOC_STATS 1607 if (rsize != NULL) 1608 #endif 1609 *rsize = usize; 1610 } 1611 } 1612 1613 *ptr = p; 1614 #ifdef JEMALLOC_STATS 1615 assert(usize == isalloc(p)); 1616 ALLOCATED_ADD(usize, 0); 1617 #endif 1618 return (ALLOCM_SUCCESS); 1619 OOM: 1620 #ifdef JEMALLOC_XMALLOC 1621 if (opt_xmalloc) { 1622 malloc_write("<jemalloc>: Error in allocm(): " 1623 "out of memory\n"); 1624 abort(); 1625 } 1626 #endif 1627 *ptr = NULL; 1628 return (ALLOCM_ERR_OOM); 1629 } 1630 1631 JEMALLOC_ATTR(nonnull(1)) 1632 JEMALLOC_ATTR(visibility("default")) 1633 int 1634 JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, size_t extra, 1635 int flags) 1636 { 1637 void *p, *q; 1638 size_t usize; 1639 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1640 size_t old_size; 1641 #endif 1642 size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1643 & (SIZE_T_MAX-1)); 1644 bool zero = flags & ALLOCM_ZERO; 1645 bool no_move = flags & ALLOCM_NO_MOVE; 1646 #ifdef JEMALLOC_PROF 1647 prof_thr_cnt_t *cnt; 1648 prof_ctx_t *old_ctx; 1649 #endif 1650 1651 assert(ptr != NULL); 1652 assert(*ptr != NULL); 1653 assert(size != 0); 1654 assert(SIZE_T_MAX - size >= extra); 1655 assert(malloc_initialized || malloc_initializer == pthread_self()); 1656 1657 p = *ptr; 1658 #ifdef JEMALLOC_PROF 1659 if (opt_prof) { 1660 /* 1661 * usize isn't knowable before iralloc() returns when extra is 1662 * non-zero. Therefore, compute its maximum possible value and 1663 * use that in prof_alloc_prep() to decide whether to capture a 1664 * backtrace. prof_realloc() will use the actual usize to 1665 * decide whether to sample. 1666 */ 1667 size_t max_usize = (alignment == 0) ? s2u(size+extra) : 1668 sa2u(size+extra, alignment, NULL); 1669 old_size = isalloc(p); 1670 old_ctx = prof_ctx_get(p); 1671 if ((cnt = prof_alloc_prep(max_usize)) == NULL) 1672 goto OOM; 1673 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && max_usize 1674 <= small_maxclass) { 1675 q = iralloc(p, small_maxclass+1, (small_maxclass+1 >= 1676 size+extra) ? 0 : size+extra - (small_maxclass+1), 1677 alignment, zero, no_move); 1678 if (q == NULL) 1679 goto ERR; 1680 usize = isalloc(q); 1681 arena_prof_promoted(q, usize); 1682 } else { 1683 q = iralloc(p, size, extra, alignment, zero, no_move); 1684 if (q == NULL) 1685 goto ERR; 1686 usize = isalloc(q); 1687 } 1688 prof_realloc(q, usize, cnt, old_size, old_ctx); 1689 if (rsize != NULL) 1690 *rsize = usize; 1691 } else 1692 #endif 1693 { 1694 #ifdef JEMALLOC_STATS 1695 old_size = isalloc(p); 1696 #endif 1697 q = iralloc(p, size, extra, alignment, zero, no_move); 1698 if (q == NULL) 1699 goto ERR; 1700 #ifndef JEMALLOC_STATS 1701 if (rsize != NULL) 1702 #endif 1703 { 1704 usize = isalloc(q); 1705 #ifdef JEMALLOC_STATS 1706 if (rsize != NULL) 1707 #endif 1708 *rsize = usize; 1709 } 1710 } 1711 1712 *ptr = q; 1713 #ifdef JEMALLOC_STATS 1714 ALLOCATED_ADD(usize, old_size); 1715 #endif 1716 return (ALLOCM_SUCCESS); 1717 ERR: 1718 if (no_move) 1719 return (ALLOCM_ERR_NOT_MOVED); 1720 #ifdef JEMALLOC_PROF 1721 OOM: 1722 #endif 1723 #ifdef JEMALLOC_XMALLOC 1724 if (opt_xmalloc) { 1725 malloc_write("<jemalloc>: Error in rallocm(): " 1726 "out of memory\n"); 1727 abort(); 1728 } 1729 #endif 1730 return (ALLOCM_ERR_OOM); 1731 } 1732 1733 JEMALLOC_ATTR(nonnull(1)) 1734 JEMALLOC_ATTR(visibility("default")) 1735 int 1736 JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags) 1737 { 1738 size_t sz; 1739 1740 assert(malloc_initialized || malloc_initializer == pthread_self()); 1741 1742 #ifdef JEMALLOC_IVSALLOC 1743 sz = ivsalloc(ptr); 1744 #else 1745 assert(ptr != NULL); 1746 sz = isalloc(ptr); 1747 #endif 1748 assert(rsize != NULL); 1749 *rsize = sz; 1750 1751 return (ALLOCM_SUCCESS); 1752 } 1753 1754 JEMALLOC_ATTR(nonnull(1)) 1755 JEMALLOC_ATTR(visibility("default")) 1756 int 1757 JEMALLOC_P(dallocm)(void *ptr, int flags) 1758 { 1759 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1760 size_t usize; 1761 #endif 1762 1763 assert(ptr != NULL); 1764 assert(malloc_initialized || malloc_initializer == pthread_self()); 1765 1766 #ifdef JEMALLOC_STATS 1767 usize = isalloc(ptr); 1768 #endif 1769 #ifdef JEMALLOC_PROF 1770 if (opt_prof) { 1771 # ifndef JEMALLOC_STATS 1772 usize = isalloc(ptr); 1773 # endif 1774 prof_free(ptr, usize); 1775 } 1776 #endif 1777 #ifdef JEMALLOC_STATS 1778 ALLOCATED_ADD(0, usize); 1779 #endif 1780 idalloc(ptr); 1781 1782 return (ALLOCM_SUCCESS); 1783 } 1784 1785 /* 1786 * End non-standard functions. 1787 */ 1788 /******************************************************************************/ 1789 1790 /* 1791 * The following functions are used by threading libraries for protection of 1792 * malloc during fork(). 1793 */ 1794 1795 void 1796 jemalloc_prefork(void) 1797 { 1798 unsigned i; 1799 1800 /* Acquire all mutexes in a safe order. */ 1801 1802 malloc_mutex_lock(&arenas_lock); 1803 for (i = 0; i < narenas; i++) { 1804 if (arenas[i] != NULL) 1805 malloc_mutex_lock(&arenas[i]->lock); 1806 } 1807 1808 malloc_mutex_lock(&base_mtx); 1809 1810 malloc_mutex_lock(&huge_mtx); 1811 1812 #ifdef JEMALLOC_DSS 1813 malloc_mutex_lock(&dss_mtx); 1814 #endif 1815 1816 #ifdef JEMALLOC_SWAP 1817 malloc_mutex_lock(&swap_mtx); 1818 #endif 1819 } 1820 1821 void 1822 jemalloc_postfork(void) 1823 { 1824 unsigned i; 1825 1826 /* Release all mutexes, now that fork() has completed. */ 1827 1828 #ifdef JEMALLOC_SWAP 1829 malloc_mutex_unlock(&swap_mtx); 1830 #endif 1831 1832 #ifdef JEMALLOC_DSS 1833 malloc_mutex_unlock(&dss_mtx); 1834 #endif 1835 1836 malloc_mutex_unlock(&huge_mtx); 1837 1838 malloc_mutex_unlock(&base_mtx); 1839 1840 for (i = 0; i < narenas; i++) { 1841 if (arenas[i] != NULL) 1842 malloc_mutex_unlock(&arenas[i]->lock); 1843 } 1844 malloc_mutex_unlock(&arenas_lock); 1845 } 1846 1847 /******************************************************************************/ 1848