1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 malloc_mutex_t arenas_lock; 8 arena_t **arenas; 9 unsigned narenas; 10 11 pthread_key_t arenas_tsd; 12 #ifndef NO_TLS 13 __thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); 14 #endif 15 16 #ifdef JEMALLOC_STATS 17 # ifndef NO_TLS 18 __thread thread_allocated_t thread_allocated_tls; 19 # else 20 pthread_key_t thread_allocated_tsd; 21 # endif 22 #endif 23 24 /* Set to true once the allocator has been initialized. */ 25 static bool malloc_initialized = false; 26 27 /* Used to let the initializing thread recursively allocate. */ 28 static pthread_t malloc_initializer = (unsigned long)0; 29 30 /* Used to avoid initialization races. */ 31 static malloc_mutex_t init_lock = 32 #ifdef JEMALLOC_OSSPIN 33 0 34 #else 35 MALLOC_MUTEX_INITIALIZER 36 #endif 37 ; 38 39 #ifdef DYNAMIC_PAGE_SHIFT 40 size_t pagesize; 41 size_t pagesize_mask; 42 size_t lg_pagesize; 43 #endif 44 45 unsigned ncpus; 46 47 /* Runtime configuration options. */ 48 const char *JEMALLOC_P(malloc_conf) JEMALLOC_ATTR(visibility("default")); 49 #ifdef JEMALLOC_DEBUG 50 bool opt_abort = true; 51 # ifdef JEMALLOC_FILL 52 bool opt_junk = true; 53 # endif 54 #else 55 bool opt_abort = false; 56 # ifdef JEMALLOC_FILL 57 bool opt_junk = false; 58 # endif 59 #endif 60 #ifdef JEMALLOC_SYSV 61 bool opt_sysv = false; 62 #endif 63 #ifdef JEMALLOC_XMALLOC 64 bool opt_xmalloc = false; 65 #endif 66 #ifdef JEMALLOC_FILL 67 bool opt_zero = false; 68 #endif 69 size_t opt_narenas = 0; 70 71 /******************************************************************************/ 72 /* Function prototypes for non-inline static functions. */ 73 74 static void wrtmessage(void *cbopaque, const char *s); 75 static void stats_print_atexit(void); 76 static unsigned malloc_ncpus(void); 77 static void arenas_cleanup(void *arg); 78 #if (defined(JEMALLOC_STATS) && defined(NO_TLS)) 79 static void thread_allocated_cleanup(void *arg); 80 #endif 81 static bool malloc_conf_next(char const **opts_p, char const **k_p, 82 size_t *klen_p, char const **v_p, size_t *vlen_p); 83 static void malloc_conf_error(const char *msg, const char *k, size_t klen, 84 const char *v, size_t vlen); 85 static void malloc_conf_init(void); 86 static bool malloc_init_hard(void); 87 static int imemalign(void **memptr, size_t alignment, size_t size); 88 89 /******************************************************************************/ 90 /* malloc_message() setup. */ 91 92 #ifdef JEMALLOC_HAVE_ATTR 93 JEMALLOC_ATTR(visibility("hidden")) 94 #else 95 static 96 #endif 97 void 98 wrtmessage(void *cbopaque, const char *s) 99 { 100 #ifdef JEMALLOC_CC_SILENCE 101 int result = 102 #endif 103 write(STDERR_FILENO, s, strlen(s)); 104 #ifdef JEMALLOC_CC_SILENCE 105 if (result < 0) 106 result = errno; 107 #endif 108 } 109 110 void (*JEMALLOC_P(malloc_message))(void *, const char *s) 111 JEMALLOC_ATTR(visibility("default")) = wrtmessage; 112 113 /******************************************************************************/ 114 /* 115 * Begin miscellaneous support functions. 116 */ 117 118 /* Create a new arena and insert it into the arenas array at index ind. */ 119 arena_t * 120 arenas_extend(unsigned ind) 121 { 122 arena_t *ret; 123 124 /* Allocate enough space for trailing bins. */ 125 ret = (arena_t *)base_alloc(offsetof(arena_t, bins) 126 + (sizeof(arena_bin_t) * nbins)); 127 if (ret != NULL && arena_new(ret, ind) == false) { 128 arenas[ind] = ret; 129 return (ret); 130 } 131 /* Only reached if there is an OOM error. */ 132 133 /* 134 * OOM here is quite inconvenient to propagate, since dealing with it 135 * would require a check for failure in the fast path. Instead, punt 136 * by using arenas[0]. In practice, this is an extremely unlikely 137 * failure. 138 */ 139 malloc_write("<jemalloc>: Error initializing arena\n"); 140 if (opt_abort) 141 abort(); 142 143 return (arenas[0]); 144 } 145 146 /* 147 * Choose an arena based on a per-thread value (slow-path code only, called 148 * only by choose_arena()). 149 */ 150 arena_t * 151 choose_arena_hard(void) 152 { 153 arena_t *ret; 154 155 if (narenas > 1) { 156 unsigned i, choose, first_null; 157 158 choose = 0; 159 first_null = narenas; 160 malloc_mutex_lock(&arenas_lock); 161 assert(arenas[0] != NULL); 162 for (i = 1; i < narenas; i++) { 163 if (arenas[i] != NULL) { 164 /* 165 * Choose the first arena that has the lowest 166 * number of threads assigned to it. 167 */ 168 if (arenas[i]->nthreads < 169 arenas[choose]->nthreads) 170 choose = i; 171 } else if (first_null == narenas) { 172 /* 173 * Record the index of the first uninitialized 174 * arena, in case all extant arenas are in use. 175 * 176 * NB: It is possible for there to be 177 * discontinuities in terms of initialized 178 * versus uninitialized arenas, due to the 179 * "thread.arena" mallctl. 180 */ 181 first_null = i; 182 } 183 } 184 185 if (arenas[choose] == 0 || first_null == narenas) { 186 /* 187 * Use an unloaded arena, or the least loaded arena if 188 * all arenas are already initialized. 189 */ 190 ret = arenas[choose]; 191 } else { 192 /* Initialize a new arena. */ 193 ret = arenas_extend(first_null); 194 } 195 ret->nthreads++; 196 malloc_mutex_unlock(&arenas_lock); 197 } else { 198 ret = arenas[0]; 199 malloc_mutex_lock(&arenas_lock); 200 ret->nthreads++; 201 malloc_mutex_unlock(&arenas_lock); 202 } 203 204 ARENA_SET(ret); 205 206 return (ret); 207 } 208 209 /* 210 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so 211 * provide a wrapper. 212 */ 213 int 214 buferror(int errnum, char *buf, size_t buflen) 215 { 216 #ifdef _GNU_SOURCE 217 char *b = strerror_r(errno, buf, buflen); 218 if (b != buf) { 219 strncpy(buf, b, buflen); 220 buf[buflen-1] = '\0'; 221 } 222 return (0); 223 #else 224 return (strerror_r(errno, buf, buflen)); 225 #endif 226 } 227 228 static void 229 stats_print_atexit(void) 230 { 231 232 #if (defined(JEMALLOC_TCACHE) && defined(JEMALLOC_STATS)) 233 unsigned i; 234 235 /* 236 * Merge stats from extant threads. This is racy, since individual 237 * threads do not lock when recording tcache stats events. As a 238 * consequence, the final stats may be slightly out of date by the time 239 * they are reported, if other threads continue to allocate. 240 */ 241 for (i = 0; i < narenas; i++) { 242 arena_t *arena = arenas[i]; 243 if (arena != NULL) { 244 tcache_t *tcache; 245 246 /* 247 * tcache_stats_merge() locks bins, so if any code is 248 * introduced that acquires both arena and bin locks in 249 * the opposite order, deadlocks may result. 250 */ 251 malloc_mutex_lock(&arena->lock); 252 ql_foreach(tcache, &arena->tcache_ql, link) { 253 tcache_stats_merge(tcache, arena); 254 } 255 malloc_mutex_unlock(&arena->lock); 256 } 257 } 258 #endif 259 JEMALLOC_P(malloc_stats_print)(NULL, NULL, NULL); 260 } 261 262 #if (defined(JEMALLOC_STATS) && defined(NO_TLS)) 263 thread_allocated_t * 264 thread_allocated_get_hard(void) 265 { 266 thread_allocated_t *thread_allocated = (thread_allocated_t *) 267 imalloc(sizeof(thread_allocated_t)); 268 if (thread_allocated == NULL) { 269 static thread_allocated_t static_thread_allocated = {0, 0}; 270 malloc_write("<jemalloc>: Error allocating TSD;" 271 " mallctl(\"thread.{de,}allocated[p]\", ...)" 272 " will be inaccurate\n"); 273 if (opt_abort) 274 abort(); 275 return (&static_thread_allocated); 276 } 277 pthread_setspecific(thread_allocated_tsd, thread_allocated); 278 thread_allocated->allocated = 0; 279 thread_allocated->deallocated = 0; 280 return (thread_allocated); 281 } 282 #endif 283 284 /* 285 * End miscellaneous support functions. 286 */ 287 /******************************************************************************/ 288 /* 289 * Begin initialization functions. 290 */ 291 292 static unsigned 293 malloc_ncpus(void) 294 { 295 unsigned ret; 296 long result; 297 298 result = sysconf(_SC_NPROCESSORS_ONLN); 299 if (result == -1) { 300 /* Error. */ 301 ret = 1; 302 } 303 ret = (unsigned)result; 304 305 return (ret); 306 } 307 308 static void 309 arenas_cleanup(void *arg) 310 { 311 arena_t *arena = (arena_t *)arg; 312 313 malloc_mutex_lock(&arenas_lock); 314 arena->nthreads--; 315 malloc_mutex_unlock(&arenas_lock); 316 } 317 318 #if (defined(JEMALLOC_STATS) && defined(NO_TLS)) 319 static void 320 thread_allocated_cleanup(void *arg) 321 { 322 uint64_t *allocated = (uint64_t *)arg; 323 324 if (allocated != NULL) 325 idalloc(allocated); 326 } 327 #endif 328 329 /* 330 * FreeBSD's pthreads implementation calls malloc(3), so the malloc 331 * implementation has to take pains to avoid infinite recursion during 332 * initialization. 333 */ 334 static inline bool 335 malloc_init(void) 336 { 337 338 if (malloc_initialized == false) 339 return (malloc_init_hard()); 340 341 return (false); 342 } 343 344 static bool 345 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 346 char const **v_p, size_t *vlen_p) 347 { 348 bool accept; 349 const char *opts = *opts_p; 350 351 *k_p = opts; 352 353 for (accept = false; accept == false;) { 354 switch (*opts) { 355 case 'A': case 'B': case 'C': case 'D': case 'E': 356 case 'F': case 'G': case 'H': case 'I': case 'J': 357 case 'K': case 'L': case 'M': case 'N': case 'O': 358 case 'P': case 'Q': case 'R': case 'S': case 'T': 359 case 'U': case 'V': case 'W': case 'X': case 'Y': 360 case 'Z': 361 case 'a': case 'b': case 'c': case 'd': case 'e': 362 case 'f': case 'g': case 'h': case 'i': case 'j': 363 case 'k': case 'l': case 'm': case 'n': case 'o': 364 case 'p': case 'q': case 'r': case 's': case 't': 365 case 'u': case 'v': case 'w': case 'x': case 'y': 366 case 'z': 367 case '0': case '1': case '2': case '3': case '4': 368 case '5': case '6': case '7': case '8': case '9': 369 case '_': 370 opts++; 371 break; 372 case ':': 373 opts++; 374 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 375 *v_p = opts; 376 accept = true; 377 break; 378 case '\0': 379 if (opts != *opts_p) { 380 malloc_write("<jemalloc>: Conf string " 381 "ends with key\n"); 382 } 383 return (true); 384 default: 385 malloc_write("<jemalloc>: Malformed conf " 386 "string\n"); 387 return (true); 388 } 389 } 390 391 for (accept = false; accept == false;) { 392 switch (*opts) { 393 case ',': 394 opts++; 395 /* 396 * Look ahead one character here, because the 397 * next time this function is called, it will 398 * assume that end of input has been cleanly 399 * reached if no input remains, but we have 400 * optimistically already consumed the comma if 401 * one exists. 402 */ 403 if (*opts == '\0') { 404 malloc_write("<jemalloc>: Conf string " 405 "ends with comma\n"); 406 } 407 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 408 accept = true; 409 break; 410 case '\0': 411 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 412 accept = true; 413 break; 414 default: 415 opts++; 416 break; 417 } 418 } 419 420 *opts_p = opts; 421 return (false); 422 } 423 424 static void 425 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 426 size_t vlen) 427 { 428 char buf[PATH_MAX + 1]; 429 430 malloc_write("<jemalloc>: "); 431 malloc_write(msg); 432 malloc_write(": "); 433 memcpy(buf, k, klen); 434 memcpy(&buf[klen], ":", 1); 435 memcpy(&buf[klen+1], v, vlen); 436 buf[klen+1+vlen] = '\0'; 437 malloc_write(buf); 438 malloc_write("\n"); 439 } 440 441 static void 442 malloc_conf_init(void) 443 { 444 unsigned i; 445 char buf[PATH_MAX + 1]; 446 const char *opts, *k, *v; 447 size_t klen, vlen; 448 449 for (i = 0; i < 3; i++) { 450 /* Get runtime configuration. */ 451 switch (i) { 452 case 0: 453 if (JEMALLOC_P(malloc_conf) != NULL) { 454 /* 455 * Use options that were compiled into the 456 * program. 457 */ 458 opts = JEMALLOC_P(malloc_conf); 459 } else { 460 /* No configuration specified. */ 461 buf[0] = '\0'; 462 opts = buf; 463 } 464 break; 465 case 1: { 466 int linklen; 467 const char *linkname = 468 #ifdef JEMALLOC_PREFIX 469 "/etc/"JEMALLOC_PREFIX"malloc.conf" 470 #else 471 "/etc/malloc.conf" 472 #endif 473 ; 474 475 if ((linklen = readlink(linkname, buf, 476 sizeof(buf) - 1)) != -1) { 477 /* 478 * Use the contents of the "/etc/malloc.conf" 479 * symbolic link's name. 480 */ 481 buf[linklen] = '\0'; 482 opts = buf; 483 } else { 484 /* No configuration specified. */ 485 buf[0] = '\0'; 486 opts = buf; 487 } 488 break; 489 } 490 case 2: { 491 const char *envname = 492 #ifdef JEMALLOC_PREFIX 493 JEMALLOC_CPREFIX"MALLOC_CONF" 494 #else 495 "MALLOC_CONF" 496 #endif 497 ; 498 499 if ((opts = getenv(envname)) != NULL) { 500 /* 501 * Do nothing; opts is already initialized to 502 * the value of the MALLOC_CONF environment 503 * variable. 504 */ 505 } else { 506 /* No configuration specified. */ 507 buf[0] = '\0'; 508 opts = buf; 509 } 510 break; 511 } 512 default: 513 /* NOTREACHED */ 514 assert(false); 515 buf[0] = '\0'; 516 opts = buf; 517 } 518 519 while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, 520 &vlen) == false) { 521 #define CONF_HANDLE_BOOL(n) \ 522 if (sizeof(#n)-1 == klen && strncmp(#n, k, \ 523 klen) == 0) { \ 524 if (strncmp("true", v, vlen) == 0 && \ 525 vlen == sizeof("true")-1) \ 526 opt_##n = true; \ 527 else if (strncmp("false", v, vlen) == \ 528 0 && vlen == sizeof("false")-1) \ 529 opt_##n = false; \ 530 else { \ 531 malloc_conf_error( \ 532 "Invalid conf value", \ 533 k, klen, v, vlen); \ 534 } \ 535 continue; \ 536 } 537 #define CONF_HANDLE_SIZE_T(n, min, max) \ 538 if (sizeof(#n)-1 == klen && strncmp(#n, k, \ 539 klen) == 0) { \ 540 unsigned long ul; \ 541 char *end; \ 542 \ 543 errno = 0; \ 544 ul = strtoul(v, &end, 0); \ 545 if (errno != 0 || (uintptr_t)end - \ 546 (uintptr_t)v != vlen) { \ 547 malloc_conf_error( \ 548 "Invalid conf value", \ 549 k, klen, v, vlen); \ 550 } else if (ul < min || ul > max) { \ 551 malloc_conf_error( \ 552 "Out-of-range conf value", \ 553 k, klen, v, vlen); \ 554 } else \ 555 opt_##n = ul; \ 556 continue; \ 557 } 558 #define CONF_HANDLE_SSIZE_T(n, min, max) \ 559 if (sizeof(#n)-1 == klen && strncmp(#n, k, \ 560 klen) == 0) { \ 561 long l; \ 562 char *end; \ 563 \ 564 errno = 0; \ 565 l = strtol(v, &end, 0); \ 566 if (errno != 0 || (uintptr_t)end - \ 567 (uintptr_t)v != vlen) { \ 568 malloc_conf_error( \ 569 "Invalid conf value", \ 570 k, klen, v, vlen); \ 571 } else if (l < (ssize_t)min || l > \ 572 (ssize_t)max) { \ 573 malloc_conf_error( \ 574 "Out-of-range conf value", \ 575 k, klen, v, vlen); \ 576 } else \ 577 opt_##n = l; \ 578 continue; \ 579 } 580 #define CONF_HANDLE_CHAR_P(n, d) \ 581 if (sizeof(#n)-1 == klen && strncmp(#n, k, \ 582 klen) == 0) { \ 583 size_t cpylen = (vlen <= \ 584 sizeof(opt_##n)-1) ? vlen : \ 585 sizeof(opt_##n)-1; \ 586 strncpy(opt_##n, v, cpylen); \ 587 opt_##n[cpylen] = '\0'; \ 588 continue; \ 589 } 590 591 CONF_HANDLE_BOOL(abort) 592 CONF_HANDLE_SIZE_T(lg_qspace_max, LG_QUANTUM, 593 PAGE_SHIFT-1) 594 CONF_HANDLE_SIZE_T(lg_cspace_max, LG_QUANTUM, 595 PAGE_SHIFT-1) 596 /* 597 * Chunks always require at least one * header page, 598 * plus one data page. 599 */ 600 CONF_HANDLE_SIZE_T(lg_chunk, PAGE_SHIFT+1, 601 (sizeof(size_t) << 3) - 1) 602 CONF_HANDLE_SIZE_T(narenas, 1, SIZE_T_MAX) 603 CONF_HANDLE_SSIZE_T(lg_dirty_mult, -1, 604 (sizeof(size_t) << 3) - 1) 605 CONF_HANDLE_BOOL(stats_print) 606 #ifdef JEMALLOC_FILL 607 CONF_HANDLE_BOOL(junk) 608 CONF_HANDLE_BOOL(zero) 609 #endif 610 #ifdef JEMALLOC_SYSV 611 CONF_HANDLE_BOOL(sysv) 612 #endif 613 #ifdef JEMALLOC_XMALLOC 614 CONF_HANDLE_BOOL(xmalloc) 615 #endif 616 #ifdef JEMALLOC_TCACHE 617 CONF_HANDLE_BOOL(tcache) 618 CONF_HANDLE_SSIZE_T(lg_tcache_gc_sweep, -1, 619 (sizeof(size_t) << 3) - 1) 620 CONF_HANDLE_SSIZE_T(lg_tcache_max, -1, 621 (sizeof(size_t) << 3) - 1) 622 #endif 623 #ifdef JEMALLOC_PROF 624 CONF_HANDLE_BOOL(prof) 625 CONF_HANDLE_CHAR_P(prof_prefix, "jeprof") 626 CONF_HANDLE_SIZE_T(lg_prof_bt_max, 0, LG_PROF_BT_MAX) 627 CONF_HANDLE_BOOL(prof_active) 628 CONF_HANDLE_SSIZE_T(lg_prof_sample, 0, 629 (sizeof(uint64_t) << 3) - 1) 630 CONF_HANDLE_BOOL(prof_accum) 631 CONF_HANDLE_SSIZE_T(lg_prof_tcmax, -1, 632 (sizeof(size_t) << 3) - 1) 633 CONF_HANDLE_SSIZE_T(lg_prof_interval, -1, 634 (sizeof(uint64_t) << 3) - 1) 635 CONF_HANDLE_BOOL(prof_gdump) 636 CONF_HANDLE_BOOL(prof_leak) 637 #endif 638 #ifdef JEMALLOC_SWAP 639 CONF_HANDLE_BOOL(overcommit) 640 #endif 641 malloc_conf_error("Invalid conf pair", k, klen, v, 642 vlen); 643 #undef CONF_HANDLE_BOOL 644 #undef CONF_HANDLE_SIZE_T 645 #undef CONF_HANDLE_SSIZE_T 646 #undef CONF_HANDLE_CHAR_P 647 } 648 649 /* Validate configuration of options that are inter-related. */ 650 if (opt_lg_qspace_max+1 >= opt_lg_cspace_max) { 651 malloc_write("<jemalloc>: Invalid lg_[qc]space_max " 652 "relationship; restoring defaults\n"); 653 opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; 654 opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; 655 } 656 } 657 } 658 659 static bool 660 malloc_init_hard(void) 661 { 662 arena_t *init_arenas[1]; 663 664 malloc_mutex_lock(&init_lock); 665 if (malloc_initialized || malloc_initializer == pthread_self()) { 666 /* 667 * Another thread initialized the allocator before this one 668 * acquired init_lock, or this thread is the initializing 669 * thread, and it is recursively allocating. 670 */ 671 malloc_mutex_unlock(&init_lock); 672 return (false); 673 } 674 if (malloc_initializer != (unsigned long)0) { 675 /* Busy-wait until the initializing thread completes. */ 676 do { 677 malloc_mutex_unlock(&init_lock); 678 CPU_SPINWAIT; 679 malloc_mutex_lock(&init_lock); 680 } while (malloc_initialized == false); 681 malloc_mutex_unlock(&init_lock); 682 return (false); 683 } 684 685 #ifdef DYNAMIC_PAGE_SHIFT 686 /* Get page size. */ 687 { 688 long result; 689 690 result = sysconf(_SC_PAGESIZE); 691 assert(result != -1); 692 pagesize = (size_t)result; 693 694 /* 695 * We assume that pagesize is a power of 2 when calculating 696 * pagesize_mask and lg_pagesize. 697 */ 698 assert(((result - 1) & result) == 0); 699 pagesize_mask = result - 1; 700 lg_pagesize = ffs((int)result) - 1; 701 } 702 #endif 703 704 #ifdef JEMALLOC_PROF 705 prof_boot0(); 706 #endif 707 708 malloc_conf_init(); 709 710 /* Register fork handlers. */ 711 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork, 712 jemalloc_postfork) != 0) { 713 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 714 if (opt_abort) 715 abort(); 716 } 717 718 if (ctl_boot()) { 719 malloc_mutex_unlock(&init_lock); 720 return (true); 721 } 722 723 if (opt_stats_print) { 724 /* Print statistics at exit. */ 725 if (atexit(stats_print_atexit) != 0) { 726 malloc_write("<jemalloc>: Error in atexit()\n"); 727 if (opt_abort) 728 abort(); 729 } 730 } 731 732 if (chunk_boot()) { 733 malloc_mutex_unlock(&init_lock); 734 return (true); 735 } 736 737 if (base_boot()) { 738 malloc_mutex_unlock(&init_lock); 739 return (true); 740 } 741 742 #ifdef JEMALLOC_PROF 743 prof_boot1(); 744 #endif 745 746 if (arena_boot()) { 747 malloc_mutex_unlock(&init_lock); 748 return (true); 749 } 750 751 #ifdef JEMALLOC_TCACHE 752 if (tcache_boot()) { 753 malloc_mutex_unlock(&init_lock); 754 return (true); 755 } 756 #endif 757 758 if (huge_boot()) { 759 malloc_mutex_unlock(&init_lock); 760 return (true); 761 } 762 763 #if (defined(JEMALLOC_STATS) && defined(NO_TLS)) 764 /* Initialize allocation counters before any allocations can occur. */ 765 if (pthread_key_create(&thread_allocated_tsd, thread_allocated_cleanup) 766 != 0) { 767 malloc_mutex_unlock(&init_lock); 768 return (true); 769 } 770 #endif 771 772 if (malloc_mutex_init(&arenas_lock)) 773 return (true); 774 775 if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) { 776 malloc_mutex_unlock(&init_lock); 777 return (true); 778 } 779 780 /* 781 * Create enough scaffolding to allow recursive allocation in 782 * malloc_ncpus(). 783 */ 784 narenas = 1; 785 arenas = init_arenas; 786 memset(arenas, 0, sizeof(arena_t *) * narenas); 787 788 /* 789 * Initialize one arena here. The rest are lazily created in 790 * choose_arena_hard(). 791 */ 792 arenas_extend(0); 793 if (arenas[0] == NULL) { 794 malloc_mutex_unlock(&init_lock); 795 return (true); 796 } 797 798 /* 799 * Assign the initial arena to the initial thread, in order to avoid 800 * spurious creation of an extra arena if the application switches to 801 * threaded mode. 802 */ 803 ARENA_SET(arenas[0]); 804 arenas[0]->nthreads++; 805 806 #ifdef JEMALLOC_PROF 807 if (prof_boot2()) { 808 malloc_mutex_unlock(&init_lock); 809 return (true); 810 } 811 #endif 812 813 /* Get number of CPUs. */ 814 malloc_initializer = pthread_self(); 815 malloc_mutex_unlock(&init_lock); 816 ncpus = malloc_ncpus(); 817 malloc_mutex_lock(&init_lock); 818 819 if (opt_narenas == 0) { 820 /* 821 * For SMP systems, create more than one arena per CPU by 822 * default. 823 */ 824 if (ncpus > 1) 825 opt_narenas = ncpus << 2; 826 else 827 opt_narenas = 1; 828 } 829 narenas = opt_narenas; 830 /* 831 * Make sure that the arenas array can be allocated. In practice, this 832 * limit is enough to allow the allocator to function, but the ctl 833 * machinery will fail to allocate memory at far lower limits. 834 */ 835 if (narenas > chunksize / sizeof(arena_t *)) { 836 char buf[UMAX2S_BUFSIZE]; 837 838 narenas = chunksize / sizeof(arena_t *); 839 malloc_write("<jemalloc>: Reducing narenas to limit ("); 840 malloc_write(u2s(narenas, 10, buf)); 841 malloc_write(")\n"); 842 } 843 844 /* Allocate and initialize arenas. */ 845 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas); 846 if (arenas == NULL) { 847 malloc_mutex_unlock(&init_lock); 848 return (true); 849 } 850 /* 851 * Zero the array. In practice, this should always be pre-zeroed, 852 * since it was just mmap()ed, but let's be sure. 853 */ 854 memset(arenas, 0, sizeof(arena_t *) * narenas); 855 /* Copy the pointer to the one arena that was already initialized. */ 856 arenas[0] = init_arenas[0]; 857 858 #ifdef JEMALLOC_ZONE 859 /* Register the custom zone. */ 860 malloc_zone_register(create_zone()); 861 862 /* 863 * Convert the default szone to an "overlay zone" that is capable of 864 * deallocating szone-allocated objects, but allocating new objects 865 * from jemalloc. 866 */ 867 szone2ozone(malloc_default_zone()); 868 #endif 869 870 malloc_initialized = true; 871 malloc_mutex_unlock(&init_lock); 872 return (false); 873 } 874 875 #ifdef JEMALLOC_ZONE 876 JEMALLOC_ATTR(constructor) 877 void 878 jemalloc_darwin_init(void) 879 { 880 881 if (malloc_init_hard()) 882 abort(); 883 } 884 #endif 885 886 /* 887 * End initialization functions. 888 */ 889 /******************************************************************************/ 890 /* 891 * Begin malloc(3)-compatible functions. 892 */ 893 894 JEMALLOC_ATTR(malloc) 895 JEMALLOC_ATTR(visibility("default")) 896 void * 897 JEMALLOC_P(malloc)(size_t size) 898 { 899 void *ret; 900 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 901 size_t usize 902 # ifdef JEMALLOC_CC_SILENCE 903 = 0 904 # endif 905 ; 906 #endif 907 #ifdef JEMALLOC_PROF 908 prof_thr_cnt_t *cnt 909 # ifdef JEMALLOC_CC_SILENCE 910 = NULL 911 # endif 912 ; 913 #endif 914 915 if (malloc_init()) { 916 ret = NULL; 917 goto OOM; 918 } 919 920 if (size == 0) { 921 #ifdef JEMALLOC_SYSV 922 if (opt_sysv == false) 923 #endif 924 size = 1; 925 #ifdef JEMALLOC_SYSV 926 else { 927 # ifdef JEMALLOC_XMALLOC 928 if (opt_xmalloc) { 929 malloc_write("<jemalloc>: Error in malloc(): " 930 "invalid size 0\n"); 931 abort(); 932 } 933 # endif 934 ret = NULL; 935 goto RETURN; 936 } 937 #endif 938 } 939 940 #ifdef JEMALLOC_PROF 941 if (opt_prof) { 942 usize = s2u(size); 943 PROF_ALLOC_PREP(1, usize, cnt); 944 if (cnt == NULL) { 945 ret = NULL; 946 goto OOM; 947 } 948 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= 949 small_maxclass) { 950 ret = imalloc(small_maxclass+1); 951 if (ret != NULL) 952 arena_prof_promoted(ret, usize); 953 } else 954 ret = imalloc(size); 955 } else 956 #endif 957 { 958 #ifdef JEMALLOC_STATS 959 usize = s2u(size); 960 #endif 961 ret = imalloc(size); 962 } 963 964 OOM: 965 if (ret == NULL) { 966 #ifdef JEMALLOC_XMALLOC 967 if (opt_xmalloc) { 968 malloc_write("<jemalloc>: Error in malloc(): " 969 "out of memory\n"); 970 abort(); 971 } 972 #endif 973 errno = ENOMEM; 974 } 975 976 #ifdef JEMALLOC_SYSV 977 RETURN: 978 #endif 979 #ifdef JEMALLOC_PROF 980 if (opt_prof && ret != NULL) 981 prof_malloc(ret, usize, cnt); 982 #endif 983 #ifdef JEMALLOC_STATS 984 if (ret != NULL) { 985 assert(usize == isalloc(ret)); 986 ALLOCATED_ADD(usize, 0); 987 } 988 #endif 989 return (ret); 990 } 991 992 JEMALLOC_ATTR(nonnull(1)) 993 #ifdef JEMALLOC_PROF 994 /* 995 * Avoid any uncertainty as to how many backtrace frames to ignore in 996 * PROF_ALLOC_PREP(). 997 */ 998 JEMALLOC_ATTR(noinline) 999 #endif 1000 static int 1001 imemalign(void **memptr, size_t alignment, size_t size) 1002 { 1003 int ret; 1004 size_t usize 1005 #ifdef JEMALLOC_CC_SILENCE 1006 = 0 1007 #endif 1008 ; 1009 void *result; 1010 #ifdef JEMALLOC_PROF 1011 prof_thr_cnt_t *cnt 1012 # ifdef JEMALLOC_CC_SILENCE 1013 = NULL 1014 # endif 1015 ; 1016 #endif 1017 1018 if (malloc_init()) 1019 result = NULL; 1020 else { 1021 if (size == 0) { 1022 #ifdef JEMALLOC_SYSV 1023 if (opt_sysv == false) 1024 #endif 1025 size = 1; 1026 #ifdef JEMALLOC_SYSV 1027 else { 1028 # ifdef JEMALLOC_XMALLOC 1029 if (opt_xmalloc) { 1030 malloc_write("<jemalloc>: Error in " 1031 "posix_memalign(): invalid size " 1032 "0\n"); 1033 abort(); 1034 } 1035 # endif 1036 result = NULL; 1037 *memptr = NULL; 1038 ret = 0; 1039 goto RETURN; 1040 } 1041 #endif 1042 } 1043 1044 /* Make sure that alignment is a large enough power of 2. */ 1045 if (((alignment - 1) & alignment) != 0 1046 || alignment < sizeof(void *)) { 1047 #ifdef JEMALLOC_XMALLOC 1048 if (opt_xmalloc) { 1049 malloc_write("<jemalloc>: Error in " 1050 "posix_memalign(): invalid alignment\n"); 1051 abort(); 1052 } 1053 #endif 1054 result = NULL; 1055 ret = EINVAL; 1056 goto RETURN; 1057 } 1058 1059 usize = sa2u(size, alignment, NULL); 1060 if (usize == 0) { 1061 result = NULL; 1062 ret = ENOMEM; 1063 goto RETURN; 1064 } 1065 1066 #ifdef JEMALLOC_PROF 1067 if (opt_prof) { 1068 PROF_ALLOC_PREP(2, usize, cnt); 1069 if (cnt == NULL) { 1070 result = NULL; 1071 ret = EINVAL; 1072 } else { 1073 if (prof_promote && (uintptr_t)cnt != 1074 (uintptr_t)1U && usize <= small_maxclass) { 1075 assert(sa2u(small_maxclass+1, 1076 alignment, NULL) != 0); 1077 result = ipalloc(sa2u(small_maxclass+1, 1078 alignment, NULL), alignment, false); 1079 if (result != NULL) { 1080 arena_prof_promoted(result, 1081 usize); 1082 } 1083 } else { 1084 result = ipalloc(usize, alignment, 1085 false); 1086 } 1087 } 1088 } else 1089 #endif 1090 result = ipalloc(usize, alignment, false); 1091 } 1092 1093 if (result == NULL) { 1094 #ifdef JEMALLOC_XMALLOC 1095 if (opt_xmalloc) { 1096 malloc_write("<jemalloc>: Error in posix_memalign(): " 1097 "out of memory\n"); 1098 abort(); 1099 } 1100 #endif 1101 ret = ENOMEM; 1102 goto RETURN; 1103 } 1104 1105 *memptr = result; 1106 ret = 0; 1107 1108 RETURN: 1109 #ifdef JEMALLOC_STATS 1110 if (result != NULL) { 1111 assert(usize == isalloc(result)); 1112 ALLOCATED_ADD(usize, 0); 1113 } 1114 #endif 1115 #ifdef JEMALLOC_PROF 1116 if (opt_prof && result != NULL) 1117 prof_malloc(result, usize, cnt); 1118 #endif 1119 return (ret); 1120 } 1121 1122 JEMALLOC_ATTR(nonnull(1)) 1123 JEMALLOC_ATTR(visibility("default")) 1124 int 1125 JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size) 1126 { 1127 1128 return imemalign(memptr, alignment, size); 1129 } 1130 1131 JEMALLOC_ATTR(malloc) 1132 JEMALLOC_ATTR(visibility("default")) 1133 void * 1134 JEMALLOC_P(calloc)(size_t num, size_t size) 1135 { 1136 void *ret; 1137 size_t num_size; 1138 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1139 size_t usize 1140 # ifdef JEMALLOC_CC_SILENCE 1141 = 0 1142 # endif 1143 ; 1144 #endif 1145 #ifdef JEMALLOC_PROF 1146 prof_thr_cnt_t *cnt 1147 # ifdef JEMALLOC_CC_SILENCE 1148 = NULL 1149 # endif 1150 ; 1151 #endif 1152 1153 if (malloc_init()) { 1154 num_size = 0; 1155 ret = NULL; 1156 goto RETURN; 1157 } 1158 1159 num_size = num * size; 1160 if (num_size == 0) { 1161 #ifdef JEMALLOC_SYSV 1162 if ((opt_sysv == false) && ((num == 0) || (size == 0))) 1163 #endif 1164 num_size = 1; 1165 #ifdef JEMALLOC_SYSV 1166 else { 1167 ret = NULL; 1168 goto RETURN; 1169 } 1170 #endif 1171 /* 1172 * Try to avoid division here. We know that it isn't possible to 1173 * overflow during multiplication if neither operand uses any of the 1174 * most significant half of the bits in a size_t. 1175 */ 1176 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) 1177 && (num_size / size != num)) { 1178 /* size_t overflow. */ 1179 ret = NULL; 1180 goto RETURN; 1181 } 1182 1183 #ifdef JEMALLOC_PROF 1184 if (opt_prof) { 1185 usize = s2u(num_size); 1186 PROF_ALLOC_PREP(1, usize, cnt); 1187 if (cnt == NULL) { 1188 ret = NULL; 1189 goto RETURN; 1190 } 1191 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize 1192 <= small_maxclass) { 1193 ret = icalloc(small_maxclass+1); 1194 if (ret != NULL) 1195 arena_prof_promoted(ret, usize); 1196 } else 1197 ret = icalloc(num_size); 1198 } else 1199 #endif 1200 { 1201 #ifdef JEMALLOC_STATS 1202 usize = s2u(num_size); 1203 #endif 1204 ret = icalloc(num_size); 1205 } 1206 1207 RETURN: 1208 if (ret == NULL) { 1209 #ifdef JEMALLOC_XMALLOC 1210 if (opt_xmalloc) { 1211 malloc_write("<jemalloc>: Error in calloc(): out of " 1212 "memory\n"); 1213 abort(); 1214 } 1215 #endif 1216 errno = ENOMEM; 1217 } 1218 1219 #ifdef JEMALLOC_PROF 1220 if (opt_prof && ret != NULL) 1221 prof_malloc(ret, usize, cnt); 1222 #endif 1223 #ifdef JEMALLOC_STATS 1224 if (ret != NULL) { 1225 assert(usize == isalloc(ret)); 1226 ALLOCATED_ADD(usize, 0); 1227 } 1228 #endif 1229 return (ret); 1230 } 1231 1232 JEMALLOC_ATTR(visibility("default")) 1233 void * 1234 JEMALLOC_P(realloc)(void *ptr, size_t size) 1235 { 1236 void *ret; 1237 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1238 size_t usize 1239 # ifdef JEMALLOC_CC_SILENCE 1240 = 0 1241 # endif 1242 ; 1243 size_t old_size = 0; 1244 #endif 1245 #ifdef JEMALLOC_PROF 1246 prof_thr_cnt_t *cnt 1247 # ifdef JEMALLOC_CC_SILENCE 1248 = NULL 1249 # endif 1250 ; 1251 prof_ctx_t *old_ctx 1252 # ifdef JEMALLOC_CC_SILENCE 1253 = NULL 1254 # endif 1255 ; 1256 #endif 1257 1258 if (size == 0) { 1259 #ifdef JEMALLOC_SYSV 1260 if (opt_sysv == false) 1261 #endif 1262 size = 1; 1263 #ifdef JEMALLOC_SYSV 1264 else { 1265 if (ptr != NULL) { 1266 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1267 old_size = isalloc(ptr); 1268 #endif 1269 #ifdef JEMALLOC_PROF 1270 if (opt_prof) { 1271 old_ctx = prof_ctx_get(ptr); 1272 cnt = NULL; 1273 } 1274 #endif 1275 idalloc(ptr); 1276 } 1277 #ifdef JEMALLOC_PROF 1278 else if (opt_prof) { 1279 old_ctx = NULL; 1280 cnt = NULL; 1281 } 1282 #endif 1283 ret = NULL; 1284 goto RETURN; 1285 } 1286 #endif 1287 } 1288 1289 if (ptr != NULL) { 1290 assert(malloc_initialized || malloc_initializer == 1291 pthread_self()); 1292 1293 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1294 old_size = isalloc(ptr); 1295 #endif 1296 #ifdef JEMALLOC_PROF 1297 if (opt_prof) { 1298 usize = s2u(size); 1299 old_ctx = prof_ctx_get(ptr); 1300 PROF_ALLOC_PREP(1, usize, cnt); 1301 if (cnt == NULL) { 1302 old_ctx = NULL; 1303 ret = NULL; 1304 goto OOM; 1305 } 1306 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && 1307 usize <= small_maxclass) { 1308 ret = iralloc(ptr, small_maxclass+1, 0, 0, 1309 false, false); 1310 if (ret != NULL) 1311 arena_prof_promoted(ret, usize); 1312 else 1313 old_ctx = NULL; 1314 } else { 1315 ret = iralloc(ptr, size, 0, 0, false, false); 1316 if (ret == NULL) 1317 old_ctx = NULL; 1318 } 1319 } else 1320 #endif 1321 { 1322 #ifdef JEMALLOC_STATS 1323 usize = s2u(size); 1324 #endif 1325 ret = iralloc(ptr, size, 0, 0, false, false); 1326 } 1327 1328 #ifdef JEMALLOC_PROF 1329 OOM: 1330 #endif 1331 if (ret == NULL) { 1332 #ifdef JEMALLOC_XMALLOC 1333 if (opt_xmalloc) { 1334 malloc_write("<jemalloc>: Error in realloc(): " 1335 "out of memory\n"); 1336 abort(); 1337 } 1338 #endif 1339 errno = ENOMEM; 1340 } 1341 } else { 1342 #ifdef JEMALLOC_PROF 1343 if (opt_prof) 1344 old_ctx = NULL; 1345 #endif 1346 if (malloc_init()) { 1347 #ifdef JEMALLOC_PROF 1348 if (opt_prof) 1349 cnt = NULL; 1350 #endif 1351 ret = NULL; 1352 } else { 1353 #ifdef JEMALLOC_PROF 1354 if (opt_prof) { 1355 usize = s2u(size); 1356 PROF_ALLOC_PREP(1, usize, cnt); 1357 if (cnt == NULL) 1358 ret = NULL; 1359 else { 1360 if (prof_promote && (uintptr_t)cnt != 1361 (uintptr_t)1U && usize <= 1362 small_maxclass) { 1363 ret = imalloc(small_maxclass+1); 1364 if (ret != NULL) { 1365 arena_prof_promoted(ret, 1366 usize); 1367 } 1368 } else 1369 ret = imalloc(size); 1370 } 1371 } else 1372 #endif 1373 { 1374 #ifdef JEMALLOC_STATS 1375 usize = s2u(size); 1376 #endif 1377 ret = imalloc(size); 1378 } 1379 } 1380 1381 if (ret == NULL) { 1382 #ifdef JEMALLOC_XMALLOC 1383 if (opt_xmalloc) { 1384 malloc_write("<jemalloc>: Error in realloc(): " 1385 "out of memory\n"); 1386 abort(); 1387 } 1388 #endif 1389 errno = ENOMEM; 1390 } 1391 } 1392 1393 #ifdef JEMALLOC_SYSV 1394 RETURN: 1395 #endif 1396 #ifdef JEMALLOC_PROF 1397 if (opt_prof) 1398 prof_realloc(ret, usize, cnt, old_size, old_ctx); 1399 #endif 1400 #ifdef JEMALLOC_STATS 1401 if (ret != NULL) { 1402 assert(usize == isalloc(ret)); 1403 ALLOCATED_ADD(usize, old_size); 1404 } 1405 #endif 1406 return (ret); 1407 } 1408 1409 JEMALLOC_ATTR(visibility("default")) 1410 void 1411 JEMALLOC_P(free)(void *ptr) 1412 { 1413 1414 if (ptr != NULL) { 1415 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1416 size_t usize; 1417 #endif 1418 1419 assert(malloc_initialized || malloc_initializer == 1420 pthread_self()); 1421 1422 #ifdef JEMALLOC_STATS 1423 usize = isalloc(ptr); 1424 #endif 1425 #ifdef JEMALLOC_PROF 1426 if (opt_prof) { 1427 # ifndef JEMALLOC_STATS 1428 usize = isalloc(ptr); 1429 # endif 1430 prof_free(ptr, usize); 1431 } 1432 #endif 1433 #ifdef JEMALLOC_STATS 1434 ALLOCATED_ADD(0, usize); 1435 #endif 1436 idalloc(ptr); 1437 } 1438 } 1439 1440 /* 1441 * End malloc(3)-compatible functions. 1442 */ 1443 /******************************************************************************/ 1444 /* 1445 * Begin non-standard override functions. 1446 * 1447 * These overrides are omitted if the JEMALLOC_PREFIX is defined, since the 1448 * entire point is to avoid accidental mixed allocator usage. 1449 */ 1450 #ifndef JEMALLOC_PREFIX 1451 1452 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 1453 JEMALLOC_ATTR(malloc) 1454 JEMALLOC_ATTR(visibility("default")) 1455 void * 1456 JEMALLOC_P(memalign)(size_t alignment, size_t size) 1457 { 1458 void *ret; 1459 #ifdef JEMALLOC_CC_SILENCE 1460 int result = 1461 #endif 1462 imemalign(&ret, alignment, size); 1463 #ifdef JEMALLOC_CC_SILENCE 1464 if (result != 0) 1465 return (NULL); 1466 #endif 1467 return (ret); 1468 } 1469 #endif 1470 1471 #ifdef JEMALLOC_OVERRIDE_VALLOC 1472 JEMALLOC_ATTR(malloc) 1473 JEMALLOC_ATTR(visibility("default")) 1474 void * 1475 JEMALLOC_P(valloc)(size_t size) 1476 { 1477 void *ret; 1478 #ifdef JEMALLOC_CC_SILENCE 1479 int result = 1480 #endif 1481 imemalign(&ret, PAGE_SIZE, size); 1482 #ifdef JEMALLOC_CC_SILENCE 1483 if (result != 0) 1484 return (NULL); 1485 #endif 1486 return (ret); 1487 } 1488 #endif 1489 1490 #endif /* JEMALLOC_PREFIX */ 1491 /* 1492 * End non-standard override functions. 1493 */ 1494 /******************************************************************************/ 1495 /* 1496 * Begin non-standard functions. 1497 */ 1498 1499 JEMALLOC_ATTR(visibility("default")) 1500 size_t 1501 JEMALLOC_P(malloc_usable_size)(const void *ptr) 1502 { 1503 size_t ret; 1504 1505 assert(malloc_initialized || malloc_initializer == pthread_self()); 1506 1507 #ifdef JEMALLOC_IVSALLOC 1508 ret = ivsalloc(ptr); 1509 #else 1510 assert(ptr != NULL); 1511 ret = isalloc(ptr); 1512 #endif 1513 1514 return (ret); 1515 } 1516 1517 JEMALLOC_ATTR(visibility("default")) 1518 void 1519 JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *), 1520 void *cbopaque, const char *opts) 1521 { 1522 1523 stats_print(write_cb, cbopaque, opts); 1524 } 1525 1526 JEMALLOC_ATTR(visibility("default")) 1527 int 1528 JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, void *newp, 1529 size_t newlen) 1530 { 1531 1532 if (malloc_init()) 1533 return (EAGAIN); 1534 1535 return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 1536 } 1537 1538 JEMALLOC_ATTR(visibility("default")) 1539 int 1540 JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, size_t *miblenp) 1541 { 1542 1543 if (malloc_init()) 1544 return (EAGAIN); 1545 1546 return (ctl_nametomib(name, mibp, miblenp)); 1547 } 1548 1549 JEMALLOC_ATTR(visibility("default")) 1550 int 1551 JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, 1552 size_t *oldlenp, void *newp, size_t newlen) 1553 { 1554 1555 if (malloc_init()) 1556 return (EAGAIN); 1557 1558 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 1559 } 1560 1561 JEMALLOC_INLINE void * 1562 iallocm(size_t usize, size_t alignment, bool zero) 1563 { 1564 1565 assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, alignment, 1566 NULL))); 1567 1568 if (alignment != 0) 1569 return (ipalloc(usize, alignment, zero)); 1570 else if (zero) 1571 return (icalloc(usize)); 1572 else 1573 return (imalloc(usize)); 1574 } 1575 1576 JEMALLOC_ATTR(nonnull(1)) 1577 JEMALLOC_ATTR(visibility("default")) 1578 int 1579 JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags) 1580 { 1581 void *p; 1582 size_t usize; 1583 size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1584 & (SIZE_T_MAX-1)); 1585 bool zero = flags & ALLOCM_ZERO; 1586 #ifdef JEMALLOC_PROF 1587 prof_thr_cnt_t *cnt; 1588 #endif 1589 1590 assert(ptr != NULL); 1591 assert(size != 0); 1592 1593 if (malloc_init()) 1594 goto OOM; 1595 1596 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment, NULL); 1597 if (usize == 0) 1598 goto OOM; 1599 1600 #ifdef JEMALLOC_PROF 1601 if (opt_prof) { 1602 PROF_ALLOC_PREP(1, usize, cnt); 1603 if (cnt == NULL) 1604 goto OOM; 1605 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= 1606 small_maxclass) { 1607 size_t usize_promoted = (alignment == 0) ? 1608 s2u(small_maxclass+1) : sa2u(small_maxclass+1, 1609 alignment, NULL); 1610 assert(usize_promoted != 0); 1611 p = iallocm(usize_promoted, alignment, zero); 1612 if (p == NULL) 1613 goto OOM; 1614 arena_prof_promoted(p, usize); 1615 } else { 1616 p = iallocm(usize, alignment, zero); 1617 if (p == NULL) 1618 goto OOM; 1619 } 1620 prof_malloc(p, usize, cnt); 1621 if (rsize != NULL) 1622 *rsize = usize; 1623 } else 1624 #endif 1625 { 1626 p = iallocm(usize, alignment, zero); 1627 if (p == NULL) 1628 goto OOM; 1629 #ifndef JEMALLOC_STATS 1630 if (rsize != NULL) 1631 #endif 1632 { 1633 #ifdef JEMALLOC_STATS 1634 if (rsize != NULL) 1635 #endif 1636 *rsize = usize; 1637 } 1638 } 1639 1640 *ptr = p; 1641 #ifdef JEMALLOC_STATS 1642 assert(usize == isalloc(p)); 1643 ALLOCATED_ADD(usize, 0); 1644 #endif 1645 return (ALLOCM_SUCCESS); 1646 OOM: 1647 #ifdef JEMALLOC_XMALLOC 1648 if (opt_xmalloc) { 1649 malloc_write("<jemalloc>: Error in allocm(): " 1650 "out of memory\n"); 1651 abort(); 1652 } 1653 #endif 1654 *ptr = NULL; 1655 return (ALLOCM_ERR_OOM); 1656 } 1657 1658 JEMALLOC_ATTR(nonnull(1)) 1659 JEMALLOC_ATTR(visibility("default")) 1660 int 1661 JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, size_t extra, 1662 int flags) 1663 { 1664 void *p, *q; 1665 size_t usize; 1666 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1667 size_t old_size; 1668 #endif 1669 size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1670 & (SIZE_T_MAX-1)); 1671 bool zero = flags & ALLOCM_ZERO; 1672 bool no_move = flags & ALLOCM_NO_MOVE; 1673 #ifdef JEMALLOC_PROF 1674 prof_thr_cnt_t *cnt; 1675 #endif 1676 1677 assert(ptr != NULL); 1678 assert(*ptr != NULL); 1679 assert(size != 0); 1680 assert(SIZE_T_MAX - size >= extra); 1681 assert(malloc_initialized || malloc_initializer == pthread_self()); 1682 1683 p = *ptr; 1684 #ifdef JEMALLOC_PROF 1685 if (opt_prof) { 1686 /* 1687 * usize isn't knowable before iralloc() returns when extra is 1688 * non-zero. Therefore, compute its maximum possible value and 1689 * use that in PROF_ALLOC_PREP() to decide whether to capture a 1690 * backtrace. prof_realloc() will use the actual usize to 1691 * decide whether to sample. 1692 */ 1693 size_t max_usize = (alignment == 0) ? s2u(size+extra) : 1694 sa2u(size+extra, alignment, NULL); 1695 prof_ctx_t *old_ctx = prof_ctx_get(p); 1696 old_size = isalloc(p); 1697 PROF_ALLOC_PREP(1, max_usize, cnt); 1698 if (cnt == NULL) 1699 goto OOM; 1700 /* 1701 * Use minimum usize to determine whether promotion may happen. 1702 */ 1703 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U 1704 && ((alignment == 0) ? s2u(size) : sa2u(size, 1705 alignment, NULL)) <= small_maxclass) { 1706 q = iralloc(p, small_maxclass+1, (small_maxclass+1 >= 1707 size+extra) ? 0 : size+extra - (small_maxclass+1), 1708 alignment, zero, no_move); 1709 if (q == NULL) 1710 goto ERR; 1711 if (max_usize < PAGE_SIZE) { 1712 usize = max_usize; 1713 arena_prof_promoted(q, usize); 1714 } else 1715 usize = isalloc(q); 1716 } else { 1717 q = iralloc(p, size, extra, alignment, zero, no_move); 1718 if (q == NULL) 1719 goto ERR; 1720 usize = isalloc(q); 1721 } 1722 prof_realloc(q, usize, cnt, old_size, old_ctx); 1723 if (rsize != NULL) 1724 *rsize = usize; 1725 } else 1726 #endif 1727 { 1728 #ifdef JEMALLOC_STATS 1729 old_size = isalloc(p); 1730 #endif 1731 q = iralloc(p, size, extra, alignment, zero, no_move); 1732 if (q == NULL) 1733 goto ERR; 1734 #ifndef JEMALLOC_STATS 1735 if (rsize != NULL) 1736 #endif 1737 { 1738 usize = isalloc(q); 1739 #ifdef JEMALLOC_STATS 1740 if (rsize != NULL) 1741 #endif 1742 *rsize = usize; 1743 } 1744 } 1745 1746 *ptr = q; 1747 #ifdef JEMALLOC_STATS 1748 ALLOCATED_ADD(usize, old_size); 1749 #endif 1750 return (ALLOCM_SUCCESS); 1751 ERR: 1752 if (no_move) 1753 return (ALLOCM_ERR_NOT_MOVED); 1754 #ifdef JEMALLOC_PROF 1755 OOM: 1756 #endif 1757 #ifdef JEMALLOC_XMALLOC 1758 if (opt_xmalloc) { 1759 malloc_write("<jemalloc>: Error in rallocm(): " 1760 "out of memory\n"); 1761 abort(); 1762 } 1763 #endif 1764 return (ALLOCM_ERR_OOM); 1765 } 1766 1767 JEMALLOC_ATTR(nonnull(1)) 1768 JEMALLOC_ATTR(visibility("default")) 1769 int 1770 JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags) 1771 { 1772 size_t sz; 1773 1774 assert(malloc_initialized || malloc_initializer == pthread_self()); 1775 1776 #ifdef JEMALLOC_IVSALLOC 1777 sz = ivsalloc(ptr); 1778 #else 1779 assert(ptr != NULL); 1780 sz = isalloc(ptr); 1781 #endif 1782 assert(rsize != NULL); 1783 *rsize = sz; 1784 1785 return (ALLOCM_SUCCESS); 1786 } 1787 1788 JEMALLOC_ATTR(nonnull(1)) 1789 JEMALLOC_ATTR(visibility("default")) 1790 int 1791 JEMALLOC_P(dallocm)(void *ptr, int flags) 1792 { 1793 #if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 1794 size_t usize; 1795 #endif 1796 1797 assert(ptr != NULL); 1798 assert(malloc_initialized || malloc_initializer == pthread_self()); 1799 1800 #ifdef JEMALLOC_STATS 1801 usize = isalloc(ptr); 1802 #endif 1803 #ifdef JEMALLOC_PROF 1804 if (opt_prof) { 1805 # ifndef JEMALLOC_STATS 1806 usize = isalloc(ptr); 1807 # endif 1808 prof_free(ptr, usize); 1809 } 1810 #endif 1811 #ifdef JEMALLOC_STATS 1812 ALLOCATED_ADD(0, usize); 1813 #endif 1814 idalloc(ptr); 1815 1816 return (ALLOCM_SUCCESS); 1817 } 1818 1819 /* 1820 * End non-standard functions. 1821 */ 1822 /******************************************************************************/ 1823 1824 /* 1825 * The following functions are used by threading libraries for protection of 1826 * malloc during fork(). 1827 */ 1828 1829 void 1830 jemalloc_prefork(void) 1831 { 1832 unsigned i; 1833 1834 /* Acquire all mutexes in a safe order. */ 1835 1836 malloc_mutex_lock(&arenas_lock); 1837 for (i = 0; i < narenas; i++) { 1838 if (arenas[i] != NULL) 1839 malloc_mutex_lock(&arenas[i]->lock); 1840 } 1841 1842 malloc_mutex_lock(&base_mtx); 1843 1844 malloc_mutex_lock(&huge_mtx); 1845 1846 #ifdef JEMALLOC_DSS 1847 malloc_mutex_lock(&dss_mtx); 1848 #endif 1849 1850 #ifdef JEMALLOC_SWAP 1851 malloc_mutex_lock(&swap_mtx); 1852 #endif 1853 } 1854 1855 void 1856 jemalloc_postfork(void) 1857 { 1858 unsigned i; 1859 1860 /* Release all mutexes, now that fork() has completed. */ 1861 1862 #ifdef JEMALLOC_SWAP 1863 malloc_mutex_unlock(&swap_mtx); 1864 #endif 1865 1866 #ifdef JEMALLOC_DSS 1867 malloc_mutex_unlock(&dss_mtx); 1868 #endif 1869 1870 malloc_mutex_unlock(&huge_mtx); 1871 1872 malloc_mutex_unlock(&base_mtx); 1873 1874 for (i = 0; i < narenas; i++) { 1875 if (arenas[i] != NULL) 1876 malloc_mutex_unlock(&arenas[i]->lock); 1877 } 1878 malloc_mutex_unlock(&arenas_lock); 1879 } 1880 1881 /******************************************************************************/ 1882