Lines Matching refs:zone
232 uma_zone_t zone; member
295 static void bucket_cache_reclaim(uma_zone_t zone, bool, int);
302 static inline void item_dtor(uma_zone_t zone, void *item, int size,
305 static void zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata,
309 static void zone_timeout(uma_zone_t zone, void *);
317 static int zone_alloc_limit(uma_zone_t zone, int count, int flags);
318 static void zone_free_limit(uma_zone_t zone, int count);
321 static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
322 static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
326 static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item);
328 static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
343 static uint64_t uma_zone_get_allocs(uma_zone_t zone);
349 static uint64_t uma_keg_get_allocs(uma_keg_t zone);
353 static bool uma_dbg_zskip(uma_zone_t zone, void *mem);
354 static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
355 static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
465 bucket_alloc(uma_zone_t zone, void *udata, int flags) in bucket_alloc() argument
486 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) in bucket_alloc()
487 udata = (void *)(uintptr_t)zone->uz_flags; in bucket_alloc()
495 ubz = bucket_zone_lookup(atomic_load_16(&zone->uz_bucket_size)); in bucket_alloc()
496 if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) in bucket_alloc()
505 zone->uz_bucket_size_max); in bucket_alloc()
508 zone->uz_name, zone, bucket); in bucket_alloc()
515 bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata) in bucket_free() argument
520 bucket_drain(zone, bucket); in bucket_free()
526 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) in bucket_free()
527 udata = (void *)(uintptr_t)zone->uz_flags; in bucket_free()
547 kasan_mark_item_valid(uma_zone_t zone, void *item) in kasan_mark_item_valid() argument
553 if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0) in kasan_mark_item_valid()
556 sz = zone->uz_size; in kasan_mark_item_valid()
558 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { in kasan_mark_item_valid()
569 kasan_mark_item_invalid(uma_zone_t zone, void *item) in kasan_mark_item_invalid() argument
575 if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0) in kasan_mark_item_invalid()
578 sz = roundup2(zone->uz_size, KASAN_SHADOW_SCALE); in kasan_mark_item_invalid()
579 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { in kasan_mark_item_invalid()
615 kasan_mark_item_valid(uma_zone_t zone __unused, void *item __unused) in kasan_mark_item_valid()
620 kasan_mark_item_invalid(uma_zone_t zone __unused, void *item __unused) in kasan_mark_item_invalid()
639 zone_domain_lock(uma_zone_t zone, int domain) in zone_domain_lock() argument
644 zdom = ZDOM_GET(zone, domain); in zone_domain_lock()
650 if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max) in zone_domain_lock()
651 zone->uz_bucket_size++; in zone_domain_lock()
660 zone_domain_lowest(uma_zone_t zone, int pref) in zone_domain_lowest() argument
669 nitems = ZDOM_GET(zone, i)->uzd_nitems; in zone_domain_lowest()
688 zone_domain_highest(uma_zone_t zone, int pref) in zone_domain_highest() argument
694 if (ZDOM_GET(zone, pref)->uzd_nitems > BUCKET_MAX) in zone_domain_highest()
700 nitems = ZDOM_GET(zone, i)->uzd_nitems; in zone_domain_highest()
738 zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim) in zone_fetch_bucket() argument
751 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && in zone_fetch_bucket()
753 if (!smr_poll(zone->uz_smr, bucket->ub_seq, false)) in zone_fetch_bucket()
756 dtor = (zone->uz_dtor != NULL) || UMA_ALWAYS_CTORDTOR; in zone_fetch_bucket()
793 item_dtor(zone, bucket->ub_bucket[i], zone->uz_size, in zone_fetch_bucket()
805 zone_put_bucket(uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata, in zone_put_bucket() argument
813 zdom = zone_domain_lock(zone, domain); in zone_put_bucket()
819 if (__predict_true(zdom->uzd_nitems < zone->uz_bucket_max)) { in zone_put_bucket()
849 bucket_free(zone, bucket, udata); in zone_put_bucket()
995 cache_fetch_bucket(uma_zone_t zone, uma_cache_t cache, int domain) in cache_fetch_bucket() argument
1004 zdom = ZDOM_GET(zone, domain); in cache_fetch_bucket()
1010 !smr_poll(zone->uz_smr, seq, false)) in cache_fetch_bucket()
1016 zdom = zone_domain_lock(zone, domain); in cache_fetch_bucket()
1017 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) in cache_fetch_bucket()
1025 zone_log_warning(uma_zone_t zone) in zone_log_warning() argument
1029 if (!zone_warnings || zone->uz_warning == NULL) in zone_log_warning()
1032 if (ratecheck(&zone->uz_ratecheck, &warninterval)) in zone_log_warning()
1033 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning); in zone_log_warning()
1037 zone_maxaction(uma_zone_t zone) in zone_maxaction() argument
1040 if (zone->uz_maxaction.ta_func != NULL) in zone_maxaction()
1041 taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction); in zone_maxaction()
1117 zone_timeout(uma_zone_t zone, void *unused) in zone_timeout() argument
1122 if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) in zone_timeout()
1125 keg = zone->uz_keg; in zone_timeout()
1171 if ((zone->uz_flags & UMA_ZONE_UNMANAGED) == 0) { in zone_timeout()
1173 if (bucket_cache_reclaim_domain(zone, false, false, i) && in zone_timeout()
1174 (zone->uz_flags & UMA_ZFLAG_CACHE) == 0) in zone_timeout()
1175 keg_drain(zone->uz_keg, i); in zone_timeout()
1291 bucket_drain(uma_zone_t zone, uma_bucket_t bucket) in bucket_drain() argument
1298 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && in bucket_drain()
1300 smr_wait(zone->uz_smr, bucket->ub_seq); in bucket_drain()
1303 item_dtor(zone, bucket->ub_bucket[i], in bucket_drain()
1304 zone->uz_size, NULL, SKIP_NONE); in bucket_drain()
1306 if (zone->uz_fini) in bucket_drain()
1308 kasan_mark_item_valid(zone, bucket->ub_bucket[i]); in bucket_drain()
1309 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); in bucket_drain()
1310 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); in bucket_drain()
1312 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); in bucket_drain()
1313 if (zone->uz_max_items > 0) in bucket_drain()
1314 zone_free_limit(zone, bucket->ub_cnt); in bucket_drain()
1335 cache_drain(uma_zone_t zone) in cache_drain() argument
1351 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in cache_drain()
1352 seq = smr_advance(zone->uz_smr); in cache_drain()
1354 cache = &zone->uz_cpu[cpu]; in cache_drain()
1357 bucket_free(zone, bucket, NULL); in cache_drain()
1361 bucket_free(zone, bucket, NULL); in cache_drain()
1366 bucket_free(zone, bucket, NULL); in cache_drain()
1369 bucket_cache_reclaim(zone, true, UMA_ANYDOMAIN); in cache_drain()
1373 cache_shrink(uma_zone_t zone, void *unused) in cache_shrink() argument
1376 if (zone->uz_flags & UMA_ZFLAG_INTERNAL) in cache_shrink()
1379 ZONE_LOCK(zone); in cache_shrink()
1380 zone->uz_bucket_size = in cache_shrink()
1381 (zone->uz_bucket_size_min + zone->uz_bucket_size) / 2; in cache_shrink()
1382 ZONE_UNLOCK(zone); in cache_shrink()
1386 cache_drain_safe_cpu(uma_zone_t zone, void *unused) in cache_drain_safe_cpu() argument
1392 if (zone->uz_flags & UMA_ZFLAG_INTERNAL) in cache_drain_safe_cpu()
1397 cache = &zone->uz_cpu[curcpu]; in cache_drain_safe_cpu()
1405 if ((zone->uz_flags & UMA_ZONE_SMR) == 0) { in cache_drain_safe_cpu()
1412 zone_free_bucket(zone, b1, NULL, domain, false); in cache_drain_safe_cpu()
1414 zone_free_bucket(zone, b2, NULL, domain, false); in cache_drain_safe_cpu()
1418 zone_free_bucket(zone, b3, NULL, domain, false); in cache_drain_safe_cpu()
1430 pcpu_cache_drain_safe(uma_zone_t zone) in pcpu_cache_drain_safe() argument
1437 if (zone) in pcpu_cache_drain_safe()
1438 cache_shrink(zone, NULL); in pcpu_cache_drain_safe()
1447 if (zone) in pcpu_cache_drain_safe()
1448 cache_drain_safe_cpu(zone, NULL); in pcpu_cache_drain_safe()
1463 bucket_cache_reclaim_domain(uma_zone_t zone, bool drain, bool trim, int domain) in bucket_cache_reclaim_domain() argument
1474 zdom = ZDOM_GET(zone, domain); in bucket_cache_reclaim_domain()
1475 if ((zone->uz_flags & UMA_ZONE_SMR) == 0 || drain) { in bucket_cache_reclaim_domain()
1476 ZONE_CROSS_LOCK(zone); in bucket_cache_reclaim_domain()
1479 ZONE_CROSS_UNLOCK(zone); in bucket_cache_reclaim_domain()
1481 bucket_free(zone, bucket, NULL); in bucket_cache_reclaim_domain()
1506 bucket = zone_fetch_bucket(zone, zdom, true); in bucket_cache_reclaim_domain()
1509 bucket_free(zone, bucket, NULL); in bucket_cache_reclaim_domain()
1518 bucket_cache_reclaim(uma_zone_t zone, bool drain, int domain) in bucket_cache_reclaim() argument
1526 if (zone->uz_bucket_size > zone->uz_bucket_size_min) in bucket_cache_reclaim()
1527 zone->uz_bucket_size--; in bucket_cache_reclaim()
1530 (zone->uz_flags & UMA_ZONE_ROUNDROBIN) == 0) { in bucket_cache_reclaim()
1531 bucket_cache_reclaim_domain(zone, drain, true, domain); in bucket_cache_reclaim()
1534 bucket_cache_reclaim_domain(zone, drain, true, i); in bucket_cache_reclaim()
1657 zone_reclaim(uma_zone_t zone, int domain, int waitok, bool drain) in zone_reclaim() argument
1667 ZONE_LOCK(zone); in zone_reclaim()
1669 while (zone->uz_reclaimers > 0) in zone_reclaim()
1670 msleep(zone, ZONE_LOCKPTR(zone), PVM, "zonedrain", 1); in zone_reclaim()
1672 zone->uz_reclaimers++; in zone_reclaim()
1673 ZONE_UNLOCK(zone); in zone_reclaim()
1674 bucket_cache_reclaim(zone, drain, domain); in zone_reclaim()
1676 if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) in zone_reclaim()
1677 keg_drain(zone->uz_keg, domain); in zone_reclaim()
1678 ZONE_LOCK(zone); in zone_reclaim()
1679 zone->uz_reclaimers--; in zone_reclaim()
1680 if (zone->uz_reclaimers == 0) in zone_reclaim()
1681 wakeup(zone); in zone_reclaim()
1682 ZONE_UNLOCK(zone); in zone_reclaim()
1699 keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, in keg_alloc_slab() argument
1740 mem = keg->uk_allocf(zone, size, domain, &sflags, aflags); in keg_alloc_slab()
1762 zone, slab); in keg_alloc_slab()
1814 startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in startup_alloc() argument
1882 page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in page_alloc() argument
1894 pcpu_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in pcpu_page_alloc() argument
1959 noobj_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, in noobj_alloc() argument
1970 keg = zone->uz_keg; in noobj_alloc()
2013 contig_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, in contig_alloc() argument
2376 uma_zone_t zone; in keg_ctor() local
2398 zone = arg->zone; in keg_ctor()
2399 keg->uk_name = zone->uz_name; in keg_ctor()
2479 zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); in keg_ctor()
2485 CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)", keg, zone->uz_name, zone); in keg_ctor()
2487 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); in keg_ctor()
2496 zone_kva_available(uma_zone_t zone, void *unused) in zone_kva_available() argument
2500 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) in zone_kva_available()
2502 KEG_GET(zone, keg); in zone_kva_available()
2517 zone_alloc_counters(uma_zone_t zone, void *unused) in zone_alloc_counters() argument
2520 zone->uz_allocs = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2521 zone->uz_frees = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2522 zone->uz_fails = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2523 zone->uz_xdomain = counter_u64_alloc(M_WAITOK); in zone_alloc_counters()
2527 zone_alloc_sysctl(uma_zone_t zone, void *unused) in zone_alloc_sysctl() argument
2542 if (zone->uz_namecnt != 0) { in zone_alloc_sysctl()
2544 for (i = 1, cnt = zone->uz_namecnt; cnt != 0; i++) in zone_alloc_sysctl()
2546 zone->uz_ctlname = malloc(strlen(zone->uz_name) + i + 1, in zone_alloc_sysctl()
2548 sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name, in zone_alloc_sysctl()
2549 zone->uz_namecnt); in zone_alloc_sysctl()
2551 zone->uz_ctlname = strdup(zone->uz_name, M_UMA); in zone_alloc_sysctl()
2552 for (c = zone->uz_ctlname; *c != '\0'; c++) in zone_alloc_sysctl()
2559 zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma), in zone_alloc_sysctl()
2560 OID_AUTO, zone->uz_ctlname, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); in zone_alloc_sysctl()
2561 oid = zone->uz_oid; in zone_alloc_sysctl()
2563 "size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size"); in zone_alloc_sysctl()
2566 zone, 0, sysctl_handle_uma_zone_flags, "A", in zone_alloc_sysctl()
2569 "bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0, in zone_alloc_sysctl()
2572 "bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0, in zone_alloc_sysctl()
2578 if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) in zone_alloc_sysctl()
2582 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, in zone_alloc_sysctl()
2584 keg = zone->uz_keg; in zone_alloc_sysctl()
2585 if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) { in zone_alloc_sysctl()
2631 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, in zone_alloc_sysctl()
2635 zone, 0, sysctl_handle_uma_zone_items, "QU", in zone_alloc_sysctl()
2638 "max_items", CTLFLAG_RD, &zone->uz_max_items, 0, in zone_alloc_sysctl()
2641 "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0, in zone_alloc_sysctl()
2644 "sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0, in zone_alloc_sysctl()
2647 "bucket_max", CTLFLAG_RD, &zone->uz_bucket_max, 0, in zone_alloc_sysctl()
2653 domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), in zone_alloc_sysctl()
2656 zdom = ZDOM_GET(zone, i); in zone_alloc_sysctl()
2686 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, in zone_alloc_sysctl()
2690 zone, 1, sysctl_handle_uma_zone_cur, "I", in zone_alloc_sysctl()
2694 zone, 0, sysctl_handle_uma_zone_allocs, "QU", in zone_alloc_sysctl()
2698 zone, 0, sysctl_handle_uma_zone_frees, "QU", in zone_alloc_sysctl()
2701 "fails", CTLFLAG_RD, &zone->uz_fails, in zone_alloc_sysctl()
2704 "xdomain", CTLFLAG_RD, &zone->uz_xdomain, in zone_alloc_sysctl()
2714 zone_count(uma_zone_t zone, void *arg) in zone_count() argument
2724 if (strcmp(zone->uz_name, cnt->name) == 0) in zone_count()
2726 zone->uz_namecnt + 1); in zone_count()
2730 zone_update_caches(uma_zone_t zone) in zone_update_caches() argument
2735 cache_set_uz_size(&zone->uz_cpu[i], zone->uz_size); in zone_update_caches()
2736 cache_set_uz_flags(&zone->uz_cpu[i], zone->uz_flags); in zone_update_caches()
2752 uma_zone_t zone = mem; in zone_ctor() local
2757 bzero(zone, size); in zone_ctor()
2758 zone->uz_name = arg->name; in zone_ctor()
2759 zone->uz_ctor = arg->ctor; in zone_ctor()
2760 zone->uz_dtor = arg->dtor; in zone_ctor()
2761 zone->uz_init = NULL; in zone_ctor()
2762 zone->uz_fini = NULL; in zone_ctor()
2763 zone->uz_sleeps = 0; in zone_ctor()
2764 zone->uz_bucket_size = 0; in zone_ctor()
2765 zone->uz_bucket_size_min = 0; in zone_ctor()
2766 zone->uz_bucket_size_max = BUCKET_MAX; in zone_ctor()
2767 zone->uz_flags = (arg->flags & UMA_ZONE_SMR); in zone_ctor()
2768 zone->uz_warning = NULL; in zone_ctor()
2770 zone->uz_bucket_max = ULONG_MAX; in zone_ctor()
2771 timevalclear(&zone->uz_ratecheck); in zone_ctor()
2777 zone->uz_namecnt = cnt.count; in zone_ctor()
2778 ZONE_CROSS_LOCK_INIT(zone); in zone_ctor()
2781 zdom = ZDOM_GET(zone, i); in zone_ctor()
2782 ZDOM_LOCK_INIT(zone, zdom, (arg->flags & UMA_ZONE_MTXCLASS)); in zone_ctor()
2788 zone->uz_flags |= UMA_ZFLAG_TRASH | UMA_ZFLAG_CTORDTOR; in zone_ctor()
2800 zone->uz_flags = arg->flags; in zone_ctor()
2801 zone->uz_size = arg->size; in zone_ctor()
2802 zone->uz_import = arg->import; in zone_ctor()
2803 zone->uz_release = arg->release; in zone_ctor()
2804 zone->uz_arg = arg->arg; in zone_ctor()
2811 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) == 0) in zone_ctor()
2812 zone->uz_flags |= UMA_ZONE_ROUNDROBIN; in zone_ctor()
2815 LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link); in zone_ctor()
2823 zone->uz_import = zone_import; in zone_ctor()
2824 zone->uz_release = zone_release; in zone_ctor()
2825 zone->uz_arg = zone; in zone_ctor()
2829 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0, in zone_ctor()
2832 zone->uz_init = arg->uminit; in zone_ctor()
2833 zone->uz_fini = arg->fini; in zone_ctor()
2834 zone->uz_flags |= UMA_ZONE_SECONDARY; in zone_ctor()
2836 ZONE_LOCK(zone); in zone_ctor()
2839 LIST_INSERT_AFTER(z, zone, uz_link); in zone_ctor()
2843 ZONE_UNLOCK(zone); in zone_ctor()
2846 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini, in zone_ctor()
2859 karg.zone = zone; in zone_ctor()
2867 zone->uz_keg = keg; in zone_ctor()
2868 zone->uz_size = keg->uk_size; in zone_ctor()
2869 zone->uz_flags |= (keg->uk_flags & in zone_ctor()
2874 zone_alloc_counters(zone, NULL); in zone_ctor()
2876 zone_alloc_sysctl(zone, NULL); in zone_ctor()
2878 zone->uz_allocs = EARLY_COUNTER; in zone_ctor()
2879 zone->uz_frees = EARLY_COUNTER; in zone_ctor()
2880 zone->uz_fails = EARLY_COUNTER; in zone_ctor()
2884 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in zone_ctor()
2885 zone->uz_smr = smr_create(zone->uz_name, 0, 0); in zone_ctor()
2891 zone->uz_bucket_size_max = zone->uz_bucket_size = 0; in zone_ctor()
2893 zone->uz_bucket_size = BUCKET_MAX; in zone_ctor()
2895 zone->uz_bucket_size = 0; in zone_ctor()
2897 zone->uz_bucket_size = bucket_select(zone->uz_size); in zone_ctor()
2898 zone->uz_bucket_size_min = zone->uz_bucket_size; in zone_ctor()
2899 if (zone->uz_dtor != NULL || zone->uz_ctor != NULL) in zone_ctor()
2900 zone->uz_flags |= UMA_ZFLAG_CTORDTOR; in zone_ctor()
2901 zone_update_caches(zone); in zone_ctor()
2945 uma_zone_t zone; in zone_dtor() local
2949 zone = (uma_zone_t)arg; in zone_dtor()
2951 sysctl_remove_oid(zone->uz_oid, 1, 1); in zone_dtor()
2953 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) in zone_dtor()
2954 cache_drain(zone); in zone_dtor()
2957 LIST_REMOVE(zone, uz_link); in zone_dtor()
2959 if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) { in zone_dtor()
2960 keg = zone->uz_keg; in zone_dtor()
2963 zone_reclaim(zone, UMA_ANYDOMAIN, M_WAITOK, true); in zone_dtor()
2968 if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) { in zone_dtor()
2969 keg = zone->uz_keg; in zone_dtor()
2975 counter_u64_free(zone->uz_allocs); in zone_dtor()
2976 counter_u64_free(zone->uz_frees); in zone_dtor()
2977 counter_u64_free(zone->uz_fails); in zone_dtor()
2978 counter_u64_free(zone->uz_xdomain); in zone_dtor()
2979 free(zone->uz_ctlname, M_UMA); in zone_dtor()
2981 ZDOM_LOCK_FINI(ZDOM_GET(zone, i)); in zone_dtor()
2982 ZONE_CROSS_LOCK_FINI(zone); in zone_dtor()
2989 uma_zone_t zone; in zone_foreach_unlocked() local
2992 LIST_FOREACH(zone, &keg->uk_zones, uz_link) in zone_foreach_unlocked()
2993 zfunc(zone, arg); in zone_foreach_unlocked()
2995 LIST_FOREACH(zone, &uma_cachezones, uz_link) in zone_foreach_unlocked()
2996 zfunc(zone, arg); in zone_foreach_unlocked()
3170 uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, in uma_kcreate() argument
3180 args.zone = zone; in uma_kcreate()
3294 uma_zdestroy(uma_zone_t zone) in uma_zdestroy() argument
3302 zone->uz_fini == NULL && zone->uz_release == zone_release) in uma_zdestroy()
3305 zone_free_item(zones, zone, NULL, SKIP_NONE); in uma_zdestroy()
3310 uma_zwait(uma_zone_t zone) in uma_zwait() argument
3313 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in uma_zwait()
3314 uma_zfree_smr(zone, uma_zalloc_smr(zone, M_WAITOK)); in uma_zwait()
3315 else if ((zone->uz_flags & UMA_ZONE_PCPU) != 0) in uma_zwait()
3316 uma_zfree_pcpu(zone, uma_zalloc_pcpu(zone, M_WAITOK)); in uma_zwait()
3318 uma_zfree(zone, uma_zalloc(zone, M_WAITOK)); in uma_zwait()
3322 uma_zalloc_pcpu_arg(uma_zone_t zone, void *udata, int flags) in uma_zalloc_pcpu_arg() argument
3328 MPASS(zone->uz_flags & UMA_ZONE_PCPU); in uma_zalloc_pcpu_arg()
3330 item = uma_zalloc_arg(zone, udata, flags & ~M_ZERO); in uma_zalloc_pcpu_arg()
3337 bzero(zpcpu_get_cpu(pcpu_item, i), zone->uz_size); in uma_zalloc_pcpu_arg()
3339 bzero(item, zone->uz_size); in uma_zalloc_pcpu_arg()
3349 uma_zfree_pcpu_arg(uma_zone_t zone, void *pcpu_item, void *udata) in uma_zfree_pcpu_arg() argument
3354 MPASS(zone->uz_flags & UMA_ZONE_PCPU); in uma_zfree_pcpu_arg()
3362 uma_zfree_arg(zone, item, udata); in uma_zfree_pcpu_arg()
3366 item_ctor(uma_zone_t zone, int uz_flags, int size, void *udata, int flags, in item_ctor() argument
3373 kasan_mark_item_valid(zone, item); in item_ctor()
3376 skipdbg = uma_dbg_zskip(zone, item); in item_ctor()
3378 zone->uz_ctor != trash_ctor) in item_ctor()
3384 __predict_false(zone->uz_ctor != NULL) && in item_ctor()
3385 zone->uz_ctor(item, size, udata, flags) != 0) { in item_ctor()
3386 counter_u64_add(zone->uz_fails, 1); in item_ctor()
3387 zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); in item_ctor()
3392 uma_dbg_alloc(zone, NULL, item); in item_ctor()
3401 item_dtor(uma_zone_t zone, void *item, int size, void *udata, in item_dtor() argument
3407 skipdbg = uma_dbg_zskip(zone, item); in item_dtor()
3409 if ((zone->uz_flags & UMA_ZONE_MALLOC) != 0) in item_dtor()
3410 uma_dbg_free(zone, udata, item); in item_dtor()
3412 uma_dbg_free(zone, NULL, item); in item_dtor()
3416 if (zone->uz_dtor != NULL) in item_dtor()
3417 zone->uz_dtor(item, size, udata); in item_dtor()
3419 if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && in item_dtor()
3420 zone->uz_dtor != trash_dtor) in item_dtor()
3424 kasan_mark_item_invalid(zone, item); in item_dtor()
3443 uma_zalloc_debug(uma_zone_t zone, void **itemp, void *udata, int flags) in uma_zalloc_debug() argument
3451 "uma_zalloc_debug: zone \"%s\"", zone->uz_name); in uma_zalloc_debug()
3460 KASSERT((zone->uz_flags & UMA_ZONE_PCPU) == 0 || (flags & M_ZERO) == 0, in uma_zalloc_debug()
3465 if ((zone->uz_flags & UMA_ZONE_SMR) == 0 && memguard_cmp_zone(zone)) { in uma_zalloc_debug()
3467 item = memguard_alloc(zone->uz_size, flags); in uma_zalloc_debug()
3470 if (zone->uz_init != NULL && in uma_zalloc_debug()
3471 zone->uz_init(item, zone->uz_size, flags) != 0) { in uma_zalloc_debug()
3475 if (zone->uz_ctor != NULL && in uma_zalloc_debug()
3476 zone->uz_ctor(item, zone->uz_size, udata, in uma_zalloc_debug()
3478 counter_u64_add(zone->uz_fails, 1); in uma_zalloc_debug()
3479 if (zone->uz_fini != NULL) in uma_zalloc_debug()
3480 zone->uz_fini(item, zone->uz_size); in uma_zalloc_debug()
3494 uma_zfree_debug(uma_zone_t zone, void *item, void *udata) in uma_zfree_debug() argument
3500 if ((zone->uz_flags & UMA_ZONE_SMR) == 0 && is_memguard_addr(item)) { in uma_zfree_debug()
3501 if (zone->uz_dtor != NULL) in uma_zfree_debug()
3502 zone->uz_dtor(item, zone->uz_size, udata); in uma_zfree_debug()
3503 if (zone->uz_fini != NULL) in uma_zfree_debug()
3504 zone->uz_fini(item, zone->uz_size); in uma_zfree_debug()
3514 cache_alloc_item(uma_zone_t zone, uma_cache_t cache, uma_cache_bucket_t bucket, in cache_alloc_item() argument
3524 return (item_ctor(zone, uz_flags, size, udata, flags, item)); in cache_alloc_item()
3528 cache_alloc_retry(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) in cache_alloc_retry() argument
3533 while (cache_alloc(zone, cache, udata, flags)) { in cache_alloc_retry()
3534 cache = &zone->uz_cpu[curcpu]; in cache_alloc_retry()
3538 return (cache_alloc_item(zone, cache, bucket, udata, flags)); in cache_alloc_retry()
3545 if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) in cache_alloc_retry()
3549 return (zone_alloc_item(zone, udata, domain, flags)); in cache_alloc_retry()
3554 uma_zalloc_smr(uma_zone_t zone, int flags) in uma_zalloc_smr() argument
3562 KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, in uma_zalloc_smr()
3564 if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) in uma_zalloc_smr()
3569 cache = &zone->uz_cpu[curcpu]; in uma_zalloc_smr()
3572 return (cache_alloc_retry(zone, cache, NULL, flags)); in uma_zalloc_smr()
3573 return (cache_alloc_item(zone, cache, bucket, NULL, flags)); in uma_zalloc_smr()
3578 uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) in uma_zalloc_arg() argument
3584 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); in uma_zalloc_arg()
3587 CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name, in uma_zalloc_arg()
3588 zone, flags); in uma_zalloc_arg()
3593 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zalloc_arg()
3595 if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) in uma_zalloc_arg()
3611 cache = &zone->uz_cpu[curcpu]; in uma_zalloc_arg()
3614 return (cache_alloc_retry(zone, cache, udata, flags)); in uma_zalloc_arg()
3615 return (cache_alloc_item(zone, cache, bucket, udata, flags)); in uma_zalloc_arg()
3626 cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) in cache_alloc() argument
3657 bucket_free(zone, bucket, udata); in cache_alloc()
3672 domain = zone_domain_highest(zone, domain); in cache_alloc()
3673 bucket = cache_fetch_bucket(zone, cache, domain); in cache_alloc()
3674 if (bucket == NULL && zone->uz_bucket_size != 0 && !bucketdisable) { in cache_alloc()
3675 bucket = zone_alloc_bucket(zone, udata, domain, flags); in cache_alloc()
3682 zone->uz_name, zone, bucket); in cache_alloc()
3694 cache = &zone->uz_cpu[curcpu]; in cache_alloc()
3700 atomic_add_long(&ZDOM_GET(zone, domain)->uzd_imax, in cache_alloc()
3710 zone_put_bucket(zone, domain, bucket, udata, !new); in cache_alloc()
3717 uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) in uma_zalloc_domain() argument
3726 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); in uma_zalloc_domain()
3730 zone->uz_name, zone, domain, flags); in uma_zalloc_domain()
3734 "uma_zalloc_domain: zone \"%s\"", zone->uz_name); in uma_zalloc_domain()
3738 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zalloc_domain()
3741 KASSERT((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0, in uma_zalloc_domain()
3745 return (uma_zalloc_arg(zone, udata, flags)); in uma_zalloc_domain()
3754 zdom = zone_domain_lock(zone, domain); in uma_zalloc_domain()
3755 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) { in uma_zalloc_domain()
3761 zone_put_bucket(zone, domain, bucket, udata, true); in uma_zalloc_domain()
3762 item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, in uma_zalloc_domain()
3768 counter_u64_add(zone->uz_allocs, 1); in uma_zalloc_domain()
3773 return (zone_alloc_item(zone, udata, domain, flags)); in uma_zalloc_domain()
3775 return (uma_zalloc_arg(zone, udata, flags)); in uma_zalloc_domain()
3841 keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) in keg_fetch_slab() argument
3880 slab = keg_alloc_slab(keg, zone, domain, flags, aflags); in keg_fetch_slab()
3951 uma_zone_t zone; in zone_import() local
3959 zone = arg; in zone_import()
3961 keg = zone->uz_keg; in zone_import()
3964 if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) in zone_import()
3991 if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0 && in zone_import()
4007 zone_alloc_limit_hard(uma_zone_t zone, int count, int flags) in zone_alloc_limit_hard() argument
4019 zone_free_limit(zone, count); in zone_alloc_limit_hard()
4020 zone_log_warning(zone); in zone_alloc_limit_hard()
4021 zone_maxaction(zone); in zone_alloc_limit_hard()
4030 sleepq_lock(&zone->uz_max_items); in zone_alloc_limit_hard()
4031 old = zone->uz_items; in zone_alloc_limit_hard()
4035 max = zone->uz_max_items; in zone_alloc_limit_hard()
4041 } while (atomic_fcmpset_64(&zone->uz_items, &old, new) == 0); in zone_alloc_limit_hard()
4045 sleepq_release(&zone->uz_max_items); in zone_alloc_limit_hard()
4055 atomic_add_32(&zone->uz_sleepers, 1); in zone_alloc_limit_hard()
4056 atomic_add_64(&zone->uz_sleeps, 1); in zone_alloc_limit_hard()
4062 sleepq_add(&zone->uz_max_items, NULL, "zonelimit", 0, 0); in zone_alloc_limit_hard()
4063 sleepq_wait(&zone->uz_max_items, PVM); in zone_alloc_limit_hard()
4072 atomic_subtract_32(&zone->uz_sleepers, 1); in zone_alloc_limit_hard()
4073 old = atomic_fetchadd_64(&zone->uz_items, in zone_alloc_limit_hard()
4084 max = zone->uz_max_items; in zone_alloc_limit_hard()
4089 zone_free_limit(zone, total + count - max); in zone_alloc_limit_hard()
4092 wakeup_one(&zone->uz_max_items); in zone_alloc_limit_hard()
4104 zone_alloc_limit(uma_zone_t zone, int count, int flags) in zone_alloc_limit() argument
4109 max = zone->uz_max_items; in zone_alloc_limit()
4116 old = atomic_fetchadd_64(&zone->uz_items, count); in zone_alloc_limit()
4127 zone_free_limit(zone, (old + count) - max); in zone_alloc_limit()
4130 return (zone_alloc_limit_hard(zone, count, flags)); in zone_alloc_limit()
4137 zone_free_limit(uma_zone_t zone, int count) in zone_free_limit() argument
4147 old = atomic_fetchadd_64(&zone->uz_items, -count); in zone_free_limit()
4149 UZ_ITEMS_COUNT(old) - count >= zone->uz_max_items)) in zone_free_limit()
4156 wakeup_one(&zone->uz_max_items); in zone_free_limit()
4160 zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) in zone_alloc_bucket() argument
4165 CTR3(KTR_UMA, "zone_alloc_bucket zone %s(%p) domain %d", zone->uz_name, in zone_alloc_bucket()
4166 zone, domain); in zone_alloc_bucket()
4171 else if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0) in zone_alloc_bucket()
4174 if (zone->uz_max_items > 0) in zone_alloc_bucket()
4175 maxbucket = zone_alloc_limit(zone, zone->uz_bucket_size, in zone_alloc_bucket()
4178 maxbucket = zone->uz_bucket_size; in zone_alloc_bucket()
4183 bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); in zone_alloc_bucket()
4189 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, in zone_alloc_bucket()
4195 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) { in zone_alloc_bucket()
4199 kasan_mark_item_valid(zone, bucket->ub_bucket[i]); in zone_alloc_bucket()
4200 error = zone->uz_init(bucket->ub_bucket[i], in zone_alloc_bucket()
4201 zone->uz_size, flags); in zone_alloc_bucket()
4202 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); in zone_alloc_bucket()
4212 zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i], in zone_alloc_bucket()
4224 bucket_free(zone, bucket, udata); in zone_alloc_bucket()
4225 counter_u64_add(zone->uz_fails, 1); in zone_alloc_bucket()
4229 if (zone->uz_max_items > 0 && cnt < maxbucket) in zone_alloc_bucket()
4230 zone_free_limit(zone, maxbucket - cnt); in zone_alloc_bucket()
4250 zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags) in zone_alloc_item() argument
4254 if (zone->uz_max_items > 0 && zone_alloc_limit(zone, 1, flags) == 0) { in zone_alloc_item()
4255 counter_u64_add(zone->uz_fails, 1); in zone_alloc_item()
4263 if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) in zone_alloc_item()
4272 if (zone->uz_init != NULL) { in zone_alloc_item()
4275 kasan_mark_item_valid(zone, item); in zone_alloc_item()
4276 error = zone->uz_init(item, zone->uz_size, flags); in zone_alloc_item()
4277 kasan_mark_item_invalid(zone, item); in zone_alloc_item()
4279 zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT); in zone_alloc_item()
4283 item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, flags, in zone_alloc_item()
4288 counter_u64_add(zone->uz_allocs, 1); in zone_alloc_item()
4290 zone->uz_name, zone); in zone_alloc_item()
4295 counter_u64_add(zone->uz_fails, 1); in zone_alloc_item()
4297 if (zone->uz_max_items > 0) in zone_alloc_item()
4298 zone_free_limit(zone, 1); in zone_alloc_item()
4300 zone->uz_name, zone); in zone_alloc_item()
4307 uma_zfree_smr(uma_zone_t zone, void *item) in uma_zfree_smr() argument
4314 KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, in uma_zfree_smr()
4317 SMR_ASSERT_NOT_ENTERED(zone->uz_smr); in uma_zfree_smr()
4318 if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN) in uma_zfree_smr()
4321 cache = &zone->uz_cpu[curcpu]; in uma_zfree_smr()
4330 cache = &zone->uz_cpu[curcpu]; in uma_zfree_smr()
4344 } while (cache_free(zone, cache, NULL, item, itemdomain)); in uma_zfree_smr()
4350 zone_free_item(zone, item, NULL, SKIP_NONE); in uma_zfree_smr()
4355 uma_zfree_arg(uma_zone_t zone, void *item, void *udata) in uma_zfree_arg() argument
4362 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); in uma_zfree_arg()
4364 CTR2(KTR_UMA, "uma_zfree_arg zone %s(%p)", zone->uz_name, zone); in uma_zfree_arg()
4367 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zfree_arg()
4369 if (uma_zfree_debug(zone, item, udata) == EJUSTRETURN) in uma_zfree_arg()
4381 cache = &zone->uz_cpu[curcpu]; in uma_zfree_arg()
4385 item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE); in uma_zfree_arg()
4392 if (atomic_load_32(&zone->uz_sleepers) > 0) in uma_zfree_arg()
4414 cache = &zone->uz_cpu[curcpu]; in uma_zfree_arg()
4438 } while (cache_free(zone, cache, udata, item, itemdomain)); in uma_zfree_arg()
4445 zone_free_item(zone, item, udata, SKIP_DTOR); in uma_zfree_arg()
4454 zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata) in zone_free_cross() argument
4465 zone->uz_name, zone, bucket); in zone_free_cross()
4472 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) in zone_free_cross()
4473 seq = smr_advance(zone->uz_smr); in zone_free_cross()
4482 ZONE_CROSS_LOCK(zone); in zone_free_cross()
4486 zdom = ZDOM_GET(zone, domain); in zone_free_cross()
4498 ZONE_CROSS_UNLOCK(zone); in zone_free_cross()
4499 b = bucket_alloc(zone, udata, M_NOWAIT); in zone_free_cross()
4502 ZONE_CROSS_LOCK(zone); in zone_free_cross()
4521 ZONE_CROSS_UNLOCK(zone); in zone_free_cross()
4525 bucket_free(zone, bucket, udata); in zone_free_cross()
4529 bucket_free(zone, b, udata); in zone_free_cross()
4534 zone_put_bucket(zone, domain, b, udata, true); in zone_free_cross()
4540 zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, in zone_free_bucket() argument
4551 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && in zone_free_bucket()
4553 zone_free_cross(zone, bucket, udata); in zone_free_bucket()
4563 zone->uz_name, zone, bucket); in zone_free_bucket()
4565 if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0) in zone_free_bucket()
4566 itemdomain = zone_domain_lowest(zone, itemdomain); in zone_free_bucket()
4567 zone_put_bucket(zone, itemdomain, bucket, udata, ws); in zone_free_bucket()
4579 cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, void *item, in cache_free() argument
4587 if (zone->uz_bucket_size == 0) in cache_free()
4590 cache = &zone->uz_cpu[curcpu]; in cache_free()
4604 counter_u64_add(zone->uz_xdomain, in cache_free()
4622 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) { in cache_free()
4624 bucket->ub_seq = smr_advance(zone->uz_smr); in cache_free()
4625 newbucket = bucket_alloc(zone, udata, M_NOWAIT); in cache_free()
4627 bucket_drain(zone, bucket); in cache_free()
4632 newbucket = bucket_alloc(zone, udata, M_NOWAIT); in cache_free()
4635 zone_free_bucket(zone, bucket, udata, itemdomain, true); in cache_free()
4640 cache = &zone->uz_cpu[curcpu]; in cache_free()
4660 bucket_free(zone, bucket, udata); in cache_free()
4669 slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item) in slab_free_item() argument
4675 keg = zone->uz_keg; in slab_free_item()
4702 uma_zone_t zone; in zone_release() local
4709 zone = arg; in zone_release()
4710 keg = zone->uz_keg; in zone_release()
4712 if (__predict_false((zone->uz_flags & UMA_ZFLAG_HASH) != 0)) in zone_release()
4716 if (__predict_true((zone->uz_flags & UMA_ZFLAG_VTOSLAB) != 0)) { in zone_release()
4720 if ((zone->uz_flags & UMA_ZFLAG_HASH) != 0) in zone_release()
4730 slab_free_item(zone, slab, item); in zone_release()
4746 zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) in zone_free_item() argument
4755 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && skip == SKIP_NONE) in zone_free_item()
4756 smr_synchronize(zone->uz_smr); in zone_free_item()
4758 item_dtor(zone, item, zone->uz_size, udata, skip); in zone_free_item()
4760 if (skip < SKIP_FINI && zone->uz_fini) { in zone_free_item()
4761 kasan_mark_item_valid(zone, item); in zone_free_item()
4762 zone->uz_fini(item, zone->uz_size); in zone_free_item()
4763 kasan_mark_item_invalid(zone, item); in zone_free_item()
4766 zone->uz_release(zone->uz_arg, &item, 1); in zone_free_item()
4771 counter_u64_add(zone->uz_frees, 1); in zone_free_item()
4773 if (zone->uz_max_items > 0) in zone_free_item()
4774 zone_free_limit(zone, 1); in zone_free_item()
4779 uma_zone_set_max(uma_zone_t zone, int nitems) in uma_zone_set_max() argument
4786 uma_zone_set_maxcache(zone, nitems); in uma_zone_set_max()
4793 ZONE_LOCK(zone); in uma_zone_set_max()
4794 zone->uz_max_items = nitems; in uma_zone_set_max()
4795 zone->uz_flags |= UMA_ZFLAG_LIMIT; in uma_zone_set_max()
4796 zone_update_caches(zone); in uma_zone_set_max()
4798 wakeup(&zone->uz_max_items); in uma_zone_set_max()
4799 ZONE_UNLOCK(zone); in uma_zone_set_max()
4806 uma_zone_set_maxcache(uma_zone_t zone, int nitems) in uma_zone_set_maxcache() argument
4810 ZONE_LOCK(zone); in uma_zone_set_maxcache()
4822 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && vm_ndomains > 1) { in uma_zone_set_maxcache()
4833 zone->uz_bucket_size_max = zone->uz_bucket_size = bsize; in uma_zone_set_maxcache()
4834 if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) in uma_zone_set_maxcache()
4835 zone->uz_bucket_size_min = zone->uz_bucket_size_max; in uma_zone_set_maxcache()
4836 zone->uz_bucket_max = nitems - nb * bsize; in uma_zone_set_maxcache()
4837 ZONE_UNLOCK(zone); in uma_zone_set_maxcache()
4842 uma_zone_get_max(uma_zone_t zone) in uma_zone_get_max() argument
4846 nitems = atomic_load_64(&zone->uz_max_items); in uma_zone_get_max()
4853 uma_zone_set_warning(uma_zone_t zone, const char *warning) in uma_zone_set_warning() argument
4856 ZONE_ASSERT_COLD(zone); in uma_zone_set_warning()
4857 zone->uz_warning = warning; in uma_zone_set_warning()
4862 uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction) in uma_zone_set_maxaction() argument
4865 ZONE_ASSERT_COLD(zone); in uma_zone_set_maxaction()
4866 TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone); in uma_zone_set_maxaction()
4871 uma_zone_get_cur(uma_zone_t zone) in uma_zone_get_cur() argument
4877 if (zone->uz_allocs != EARLY_COUNTER && zone->uz_frees != EARLY_COUNTER) in uma_zone_get_cur()
4878 nitems = counter_u64_fetch(zone->uz_allocs) - in uma_zone_get_cur()
4879 counter_u64_fetch(zone->uz_frees); in uma_zone_get_cur()
4881 nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs) - in uma_zone_get_cur()
4882 atomic_load_64(&zone->uz_cpu[i].uc_frees); in uma_zone_get_cur()
4888 uma_zone_get_allocs(uma_zone_t zone) in uma_zone_get_allocs() argument
4894 if (zone->uz_allocs != EARLY_COUNTER) in uma_zone_get_allocs()
4895 nitems = counter_u64_fetch(zone->uz_allocs); in uma_zone_get_allocs()
4897 nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs); in uma_zone_get_allocs()
4903 uma_zone_get_frees(uma_zone_t zone) in uma_zone_get_frees() argument
4909 if (zone->uz_frees != EARLY_COUNTER) in uma_zone_get_frees()
4910 nitems = counter_u64_fetch(zone->uz_frees); in uma_zone_get_frees()
4912 nitems += atomic_load_64(&zone->uz_cpu[i].uc_frees); in uma_zone_get_frees()
4935 uma_zone_set_init(uma_zone_t zone, uma_init uminit) in uma_zone_set_init() argument
4939 KEG_GET(zone, keg); in uma_zone_set_init()
4946 uma_zone_set_fini(uma_zone_t zone, uma_fini fini) in uma_zone_set_fini() argument
4950 KEG_GET(zone, keg); in uma_zone_set_fini()
4957 uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) in uma_zone_set_zinit() argument
4960 ZONE_ASSERT_COLD(zone); in uma_zone_set_zinit()
4961 zone->uz_init = zinit; in uma_zone_set_zinit()
4966 uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) in uma_zone_set_zfini() argument
4969 ZONE_ASSERT_COLD(zone); in uma_zone_set_zfini()
4970 zone->uz_fini = zfini; in uma_zone_set_zfini()
4975 uma_zone_set_freef(uma_zone_t zone, uma_free freef) in uma_zone_set_freef() argument
4979 KEG_GET(zone, keg); in uma_zone_set_freef()
4986 uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) in uma_zone_set_allocf() argument
4990 KEG_GET(zone, keg); in uma_zone_set_allocf()
4997 uma_zone_set_smr(uma_zone_t zone, smr_t smr) in uma_zone_set_smr() argument
5000 ZONE_ASSERT_COLD(zone); in uma_zone_set_smr()
5003 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, in uma_zone_set_smr()
5004 ("zone %p (%s) already uses SMR", zone, zone->uz_name)); in uma_zone_set_smr()
5005 zone->uz_flags |= UMA_ZONE_SMR; in uma_zone_set_smr()
5006 zone->uz_smr = smr; in uma_zone_set_smr()
5007 zone_update_caches(zone); in uma_zone_set_smr()
5011 uma_zone_get_smr(uma_zone_t zone) in uma_zone_get_smr() argument
5014 return (zone->uz_smr); in uma_zone_get_smr()
5019 uma_zone_reserve(uma_zone_t zone, int items) in uma_zone_reserve() argument
5023 KEG_GET(zone, keg); in uma_zone_reserve()
5030 uma_zone_reserve_kva(uma_zone_t zone, int count) argument
5036 KEG_GET(zone, keg);
5038 ZONE_ASSERT_COLD(zone);
5056 zone->uz_max_items = pages * keg->uk_ipers;
5063 zone->uz_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE;
5064 zone_update_caches(zone);
5071 uma_prealloc(uma_zone_t zone, int items) argument
5079 KEG_GET(zone, keg);
5086 slab = keg_alloc_slab(keg, zone, domain, M_WAITOK,
5111 uma_zone_memory(uma_zone_t zone) argument
5117 if (zone->uz_flags & UMA_ZFLAG_CACHE) {
5119 sz += ZDOM_GET(zone, i)->uzd_nitems;
5120 return (sz * zone->uz_size);
5123 sz += zone->uz_keg->uk_domain[i].ud_pages;
5134 uma_reclaim_domain_cb(uma_zone_t zone, void *arg) argument
5139 if ((zone->uz_flags & UMA_ZONE_UNMANAGED) == 0)
5140 uma_zone_reclaim_domain(zone, args->req, args->domain);
5216 uma_zone_reclaim(uma_zone_t zone, int req) argument
5218 uma_zone_reclaim_domain(zone, req, UMA_ANYDOMAIN);
5222 uma_zone_reclaim_domain(uma_zone_t zone, int req, int domain) argument
5226 zone_reclaim(zone, domain, M_NOWAIT, false);
5229 zone_reclaim(zone, domain, M_NOWAIT, true);
5232 pcpu_cache_drain_safe(zone);
5233 zone_reclaim(zone, domain, M_NOWAIT, true);
5242 uma_zone_exhausted(uma_zone_t zone) argument
5245 return (atomic_load_32(&zone->uz_sleepers) > 0);
5468 uma_zone_t zone = *(uma_zone_t *)arg1; local
5471 max = uma_zone_get_max(zone);
5476 uma_zone_set_max(zone, max);
5484 uma_zone_t zone; local
5492 zone = *(uma_zone_t *)arg1;
5494 zone = arg1;
5495 cur = uma_zone_get_cur(zone);
5502 uma_zone_t zone = arg1; local
5505 cur = uma_zone_get_allocs(zone);
5512 uma_zone_t zone = arg1; local
5515 cur = uma_zone_get_frees(zone);
5523 uma_zone_t zone = arg1; local
5527 if (zone->uz_flags != 0)
5528 sbuf_printf(&sbuf, "0x%b", zone->uz_flags, PRINT_UMA_ZFLAGS);
5561 uma_zone_t zone = arg1; local
5564 cur = UZ_ITEMS_COUNT(atomic_load_64(&zone->uz_items));
5570 uma_dbg_getslab(uma_zone_t zone, void *item) argument
5582 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
5584 if (zone->uz_flags & UMA_ZFLAG_VTOSLAB)
5586 keg = zone->uz_keg;
5597 uma_dbg_zskip(uma_zone_t zone, void *mem) argument
5600 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
5603 return (uma_dbg_kskip(zone->uz_keg, mem));
5637 uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item) argument
5643 slab = uma_dbg_getslab(zone, item);
5646 item, zone->uz_name);
5648 keg = zone->uz_keg;
5654 item, zone, zone->uz_name, slab, freei);
5663 uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item) argument
5669 slab = uma_dbg_getslab(zone, item);
5672 item, zone->uz_name);
5674 keg = zone->uz_keg;
5679 item, zone, zone->uz_name, slab, freei);
5683 item, zone, zone->uz_name, slab, freei);
5688 item, zone, zone->uz_name, slab, freei);