34 void kmem_cache_set_callback(struct kmem_cache *cachep, void (*callback)(void *))  in kmem_cache_set_callback()  argument
36 	cachep->callback = callback;  in kmem_cache_set_callback()
39 void kmem_cache_set_private(struct kmem_cache *cachep, void *private)  in kmem_cache_set_private()  argument
41 	cachep->private = private;  in kmem_cache_set_private()
44 void kmem_cache_set_non_kernel(struct kmem_cache *cachep, unsigned int val)  in kmem_cache_set_non_kernel()  argument
46 	cachep->non_kernel = val;  in kmem_cache_set_non_kernel()
49 unsigned long kmem_cache_get_alloc(struct kmem_cache *cachep)  in kmem_cache_get_alloc()  argument
51 	return cachep->size * cachep->nr_allocated;  in kmem_cache_get_alloc()
54 unsigned long kmem_cache_nr_allocated(struct kmem_cache *cachep)  in kmem_cache_nr_allocated()  argument
56 	return cachep->nr_allocated;  in kmem_cache_nr_allocated()
59 unsigned long kmem_cache_nr_tallocated(struct kmem_cache *cachep)  in kmem_cache_nr_tallocated()  argument
61 	return cachep->nr_tallocated;  in kmem_cache_nr_tallocated()
64 void kmem_cache_zero_nr_tallocated(struct kmem_cache *cachep)  in kmem_cache_zero_nr_tallocated()  argument
66 	cachep->nr_tallocated = 0;  in kmem_cache_zero_nr_tallocated()
69 void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru,  in kmem_cache_alloc_lru()  argument
74 	if (cachep->exec_callback) {  in kmem_cache_alloc_lru()
75 		if (cachep->callback)  in kmem_cache_alloc_lru()
76 			cachep->callback(cachep->private);  in kmem_cache_alloc_lru()
77 		cachep->exec_callback = false;  in kmem_cache_alloc_lru()
81 		if (!cachep->non_kernel) {  in kmem_cache_alloc_lru()
82 			cachep->exec_callback = true;  in kmem_cache_alloc_lru()
86 		cachep->non_kernel--;  in kmem_cache_alloc_lru()
89 	pthread_mutex_lock(&cachep->lock);  in kmem_cache_alloc_lru()
90 	if (cachep->nr_objs) {  in kmem_cache_alloc_lru()
91 		struct radix_tree_node *node = cachep->objs;  in kmem_cache_alloc_lru()
92 		cachep->nr_objs--;  in kmem_cache_alloc_lru()
93 		cachep->objs = node->parent;  in kmem_cache_alloc_lru()
94 		pthread_mutex_unlock(&cachep->lock);  in kmem_cache_alloc_lru()
98 		pthread_mutex_unlock(&cachep->lock);  in kmem_cache_alloc_lru()
99 		if (cachep->align) {  in kmem_cache_alloc_lru()
100 			if (posix_memalign(&p, cachep->align, cachep->size) < 0)  in kmem_cache_alloc_lru()
103 			p = malloc(cachep->size);  in kmem_cache_alloc_lru()
106 		if (cachep->ctor)  in kmem_cache_alloc_lru()
107 			cachep->ctor(p);  in kmem_cache_alloc_lru()
109 			memset(p, 0, cachep->size);  in kmem_cache_alloc_lru()
112 	uatomic_inc(&cachep->nr_allocated);  in kmem_cache_alloc_lru()
114 	uatomic_inc(&cachep->nr_tallocated);  in kmem_cache_alloc_lru()
120 void __kmem_cache_free_locked(struct kmem_cache *cachep, void *objp)  in __kmem_cache_free_locked()  argument
123 	if (cachep->nr_objs > 10 || cachep->align) {  in __kmem_cache_free_locked()
124 		memset(objp, POISON_FREE, cachep->size);  in __kmem_cache_free_locked()
128 		cachep->nr_objs++;  in __kmem_cache_free_locked()
129 		node->parent = cachep->objs;  in __kmem_cache_free_locked()
130 		cachep->objs = node;  in __kmem_cache_free_locked()
134 void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp)  in kmem_cache_free_locked()  argument
137 	uatomic_dec(&cachep->nr_allocated);  in kmem_cache_free_locked()
140 	__kmem_cache_free_locked(cachep, objp);  in kmem_cache_free_locked()
143 void kmem_cache_free(struct kmem_cache *cachep, void *objp)  in kmem_cache_free()  argument
145 	pthread_mutex_lock(&cachep->lock);  in kmem_cache_free()
146 	kmem_cache_free_locked(cachep, objp);  in kmem_cache_free()
147 	pthread_mutex_unlock(&cachep->lock);  in kmem_cache_free()
150 void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list)  in kmem_cache_free_bulk()  argument
155 	pthread_mutex_lock(&cachep->lock);  in kmem_cache_free_bulk()
157 		kmem_cache_free_locked(cachep, list[i]);  in kmem_cache_free_bulk()
158 	pthread_mutex_unlock(&cachep->lock);  in kmem_cache_free_bulk()
161 void kmem_cache_shrink(struct kmem_cache *cachep)  in kmem_cache_shrink()  argument
165 int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,  in kmem_cache_alloc_bulk()  argument
173 	pthread_mutex_lock(&cachep->lock);  in kmem_cache_alloc_bulk()
174 	if (cachep->nr_objs >= size) {  in kmem_cache_alloc_bulk()
179 				if (!cachep->non_kernel)  in kmem_cache_alloc_bulk()
181 				cachep->non_kernel--;  in kmem_cache_alloc_bulk()
184 			node = cachep->objs;  in kmem_cache_alloc_bulk()
185 			cachep->nr_objs--;  in kmem_cache_alloc_bulk()
186 			cachep->objs = node->parent;  in kmem_cache_alloc_bulk()
190 		pthread_mutex_unlock(&cachep->lock);  in kmem_cache_alloc_bulk()
192 		pthread_mutex_unlock(&cachep->lock);  in kmem_cache_alloc_bulk()
195 				if (!cachep->non_kernel)  in kmem_cache_alloc_bulk()
197 				cachep->non_kernel--;  in kmem_cache_alloc_bulk()
200 			if (cachep->align) {  in kmem_cache_alloc_bulk()
201 				if (posix_memalign(&p[i], cachep->align,  in kmem_cache_alloc_bulk()
202 					       cachep->size) < 0)  in kmem_cache_alloc_bulk()
205 				p[i] = malloc(cachep->size);  in kmem_cache_alloc_bulk()
209 			if (cachep->ctor)  in kmem_cache_alloc_bulk()
210 				cachep->ctor(p[i]);  in kmem_cache_alloc_bulk()
212 				memset(p[i], 0, cachep->size);  in kmem_cache_alloc_bulk()
218 		pthread_mutex_lock(&cachep->lock);  in kmem_cache_alloc_bulk()
220 			__kmem_cache_free_locked(cachep, p[i]);  in kmem_cache_alloc_bulk()
221 		pthread_mutex_unlock(&cachep->lock);  in kmem_cache_alloc_bulk()
227 		uatomic_inc(&cachep->nr_allocated);  in kmem_cache_alloc_bulk()
228 		uatomic_inc(&cachep->nr_tallocated);  in kmem_cache_alloc_bulk()