1 #ifndef IOU_ALLOC_CACHE_H 2 #define IOU_ALLOC_CACHE_H 3 4 #include <linux/io_uring_types.h> 5 6 /* 7 * Don't allow the cache to grow beyond this size. 8 */ 9 #define IO_ALLOC_CACHE_MAX 128 10 11 static inline void io_alloc_cache_kasan(struct iovec **iov, int *nr) 12 { 13 if (IS_ENABLED(CONFIG_KASAN)) { 14 kfree(*iov); 15 *iov = NULL; 16 *nr = 0; 17 } 18 } 19 20 static inline bool io_alloc_cache_put(struct io_alloc_cache *cache, 21 void *entry) 22 { 23 if (cache->nr_cached < cache->max_cached) { 24 if (!kasan_mempool_poison_object(entry)) 25 return false; 26 cache->entries[cache->nr_cached++] = entry; 27 return true; 28 } 29 return false; 30 } 31 32 static inline void *io_alloc_cache_get(struct io_alloc_cache *cache) 33 { 34 if (cache->nr_cached) { 35 void *entry = cache->entries[--cache->nr_cached]; 36 37 /* 38 * If KASAN is enabled, always clear the initial bytes that 39 * must be zeroed post alloc, in case any of them overlap 40 * with KASAN storage. 41 */ 42 #if defined(CONFIG_KASAN) 43 kasan_mempool_unpoison_object(entry, cache->elem_size); 44 if (cache->init_clear) 45 memset(entry, 0, cache->init_clear); 46 #endif 47 return entry; 48 } 49 50 return NULL; 51 } 52 53 static inline void *io_cache_alloc(struct io_alloc_cache *cache, gfp_t gfp) 54 { 55 void *obj; 56 57 obj = io_alloc_cache_get(cache); 58 if (obj) 59 return obj; 60 61 obj = kmalloc(cache->elem_size, gfp); 62 if (obj && cache->init_clear) 63 memset(obj, 0, cache->init_clear); 64 return obj; 65 } 66 67 /* returns false if the cache was initialized properly */ 68 static inline bool io_alloc_cache_init(struct io_alloc_cache *cache, 69 unsigned max_nr, unsigned int size, 70 unsigned int init_bytes) 71 { 72 cache->entries = kvmalloc_array(max_nr, sizeof(void *), GFP_KERNEL); 73 if (cache->entries) { 74 cache->nr_cached = 0; 75 cache->max_cached = max_nr; 76 cache->elem_size = size; 77 cache->init_clear = init_bytes; 78 return false; 79 } 80 return true; 81 } 82 83 static inline void io_alloc_cache_free(struct io_alloc_cache *cache, 84 void (*free)(const void *)) 85 { 86 void *entry; 87 88 if (!cache->entries) 89 return; 90 91 while ((entry = io_alloc_cache_get(cache)) != NULL) 92 free(entry); 93 94 kvfree(cache->entries); 95 cache->entries = NULL; 96 } 97 #endif 98