1 /*-
2 * Copyright (c) 2017 Mellanox Technologies, Ltd.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29
30 #include <linux/slab.h>
31 #include <linux/rcupdate.h>
32 #include <linux/kernel.h>
33
34 struct linux_kmem_rcu {
35 struct rcu_head rcu_head;
36 struct linux_kmem_cache *cache;
37 };
38
39 #define LINUX_KMEM_TO_RCU(c, m) \
40 ((struct linux_kmem_rcu *)((char *)(m) + \
41 (c)->cache_size - sizeof(struct linux_kmem_rcu)))
42
43 #define LINUX_RCU_TO_KMEM(r) \
44 ((void *)((char *)(r) + sizeof(struct linux_kmem_rcu) - \
45 (r)->cache->cache_size))
46
47 static int
linux_kmem_ctor(void * mem,int size,void * arg,int flags)48 linux_kmem_ctor(void *mem, int size, void *arg, int flags)
49 {
50 struct linux_kmem_cache *c = arg;
51
52 if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU)) {
53 struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, mem);
54
55 /* duplicate cache pointer */
56 rcu->cache = c;
57 }
58
59 /* check for constructor */
60 if (likely(c->cache_ctor != NULL))
61 c->cache_ctor(mem);
62
63 return (0);
64 }
65
66 static void
linux_kmem_cache_free_rcu_callback(struct rcu_head * head)67 linux_kmem_cache_free_rcu_callback(struct rcu_head *head)
68 {
69 struct linux_kmem_rcu *rcu =
70 container_of(head, struct linux_kmem_rcu, rcu_head);
71
72 uma_zfree(rcu->cache->cache_zone, LINUX_RCU_TO_KMEM(rcu));
73 }
74
75 struct linux_kmem_cache *
linux_kmem_cache_create(const char * name,size_t size,size_t align,unsigned flags,linux_kmem_ctor_t * ctor)76 linux_kmem_cache_create(const char *name, size_t size, size_t align,
77 unsigned flags, linux_kmem_ctor_t *ctor)
78 {
79 struct linux_kmem_cache *c;
80
81 c = malloc(sizeof(*c), M_KMALLOC, M_WAITOK);
82
83 if (flags & SLAB_HWCACHE_ALIGN)
84 align = UMA_ALIGN_CACHE;
85 else if (align != 0)
86 align--;
87
88 if (flags & SLAB_TYPESAFE_BY_RCU) {
89 /* make room for RCU structure */
90 size = ALIGN(size, sizeof(void *));
91 size += sizeof(struct linux_kmem_rcu);
92
93 /* create cache_zone */
94 c->cache_zone = uma_zcreate(name, size,
95 linux_kmem_ctor, NULL, NULL, NULL,
96 align, UMA_ZONE_ZINIT);
97 } else {
98 /* create cache_zone */
99 c->cache_zone = uma_zcreate(name, size,
100 ctor ? linux_kmem_ctor : NULL, NULL,
101 NULL, NULL, align, 0);
102 }
103
104 c->cache_flags = flags;
105 c->cache_ctor = ctor;
106 c->cache_size = size;
107 return (c);
108 }
109
110 void
linux_kmem_cache_free_rcu(struct linux_kmem_cache * c,void * m)111 linux_kmem_cache_free_rcu(struct linux_kmem_cache *c, void *m)
112 {
113 struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, m);
114
115 call_rcu(&rcu->rcu_head, linux_kmem_cache_free_rcu_callback);
116 }
117
118 void
linux_kmem_cache_destroy(struct linux_kmem_cache * c)119 linux_kmem_cache_destroy(struct linux_kmem_cache *c)
120 {
121 if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU)) {
122 /* make sure all free callbacks have been called */
123 rcu_barrier();
124 }
125
126 uma_zdestroy(c->cache_zone);
127 free(c, M_KMALLOC);
128 }
129