1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies, Ltd
3 */
4
5 #include <errno.h>
6 #include <rte_malloc.h>
7 #include <malloc.h>
8 #include <stdbool.h>
9 #include <string.h>
10
11 #include "mlx5_common_utils.h"
12 #include "mlx5_malloc.h"
13
14 struct mlx5_sys_mem {
15 uint32_t init:1; /* Memory allocator initialized. */
16 uint32_t enable:1; /* System memory select. */
17 uint32_t reserve:30; /* Reserve. */
18 struct rte_memseg_list *last_msl;
19 /* last allocated rte memory memseg list. */
20 #ifdef RTE_LIBRTE_MLX5_DEBUG
21 uint64_t malloc_sys;
22 /* Memory allocated from system count. */
23 uint64_t malloc_rte;
24 /* Memory allocated from hugepage count. */
25 uint64_t realloc_sys;
26 /* Memory reallocate from system count. */
27 uint64_t realloc_rte;
28 /* Memory reallocate from hugepage count. */
29 uint64_t free_sys;
30 /* Memory free to system count. */
31 uint64_t free_rte;
32 /* Memory free to hugepage count. */
33 uint64_t msl_miss;
34 /* MSL miss count. */
35 uint64_t msl_update;
36 /* MSL update count. */
37 #endif
38 };
39
40 /* Initialize default as not */
41 static struct mlx5_sys_mem mlx5_sys_mem = {
42 .init = 0,
43 .enable = 0,
44 #ifdef RTE_LIBRTE_MLX5_DEBUG
45 .malloc_sys = 0,
46 .malloc_rte = 0,
47 .realloc_sys = 0,
48 .realloc_rte = 0,
49 .free_sys = 0,
50 .free_rte = 0,
51 .msl_miss = 0,
52 .msl_update = 0,
53 #endif
54 };
55
56 /**
57 * Check if the address belongs to memory seg list.
58 *
59 * @param addr
60 * Memory address to be ckeced.
61 * @param msl
62 * Memory seg list.
63 *
64 * @return
65 * True if it belongs, false otherwise.
66 */
67 static bool
mlx5_mem_check_msl(void * addr,struct rte_memseg_list * msl)68 mlx5_mem_check_msl(void *addr, struct rte_memseg_list *msl)
69 {
70 void *start, *end;
71
72 if (!msl)
73 return false;
74 start = msl->base_va;
75 end = RTE_PTR_ADD(start, msl->len);
76 if (addr >= start && addr < end)
77 return true;
78 return false;
79 }
80
81 /**
82 * Update the msl if memory belongs to new msl.
83 *
84 * @param addr
85 * Memory address.
86 */
87 static void
mlx5_mem_update_msl(void * addr)88 mlx5_mem_update_msl(void *addr)
89 {
90 /*
91 * Update the cache msl if the new addr comes from the new msl
92 * different with the cached msl.
93 */
94 if (addr && !mlx5_mem_check_msl(addr,
95 (struct rte_memseg_list *)__atomic_load_n
96 (&mlx5_sys_mem.last_msl, __ATOMIC_RELAXED))) {
97 __atomic_store_n(&mlx5_sys_mem.last_msl,
98 rte_mem_virt2memseg_list(addr),
99 __ATOMIC_RELAXED);
100 #ifdef RTE_LIBRTE_MLX5_DEBUG
101 __atomic_add_fetch(&mlx5_sys_mem.msl_update, 1,
102 __ATOMIC_RELAXED);
103 #endif
104 }
105 }
106
107 /**
108 * Check if the address belongs to rte memory.
109 *
110 * @param addr
111 * Memory address to be ckeced.
112 *
113 * @return
114 * True if it belongs, false otherwise.
115 */
116 static bool
mlx5_mem_is_rte(void * addr)117 mlx5_mem_is_rte(void *addr)
118 {
119 /*
120 * Check if the last cache msl matches. Drop to slow path
121 * to check if the memory belongs to rte memory.
122 */
123 if (!mlx5_mem_check_msl(addr, (struct rte_memseg_list *)
124 __atomic_load_n(&mlx5_sys_mem.last_msl, __ATOMIC_RELAXED))) {
125 if (!rte_mem_virt2memseg_list(addr))
126 return false;
127 #ifdef RTE_LIBRTE_MLX5_DEBUG
128 __atomic_add_fetch(&mlx5_sys_mem.msl_miss, 1, __ATOMIC_RELAXED);
129 #endif
130 }
131 return true;
132 }
133
134 /**
135 * Allocate memory with alignment.
136 *
137 * @param size
138 * Memory size to be allocated.
139 * @param align
140 * Memory alignment.
141 * @param zero
142 * Clear the allocated memory or not.
143 *
144 * @return
145 * Pointer of the allocated memory, NULL otherwise.
146 */
147 static void *
mlx5_alloc_align(size_t size,unsigned int align,unsigned int zero)148 mlx5_alloc_align(size_t size, unsigned int align, unsigned int zero)
149 {
150 void *buf;
151 int ret;
152
153 ret = posix_memalign(&buf, align, size);
154 if (ret) {
155 DRV_LOG(ERR,
156 "Couldn't allocate buf size=%zu align=%u. Err=%d\n",
157 size, align, ret);
158
159 return NULL;
160 }
161 if (zero)
162 memset(buf, 0, size);
163 return buf;
164 }
165
166 void *
mlx5_malloc(uint32_t flags,size_t size,unsigned int align,int socket)167 mlx5_malloc(uint32_t flags, size_t size, unsigned int align, int socket)
168 {
169 void *addr;
170 bool rte_mem;
171
172 /*
173 * If neither system memory nor rte memory is required, allocate
174 * memory according to mlx5_sys_mem.enable.
175 */
176 if (flags & MLX5_MEM_RTE)
177 rte_mem = true;
178 else if (flags & MLX5_MEM_SYS)
179 rte_mem = false;
180 else
181 rte_mem = mlx5_sys_mem.enable ? false : true;
182 if (rte_mem) {
183 if (flags & MLX5_MEM_ZERO)
184 addr = rte_zmalloc_socket(NULL, size, align, socket);
185 else
186 addr = rte_malloc_socket(NULL, size, align, socket);
187 mlx5_mem_update_msl(addr);
188 #ifdef RTE_LIBRTE_MLX5_DEBUG
189 if (addr)
190 __atomic_add_fetch(&mlx5_sys_mem.malloc_rte, 1,
191 __ATOMIC_RELAXED);
192 #endif
193 return addr;
194 }
195 /* The memory will be allocated from system. */
196 if (align > MLX5_MALLOC_ALIGNMENT)
197 addr = mlx5_alloc_align(size, align, !!(flags & MLX5_MEM_ZERO));
198 else if (flags & MLX5_MEM_ZERO)
199 addr = calloc(1, size);
200 else
201 addr = malloc(size);
202 #ifdef RTE_LIBRTE_MLX5_DEBUG
203 if (addr)
204 __atomic_add_fetch(&mlx5_sys_mem.malloc_sys, 1,
205 __ATOMIC_RELAXED);
206 #endif
207 return addr;
208 }
209
210 void *
mlx5_realloc(void * addr,uint32_t flags,size_t size,unsigned int align,int socket)211 mlx5_realloc(void *addr, uint32_t flags, size_t size, unsigned int align,
212 int socket)
213 {
214 void *new_addr;
215 bool rte_mem;
216
217 /* Allocate directly if old memory address is NULL. */
218 if (!addr)
219 return mlx5_malloc(flags, size, align, socket);
220 /* Get the memory type. */
221 if (flags & MLX5_MEM_RTE)
222 rte_mem = true;
223 else if (flags & MLX5_MEM_SYS)
224 rte_mem = false;
225 else
226 rte_mem = mlx5_sys_mem.enable ? false : true;
227 /* Check if old memory and to be allocated memory are the same type. */
228 if (rte_mem != mlx5_mem_is_rte(addr)) {
229 DRV_LOG(ERR, "Couldn't reallocate to different memory type.");
230 return NULL;
231 }
232 /* Allocate memory from rte memory. */
233 if (rte_mem) {
234 new_addr = rte_realloc_socket(addr, size, align, socket);
235 mlx5_mem_update_msl(new_addr);
236 #ifdef RTE_LIBRTE_MLX5_DEBUG
237 if (new_addr)
238 __atomic_add_fetch(&mlx5_sys_mem.realloc_rte, 1,
239 __ATOMIC_RELAXED);
240 #endif
241 return new_addr;
242 }
243 /* Align is not supported for system memory. */
244 if (align) {
245 DRV_LOG(ERR, "Couldn't reallocate with alignment");
246 return NULL;
247 }
248 new_addr = realloc(addr, size);
249 #ifdef RTE_LIBRTE_MLX5_DEBUG
250 if (new_addr)
251 __atomic_add_fetch(&mlx5_sys_mem.realloc_sys, 1,
252 __ATOMIC_RELAXED);
253 #endif
254 return new_addr;
255 }
256
257 void
mlx5_free(void * addr)258 mlx5_free(void *addr)
259 {
260 if (addr == NULL)
261 return;
262 if (!mlx5_mem_is_rte(addr)) {
263 #ifdef RTE_LIBRTE_MLX5_DEBUG
264 __atomic_add_fetch(&mlx5_sys_mem.free_sys, 1,
265 __ATOMIC_RELAXED);
266 #endif
267 free(addr);
268 } else {
269 #ifdef RTE_LIBRTE_MLX5_DEBUG
270 __atomic_add_fetch(&mlx5_sys_mem.free_rte, 1,
271 __ATOMIC_RELAXED);
272 #endif
273 rte_free(addr);
274 }
275 }
276
277 void
mlx5_memory_stat_dump(void)278 mlx5_memory_stat_dump(void)
279 {
280 #ifdef RTE_LIBRTE_MLX5_DEBUG
281 DRV_LOG(INFO, "System memory malloc:%"PRIi64", realloc:%"PRIi64","
282 " free:%"PRIi64"\nRTE memory malloc:%"PRIi64","
283 " realloc:%"PRIi64", free:%"PRIi64"\nMSL miss:%"PRIi64","
284 " update:%"PRIi64"",
285 __atomic_load_n(&mlx5_sys_mem.malloc_sys, __ATOMIC_RELAXED),
286 __atomic_load_n(&mlx5_sys_mem.realloc_sys, __ATOMIC_RELAXED),
287 __atomic_load_n(&mlx5_sys_mem.free_sys, __ATOMIC_RELAXED),
288 __atomic_load_n(&mlx5_sys_mem.malloc_rte, __ATOMIC_RELAXED),
289 __atomic_load_n(&mlx5_sys_mem.realloc_rte, __ATOMIC_RELAXED),
290 __atomic_load_n(&mlx5_sys_mem.free_rte, __ATOMIC_RELAXED),
291 __atomic_load_n(&mlx5_sys_mem.msl_miss, __ATOMIC_RELAXED),
292 __atomic_load_n(&mlx5_sys_mem.msl_update, __ATOMIC_RELAXED));
293 #endif
294 }
295
296 void
mlx5_malloc_mem_select(uint32_t sys_mem_en)297 mlx5_malloc_mem_select(uint32_t sys_mem_en)
298 {
299 /*
300 * The initialization should be called only once and all devices
301 * should use the same memory type. Otherwise, when new device is
302 * being attached with some different memory allocation configuration,
303 * the memory will get wrong behavior or a failure will be raised.
304 */
305 if (!mlx5_sys_mem.init) {
306 if (sys_mem_en)
307 mlx5_sys_mem.enable = 1;
308 mlx5_sys_mem.init = 1;
309 DRV_LOG(INFO, "%s is selected.", sys_mem_en ? "SYS_MEM" : "RTE_MEM");
310 } else if (mlx5_sys_mem.enable != sys_mem_en) {
311 DRV_LOG(WARNING, "%s is already selected.",
312 mlx5_sys_mem.enable ? "SYS_MEM" : "RTE_MEM");
313 }
314 }
315