1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
3 */
4
5 #include <string.h>
6
7 #include <rte_errno.h>
8 #include <rte_lcore.h>
9 #include <rte_fbarray.h>
10 #include <rte_memzone.h>
11 #include <rte_memory.h>
12 #include <rte_string_fns.h>
13 #include <rte_rwlock.h>
14
15 #include "eal_private.h"
16 #include "eal_internal_cfg.h"
17 #include "eal_memalloc.h"
18
19 struct mem_event_callback_entry {
20 TAILQ_ENTRY(mem_event_callback_entry) next;
21 char name[RTE_MEM_EVENT_CALLBACK_NAME_LEN];
22 rte_mem_event_callback_t clb;
23 void *arg;
24 };
25
26 struct mem_alloc_validator_entry {
27 TAILQ_ENTRY(mem_alloc_validator_entry) next;
28 char name[RTE_MEM_ALLOC_VALIDATOR_NAME_LEN];
29 rte_mem_alloc_validator_t clb;
30 int socket_id;
31 size_t limit;
32 };
33
34 /** Double linked list of actions. */
35 TAILQ_HEAD(mem_event_callback_entry_list, mem_event_callback_entry);
36 TAILQ_HEAD(mem_alloc_validator_entry_list, mem_alloc_validator_entry);
37
38 static struct mem_event_callback_entry_list mem_event_callback_list =
39 TAILQ_HEAD_INITIALIZER(mem_event_callback_list);
40 static rte_rwlock_t mem_event_rwlock = RTE_RWLOCK_INITIALIZER;
41
42 static struct mem_alloc_validator_entry_list mem_alloc_validator_list =
43 TAILQ_HEAD_INITIALIZER(mem_alloc_validator_list);
44 static rte_rwlock_t mem_alloc_validator_rwlock = RTE_RWLOCK_INITIALIZER;
45
46 static struct mem_event_callback_entry *
find_mem_event_callback(const char * name,void * arg)47 find_mem_event_callback(const char *name, void *arg)
48 {
49 struct mem_event_callback_entry *r;
50
51 TAILQ_FOREACH(r, &mem_event_callback_list, next) {
52 if (!strcmp(r->name, name) && r->arg == arg)
53 break;
54 }
55 return r;
56 }
57
58 static struct mem_alloc_validator_entry *
find_mem_alloc_validator(const char * name,int socket_id)59 find_mem_alloc_validator(const char *name, int socket_id)
60 {
61 struct mem_alloc_validator_entry *r;
62
63 TAILQ_FOREACH(r, &mem_alloc_validator_list, next) {
64 if (!strcmp(r->name, name) && r->socket_id == socket_id)
65 break;
66 }
67 return r;
68 }
69
70 bool
eal_memalloc_is_contig(const struct rte_memseg_list * msl,void * start,size_t len)71 eal_memalloc_is_contig(const struct rte_memseg_list *msl, void *start,
72 size_t len)
73 {
74 void *end, *aligned_start, *aligned_end;
75 size_t pgsz = (size_t)msl->page_sz;
76 const struct rte_memseg *ms;
77 const struct internal_config *internal_conf =
78 eal_get_internal_configuration();
79
80 /* for IOVA_VA, it's always contiguous */
81 if (rte_eal_iova_mode() == RTE_IOVA_VA && !msl->external)
82 return true;
83
84 /* for legacy memory, it's always contiguous */
85 if (internal_conf->legacy_mem)
86 return true;
87
88 end = RTE_PTR_ADD(start, len);
89
90 /* for nohuge, we check pagemap, otherwise check memseg */
91 if (!rte_eal_has_hugepages()) {
92 rte_iova_t cur, expected;
93
94 aligned_start = RTE_PTR_ALIGN_FLOOR(start, pgsz);
95 aligned_end = RTE_PTR_ALIGN_CEIL(end, pgsz);
96
97 /* if start and end are on the same page, bail out early */
98 if (RTE_PTR_DIFF(aligned_end, aligned_start) == pgsz)
99 return true;
100
101 /* skip first iteration */
102 cur = rte_mem_virt2iova(aligned_start);
103 expected = cur + pgsz;
104 aligned_start = RTE_PTR_ADD(aligned_start, pgsz);
105
106 while (aligned_start < aligned_end) {
107 cur = rte_mem_virt2iova(aligned_start);
108 if (cur != expected)
109 return false;
110 aligned_start = RTE_PTR_ADD(aligned_start, pgsz);
111 expected += pgsz;
112 }
113 } else {
114 int start_seg, end_seg, cur_seg;
115 rte_iova_t cur, expected;
116
117 aligned_start = RTE_PTR_ALIGN_FLOOR(start, pgsz);
118 aligned_end = RTE_PTR_ALIGN_CEIL(end, pgsz);
119
120 start_seg = RTE_PTR_DIFF(aligned_start, msl->base_va) /
121 pgsz;
122 end_seg = RTE_PTR_DIFF(aligned_end, msl->base_va) /
123 pgsz;
124
125 /* if start and end are on the same page, bail out early */
126 if (RTE_PTR_DIFF(aligned_end, aligned_start) == pgsz)
127 return true;
128
129 /* skip first iteration */
130 ms = rte_fbarray_get(&msl->memseg_arr, start_seg);
131 cur = ms->iova;
132 expected = cur + pgsz;
133
134 /* if we can't access IOVA addresses, assume non-contiguous */
135 if (cur == RTE_BAD_IOVA)
136 return false;
137
138 for (cur_seg = start_seg + 1; cur_seg < end_seg;
139 cur_seg++, expected += pgsz) {
140 ms = rte_fbarray_get(&msl->memseg_arr, cur_seg);
141
142 if (ms->iova != expected)
143 return false;
144 }
145 }
146 return true;
147 }
148
149 int
eal_memalloc_mem_event_callback_register(const char * name,rte_mem_event_callback_t clb,void * arg)150 eal_memalloc_mem_event_callback_register(const char *name,
151 rte_mem_event_callback_t clb, void *arg)
152 {
153 struct mem_event_callback_entry *entry;
154 int ret, len;
155 if (name == NULL || clb == NULL) {
156 rte_errno = EINVAL;
157 return -1;
158 }
159 len = strnlen(name, RTE_MEM_EVENT_CALLBACK_NAME_LEN);
160 if (len == 0) {
161 rte_errno = EINVAL;
162 return -1;
163 } else if (len == RTE_MEM_EVENT_CALLBACK_NAME_LEN) {
164 rte_errno = ENAMETOOLONG;
165 return -1;
166 }
167 rte_rwlock_write_lock(&mem_event_rwlock);
168
169 entry = find_mem_event_callback(name, arg);
170 if (entry != NULL) {
171 rte_errno = EEXIST;
172 ret = -1;
173 goto unlock;
174 }
175
176 entry = malloc(sizeof(*entry));
177 if (entry == NULL) {
178 rte_errno = ENOMEM;
179 ret = -1;
180 goto unlock;
181 }
182
183 /* callback successfully created and is valid, add it to the list */
184 entry->clb = clb;
185 entry->arg = arg;
186 strlcpy(entry->name, name, RTE_MEM_EVENT_CALLBACK_NAME_LEN);
187 TAILQ_INSERT_TAIL(&mem_event_callback_list, entry, next);
188
189 ret = 0;
190
191 RTE_LOG(DEBUG, EAL, "Mem event callback '%s:%p' registered\n",
192 name, arg);
193
194 unlock:
195 rte_rwlock_write_unlock(&mem_event_rwlock);
196 return ret;
197 }
198
199 int
eal_memalloc_mem_event_callback_unregister(const char * name,void * arg)200 eal_memalloc_mem_event_callback_unregister(const char *name, void *arg)
201 {
202 struct mem_event_callback_entry *entry;
203 int ret, len;
204
205 if (name == NULL) {
206 rte_errno = EINVAL;
207 return -1;
208 }
209 len = strnlen(name, RTE_MEM_EVENT_CALLBACK_NAME_LEN);
210 if (len == 0) {
211 rte_errno = EINVAL;
212 return -1;
213 } else if (len == RTE_MEM_EVENT_CALLBACK_NAME_LEN) {
214 rte_errno = ENAMETOOLONG;
215 return -1;
216 }
217 rte_rwlock_write_lock(&mem_event_rwlock);
218
219 entry = find_mem_event_callback(name, arg);
220 if (entry == NULL) {
221 rte_errno = ENOENT;
222 ret = -1;
223 goto unlock;
224 }
225 TAILQ_REMOVE(&mem_event_callback_list, entry, next);
226 free(entry);
227
228 ret = 0;
229
230 RTE_LOG(DEBUG, EAL, "Mem event callback '%s:%p' unregistered\n",
231 name, arg);
232
233 unlock:
234 rte_rwlock_write_unlock(&mem_event_rwlock);
235 return ret;
236 }
237
238 void
eal_memalloc_mem_event_notify(enum rte_mem_event event,const void * start,size_t len)239 eal_memalloc_mem_event_notify(enum rte_mem_event event, const void *start,
240 size_t len)
241 {
242 struct mem_event_callback_entry *entry;
243
244 rte_rwlock_read_lock(&mem_event_rwlock);
245
246 TAILQ_FOREACH(entry, &mem_event_callback_list, next) {
247 RTE_LOG(DEBUG, EAL, "Calling mem event callback '%s:%p'\n",
248 entry->name, entry->arg);
249 entry->clb(event, start, len, entry->arg);
250 }
251
252 rte_rwlock_read_unlock(&mem_event_rwlock);
253 }
254
255 int
eal_memalloc_mem_alloc_validator_register(const char * name,rte_mem_alloc_validator_t clb,int socket_id,size_t limit)256 eal_memalloc_mem_alloc_validator_register(const char *name,
257 rte_mem_alloc_validator_t clb, int socket_id, size_t limit)
258 {
259 struct mem_alloc_validator_entry *entry;
260 int ret, len;
261 if (name == NULL || clb == NULL || socket_id < 0) {
262 rte_errno = EINVAL;
263 return -1;
264 }
265 len = strnlen(name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN);
266 if (len == 0) {
267 rte_errno = EINVAL;
268 return -1;
269 } else if (len == RTE_MEM_ALLOC_VALIDATOR_NAME_LEN) {
270 rte_errno = ENAMETOOLONG;
271 return -1;
272 }
273 rte_rwlock_write_lock(&mem_alloc_validator_rwlock);
274
275 entry = find_mem_alloc_validator(name, socket_id);
276 if (entry != NULL) {
277 rte_errno = EEXIST;
278 ret = -1;
279 goto unlock;
280 }
281
282 entry = malloc(sizeof(*entry));
283 if (entry == NULL) {
284 rte_errno = ENOMEM;
285 ret = -1;
286 goto unlock;
287 }
288
289 /* callback successfully created and is valid, add it to the list */
290 entry->clb = clb;
291 entry->socket_id = socket_id;
292 entry->limit = limit;
293 strlcpy(entry->name, name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN);
294 TAILQ_INSERT_TAIL(&mem_alloc_validator_list, entry, next);
295
296 ret = 0;
297
298 RTE_LOG(DEBUG, EAL, "Mem alloc validator '%s' on socket %i with limit %zu registered\n",
299 name, socket_id, limit);
300
301 unlock:
302 rte_rwlock_write_unlock(&mem_alloc_validator_rwlock);
303 return ret;
304 }
305
306 int
eal_memalloc_mem_alloc_validator_unregister(const char * name,int socket_id)307 eal_memalloc_mem_alloc_validator_unregister(const char *name, int socket_id)
308 {
309 struct mem_alloc_validator_entry *entry;
310 int ret, len;
311
312 if (name == NULL || socket_id < 0) {
313 rte_errno = EINVAL;
314 return -1;
315 }
316 len = strnlen(name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN);
317 if (len == 0) {
318 rte_errno = EINVAL;
319 return -1;
320 } else if (len == RTE_MEM_ALLOC_VALIDATOR_NAME_LEN) {
321 rte_errno = ENAMETOOLONG;
322 return -1;
323 }
324 rte_rwlock_write_lock(&mem_alloc_validator_rwlock);
325
326 entry = find_mem_alloc_validator(name, socket_id);
327 if (entry == NULL) {
328 rte_errno = ENOENT;
329 ret = -1;
330 goto unlock;
331 }
332 TAILQ_REMOVE(&mem_alloc_validator_list, entry, next);
333 free(entry);
334
335 ret = 0;
336
337 RTE_LOG(DEBUG, EAL, "Mem alloc validator '%s' on socket %i unregistered\n",
338 name, socket_id);
339
340 unlock:
341 rte_rwlock_write_unlock(&mem_alloc_validator_rwlock);
342 return ret;
343 }
344
345 int
eal_memalloc_mem_alloc_validate(int socket_id,size_t new_len)346 eal_memalloc_mem_alloc_validate(int socket_id, size_t new_len)
347 {
348 struct mem_alloc_validator_entry *entry;
349 int ret = 0;
350
351 rte_rwlock_read_lock(&mem_alloc_validator_rwlock);
352
353 TAILQ_FOREACH(entry, &mem_alloc_validator_list, next) {
354 if (entry->socket_id != socket_id || entry->limit > new_len)
355 continue;
356 RTE_LOG(DEBUG, EAL, "Calling mem alloc validator '%s' on socket %i\n",
357 entry->name, entry->socket_id);
358 if (entry->clb(socket_id, entry->limit, new_len) < 0)
359 ret = -1;
360 }
361
362 rte_rwlock_read_unlock(&mem_alloc_validator_rwlock);
363
364 return ret;
365 }
366