1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2019 6WIND S.A.
3 */
4
5 #include <sys/queue.h>
6 #include <stdint.h>
7 #include <limits.h>
8
9 #include <rte_common.h>
10 #include <rte_eal.h>
11 #include <rte_eal_memconfig.h>
12 #include <rte_tailq.h>
13 #include <rte_errno.h>
14 #include <rte_malloc.h>
15 #include <rte_string_fns.h>
16 #include <rte_bitops.h>
17 #include <rte_mbuf.h>
18 #include <rte_mbuf_dyn.h>
19
20 #define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn"
21
22 struct mbuf_dynfield_elt {
23 struct rte_mbuf_dynfield params;
24 size_t offset;
25 };
26 TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry);
27
28 static struct rte_tailq_elem mbuf_dynfield_tailq = {
29 .name = "RTE_MBUF_DYNFIELD",
30 };
31 EAL_REGISTER_TAILQ(mbuf_dynfield_tailq);
32
33 struct mbuf_dynflag_elt {
34 struct rte_mbuf_dynflag params;
35 unsigned int bitnum;
36 };
37 TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry);
38
39 static struct rte_tailq_elem mbuf_dynflag_tailq = {
40 .name = "RTE_MBUF_DYNFLAG",
41 };
42 EAL_REGISTER_TAILQ(mbuf_dynflag_tailq);
43
44 struct mbuf_dyn_shm {
45 /**
46 * For each mbuf byte, free_space[i] != 0 if space is free.
47 * The value is the size of the biggest aligned element that
48 * can fit in the zone.
49 */
50 uint8_t free_space[sizeof(struct rte_mbuf)];
51 /** Bitfield of available flags. */
52 uint64_t free_flags;
53 };
54 static struct mbuf_dyn_shm *shm;
55
56 /* Set the value of free_space[] according to the size and alignment of
57 * the free areas. This helps to select the best place when reserving a
58 * dynamic field. Assume tailq is locked.
59 */
60 static void
process_score(void)61 process_score(void)
62 {
63 size_t off, align, size, i;
64
65 /* first, erase previous info */
66 for (i = 0; i < sizeof(struct rte_mbuf); i++) {
67 if (shm->free_space[i])
68 shm->free_space[i] = 1;
69 }
70
71 off = 0;
72 while (off < sizeof(struct rte_mbuf)) {
73 /* get the size of the free zone */
74 for (size = 0; (off + size) < sizeof(struct rte_mbuf) &&
75 shm->free_space[off + size]; size++)
76 ;
77 if (size == 0) {
78 off++;
79 continue;
80 }
81
82 /* get the alignment of biggest object that can fit in
83 * the zone at this offset.
84 */
85 for (align = 1;
86 (off % (align << 1)) == 0 && (align << 1) <= size;
87 align <<= 1)
88 ;
89
90 /* save it in free_space[] */
91 for (i = off; i < off + align; i++)
92 shm->free_space[i] = RTE_MAX(align, shm->free_space[i]);
93
94 off += align;
95 }
96 }
97
98 /* Mark the area occupied by a mbuf field as available in the shm. */
99 #define mark_free(field) \
100 memset(&shm->free_space[offsetof(struct rte_mbuf, field)], \
101 1, sizeof(((struct rte_mbuf *)0)->field))
102
103 /* Allocate and initialize the shared memory. Assume tailq is locked */
104 static int
init_shared_mem(void)105 init_shared_mem(void)
106 {
107 const struct rte_memzone *mz;
108 uint64_t mask;
109
110 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
111 mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME,
112 sizeof(struct mbuf_dyn_shm),
113 SOCKET_ID_ANY, 0,
114 RTE_CACHE_LINE_SIZE);
115 } else {
116 mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME);
117 }
118 if (mz == NULL) {
119 RTE_LOG(ERR, MBUF, "Failed to get mbuf dyn shared memory\n");
120 return -1;
121 }
122
123 shm = mz->addr;
124
125 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
126 /* init free_space, keep it sync'd with
127 * rte_mbuf_dynfield_copy().
128 */
129 memset(shm, 0, sizeof(*shm));
130 mark_free(dynfield1);
131
132 /* init free_flags */
133 for (mask = RTE_MBUF_F_FIRST_FREE; mask <= RTE_MBUF_F_LAST_FREE; mask <<= 1)
134 shm->free_flags |= mask;
135
136 process_score();
137 }
138
139 return 0;
140 }
141
142 /* check if this offset can be used */
143 static int
check_offset(size_t offset,size_t size,size_t align)144 check_offset(size_t offset, size_t size, size_t align)
145 {
146 size_t i;
147
148 if ((offset & (align - 1)) != 0)
149 return -1;
150 if (offset + size > sizeof(struct rte_mbuf))
151 return -1;
152
153 for (i = 0; i < size; i++) {
154 if (!shm->free_space[i + offset])
155 return -1;
156 }
157
158 return 0;
159 }
160
161 /* assume tailq is locked */
162 static struct mbuf_dynfield_elt *
__mbuf_dynfield_lookup(const char * name)163 __mbuf_dynfield_lookup(const char *name)
164 {
165 struct mbuf_dynfield_list *mbuf_dynfield_list;
166 struct mbuf_dynfield_elt *mbuf_dynfield;
167 struct rte_tailq_entry *te;
168
169 mbuf_dynfield_list = RTE_TAILQ_CAST(
170 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
171
172 TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
173 mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data;
174 if (strcmp(name, mbuf_dynfield->params.name) == 0)
175 break;
176 }
177
178 if (te == NULL || mbuf_dynfield == NULL) {
179 rte_errno = ENOENT;
180 return NULL;
181 }
182
183 return mbuf_dynfield;
184 }
185
186 int
rte_mbuf_dynfield_lookup(const char * name,struct rte_mbuf_dynfield * params)187 rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params)
188 {
189 struct mbuf_dynfield_elt *mbuf_dynfield;
190
191 rte_mcfg_tailq_read_lock();
192 if (shm == NULL && init_shared_mem() < 0)
193 mbuf_dynfield = NULL;
194 else
195 mbuf_dynfield = __mbuf_dynfield_lookup(name);
196 rte_mcfg_tailq_read_unlock();
197
198 if (mbuf_dynfield == NULL)
199 return -1;
200
201 if (params != NULL)
202 memcpy(params, &mbuf_dynfield->params, sizeof(*params));
203
204 return mbuf_dynfield->offset;
205 }
206
mbuf_dynfield_cmp(const struct rte_mbuf_dynfield * params1,const struct rte_mbuf_dynfield * params2)207 static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1,
208 const struct rte_mbuf_dynfield *params2)
209 {
210 if (strcmp(params1->name, params2->name))
211 return -1;
212 if (params1->size != params2->size)
213 return -1;
214 if (params1->align != params2->align)
215 return -1;
216 if (params1->flags != params2->flags)
217 return -1;
218 return 0;
219 }
220
221 /* assume tailq is locked */
222 static int
__rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield * params,size_t req)223 __rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
224 size_t req)
225 {
226 struct mbuf_dynfield_list *mbuf_dynfield_list;
227 struct mbuf_dynfield_elt *mbuf_dynfield = NULL;
228 struct rte_tailq_entry *te = NULL;
229 unsigned int best_zone = UINT_MAX;
230 size_t i, offset;
231 int ret;
232
233 if (shm == NULL && init_shared_mem() < 0)
234 return -1;
235
236 mbuf_dynfield = __mbuf_dynfield_lookup(params->name);
237 if (mbuf_dynfield != NULL) {
238 if (req != SIZE_MAX && req != mbuf_dynfield->offset) {
239 rte_errno = EEXIST;
240 return -1;
241 }
242 if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) {
243 rte_errno = EEXIST;
244 return -1;
245 }
246 return mbuf_dynfield->offset;
247 }
248
249 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
250 rte_errno = EPERM;
251 return -1;
252 }
253
254 if (req == SIZE_MAX) {
255 /* Find the best place to put this field: we search the
256 * lowest value of shm->free_space[offset]: the zones
257 * containing room for larger fields are kept for later.
258 */
259 for (offset = 0;
260 offset < sizeof(struct rte_mbuf);
261 offset++) {
262 if (check_offset(offset, params->size,
263 params->align) == 0 &&
264 shm->free_space[offset] < best_zone) {
265 best_zone = shm->free_space[offset];
266 req = offset;
267 }
268 }
269 if (req == SIZE_MAX) {
270 rte_errno = ENOENT;
271 return -1;
272 }
273 } else {
274 if (check_offset(req, params->size, params->align) < 0) {
275 rte_errno = EBUSY;
276 return -1;
277 }
278 }
279
280 offset = req;
281 mbuf_dynfield_list = RTE_TAILQ_CAST(
282 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
283
284 te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0);
285 if (te == NULL) {
286 rte_errno = ENOMEM;
287 return -1;
288 }
289
290 mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0);
291 if (mbuf_dynfield == NULL) {
292 rte_free(te);
293 rte_errno = ENOMEM;
294 return -1;
295 }
296
297 ret = strlcpy(mbuf_dynfield->params.name, params->name,
298 sizeof(mbuf_dynfield->params.name));
299 if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) {
300 rte_errno = ENAMETOOLONG;
301 rte_free(mbuf_dynfield);
302 rte_free(te);
303 return -1;
304 }
305 memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params));
306 mbuf_dynfield->offset = offset;
307 te->data = mbuf_dynfield;
308
309 TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next);
310
311 for (i = offset; i < offset + params->size; i++)
312 shm->free_space[i] = 0;
313 process_score();
314
315 RTE_LOG(DEBUG, MBUF, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd\n",
316 params->name, params->size, params->align, params->flags,
317 offset);
318
319 return offset;
320 }
321
322 int
rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield * params,size_t req)323 rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
324 size_t req)
325 {
326 int ret;
327
328 if (params->size >= sizeof(struct rte_mbuf)) {
329 rte_errno = EINVAL;
330 return -1;
331 }
332 if (!rte_is_power_of_2(params->align)) {
333 rte_errno = EINVAL;
334 return -1;
335 }
336 if (params->flags != 0) {
337 rte_errno = EINVAL;
338 return -1;
339 }
340
341 rte_mcfg_tailq_write_lock();
342 ret = __rte_mbuf_dynfield_register_offset(params, req);
343 rte_mcfg_tailq_write_unlock();
344
345 return ret;
346 }
347
348 int
rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield * params)349 rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params)
350 {
351 return rte_mbuf_dynfield_register_offset(params, SIZE_MAX);
352 }
353
354 /* assume tailq is locked */
355 static struct mbuf_dynflag_elt *
__mbuf_dynflag_lookup(const char * name)356 __mbuf_dynflag_lookup(const char *name)
357 {
358 struct mbuf_dynflag_list *mbuf_dynflag_list;
359 struct mbuf_dynflag_elt *mbuf_dynflag;
360 struct rte_tailq_entry *te;
361
362 mbuf_dynflag_list = RTE_TAILQ_CAST(
363 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
364
365 TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
366 mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data;
367 if (strncmp(name, mbuf_dynflag->params.name,
368 RTE_MBUF_DYN_NAMESIZE) == 0)
369 break;
370 }
371
372 if (te == NULL) {
373 rte_errno = ENOENT;
374 return NULL;
375 }
376
377 return mbuf_dynflag;
378 }
379
380 int
rte_mbuf_dynflag_lookup(const char * name,struct rte_mbuf_dynflag * params)381 rte_mbuf_dynflag_lookup(const char *name,
382 struct rte_mbuf_dynflag *params)
383 {
384 struct mbuf_dynflag_elt *mbuf_dynflag;
385
386 rte_mcfg_tailq_read_lock();
387 if (shm == NULL && init_shared_mem() < 0)
388 mbuf_dynflag = NULL;
389 else
390 mbuf_dynflag = __mbuf_dynflag_lookup(name);
391 rte_mcfg_tailq_read_unlock();
392
393 if (mbuf_dynflag == NULL)
394 return -1;
395
396 if (params != NULL)
397 memcpy(params, &mbuf_dynflag->params, sizeof(*params));
398
399 return mbuf_dynflag->bitnum;
400 }
401
mbuf_dynflag_cmp(const struct rte_mbuf_dynflag * params1,const struct rte_mbuf_dynflag * params2)402 static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1,
403 const struct rte_mbuf_dynflag *params2)
404 {
405 if (strcmp(params1->name, params2->name))
406 return -1;
407 if (params1->flags != params2->flags)
408 return -1;
409 return 0;
410 }
411
412 /* assume tailq is locked */
413 static int
__rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag * params,unsigned int req)414 __rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
415 unsigned int req)
416 {
417 struct mbuf_dynflag_list *mbuf_dynflag_list;
418 struct mbuf_dynflag_elt *mbuf_dynflag = NULL;
419 struct rte_tailq_entry *te = NULL;
420 unsigned int bitnum;
421 int ret;
422
423 if (shm == NULL && init_shared_mem() < 0)
424 return -1;
425
426 mbuf_dynflag = __mbuf_dynflag_lookup(params->name);
427 if (mbuf_dynflag != NULL) {
428 if (req != UINT_MAX && req != mbuf_dynflag->bitnum) {
429 rte_errno = EEXIST;
430 return -1;
431 }
432 if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) {
433 rte_errno = EEXIST;
434 return -1;
435 }
436 return mbuf_dynflag->bitnum;
437 }
438
439 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
440 rte_errno = EPERM;
441 return -1;
442 }
443
444 if (req == UINT_MAX) {
445 if (shm->free_flags == 0) {
446 rte_errno = ENOENT;
447 return -1;
448 }
449 bitnum = rte_bsf64(shm->free_flags);
450 } else {
451 if ((shm->free_flags & (1ULL << req)) == 0) {
452 rte_errno = EBUSY;
453 return -1;
454 }
455 bitnum = req;
456 }
457
458 mbuf_dynflag_list = RTE_TAILQ_CAST(
459 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
460
461 te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0);
462 if (te == NULL) {
463 rte_errno = ENOMEM;
464 return -1;
465 }
466
467 mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0);
468 if (mbuf_dynflag == NULL) {
469 rte_free(te);
470 rte_errno = ENOMEM;
471 return -1;
472 }
473
474 ret = strlcpy(mbuf_dynflag->params.name, params->name,
475 sizeof(mbuf_dynflag->params.name));
476 if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) {
477 rte_free(mbuf_dynflag);
478 rte_free(te);
479 rte_errno = ENAMETOOLONG;
480 return -1;
481 }
482 mbuf_dynflag->bitnum = bitnum;
483 te->data = mbuf_dynflag;
484
485 TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next);
486
487 shm->free_flags &= ~(1ULL << bitnum);
488
489 RTE_LOG(DEBUG, MBUF, "Registered dynamic flag %s (fl=0x%x) -> %u\n",
490 params->name, params->flags, bitnum);
491
492 return bitnum;
493 }
494
495 int
rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag * params,unsigned int req)496 rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
497 unsigned int req)
498 {
499 int ret;
500
501 if (params->flags != 0) {
502 rte_errno = EINVAL;
503 return -1;
504 }
505 if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT &&
506 req != UINT_MAX) {
507 rte_errno = EINVAL;
508 return -1;
509 }
510
511 rte_mcfg_tailq_write_lock();
512 ret = __rte_mbuf_dynflag_register_bitnum(params, req);
513 rte_mcfg_tailq_write_unlock();
514
515 return ret;
516 }
517
518 int
rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag * params)519 rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params)
520 {
521 return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX);
522 }
523
rte_mbuf_dyn_dump(FILE * out)524 void rte_mbuf_dyn_dump(FILE *out)
525 {
526 struct mbuf_dynfield_list *mbuf_dynfield_list;
527 struct mbuf_dynfield_elt *dynfield;
528 struct mbuf_dynflag_list *mbuf_dynflag_list;
529 struct mbuf_dynflag_elt *dynflag;
530 struct rte_tailq_entry *te;
531 size_t i;
532
533 rte_mcfg_tailq_write_lock();
534 if (shm == NULL && init_shared_mem() < 0) {
535 rte_mcfg_tailq_write_unlock();
536 return;
537 }
538
539 fprintf(out, "Reserved fields:\n");
540 mbuf_dynfield_list = RTE_TAILQ_CAST(
541 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
542 TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
543 dynfield = (struct mbuf_dynfield_elt *)te->data;
544 fprintf(out, " name=%s offset=%zd size=%zd align=%zd flags=%x\n",
545 dynfield->params.name, dynfield->offset,
546 dynfield->params.size, dynfield->params.align,
547 dynfield->params.flags);
548 }
549 fprintf(out, "Reserved flags:\n");
550 mbuf_dynflag_list = RTE_TAILQ_CAST(
551 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
552 TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
553 dynflag = (struct mbuf_dynflag_elt *)te->data;
554 fprintf(out, " name=%s bitnum=%u flags=%x\n",
555 dynflag->params.name, dynflag->bitnum,
556 dynflag->params.flags);
557 }
558 fprintf(out, "Free space in mbuf (0 = occupied, value = free zone alignment):\n");
559 for (i = 0; i < sizeof(struct rte_mbuf); i++) {
560 if ((i % 8) == 0)
561 fprintf(out, " %4.4zx: ", i);
562 fprintf(out, "%2.2x%s", shm->free_space[i],
563 (i % 8 != 7) ? " " : "\n");
564 }
565 fprintf(out, "Free bit in mbuf->ol_flags (0 = occupied, 1 = free):\n");
566 for (i = 0; i < sizeof(uint64_t) * CHAR_BIT; i++) {
567 if ((i % 8) == 0)
568 fprintf(out, " %4.4zx: ", i);
569 fprintf(out, "%1.1x%s", (shm->free_flags & (1ULL << i)) ? 1 : 0,
570 (i % 8 != 7) ? " " : "\n");
571 }
572
573 rte_mcfg_tailq_write_unlock();
574 }
575
576 static int
rte_mbuf_dyn_timestamp_register(int * field_offset,uint64_t * flag,const char * direction,const char * flag_name)577 rte_mbuf_dyn_timestamp_register(int *field_offset, uint64_t *flag,
578 const char *direction, const char *flag_name)
579 {
580 static const struct rte_mbuf_dynfield field_desc = {
581 .name = RTE_MBUF_DYNFIELD_TIMESTAMP_NAME,
582 .size = sizeof(rte_mbuf_timestamp_t),
583 .align = __alignof__(rte_mbuf_timestamp_t),
584 };
585 struct rte_mbuf_dynflag flag_desc = {};
586 int offset;
587
588 offset = rte_mbuf_dynfield_register(&field_desc);
589 if (offset < 0) {
590 RTE_LOG(ERR, MBUF,
591 "Failed to register mbuf field for timestamp\n");
592 return -1;
593 }
594 if (field_offset != NULL)
595 *field_offset = offset;
596
597 strlcpy(flag_desc.name, flag_name, sizeof(flag_desc.name));
598 offset = rte_mbuf_dynflag_register(&flag_desc);
599 if (offset < 0) {
600 RTE_LOG(ERR, MBUF,
601 "Failed to register mbuf flag for %s timestamp\n",
602 direction);
603 return -1;
604 }
605 if (flag != NULL)
606 *flag = RTE_BIT64(offset);
607
608 return 0;
609 }
610
611 int
rte_mbuf_dyn_rx_timestamp_register(int * field_offset,uint64_t * rx_flag)612 rte_mbuf_dyn_rx_timestamp_register(int *field_offset, uint64_t *rx_flag)
613 {
614 return rte_mbuf_dyn_timestamp_register(field_offset, rx_flag,
615 "Rx", RTE_MBUF_DYNFLAG_RX_TIMESTAMP_NAME);
616 }
617
618 int
rte_mbuf_dyn_tx_timestamp_register(int * field_offset,uint64_t * tx_flag)619 rte_mbuf_dyn_tx_timestamp_register(int *field_offset, uint64_t *tx_flag)
620 {
621 return rte_mbuf_dyn_timestamp_register(field_offset, tx_flag,
622 "Tx", RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME);
623 }
624