1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
3 */
4
5 #ifndef MALLOC_ELEM_H_
6 #define MALLOC_ELEM_H_
7
8 #include <stdbool.h>
9
10 #define MIN_DATA_SIZE (RTE_CACHE_LINE_SIZE)
11
12 /* dummy definition of struct so we can use pointers to it in malloc_elem struct */
13 struct malloc_heap;
14
15 enum elem_state {
16 ELEM_FREE = 0,
17 ELEM_BUSY,
18 ELEM_PAD /* element is a padding-only header */
19 };
20
21 struct malloc_elem {
22 struct malloc_heap *heap;
23 struct malloc_elem *volatile prev;
24 /**< points to prev elem in memseg */
25 struct malloc_elem *volatile next;
26 /**< points to next elem in memseg */
27 LIST_ENTRY(malloc_elem) free_list;
28 /**< list of free elements in heap */
29 struct rte_memseg_list *msl;
30 volatile enum elem_state state;
31 uint32_t pad;
32 size_t size;
33 struct malloc_elem *orig_elem;
34 size_t orig_size;
35 #ifdef RTE_MALLOC_DEBUG
36 uint64_t header_cookie; /* Cookie marking start of data */
37 /* trailer cookie at start + size */
38 #endif
39 } __rte_cache_aligned;
40
41 #ifndef RTE_MALLOC_DEBUG
42 static const unsigned MALLOC_ELEM_TRAILER_LEN = 0;
43
44 /* dummy function - just check if pointer is non-null */
45 static inline int
malloc_elem_cookies_ok(const struct malloc_elem * elem)46 malloc_elem_cookies_ok(const struct malloc_elem *elem){ return elem != NULL; }
47
48 /* dummy function - no header if malloc_debug is not enabled */
49 static inline void
set_header(struct malloc_elem * elem __rte_unused)50 set_header(struct malloc_elem *elem __rte_unused){ }
51
52 /* dummy function - no trailer if malloc_debug is not enabled */
53 static inline void
set_trailer(struct malloc_elem * elem __rte_unused)54 set_trailer(struct malloc_elem *elem __rte_unused){ }
55
56
57 #else
58 static const unsigned MALLOC_ELEM_TRAILER_LEN = RTE_CACHE_LINE_SIZE;
59
60 #define MALLOC_HEADER_COOKIE 0xbadbadbadadd2e55ULL /**< Header cookie. */
61 #define MALLOC_TRAILER_COOKIE 0xadd2e55badbadbadULL /**< Trailer cookie.*/
62
63 /* define macros to make referencing the header and trailer cookies easier */
64 #define MALLOC_ELEM_TRAILER(elem) (*((uint64_t*)RTE_PTR_ADD(elem, \
65 elem->size - MALLOC_ELEM_TRAILER_LEN)))
66 #define MALLOC_ELEM_HEADER(elem) (elem->header_cookie)
67
68 static inline void
set_header(struct malloc_elem * elem)69 set_header(struct malloc_elem *elem)
70 {
71 if (elem != NULL)
72 MALLOC_ELEM_HEADER(elem) = MALLOC_HEADER_COOKIE;
73 }
74
75 static inline void
set_trailer(struct malloc_elem * elem)76 set_trailer(struct malloc_elem *elem)
77 {
78 if (elem != NULL)
79 MALLOC_ELEM_TRAILER(elem) = MALLOC_TRAILER_COOKIE;
80 }
81
82 /* check that the header and trailer cookies are set correctly */
83 static inline int
malloc_elem_cookies_ok(const struct malloc_elem * elem)84 malloc_elem_cookies_ok(const struct malloc_elem *elem)
85 {
86 return elem != NULL &&
87 MALLOC_ELEM_HEADER(elem) == MALLOC_HEADER_COOKIE &&
88 MALLOC_ELEM_TRAILER(elem) == MALLOC_TRAILER_COOKIE;
89 }
90
91 #endif
92
93 static const unsigned MALLOC_ELEM_HEADER_LEN = sizeof(struct malloc_elem);
94 #define MALLOC_ELEM_OVERHEAD (MALLOC_ELEM_HEADER_LEN + MALLOC_ELEM_TRAILER_LEN)
95
96 /*
97 * Given a pointer to the start of a memory block returned by malloc, get
98 * the actual malloc_elem header for that block.
99 */
100 static inline struct malloc_elem *
malloc_elem_from_data(const void * data)101 malloc_elem_from_data(const void *data)
102 {
103 if (data == NULL)
104 return NULL;
105
106 struct malloc_elem *elem = RTE_PTR_SUB(data, MALLOC_ELEM_HEADER_LEN);
107 if (!malloc_elem_cookies_ok(elem))
108 return NULL;
109 return elem->state != ELEM_PAD ? elem: RTE_PTR_SUB(elem, elem->pad);
110 }
111
112 /*
113 * initialise a malloc_elem header
114 */
115 void
116 malloc_elem_init(struct malloc_elem *elem,
117 struct malloc_heap *heap,
118 struct rte_memseg_list *msl,
119 size_t size,
120 struct malloc_elem *orig_elem,
121 size_t orig_size);
122
123 void
124 malloc_elem_insert(struct malloc_elem *elem);
125
126 /*
127 * return true if the current malloc_elem can hold a block of data
128 * of the requested size and with the requested alignment
129 */
130 int
131 malloc_elem_can_hold(struct malloc_elem *elem, size_t size,
132 unsigned int align, size_t bound, bool contig);
133
134 /*
135 * reserve a block of data in an existing malloc_elem. If the malloc_elem
136 * is much larger than the data block requested, we split the element in two.
137 */
138 struct malloc_elem *
139 malloc_elem_alloc(struct malloc_elem *elem, size_t size,
140 unsigned int align, size_t bound, bool contig);
141
142 /*
143 * free a malloc_elem block by adding it to the free list. If the
144 * blocks either immediately before or immediately after newly freed block
145 * are also free, the blocks are merged together.
146 */
147 struct malloc_elem *
148 malloc_elem_free(struct malloc_elem *elem);
149
150 struct malloc_elem *
151 malloc_elem_join_adjacent_free(struct malloc_elem *elem);
152
153 /*
154 * attempt to resize a malloc_elem by expanding into any free space
155 * immediately after it in memory.
156 */
157 int
158 malloc_elem_resize(struct malloc_elem *elem, size_t size);
159
160 void
161 malloc_elem_hide_region(struct malloc_elem *elem, void *start, size_t len);
162
163 void
164 malloc_elem_free_list_remove(struct malloc_elem *elem);
165
166 /*
167 * dump contents of malloc elem to a file.
168 */
169 void
170 malloc_elem_dump(const struct malloc_elem *elem, FILE *f);
171
172 /*
173 * Given an element size, compute its freelist index.
174 */
175 size_t
176 malloc_elem_free_list_index(size_t size);
177
178 /*
179 * Add element to its heap's free list.
180 */
181 void
182 malloc_elem_free_list_insert(struct malloc_elem *elem);
183
184 /*
185 * Find biggest IOVA-contiguous zone within an element with specified alignment.
186 */
187 size_t
188 malloc_elem_find_max_iova_contig(struct malloc_elem *elem, size_t align);
189
190 #endif /* MALLOC_ELEM_H_ */
191