1 /******************************************************************************/
2 #ifdef JEMALLOC_H_TYPES
3
4 #ifdef _WIN32
5 # ifdef _WIN64
6 # define FMT64_PREFIX "ll"
7 # define FMTPTR_PREFIX "ll"
8 # else
9 # define FMT64_PREFIX "ll"
10 # define FMTPTR_PREFIX ""
11 # endif
12 # define FMTd32 "d"
13 # define FMTu32 "u"
14 # define FMTx32 "x"
15 # define FMTd64 FMT64_PREFIX "d"
16 # define FMTu64 FMT64_PREFIX "u"
17 # define FMTx64 FMT64_PREFIX "x"
18 # define FMTdPTR FMTPTR_PREFIX "d"
19 # define FMTuPTR FMTPTR_PREFIX "u"
20 # define FMTxPTR FMTPTR_PREFIX "x"
21 #else
22 # include <inttypes.h>
23 # define FMTd32 PRId32
24 # define FMTu32 PRIu32
25 # define FMTx32 PRIx32
26 # define FMTd64 PRId64
27 # define FMTu64 PRIu64
28 # define FMTx64 PRIx64
29 # define FMTdPTR PRIdPTR
30 # define FMTuPTR PRIuPTR
31 # define FMTxPTR PRIxPTR
32 #endif
33
34 /* Size of stack-allocated buffer passed to buferror(). */
35 #define BUFERROR_BUF 64
36
37 /*
38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be
39 * large enough for all possible uses within jemalloc.
40 */
41 #define MALLOC_PRINTF_BUFSIZE 4096
42
43 /*
44 * Wrap a cpp argument that contains commas such that it isn't broken up into
45 * multiple arguments.
46 */
47 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
48
49 /*
50 * Silence compiler warnings due to uninitialized values. This is used
51 * wherever the compiler fails to recognize that the variable is never used
52 * uninitialized.
53 */
54 #ifdef JEMALLOC_CC_SILENCE
55 # define JEMALLOC_CC_SILENCE_INIT(v) = v
56 #else
57 # define JEMALLOC_CC_SILENCE_INIT(v)
58 #endif
59
60 #define JEMALLOC_GNUC_PREREQ(major, minor) \
61 (!defined(__clang__) && \
62 (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor))))
63 #ifndef __has_builtin
64 # define __has_builtin(builtin) (0)
65 #endif
66 #define JEMALLOC_CLANG_HAS_BUILTIN(builtin) \
67 (defined(__clang__) && __has_builtin(builtin))
68
69 #ifdef __GNUC__
70 # define likely(x) __builtin_expect(!!(x), 1)
71 # define unlikely(x) __builtin_expect(!!(x), 0)
72 # if JEMALLOC_GNUC_PREREQ(4, 6) || \
73 JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable)
74 # define unreachable() __builtin_unreachable()
75 # else
76 # define unreachable()
77 # endif
78 #else
79 # define likely(x) !!(x)
80 # define unlikely(x) !!(x)
81 # define unreachable()
82 #endif
83
84 /*
85 * Define a custom assert() in order to reduce the chances of deadlock during
86 * assertion failure.
87 */
88 #ifndef assert
89 #define assert(e) do { \
90 if (unlikely(config_debug && !(e))) { \
91 malloc_printf( \
92 "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", \
93 __FILE__, __LINE__, #e); \
94 abort(); \
95 } \
96 } while (0)
97 #endif
98
99 #ifndef not_reached
100 #define not_reached() do { \
101 if (config_debug) { \
102 malloc_printf( \
103 "<jemalloc>: %s:%d: Unreachable code reached\n", \
104 __FILE__, __LINE__); \
105 abort(); \
106 } \
107 unreachable(); \
108 } while (0)
109 #endif
110
111 #ifndef not_implemented
112 #define not_implemented() do { \
113 if (config_debug) { \
114 malloc_printf("<jemalloc>: %s:%d: Not implemented\n", \
115 __FILE__, __LINE__); \
116 abort(); \
117 } \
118 } while (0)
119 #endif
120
121 #ifndef assert_not_implemented
122 #define assert_not_implemented(e) do { \
123 if (unlikely(config_debug && !(e))) \
124 not_implemented(); \
125 } while (0)
126 #endif
127
128 /* Use to assert a particular configuration, e.g., cassert(config_debug). */
129 #define cassert(c) do { \
130 if (unlikely(!(c))) \
131 not_reached(); \
132 } while (0)
133
134 #endif /* JEMALLOC_H_TYPES */
135 /******************************************************************************/
136 #ifdef JEMALLOC_H_STRUCTS
137
138 #endif /* JEMALLOC_H_STRUCTS */
139 /******************************************************************************/
140 #ifdef JEMALLOC_H_EXTERNS
141
142 int buferror(int err, char *buf, size_t buflen);
143 uintmax_t malloc_strtoumax(const char *restrict nptr,
144 char **restrict endptr, int base);
145 void malloc_write(const char *s);
146
147 /*
148 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
149 * point math.
150 */
151 int malloc_vsnprintf(char *str, size_t size, const char *format,
152 va_list ap);
153 int malloc_snprintf(char *str, size_t size, const char *format, ...)
154 JEMALLOC_FORMAT_PRINTF(3, 4);
155 void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
156 const char *format, va_list ap);
157 void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
158 const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4);
159 void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
160
161 #endif /* JEMALLOC_H_EXTERNS */
162 /******************************************************************************/
163 #ifdef JEMALLOC_H_INLINES
164
165 #ifndef JEMALLOC_ENABLE_INLINE
166 int jemalloc_ffsl(long bitmap);
167 int jemalloc_ffs(int bitmap);
168 size_t pow2_ceil(size_t x);
169 size_t lg_floor(size_t x);
170 void set_errno(int errnum);
171 int get_errno(void);
172 #endif
173
174 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_))
175
176 /* Sanity check. */
177 #if !defined(JEMALLOC_INTERNAL_FFSL) || !defined(JEMALLOC_INTERNAL_FFS)
178 # error Both JEMALLOC_INTERNAL_FFSL && JEMALLOC_INTERNAL_FFS should have been defined by configure
179 #endif
180
181 JEMALLOC_ALWAYS_INLINE int
jemalloc_ffsl(long bitmap)182 jemalloc_ffsl(long bitmap)
183 {
184
185 return (JEMALLOC_INTERNAL_FFSL(bitmap));
186 }
187
188 JEMALLOC_ALWAYS_INLINE int
jemalloc_ffs(int bitmap)189 jemalloc_ffs(int bitmap)
190 {
191
192 return (JEMALLOC_INTERNAL_FFS(bitmap));
193 }
194
195 /* Compute the smallest power of 2 that is >= x. */
196 JEMALLOC_INLINE size_t
pow2_ceil(size_t x)197 pow2_ceil(size_t x)
198 {
199
200 x--;
201 x |= x >> 1;
202 x |= x >> 2;
203 x |= x >> 4;
204 x |= x >> 8;
205 x |= x >> 16;
206 #if (LG_SIZEOF_PTR == 3)
207 x |= x >> 32;
208 #endif
209 x++;
210 return (x);
211 }
212
213 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
214 JEMALLOC_INLINE size_t
lg_floor(size_t x)215 lg_floor(size_t x)
216 {
217 size_t ret;
218
219 assert(x != 0);
220
221 asm ("bsr %1, %0"
222 : "=r"(ret) // Outputs.
223 : "r"(x) // Inputs.
224 );
225 return (ret);
226 }
227 #elif (defined(_MSC_VER))
228 JEMALLOC_INLINE size_t
lg_floor(size_t x)229 lg_floor(size_t x)
230 {
231 unsigned long ret;
232
233 assert(x != 0);
234
235 #if (LG_SIZEOF_PTR == 3)
236 _BitScanReverse64(&ret, x);
237 #elif (LG_SIZEOF_PTR == 2)
238 _BitScanReverse(&ret, x);
239 #else
240 # error "Unsupported type sizes for lg_floor()"
241 #endif
242 return (ret);
243 }
244 #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
245 JEMALLOC_INLINE size_t
lg_floor(size_t x)246 lg_floor(size_t x)
247 {
248
249 assert(x != 0);
250
251 #if (LG_SIZEOF_PTR == LG_SIZEOF_INT)
252 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x));
253 #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG)
254 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x));
255 #else
256 # error "Unsupported type sizes for lg_floor()"
257 #endif
258 }
259 #else
260 JEMALLOC_INLINE size_t
lg_floor(size_t x)261 lg_floor(size_t x)
262 {
263
264 assert(x != 0);
265
266 x |= (x >> 1);
267 x |= (x >> 2);
268 x |= (x >> 4);
269 x |= (x >> 8);
270 x |= (x >> 16);
271 #if (LG_SIZEOF_PTR == 3 && LG_SIZEOF_PTR == LG_SIZEOF_LONG)
272 x |= (x >> 32);
273 if (x == KZU(0xffffffffffffffff))
274 return (63);
275 x++;
276 return (jemalloc_ffsl(x) - 2);
277 #elif (LG_SIZEOF_PTR == 2)
278 if (x == KZU(0xffffffff))
279 return (31);
280 x++;
281 return (jemalloc_ffs(x) - 2);
282 #else
283 # error "Unsupported type sizes for lg_floor()"
284 #endif
285 }
286 #endif
287
288 /* Set error code. */
289 JEMALLOC_INLINE void
set_errno(int errnum)290 set_errno(int errnum)
291 {
292
293 #ifdef _WIN32
294 SetLastError(errnum);
295 #else
296 errno = errnum;
297 #endif
298 }
299
300 /* Get last error code. */
301 JEMALLOC_INLINE int
get_errno(void)302 get_errno(void)
303 {
304
305 #ifdef _WIN32
306 return (GetLastError());
307 #else
308 return (errno);
309 #endif
310 }
311 #endif
312
313 #endif /* JEMALLOC_H_INLINES */
314 /******************************************************************************/
315