1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2001 Daniel Eischen <[email protected]>
5 * Copyright (c) 2000-2001 Jason Evans <[email protected]>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include <sys/types.h>
34 #include <sys/mman.h>
35 #include <sys/queue.h>
36 #include <sys/resource.h>
37 #include <sys/sysctl.h>
38 #include <stdlib.h>
39 #include <pthread.h>
40 #include <link.h>
41
42 #include "thr_private.h"
43
44 /* Spare thread stack. */
45 struct stack {
46 LIST_ENTRY(stack) qe; /* Stack queue linkage. */
47 size_t stacksize; /* Stack size (rounded up). */
48 size_t guardsize; /* Guard size. */
49 void *stackaddr; /* Stack address. */
50 };
51
52 /*
53 * Default sized (stack and guard) spare stack queue. Stacks are cached
54 * to avoid additional complexity managing mmap()ed stack regions. Spare
55 * stacks are used in LIFO order to increase cache locality.
56 */
57 static LIST_HEAD(, stack) dstackq = LIST_HEAD_INITIALIZER(dstackq);
58
59 /*
60 * Miscellaneous sized (non-default stack and/or guard) spare stack queue.
61 * Stacks are cached to avoid additional complexity managing mmap()ed
62 * stack regions. This list is unordered, since ordering on both stack
63 * size and guard size would be more trouble than it's worth. Stacks are
64 * allocated from this cache on a first size match basis.
65 */
66 static LIST_HEAD(, stack) mstackq = LIST_HEAD_INITIALIZER(mstackq);
67
68 /**
69 * Base address of the last stack allocated (including its red zone, if
70 * there is one). Stacks are allocated contiguously, starting beyond the
71 * top of the main stack. When a new stack is created, a red zone is
72 * typically created (actually, the red zone is mapped with PROT_NONE) above
73 * the top of the stack, such that the stack will not be able to grow all
74 * the way to the bottom of the next stack. This isn't fool-proof. It is
75 * possible for a stack to grow by a large amount, such that it grows into
76 * the next stack, and as long as the memory within the red zone is never
77 * accessed, nothing will prevent one thread stack from trouncing all over
78 * the next.
79 *
80 * low memory
81 * . . . . . . . . . . . . . . . . . .
82 * | |
83 * | stack 3 | start of 3rd thread stack
84 * +-----------------------------------+
85 * | |
86 * | Red Zone (guard page) | red zone for 2nd thread
87 * | |
88 * +-----------------------------------+
89 * | stack 2 - _thr_stack_default | top of 2nd thread stack
90 * | |
91 * | |
92 * | |
93 * | |
94 * | stack 2 |
95 * +-----------------------------------+ <-- start of 2nd thread stack
96 * | |
97 * | Red Zone | red zone for 1st thread
98 * | |
99 * +-----------------------------------+
100 * | stack 1 - _thr_stack_default | top of 1st thread stack
101 * | |
102 * | |
103 * | |
104 * | |
105 * | stack 1 |
106 * +-----------------------------------+ <-- start of 1st thread stack
107 * | | (initial value of last_stack)
108 * | Red Zone |
109 * | | red zone for main thread
110 * +-----------------------------------+
111 * | USRSTACK - _thr_stack_initial | top of main thread stack
112 * | | ^
113 * | | |
114 * | | |
115 * | | | stack growth
116 * | |
117 * +-----------------------------------+ <-- start of main thread stack
118 * (USRSTACK)
119 * high memory
120 *
121 */
122 static char *last_stack = NULL;
123
124 /*
125 * Round size up to the nearest multiple of
126 * _thr_page_size.
127 */
128 static inline size_t
round_up(size_t size)129 round_up(size_t size)
130 {
131 if (size % _thr_page_size != 0)
132 size = ((size / _thr_page_size) + 1) *
133 _thr_page_size;
134 return size;
135 }
136
137 void
_thr_stack_fix_protection(struct pthread * thrd)138 _thr_stack_fix_protection(struct pthread *thrd)
139 {
140
141 mprotect((char *)thrd->attr.stackaddr_attr +
142 round_up(thrd->attr.guardsize_attr),
143 round_up(thrd->attr.stacksize_attr),
144 _rtld_get_stack_prot());
145 }
146
147 static void
singlethread_map_stacks_exec(void)148 singlethread_map_stacks_exec(void)
149 {
150 int mib[2];
151 struct rlimit rlim;
152 u_long usrstack;
153 size_t len;
154
155 mib[0] = CTL_KERN;
156 mib[1] = KERN_USRSTACK;
157 len = sizeof(usrstack);
158 if (sysctl(mib, sizeof(mib) / sizeof(mib[0]), &usrstack, &len, NULL, 0)
159 == -1)
160 return;
161 if (getrlimit(RLIMIT_STACK, &rlim) == -1)
162 return;
163 mprotect((void *)(uintptr_t)(usrstack - rlim.rlim_cur),
164 rlim.rlim_cur, _rtld_get_stack_prot());
165 }
166
167 void
__thr_map_stacks_exec(void)168 __thr_map_stacks_exec(void)
169 {
170 struct pthread *curthread, *thrd;
171 struct stack *st;
172
173 if (!_thr_is_inited()) {
174 singlethread_map_stacks_exec();
175 return;
176 }
177 curthread = _get_curthread();
178 THREAD_LIST_RDLOCK(curthread);
179 LIST_FOREACH(st, &mstackq, qe)
180 mprotect((char *)st->stackaddr + st->guardsize, st->stacksize,
181 _rtld_get_stack_prot());
182 LIST_FOREACH(st, &dstackq, qe)
183 mprotect((char *)st->stackaddr + st->guardsize, st->stacksize,
184 _rtld_get_stack_prot());
185 TAILQ_FOREACH(thrd, &_thread_gc_list, gcle)
186 _thr_stack_fix_protection(thrd);
187 TAILQ_FOREACH(thrd, &_thread_list, tle)
188 _thr_stack_fix_protection(thrd);
189 THREAD_LIST_UNLOCK(curthread);
190 }
191
192 int
_thr_stack_alloc(struct pthread_attr * attr)193 _thr_stack_alloc(struct pthread_attr *attr)
194 {
195 struct pthread *curthread = _get_curthread();
196 struct stack *spare_stack;
197 size_t stacksize;
198 size_t guardsize;
199 char *stackaddr;
200
201 /*
202 * Round up stack size to nearest multiple of _thr_page_size so
203 * that mmap() * will work. If the stack size is not an even
204 * multiple, we end up initializing things such that there is
205 * unused space above the beginning of the stack, so the stack
206 * sits snugly against its guard.
207 */
208 stacksize = round_up(attr->stacksize_attr);
209 guardsize = round_up(attr->guardsize_attr);
210
211 attr->stackaddr_attr = NULL;
212 attr->flags &= ~THR_STACK_USER;
213
214 /*
215 * Use the garbage collector lock for synchronization of the
216 * spare stack lists and allocations from usrstack.
217 */
218 THREAD_LIST_WRLOCK(curthread);
219 /*
220 * If the stack and guard sizes are default, try to allocate a stack
221 * from the default-size stack cache:
222 */
223 if ((stacksize == THR_STACK_DEFAULT) &&
224 (guardsize == _thr_guard_default)) {
225 if ((spare_stack = LIST_FIRST(&dstackq)) != NULL) {
226 /* Use the spare stack. */
227 LIST_REMOVE(spare_stack, qe);
228 attr->stackaddr_attr = spare_stack->stackaddr;
229 }
230 }
231 /*
232 * The user specified a non-default stack and/or guard size, so try to
233 * allocate a stack from the non-default size stack cache, using the
234 * rounded up stack size (stack_size) in the search:
235 */
236 else {
237 LIST_FOREACH(spare_stack, &mstackq, qe) {
238 if (spare_stack->stacksize == stacksize &&
239 spare_stack->guardsize == guardsize) {
240 LIST_REMOVE(spare_stack, qe);
241 attr->stackaddr_attr = spare_stack->stackaddr;
242 break;
243 }
244 }
245 }
246 if (attr->stackaddr_attr != NULL) {
247 /* A cached stack was found. Release the lock. */
248 THREAD_LIST_UNLOCK(curthread);
249 }
250 else {
251 /*
252 * Allocate a stack from or below usrstack, depending
253 * on the LIBPTHREAD_BIGSTACK_MAIN env variable.
254 */
255 if (last_stack == NULL)
256 last_stack = _usrstack - _thr_stack_initial -
257 _thr_guard_default;
258
259 /* Allocate a new stack. */
260 stackaddr = last_stack - stacksize - guardsize;
261
262 /*
263 * Even if stack allocation fails, we don't want to try to
264 * use this location again, so unconditionally decrement
265 * last_stack. Under normal operating conditions, the most
266 * likely reason for an mmap() error is a stack overflow of
267 * the adjacent thread stack.
268 */
269 last_stack -= (stacksize + guardsize);
270
271 /* Release the lock before mmap'ing it. */
272 THREAD_LIST_UNLOCK(curthread);
273
274 /* Map the stack and guard page together, and split guard
275 page from allocated space: */
276 if ((stackaddr = mmap(stackaddr, stacksize + guardsize,
277 _rtld_get_stack_prot(), MAP_STACK,
278 -1, 0)) != MAP_FAILED &&
279 (guardsize == 0 ||
280 mprotect(stackaddr, guardsize, PROT_NONE) == 0)) {
281 stackaddr += guardsize;
282 } else {
283 if (stackaddr != MAP_FAILED)
284 munmap(stackaddr, stacksize + guardsize);
285 stackaddr = NULL;
286 }
287 attr->stackaddr_attr = stackaddr;
288 }
289 if (attr->stackaddr_attr != NULL)
290 return (0);
291 else
292 return (-1);
293 }
294
295 /* This function must be called with _thread_list_lock held. */
296 void
_thr_stack_free(struct pthread_attr * attr)297 _thr_stack_free(struct pthread_attr *attr)
298 {
299 struct stack *spare_stack;
300
301 if ((attr != NULL) && ((attr->flags & THR_STACK_USER) == 0)
302 && (attr->stackaddr_attr != NULL)) {
303 spare_stack = (struct stack *)
304 ((char *)attr->stackaddr_attr +
305 attr->stacksize_attr - sizeof(struct stack));
306 spare_stack->stacksize = round_up(attr->stacksize_attr);
307 spare_stack->guardsize = round_up(attr->guardsize_attr);
308 spare_stack->stackaddr = attr->stackaddr_attr;
309
310 if (spare_stack->stacksize == THR_STACK_DEFAULT &&
311 spare_stack->guardsize == _thr_guard_default) {
312 /* Default stack/guard size. */
313 LIST_INSERT_HEAD(&dstackq, spare_stack, qe);
314 } else {
315 /* Non-default stack/guard size. */
316 LIST_INSERT_HEAD(&mstackq, spare_stack, qe);
317 }
318 attr->stackaddr_attr = NULL;
319 }
320 }
321