1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright 1999, 2000 John D. Polstra.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 * from: FreeBSD: src/libexec/rtld-elf/sparc64/lockdflt.c,v 1.3 2002/10/09
28 */
29
30 /*
31 * Thread locking implementation for the dynamic linker.
32 *
33 * We use the "simple, non-scalable reader-preference lock" from:
34 *
35 * J. M. Mellor-Crummey and M. L. Scott. "Scalable Reader-Writer
36 * Synchronization for Shared-Memory Multiprocessors." 3rd ACM Symp. on
37 * Principles and Practice of Parallel Programming, April 1991.
38 *
39 * In this algorithm the lock is a single word. Its low-order bit is
40 * set when a writer holds the lock. The remaining high-order bits
41 * contain a count of readers desiring the lock. The algorithm requires
42 * atomic "compare_and_store" and "add" operations, which we take
43 * from machine/atomic.h.
44 */
45
46 #include <sys/param.h>
47 #include <sys/signalvar.h>
48 #include <signal.h>
49 #include <stdlib.h>
50 #include <time.h>
51
52 #include "debug.h"
53 #include "rtld.h"
54 #include "rtld_machdep.h"
55 #include "rtld_libc.h"
56
57 void _rtld_thread_init(struct RtldLockInfo *) __exported;
58 void _rtld_atfork_pre(int *) __exported;
59 void _rtld_atfork_post(int *) __exported;
60
61 static char def_dlerror_msg[512];
62 static int def_dlerror_seen_val = 1;
63
64 static char *
def_dlerror_loc(void)65 def_dlerror_loc(void)
66 {
67 return (def_dlerror_msg);
68 }
69
70 static int *
def_dlerror_seen(void)71 def_dlerror_seen(void)
72 {
73 return (&def_dlerror_seen_val);
74 }
75
76 #define WAFLAG 0x1 /* A writer holds the lock */
77 #define RC_INCR 0x2 /* Adjusts count of readers desiring lock */
78
79 typedef struct Struct_Lock {
80 volatile u_int lock;
81 void *base;
82 } Lock;
83
84 static sigset_t fullsigmask, oldsigmask;
85 static int thread_flag, wnested;
86 static uint32_t fsigblock;
87
88 static void *
def_lock_create(void)89 def_lock_create(void)
90 {
91 void *base;
92 char *p;
93 uintptr_t r;
94 Lock *l;
95
96 /*
97 * Arrange for the lock to occupy its own cache line. First, we
98 * optimistically allocate just a cache line, hoping that malloc
99 * will give us a well-aligned block of memory. If that doesn't
100 * work, we allocate a larger block and take a well-aligned cache
101 * line from it.
102 */
103 base = xmalloc(CACHE_LINE_SIZE);
104 p = base;
105 if ((uintptr_t)p % CACHE_LINE_SIZE != 0) {
106 free(base);
107 base = xmalloc(2 * CACHE_LINE_SIZE);
108 p = base;
109 if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0)
110 p += CACHE_LINE_SIZE - r;
111 }
112 l = (Lock *)p;
113 l->base = base;
114 l->lock = 0;
115 return (l);
116 }
117
118 static void
def_lock_destroy(void * lock)119 def_lock_destroy(void *lock)
120 {
121 Lock *l = lock;
122
123 free(l->base);
124 }
125
126 static void
sig_fastunblock(void)127 sig_fastunblock(void)
128 {
129 uint32_t oldval;
130
131 assert((fsigblock & ~SIGFASTBLOCK_FLAGS) >= SIGFASTBLOCK_INC);
132 oldval = atomic_fetchadd_32(&fsigblock, -SIGFASTBLOCK_INC);
133 if (oldval == (SIGFASTBLOCK_PEND | SIGFASTBLOCK_INC))
134 __sys_sigfastblock(SIGFASTBLOCK_UNBLOCK, NULL);
135 }
136
137 static bool
def_lock_acquire_set(Lock * l,bool wlock)138 def_lock_acquire_set(Lock *l, bool wlock)
139 {
140 if (wlock) {
141 if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG))
142 return (true);
143 } else {
144 atomic_add_acq_int(&l->lock, RC_INCR);
145 if ((l->lock & WAFLAG) == 0)
146 return (true);
147 atomic_add_int(&l->lock, -RC_INCR);
148 }
149 return (false);
150 }
151
152 static void
def_lock_acquire(Lock * l,bool wlock)153 def_lock_acquire(Lock *l, bool wlock)
154 {
155 sigset_t tmp_oldsigmask;
156
157 if (ld_fast_sigblock) {
158 for (;;) {
159 atomic_add_32(&fsigblock, SIGFASTBLOCK_INC);
160 if (def_lock_acquire_set(l, wlock))
161 break;
162 sig_fastunblock();
163 }
164 } else {
165 for (;;) {
166 sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask);
167 if (def_lock_acquire_set(l, wlock))
168 break;
169 sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL);
170 }
171 if (atomic_fetchadd_int(&wnested, 1) == 0)
172 oldsigmask = tmp_oldsigmask;
173 }
174 }
175
176 static void
def_rlock_acquire(void * lock)177 def_rlock_acquire(void *lock)
178 {
179 def_lock_acquire(lock, false);
180 }
181
182 static void
def_wlock_acquire(void * lock)183 def_wlock_acquire(void *lock)
184 {
185 def_lock_acquire(lock, true);
186 }
187
188 static void
def_lock_release(void * lock)189 def_lock_release(void *lock)
190 {
191 Lock *l = lock;
192
193 atomic_add_rel_int(&l->lock, -((l->lock & WAFLAG) == 0 ?
194 RC_INCR : WAFLAG));
195 if (ld_fast_sigblock)
196 sig_fastunblock();
197 else if (atomic_fetchadd_int(&wnested, -1) == 1)
198 sigprocmask(SIG_SETMASK, &oldsigmask, NULL);
199 }
200
201 static int
def_thread_set_flag(int mask)202 def_thread_set_flag(int mask)
203 {
204 int old_val = thread_flag;
205
206 thread_flag |= mask;
207 return (old_val);
208 }
209
210 static int
def_thread_clr_flag(int mask)211 def_thread_clr_flag(int mask)
212 {
213 int old_val = thread_flag;
214
215 thread_flag &= ~mask;
216 return (old_val);
217 }
218
219 /*
220 * Public interface exposed to the rest of the dynamic linker.
221 */
222 struct RtldLockInfo lockinfo;
223 static struct RtldLockInfo deflockinfo;
224
225 static __inline int
thread_mask_set(int mask)226 thread_mask_set(int mask)
227 {
228 return (lockinfo.thread_set_flag(mask));
229 }
230
231 static __inline void
thread_mask_clear(int mask)232 thread_mask_clear(int mask)
233 {
234 lockinfo.thread_clr_flag(mask);
235 }
236
237 #define RTLD_LOCK_CNT 3
238 static struct rtld_lock {
239 void *handle;
240 int mask;
241 } rtld_locks[RTLD_LOCK_CNT];
242
243 rtld_lock_t rtld_bind_lock = &rtld_locks[0];
244 rtld_lock_t rtld_libc_lock = &rtld_locks[1];
245 rtld_lock_t rtld_phdr_lock = &rtld_locks[2];
246
247 void
rlock_acquire(rtld_lock_t lock,RtldLockState * lockstate)248 rlock_acquire(rtld_lock_t lock, RtldLockState *lockstate)
249 {
250
251 if (lockstate == NULL)
252 return;
253
254 if (thread_mask_set(lock->mask) & lock->mask) {
255 dbg("rlock_acquire: recursed");
256 lockstate->lockstate = RTLD_LOCK_UNLOCKED;
257 return;
258 }
259 lockinfo.rlock_acquire(lock->handle);
260 lockstate->lockstate = RTLD_LOCK_RLOCKED;
261 }
262
263 void
wlock_acquire(rtld_lock_t lock,RtldLockState * lockstate)264 wlock_acquire(rtld_lock_t lock, RtldLockState *lockstate)
265 {
266
267 if (lockstate == NULL)
268 return;
269
270 if (thread_mask_set(lock->mask) & lock->mask) {
271 dbg("wlock_acquire: recursed");
272 lockstate->lockstate = RTLD_LOCK_UNLOCKED;
273 return;
274 }
275 lockinfo.wlock_acquire(lock->handle);
276 lockstate->lockstate = RTLD_LOCK_WLOCKED;
277 }
278
279 void
lock_release(rtld_lock_t lock,RtldLockState * lockstate)280 lock_release(rtld_lock_t lock, RtldLockState *lockstate)
281 {
282
283 if (lockstate == NULL)
284 return;
285
286 switch (lockstate->lockstate) {
287 case RTLD_LOCK_UNLOCKED:
288 break;
289 case RTLD_LOCK_RLOCKED:
290 case RTLD_LOCK_WLOCKED:
291 thread_mask_clear(lock->mask);
292 lockinfo.lock_release(lock->handle);
293 break;
294 default:
295 assert(0);
296 }
297 }
298
299 void
lock_upgrade(rtld_lock_t lock,RtldLockState * lockstate)300 lock_upgrade(rtld_lock_t lock, RtldLockState *lockstate)
301 {
302
303 if (lockstate == NULL)
304 return;
305
306 lock_release(lock, lockstate);
307 wlock_acquire(lock, lockstate);
308 }
309
310 void
lock_restart_for_upgrade(RtldLockState * lockstate)311 lock_restart_for_upgrade(RtldLockState *lockstate)
312 {
313
314 if (lockstate == NULL)
315 return;
316
317 switch (lockstate->lockstate) {
318 case RTLD_LOCK_UNLOCKED:
319 case RTLD_LOCK_WLOCKED:
320 break;
321 case RTLD_LOCK_RLOCKED:
322 siglongjmp(lockstate->env, 1);
323 break;
324 default:
325 assert(0);
326 }
327 }
328
329 void
dlerror_dflt_init(void)330 dlerror_dflt_init(void)
331 {
332 lockinfo.dlerror_loc = def_dlerror_loc;
333 lockinfo.dlerror_loc_sz = sizeof(def_dlerror_msg);
334 lockinfo.dlerror_seen = def_dlerror_seen;
335 }
336
337 void
lockdflt_init(void)338 lockdflt_init(void)
339 {
340 int i;
341
342 deflockinfo.rtli_version = RTLI_VERSION;
343 deflockinfo.lock_create = def_lock_create;
344 deflockinfo.lock_destroy = def_lock_destroy;
345 deflockinfo.rlock_acquire = def_rlock_acquire;
346 deflockinfo.wlock_acquire = def_wlock_acquire;
347 deflockinfo.lock_release = def_lock_release;
348 deflockinfo.thread_set_flag = def_thread_set_flag;
349 deflockinfo.thread_clr_flag = def_thread_clr_flag;
350 deflockinfo.at_fork = NULL;
351 deflockinfo.dlerror_loc = def_dlerror_loc;
352 deflockinfo.dlerror_loc_sz = sizeof(def_dlerror_msg);
353 deflockinfo.dlerror_seen = def_dlerror_seen;
354
355 for (i = 0; i < RTLD_LOCK_CNT; i++) {
356 rtld_locks[i].mask = (1 << i);
357 rtld_locks[i].handle = NULL;
358 }
359
360 memcpy(&lockinfo, &deflockinfo, sizeof(lockinfo));
361 _rtld_thread_init(NULL);
362 if (ld_fast_sigblock) {
363 __sys_sigfastblock(SIGFASTBLOCK_SETPTR, &fsigblock);
364 } else {
365 /*
366 * Construct a mask to block all signals. Note that
367 * blocked traps mean that the process is terminated
368 * if trap occurs while we are in locked section, with
369 * the default settings for kern.forcesigexit.
370 */
371 sigfillset(&fullsigmask);
372 }
373 }
374
375 /*
376 * Callback function to allow threads implementation to
377 * register their own locking primitives if the default
378 * one is not suitable.
379 * The current context should be the only context
380 * executing at the invocation time.
381 */
382 void
_rtld_thread_init(struct RtldLockInfo * pli)383 _rtld_thread_init(struct RtldLockInfo *pli)
384 {
385 const Obj_Entry *obj;
386 SymLook req;
387 void *locks[RTLD_LOCK_CNT];
388 int flags, i, res;
389
390 if (pli == NULL) {
391 lockinfo.rtli_version = RTLI_VERSION;
392 } else {
393 lockinfo.rtli_version = RTLI_VERSION_ONE;
394 obj = obj_from_addr(pli->lock_create);
395 if (obj != NULL) {
396 symlook_init(&req, "_pli_rtli_version");
397 res = symlook_obj(&req, obj);
398 if (res == 0)
399 lockinfo.rtli_version = pli->rtli_version;
400 }
401 }
402
403 /* disable all locking while this function is running */
404 flags = thread_mask_set(~0);
405
406 if (pli == NULL)
407 pli = &deflockinfo;
408 else if (ld_fast_sigblock) {
409 fsigblock = 0;
410 __sys_sigfastblock(SIGFASTBLOCK_UNSETPTR, NULL);
411 }
412
413 for (i = 0; i < RTLD_LOCK_CNT; i++)
414 if ((locks[i] = pli->lock_create()) == NULL)
415 break;
416
417 if (i < RTLD_LOCK_CNT) {
418 while (--i >= 0)
419 pli->lock_destroy(locks[i]);
420 abort();
421 }
422
423 for (i = 0; i < RTLD_LOCK_CNT; i++) {
424 if (rtld_locks[i].handle == NULL)
425 continue;
426 if (flags & rtld_locks[i].mask)
427 lockinfo.lock_release(rtld_locks[i].handle);
428 lockinfo.lock_destroy(rtld_locks[i].handle);
429 }
430
431 for (i = 0; i < RTLD_LOCK_CNT; i++) {
432 rtld_locks[i].handle = locks[i];
433 if (flags & rtld_locks[i].mask)
434 pli->wlock_acquire(rtld_locks[i].handle);
435 }
436
437 lockinfo.lock_create = pli->lock_create;
438 lockinfo.lock_destroy = pli->lock_destroy;
439 lockinfo.rlock_acquire = pli->rlock_acquire;
440 lockinfo.wlock_acquire = pli->wlock_acquire;
441 lockinfo.lock_release = pli->lock_release;
442 lockinfo.thread_set_flag = pli->thread_set_flag;
443 lockinfo.thread_clr_flag = pli->thread_clr_flag;
444 lockinfo.at_fork = pli->at_fork;
445 if (lockinfo.rtli_version > RTLI_VERSION_ONE && pli != NULL) {
446 strlcpy(pli->dlerror_loc(), lockinfo.dlerror_loc(),
447 lockinfo.dlerror_loc_sz);
448 lockinfo.dlerror_loc = pli->dlerror_loc;
449 lockinfo.dlerror_loc_sz = pli->dlerror_loc_sz;
450 lockinfo.dlerror_seen = pli->dlerror_seen;
451 }
452
453 /* restore thread locking state, this time with new locks */
454 thread_mask_clear(~0);
455 thread_mask_set(flags);
456 dbg("_rtld_thread_init: done");
457 }
458
459 void
_rtld_atfork_pre(int * locks)460 _rtld_atfork_pre(int *locks)
461 {
462 RtldLockState ls[2];
463
464 if (locks == NULL)
465 return;
466 bzero(ls, sizeof(ls));
467
468 /*
469 * Warning: this did not worked well with the rtld compat
470 * locks above, when the thread signal mask was corrupted (set
471 * to all signals blocked) if two locks were taken
472 * simultaneously in the write mode. The caller of the
473 * _rtld_atfork_pre() must provide the working implementation
474 * of the locks anyway, and libthr locks are fine.
475 */
476 if (ld_get_env_var(LD_NO_DL_ITERATE_PHDR_AFTER_FORK) == NULL)
477 wlock_acquire(rtld_phdr_lock, &ls[0]);
478 wlock_acquire(rtld_bind_lock, &ls[1]);
479
480 /* XXXKIB: I am really sorry for this. */
481 locks[0] = ls[1].lockstate;
482 locks[2] = ls[0].lockstate;
483 }
484
485 void
_rtld_atfork_post(int * locks)486 _rtld_atfork_post(int *locks)
487 {
488 RtldLockState ls[2];
489
490 if (locks == NULL)
491 return;
492
493 bzero(ls, sizeof(ls));
494 ls[0].lockstate = locks[2];
495 ls[1].lockstate = locks[0];
496 lock_release(rtld_bind_lock, &ls[1]);
497 if (ld_get_env_var(LD_NO_DL_ITERATE_PHDR_AFTER_FORK) == NULL)
498 lock_release(rtld_phdr_lock, &ls[0]);
499 }
500