1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2006, David Xu <[email protected]>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /*
30 * A lockless rwlock for rtld.
31 */
32 #include <sys/mman.h>
33 #include <sys/syscall.h>
34 #include <link.h>
35 #include <stdlib.h>
36 #include <string.h>
37
38 #include "libc_private.h"
39 #include "rtld_lock.h"
40 #include "thr_private.h"
41
42 #undef errno
43 extern int errno;
44
45 static int _thr_rtld_clr_flag(int);
46 static void *_thr_rtld_lock_create(void);
47 static void _thr_rtld_lock_destroy(void *);
48 static void _thr_rtld_lock_release(void *);
49 static void _thr_rtld_rlock_acquire(void *);
50 static int _thr_rtld_set_flag(int);
51 static void _thr_rtld_wlock_acquire(void *);
52
53 struct rtld_lock {
54 struct urwlock lock;
55 char _pad[CACHE_LINE_SIZE - sizeof(struct urwlock)];
56 };
57
58 static struct rtld_lock lock_place[MAX_RTLD_LOCKS] __aligned(CACHE_LINE_SIZE);
59 static int busy_places;
60
61 static void *
_thr_rtld_lock_create(void)62 _thr_rtld_lock_create(void)
63 {
64 int locki;
65 struct rtld_lock *l;
66 static const char fail[] = "_thr_rtld_lock_create failed\n";
67
68 for (locki = 0; locki < MAX_RTLD_LOCKS; locki++) {
69 if ((busy_places & (1 << locki)) == 0)
70 break;
71 }
72 if (locki == MAX_RTLD_LOCKS) {
73 write(2, fail, sizeof(fail) - 1);
74 return (NULL);
75 }
76 busy_places |= (1 << locki);
77
78 l = &lock_place[locki];
79 l->lock.rw_flags = URWLOCK_PREFER_READER;
80 return (l);
81 }
82
83 static void
_thr_rtld_lock_destroy(void * lock)84 _thr_rtld_lock_destroy(void *lock)
85 {
86 int locki;
87 size_t i;
88
89 locki = (struct rtld_lock *)lock - &lock_place[0];
90 for (i = 0; i < sizeof(struct rtld_lock); ++i)
91 ((char *)lock)[i] = 0;
92 busy_places &= ~(1 << locki);
93 }
94
95 #define SAVE_ERRNO() { \
96 if (curthread != _thr_initial) \
97 errsave = curthread->error; \
98 else \
99 errsave = errno; \
100 }
101
102 #define RESTORE_ERRNO() { \
103 if (curthread != _thr_initial) \
104 curthread->error = errsave; \
105 else \
106 errno = errsave; \
107 }
108
109 static void
_thr_rtld_rlock_acquire(void * lock)110 _thr_rtld_rlock_acquire(void *lock)
111 {
112 struct pthread *curthread;
113 struct rtld_lock *l;
114 int errsave;
115
116 curthread = _get_curthread();
117 SAVE_ERRNO();
118 l = (struct rtld_lock *)lock;
119
120 THR_CRITICAL_ENTER(curthread);
121 while (_thr_rwlock_rdlock(&l->lock, 0, NULL) != 0)
122 ;
123 curthread->rdlock_count++;
124 RESTORE_ERRNO();
125 }
126
127 static void
_thr_rtld_wlock_acquire(void * lock)128 _thr_rtld_wlock_acquire(void *lock)
129 {
130 struct pthread *curthread;
131 struct rtld_lock *l;
132 int errsave;
133
134 curthread = _get_curthread();
135 SAVE_ERRNO();
136 l = (struct rtld_lock *)lock;
137
138 THR_CRITICAL_ENTER(curthread);
139 while (_thr_rwlock_wrlock(&l->lock, NULL) != 0)
140 ;
141 RESTORE_ERRNO();
142 }
143
144 static void
_thr_rtld_lock_release(void * lock)145 _thr_rtld_lock_release(void *lock)
146 {
147 struct pthread *curthread;
148 struct rtld_lock *l;
149 int32_t state;
150 int errsave;
151
152 curthread = _get_curthread();
153 SAVE_ERRNO();
154 l = (struct rtld_lock *)lock;
155
156 state = l->lock.rw_state;
157 if (__predict_false(_thr_after_fork)) {
158 /*
159 * After fork, only this thread is running, there is no
160 * waiters. Keeping waiters recorded in rwlock breaks
161 * wake logic.
162 */
163 atomic_clear_int(&l->lock.rw_state,
164 URWLOCK_WRITE_WAITERS | URWLOCK_READ_WAITERS);
165 l->lock.rw_blocked_readers = 0;
166 l->lock.rw_blocked_writers = 0;
167 }
168 if (_thr_rwlock_unlock(&l->lock) == 0) {
169 if ((state & URWLOCK_WRITE_OWNER) == 0)
170 curthread->rdlock_count--;
171 THR_CRITICAL_LEAVE(curthread);
172 }
173 RESTORE_ERRNO();
174 }
175
176 static int
_thr_rtld_set_flag(int mask __unused)177 _thr_rtld_set_flag(int mask __unused)
178 {
179 /*
180 * The caller's code in rtld-elf is broken, it is not signal safe,
181 * just return zero to fool it.
182 */
183 return (0);
184 }
185
186 static int
_thr_rtld_clr_flag(int mask __unused)187 _thr_rtld_clr_flag(int mask __unused)
188 {
189 return (0);
190 }
191
192 /*
193 * ABI bug workaround: This symbol must be present for rtld to accept
194 * RTLI_VERSION from RtldLockInfo
195 */
196 extern char _pli_rtli_version;
197 char _pli_rtli_version;
198
199 static char *
_thr_dlerror_loc(void)200 _thr_dlerror_loc(void)
201 {
202 struct pthread *curthread;
203
204 curthread = _get_curthread();
205 return (curthread->dlerror_msg);
206 }
207
208 static int *
_thr_dlerror_seen(void)209 _thr_dlerror_seen(void)
210 {
211 struct pthread *curthread;
212
213 curthread = _get_curthread();
214 return (&curthread->dlerror_seen);
215 }
216
217 void
_thr_rtld_init(void)218 _thr_rtld_init(void)
219 {
220 struct RtldLockInfo li;
221 struct pthread *curthread;
222 ucontext_t *uc;
223 long dummy = -1;
224 int uc_len;
225
226 curthread = _get_curthread();
227
228 /* force to resolve _umtx_op PLT */
229 _umtx_op_err((struct umtx *)&dummy, UMTX_OP_WAKE, 1, 0, 0);
230
231 /* force to resolve errno() PLT */
232 __error();
233
234 /* force to resolve memcpy PLT */
235 memcpy(&dummy, &dummy, sizeof(dummy));
236
237 mprotect(NULL, 0, 0);
238 _rtld_get_stack_prot();
239 thr_wake(-1);
240
241 li.rtli_version = RTLI_VERSION;
242 li.lock_create = _thr_rtld_lock_create;
243 li.lock_destroy = _thr_rtld_lock_destroy;
244 li.rlock_acquire = _thr_rtld_rlock_acquire;
245 li.wlock_acquire = _thr_rtld_wlock_acquire;
246 li.lock_release = _thr_rtld_lock_release;
247 li.thread_set_flag = _thr_rtld_set_flag;
248 li.thread_clr_flag = _thr_rtld_clr_flag;
249 li.at_fork = NULL;
250 li.dlerror_loc = _thr_dlerror_loc;
251 li.dlerror_loc_sz = sizeof(curthread->dlerror_msg);
252 li.dlerror_seen = _thr_dlerror_seen;
253
254 /*
255 * Preresolve the symbols needed for the fork interposer. We
256 * call _rtld_atfork_pre() and _rtld_atfork_post() with NULL
257 * argument to indicate that no actual locking inside the
258 * functions should happen. Neither rtld compat locks nor
259 * libthr rtld locks cannot work there:
260 * - compat locks do not handle the case of two locks taken
261 * in write mode (the signal mask for the thread is corrupted);
262 * - libthr locks would work, but locked rtld_bind_lock prevents
263 * symbol resolution for _rtld_atfork_post.
264 */
265 _rtld_atfork_pre(NULL);
266 _rtld_atfork_post(NULL);
267 _malloc_prefork();
268 _malloc_postfork();
269 getpid();
270 syscall(SYS_getpid);
271
272 /* mask signals, also force to resolve __sys_sigprocmask PLT */
273 _thr_signal_block(curthread);
274 _rtld_thread_init(&li);
275 _thr_signal_unblock(curthread);
276 _thr_signal_block_check_fast();
277 _thr_signal_block_setup(curthread);
278
279 /* resolve machine depended functions, if any */
280 _thr_resolve_machdep();
281
282 uc_len = __getcontextx_size();
283 uc = alloca(uc_len);
284 getcontext(uc);
285 __fillcontextx2((char *)uc);
286 }
287