xref: /freebsd-14.2/sys/kern/subr_trap.c (revision 7fdef9cd)
1 /*-
2  * SPDX-License-Identifier: BSD-4-Clause
3  *
4  * Copyright (C) 1994, David Greenman
5  * Copyright (c) 1990, 1993
6  *	The Regents of the University of California.  All rights reserved.
7  * Copyright (c) 2007, 2022 The FreeBSD Foundation
8  *
9  * This code is derived from software contributed to Berkeley by
10  * the University of Utah, and William Jolitz.
11  *
12  * Portions of this software were developed by A. Joseph Koshy under
13  * sponsorship from the FreeBSD Foundation and Google, Inc.
14  *
15  * Redistribution and use in source and binary forms, with or without
16  * modification, are permitted provided that the following conditions
17  * are met:
18  * 1. Redistributions of source code must retain the above copyright
19  *    notice, this list of conditions and the following disclaimer.
20  * 2. Redistributions in binary form must reproduce the above copyright
21  *    notice, this list of conditions and the following disclaimer in the
22  *    documentation and/or other materials provided with the distribution.
23  * 3. All advertising materials mentioning features or use of this software
24  *    must display the following acknowledgement:
25  *	This product includes software developed by the University of
26  *	California, Berkeley and its contributors.
27  * 4. Neither the name of the University nor the names of its contributors
28  *    may be used to endorse or promote products derived from this software
29  *    without specific prior written permission.
30  *
31  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
32  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
35  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
36  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
37  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
38  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
39  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
40  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
41  * SUCH DAMAGE.
42  *
43  *	from: @(#)trap.c	7.4 (Berkeley) 5/13/91
44  */
45 
46 #include <sys/cdefs.h>
47 #include "opt_hwpmc_hooks.h"
48 
49 #include <sys/param.h>
50 #include <sys/kernel.h>
51 #include <sys/limits.h>
52 #include <sys/lock.h>
53 #include <sys/msan.h>
54 #include <sys/mutex.h>
55 #include <sys/proc.h>
56 #include <sys/ktr.h>
57 #include <sys/resourcevar.h>
58 #include <sys/sched.h>
59 #include <sys/syscall.h>
60 #include <sys/syscallsubr.h>
61 #include <sys/sysent.h>
62 #include <sys/systm.h>
63 #include <sys/vmmeter.h>
64 
65 #include <machine/cpu.h>
66 
67 #ifdef VIMAGE
68 #include <net/vnet.h>
69 #endif
70 
71 #ifdef	HWPMC_HOOKS
72 #include <sys/pmckern.h>
73 #endif
74 
75 #ifdef EPOCH_TRACE
76 #include <sys/epoch.h>
77 #endif
78 
79 volatile uint32_t __read_frequently hpts_that_need_softclock = 0;
80 
81 void	(*tcp_hpts_softclock)(void);
82 
83 /*
84  * Define the code needed before returning to user mode, for trap and
85  * syscall.
86  */
87 void
userret(struct thread * td,struct trapframe * frame)88 userret(struct thread *td, struct trapframe *frame)
89 {
90 	struct proc *p = td->td_proc;
91 
92 	CTR3(KTR_SYSC, "userret: thread %p (pid %d, %s)", td, p->p_pid,
93             td->td_name);
94 	KASSERT((p->p_flag & P_WEXIT) == 0,
95 	    ("Exiting process returns to usermode"));
96 #ifdef DIAGNOSTIC
97 	/*
98 	 * Check that we called signotify() enough.  For
99 	 * multi-threaded processes, where signal distribution might
100 	 * change due to other threads changing sigmask, the check is
101 	 * racy and cannot be performed reliably.
102 	 * If current process is vfork child, indicated by P_PPWAIT, then
103 	 * issignal() ignores stops, so we block the check to avoid
104 	 * classifying pending signals.
105 	 */
106 	if (p->p_numthreads == 1) {
107 		PROC_LOCK(p);
108 		thread_lock(td);
109 		if ((p->p_flag & P_PPWAIT) == 0 &&
110 		    (td->td_pflags & TDP_SIGFASTBLOCK) == 0 &&
111 		    SIGPENDING(td) && !td_ast_pending(td, TDA_AST) &&
112 		    !td_ast_pending(td, TDA_SIG)) {
113 			thread_unlock(td);
114 			panic(
115 			    "failed to set signal flags for ast p %p "
116 			    "td %p td_ast %#x fl %#x",
117 			    p, td, td->td_ast, td->td_flags);
118 		}
119 		thread_unlock(td);
120 		PROC_UNLOCK(p);
121 	}
122 #endif
123 
124 	/*
125 	 * Charge system time if profiling.
126 	 */
127 	if (__predict_false(p->p_flag & P_PROFIL))
128 		addupc_task(td, TRAPF_PC(frame), td->td_pticks * psratio);
129 
130 #ifdef HWPMC_HOOKS
131 	if (PMC_THREAD_HAS_SAMPLES(td))
132 		PMC_CALL_HOOK(td, PMC_FN_THR_USERRET, NULL);
133 #endif
134 	/*
135 	 * Calling tcp_hpts_softclock() here allows us to avoid frequent,
136 	 * expensive callouts that trash the cache and lead to a much higher
137 	 * number of interrupts and context switches.  Testing on busy web
138 	 * servers at Netflix has shown that this improves CPU use by 7% over
139 	 * relying only on callouts to drive HPTS, and also results in idle
140 	 * power savings on mostly idle servers.
141 	 * This was inspired by the paper "Soft Timers: Efficient Microsecond
142 	 * Software Timer Support for Network Processing"
143 	 * by Mohit Aron and Peter Druschel.
144 	 */
145 	tcp_hpts_softclock();
146 	/*
147 	 * Let the scheduler adjust our priority etc.
148 	 */
149 	sched_userret(td);
150 
151 	/*
152 	 * Check for misbehavior.
153 	 *
154 	 * In case there is a callchain tracing ongoing because of
155 	 * hwpmc(4), skip the scheduler pinning check.
156 	 * hwpmc(4) subsystem, infact, will collect callchain informations
157 	 * at ast() checkpoint, which is past userret().
158 	 */
159 	WITNESS_WARN(WARN_PANIC, NULL, "userret: returning");
160 	KASSERT(td->td_critnest == 0,
161 	    ("userret: Returning in a critical section"));
162 	KASSERT(td->td_locks == 0,
163 	    ("userret: Returning with %d locks held", td->td_locks));
164 	KASSERT(td->td_rw_rlocks == 0,
165 	    ("userret: Returning with %d rwlocks held in read mode",
166 	    td->td_rw_rlocks));
167 	KASSERT(td->td_sx_slocks == 0,
168 	    ("userret: Returning with %d sx locks held in shared mode",
169 	    td->td_sx_slocks));
170 	KASSERT(td->td_lk_slocks == 0,
171 	    ("userret: Returning with %d lockmanager locks held in shared mode",
172 	    td->td_lk_slocks));
173 	KASSERT((td->td_pflags & TDP_NOFAULTING) == 0,
174 	    ("userret: Returning with pagefaults disabled"));
175 	if (__predict_false(!THREAD_CAN_SLEEP())) {
176 #ifdef EPOCH_TRACE
177 		epoch_trace_list(curthread);
178 #endif
179 		KASSERT(0, ("userret: Returning with sleep disabled"));
180 	}
181 	KASSERT(td->td_pinned == 0 || (td->td_pflags & TDP_CALLCHAIN) != 0,
182 	    ("userret: Returning with pinned thread"));
183 	KASSERT(td->td_vp_reserved == NULL,
184 	    ("userret: Returning with preallocated vnode"));
185 	KASSERT((td->td_flags & (TDF_SBDRY | TDF_SEINTR | TDF_SERESTART)) == 0,
186 	    ("userret: Returning with stop signals deferred"));
187 	KASSERT(td->td_vslock_sz == 0,
188 	    ("userret: Returning with vslock-wired space"));
189 #ifdef VIMAGE
190 	/* Unfortunately td_vnet_lpush needs VNET_DEBUG. */
191 	VNET_ASSERT(curvnet == NULL,
192 	    ("%s: Returning on td %p (pid %d, %s) with vnet %p set in %s",
193 	    __func__, td, p->p_pid, td->td_name, curvnet,
194 	    (td->td_vnet_lpush != NULL) ? td->td_vnet_lpush : "N/A"));
195 #endif
196 }
197 
198 static void
ast_prep(struct thread * td,int tda __unused)199 ast_prep(struct thread *td, int tda __unused)
200 {
201 	VM_CNT_INC(v_trap);
202 	td->td_pticks = 0;
203 	if (td->td_cowgen != atomic_load_int(&td->td_proc->p_cowgen))
204 		thread_cow_update(td);
205 
206 }
207 
208 struct ast_entry {
209 	int	ae_flags;
210 	int	ae_tdp;
211 	void	(*ae_f)(struct thread *td, int ast);
212 };
213 
214 _Static_assert(TDAI(TDA_MAX) <= UINT_MAX, "Too many ASTs");
215 
216 static struct ast_entry ast_entries[TDA_MAX] __read_mostly = {
217 	[TDA_AST] = { .ae_f = ast_prep, .ae_flags = ASTR_UNCOND},
218 };
219 
220 void
ast_register(int ast,int flags,int tdp,void (* f)(struct thread *,int asts))221 ast_register(int ast, int flags, int tdp,
222     void (*f)(struct thread *, int asts))
223 {
224 	struct ast_entry *ae;
225 
226 	MPASS(ast < TDA_MAX);
227 	MPASS((flags & ASTR_TDP) == 0 || ((flags & ASTR_ASTF_REQUIRED) != 0
228 	    && __bitcount(tdp) == 1));
229 	ae = &ast_entries[ast];
230 	MPASS(ae->ae_f == NULL);
231 	ae->ae_flags = flags;
232 	ae->ae_tdp = tdp;
233 	atomic_interrupt_fence();
234 	ae->ae_f = f;
235 }
236 
237 /*
238  * XXXKIB Note that the deregistration of an AST handler does not
239  * drain threads possibly executing it, which affects unloadable
240  * modules.  The issue is either handled by the subsystem using
241  * handlers, or simply ignored.  Fixing the problem is considered not
242  * worth the overhead.
243  */
244 void
ast_deregister(int ast)245 ast_deregister(int ast)
246 {
247 	struct ast_entry *ae;
248 
249 	MPASS(ast < TDA_MAX);
250 	ae = &ast_entries[ast];
251 	MPASS(ae->ae_f != NULL);
252 	ae->ae_f = NULL;
253 	atomic_interrupt_fence();
254 	ae->ae_flags = 0;
255 	ae->ae_tdp = 0;
256 }
257 
258 void
ast_sched_locked(struct thread * td,int tda)259 ast_sched_locked(struct thread *td, int tda)
260 {
261 	THREAD_LOCK_ASSERT(td, MA_OWNED);
262 	MPASS(tda < TDA_MAX);
263 
264 	td->td_ast |= TDAI(tda);
265 }
266 
267 void
ast_unsched_locked(struct thread * td,int tda)268 ast_unsched_locked(struct thread *td, int tda)
269 {
270 	THREAD_LOCK_ASSERT(td, MA_OWNED);
271 	MPASS(tda < TDA_MAX);
272 
273 	td->td_ast &= ~TDAI(tda);
274 }
275 
276 void
ast_sched(struct thread * td,int tda)277 ast_sched(struct thread *td, int tda)
278 {
279 	thread_lock(td);
280 	ast_sched_locked(td, tda);
281 	thread_unlock(td);
282 }
283 
284 void
ast_sched_mask(struct thread * td,int ast)285 ast_sched_mask(struct thread *td, int ast)
286 {
287 	thread_lock(td);
288 	td->td_ast |= ast;
289 	thread_unlock(td);
290 }
291 
292 static bool
ast_handler_calc_tdp_run(struct thread * td,const struct ast_entry * ae)293 ast_handler_calc_tdp_run(struct thread *td, const struct ast_entry *ae)
294 {
295 	return ((ae->ae_flags & ASTR_TDP) == 0 ||
296 	    (td->td_pflags & ae->ae_tdp) != 0);
297 }
298 
299 /*
300  * Process an asynchronous software trap.
301  */
302 static void
ast_handler(struct thread * td,struct trapframe * framep,bool dtor)303 ast_handler(struct thread *td, struct trapframe *framep, bool dtor)
304 {
305 	struct ast_entry *ae;
306 	void (*f)(struct thread *td, int asts);
307 	int a, td_ast;
308 	bool run;
309 
310 	if (framep != NULL) {
311 		kmsan_mark(framep, sizeof(*framep), KMSAN_STATE_INITED);
312 		td->td_frame = framep;
313 	}
314 
315 	if (__predict_true(!dtor)) {
316 		WITNESS_WARN(WARN_PANIC, NULL, "Returning to user mode");
317 		mtx_assert(&Giant, MA_NOTOWNED);
318 		THREAD_LOCK_ASSERT(td, MA_NOTOWNED);
319 
320 		/*
321 		 * This updates the td_ast for the checks below in one
322 		 * atomic operation with turning off all scheduled AST's.
323 		 * If another AST is triggered while we are handling the
324 		 * AST's saved in td_ast, the td_ast is again non-zero and
325 		 * ast() will be called again.
326 		 */
327 		thread_lock(td);
328 		td_ast = td->td_ast;
329 		td->td_ast = 0;
330 		thread_unlock(td);
331 	} else {
332 		/*
333 		 * The td thread's td_lock is not guaranteed to exist,
334 		 * the thread might be not initialized enough when it's
335 		 * destructor is called.  It is safe to read and
336 		 * update td_ast without locking since the thread is
337 		 * not runnable or visible to other threads.
338 		 */
339 		td_ast = td->td_ast;
340 		td->td_ast = 0;
341 	}
342 
343 	CTR3(KTR_SYSC, "ast: thread %p (pid %d, %s)", td, td->td_proc->p_pid,
344             td->td_proc->p_comm);
345 	KASSERT(framep == NULL || TRAPF_USERMODE(framep),
346 	    ("ast in kernel mode"));
347 
348 	for (a = 0; a < nitems(ast_entries); a++) {
349 		ae = &ast_entries[a];
350 		f = ae->ae_f;
351 		if (f == NULL)
352 			continue;
353 		atomic_interrupt_fence();
354 
355 		run = false;
356 		if (__predict_false(framep == NULL)) {
357 			if ((ae->ae_flags & ASTR_KCLEAR) != 0)
358 				run = ast_handler_calc_tdp_run(td, ae);
359 		} else {
360 			if ((ae->ae_flags & ASTR_UNCOND) != 0)
361 				run = true;
362 			else if ((ae->ae_flags & ASTR_ASTF_REQUIRED) != 0 &&
363 			    (td_ast & TDAI(a)) != 0)
364 				run = ast_handler_calc_tdp_run(td, ae);
365 		}
366 		if (run)
367 			f(td, td_ast);
368 	}
369 }
370 
371 void
ast(struct trapframe * framep)372 ast(struct trapframe *framep)
373 {
374 	struct thread *td;
375 
376 	td = curthread;
377 	ast_handler(td, framep, false);
378 	userret(td, framep);
379 }
380 
381 void
ast_kclear(struct thread * td)382 ast_kclear(struct thread *td)
383 {
384 	ast_handler(td, NULL, td != curthread);
385 }
386 
387 const char *
syscallname(struct proc * p,u_int code)388 syscallname(struct proc *p, u_int code)
389 {
390 	static const char unknown[] = "unknown";
391 	struct sysentvec *sv;
392 
393 	sv = p->p_sysent;
394 	if (sv->sv_syscallnames == NULL || code >= sv->sv_size)
395 		return (unknown);
396 	return (sv->sv_syscallnames[code]);
397 }
398