1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_ENTRYCOMMON_H 3 #define __LINUX_ENTRYCOMMON_H 4 5 #include <linux/tracehook.h> 6 #include <linux/syscalls.h> 7 #include <linux/seccomp.h> 8 #include <linux/sched.h> 9 10 #include <asm/entry-common.h> 11 12 /* 13 * Define dummy _TIF work flags if not defined by the architecture or for 14 * disabled functionality. 15 */ 16 #ifndef _TIF_SYSCALL_EMU 17 # define _TIF_SYSCALL_EMU (0) 18 #endif 19 20 #ifndef _TIF_SYSCALL_TRACEPOINT 21 # define _TIF_SYSCALL_TRACEPOINT (0) 22 #endif 23 24 #ifndef _TIF_SECCOMP 25 # define _TIF_SECCOMP (0) 26 #endif 27 28 #ifndef _TIF_SYSCALL_AUDIT 29 # define _TIF_SYSCALL_AUDIT (0) 30 #endif 31 32 #ifndef _TIF_PATCH_PENDING 33 # define _TIF_PATCH_PENDING (0) 34 #endif 35 36 #ifndef _TIF_UPROBE 37 # define _TIF_UPROBE (0) 38 #endif 39 40 /* 41 * TIF flags handled in syscall_enter_from_user_mode() 42 */ 43 #ifndef ARCH_SYSCALL_ENTER_WORK 44 # define ARCH_SYSCALL_ENTER_WORK (0) 45 #endif 46 47 #define SYSCALL_ENTER_WORK \ 48 (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ 49 _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_EMU | \ 50 ARCH_SYSCALL_ENTER_WORK) 51 52 /* 53 * TIF flags handled in syscall_exit_to_user_mode() 54 */ 55 #ifndef ARCH_SYSCALL_EXIT_WORK 56 # define ARCH_SYSCALL_EXIT_WORK (0) 57 #endif 58 59 #define SYSCALL_EXIT_WORK \ 60 (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ 61 _TIF_SYSCALL_TRACEPOINT | ARCH_SYSCALL_EXIT_WORK) 62 63 /* 64 * TIF flags handled in exit_to_user_mode_loop() 65 */ 66 #ifndef ARCH_EXIT_TO_USER_MODE_WORK 67 # define ARCH_EXIT_TO_USER_MODE_WORK (0) 68 #endif 69 70 #define EXIT_TO_USER_MODE_WORK \ 71 (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ 72 _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | \ 73 ARCH_EXIT_TO_USER_MODE_WORK) 74 75 /** 76 * arch_check_user_regs - Architecture specific sanity check for user mode regs 77 * @regs: Pointer to currents pt_regs 78 * 79 * Defaults to an empty implementation. Can be replaced by architecture 80 * specific code. 81 * 82 * Invoked from syscall_enter_from_user_mode() in the non-instrumentable 83 * section. Use __always_inline so the compiler cannot push it out of line 84 * and make it instrumentable. 85 */ 86 static __always_inline void arch_check_user_regs(struct pt_regs *regs); 87 88 #ifndef arch_check_user_regs 89 static __always_inline void arch_check_user_regs(struct pt_regs *regs) {} 90 #endif 91 92 /** 93 * arch_syscall_enter_tracehook - Wrapper around tracehook_report_syscall_entry() 94 * @regs: Pointer to currents pt_regs 95 * 96 * Returns: 0 on success or an error code to skip the syscall. 97 * 98 * Defaults to tracehook_report_syscall_entry(). Can be replaced by 99 * architecture specific code. 100 * 101 * Invoked from syscall_enter_from_user_mode() 102 */ 103 static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs); 104 105 #ifndef arch_syscall_enter_tracehook 106 static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs) 107 { 108 return tracehook_report_syscall_entry(regs); 109 } 110 #endif 111 112 /** 113 * syscall_enter_from_user_mode_prepare - Establish state and enable interrupts 114 * @regs: Pointer to currents pt_regs 115 * 116 * Invoked from architecture specific syscall entry code with interrupts 117 * disabled. The calling code has to be non-instrumentable. When the 118 * function returns all state is correct, interrupts are enabled and the 119 * subsequent functions can be instrumented. 120 * 121 * This handles lockdep, RCU (context tracking) and tracing state. 122 * 123 * This is invoked when there is extra architecture specific functionality 124 * to be done between establishing state and handling user mode entry work. 125 */ 126 void syscall_enter_from_user_mode_prepare(struct pt_regs *regs); 127 128 /** 129 * syscall_enter_from_user_mode_work - Check and handle work before invoking 130 * a syscall 131 * @regs: Pointer to currents pt_regs 132 * @syscall: The syscall number 133 * 134 * Invoked from architecture specific syscall entry code with interrupts 135 * enabled after invoking syscall_enter_from_user_mode_prepare() and extra 136 * architecture specific work. 137 * 138 * Returns: The original or a modified syscall number 139 * 140 * If the returned syscall number is -1 then the syscall should be 141 * skipped. In this case the caller may invoke syscall_set_error() or 142 * syscall_set_return_value() first. If neither of those are called and -1 143 * is returned, then the syscall will fail with ENOSYS. 144 * 145 * It handles the following work items: 146 * 147 * 1) TIF flag dependent invocations of arch_syscall_enter_tracehook(), 148 * __secure_computing(), trace_sys_enter() 149 * 2) Invocation of audit_syscall_entry() 150 */ 151 long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall); 152 153 /** 154 * syscall_enter_from_user_mode - Establish state and check and handle work 155 * before invoking a syscall 156 * @regs: Pointer to currents pt_regs 157 * @syscall: The syscall number 158 * 159 * Invoked from architecture specific syscall entry code with interrupts 160 * disabled. The calling code has to be non-instrumentable. When the 161 * function returns all state is correct, interrupts are enabled and the 162 * subsequent functions can be instrumented. 163 * 164 * This is combination of syscall_enter_from_user_mode_prepare() and 165 * syscall_enter_from_user_mode_work(). 166 * 167 * Returns: The original or a modified syscall number. See 168 * syscall_enter_from_user_mode_work() for further explanation. 169 */ 170 long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall); 171 172 /** 173 * local_irq_enable_exit_to_user - Exit to user variant of local_irq_enable() 174 * @ti_work: Cached TIF flags gathered with interrupts disabled 175 * 176 * Defaults to local_irq_enable(). Can be supplied by architecture specific 177 * code. 178 */ 179 static inline void local_irq_enable_exit_to_user(unsigned long ti_work); 180 181 #ifndef local_irq_enable_exit_to_user 182 static inline void local_irq_enable_exit_to_user(unsigned long ti_work) 183 { 184 local_irq_enable(); 185 } 186 #endif 187 188 /** 189 * local_irq_disable_exit_to_user - Exit to user variant of local_irq_disable() 190 * 191 * Defaults to local_irq_disable(). Can be supplied by architecture specific 192 * code. 193 */ 194 static inline void local_irq_disable_exit_to_user(void); 195 196 #ifndef local_irq_disable_exit_to_user 197 static inline void local_irq_disable_exit_to_user(void) 198 { 199 local_irq_disable(); 200 } 201 #endif 202 203 /** 204 * arch_exit_to_user_mode_work - Architecture specific TIF work for exit 205 * to user mode. 206 * @regs: Pointer to currents pt_regs 207 * @ti_work: Cached TIF flags gathered with interrupts disabled 208 * 209 * Invoked from exit_to_user_mode_loop() with interrupt enabled 210 * 211 * Defaults to NOOP. Can be supplied by architecture specific code. 212 */ 213 static inline void arch_exit_to_user_mode_work(struct pt_regs *regs, 214 unsigned long ti_work); 215 216 #ifndef arch_exit_to_user_mode_work 217 static inline void arch_exit_to_user_mode_work(struct pt_regs *regs, 218 unsigned long ti_work) 219 { 220 } 221 #endif 222 223 /** 224 * arch_exit_to_user_mode_prepare - Architecture specific preparation for 225 * exit to user mode. 226 * @regs: Pointer to currents pt_regs 227 * @ti_work: Cached TIF flags gathered with interrupts disabled 228 * 229 * Invoked from exit_to_user_mode_prepare() with interrupt disabled as the last 230 * function before return. Defaults to NOOP. 231 */ 232 static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, 233 unsigned long ti_work); 234 235 #ifndef arch_exit_to_user_mode_prepare 236 static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, 237 unsigned long ti_work) 238 { 239 } 240 #endif 241 242 /** 243 * arch_exit_to_user_mode - Architecture specific final work before 244 * exit to user mode. 245 * 246 * Invoked from exit_to_user_mode() with interrupt disabled as the last 247 * function before return. Defaults to NOOP. 248 * 249 * This needs to be __always_inline because it is non-instrumentable code 250 * invoked after context tracking switched to user mode. 251 * 252 * An architecture implementation must not do anything complex, no locking 253 * etc. The main purpose is for speculation mitigations. 254 */ 255 static __always_inline void arch_exit_to_user_mode(void); 256 257 #ifndef arch_exit_to_user_mode 258 static __always_inline void arch_exit_to_user_mode(void) { } 259 #endif 260 261 /** 262 * arch_do_signal_or_restart - Architecture specific signal delivery function 263 * @regs: Pointer to currents pt_regs 264 * @has_signal: actual signal to handle 265 * 266 * Invoked from exit_to_user_mode_loop(). 267 */ 268 void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal); 269 270 /** 271 * arch_syscall_exit_tracehook - Wrapper around tracehook_report_syscall_exit() 272 * @regs: Pointer to currents pt_regs 273 * @step: Indicator for single step 274 * 275 * Defaults to tracehook_report_syscall_exit(). Can be replaced by 276 * architecture specific code. 277 * 278 * Invoked from syscall_exit_to_user_mode() 279 */ 280 static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step); 281 282 #ifndef arch_syscall_exit_tracehook 283 static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step) 284 { 285 tracehook_report_syscall_exit(regs, step); 286 } 287 #endif 288 289 /** 290 * syscall_exit_to_user_mode - Handle work before returning to user mode 291 * @regs: Pointer to currents pt_regs 292 * 293 * Invoked with interrupts enabled and fully valid regs. Returns with all 294 * work handled, interrupts disabled such that the caller can immediately 295 * switch to user mode. Called from architecture specific syscall and ret 296 * from fork code. 297 * 298 * The call order is: 299 * 1) One-time syscall exit work: 300 * - rseq syscall exit 301 * - audit 302 * - syscall tracing 303 * - tracehook (single stepping) 304 * 305 * 2) Preparatory work 306 * - Exit to user mode loop (common TIF handling). Invokes 307 * arch_exit_to_user_mode_work() for architecture specific TIF work 308 * - Architecture specific one time work arch_exit_to_user_mode_prepare() 309 * - Address limit and lockdep checks 310 * 311 * 3) Final transition (lockdep, tracing, context tracking, RCU). Invokes 312 * arch_exit_to_user_mode() to handle e.g. speculation mitigations 313 */ 314 void syscall_exit_to_user_mode(struct pt_regs *regs); 315 316 /** 317 * irqentry_enter_from_user_mode - Establish state before invoking the irq handler 318 * @regs: Pointer to currents pt_regs 319 * 320 * Invoked from architecture specific entry code with interrupts disabled. 321 * Can only be called when the interrupt entry came from user mode. The 322 * calling code must be non-instrumentable. When the function returns all 323 * state is correct and the subsequent functions can be instrumented. 324 * 325 * The function establishes state (lockdep, RCU (context tracking), tracing) 326 */ 327 void irqentry_enter_from_user_mode(struct pt_regs *regs); 328 329 /** 330 * irqentry_exit_to_user_mode - Interrupt exit work 331 * @regs: Pointer to current's pt_regs 332 * 333 * Invoked with interrupts disbled and fully valid regs. Returns with all 334 * work handled, interrupts disabled such that the caller can immediately 335 * switch to user mode. Called from architecture specific interrupt 336 * handling code. 337 * 338 * The call order is #2 and #3 as described in syscall_exit_to_user_mode(). 339 * Interrupt exit is not invoking #1 which is the syscall specific one time 340 * work. 341 */ 342 void irqentry_exit_to_user_mode(struct pt_regs *regs); 343 344 #ifndef irqentry_state 345 typedef struct irqentry_state { 346 bool exit_rcu; 347 } irqentry_state_t; 348 #endif 349 350 /** 351 * irqentry_enter - Handle state tracking on ordinary interrupt entries 352 * @regs: Pointer to pt_regs of interrupted context 353 * 354 * Invokes: 355 * - lockdep irqflag state tracking as low level ASM entry disabled 356 * interrupts. 357 * 358 * - Context tracking if the exception hit user mode. 359 * 360 * - The hardirq tracer to keep the state consistent as low level ASM 361 * entry disabled interrupts. 362 * 363 * As a precondition, this requires that the entry came from user mode, 364 * idle, or a kernel context in which RCU is watching. 365 * 366 * For kernel mode entries RCU handling is done conditional. If RCU is 367 * watching then the only RCU requirement is to check whether the tick has 368 * to be restarted. If RCU is not watching then rcu_irq_enter() has to be 369 * invoked on entry and rcu_irq_exit() on exit. 370 * 371 * Avoiding the rcu_irq_enter/exit() calls is an optimization but also 372 * solves the problem of kernel mode pagefaults which can schedule, which 373 * is not possible after invoking rcu_irq_enter() without undoing it. 374 * 375 * For user mode entries irqentry_enter_from_user_mode() is invoked to 376 * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit 377 * would not be possible. 378 * 379 * Returns: An opaque object that must be passed to idtentry_exit() 380 */ 381 irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs); 382 383 /** 384 * irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt 385 * 386 * Conditional reschedule with additional sanity checks. 387 */ 388 void irqentry_exit_cond_resched(void); 389 390 /** 391 * irqentry_exit - Handle return from exception that used irqentry_enter() 392 * @regs: Pointer to pt_regs (exception entry regs) 393 * @state: Return value from matching call to irqentry_enter() 394 * 395 * Depending on the return target (kernel/user) this runs the necessary 396 * preemption and work checks if possible and reguired and returns to 397 * the caller with interrupts disabled and no further work pending. 398 * 399 * This is the last action before returning to the low level ASM code which 400 * just needs to return to the appropriate context. 401 * 402 * Counterpart to irqentry_enter(). 403 */ 404 void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t state); 405 406 #endif 407