1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_ENTRYCOMMON_H 3 #define __LINUX_ENTRYCOMMON_H 4 5 #include <linux/tracehook.h> 6 #include <linux/syscalls.h> 7 #include <linux/seccomp.h> 8 #include <linux/sched.h> 9 10 #include <asm/entry-common.h> 11 12 /* 13 * Define dummy _TIF work flags if not defined by the architecture or for 14 * disabled functionality. 15 */ 16 #ifndef _TIF_PATCH_PENDING 17 # define _TIF_PATCH_PENDING (0) 18 #endif 19 20 #ifndef _TIF_UPROBE 21 # define _TIF_UPROBE (0) 22 #endif 23 24 #ifndef _TIF_NOTIFY_SIGNAL 25 # define _TIF_NOTIFY_SIGNAL (0) 26 #endif 27 28 /* 29 * TIF flags handled in syscall_enter_from_user_mode() 30 */ 31 #ifndef ARCH_SYSCALL_ENTER_WORK 32 # define ARCH_SYSCALL_ENTER_WORK (0) 33 #endif 34 35 #define SYSCALL_ENTER_WORK ARCH_SYSCALL_ENTER_WORK 36 37 /* 38 * TIF flags handled in syscall_exit_to_user_mode() 39 */ 40 #ifndef ARCH_SYSCALL_EXIT_WORK 41 # define ARCH_SYSCALL_EXIT_WORK (0) 42 #endif 43 44 #define SYSCALL_EXIT_WORK ARCH_SYSCALL_EXIT_WORK 45 46 #define SYSCALL_WORK_ENTER (SYSCALL_WORK_SECCOMP | \ 47 SYSCALL_WORK_SYSCALL_TRACEPOINT | \ 48 SYSCALL_WORK_SYSCALL_TRACE | \ 49 SYSCALL_WORK_SYSCALL_EMU | \ 50 SYSCALL_WORK_SYSCALL_AUDIT) 51 #define SYSCALL_WORK_EXIT (SYSCALL_WORK_SYSCALL_TRACEPOINT | \ 52 SYSCALL_WORK_SYSCALL_TRACE | \ 53 SYSCALL_WORK_SYSCALL_AUDIT) 54 55 /* 56 * TIF flags handled in exit_to_user_mode_loop() 57 */ 58 #ifndef ARCH_EXIT_TO_USER_MODE_WORK 59 # define ARCH_EXIT_TO_USER_MODE_WORK (0) 60 #endif 61 62 #define EXIT_TO_USER_MODE_WORK \ 63 (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ 64 _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | \ 65 ARCH_EXIT_TO_USER_MODE_WORK) 66 67 /** 68 * arch_check_user_regs - Architecture specific sanity check for user mode regs 69 * @regs: Pointer to currents pt_regs 70 * 71 * Defaults to an empty implementation. Can be replaced by architecture 72 * specific code. 73 * 74 * Invoked from syscall_enter_from_user_mode() in the non-instrumentable 75 * section. Use __always_inline so the compiler cannot push it out of line 76 * and make it instrumentable. 77 */ 78 static __always_inline void arch_check_user_regs(struct pt_regs *regs); 79 80 #ifndef arch_check_user_regs 81 static __always_inline void arch_check_user_regs(struct pt_regs *regs) {} 82 #endif 83 84 /** 85 * arch_syscall_enter_tracehook - Wrapper around tracehook_report_syscall_entry() 86 * @regs: Pointer to currents pt_regs 87 * 88 * Returns: 0 on success or an error code to skip the syscall. 89 * 90 * Defaults to tracehook_report_syscall_entry(). Can be replaced by 91 * architecture specific code. 92 * 93 * Invoked from syscall_enter_from_user_mode() 94 */ 95 static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs); 96 97 #ifndef arch_syscall_enter_tracehook 98 static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs) 99 { 100 return tracehook_report_syscall_entry(regs); 101 } 102 #endif 103 104 /** 105 * syscall_enter_from_user_mode_prepare - Establish state and enable interrupts 106 * @regs: Pointer to currents pt_regs 107 * 108 * Invoked from architecture specific syscall entry code with interrupts 109 * disabled. The calling code has to be non-instrumentable. When the 110 * function returns all state is correct, interrupts are enabled and the 111 * subsequent functions can be instrumented. 112 * 113 * This handles lockdep, RCU (context tracking) and tracing state. 114 * 115 * This is invoked when there is extra architecture specific functionality 116 * to be done between establishing state and handling user mode entry work. 117 */ 118 void syscall_enter_from_user_mode_prepare(struct pt_regs *regs); 119 120 /** 121 * syscall_enter_from_user_mode_work - Check and handle work before invoking 122 * a syscall 123 * @regs: Pointer to currents pt_regs 124 * @syscall: The syscall number 125 * 126 * Invoked from architecture specific syscall entry code with interrupts 127 * enabled after invoking syscall_enter_from_user_mode_prepare() and extra 128 * architecture specific work. 129 * 130 * Returns: The original or a modified syscall number 131 * 132 * If the returned syscall number is -1 then the syscall should be 133 * skipped. In this case the caller may invoke syscall_set_error() or 134 * syscall_set_return_value() first. If neither of those are called and -1 135 * is returned, then the syscall will fail with ENOSYS. 136 * 137 * It handles the following work items: 138 * 139 * 1) TIF flag dependent invocations of arch_syscall_enter_tracehook(), 140 * __secure_computing(), trace_sys_enter() 141 * 2) Invocation of audit_syscall_entry() 142 */ 143 long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall); 144 145 /** 146 * syscall_enter_from_user_mode - Establish state and check and handle work 147 * before invoking a syscall 148 * @regs: Pointer to currents pt_regs 149 * @syscall: The syscall number 150 * 151 * Invoked from architecture specific syscall entry code with interrupts 152 * disabled. The calling code has to be non-instrumentable. When the 153 * function returns all state is correct, interrupts are enabled and the 154 * subsequent functions can be instrumented. 155 * 156 * This is combination of syscall_enter_from_user_mode_prepare() and 157 * syscall_enter_from_user_mode_work(). 158 * 159 * Returns: The original or a modified syscall number. See 160 * syscall_enter_from_user_mode_work() for further explanation. 161 */ 162 long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall); 163 164 /** 165 * local_irq_enable_exit_to_user - Exit to user variant of local_irq_enable() 166 * @ti_work: Cached TIF flags gathered with interrupts disabled 167 * 168 * Defaults to local_irq_enable(). Can be supplied by architecture specific 169 * code. 170 */ 171 static inline void local_irq_enable_exit_to_user(unsigned long ti_work); 172 173 #ifndef local_irq_enable_exit_to_user 174 static inline void local_irq_enable_exit_to_user(unsigned long ti_work) 175 { 176 local_irq_enable(); 177 } 178 #endif 179 180 /** 181 * local_irq_disable_exit_to_user - Exit to user variant of local_irq_disable() 182 * 183 * Defaults to local_irq_disable(). Can be supplied by architecture specific 184 * code. 185 */ 186 static inline void local_irq_disable_exit_to_user(void); 187 188 #ifndef local_irq_disable_exit_to_user 189 static inline void local_irq_disable_exit_to_user(void) 190 { 191 local_irq_disable(); 192 } 193 #endif 194 195 /** 196 * arch_exit_to_user_mode_work - Architecture specific TIF work for exit 197 * to user mode. 198 * @regs: Pointer to currents pt_regs 199 * @ti_work: Cached TIF flags gathered with interrupts disabled 200 * 201 * Invoked from exit_to_user_mode_loop() with interrupt enabled 202 * 203 * Defaults to NOOP. Can be supplied by architecture specific code. 204 */ 205 static inline void arch_exit_to_user_mode_work(struct pt_regs *regs, 206 unsigned long ti_work); 207 208 #ifndef arch_exit_to_user_mode_work 209 static inline void arch_exit_to_user_mode_work(struct pt_regs *regs, 210 unsigned long ti_work) 211 { 212 } 213 #endif 214 215 /** 216 * arch_exit_to_user_mode_prepare - Architecture specific preparation for 217 * exit to user mode. 218 * @regs: Pointer to currents pt_regs 219 * @ti_work: Cached TIF flags gathered with interrupts disabled 220 * 221 * Invoked from exit_to_user_mode_prepare() with interrupt disabled as the last 222 * function before return. Defaults to NOOP. 223 */ 224 static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, 225 unsigned long ti_work); 226 227 #ifndef arch_exit_to_user_mode_prepare 228 static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, 229 unsigned long ti_work) 230 { 231 } 232 #endif 233 234 /** 235 * arch_exit_to_user_mode - Architecture specific final work before 236 * exit to user mode. 237 * 238 * Invoked from exit_to_user_mode() with interrupt disabled as the last 239 * function before return. Defaults to NOOP. 240 * 241 * This needs to be __always_inline because it is non-instrumentable code 242 * invoked after context tracking switched to user mode. 243 * 244 * An architecture implementation must not do anything complex, no locking 245 * etc. The main purpose is for speculation mitigations. 246 */ 247 static __always_inline void arch_exit_to_user_mode(void); 248 249 #ifndef arch_exit_to_user_mode 250 static __always_inline void arch_exit_to_user_mode(void) { } 251 #endif 252 253 /** 254 * arch_do_signal_or_restart - Architecture specific signal delivery function 255 * @regs: Pointer to currents pt_regs 256 * @has_signal: actual signal to handle 257 * 258 * Invoked from exit_to_user_mode_loop(). 259 */ 260 void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal); 261 262 /** 263 * arch_syscall_exit_tracehook - Wrapper around tracehook_report_syscall_exit() 264 * @regs: Pointer to currents pt_regs 265 * @step: Indicator for single step 266 * 267 * Defaults to tracehook_report_syscall_exit(). Can be replaced by 268 * architecture specific code. 269 * 270 * Invoked from syscall_exit_to_user_mode() 271 */ 272 static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step); 273 274 #ifndef arch_syscall_exit_tracehook 275 static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step) 276 { 277 tracehook_report_syscall_exit(regs, step); 278 } 279 #endif 280 281 /** 282 * syscall_exit_to_user_mode - Handle work before returning to user mode 283 * @regs: Pointer to currents pt_regs 284 * 285 * Invoked with interrupts enabled and fully valid regs. Returns with all 286 * work handled, interrupts disabled such that the caller can immediately 287 * switch to user mode. Called from architecture specific syscall and ret 288 * from fork code. 289 * 290 * The call order is: 291 * 1) One-time syscall exit work: 292 * - rseq syscall exit 293 * - audit 294 * - syscall tracing 295 * - tracehook (single stepping) 296 * 297 * 2) Preparatory work 298 * - Exit to user mode loop (common TIF handling). Invokes 299 * arch_exit_to_user_mode_work() for architecture specific TIF work 300 * - Architecture specific one time work arch_exit_to_user_mode_prepare() 301 * - Address limit and lockdep checks 302 * 303 * 3) Final transition (lockdep, tracing, context tracking, RCU). Invokes 304 * arch_exit_to_user_mode() to handle e.g. speculation mitigations 305 */ 306 void syscall_exit_to_user_mode(struct pt_regs *regs); 307 308 /** 309 * irqentry_enter_from_user_mode - Establish state before invoking the irq handler 310 * @regs: Pointer to currents pt_regs 311 * 312 * Invoked from architecture specific entry code with interrupts disabled. 313 * Can only be called when the interrupt entry came from user mode. The 314 * calling code must be non-instrumentable. When the function returns all 315 * state is correct and the subsequent functions can be instrumented. 316 * 317 * The function establishes state (lockdep, RCU (context tracking), tracing) 318 */ 319 void irqentry_enter_from_user_mode(struct pt_regs *regs); 320 321 /** 322 * irqentry_exit_to_user_mode - Interrupt exit work 323 * @regs: Pointer to current's pt_regs 324 * 325 * Invoked with interrupts disbled and fully valid regs. Returns with all 326 * work handled, interrupts disabled such that the caller can immediately 327 * switch to user mode. Called from architecture specific interrupt 328 * handling code. 329 * 330 * The call order is #2 and #3 as described in syscall_exit_to_user_mode(). 331 * Interrupt exit is not invoking #1 which is the syscall specific one time 332 * work. 333 */ 334 void irqentry_exit_to_user_mode(struct pt_regs *regs); 335 336 #ifndef irqentry_state 337 /** 338 * struct irqentry_state - Opaque object for exception state storage 339 * @exit_rcu: Used exclusively in the irqentry_*() calls; signals whether the 340 * exit path has to invoke rcu_irq_exit(). 341 * @lockdep: Used exclusively in the irqentry_nmi_*() calls; ensures that 342 * lockdep state is restored correctly on exit from nmi. 343 * 344 * This opaque object is filled in by the irqentry_*_enter() functions and 345 * must be passed back into the corresponding irqentry_*_exit() functions 346 * when the exception is complete. 347 * 348 * Callers of irqentry_*_[enter|exit]() must consider this structure opaque 349 * and all members private. Descriptions of the members are provided to aid in 350 * the maintenance of the irqentry_*() functions. 351 */ 352 typedef struct irqentry_state { 353 union { 354 bool exit_rcu; 355 bool lockdep; 356 }; 357 } irqentry_state_t; 358 #endif 359 360 /** 361 * irqentry_enter - Handle state tracking on ordinary interrupt entries 362 * @regs: Pointer to pt_regs of interrupted context 363 * 364 * Invokes: 365 * - lockdep irqflag state tracking as low level ASM entry disabled 366 * interrupts. 367 * 368 * - Context tracking if the exception hit user mode. 369 * 370 * - The hardirq tracer to keep the state consistent as low level ASM 371 * entry disabled interrupts. 372 * 373 * As a precondition, this requires that the entry came from user mode, 374 * idle, or a kernel context in which RCU is watching. 375 * 376 * For kernel mode entries RCU handling is done conditional. If RCU is 377 * watching then the only RCU requirement is to check whether the tick has 378 * to be restarted. If RCU is not watching then rcu_irq_enter() has to be 379 * invoked on entry and rcu_irq_exit() on exit. 380 * 381 * Avoiding the rcu_irq_enter/exit() calls is an optimization but also 382 * solves the problem of kernel mode pagefaults which can schedule, which 383 * is not possible after invoking rcu_irq_enter() without undoing it. 384 * 385 * For user mode entries irqentry_enter_from_user_mode() is invoked to 386 * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit 387 * would not be possible. 388 * 389 * Returns: An opaque object that must be passed to idtentry_exit() 390 */ 391 irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs); 392 393 /** 394 * irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt 395 * 396 * Conditional reschedule with additional sanity checks. 397 */ 398 void irqentry_exit_cond_resched(void); 399 400 /** 401 * irqentry_exit - Handle return from exception that used irqentry_enter() 402 * @regs: Pointer to pt_regs (exception entry regs) 403 * @state: Return value from matching call to irqentry_enter() 404 * 405 * Depending on the return target (kernel/user) this runs the necessary 406 * preemption and work checks if possible and required and returns to 407 * the caller with interrupts disabled and no further work pending. 408 * 409 * This is the last action before returning to the low level ASM code which 410 * just needs to return to the appropriate context. 411 * 412 * Counterpart to irqentry_enter(). 413 */ 414 void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t state); 415 416 /** 417 * irqentry_nmi_enter - Handle NMI entry 418 * @regs: Pointer to currents pt_regs 419 * 420 * Similar to irqentry_enter() but taking care of the NMI constraints. 421 */ 422 irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs); 423 424 /** 425 * irqentry_nmi_exit - Handle return from NMI handling 426 * @regs: Pointer to pt_regs (NMI entry regs) 427 * @irq_state: Return value from matching call to irqentry_nmi_enter() 428 * 429 * Last action before returning to the low level assembly code. 430 * 431 * Counterpart to irqentry_nmi_enter(). 432 */ 433 void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state); 434 435 #endif 436