187 		  struct pt_regs *regs,	long error_code)  in do_trap_no_signal()  argument
189 	if (v8086_mode(regs)) {  in do_trap_no_signal()
195 			if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,  in do_trap_no_signal()
199 	} else if (!user_mode(regs)) {  in do_trap_no_signal()
200 		if (fixup_exception(regs, trapnr, error_code, 0))  in do_trap_no_signal()
205 		die(str, regs, error_code);  in do_trap_no_signal()
207 		if (fixup_vdso_exception(regs, trapnr, error_code, 0))  in do_trap_no_signal()
228 			struct pt_regs *regs, long error_code)  in show_signal()  argument
234 			regs->ip, regs->sp, error_code);  in show_signal()
235 		print_vma_addr(KERN_CONT " in ", regs->ip);  in show_signal()
241 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,  in do_trap()  argument
246 	if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))  in do_trap()
249 	show_signal(tsk, signr, "trap ", str, regs, error_code);  in do_trap()
258 static void do_error_trap(struct pt_regs *regs, long error_code, char *str,  in do_error_trap()  argument
263 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=  in do_error_trap()
265 		cond_local_irq_enable(regs);  in do_error_trap()
266 		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);  in do_error_trap()
267 		cond_local_irq_disable(regs);  in do_error_trap()
281 static __always_inline void __user *error_get_trap_addr(struct pt_regs *regs)  in error_get_trap_addr()  argument
283 	return (void __user *)uprobe_get_trap_addr(regs);  in error_get_trap_addr()
288 	do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE,  in DEFINE_IDTENTRY()
289 		      FPE_INTDIV, error_get_trap_addr(regs));  in DEFINE_IDTENTRY()
294 	do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);  in DEFINE_IDTENTRY()
298 void handle_invalid_op(struct pt_regs *regs)  in handle_invalid_op()  argument
300 static inline void handle_invalid_op(struct pt_regs *regs)  in handle_invalid_op()
303 	do_error_trap(regs, 0, "invalid opcode", X86_TRAP_UD, SIGILL,  in handle_invalid_op()
304 		      ILL_ILLOPN, error_get_trap_addr(regs));  in handle_invalid_op()
307 static noinstr bool handle_bug(struct pt_regs *regs)  in handle_bug()  argument
309 	unsigned long addr = regs->ip;  in handle_bug()
327 	kmsan_unpoison_entry_regs(regs);  in handle_bug()
332 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
337 		if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {  in handle_bug()
345 		if (handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {  in handle_bug()
354 				report_ubsan_failure(regs, ud_imm),  in handle_bug()
355 				(void *)regs->ip);  in handle_bug()
369 		if (regs->ip == addr)  in handle_bug()
370 			regs->ip += ud_len;  in handle_bug()
372 		regs->ip = addr;  in handle_bug()
375 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
391 	if (!user_mode(regs) && handle_bug(regs))  in DEFINE_IDTENTRY_RAW()
394 	state = irqentry_enter(regs);  in DEFINE_IDTENTRY_RAW()
396 	handle_invalid_op(regs);  in DEFINE_IDTENTRY_RAW()
398 	irqentry_exit(regs, state);  in DEFINE_IDTENTRY_RAW()
403 	do_error_trap(regs, 0, "coprocessor segment overrun",  in DEFINE_IDTENTRY()
409 	do_error_trap(regs, error_code, "invalid TSS", X86_TRAP_TS, SIGSEGV,  in DEFINE_IDTENTRY_ERRORCODE()
415 	do_error_trap(regs, error_code, "segment not present", X86_TRAP_NP,  in DEFINE_IDTENTRY_ERRORCODE()
421 	do_error_trap(regs, error_code, "stack segment", X86_TRAP_SS, SIGBUS,  in DEFINE_IDTENTRY_ERRORCODE()
429 	if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP)  in DEFINE_IDTENTRY_ERRORCODE()
432 	if (!user_mode(regs))  in DEFINE_IDTENTRY_ERRORCODE()
433 		die("Split lock detected\n", regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
437 	if (handle_user_split_lock(regs, error_code))  in DEFINE_IDTENTRY_ERRORCODE()
440 	do_trap(X86_TRAP_AC, SIGBUS, "alignment check", regs,  in DEFINE_IDTENTRY_ERRORCODE()
448 __visible void __noreturn handle_stack_overflow(struct pt_regs *regs,  in handle_stack_overflow()  argument
457 	die("stack guard page", regs, 0);  in handle_stack_overflow()
525 	if (((long)regs->sp >> P4D_SHIFT) == ESPFIX_PGD_ENTRY &&  in DEFINE_IDTENTRY_DF()
526 		regs->cs == __KERNEL_CS &&  in DEFINE_IDTENTRY_DF()
527 		regs->ip == (unsigned long)native_irq_return_iret)  in DEFINE_IDTENTRY_DF()
530 		unsigned long *p = (unsigned long *)regs->sp;  in DEFINE_IDTENTRY_DF()
556 		regs->ip = (unsigned long)asm_exc_general_protection;  in DEFINE_IDTENTRY_DF()
557 		regs->sp = (unsigned long)&gpregs->orig_ax;  in DEFINE_IDTENTRY_DF()
563 	irqentry_nmi_enter(regs);  in DEFINE_IDTENTRY_DF()
565 	notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);  in DEFINE_IDTENTRY_DF()
609 		handle_stack_overflow(regs, address, &info);  in DEFINE_IDTENTRY_DF()
613 	die("double fault", regs, error_code);  in DEFINE_IDTENTRY_DF()
621 	if (notify_die(DIE_TRAP, "bounds", regs, 0,  in DEFINE_IDTENTRY()
624 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
626 	if (!user_mode(regs))  in DEFINE_IDTENTRY()
627 		die("bounds", regs, 0);  in DEFINE_IDTENTRY()
629 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL);  in DEFINE_IDTENTRY()
631 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
645 static enum kernel_gp_hint get_kernel_gp_address(struct pt_regs *regs,  in get_kernel_gp_address()  argument
652 	if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip,  in get_kernel_gp_address()
660 	*addr = (unsigned long)insn_get_addr_ref(&insn, regs);  in get_kernel_gp_address()
680 static bool fixup_iopl_exception(struct pt_regs *regs)  in fixup_iopl_exception()  argument
689 	if (insn_get_effective_ip(regs, &ip))  in fixup_iopl_exception()
706 	regs->ip += 1;  in fixup_iopl_exception()
761 static bool gp_try_fixup_and_notify(struct pt_regs *regs, int trapnr,  in gp_try_fixup_and_notify()  argument
765 	if (fixup_exception(regs, trapnr, error_code, address))  in gp_try_fixup_and_notify()
776 	    kprobe_fault_handler(regs, trapnr))  in gp_try_fixup_and_notify()
779 	return notify_die(DIE_GPF, str, regs, error_code, trapnr, SIGSEGV) == NOTIFY_STOP;  in gp_try_fixup_and_notify()
782 static void gp_user_force_sig_segv(struct pt_regs *regs, int trapnr,  in gp_user_force_sig_segv()  argument
787 	show_signal(current, SIGSEGV, "", str, regs, error_code);  in gp_user_force_sig_segv()
797 	if (user_mode(regs) && try_fixup_enqcmd_gp())  in DEFINE_IDTENTRY_ERRORCODE()
800 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY_ERRORCODE()
803 		if (user_mode(regs) && fixup_umip_exception(regs))  in DEFINE_IDTENTRY_ERRORCODE()
807 	if (v8086_mode(regs)) {  in DEFINE_IDTENTRY_ERRORCODE()
809 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
814 	if (user_mode(regs)) {  in DEFINE_IDTENTRY_ERRORCODE()
815 		if (fixup_iopl_exception(regs))  in DEFINE_IDTENTRY_ERRORCODE()
818 		if (fixup_vdso_exception(regs, X86_TRAP_GP, error_code, 0))  in DEFINE_IDTENTRY_ERRORCODE()
821 		gp_user_force_sig_segv(regs, X86_TRAP_GP, error_code, desc);  in DEFINE_IDTENTRY_ERRORCODE()
825 	if (gp_try_fixup_and_notify(regs, X86_TRAP_GP, error_code, desc, 0))  in DEFINE_IDTENTRY_ERRORCODE()
831 		hint = get_kernel_gp_address(regs, &gp_addr);  in DEFINE_IDTENTRY_ERRORCODE()
846 	die_addr(desc, regs, error_code, gp_addr);  in DEFINE_IDTENTRY_ERRORCODE()
849 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY_ERRORCODE()
852 static bool do_int3(struct pt_regs *regs)  in do_int3()  argument
857 	if (kgdb_ll_trap(DIE_INT3, "int3", regs, 0, X86_TRAP_BP,  in do_int3()
863 	if (kprobe_int3_handler(regs))  in do_int3()
866 	res = notify_die(DIE_INT3, "int3", regs, 0, X86_TRAP_BP, SIGTRAP);  in do_int3()
872 static void do_int3_user(struct pt_regs *regs)  in do_int3_user()  argument
874 	if (do_int3(regs))  in do_int3_user()
877 	cond_local_irq_enable(regs);  in do_int3_user()
878 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, 0, 0, NULL);  in do_int3_user()
879 	cond_local_irq_disable(regs);  in do_int3_user()
889 	if (poke_int3_handler(regs))  in DEFINE_IDTENTRY_RAW()
899 	if (user_mode(regs)) {  in DEFINE_IDTENTRY_RAW()
900 		irqentry_enter_from_user_mode(regs);  in DEFINE_IDTENTRY_RAW()
902 		do_int3_user(regs);  in DEFINE_IDTENTRY_RAW()
904 		irqentry_exit_to_user_mode(regs);  in DEFINE_IDTENTRY_RAW()
906 		irqentry_state_t irq_state = irqentry_nmi_enter(regs);  in DEFINE_IDTENTRY_RAW()
909 		if (!do_int3(regs))  in DEFINE_IDTENTRY_RAW()
910 			die("int3", regs, 0);  in DEFINE_IDTENTRY_RAW()
912 		irqentry_nmi_exit(regs, irq_state);  in DEFINE_IDTENTRY_RAW()
924 	struct pt_regs *regs = (struct pt_regs *)current_top_of_stack() - 1;  in sync_regs()  local
925 	if (regs != eregs)  in sync_regs()
926 		*regs = *eregs;  in sync_regs()
927 	return regs;  in sync_regs()
931 asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *regs)  in vc_switch_off_ist()  argument
941 	if (ip_within_syscall_gap(regs)) {  in vc_switch_off_ist()
951 	sp    = regs->sp;  in vc_switch_off_ist()
967 	*regs_ret = *regs;  in vc_switch_off_ist()
1001 static bool is_sysenter_singlestep(struct pt_regs *regs)  in is_sysenter_singlestep()  argument
1012 	return (regs->ip - (unsigned long)__begin_SYSENTER_singlestep_region) <  in is_sysenter_singlestep()
1016 	return (regs->ip - (unsigned long)entry_SYSENTER_compat) <  in is_sysenter_singlestep()
1071 static bool notify_debug(struct pt_regs *regs, unsigned long *dr6)  in notify_debug()  argument
1080 	if (notify_die(DIE_DEBUG, "debug", regs, (long)dr6, 0, SIGTRAP) == NOTIFY_STOP)  in notify_debug()
1086 static noinstr void exc_debug_kernel(struct pt_regs *regs, unsigned long dr6)  in exc_debug_kernel()  argument
1105 	irqentry_state_t irq_state = irqentry_nmi_enter(regs);  in exc_debug_kernel()
1112 	WARN_ON_ONCE(user_mode(regs));  in exc_debug_kernel()
1133 	    (dr6 & DR_STEP) && is_sysenter_singlestep(regs))  in exc_debug_kernel()
1142 	if (notify_debug(regs, &dr6))  in exc_debug_kernel()
1157 		regs->flags &= ~X86_EFLAGS_TF;  in exc_debug_kernel()
1160 	irqentry_nmi_exit(regs, irq_state);  in exc_debug_kernel()
1165 static noinstr void exc_debug_user(struct pt_regs *regs, unsigned long dr6)  in exc_debug_user()  argument
1173 	WARN_ON_ONCE(!user_mode(regs));  in exc_debug_user()
1184 	irqentry_enter_from_user_mode(regs);  in exc_debug_user()
1210 	if (notify_debug(regs, &dr6))  in exc_debug_user()
1216 	if (v8086_mode(regs)) {  in exc_debug_user()
1217 		handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);  in exc_debug_user()
1223 		handle_bus_lock(regs);  in exc_debug_user()
1228 		send_sigtrap(regs, 0, get_si_code(dr6));  in exc_debug_user()
1234 	irqentry_exit_to_user_mode(regs);  in exc_debug_user()
1241 	exc_debug_kernel(regs, debug_read_clear_dr6());  in DEFINE_IDTENTRY_DEBUG()
1247 	exc_debug_user(regs, debug_read_clear_dr6());  in DEFINE_IDTENTRY_DEBUG_USER()
1268 	unsigned long dr6 = fred_event_data(regs);  in DEFINE_FREDENTRY_DEBUG()
1270 	if (user_mode(regs))  in DEFINE_FREDENTRY_DEBUG()
1271 		exc_debug_user(regs, dr6);  in DEFINE_FREDENTRY_DEBUG()
1273 		exc_debug_kernel(regs, dr6);  in DEFINE_FREDENTRY_DEBUG()
1283 	if (user_mode(regs))  in DEFINE_IDTENTRY_RAW()
1284 		exc_debug_user(regs, dr6);  in DEFINE_IDTENTRY_RAW()
1286 		exc_debug_kernel(regs, dr6);  in DEFINE_IDTENTRY_RAW()
1295 static void math_error(struct pt_regs *regs, int trapnr)  in math_error()  argument
1303 	cond_local_irq_enable(regs);  in math_error()
1305 	if (!user_mode(regs)) {  in math_error()
1306 		if (fixup_exception(regs, trapnr, 0, 0))  in math_error()
1312 		if (notify_die(DIE_TRAP, str, regs, 0, trapnr,  in math_error()
1314 			die(str, regs, 0);  in math_error()
1332 	if (fixup_vdso_exception(regs, trapnr, 0, 0))  in math_error()
1336 			(void __user *)uprobe_get_trap_addr(regs));  in math_error()
1338 	cond_local_irq_disable(regs);  in math_error()
1343 	math_error(regs, X86_TRAP_MF);  in DEFINE_IDTENTRY()
1351 			__exc_general_protection(regs, 0);  in DEFINE_IDTENTRY()
1355 	math_error(regs, X86_TRAP_XF);  in DEFINE_IDTENTRY()
1381 static bool handle_xfd_event(struct pt_regs *regs)  in handle_xfd_event()  argument
1396 	if (WARN_ON(!user_mode(regs)))  in handle_xfd_event()
1405 		force_sig_fault(SIGILL, ILL_ILLOPC, error_get_trap_addr(regs));  in handle_xfd_event()
1420 	if (handle_xfd_event(regs))  in DEFINE_IDTENTRY()
1427 		cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
1429 		info.regs = regs;  in DEFINE_IDTENTRY()
1432 		cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
1447 		die("unexpected #NM exception", regs, 0);  in DEFINE_IDTENTRY()
1455 static void ve_raise_fault(struct pt_regs *regs, long error_code,  in ve_raise_fault()  argument
1458 	if (user_mode(regs)) {  in ve_raise_fault()
1459 		gp_user_force_sig_segv(regs, X86_TRAP_VE, error_code, VE_FAULT_STR);  in ve_raise_fault()
1463 	if (gp_try_fixup_and_notify(regs, X86_TRAP_VE, error_code,  in ve_raise_fault()
1468 	die_addr(VE_FAULT_STR, regs, error_code, address);  in ve_raise_fault()
1525 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
1531 	if (!tdx_handle_virt_exception(regs, &ve))  in DEFINE_IDTENTRY()
1532 		ve_raise_fault(regs, 0, ve.gla);  in DEFINE_IDTENTRY()
1534 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
1543 	if (notify_die(DIE_TRAP, "iret exception", regs, 0,  in DEFINE_IDTENTRY_SW()
1545 		do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, 0,  in DEFINE_IDTENTRY_SW()