1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * x86 implementation of rethook. Mostly copied from arch/x86/kernel/kprobes/core.c. 4 */ 5 #include <linux/bug.h> 6 #include <linux/rethook.h> 7 #include <linux/kprobes.h> 8 #include <linux/objtool.h> 9 10 #include "kprobes/common.h" 11 12 __visible void arch_rethook_trampoline_callback(struct pt_regs *regs); 13 14 #ifndef ANNOTATE_NOENDBR 15 #define ANNOTATE_NOENDBR 16 #endif 17 18 /* 19 * When a target function returns, this code saves registers and calls 20 * arch_rethook_trampoline_callback(), which calls the rethook handler. 21 */ 22 asm( 23 ".text\n" 24 ".global arch_rethook_trampoline\n" 25 ".type arch_rethook_trampoline, @function\n" 26 "arch_rethook_trampoline:\n" 27 #ifdef CONFIG_X86_64 28 ANNOTATE_NOENDBR /* This is only jumped from ret instruction */ 29 /* Push a fake return address to tell the unwinder it's a rethook. */ 30 " pushq $arch_rethook_trampoline\n" 31 UNWIND_HINT_FUNC 32 /* Save the 'sp - 8', this will be fixed later. */ 33 " pushq %rsp\n" 34 " pushfq\n" 35 SAVE_REGS_STRING 36 " movq %rsp, %rdi\n" 37 " call arch_rethook_trampoline_callback\n" 38 RESTORE_REGS_STRING 39 /* In the callback function, 'regs->flags' is copied to 'regs->sp'. */ 40 " addq $8, %rsp\n" 41 " popfq\n" 42 #else 43 /* Push a fake return address to tell the unwinder it's a rethook. */ 44 " pushl $arch_rethook_trampoline\n" 45 UNWIND_HINT_FUNC 46 /* Save the 'sp - 4', this will be fixed later. */ 47 " pushl %esp\n" 48 " pushfl\n" 49 SAVE_REGS_STRING 50 " movl %esp, %eax\n" 51 " call arch_rethook_trampoline_callback\n" 52 RESTORE_REGS_STRING 53 /* In the callback function, 'regs->flags' is copied to 'regs->sp'. */ 54 " addl $4, %esp\n" 55 " popfl\n" 56 #endif 57 ASM_RET 58 ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n" 59 ); 60 NOKPROBE_SYMBOL(arch_rethook_trampoline); 61 62 /* 63 * Called from arch_rethook_trampoline 64 */ 65 __used __visible void arch_rethook_trampoline_callback(struct pt_regs *regs) 66 { 67 unsigned long *frame_pointer; 68 69 /* fixup registers */ 70 regs->cs = __KERNEL_CS; 71 #ifdef CONFIG_X86_32 72 regs->gs = 0; 73 #endif 74 regs->ip = (unsigned long)&arch_rethook_trampoline; 75 regs->orig_ax = ~0UL; 76 regs->sp += sizeof(long); 77 frame_pointer = ®s->sp + 1; 78 79 /* 80 * The return address at 'frame_pointer' is recovered by the 81 * arch_rethook_fixup_return() which called from this 82 * rethook_trampoline_handler(). 83 */ 84 rethook_trampoline_handler(regs, (unsigned long)frame_pointer); 85 86 /* 87 * Copy FLAGS to 'pt_regs::sp' so that arch_rethook_trapmoline() 88 * can do RET right after POPF. 89 */ 90 regs->sp = regs->flags; 91 } 92 NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); 93 94 /* 95 * arch_rethook_trampoline() skips updating frame pointer. The frame pointer 96 * saved in arch_rethook_trampoline_callback() points to the real caller 97 * function's frame pointer. Thus the arch_rethook_trampoline() doesn't have 98 * a standard stack frame with CONFIG_FRAME_POINTER=y. 99 * Let's mark it non-standard function. Anyway, FP unwinder can correctly 100 * unwind without the hint. 101 */ 102 STACK_FRAME_NON_STANDARD_FP(arch_rethook_trampoline); 103 104 /* This is called from rethook_trampoline_handler(). */ 105 void arch_rethook_fixup_return(struct pt_regs *regs, 106 unsigned long correct_ret_addr) 107 { 108 unsigned long *frame_pointer = ®s->sp + 1; 109 110 /* Replace fake return address with real one. */ 111 *frame_pointer = correct_ret_addr; 112 } 113 NOKPROBE_SYMBOL(arch_rethook_fixup_return); 114 115 void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) 116 { 117 unsigned long *stack = (unsigned long *)regs->sp; 118 119 rh->ret_addr = stack[0]; 120 rh->frame = regs->sp; 121 122 /* Replace the return addr with trampoline addr */ 123 stack[0] = (unsigned long) arch_rethook_trampoline; 124 } 125 NOKPROBE_SYMBOL(arch_rethook_prepare); 126