1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_STATIC_CALL_H 3 #define _LINUX_STATIC_CALL_H 4 5 /* 6 * Static call support 7 * 8 * Static calls use code patching to hard-code function pointers into direct 9 * branch instructions. They give the flexibility of function pointers, but 10 * with improved performance. This is especially important for cases where 11 * retpolines would otherwise be used, as retpolines can significantly impact 12 * performance. 13 * 14 * 15 * API overview: 16 * 17 * DECLARE_STATIC_CALL(name, func); 18 * DEFINE_STATIC_CALL(name, func); 19 * static_call(name)(args...); 20 * static_call_update(name, func); 21 * 22 * Usage example: 23 * 24 * # Start with the following functions (with identical prototypes): 25 * int func_a(int arg1, int arg2); 26 * int func_b(int arg1, int arg2); 27 * 28 * # Define a 'my_name' reference, associated with func_a() by default 29 * DEFINE_STATIC_CALL(my_name, func_a); 30 * 31 * # Call func_a() 32 * static_call(my_name)(arg1, arg2); 33 * 34 * # Update 'my_name' to point to func_b() 35 * static_call_update(my_name, &func_b); 36 * 37 * # Call func_b() 38 * static_call(my_name)(arg1, arg2); 39 * 40 * 41 * Implementation details: 42 * 43 * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). 44 * Otherwise basic indirect calls are used (with function pointers). 45 * 46 * Each static_call() site calls into a trampoline associated with the name. 47 * The trampoline has a direct branch to the default function. Updates to a 48 * name will modify the trampoline's branch destination. 49 * 50 * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites 51 * themselves will be patched at runtime to call the functions directly, 52 * rather than calling through the trampoline. This requires objtool or a 53 * compiler plugin to detect all the static_call() sites and annotate them 54 * in the .static_call_sites section. 55 */ 56 57 #include <linux/types.h> 58 #include <linux/cpu.h> 59 #include <linux/static_call_types.h> 60 61 #ifdef CONFIG_HAVE_STATIC_CALL 62 #include <asm/static_call.h> 63 64 /* 65 * Either @site or @tramp can be NULL. 66 */ 67 extern void arch_static_call_transform(void *site, void *tramp, void *func); 68 69 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name) 70 71 /* 72 * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from 73 * the symbol table so that objtool can reference it when it generates the 74 * .static_call_sites section. 75 */ 76 #define __static_call(name) \ 77 ({ \ 78 __ADDRESSABLE(STATIC_CALL_KEY(name)); \ 79 &STATIC_CALL_TRAMP(name); \ 80 }) 81 82 #else 83 #define STATIC_CALL_TRAMP_ADDR(name) NULL 84 #endif 85 86 87 #define DECLARE_STATIC_CALL(name, func) \ 88 extern struct static_call_key STATIC_CALL_KEY(name); \ 89 extern typeof(func) STATIC_CALL_TRAMP(name); 90 91 #define static_call_update(name, func) \ 92 ({ \ 93 BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name))); \ 94 __static_call_update(&STATIC_CALL_KEY(name), \ 95 STATIC_CALL_TRAMP_ADDR(name), func); \ 96 }) 97 98 #if defined(CONFIG_HAVE_STATIC_CALL) 99 100 struct static_call_key { 101 void *func; 102 }; 103 104 #define DEFINE_STATIC_CALL(name, _func) \ 105 DECLARE_STATIC_CALL(name, _func); \ 106 struct static_call_key STATIC_CALL_KEY(name) = { \ 107 .func = _func, \ 108 }; \ 109 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) 110 111 #define static_call(name) __static_call(name) 112 113 static inline 114 void __static_call_update(struct static_call_key *key, void *tramp, void *func) 115 { 116 cpus_read_lock(); 117 WRITE_ONCE(key->func, func); 118 arch_static_call_transform(NULL, tramp, func); 119 cpus_read_unlock(); 120 } 121 122 #define EXPORT_STATIC_CALL(name) \ 123 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ 124 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 125 126 #define EXPORT_STATIC_CALL_GPL(name) \ 127 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ 128 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 129 130 #else /* Generic implementation */ 131 132 struct static_call_key { 133 void *func; 134 }; 135 136 #define DEFINE_STATIC_CALL(name, _func) \ 137 DECLARE_STATIC_CALL(name, _func); \ 138 struct static_call_key STATIC_CALL_KEY(name) = { \ 139 .func = _func, \ 140 } 141 142 #define static_call(name) \ 143 ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func)) 144 145 static inline 146 void __static_call_update(struct static_call_key *key, void *tramp, void *func) 147 { 148 WRITE_ONCE(key->func, func); 149 } 150 151 #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name)) 152 #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)) 153 154 #endif /* CONFIG_HAVE_STATIC_CALL */ 155 156 #endif /* _LINUX_STATIC_CALL_H */ 157