1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_STATIC_CALL_H 3 #define _LINUX_STATIC_CALL_H 4 5 /* 6 * Static call support 7 * 8 * Static calls use code patching to hard-code function pointers into direct 9 * branch instructions. They give the flexibility of function pointers, but 10 * with improved performance. This is especially important for cases where 11 * retpolines would otherwise be used, as retpolines can significantly impact 12 * performance. 13 * 14 * 15 * API overview: 16 * 17 * DECLARE_STATIC_CALL(name, func); 18 * DEFINE_STATIC_CALL(name, func); 19 * DEFINE_STATIC_CALL_NULL(name, typename); 20 * static_call(name)(args...); 21 * static_call_cond(name)(args...); 22 * static_call_update(name, func); 23 * 24 * Usage example: 25 * 26 * # Start with the following functions (with identical prototypes): 27 * int func_a(int arg1, int arg2); 28 * int func_b(int arg1, int arg2); 29 * 30 * # Define a 'my_name' reference, associated with func_a() by default 31 * DEFINE_STATIC_CALL(my_name, func_a); 32 * 33 * # Call func_a() 34 * static_call(my_name)(arg1, arg2); 35 * 36 * # Update 'my_name' to point to func_b() 37 * static_call_update(my_name, &func_b); 38 * 39 * # Call func_b() 40 * static_call(my_name)(arg1, arg2); 41 * 42 * 43 * Implementation details: 44 * 45 * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). 46 * Otherwise basic indirect calls are used (with function pointers). 47 * 48 * Each static_call() site calls into a trampoline associated with the name. 49 * The trampoline has a direct branch to the default function. Updates to a 50 * name will modify the trampoline's branch destination. 51 * 52 * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites 53 * themselves will be patched at runtime to call the functions directly, 54 * rather than calling through the trampoline. This requires objtool or a 55 * compiler plugin to detect all the static_call() sites and annotate them 56 * in the .static_call_sites section. 57 * 58 * 59 * Notes on NULL function pointers: 60 * 61 * Static_call()s support NULL functions, with many of the caveats that 62 * regular function pointers have. 63 * 64 * Clearly calling a NULL function pointer is 'BAD', so too for 65 * static_call()s (although when HAVE_STATIC_CALL it might not be immediately 66 * fatal). A NULL static_call can be the result of: 67 * 68 * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int)); 69 * 70 * which is equivalent to declaring a NULL function pointer with just a 71 * typename: 72 * 73 * void (*my_func_ptr)(int arg1) = NULL; 74 * 75 * or using static_call_update() with a NULL function. In both cases the 76 * HAVE_STATIC_CALL implementation will patch the trampoline with a RET 77 * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE 78 * architectures can patch the trampoline call to a NOP. 79 * 80 * In all cases, any argument evaluation is unconditional. Unlike a regular 81 * conditional function pointer call: 82 * 83 * if (my_func_ptr) 84 * my_func_ptr(arg1) 85 * 86 * where the argument evaludation also depends on the pointer value. 87 * 88 * When calling a static_call that can be NULL, use: 89 * 90 * static_call_cond(name)(arg1); 91 * 92 * which will include the required value tests to avoid NULL-pointer 93 * dereferences. 94 */ 95 96 #include <linux/types.h> 97 #include <linux/cpu.h> 98 #include <linux/static_call_types.h> 99 100 #ifdef CONFIG_HAVE_STATIC_CALL 101 #include <asm/static_call.h> 102 103 /* 104 * Either @site or @tramp can be NULL. 105 */ 106 extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail); 107 108 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name) 109 110 #else 111 #define STATIC_CALL_TRAMP_ADDR(name) NULL 112 #endif 113 114 #define static_call_update(name, func) \ 115 ({ \ 116 BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name))); \ 117 __static_call_update(&STATIC_CALL_KEY(name), \ 118 STATIC_CALL_TRAMP_ADDR(name), func); \ 119 }) 120 121 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE 122 123 extern int __init static_call_init(void); 124 125 struct static_call_mod { 126 struct static_call_mod *next; 127 struct module *mod; /* for vmlinux, mod == NULL */ 128 struct static_call_site *sites; 129 }; 130 131 /* For finding the key associated with a trampoline */ 132 struct static_call_tramp_key { 133 s32 tramp; 134 s32 key; 135 }; 136 137 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); 138 extern int static_call_mod_init(struct module *mod); 139 extern int static_call_text_reserved(void *start, void *end); 140 141 extern long __static_call_return0(void); 142 143 #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ 144 DECLARE_STATIC_CALL(name, _func); \ 145 struct static_call_key STATIC_CALL_KEY(name) = { \ 146 .func = _func_init, \ 147 .type = 1, \ 148 }; \ 149 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init) 150 151 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 152 DECLARE_STATIC_CALL(name, _func); \ 153 struct static_call_key STATIC_CALL_KEY(name) = { \ 154 .func = NULL, \ 155 .type = 1, \ 156 }; \ 157 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) 158 159 #define static_call_cond(name) (void)__static_call(name) 160 161 #define EXPORT_STATIC_CALL(name) \ 162 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ 163 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 164 #define EXPORT_STATIC_CALL_GPL(name) \ 165 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ 166 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 167 168 /* Leave the key unexported, so modules can't change static call targets: */ 169 #define EXPORT_STATIC_CALL_TRAMP(name) \ 170 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \ 171 ARCH_ADD_TRAMP_KEY(name) 172 #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ 173 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \ 174 ARCH_ADD_TRAMP_KEY(name) 175 176 #elif defined(CONFIG_HAVE_STATIC_CALL) 177 178 static inline int static_call_init(void) { return 0; } 179 180 #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ 181 DECLARE_STATIC_CALL(name, _func); \ 182 struct static_call_key STATIC_CALL_KEY(name) = { \ 183 .func = _func_init, \ 184 }; \ 185 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init) 186 187 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 188 DECLARE_STATIC_CALL(name, _func); \ 189 struct static_call_key STATIC_CALL_KEY(name) = { \ 190 .func = NULL, \ 191 }; \ 192 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) 193 194 #define static_call_cond(name) (void)__static_call(name) 195 196 static inline 197 void __static_call_update(struct static_call_key *key, void *tramp, void *func) 198 { 199 cpus_read_lock(); 200 WRITE_ONCE(key->func, func); 201 arch_static_call_transform(NULL, tramp, func, false); 202 cpus_read_unlock(); 203 } 204 205 static inline int static_call_text_reserved(void *start, void *end) 206 { 207 return 0; 208 } 209 210 static inline long __static_call_return0(void) 211 { 212 return 0; 213 } 214 215 #define EXPORT_STATIC_CALL(name) \ 216 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ 217 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 218 #define EXPORT_STATIC_CALL_GPL(name) \ 219 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ 220 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 221 222 /* Leave the key unexported, so modules can't change static call targets: */ 223 #define EXPORT_STATIC_CALL_TRAMP(name) \ 224 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 225 #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ 226 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 227 228 #else /* Generic implementation */ 229 230 static inline int static_call_init(void) { return 0; } 231 232 static inline long __static_call_return0(void) 233 { 234 return 0; 235 } 236 237 #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ 238 DECLARE_STATIC_CALL(name, _func); \ 239 struct static_call_key STATIC_CALL_KEY(name) = { \ 240 .func = _func_init, \ 241 } 242 243 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 244 DECLARE_STATIC_CALL(name, _func); \ 245 struct static_call_key STATIC_CALL_KEY(name) = { \ 246 .func = NULL, \ 247 } 248 249 static inline void __static_call_nop(void) { } 250 251 /* 252 * This horrific hack takes care of two things: 253 * 254 * - it ensures the compiler will only load the function pointer ONCE, 255 * which avoids a reload race. 256 * 257 * - it ensures the argument evaluation is unconditional, similar 258 * to the HAVE_STATIC_CALL variant. 259 * 260 * Sadly current GCC/Clang (10 for both) do not optimize this properly 261 * and will emit an indirect call for the NULL case :-( 262 */ 263 #define __static_call_cond(name) \ 264 ({ \ 265 void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \ 266 if (!func) \ 267 func = &__static_call_nop; \ 268 (typeof(STATIC_CALL_TRAMP(name))*)func; \ 269 }) 270 271 #define static_call_cond(name) (void)__static_call_cond(name) 272 273 static inline 274 void __static_call_update(struct static_call_key *key, void *tramp, void *func) 275 { 276 WRITE_ONCE(key->func, func); 277 } 278 279 static inline int static_call_text_reserved(void *start, void *end) 280 { 281 return 0; 282 } 283 284 #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name)) 285 #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)) 286 287 #endif /* CONFIG_HAVE_STATIC_CALL */ 288 289 #define DEFINE_STATIC_CALL(name, _func) \ 290 __DEFINE_STATIC_CALL(name, _func, _func) 291 292 #define DEFINE_STATIC_CALL_RET0(name, _func) \ 293 __DEFINE_STATIC_CALL(name, _func, __static_call_return0) 294 295 #endif /* _LINUX_STATIC_CALL_H */ 296