1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_STATIC_CALL_H 3 #define _LINUX_STATIC_CALL_H 4 5 /* 6 * Static call support 7 * 8 * Static calls use code patching to hard-code function pointers into direct 9 * branch instructions. They give the flexibility of function pointers, but 10 * with improved performance. This is especially important for cases where 11 * retpolines would otherwise be used, as retpolines can significantly impact 12 * performance. 13 * 14 * 15 * API overview: 16 * 17 * DECLARE_STATIC_CALL(name, func); 18 * DEFINE_STATIC_CALL(name, func); 19 * DEFINE_STATIC_CALL_NULL(name, typename); 20 * static_call(name)(args...); 21 * static_call_cond(name)(args...); 22 * static_call_update(name, func); 23 * 24 * Usage example: 25 * 26 * # Start with the following functions (with identical prototypes): 27 * int func_a(int arg1, int arg2); 28 * int func_b(int arg1, int arg2); 29 * 30 * # Define a 'my_name' reference, associated with func_a() by default 31 * DEFINE_STATIC_CALL(my_name, func_a); 32 * 33 * # Call func_a() 34 * static_call(my_name)(arg1, arg2); 35 * 36 * # Update 'my_name' to point to func_b() 37 * static_call_update(my_name, &func_b); 38 * 39 * # Call func_b() 40 * static_call(my_name)(arg1, arg2); 41 * 42 * 43 * Implementation details: 44 * 45 * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). 46 * Otherwise basic indirect calls are used (with function pointers). 47 * 48 * Each static_call() site calls into a trampoline associated with the name. 49 * The trampoline has a direct branch to the default function. Updates to a 50 * name will modify the trampoline's branch destination. 51 * 52 * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites 53 * themselves will be patched at runtime to call the functions directly, 54 * rather than calling through the trampoline. This requires objtool or a 55 * compiler plugin to detect all the static_call() sites and annotate them 56 * in the .static_call_sites section. 57 * 58 * 59 * Notes on NULL function pointers: 60 * 61 * Static_call()s support NULL functions, with many of the caveats that 62 * regular function pointers have. 63 * 64 * Clearly calling a NULL function pointer is 'BAD', so too for 65 * static_call()s (although when HAVE_STATIC_CALL it might not be immediately 66 * fatal). A NULL static_call can be the result of: 67 * 68 * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int)); 69 * 70 * which is equivalent to declaring a NULL function pointer with just a 71 * typename: 72 * 73 * void (*my_func_ptr)(int arg1) = NULL; 74 * 75 * or using static_call_update() with a NULL function. In both cases the 76 * HAVE_STATIC_CALL implementation will patch the trampoline with a RET 77 * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE 78 * architectures can patch the trampoline call to a NOP. 79 * 80 * In all cases, any argument evaluation is unconditional. Unlike a regular 81 * conditional function pointer call: 82 * 83 * if (my_func_ptr) 84 * my_func_ptr(arg1) 85 * 86 * where the argument evaludation also depends on the pointer value. 87 * 88 * When calling a static_call that can be NULL, use: 89 * 90 * static_call_cond(name)(arg1); 91 * 92 * which will include the required value tests to avoid NULL-pointer 93 * dereferences. 94 */ 95 96 #include <linux/types.h> 97 #include <linux/cpu.h> 98 #include <linux/static_call_types.h> 99 100 #ifdef CONFIG_HAVE_STATIC_CALL 101 #include <asm/static_call.h> 102 103 /* 104 * Either @site or @tramp can be NULL. 105 */ 106 extern void arch_static_call_transform(void *site, void *tramp, void *func); 107 108 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name) 109 110 /* 111 * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from 112 * the symbol table so that objtool can reference it when it generates the 113 * .static_call_sites section. 114 */ 115 #define __static_call(name) \ 116 ({ \ 117 __ADDRESSABLE(STATIC_CALL_KEY(name)); \ 118 &STATIC_CALL_TRAMP(name); \ 119 }) 120 121 #else 122 #define STATIC_CALL_TRAMP_ADDR(name) NULL 123 #endif 124 125 126 #define DECLARE_STATIC_CALL(name, func) \ 127 extern struct static_call_key STATIC_CALL_KEY(name); \ 128 extern typeof(func) STATIC_CALL_TRAMP(name); 129 130 #define static_call_update(name, func) \ 131 ({ \ 132 BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name))); \ 133 __static_call_update(&STATIC_CALL_KEY(name), \ 134 STATIC_CALL_TRAMP_ADDR(name), func); \ 135 }) 136 137 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE 138 139 struct static_call_mod { 140 struct static_call_mod *next; 141 struct module *mod; /* for vmlinux, mod == NULL */ 142 struct static_call_site *sites; 143 }; 144 145 struct static_call_key { 146 void *func; 147 struct static_call_mod *mods; 148 }; 149 150 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); 151 extern int static_call_mod_init(struct module *mod); 152 extern int static_call_text_reserved(void *start, void *end); 153 154 #define DEFINE_STATIC_CALL(name, _func) \ 155 DECLARE_STATIC_CALL(name, _func); \ 156 struct static_call_key STATIC_CALL_KEY(name) = { \ 157 .func = _func, \ 158 .mods = NULL, \ 159 }; \ 160 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) 161 162 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 163 DECLARE_STATIC_CALL(name, _func); \ 164 struct static_call_key STATIC_CALL_KEY(name) = { \ 165 .func = NULL, \ 166 .type = 1, \ 167 }; \ 168 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) 169 170 #define static_call(name) __static_call(name) 171 #define static_call_cond(name) (void)__static_call(name) 172 173 #define EXPORT_STATIC_CALL(name) \ 174 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ 175 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 176 177 #define EXPORT_STATIC_CALL_GPL(name) \ 178 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ 179 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 180 181 #elif defined(CONFIG_HAVE_STATIC_CALL) 182 183 struct static_call_key { 184 void *func; 185 }; 186 187 #define DEFINE_STATIC_CALL(name, _func) \ 188 DECLARE_STATIC_CALL(name, _func); \ 189 struct static_call_key STATIC_CALL_KEY(name) = { \ 190 .func = _func, \ 191 }; \ 192 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) 193 194 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 195 DECLARE_STATIC_CALL(name, _func); \ 196 struct static_call_key STATIC_CALL_KEY(name) = { \ 197 .func = NULL, \ 198 }; \ 199 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) 200 201 #define static_call(name) __static_call(name) 202 #define static_call_cond(name) (void)__static_call(name) 203 204 static inline 205 void __static_call_update(struct static_call_key *key, void *tramp, void *func) 206 { 207 cpus_read_lock(); 208 WRITE_ONCE(key->func, func); 209 arch_static_call_transform(NULL, tramp, func); 210 cpus_read_unlock(); 211 } 212 213 static inline int static_call_text_reserved(void *start, void *end) 214 { 215 return 0; 216 } 217 218 #define EXPORT_STATIC_CALL(name) \ 219 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ 220 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 221 222 #define EXPORT_STATIC_CALL_GPL(name) \ 223 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ 224 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 225 226 #else /* Generic implementation */ 227 228 struct static_call_key { 229 void *func; 230 }; 231 232 #define DEFINE_STATIC_CALL(name, _func) \ 233 DECLARE_STATIC_CALL(name, _func); \ 234 struct static_call_key STATIC_CALL_KEY(name) = { \ 235 .func = _func, \ 236 } 237 238 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 239 DECLARE_STATIC_CALL(name, _func); \ 240 struct static_call_key STATIC_CALL_KEY(name) = { \ 241 .func = NULL, \ 242 } 243 244 #define static_call(name) \ 245 ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func)) 246 247 static inline void __static_call_nop(void) { } 248 249 /* 250 * This horrific hack takes care of two things: 251 * 252 * - it ensures the compiler will only load the function pointer ONCE, 253 * which avoids a reload race. 254 * 255 * - it ensures the argument evaluation is unconditional, similar 256 * to the HAVE_STATIC_CALL variant. 257 * 258 * Sadly current GCC/Clang (10 for both) do not optimize this properly 259 * and will emit an indirect call for the NULL case :-( 260 */ 261 #define __static_call_cond(name) \ 262 ({ \ 263 void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \ 264 if (!func) \ 265 func = &__static_call_nop; \ 266 (typeof(STATIC_CALL_TRAMP(name))*)func; \ 267 }) 268 269 #define static_call_cond(name) (void)__static_call_cond(name) 270 271 static inline 272 void __static_call_update(struct static_call_key *key, void *tramp, void *func) 273 { 274 WRITE_ONCE(key->func, func); 275 } 276 277 static inline int static_call_text_reserved(void *start, void *end) 278 { 279 return 0; 280 } 281 282 #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name)) 283 #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)) 284 285 #endif /* CONFIG_HAVE_STATIC_CALL */ 286 287 #endif /* _LINUX_STATIC_CALL_H */ 288