1 #ifndef _LINUX_JUMP_LABEL_H 2 #define _LINUX_JUMP_LABEL_H 3 4 /* 5 * Jump label support 6 * 7 * Copyright (C) 2009-2012 Jason Baron <[email protected]> 8 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra 9 * 10 * DEPRECATED API: 11 * 12 * The use of 'struct static_key' directly, is now DEPRECATED. In addition 13 * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following: 14 * 15 * struct static_key false = STATIC_KEY_INIT_FALSE; 16 * struct static_key true = STATIC_KEY_INIT_TRUE; 17 * static_key_true() 18 * static_key_false() 19 * 20 * The updated API replacements are: 21 * 22 * DEFINE_STATIC_KEY_TRUE(key); 23 * DEFINE_STATIC_KEY_FALSE(key); 24 * static_branch_likely() 25 * static_branch_unlikely() 26 * 27 * Jump labels provide an interface to generate dynamic branches using 28 * self-modifying code. Assuming toolchain and architecture support, if we 29 * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)", 30 * an "if (static_branch_unlikely(&key))" statement is an unconditional branch 31 * (which defaults to false - and the true block is placed out of line). 32 * Similarly, we can define an initially true key via 33 * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same 34 * "if (static_branch_unlikely(&key))", in which case we will generate an 35 * unconditional branch to the out-of-line true branch. Keys that are 36 * initially true or false can be using in both static_branch_unlikely() 37 * and static_branch_likely() statements. 38 * 39 * At runtime we can change the branch target by setting the key 40 * to true via a call to static_branch_enable(), or false using 41 * static_branch_disable(). If the direction of the branch is switched by 42 * these calls then we run-time modify the branch target via a 43 * no-op -> jump or jump -> no-op conversion. For example, for an 44 * initially false key that is used in an "if (static_branch_unlikely(&key))" 45 * statement, setting the key to true requires us to patch in a jump 46 * to the out-of-line of true branch. 47 * 48 * In addition to static_branch_{enable,disable}, we can also reference count 49 * the key or branch direction via static_branch_{inc,dec}. Thus, 50 * static_branch_inc() can be thought of as a 'make more true' and 51 * static_branch_dec() as a 'make more false'. 52 * 53 * Since this relies on modifying code, the branch modifying functions 54 * must be considered absolute slow paths (machine wide synchronization etc.). 55 * OTOH, since the affected branches are unconditional, their runtime overhead 56 * will be absolutely minimal, esp. in the default (off) case where the total 57 * effect is a single NOP of appropriate size. The on case will patch in a jump 58 * to the out-of-line block. 59 * 60 * When the control is directly exposed to userspace, it is prudent to delay the 61 * decrement to avoid high frequency code modifications which can (and do) 62 * cause significant performance degradation. Struct static_key_deferred and 63 * static_key_slow_dec_deferred() provide for this. 64 * 65 * Lacking toolchain and or architecture support, static keys fall back to a 66 * simple conditional branch. 67 * 68 * Additional babbling in: Documentation/static-keys.txt 69 */ 70 71 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) 72 # define HAVE_JUMP_LABEL 73 #endif 74 75 #ifndef __ASSEMBLY__ 76 77 #include <linux/types.h> 78 #include <linux/compiler.h> 79 80 extern bool static_key_initialized; 81 82 #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \ 83 "%s used before call to jump_label_init", \ 84 __func__) 85 86 #ifdef HAVE_JUMP_LABEL 87 88 struct static_key { 89 atomic_t enabled; 90 /* Set lsb bit to 1 if branch is default true, 0 ot */ 91 struct jump_entry *entries; 92 #ifdef CONFIG_MODULES 93 struct static_key_mod *next; 94 #endif 95 }; 96 97 #else 98 struct static_key { 99 atomic_t enabled; 100 }; 101 #endif /* HAVE_JUMP_LABEL */ 102 #endif /* __ASSEMBLY__ */ 103 104 #ifdef HAVE_JUMP_LABEL 105 #include <asm/jump_label.h> 106 #endif 107 108 #ifndef __ASSEMBLY__ 109 110 enum jump_label_type { 111 JUMP_LABEL_NOP = 0, 112 JUMP_LABEL_JMP, 113 }; 114 115 struct module; 116 117 #ifdef HAVE_JUMP_LABEL 118 119 #define JUMP_TYPE_FALSE 0UL 120 #define JUMP_TYPE_TRUE 1UL 121 #define JUMP_TYPE_MASK 1UL 122 123 static __always_inline bool static_key_false(struct static_key *key) 124 { 125 return arch_static_branch(key, false); 126 } 127 128 static __always_inline bool static_key_true(struct static_key *key) 129 { 130 return !arch_static_branch(key, true); 131 } 132 133 extern struct jump_entry __start___jump_table[]; 134 extern struct jump_entry __stop___jump_table[]; 135 136 extern void jump_label_init(void); 137 extern void jump_label_lock(void); 138 extern void jump_label_unlock(void); 139 extern void arch_jump_label_transform(struct jump_entry *entry, 140 enum jump_label_type type); 141 extern void arch_jump_label_transform_static(struct jump_entry *entry, 142 enum jump_label_type type); 143 extern int jump_label_text_reserved(void *start, void *end); 144 extern void static_key_slow_inc(struct static_key *key); 145 extern void static_key_slow_dec(struct static_key *key); 146 extern void jump_label_apply_nops(struct module *mod); 147 extern int static_key_count(struct static_key *key); 148 extern void static_key_enable(struct static_key *key); 149 extern void static_key_disable(struct static_key *key); 150 151 /* 152 * We should be using ATOMIC_INIT() for initializing .enabled, but 153 * the inclusion of atomic.h is problematic for inclusion of jump_label.h 154 * in 'low-level' headers. Thus, we are initializing .enabled with a 155 * raw value, but have added a BUILD_BUG_ON() to catch any issues in 156 * jump_label_init() see: kernel/jump_label.c. 157 */ 158 #define STATIC_KEY_INIT_TRUE \ 159 { .enabled = { 1 }, \ 160 .entries = (void *)JUMP_TYPE_TRUE } 161 #define STATIC_KEY_INIT_FALSE \ 162 { .enabled = { 0 }, \ 163 .entries = (void *)JUMP_TYPE_FALSE } 164 165 #else /* !HAVE_JUMP_LABEL */ 166 167 #include <linux/atomic.h> 168 #include <linux/bug.h> 169 170 static inline int static_key_count(struct static_key *key) 171 { 172 return atomic_read(&key->enabled); 173 } 174 175 static __always_inline void jump_label_init(void) 176 { 177 static_key_initialized = true; 178 } 179 180 static __always_inline bool static_key_false(struct static_key *key) 181 { 182 if (unlikely(static_key_count(key) > 0)) 183 return true; 184 return false; 185 } 186 187 static __always_inline bool static_key_true(struct static_key *key) 188 { 189 if (likely(static_key_count(key) > 0)) 190 return true; 191 return false; 192 } 193 194 static inline void static_key_slow_inc(struct static_key *key) 195 { 196 STATIC_KEY_CHECK_USE(); 197 atomic_inc(&key->enabled); 198 } 199 200 static inline void static_key_slow_dec(struct static_key *key) 201 { 202 STATIC_KEY_CHECK_USE(); 203 atomic_dec(&key->enabled); 204 } 205 206 static inline int jump_label_text_reserved(void *start, void *end) 207 { 208 return 0; 209 } 210 211 static inline void jump_label_lock(void) {} 212 static inline void jump_label_unlock(void) {} 213 214 static inline int jump_label_apply_nops(struct module *mod) 215 { 216 return 0; 217 } 218 219 static inline void static_key_enable(struct static_key *key) 220 { 221 int count = static_key_count(key); 222 223 WARN_ON_ONCE(count < 0 || count > 1); 224 225 if (!count) 226 static_key_slow_inc(key); 227 } 228 229 static inline void static_key_disable(struct static_key *key) 230 { 231 int count = static_key_count(key); 232 233 WARN_ON_ONCE(count < 0 || count > 1); 234 235 if (count) 236 static_key_slow_dec(key); 237 } 238 239 #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) } 240 #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) } 241 242 #endif /* HAVE_JUMP_LABEL */ 243 244 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE 245 #define jump_label_enabled static_key_enabled 246 247 /* -------------------------------------------------------------------------- */ 248 249 /* 250 * Two type wrappers around static_key, such that we can use compile time 251 * type differentiation to emit the right code. 252 * 253 * All the below code is macros in order to play type games. 254 */ 255 256 struct static_key_true { 257 struct static_key key; 258 }; 259 260 struct static_key_false { 261 struct static_key key; 262 }; 263 264 #define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, } 265 #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, } 266 267 #define DEFINE_STATIC_KEY_TRUE(name) \ 268 struct static_key_true name = STATIC_KEY_TRUE_INIT 269 270 #define DEFINE_STATIC_KEY_FALSE(name) \ 271 struct static_key_false name = STATIC_KEY_FALSE_INIT 272 273 extern bool ____wrong_branch_error(void); 274 275 #define static_key_enabled(x) \ 276 ({ \ 277 if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \ 278 !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\ 279 !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 280 ____wrong_branch_error(); \ 281 static_key_count((struct static_key *)x) > 0; \ 282 }) 283 284 #ifdef HAVE_JUMP_LABEL 285 286 /* 287 * Combine the right initial value (type) with the right branch order 288 * to generate the desired result. 289 * 290 * 291 * type\branch| likely (1) | unlikely (0) 292 * -----------+-----------------------+------------------ 293 * | | 294 * true (1) | ... | ... 295 * | NOP | JMP L 296 * | <br-stmts> | 1: ... 297 * | L: ... | 298 * | | 299 * | | L: <br-stmts> 300 * | | jmp 1b 301 * | | 302 * -----------+-----------------------+------------------ 303 * | | 304 * false (0) | ... | ... 305 * | JMP L | NOP 306 * | <br-stmts> | 1: ... 307 * | L: ... | 308 * | | 309 * | | L: <br-stmts> 310 * | | jmp 1b 311 * | | 312 * -----------+-----------------------+------------------ 313 * 314 * The initial value is encoded in the LSB of static_key::entries, 315 * type: 0 = false, 1 = true. 316 * 317 * The branch type is encoded in the LSB of jump_entry::key, 318 * branch: 0 = unlikely, 1 = likely. 319 * 320 * This gives the following logic table: 321 * 322 * enabled type branch instuction 323 * -----------------------------+----------- 324 * 0 0 0 | NOP 325 * 0 0 1 | JMP 326 * 0 1 0 | NOP 327 * 0 1 1 | JMP 328 * 329 * 1 0 0 | JMP 330 * 1 0 1 | NOP 331 * 1 1 0 | JMP 332 * 1 1 1 | NOP 333 * 334 * Which gives the following functions: 335 * 336 * dynamic: instruction = enabled ^ branch 337 * static: instruction = type ^ branch 338 * 339 * See jump_label_type() / jump_label_init_type(). 340 */ 341 342 #define static_branch_likely(x) \ 343 ({ \ 344 bool branch; \ 345 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \ 346 branch = !arch_static_branch(&(x)->key, true); \ 347 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 348 branch = !arch_static_branch_jump(&(x)->key, true); \ 349 else \ 350 branch = ____wrong_branch_error(); \ 351 branch; \ 352 }) 353 354 #define static_branch_unlikely(x) \ 355 ({ \ 356 bool branch; \ 357 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \ 358 branch = arch_static_branch_jump(&(x)->key, false); \ 359 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 360 branch = arch_static_branch(&(x)->key, false); \ 361 else \ 362 branch = ____wrong_branch_error(); \ 363 branch; \ 364 }) 365 366 #else /* !HAVE_JUMP_LABEL */ 367 368 #define static_branch_likely(x) likely(static_key_enabled(&(x)->key)) 369 #define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key)) 370 371 #endif /* HAVE_JUMP_LABEL */ 372 373 /* 374 * Advanced usage; refcount, branch is enabled when: count != 0 375 */ 376 377 #define static_branch_inc(x) static_key_slow_inc(&(x)->key) 378 #define static_branch_dec(x) static_key_slow_dec(&(x)->key) 379 380 /* 381 * Normal usage; boolean enable/disable. 382 */ 383 384 #define static_branch_enable(x) static_key_enable(&(x)->key) 385 #define static_branch_disable(x) static_key_disable(&(x)->key) 386 387 #endif /* _LINUX_JUMP_LABEL_H */ 388 389 #endif /* __ASSEMBLY__ */ 390