1 #ifndef _LINUX_JUMP_LABEL_H 2 #define _LINUX_JUMP_LABEL_H 3 4 /* 5 * Jump label support 6 * 7 * Copyright (C) 2009-2012 Jason Baron <[email protected]> 8 * Copyright (C) 2011-2012 Peter Zijlstra <[email protected]> 9 * 10 * DEPRECATED API: 11 * 12 * The use of 'struct static_key' directly, is now DEPRECATED. In addition 13 * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following: 14 * 15 * struct static_key false = STATIC_KEY_INIT_FALSE; 16 * struct static_key true = STATIC_KEY_INIT_TRUE; 17 * static_key_true() 18 * static_key_false() 19 * 20 * The updated API replacements are: 21 * 22 * DEFINE_STATIC_KEY_TRUE(key); 23 * DEFINE_STATIC_KEY_FALSE(key); 24 * static_branch_likely() 25 * static_branch_unlikely() 26 * 27 * Jump labels provide an interface to generate dynamic branches using 28 * self-modifying code. Assuming toolchain and architecture support, if we 29 * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)", 30 * an "if (static_branch_unlikely(&key))" statement is an unconditional branch 31 * (which defaults to false - and the true block is placed out of line). 32 * Similarly, we can define an initially true key via 33 * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same 34 * "if (static_branch_unlikely(&key))", in which case we will generate an 35 * unconditional branch to the out-of-line true branch. Keys that are 36 * initially true or false can be using in both static_branch_unlikely() 37 * and static_branch_likely() statements. 38 * 39 * At runtime we can change the branch target by setting the key 40 * to true via a call to static_branch_enable(), or false using 41 * static_branch_disable(). If the direction of the branch is switched by 42 * these calls then we run-time modify the branch target via a 43 * no-op -> jump or jump -> no-op conversion. For example, for an 44 * initially false key that is used in an "if (static_branch_unlikely(&key))" 45 * statement, setting the key to true requires us to patch in a jump 46 * to the out-of-line of true branch. 47 * 48 * In addition to static_branch_{enable,disable}, we can also reference count 49 * the key or branch direction via static_branch_{inc,dec}. Thus, 50 * static_branch_inc() can be thought of as a 'make more true' and 51 * static_branch_dec() as a 'make more false'. 52 * 53 * Since this relies on modifying code, the branch modifying functions 54 * must be considered absolute slow paths (machine wide synchronization etc.). 55 * OTOH, since the affected branches are unconditional, their runtime overhead 56 * will be absolutely minimal, esp. in the default (off) case where the total 57 * effect is a single NOP of appropriate size. The on case will patch in a jump 58 * to the out-of-line block. 59 * 60 * When the control is directly exposed to userspace, it is prudent to delay the 61 * decrement to avoid high frequency code modifications which can (and do) 62 * cause significant performance degradation. Struct static_key_deferred and 63 * static_key_slow_dec_deferred() provide for this. 64 * 65 * Lacking toolchain and or architecture support, static keys fall back to a 66 * simple conditional branch. 67 * 68 * Additional babbling in: Documentation/static-keys.txt 69 */ 70 71 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) 72 # define HAVE_JUMP_LABEL 73 #endif 74 75 #ifndef __ASSEMBLY__ 76 77 #include <linux/types.h> 78 #include <linux/compiler.h> 79 #include <linux/bug.h> 80 81 extern bool static_key_initialized; 82 83 #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \ 84 "%s used before call to jump_label_init", \ 85 __func__) 86 87 #ifdef HAVE_JUMP_LABEL 88 89 struct static_key { 90 atomic_t enabled; 91 /* Set lsb bit to 1 if branch is default true, 0 ot */ 92 struct jump_entry *entries; 93 #ifdef CONFIG_MODULES 94 struct static_key_mod *next; 95 #endif 96 }; 97 98 #else 99 struct static_key { 100 atomic_t enabled; 101 }; 102 #endif /* HAVE_JUMP_LABEL */ 103 #endif /* __ASSEMBLY__ */ 104 105 #ifdef HAVE_JUMP_LABEL 106 #include <asm/jump_label.h> 107 #endif 108 109 #ifndef __ASSEMBLY__ 110 111 enum jump_label_type { 112 JUMP_LABEL_NOP = 0, 113 JUMP_LABEL_JMP, 114 }; 115 116 struct module; 117 118 #include <linux/atomic.h> 119 120 static inline int static_key_count(struct static_key *key) 121 { 122 return atomic_read(&key->enabled); 123 } 124 125 #ifdef HAVE_JUMP_LABEL 126 127 #define JUMP_TYPE_FALSE 0UL 128 #define JUMP_TYPE_TRUE 1UL 129 #define JUMP_TYPE_MASK 1UL 130 131 static __always_inline bool static_key_false(struct static_key *key) 132 { 133 return arch_static_branch(key, false); 134 } 135 136 static __always_inline bool static_key_true(struct static_key *key) 137 { 138 return !arch_static_branch(key, true); 139 } 140 141 extern struct jump_entry __start___jump_table[]; 142 extern struct jump_entry __stop___jump_table[]; 143 144 extern void jump_label_init(void); 145 extern void jump_label_lock(void); 146 extern void jump_label_unlock(void); 147 extern void arch_jump_label_transform(struct jump_entry *entry, 148 enum jump_label_type type); 149 extern void arch_jump_label_transform_static(struct jump_entry *entry, 150 enum jump_label_type type); 151 extern int jump_label_text_reserved(void *start, void *end); 152 extern void static_key_slow_inc(struct static_key *key); 153 extern void static_key_slow_dec(struct static_key *key); 154 extern void jump_label_apply_nops(struct module *mod); 155 156 #define STATIC_KEY_INIT_TRUE \ 157 { .enabled = ATOMIC_INIT(1), \ 158 .entries = (void *)JUMP_TYPE_TRUE } 159 #define STATIC_KEY_INIT_FALSE \ 160 { .enabled = ATOMIC_INIT(0), \ 161 .entries = (void *)JUMP_TYPE_FALSE } 162 163 #else /* !HAVE_JUMP_LABEL */ 164 165 static __always_inline void jump_label_init(void) 166 { 167 static_key_initialized = true; 168 } 169 170 static __always_inline bool static_key_false(struct static_key *key) 171 { 172 if (unlikely(static_key_count(key) > 0)) 173 return true; 174 return false; 175 } 176 177 static __always_inline bool static_key_true(struct static_key *key) 178 { 179 if (likely(static_key_count(key) > 0)) 180 return true; 181 return false; 182 } 183 184 static inline void static_key_slow_inc(struct static_key *key) 185 { 186 STATIC_KEY_CHECK_USE(); 187 atomic_inc(&key->enabled); 188 } 189 190 static inline void static_key_slow_dec(struct static_key *key) 191 { 192 STATIC_KEY_CHECK_USE(); 193 atomic_dec(&key->enabled); 194 } 195 196 static inline int jump_label_text_reserved(void *start, void *end) 197 { 198 return 0; 199 } 200 201 static inline void jump_label_lock(void) {} 202 static inline void jump_label_unlock(void) {} 203 204 static inline int jump_label_apply_nops(struct module *mod) 205 { 206 return 0; 207 } 208 209 #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) } 210 #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) } 211 212 #endif /* HAVE_JUMP_LABEL */ 213 214 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE 215 #define jump_label_enabled static_key_enabled 216 217 static inline bool static_key_enabled(struct static_key *key) 218 { 219 return static_key_count(key) > 0; 220 } 221 222 static inline void static_key_enable(struct static_key *key) 223 { 224 int count = static_key_count(key); 225 226 WARN_ON_ONCE(count < 0 || count > 1); 227 228 if (!count) 229 static_key_slow_inc(key); 230 } 231 232 static inline void static_key_disable(struct static_key *key) 233 { 234 int count = static_key_count(key); 235 236 WARN_ON_ONCE(count < 0 || count > 1); 237 238 if (count) 239 static_key_slow_dec(key); 240 } 241 242 /* -------------------------------------------------------------------------- */ 243 244 /* 245 * Two type wrappers around static_key, such that we can use compile time 246 * type differentiation to emit the right code. 247 * 248 * All the below code is macros in order to play type games. 249 */ 250 251 struct static_key_true { 252 struct static_key key; 253 }; 254 255 struct static_key_false { 256 struct static_key key; 257 }; 258 259 #define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, } 260 #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, } 261 262 #define DEFINE_STATIC_KEY_TRUE(name) \ 263 struct static_key_true name = STATIC_KEY_TRUE_INIT 264 265 #define DEFINE_STATIC_KEY_FALSE(name) \ 266 struct static_key_false name = STATIC_KEY_FALSE_INIT 267 268 #ifdef HAVE_JUMP_LABEL 269 270 /* 271 * Combine the right initial value (type) with the right branch order 272 * to generate the desired result. 273 * 274 * 275 * type\branch| likely (1) | unlikely (0) 276 * -----------+-----------------------+------------------ 277 * | | 278 * true (1) | ... | ... 279 * | NOP | JMP L 280 * | <br-stmts> | 1: ... 281 * | L: ... | 282 * | | 283 * | | L: <br-stmts> 284 * | | jmp 1b 285 * | | 286 * -----------+-----------------------+------------------ 287 * | | 288 * false (0) | ... | ... 289 * | JMP L | NOP 290 * | <br-stmts> | 1: ... 291 * | L: ... | 292 * | | 293 * | | L: <br-stmts> 294 * | | jmp 1b 295 * | | 296 * -----------+-----------------------+------------------ 297 * 298 * The initial value is encoded in the LSB of static_key::entries, 299 * type: 0 = false, 1 = true. 300 * 301 * The branch type is encoded in the LSB of jump_entry::key, 302 * branch: 0 = unlikely, 1 = likely. 303 * 304 * This gives the following logic table: 305 * 306 * enabled type branch instuction 307 * -----------------------------+----------- 308 * 0 0 0 | NOP 309 * 0 0 1 | JMP 310 * 0 1 0 | NOP 311 * 0 1 1 | JMP 312 * 313 * 1 0 0 | JMP 314 * 1 0 1 | NOP 315 * 1 1 0 | JMP 316 * 1 1 1 | NOP 317 * 318 * Which gives the following functions: 319 * 320 * dynamic: instruction = enabled ^ branch 321 * static: instruction = type ^ branch 322 * 323 * See jump_label_type() / jump_label_init_type(). 324 */ 325 326 extern bool ____wrong_branch_error(void); 327 328 #define static_branch_likely(x) \ 329 ({ \ 330 bool branch; \ 331 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \ 332 branch = !arch_static_branch(&(x)->key, true); \ 333 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 334 branch = !arch_static_branch_jump(&(x)->key, true); \ 335 else \ 336 branch = ____wrong_branch_error(); \ 337 branch; \ 338 }) 339 340 #define static_branch_unlikely(x) \ 341 ({ \ 342 bool branch; \ 343 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \ 344 branch = arch_static_branch_jump(&(x)->key, false); \ 345 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 346 branch = arch_static_branch(&(x)->key, false); \ 347 else \ 348 branch = ____wrong_branch_error(); \ 349 branch; \ 350 }) 351 352 #else /* !HAVE_JUMP_LABEL */ 353 354 #define static_branch_likely(x) likely(static_key_enabled(&(x)->key)) 355 #define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key)) 356 357 #endif /* HAVE_JUMP_LABEL */ 358 359 /* 360 * Advanced usage; refcount, branch is enabled when: count != 0 361 */ 362 363 #define static_branch_inc(x) static_key_slow_inc(&(x)->key) 364 #define static_branch_dec(x) static_key_slow_dec(&(x)->key) 365 366 /* 367 * Normal usage; boolean enable/disable. 368 */ 369 370 #define static_branch_enable(x) static_key_enable(&(x)->key) 371 #define static_branch_disable(x) static_key_disable(&(x)->key) 372 373 #endif /* _LINUX_JUMP_LABEL_H */ 374 375 #endif /* __ASSEMBLY__ */ 376