1 #ifndef _LINUX_JUMP_LABEL_H 2 #define _LINUX_JUMP_LABEL_H 3 4 /* 5 * Jump label support 6 * 7 * Copyright (C) 2009-2012 Jason Baron <[email protected]> 8 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra 9 * 10 * DEPRECATED API: 11 * 12 * The use of 'struct static_key' directly, is now DEPRECATED. In addition 13 * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following: 14 * 15 * struct static_key false = STATIC_KEY_INIT_FALSE; 16 * struct static_key true = STATIC_KEY_INIT_TRUE; 17 * static_key_true() 18 * static_key_false() 19 * 20 * The updated API replacements are: 21 * 22 * DEFINE_STATIC_KEY_TRUE(key); 23 * DEFINE_STATIC_KEY_FALSE(key); 24 * static_branch_likely() 25 * static_branch_unlikely() 26 * 27 * Jump labels provide an interface to generate dynamic branches using 28 * self-modifying code. Assuming toolchain and architecture support, if we 29 * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)", 30 * an "if (static_branch_unlikely(&key))" statement is an unconditional branch 31 * (which defaults to false - and the true block is placed out of line). 32 * Similarly, we can define an initially true key via 33 * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same 34 * "if (static_branch_unlikely(&key))", in which case we will generate an 35 * unconditional branch to the out-of-line true branch. Keys that are 36 * initially true or false can be using in both static_branch_unlikely() 37 * and static_branch_likely() statements. 38 * 39 * At runtime we can change the branch target by setting the key 40 * to true via a call to static_branch_enable(), or false using 41 * static_branch_disable(). If the direction of the branch is switched by 42 * these calls then we run-time modify the branch target via a 43 * no-op -> jump or jump -> no-op conversion. For example, for an 44 * initially false key that is used in an "if (static_branch_unlikely(&key))" 45 * statement, setting the key to true requires us to patch in a jump 46 * to the out-of-line of true branch. 47 * 48 * In addition to static_branch_{enable,disable}, we can also reference count 49 * the key or branch direction via static_branch_{inc,dec}. Thus, 50 * static_branch_inc() can be thought of as a 'make more true' and 51 * static_branch_dec() as a 'make more false'. 52 * 53 * Since this relies on modifying code, the branch modifying functions 54 * must be considered absolute slow paths (machine wide synchronization etc.). 55 * OTOH, since the affected branches are unconditional, their runtime overhead 56 * will be absolutely minimal, esp. in the default (off) case where the total 57 * effect is a single NOP of appropriate size. The on case will patch in a jump 58 * to the out-of-line block. 59 * 60 * When the control is directly exposed to userspace, it is prudent to delay the 61 * decrement to avoid high frequency code modifications which can (and do) 62 * cause significant performance degradation. Struct static_key_deferred and 63 * static_key_slow_dec_deferred() provide for this. 64 * 65 * Lacking toolchain and or architecture support, static keys fall back to a 66 * simple conditional branch. 67 * 68 * Additional babbling in: Documentation/static-keys.txt 69 */ 70 71 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) 72 # define HAVE_JUMP_LABEL 73 #endif 74 75 #ifndef __ASSEMBLY__ 76 77 #include <linux/types.h> 78 #include <linux/compiler.h> 79 #include <linux/bug.h> 80 81 extern bool static_key_initialized; 82 83 #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \ 84 "%s used before call to jump_label_init", \ 85 __func__) 86 87 #ifdef HAVE_JUMP_LABEL 88 89 struct static_key { 90 atomic_t enabled; 91 /* Set lsb bit to 1 if branch is default true, 0 ot */ 92 struct jump_entry *entries; 93 #ifdef CONFIG_MODULES 94 struct static_key_mod *next; 95 #endif 96 }; 97 98 #else 99 struct static_key { 100 atomic_t enabled; 101 }; 102 #endif /* HAVE_JUMP_LABEL */ 103 #endif /* __ASSEMBLY__ */ 104 105 #ifdef HAVE_JUMP_LABEL 106 #include <asm/jump_label.h> 107 #endif 108 109 #ifndef __ASSEMBLY__ 110 111 enum jump_label_type { 112 JUMP_LABEL_NOP = 0, 113 JUMP_LABEL_JMP, 114 }; 115 116 struct module; 117 118 #include <linux/atomic.h> 119 120 static inline int static_key_count(struct static_key *key) 121 { 122 return atomic_read(&key->enabled); 123 } 124 125 #ifdef HAVE_JUMP_LABEL 126 127 #define JUMP_TYPE_FALSE 0UL 128 #define JUMP_TYPE_TRUE 1UL 129 #define JUMP_TYPE_MASK 1UL 130 131 static __always_inline bool static_key_false(struct static_key *key) 132 { 133 return arch_static_branch(key, false); 134 } 135 136 static __always_inline bool static_key_true(struct static_key *key) 137 { 138 return !arch_static_branch(key, true); 139 } 140 141 extern struct jump_entry __start___jump_table[]; 142 extern struct jump_entry __stop___jump_table[]; 143 144 extern void jump_label_init(void); 145 extern void jump_label_lock(void); 146 extern void jump_label_unlock(void); 147 extern void arch_jump_label_transform(struct jump_entry *entry, 148 enum jump_label_type type); 149 extern void arch_jump_label_transform_static(struct jump_entry *entry, 150 enum jump_label_type type); 151 extern int jump_label_text_reserved(void *start, void *end); 152 extern void static_key_slow_inc(struct static_key *key); 153 extern void static_key_slow_dec(struct static_key *key); 154 extern void jump_label_apply_nops(struct module *mod); 155 156 #define STATIC_KEY_INIT_TRUE \ 157 { .enabled = ATOMIC_INIT(1), \ 158 .entries = (void *)JUMP_TYPE_TRUE } 159 #define STATIC_KEY_INIT_FALSE \ 160 { .enabled = ATOMIC_INIT(0), \ 161 .entries = (void *)JUMP_TYPE_FALSE } 162 163 #else /* !HAVE_JUMP_LABEL */ 164 165 static __always_inline void jump_label_init(void) 166 { 167 static_key_initialized = true; 168 } 169 170 static __always_inline bool static_key_false(struct static_key *key) 171 { 172 if (unlikely(static_key_count(key) > 0)) 173 return true; 174 return false; 175 } 176 177 static __always_inline bool static_key_true(struct static_key *key) 178 { 179 if (likely(static_key_count(key) > 0)) 180 return true; 181 return false; 182 } 183 184 static inline void static_key_slow_inc(struct static_key *key) 185 { 186 STATIC_KEY_CHECK_USE(); 187 atomic_inc(&key->enabled); 188 } 189 190 static inline void static_key_slow_dec(struct static_key *key) 191 { 192 STATIC_KEY_CHECK_USE(); 193 atomic_dec(&key->enabled); 194 } 195 196 static inline int jump_label_text_reserved(void *start, void *end) 197 { 198 return 0; 199 } 200 201 static inline void jump_label_lock(void) {} 202 static inline void jump_label_unlock(void) {} 203 204 static inline int jump_label_apply_nops(struct module *mod) 205 { 206 return 0; 207 } 208 209 #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) } 210 #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) } 211 212 #endif /* HAVE_JUMP_LABEL */ 213 214 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE 215 #define jump_label_enabled static_key_enabled 216 217 static inline void static_key_enable(struct static_key *key) 218 { 219 int count = static_key_count(key); 220 221 WARN_ON_ONCE(count < 0 || count > 1); 222 223 if (!count) 224 static_key_slow_inc(key); 225 } 226 227 static inline void static_key_disable(struct static_key *key) 228 { 229 int count = static_key_count(key); 230 231 WARN_ON_ONCE(count < 0 || count > 1); 232 233 if (count) 234 static_key_slow_dec(key); 235 } 236 237 /* -------------------------------------------------------------------------- */ 238 239 /* 240 * Two type wrappers around static_key, such that we can use compile time 241 * type differentiation to emit the right code. 242 * 243 * All the below code is macros in order to play type games. 244 */ 245 246 struct static_key_true { 247 struct static_key key; 248 }; 249 250 struct static_key_false { 251 struct static_key key; 252 }; 253 254 #define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, } 255 #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, } 256 257 #define DEFINE_STATIC_KEY_TRUE(name) \ 258 struct static_key_true name = STATIC_KEY_TRUE_INIT 259 260 #define DEFINE_STATIC_KEY_FALSE(name) \ 261 struct static_key_false name = STATIC_KEY_FALSE_INIT 262 263 extern bool ____wrong_branch_error(void); 264 265 #define static_key_enabled(x) \ 266 ({ \ 267 if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \ 268 !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\ 269 !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 270 ____wrong_branch_error(); \ 271 static_key_count((struct static_key *)x) > 0; \ 272 }) 273 274 #ifdef HAVE_JUMP_LABEL 275 276 /* 277 * Combine the right initial value (type) with the right branch order 278 * to generate the desired result. 279 * 280 * 281 * type\branch| likely (1) | unlikely (0) 282 * -----------+-----------------------+------------------ 283 * | | 284 * true (1) | ... | ... 285 * | NOP | JMP L 286 * | <br-stmts> | 1: ... 287 * | L: ... | 288 * | | 289 * | | L: <br-stmts> 290 * | | jmp 1b 291 * | | 292 * -----------+-----------------------+------------------ 293 * | | 294 * false (0) | ... | ... 295 * | JMP L | NOP 296 * | <br-stmts> | 1: ... 297 * | L: ... | 298 * | | 299 * | | L: <br-stmts> 300 * | | jmp 1b 301 * | | 302 * -----------+-----------------------+------------------ 303 * 304 * The initial value is encoded in the LSB of static_key::entries, 305 * type: 0 = false, 1 = true. 306 * 307 * The branch type is encoded in the LSB of jump_entry::key, 308 * branch: 0 = unlikely, 1 = likely. 309 * 310 * This gives the following logic table: 311 * 312 * enabled type branch instuction 313 * -----------------------------+----------- 314 * 0 0 0 | NOP 315 * 0 0 1 | JMP 316 * 0 1 0 | NOP 317 * 0 1 1 | JMP 318 * 319 * 1 0 0 | JMP 320 * 1 0 1 | NOP 321 * 1 1 0 | JMP 322 * 1 1 1 | NOP 323 * 324 * Which gives the following functions: 325 * 326 * dynamic: instruction = enabled ^ branch 327 * static: instruction = type ^ branch 328 * 329 * See jump_label_type() / jump_label_init_type(). 330 */ 331 332 #define static_branch_likely(x) \ 333 ({ \ 334 bool branch; \ 335 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \ 336 branch = !arch_static_branch(&(x)->key, true); \ 337 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 338 branch = !arch_static_branch_jump(&(x)->key, true); \ 339 else \ 340 branch = ____wrong_branch_error(); \ 341 branch; \ 342 }) 343 344 #define static_branch_unlikely(x) \ 345 ({ \ 346 bool branch; \ 347 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \ 348 branch = arch_static_branch_jump(&(x)->key, false); \ 349 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 350 branch = arch_static_branch(&(x)->key, false); \ 351 else \ 352 branch = ____wrong_branch_error(); \ 353 branch; \ 354 }) 355 356 #else /* !HAVE_JUMP_LABEL */ 357 358 #define static_branch_likely(x) likely(static_key_enabled(&(x)->key)) 359 #define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key)) 360 361 #endif /* HAVE_JUMP_LABEL */ 362 363 /* 364 * Advanced usage; refcount, branch is enabled when: count != 0 365 */ 366 367 #define static_branch_inc(x) static_key_slow_inc(&(x)->key) 368 #define static_branch_dec(x) static_key_slow_dec(&(x)->key) 369 370 /* 371 * Normal usage; boolean enable/disable. 372 */ 373 374 #define static_branch_enable(x) static_key_enable(&(x)->key) 375 #define static_branch_disable(x) static_key_disable(&(x)->key) 376 377 #endif /* _LINUX_JUMP_LABEL_H */ 378 379 #endif /* __ASSEMBLY__ */ 380