1 #ifndef _LINUX_JUMP_LABEL_H 2 #define _LINUX_JUMP_LABEL_H 3 4 /* 5 * Jump label support 6 * 7 * Copyright (C) 2009-2012 Jason Baron <[email protected]> 8 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra 9 * 10 * DEPRECATED API: 11 * 12 * The use of 'struct static_key' directly, is now DEPRECATED. In addition 13 * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following: 14 * 15 * struct static_key false = STATIC_KEY_INIT_FALSE; 16 * struct static_key true = STATIC_KEY_INIT_TRUE; 17 * static_key_true() 18 * static_key_false() 19 * 20 * The updated API replacements are: 21 * 22 * DEFINE_STATIC_KEY_TRUE(key); 23 * DEFINE_STATIC_KEY_FALSE(key); 24 * static_branch_likely() 25 * static_branch_unlikely() 26 * 27 * Jump labels provide an interface to generate dynamic branches using 28 * self-modifying code. Assuming toolchain and architecture support, if we 29 * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)", 30 * an "if (static_branch_unlikely(&key))" statement is an unconditional branch 31 * (which defaults to false - and the true block is placed out of line). 32 * Similarly, we can define an initially true key via 33 * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same 34 * "if (static_branch_unlikely(&key))", in which case we will generate an 35 * unconditional branch to the out-of-line true branch. Keys that are 36 * initially true or false can be using in both static_branch_unlikely() 37 * and static_branch_likely() statements. 38 * 39 * At runtime we can change the branch target by setting the key 40 * to true via a call to static_branch_enable(), or false using 41 * static_branch_disable(). If the direction of the branch is switched by 42 * these calls then we run-time modify the branch target via a 43 * no-op -> jump or jump -> no-op conversion. For example, for an 44 * initially false key that is used in an "if (static_branch_unlikely(&key))" 45 * statement, setting the key to true requires us to patch in a jump 46 * to the out-of-line of true branch. 47 * 48 * In addition to static_branch_{enable,disable}, we can also reference count 49 * the key or branch direction via static_branch_{inc,dec}. Thus, 50 * static_branch_inc() can be thought of as a 'make more true' and 51 * static_branch_dec() as a 'make more false'. 52 * 53 * Since this relies on modifying code, the branch modifying functions 54 * must be considered absolute slow paths (machine wide synchronization etc.). 55 * OTOH, since the affected branches are unconditional, their runtime overhead 56 * will be absolutely minimal, esp. in the default (off) case where the total 57 * effect is a single NOP of appropriate size. The on case will patch in a jump 58 * to the out-of-line block. 59 * 60 * When the control is directly exposed to userspace, it is prudent to delay the 61 * decrement to avoid high frequency code modifications which can (and do) 62 * cause significant performance degradation. Struct static_key_deferred and 63 * static_key_slow_dec_deferred() provide for this. 64 * 65 * Lacking toolchain and or architecture support, static keys fall back to a 66 * simple conditional branch. 67 * 68 * Additional babbling in: Documentation/static-keys.txt 69 */ 70 71 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) 72 # define HAVE_JUMP_LABEL 73 #endif 74 75 #ifndef __ASSEMBLY__ 76 77 #include <linux/types.h> 78 #include <linux/compiler.h> 79 #include <linux/bug.h> 80 81 extern bool static_key_initialized; 82 83 #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \ 84 "%s used before call to jump_label_init", \ 85 __func__) 86 87 #ifdef HAVE_JUMP_LABEL 88 89 struct static_key { 90 atomic_t enabled; 91 /* Set lsb bit to 1 if branch is default true, 0 ot */ 92 struct jump_entry *entries; 93 #ifdef CONFIG_MODULES 94 struct static_key_mod *next; 95 #endif 96 }; 97 98 #else 99 struct static_key { 100 atomic_t enabled; 101 }; 102 #endif /* HAVE_JUMP_LABEL */ 103 #endif /* __ASSEMBLY__ */ 104 105 #ifdef HAVE_JUMP_LABEL 106 #include <asm/jump_label.h> 107 #endif 108 109 #ifndef __ASSEMBLY__ 110 111 enum jump_label_type { 112 JUMP_LABEL_NOP = 0, 113 JUMP_LABEL_JMP, 114 }; 115 116 struct module; 117 118 #include <linux/atomic.h> 119 120 #ifdef HAVE_JUMP_LABEL 121 122 static inline int static_key_count(struct static_key *key) 123 { 124 /* 125 * -1 means the first static_key_slow_inc() is in progress. 126 * static_key_enabled() must return true, so return 1 here. 127 */ 128 int n = atomic_read(&key->enabled); 129 return n >= 0 ? n : 1; 130 } 131 132 #define JUMP_TYPE_FALSE 0UL 133 #define JUMP_TYPE_TRUE 1UL 134 #define JUMP_TYPE_MASK 1UL 135 136 static __always_inline bool static_key_false(struct static_key *key) 137 { 138 return arch_static_branch(key, false); 139 } 140 141 static __always_inline bool static_key_true(struct static_key *key) 142 { 143 return !arch_static_branch(key, true); 144 } 145 146 extern struct jump_entry __start___jump_table[]; 147 extern struct jump_entry __stop___jump_table[]; 148 149 extern void jump_label_init(void); 150 extern void jump_label_lock(void); 151 extern void jump_label_unlock(void); 152 extern void arch_jump_label_transform(struct jump_entry *entry, 153 enum jump_label_type type); 154 extern void arch_jump_label_transform_static(struct jump_entry *entry, 155 enum jump_label_type type); 156 extern int jump_label_text_reserved(void *start, void *end); 157 extern void static_key_slow_inc(struct static_key *key); 158 extern void static_key_slow_dec(struct static_key *key); 159 extern void jump_label_apply_nops(struct module *mod); 160 161 #define STATIC_KEY_INIT_TRUE \ 162 { .enabled = ATOMIC_INIT(1), \ 163 .entries = (void *)JUMP_TYPE_TRUE } 164 #define STATIC_KEY_INIT_FALSE \ 165 { .enabled = ATOMIC_INIT(0), \ 166 .entries = (void *)JUMP_TYPE_FALSE } 167 168 #else /* !HAVE_JUMP_LABEL */ 169 170 static inline int static_key_count(struct static_key *key) 171 { 172 return atomic_read(&key->enabled); 173 } 174 175 static __always_inline void jump_label_init(void) 176 { 177 static_key_initialized = true; 178 } 179 180 static __always_inline bool static_key_false(struct static_key *key) 181 { 182 if (unlikely(static_key_count(key) > 0)) 183 return true; 184 return false; 185 } 186 187 static __always_inline bool static_key_true(struct static_key *key) 188 { 189 if (likely(static_key_count(key) > 0)) 190 return true; 191 return false; 192 } 193 194 static inline void static_key_slow_inc(struct static_key *key) 195 { 196 STATIC_KEY_CHECK_USE(); 197 atomic_inc(&key->enabled); 198 } 199 200 static inline void static_key_slow_dec(struct static_key *key) 201 { 202 STATIC_KEY_CHECK_USE(); 203 atomic_dec(&key->enabled); 204 } 205 206 static inline int jump_label_text_reserved(void *start, void *end) 207 { 208 return 0; 209 } 210 211 static inline void jump_label_lock(void) {} 212 static inline void jump_label_unlock(void) {} 213 214 static inline int jump_label_apply_nops(struct module *mod) 215 { 216 return 0; 217 } 218 219 #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) } 220 #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) } 221 222 #endif /* HAVE_JUMP_LABEL */ 223 224 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE 225 #define jump_label_enabled static_key_enabled 226 227 static inline void static_key_enable(struct static_key *key) 228 { 229 int count = static_key_count(key); 230 231 WARN_ON_ONCE(count < 0 || count > 1); 232 233 if (!count) 234 static_key_slow_inc(key); 235 } 236 237 static inline void static_key_disable(struct static_key *key) 238 { 239 int count = static_key_count(key); 240 241 WARN_ON_ONCE(count < 0 || count > 1); 242 243 if (count) 244 static_key_slow_dec(key); 245 } 246 247 /* -------------------------------------------------------------------------- */ 248 249 /* 250 * Two type wrappers around static_key, such that we can use compile time 251 * type differentiation to emit the right code. 252 * 253 * All the below code is macros in order to play type games. 254 */ 255 256 struct static_key_true { 257 struct static_key key; 258 }; 259 260 struct static_key_false { 261 struct static_key key; 262 }; 263 264 #define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, } 265 #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, } 266 267 #define DEFINE_STATIC_KEY_TRUE(name) \ 268 struct static_key_true name = STATIC_KEY_TRUE_INIT 269 270 #define DEFINE_STATIC_KEY_FALSE(name) \ 271 struct static_key_false name = STATIC_KEY_FALSE_INIT 272 273 extern bool ____wrong_branch_error(void); 274 275 #define static_key_enabled(x) \ 276 ({ \ 277 if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \ 278 !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\ 279 !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 280 ____wrong_branch_error(); \ 281 static_key_count((struct static_key *)x) > 0; \ 282 }) 283 284 #ifdef HAVE_JUMP_LABEL 285 286 /* 287 * Combine the right initial value (type) with the right branch order 288 * to generate the desired result. 289 * 290 * 291 * type\branch| likely (1) | unlikely (0) 292 * -----------+-----------------------+------------------ 293 * | | 294 * true (1) | ... | ... 295 * | NOP | JMP L 296 * | <br-stmts> | 1: ... 297 * | L: ... | 298 * | | 299 * | | L: <br-stmts> 300 * | | jmp 1b 301 * | | 302 * -----------+-----------------------+------------------ 303 * | | 304 * false (0) | ... | ... 305 * | JMP L | NOP 306 * | <br-stmts> | 1: ... 307 * | L: ... | 308 * | | 309 * | | L: <br-stmts> 310 * | | jmp 1b 311 * | | 312 * -----------+-----------------------+------------------ 313 * 314 * The initial value is encoded in the LSB of static_key::entries, 315 * type: 0 = false, 1 = true. 316 * 317 * The branch type is encoded in the LSB of jump_entry::key, 318 * branch: 0 = unlikely, 1 = likely. 319 * 320 * This gives the following logic table: 321 * 322 * enabled type branch instuction 323 * -----------------------------+----------- 324 * 0 0 0 | NOP 325 * 0 0 1 | JMP 326 * 0 1 0 | NOP 327 * 0 1 1 | JMP 328 * 329 * 1 0 0 | JMP 330 * 1 0 1 | NOP 331 * 1 1 0 | JMP 332 * 1 1 1 | NOP 333 * 334 * Which gives the following functions: 335 * 336 * dynamic: instruction = enabled ^ branch 337 * static: instruction = type ^ branch 338 * 339 * See jump_label_type() / jump_label_init_type(). 340 */ 341 342 #define static_branch_likely(x) \ 343 ({ \ 344 bool branch; \ 345 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \ 346 branch = !arch_static_branch(&(x)->key, true); \ 347 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 348 branch = !arch_static_branch_jump(&(x)->key, true); \ 349 else \ 350 branch = ____wrong_branch_error(); \ 351 branch; \ 352 }) 353 354 #define static_branch_unlikely(x) \ 355 ({ \ 356 bool branch; \ 357 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \ 358 branch = arch_static_branch_jump(&(x)->key, false); \ 359 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ 360 branch = arch_static_branch(&(x)->key, false); \ 361 else \ 362 branch = ____wrong_branch_error(); \ 363 branch; \ 364 }) 365 366 #else /* !HAVE_JUMP_LABEL */ 367 368 #define static_branch_likely(x) likely(static_key_enabled(&(x)->key)) 369 #define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key)) 370 371 #endif /* HAVE_JUMP_LABEL */ 372 373 /* 374 * Advanced usage; refcount, branch is enabled when: count != 0 375 */ 376 377 #define static_branch_inc(x) static_key_slow_inc(&(x)->key) 378 #define static_branch_dec(x) static_key_slow_dec(&(x)->key) 379 380 /* 381 * Normal usage; boolean enable/disable. 382 */ 383 384 #define static_branch_enable(x) static_key_enable(&(x)->key) 385 #define static_branch_disable(x) static_key_disable(&(x)->key) 386 387 #endif /* _LINUX_JUMP_LABEL_H */ 388 389 #endif /* __ASSEMBLY__ */ 390