1 #ifndef _LINUX_JUMP_LABEL_H 2 #define _LINUX_JUMP_LABEL_H 3 4 /* 5 * Jump label support 6 * 7 * Copyright (C) 2009-2012 Jason Baron <[email protected]> 8 * Copyright (C) 2011-2012 Peter Zijlstra <[email protected]> 9 * 10 * Jump labels provide an interface to generate dynamic branches using 11 * self-modifying code. Assuming toolchain and architecture support the result 12 * of a "if (static_key_false(&key))" statement is a unconditional branch (which 13 * defaults to false - and the true block is placed out of line). 14 * 15 * However at runtime we can change the branch target using 16 * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key 17 * object and for as long as there are references all branches referring to 18 * that particular key will point to the (out of line) true block. 19 * 20 * Since this relies on modifying code the static_key_slow_{inc,dec}() functions 21 * must be considered absolute slow paths (machine wide synchronization etc.). 22 * OTOH, since the affected branches are unconditional their runtime overhead 23 * will be absolutely minimal, esp. in the default (off) case where the total 24 * effect is a single NOP of appropriate size. The on case will patch in a jump 25 * to the out-of-line block. 26 * 27 * When the control is directly exposed to userspace it is prudent to delay the 28 * decrement to avoid high frequency code modifications which can (and do) 29 * cause significant performance degradation. Struct static_key_deferred and 30 * static_key_slow_dec_deferred() provide for this. 31 * 32 * Lacking toolchain and or architecture support, it falls back to a simple 33 * conditional branch. 34 * 35 * struct static_key my_key = STATIC_KEY_INIT_TRUE; 36 * 37 * if (static_key_true(&my_key)) { 38 * } 39 * 40 * will result in the true case being in-line and starts the key with a single 41 * reference. Mixing static_key_true() and static_key_false() on the same key is not 42 * allowed. 43 * 44 * Not initializing the key (static data is initialized to 0s anyway) is the 45 * same as using STATIC_KEY_INIT_FALSE and static_key_false() is 46 * equivalent with static_branch(). 47 * 48 */ 49 50 #include <linux/types.h> 51 #include <linux/compiler.h> 52 #include <linux/workqueue.h> 53 54 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) 55 56 struct static_key { 57 atomic_t enabled; 58 /* Set lsb bit to 1 if branch is default true, 0 ot */ 59 struct jump_entry *entries; 60 #ifdef CONFIG_MODULES 61 struct static_key_mod *next; 62 #endif 63 }; 64 65 struct static_key_deferred { 66 struct static_key key; 67 unsigned long timeout; 68 struct delayed_work work; 69 }; 70 71 # include <asm/jump_label.h> 72 # define HAVE_JUMP_LABEL 73 #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */ 74 75 enum jump_label_type { 76 JUMP_LABEL_DISABLE = 0, 77 JUMP_LABEL_ENABLE, 78 }; 79 80 struct module; 81 82 #ifdef HAVE_JUMP_LABEL 83 84 #define JUMP_LABEL_TRUE_BRANCH 1UL 85 86 static 87 inline struct jump_entry *jump_label_get_entries(struct static_key *key) 88 { 89 return (struct jump_entry *)((unsigned long)key->entries 90 & ~JUMP_LABEL_TRUE_BRANCH); 91 } 92 93 static inline bool jump_label_get_branch_default(struct static_key *key) 94 { 95 if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH) 96 return true; 97 return false; 98 } 99 100 static __always_inline bool static_key_false(struct static_key *key) 101 { 102 return arch_static_branch(key); 103 } 104 105 static __always_inline bool static_key_true(struct static_key *key) 106 { 107 return !static_key_false(key); 108 } 109 110 /* Deprecated. Please use 'static_key_false() instead. */ 111 static __always_inline bool static_branch(struct static_key *key) 112 { 113 return arch_static_branch(key); 114 } 115 116 extern struct jump_entry __start___jump_table[]; 117 extern struct jump_entry __stop___jump_table[]; 118 119 extern void jump_label_init(void); 120 extern void jump_label_lock(void); 121 extern void jump_label_unlock(void); 122 extern void arch_jump_label_transform(struct jump_entry *entry, 123 enum jump_label_type type); 124 extern void arch_jump_label_transform_static(struct jump_entry *entry, 125 enum jump_label_type type); 126 extern int jump_label_text_reserved(void *start, void *end); 127 extern void static_key_slow_inc(struct static_key *key); 128 extern void static_key_slow_dec(struct static_key *key); 129 extern void static_key_slow_dec_deferred(struct static_key_deferred *key); 130 extern void jump_label_apply_nops(struct module *mod); 131 extern void 132 jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl); 133 134 #define STATIC_KEY_INIT_TRUE ((struct static_key) \ 135 { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) 136 #define STATIC_KEY_INIT_FALSE ((struct static_key) \ 137 { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) 138 139 #else /* !HAVE_JUMP_LABEL */ 140 141 #include <linux/atomic.h> 142 143 struct static_key { 144 atomic_t enabled; 145 }; 146 147 static __always_inline void jump_label_init(void) 148 { 149 } 150 151 struct static_key_deferred { 152 struct static_key key; 153 }; 154 155 static __always_inline bool static_key_false(struct static_key *key) 156 { 157 if (unlikely(atomic_read(&key->enabled)) > 0) 158 return true; 159 return false; 160 } 161 162 static __always_inline bool static_key_true(struct static_key *key) 163 { 164 if (likely(atomic_read(&key->enabled)) > 0) 165 return true; 166 return false; 167 } 168 169 /* Deprecated. Please use 'static_key_false() instead. */ 170 static __always_inline bool static_branch(struct static_key *key) 171 { 172 if (unlikely(atomic_read(&key->enabled)) > 0) 173 return true; 174 return false; 175 } 176 177 static inline void static_key_slow_inc(struct static_key *key) 178 { 179 atomic_inc(&key->enabled); 180 } 181 182 static inline void static_key_slow_dec(struct static_key *key) 183 { 184 atomic_dec(&key->enabled); 185 } 186 187 static inline void static_key_slow_dec_deferred(struct static_key_deferred *key) 188 { 189 static_key_slow_dec(&key->key); 190 } 191 192 static inline int jump_label_text_reserved(void *start, void *end) 193 { 194 return 0; 195 } 196 197 static inline void jump_label_lock(void) {} 198 static inline void jump_label_unlock(void) {} 199 200 static inline int jump_label_apply_nops(struct module *mod) 201 { 202 return 0; 203 } 204 205 static inline void 206 jump_label_rate_limit(struct static_key_deferred *key, 207 unsigned long rl) 208 { 209 } 210 211 #define STATIC_KEY_INIT_TRUE ((struct static_key) \ 212 { .enabled = ATOMIC_INIT(1) }) 213 #define STATIC_KEY_INIT_FALSE ((struct static_key) \ 214 { .enabled = ATOMIC_INIT(0) }) 215 216 #endif /* HAVE_JUMP_LABEL */ 217 218 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE 219 #define jump_label_enabled static_key_enabled 220 221 static inline bool static_key_enabled(struct static_key *key) 222 { 223 return (atomic_read(&key->enabled) > 0); 224 } 225 226 #endif /* _LINUX_JUMP_LABEL_H */ 227