1 #ifndef _LINUX_JUMP_LABEL_H 2 #define _LINUX_JUMP_LABEL_H 3 4 /* 5 * Jump label support 6 * 7 * Copyright (C) 2009-2012 Jason Baron <[email protected]> 8 * Copyright (C) 2011-2012 Peter Zijlstra <[email protected]> 9 * 10 * Jump labels provide an interface to generate dynamic branches using 11 * self-modifying code. Assuming toolchain and architecture support, the result 12 * of a "if (static_key_false(&key))" statement is an unconditional branch (which 13 * defaults to false - and the true block is placed out of line). 14 * 15 * However at runtime we can change the branch target using 16 * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key 17 * object, and for as long as there are references all branches referring to 18 * that particular key will point to the (out of line) true block. 19 * 20 * Since this relies on modifying code, the static_key_slow_{inc,dec}() functions 21 * must be considered absolute slow paths (machine wide synchronization etc.). 22 * OTOH, since the affected branches are unconditional, their runtime overhead 23 * will be absolutely minimal, esp. in the default (off) case where the total 24 * effect is a single NOP of appropriate size. The on case will patch in a jump 25 * to the out-of-line block. 26 * 27 * When the control is directly exposed to userspace, it is prudent to delay the 28 * decrement to avoid high frequency code modifications which can (and do) 29 * cause significant performance degradation. Struct static_key_deferred and 30 * static_key_slow_dec_deferred() provide for this. 31 * 32 * Lacking toolchain and or architecture support, jump labels fall back to a simple 33 * conditional branch. 34 * 35 * struct static_key my_key = STATIC_KEY_INIT_TRUE; 36 * 37 * if (static_key_true(&my_key)) { 38 * } 39 * 40 * will result in the true case being in-line and starts the key with a single 41 * reference. Mixing static_key_true() and static_key_false() on the same key is not 42 * allowed. 43 * 44 * Not initializing the key (static data is initialized to 0s anyway) is the 45 * same as using STATIC_KEY_INIT_FALSE. 46 */ 47 48 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) 49 # define HAVE_JUMP_LABEL 50 #endif 51 52 #ifndef __ASSEMBLY__ 53 54 #include <linux/types.h> 55 #include <linux/compiler.h> 56 #include <linux/bug.h> 57 58 extern bool static_key_initialized; 59 60 #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \ 61 "%s used before call to jump_label_init", \ 62 __func__) 63 64 #ifdef HAVE_JUMP_LABEL 65 66 struct static_key { 67 atomic_t enabled; 68 /* Set lsb bit to 1 if branch is default true, 0 ot */ 69 struct jump_entry *entries; 70 #ifdef CONFIG_MODULES 71 struct static_key_mod *next; 72 #endif 73 }; 74 75 #else 76 struct static_key { 77 atomic_t enabled; 78 }; 79 #endif /* HAVE_JUMP_LABEL */ 80 #endif /* __ASSEMBLY__ */ 81 82 #ifdef HAVE_JUMP_LABEL 83 #include <asm/jump_label.h> 84 #endif 85 86 #ifndef __ASSEMBLY__ 87 88 enum jump_label_type { 89 JUMP_LABEL_DISABLE = 0, 90 JUMP_LABEL_ENABLE, 91 }; 92 93 struct module; 94 95 #include <linux/atomic.h> 96 97 static inline int static_key_count(struct static_key *key) 98 { 99 return atomic_read(&key->enabled); 100 } 101 102 #ifdef HAVE_JUMP_LABEL 103 104 #define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL 105 #define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL 106 #define JUMP_LABEL_TYPE_MASK 1UL 107 108 static 109 inline struct jump_entry *jump_label_get_entries(struct static_key *key) 110 { 111 return (struct jump_entry *)((unsigned long)key->entries 112 & ~JUMP_LABEL_TYPE_MASK); 113 } 114 115 static inline bool jump_label_get_branch_default(struct static_key *key) 116 { 117 if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) == 118 JUMP_LABEL_TYPE_TRUE_BRANCH) 119 return true; 120 return false; 121 } 122 123 static __always_inline bool static_key_false(struct static_key *key) 124 { 125 return arch_static_branch(key); 126 } 127 128 static __always_inline bool static_key_true(struct static_key *key) 129 { 130 return !static_key_false(key); 131 } 132 133 extern struct jump_entry __start___jump_table[]; 134 extern struct jump_entry __stop___jump_table[]; 135 136 extern void jump_label_init(void); 137 extern void jump_label_lock(void); 138 extern void jump_label_unlock(void); 139 extern void arch_jump_label_transform(struct jump_entry *entry, 140 enum jump_label_type type); 141 extern void arch_jump_label_transform_static(struct jump_entry *entry, 142 enum jump_label_type type); 143 extern int jump_label_text_reserved(void *start, void *end); 144 extern void static_key_slow_inc(struct static_key *key); 145 extern void static_key_slow_dec(struct static_key *key); 146 extern void jump_label_apply_nops(struct module *mod); 147 148 #define STATIC_KEY_INIT_TRUE ((struct static_key) \ 149 { .enabled = ATOMIC_INIT(1), \ 150 .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH }) 151 #define STATIC_KEY_INIT_FALSE ((struct static_key) \ 152 { .enabled = ATOMIC_INIT(0), \ 153 .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH }) 154 155 #else /* !HAVE_JUMP_LABEL */ 156 157 static __always_inline void jump_label_init(void) 158 { 159 static_key_initialized = true; 160 } 161 162 static __always_inline bool static_key_false(struct static_key *key) 163 { 164 if (unlikely(static_key_count(key) > 0)) 165 return true; 166 return false; 167 } 168 169 static __always_inline bool static_key_true(struct static_key *key) 170 { 171 if (likely(static_key_count(key) > 0)) 172 return true; 173 return false; 174 } 175 176 static inline void static_key_slow_inc(struct static_key *key) 177 { 178 STATIC_KEY_CHECK_USE(); 179 atomic_inc(&key->enabled); 180 } 181 182 static inline void static_key_slow_dec(struct static_key *key) 183 { 184 STATIC_KEY_CHECK_USE(); 185 atomic_dec(&key->enabled); 186 } 187 188 static inline int jump_label_text_reserved(void *start, void *end) 189 { 190 return 0; 191 } 192 193 static inline void jump_label_lock(void) {} 194 static inline void jump_label_unlock(void) {} 195 196 static inline int jump_label_apply_nops(struct module *mod) 197 { 198 return 0; 199 } 200 201 #define STATIC_KEY_INIT_TRUE ((struct static_key) \ 202 { .enabled = ATOMIC_INIT(1) }) 203 #define STATIC_KEY_INIT_FALSE ((struct static_key) \ 204 { .enabled = ATOMIC_INIT(0) }) 205 206 #endif /* HAVE_JUMP_LABEL */ 207 208 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE 209 #define jump_label_enabled static_key_enabled 210 211 static inline bool static_key_enabled(struct static_key *key) 212 { 213 return static_key_count(key) > 0; 214 } 215 216 #endif /* _LINUX_JUMP_LABEL_H */ 217 218 #endif /* __ASSEMBLY__ */ 219