1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2019 Facebook */ 3 #include <linux/hash.h> 4 #include <linux/bpf.h> 5 #include <linux/filter.h> 6 #include <linux/ftrace.h> 7 8 /* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */ 9 #define TRAMPOLINE_HASH_BITS 10 10 #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS) 11 12 static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; 13 14 /* serializes access to trampoline_table */ 15 static DEFINE_MUTEX(trampoline_mutex); 16 17 void *bpf_jit_alloc_exec_page(void) 18 { 19 void *image; 20 21 image = bpf_jit_alloc_exec(PAGE_SIZE); 22 if (!image) 23 return NULL; 24 25 set_vm_flush_reset_perms(image); 26 /* Keep image as writeable. The alternative is to keep flipping ro/rw 27 * everytime new program is attached or detached. 28 */ 29 set_memory_x((long)image, 1); 30 return image; 31 } 32 33 struct bpf_trampoline *bpf_trampoline_lookup(u64 key) 34 { 35 struct bpf_trampoline *tr; 36 struct hlist_head *head; 37 void *image; 38 int i; 39 40 mutex_lock(&trampoline_mutex); 41 head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; 42 hlist_for_each_entry(tr, head, hlist) { 43 if (tr->key == key) { 44 refcount_inc(&tr->refcnt); 45 goto out; 46 } 47 } 48 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 49 if (!tr) 50 goto out; 51 52 /* is_root was checked earlier. No need for bpf_jit_charge_modmem() */ 53 image = bpf_jit_alloc_exec_page(); 54 if (!image) { 55 kfree(tr); 56 tr = NULL; 57 goto out; 58 } 59 60 tr->key = key; 61 INIT_HLIST_NODE(&tr->hlist); 62 hlist_add_head(&tr->hlist, head); 63 refcount_set(&tr->refcnt, 1); 64 mutex_init(&tr->mutex); 65 for (i = 0; i < BPF_TRAMP_MAX; i++) 66 INIT_HLIST_HEAD(&tr->progs_hlist[i]); 67 tr->image = image; 68 out: 69 mutex_unlock(&trampoline_mutex); 70 return tr; 71 } 72 73 static int is_ftrace_location(void *ip) 74 { 75 long addr; 76 77 addr = ftrace_location((long)ip); 78 if (!addr) 79 return 0; 80 if (WARN_ON_ONCE(addr != (long)ip)) 81 return -EFAULT; 82 return 1; 83 } 84 85 static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr) 86 { 87 void *ip = tr->func.addr; 88 int ret; 89 90 if (tr->func.ftrace_managed) 91 ret = unregister_ftrace_direct((long)ip, (long)old_addr); 92 else 93 ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL); 94 return ret; 95 } 96 97 static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr) 98 { 99 void *ip = tr->func.addr; 100 int ret; 101 102 if (tr->func.ftrace_managed) 103 ret = modify_ftrace_direct((long)ip, (long)old_addr, (long)new_addr); 104 else 105 ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr); 106 return ret; 107 } 108 109 /* first time registering */ 110 static int register_fentry(struct bpf_trampoline *tr, void *new_addr) 111 { 112 void *ip = tr->func.addr; 113 int ret; 114 115 ret = is_ftrace_location(ip); 116 if (ret < 0) 117 return ret; 118 tr->func.ftrace_managed = ret; 119 120 if (tr->func.ftrace_managed) 121 ret = register_ftrace_direct((long)ip, (long)new_addr); 122 else 123 ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr); 124 return ret; 125 } 126 127 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 128 * bytes on x86. Pick a number to fit into PAGE_SIZE / 2 129 */ 130 #define BPF_MAX_TRAMP_PROGS 40 131 132 static int bpf_trampoline_update(struct bpf_trampoline *tr) 133 { 134 void *old_image = tr->image + ((tr->selector + 1) & 1) * PAGE_SIZE/2; 135 void *new_image = tr->image + (tr->selector & 1) * PAGE_SIZE/2; 136 struct bpf_prog *progs_to_run[BPF_MAX_TRAMP_PROGS]; 137 int fentry_cnt = tr->progs_cnt[BPF_TRAMP_FENTRY]; 138 int fexit_cnt = tr->progs_cnt[BPF_TRAMP_FEXIT]; 139 struct bpf_prog **progs, **fentry, **fexit; 140 u32 flags = BPF_TRAMP_F_RESTORE_REGS; 141 struct bpf_prog_aux *aux; 142 int err; 143 144 if (fentry_cnt + fexit_cnt == 0) { 145 err = unregister_fentry(tr, old_image); 146 tr->selector = 0; 147 goto out; 148 } 149 150 /* populate fentry progs */ 151 fentry = progs = progs_to_run; 152 hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FENTRY], tramp_hlist) 153 *progs++ = aux->prog; 154 155 /* populate fexit progs */ 156 fexit = progs; 157 hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FEXIT], tramp_hlist) 158 *progs++ = aux->prog; 159 160 if (fexit_cnt) 161 flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; 162 163 err = arch_prepare_bpf_trampoline(new_image, &tr->func.model, flags, 164 fentry, fentry_cnt, 165 fexit, fexit_cnt, 166 tr->func.addr); 167 if (err) 168 goto out; 169 170 if (tr->selector) 171 /* progs already running at this address */ 172 err = modify_fentry(tr, old_image, new_image); 173 else 174 /* first time registering */ 175 err = register_fentry(tr, new_image); 176 if (err) 177 goto out; 178 tr->selector++; 179 out: 180 return err; 181 } 182 183 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(enum bpf_attach_type t) 184 { 185 switch (t) { 186 case BPF_TRACE_FENTRY: 187 return BPF_TRAMP_FENTRY; 188 default: 189 return BPF_TRAMP_FEXIT; 190 } 191 } 192 193 int bpf_trampoline_link_prog(struct bpf_prog *prog) 194 { 195 enum bpf_tramp_prog_type kind; 196 struct bpf_trampoline *tr; 197 int err = 0; 198 199 tr = prog->aux->trampoline; 200 kind = bpf_attach_type_to_tramp(prog->expected_attach_type); 201 mutex_lock(&tr->mutex); 202 if (tr->progs_cnt[BPF_TRAMP_FENTRY] + tr->progs_cnt[BPF_TRAMP_FEXIT] 203 >= BPF_MAX_TRAMP_PROGS) { 204 err = -E2BIG; 205 goto out; 206 } 207 if (!hlist_unhashed(&prog->aux->tramp_hlist)) { 208 /* prog already linked */ 209 err = -EBUSY; 210 goto out; 211 } 212 hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]); 213 tr->progs_cnt[kind]++; 214 err = bpf_trampoline_update(prog->aux->trampoline); 215 if (err) { 216 hlist_del(&prog->aux->tramp_hlist); 217 tr->progs_cnt[kind]--; 218 } 219 out: 220 mutex_unlock(&tr->mutex); 221 return err; 222 } 223 224 /* bpf_trampoline_unlink_prog() should never fail. */ 225 int bpf_trampoline_unlink_prog(struct bpf_prog *prog) 226 { 227 enum bpf_tramp_prog_type kind; 228 struct bpf_trampoline *tr; 229 int err; 230 231 tr = prog->aux->trampoline; 232 kind = bpf_attach_type_to_tramp(prog->expected_attach_type); 233 mutex_lock(&tr->mutex); 234 hlist_del(&prog->aux->tramp_hlist); 235 tr->progs_cnt[kind]--; 236 err = bpf_trampoline_update(prog->aux->trampoline); 237 mutex_unlock(&tr->mutex); 238 return err; 239 } 240 241 void bpf_trampoline_put(struct bpf_trampoline *tr) 242 { 243 if (!tr) 244 return; 245 mutex_lock(&trampoline_mutex); 246 if (!refcount_dec_and_test(&tr->refcnt)) 247 goto out; 248 WARN_ON_ONCE(mutex_is_locked(&tr->mutex)); 249 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FENTRY]))) 250 goto out; 251 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FEXIT]))) 252 goto out; 253 bpf_jit_free_exec(tr->image); 254 hlist_del(&tr->hlist); 255 kfree(tr); 256 out: 257 mutex_unlock(&trampoline_mutex); 258 } 259 260 /* The logic is similar to BPF_PROG_RUN, but with explicit rcu and preempt that 261 * are needed for trampoline. The macro is split into 262 * call _bpf_prog_enter 263 * call prog->bpf_func 264 * call __bpf_prog_exit 265 */ 266 u64 notrace __bpf_prog_enter(void) 267 { 268 u64 start = 0; 269 270 rcu_read_lock(); 271 preempt_disable(); 272 if (static_branch_unlikely(&bpf_stats_enabled_key)) 273 start = sched_clock(); 274 return start; 275 } 276 277 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) 278 { 279 struct bpf_prog_stats *stats; 280 281 if (static_branch_unlikely(&bpf_stats_enabled_key) && 282 /* static_key could be enabled in __bpf_prog_enter 283 * and disabled in __bpf_prog_exit. 284 * And vice versa. 285 * Hence check that 'start' is not zero. 286 */ 287 start) { 288 stats = this_cpu_ptr(prog->aux->stats); 289 u64_stats_update_begin(&stats->syncp); 290 stats->cnt++; 291 stats->nsecs += sched_clock() - start; 292 u64_stats_update_end(&stats->syncp); 293 } 294 preempt_enable(); 295 rcu_read_unlock(); 296 } 297 298 int __weak 299 arch_prepare_bpf_trampoline(void *image, struct btf_func_model *m, u32 flags, 300 struct bpf_prog **fentry_progs, int fentry_cnt, 301 struct bpf_prog **fexit_progs, int fexit_cnt, 302 void *orig_call) 303 { 304 return -ENOTSUPP; 305 } 306 307 static int __init init_trampolines(void) 308 { 309 int i; 310 311 for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++) 312 INIT_HLIST_HEAD(&trampoline_table[i]); 313 return 0; 314 } 315 late_initcall(init_trampolines); 316