93 	if (READ_ONCE(node->fp) != NULL) {  in delete_fprobe_node()
94 		WRITE_ONCE(node->fp, NULL);  in delete_fprobe_node()
101 static bool is_fprobe_still_exist(struct fprobe *fp)  in is_fprobe_still_exist()  argument
106 	head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];  in is_fprobe_still_exist()
109 		if (fph->fp == fp)  in is_fprobe_still_exist()
116 static int add_fprobe_hash(struct fprobe *fp)  in add_fprobe_hash()  argument
118 	struct fprobe_hlist *fph = fp->hlist_array;  in add_fprobe_hash()
126 	if (is_fprobe_still_exist(fp))  in add_fprobe_hash()
129 	head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];  in add_fprobe_hash()
130 	hlist_add_head_rcu(&fp->hlist_array->hlist, head);  in add_fprobe_hash()
134 static int del_fprobe_hash(struct fprobe *fp)  in del_fprobe_hash()  argument
136 	struct fprobe_hlist *fph = fp->hlist_array;  in del_fprobe_hash()
143 	if (!is_fprobe_still_exist(fp))  in del_fprobe_hash()
146 	fph->fp = NULL;  in del_fprobe_hash()
157 					struct fprobe *fp, unsigned int size_words)  in write_fprobe_header()  argument
160 			 !arch_fprobe_header_encodable(fp)))  in write_fprobe_header()
163 	*stack = arch_encode_fprobe_header(fp, size_words);  in write_fprobe_header()
168 					struct fprobe **fp, unsigned int *size_words)  in read_fprobe_header()  argument
170 	*fp = arch_decode_fprobe_header_fp(*stack);  in read_fprobe_header()
178 	struct fprobe *fp;  member
185 					struct fprobe *fp, unsigned int size_words)  in write_fprobe_header()  argument
192 	fph->fp = fp;  in write_fprobe_header()
198 					struct fprobe **fp, unsigned int *size_words)  in read_fprobe_header()  argument
202 	*fp = fph->fp;  in read_fprobe_header()
218 				   struct fprobe *fp, struct ftrace_regs *fregs,  in __fprobe_handler()  argument
221 	if (!fp->entry_handler)  in __fprobe_handler()
224 	return fp->entry_handler(fp, ip, parent_ip, fregs, data);  in __fprobe_handler()
228 					  struct fprobe *fp, struct ftrace_regs *fregs,  in __fprobe_kprobe_handler()  argument
239 		fp->nmissed++;  in __fprobe_kprobe_handler()
244 	ret = __fprobe_handler(ip, parent_ip, fp, fregs, data);  in __fprobe_kprobe_handler()
257 	struct fprobe *fp;  in fprobe_entry()  local
271 		fp = READ_ONCE(node->fp);  in fprobe_entry()
272 		if (!fp || !fp->exit_handler)  in fprobe_entry()
279 			FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size);  in fprobe_entry()
288 				fp = READ_ONCE(node->fp);  in fprobe_entry()
289 				if (fp && !fprobe_disabled(fp))  in fprobe_entry()
290 					fp->nmissed++;  in fprobe_entry()
308 		fp = READ_ONCE(node->fp);  in fprobe_entry()
309 		if (!fp || fprobe_disabled(fp))  in fprobe_entry()
312 		data_size = fp->entry_data_size;  in fprobe_entry()
313 		if (data_size && fp->exit_handler)  in fprobe_entry()
318 		if (fprobe_shared_with_kprobes(fp))  in fprobe_entry()
319 			ret = __fprobe_kprobe_handler(func, ret_ip, fp, fregs, data);  in fprobe_entry()
321 			ret = __fprobe_handler(func, ret_ip, fp, fregs, data);  in fprobe_entry()
324 		if (!ret && fp->exit_handler) {  in fprobe_entry()
327 			if (write_fprobe_header(&fgraph_data[used], fp, size_words))  in fprobe_entry()
345 	struct fprobe *fp;  in fprobe_return()  local
359 		read_fprobe_header(&fgraph_data[curr], &fp, &size);  in fprobe_return()
360 		if (!fp)  in fprobe_return()
363 		if (is_fprobe_still_exist(fp) && !fprobe_disabled(fp)) {  in fprobe_return()
366 			fp->exit_handler(fp, trace->func, ret_ip, fregs,  in fprobe_return()
607 static void fprobe_fail_cleanup(struct fprobe *fp)  in fprobe_fail_cleanup()  argument
609 	kfree(fp->hlist_array);  in fprobe_fail_cleanup()
610 	fp->hlist_array = NULL;  in fprobe_fail_cleanup()
614 static int fprobe_init(struct fprobe *fp, unsigned long *addrs, int num)  in fprobe_init()  argument
620 	if (!fp || !addrs || num <= 0)  in fprobe_init()
623 	size = ALIGN(fp->entry_data_size, sizeof(long));  in fprobe_init()
626 	fp->entry_data_size = size;  in fprobe_init()
632 	fp->nmissed = 0;  in fprobe_init()
635 	fp->hlist_array = hlist_array;  in fprobe_init()
636 	hlist_array->fp = fp;  in fprobe_init()
638 		hlist_array->array[i].fp = fp;  in fprobe_init()
641 			fprobe_fail_cleanup(fp);  in fprobe_init()
662 int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)  in register_fprobe()  argument
668 	if (!fp || !filter)  in register_fprobe()
687 	ret = register_fprobe_ips(fp, addrs, ret);  in register_fprobe()
710 int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)  in register_fprobe_ips()  argument
715 	ret = fprobe_init(fp, addrs, num);  in register_fprobe_ips()
721 	hlist_array = fp->hlist_array;  in register_fprobe_ips()
724 		add_fprobe_hash(fp);  in register_fprobe_ips()
731 		fprobe_fail_cleanup(fp);  in register_fprobe_ips()
748 int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)  in register_fprobe_syms()  argument
753 	if (!fp || !syms || num <= 0)  in register_fprobe_syms()
760 	ret = register_fprobe_ips(fp, addrs, num);  in register_fprobe_syms()
768 bool fprobe_is_registered(struct fprobe *fp)  in fprobe_is_registered()  argument
770 	if (!fp || !fp->hlist_array)  in fprobe_is_registered()
783 int unregister_fprobe(struct fprobe *fp)  in unregister_fprobe()  argument
790 	if (!fp || !is_fprobe_still_exist(fp)) {  in unregister_fprobe()
795 	hlist_array = fp->hlist_array;  in unregister_fprobe()
808 	del_fprobe_hash(fp);  in unregister_fprobe()
813 	fp->hlist_array = NULL;  in unregister_fprobe()