72 static bool trace_fprobe_is_return(struct trace_fprobe *tf)  in trace_fprobe_is_return()  argument
74 	return tf->fp.exit_handler != NULL;  in trace_fprobe_is_return()
77 static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf)  in trace_fprobe_is_tracepoint()  argument
79 	return tf->tpoint != NULL;  in trace_fprobe_is_tracepoint()
82 static const char *trace_fprobe_symbol(struct trace_fprobe *tf)  in trace_fprobe_symbol()  argument
84 	return tf->symbol ? tf->symbol : "unknown";  in trace_fprobe_symbol()
89 	struct trace_fprobe *tf = to_trace_fprobe(ev);  in trace_fprobe_is_busy()  local
91 	return trace_probe_is_enabled(&tf->tp);  in trace_fprobe_is_busy()
94 static bool trace_fprobe_match_command_head(struct trace_fprobe *tf,  in trace_fprobe_match_command_head()  argument
102 	snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf));  in trace_fprobe_match_command_head()
107 	return trace_probe_match_command_args(&tf->tp, argc, argv);  in trace_fprobe_match_command_head()
113 	struct trace_fprobe *tf = to_trace_fprobe(ev);  in trace_fprobe_match()  local
115 	if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event))  in trace_fprobe_match()
118 	if (system && strcmp(trace_probe_group_name(&tf->tp), system))  in trace_fprobe_match()
121 	return trace_fprobe_match_command_head(tf, argc, argv);  in trace_fprobe_match()
124 static bool trace_fprobe_is_registered(struct trace_fprobe *tf)  in trace_fprobe_is_registered()  argument
126 	return fprobe_is_registered(&tf->fp);  in trace_fprobe_is_registered()
177 __fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,  in NOKPROBE_SYMBOL()
182 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in NOKPROBE_SYMBOL()
192 	dsize = __get_data_size(&tf->tp, fregs, NULL);  in NOKPROBE_SYMBOL()
195 					   sizeof(*entry) + tf->tp.size + dsize);  in NOKPROBE_SYMBOL()
202 	store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize);  in NOKPROBE_SYMBOL()
208 fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,  in fentry_trace_func()  argument
213 	trace_probe_for_each_link_rcu(link, &tf->tp)  in fentry_trace_func()
214 		__fentry_trace_func(tf, entry_ip, fregs, link->file);  in fentry_trace_func()
253 	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);  in trace_fprobe_entry_handler()  local
255 	if (tf->tp.entry_arg)  in trace_fprobe_entry_handler()
256 		store_fprobe_entry_data(entry_data, &tf->tp, fregs);  in trace_fprobe_entry_handler()
263 __fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,  in NOKPROBE_SYMBOL()
269 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in NOKPROBE_SYMBOL()
278 	dsize = __get_data_size(&tf->tp, fregs, entry_data);  in NOKPROBE_SYMBOL()
281 					   sizeof(*entry) + tf->tp.size + dsize);  in NOKPROBE_SYMBOL()
289 	store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize);  in NOKPROBE_SYMBOL()
295 fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,  in fexit_trace_func()  argument
300 	trace_probe_for_each_link_rcu(link, &tf->tp)  in fexit_trace_func()
301 		__fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data, link->file);  in fexit_trace_func()
307 static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,  in fentry_perf_func()  argument
310 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in fentry_perf_func()
321 	dsize = __get_data_size(&tf->tp, fregs, NULL);  in fentry_perf_func()
322 	__size = sizeof(*entry) + tf->tp.size + dsize;  in fentry_perf_func()
334 	store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize);  in fentry_perf_func()
342 fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,  in fexit_perf_func()  argument
346 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in fexit_perf_func()
357 	dsize = __get_data_size(&tf->tp, fregs, entry_data);  in fexit_perf_func()
358 	__size = sizeof(*entry) + tf->tp.size + dsize;  in fexit_perf_func()
370 	store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize);  in fexit_perf_func()
381 	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);  in fentry_dispatcher()  local
384 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))  in fentry_dispatcher()
385 		fentry_trace_func(tf, entry_ip, fregs);  in fentry_dispatcher()
388 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))  in fentry_dispatcher()
389 		ret = fentry_perf_func(tf, entry_ip, fregs);  in fentry_dispatcher()
399 	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);  in fexit_dispatcher()  local
401 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))  in fexit_dispatcher()
402 		fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data);  in fexit_dispatcher()
404 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))  in fexit_dispatcher()
405 		fexit_perf_func(tf, entry_ip, ret_ip, fregs, entry_data);  in fexit_dispatcher()
410 static void free_trace_fprobe(struct trace_fprobe *tf)  in free_trace_fprobe()  argument
412 	if (tf) {  in free_trace_fprobe()
413 		trace_probe_cleanup(&tf->tp);  in free_trace_fprobe()
414 		kfree(tf->symbol);  in free_trace_fprobe()
415 		kfree(tf);  in free_trace_fprobe()
432 	struct trace_fprobe *tf __free(free_trace_fprobe) = NULL;  in alloc_trace_fprobe()
435 	tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL);  in alloc_trace_fprobe()
436 	if (!tf)  in alloc_trace_fprobe()
439 	tf->symbol = kstrdup(symbol, GFP_KERNEL);  in alloc_trace_fprobe()
440 	if (!tf->symbol)  in alloc_trace_fprobe()
444 		tf->fp.exit_handler = fexit_dispatcher;  in alloc_trace_fprobe()
446 		tf->fp.entry_handler = fentry_dispatcher;  in alloc_trace_fprobe()
448 	tf->tpoint = tpoint;  in alloc_trace_fprobe()
449 	tf->mod = mod;  in alloc_trace_fprobe()
451 	ret = trace_probe_init(&tf->tp, event, group, false, nargs);  in alloc_trace_fprobe()
455 	dyn_event_init(&tf->devent, &trace_fprobe_ops);  in alloc_trace_fprobe()
456 	return_ptr(tf);  in alloc_trace_fprobe()
463 	struct trace_fprobe *tf;  in find_trace_fprobe()  local
465 	for_each_trace_fprobe(tf, pos)  in find_trace_fprobe()
466 		if (strcmp(trace_probe_name(&tf->tp), event) == 0 &&  in find_trace_fprobe()
467 		    strcmp(trace_probe_group_name(&tf->tp), group) == 0)  in find_trace_fprobe()
468 			return tf;  in find_trace_fprobe()
472 static inline int __enable_trace_fprobe(struct trace_fprobe *tf)  in __enable_trace_fprobe()  argument
474 	if (trace_fprobe_is_registered(tf))  in __enable_trace_fprobe()
475 		enable_fprobe(&tf->fp);  in __enable_trace_fprobe()
482 	struct trace_fprobe *tf;  in __disable_trace_fprobe()  local
484 	list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {  in __disable_trace_fprobe()
485 		if (!trace_fprobe_is_registered(tf))  in __disable_trace_fprobe()
487 		disable_fprobe(&tf->fp);  in __disable_trace_fprobe()
499 	struct trace_fprobe *tf;  in enable_trace_fprobe()  local
517 		list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {  in enable_trace_fprobe()
519 			__enable_trace_fprobe(tf);  in enable_trace_fprobe()
685 static inline void init_trace_event_call(struct trace_fprobe *tf)  in init_trace_event_call()  argument
687 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in init_trace_event_call()
689 	if (trace_fprobe_is_return(tf)) {  in init_trace_event_call()
701 static int register_fprobe_event(struct trace_fprobe *tf)  in register_fprobe_event()  argument
703 	init_trace_event_call(tf);  in register_fprobe_event()
705 	return trace_probe_register_event_call(&tf->tp);  in register_fprobe_event()
708 static int unregister_fprobe_event(struct trace_fprobe *tf)  in unregister_fprobe_event()  argument
710 	return trace_probe_unregister_event_call(&tf->tp);  in unregister_fprobe_event()
713 static int __regsiter_tracepoint_fprobe(struct trace_fprobe *tf)  in __regsiter_tracepoint_fprobe()  argument
715 	struct tracepoint *tpoint = tf->tpoint;  in __regsiter_tracepoint_fprobe()
728 	return register_fprobe_ips(&tf->fp, &ip, 1);  in __regsiter_tracepoint_fprobe()
732 static int __register_trace_fprobe(struct trace_fprobe *tf)  in __register_trace_fprobe()  argument
741 	if (trace_fprobe_is_registered(tf))  in __register_trace_fprobe()
744 	for (i = 0; i < tf->tp.nr_args; i++) {  in __register_trace_fprobe()
745 		ret = traceprobe_update_arg(&tf->tp.args[i]);  in __register_trace_fprobe()
751 	if (trace_probe_is_enabled(&tf->tp))  in __register_trace_fprobe()
752 		tf->fp.flags &= ~FPROBE_FL_DISABLED;  in __register_trace_fprobe()
754 		tf->fp.flags |= FPROBE_FL_DISABLED;  in __register_trace_fprobe()
756 	if (trace_fprobe_is_tracepoint(tf)) {  in __register_trace_fprobe()
759 		if (tf->tpoint == TRACEPOINT_STUB)  in __register_trace_fprobe()
762 		return __regsiter_tracepoint_fprobe(tf);  in __register_trace_fprobe()
766 	return register_fprobe(&tf->fp, tf->symbol, NULL);  in __register_trace_fprobe()
770 static void __unregister_trace_fprobe(struct trace_fprobe *tf)  in __unregister_trace_fprobe()  argument
772 	if (trace_fprobe_is_registered(tf)) {  in __unregister_trace_fprobe()
773 		unregister_fprobe(&tf->fp);  in __unregister_trace_fprobe()
774 		memset(&tf->fp, 0, sizeof(tf->fp));  in __unregister_trace_fprobe()
775 		if (trace_fprobe_is_tracepoint(tf)) {  in __unregister_trace_fprobe()
776 			tracepoint_probe_unregister(tf->tpoint,  in __unregister_trace_fprobe()
777 					tf->tpoint->probestub, NULL);  in __unregister_trace_fprobe()
778 			tf->tpoint = NULL;  in __unregister_trace_fprobe()
779 			tf->mod = NULL;  in __unregister_trace_fprobe()
786 static int unregister_trace_fprobe(struct trace_fprobe *tf)  in unregister_trace_fprobe()  argument
789 	if (trace_probe_has_sibling(&tf->tp))  in unregister_trace_fprobe()
793 	if (trace_probe_is_enabled(&tf->tp))  in unregister_trace_fprobe()
797 	if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp)))  in unregister_trace_fprobe()
801 	if (unregister_fprobe_event(tf))  in unregister_trace_fprobe()
805 	__unregister_trace_fprobe(tf);  in unregister_trace_fprobe()
806 	dyn_event_remove(&tf->devent);  in unregister_trace_fprobe()
807 	trace_probe_unlink(&tf->tp);  in unregister_trace_fprobe()
840 static int append_trace_fprobe(struct trace_fprobe *tf, struct trace_fprobe *to)  in append_trace_fprobe()  argument
844 	if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) ||  in append_trace_fprobe()
845 	    trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) {  in append_trace_fprobe()
850 	ret = trace_probe_compare_arg_type(&tf->tp, &to->tp);  in append_trace_fprobe()
857 	if (trace_fprobe_has_same_fprobe(to, tf)) {  in append_trace_fprobe()
864 	ret = trace_probe_append(&tf->tp, &to->tp);  in append_trace_fprobe()
868 	ret = __register_trace_fprobe(tf);  in append_trace_fprobe()
870 		trace_probe_unlink(&tf->tp);  in append_trace_fprobe()
872 		dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));  in append_trace_fprobe()
878 static int register_trace_fprobe(struct trace_fprobe *tf)  in register_trace_fprobe()  argument
885 	old_tf = find_trace_fprobe(trace_probe_name(&tf->tp),  in register_trace_fprobe()
886 				   trace_probe_group_name(&tf->tp));  in register_trace_fprobe()
888 		return append_trace_fprobe(tf, old_tf);  in register_trace_fprobe()
891 	ret = register_fprobe_event(tf);  in register_trace_fprobe()
902 	ret = __register_trace_fprobe(tf);  in register_trace_fprobe()
904 		unregister_fprobe_event(tf);  in register_trace_fprobe()
906 		dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));  in register_trace_fprobe()
966 static void reenable_trace_fprobe(struct trace_fprobe *tf)  in reenable_trace_fprobe()  argument
968 	struct trace_probe *tp = &tf->tp;  in reenable_trace_fprobe()
970 	list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {  in reenable_trace_fprobe()
971 		__enable_trace_fprobe(tf);  in reenable_trace_fprobe()
996 	struct trace_fprobe *tf;  in __tracepoint_probe_module_cb()  local
1003 	for_each_trace_fprobe(tf, pos) {  in __tracepoint_probe_module_cb()
1004 		if (val == MODULE_STATE_COMING && tf->tpoint == TRACEPOINT_STUB) {  in __tracepoint_probe_module_cb()
1005 			tpoint = find_tracepoint_in_module(tp_mod->mod, tf->symbol);  in __tracepoint_probe_module_cb()
1007 				tf->tpoint = tpoint;  in __tracepoint_probe_module_cb()
1008 				tf->mod = tp_mod->mod;  in __tracepoint_probe_module_cb()
1009 				if (!WARN_ON_ONCE(__regsiter_tracepoint_fprobe(tf)) &&  in __tracepoint_probe_module_cb()
1010 				    trace_probe_is_enabled(&tf->tp))  in __tracepoint_probe_module_cb()
1011 					reenable_trace_fprobe(tf);  in __tracepoint_probe_module_cb()
1013 		} else if (val == MODULE_STATE_GOING && tp_mod->mod == tf->mod) {  in __tracepoint_probe_module_cb()
1014 			unregister_fprobe(&tf->fp);  in __tracepoint_probe_module_cb()
1015 			if (trace_fprobe_is_tracepoint(tf)) {  in __tracepoint_probe_module_cb()
1016 				tracepoint_probe_unregister(tf->tpoint,  in __tracepoint_probe_module_cb()
1017 					tf->tpoint->probestub, NULL);  in __tracepoint_probe_module_cb()
1018 				tf->tpoint = TRACEPOINT_STUB;  in __tracepoint_probe_module_cb()
1019 				tf->mod = NULL;  in __tracepoint_probe_module_cb()
1118 	struct trace_fprobe *tf __free(free_trace_fprobe) = NULL;  in DEFINE_FREE()
1221 	tf = alloc_trace_fprobe(group, event, symbol, tpoint, tp_mod,  in DEFINE_FREE()
1223 	if (IS_ERR(tf)) {  in DEFINE_FREE()
1224 		ret = PTR_ERR(tf);  in DEFINE_FREE()
1234 		ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], ctx);  in DEFINE_FREE()
1239 	if (is_return && tf->tp.entry_arg) {  in DEFINE_FREE()
1240 		tf->fp.entry_handler = trace_fprobe_entry_handler;  in DEFINE_FREE()
1241 		tf->fp.entry_data_size = traceprobe_get_entry_data_size(&tf->tp);  in DEFINE_FREE()
1242 		if (ALIGN(tf->fp.entry_data_size, sizeof(long)) > MAX_FPROBE_DATA_SIZE) {  in DEFINE_FREE()
1249 	ret = traceprobe_set_print_fmt(&tf->tp,  in DEFINE_FREE()
1254 	ret = register_trace_fprobe(tf);  in DEFINE_FREE()
1267 	tf = NULL;  in DEFINE_FREE()
1293 	struct trace_fprobe *tf = to_trace_fprobe(ev);  in trace_fprobe_release()  local
1294 	int ret = unregister_trace_fprobe(tf);  in trace_fprobe_release()
1297 		free_trace_fprobe(tf);  in trace_fprobe_release()
1303 	struct trace_fprobe *tf = to_trace_fprobe(ev);  in trace_fprobe_show()  local
1306 	if (trace_fprobe_is_tracepoint(tf))  in trace_fprobe_show()
1310 	seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp),  in trace_fprobe_show()
1311 				trace_probe_name(&tf->tp));  in trace_fprobe_show()
1313 	seq_printf(m, " %s%s", trace_fprobe_symbol(tf),  in trace_fprobe_show()
1314 			       trace_fprobe_is_return(tf) ? "%return" : "");  in trace_fprobe_show()
1316 	for (i = 0; i < tf->tp.nr_args; i++)  in trace_fprobe_show()
1317 		seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);  in trace_fprobe_show()