22 static void tracing_start_function_trace(struct trace_array *tr);
23 static void tracing_stop_function_trace(struct trace_array *tr);
56 int ftrace_allocate_ftrace_ops(struct trace_array *tr)  in ftrace_allocate_ftrace_ops()  argument
61 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)  in ftrace_allocate_ftrace_ops()
72 	tr->ops = ops;  in ftrace_allocate_ftrace_ops()
73 	ops->private = tr;  in ftrace_allocate_ftrace_ops()
78 void ftrace_free_ftrace_ops(struct trace_array *tr)  in ftrace_free_ftrace_ops()  argument
80 	kfree(tr->ops);  in ftrace_free_ftrace_ops()
81 	tr->ops = NULL;  in ftrace_free_ftrace_ops()
84 int ftrace_create_function_files(struct trace_array *tr,  in ftrace_create_function_files()  argument
92 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)  in ftrace_create_function_files()
95 	if (!tr->ops)  in ftrace_create_function_files()
98 	ret = allocate_fgraph_ops(tr, tr->ops);  in ftrace_create_function_files()
100 		kfree(tr->ops);  in ftrace_create_function_files()
104 	ftrace_create_filter_files(tr->ops, parent);  in ftrace_create_function_files()
109 void ftrace_destroy_function_files(struct trace_array *tr)  in ftrace_destroy_function_files()  argument
111 	ftrace_destroy_filter_files(tr->ops);  in ftrace_destroy_function_files()
112 	ftrace_free_ftrace_ops(tr);  in ftrace_destroy_function_files()
113 	free_fgraph_ops(tr);  in ftrace_destroy_function_files()
134 static bool handle_func_repeats(struct trace_array *tr, u32 flags_val)  in handle_func_repeats()  argument
136 	if (!tr->last_func_repeats &&  in handle_func_repeats()
138 		tr->last_func_repeats = alloc_percpu(struct trace_func_repeats);  in handle_func_repeats()
139 		if (!tr->last_func_repeats)  in handle_func_repeats()
146 static int function_trace_init(struct trace_array *tr)  in function_trace_init()  argument
154 	if (!tr->ops)  in function_trace_init()
161 	if (!handle_func_repeats(tr, func_flags.val))  in function_trace_init()
164 	ftrace_init_array_ops(tr, func);  in function_trace_init()
166 	tr->array_buffer.cpu = raw_smp_processor_id();  in function_trace_init()
169 	tracing_start_function_trace(tr);  in function_trace_init()
173 static void function_trace_reset(struct trace_array *tr)  in function_trace_reset()  argument
175 	tracing_stop_function_trace(tr);  in function_trace_reset()
177 	ftrace_reset_array_ops(tr);  in function_trace_reset()
180 static void function_trace_start(struct trace_array *tr)  in function_trace_start()  argument
182 	tracing_reset_online_cpus(&tr->array_buffer);  in function_trace_start()
211 	struct trace_array *tr = op->private;  in function_trace_call()  local
216 	if (unlikely(!tr->function_enabled))  in function_trace_call()
227 	data = this_cpu_ptr(tr->array_buffer.data);  in function_trace_call()
229 		trace_function(tr, ip, parent_ip, trace_ctx, NULL);  in function_trace_call()
238 	struct trace_array *tr = op->private;  in function_args_trace_call()  local
244 	if (unlikely(!tr->function_enabled))  in function_args_trace_call()
254 	data = per_cpu_ptr(tr->array_buffer.data, cpu);  in function_args_trace_call()
256 		trace_function(tr, ip, parent_ip, trace_ctx, fregs);  in function_args_trace_call()
283 	struct trace_array *tr = op->private;  in function_stack_trace_call()  local
291 	if (unlikely(!tr->function_enabled))  in function_stack_trace_call()
301 	data = per_cpu_ptr(tr->array_buffer.data, cpu);  in function_stack_trace_call()
306 		trace_function(tr, ip, parent_ip, trace_ctx, NULL);  in function_stack_trace_call()
311 		__trace_stack(tr, trace_ctx, skip);  in function_stack_trace_call()
318 static inline bool is_repeat_check(struct trace_array *tr,  in is_repeat_check()  argument
326 			ring_buffer_time_stamp(tr->array_buffer.buffer);  in is_repeat_check()
334 static inline void process_repeats(struct trace_array *tr,  in process_repeats()  argument
340 		trace_last_func_repeats(tr, last_info, trace_ctx);  in process_repeats()
354 	struct trace_array *tr = op->private;  in function_no_repeats_trace_call()  local
359 	if (unlikely(!tr->function_enabled))  in function_no_repeats_trace_call()
367 	data = this_cpu_ptr(tr->array_buffer.data);  in function_no_repeats_trace_call()
378 	last_info = this_cpu_ptr(tr->last_func_repeats);  in function_no_repeats_trace_call()
379 	if (is_repeat_check(tr, last_info, ip, parent_ip))  in function_no_repeats_trace_call()
383 	process_repeats(tr, ip, parent_ip, last_info, trace_ctx);  in function_no_repeats_trace_call()
385 	trace_function(tr, ip, parent_ip, trace_ctx, NULL);  in function_no_repeats_trace_call()
397 	struct trace_array *tr = op->private;  in function_stack_no_repeats_trace_call()  local
404 	if (unlikely(!tr->function_enabled))  in function_stack_no_repeats_trace_call()
414 	data = per_cpu_ptr(tr->array_buffer.data, cpu);  in function_stack_no_repeats_trace_call()
418 		last_info = per_cpu_ptr(tr->last_func_repeats, cpu);  in function_stack_no_repeats_trace_call()
419 		if (is_repeat_check(tr, last_info, ip, parent_ip))  in function_stack_no_repeats_trace_call()
423 		process_repeats(tr, ip, parent_ip, last_info, trace_ctx);  in function_stack_no_repeats_trace_call()
425 		trace_function(tr, ip, parent_ip, trace_ctx, NULL);  in function_stack_no_repeats_trace_call()
426 		__trace_stack(tr, trace_ctx, STACK_SKIP);  in function_stack_no_repeats_trace_call()
450 static void tracing_start_function_trace(struct trace_array *tr)  in tracing_start_function_trace()  argument
452 	tr->function_enabled = 0;  in tracing_start_function_trace()
453 	register_ftrace_function(tr->ops);  in tracing_start_function_trace()
454 	tr->function_enabled = 1;  in tracing_start_function_trace()
457 static void tracing_stop_function_trace(struct trace_array *tr)  in tracing_stop_function_trace()  argument
459 	tr->function_enabled = 0;  in tracing_stop_function_trace()
460 	unregister_ftrace_function(tr->ops);  in tracing_stop_function_trace()
466 func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)  in func_set_flag()  argument
476 	if (tr->current_trace != &function_trace)  in func_set_flag()
485 	if (tr->ops->func == func)  in func_set_flag()
488 	if (!handle_func_repeats(tr, new_flags))  in func_set_flag()
491 	unregister_ftrace_function(tr->ops);  in func_set_flag()
492 	tr->ops->func = func;  in func_set_flag()
493 	register_ftrace_function(tr->ops);  in func_set_flag()
515 				 struct trace_array *tr, bool on,  in update_traceon_count()  argument
560 	if (on == !!tracer_tracing_is_on(tr))  in update_traceon_count()
564 		tracer_tracing_on(tr);  in update_traceon_count()
566 		tracer_tracing_off(tr);  in update_traceon_count()
576 		     struct trace_array *tr, struct ftrace_probe_ops *ops,  in ftrace_traceon_count()  argument
579 	update_traceon_count(ops, ip, tr, 1, data);  in ftrace_traceon_count()
584 		      struct trace_array *tr, struct ftrace_probe_ops *ops,  in ftrace_traceoff_count()  argument
587 	update_traceon_count(ops, ip, tr, 0, data);  in ftrace_traceoff_count()
592 	       struct trace_array *tr, struct ftrace_probe_ops *ops,  in ftrace_traceon()  argument
595 	if (tracer_tracing_is_on(tr))  in ftrace_traceon()
598 	tracer_tracing_on(tr);  in ftrace_traceon()
603 		struct trace_array *tr, struct ftrace_probe_ops *ops,  in ftrace_traceoff()  argument
606 	if (!tracer_tracing_is_on(tr))  in ftrace_traceoff()
609 	tracer_tracing_off(tr);  in ftrace_traceoff()
634 static __always_inline void trace_stack(struct trace_array *tr)  in trace_stack()  argument
636 	__trace_stack(tr, tracing_gen_ctx_dec(), FTRACE_STACK_SKIP);  in trace_stack()
641 		  struct trace_array *tr, struct ftrace_probe_ops *ops,  in ftrace_stacktrace()  argument
644 	trace_stack(tr);  in ftrace_stacktrace()
649 			struct trace_array *tr, struct ftrace_probe_ops *ops,  in ftrace_stacktrace_count()  argument
662 		trace_stack(tr);  in ftrace_stacktrace_count()
681 			trace_stack(tr);  in ftrace_stacktrace_count()
709 		  struct trace_array *tr, struct ftrace_probe_ops *ops,  in ftrace_dump_probe()  argument
719 		     struct trace_array *tr, struct ftrace_probe_ops *ops,  in ftrace_cpudump_probe()  argument
785 ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr,  in ftrace_count_init()  argument
801 ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr,  in ftrace_count_free()  argument
863 ftrace_trace_probe_callback(struct trace_array *tr,  in ftrace_trace_probe_callback()  argument
877 		return unregister_ftrace_function_probe_func(glob+1, tr, ops);  in ftrace_trace_probe_callback()
896 	ret = register_ftrace_function_probe(glob, tr, ops, count);  in ftrace_trace_probe_callback()
902 ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash,  in ftrace_trace_onoff_callback()  argument
907 	if (!tr)  in ftrace_trace_onoff_callback()
916 	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,  in ftrace_trace_onoff_callback()
921 ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash,  in ftrace_stacktrace_callback()  argument
926 	if (!tr)  in ftrace_stacktrace_callback()
931 	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,  in ftrace_stacktrace_callback()
936 ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash,  in ftrace_dump_callback()  argument
941 	if (!tr)  in ftrace_dump_callback()
947 	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,  in ftrace_dump_callback()
952 ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash,  in ftrace_cpudump_callback()  argument
957 	if (!tr)  in ftrace_cpudump_callback()
963 	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,  in ftrace_cpudump_callback()