291 	struct osnoise_variables *osn_var;  in osn_var_reset()  local
299 		osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in osn_var_reset()
300 		memset(osn_var, 0, sizeof(*osn_var));  in osn_var_reset()
350 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)  in timerlat_softirq_exit()  argument
358 		osn_var->softirq.arrival_time = 0;  in timerlat_softirq_exit()
359 		osn_var->softirq.delta_start = 0;  in timerlat_softirq_exit()
365 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)  in timerlat_thread_exit()  argument
373 		osn_var->thread.delta_start = 0;  in timerlat_thread_exit()
374 		osn_var->thread.arrival_time = 0;  in timerlat_thread_exit()
385 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)  in timerlat_softirq_exit()  argument
389 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)  in timerlat_thread_exit()  argument
697 cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_irq_delta_start()  argument
699 	if (osn_var->irq.delta_start)  in cond_move_irq_delta_start()
700 		osn_var->irq.delta_start += duration;  in cond_move_irq_delta_start()
713 cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_softirq_delta_start()  argument
715 	if (osn_var->softirq.delta_start)  in cond_move_softirq_delta_start()
716 		osn_var->softirq.delta_start += duration;  in cond_move_softirq_delta_start()
719 #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)  argument
731 cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_thread_delta_start()  argument
733 	if (osn_var->thread.delta_start)  in cond_move_thread_delta_start()
734 		osn_var->thread.delta_start += duration;  in cond_move_thread_delta_start()
765 get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)  in get_int_safe_duration()  argument
771 		int_counter = local_read(&osn_var->int_counter);  in get_int_safe_duration()
780 	} while (int_counter != local_read(&osn_var->int_counter));  in get_int_safe_duration()
804 set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)  in set_int_safe_time()  argument
809 		int_counter = local_read(&osn_var->int_counter);  in set_int_safe_time()
817 	} while (int_counter != local_read(&osn_var->int_counter));  in set_int_safe_time()
827 copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)  in copy_int_safe_time()  argument
832 		int_counter = local_read(&osn_var->int_counter);  in copy_int_safe_time()
840 	} while (int_counter != local_read(&osn_var->int_counter));  in copy_int_safe_time()
856 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_osnoise_callback()  local
859 	if (!osn_var->sampling)  in trace_osnoise_callback()
868 			osn_var->nmi.delta_start = time_get();  in trace_osnoise_callback()
869 			local_inc(&osn_var->int_counter);  in trace_osnoise_callback()
871 			duration = time_get() - osn_var->nmi.delta_start;  in trace_osnoise_callback()
873 			trace_nmi_noise(osn_var->nmi.delta_start, duration);  in trace_osnoise_callback()
875 			cond_move_irq_delta_start(osn_var, duration);  in trace_osnoise_callback()
876 			cond_move_softirq_delta_start(osn_var, duration);  in trace_osnoise_callback()
877 			cond_move_thread_delta_start(osn_var, duration);  in trace_osnoise_callback()
882 		osn_var->nmi.count++;  in trace_osnoise_callback()
896 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in osnoise_trace_irq_entry()  local
898 	if (!osn_var->sampling)  in osnoise_trace_irq_entry()
904 	osn_var->irq.arrival_time = time_get();  in osnoise_trace_irq_entry()
905 	set_int_safe_time(osn_var, &osn_var->irq.delta_start);  in osnoise_trace_irq_entry()
906 	osn_var->irq.count++;  in osnoise_trace_irq_entry()
908 	local_inc(&osn_var->int_counter);  in osnoise_trace_irq_entry()
919 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in osnoise_trace_irq_exit()  local
922 	if (!osn_var->sampling)  in osnoise_trace_irq_exit()
925 	duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);  in osnoise_trace_irq_exit()
926 	trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);  in osnoise_trace_irq_exit()
927 	osn_var->irq.arrival_time = 0;  in osnoise_trace_irq_exit()
928 	cond_move_softirq_delta_start(osn_var, duration);  in osnoise_trace_irq_exit()
929 	cond_move_thread_delta_start(osn_var, duration);  in osnoise_trace_irq_exit()
1027 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_softirq_entry_callback()  local
1029 	if (!osn_var->sampling)  in trace_softirq_entry_callback()
1035 	osn_var->softirq.arrival_time = time_get();  in trace_softirq_entry_callback()
1036 	set_int_safe_time(osn_var, &osn_var->softirq.delta_start);  in trace_softirq_entry_callback()
1037 	osn_var->softirq.count++;  in trace_softirq_entry_callback()
1039 	local_inc(&osn_var->int_counter);  in trace_softirq_entry_callback()
1050 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_softirq_exit_callback()  local
1053 	if (!osn_var->sampling)  in trace_softirq_exit_callback()
1057 		if (!timerlat_softirq_exit(osn_var))  in trace_softirq_exit_callback()
1060 	duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);  in trace_softirq_exit_callback()
1061 	trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);  in trace_softirq_exit_callback()
1062 	cond_move_thread_delta_start(osn_var, duration);  in trace_softirq_exit_callback()
1063 	osn_var->softirq.arrival_time = 0;  in trace_softirq_exit_callback()
1123 thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)  in thread_entry()  argument
1125 	if (!osn_var->sampling)  in thread_entry()
1131 	osn_var->thread.arrival_time = time_get();  in thread_entry()
1133 	set_int_safe_time(osn_var, &osn_var->thread.delta_start);  in thread_entry()
1135 	osn_var->thread.count++;  in thread_entry()
1136 	local_inc(&osn_var->int_counter);  in thread_entry()
1145 thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)  in thread_exit()  argument
1149 	if (!osn_var->sampling)  in thread_exit()
1153 		if (!timerlat_thread_exit(osn_var))  in thread_exit()
1156 	duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);  in thread_exit()
1158 	trace_thread_noise(t, osn_var->thread.arrival_time, duration);  in thread_exit()
1160 	osn_var->thread.arrival_time = 0;  in thread_exit()
1198 	struct osnoise_variables *osn_var;  in trace_sched_migrate_callback()  local
1201 	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in trace_sched_migrate_callback()
1202 	if (osn_var->pid == p->pid && dest_cpu != cpu) {  in trace_sched_migrate_callback()
1258 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_sched_switch_callback()  local
1261 	if ((p->pid != osn_var->pid) || !workload)  in trace_sched_switch_callback()
1262 		thread_exit(osn_var, p);  in trace_sched_switch_callback()
1264 	if ((n->pid != osn_var->pid) || !workload)  in trace_sched_switch_callback()
1265 		thread_entry(osn_var, n);  in trace_sched_switch_callback()
1313 save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)  in save_osn_sample_stats()  argument
1315 	s->nmi_count = osn_var->nmi.count;  in save_osn_sample_stats()
1316 	s->irq_count = osn_var->irq.count;  in save_osn_sample_stats()
1317 	s->softirq_count = osn_var->softirq.count;  in save_osn_sample_stats()
1318 	s->thread_count = osn_var->thread.count;  in save_osn_sample_stats()
1329 diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)  in diff_osn_sample_stats()  argument
1331 	s->nmi_count = osn_var->nmi.count - s->nmi_count;  in diff_osn_sample_stats()
1332 	s->irq_count = osn_var->irq.count - s->irq_count;  in diff_osn_sample_stats()
1333 	s->softirq_count = osn_var->softirq.count - s->softirq_count;  in diff_osn_sample_stats()
1334 	s->thread_count = osn_var->thread.count - s->thread_count;  in diff_osn_sample_stats()
1405 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in run_osnoise()  local
1427 	osn_var->pid = current->pid;  in run_osnoise()
1432 	save_osn_sample_stats(osn_var, &s);  in run_osnoise()
1451 	osn_var->sampling = true;  in run_osnoise()
1469 	last_int_count = set_int_safe_time(osn_var, &last_sample);  in run_osnoise()
1475 		int_count = set_int_safe_time(osn_var, &sample);  in run_osnoise()
1559 	osn_var->sampling = false;  in run_osnoise()
1584 	diff_osn_sample_stats(osn_var, &s);  in run_osnoise()
1717 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in timerlat_irq()  local
1737 	osn_var->thread.arrival_time = time_get();  in timerlat_irq()
1760 	if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {  in timerlat_irq()
1761 		copy_int_safe_time(osn_var, &osn_var->thread.delta_start,  in timerlat_irq()
1762 				   &osn_var->softirq.delta_start);  in timerlat_irq()
1764 		copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,  in timerlat_irq()
1765 				    &osn_var->irq.delta_start);  in timerlat_irq()
1767 		copy_int_safe_time(osn_var, &osn_var->thread.delta_start,  in timerlat_irq()
1768 				    &osn_var->irq.delta_start);  in timerlat_irq()
1851 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in timerlat_main()  local
1881 	osn_var->pid = current->pid;  in timerlat_main()
1889 	osn_var->sampling = 1;  in timerlat_main()
2373 	struct osnoise_variables *osn_var;  in timerlat_fd_open()  local
2390 	osn_var = this_cpu_osn_var();  in timerlat_fd_open()
2395 	if (osn_var->pid) {  in timerlat_fd_open()
2421 	osn_var->kthread = current;  in timerlat_fd_open()
2422 	osn_var->pid = current->pid;  in timerlat_fd_open()
2452 	struct osnoise_variables *osn_var;  in timerlat_fd_read()  local
2480 	osn_var = this_cpu_osn_var();  in timerlat_fd_read()
2490 	if (likely(osn_var->sampling)) {  in timerlat_fd_read()
2519 		osn_var->sampling = 1;  in timerlat_fd_read()
2556 	struct osnoise_variables *osn_var;  in timerlat_fd_release()  local
2563 	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in timerlat_fd_release()
2570 	osn_var->sampling = 0;  in timerlat_fd_release()
2571 	osn_var->pid = 0;  in timerlat_fd_release()
2576 	if (osn_var->kthread) {  in timerlat_fd_release()
2577 		put_task_struct(osn_var->kthread);  in timerlat_fd_release()
2578 		osn_var->kthread = NULL;  in timerlat_fd_release()