643 static u64 vtime_delta(struct vtime *vtime)  in vtime_delta()  argument
648 	if (clock < vtime->starttime)  in vtime_delta()
651 	return clock - vtime->starttime;  in vtime_delta()
654 static u64 get_vtime_delta(struct vtime *vtime)  in get_vtime_delta()  argument
656 	u64 delta = vtime_delta(vtime);  in get_vtime_delta()
667 	WARN_ON_ONCE(vtime->state == VTIME_INACTIVE);  in get_vtime_delta()
668 	vtime->starttime += delta;  in get_vtime_delta()
674 				 struct vtime *vtime)  in vtime_account_system()  argument
676 	vtime->stime += get_vtime_delta(vtime);  in vtime_account_system()
677 	if (vtime->stime >= TICK_NSEC) {  in vtime_account_system()
678 		account_system_time(tsk, irq_count(), vtime->stime);  in vtime_account_system()
679 		vtime->stime = 0;  in vtime_account_system()
684 				struct vtime *vtime)  in vtime_account_guest()  argument
686 	vtime->gtime += get_vtime_delta(vtime);  in vtime_account_guest()
687 	if (vtime->gtime >= TICK_NSEC) {  in vtime_account_guest()
688 		account_guest_time(tsk, vtime->gtime);  in vtime_account_guest()
689 		vtime->gtime = 0;  in vtime_account_guest()
694 				   struct vtime *vtime)  in __vtime_account_kernel()  argument
697 	if (vtime->state == VTIME_GUEST)  in __vtime_account_kernel()
698 		vtime_account_guest(tsk, vtime);  in __vtime_account_kernel()
700 		vtime_account_system(tsk, vtime);  in __vtime_account_kernel()
705 	struct vtime *vtime = &tsk->vtime;  in vtime_account_kernel()  local
707 	if (!vtime_delta(vtime))  in vtime_account_kernel()
710 	write_seqcount_begin(&vtime->seqcount);  in vtime_account_kernel()
711 	__vtime_account_kernel(tsk, vtime);  in vtime_account_kernel()
712 	write_seqcount_end(&vtime->seqcount);  in vtime_account_kernel()
717 	struct vtime *vtime = &tsk->vtime;  in vtime_user_enter()  local
719 	write_seqcount_begin(&vtime->seqcount);  in vtime_user_enter()
720 	vtime_account_system(tsk, vtime);  in vtime_user_enter()
721 	vtime->state = VTIME_USER;  in vtime_user_enter()
722 	write_seqcount_end(&vtime->seqcount);  in vtime_user_enter()
727 	struct vtime *vtime = &tsk->vtime;  in vtime_user_exit()  local
729 	write_seqcount_begin(&vtime->seqcount);  in vtime_user_exit()
730 	vtime->utime += get_vtime_delta(vtime);  in vtime_user_exit()
731 	if (vtime->utime >= TICK_NSEC) {  in vtime_user_exit()
732 		account_user_time(tsk, vtime->utime);  in vtime_user_exit()
733 		vtime->utime = 0;  in vtime_user_exit()
735 	vtime->state = VTIME_SYS;  in vtime_user_exit()
736 	write_seqcount_end(&vtime->seqcount);  in vtime_user_exit()
741 	struct vtime *vtime = &tsk->vtime;  in vtime_guest_enter()  local
749 	write_seqcount_begin(&vtime->seqcount);  in vtime_guest_enter()
750 	vtime_account_system(tsk, vtime);  in vtime_guest_enter()
752 	vtime->state = VTIME_GUEST;  in vtime_guest_enter()
753 	write_seqcount_end(&vtime->seqcount);  in vtime_guest_enter()
759 	struct vtime *vtime = &tsk->vtime;  in vtime_guest_exit()  local
761 	write_seqcount_begin(&vtime->seqcount);  in vtime_guest_exit()
762 	vtime_account_guest(tsk, vtime);  in vtime_guest_exit()
764 	vtime->state = VTIME_SYS;  in vtime_guest_exit()
765 	write_seqcount_end(&vtime->seqcount);  in vtime_guest_exit()
771 	account_idle_time(get_vtime_delta(&tsk->vtime));  in vtime_account_idle()
776 	struct vtime *vtime = &prev->vtime;  in vtime_task_switch_generic()  local
778 	write_seqcount_begin(&vtime->seqcount);  in vtime_task_switch_generic()
779 	if (vtime->state == VTIME_IDLE)  in vtime_task_switch_generic()
782 		__vtime_account_kernel(prev, vtime);  in vtime_task_switch_generic()
783 	vtime->state = VTIME_INACTIVE;  in vtime_task_switch_generic()
784 	vtime->cpu = -1;  in vtime_task_switch_generic()
785 	write_seqcount_end(&vtime->seqcount);  in vtime_task_switch_generic()
787 	vtime = &current->vtime;  in vtime_task_switch_generic()
789 	write_seqcount_begin(&vtime->seqcount);  in vtime_task_switch_generic()
791 		vtime->state = VTIME_IDLE;  in vtime_task_switch_generic()
793 		vtime->state = VTIME_GUEST;  in vtime_task_switch_generic()
795 		vtime->state = VTIME_SYS;  in vtime_task_switch_generic()
796 	vtime->starttime = sched_clock();  in vtime_task_switch_generic()
797 	vtime->cpu = smp_processor_id();  in vtime_task_switch_generic()
798 	write_seqcount_end(&vtime->seqcount);  in vtime_task_switch_generic()
803 	struct vtime *vtime = &t->vtime;  in vtime_init_idle()  local
807 	write_seqcount_begin(&vtime->seqcount);  in vtime_init_idle()
808 	vtime->state = VTIME_IDLE;  in vtime_init_idle()
809 	vtime->starttime = sched_clock();  in vtime_init_idle()
810 	vtime->cpu = cpu;  in vtime_init_idle()
811 	write_seqcount_end(&vtime->seqcount);  in vtime_init_idle()
817 	struct vtime *vtime = &t->vtime;  in task_gtime()  local
825 		seq = read_seqcount_begin(&vtime->seqcount);  in task_gtime()
828 		if (vtime->state == VTIME_GUEST)  in task_gtime()
829 			gtime += vtime->gtime + vtime_delta(vtime);  in task_gtime()
831 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in task_gtime()
843 	struct vtime *vtime = &t->vtime;  in task_cputime()  local
856 		seq = read_seqcount_begin(&vtime->seqcount);  in task_cputime()
862 		if (vtime->state < VTIME_SYS)  in task_cputime()
866 		delta = vtime_delta(vtime);  in task_cputime()
872 		if (vtime->state == VTIME_SYS)  in task_cputime()
873 			*stime += vtime->stime + delta;  in task_cputime()
875 			*utime += vtime->utime + delta;  in task_cputime()
876 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in task_cputime()
881 static int vtime_state_fetch(struct vtime *vtime, int cpu)  in vtime_state_fetch()  argument
883 	int state = READ_ONCE(vtime->state);  in vtime_state_fetch()
889 	if (vtime->cpu != cpu && vtime->cpu != -1)  in vtime_state_fetch()
907 static u64 kcpustat_user_vtime(struct vtime *vtime)  in kcpustat_user_vtime()  argument
909 	if (vtime->state == VTIME_USER)  in kcpustat_user_vtime()
910 		return vtime->utime + vtime_delta(vtime);  in kcpustat_user_vtime()
911 	else if (vtime->state == VTIME_GUEST)  in kcpustat_user_vtime()
912 		return vtime->gtime + vtime_delta(vtime);  in kcpustat_user_vtime()
921 	struct vtime *vtime = &tsk->vtime;  in kcpustat_field_vtime()  local
927 		seq = read_seqcount_begin(&vtime->seqcount);  in kcpustat_field_vtime()
929 		state = vtime_state_fetch(vtime, cpu);  in kcpustat_field_vtime()
945 				*val += vtime->stime + vtime_delta(vtime);  in kcpustat_field_vtime()
949 				*val += kcpustat_user_vtime(vtime);  in kcpustat_field_vtime()
953 				*val += kcpustat_user_vtime(vtime);  in kcpustat_field_vtime()
957 				*val += vtime->gtime + vtime_delta(vtime);  in kcpustat_field_vtime()
961 				*val += vtime->gtime + vtime_delta(vtime);  in kcpustat_field_vtime()
966 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in kcpustat_field_vtime()
1009 	struct vtime *vtime = &tsk->vtime;  in kcpustat_cpu_fetch_vtime()  local
1017 		seq = read_seqcount_begin(&vtime->seqcount);  in kcpustat_cpu_fetch_vtime()
1019 		state = vtime_state_fetch(vtime, cpu);  in kcpustat_cpu_fetch_vtime()
1030 		delta = vtime_delta(vtime);  in kcpustat_cpu_fetch_vtime()
1037 			cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;  in kcpustat_cpu_fetch_vtime()
1040 				cpustat[CPUTIME_NICE] += vtime->utime + delta;  in kcpustat_cpu_fetch_vtime()
1042 				cpustat[CPUTIME_USER] += vtime->utime + delta;  in kcpustat_cpu_fetch_vtime()
1046 				cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1047 				cpustat[CPUTIME_NICE] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1049 				cpustat[CPUTIME_GUEST] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1050 				cpustat[CPUTIME_USER] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1053 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in kcpustat_cpu_fetch_vtime()