28 struct priv_timer {  struct
52 	struct priv_timer priv_timer[RTE_MAX_LCORE];  argument
65 #define __TIMER_STAT_ADD(priv_timer, name, n) do {			\  argument
68 			priv_timer[__lcore_id].stats.name += (n);	\
71 #define __TIMER_STAT_ADD(priv_timer, name, n) do {} while (0)  argument
171 					&data->priv_timer[lcore_id].list_lock);  in rte_timer_subsystem_init()
172 				data->priv_timer[lcore_id].prev_lcore =  in rte_timer_subsystem_init()
225 		       struct priv_timer *priv_timer)  in timer_set_config_state()  argument
243 		     tim != priv_timer[lcore_id].running_tim))  in timer_set_config_state()
350 		       struct rte_timer **prev, struct priv_timer *priv_timer)  in timer_get_prev_entries()  argument
352 	unsigned lvl = priv_timer[tim_lcore].curr_skiplist_depth;  in timer_get_prev_entries()
353 	prev[lvl] = &priv_timer[tim_lcore].pending_head;  in timer_get_prev_entries()
370 				struct priv_timer *priv_timer)  in timer_get_prev_entries_for_node()  argument
377 	timer_get_prev_entries(tim->expire - 1, tim_lcore, prev, priv_timer);  in timer_get_prev_entries_for_node()
378 	for (i = priv_timer[tim_lcore].curr_skiplist_depth - 1; i >= 0; i--) {  in timer_get_prev_entries_for_node()
393 	  struct priv_timer *priv_timer)  in timer_add()  argument
400 	timer_get_prev_entries(tim->expire, tim_lcore, prev, priv_timer);  in timer_add()
404 			priv_timer[tim_lcore].curr_skiplist_depth);  in timer_add()
405 	if (tim_level == priv_timer[tim_lcore].curr_skiplist_depth)  in timer_add()
406 		priv_timer[tim_lcore].curr_skiplist_depth++;  in timer_add()
419 	priv_timer[tim_lcore].pending_head.expire = priv_timer[tim_lcore].\  in timer_add()
430 	  int local_is_locked, struct priv_timer *priv_timer)  in timer_del()  argument
441 		rte_spinlock_lock(&priv_timer[prev_owner].list_lock);  in timer_del()
445 	if (tim == priv_timer[prev_owner].pending_head.sl_next[0])  in timer_del()
446 		priv_timer[prev_owner].pending_head.expire =  in timer_del()
450 	timer_get_prev_entries_for_node(tim, prev_owner, prev, priv_timer);  in timer_del()
451 	for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--) {  in timer_del()
457 	for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--)  in timer_del()
458 		if (priv_timer[prev_owner].pending_head.sl_next[i] == NULL)  in timer_del()
459 			priv_timer[prev_owner].curr_skiplist_depth --;  in timer_del()
464 		rte_spinlock_unlock(&priv_timer[prev_owner].list_lock);  in timer_del()
478 	struct priv_timer *priv_timer = timer_data->priv_timer;  in __rte_timer_reset()  local
485 				priv_timer[lcore_id].prev_lcore,  in __rte_timer_reset()
487 			priv_timer[lcore_id].prev_lcore = tim_lcore;  in __rte_timer_reset()
496 	ret = timer_set_config_state(tim, &prev_status, priv_timer);  in __rte_timer_reset()
500 	__TIMER_STAT_ADD(priv_timer, reset, 1);  in __rte_timer_reset()
503 		priv_timer[lcore_id].updated = 1;  in __rte_timer_reset()
508 		timer_del(tim, prev_status, local_is_locked, priv_timer);  in __rte_timer_reset()
509 		__TIMER_STAT_ADD(priv_timer, pending, -1);  in __rte_timer_reset()
522 		rte_spinlock_lock(&priv_timer[tim_lcore].list_lock);  in __rte_timer_reset()
524 	__TIMER_STAT_ADD(priv_timer, pending, 1);  in __rte_timer_reset()
525 	timer_add(tim, tim_lcore, priv_timer);  in __rte_timer_reset()
537 		rte_spinlock_unlock(&priv_timer[tim_lcore].list_lock);  in __rte_timer_reset()
590 	struct priv_timer *priv_timer = timer_data->priv_timer;  in __rte_timer_stop()  local
594 	ret = timer_set_config_state(tim, &prev_status, priv_timer);  in __rte_timer_stop()
598 	__TIMER_STAT_ADD(priv_timer, stop, 1);  in __rte_timer_stop()
601 		priv_timer[lcore_id].updated = 1;  in __rte_timer_stop()
606 		timer_del(tim, prev_status, local_is_locked, priv_timer);  in __rte_timer_stop()
607 		__TIMER_STAT_ADD(priv_timer, pending, -1);  in __rte_timer_stop()
665 	struct priv_timer *priv_timer = timer_data->priv_timer;  in __rte_timer_manage()  local
670 	__TIMER_STAT_ADD(priv_timer, manage, 1);  in __rte_timer_manage()
672 	if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL)  in __rte_timer_manage()
680 	if (likely(priv_timer[lcore_id].pending_head.expire > cur_time))  in __rte_timer_manage()
685 	rte_spinlock_lock(&priv_timer[lcore_id].list_lock);  in __rte_timer_manage()
688 	if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL ||  in __rte_timer_manage()
689 	    priv_timer[lcore_id].pending_head.sl_next[0]->expire > cur_time) {  in __rte_timer_manage()
690 		rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);  in __rte_timer_manage()
695 	tim = priv_timer[lcore_id].pending_head.sl_next[0];  in __rte_timer_manage()
698 	timer_get_prev_entries(cur_time, lcore_id, prev, priv_timer);  in __rte_timer_manage()
699 	for (i = priv_timer[lcore_id].curr_skiplist_depth -1; i >= 0; i--) {  in __rte_timer_manage()
700 		if (prev[i] == &priv_timer[lcore_id].pending_head)  in __rte_timer_manage()
702 		priv_timer[lcore_id].pending_head.sl_next[i] =  in __rte_timer_manage()
705 			priv_timer[lcore_id].curr_skiplist_depth--;  in __rte_timer_manage()
728 	priv_timer[lcore_id].pending_head.expire =  in __rte_timer_manage()
729 	    (priv_timer[lcore_id].pending_head.sl_next[0] == NULL) ? 0 :  in __rte_timer_manage()
730 		priv_timer[lcore_id].pending_head.sl_next[0]->expire;  in __rte_timer_manage()
732 	rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);  in __rte_timer_manage()
737 		priv_timer[lcore_id].updated = 0;  in __rte_timer_manage()
738 		priv_timer[lcore_id].running_tim = tim;  in __rte_timer_manage()
743 		__TIMER_STAT_ADD(priv_timer, pending, -1);  in __rte_timer_manage()
746 		if (priv_timer[lcore_id].updated == 1)  in __rte_timer_manage()
762 			rte_spinlock_lock(&priv_timer[lcore_id].list_lock);  in __rte_timer_manage()
764 			__TIMER_STAT_ADD(priv_timer, pending, 1);  in __rte_timer_manage()
775 			rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);  in __rte_timer_manage()
778 	priv_timer[lcore_id].running_tim = NULL;  in __rte_timer_manage()
809 	struct priv_timer *privp;  in rte_timer_alt_manage()
817 	__TIMER_STAT_ADD(data->priv_timer, manage, 1);  in rte_timer_alt_manage()
826 		privp = &data->priv_timer[poll_lcore];  in rte_timer_alt_manage()
857 				       data->priv_timer);  in rte_timer_alt_manage()
923 		data->priv_timer[this_lcore].updated = 0;  in rte_timer_alt_manage()
924 		data->priv_timer[this_lcore].running_tim = tim;  in rte_timer_alt_manage()
929 		__TIMER_STAT_ADD(data->priv_timer, pending, -1);  in rte_timer_alt_manage()
934 		if (data->priv_timer[this_lcore].updated == 1)  in rte_timer_alt_manage()
950 				&data->priv_timer[this_lcore].list_lock);  in rte_timer_alt_manage()
952 			__TIMER_STAT_ADD(data->priv_timer, pending, 1);  in rte_timer_alt_manage()
964 				&data->priv_timer[this_lcore].list_lock);  in rte_timer_alt_manage()
967 		data->priv_timer[this_lcore].running_tim = NULL;  in rte_timer_alt_manage()
980 	struct priv_timer *priv_timer;  in rte_timer_stop_all()  local
989 		priv_timer = &timer_data->priv_timer[walk_lcore];  in rte_timer_stop_all()
991 		rte_spinlock_lock(&priv_timer->list_lock);  in rte_timer_stop_all()
993 		for (tim = priv_timer->pending_head.sl_next[0];  in rte_timer_stop_all()
1005 		rte_spinlock_unlock(&priv_timer->list_lock);  in rte_timer_stop_all()
1016 	struct priv_timer *priv_timer;  in rte_timer_next_ticks()  local
1023 	priv_timer = timer_data->priv_timer;  in rte_timer_next_ticks()
1026 	rte_spinlock_lock(&priv_timer[lcore_id].list_lock);  in rte_timer_next_ticks()
1027 	tm = priv_timer[lcore_id].pending_head.sl_next[0];  in rte_timer_next_ticks()
1033 	rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);  in rte_timer_next_ticks()
1045 	struct priv_timer *priv_timer = timer_data->priv_timer;  in __rte_timer_dump_stats()  local
1049 		sum.reset += priv_timer[lcore_id].stats.reset;  in __rte_timer_dump_stats()
1050 		sum.stop += priv_timer[lcore_id].stats.stop;  in __rte_timer_dump_stats()
1051 		sum.manage += priv_timer[lcore_id].stats.manage;  in __rte_timer_dump_stats()
1052 		sum.pending += priv_timer[lcore_id].stats.pending;  in __rte_timer_dump_stats()