34 struct priv_timer {  struct
58 	struct priv_timer priv_timer[RTE_MAX_LCORE];  member
71 #define __TIMER_STAT_ADD(priv_timer, name, n) do {			\  argument
74 			priv_timer[__lcore_id].stats.name += (n);	\
77 #define __TIMER_STAT_ADD(priv_timer, name, n) do {} while (0)  argument
177 					&data->priv_timer[lcore_id].list_lock);  in rte_timer_subsystem_init()
178 				data->priv_timer[lcore_id].prev_lcore =  in rte_timer_subsystem_init()
231 		       struct priv_timer *priv_timer)  in timer_set_config_state()  argument
249 		     tim != priv_timer[lcore_id].running_tim))  in timer_set_config_state()
356 		       struct rte_timer **prev, struct priv_timer *priv_timer)  in timer_get_prev_entries()  argument
358 	unsigned lvl = priv_timer[tim_lcore].curr_skiplist_depth;  in timer_get_prev_entries()
359 	prev[lvl] = &priv_timer[tim_lcore].pending_head;  in timer_get_prev_entries()
376 				struct priv_timer *priv_timer)  in timer_get_prev_entries_for_node()  argument
383 	timer_get_prev_entries(tim->expire - 1, tim_lcore, prev, priv_timer);  in timer_get_prev_entries_for_node()
384 	for (i = priv_timer[tim_lcore].curr_skiplist_depth - 1; i >= 0; i--) {  in timer_get_prev_entries_for_node()
399 	  struct priv_timer *priv_timer)  in timer_add()  argument
406 	timer_get_prev_entries(tim->expire, tim_lcore, prev, priv_timer);  in timer_add()
410 			priv_timer[tim_lcore].curr_skiplist_depth);  in timer_add()
411 	if (tim_level == priv_timer[tim_lcore].curr_skiplist_depth)  in timer_add()
412 		priv_timer[tim_lcore].curr_skiplist_depth++;  in timer_add()
425 	priv_timer[tim_lcore].pending_head.expire = priv_timer[tim_lcore].\  in timer_add()
436 	  int local_is_locked, struct priv_timer *priv_timer)  in timer_del()  argument
447 		rte_spinlock_lock(&priv_timer[prev_owner].list_lock);  in timer_del()
451 	if (tim == priv_timer[prev_owner].pending_head.sl_next[0])  in timer_del()
452 		priv_timer[prev_owner].pending_head.expire =  in timer_del()
456 	timer_get_prev_entries_for_node(tim, prev_owner, prev, priv_timer);  in timer_del()
457 	for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--) {  in timer_del()
463 	for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--)  in timer_del()
464 		if (priv_timer[prev_owner].pending_head.sl_next[i] == NULL)  in timer_del()
465 			priv_timer[prev_owner].curr_skiplist_depth --;  in timer_del()
470 		rte_spinlock_unlock(&priv_timer[prev_owner].list_lock);  in timer_del()
484 	struct priv_timer *priv_timer = timer_data->priv_timer;  in __rte_timer_reset()  local
491 				priv_timer[lcore_id].prev_lcore,  in __rte_timer_reset()
493 			priv_timer[lcore_id].prev_lcore = tim_lcore;  in __rte_timer_reset()
502 	ret = timer_set_config_state(tim, &prev_status, priv_timer);  in __rte_timer_reset()
506 	__TIMER_STAT_ADD(priv_timer, reset, 1);  in __rte_timer_reset()
509 		priv_timer[lcore_id].updated = 1;  in __rte_timer_reset()
514 		timer_del(tim, prev_status, local_is_locked, priv_timer);  in __rte_timer_reset()
515 		__TIMER_STAT_ADD(priv_timer, pending, -1);  in __rte_timer_reset()
528 		rte_spinlock_lock(&priv_timer[tim_lcore].list_lock);  in __rte_timer_reset()
530 	__TIMER_STAT_ADD(priv_timer, pending, 1);  in __rte_timer_reset()
531 	timer_add(tim, tim_lcore, priv_timer);  in __rte_timer_reset()
543 		rte_spinlock_unlock(&priv_timer[tim_lcore].list_lock);  in __rte_timer_reset()
596 	struct priv_timer *priv_timer = timer_data->priv_timer;  in __rte_timer_stop()  local
600 	ret = timer_set_config_state(tim, &prev_status, priv_timer);  in __rte_timer_stop()
604 	__TIMER_STAT_ADD(priv_timer, stop, 1);  in __rte_timer_stop()
607 		priv_timer[lcore_id].updated = 1;  in __rte_timer_stop()
612 		timer_del(tim, prev_status, local_is_locked, priv_timer);  in __rte_timer_stop()
613 		__TIMER_STAT_ADD(priv_timer, pending, -1);  in __rte_timer_stop()
671 	struct priv_timer *priv_timer = timer_data->priv_timer;  in __rte_timer_manage()  local
676 	__TIMER_STAT_ADD(priv_timer, manage, 1);  in __rte_timer_manage()
678 	if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL)  in __rte_timer_manage()
686 	if (likely(priv_timer[lcore_id].pending_head.expire > cur_time))  in __rte_timer_manage()
691 	rte_spinlock_lock(&priv_timer[lcore_id].list_lock);  in __rte_timer_manage()
694 	if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL ||  in __rte_timer_manage()
695 	    priv_timer[lcore_id].pending_head.sl_next[0]->expire > cur_time) {  in __rte_timer_manage()
696 		rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);  in __rte_timer_manage()
701 	tim = priv_timer[lcore_id].pending_head.sl_next[0];  in __rte_timer_manage()
704 	timer_get_prev_entries(cur_time, lcore_id, prev, priv_timer);  in __rte_timer_manage()
705 	for (i = priv_timer[lcore_id].curr_skiplist_depth -1; i >= 0; i--) {  in __rte_timer_manage()
706 		if (prev[i] == &priv_timer[lcore_id].pending_head)  in __rte_timer_manage()
708 		priv_timer[lcore_id].pending_head.sl_next[i] =  in __rte_timer_manage()
711 			priv_timer[lcore_id].curr_skiplist_depth--;  in __rte_timer_manage()
734 	priv_timer[lcore_id].pending_head.expire =  in __rte_timer_manage()
735 	    (priv_timer[lcore_id].pending_head.sl_next[0] == NULL) ? 0 :  in __rte_timer_manage()
736 		priv_timer[lcore_id].pending_head.sl_next[0]->expire;  in __rte_timer_manage()
738 	rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);  in __rte_timer_manage()
743 		priv_timer[lcore_id].updated = 0;  in __rte_timer_manage()
744 		priv_timer[lcore_id].running_tim = tim;  in __rte_timer_manage()
749 		__TIMER_STAT_ADD(priv_timer, pending, -1);  in __rte_timer_manage()
752 		if (priv_timer[lcore_id].updated == 1)  in __rte_timer_manage()
768 			rte_spinlock_lock(&priv_timer[lcore_id].list_lock);  in __rte_timer_manage()
770 			__TIMER_STAT_ADD(priv_timer, pending, 1);  in __rte_timer_manage()
781 			rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);  in __rte_timer_manage()
784 	priv_timer[lcore_id].running_tim = NULL;  in __rte_timer_manage()
815 	struct priv_timer *privp;  in rte_timer_alt_manage()
823 	__TIMER_STAT_ADD(data->priv_timer, manage, 1);  in rte_timer_alt_manage()
832 		privp = &data->priv_timer[poll_lcore];  in rte_timer_alt_manage()
863 				       data->priv_timer);  in rte_timer_alt_manage()
929 		data->priv_timer[this_lcore].updated = 0;  in rte_timer_alt_manage()
930 		data->priv_timer[this_lcore].running_tim = tim;  in rte_timer_alt_manage()
935 		__TIMER_STAT_ADD(data->priv_timer, pending, -1);  in rte_timer_alt_manage()
940 		if (data->priv_timer[this_lcore].updated == 1)  in rte_timer_alt_manage()
956 				&data->priv_timer[this_lcore].list_lock);  in rte_timer_alt_manage()
958 			__TIMER_STAT_ADD(data->priv_timer, pending, 1);  in rte_timer_alt_manage()
970 				&data->priv_timer[this_lcore].list_lock);  in rte_timer_alt_manage()
973 		data->priv_timer[this_lcore].running_tim = NULL;  in rte_timer_alt_manage()
986 	struct priv_timer *priv_timer;  in rte_timer_stop_all()  local
995 		priv_timer = &timer_data->priv_timer[walk_lcore];  in rte_timer_stop_all()
997 		rte_spinlock_lock(&priv_timer->list_lock);  in rte_timer_stop_all()
999 		for (tim = priv_timer->pending_head.sl_next[0];  in rte_timer_stop_all()
1011 		rte_spinlock_unlock(&priv_timer->list_lock);  in rte_timer_stop_all()
1022 	struct priv_timer *priv_timer;  in rte_timer_next_ticks()  local
1029 	priv_timer = timer_data->priv_timer;  in rte_timer_next_ticks()
1032 	rte_spinlock_lock(&priv_timer[lcore_id].list_lock);  in rte_timer_next_ticks()
1033 	tm = priv_timer[lcore_id].pending_head.sl_next[0];  in rte_timer_next_ticks()
1039 	rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);  in rte_timer_next_ticks()
1051 	struct priv_timer *priv_timer = timer_data->priv_timer;  in __rte_timer_dump_stats()  local
1055 		sum.reset += priv_timer[lcore_id].stats.reset;  in __rte_timer_dump_stats()
1056 		sum.stop += priv_timer[lcore_id].stats.stop;  in __rte_timer_dump_stats()
1057 		sum.manage += priv_timer[lcore_id].stats.manage;  in __rte_timer_dump_stats()
1058 		sum.pending += priv_timer[lcore_id].stats.pending;  in __rte_timer_dump_stats()