214 rte_timer_init(struct rte_timer *tim)  in rte_timer_init()  argument
220 	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELAXED);  in rte_timer_init()
229 timer_set_config_state(struct rte_timer *tim,  in timer_set_config_state()  argument
241 	prev_status.u32 = __atomic_load_n(&tim->status.u32, __ATOMIC_RELAXED);  in timer_set_config_state()
249 		     tim != priv_timer[lcore_id].running_tim))  in timer_set_config_state()
264 		success = __atomic_compare_exchange_n(&tim->status.u32,  in timer_set_config_state()
279 timer_set_running_state(struct rte_timer *tim)  in timer_set_running_state()  argument
287 	prev_status.u32 = __atomic_load_n(&tim->status.u32, __ATOMIC_RELAXED);  in timer_set_running_state()
303 		success = __atomic_compare_exchange_n(&tim->status.u32,  in timer_set_running_state()
374 timer_get_prev_entries_for_node(struct rte_timer *tim, unsigned tim_lcore,  in timer_get_prev_entries_for_node()  argument
383 	timer_get_prev_entries(tim->expire - 1, tim_lcore, prev, priv_timer);  in timer_get_prev_entries_for_node()
386 				prev[i]->sl_next[i] != tim &&  in timer_get_prev_entries_for_node()
387 				prev[i]->sl_next[i]->expire <= tim->expire)  in timer_get_prev_entries_for_node()
398 timer_add(struct rte_timer *tim, unsigned int tim_lcore,  in timer_add()  argument
406 	timer_get_prev_entries(tim->expire, tim_lcore, prev, priv_timer);  in timer_add()
416 		tim->sl_next[lvl] = prev[lvl]->sl_next[lvl];  in timer_add()
417 		prev[lvl]->sl_next[lvl] = tim;  in timer_add()
420 	tim->sl_next[0] = prev[0]->sl_next[0];  in timer_add()
421 	prev[0]->sl_next[0] = tim;  in timer_add()
435 timer_del(struct rte_timer *tim, union rte_timer_status prev_status,  in timer_del()  argument
451 	if (tim == priv_timer[prev_owner].pending_head.sl_next[0])  in timer_del()
453 				((tim->sl_next[0] == NULL) ? 0 : tim->sl_next[0]->expire);  in timer_del()
456 	timer_get_prev_entries_for_node(tim, prev_owner, prev, priv_timer);  in timer_del()
458 		if (prev[i]->sl_next[i] == tim)  in timer_del()
459 			prev[i]->sl_next[i] = tim->sl_next[i];  in timer_del()
475 __rte_timer_reset(struct rte_timer *tim, uint64_t expire,  in __rte_timer_reset()  argument
502 	ret = timer_set_config_state(tim, &prev_status, priv_timer);  in __rte_timer_reset()
514 		timer_del(tim, prev_status, local_is_locked, priv_timer);  in __rte_timer_reset()
518 	tim->period = period;  in __rte_timer_reset()
519 	tim->expire = expire;  in __rte_timer_reset()
520 	tim->f = fct;  in __rte_timer_reset()
521 	tim->arg = arg;  in __rte_timer_reset()
531 	timer_add(tim, tim_lcore, priv_timer);  in __rte_timer_reset()
540 	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELEASE);  in __rte_timer_reset()
550 rte_timer_reset(struct rte_timer *tim, uint64_t ticks,  in rte_timer_reset()  argument
554 	return rte_timer_alt_reset(default_data_id, tim, ticks, type,  in rte_timer_reset()
559 rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim,  in rte_timer_alt_reset()  argument
574 	return __rte_timer_reset(tim,  cur_time + ticks, period, tim_lcore,  in rte_timer_alt_reset()
580 rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks,  in rte_timer_reset_sync()  argument
584 	while (rte_timer_reset(tim, ticks, type, tim_lcore,  in rte_timer_reset_sync()
590 __rte_timer_stop(struct rte_timer *tim, int local_is_locked,  in __rte_timer_stop()  argument
600 	ret = timer_set_config_state(tim, &prev_status, priv_timer);  in __rte_timer_stop()
612 		timer_del(tim, prev_status, local_is_locked, priv_timer);  in __rte_timer_stop()
622 	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELEASE);  in __rte_timer_stop()
629 rte_timer_stop(struct rte_timer *tim)  in rte_timer_stop()  argument
631 	return rte_timer_alt_stop(default_data_id, tim);  in rte_timer_stop()
635 rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim)  in rte_timer_alt_stop()  argument
641 	return __rte_timer_stop(tim, 0, timer_data);  in rte_timer_alt_stop()
646 rte_timer_stop_sync(struct rte_timer *tim)  in rte_timer_stop_sync()  argument
648 	while (rte_timer_stop(tim) != 0)  in rte_timer_stop_sync()
654 rte_timer_pending(struct rte_timer *tim)  in rte_timer_pending()  argument
656 	return __atomic_load_n(&tim->status.state,  in rte_timer_pending()
665 	struct rte_timer *tim, *next_tim;  in __rte_timer_manage()  local
701 	tim = priv_timer[lcore_id].pending_head.sl_next[0];  in __rte_timer_manage()
716 	run_first_tim = tim;  in __rte_timer_manage()
719 	for ( ; tim != NULL; tim = next_tim) {  in __rte_timer_manage()
720 		next_tim = tim->sl_next[0];  in __rte_timer_manage()
722 		ret = timer_set_running_state(tim);  in __rte_timer_manage()
724 			pprev = &tim->sl_next[0];  in __rte_timer_manage()
741 	for (tim = run_first_tim; tim != NULL; tim = next_tim) {  in __rte_timer_manage()
742 		next_tim = tim->sl_next[0];  in __rte_timer_manage()
744 		priv_timer[lcore_id].running_tim = tim;  in __rte_timer_manage()
747 		tim->f(tim, tim->arg);  in __rte_timer_manage()
755 		if (tim->period == 0) {  in __rte_timer_manage()
763 			__atomic_store_n(&tim->status.u32, status.u32,  in __rte_timer_manage()
776 			__atomic_store_n(&tim->status.u32, status.u32,  in __rte_timer_manage()
778 			__rte_timer_reset(tim, tim->expire + tim->period,  in __rte_timer_manage()
779 				tim->period, lcore_id, tim->f, tim->arg, 1,  in __rte_timer_manage()
807 	struct rte_timer *tim, *next_tim, **pprev;  in rte_timer_alt_manage()  local
859 		tim = privp->pending_head.sl_next[0];  in rte_timer_alt_manage()
876 		run_first_tims[nb_runlists] = tim;  in rte_timer_alt_manage()
880 		for ( ; tim != NULL; tim = next_tim) {  in rte_timer_alt_manage()
881 			next_tim = tim->sl_next[0];  in rte_timer_alt_manage()
883 			ret = timer_set_running_state(tim);  in rte_timer_alt_manage()
885 				pprev = &tim->sl_next[0];  in rte_timer_alt_manage()
910 			tim = run_first_tims[i];  in rte_timer_alt_manage()
912 			if (tim != NULL && tim->expire < min_expire) {  in rte_timer_alt_manage()
913 				min_expire = tim->expire;  in rte_timer_alt_manage()
922 		tim = run_first_tims[min_idx];  in rte_timer_alt_manage()
930 		data->priv_timer[this_lcore].running_tim = tim;  in rte_timer_alt_manage()
933 		f(tim);  in rte_timer_alt_manage()
943 		if (tim->period == 0) {  in rte_timer_alt_manage()
951 			__atomic_store_n(&tim->status.u32, status.u32,  in rte_timer_alt_manage()
964 			__atomic_store_n(&tim->status.u32, status.u32,  in rte_timer_alt_manage()
966 			__rte_timer_reset(tim, tim->expire + tim->period,  in rte_timer_alt_manage()
967 				tim->period, this_lcore, tim->f, tim->arg, 1,  in rte_timer_alt_manage()
988 	struct rte_timer *tim, *next_tim;  in rte_timer_stop_all()  local
999 		for (tim = priv_timer->pending_head.sl_next[0];  in rte_timer_stop_all()
1000 		     tim != NULL;  in rte_timer_stop_all()
1001 		     tim = next_tim) {  in rte_timer_stop_all()
1002 			next_tim = tim->sl_next[0];  in rte_timer_stop_all()
1005 			__rte_timer_stop(tim, 1, timer_data);  in rte_timer_stop_all()
1008 				f(tim, f_arg);  in rte_timer_stop_all()