208 rte_timer_init(struct rte_timer *tim)  in rte_timer_init()  argument
214 	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELAXED);  in rte_timer_init()
223 timer_set_config_state(struct rte_timer *tim,  in timer_set_config_state()  argument
235 	prev_status.u32 = __atomic_load_n(&tim->status.u32, __ATOMIC_RELAXED);  in timer_set_config_state()
243 		     tim != priv_timer[lcore_id].running_tim))  in timer_set_config_state()
258 		success = __atomic_compare_exchange_n(&tim->status.u32,  in timer_set_config_state()
273 timer_set_running_state(struct rte_timer *tim)  in timer_set_running_state()  argument
281 	prev_status.u32 = __atomic_load_n(&tim->status.u32, __ATOMIC_RELAXED);  in timer_set_running_state()
297 		success = __atomic_compare_exchange_n(&tim->status.u32,  in timer_set_running_state()
368 timer_get_prev_entries_for_node(struct rte_timer *tim, unsigned tim_lcore,  in timer_get_prev_entries_for_node()  argument
377 	timer_get_prev_entries(tim->expire - 1, tim_lcore, prev, priv_timer);  in timer_get_prev_entries_for_node()
380 				prev[i]->sl_next[i] != tim &&  in timer_get_prev_entries_for_node()
381 				prev[i]->sl_next[i]->expire <= tim->expire)  in timer_get_prev_entries_for_node()
392 timer_add(struct rte_timer *tim, unsigned int tim_lcore,  in timer_add()  argument
400 	timer_get_prev_entries(tim->expire, tim_lcore, prev, priv_timer);  in timer_add()
410 		tim->sl_next[lvl] = prev[lvl]->sl_next[lvl];  in timer_add()
411 		prev[lvl]->sl_next[lvl] = tim;  in timer_add()
414 	tim->sl_next[0] = prev[0]->sl_next[0];  in timer_add()
415 	prev[0]->sl_next[0] = tim;  in timer_add()
429 timer_del(struct rte_timer *tim, union rte_timer_status prev_status,  in timer_del()  argument
445 	if (tim == priv_timer[prev_owner].pending_head.sl_next[0])  in timer_del()
447 				((tim->sl_next[0] == NULL) ? 0 : tim->sl_next[0]->expire);  in timer_del()
450 	timer_get_prev_entries_for_node(tim, prev_owner, prev, priv_timer);  in timer_del()
452 		if (prev[i]->sl_next[i] == tim)  in timer_del()
453 			prev[i]->sl_next[i] = tim->sl_next[i];  in timer_del()
469 __rte_timer_reset(struct rte_timer *tim, uint64_t expire,  in __rte_timer_reset()  argument
496 	ret = timer_set_config_state(tim, &prev_status, priv_timer);  in __rte_timer_reset()
508 		timer_del(tim, prev_status, local_is_locked, priv_timer);  in __rte_timer_reset()
512 	tim->period = period;  in __rte_timer_reset()
513 	tim->expire = expire;  in __rte_timer_reset()
514 	tim->f = fct;  in __rte_timer_reset()
515 	tim->arg = arg;  in __rte_timer_reset()
525 	timer_add(tim, tim_lcore, priv_timer);  in __rte_timer_reset()
534 	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELEASE);  in __rte_timer_reset()
544 rte_timer_reset(struct rte_timer *tim, uint64_t ticks,  in rte_timer_reset()  argument
548 	return rte_timer_alt_reset(default_data_id, tim, ticks, type,  in rte_timer_reset()
553 rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim,  in rte_timer_alt_reset()  argument
568 	return __rte_timer_reset(tim,  cur_time + ticks, period, tim_lcore,  in rte_timer_alt_reset()
574 rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks,  in rte_timer_reset_sync()  argument
578 	while (rte_timer_reset(tim, ticks, type, tim_lcore,  in rte_timer_reset_sync()
584 __rte_timer_stop(struct rte_timer *tim, int local_is_locked,  in __rte_timer_stop()  argument
594 	ret = timer_set_config_state(tim, &prev_status, priv_timer);  in __rte_timer_stop()
606 		timer_del(tim, prev_status, local_is_locked, priv_timer);  in __rte_timer_stop()
616 	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELEASE);  in __rte_timer_stop()
623 rte_timer_stop(struct rte_timer *tim)  in rte_timer_stop()  argument
625 	return rte_timer_alt_stop(default_data_id, tim);  in rte_timer_stop()
629 rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim)  in rte_timer_alt_stop()  argument
635 	return __rte_timer_stop(tim, 0, timer_data);  in rte_timer_alt_stop()
640 rte_timer_stop_sync(struct rte_timer *tim)  in rte_timer_stop_sync()  argument
642 	while (rte_timer_stop(tim) != 0)  in rte_timer_stop_sync()
648 rte_timer_pending(struct rte_timer *tim)  in rte_timer_pending()  argument
650 	return __atomic_load_n(&tim->status.state,  in rte_timer_pending()
659 	struct rte_timer *tim, *next_tim;  in __rte_timer_manage()  local
695 	tim = priv_timer[lcore_id].pending_head.sl_next[0];  in __rte_timer_manage()
710 	run_first_tim = tim;  in __rte_timer_manage()
713 	for ( ; tim != NULL; tim = next_tim) {  in __rte_timer_manage()
714 		next_tim = tim->sl_next[0];  in __rte_timer_manage()
716 		ret = timer_set_running_state(tim);  in __rte_timer_manage()
718 			pprev = &tim->sl_next[0];  in __rte_timer_manage()
735 	for (tim = run_first_tim; tim != NULL; tim = next_tim) {  in __rte_timer_manage()
736 		next_tim = tim->sl_next[0];  in __rte_timer_manage()
738 		priv_timer[lcore_id].running_tim = tim;  in __rte_timer_manage()
741 		tim->f(tim, tim->arg);  in __rte_timer_manage()
749 		if (tim->period == 0) {  in __rte_timer_manage()
757 			__atomic_store_n(&tim->status.u32, status.u32,  in __rte_timer_manage()
770 			__atomic_store_n(&tim->status.u32, status.u32,  in __rte_timer_manage()
772 			__rte_timer_reset(tim, tim->expire + tim->period,  in __rte_timer_manage()
773 				tim->period, lcore_id, tim->f, tim->arg, 1,  in __rte_timer_manage()
801 	struct rte_timer *tim, *next_tim, **pprev;  in rte_timer_alt_manage()  local
853 		tim = privp->pending_head.sl_next[0];  in rte_timer_alt_manage()
870 		run_first_tims[nb_runlists] = tim;  in rte_timer_alt_manage()
874 		for ( ; tim != NULL; tim = next_tim) {  in rte_timer_alt_manage()
875 			next_tim = tim->sl_next[0];  in rte_timer_alt_manage()
877 			ret = timer_set_running_state(tim);  in rte_timer_alt_manage()
879 				pprev = &tim->sl_next[0];  in rte_timer_alt_manage()
904 			tim = run_first_tims[i];  in rte_timer_alt_manage()
906 			if (tim != NULL && tim->expire < min_expire) {  in rte_timer_alt_manage()
907 				min_expire = tim->expire;  in rte_timer_alt_manage()
916 		tim = run_first_tims[min_idx];  in rte_timer_alt_manage()
924 		data->priv_timer[this_lcore].running_tim = tim;  in rte_timer_alt_manage()
927 		f(tim);  in rte_timer_alt_manage()
937 		if (tim->period == 0) {  in rte_timer_alt_manage()
945 			__atomic_store_n(&tim->status.u32, status.u32,  in rte_timer_alt_manage()
958 			__atomic_store_n(&tim->status.u32, status.u32,  in rte_timer_alt_manage()
960 			__rte_timer_reset(tim, tim->expire + tim->period,  in rte_timer_alt_manage()
961 				tim->period, this_lcore, tim->f, tim->arg, 1,  in rte_timer_alt_manage()
982 	struct rte_timer *tim, *next_tim;  in rte_timer_stop_all()  local
993 		for (tim = priv_timer->pending_head.sl_next[0];  in rte_timer_stop_all()
994 		     tim != NULL;  in rte_timer_stop_all()
995 		     tim = next_tim) {  in rte_timer_stop_all()
996 			next_tim = tim->sl_next[0];  in rte_timer_stop_all()
999 			__rte_timer_stop(tim, 1, timer_data);  in rte_timer_stop_all()
1002 				f(tim, f_arg);  in rte_timer_stop_all()