18 atq_mark_fwd_latency(struct rte_event *const ev)  in atq_mark_fwd_latency()  argument
20 	if (unlikely(ev->sub_event_type == 0)) {  in atq_mark_fwd_latency()
21 		struct perf_elt *const m = ev->event_ptr;  in atq_mark_fwd_latency()
28 atq_fwd_event(struct rte_event *const ev, uint8_t *const sched_type_list,  in atq_fwd_event()  argument
31 	ev->sub_event_type++;  in atq_fwd_event()
32 	ev->sched_type = sched_type_list[ev->sub_event_type % nb_stages];  in atq_fwd_event()
33 	ev->op = RTE_EVENT_OP_FORWARD;  in atq_fwd_event()
34 	ev->event_type = RTE_EVENT_TYPE_CPU;  in atq_fwd_event()
41 	struct rte_event ev;  in perf_atq_worker()  local
44 		uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);  in perf_atq_worker()
53 			atq_mark_fwd_latency(&ev);  in perf_atq_worker()
56 		if (unlikely((ev.sub_event_type % nb_stages) == laststage)) {  in perf_atq_worker()
59 					&ev, w, bufs, sz, cnt);  in perf_atq_worker()
61 				cnt = perf_process_last_stage(pool, &ev, w,  in perf_atq_worker()
64 			atq_fwd_event(&ev, sched_type_list, nb_stages);  in perf_atq_worker()
65 			while (rte_event_enqueue_burst(dev, port, &ev, 1) != 1)  in perf_atq_worker()
78 	struct rte_event ev[BURST_SIZE + 1];  in perf_atq_worker_burst()  local
81 		uint16_t const nb_rx = rte_event_dequeue_burst(dev, port, ev,  in perf_atq_worker_burst()
91 				rte_prefetch0(ev[i+1].event_ptr);  in perf_atq_worker_burst()
95 				atq_mark_fwd_latency(&ev[i]);  in perf_atq_worker_burst()
98 			if (unlikely((ev[i].sub_event_type % nb_stages)  in perf_atq_worker_burst()
102 						pool, &ev[i], w, bufs, sz, cnt);  in perf_atq_worker_burst()
105 						&ev[i], w, bufs, sz, cnt);  in perf_atq_worker_burst()
107 				ev[i].op = RTE_EVENT_OP_RELEASE;  in perf_atq_worker_burst()
109 				atq_fwd_event(&ev[i], sched_type_list,  in perf_atq_worker_burst()
116 		enq = rte_event_enqueue_burst(dev, port, ev, nb_rx);  in perf_atq_worker_burst()
119 							ev + enq, nb_rx - enq);  in perf_atq_worker_burst()