27 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);  in pipeline_queue_worker_single_stage_tx()
34 		if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {  in pipeline_queue_worker_single_stage_tx()
35 			enq = pipeline_event_tx(dev, port, &ev, t);  in pipeline_queue_worker_single_stage_tx()
36 			ev.op = RTE_EVENT_OP_RELEASE;  in pipeline_queue_worker_single_stage_tx()
39 			ev.queue_id++;  in pipeline_queue_worker_single_stage_tx()
40 			pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);  in pipeline_queue_worker_single_stage_tx()
41 			enq = pipeline_event_enqueue(dev, port, &ev, t);  in pipeline_queue_worker_single_stage_tx()
44 	pipeline_worker_cleanup(dev, port, &ev, enq, deq);  in pipeline_queue_worker_single_stage_tx()
57 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);  in pipeline_queue_worker_single_stage_fwd()
64 		ev.queue_id = tx_queue[ev.mbuf->port];  in pipeline_queue_worker_single_stage_fwd()
65 		rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0);  in pipeline_queue_worker_single_stage_fwd()
66 		pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);  in pipeline_queue_worker_single_stage_fwd()
67 		enq = pipeline_event_enqueue(dev, port, &ev, t);  in pipeline_queue_worker_single_stage_fwd()
70 	pipeline_worker_cleanup(dev, port, &ev, enq, deq);  in pipeline_queue_worker_single_stage_fwd()
82 		nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);  in pipeline_queue_worker_single_stage_burst_tx()
90 			rte_prefetch0(ev[i + 1].mbuf);  in pipeline_queue_worker_single_stage_burst_tx()
91 			if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {  in pipeline_queue_worker_single_stage_burst_tx()
92 				pipeline_event_tx(dev, port, &ev[i], t);  in pipeline_queue_worker_single_stage_burst_tx()
93 				ev[i].op = RTE_EVENT_OP_RELEASE;  in pipeline_queue_worker_single_stage_burst_tx()
96 				ev[i].queue_id++;  in pipeline_queue_worker_single_stage_burst_tx()
97 				pipeline_fwd_event(&ev[i],  in pipeline_queue_worker_single_stage_burst_tx()
101 		nb_tx = pipeline_event_enqueue_burst(dev, port, ev, nb_rx, t);  in pipeline_queue_worker_single_stage_burst_tx()
103 	pipeline_worker_cleanup(dev, port, ev, nb_tx, nb_rx);  in pipeline_queue_worker_single_stage_burst_tx()
116 		nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);  in pipeline_queue_worker_single_stage_burst_fwd()
124 			rte_prefetch0(ev[i + 1].mbuf);  in pipeline_queue_worker_single_stage_burst_fwd()
125 			ev[i].queue_id = tx_queue[ev[i].mbuf->port];  in pipeline_queue_worker_single_stage_burst_fwd()
126 			rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0);  in pipeline_queue_worker_single_stage_burst_fwd()
127 			pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC);  in pipeline_queue_worker_single_stage_burst_fwd()
130 		nb_tx = pipeline_event_enqueue_burst(dev, port, ev, nb_rx, t);  in pipeline_queue_worker_single_stage_burst_fwd()
133 	pipeline_worker_cleanup(dev, port, ev, nb_tx, nb_rx);  in pipeline_queue_worker_single_stage_burst_fwd()
146 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);  in pipeline_queue_worker_single_stage_tx_vector()
153 		if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {  in pipeline_queue_worker_single_stage_tx_vector()
154 			vector_sz = ev.vec->nb_elem;  in pipeline_queue_worker_single_stage_tx_vector()
155 			enq = pipeline_event_tx_vector(dev, port, &ev, t);  in pipeline_queue_worker_single_stage_tx_vector()
156 			ev.op = RTE_EVENT_OP_RELEASE;  in pipeline_queue_worker_single_stage_tx_vector()
159 			ev.queue_id++;  in pipeline_queue_worker_single_stage_tx_vector()
160 			pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC);  in pipeline_queue_worker_single_stage_tx_vector()
161 			enq = pipeline_event_enqueue(dev, port, &ev, t);  in pipeline_queue_worker_single_stage_tx_vector()
164 	pipeline_worker_cleanup(dev, port, &ev, enq, deq);  in pipeline_queue_worker_single_stage_tx_vector()
178 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);  in pipeline_queue_worker_single_stage_fwd_vector()
185 		ev.queue_id = tx_queue[ev.vec->port];  in pipeline_queue_worker_single_stage_fwd_vector()
186 		ev.vec->queue = 0;  in pipeline_queue_worker_single_stage_fwd_vector()
187 		vector_sz = ev.vec->nb_elem;  in pipeline_queue_worker_single_stage_fwd_vector()
188 		pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC);  in pipeline_queue_worker_single_stage_fwd_vector()
189 		enq = pipeline_event_enqueue(dev, port, &ev, t);  in pipeline_queue_worker_single_stage_fwd_vector()
192 	pipeline_worker_cleanup(dev, port, &ev, enq, deq);  in pipeline_queue_worker_single_stage_fwd_vector()
205 		nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);  in pipeline_queue_worker_single_stage_burst_tx_vector()
213 			if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {  in pipeline_queue_worker_single_stage_burst_tx_vector()
214 				vector_sz = ev[i].vec->nb_elem;  in pipeline_queue_worker_single_stage_burst_tx_vector()
215 				pipeline_event_tx_vector(dev, port, &ev[i], t);  in pipeline_queue_worker_single_stage_burst_tx_vector()
216 				ev[i].op = RTE_EVENT_OP_RELEASE;  in pipeline_queue_worker_single_stage_burst_tx_vector()
219 				ev[i].queue_id++;  in pipeline_queue_worker_single_stage_burst_tx_vector()
221 					&ev[i], RTE_SCHED_TYPE_ATOMIC);  in pipeline_queue_worker_single_stage_burst_tx_vector()
225 		nb_tx = pipeline_event_enqueue_burst(dev, port, ev, nb_rx, t);  in pipeline_queue_worker_single_stage_burst_tx_vector()
227 	pipeline_worker_cleanup(dev, port, ev, nb_tx, nb_rx);  in pipeline_queue_worker_single_stage_burst_tx_vector()
241 		nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);  in pipeline_queue_worker_single_stage_burst_fwd_vector()
250 			ev[i].queue_id = tx_queue[ev[i].vec->port];  in pipeline_queue_worker_single_stage_burst_fwd_vector()
251 			ev[i].vec->queue = 0;  in pipeline_queue_worker_single_stage_burst_fwd_vector()
252 			vector_sz += ev[i].vec->nb_elem;  in pipeline_queue_worker_single_stage_burst_fwd_vector()
253 			pipeline_fwd_event_vector(&ev[i],  in pipeline_queue_worker_single_stage_burst_fwd_vector()
257 		nb_tx = pipeline_event_enqueue_burst(dev, port, ev, nb_rx, t);  in pipeline_queue_worker_single_stage_burst_fwd_vector()
260 	pipeline_worker_cleanup(dev, port, ev, nb_tx, nb_rx);  in pipeline_queue_worker_single_stage_burst_fwd_vector()
273 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);  in pipeline_queue_worker_multi_stage_tx()
280 		cq_id = ev.queue_id % nb_stages;  in pipeline_queue_worker_multi_stage_tx()
282 		if (ev.queue_id == tx_queue[ev.mbuf->port]) {  in pipeline_queue_worker_multi_stage_tx()
283 			enq = pipeline_event_tx(dev, port, &ev, t);  in pipeline_queue_worker_multi_stage_tx()
284 			ev.op = RTE_EVENT_OP_RELEASE;  in pipeline_queue_worker_multi_stage_tx()
289 		ev.queue_id++;  in pipeline_queue_worker_multi_stage_tx()
290 		pipeline_fwd_event(&ev, cq_id != last_queue ?  in pipeline_queue_worker_multi_stage_tx()
293 		enq = pipeline_event_enqueue(dev, port, &ev, t);  in pipeline_queue_worker_multi_stage_tx()
295 	pipeline_worker_cleanup(dev, port, &ev, enq, deq);  in pipeline_queue_worker_multi_stage_tx()
308 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);  in pipeline_queue_worker_multi_stage_fwd()
315 		cq_id = ev.queue_id % nb_stages;  in pipeline_queue_worker_multi_stage_fwd()
318 			ev.queue_id = tx_queue[ev.mbuf->port];  in pipeline_queue_worker_multi_stage_fwd()
319 			rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0);  in pipeline_queue_worker_multi_stage_fwd()
320 			pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);  in pipeline_queue_worker_multi_stage_fwd()
321 			enq = pipeline_event_enqueue(dev, port, &ev, t);  in pipeline_queue_worker_multi_stage_fwd()
324 			ev.queue_id++;  in pipeline_queue_worker_multi_stage_fwd()
325 			pipeline_fwd_event(&ev, sched_type_list[cq_id]);  in pipeline_queue_worker_multi_stage_fwd()
326 			enq = pipeline_event_enqueue(dev, port, &ev, t);  in pipeline_queue_worker_multi_stage_fwd()
329 	pipeline_worker_cleanup(dev, port, &ev, enq, deq);  in pipeline_queue_worker_multi_stage_fwd()
342 		nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);  in pipeline_queue_worker_multi_stage_burst_tx()
350 			rte_prefetch0(ev[i + 1].mbuf);  in pipeline_queue_worker_multi_stage_burst_tx()
351 			cq_id = ev[i].queue_id % nb_stages;  in pipeline_queue_worker_multi_stage_burst_tx()
353 			if (ev[i].queue_id == tx_queue[ev[i].mbuf->port]) {  in pipeline_queue_worker_multi_stage_burst_tx()
354 				pipeline_event_tx(dev, port, &ev[i], t);  in pipeline_queue_worker_multi_stage_burst_tx()
355 				ev[i].op = RTE_EVENT_OP_RELEASE;  in pipeline_queue_worker_multi_stage_burst_tx()
360 			ev[i].queue_id++;  in pipeline_queue_worker_multi_stage_burst_tx()
361 			pipeline_fwd_event(&ev[i], cq_id != last_queue ?  in pipeline_queue_worker_multi_stage_burst_tx()
365 		nb_tx = pipeline_event_enqueue_burst(dev, port, ev, nb_rx, t);  in pipeline_queue_worker_multi_stage_burst_tx()
367 	pipeline_worker_cleanup(dev, port, ev, nb_tx, nb_rx);  in pipeline_queue_worker_multi_stage_burst_tx()
381 		nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);  in pipeline_queue_worker_multi_stage_burst_fwd()
389 			rte_prefetch0(ev[i + 1].mbuf);  in pipeline_queue_worker_multi_stage_burst_fwd()
390 			cq_id = ev[i].queue_id % nb_stages;  in pipeline_queue_worker_multi_stage_burst_fwd()
393 				ev[i].queue_id = tx_queue[ev[i].mbuf->port];  in pipeline_queue_worker_multi_stage_burst_fwd()
394 				rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0);  in pipeline_queue_worker_multi_stage_burst_fwd()
395 				pipeline_fwd_event(&ev[i],  in pipeline_queue_worker_multi_stage_burst_fwd()
399 				ev[i].queue_id++;  in pipeline_queue_worker_multi_stage_burst_fwd()
400 				pipeline_fwd_event(&ev[i],  in pipeline_queue_worker_multi_stage_burst_fwd()
405 		nb_tx = pipeline_event_enqueue_burst(dev, port, ev, nb_rx, t);  in pipeline_queue_worker_multi_stage_burst_fwd()
408 	pipeline_worker_cleanup(dev, port, ev, nb_tx, nb_rx);  in pipeline_queue_worker_multi_stage_burst_fwd()
422 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);  in pipeline_queue_worker_multi_stage_tx_vector()
429 		cq_id = ev.queue_id % nb_stages;  in pipeline_queue_worker_multi_stage_tx_vector()
431 		if (ev.queue_id == tx_queue[ev.vec->port]) {  in pipeline_queue_worker_multi_stage_tx_vector()
432 			vector_sz = ev.vec->nb_elem;  in pipeline_queue_worker_multi_stage_tx_vector()
433 			enq = pipeline_event_tx_vector(dev, port, &ev, t);  in pipeline_queue_worker_multi_stage_tx_vector()
435 			ev.op = RTE_EVENT_OP_RELEASE;  in pipeline_queue_worker_multi_stage_tx_vector()
439 		ev.queue_id++;  in pipeline_queue_worker_multi_stage_tx_vector()
440 		pipeline_fwd_event_vector(&ev, cq_id != last_queue  in pipeline_queue_worker_multi_stage_tx_vector()
443 		enq = pipeline_event_enqueue(dev, port, &ev, t);  in pipeline_queue_worker_multi_stage_tx_vector()
445 	pipeline_worker_cleanup(dev, port, &ev, enq, deq);  in pipeline_queue_worker_multi_stage_tx_vector()
459 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);  in pipeline_queue_worker_multi_stage_fwd_vector()
466 		cq_id = ev.queue_id % nb_stages;  in pipeline_queue_worker_multi_stage_fwd_vector()
469 			vector_sz = ev.vec->nb_elem;  in pipeline_queue_worker_multi_stage_fwd_vector()
470 			ev.queue_id = tx_queue[ev.vec->port];  in pipeline_queue_worker_multi_stage_fwd_vector()
471 			pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC);  in pipeline_queue_worker_multi_stage_fwd_vector()
474 			ev.queue_id++;  in pipeline_queue_worker_multi_stage_fwd_vector()
475 			pipeline_fwd_event_vector(&ev, sched_type_list[cq_id]);  in pipeline_queue_worker_multi_stage_fwd_vector()
478 		enq = pipeline_event_enqueue(dev, port, &ev, t);  in pipeline_queue_worker_multi_stage_fwd_vector()
480 	pipeline_worker_cleanup(dev, port, &ev, enq, deq);  in pipeline_queue_worker_multi_stage_fwd_vector()
494 		nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);  in pipeline_queue_worker_multi_stage_burst_tx_vector()
502 			cq_id = ev[i].queue_id % nb_stages;  in pipeline_queue_worker_multi_stage_burst_tx_vector()
504 			if (ev[i].queue_id == tx_queue[ev[i].vec->port]) {  in pipeline_queue_worker_multi_stage_burst_tx_vector()
505 				vector_sz = ev[i].vec->nb_elem;  in pipeline_queue_worker_multi_stage_burst_tx_vector()
506 				pipeline_event_tx_vector(dev, port, &ev[i], t);  in pipeline_queue_worker_multi_stage_burst_tx_vector()
507 				ev[i].op = RTE_EVENT_OP_RELEASE;  in pipeline_queue_worker_multi_stage_burst_tx_vector()
512 			ev[i].queue_id++;  in pipeline_queue_worker_multi_stage_burst_tx_vector()
514 				&ev[i], cq_id != last_queue  in pipeline_queue_worker_multi_stage_burst_tx_vector()
519 		nb_tx = pipeline_event_enqueue_burst(dev, port, ev, nb_rx, t);  in pipeline_queue_worker_multi_stage_burst_tx_vector()
521 	pipeline_worker_cleanup(dev, port, ev, nb_tx, nb_rx);  in pipeline_queue_worker_multi_stage_burst_tx_vector()
535 		nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);  in pipeline_queue_worker_multi_stage_burst_fwd_vector()
543 			cq_id = ev[i].queue_id % nb_stages;  in pipeline_queue_worker_multi_stage_burst_fwd_vector()
546 				ev[i].queue_id = tx_queue[ev[i].vec->port];  in pipeline_queue_worker_multi_stage_burst_fwd_vector()
547 				vector_sz = ev[i].vec->nb_elem;  in pipeline_queue_worker_multi_stage_burst_fwd_vector()
549 					&ev[i], RTE_SCHED_TYPE_ATOMIC);  in pipeline_queue_worker_multi_stage_burst_fwd_vector()
552 				ev[i].queue_id++;  in pipeline_queue_worker_multi_stage_burst_fwd_vector()
554 					&ev[i], sched_type_list[cq_id]);  in pipeline_queue_worker_multi_stage_burst_fwd_vector()
558 		nb_tx = pipeline_event_enqueue_burst(dev, port, ev, nb_rx, t);  in pipeline_queue_worker_multi_stage_burst_fwd_vector()
560 	pipeline_worker_cleanup(dev, port, ev, nb_tx, nb_rx);  in pipeline_queue_worker_multi_stage_burst_fwd_vector()