56 static int eth_ark_tx_jumbo(struct ark_tx_queue *queue,
59 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue);
60 static void free_completed_tx(struct ark_tx_queue *queue);
63 ark_tx_hw_queue_stop(struct ark_tx_queue *queue)  in ark_tx_hw_queue_stop()  argument
65 	ark_mpu_stop(queue->mpu);  in ark_tx_hw_queue_stop()
70 eth_ark_tx_desc_fill(struct ark_tx_queue *queue,  in eth_ark_tx_desc_fill()  argument
82 	tx_idx = queue->prod_index & queue->queue_mask;  in eth_ark_tx_desc_fill()
83 	meta = &queue->meta_q[tx_idx];  in eth_ark_tx_desc_fill()
88 	queue->prod_index++;  in eth_ark_tx_desc_fill()
90 	queue->bufs[tx_idx] = mbuf;  in eth_ark_tx_desc_fill()
94 		tx_idx = queue->prod_index & queue->queue_mask;  in eth_ark_tx_desc_fill()
95 		meta = &queue->meta_q[tx_idx];  in eth_ark_tx_desc_fill()
98 		queue->prod_index++;  in eth_ark_tx_desc_fill()
101 	tx_idx = queue->prod_index & queue->queue_mask;  in eth_ark_tx_desc_fill()
102 	meta = &queue->meta_q[tx_idx];  in eth_ark_tx_desc_fill()
104 	queue->prod_index++;  in eth_ark_tx_desc_fill()
112 	struct ark_tx_queue *queue;  in eth_ark_xmit_pkts()  local
123 	queue = (struct ark_tx_queue *)vtxq;  in eth_ark_xmit_pkts()
124 	tx_user_meta_hook = queue->tx_user_meta_hook;  in eth_ark_xmit_pkts()
127 	free_completed_tx(queue);  in eth_ark_xmit_pkts()
130 	prod_index_limit = queue->queue_size + queue->free_index - 4;  in eth_ark_xmit_pkts()
133 	     (nb < nb_pkts) && (prod_index_limit - queue->prod_index) > 0;  in eth_ark_xmit_pkts()
152 				queue->tx_errors += 1;  in eth_ark_xmit_pkts()
161 					  queue->ext_user_data);  in eth_ark_xmit_pkts()
163 			stat = eth_ark_tx_jumbo(queue, mbuf,  in eth_ark_xmit_pkts()
168 			eth_ark_tx_desc_fill(queue, mbuf,  in eth_ark_xmit_pkts()
182 			   queue->prod_index,  in eth_ark_xmit_pkts()
183 			   queue->cons_index,  in eth_ark_xmit_pkts()
184 			   queue->free_index);  in eth_ark_xmit_pkts()
185 		ark_mpu_dump(queue->mpu,  in eth_ark_xmit_pkts()
187 			     queue->phys_qid);  in eth_ark_xmit_pkts()
192 		ark_mpu_set_producer(queue->mpu, queue->prod_index);  in eth_ark_xmit_pkts()
199 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf,  in eth_ark_tx_jumbo()  argument
206 	free_queue_space = queue->queue_mask -  in eth_ark_tx_jumbo()
207 		(queue->prod_index - queue->free_index);  in eth_ark_tx_jumbo()
216 		eth_ark_tx_desc_fill(queue, mbuf, flags, user_meta, meta_cnt);  in eth_ark_tx_jumbo()
235 	struct ark_tx_queue *queue;  in eth_ark_tx_queue_setup()  local
252 	queue =	rte_zmalloc_socket("Ark_txqueue",  in eth_ark_tx_queue_setup()
256 	if (queue == 0) {  in eth_ark_tx_queue_setup()
264 	queue->queue_size = nb_desc;  in eth_ark_tx_queue_setup()
265 	queue->queue_mask = nb_desc - 1;  in eth_ark_tx_queue_setup()
266 	queue->phys_qid = qidx;  in eth_ark_tx_queue_setup()
267 	queue->queue_index = queue_idx;  in eth_ark_tx_queue_setup()
268 	dev->data->tx_queues[queue_idx] = queue;  in eth_ark_tx_queue_setup()
269 	queue->tx_user_meta_hook = ark->user_ext.tx_user_meta_hook;  in eth_ark_tx_queue_setup()
270 	queue->ext_user_data = ark->user_data[dev->data->port_id];  in eth_ark_tx_queue_setup()
272 	queue->meta_q =  in eth_ark_tx_queue_setup()
277 	queue->bufs =  in eth_ark_tx_queue_setup()
283 	if (queue->meta_q == 0 || queue->bufs == 0) {  in eth_ark_tx_queue_setup()
286 		rte_free(queue->meta_q);  in eth_ark_tx_queue_setup()
287 		rte_free(queue->bufs);  in eth_ark_tx_queue_setup()
288 		rte_free(queue);  in eth_ark_tx_queue_setup()
292 	queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET);  in eth_ark_tx_queue_setup()
293 	queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET);  in eth_ark_tx_queue_setup()
295 	status = eth_ark_tx_hw_queue_config(queue);  in eth_ark_tx_queue_setup()
298 		rte_free(queue->meta_q);  in eth_ark_tx_queue_setup()
299 		rte_free(queue->bufs);  in eth_ark_tx_queue_setup()
300 		rte_free(queue);  in eth_ark_tx_queue_setup()
309 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue)  in eth_ark_tx_hw_queue_config()  argument
315 	if (ark_mpu_verify(queue->mpu, sizeof(union ark_tx_meta)))  in eth_ark_tx_hw_queue_config()
318 	queue_base = rte_malloc_virt2iova(queue);  in eth_ark_tx_hw_queue_config()
319 	ring_base = rte_malloc_virt2iova(queue->meta_q);  in eth_ark_tx_hw_queue_config()
323 	ark_mpu_stop(queue->mpu);  in eth_ark_tx_hw_queue_config()
324 	ark_mpu_reset(queue->mpu);  in eth_ark_tx_hw_queue_config()
327 	ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1);  in eth_ark_tx_hw_queue_config()
334 	switch (queue->queue_size) {  in eth_ark_tx_hw_queue_config()
350 	ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns);  in eth_ark_tx_hw_queue_config()
359 	struct ark_tx_queue *queue;  in eth_ark_tx_queue_release()  local
361 	queue = (struct ark_tx_queue *)vtx_queue;  in eth_ark_tx_queue_release()
363 	ark_tx_hw_queue_stop(queue);  in eth_ark_tx_queue_release()
365 	queue->cons_index = queue->prod_index;  in eth_ark_tx_queue_release()
366 	free_completed_tx(queue);  in eth_ark_tx_queue_release()
368 	rte_free(queue->meta_q);  in eth_ark_tx_queue_release()
369 	rte_free(queue->bufs);  in eth_ark_tx_queue_release()
370 	rte_free(queue);  in eth_ark_tx_queue_release()
377 	struct ark_tx_queue *queue;  in eth_ark_tx_queue_stop()  local
380 	queue = dev->data->tx_queues[queue_id];  in eth_ark_tx_queue_stop()
383 	while (queue->cons_index != queue->prod_index) {  in eth_ark_tx_queue_stop()
389 	ark_mpu_stop(queue->mpu);  in eth_ark_tx_queue_stop()
390 	free_completed_tx(queue);  in eth_ark_tx_queue_stop()
400 	struct ark_tx_queue *queue;  in eth_ark_tx_queue_start()  local
402 	queue = dev->data->tx_queues[queue_id];  in eth_ark_tx_queue_start()
406 	ark_mpu_start(queue->mpu);  in eth_ark_tx_queue_start()
414 free_completed_tx(struct ark_tx_queue *queue)  in free_completed_tx()  argument
420 	top_index = queue->cons_index;	/* read once */  in free_completed_tx()
421 	while ((top_index - queue->free_index) > 0) {  in free_completed_tx()
422 		meta = &queue->meta_q[queue->free_index & queue->queue_mask];  in free_completed_tx()
424 			mbuf = queue->bufs[queue->free_index &  in free_completed_tx()
425 					   queue->queue_mask];  in free_completed_tx()
429 		queue->free_index += (meta->meta_cnt + 2);  in free_completed_tx()
437 	struct ark_tx_queue *queue;  in eth_tx_queue_stats_get()  local
441 	queue = vqueue;  in eth_tx_queue_stats_get()
442 	ddm = queue->ddm;  in eth_tx_queue_stats_get()
447 	stats->q_opackets[queue->queue_index] = pkts;  in eth_tx_queue_stats_get()
448 	stats->q_obytes[queue->queue_index] = bytes;  in eth_tx_queue_stats_get()
451 	stats->oerrors += queue->tx_errors;  in eth_tx_queue_stats_get()
457 	struct ark_tx_queue *queue;  in eth_tx_queue_stats_reset()  local
460 	queue = vqueue;  in eth_tx_queue_stats_reset()
461 	ddm = queue->ddm;  in eth_tx_queue_stats_reset()
464 	queue->tx_errors = 0;  in eth_tx_queue_stats_reset()