22 static void ark_ethdev_rx_dump(const char *name, struct ark_rx_queue *queue);
23 static uint32_t eth_ark_rx_jumbo(struct ark_rx_queue *queue,
27 static inline int eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue);
70 		    struct ark_rx_queue *queue,  in eth_ark_rx_hw_setup()  argument
77 	queue_base = rte_malloc_virt2iova(queue);  in eth_ark_rx_hw_setup()
81 	phys_addr_q_base = rte_malloc_virt2iova(queue->paddress_q);  in eth_ark_rx_hw_setup()
84 	if (ark_mpu_verify(queue->mpu, sizeof(rte_iova_t))) {  in eth_ark_rx_hw_setup()
90 	ark_mpu_configure(queue->mpu, phys_addr_q_base, queue->queue_size, 0);  in eth_ark_rx_hw_setup()
92 	ark_udm_write_addr(queue->udm, phys_addr_prod_index);  in eth_ark_rx_hw_setup()
95 	ark_mpu_reset_stats(queue->mpu);  in eth_ark_rx_hw_setup()
98 	ark_mpu_set_producer(queue->mpu, queue->seed_index);  in eth_ark_rx_hw_setup()
105 eth_ark_rx_update_cons_index(struct ark_rx_queue *queue, uint32_t cons_index)  in eth_ark_rx_update_cons_index()  argument
107 	queue->cons_index = cons_index;  in eth_ark_rx_update_cons_index()
108 	if ((cons_index + queue->queue_size - queue->seed_index) >= ARK_RX_MPU_CHUNK) {  in eth_ark_rx_update_cons_index()
109 		eth_ark_rx_seed_mbufs(queue);  in eth_ark_rx_update_cons_index()
110 		ark_mpu_set_producer(queue->mpu, queue->seed_index);  in eth_ark_rx_update_cons_index()
126 	struct ark_rx_queue *queue;  in eth_ark_dev_rx_queue_setup()  local
159 	queue = rte_zmalloc_socket("Ark_rxqueue",  in eth_ark_dev_rx_queue_setup()
163 	if (queue == 0) {  in eth_ark_dev_rx_queue_setup()
169 	queue->mb_pool = mb_pool;  in eth_ark_dev_rx_queue_setup()
170 	queue->dataroom = rte_pktmbuf_data_room_size(mb_pool) -  in eth_ark_dev_rx_queue_setup()
172 	queue->headroom = RTE_PKTMBUF_HEADROOM;  in eth_ark_dev_rx_queue_setup()
173 	queue->phys_qid = qidx;  in eth_ark_dev_rx_queue_setup()
174 	queue->queue_index = queue_idx;  in eth_ark_dev_rx_queue_setup()
175 	queue->queue_size = nb_desc;  in eth_ark_dev_rx_queue_setup()
176 	queue->queue_mask = nb_desc - 1;  in eth_ark_dev_rx_queue_setup()
177 	queue->rx_user_meta_hook = ark->user_ext.rx_user_meta_hook;  in eth_ark_dev_rx_queue_setup()
178 	queue->ext_user_data = ark->user_data[dev->data->port_id];  in eth_ark_dev_rx_queue_setup()
180 	queue->reserve_q =  in eth_ark_dev_rx_queue_setup()
185 	queue->paddress_q =  in eth_ark_dev_rx_queue_setup()
191 	if (queue->reserve_q == 0 || queue->paddress_q == 0) {  in eth_ark_dev_rx_queue_setup()
195 		rte_free(queue->reserve_q);  in eth_ark_dev_rx_queue_setup()
196 		rte_free(queue->paddress_q);  in eth_ark_dev_rx_queue_setup()
197 		rte_free(queue);  in eth_ark_dev_rx_queue_setup()
201 	dev->data->rx_queues[queue_idx] = queue;  in eth_ark_dev_rx_queue_setup()
202 	queue->udm = RTE_PTR_ADD(ark->udm.v, qidx * ARK_UDM_QOFFSET);  in eth_ark_dev_rx_queue_setup()
203 	queue->mpu = RTE_PTR_ADD(ark->mpurx.v, qidx * ARK_MPU_QOFFSET);  in eth_ark_dev_rx_queue_setup()
206 	ark_udm_stop(queue->udm, 0);  in eth_ark_dev_rx_queue_setup()
207 	ark_udm_configure(queue->udm,  in eth_ark_dev_rx_queue_setup()
209 			  queue->dataroom,  in eth_ark_dev_rx_queue_setup()
211 	ark_udm_stats_reset(queue->udm);  in eth_ark_dev_rx_queue_setup()
212 	ark_udm_stop(queue->udm, 0);  in eth_ark_dev_rx_queue_setup()
215 	status = eth_ark_rx_seed_mbufs(queue);  in eth_ark_dev_rx_queue_setup()
217 	if (queue->seed_index != nb_desc) {  in eth_ark_dev_rx_queue_setup()
224 		status = eth_ark_rx_hw_setup(dev, queue, qidx, queue_idx);  in eth_ark_dev_rx_queue_setup()
233 		for (i = 0, mbuf = queue->reserve_q;  in eth_ark_dev_rx_queue_setup()
234 		     i < queue->seed_index; ++i, mbuf++) {  in eth_ark_dev_rx_queue_setup()
237 		rte_free(queue->reserve_q);  in eth_ark_dev_rx_queue_setup()
238 		rte_free(queue->paddress_q);  in eth_ark_dev_rx_queue_setup()
239 		rte_free(queue);  in eth_ark_dev_rx_queue_setup()
252 	struct ark_rx_queue *queue;  in eth_ark_recv_pkts()  local
261 	queue = (struct ark_rx_queue *)rx_queue;  in eth_ark_recv_pkts()
262 	if (unlikely(queue == 0))  in eth_ark_recv_pkts()
266 	prod_index = queue->prod_index;  in eth_ark_recv_pkts()
267 	cons_index = queue->cons_index;  in eth_ark_recv_pkts()
273 		mbuf = queue->reserve_q[cons_index & queue->queue_mask];  in eth_ark_recv_pkts()
293 					   queue->phys_qid,  in eth_ark_recv_pkts()
295 					   queue->prod_index,  in eth_ark_recv_pkts()
296 					   queue->seed_index);  in eth_ark_recv_pkts()
302 					   queue->udm->rt_cfg.prod_idx,  in eth_ark_recv_pkts()
304 				ark_mpu_dump(queue->mpu,  in eth_ark_recv_pkts()
306 					     queue->phys_qid);  in eth_ark_recv_pkts()
314 		if (unlikely(meta->pkt_len > queue->dataroom))  in eth_ark_recv_pkts()
316 				(queue, meta, mbuf, cons_index + 1);  in eth_ark_recv_pkts()
326 	rx_user_meta_hook = queue->rx_user_meta_hook;  in eth_ark_recv_pkts()
330 		rx_user_meta_hook(mbuf, meta->user_meta, queue->ext_user_data);  in eth_ark_recv_pkts()
333 	eth_ark_rx_update_cons_index(queue, cons_index);  in eth_ark_recv_pkts()
340 eth_ark_rx_jumbo(struct ark_rx_queue *queue,  in eth_ark_rx_jumbo()  argument
355 	data_len = RTE_MIN(meta->pkt_len, queue->dataroom);  in eth_ark_rx_jumbo()
362 				   queue->dataroom);  in eth_ark_rx_jumbo()
367 		mbuf = queue->reserve_q[cons_index & queue->queue_mask];  in eth_ark_rx_jumbo()
381 eth_ark_rx_queue_drain(struct ark_rx_queue *queue)  in eth_ark_rx_queue_drain()  argument
386 	cons_index = queue->cons_index;  in eth_ark_rx_queue_drain()
389 	while ((cons_index ^ queue->prod_index) & queue->queue_mask) {  in eth_ark_rx_queue_drain()
390 		mbuf = queue->reserve_q[cons_index & queue->queue_mask];  in eth_ark_rx_queue_drain()
393 		eth_ark_rx_update_cons_index(queue, cons_index);  in eth_ark_rx_queue_drain()
400 	struct ark_rx_queue *queue;  in eth_ark_dev_rx_queue_count()  local
402 	queue = rx_queue;  in eth_ark_dev_rx_queue_count()
403 	return (queue->prod_index - queue->cons_index);	/* mod arith */  in eth_ark_dev_rx_queue_count()
410 	struct ark_rx_queue *queue;  in eth_ark_rx_start_queue()  local
412 	queue = dev->data->rx_queues[queue_id];  in eth_ark_rx_start_queue()
413 	if (queue == 0)  in eth_ark_rx_start_queue()
418 	ark_mpu_set_producer(queue->mpu, queue->seed_index);  in eth_ark_rx_start_queue()
419 	ark_mpu_start(queue->mpu);  in eth_ark_rx_start_queue()
421 	ark_udm_queue_enable(queue->udm, 1);  in eth_ark_rx_start_queue()
433 	struct ark_rx_queue *queue;  in eth_ark_rx_stop_queue()  local
435 	queue = dev->data->rx_queues[queue_id];  in eth_ark_rx_stop_queue()
436 	if (queue == 0)  in eth_ark_rx_stop_queue()
439 	ark_udm_queue_enable(queue->udm, 0);  in eth_ark_rx_stop_queue()
448 eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue)  in eth_ark_rx_seed_mbufs()  argument
450 	uint32_t limit = (queue->cons_index & ~(ARK_RX_MPU_CHUNK - 1)) +  in eth_ark_rx_seed_mbufs()
451 		queue->queue_size;  in eth_ark_rx_seed_mbufs()
452 	uint32_t seed_index = queue->seed_index;  in eth_ark_rx_seed_mbufs()
455 	uint32_t seed_m = queue->seed_index & queue->queue_mask;  in eth_ark_rx_seed_mbufs()
460 	if (unlikely(seed_m + nb > queue->queue_size))  in eth_ark_rx_seed_mbufs()
461 		nb = queue->queue_size - seed_m;  in eth_ark_rx_seed_mbufs()
463 	struct rte_mbuf **mbufs = &queue->reserve_q[seed_m];  in eth_ark_rx_seed_mbufs()
464 	int status = rte_pktmbuf_alloc_bulk(queue->mb_pool, mbufs, nb);  in eth_ark_rx_seed_mbufs()
471 			    nb, queue->queue_index,  in eth_ark_rx_seed_mbufs()
472 			    queue->seed_index - queue->cons_index);  in eth_ark_rx_seed_mbufs()
479 				queue->reserve_q[seed_m + count];  in eth_ark_rx_seed_mbufs()
485 				queue->phys_qid;  in eth_ark_rx_seed_mbufs()
490 	queue->seed_index += nb;  in eth_ark_rx_seed_mbufs()
496 			queue->paddress_q[seed_m++] =  in eth_ark_rx_seed_mbufs()
501 		queue->paddress_q[seed_m++] =  in eth_ark_rx_seed_mbufs()
506 		queue->paddress_q[seed_m++] =  in eth_ark_rx_seed_mbufs()
511 		queue->paddress_q[seed_m++] =  in eth_ark_rx_seed_mbufs()
526 	struct ark_rx_queue *queue;  in eth_ark_rx_dump_queue()  local
528 	queue = dev->data->rx_queues[queue_id];  in eth_ark_rx_dump_queue()
530 	ark_ethdev_rx_dump(msg, queue);  in eth_ark_rx_dump_queue()
538 	struct ark_rx_queue *queue;  in eth_ark_dev_rx_queue_release()  local
541 	queue = (struct ark_rx_queue *)vqueue;  in eth_ark_dev_rx_queue_release()
542 	if (queue == 0)  in eth_ark_dev_rx_queue_release()
545 	ark_udm_queue_enable(queue->udm, 0);  in eth_ark_dev_rx_queue_release()
547 	ark_mpu_stop(queue->mpu);  in eth_ark_dev_rx_queue_release()
550 	eth_ark_rx_queue_drain(queue);  in eth_ark_dev_rx_queue_release()
552 	for (i = 0; i < queue->queue_size; ++i)  in eth_ark_dev_rx_queue_release()
553 		rte_pktmbuf_free(queue->reserve_q[i]);  in eth_ark_dev_rx_queue_release()
555 	rte_free(queue->reserve_q);  in eth_ark_dev_rx_queue_release()
556 	rte_free(queue->paddress_q);  in eth_ark_dev_rx_queue_release()
557 	rte_free(queue);  in eth_ark_dev_rx_queue_release()
563 	struct ark_rx_queue *queue;  in eth_rx_queue_stats_get()  local
566 	queue = vqueue;  in eth_rx_queue_stats_get()
567 	if (queue == 0)  in eth_rx_queue_stats_get()
569 	udm = queue->udm;  in eth_rx_queue_stats_get()
573 	uint64_t idropped = ark_udm_dropped(queue->udm);  in eth_rx_queue_stats_get()
575 	stats->q_ipackets[queue->queue_index] = ipackets;  in eth_rx_queue_stats_get()
576 	stats->q_ibytes[queue->queue_index] = ibytes;  in eth_rx_queue_stats_get()
577 	stats->q_errors[queue->queue_index] = idropped;  in eth_rx_queue_stats_get()
586 	struct ark_rx_queue *queue;  in eth_rx_queue_stats_reset()  local
588 	queue = vqueue;  in eth_rx_queue_stats_reset()
589 	if (queue == 0)  in eth_rx_queue_stats_reset()
592 	ark_mpu_reset_stats(queue->mpu);  in eth_rx_queue_stats_reset()
593 	ark_udm_queue_stats_reset(queue->udm);  in eth_rx_queue_stats_reset()
600 	struct ark_rx_queue *queue;  in eth_ark_udm_force_close()  local
608 			queue = (struct ark_rx_queue *)dev->data->rx_queues[i];  in eth_ark_udm_force_close()
609 			if (queue == 0)  in eth_ark_udm_force_close()
612 			ark_mpu_start(queue->mpu);  in eth_ark_udm_force_close()
614 			index = ARK_RX_MPU_CHUNK + queue->seed_index;  in eth_ark_udm_force_close()
615 			ark_mpu_set_producer(queue->mpu, index);  in eth_ark_udm_force_close()
627 ark_ethdev_rx_dump(const char *name, struct ark_rx_queue *queue)  in ark_ethdev_rx_dump()  argument
629 	if (queue == NULL)  in ark_ethdev_rx_dump()
631 	ARK_PMD_LOG(DEBUG, "RX QUEUE %d -- %s", queue->phys_qid, name);  in ark_ethdev_rx_dump()
633 			"queue_size", queue->queue_size,  in ark_ethdev_rx_dump()
634 			"seed_index", queue->seed_index,  in ark_ethdev_rx_dump()
635 			"prod_index", queue->prod_index,  in ark_ethdev_rx_dump()
636 			"cons_index", queue->cons_index);  in ark_ethdev_rx_dump()
638 	ark_mpu_dump(queue->mpu, name, queue->phys_qid);  in ark_ethdev_rx_dump()
639 	ark_mpu_dump_setup(queue->mpu, queue->phys_qid);  in ark_ethdev_rx_dump()
640 	ark_udm_dump(queue->udm, name);  in ark_ethdev_rx_dump()
641 	ark_udm_dump_setup(queue->udm, queue->phys_qid);  in ark_ethdev_rx_dump()