61 	struct ionic_queue *q = &txq->qcq.q;  in ionic_txq_info_get()  local
63 	qinfo->nb_desc = q->num_descs;  in ionic_txq_info_get()
72 	struct ionic_queue *q = &txq->qcq.q;  in ionic_tx_flush()  local
96 		while (q->tail_idx != comp_index) {  in ionic_tx_flush()
97 			info = IONIC_INFO_PTR(q, q->tail_idx);  in ionic_tx_flush()
99 			q->tail_idx = Q_NEXT_TO_SRVC(q, 1);  in ionic_tx_flush()
102 			if ((q->tail_idx & 0x3) == 0)  in ionic_tx_flush()
104 				rte_prefetch0(&q->info[q->tail_idx]);  in ionic_tx_flush()
129 		txq->qcq.q.index, stats->packets, stats->tso);  in ionic_dev_tx_queue_release()
237 		tx_queue_id, txq->qcq.q.num_descs);  in ionic_dev_tx_queue_start()
294 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,  in ionic_tx_tso_post()  argument
317 		info = IONIC_INFO_PTR(q, q->head_idx);  in ionic_tx_tso_post()
321 	q->head_idx = Q_NEXT_TO_POST(q, 1);  in ionic_tx_tso_post()
327 	struct ionic_queue *q = &txq->qcq.q;  in ionic_tx_tso_next()  local
328 	struct ionic_txq_desc *desc_base = q->base;  in ionic_tx_tso_next()
329 	struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;  in ionic_tx_tso_next()
330 	struct ionic_txq_desc *desc = &desc_base[q->head_idx];  in ionic_tx_tso_next()
331 	struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx];  in ionic_tx_tso_next()
340 	struct ionic_queue *q = &txq->qcq.q;  in ionic_tx_tso()  local
401 		ionic_tx_tso_post(q, desc, txm,  in ionic_tx_tso()
442 			ionic_tx_tso_post(q, desc, txm_seg,  in ionic_tx_tso()
463 	struct ionic_queue *q = &txq->qcq.q;  in ionic_tx()  local
464 	struct ionic_txq_desc *desc, *desc_base = q->base;  in ionic_tx()
465 	struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;  in ionic_tx()
477 	desc = &desc_base[q->head_idx];  in ionic_tx()
478 	info = IONIC_INFO_PTR(q, q->head_idx);  in ionic_tx()
514 	elem = sg_desc_base[q->head_idx].elems;  in ionic_tx()
524 	q->head_idx = Q_NEXT_TO_POST(q, 1);  in ionic_tx()
534 	struct ionic_queue *q = &txq->qcq.q;  in ionic_xmit_pkts()  local
544 	nb_avail = ionic_q_space_avail(q);  in ionic_xmit_pkts()
551 		next_q_head_idx = Q_NEXT_TO_POST(q, 1);  in ionic_xmit_pkts()
553 			struct ionic_txq_desc *desc_base = q->base;  in ionic_xmit_pkts()
555 			rte_prefetch0(&q->info[next_q_head_idx]);  in ionic_xmit_pkts()
573 		ionic_q_flush(q);  in ionic_xmit_pkts()
631 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
639 	struct ionic_queue *q = &rxq->qcq.q;  in ionic_rxq_info_get()  local
643 	qinfo->nb_desc = q->num_descs;  in ionic_rxq_info_get()
651 	struct ionic_queue *q = &rxq->qcq.q;  in ionic_rx_empty()  local
655 	while (q->tail_idx != q->head_idx) {  in ionic_rx_empty()
656 		info = IONIC_INFO_PTR(q, q->tail_idx);  in ionic_rx_empty()
660 		q->tail_idx = Q_NEXT_TO_SRVC(q, 1);  in ionic_rx_empty()
678 		rxq->qcq.q.index, stats->packets, stats->mtods);  in ionic_dev_rx_queue_release()
767 	struct ionic_queue *q = &rxq->qcq.q;  in ionic_rx_clean()  local
787 	info = IONIC_INFO_PTR(q, cq_desc->comp_index);  in ionic_rx_clean()
804 		ionic_rx_recycle(q, q_desc_index, rxm);  in ionic_rx_clean()
810 		ionic_rx_recycle(q, q_desc_index, rxm);  in ionic_rx_clean()
817 		ionic_rx_recycle(q, q_desc_index, rxm);  in ionic_rx_clean()
918 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,  in ionic_rx_recycle()  argument
921 	struct ionic_rxq_desc *desc_base = q->base;  in ionic_rx_recycle()
923 	struct ionic_rxq_desc *new = &desc_base[q->head_idx];  in ionic_rx_recycle()
928 	q->info[q->head_idx] = mbuf;  in ionic_rx_recycle()
930 	q->head_idx = Q_NEXT_TO_POST(q, 1);  in ionic_rx_recycle()
932 	ionic_q_flush(q);  in ionic_rx_recycle()
938 	struct ionic_queue *q = &rxq->qcq.q;  in ionic_rx_fill()  local
939 	struct ionic_rxq_desc *desc, *desc_base = q->base;  in ionic_rx_fill()
940 	struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base;  in ionic_rx_fill()
950 	for (i = ionic_q_space_avail(q); i; i--) {  in ionic_rx_fill()
959 		info = IONIC_INFO_PTR(q, q->head_idx);  in ionic_rx_fill()
963 		desc = &desc_base[q->head_idx];  in ionic_rx_fill()
973 		sg_desc = &sg_desc_base[q->head_idx];  in ionic_rx_fill()
1002 		q->head_idx = Q_NEXT_TO_POST(q, 1);  in ionic_rx_fill()
1005 	ionic_q_flush(q);  in ionic_rx_fill()
1030 		rx_queue_id, rxq->qcq.q.num_descs, frame_size);  in ionic_dev_rx_queue_start()
1057 	struct ionic_queue *q = &rxq->qcq.q;  in ionic_rxq_service()  local
1079 			more = (q->tail_idx != cq_desc->comp_index);  in ionic_rxq_service()
1081 			curr_q_tail_idx = q->tail_idx;  in ionic_rxq_service()
1082 			q->tail_idx = Q_NEXT_TO_SRVC(q, 1);  in ionic_rxq_service()
1085 			if ((q->tail_idx & 0x3) == 0)  in ionic_rxq_service()
1087 				rte_prefetch0(&q->info[q->tail_idx]);  in ionic_rxq_service()