63 	struct ionic_queue *q = &txq->q;  in ionic_txq_info_get()  local
65 	qinfo->nb_desc = q->num_descs;  in ionic_txq_info_get()
73 	struct ionic_queue *q = cq->bound_q;  in ionic_tx_flush()  local
97 		while (q->tail_idx != comp_index) {  in ionic_tx_flush()
98 			q_desc_info = &q->info[q->tail_idx];  in ionic_tx_flush()
100 			q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);  in ionic_tx_flush()
103 			if ((q->tail_idx & 0x3) == 0)  in ionic_tx_flush()
105 				rte_prefetch0(&q->info[q->tail_idx]);  in ionic_tx_flush()
275 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,  in ionic_tx_tso_post()  argument
296 	ionic_q_post(q, done, NULL, done ? txm : NULL);  in ionic_tx_tso_post()
300 ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem)  in ionic_tx_tso_next()  argument
302 	struct ionic_txq_desc *desc_base = q->base;  in ionic_tx_tso_next()
303 	struct ionic_txq_sg_desc *sg_desc_base = q->sg_base;  in ionic_tx_tso_next()
304 	struct ionic_txq_desc *desc = &desc_base[q->head_idx];  in ionic_tx_tso_next()
305 	struct ionic_txq_sg_desc *sg_desc = &sg_desc_base[q->head_idx];  in ionic_tx_tso_next()
312 ionic_tx_tso(struct ionic_queue *q, struct rte_mbuf *txm,  in ionic_tx_tso()  argument
315 	struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);  in ionic_tx_tso()
357 	desc = ionic_tx_tso_next(q, &elem);  in ionic_tx_tso()
373 		ionic_tx_tso_post(q, desc, txm,  in ionic_tx_tso()
379 		desc = ionic_tx_tso_next(q, &elem);  in ionic_tx_tso()
415 			ionic_tx_tso_post(q, desc, txm_seg,  in ionic_tx_tso()
421 			desc = ionic_tx_tso_next(q, &elem);  in ionic_tx_tso()
434 ionic_tx(struct ionic_queue *q, struct rte_mbuf *txm,  in ionic_tx()  argument
437 	struct ionic_txq_desc *desc_base = q->base;  in ionic_tx()
438 	struct ionic_txq_sg_desc *sg_desc_base = q->sg_base;  in ionic_tx()
439 	struct ionic_txq_desc *desc = &desc_base[q->head_idx];  in ionic_tx()
440 	struct ionic_txq_sg_desc *sg_desc = &sg_desc_base[q->head_idx];  in ionic_tx()
442 	struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);  in ionic_tx()
486 	ionic_q_post(q, not_xmit_more, NULL, txm);  in ionic_tx()
496 	struct ionic_queue *q = &txq->q;  in ionic_xmit_pkts()  local
498 	struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);  in ionic_xmit_pkts()
508 	if (unlikely(ionic_q_space_avail(q) < nb_pkts)) {  in ionic_xmit_pkts()
516 		next_q_head_idx = (q->head_idx + 1) & (q->num_descs - 1);  in ionic_xmit_pkts()
518 			struct ionic_txq_desc *desc_base = q->base;  in ionic_xmit_pkts()
520 			rte_prefetch0(&q->info[next_q_head_idx]);  in ionic_xmit_pkts()
524 			err = ionic_tx_tso(q, tx_pkts[nb_tx], txq->offloads,  in ionic_xmit_pkts()
527 			err = ionic_tx(q, tx_pkts[nb_tx], txq->offloads, last);  in ionic_xmit_pkts()
531 				ionic_q_flush(q);  in ionic_xmit_pkts()
595 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
603 	struct ionic_queue *q = &rxq->q;  in ionic_rxq_info_get()  local
607 	qinfo->nb_desc = q->num_descs;  in ionic_rxq_info_get()
613 ionic_rx_empty(struct ionic_queue *q)  in ionic_rx_empty()  argument
615 	struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q);  in ionic_rx_empty()
619 	while (q->tail_idx != q->head_idx) {  in ionic_rx_empty()
620 		cur = &q->info[q->tail_idx];  in ionic_rx_empty()
624 		q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);  in ionic_rx_empty()
635 	ionic_rx_empty(&rxq->q);  in ionic_dev_rx_queue_release()
716 ionic_rx_clean(struct ionic_queue *q,  in ionic_rx_clean()  argument
720 	struct ionic_rxq_comp *cq_desc_base = q->bound_cq->base;  in ionic_rx_clean()
724 	struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q);  in ionic_rx_clean()
729 	struct ionic_rx_stats *stats = IONIC_Q_TO_RX_STATS(q);  in ionic_rx_clean()
750 		ionic_rx_recycle(q, q_desc_index, rxm);  in ionic_rx_clean()
756 		ionic_rx_recycle(q, q_desc_index, rxm);  in ionic_rx_clean()
763 		ionic_rx_recycle(q, q_desc_index, rxm);  in ionic_rx_clean()
863 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,  in ionic_rx_recycle()  argument
866 	struct ionic_rxq_desc *desc_base = q->base;  in ionic_rx_recycle()
868 	struct ionic_rxq_desc *new = &desc_base[q->head_idx];  in ionic_rx_recycle()
873 	ionic_q_post(q, true, ionic_rx_clean, mbuf);  in ionic_rx_recycle()
879 	struct ionic_queue *q = &rxq->q;  in ionic_rx_fill()  local
880 	struct ionic_rxq_desc *desc_base = q->base;  in ionic_rx_fill()
881 	struct ionic_rxq_sg_desc *sg_desc_base = q->sg_base;  in ionic_rx_fill()
893 	for (i = ionic_q_space_avail(q); i; i--) {  in ionic_rx_fill()
904 		desc = &desc_base[q->head_idx];  in ionic_rx_fill()
914 		sg_desc = &sg_desc_base[q->head_idx];  in ionic_rx_fill()
941 		ring_doorbell = ((q->head_idx + 1) &  in ionic_rx_fill()
944 		ionic_q_post(q, ring_doorbell, ionic_rx_clean, rxm);  in ionic_rx_fill()
990 	struct ionic_queue *q = cq->bound_q;  in ionic_rxq_service()  local
1014 			more = (q->tail_idx != cq_desc->comp_index);  in ionic_rxq_service()
1016 			q_desc_info = &q->info[q->tail_idx];  in ionic_rxq_service()
1018 			curr_q_tail_idx = q->tail_idx;  in ionic_rxq_service()
1019 			q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);  in ionic_rxq_service()
1022 			if ((q->tail_idx & 0x3) == 0)  in ionic_rxq_service()
1024 				rte_prefetch0(&q->info[q->tail_idx]);  in ionic_rxq_service()
1026 			ionic_rx_clean(q, curr_q_tail_idx, curr_cq_tail_idx,  in ionic_rxq_service()