51 static inline int ena_get_tx_req_id(struct ena_ring *tx_ring,
60 static int	ena_check_and_collapse_mbuf(struct ena_ring *tx_ring,
75 	struct ena_ring *tx_ring;  in ena_cleanup()  local
87 	tx_ring = que->tx_ring;  in ena_cleanup()
93 	tx_ring->first_interrupt = true;  in ena_cleanup()
98 		txc = ena_tx_cleanup(tx_ring);  in ena_cleanup()
112 	counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1);  in ena_cleanup()
119 	struct ena_ring *tx_ring = (struct ena_ring *)arg;  in ena_deferred_mq_start()  local
120 	struct ifnet *ifp = tx_ring->adapter->ifp;  in ena_deferred_mq_start()
122 	while (!drbr_empty(ifp, tx_ring->br) &&  in ena_deferred_mq_start()
123 	    tx_ring->running &&  in ena_deferred_mq_start()
125 		ENA_RING_MTX_LOCK(tx_ring);  in ena_deferred_mq_start()
126 		ena_start_xmit(tx_ring);  in ena_deferred_mq_start()
127 		ENA_RING_MTX_UNLOCK(tx_ring);  in ena_deferred_mq_start()
135 	struct ena_ring *tx_ring;  in ena_mq_start()  local
162 	tx_ring = &adapter->tx_ring[i];  in ena_mq_start()
165 	is_drbr_empty = drbr_empty(ifp, tx_ring->br);  in ena_mq_start()
166 	ret = drbr_enqueue(ifp, tx_ring->br, m);  in ena_mq_start()
168 		taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);  in ena_mq_start()
172 	if (is_drbr_empty && (ENA_RING_MTX_TRYLOCK(tx_ring) != 0)) {  in ena_mq_start()
173 		ena_start_xmit(tx_ring);  in ena_mq_start()
174 		ENA_RING_MTX_UNLOCK(tx_ring);  in ena_mq_start()
176 		taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);  in ena_mq_start()
186 	struct ena_ring *tx_ring = adapter->tx_ring;  in ena_qflush()  local
189 	for(i = 0; i < adapter->num_io_queues; ++i, ++tx_ring)  in ena_qflush()
190 		if (!drbr_empty(ifp, tx_ring->br)) {  in ena_qflush()
191 			ENA_RING_MTX_LOCK(tx_ring);  in ena_qflush()
192 			drbr_flush(ifp, tx_ring->br);  in ena_qflush()
193 			ENA_RING_MTX_UNLOCK(tx_ring);  in ena_qflush()
204 ena_get_tx_req_id(struct ena_ring *tx_ring, struct ena_com_io_cq *io_cq,  in ena_get_tx_req_id()  argument
207 	struct ena_adapter *adapter = tx_ring->adapter;  in ena_get_tx_req_id()
216 		counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);  in ena_get_tx_req_id()
220 	if (tx_ring->tx_buffer_info[*req_id].mbuf != NULL)  in ena_get_tx_req_id()
242 ena_tx_cleanup(struct ena_ring *tx_ring)  in ena_tx_cleanup()  argument
256 	adapter = tx_ring->que->adapter;  in ena_tx_cleanup()
257 	ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);  in ena_tx_cleanup()
259 	next_to_clean = tx_ring->next_to_clean;  in ena_tx_cleanup()
262 	if (netmap_tx_irq(adapter->ifp, tx_ring->qid) != NM_IRQ_PASS)  in ena_tx_cleanup()
270 		rc = ena_get_tx_req_id(tx_ring, io_cq, &req_id);  in ena_tx_cleanup()
274 		tx_info = &tx_ring->tx_buffer_info[req_id];  in ena_tx_cleanup()
287 		    tx_ring->qid, mbuf);  in ena_tx_cleanup()
293 		tx_ring->free_tx_ids[next_to_clean] = req_id;  in ena_tx_cleanup()
295 		    tx_ring->ring_size);  in ena_tx_cleanup()
300 			tx_ring->next_to_clean = next_to_clean;  in ena_tx_cleanup()
312 	    tx_ring->qid, work_done);  in ena_tx_cleanup()
316 		tx_ring->next_to_clean = next_to_clean;  in ena_tx_cleanup()
328 	above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_tx_cleanup()
330 	if (unlikely(!tx_ring->running && above_thresh)) {  in ena_tx_cleanup()
331 		ENA_RING_MTX_LOCK(tx_ring);  in ena_tx_cleanup()
333 		    ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_tx_cleanup()
335 		if (!tx_ring->running && above_thresh) {  in ena_tx_cleanup()
336 			tx_ring->running = true;  in ena_tx_cleanup()
337 			counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);  in ena_tx_cleanup()
338 			taskqueue_enqueue(tx_ring->enqueue_tq,  in ena_tx_cleanup()
339 			    &tx_ring->enqueue_task);  in ena_tx_cleanup()
341 		ENA_RING_MTX_UNLOCK(tx_ring);  in ena_tx_cleanup()
819 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)  in ena_check_and_collapse_mbuf()  argument
825 	adapter = tx_ring->adapter;  in ena_check_and_collapse_mbuf()
833 	    ((*mbuf)->m_pkthdr.len < tx_ring->tx_max_header_size))  in ena_check_and_collapse_mbuf()
836 	counter_u64_add(tx_ring->tx_stats.collapse, 1);  in ena_check_and_collapse_mbuf()
841 		counter_u64_add(tx_ring->tx_stats.collapse_err, 1);  in ena_check_and_collapse_mbuf()
852 ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info,  in ena_tx_map_mbuf()  argument
855 	struct ena_adapter *adapter = tx_ring->adapter;  in ena_tx_map_mbuf()
879 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {  in ena_tx_map_mbuf()
890 		*header_len = min_t(uint32_t, mbuf->m_pkthdr.len, tx_ring->tx_max_header_size);  in ena_tx_map_mbuf()
900 			m_copydata(mbuf, 0, *header_len, tx_ring->push_buf_intermediate_buf);  in ena_tx_map_mbuf()
901 			*push_hdr = tx_ring->push_buf_intermediate_buf;  in ena_tx_map_mbuf()
903 			counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);  in ena_tx_map_mbuf()
910 		if (mbuf->m_pkthdr.len <= tx_ring->tx_max_header_size) {  in ena_tx_map_mbuf()
913 			offset = tx_ring->tx_max_header_size;  in ena_tx_map_mbuf()
955 	counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1);  in ena_tx_map_mbuf()
961 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)  in ena_xmit_mbuf()  argument
977 	ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);  in ena_xmit_mbuf()
978 	adapter = tx_ring->que->adapter;  in ena_xmit_mbuf()
983 	rc = ena_check_and_collapse_mbuf(tx_ring, mbuf);  in ena_xmit_mbuf()
992 	next_to_use = tx_ring->next_to_use;  in ena_xmit_mbuf()
993 	req_id = tx_ring->free_tx_ids[next_to_use];  in ena_xmit_mbuf()
994 	tx_info = &tx_ring->tx_buffer_info[req_id];  in ena_xmit_mbuf()
1000 	rc = ena_tx_map_mbuf(tx_ring, tx_info, *mbuf, &push_hdr, &header_len);  in ena_xmit_mbuf()
1015 	if (tx_ring->acum_pkts == DB_THRESHOLD ||  in ena_xmit_mbuf()
1016 	    ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, &ena_tx_ctx)) {  in ena_xmit_mbuf()
1019 		    tx_ring->que->id);  in ena_xmit_mbuf()
1020 		ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);  in ena_xmit_mbuf()
1021 		counter_u64_add(tx_ring->tx_stats.doorbells, 1);  in ena_xmit_mbuf()
1022 		tx_ring->acum_pkts = 0;  in ena_xmit_mbuf()
1030 			    tx_ring->que->id);  in ena_xmit_mbuf()
1036 		counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);  in ena_xmit_mbuf()
1041 	counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);  in ena_xmit_mbuf()
1042 	counter_u64_add_protected(tx_ring->tx_stats.bytes,  in ena_xmit_mbuf()
1054 	tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,  in ena_xmit_mbuf()
1055 	    tx_ring->ring_size);  in ena_xmit_mbuf()
1061 	if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_xmit_mbuf()
1063 		ena_log_io(pdev, DBG, "Stop queue %d\n", tx_ring->que->id);  in ena_xmit_mbuf()
1065 		tx_ring->running = false;  in ena_xmit_mbuf()
1066 		counter_u64_add(tx_ring->tx_stats.queue_stop, 1);  in ena_xmit_mbuf()
1077 		if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_xmit_mbuf()
1079 			tx_ring->running = true;  in ena_xmit_mbuf()
1080 			counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);  in ena_xmit_mbuf()
1097 ena_start_xmit(struct ena_ring *tx_ring)  in ena_start_xmit()  argument
1100 	struct ena_adapter *adapter = tx_ring->adapter;  in ena_start_xmit()
1105 	ENA_RING_MTX_ASSERT(tx_ring);  in ena_start_xmit()
1113 	ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);  in ena_start_xmit()
1116 	while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) {  in ena_start_xmit()
1121 		if (unlikely(!tx_ring->running)) {  in ena_start_xmit()
1122 			drbr_putback(adapter->ifp, tx_ring->br, mbuf);  in ena_start_xmit()
1126 		if (unlikely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0)) {  in ena_start_xmit()
1128 				drbr_putback(adapter->ifp, tx_ring->br, mbuf);  in ena_start_xmit()
1130 				drbr_putback(adapter->ifp, tx_ring->br, mbuf);  in ena_start_xmit()
1133 				drbr_advance(adapter->ifp, tx_ring->br);  in ena_start_xmit()
1139 		drbr_advance(adapter->ifp, tx_ring->br);  in ena_start_xmit()
1145 		tx_ring->acum_pkts++;  in ena_start_xmit()
1150 	if (likely(tx_ring->acum_pkts != 0)) {  in ena_start_xmit()
1153 		counter_u64_add(tx_ring->tx_stats.doorbells, 1);  in ena_start_xmit()
1154 		tx_ring->acum_pkts = 0;  in ena_start_xmit()
1157 	if (unlikely(!tx_ring->running))  in ena_start_xmit()
1158 		taskqueue_enqueue(tx_ring->que->cleanup_tq,  in ena_start_xmit()
1159 		    &tx_ring->que->cleanup_task);  in ena_start_xmit()