Lines Matching refs:tx_ring
160 static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring,
436 txr = &adapter->tx_ring[i]; in ena_init_io_rings()
479 que->tx_ring = txr; in ena_init_io_rings()
493 struct ena_ring *txr = &adapter->tx_ring[qid]; in ena_free_io_ring_resources()
600 struct ena_ring *tx_ring = que->tx_ring; in ena_setup_tx_resources() local
606 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size; in ena_setup_tx_resources()
608 tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); in ena_setup_tx_resources()
609 if (unlikely(tx_ring->tx_buffer_info == NULL)) in ena_setup_tx_resources()
612 size = sizeof(uint16_t) * tx_ring->ring_size; in ena_setup_tx_resources()
613 tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); in ena_setup_tx_resources()
614 if (unlikely(tx_ring->free_tx_ids == NULL)) in ena_setup_tx_resources()
618 for (i = 0; i < tx_ring->ring_size; i++) in ena_setup_tx_resources()
619 tx_ring->free_tx_ids[i] = i; in ena_setup_tx_resources()
622 ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats, in ena_setup_tx_resources()
623 sizeof(tx_ring->tx_stats)); in ena_setup_tx_resources()
625 tx_ring->next_to_use = 0; in ena_setup_tx_resources()
626 tx_ring->next_to_clean = 0; in ena_setup_tx_resources()
629 ENA_RING_MTX_LOCK(tx_ring); in ena_setup_tx_resources()
630 drbr_flush(adapter->ifp, tx_ring->br); in ena_setup_tx_resources()
631 ENA_RING_MTX_UNLOCK(tx_ring); in ena_setup_tx_resources()
634 for (i = 0; i < tx_ring->ring_size; i++) { in ena_setup_tx_resources()
636 &tx_ring->tx_buffer_info[i].map); in ena_setup_tx_resources()
645 TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring); in ena_setup_tx_resources()
646 tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT, in ena_setup_tx_resources()
647 taskqueue_thread_enqueue, &tx_ring->enqueue_tq); in ena_setup_tx_resources()
648 if (unlikely(tx_ring->enqueue_tq == NULL)) { in ena_setup_tx_resources()
651 i = tx_ring->ring_size; in ena_setup_tx_resources()
658 taskqueue_start_threads_cpuset(&tx_ring->enqueue_tq, 1, PI_NET, in ena_setup_tx_resources()
662 taskqueue_start_threads(&tx_ring->enqueue_tq, 1, PI_NET, in ena_setup_tx_resources()
671 tx_ring->tx_buffer_info[i].map); in ena_setup_tx_resources()
673 free(tx_ring->free_tx_ids, M_DEVBUF); in ena_setup_tx_resources()
674 tx_ring->free_tx_ids = NULL; in ena_setup_tx_resources()
676 free(tx_ring->tx_buffer_info, M_DEVBUF); in ena_setup_tx_resources()
677 tx_ring->tx_buffer_info = NULL; in ena_setup_tx_resources()
692 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; in ena_free_tx_resources() local
694 while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task, in ena_free_tx_resources()
696 taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task); in ena_free_tx_resources()
698 taskqueue_free(tx_ring->enqueue_tq); in ena_free_tx_resources()
700 ENA_RING_MTX_LOCK(tx_ring); in ena_free_tx_resources()
702 drbr_flush(adapter->ifp, tx_ring->br); in ena_free_tx_resources()
705 for (int i = 0; i < tx_ring->ring_size; i++) { in ena_free_tx_resources()
706 m_freem(tx_ring->tx_buffer_info[i].mbuf); in ena_free_tx_resources()
707 tx_ring->tx_buffer_info[i].mbuf = NULL; in ena_free_tx_resources()
709 tx_ring->tx_buffer_info[i].map); in ena_free_tx_resources()
711 tx_ring->tx_buffer_info[i].map); in ena_free_tx_resources()
713 ENA_RING_MTX_UNLOCK(tx_ring); in ena_free_tx_resources()
716 free(tx_ring->tx_buffer_info, M_DEVBUF); in ena_free_tx_resources()
717 tx_ring->tx_buffer_info = NULL; in ena_free_tx_resources()
719 free(tx_ring->free_tx_ids, M_DEVBUF); in ena_free_tx_resources()
720 tx_ring->free_tx_ids = NULL; in ena_free_tx_resources()
1154 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; in ena_free_tx_bufs() local
1156 ENA_RING_MTX_LOCK(tx_ring); in ena_free_tx_bufs()
1157 for (int i = 0; i < tx_ring->ring_size; i++) { in ena_free_tx_bufs()
1158 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i]; in ena_free_tx_bufs()
1178 ENA_RING_MTX_UNLOCK(tx_ring); in ena_free_tx_bufs()
1221 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id) in validate_tx_req_id() argument
1223 struct ena_adapter *adapter = tx_ring->adapter; in validate_tx_req_id()
1226 if (likely(req_id < tx_ring->ring_size)) { in validate_tx_req_id()
1227 tx_info = &tx_ring->tx_buffer_info[req_id]; in validate_tx_req_id()
1238 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1); in validate_tx_req_id()
1268 ring = &adapter->tx_ring[i]; in ena_create_io_queues()
1335 ena_tx_cleanup(struct ena_ring *tx_ring) in ena_tx_cleanup() argument
1348 adapter = tx_ring->que->adapter; in ena_tx_cleanup()
1349 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); in ena_tx_cleanup()
1351 next_to_clean = tx_ring->next_to_clean; in ena_tx_cleanup()
1361 rc = validate_tx_req_id(tx_ring, req_id); in ena_tx_cleanup()
1365 tx_info = &tx_ring->tx_buffer_info[req_id]; in ena_tx_cleanup()
1378 tx_ring->qid, mbuf); in ena_tx_cleanup()
1384 tx_ring->free_tx_ids[next_to_clean] = req_id; in ena_tx_cleanup()
1386 tx_ring->ring_size); in ena_tx_cleanup()
1391 tx_ring->next_to_clean = next_to_clean; in ena_tx_cleanup()
1403 tx_ring->qid, work_done); in ena_tx_cleanup()
1407 tx_ring->next_to_clean = next_to_clean; in ena_tx_cleanup()
1413 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task); in ena_tx_cleanup()
1792 struct ena_ring *tx_ring; in ena_handle_msix() local
1804 tx_ring = que->tx_ring; in ena_handle_msix()
1824 ENA_RING_MTX_LOCK(tx_ring); in ena_handle_msix()
1825 txc = ena_tx_cleanup(tx_ring); in ena_handle_msix()
1826 ENA_RING_MTX_UNLOCK(tx_ring); in ena_handle_msix()
2719 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) in ena_check_and_collapse_mbuf() argument
2725 adapter = tx_ring->adapter; in ena_check_and_collapse_mbuf()
2731 counter_u64_add(tx_ring->tx_stats.collapse, 1); in ena_check_and_collapse_mbuf()
2736 counter_u64_add(tx_ring->tx_stats.collapse_err, 1); in ena_check_and_collapse_mbuf()
2747 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) in ena_xmit_mbuf() argument
2765 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); in ena_xmit_mbuf()
2766 adapter = tx_ring->que->adapter; in ena_xmit_mbuf()
2770 rc = ena_check_and_collapse_mbuf(tx_ring, mbuf); in ena_xmit_mbuf()
2777 next_to_use = tx_ring->next_to_use; in ena_xmit_mbuf()
2778 req_id = tx_ring->free_tx_ids[next_to_use]; in ena_xmit_mbuf()
2779 tx_info = &tx_ring->tx_buffer_info[req_id]; in ena_xmit_mbuf()
2805 counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1); in ena_xmit_mbuf()
2833 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1); in ena_xmit_mbuf()
2838 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1); in ena_xmit_mbuf()
2839 counter_u64_add_protected(tx_ring->tx_stats.bytes, in ena_xmit_mbuf()
2851 tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, in ena_xmit_mbuf()
2852 tx_ring->ring_size); in ena_xmit_mbuf()
2867 ena_start_xmit(struct ena_ring *tx_ring) in ena_start_xmit() argument
2870 struct ena_adapter *adapter = tx_ring->adapter; in ena_start_xmit()
2882 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); in ena_start_xmit()
2885 while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) { in ena_start_xmit()
2892 ena_tx_cleanup(tx_ring); in ena_start_xmit()
2894 if (unlikely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0)) { in ena_start_xmit()
2896 drbr_putback(adapter->ifp, tx_ring->br, mbuf); in ena_start_xmit()
2898 drbr_putback(adapter->ifp, tx_ring->br, mbuf); in ena_start_xmit()
2901 drbr_advance(adapter->ifp, tx_ring->br); in ena_start_xmit()
2907 drbr_advance(adapter->ifp, tx_ring->br); in ena_start_xmit()
2922 counter_u64_add(tx_ring->tx_stats.doorbells, 1); in ena_start_xmit()
2931 counter_u64_add(tx_ring->tx_stats.doorbells, 1); in ena_start_xmit()
2935 ena_tx_cleanup(tx_ring); in ena_start_xmit()
2941 struct ena_ring *tx_ring = (struct ena_ring *)arg; in ena_deferred_mq_start() local
2942 struct ifnet *ifp = tx_ring->adapter->ifp; in ena_deferred_mq_start()
2944 while (!drbr_empty(ifp, tx_ring->br) && in ena_deferred_mq_start()
2946 ENA_RING_MTX_LOCK(tx_ring); in ena_deferred_mq_start()
2947 ena_start_xmit(tx_ring); in ena_deferred_mq_start()
2948 ENA_RING_MTX_UNLOCK(tx_ring); in ena_deferred_mq_start()
2956 struct ena_ring *tx_ring; in ena_mq_start() local
2983 tx_ring = &adapter->tx_ring[i]; in ena_mq_start()
2986 is_drbr_empty = drbr_empty(ifp, tx_ring->br); in ena_mq_start()
2987 ret = drbr_enqueue(ifp, tx_ring->br, m); in ena_mq_start()
2989 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task); in ena_mq_start()
2993 if ((is_drbr_empty != 0) && (ENA_RING_MTX_TRYLOCK(tx_ring) != 0)) { in ena_mq_start()
2994 ena_start_xmit(tx_ring); in ena_mq_start()
2995 ENA_RING_MTX_UNLOCK(tx_ring); in ena_mq_start()
2997 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task); in ena_mq_start()
3007 struct ena_ring *tx_ring = adapter->tx_ring; in ena_qflush() local
3010 for(i = 0; i < adapter->num_queues; ++i, ++tx_ring) in ena_qflush()
3011 if (!drbr_empty(ifp, tx_ring->br)) { in ena_qflush()
3012 ENA_RING_MTX_LOCK(tx_ring); in ena_qflush()
3013 drbr_flush(ifp, tx_ring->br); in ena_qflush()
3014 ENA_RING_MTX_UNLOCK(tx_ring); in ena_qflush()
3382 struct ena_ring *tx_ring) in check_missing_comp_in_queue() argument
3391 for (i = 0; i < tx_ring->ring_size; i++) { in check_missing_comp_in_queue()
3392 tx_buf = &tx_ring->tx_buffer_info[i]; in check_missing_comp_in_queue()
3406 tx_ring->qid, i); in check_missing_comp_in_queue()
3410 counter_u64_add(tx_ring->tx_stats.missing_tx_comp, 1); in check_missing_comp_in_queue()
3439 struct ena_ring *tx_ring; in check_for_missing_tx_completions() local
3457 tx_ring = &adapter->tx_ring[i]; in check_for_missing_tx_completions()
3459 rc = check_missing_comp_in_queue(adapter, tx_ring); in check_for_missing_tx_completions()