1003 	struct mvneta_tx_ring *tx;  in mvneta_portdown()  local
1015 		tx = MVNETA_TX_RING(sc, q);  in mvneta_portdown()
1017 		tx->queue_status = MVNETA_QUEUE_DISABLED;  in mvneta_portdown()
1246 	struct mvneta_tx_ring *tx;  in mvneta_ring_alloc_tx_queue()  local
1251 	tx = MVNETA_TX_RING(sc, q);  in mvneta_ring_alloc_tx_queue()
1252 	mtx_init(&tx->ring_mtx, "mvneta_tx", NULL, MTX_DEF);  in mvneta_ring_alloc_tx_queue()
1254 	    (void**)&(tx->desc),  in mvneta_ring_alloc_tx_queue()
1256 	    &tx->desc_map);  in mvneta_ring_alloc_tx_queue()
1257 	if (error != 0 || tx->desc == NULL)  in mvneta_ring_alloc_tx_queue()
1259 	error = bus_dmamap_load(sc->tx_dtag, tx->desc_map,  in mvneta_ring_alloc_tx_queue()
1260 	    tx->desc,  in mvneta_ring_alloc_tx_queue()
1262 	    mvneta_dmamap_cb, &tx->desc_pa, BUS_DMA_NOWAIT);  in mvneta_ring_alloc_tx_queue()
1267 	tx->br = buf_ring_alloc(MVNETA_BUFRING_SIZE, M_DEVBUF, M_NOWAIT,  in mvneta_ring_alloc_tx_queue()
1268 	    &tx->ring_mtx);  in mvneta_ring_alloc_tx_queue()
1269 	if (tx->br == NULL) {  in mvneta_ring_alloc_tx_queue()
1287 	struct mvneta_tx_ring *tx;  in mvneta_ring_dealloc_tx_queue()  local
1295 	tx = MVNETA_TX_RING(sc, q);  in mvneta_ring_dealloc_tx_queue()
1297 	if (tx->taskq != NULL) {  in mvneta_ring_dealloc_tx_queue()
1299 		while (taskqueue_cancel(tx->taskq, &tx->task, NULL) != 0)  in mvneta_ring_dealloc_tx_queue()
1300 			taskqueue_drain(tx->taskq, &tx->task);  in mvneta_ring_dealloc_tx_queue()
1303 	if (tx->br != NULL)  in mvneta_ring_dealloc_tx_queue()
1304 		drbr_free(tx->br, M_DEVBUF);  in mvneta_ring_dealloc_tx_queue()
1308 		if (mtx_name(&tx->ring_mtx) != NULL) {  in mvneta_ring_dealloc_tx_queue()
1319 			txbuf = &tx->txbuf[i];  in mvneta_ring_dealloc_tx_queue()
1331 	if (tx->desc_pa != 0)  in mvneta_ring_dealloc_tx_queue()
1332 		bus_dmamap_unload(sc->tx_dtag, tx->desc_map);  in mvneta_ring_dealloc_tx_queue()
1334 	kva = (void *)tx->desc;  in mvneta_ring_dealloc_tx_queue()
1336 		bus_dmamem_free(sc->tx_dtag, tx->desc, tx->desc_map);  in mvneta_ring_dealloc_tx_queue()
1338 	if (mtx_name(&tx->ring_mtx) != NULL)  in mvneta_ring_dealloc_tx_queue()
1339 		mtx_destroy(&tx->ring_mtx);  in mvneta_ring_dealloc_tx_queue()
1341 	memset(tx, 0, sizeof(*tx));  in mvneta_ring_dealloc_tx_queue()
1408 	struct mvneta_tx_ring *tx;  in mvneta_ring_init_tx_queue()  local
1415 	tx = MVNETA_TX_RING(sc, q);  in mvneta_ring_init_tx_queue()
1419 		txbuf = &tx->txbuf[i];  in mvneta_ring_init_tx_queue()
1430 	tx->dma = tx->cpu = 0;  in mvneta_ring_init_tx_queue()
1431 	tx->used = 0;  in mvneta_ring_init_tx_queue()
1432 	tx->drv_error = 0;  in mvneta_ring_init_tx_queue()
1433 	tx->queue_status = MVNETA_QUEUE_DISABLED;  in mvneta_ring_init_tx_queue()
1434 	tx->queue_hung = FALSE;  in mvneta_ring_init_tx_queue()
1436 	tx->ifp = sc->ifp;  in mvneta_ring_init_tx_queue()
1437 	tx->qidx = q;  in mvneta_ring_init_tx_queue()
1438 	TASK_INIT(&tx->task, 0, mvneta_tx_task, tx);  in mvneta_ring_init_tx_queue()
1439 	tx->taskq = taskqueue_create_fast("mvneta_tx_taskq", M_WAITOK,  in mvneta_ring_init_tx_queue()
1440 	    taskqueue_thread_enqueue, &tx->taskq);  in mvneta_ring_init_tx_queue()
1441 	taskqueue_start_threads(&tx->taskq, 1, PI_NET, "%s: tx_taskq(%d)",  in mvneta_ring_init_tx_queue()
1450 	struct mvneta_tx_ring *tx;  in mvneta_ring_flush_tx_queue()  local
1454 	tx = MVNETA_TX_RING(sc, q);  in mvneta_ring_flush_tx_queue()
1459 		txbuf = &tx->txbuf[i];  in mvneta_ring_flush_tx_queue()
1466 	tx->dma = tx->cpu = 0;  in mvneta_ring_flush_tx_queue()
1467 	tx->used = 0;  in mvneta_ring_flush_tx_queue()
1531 	struct mvneta_tx_ring *tx;  in mvneta_tx_queue_init()  local
1536 	tx = MVNETA_TX_RING(sc, q);  in mvneta_tx_queue_init()
1537 	DASSERT(tx->desc_pa != 0);  in mvneta_tx_queue_init()
1540 	MVNETA_WRITE(sc, MVNETA_PTXDQA(q), tx->desc_pa);  in mvneta_tx_queue_init()
1547 	DASSERT(MVNETA_READ(sc, MVNETA_PTXDQA(q)) == tx->desc_pa);  in mvneta_tx_queue_init()
1587 	struct mvneta_tx_ring *tx;  in mvneta_tx_queue_enable()  local
1590 	tx = MVNETA_TX_RING(sc, q);  in mvneta_tx_queue_enable()
1596 	tx->queue_status = MVNETA_QUEUE_IDLE;  in mvneta_tx_queue_enable()
1597 	tx->queue_hung = FALSE;  in mvneta_tx_queue_enable()
1755 	struct mvneta_tx_ring *tx;  in mvneta_tick()  local
1810 		tx = MVNETA_TX_RING(sc, q);  in mvneta_tick()
1812 		if (tx->queue_hung && (fc_curr - fc_prev) == 0)  in mvneta_tick()
1836 	struct mvneta_tx_ring *tx;  in mvneta_qflush()  local
1843 		tx = MVNETA_TX_RING(sc, q);  in mvneta_qflush()
1845 		while ((m = buf_ring_dequeue_sc(tx->br)) != NULL)  in mvneta_qflush()
1857 	struct mvneta_tx_ring *tx;  in mvneta_tx_task()  local
1861 	tx = arg;  in mvneta_tx_task()
1862 	ifp = tx->ifp;  in mvneta_tx_task()
1865 	mvneta_tx_lockq(sc, tx->qidx);  in mvneta_tx_task()
1866 	error = mvneta_xmit_locked(sc, tx->qidx);  in mvneta_tx_task()
1867 	mvneta_tx_unlockq(sc, tx->qidx);  in mvneta_tx_task()
1872 		taskqueue_enqueue(tx->taskq, &tx->task);  in mvneta_tx_task()
1879 	struct mvneta_tx_ring *tx;  in mvneta_xmitfast_locked()  local
1884 	tx = MVNETA_TX_RING(sc, q);  in mvneta_xmitfast_locked()
1890 	if (__predict_false(tx->queue_status == MVNETA_QUEUE_DISABLED)) {  in mvneta_xmitfast_locked()
1897 	if (__predict_true(tx->used > MVNETA_TX_RECLAIM_COUNT))  in mvneta_xmitfast_locked()
1901 	if (__predict_false(tx->used >  in mvneta_xmitfast_locked()
1913 	tx->watchdog_time = ticks;  in mvneta_xmitfast_locked()
1914 	tx->queue_status = MVNETA_QUEUE_WORKING;  in mvneta_xmitfast_locked()
1924 	struct mvneta_tx_ring *tx;  in mvneta_transmit()  local
1936 	tx = MVNETA_TX_RING(sc, q);  in mvneta_transmit()
1939 	if (buf_ring_full(tx->br)) {  in mvneta_transmit()
1949 	if (buf_ring_empty(tx->br) && mvneta_tx_trylockq(sc, q) != 0) {  in mvneta_transmit()
1961 	error = drbr_enqueue(ifp, tx->br, m);  in mvneta_transmit()
1965 	taskqueue_enqueue(tx->taskq, &tx->task);  in mvneta_transmit()
1973 	struct mvneta_tx_ring *tx;  in mvneta_xmit_locked()  local
1979 	tx = MVNETA_TX_RING(sc, q);  in mvneta_xmit_locked()
1982 	while ((m = drbr_peek(ifp, tx->br)) != NULL) {  in mvneta_xmit_locked()
1986 				drbr_putback(ifp, tx->br, m);  in mvneta_xmit_locked()
1988 				drbr_advance(ifp, tx->br);  in mvneta_xmit_locked()
1991 		drbr_advance(ifp, tx->br);  in mvneta_xmit_locked()
2001 	struct mvneta_tx_ring *tx;  in mvneta_start()  local
2005 	tx = MVNETA_TX_RING(sc, 0);  in mvneta_start()
2012 		taskqueue_enqueue(tx->taskq, &tx->task);  in mvneta_start()
2019 	struct mvneta_tx_ring *tx;  in mvneta_xmit_locked()  local
2025 	tx = MVNETA_TX_RING(sc, 0);  in mvneta_xmit_locked()
2240 	struct mvneta_tx_ring *tx;  in mvneta_stop_locked()  local
2280 		tx = MVNETA_TX_RING(sc, q);  in mvneta_stop_locked()
2656 	struct mvneta_tx_ring *tx;  in mvneta_tx_queue()  local
2663 	tx = MVNETA_TX_RING(sc, q);  in mvneta_tx_queue()
2664 	DASSERT(tx->used >= 0);  in mvneta_tx_queue()
2665 	DASSERT(tx->used <= MVNETA_TX_RING_CNT);  in mvneta_tx_queue()
2672 			tx->drv_error++;  in mvneta_tx_queue()
2687 				tx->drv_error++;  in mvneta_tx_queue()
2696 	txbuf = &tx->txbuf[tx->cpu];  in mvneta_tx_queue()
2706 			tx->drv_error++;  in mvneta_tx_queue()
2715 	    || (txnsegs + tx->used) > MVNETA_TX_RING_CNT)) {  in mvneta_tx_queue()
2732 	start = tx->cpu;  in mvneta_tx_queue()
2735 		t = &tx->desc[tx->cpu];  in mvneta_tx_queue()
2747 		tx->cpu = tx_counter_adv(tx->cpu, 1);  in mvneta_tx_queue()
2749 		tx->used++;  in mvneta_tx_queue()
2756 	bus_dmamap_sync(sc->tx_dtag, tx->desc_map,  in mvneta_tx_queue()
2827 	struct mvneta_tx_ring *tx;  in mvneta_tx_queue_complete()  local
2835 	tx = MVNETA_TX_RING(sc, q);  in mvneta_tx_queue_complete()
2836 	if (__predict_false(tx->queue_status == MVNETA_QUEUE_DISABLED))  in mvneta_tx_queue_complete()
2843 		if (tx->used == 0)  in mvneta_tx_queue_complete()
2844 			tx->queue_status = MVNETA_QUEUE_IDLE;  in mvneta_tx_queue_complete()
2845 		else if (tx->queue_status == MVNETA_QUEUE_WORKING &&  in mvneta_tx_queue_complete()
2846 		    ((ticks - tx->watchdog_time) > MVNETA_WATCHDOG))  in mvneta_tx_queue_complete()
2847 			tx->queue_hung = TRUE;  in mvneta_tx_queue_complete()
2856 	bus_dmamap_sync(sc->tx_dtag, tx->desc_map,  in mvneta_tx_queue_complete()
2860 		t = &tx->desc[tx->dma];  in mvneta_tx_queue_complete()
2864 			    sc->ifp->if_xname, q, tx->dma);  in mvneta_tx_queue_complete()
2866 		txbuf = &tx->txbuf[tx->dma];  in mvneta_tx_queue_complete()
2875 		tx->dma = tx_counter_adv(tx->dma, 1);  in mvneta_tx_queue_complete()
2876 		tx->used--;  in mvneta_tx_queue_complete()
2878 	DASSERT(tx->used >= 0);  in mvneta_tx_queue_complete()
2879 	DASSERT(tx->used <= MVNETA_TX_RING_CNT);  in mvneta_tx_queue_complete()
2891 	    sc->ifp->if_xname, q, tx->cpu, tx->dma, tx->used);  in mvneta_tx_queue_complete()
2894 	tx->watchdog_time = ticks;  in mvneta_tx_queue_complete()
2896 	if (tx->used == 0)  in mvneta_tx_queue_complete()
2897 		tx->queue_status = MVNETA_QUEUE_IDLE;  in mvneta_tx_queue_complete()
2906 	struct mvneta_tx_ring *tx;  in mvneta_tx_drain()  local
2914 		tx = MVNETA_TX_RING(sc, q);  in mvneta_tx_drain()
2915 		if ((ticks - tx->watchdog_time) > MVNETA_WATCHDOG_TXCOMP &&  in mvneta_tx_drain()
2916 		    tx->used > 0) {  in mvneta_tx_drain()
3516 	struct mvneta_tx_ring *tx;  in mvneta_update_mib()  local
3586 		tx = MVNETA_TX_RING(sc, i);  in mvneta_update_mib()
3588 		if (tx->drv_error > 0) {  in mvneta_update_mib()
3589 			if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, tx->drv_error);  in mvneta_update_mib()
3590 			tx->drv_error = 0;  in mvneta_update_mib()