3531 static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)  in netif_tx_start_queue()  argument
3533 	clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);  in netif_tx_start_queue()
3557 void netif_tx_wake_queue(struct netdev_queue *dev_queue);
3581 static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)  in netif_tx_stop_queue()  argument
3584 	WRITE_ONCE(dev_queue->trans_start, jiffies);  in netif_tx_stop_queue()
3590 	set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);  in netif_tx_stop_queue()
3607 static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)  in netif_tx_queue_stopped()  argument
3609 	return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);  in netif_tx_queue_stopped()
3623 static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)  in netif_xmit_stopped()  argument
3625 	return dev_queue->state & QUEUE_STATE_ANY_XOFF;  in netif_xmit_stopped()
3629 netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)  in netif_xmit_frozen_or_stopped()  argument
3631 	return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;  in netif_xmit_frozen_or_stopped()
3635 netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)  in netif_xmit_frozen_or_drv_stopped()  argument
3637 	return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;  in netif_xmit_frozen_or_drv_stopped()
3650 static inline void netdev_queue_set_dql_min_limit(struct netdev_queue *dev_queue,  in netdev_queue_set_dql_min_limit()  argument
3654 	dev_queue->dql.min_limit = min_limit;  in netdev_queue_set_dql_min_limit()
3675 static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)  in netdev_txq_bql_enqueue_prefetchw()  argument
3678 	prefetchw(&dev_queue->dql.num_queued);  in netdev_txq_bql_enqueue_prefetchw()
3689 static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)  in netdev_txq_bql_complete_prefetchw()  argument
3692 	prefetchw(&dev_queue->dql.limit);  in netdev_txq_bql_complete_prefetchw()
3706 static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,  in netdev_tx_sent_queue()  argument
3710 	dql_queued(&dev_queue->dql, bytes);  in netdev_tx_sent_queue()
3712 	if (likely(dql_avail(&dev_queue->dql) >= 0))  in netdev_tx_sent_queue()
3716 	WRITE_ONCE(dev_queue->trans_start, jiffies);  in netdev_tx_sent_queue()
3721 	set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);  in netdev_tx_sent_queue()
3731 	if (unlikely(dql_avail(&dev_queue->dql) >= 0))  in netdev_tx_sent_queue()
3732 		clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);  in netdev_tx_sent_queue()
3742 static inline bool __netdev_tx_sent_queue(struct netdev_queue *dev_queue,  in __netdev_tx_sent_queue()  argument
3748 		dql_queued(&dev_queue->dql, bytes);  in __netdev_tx_sent_queue()
3750 		return netif_tx_queue_stopped(dev_queue);  in __netdev_tx_sent_queue()
3752 	netdev_tx_sent_queue(dev_queue, bytes);  in __netdev_tx_sent_queue()
3788 static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,  in netdev_tx_completed_queue()  argument
3795 	dql_completed(&dev_queue->dql, bytes);  in netdev_tx_completed_queue()
3804 	if (unlikely(dql_avail(&dev_queue->dql) < 0))  in netdev_tx_completed_queue()
3807 	if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))  in netdev_tx_completed_queue()
3808 		netif_schedule_queue(dev_queue);  in netdev_tx_completed_queue()
3854 static inline void netdev_reset_queue(struct net_device *dev_queue)  in netdev_reset_queue()  argument
3856 	netdev_tx_reset_subqueue(dev_queue, 0);  in netdev_reset_queue()