17 static unsigned int efx_tx_cb_page_count(struct efx_tx_queue *tx_queue)  in efx_tx_cb_page_count()  argument
19 	return DIV_ROUND_UP(tx_queue->ptr_mask + 1,  in efx_tx_cb_page_count()
23 int efx_probe_tx_queue(struct efx_tx_queue *tx_queue)  in efx_probe_tx_queue()  argument
25 	struct efx_nic *efx = tx_queue->efx;  in efx_probe_tx_queue()
32 	tx_queue->ptr_mask = entries - 1;  in efx_probe_tx_queue()
36 		  tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask);  in efx_probe_tx_queue()
39 	tx_queue->buffer = kcalloc(entries, sizeof(*tx_queue->buffer),  in efx_probe_tx_queue()
41 	if (!tx_queue->buffer)  in efx_probe_tx_queue()
44 	tx_queue->cb_page = kcalloc(efx_tx_cb_page_count(tx_queue),  in efx_probe_tx_queue()
45 				    sizeof(tx_queue->cb_page[0]), GFP_KERNEL);  in efx_probe_tx_queue()
46 	if (!tx_queue->cb_page) {  in efx_probe_tx_queue()
52 	rc = efx_nic_probe_tx(tx_queue);  in efx_probe_tx_queue()
56 	tx_queue->channel->tx_queue_by_type[tx_queue->type] = tx_queue;  in efx_probe_tx_queue()
60 	kfree(tx_queue->cb_page);  in efx_probe_tx_queue()
61 	tx_queue->cb_page = NULL;  in efx_probe_tx_queue()
63 	kfree(tx_queue->buffer);  in efx_probe_tx_queue()
64 	tx_queue->buffer = NULL;  in efx_probe_tx_queue()
68 void efx_init_tx_queue(struct efx_tx_queue *tx_queue)  in efx_init_tx_queue()  argument
70 	struct efx_nic *efx = tx_queue->efx;  in efx_init_tx_queue()
73 		  "initialising TX queue %d\n", tx_queue->queue);  in efx_init_tx_queue()
75 	tx_queue->insert_count = 0;  in efx_init_tx_queue()
76 	tx_queue->notify_count = 0;  in efx_init_tx_queue()
77 	tx_queue->write_count = 0;  in efx_init_tx_queue()
78 	tx_queue->packet_write_count = 0;  in efx_init_tx_queue()
79 	tx_queue->old_write_count = 0;  in efx_init_tx_queue()
80 	tx_queue->read_count = 0;  in efx_init_tx_queue()
81 	tx_queue->old_read_count = 0;  in efx_init_tx_queue()
82 	tx_queue->empty_read_count = 0 | EFX_EMPTY_COUNT_VALID;  in efx_init_tx_queue()
83 	tx_queue->xmit_pending = false;  in efx_init_tx_queue()
84 	tx_queue->timestamping = (efx_ptp_use_mac_tx_timestamps(efx) &&  in efx_init_tx_queue()
85 				  tx_queue->channel == efx_ptp_channel(efx));  in efx_init_tx_queue()
86 	tx_queue->completed_timestamp_major = 0;  in efx_init_tx_queue()
87 	tx_queue->completed_timestamp_minor = 0;  in efx_init_tx_queue()
89 	tx_queue->old_complete_packets = tx_queue->complete_packets;  in efx_init_tx_queue()
90 	tx_queue->old_complete_bytes = tx_queue->complete_bytes;  in efx_init_tx_queue()
91 	tx_queue->old_tso_bursts = tx_queue->tso_bursts;  in efx_init_tx_queue()
92 	tx_queue->old_tso_packets = tx_queue->tso_packets;  in efx_init_tx_queue()
94 	tx_queue->xdp_tx = efx_channel_is_xdp_tx(tx_queue->channel);  in efx_init_tx_queue()
95 	tx_queue->tso_version = 0;  in efx_init_tx_queue()
98 	efx_nic_init_tx(tx_queue);  in efx_init_tx_queue()
100 	tx_queue->initialised = true;  in efx_init_tx_queue()
103 void efx_fini_tx_queue(struct efx_tx_queue *tx_queue)  in efx_fini_tx_queue()  argument
107 	netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,  in efx_fini_tx_queue()
108 		  "shutting down TX queue %d\n", tx_queue->queue);  in efx_fini_tx_queue()
110 	tx_queue->initialised = false;  in efx_fini_tx_queue()
112 	if (!tx_queue->buffer)  in efx_fini_tx_queue()
116 	while (tx_queue->read_count != tx_queue->write_count) {  in efx_fini_tx_queue()
121 		buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask];  in efx_fini_tx_queue()
122 		efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl,  in efx_fini_tx_queue()
126 		++tx_queue->read_count;  in efx_fini_tx_queue()
128 	tx_queue->xmit_pending = false;  in efx_fini_tx_queue()
129 	netdev_tx_reset_queue(tx_queue->core_txq);  in efx_fini_tx_queue()
132 void efx_remove_tx_queue(struct efx_tx_queue *tx_queue)  in efx_remove_tx_queue()  argument
136 	if (!tx_queue->buffer)  in efx_remove_tx_queue()
139 	netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,  in efx_remove_tx_queue()
140 		  "destroying TX queue %d\n", tx_queue->queue);  in efx_remove_tx_queue()
141 	efx_nic_remove_tx(tx_queue);  in efx_remove_tx_queue()
143 	if (tx_queue->cb_page) {  in efx_remove_tx_queue()
144 		for (i = 0; i < efx_tx_cb_page_count(tx_queue); i++)  in efx_remove_tx_queue()
145 			efx_nic_free_buffer(tx_queue->efx,  in efx_remove_tx_queue()
146 					    &tx_queue->cb_page[i]);  in efx_remove_tx_queue()
147 		kfree(tx_queue->cb_page);  in efx_remove_tx_queue()
148 		tx_queue->cb_page = NULL;  in efx_remove_tx_queue()
151 	kfree(tx_queue->buffer);  in efx_remove_tx_queue()
152 	tx_queue->buffer = NULL;  in efx_remove_tx_queue()
153 	tx_queue->channel->tx_queue_by_type[tx_queue->type] = NULL;  in efx_remove_tx_queue()
156 void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,  in efx_dequeue_buffer()  argument
165 		struct device *dma_dev = &tx_queue->efx->pci_dev->dev;  in efx_dequeue_buffer()
189 		if (tx_queue->timestamping &&  in efx_dequeue_buffer()
190 		    (tx_queue->completed_timestamp_major ||  in efx_dequeue_buffer()
191 		     tx_queue->completed_timestamp_minor)) {  in efx_dequeue_buffer()
195 				efx_ptp_nic_to_kernel_time(tx_queue);  in efx_dequeue_buffer()
198 			tx_queue->completed_timestamp_major = 0;  in efx_dequeue_buffer()
199 			tx_queue->completed_timestamp_minor = 0;  in efx_dequeue_buffer()
202 		netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev,  in efx_dequeue_buffer()
204 			   tx_queue->queue, tx_queue->read_count);  in efx_dequeue_buffer()
222 static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue,  in efx_dequeue_buffers()  argument
230 	struct efx_nic *efx = tx_queue->efx;  in efx_dequeue_buffers()
233 	stop_index = (index + 1) & tx_queue->ptr_mask;  in efx_dequeue_buffers()
234 	read_ptr = tx_queue->read_count & tx_queue->ptr_mask;  in efx_dequeue_buffers()
237 		struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];  in efx_dequeue_buffers()
242 				  tx_queue->queue, read_ptr);  in efx_dequeue_buffers()
247 		efx_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl,  in efx_dequeue_buffers()
250 		++tx_queue->read_count;  in efx_dequeue_buffers()
251 		read_ptr = tx_queue->read_count & tx_queue->ptr_mask;  in efx_dequeue_buffers()
255 void efx_xmit_done_check_empty(struct efx_tx_queue *tx_queue)  in efx_xmit_done_check_empty()  argument
257 	if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) {  in efx_xmit_done_check_empty()
258 		tx_queue->old_write_count = READ_ONCE(tx_queue->write_count);  in efx_xmit_done_check_empty()
259 		if (tx_queue->read_count == tx_queue->old_write_count) {  in efx_xmit_done_check_empty()
262 			tx_queue->empty_read_count =  in efx_xmit_done_check_empty()
263 				tx_queue->read_count | EFX_EMPTY_COUNT_VALID;  in efx_xmit_done_check_empty()
268 int efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)  in efx_xmit_done()  argument
273 	struct efx_nic *efx = tx_queue->efx;  in efx_xmit_done()
275 	EFX_WARN_ON_ONCE_PARANOID(index > tx_queue->ptr_mask);  in efx_xmit_done()
277 	efx_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl,  in efx_xmit_done()
279 	tx_queue->pkts_compl += pkts_compl;  in efx_xmit_done()
280 	tx_queue->bytes_compl += bytes_compl;  in efx_xmit_done()
281 	tx_queue->complete_xdp_packets += xdp_pkts_compl;  in efx_xmit_done()
282 	tx_queue->complete_xdp_bytes += xdp_bytes_compl;  in efx_xmit_done()
285 		++tx_queue->merge_events;  in efx_xmit_done()
292 	if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) &&  in efx_xmit_done()
295 		fill_level = efx_channel_tx_fill_level(tx_queue->channel);  in efx_xmit_done()
297 			netif_tx_wake_queue(tx_queue->core_txq);  in efx_xmit_done()
300 	efx_xmit_done_check_empty(tx_queue);  in efx_xmit_done()
308 void efx_enqueue_unwind(struct efx_tx_queue *tx_queue,  in efx_enqueue_unwind()  argument
319 	while (tx_queue->insert_count != insert_count) {  in efx_enqueue_unwind()
320 		--tx_queue->insert_count;  in efx_enqueue_unwind()
321 		buffer = __efx_tx_queue_get_insert_buffer(tx_queue);  in efx_enqueue_unwind()
322 		efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl,  in efx_enqueue_unwind()
328 struct efx_tx_buffer *efx_tx_map_chunk(struct efx_tx_queue *tx_queue,  in efx_tx_map_chunk()  argument
331 	const struct efx_nic_type *nic_type = tx_queue->efx->type;  in efx_tx_map_chunk()
337 		buffer = efx_tx_queue_get_insert_buffer(tx_queue);  in efx_tx_map_chunk()
340 			dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len);  in efx_tx_map_chunk()
349 		++tx_queue->insert_count;  in efx_tx_map_chunk()
369 int efx_tx_map_data(struct efx_tx_queue *tx_queue, struct sk_buff *skb,  in efx_tx_map_data()  argument
372 	struct efx_nic *efx = tx_queue->efx;  in efx_tx_map_data()
399 			tx_queue->tso_long_headers++;  in efx_tx_map_data()
400 			efx_tx_map_chunk(tx_queue, dma_addr, header_len);  in efx_tx_map_data()
411 		buffer = efx_tx_map_chunk(tx_queue, dma_addr, len);  in efx_tx_map_data()
472 int efx_tx_tso_fallback(struct efx_tx_queue *tx_queue, struct sk_buff *skb)  in efx_tx_tso_fallback()  argument
484 		efx_enqueue_skb(tx_queue, skb);  in efx_tx_tso_fallback()