400 		rxr = &adapter->rx_ring[i];  in ena_init_io_rings_basic()
414 		que->rx_ring = rxr;  in ena_init_io_rings_basic()
432 		rxr = &adapter->rx_ring[i];  in ena_init_io_rings_advanced()
475 	struct ena_ring *rxr = &adapter->rx_ring[qid];  in ena_free_io_ring_resources()
873 	struct ena_ring *rx_ring = que->rx_ring;  in ena_setup_rx_resources()  local
876 	size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;  in ena_setup_rx_resources()
880 	rx_ring->initialized = false;  in ena_setup_rx_resources()
889 	rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);  in ena_setup_rx_resources()
891 	size = sizeof(uint16_t) * rx_ring->ring_size;  in ena_setup_rx_resources()
892 	rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK);  in ena_setup_rx_resources()
894 	for (i = 0; i < rx_ring->ring_size; i++)  in ena_setup_rx_resources()
895 		rx_ring->free_rx_ids[i] = i;  in ena_setup_rx_resources()
898 	ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,  in ena_setup_rx_resources()
899 	    sizeof(rx_ring->rx_stats));  in ena_setup_rx_resources()
901 	rx_ring->next_to_clean = 0;  in ena_setup_rx_resources()
902 	rx_ring->next_to_use = 0;  in ena_setup_rx_resources()
905 	for (i = 0; i < rx_ring->ring_size; i++) {  in ena_setup_rx_resources()
907 		    &(rx_ring->rx_buffer_info[i].map));  in ena_setup_rx_resources()
917 		int err = tcp_lro_init(&rx_ring->lro);  in ena_setup_rx_resources()
924 			rx_ring->lro.ifp = adapter->ifp;  in ena_setup_rx_resources()
933 		    rx_ring->rx_buffer_info[i].map);  in ena_setup_rx_resources()
936 	free(rx_ring->free_rx_ids, M_DEVBUF);  in ena_setup_rx_resources()
937 	rx_ring->free_rx_ids = NULL;  in ena_setup_rx_resources()
938 	free(rx_ring->rx_buffer_info, M_DEVBUF);  in ena_setup_rx_resources()
939 	rx_ring->rx_buffer_info = NULL;  in ena_setup_rx_resources()
953 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];  in ena_free_rx_resources()  local
956 	for (int i = 0; i < rx_ring->ring_size; i++) {  in ena_free_rx_resources()
958 		    rx_ring->rx_buffer_info[i].map, BUS_DMASYNC_POSTREAD);  in ena_free_rx_resources()
959 		m_freem(rx_ring->rx_buffer_info[i].mbuf);  in ena_free_rx_resources()
960 		rx_ring->rx_buffer_info[i].mbuf = NULL;  in ena_free_rx_resources()
962 		    rx_ring->rx_buffer_info[i].map);  in ena_free_rx_resources()
964 		    rx_ring->rx_buffer_info[i].map);  in ena_free_rx_resources()
968 	tcp_lro_free(&rx_ring->lro);  in ena_free_rx_resources()
971 	free(rx_ring->rx_buffer_info, M_DEVBUF);  in ena_free_rx_resources()
972 	rx_ring->rx_buffer_info = NULL;  in ena_free_rx_resources()
974 	free(rx_ring->free_rx_ids, M_DEVBUF);  in ena_free_rx_resources()
975 	rx_ring->free_rx_ids = NULL;  in ena_free_rx_resources()
1022 ena_alloc_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,  in ena_alloc_rx_mbuf()  argument
1037 	    rx_ring->rx_mbuf_sz);  in ena_alloc_rx_mbuf()
1040 		counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1);  in ena_alloc_rx_mbuf()
1043 			counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);  in ena_alloc_rx_mbuf()
1048 		mlen = rx_ring->rx_mbuf_sz;  in ena_alloc_rx_mbuf()
1062 		counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1);  in ena_alloc_rx_mbuf()
1085 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,  in ena_free_rx_mbuf()  argument
1108 ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)  in ena_refill_rx_bufs()  argument
1110 	struct ena_adapter *adapter = rx_ring->adapter;  in ena_refill_rx_bufs()
1116 	ena_log_io(adapter->pdev, DBG, "refill qid: %d\n", rx_ring->qid);  in ena_refill_rx_bufs()
1118 	next_to_use = rx_ring->next_to_use;  in ena_refill_rx_bufs()
1126 		req_id = rx_ring->free_rx_ids[next_to_use];  in ena_refill_rx_bufs()
1127 		rx_info = &rx_ring->rx_buffer_info[req_id];  in ena_refill_rx_bufs()
1129 		if (ena_rx_ring_in_netmap(adapter, rx_ring->qid))  in ena_refill_rx_bufs()
1130 			rc = ena_netmap_alloc_rx_slot(adapter, rx_ring,  in ena_refill_rx_bufs()
1134 			rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);  in ena_refill_rx_bufs()
1138 			    rx_ring->qid);  in ena_refill_rx_bufs()
1141 		rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,  in ena_refill_rx_bufs()
1146 			    rx_ring->qid);  in ena_refill_rx_bufs()
1150 		    rx_ring->ring_size);  in ena_refill_rx_bufs()
1154 		counter_u64_add(rx_ring->rx_stats.refil_partial, 1);  in ena_refill_rx_bufs()
1157 		    rx_ring->qid, i, num);  in ena_refill_rx_bufs()
1161 		ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);  in ena_refill_rx_bufs()
1163 	rx_ring->next_to_use = next_to_use;  in ena_refill_rx_bufs()
1441 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];  in ena_free_rx_bufs()  local
1444 	for (i = 0; i < rx_ring->ring_size; i++) {  in ena_free_rx_bufs()
1445 		struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];  in ena_free_rx_bufs()
1448 			ena_free_rx_mbuf(adapter, rx_ring, rx_info);  in ena_free_rx_bufs()
1453 				ena_netmap_free_rx_slot(adapter, rx_ring,  in ena_free_rx_bufs()
1468 	struct ena_ring *rx_ring;  in ena_refill_all_rx_bufs()  local
1472 		rx_ring = &adapter->rx_ring[i];  in ena_refill_all_rx_bufs()
1473 		bufs_num = rx_ring->ring_size - 1;  in ena_refill_all_rx_bufs()
1474 		rc = ena_refill_rx_bufs(rx_ring, bufs_num);  in ena_refill_all_rx_bufs()
1481 		rx_ring->initialized = true;  in ena_refill_all_rx_bufs()
1647 		ring = &adapter->rx_ring[i];  in ena_create_io_queues()
2148 		adapter->rx_ring[i].ring_size = new_rx_size;  in set_io_rings_size()
2207 		cur_rx_ring_size = adapter->rx_ring[0].ring_size;  in create_queues_with_size_backoff()
3131     struct ena_ring *rx_ring)  in check_for_rx_interrupt_queue()  argument
3133 	if (likely(atomic_load_8(&rx_ring->first_interrupt)))  in check_for_rx_interrupt_queue()
3136 	if (ena_com_cq_empty(rx_ring->ena_com_io_cq))  in check_for_rx_interrupt_queue()
3139 	rx_ring->no_interrupt_event_cnt++;  in check_for_rx_interrupt_queue()
3141 	if (rx_ring->no_interrupt_event_cnt ==  in check_for_rx_interrupt_queue()
3145 		    rx_ring->qid);  in check_for_rx_interrupt_queue()
3278 	struct ena_ring *rx_ring;  in check_for_missing_completions()  local
3297 		rx_ring = &adapter->rx_ring[i];  in check_for_missing_completions()
3303 		rc = check_for_rx_interrupt_queue(adapter, rx_ring);  in check_for_missing_completions()
3332 	struct ena_ring *rx_ring;  in check_for_empty_rx_ring()  local
3342 		rx_ring = &adapter->rx_ring[i];  in check_for_empty_rx_ring()
3345 		    rx_ring->ena_com_io_sq);  in check_for_empty_rx_ring()
3346 		if (unlikely(refill_required == (rx_ring->ring_size - 1))) {  in check_for_empty_rx_ring()
3347 			rx_ring->empty_rx_queue++;  in check_for_empty_rx_ring()
3349 			if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {  in check_for_empty_rx_ring()
3350 				counter_u64_add(rx_ring->rx_stats.empty_rx_ring,  in check_for_empty_rx_ring()
3357 				taskqueue_enqueue(rx_ring->que->cleanup_tq,  in check_for_empty_rx_ring()
3358 				    &rx_ring->que->cleanup_task);  in check_for_empty_rx_ring()
3359 				rx_ring->empty_rx_queue = 0;  in check_for_empty_rx_ring()
3362 			rx_ring->empty_rx_queue = 0;  in check_for_empty_rx_ring()