197 static void ena_init_rings(struct ena_adapter *adapter,
246 static int ena_parse_devargs(struct ena_adapter *adapter,
248 static int ena_copy_eni_stats(struct ena_adapter *adapter);
390 	rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;  in validate_rx_req_id()
391 	rx_ring->adapter->trigger_reset = true;  in validate_rx_req_id()
414 	tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;  in validate_tx_req_id()
415 	tx_ring->adapter->trigger_reset	= true;  in validate_tx_req_id()
474 static void ena_config_debug_area(struct ena_adapter *adapter)  in ena_config_debug_area()  argument
479 	ss_count = ena_xstats_calc_num(adapter->rte_dev);  in ena_config_debug_area()
484 	rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size);  in ena_config_debug_area()
490 	rc = ena_com_set_host_attributes(&adapter->ena_dev);  in ena_config_debug_area()
502 	ena_com_delete_debug_area(&adapter->ena_dev);  in ena_config_debug_area()
509 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_close()  local
515 	if (adapter->state == ENA_ADAPTER_STATE_RUNNING)  in ena_close()
517 	adapter->state = ENA_ADAPTER_STATE_CLOSED;  in ena_close()
522 	rte_free(adapter->drv_stats);  in ena_close()
523 	adapter->drv_stats = NULL;  in ena_close()
528 				     adapter);  in ena_close()
556 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_rss_reta_update()  local
557 	struct ena_com_dev *ena_dev = &adapter->ena_dev;  in ena_rss_reta_update()
594 	rte_spinlock_lock(&adapter->admin_lock);  in ena_rss_reta_update()
596 	rte_spinlock_unlock(&adapter->admin_lock);  in ena_rss_reta_update()
603 		__func__, reta_size, adapter->rte_dev->data->port_id);  in ena_rss_reta_update()
613 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_rss_reta_query()  local
614 	struct ena_com_dev *ena_dev = &adapter->ena_dev;  in ena_rss_reta_query()
625 	rte_spinlock_lock(&adapter->admin_lock);  in ena_rss_reta_query()
627 	rte_spinlock_unlock(&adapter->admin_lock);  in ena_rss_reta_query()
644 static int ena_rss_init_default(struct ena_adapter *adapter)  in ena_rss_init_default()  argument
646 	struct ena_com_dev *ena_dev = &adapter->ena_dev;  in ena_rss_init_default()
647 	uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues;  in ena_rss_init_default()
686 		adapter->rte_dev->data->port_id);  in ena_rss_init_default()
793 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_link_update()  local
795 	link->link_status = adapter->link_status ? ETH_LINK_UP : ETH_LINK_DOWN;  in ena_link_update()
805 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_queue_start_all()  local
812 		queues = adapter->rx_ring;  in ena_queue_start_all()
815 		queues = adapter->tx_ring;  in ena_queue_start_all()
851 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter)  in ena_get_mtu_conf()  argument
853 	uint32_t max_frame_len = adapter->max_mtu;  in ena_get_mtu_conf()
855 	if (adapter->rte_eth_dev_data->dev_conf.rxmode.offloads &  in ena_get_mtu_conf()
858 			adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len;  in ena_get_mtu_conf()
863 static int ena_check_valid_conf(struct ena_adapter *adapter)  in ena_check_valid_conf()  argument
865 	uint32_t max_frame_len = ena_get_mtu_conf(adapter);  in ena_check_valid_conf()
867 	if (max_frame_len > adapter->max_mtu || max_frame_len < ENA_MIN_MTU) {  in ena_check_valid_conf()
870 			     max_frame_len, adapter->max_mtu, ENA_MIN_MTU);  in ena_check_valid_conf()
960 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_stats_restart()  local
962 	rte_atomic64_init(&adapter->drv_stats->ierrors);  in ena_stats_restart()
963 	rte_atomic64_init(&adapter->drv_stats->oerrors);  in ena_stats_restart()
964 	rte_atomic64_init(&adapter->drv_stats->rx_nombuf);  in ena_stats_restart()
965 	adapter->drv_stats->rx_drops = 0;  in ena_stats_restart()
972 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_stats_get()  local
973 	struct ena_com_dev *ena_dev = &adapter->ena_dev;  in ena_stats_get()
983 	rte_spinlock_lock(&adapter->admin_lock);  in ena_stats_get()
985 	rte_spinlock_unlock(&adapter->admin_lock);  in ena_stats_get()
1002 	stats->imissed = adapter->drv_stats->rx_drops;  in ena_stats_get()
1003 	stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors);  in ena_stats_get()
1004 	stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors);  in ena_stats_get()
1005 	stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf);  in ena_stats_get()
1010 		struct ena_stats_rx *rx_stats = &adapter->rx_ring[i].rx_stats;  in ena_stats_get()
1021 		struct ena_stats_tx *tx_stats = &adapter->tx_ring[i].tx_stats;  in ena_stats_get()
1032 	struct ena_adapter *adapter;  in ena_mtu_set()  local
1038 	adapter = dev->data->dev_private;  in ena_mtu_set()
1040 	ena_dev = &adapter->ena_dev;  in ena_mtu_set()
1043 	if (mtu > ena_get_mtu_conf(adapter) || mtu < ENA_MIN_MTU) {  in ena_mtu_set()
1047 			mtu, ena_get_mtu_conf(adapter), ENA_MIN_MTU);  in ena_mtu_set()
1062 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_start()  local
1066 	rc = ena_check_valid_conf(adapter);  in ena_start()
1078 	if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode &  in ena_start()
1079 	    ETH_MQ_RX_RSS_FLAG && adapter->rte_dev->data->nb_rx_queues > 0) {  in ena_start()
1080 		rc = ena_rss_init_default(adapter);  in ena_start()
1087 	adapter->timestamp_wd = rte_get_timer_cycles();  in ena_start()
1088 	adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT;  in ena_start()
1091 	rte_timer_reset(&adapter->timer_wd, ticks, PERIODICAL, rte_lcore_id(),  in ena_start()
1092 			ena_timer_wd_callback, adapter);  in ena_start()
1094 	++adapter->dev_stats.dev_start;  in ena_start()
1095 	adapter->state = ENA_ADAPTER_STATE_RUNNING;  in ena_start()
1108 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_stop()  local
1109 	struct ena_com_dev *ena_dev = &adapter->ena_dev;  in ena_stop()
1112 	rte_timer_stop_sync(&adapter->timer_wd);  in ena_stop()
1116 	if (adapter->trigger_reset) {  in ena_stop()
1117 		rc = ena_com_dev_reset(ena_dev, adapter->reset_reason);  in ena_stop()
1122 	++adapter->dev_stats.dev_stop;  in ena_stop()
1123 	adapter->state = ENA_ADAPTER_STATE_STOPPED;  in ena_stop()
1131 	struct ena_adapter *adapter;  in ena_create_io_queue()  local
1141 	adapter = ring->adapter;  in ena_create_io_queue()
1142 	ena_dev = &adapter->ena_dev;  in ena_create_io_queue()
1188 	struct ena_com_dev *ena_dev = &ring->adapter->ena_dev;  in ena_queue_stop()
1202 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_queue_stop_all()  local
1207 		queues = adapter->rx_ring;  in ena_queue_stop_all()
1210 		queues = adapter->tx_ring;  in ena_queue_stop_all()
1244 		ena_com_destroy_io_queue(&ring->adapter->ena_dev,  in ena_queue_start()
1260 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_tx_queue_setup()  local
1263 	txq = &adapter->tx_ring[queue_idx];  in ena_tx_queue_setup()
1279 	if (nb_desc > adapter->max_tx_ring_size) {  in ena_tx_queue_setup()
1282 			adapter->max_tx_ring_size);  in ena_tx_queue_setup()
1287 		nb_desc = adapter->max_tx_ring_size;  in ena_tx_queue_setup()
1346 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_rx_queue_setup()  local
1351 	rxq = &adapter->rx_ring[queue_idx];  in ena_rx_queue_setup()
1360 		nb_desc = adapter->max_rx_ring_size;  in ena_rx_queue_setup()
1369 	if (nb_desc > adapter->max_rx_ring_size) {  in ena_rx_queue_setup()
1372 			adapter->max_rx_ring_size);  in ena_rx_queue_setup()
1471 		rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf);  in ena_populate_rx_queue()
1609 	struct ena_adapter *adapter = cb_arg;  in ena_interrupt_handler_rte()  local
1610 	struct ena_com_dev *ena_dev = &adapter->ena_dev;  in ena_interrupt_handler_rte()
1613 	if (likely(adapter->state != ENA_ADAPTER_STATE_CLOSED))  in ena_interrupt_handler_rte()
1614 		ena_com_aenq_intr_handler(ena_dev, adapter);  in ena_interrupt_handler_rte()
1617 static void check_for_missing_keep_alive(struct ena_adapter *adapter)  in check_for_missing_keep_alive()  argument
1619 	if (!adapter->wd_state)  in check_for_missing_keep_alive()
1622 	if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)  in check_for_missing_keep_alive()
1625 	if (unlikely((rte_get_timer_cycles() - adapter->timestamp_wd) >=  in check_for_missing_keep_alive()
1626 	    adapter->keep_alive_timeout)) {  in check_for_missing_keep_alive()
1628 		adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;  in check_for_missing_keep_alive()
1629 		adapter->trigger_reset = true;  in check_for_missing_keep_alive()
1630 		++adapter->dev_stats.wd_expired;  in check_for_missing_keep_alive()
1635 static void check_for_admin_com_state(struct ena_adapter *adapter)  in check_for_admin_com_state()  argument
1637 	if (unlikely(!ena_com_get_admin_running_state(&adapter->ena_dev))) {  in check_for_admin_com_state()
1639 		adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO;  in check_for_admin_com_state()
1640 		adapter->trigger_reset = true;  in check_for_admin_com_state()
1647 	struct ena_adapter *adapter = arg;  in ena_timer_wd_callback()  local
1648 	struct rte_eth_dev *dev = adapter->rte_dev;  in ena_timer_wd_callback()
1650 	check_for_missing_keep_alive(adapter);  in ena_timer_wd_callback()
1651 	check_for_admin_com_state(adapter);  in ena_timer_wd_callback()
1653 	if (unlikely(adapter->trigger_reset)) {  in ena_timer_wd_callback()
1683 ena_set_queues_placement_policy(struct ena_adapter *adapter,  in ena_set_queues_placement_policy()  argument
1711 	if (!adapter->dev_mem_base) {  in ena_set_queues_placement_policy()
1718 	ena_dev->mem_bar = adapter->dev_mem_base;  in ena_set_queues_placement_policy()
1765 	struct ena_adapter *adapter = eth_dev->data->dev_private;  in eth_ena_dev_init()  local
1766 	struct ena_com_dev *ena_dev = &adapter->ena_dev;  in eth_ena_dev_init()
1786 	memset(adapter, 0, sizeof(struct ena_adapter));  in eth_ena_dev_init()
1787 	ena_dev = &adapter->ena_dev;  in eth_ena_dev_init()
1789 	adapter->rte_eth_dev_data = eth_dev->data;  in eth_ena_dev_init()
1790 	adapter->rte_dev = eth_dev;  in eth_ena_dev_init()
1793 	adapter->pdev = pci_dev;  in eth_ena_dev_init()
1803 	adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr;  in eth_ena_dev_init()
1804 	adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr;  in eth_ena_dev_init()
1806 	if (!adapter->regs) {  in eth_ena_dev_init()
1812 	ena_dev->reg_bar = adapter->regs;  in eth_ena_dev_init()
1813 	ena_dev->dmadev = adapter->pdev;  in eth_ena_dev_init()
1815 	adapter->id_number = adapters_found;  in eth_ena_dev_init()
1817 	snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d",  in eth_ena_dev_init()
1818 		 adapter->id_number);  in eth_ena_dev_init()
1820 	rc = ena_parse_devargs(adapter, pci_dev->device.devargs);  in eth_ena_dev_init()
1832 	adapter->wd_state = wd_state;  in eth_ena_dev_init()
1835 		adapter->use_large_llq_hdr);  in eth_ena_dev_init()
1836 	rc = ena_set_queues_placement_policy(adapter, ena_dev,  in eth_ena_dev_init()
1854 		adapter->use_large_llq_hdr);  in eth_ena_dev_init()
1860 	adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size;  in eth_ena_dev_init()
1861 	adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size;  in eth_ena_dev_init()
1862 	adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size;  in eth_ena_dev_init()
1863 	adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size;  in eth_ena_dev_init()
1864 	adapter->max_num_io_queues = max_num_io_queues;  in eth_ena_dev_init()
1875 	ena_init_rings(adapter, disable_meta_caching);  in eth_ena_dev_init()
1877 	ena_config_debug_area(adapter);  in eth_ena_dev_init()
1880 	adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu;  in eth_ena_dev_init()
1883 	adapter->offloads.tso4_supported = (get_feat_ctx.offload.tx &  in eth_ena_dev_init()
1885 	adapter->offloads.tx_csum_supported = (get_feat_ctx.offload.tx &  in eth_ena_dev_init()
1887 	adapter->offloads.rx_csum_supported =  in eth_ena_dev_init()
1892 	eth_dev->data->mac_addrs = (struct rte_ether_addr *)adapter->mac_addr;  in eth_ena_dev_init()
1895 			(struct rte_ether_addr *)adapter->mac_addr);  in eth_ena_dev_init()
1897 	adapter->drv_stats = rte_zmalloc("adapter stats",  in eth_ena_dev_init()
1898 					 sizeof(*adapter->drv_stats),  in eth_ena_dev_init()
1900 	if (!adapter->drv_stats) {  in eth_ena_dev_init()
1906 	rte_spinlock_init(&adapter->admin_lock);  in eth_ena_dev_init()
1910 				   adapter);  in eth_ena_dev_init()
1917 	rte_timer_init(&adapter->timer_wd);  in eth_ena_dev_init()
1920 	adapter->state = ENA_ADAPTER_STATE_INIT;  in eth_ena_dev_init()
1937 	struct ena_adapter *adapter = eth_dev->data->dev_private;  in ena_destroy_device()  local
1938 	struct ena_com_dev *ena_dev = &adapter->ena_dev;  in ena_destroy_device()
1940 	if (adapter->state == ENA_ADAPTER_STATE_FREE)  in ena_destroy_device()
1945 	if (adapter->state != ENA_ADAPTER_STATE_CLOSED)  in ena_destroy_device()
1956 	adapter->state = ENA_ADAPTER_STATE_FREE;  in ena_destroy_device()
1971 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_dev_configure()  local
1973 	adapter->state = ENA_ADAPTER_STATE_CONFIG;  in ena_dev_configure()
1975 	adapter->tx_selected_offloads = dev->data->dev_conf.txmode.offloads;  in ena_dev_configure()
1976 	adapter->rx_selected_offloads = dev->data->dev_conf.rxmode.offloads;  in ena_dev_configure()
1980 static void ena_init_rings(struct ena_adapter *adapter,  in ena_init_rings()  argument
1985 	for (i = 0; i < adapter->max_num_io_queues; i++) {  in ena_init_rings()
1986 		struct ena_ring *ring = &adapter->tx_ring[i];  in ena_init_rings()
1990 		ring->adapter = adapter;  in ena_init_rings()
1992 		ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type;  in ena_init_rings()
1993 		ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size;  in ena_init_rings()
1994 		ring->sgl_size = adapter->max_tx_sgl_size;  in ena_init_rings()
1998 	for (i = 0; i < adapter->max_num_io_queues; i++) {  in ena_init_rings()
1999 		struct ena_ring *ring = &adapter->rx_ring[i];  in ena_init_rings()
2003 		ring->adapter = adapter;  in ena_init_rings()
2005 		ring->sgl_size = adapter->max_rx_sgl_size;  in ena_init_rings()
2012 	struct ena_adapter *adapter;  in ena_infos_get()  local
2018 	adapter = dev->data->dev_private;  in ena_infos_get()
2020 	ena_dev = &adapter->ena_dev;  in ena_infos_get()
2034 	if (adapter->offloads.tso4_supported)  in ena_infos_get()
2037 	if (adapter->offloads.tx_csum_supported)  in ena_infos_get()
2042 	if (adapter->offloads.rx_csum_supported)  in ena_infos_get()
2059 	dev_info->max_rx_pktlen  = adapter->max_mtu;  in ena_infos_get()
2062 	dev_info->max_rx_queues = adapter->max_num_io_queues;  in ena_infos_get()
2063 	dev_info->max_tx_queues = adapter->max_num_io_queues;  in ena_infos_get()
2066 	adapter->tx_supported_offloads = tx_feat;  in ena_infos_get()
2067 	adapter->rx_supported_offloads = rx_feat;  in ena_infos_get()
2069 	dev_info->rx_desc_lim.nb_max = adapter->max_rx_ring_size;  in ena_infos_get()
2072 					adapter->max_rx_sgl_size);  in ena_infos_get()
2074 					adapter->max_rx_sgl_size);  in ena_infos_get()
2076 	dev_info->tx_desc_lim.nb_max = adapter->max_tx_ring_size;  in ena_infos_get()
2079 					adapter->max_tx_sgl_size);  in ena_infos_get()
2081 					adapter->max_tx_sgl_size);  in ena_infos_get()
2208 	if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) {  in eth_ena_recv_pkts()
2229 			rx_ring->adapter->reset_reason =  in eth_ena_recv_pkts()
2231 			rx_ring->adapter->trigger_reset = true;  in eth_ena_recv_pkts()
2256 			rte_atomic64_inc(&rx_ring->adapter->drv_stats->ierrors);  in eth_ena_recv_pkts()
2318 			if (!tx_ring->adapter->offloads.tso4_supported)  in eth_ena_prep_pkts()
2353 static void ena_update_hints(struct ena_adapter *adapter,  in ena_update_hints()  argument
2357 		adapter->ena_dev.admin_queue.completion_timeout =  in ena_update_hints()
2362 		adapter->ena_dev.mmio_read.reg_read_to =  in ena_update_hints()
2367 			adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;  in ena_update_hints()
2370 			adapter->keep_alive_timeout =  in ena_update_hints()
2382 	ena_dev = &tx_ring->adapter->ena_dev;  in ena_check_and_linearize_mbuf()
2398 		rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors);  in ena_check_and_linearize_mbuf()
2590 	if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) {  in eth_ena_xmit_pkts()
2626 int ena_copy_eni_stats(struct ena_adapter *adapter)  in ena_copy_eni_stats()  argument
2631 	rte_spinlock_lock(&adapter->admin_lock);  in ena_copy_eni_stats()
2632 	rc = ena_com_get_eni_stats(&adapter->ena_dev, &admin_eni_stats);  in ena_copy_eni_stats()
2633 	rte_spinlock_unlock(&adapter->admin_lock);  in ena_copy_eni_stats()
2645 	rte_memcpy(&adapter->eni_stats, &admin_eni_stats,  in ena_copy_eni_stats()
2716 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_xstats_get()  local
2730 		stats_begin = &adapter->dev_stats;  in ena_xstats_get()
2740 	ena_copy_eni_stats(adapter);  in ena_xstats_get()
2743 		stats_begin = &adapter->eni_stats;  in ena_xstats_get()
2753 			stats_begin = &adapter->rx_ring[i].rx_stats;  in ena_xstats_get()
2764 			stats_begin = &adapter->tx_ring[i].rx_stats;  in ena_xstats_get()
2780 	struct ena_adapter *adapter = dev->data->dev_private;  in ena_xstats_get_by_id()  local
2792 			values[i] = *((uint64_t *)&adapter->dev_stats + id);  in ena_xstats_get_by_id()
2806 				ena_copy_eni_stats(adapter);  in ena_xstats_get_by_id()
2808 			values[i] = *((uint64_t *)&adapter->eni_stats + id);  in ena_xstats_get_by_id()
2820 				&adapter->rx_ring[qid].rx_stats + id);  in ena_xstats_get_by_id()
2831 				&adapter->tx_ring[qid].tx_stats + id);  in ena_xstats_get_by_id()
2844 	struct ena_adapter *adapter = opaque;  in ena_process_bool_devarg()  local
2861 		adapter->use_large_llq_hdr = bool_value;  in ena_process_bool_devarg()
2866 static int ena_parse_devargs(struct ena_adapter *adapter,  in ena_parse_devargs()  argument
2886 		ena_process_bool_devarg, adapter);  in ena_parse_devargs()
2942 	struct ena_adapter *adapter;  in ena_update_on_link_change()  local
2946 	adapter = adapter_data;  in ena_update_on_link_change()
2948 	eth_dev = adapter->rte_dev;  in ena_update_on_link_change()
2951 	adapter->link_status = status;  in ena_update_on_link_change()
2960 	struct ena_adapter *adapter = data;  in ena_notification()  local
2972 		ena_update_hints(adapter, hints);  in ena_notification()
2983 	struct ena_adapter *adapter = adapter_data;  in ena_keep_alive()  local
2988 	adapter->timestamp_wd = rte_get_timer_cycles();  in ena_keep_alive()
2994 	adapter->drv_stats->rx_drops = rx_drops;  in ena_keep_alive()
2995 	adapter->dev_stats.tx_drops = tx_drops;  in ena_keep_alive()