Lines Matching refs:adapter
204 struct ena_adapter* adapter = device_get_softc(dmadev); in ena_dma_alloc() local
205 device_t pdev = adapter->pdev; in ena_dma_alloc()
212 dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width); in ena_dma_alloc()
274 ena_free_pci_resources(struct ena_adapter *adapter) in ena_free_pci_resources() argument
276 device_t pdev = adapter->pdev; in ena_free_pci_resources()
278 if (adapter->memory != NULL) { in ena_free_pci_resources()
280 PCIR_BAR(ENA_MEM_BAR), adapter->memory); in ena_free_pci_resources()
283 if (adapter->registers != NULL) { in ena_free_pci_resources()
285 PCIR_BAR(ENA_REG_BAR), adapter->registers); in ena_free_pci_resources()
288 if (adapter->msix != NULL) { in ena_free_pci_resources()
290 adapter->msix_rid, adapter->msix); in ena_free_pci_resources()
327 struct ena_adapter *adapter = if_getsoftc(ifp); in ena_change_mtu() local
328 device_t pdev = adapter->pdev; in ena_change_mtu()
331 if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) { in ena_change_mtu()
334 new_mtu, adapter->max_mtu, ENA_MIN_MTU); in ena_change_mtu()
338 rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); in ena_change_mtu()
377 ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring, in ena_init_io_rings_common() argument
382 ring->adapter = adapter; in ena_init_io_rings_common()
383 ring->ena_dev = adapter->ena_dev; in ena_init_io_rings_common()
389 ena_init_io_rings_basic(struct ena_adapter *adapter) in ena_init_io_rings_basic() argument
396 ena_dev = adapter->ena_dev; in ena_init_io_rings_basic()
398 for (i = 0; i < adapter->num_io_queues; i++) { in ena_init_io_rings_basic()
399 txr = &adapter->tx_ring[i]; in ena_init_io_rings_basic()
400 rxr = &adapter->rx_ring[i]; in ena_init_io_rings_basic()
403 ena_init_io_rings_common(adapter, txr, i); in ena_init_io_rings_basic()
404 ena_init_io_rings_common(adapter, rxr, i); in ena_init_io_rings_basic()
410 que = &adapter->que[i]; in ena_init_io_rings_basic()
411 que->adapter = adapter; in ena_init_io_rings_basic()
425 ena_init_io_rings_advanced(struct ena_adapter *adapter) in ena_init_io_rings_advanced() argument
430 for (i = 0; i < adapter->num_io_queues; i++) { in ena_init_io_rings_advanced()
431 txr = &adapter->tx_ring[i]; in ena_init_io_rings_advanced()
432 rxr = &adapter->rx_ring[i]; in ena_init_io_rings_advanced()
435 txr->buf_ring_size = adapter->buf_ring_size; in ena_init_io_rings_advanced()
449 device_get_nameunit(adapter->pdev), i); in ena_init_io_rings_advanced()
451 device_get_nameunit(adapter->pdev), i); in ena_init_io_rings_advanced()
458 ena_init_io_rings(struct ena_adapter *adapter) in ena_init_io_rings() argument
466 ena_init_io_rings_basic(adapter); in ena_init_io_rings()
467 ena_init_io_rings_advanced(adapter); in ena_init_io_rings()
471 ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid) in ena_free_io_ring_resources() argument
473 struct ena_ring *txr = &adapter->tx_ring[qid]; in ena_free_io_ring_resources()
474 struct ena_ring *rxr = &adapter->rx_ring[qid]; in ena_free_io_ring_resources()
489 ena_free_all_io_rings_resources(struct ena_adapter *adapter) in ena_free_all_io_rings_resources() argument
493 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_io_rings_resources()
494 ena_free_io_ring_resources(adapter, i); in ena_free_all_io_rings_resources()
499 ena_setup_tx_dma_tag(struct ena_adapter *adapter) in ena_setup_tx_dma_tag() argument
504 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), in ena_setup_tx_dma_tag()
506 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ in ena_setup_tx_dma_tag()
510 adapter->max_tx_sgl_size - 1, /* nsegments */ in ena_setup_tx_dma_tag()
515 &adapter->tx_buf_tag); in ena_setup_tx_dma_tag()
521 ena_free_tx_dma_tag(struct ena_adapter *adapter) in ena_free_tx_dma_tag() argument
525 ret = bus_dma_tag_destroy(adapter->tx_buf_tag); in ena_free_tx_dma_tag()
528 adapter->tx_buf_tag = NULL; in ena_free_tx_dma_tag()
534 ena_setup_rx_dma_tag(struct ena_adapter *adapter) in ena_setup_rx_dma_tag() argument
539 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */ in ena_setup_rx_dma_tag()
541 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ in ena_setup_rx_dma_tag()
545 adapter->max_rx_sgl_size, /* nsegments */ in ena_setup_rx_dma_tag()
550 &adapter->rx_buf_tag); in ena_setup_rx_dma_tag()
556 ena_free_rx_dma_tag(struct ena_adapter *adapter) in ena_free_rx_dma_tag() argument
560 ret = bus_dma_tag_destroy(adapter->rx_buf_tag); in ena_free_rx_dma_tag()
563 adapter->rx_buf_tag = NULL; in ena_free_rx_dma_tag()
571 struct ena_adapter *adapter = tx_ring->adapter; in ena_release_all_tx_dmamap() local
573 bus_dma_tag_t tx_tag = adapter->tx_buf_tag;; in ena_release_all_tx_dmamap()
583 if (adapter->ifp->if_capenable & IFCAP_NETMAP) { in ena_release_all_tx_dmamap()
609 ena_setup_tx_resources(struct ena_adapter *adapter, int qid) in ena_setup_tx_resources() argument
611 device_t pdev = adapter->pdev; in ena_setup_tx_resources()
613 struct ena_que *que = &adapter->que[qid]; in ena_setup_tx_resources()
621 ena_netmap_reset_tx_ring(adapter, qid); in ena_setup_tx_resources()
655 drbr_flush(adapter->ifp, tx_ring->br); in ena_setup_tx_resources()
660 err = bus_dmamap_create(adapter->tx_buf_tag, 0, in ena_setup_tx_resources()
670 if (adapter->ifp->if_capenable & IFCAP_NETMAP) { in ena_setup_tx_resources()
673 err = bus_dmamap_create(adapter->tx_buf_tag, 0, in ena_setup_tx_resources()
702 device_get_nameunit(adapter->pdev), que->cpu); in ena_setup_tx_resources()
705 device_get_nameunit(adapter->pdev), que->id); in ena_setup_tx_resources()
732 ena_free_tx_resources(struct ena_adapter *adapter, int qid) in ena_free_tx_resources() argument
734 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; in ena_free_tx_resources()
748 drbr_flush(adapter->ifp, tx_ring->br); in ena_free_tx_resources()
752 bus_dmamap_sync(adapter->tx_buf_tag, in ena_free_tx_resources()
754 bus_dmamap_unload(adapter->tx_buf_tag, in ena_free_tx_resources()
756 bus_dmamap_destroy(adapter->tx_buf_tag, in ena_free_tx_resources()
760 if (adapter->ifp->if_capenable & IFCAP_NETMAP) { in ena_free_tx_resources()
764 bus_dmamap_sync(adapter->tx_buf_tag, in ena_free_tx_resources()
767 ena_netmap_unload(adapter, in ena_free_tx_resources()
770 bus_dmamap_destroy(adapter->tx_buf_tag, in ena_free_tx_resources()
800 ena_setup_all_tx_resources(struct ena_adapter *adapter) in ena_setup_all_tx_resources() argument
804 for (i = 0; i < adapter->num_io_queues; i++) { in ena_setup_all_tx_resources()
805 rc = ena_setup_tx_resources(adapter, i); in ena_setup_all_tx_resources()
807 ena_log(adapter->pdev, ERR, in ena_setup_all_tx_resources()
818 ena_free_tx_resources(adapter, i); in ena_setup_all_tx_resources()
829 ena_free_all_tx_resources(struct ena_adapter *adapter) in ena_free_all_tx_resources() argument
833 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_tx_resources()
834 ena_free_tx_resources(adapter, i); in ena_free_all_tx_resources()
845 ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid) in ena_setup_rx_resources() argument
847 device_t pdev = adapter->pdev; in ena_setup_rx_resources()
848 struct ena_que *que = &adapter->que[qid]; in ena_setup_rx_resources()
855 ena_netmap_reset_rx_ring(adapter, qid); in ena_setup_rx_resources()
882 err = bus_dmamap_create(adapter->rx_buf_tag, 0, in ena_setup_rx_resources()
892 if ((adapter->ifp->if_capenable & IFCAP_LRO) != 0) { in ena_setup_rx_resources()
900 rx_ring->lro.ifp = adapter->ifp; in ena_setup_rx_resources()
908 bus_dmamap_destroy(adapter->rx_buf_tag, in ena_setup_rx_resources()
927 ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid) in ena_free_rx_resources() argument
929 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_free_rx_resources()
933 bus_dmamap_sync(adapter->rx_buf_tag, in ena_free_rx_resources()
937 bus_dmamap_unload(adapter->rx_buf_tag, in ena_free_rx_resources()
939 bus_dmamap_destroy(adapter->rx_buf_tag, in ena_free_rx_resources()
961 ena_setup_all_rx_resources(struct ena_adapter *adapter) in ena_setup_all_rx_resources() argument
965 for (i = 0; i < adapter->num_io_queues; i++) { in ena_setup_all_rx_resources()
966 rc = ena_setup_rx_resources(adapter, i); in ena_setup_all_rx_resources()
968 ena_log(adapter->pdev, ERR, in ena_setup_all_rx_resources()
978 ena_free_rx_resources(adapter, i); in ena_setup_all_rx_resources()
989 ena_free_all_rx_resources(struct ena_adapter *adapter) in ena_free_all_rx_resources() argument
993 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_rx_resources()
994 ena_free_rx_resources(adapter, i); in ena_free_all_rx_resources()
998 ena_alloc_rx_mbuf(struct ena_adapter *adapter, in ena_alloc_rx_mbuf() argument
1001 device_t pdev = adapter->pdev; in ena_alloc_rx_mbuf()
1031 adapter->rx_buf_tag,rx_info->mbuf, rx_info->mbuf->m_len); in ena_alloc_rx_mbuf()
1032 error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map, in ena_alloc_rx_mbuf()
1042 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD); in ena_alloc_rx_mbuf()
1060 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring, in ena_free_rx_mbuf() argument
1065 ena_log(adapter->pdev, WARN, in ena_free_rx_mbuf()
1070 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, in ena_free_rx_mbuf()
1072 bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map); in ena_free_rx_mbuf()
1086 struct ena_adapter *adapter = rx_ring->adapter; in ena_refill_rx_bufs() local
1087 device_t pdev = adapter->pdev; in ena_refill_rx_bufs()
1092 ena_log_io(adapter->pdev, DBG, "refill qid: %d\n", rx_ring->qid); in ena_refill_rx_bufs()
1105 if (ena_rx_ring_in_netmap(adapter, rx_ring->qid)) in ena_refill_rx_bufs()
1106 rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, rx_info); in ena_refill_rx_bufs()
1109 rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info); in ena_refill_rx_bufs()
1143 ena_update_buf_ring_size(struct ena_adapter *adapter, in ena_update_buf_ring_size() argument
1150 old_buf_ring_size = adapter->buf_ring_size; in ena_update_buf_ring_size()
1151 adapter->buf_ring_size = new_buf_ring_size; in ena_update_buf_ring_size()
1153 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); in ena_update_buf_ring_size()
1154 ena_down(adapter); in ena_update_buf_ring_size()
1157 ena_free_all_io_rings_resources(adapter); in ena_update_buf_ring_size()
1158 ena_init_io_rings_advanced(adapter); in ena_update_buf_ring_size()
1166 rc = ena_up(adapter); in ena_update_buf_ring_size()
1168 ena_log(adapter->pdev, ERR, in ena_update_buf_ring_size()
1173 adapter->buf_ring_size = old_buf_ring_size; in ena_update_buf_ring_size()
1174 ena_free_all_io_rings_resources(adapter); in ena_update_buf_ring_size()
1175 ena_init_io_rings_advanced(adapter); in ena_update_buf_ring_size()
1178 adapter); in ena_update_buf_ring_size()
1179 ena_trigger_reset(adapter, ENA_REGS_RESET_OS_TRIGGER); in ena_update_buf_ring_size()
1188 ena_update_queue_size(struct ena_adapter *adapter, uint32_t new_tx_size, in ena_update_queue_size() argument
1195 old_tx_size = adapter->requested_tx_ring_size; in ena_update_queue_size()
1196 old_rx_size = adapter->requested_rx_ring_size; in ena_update_queue_size()
1197 adapter->requested_tx_ring_size = new_tx_size; in ena_update_queue_size()
1198 adapter->requested_rx_ring_size = new_rx_size; in ena_update_queue_size()
1200 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); in ena_update_queue_size()
1201 ena_down(adapter); in ena_update_queue_size()
1204 ena_init_io_rings_basic(adapter); in ena_update_queue_size()
1206 rc = ena_up(adapter); in ena_update_queue_size()
1208 ena_log(adapter->pdev, ERR, in ena_update_queue_size()
1213 adapter->requested_tx_ring_size = old_tx_size; in ena_update_queue_size()
1214 adapter->requested_rx_ring_size = old_rx_size; in ena_update_queue_size()
1215 ena_init_io_rings_basic(adapter); in ena_update_queue_size()
1218 rc = ena_up(adapter); in ena_update_queue_size()
1220 ena_log(adapter->pdev, ERR, in ena_update_queue_size()
1228 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); in ena_update_queue_size()
1229 ena_trigger_reset(adapter, in ena_update_queue_size()
1239 ena_update_io_rings(struct ena_adapter *adapter, uint32_t num) in ena_update_io_rings() argument
1241 ena_free_all_io_rings_resources(adapter); in ena_update_io_rings()
1243 ena_com_rss_destroy(adapter->ena_dev); in ena_update_io_rings()
1245 adapter->num_io_queues = num; in ena_update_io_rings()
1246 ena_init_io_rings(adapter); in ena_update_io_rings()
1251 ena_update_io_queue_nb(struct ena_adapter *adapter, uint32_t new_num) in ena_update_io_queue_nb() argument
1257 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); in ena_update_io_queue_nb()
1258 old_num = adapter->num_io_queues; in ena_update_io_queue_nb()
1259 ena_down(adapter); in ena_update_io_queue_nb()
1261 ena_update_io_rings(adapter, new_num); in ena_update_io_queue_nb()
1264 rc = ena_up(adapter); in ena_update_io_queue_nb()
1266 ena_log(adapter->pdev, ERR, in ena_update_io_queue_nb()
1271 ena_update_io_rings(adapter, old_num); in ena_update_io_queue_nb()
1273 rc = ena_up(adapter); in ena_update_io_queue_nb()
1275 ena_log(adapter->pdev, ERR, in ena_update_io_queue_nb()
1279 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); in ena_update_io_queue_nb()
1280 ena_trigger_reset(adapter, in ena_update_io_queue_nb()
1290 ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid) in ena_free_rx_bufs() argument
1292 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_free_rx_bufs()
1299 ena_free_rx_mbuf(adapter, rx_ring, rx_info); in ena_free_rx_bufs()
1301 if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) && in ena_free_rx_bufs()
1302 (adapter->ifp->if_capenable & IFCAP_NETMAP)) { in ena_free_rx_bufs()
1304 ena_netmap_free_rx_slot(adapter, rx_ring, in ena_free_rx_bufs()
1317 ena_refill_all_rx_bufs(struct ena_adapter *adapter) in ena_refill_all_rx_bufs() argument
1322 for (i = 0; i < adapter->num_io_queues; i++) { in ena_refill_all_rx_bufs()
1323 rx_ring = &adapter->rx_ring[i]; in ena_refill_all_rx_bufs()
1327 ena_log_io(adapter->pdev, WARN, in ena_refill_all_rx_bufs()
1337 ena_free_all_rx_bufs(struct ena_adapter *adapter) in ena_free_all_rx_bufs() argument
1341 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_rx_bufs()
1342 ena_free_rx_bufs(adapter, i); in ena_free_all_rx_bufs()
1351 ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid) in ena_free_tx_bufs() argument
1354 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; in ena_free_tx_bufs()
1364 ena_log(adapter->pdev, WARN, in ena_free_tx_bufs()
1369 ena_log(adapter->pdev, DBG, in ena_free_tx_bufs()
1374 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, in ena_free_tx_bufs()
1376 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); in ena_free_tx_bufs()
1385 ena_free_all_tx_bufs(struct ena_adapter *adapter) in ena_free_all_tx_bufs() argument
1388 for (int i = 0; i < adapter->num_io_queues; i++) in ena_free_all_tx_bufs()
1389 ena_free_tx_bufs(adapter, i); in ena_free_all_tx_bufs()
1393 ena_destroy_all_tx_queues(struct ena_adapter *adapter) in ena_destroy_all_tx_queues() argument
1398 for (i = 0; i < adapter->num_io_queues; i++) { in ena_destroy_all_tx_queues()
1400 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); in ena_destroy_all_tx_queues()
1405 ena_destroy_all_rx_queues(struct ena_adapter *adapter) in ena_destroy_all_rx_queues() argument
1410 for (i = 0; i < adapter->num_io_queues; i++) { in ena_destroy_all_rx_queues()
1412 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); in ena_destroy_all_rx_queues()
1417 ena_destroy_all_io_queues(struct ena_adapter *adapter) in ena_destroy_all_io_queues() argument
1422 for (i = 0; i < adapter->num_io_queues; i++) { in ena_destroy_all_io_queues()
1423 queue = &adapter->que[i]; in ena_destroy_all_io_queues()
1431 ena_destroy_all_tx_queues(adapter); in ena_destroy_all_io_queues()
1432 ena_destroy_all_rx_queues(adapter); in ena_destroy_all_io_queues()
1436 ena_create_io_queues(struct ena_adapter *adapter) in ena_create_io_queues() argument
1438 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_create_io_queues()
1448 for (i = 0; i < adapter->num_io_queues; i++) { in ena_create_io_queues()
1453 ctx.queue_size = adapter->requested_tx_ring_size; in ena_create_io_queues()
1456 ctx.numa_node = adapter->que[i].domain; in ena_create_io_queues()
1460 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1464 ring = &adapter->tx_ring[i]; in ena_create_io_queues()
1469 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1483 for (i = 0; i < adapter->num_io_queues; i++) { in ena_create_io_queues()
1488 ctx.queue_size = adapter->requested_rx_ring_size; in ena_create_io_queues()
1491 ctx.numa_node = adapter->que[i].domain; in ena_create_io_queues()
1495 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1500 ring = &adapter->rx_ring[i]; in ena_create_io_queues()
1505 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1518 for (i = 0; i < adapter->num_io_queues; i++) { in ena_create_io_queues()
1519 queue = &adapter->que[i]; in ena_create_io_queues()
1531 device_get_nameunit(adapter->pdev), i); in ena_create_io_queues()
1539 i = adapter->num_io_queues; in ena_create_io_queues()
1560 struct ena_adapter *adapter = (struct ena_adapter *)arg; in ena_intr_msix_mgmnt() local
1562 ena_com_admin_q_comp_intr_handler(adapter->ena_dev); in ena_intr_msix_mgmnt()
1563 if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))) in ena_intr_msix_mgmnt()
1564 ena_com_aenq_intr_handler(adapter->ena_dev, arg); in ena_intr_msix_mgmnt()
1575 struct ena_adapter *adapter = queue->adapter; in ena_handle_msix() local
1576 if_t ifp = adapter->ifp; in ena_handle_msix()
1587 ena_enable_msix(struct ena_adapter *adapter) in ena_enable_msix() argument
1589 device_t dev = adapter->pdev; in ena_enable_msix()
1593 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { in ena_enable_msix()
1599 msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues); in ena_enable_msix()
1601 adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry), in ena_enable_msix()
1608 adapter->msix_entries[i].entry = i; in ena_enable_msix()
1610 adapter->msix_entries[i].vector = i + 1; in ena_enable_msix()
1636 adapter->msix_vecs = msix_vecs; in ena_enable_msix()
1637 ENA_FLAG_SET_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); in ena_enable_msix()
1642 free(adapter->msix_entries, M_DEVBUF); in ena_enable_msix()
1643 adapter->msix_entries = NULL; in ena_enable_msix()
1649 ena_setup_mgmnt_intr(struct ena_adapter *adapter) in ena_setup_mgmnt_intr() argument
1652 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, in ena_setup_mgmnt_intr()
1654 device_get_nameunit(adapter->pdev)); in ena_setup_mgmnt_intr()
1659 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL; in ena_setup_mgmnt_intr()
1660 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter; in ena_setup_mgmnt_intr()
1661 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector = in ena_setup_mgmnt_intr()
1662 adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector; in ena_setup_mgmnt_intr()
1666 ena_setup_io_intr(struct ena_adapter *adapter) in ena_setup_io_intr() argument
1676 if (adapter->msix_entries == NULL) in ena_setup_io_intr()
1680 if (adapter->first_bind < 0) { in ena_setup_io_intr()
1681 adapter->first_bind = last_bind; in ena_setup_io_intr()
1682 last_bind = (last_bind + adapter->num_io_queues) % num_buckets; in ena_setup_io_intr()
1684 cur_bind = adapter->first_bind; in ena_setup_io_intr()
1687 for (int i = 0; i < adapter->num_io_queues; i++) { in ena_setup_io_intr()
1690 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE, in ena_setup_io_intr()
1691 "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i); in ena_setup_io_intr()
1692 adapter->irq_tbl[irq_idx].handler = ena_handle_msix; in ena_setup_io_intr()
1693 adapter->irq_tbl[irq_idx].data = &adapter->que[i]; in ena_setup_io_intr()
1694 adapter->irq_tbl[irq_idx].vector = in ena_setup_io_intr()
1695 adapter->msix_entries[irq_idx].vector; in ena_setup_io_intr()
1696 ena_log(adapter->pdev, DBG, "ena_setup_io_intr vector: %d\n", in ena_setup_io_intr()
1697 adapter->msix_entries[irq_idx].vector); in ena_setup_io_intr()
1700 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = in ena_setup_io_intr()
1703 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask); in ena_setup_io_intr()
1706 if (CPU_ISSET(adapter->que[i].cpu, &cpuset_domain[idx])) in ena_setup_io_intr()
1709 adapter->que[i].domain = idx; in ena_setup_io_intr()
1711 adapter->que[i].domain = -1; in ena_setup_io_intr()
1719 ena_request_mgmnt_irq(struct ena_adapter *adapter) in ena_request_mgmnt_irq() argument
1721 device_t pdev = adapter->pdev; in ena_request_mgmnt_irq()
1728 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; in ena_request_mgmnt_irq()
1729 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, in ena_request_mgmnt_irq()
1738 rc = bus_setup_intr(adapter->pdev, irq->res, in ena_request_mgmnt_irq()
1753 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, in ena_request_mgmnt_irq()
1764 ena_request_io_irq(struct ena_adapter *adapter) in ena_request_io_irq() argument
1766 device_t pdev = adapter->pdev; in ena_request_io_irq()
1771 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter))) { in ena_request_io_irq()
1779 for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { in ena_request_io_irq()
1780 irq = &adapter->irq_tbl[i]; in ena_request_io_irq()
1785 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, in ena_request_io_irq()
1794 rc = bus_setup_intr(adapter->pdev, irq->res, in ena_request_io_irq()
1806 rc = bus_bind_intr(adapter->pdev, irq->res, irq->cpu); in ena_request_io_irq()
1824 irq = &adapter->irq_tbl[i]; in ena_request_io_irq()
1830 rcc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); in ena_request_io_irq()
1841 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, in ena_request_io_irq()
1855 ena_free_mgmnt_irq(struct ena_adapter *adapter) in ena_free_mgmnt_irq() argument
1857 device_t pdev = adapter->pdev; in ena_free_mgmnt_irq()
1861 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; in ena_free_mgmnt_irq()
1864 rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); in ena_free_mgmnt_irq()
1873 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, in ena_free_mgmnt_irq()
1883 ena_free_io_irq(struct ena_adapter *adapter) in ena_free_io_irq() argument
1885 device_t pdev = adapter->pdev; in ena_free_io_irq()
1889 for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { in ena_free_io_irq()
1890 irq = &adapter->irq_tbl[i]; in ena_free_io_irq()
1893 rc = bus_teardown_intr(adapter->pdev, irq->res, in ena_free_io_irq()
1905 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, in ena_free_io_irq()
1918 ena_free_irqs(struct ena_adapter* adapter) in ena_free_irqs() argument
1921 ena_free_io_irq(adapter); in ena_free_irqs()
1922 ena_free_mgmnt_irq(adapter); in ena_free_irqs()
1923 ena_disable_msix(adapter); in ena_free_irqs()
1927 ena_disable_msix(struct ena_adapter *adapter) in ena_disable_msix() argument
1930 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { in ena_disable_msix()
1931 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); in ena_disable_msix()
1932 pci_release_msi(adapter->pdev); in ena_disable_msix()
1935 adapter->msix_vecs = 0; in ena_disable_msix()
1936 free(adapter->msix_entries, M_DEVBUF); in ena_disable_msix()
1937 adapter->msix_entries = NULL; in ena_disable_msix()
1941 ena_unmask_all_io_irqs(struct ena_adapter *adapter) in ena_unmask_all_io_irqs() argument
1950 for (i = 0; i < adapter->num_io_queues; i++) { in ena_unmask_all_io_irqs()
1952 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; in ena_unmask_all_io_irqs()
1954 tx_ring = &adapter->tx_ring[i]; in ena_unmask_all_io_irqs()
1961 ena_up_complete(struct ena_adapter *adapter) in ena_up_complete() argument
1965 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) { in ena_up_complete()
1966 rc = ena_rss_configure(adapter); in ena_up_complete()
1968 ena_log(adapter->pdev, ERR, in ena_up_complete()
1974 rc = ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu); in ena_up_complete()
1978 ena_refill_all_rx_bufs(adapter); in ena_up_complete()
1979 ena_reset_counters((counter_u64_t *)&adapter->hw_stats, in ena_up_complete()
1980 sizeof(adapter->hw_stats)); in ena_up_complete()
1986 set_io_rings_size(struct ena_adapter *adapter, int new_tx_size, in set_io_rings_size() argument
1991 for (i = 0; i < adapter->num_io_queues; i++) { in set_io_rings_size()
1992 adapter->tx_ring[i].ring_size = new_tx_size; in set_io_rings_size()
1993 adapter->rx_ring[i].ring_size = new_rx_size; in set_io_rings_size()
1998 create_queues_with_size_backoff(struct ena_adapter *adapter) in create_queues_with_size_backoff() argument
2000 device_t pdev = adapter->pdev; in create_queues_with_size_backoff()
2009 set_io_rings_size(adapter, adapter->requested_tx_ring_size, in create_queues_with_size_backoff()
2010 adapter->requested_rx_ring_size); in create_queues_with_size_backoff()
2014 rc = ena_setup_all_tx_resources(adapter); in create_queues_with_size_backoff()
2021 rc = ena_setup_all_rx_resources(adapter); in create_queues_with_size_backoff()
2028 rc = ena_create_io_queues(adapter); in create_queues_with_size_backoff()
2038 ena_free_all_rx_resources(adapter); in create_queues_with_size_backoff()
2040 ena_free_all_tx_resources(adapter); in create_queues_with_size_backoff()
2052 cur_tx_ring_size = adapter->tx_ring[0].ring_size; in create_queues_with_size_backoff()
2053 cur_rx_ring_size = adapter->rx_ring[0].ring_size; in create_queues_with_size_backoff()
2084 set_io_rings_size(adapter, new_tx_ring_size, new_rx_ring_size); in create_queues_with_size_backoff()
2089 ena_up(struct ena_adapter *adapter) in ena_up() argument
2095 if (unlikely(device_is_attached(adapter->pdev) == 0)) { in ena_up()
2096 ena_log(adapter->pdev, ERR, "device is not attached!\n"); in ena_up()
2100 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) in ena_up()
2103 ena_log(adapter->pdev, INFO, "device is going UP\n"); in ena_up()
2110 ENA_TIMER_DRAIN(adapter); in ena_up()
2113 rc = ena_setup_io_intr(adapter); in ena_up()
2115 ena_log(adapter->pdev, ERR, "error setting up IO interrupt\n"); in ena_up()
2118 rc = ena_request_io_irq(adapter); in ena_up()
2120 ena_log(adapter->pdev, ERR, "err_req_irq\n"); in ena_up()
2124 ena_log(adapter->pdev, INFO, in ena_up()
2127 adapter->num_io_queues, in ena_up()
2128 adapter->requested_rx_ring_size, in ena_up()
2129 adapter->requested_tx_ring_size, in ena_up()
2130 (adapter->ena_dev->tx_mem_queue_type == in ena_up()
2133 rc = create_queues_with_size_backoff(adapter); in ena_up()
2135 ena_log(adapter->pdev, ERR, in ena_up()
2140 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) in ena_up()
2141 if_link_state_change(adapter->ifp, LINK_STATE_UP); in ena_up()
2143 rc = ena_up_complete(adapter); in ena_up()
2147 counter_u64_add(adapter->dev_stats.interface_up, 1); in ena_up()
2149 ena_update_hwassist(adapter); in ena_up()
2151 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, in ena_up()
2154 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter); in ena_up()
2156 ena_unmask_all_io_irqs(adapter); in ena_up()
2158 ENA_TIMER_RESET(adapter); in ena_up()
2163 ena_destroy_all_io_queues(adapter); in ena_up()
2164 ena_free_all_rx_resources(adapter); in ena_up()
2165 ena_free_all_tx_resources(adapter); in ena_up()
2167 ena_free_io_irq(adapter); in ena_up()
2169 ENA_TIMER_RESET(adapter); in ena_up()
2177 struct ena_adapter *adapter; in ena_get_counter() local
2180 adapter = if_getsoftc(ifp); in ena_get_counter()
2181 stats = &adapter->hw_stats; in ena_get_counter()
2211 struct ena_adapter *adapter = if_getsoftc(ifp); in ena_media_status() local
2212 ena_log(adapter->pdev, DBG, "Media status update\n"); in ena_media_status()
2219 if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) { in ena_media_status()
2221 ena_log(adapter->pdev, INFO, "Link is down\n"); in ena_media_status()
2234 struct ena_adapter *adapter = (struct ena_adapter *)arg; in ena_init() local
2236 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { in ena_init()
2238 ena_up(adapter); in ena_init()
2246 struct ena_adapter *adapter; in ena_ioctl() local
2250 adapter = ifp->if_softc; in ena_ioctl()
2262 ena_down(adapter); in ena_ioctl()
2266 rc = ena_up(adapter); in ena_ioctl()
2275 ena_log(adapter->pdev, INFO, in ena_ioctl()
2280 rc = ena_up(adapter); in ena_ioctl()
2286 ena_down(adapter); in ena_ioctl()
2298 rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command); in ena_ioctl()
2313 ena_down(adapter); in ena_ioctl()
2314 rc = ena_up(adapter); in ena_ioctl()
2375 ena_update_hwassist(struct ena_adapter *adapter) in ena_update_hwassist() argument
2377 if_t ifp = adapter->ifp; in ena_update_hwassist()
2378 uint32_t feat = adapter->tx_offload_cap; in ena_update_hwassist()
2407 ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter, in ena_setup_ifnet() argument
2413 ifp = adapter->ifp = if_gethandle(IFT_ETHER); in ena_setup_ifnet()
2420 if_setsoftc(ifp, adapter); in ena_setup_ifnet()
2430 if_setsendqlen(ifp, adapter->requested_tx_ring_size); in ena_setup_ifnet()
2445 ifp->if_hw_tsomaxsegcount = adapter->max_tx_sgl_size - 1; in ena_setup_ifnet()
2455 ifmedia_init(&adapter->media, IFM_IMASK, in ena_setup_ifnet()
2457 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); in ena_setup_ifnet()
2458 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); in ena_setup_ifnet()
2460 ether_ifattach(ifp, adapter->mac_addr); in ena_setup_ifnet()
2466 ena_down(struct ena_adapter *adapter) in ena_down() argument
2472 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) in ena_down()
2476 ENA_TIMER_DRAIN(adapter); in ena_down()
2478 ena_log(adapter->pdev, INFO, "device is going DOWN\n"); in ena_down()
2480 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter); in ena_down()
2481 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, in ena_down()
2484 ena_free_io_irq(adapter); in ena_down()
2486 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) { in ena_down()
2487 rc = ena_com_dev_reset(adapter->ena_dev, in ena_down()
2488 adapter->reset_reason); in ena_down()
2490 ena_log(adapter->pdev, ERR, in ena_down()
2494 ena_destroy_all_io_queues(adapter); in ena_down()
2496 ena_free_all_tx_bufs(adapter); in ena_down()
2497 ena_free_all_rx_bufs(adapter); in ena_down()
2498 ena_free_all_tx_resources(adapter); in ena_down()
2499 ena_free_all_rx_resources(adapter); in ena_down()
2501 counter_u64_add(adapter->dev_stats.interface_down, 1); in ena_down()
2503 ENA_TIMER_RESET(adapter); in ena_down()
2606 struct ena_adapter *adapter = device_get_softc(pdev); in ena_map_llq_mem_bar() local
2611 adapter->memory = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, in ena_map_llq_mem_bar()
2613 if (unlikely(adapter->memory == NULL)) { in ena_map_llq_mem_bar()
2620 rc = ena_enable_wc(adapter->pdev, adapter->memory); in ena_map_llq_mem_bar()
2630 ena_dev->mem_bar = rman_get_virtual(adapter->memory); in ena_map_llq_mem_bar()
2801 ena_device_init(struct ena_adapter *adapter, device_t pdev, in ena_device_init() argument
2805 struct ena_com_dev* ena_dev = adapter->ena_dev; in ena_device_init()
2842 adapter->dma_width = dma_width; in ena_device_init()
2904 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter) in ena_enable_msix_and_set_admin_interrupts() argument
2906 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_enable_msix_and_set_admin_interrupts()
2909 rc = ena_enable_msix(adapter); in ena_enable_msix_and_set_admin_interrupts()
2911 ena_log(adapter->pdev, ERR, "Error with MSI-X enablement\n"); in ena_enable_msix_and_set_admin_interrupts()
2915 ena_setup_mgmnt_intr(adapter); in ena_enable_msix_and_set_admin_interrupts()
2917 rc = ena_request_mgmnt_irq(adapter); in ena_enable_msix_and_set_admin_interrupts()
2919 ena_log(adapter->pdev, ERR, "Cannot setup mgmnt queue intr\n"); in ena_enable_msix_and_set_admin_interrupts()
2930 ena_disable_msix(adapter); in ena_enable_msix_and_set_admin_interrupts()
2939 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_keep_alive_wd() local
2949 counter_u64_zero(adapter->hw_stats.rx_drops); in ena_keep_alive_wd()
2950 counter_u64_add(adapter->hw_stats.rx_drops, rx_drops); in ena_keep_alive_wd()
2951 counter_u64_zero(adapter->hw_stats.tx_drops); in ena_keep_alive_wd()
2952 counter_u64_add(adapter->hw_stats.tx_drops, tx_drops); in ena_keep_alive_wd()
2955 atomic_store_rel_64(&adapter->keep_alive_timestamp, stime); in ena_keep_alive_wd()
2959 static void check_for_missing_keep_alive(struct ena_adapter *adapter) in check_for_missing_keep_alive() argument
2963 if (adapter->wd_active == 0) in check_for_missing_keep_alive()
2966 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) in check_for_missing_keep_alive()
2969 timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp); in check_for_missing_keep_alive()
2971 if (unlikely(time > adapter->keep_alive_timeout)) { in check_for_missing_keep_alive()
2972 ena_log(adapter->pdev, ERR, "Keep alive watchdog timeout.\n"); in check_for_missing_keep_alive()
2973 counter_u64_add(adapter->dev_stats.wd_expired, 1); in check_for_missing_keep_alive()
2974 ena_trigger_reset(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO); in check_for_missing_keep_alive()
2979 static void check_for_admin_com_state(struct ena_adapter *adapter) in check_for_admin_com_state() argument
2981 if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) == in check_for_admin_com_state()
2983 ena_log(adapter->pdev, ERR, in check_for_admin_com_state()
2985 counter_u64_add(adapter->dev_stats.admin_q_pause, 1); in check_for_admin_com_state()
2986 ena_trigger_reset(adapter, ENA_REGS_RESET_ADMIN_TO); in check_for_admin_com_state()
2991 check_for_rx_interrupt_queue(struct ena_adapter *adapter, in check_for_rx_interrupt_queue() argument
3003 ena_log(adapter->pdev, ERR, "Potential MSIX issue on Rx side " in check_for_rx_interrupt_queue()
3005 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_INTERRUPT); in check_for_rx_interrupt_queue()
3013 check_missing_comp_in_tx_queue(struct ena_adapter *adapter, in check_missing_comp_in_tx_queue() argument
3016 device_t pdev = adapter->pdev; in check_missing_comp_in_tx_queue()
3036 time_offset > 2 * adapter->missing_tx_timeout)) { in check_missing_comp_in_tx_queue()
3044 ena_trigger_reset(adapter, in check_missing_comp_in_tx_queue()
3050 if (unlikely(time_offset > adapter->missing_tx_timeout)) { in check_missing_comp_in_tx_queue()
3062 if (unlikely(missed_tx > adapter->missing_tx_threshold)) { in check_missing_comp_in_tx_queue()
3066 missed_tx, adapter->missing_tx_threshold); in check_missing_comp_in_tx_queue()
3067 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_TX_CMPL); in check_missing_comp_in_tx_queue()
3083 check_for_missing_completions(struct ena_adapter *adapter) in check_for_missing_completions() argument
3092 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) in check_for_missing_completions()
3095 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) in check_for_missing_completions()
3098 if (adapter->missing_tx_timeout == ENA_HW_HINTS_NO_TIMEOUT) in check_for_missing_completions()
3101 budget = adapter->missing_tx_max_queues; in check_for_missing_completions()
3103 for (i = adapter->next_monitored_tx_qid; i < adapter->num_io_queues; i++) { in check_for_missing_completions()
3104 tx_ring = &adapter->tx_ring[i]; in check_for_missing_completions()
3105 rx_ring = &adapter->rx_ring[i]; in check_for_missing_completions()
3107 rc = check_missing_comp_in_tx_queue(adapter, tx_ring); in check_for_missing_completions()
3111 rc = check_for_rx_interrupt_queue(adapter, rx_ring); in check_for_missing_completions()
3122 adapter->next_monitored_tx_qid = i % adapter->num_io_queues; in check_for_missing_completions()
3138 check_for_empty_rx_ring(struct ena_adapter *adapter) in check_for_empty_rx_ring() argument
3143 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) in check_for_empty_rx_ring()
3146 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) in check_for_empty_rx_ring()
3149 for (i = 0; i < adapter->num_io_queues; i++) { in check_for_empty_rx_ring()
3150 rx_ring = &adapter->rx_ring[i]; in check_for_empty_rx_ring()
3160 ena_log(adapter->pdev, WARN, in check_for_empty_rx_ring()
3174 static void ena_update_hints(struct ena_adapter *adapter, in ena_update_hints() argument
3177 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_update_hints()
3189 adapter->missing_tx_threshold = in ena_update_hints()
3195 adapter->missing_tx_timeout = ENA_HW_HINTS_NO_TIMEOUT; in ena_update_hints()
3197 adapter->missing_tx_timeout = in ena_update_hints()
3203 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; in ena_update_hints()
3205 adapter->keep_alive_timeout = in ena_update_hints()
3222 ena_copy_eni_metrics(struct ena_adapter *adapter) in ena_copy_eni_metrics() argument
3227 rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_metrics); in ena_copy_eni_metrics()
3232 ena_log(adapter->pdev, WARN, in ena_copy_eni_metrics()
3236 ena_log(adapter->pdev, DBG, in ena_copy_eni_metrics()
3240 ena_log(adapter->pdev, ERR, in ena_copy_eni_metrics()
3251 struct ena_adapter *adapter = (struct ena_adapter *)data; in ena_timer_service() local
3253 adapter->ena_dev->host_attr.host_info; in ena_timer_service()
3255 check_for_missing_keep_alive(adapter); in ena_timer_service()
3257 check_for_admin_com_state(adapter); in ena_timer_service()
3259 check_for_missing_completions(adapter); in ena_timer_service()
3261 check_for_empty_rx_ring(adapter); in ena_timer_service()
3272 if ((adapter->eni_metrics_sample_interval != 0) && in ena_timer_service()
3273 (++adapter->eni_metrics_sample_interval_cnt >= in ena_timer_service()
3274 adapter->eni_metrics_sample_interval)) { in ena_timer_service()
3292 (void)ena_copy_eni_metrics(adapter); in ena_timer_service()
3293 adapter->eni_metrics_sample_interval_cnt = 0; in ena_timer_service()
3298 ena_update_host_info(host_info, adapter->ifp); in ena_timer_service()
3300 if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { in ena_timer_service()
3306 if (ena_com_validate_version(adapter->ena_dev) == in ena_timer_service()
3308 ena_log(adapter->pdev, WARN, in ena_timer_service()
3310 ENA_TIMER_RESET(adapter); in ena_timer_service()
3313 ena_log(adapter->pdev, WARN, "Trigger reset is on\n"); in ena_timer_service()
3314 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task); in ena_timer_service()
3321 ENA_TIMER_RESET(adapter); in ena_timer_service()
3325 ena_destroy_device(struct ena_adapter *adapter, bool graceful) in ena_destroy_device() argument
3327 if_t ifp = adapter->ifp; in ena_destroy_device()
3328 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_destroy_device()
3331 if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) in ena_destroy_device()
3336 ENA_TIMER_DRAIN(adapter); in ena_destroy_device()
3338 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); in ena_destroy_device()
3340 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); in ena_destroy_device()
3345 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) in ena_destroy_device()
3346 ena_down(adapter); in ena_destroy_device()
3352 if (!(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter) && dev_up)) in ena_destroy_device()
3353 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); in ena_destroy_device()
3355 ena_free_mgmnt_irq(adapter); in ena_destroy_device()
3357 ena_disable_msix(adapter); in ena_destroy_device()
3366 ena_free_all_io_rings_resources(adapter); in ena_destroy_device()
3376 adapter->reset_reason = ENA_REGS_RESET_NORMAL; in ena_destroy_device()
3378 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); in ena_destroy_device()
3379 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); in ena_destroy_device()
3383 ena_device_validate_params(struct ena_adapter *adapter, in ena_device_validate_params() argument
3387 if (memcmp(get_feat_ctx->dev_attr.mac_addr, adapter->mac_addr, in ena_device_validate_params()
3389 ena_log(adapter->pdev, ERR, "Error, mac addresses differ\n"); in ena_device_validate_params()
3393 if (get_feat_ctx->dev_attr.max_mtu < if_getmtu(adapter->ifp)) { in ena_device_validate_params()
3394 ena_log(adapter->pdev, ERR, in ena_device_validate_params()
3403 ena_restore_device(struct ena_adapter *adapter) in ena_restore_device() argument
3406 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_restore_device()
3407 if_t ifp = adapter->ifp; in ena_restore_device()
3408 device_t dev = adapter->pdev; in ena_restore_device()
3412 ENA_FLAG_SET_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); in ena_restore_device()
3414 rc = ena_device_init(adapter, dev, &get_feat_ctx, &wd_active); in ena_restore_device()
3423 if (adapter->wd_active != 0) in ena_restore_device()
3424 adapter->wd_active = wd_active; in ena_restore_device()
3426 rc = ena_device_validate_params(adapter, &get_feat_ctx); in ena_restore_device()
3432 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); in ena_restore_device()
3434 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) in ena_restore_device()
3437 rc = ena_enable_msix_and_set_admin_interrupts(adapter); in ena_restore_device()
3448 if ((adapter->msix_vecs - ENA_ADMIN_MSIX_VEC) < adapter->num_io_queues) in ena_restore_device()
3449 adapter->num_io_queues = in ena_restore_device()
3450 adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; in ena_restore_device()
3453 ena_init_io_rings(adapter); in ena_restore_device()
3456 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) { in ena_restore_device()
3457 rc = ena_up(adapter); in ena_restore_device()
3465 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); in ena_restore_device()
3473 adapter->keep_alive_timestamp = getsbinuptime(); in ena_restore_device()
3474 ENA_TIMER_RESET(adapter); in ena_restore_device()
3476 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); in ena_restore_device()
3484 ena_free_mgmnt_irq(adapter); in ena_restore_device()
3485 ena_disable_msix(adapter); in ena_restore_device()
3493 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); in ena_restore_device()
3494 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); in ena_restore_device()
3497 ENA_TIMER_RESET(adapter); in ena_restore_device()
3505 struct ena_adapter *adapter = (struct ena_adapter *)arg; in ena_reset_task() local
3508 if (likely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { in ena_reset_task()
3509 ena_destroy_device(adapter, false); in ena_reset_task()
3510 ena_restore_device(adapter); in ena_reset_task()
3531 struct ena_adapter *adapter; in ena_attach() local
3537 adapter = device_get_softc(pdev); in ena_attach()
3538 adapter->pdev = pdev; in ena_attach()
3539 adapter->first_bind = -1; in ena_attach()
3545 ENA_TIMER_INIT(adapter); in ena_attach()
3546 adapter->keep_alive_timeout = DEFAULT_KEEP_ALIVE_TO; in ena_attach()
3547 adapter->missing_tx_timeout = DEFAULT_TX_CMP_TO; in ena_attach()
3548 adapter->missing_tx_max_queues = DEFAULT_TX_MONITORED_QUEUES; in ena_attach()
3549 adapter->missing_tx_threshold = DEFAULT_TX_CMP_THRESHOLD; in ena_attach()
3558 adapter->ena_dev = ena_dev; in ena_attach()
3562 adapter->memory = NULL; in ena_attach()
3563 adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, in ena_attach()
3565 if (unlikely(adapter->registers == NULL)) { in ena_attach()
3575 adapter->msix = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, in ena_attach()
3577 if (unlikely(adapter->msix == NULL)) { in ena_attach()
3583 adapter->msix_rid = msix_rid; in ena_attach()
3591 rman_get_bustag(adapter->registers); in ena_attach()
3593 rman_get_bushandle(adapter->registers); in ena_attach()
3608 ENA_FLAG_ZERO(adapter); in ena_attach()
3611 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active); in ena_attach()
3619 adapter->disable_meta_caching = in ena_attach()
3623 adapter->keep_alive_timestamp = getsbinuptime(); in ena_attach()
3625 adapter->tx_offload_cap = get_feat_ctx.offload.tx; in ena_attach()
3627 memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr, in ena_attach()
3643 adapter->requested_tx_ring_size = calc_queue_ctx.tx_queue_size; in ena_attach()
3644 adapter->requested_rx_ring_size = calc_queue_ctx.rx_queue_size; in ena_attach()
3645 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size; in ena_attach()
3646 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size; in ena_attach()
3647 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; in ena_attach()
3648 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; in ena_attach()
3650 adapter->max_num_io_queues = max_num_io_queues; in ena_attach()
3652 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE; in ena_attach()
3654 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; in ena_attach()
3656 adapter->reset_reason = ENA_REGS_RESET_NORMAL; in ena_attach()
3659 rc = ena_setup_tx_dma_tag(adapter); in ena_attach()
3665 rc = ena_setup_rx_dma_tag(adapter); in ena_attach()
3680 rc = ena_enable_msix_and_set_admin_interrupts(adapter); in ena_attach()
3687 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; in ena_attach()
3690 ena_init_io_rings(adapter); in ena_attach()
3693 rc = ena_setup_ifnet(pdev, adapter, &get_feat_ctx); in ena_attach()
3700 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter); in ena_attach()
3701 adapter->reset_tq = taskqueue_create("ena_reset_enqueue", in ena_attach()
3702 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq); in ena_attach()
3703 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, in ena_attach()
3704 "%s rstq", device_get_nameunit(adapter->pdev)); in ena_attach()
3707 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats, in ena_attach()
3709 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats, in ena_attach()
3711 ena_sysctl_add_nodes(adapter); in ena_attach()
3714 rc = ena_netmap_attach(adapter); in ena_attach()
3722 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); in ena_attach()
3723 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); in ena_attach()
3726 ENA_TIMER_RESET(adapter); in ena_attach()
3732 ether_ifdetach(adapter->ifp); in ena_attach()
3735 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR); in ena_attach()
3736 ena_free_mgmnt_irq(adapter); in ena_attach()
3737 ena_disable_msix(adapter); in ena_attach()
3739 ena_free_all_io_rings_resources(adapter); in ena_attach()
3740 ena_free_rx_dma_tag(adapter); in ena_attach()
3742 ena_free_tx_dma_tag(adapter); in ena_attach()
3750 ena_free_pci_resources(adapter); in ena_attach()
3767 struct ena_adapter *adapter = device_get_softc(pdev); in ena_detach() local
3768 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_detach()
3772 if (adapter->ifp->if_vlantrunk != NULL) { in ena_detach()
3773 ena_log(adapter->pdev, ERR, "VLAN is in use, detach first\n"); in ena_detach()
3777 ether_ifdetach(adapter->ifp); in ena_detach()
3781 ENA_TIMER_DRAIN(adapter); in ena_detach()
3785 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL)) in ena_detach()
3786 taskqueue_drain(adapter->reset_tq, &adapter->reset_task); in ena_detach()
3787 taskqueue_free(adapter->reset_tq); in ena_detach()
3790 ena_down(adapter); in ena_detach()
3791 ena_destroy_device(adapter, true); in ena_detach()
3795 ena_sysctl_update_queue_node_nb(adapter, adapter->num_io_queues, in ena_detach()
3796 adapter->max_num_io_queues); in ena_detach()
3799 netmap_detach(adapter->ifp); in ena_detach()
3802 ena_free_counters((counter_u64_t *)&adapter->hw_stats, in ena_detach()
3804 ena_free_counters((counter_u64_t *)&adapter->dev_stats, in ena_detach()
3807 rc = ena_free_rx_dma_tag(adapter); in ena_detach()
3809 ena_log(adapter->pdev, WARN, in ena_detach()
3812 rc = ena_free_tx_dma_tag(adapter); in ena_detach()
3814 ena_log(adapter->pdev, WARN, in ena_detach()
3817 ena_free_irqs(adapter); in ena_detach()
3819 ena_free_pci_resources(adapter); in ena_detach()
3821 if (adapter->rss_indir != NULL) in ena_detach()
3822 free(adapter->rss_indir, M_DEVBUF); in ena_detach()
3824 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) in ena_detach()
3829 if_free(adapter->ifp); in ena_detach()
3849 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_update_on_link_change() local
3855 ifp = adapter->ifp; in ena_update_on_link_change()
3860 ena_log(adapter->pdev, INFO, "link is UP\n"); in ena_update_on_link_change()
3861 ENA_FLAG_SET_ATOMIC(ENA_FLAG_LINK_UP, adapter); in ena_update_on_link_change()
3862 if (!ENA_FLAG_ISSET(ENA_FLAG_ONGOING_RESET, adapter)) in ena_update_on_link_change()
3865 ena_log(adapter->pdev, INFO, "link is DOWN\n"); in ena_update_on_link_change()
3867 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_LINK_UP, adapter); in ena_update_on_link_change()
3874 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_notification() local
3877 ENA_WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION, adapter->ena_dev, in ena_notification()
3885 ena_update_hints(adapter, hints); in ena_notification()
3888 ena_log(adapter->pdev, ERR, in ena_notification()
3915 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in unimplemented_aenq_handler() local
3917 ena_log(adapter->pdev, ERR, in unimplemented_aenq_handler()