59 	struct ionic_qcq *qcq;  in ionic_dim_work()  local
64 	qcq = container_of(dim, struct ionic_qcq, dim);  in ionic_dim_work()
65 	q = &qcq->q;  in ionic_dim_work()
74 	intr = &qcq->intr;  in ionic_dim_work()
227 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)  in ionic_request_irq()  argument
229 	struct ionic_intr_info *intr = &qcq->intr;  in ionic_request_irq()
231 	struct ionic_queue *q = &qcq->q;  in ionic_request_irq()
243 				0, intr->name, &qcq->napi);  in ionic_request_irq()
282 static int ionic_qcq_enable(struct ionic_qcq *qcq)  in ionic_qcq_enable()  argument
284 	struct ionic_queue *q = &qcq->q;  in ionic_qcq_enable()
307 	if (qcq->flags & IONIC_QCQ_F_INTR)  in ionic_qcq_enable()
308 		ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);  in ionic_qcq_enable()
314 	if (qcq->flags & IONIC_QCQ_F_INTR) {  in ionic_qcq_enable()
315 		napi_enable(&qcq->napi);  in ionic_qcq_enable()
316 		irq_set_affinity_notifier(qcq->intr.vector,  in ionic_qcq_enable()
317 					  &qcq->intr.aff_notify);  in ionic_qcq_enable()
318 		irq_set_affinity_hint(qcq->intr.vector,  in ionic_qcq_enable()
319 				      *qcq->intr.affinity_mask);  in ionic_qcq_enable()
320 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,  in ionic_qcq_enable()
327 static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int fw_err)  in ionic_qcq_disable()  argument
339 	if (!qcq) {  in ionic_qcq_disable()
344 	q = &qcq->q;  in ionic_qcq_disable()
346 	if (qcq->flags & IONIC_QCQ_F_INTR) {  in ionic_qcq_disable()
350 			cancel_work_sync(&qcq->doorbell_napi_work);  in ionic_qcq_disable()
351 		cancel_work_sync(&qcq->dim.work);  in ionic_qcq_disable()
352 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,  in ionic_qcq_disable()
354 		synchronize_irq(qcq->intr.vector);  in ionic_qcq_disable()
355 		irq_set_affinity_notifier(qcq->intr.vector, NULL);  in ionic_qcq_disable()
356 		irq_set_affinity_hint(qcq->intr.vector, NULL);  in ionic_qcq_disable()
357 		napi_disable(&qcq->napi);  in ionic_qcq_disable()
375 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)  in ionic_lif_qcq_deinit()  argument
379 	if (!qcq)  in ionic_lif_qcq_deinit()
382 	if (!(qcq->flags & IONIC_QCQ_F_INITED))  in ionic_lif_qcq_deinit()
385 	ionic_unregister_rxq_info(&qcq->q);  in ionic_lif_qcq_deinit()
386 	if (qcq->flags & IONIC_QCQ_F_INTR) {  in ionic_lif_qcq_deinit()
387 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,  in ionic_lif_qcq_deinit()
389 		netif_napi_del(&qcq->napi);  in ionic_lif_qcq_deinit()
392 	qcq->flags &= ~IONIC_QCQ_F_INITED;  in ionic_lif_qcq_deinit()
395 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)  in ionic_qcq_intr_free()  argument
397 	if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)  in ionic_qcq_intr_free()
400 	irq_set_affinity_hint(qcq->intr.vector, NULL);  in ionic_qcq_intr_free()
401 	devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);  in ionic_qcq_intr_free()
402 	qcq->intr.vector = 0;  in ionic_qcq_intr_free()
403 	ionic_intr_free(lif->ionic, qcq->intr.index);  in ionic_qcq_intr_free()
404 	qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;  in ionic_qcq_intr_free()
407 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)  in ionic_qcq_free()  argument
411 	if (!qcq)  in ionic_qcq_free()
414 	ionic_debugfs_del_qcq(qcq);  in ionic_qcq_free()
416 	if (qcq->q_base) {  in ionic_qcq_free()
417 		dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);  in ionic_qcq_free()
418 		qcq->q_base = NULL;  in ionic_qcq_free()
419 		qcq->q_base_pa = 0;  in ionic_qcq_free()
422 	if (qcq->cmb_q_base) {  in ionic_qcq_free()
423 		iounmap(qcq->cmb_q_base);  in ionic_qcq_free()
424 		ionic_put_cmb(lif, qcq->cmb_pgid, qcq->cmb_order);  in ionic_qcq_free()
425 		qcq->cmb_pgid = 0;  in ionic_qcq_free()
426 		qcq->cmb_order = 0;  in ionic_qcq_free()
427 		qcq->cmb_q_base = NULL;  in ionic_qcq_free()
428 		qcq->cmb_q_base_pa = 0;  in ionic_qcq_free()
431 	if (qcq->cq_base) {  in ionic_qcq_free()
432 		dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);  in ionic_qcq_free()
433 		qcq->cq_base = NULL;  in ionic_qcq_free()
434 		qcq->cq_base_pa = 0;  in ionic_qcq_free()
437 	if (qcq->sg_base) {  in ionic_qcq_free()
438 		dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);  in ionic_qcq_free()
439 		qcq->sg_base = NULL;  in ionic_qcq_free()
440 		qcq->sg_base_pa = 0;  in ionic_qcq_free()
443 	page_pool_destroy(qcq->q.page_pool);  in ionic_qcq_free()
444 	qcq->q.page_pool = NULL;  in ionic_qcq_free()
446 	ionic_qcq_intr_free(lif, qcq);  in ionic_qcq_free()
447 	vfree(qcq->q.info);  in ionic_qcq_free()
448 	qcq->q.info = NULL;  in ionic_qcq_free()
496 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)  in ionic_alloc_qcq_interrupt()  argument
501 	if (!(qcq->flags & IONIC_QCQ_F_INTR)) {  in ionic_alloc_qcq_interrupt()
502 		qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;  in ionic_alloc_qcq_interrupt()
506 	err = ionic_intr_alloc(lif, &qcq->intr);  in ionic_alloc_qcq_interrupt()
509 			    qcq->q.name, err);  in ionic_alloc_qcq_interrupt()
513 	err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);  in ionic_alloc_qcq_interrupt()
516 			    qcq->q.name, err);  in ionic_alloc_qcq_interrupt()
519 	qcq->intr.vector = err;  in ionic_alloc_qcq_interrupt()
520 	ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,  in ionic_alloc_qcq_interrupt()
523 	err = ionic_request_irq(lif, qcq);  in ionic_alloc_qcq_interrupt()
530 	affinity_mask = &lif->ionic->affinity_masks[qcq->intr.index];  in ionic_alloc_qcq_interrupt()
534 		cpu = cpumask_local_spread(qcq->intr.index,  in ionic_alloc_qcq_interrupt()
540 	qcq->intr.affinity_mask = affinity_mask;  in ionic_alloc_qcq_interrupt()
541 	qcq->intr.aff_notify.notify = ionic_irq_aff_notify;  in ionic_alloc_qcq_interrupt()
542 	qcq->intr.aff_notify.release = ionic_irq_aff_release;  in ionic_alloc_qcq_interrupt()
544 	netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);  in ionic_alloc_qcq_interrupt()
548 	ionic_intr_free(lif->ionic, qcq->intr.index);  in ionic_alloc_qcq_interrupt()
561 			   struct ionic_qcq **qcq)  in ionic_qcq_alloc()  argument
568 	*qcq = NULL;  in ionic_qcq_alloc()
728 	*qcq = new;  in ionic_qcq_alloc()
817 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)  in ionic_qcq_sanitize()  argument
819 	qcq->q.tail_idx = 0;  in ionic_qcq_sanitize()
820 	qcq->q.head_idx = 0;  in ionic_qcq_sanitize()
821 	qcq->cq.tail_idx = 0;  in ionic_qcq_sanitize()
822 	qcq->cq.done_color = 1;  in ionic_qcq_sanitize()
823 	memset(qcq->q_base, 0, qcq->q_size);  in ionic_qcq_sanitize()
824 	if (qcq->cmb_q_base)  in ionic_qcq_sanitize()
825 		memset_io(qcq->cmb_q_base, 0, qcq->cmb_q_size);  in ionic_qcq_sanitize()
826 	memset(qcq->cq_base, 0, qcq->cq_size);  in ionic_qcq_sanitize()
827 	memset(qcq->sg_base, 0, qcq->sg_size);  in ionic_qcq_sanitize()
830 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)  in ionic_lif_txq_init()  argument
833 	struct ionic_queue *q = &qcq->q;  in ionic_lif_txq_init()
834 	struct ionic_cq *cq = &qcq->cq;  in ionic_lif_txq_init()
845 			.intr_index = cpu_to_le16(qcq->intr.index),  in ionic_lif_txq_init()
856 	if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {  in ionic_lif_txq_init()
858 		ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);  in ionic_lif_txq_init()
871 	ionic_qcq_sanitize(qcq);  in ionic_lif_txq_init()
888 		netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);  in ionic_lif_txq_init()
890 	qcq->flags |= IONIC_QCQ_F_INITED;  in ionic_lif_txq_init()
895 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)  in ionic_lif_rxq_init()  argument
898 	struct ionic_queue *q = &qcq->q;  in ionic_lif_rxq_init()
899 	struct ionic_cq *cq = &qcq->cq;  in ionic_lif_rxq_init()
927 	if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {  in ionic_lif_rxq_init()
929 		ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);  in ionic_lif_rxq_init()
940 	ionic_qcq_sanitize(qcq);  in ionic_lif_rxq_init()
957 		netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);  in ionic_lif_rxq_init()
959 		netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);  in ionic_lif_rxq_init()
960 	err = ionic_register_rxq_info(q, qcq->napi.napi_id);  in ionic_lif_rxq_init()
962 		netif_napi_del(&qcq->napi);  in ionic_lif_rxq_init()
966 	qcq->flags |= IONIC_QCQ_F_INITED;  in ionic_lif_rxq_init()
3566 	struct ionic_qcq *qcq;  in ionic_lif_adminq_init()  local
3571 	qcq = lif->adminqcq;  in ionic_lif_adminq_init()
3572 	q = &qcq->q;  in ionic_lif_adminq_init()
3575 	ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);  in ionic_lif_adminq_init()
3594 	netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);  in ionic_lif_adminq_init()
3596 	napi_enable(&qcq->napi);  in ionic_lif_adminq_init()
3598 	if (qcq->flags & IONIC_QCQ_F_INTR) {  in ionic_lif_adminq_init()
3599 		irq_set_affinity_hint(qcq->intr.vector,  in ionic_lif_adminq_init()
3600 				      *qcq->intr.affinity_mask);  in ionic_lif_adminq_init()
3601 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,  in ionic_lif_adminq_init()
3605 	qcq->flags |= IONIC_QCQ_F_INITED;  in ionic_lif_adminq_init()
3612 	struct ionic_qcq *qcq = lif->notifyqcq;  in ionic_lif_notifyq_init()  local
3614 	struct ionic_queue *q = &qcq->q;  in ionic_lif_notifyq_init()
3654 	qcq->flags |= IONIC_QCQ_F_INITED;  in ionic_lif_notifyq_init()