Lines Matching refs:ctrl
84 struct nvmet_pci_epf_ctrl *ctrl; member
129 struct nvmet_pci_epf_ctrl *ctrl; member
209 struct nvmet_pci_epf_ctrl ctrl; member
226 static inline u32 nvmet_pci_epf_bar_read32(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_bar_read32() argument
229 __le32 *bar_reg = ctrl->bar + off; in nvmet_pci_epf_bar_read32()
234 static inline void nvmet_pci_epf_bar_write32(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_bar_write32() argument
237 __le32 *bar_reg = ctrl->bar + off; in nvmet_pci_epf_bar_write32()
242 static inline u64 nvmet_pci_epf_bar_read64(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_bar_read64() argument
245 return (u64)nvmet_pci_epf_bar_read32(ctrl, off) | in nvmet_pci_epf_bar_read64()
246 ((u64)nvmet_pci_epf_bar_read32(ctrl, off + 4) << 32); in nvmet_pci_epf_bar_read64()
249 static inline void nvmet_pci_epf_bar_write64(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_bar_write64() argument
252 nvmet_pci_epf_bar_write32(ctrl, off, val & 0xFFFFFFFF); in nvmet_pci_epf_bar_write64()
253 nvmet_pci_epf_bar_write32(ctrl, off + 4, (val >> 32) & 0xFFFFFFFF); in nvmet_pci_epf_bar_write64()
488 static inline int nvmet_pci_epf_transfer(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_transfer() argument
498 return nvmet_pci_epf_transfer_seg(ctrl->nvme_epf, &seg, dir); in nvmet_pci_epf_transfer()
501 static int nvmet_pci_epf_alloc_irq_vectors(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_alloc_irq_vectors() argument
503 ctrl->irq_vectors = kcalloc(ctrl->nr_queues, in nvmet_pci_epf_alloc_irq_vectors()
506 if (!ctrl->irq_vectors) in nvmet_pci_epf_alloc_irq_vectors()
509 mutex_init(&ctrl->irq_lock); in nvmet_pci_epf_alloc_irq_vectors()
514 static void nvmet_pci_epf_free_irq_vectors(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_free_irq_vectors() argument
516 if (ctrl->irq_vectors) { in nvmet_pci_epf_free_irq_vectors()
517 mutex_destroy(&ctrl->irq_lock); in nvmet_pci_epf_free_irq_vectors()
518 kfree(ctrl->irq_vectors); in nvmet_pci_epf_free_irq_vectors()
519 ctrl->irq_vectors = NULL; in nvmet_pci_epf_free_irq_vectors()
524 nvmet_pci_epf_find_irq_vector(struct nvmet_pci_epf_ctrl *ctrl, u16 vector) in nvmet_pci_epf_find_irq_vector() argument
529 lockdep_assert_held(&ctrl->irq_lock); in nvmet_pci_epf_find_irq_vector()
531 for (i = 0; i < ctrl->nr_queues; i++) { in nvmet_pci_epf_find_irq_vector()
532 iv = &ctrl->irq_vectors[i]; in nvmet_pci_epf_find_irq_vector()
541 nvmet_pci_epf_add_irq_vector(struct nvmet_pci_epf_ctrl *ctrl, u16 vector) in nvmet_pci_epf_add_irq_vector() argument
546 mutex_lock(&ctrl->irq_lock); in nvmet_pci_epf_add_irq_vector()
548 iv = nvmet_pci_epf_find_irq_vector(ctrl, vector); in nvmet_pci_epf_add_irq_vector()
554 for (i = 0; i < ctrl->nr_queues; i++) { in nvmet_pci_epf_add_irq_vector()
555 iv = &ctrl->irq_vectors[i]; in nvmet_pci_epf_add_irq_vector()
568 mutex_unlock(&ctrl->irq_lock); in nvmet_pci_epf_add_irq_vector()
573 static void nvmet_pci_epf_remove_irq_vector(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_remove_irq_vector() argument
578 mutex_lock(&ctrl->irq_lock); in nvmet_pci_epf_remove_irq_vector()
580 iv = nvmet_pci_epf_find_irq_vector(ctrl, vector); in nvmet_pci_epf_remove_irq_vector()
589 mutex_unlock(&ctrl->irq_lock); in nvmet_pci_epf_remove_irq_vector()
592 static bool nvmet_pci_epf_should_raise_irq(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_should_raise_irq() argument
609 ret = iv->nr_irqs >= ctrl->irq_vector_threshold; in nvmet_pci_epf_should_raise_irq()
617 static void nvmet_pci_epf_raise_irq(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_raise_irq() argument
620 struct nvmet_pci_epf *nvme_epf = ctrl->nvme_epf; in nvmet_pci_epf_raise_irq()
628 mutex_lock(&ctrl->irq_lock); in nvmet_pci_epf_raise_irq()
630 if (!nvmet_pci_epf_should_raise_irq(ctrl, cq, force)) in nvmet_pci_epf_raise_irq()
658 dev_err_ratelimited(ctrl->dev, in nvmet_pci_epf_raise_irq()
663 mutex_unlock(&ctrl->irq_lock); in nvmet_pci_epf_raise_irq()
676 struct nvmet_pci_epf_ctrl *ctrl = sq->ctrl; in nvmet_pci_epf_alloc_iod() local
679 iod = mempool_alloc(&ctrl->iod_pool, GFP_KERNEL); in nvmet_pci_epf_alloc_iod()
686 iod->req.port = ctrl->port; in nvmet_pci_epf_alloc_iod()
687 iod->ctrl = ctrl; in nvmet_pci_epf_alloc_iod()
689 iod->cq = &ctrl->cq[sq->qid]; in nvmet_pci_epf_alloc_iod()
731 mempool_free(iod, &iod->ctrl->iod_pool); in nvmet_pci_epf_free_iod()
736 struct nvmet_pci_epf *nvme_epf = iod->ctrl->nvme_epf; in nvmet_pci_epf_transfer_iod_data()
752 static inline u32 nvmet_pci_epf_prp_ofst(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_prp_ofst() argument
755 return prp & ctrl->mps_mask; in nvmet_pci_epf_prp_ofst()
758 static inline size_t nvmet_pci_epf_prp_size(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_prp_size() argument
761 return ctrl->mps - nvmet_pci_epf_prp_ofst(ctrl, prp); in nvmet_pci_epf_prp_size()
767 static int nvmet_pci_epf_get_prp_list(struct nvmet_pci_epf_ctrl *ctrl, u64 prp, in nvmet_pci_epf_get_prp_list() argument
770 size_t nr_prps = (xfer_len + ctrl->mps_mask) >> ctrl->mps_shift; in nvmet_pci_epf_get_prp_list()
781 length = min(nvmet_pci_epf_prp_size(ctrl, prp), nr_prps << 3); in nvmet_pci_epf_get_prp_list()
782 ret = nvmet_pci_epf_transfer(ctrl, prps, prp, length, DMA_FROM_DEVICE); in nvmet_pci_epf_get_prp_list()
789 static int nvmet_pci_epf_iod_parse_prp_list(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_iod_parse_prp_list() argument
801 prps = kzalloc(ctrl->mps, GFP_KERNEL); in nvmet_pci_epf_iod_parse_prp_list()
815 ofst = nvmet_pci_epf_prp_ofst(ctrl, prp); in nvmet_pci_epf_iod_parse_prp_list()
816 nr_segs = (transfer_len + ofst + ctrl->mps - 1) >> ctrl->mps_shift; in nvmet_pci_epf_iod_parse_prp_list()
825 seg->length = nvmet_pci_epf_prp_size(ctrl, prp); in nvmet_pci_epf_iod_parse_prp_list()
843 nr_prps = nvmet_pci_epf_get_prp_list(ctrl, prp, in nvmet_pci_epf_iod_parse_prp_list()
858 if (xfer_len > ctrl->mps && i == nr_prps - 1) { in nvmet_pci_epf_iod_parse_prp_list()
865 if (nvmet_pci_epf_prp_ofst(ctrl, prp)) in nvmet_pci_epf_iod_parse_prp_list()
880 prp_size = min_t(size_t, ctrl->mps, xfer_len); in nvmet_pci_epf_iod_parse_prp_list()
892 dev_err(ctrl->dev, in nvmet_pci_epf_iod_parse_prp_list()
903 dev_err(ctrl->dev, "PRPs list invalid offset\n"); in nvmet_pci_epf_iod_parse_prp_list()
908 dev_err(ctrl->dev, "PRPs list invalid field\n"); in nvmet_pci_epf_iod_parse_prp_list()
913 dev_err(ctrl->dev, "PRPs list internal error\n"); in nvmet_pci_epf_iod_parse_prp_list()
921 static int nvmet_pci_epf_iod_parse_prp_simple(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_iod_parse_prp_simple() argument
931 prp1_size = nvmet_pci_epf_prp_size(ctrl, prp1); in nvmet_pci_epf_iod_parse_prp_simple()
940 if (nvmet_pci_epf_prp_ofst(ctrl, prp2)) { in nvmet_pci_epf_iod_parse_prp_simple()
973 struct nvmet_pci_epf_ctrl *ctrl = iod->ctrl; in nvmet_pci_epf_iod_parse_prps() local
978 ofst = nvmet_pci_epf_prp_ofst(ctrl, prp1); in nvmet_pci_epf_iod_parse_prps()
984 if (iod->data_len + ofst <= ctrl->mps * 2) in nvmet_pci_epf_iod_parse_prps()
985 return nvmet_pci_epf_iod_parse_prp_simple(ctrl, iod); in nvmet_pci_epf_iod_parse_prps()
987 return nvmet_pci_epf_iod_parse_prp_list(ctrl, iod); in nvmet_pci_epf_iod_parse_prps()
995 nvmet_pci_epf_get_sgl_segment(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_get_sgl_segment() argument
1007 ret = nvmet_pci_epf_transfer(ctrl, buf, le64_to_cpu(desc->addr), length, in nvmet_pci_epf_get_sgl_segment()
1035 static int nvmet_pci_epf_iod_parse_sgl_segments(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_iod_parse_sgl_segments() argument
1055 sgls = nvmet_pci_epf_get_sgl_segment(ctrl, &seg, &nr_sgls); in nvmet_pci_epf_iod_parse_sgl_segments()
1097 struct nvmet_pci_epf_ctrl *ctrl = iod->ctrl; in nvmet_pci_epf_iod_parse_sgls() local
1109 return nvmet_pci_epf_iod_parse_sgl_segments(ctrl, iod); in nvmet_pci_epf_iod_parse_sgls()
1114 struct nvmet_pci_epf_ctrl *ctrl = iod->ctrl; in nvmet_pci_epf_alloc_iod_data_buf() local
1120 if (iod->data_len > ctrl->mdts) { in nvmet_pci_epf_alloc_iod_data_buf()
1176 dev_err(iod->ctrl->dev, in nvmet_pci_epf_complete_iod()
1222 nvmet_pci_epf_find_port(struct nvmet_pci_epf_ctrl *ctrl, __le16 portid) in nvmet_pci_epf_find_port() argument
1256 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_get_mdts() local
1259 return ilog2(ctrl->mdts) - page_shift; in nvmet_pci_epf_get_mdts()
1265 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_create_cq() local
1266 struct nvmet_pci_epf_queue *cq = &ctrl->cq[cqid]; in nvmet_pci_epf_create_cq()
1284 nvmet_pci_epf_bar_write32(ctrl, cq->db, 0); in nvmet_pci_epf_create_cq()
1289 cq->qes = ctrl->io_cqes; in nvmet_pci_epf_create_cq()
1293 cq->iv = nvmet_pci_epf_add_irq_vector(ctrl, vector); in nvmet_pci_epf_create_cq()
1307 ret = nvmet_pci_epf_mem_map(ctrl->nvme_epf, cq->pci_addr, cq->pci_size, in nvmet_pci_epf_create_cq()
1310 dev_err(ctrl->dev, "Failed to map CQ %u (err=%d)\n", in nvmet_pci_epf_create_cq()
1316 dev_err(ctrl->dev, "Invalid partial mapping of queue %u\n", in nvmet_pci_epf_create_cq()
1324 dev_dbg(ctrl->dev, in nvmet_pci_epf_create_cq()
1328 dev_dbg(ctrl->dev, in nvmet_pci_epf_create_cq()
1335 nvmet_pci_epf_mem_unmap(ctrl->nvme_epf, &cq->pci_map); in nvmet_pci_epf_create_cq()
1340 nvmet_pci_epf_remove_irq_vector(ctrl, cq->vector); in nvmet_pci_epf_create_cq()
1346 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_delete_cq() local
1347 struct nvmet_pci_epf_queue *cq = &ctrl->cq[cqid]; in nvmet_pci_epf_delete_cq()
1355 nvmet_pci_epf_remove_irq_vector(ctrl, cq->vector); in nvmet_pci_epf_delete_cq()
1356 nvmet_pci_epf_mem_unmap(ctrl->nvme_epf, &cq->pci_map); in nvmet_pci_epf_delete_cq()
1364 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_create_sq() local
1365 struct nvmet_pci_epf_queue *sq = &ctrl->sq[sqid]; in nvmet_pci_epf_create_sq()
1381 nvmet_pci_epf_bar_write32(ctrl, sq->db, 0); in nvmet_pci_epf_create_sq()
1385 sq->qes = ctrl->io_sqes; in nvmet_pci_epf_create_sq()
1395 dev_err(ctrl->dev, "Failed to create SQ %d work queue\n", sqid); in nvmet_pci_epf_create_sq()
1402 dev_dbg(ctrl->dev, "SQ[%u]: %u entries of %zu B\n", in nvmet_pci_epf_create_sq()
1414 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_delete_sq() local
1415 struct nvmet_pci_epf_queue *sq = &ctrl->sq[sqid]; in nvmet_pci_epf_delete_sq()
1425 if (sq->nvme_sq.ctrl) in nvmet_pci_epf_delete_sq()
1434 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_get_feat() local
1444 if (!ctrl->sq_ab) in nvmet_pci_epf_get_feat()
1447 arb->ab = ilog2(ctrl->sq_ab); in nvmet_pci_epf_get_feat()
1452 irqc->thr = ctrl->irq_vector_threshold; in nvmet_pci_epf_get_feat()
1458 mutex_lock(&ctrl->irq_lock); in nvmet_pci_epf_get_feat()
1459 iv = nvmet_pci_epf_find_irq_vector(ctrl, irqcfg->iv); in nvmet_pci_epf_get_feat()
1466 mutex_unlock(&ctrl->irq_lock); in nvmet_pci_epf_get_feat()
1477 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_set_feat() local
1488 ctrl->sq_ab = 0; in nvmet_pci_epf_set_feat()
1490 ctrl->sq_ab = 1 << arb->ab; in nvmet_pci_epf_set_feat()
1499 ctrl->irq_vector_threshold = irqc->thr + 1; in nvmet_pci_epf_set_feat()
1504 mutex_lock(&ctrl->irq_lock); in nvmet_pci_epf_set_feat()
1505 iv = nvmet_pci_epf_find_irq_vector(ctrl, irqcfg->iv); in nvmet_pci_epf_set_feat()
1512 mutex_unlock(&ctrl->irq_lock); in nvmet_pci_epf_set_feat()
1537 static void nvmet_pci_epf_init_queue(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_init_queue() argument
1543 queue = &ctrl->sq[qid]; in nvmet_pci_epf_init_queue()
1545 queue = &ctrl->cq[qid]; in nvmet_pci_epf_init_queue()
1548 queue->ctrl = ctrl; in nvmet_pci_epf_init_queue()
1554 static int nvmet_pci_epf_alloc_queues(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_alloc_queues() argument
1558 ctrl->sq = kcalloc(ctrl->nr_queues, in nvmet_pci_epf_alloc_queues()
1560 if (!ctrl->sq) in nvmet_pci_epf_alloc_queues()
1563 ctrl->cq = kcalloc(ctrl->nr_queues, in nvmet_pci_epf_alloc_queues()
1565 if (!ctrl->cq) { in nvmet_pci_epf_alloc_queues()
1566 kfree(ctrl->sq); in nvmet_pci_epf_alloc_queues()
1567 ctrl->sq = NULL; in nvmet_pci_epf_alloc_queues()
1571 for (qid = 0; qid < ctrl->nr_queues; qid++) { in nvmet_pci_epf_alloc_queues()
1572 nvmet_pci_epf_init_queue(ctrl, qid, true); in nvmet_pci_epf_alloc_queues()
1573 nvmet_pci_epf_init_queue(ctrl, qid, false); in nvmet_pci_epf_alloc_queues()
1579 static void nvmet_pci_epf_free_queues(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_free_queues() argument
1581 kfree(ctrl->sq); in nvmet_pci_epf_free_queues()
1582 ctrl->sq = NULL; in nvmet_pci_epf_free_queues()
1583 kfree(ctrl->cq); in nvmet_pci_epf_free_queues()
1584 ctrl->cq = NULL; in nvmet_pci_epf_free_queues()
1594 if (!iod->ctrl->link_up) { in nvmet_pci_epf_exec_iod_work()
1653 static int nvmet_pci_epf_process_sq(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_process_sq() argument
1660 sq->tail = nvmet_pci_epf_bar_read32(ctrl, sq->db); in nvmet_pci_epf_process_sq()
1661 while (head != sq->tail && (!ctrl->sq_ab || n < ctrl->sq_ab)) { in nvmet_pci_epf_process_sq()
1667 ret = nvmet_pci_epf_transfer(ctrl, &iod->cmd, in nvmet_pci_epf_process_sq()
1676 dev_dbg(ctrl->dev, "SQ[%u]: head %u, tail %u, command %s\n", in nvmet_pci_epf_process_sq()
1688 sq->tail = nvmet_pci_epf_bar_read32(ctrl, sq->db); in nvmet_pci_epf_process_sq()
1696 struct nvmet_pci_epf_ctrl *ctrl = in nvmet_pci_epf_poll_sqs_work() local
1703 while (ctrl->link_up && ctrl->enabled) { in nvmet_pci_epf_poll_sqs_work()
1706 for (i = 0; i < ctrl->nr_queues; i++) { in nvmet_pci_epf_poll_sqs_work()
1707 sq = &ctrl->sq[i]; in nvmet_pci_epf_poll_sqs_work()
1710 if (nvmet_pci_epf_process_sq(ctrl, sq)) in nvmet_pci_epf_poll_sqs_work()
1741 schedule_delayed_work(&ctrl->poll_sqs, NVMET_PCI_EPF_SQ_POLL_INTERVAL); in nvmet_pci_epf_poll_sqs_work()
1748 struct nvmet_pci_epf_ctrl *ctrl = cq->ctrl; in nvmet_pci_epf_cq_work() local
1754 while (test_bit(NVMET_PCI_EPF_Q_LIVE, &cq->flags) && ctrl->link_up) { in nvmet_pci_epf_cq_work()
1757 cq->head = nvmet_pci_epf_bar_read32(ctrl, cq->db); in nvmet_pci_epf_cq_work()
1786 dev_dbg(ctrl->dev, in nvmet_pci_epf_cq_work()
1804 nvmet_pci_epf_raise_irq(ctrl, cq, false); in nvmet_pci_epf_cq_work()
1814 nvmet_pci_epf_raise_irq(ctrl, cq, true); in nvmet_pci_epf_cq_work()
1821 static void nvmet_pci_epf_clear_ctrl_config(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_clear_ctrl_config() argument
1823 struct nvmet_ctrl *tctrl = ctrl->tctrl; in nvmet_pci_epf_clear_ctrl_config()
1827 ctrl->csts = 0; in nvmet_pci_epf_clear_ctrl_config()
1828 nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CSTS, ctrl->csts); in nvmet_pci_epf_clear_ctrl_config()
1832 ctrl->cc = 0; in nvmet_pci_epf_clear_ctrl_config()
1833 nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CC, ctrl->cc); in nvmet_pci_epf_clear_ctrl_config()
1836 static int nvmet_pci_epf_enable_ctrl(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_enable_ctrl() argument
1842 if (ctrl->enabled) in nvmet_pci_epf_enable_ctrl()
1845 dev_info(ctrl->dev, "Enabling controller\n"); in nvmet_pci_epf_enable_ctrl()
1847 ctrl->mps_shift = nvmet_cc_mps(ctrl->cc) + 12; in nvmet_pci_epf_enable_ctrl()
1848 ctrl->mps = 1UL << ctrl->mps_shift; in nvmet_pci_epf_enable_ctrl()
1849 ctrl->mps_mask = ctrl->mps - 1; in nvmet_pci_epf_enable_ctrl()
1851 ctrl->io_sqes = 1UL << nvmet_cc_iosqes(ctrl->cc); in nvmet_pci_epf_enable_ctrl()
1852 if (ctrl->io_sqes < sizeof(struct nvme_command)) { in nvmet_pci_epf_enable_ctrl()
1853 dev_err(ctrl->dev, "Unsupported I/O SQES %zu (need %zu)\n", in nvmet_pci_epf_enable_ctrl()
1854 ctrl->io_sqes, sizeof(struct nvme_command)); in nvmet_pci_epf_enable_ctrl()
1858 ctrl->io_cqes = 1UL << nvmet_cc_iocqes(ctrl->cc); in nvmet_pci_epf_enable_ctrl()
1859 if (ctrl->io_cqes < sizeof(struct nvme_completion)) { in nvmet_pci_epf_enable_ctrl()
1860 dev_err(ctrl->dev, "Unsupported I/O CQES %zu (need %zu)\n", in nvmet_pci_epf_enable_ctrl()
1861 ctrl->io_sqes, sizeof(struct nvme_completion)); in nvmet_pci_epf_enable_ctrl()
1866 aqa = nvmet_pci_epf_bar_read32(ctrl, NVME_REG_AQA); in nvmet_pci_epf_enable_ctrl()
1867 asq = nvmet_pci_epf_bar_read64(ctrl, NVME_REG_ASQ); in nvmet_pci_epf_enable_ctrl()
1868 acq = nvmet_pci_epf_bar_read64(ctrl, NVME_REG_ACQ); in nvmet_pci_epf_enable_ctrl()
1872 status = nvmet_pci_epf_create_cq(ctrl->tctrl, 0, in nvmet_pci_epf_enable_ctrl()
1876 dev_err(ctrl->dev, "Failed to create admin completion queue\n"); in nvmet_pci_epf_enable_ctrl()
1882 status = nvmet_pci_epf_create_sq(ctrl->tctrl, 0, NVME_QUEUE_PHYS_CONTIG, in nvmet_pci_epf_enable_ctrl()
1885 dev_err(ctrl->dev, "Failed to create admin submission queue\n"); in nvmet_pci_epf_enable_ctrl()
1886 nvmet_pci_epf_delete_cq(ctrl->tctrl, 0); in nvmet_pci_epf_enable_ctrl()
1890 ctrl->sq_ab = NVMET_PCI_EPF_SQ_AB; in nvmet_pci_epf_enable_ctrl()
1891 ctrl->irq_vector_threshold = NVMET_PCI_EPF_IV_THRESHOLD; in nvmet_pci_epf_enable_ctrl()
1892 ctrl->enabled = true; in nvmet_pci_epf_enable_ctrl()
1893 ctrl->csts = NVME_CSTS_RDY; in nvmet_pci_epf_enable_ctrl()
1896 schedule_delayed_work(&ctrl->poll_sqs, 0); in nvmet_pci_epf_enable_ctrl()
1901 nvmet_pci_epf_clear_ctrl_config(ctrl); in nvmet_pci_epf_enable_ctrl()
1905 static void nvmet_pci_epf_disable_ctrl(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_disable_ctrl() argument
1910 if (!ctrl->enabled) in nvmet_pci_epf_disable_ctrl()
1913 dev_info(ctrl->dev, "%s controller\n", in nvmet_pci_epf_disable_ctrl()
1916 ctrl->enabled = false; in nvmet_pci_epf_disable_ctrl()
1917 cancel_delayed_work_sync(&ctrl->poll_sqs); in nvmet_pci_epf_disable_ctrl()
1920 for (qid = 1; qid < ctrl->nr_queues; qid++) in nvmet_pci_epf_disable_ctrl()
1921 nvmet_pci_epf_delete_sq(ctrl->tctrl, qid); in nvmet_pci_epf_disable_ctrl()
1923 for (qid = 1; qid < ctrl->nr_queues; qid++) in nvmet_pci_epf_disable_ctrl()
1924 nvmet_pci_epf_delete_cq(ctrl->tctrl, qid); in nvmet_pci_epf_disable_ctrl()
1927 nvmet_pci_epf_delete_sq(ctrl->tctrl, 0); in nvmet_pci_epf_disable_ctrl()
1928 nvmet_pci_epf_delete_cq(ctrl->tctrl, 0); in nvmet_pci_epf_disable_ctrl()
1930 ctrl->csts &= ~NVME_CSTS_RDY; in nvmet_pci_epf_disable_ctrl()
1932 ctrl->csts |= NVME_CSTS_SHST_CMPLT; in nvmet_pci_epf_disable_ctrl()
1933 ctrl->cc &= ~NVME_CC_ENABLE; in nvmet_pci_epf_disable_ctrl()
1934 nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CC, ctrl->cc); in nvmet_pci_epf_disable_ctrl()
1940 struct nvmet_pci_epf_ctrl *ctrl = in nvmet_pci_epf_poll_cc_work() local
1945 if (!ctrl->tctrl) in nvmet_pci_epf_poll_cc_work()
1948 old_cc = ctrl->cc; in nvmet_pci_epf_poll_cc_work()
1949 new_cc = nvmet_pci_epf_bar_read32(ctrl, NVME_REG_CC); in nvmet_pci_epf_poll_cc_work()
1953 ctrl->cc = new_cc; in nvmet_pci_epf_poll_cc_work()
1956 ret = nvmet_pci_epf_enable_ctrl(ctrl); in nvmet_pci_epf_poll_cc_work()
1962 nvmet_pci_epf_disable_ctrl(ctrl, false); in nvmet_pci_epf_poll_cc_work()
1965 nvmet_pci_epf_disable_ctrl(ctrl, true); in nvmet_pci_epf_poll_cc_work()
1968 ctrl->csts &= ~NVME_CSTS_SHST_CMPLT; in nvmet_pci_epf_poll_cc_work()
1970 nvmet_update_cc(ctrl->tctrl, ctrl->cc); in nvmet_pci_epf_poll_cc_work()
1971 nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CSTS, ctrl->csts); in nvmet_pci_epf_poll_cc_work()
1974 schedule_delayed_work(&ctrl->poll_cc, NVMET_PCI_EPF_CC_POLL_INTERVAL); in nvmet_pci_epf_poll_cc_work()
1977 static void nvmet_pci_epf_init_bar(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_init_bar() argument
1979 struct nvmet_ctrl *tctrl = ctrl->tctrl; in nvmet_pci_epf_init_bar()
1981 ctrl->bar = ctrl->nvme_epf->reg_bar; in nvmet_pci_epf_init_bar()
1984 ctrl->cap = tctrl->cap; in nvmet_pci_epf_init_bar()
1987 ctrl->cap |= 0x1ULL << 16; in nvmet_pci_epf_init_bar()
1990 ctrl->cap &= ~GENMASK_ULL(35, 32); in nvmet_pci_epf_init_bar()
1993 ctrl->cap &= ~(0x1ULL << 36); in nvmet_pci_epf_init_bar()
1996 ctrl->cap &= ~(0x1ULL << 45); in nvmet_pci_epf_init_bar()
1999 ctrl->cap &= ~(0x1ULL << 56); in nvmet_pci_epf_init_bar()
2002 ctrl->cap &= ~(0x1ULL << 57); in nvmet_pci_epf_init_bar()
2004 nvmet_pci_epf_bar_write64(ctrl, NVME_REG_CAP, ctrl->cap); in nvmet_pci_epf_init_bar()
2005 nvmet_pci_epf_bar_write32(ctrl, NVME_REG_VS, tctrl->subsys->ver); in nvmet_pci_epf_init_bar()
2007 nvmet_pci_epf_clear_ctrl_config(ctrl); in nvmet_pci_epf_init_bar()
2013 struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl; in nvmet_pci_epf_create_ctrl() local
2019 memset(ctrl, 0, sizeof(*ctrl)); in nvmet_pci_epf_create_ctrl()
2020 ctrl->dev = &nvme_epf->epf->dev; in nvmet_pci_epf_create_ctrl()
2021 mutex_init(&ctrl->irq_lock); in nvmet_pci_epf_create_ctrl()
2022 ctrl->nvme_epf = nvme_epf; in nvmet_pci_epf_create_ctrl()
2023 ctrl->mdts = nvme_epf->mdts_kb * SZ_1K; in nvmet_pci_epf_create_ctrl()
2024 INIT_DELAYED_WORK(&ctrl->poll_cc, nvmet_pci_epf_poll_cc_work); in nvmet_pci_epf_create_ctrl()
2025 INIT_DELAYED_WORK(&ctrl->poll_sqs, nvmet_pci_epf_poll_sqs_work); in nvmet_pci_epf_create_ctrl()
2027 ret = mempool_init_kmalloc_pool(&ctrl->iod_pool, in nvmet_pci_epf_create_ctrl()
2031 dev_err(ctrl->dev, "Failed to initialize IOD mempool\n"); in nvmet_pci_epf_create_ctrl()
2035 ctrl->port = nvmet_pci_epf_find_port(ctrl, nvme_epf->portid); in nvmet_pci_epf_create_ctrl()
2036 if (!ctrl->port) { in nvmet_pci_epf_create_ctrl()
2037 dev_err(ctrl->dev, "Port not found\n"); in nvmet_pci_epf_create_ctrl()
2046 args.port = ctrl->port; in nvmet_pci_epf_create_ctrl()
2053 ctrl->tctrl = nvmet_alloc_ctrl(&args); in nvmet_pci_epf_create_ctrl()
2054 if (!ctrl->tctrl) { in nvmet_pci_epf_create_ctrl()
2055 dev_err(ctrl->dev, "Failed to create target controller\n"); in nvmet_pci_epf_create_ctrl()
2059 ctrl->tctrl->drvdata = ctrl; in nvmet_pci_epf_create_ctrl()
2062 if (ctrl->tctrl->pi_support) { in nvmet_pci_epf_create_ctrl()
2063 dev_err(ctrl->dev, in nvmet_pci_epf_create_ctrl()
2070 ctrl->nr_queues = min(ctrl->tctrl->subsys->max_qid + 1, max_nr_queues); in nvmet_pci_epf_create_ctrl()
2071 ret = nvmet_pci_epf_alloc_queues(ctrl); in nvmet_pci_epf_create_ctrl()
2079 ret = nvmet_pci_epf_alloc_irq_vectors(ctrl); in nvmet_pci_epf_create_ctrl()
2083 dev_info(ctrl->dev, in nvmet_pci_epf_create_ctrl()
2085 ctrl->tctrl->subsys->subsysnqn, ctrl->nr_queues - 1, in nvmet_pci_epf_create_ctrl()
2086 ctrl->mdts); in nvmet_pci_epf_create_ctrl()
2089 nvmet_pci_epf_init_bar(ctrl); in nvmet_pci_epf_create_ctrl()
2094 nvmet_pci_epf_free_queues(ctrl); in nvmet_pci_epf_create_ctrl()
2096 nvmet_ctrl_put(ctrl->tctrl); in nvmet_pci_epf_create_ctrl()
2097 ctrl->tctrl = NULL; in nvmet_pci_epf_create_ctrl()
2099 mempool_exit(&ctrl->iod_pool); in nvmet_pci_epf_create_ctrl()
2103 static void nvmet_pci_epf_start_ctrl(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_start_ctrl() argument
2106 dev_info(ctrl->dev, "PCI link up\n"); in nvmet_pci_epf_start_ctrl()
2107 ctrl->link_up = true; in nvmet_pci_epf_start_ctrl()
2109 schedule_delayed_work(&ctrl->poll_cc, NVMET_PCI_EPF_CC_POLL_INTERVAL); in nvmet_pci_epf_start_ctrl()
2112 static void nvmet_pci_epf_stop_ctrl(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_stop_ctrl() argument
2114 dev_info(ctrl->dev, "PCI link down\n"); in nvmet_pci_epf_stop_ctrl()
2115 ctrl->link_up = false; in nvmet_pci_epf_stop_ctrl()
2117 cancel_delayed_work_sync(&ctrl->poll_cc); in nvmet_pci_epf_stop_ctrl()
2119 nvmet_pci_epf_disable_ctrl(ctrl, false); in nvmet_pci_epf_stop_ctrl()
2120 nvmet_pci_epf_clear_ctrl_config(ctrl); in nvmet_pci_epf_stop_ctrl()
2123 static void nvmet_pci_epf_destroy_ctrl(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_destroy_ctrl() argument
2125 if (!ctrl->tctrl) in nvmet_pci_epf_destroy_ctrl()
2128 dev_info(ctrl->dev, "Destroying PCI ctrl \"%s\"\n", in nvmet_pci_epf_destroy_ctrl()
2129 ctrl->tctrl->subsys->subsysnqn); in nvmet_pci_epf_destroy_ctrl()
2131 nvmet_pci_epf_stop_ctrl(ctrl); in nvmet_pci_epf_destroy_ctrl()
2133 nvmet_pci_epf_free_queues(ctrl); in nvmet_pci_epf_destroy_ctrl()
2134 nvmet_pci_epf_free_irq_vectors(ctrl); in nvmet_pci_epf_destroy_ctrl()
2136 nvmet_ctrl_put(ctrl->tctrl); in nvmet_pci_epf_destroy_ctrl()
2137 ctrl->tctrl = NULL; in nvmet_pci_epf_destroy_ctrl()
2139 mempool_exit(&ctrl->iod_pool); in nvmet_pci_epf_destroy_ctrl()
2276 struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl; in nvmet_pci_epf_epc_init() local
2318 epf->header->vendorid = ctrl->tctrl->subsys->vendor_id; in nvmet_pci_epf_epc_init()
2319 epf->header->subsys_vendor_id = ctrl->tctrl->subsys->subsys_vendor_id; in nvmet_pci_epf_epc_init()
2344 nvmet_pci_epf_start_ctrl(&nvme_epf->ctrl); in nvmet_pci_epf_epc_init()
2351 nvmet_pci_epf_destroy_ctrl(&nvme_epf->ctrl); in nvmet_pci_epf_epc_init()
2358 struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl; in nvmet_pci_epf_epc_deinit() local
2360 nvmet_pci_epf_destroy_ctrl(ctrl); in nvmet_pci_epf_epc_deinit()
2369 struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl; in nvmet_pci_epf_link_up() local
2371 nvmet_pci_epf_start_ctrl(ctrl); in nvmet_pci_epf_link_up()
2379 struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl; in nvmet_pci_epf_link_down() local
2381 nvmet_pci_epf_stop_ctrl(ctrl); in nvmet_pci_epf_link_down()
2424 nvmet_pci_epf_destroy_ctrl(&nvme_epf->ctrl); in nvmet_pci_epf_unbind()
2486 if (nvme_epf->ctrl.tctrl) in nvmet_pci_epf_portid_store()
2518 if (nvme_epf->ctrl.tctrl) in nvmet_pci_epf_subsysnqn_store()
2547 if (nvme_epf->ctrl.tctrl) in nvmet_pci_epf_mdts_kb_store()