Lines Matching refs:ctrl
141 struct nvme_ctrl ctrl; member
183 static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl) in to_nvme_dev() argument
185 return container_of(ctrl, struct nvme_dev, ctrl); in to_nvme_dev()
257 if (!(dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP)) in nvme_dbbuf_dma_alloc()
341 if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) { in nvme_dbbuf_set()
342 dev_warn(dev->ctrl.device, "unable to set dbbuf\n"); in nvme_dbbuf_set()
430 nvme_req(req)->ctrl = set->driver_data; in nvme_pci_init_request()
515 if (!nvme_ctrl_meta_sgl_supported(&dev->ctrl)) in nvme_pci_metadata_use_sgls()
529 if (!nvme_ctrl_sgl_supported(&dev->ctrl)) in nvme_pci_use_sgls()
804 nvme_ctrl_sgl_supported(&dev->ctrl)) in nvme_map_data()
972 if (unlikely(!nvme_check_ready(&dev->ctrl, req, true))) in nvme_queue_rq()
973 return nvme_fail_nonready_command(&dev->ctrl, req); in nvme_queue_rq()
1010 if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true))) in nvme_prep_rq_batch()
1118 nvme_complete_async_event(&nvmeq->dev->ctrl, in nvme_handle_cqe()
1125 dev_warn(nvmeq->dev->ctrl.device, in nvme_handle_cqe()
1226 static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl) in nvme_pci_submit_async_event() argument
1228 struct nvme_dev *dev = to_nvme_dev(ctrl); in nvme_pci_submit_async_event()
1241 static int nvme_pci_subsystem_reset(struct nvme_ctrl *ctrl) in nvme_pci_subsystem_reset() argument
1243 struct nvme_dev *dev = to_nvme_dev(ctrl); in nvme_pci_subsystem_reset()
1258 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) { in nvme_pci_subsystem_reset()
1264 nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE); in nvme_pci_subsystem_reset()
1283 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); in adapter_delete_queue()
1306 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); in adapter_alloc_cq()
1312 struct nvme_ctrl *ctrl = &dev->ctrl; in adapter_alloc_sq() local
1321 if (ctrl->quirks & NVME_QUIRK_MEDIUM_PRIO_SQ) in adapter_alloc_sq()
1335 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); in adapter_alloc_sq()
1352 dev_warn(nvmeq->dev->ctrl.device, in abort_endio()
1354 atomic_inc(&nvmeq->dev->ctrl.abort_limit); in abort_endio()
1367 switch (nvme_ctrl_state(&dev->ctrl)) { in nvme_should_reset()
1393 dev_warn(dev->ctrl.device, in nvme_warn_reset()
1397 dev_warn(dev->ctrl.device, in nvme_warn_reset()
1404 dev_warn(dev->ctrl.device, in nvme_warn_reset()
1406 dev_warn(dev->ctrl.device, in nvme_warn_reset()
1430 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); in nvme_timeout()
1431 if (nvme_state_terminal(&dev->ctrl)) in nvme_timeout()
1458 dev_warn(dev->ctrl.device, in nvme_timeout()
1470 switch (nvme_ctrl_state(&dev->ctrl)) { in nvme_timeout()
1472 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); in nvme_timeout()
1475 dev_warn_ratelimited(dev->ctrl.device, in nvme_timeout()
1494 dev_warn(dev->ctrl.device, in nvme_timeout()
1502 if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) { in nvme_timeout()
1503 atomic_inc(&dev->ctrl.abort_limit); in nvme_timeout()
1512 dev_warn(nvmeq->dev->ctrl.device, in nvme_timeout()
1518 abort_req = blk_mq_alloc_request(dev->ctrl.admin_q, nvme_req_op(&cmd), in nvme_timeout()
1521 atomic_inc(&dev->ctrl.abort_limit); in nvme_timeout()
1538 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) { in nvme_timeout()
1539 if (nvme_state_terminal(&dev->ctrl)) in nvme_timeout()
1545 if (nvme_try_sched_reset(&dev->ctrl)) in nvme_timeout()
1546 nvme_unquiesce_io_queues(&dev->ctrl); in nvme_timeout()
1570 for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) { in nvme_free_queues()
1571 dev->ctrl.queue_count--; in nvme_free_queues()
1587 if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q) in nvme_suspend_queue()
1588 nvme_quiesce_admin_queue(&nvmeq->dev->ctrl); in nvme_suspend_queue()
1597 for (i = dev->ctrl.queue_count - 1; i > 0; i--) in nvme_suspend_io_queues()
1611 for (i = dev->ctrl.queue_count - 1; i > 0; i--) { in nvme_reap_pending_cqes()
1673 if (dev->ctrl.queue_count > qid) in nvme_alloc_queue()
1693 dev->ctrl.queue_count++; in nvme_alloc_queue()
1707 int nr = nvmeq->dev->ctrl.instance; in queue_request_irq()
1747 if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_CONNECTING) { in nvme_setup_io_queues_trylock()
1829 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) { in nvme_dev_remove_admin()
1835 nvme_unquiesce_admin_queue(&dev->ctrl); in nvme_dev_remove_admin()
1836 nvme_remove_admin_tag_set(&dev->ctrl); in nvme_dev_remove_admin()
1877 NVME_CAP_NSSRC(dev->ctrl.cap) : 0; in nvme_pci_configure_admin_queue()
1890 result = nvme_disable_ctrl(&dev->ctrl, false); in nvme_pci_configure_admin_queue()
1898 dev->ctrl.numa_node = dev_to_node(dev->dev); in nvme_pci_configure_admin_queue()
1908 result = nvme_enable_ctrl(&dev->ctrl); in nvme_pci_configure_admin_queue()
1929 for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) { in nvme_create_io_queues()
1936 max = min(dev->max_qid, dev->ctrl.queue_count - 1); in nvme_create_io_queues()
1983 if (NVME_CAP_CMBS(dev->ctrl.cap)) in nvme_map_cmb()
2015 if (NVME_CAP_CMBS(dev->ctrl.cap)) { in nvme_map_cmb()
2022 dev_warn(dev->ctrl.device, in nvme_map_cmb()
2053 ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); in nvme_set_host_mem()
2055 dev_warn(dev->ctrl.device, in nvme_set_host_mem()
2136 if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries) in nvme_alloc_host_mem_multi()
2137 max_entries = dev->ctrl.hmmaxd; in nvme_alloc_host_mem_multi()
2187 u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2); in nvme_alloc_host_mem()
2214 u64 preferred = (u64)dev->ctrl.hmpre * 4096; in nvme_setup_host_mem()
2215 u64 min = (u64)dev->ctrl.hmmin * 4096; in nvme_setup_host_mem()
2219 if (!dev->ctrl.hmpre) in nvme_setup_host_mem()
2224 dev_warn(dev->ctrl.device, in nvme_setup_host_mem()
2243 dev_warn(dev->ctrl.device, in nvme_setup_host_mem()
2248 dev_info(dev->ctrl.device, in nvme_setup_host_mem()
2328 struct nvme_ctrl *ctrl = in nvme_pci_attrs_are_visible() local
2330 struct nvme_dev *dev = to_nvme_dev(ctrl); in nvme_pci_attrs_are_visible()
2338 if (a == &dev_attr_hmb.attr && !ctrl->hmpre) in nvme_pci_attrs_are_visible()
2365 sysfs_update_group(&dev->ctrl.device->kobj, &nvme_pci_dev_attrs_group); in nvme_update_attrs()
2437 if (!(dev->ctrl.quirks & NVME_QUIRK_SINGLE_VECTOR)) in nvme_setup_irqs()
2439 if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI) in nvme_setup_irqs()
2451 if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) in nvme_max_io_queues()
2472 result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues); in nvme_setup_io_queues()
2497 dev->ctrl.sqsize = result - 1; in nvme_setup_io_queues()
2561 dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n", in nvme_setup_io_queues()
2594 struct request_queue *q = nvmeq->dev->ctrl.admin_q; in nvme_delete_queue()
2662 if (!dev->ctrl.tagset) { in nvme_pci_update_nr_queues()
2663 nvme_alloc_io_tag_set(&dev->ctrl, &dev->tagset, &nvme_mq_ops, in nvme_pci_update_nr_queues()
2706 if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI) in nvme_pci_enable()
2712 dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP); in nvme_pci_enable()
2714 dev->q_depth = min_t(u32, NVME_CAP_MQES(dev->ctrl.cap) + 1, in nvme_pci_enable()
2716 dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap); in nvme_pci_enable()
2724 if (dev->ctrl.quirks & NVME_QUIRK_128_BYTES_SQES) in nvme_pci_enable()
2729 if (dev->ctrl.quirks & NVME_QUIRK_QDEPTH_ONE) { in nvme_pci_enable()
2733 NVME_CAP_MQES(dev->ctrl.cap) == 0) { in nvme_pci_enable()
2735 dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, " in nvme_pci_enable()
2743 if ((dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) && in nvme_pci_enable()
2746 dev_warn(dev->ctrl.device, "IO queue depth clamped to %d\n", in nvme_pci_enable()
2749 dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */ in nvme_pci_enable()
2790 enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); in nvme_dev_disable()
2798 nvme_start_freeze(&dev->ctrl); in nvme_dev_disable()
2804 nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT); in nvme_dev_disable()
2807 nvme_quiesce_io_queues(&dev->ctrl); in nvme_dev_disable()
2809 if (!dead && dev->ctrl.queue_count > 0) { in nvme_dev_disable()
2811 nvme_disable_ctrl(&dev->ctrl, shutdown); in nvme_dev_disable()
2821 nvme_cancel_tagset(&dev->ctrl); in nvme_dev_disable()
2822 nvme_cancel_admin_tagset(&dev->ctrl); in nvme_dev_disable()
2830 nvme_unquiesce_io_queues(&dev->ctrl); in nvme_dev_disable()
2831 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) in nvme_dev_disable()
2832 nvme_unquiesce_admin_queue(&dev->ctrl); in nvme_dev_disable()
2839 if (!nvme_wait_reset(&dev->ctrl)) in nvme_disable_prepare_reset()
2855 if (dev->ctrl.quirks & NVME_QUIRK_DMAPOOL_ALIGN_512) in nvme_setup_prp_pools()
2902 nvme_remove_io_tag_set(&dev->ctrl); in nvme_free_tagset()
2903 dev->ctrl.tagset = NULL; in nvme_free_tagset()
2907 static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl) in nvme_pci_free_ctrl() argument
2909 struct nvme_dev *dev = to_nvme_dev(ctrl); in nvme_pci_free_ctrl()
2920 container_of(work, struct nvme_dev, ctrl.reset_work); in nvme_reset_work()
2921 bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL); in nvme_reset_work()
2924 if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_RESETTING) { in nvme_reset_work()
2925 dev_warn(dev->ctrl.device, "ctrl state %d is not RESETTING\n", in nvme_reset_work()
2926 dev->ctrl.state); in nvme_reset_work()
2935 if (dev->ctrl.ctrl_config & NVME_CC_ENABLE) in nvme_reset_work()
2937 nvme_sync_queues(&dev->ctrl); in nvme_reset_work()
2943 nvme_unquiesce_admin_queue(&dev->ctrl); in nvme_reset_work()
2950 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) { in nvme_reset_work()
2951 dev_warn(dev->ctrl.device, in nvme_reset_work()
2957 result = nvme_init_ctrl_finish(&dev->ctrl, was_suspend); in nvme_reset_work()
2961 if (nvme_ctrl_meta_sgl_supported(&dev->ctrl)) in nvme_reset_work()
2962 dev->ctrl.max_integrity_segments = NVME_MAX_META_SEGS; in nvme_reset_work()
2964 dev->ctrl.max_integrity_segments = 1; in nvme_reset_work()
2983 nvme_unquiesce_io_queues(&dev->ctrl); in nvme_reset_work()
2984 nvme_wait_freeze(&dev->ctrl); in nvme_reset_work()
2987 nvme_unfreeze(&dev->ctrl); in nvme_reset_work()
2989 dev_warn(dev->ctrl.device, "IO queues lost\n"); in nvme_reset_work()
2990 nvme_mark_namespaces_dead(&dev->ctrl); in nvme_reset_work()
2991 nvme_unquiesce_io_queues(&dev->ctrl); in nvme_reset_work()
2992 nvme_remove_namespaces(&dev->ctrl); in nvme_reset_work()
3000 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) { in nvme_reset_work()
3001 dev_warn(dev->ctrl.device, in nvme_reset_work()
3007 nvme_start_ctrl(&dev->ctrl); in nvme_reset_work()
3017 dev_warn(dev->ctrl.device, "Disabling device after reset failure: %d\n", in nvme_reset_work()
3019 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); in nvme_reset_work()
3021 nvme_sync_queues(&dev->ctrl); in nvme_reset_work()
3022 nvme_mark_namespaces_dead(&dev->ctrl); in nvme_reset_work()
3023 nvme_unquiesce_io_queues(&dev->ctrl); in nvme_reset_work()
3024 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD); in nvme_reset_work()
3027 static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val) in nvme_pci_reg_read32() argument
3029 *val = readl(to_nvme_dev(ctrl)->bar + off); in nvme_pci_reg_read32()
3033 static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val) in nvme_pci_reg_write32() argument
3035 writel(val, to_nvme_dev(ctrl)->bar + off); in nvme_pci_reg_write32()
3039 static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val) in nvme_pci_reg_read64() argument
3041 *val = lo_hi_readq(to_nvme_dev(ctrl)->bar + off); in nvme_pci_reg_read64()
3045 static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size) in nvme_pci_get_address() argument
3047 struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev); in nvme_pci_get_address()
3052 static void nvme_pci_print_device_info(struct nvme_ctrl *ctrl) in nvme_pci_print_device_info() argument
3054 struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev); in nvme_pci_print_device_info()
3055 struct nvme_subsystem *subsys = ctrl->subsys; in nvme_pci_print_device_info()
3057 dev_err(ctrl->device, in nvme_pci_print_device_info()
3066 static bool nvme_pci_supports_pci_p2pdma(struct nvme_ctrl *ctrl) in nvme_pci_supports_pci_p2pdma() argument
3068 struct nvme_dev *dev = to_nvme_dev(ctrl); in nvme_pci_supports_pci_p2pdma()
3191 INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work); in nvme_pci_alloc_dev()
3216 ret = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops, in nvme_pci_alloc_dev()
3221 if (dev->ctrl.quirks & NVME_QUIRK_DMA_ADDRESS_BITS_48) in nvme_pci_alloc_dev()
3232 dev->ctrl.max_hw_sectors = min_t(u32, in nvme_pci_alloc_dev()
3234 dev->ctrl.max_segments = NVME_MAX_SEGS; in nvme_pci_alloc_dev()
3235 dev->ctrl.max_integrity_segments = 1; in nvme_pci_alloc_dev()
3255 result = nvme_add_ctrl(&dev->ctrl); in nvme_probe()
3271 dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev)); in nvme_probe()
3277 result = nvme_alloc_admin_tag_set(&dev->ctrl, &dev->admin_tagset, in nvme_probe()
3286 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) { in nvme_probe()
3287 dev_warn(dev->ctrl.device, in nvme_probe()
3293 result = nvme_init_ctrl_finish(&dev->ctrl, false); in nvme_probe()
3297 if (nvme_ctrl_meta_sgl_supported(&dev->ctrl)) in nvme_probe()
3298 dev->ctrl.max_integrity_segments = NVME_MAX_META_SEGS; in nvme_probe()
3300 dev->ctrl.max_integrity_segments = 1; in nvme_probe()
3313 nvme_alloc_io_tag_set(&dev->ctrl, &dev->tagset, &nvme_mq_ops, in nvme_probe()
3318 if (!dev->ctrl.tagset) in nvme_probe()
3319 dev_warn(dev->ctrl.device, "IO queues not created\n"); in nvme_probe()
3321 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) { in nvme_probe()
3322 dev_warn(dev->ctrl.device, in nvme_probe()
3330 nvme_start_ctrl(&dev->ctrl); in nvme_probe()
3331 nvme_put_ctrl(&dev->ctrl); in nvme_probe()
3332 flush_work(&dev->ctrl.scan_work); in nvme_probe()
3336 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); in nvme_probe()
3350 nvme_uninit_ctrl(&dev->ctrl); in nvme_probe()
3352 nvme_put_ctrl(&dev->ctrl); in nvme_probe()
3366 nvme_sync_queues(&dev->ctrl); in nvme_reset_prepare()
3373 if (!nvme_try_sched_reset(&dev->ctrl)) in nvme_reset_done()
3374 flush_work(&dev->ctrl.reset_work); in nvme_reset_done()
3393 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); in nvme_remove()
3397 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD); in nvme_remove()
3401 flush_work(&dev->ctrl.reset_work); in nvme_remove()
3402 nvme_stop_ctrl(&dev->ctrl); in nvme_remove()
3403 nvme_remove_namespaces(&dev->ctrl); in nvme_remove()
3413 nvme_uninit_ctrl(&dev->ctrl); in nvme_remove()
3417 static int nvme_get_power_state(struct nvme_ctrl *ctrl, u32 *ps) in nvme_get_power_state() argument
3419 return nvme_get_features(ctrl, NVME_FEAT_POWER_MGMT, 0, NULL, 0, ps); in nvme_get_power_state()
3422 static int nvme_set_power_state(struct nvme_ctrl *ctrl, u32 ps) in nvme_set_power_state() argument
3424 return nvme_set_features(ctrl, NVME_FEAT_POWER_MGMT, ps, NULL, 0, NULL); in nvme_set_power_state()
3430 struct nvme_ctrl *ctrl = &ndev->ctrl; in nvme_resume() local
3433 nvme_set_power_state(ctrl, ndev->last_ps) != 0) in nvme_resume()
3435 if (ctrl->hmpre && nvme_setup_host_mem(ndev)) in nvme_resume()
3440 return nvme_try_sched_reset(ctrl); in nvme_resume()
3447 struct nvme_ctrl *ctrl = &ndev->ctrl; in nvme_suspend() local
3465 if (pm_suspend_via_firmware() || !ctrl->npss || in nvme_suspend()
3467 (ndev->ctrl.quirks & NVME_QUIRK_SIMPLE_SUSPEND)) in nvme_suspend()
3470 nvme_start_freeze(ctrl); in nvme_suspend()
3471 nvme_wait_freeze(ctrl); in nvme_suspend()
3472 nvme_sync_queues(ctrl); in nvme_suspend()
3474 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) in nvme_suspend()
3488 ret = nvme_get_power_state(ctrl, &ndev->last_ps); in nvme_suspend()
3499 ret = nvme_set_power_state(ctrl, ctrl->npss); in nvme_suspend()
3512 ctrl->npss = 0; in nvme_suspend()
3515 nvme_unfreeze(ctrl); in nvme_suspend()
3531 return nvme_try_sched_reset(&ndev->ctrl); in nvme_simple_resume()
3558 dev_warn(dev->ctrl.device, in nvme_error_detected()
3560 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) { in nvme_error_detected()
3567 dev_warn(dev->ctrl.device, in nvme_error_detected()
3578 dev_info(dev->ctrl.device, "restart after slot reset\n"); in nvme_slot_reset()
3580 if (nvme_try_sched_reset(&dev->ctrl)) in nvme_slot_reset()
3581 nvme_unquiesce_io_queues(&dev->ctrl); in nvme_slot_reset()
3589 flush_work(&dev->ctrl.reset_work); in nvme_error_resume()