884 	struct ssam_controller *ctrl = to_ssam_controller(kref, kref);  in __ssam_controller_release()  local
891 	ssam_controller_lock(ctrl);  in __ssam_controller_release()
892 	ssam_controller_destroy(ctrl);  in __ssam_controller_release()
893 	ssam_controller_unlock(ctrl);  in __ssam_controller_release()
895 	kfree(ctrl);  in __ssam_controller_release()
988 	struct ssam_controller *ctrl = to_ssam_controller(rtl, rtl);  in ssam_handle_event()  local
1002 	if (WARN_ON(ssam_cplt_submit_event(&ctrl->cplt, item)))  in ssam_handle_event()
1010 static bool ssam_notifier_is_empty(struct ssam_controller *ctrl);
1011 static void ssam_notifier_unregister_all(struct ssam_controller *ctrl);
1204 int ssam_controller_init(struct ssam_controller *ctrl,  in ssam_controller_init()  argument
1209 	init_rwsem(&ctrl->lock);  in ssam_controller_init()
1210 	kref_init(&ctrl->kref);  in ssam_controller_init()
1212 	status = ssam_controller_caps_load(&serdev->dev, &ctrl->caps);  in ssam_controller_init()
1223 		ctrl->caps.ssh_power_profile,  in ssam_controller_init()
1224 		ctrl->caps.ssh_buffer_size,  in ssam_controller_init()
1225 		ctrl->caps.screen_on_sleep_idle_timeout,  in ssam_controller_init()
1226 		ctrl->caps.screen_off_sleep_idle_timeout,  in ssam_controller_init()
1227 		ctrl->caps.d3_closes_handle);  in ssam_controller_init()
1229 	ssh_seq_reset(&ctrl->counter.seq);  in ssam_controller_init()
1230 	ssh_rqid_reset(&ctrl->counter.rqid);  in ssam_controller_init()
1233 	status = ssam_cplt_init(&ctrl->cplt, &serdev->dev);  in ssam_controller_init()
1238 	status = ssh_rtl_init(&ctrl->rtl, serdev, &ssam_rtl_ops);  in ssam_controller_init()
1240 		ssam_cplt_destroy(&ctrl->cplt);  in ssam_controller_init()
1249 	WRITE_ONCE(ctrl->state, SSAM_CONTROLLER_INITIALIZED);  in ssam_controller_init()
1265 int ssam_controller_start(struct ssam_controller *ctrl)  in ssam_controller_start()  argument
1269 	lockdep_assert_held_write(&ctrl->lock);  in ssam_controller_start()
1271 	if (ctrl->state != SSAM_CONTROLLER_INITIALIZED)  in ssam_controller_start()
1274 	status = ssh_rtl_start(&ctrl->rtl);  in ssam_controller_start()
1283 	WRITE_ONCE(ctrl->state, SSAM_CONTROLLER_STARTED);  in ssam_controller_start()
1319 void ssam_controller_shutdown(struct ssam_controller *ctrl)  in ssam_controller_shutdown()  argument
1321 	enum ssam_controller_state s = ctrl->state;  in ssam_controller_shutdown()
1324 	lockdep_assert_held_write(&ctrl->lock);  in ssam_controller_shutdown()
1336 	status = ssh_rtl_flush(&ctrl->rtl, SSAM_CTRL_SHUTDOWN_FLUSH_TIMEOUT);  in ssam_controller_shutdown()
1338 		ssam_err(ctrl, "failed to flush request transport layer: %d\n",  in ssam_controller_shutdown()
1343 	ssam_cplt_flush(&ctrl->cplt);  in ssam_controller_shutdown()
1350 	WARN_ON(!ssam_notifier_is_empty(ctrl));  in ssam_controller_shutdown()
1356 	ssam_notifier_unregister_all(ctrl);  in ssam_controller_shutdown()
1362 	ssh_rtl_shutdown(&ctrl->rtl);  in ssam_controller_shutdown()
1369 	WRITE_ONCE(ctrl->state, SSAM_CONTROLLER_STOPPED);  in ssam_controller_shutdown()
1370 	ctrl->rtl.ptl.serdev = NULL;  in ssam_controller_shutdown()
1388 void ssam_controller_destroy(struct ssam_controller *ctrl)  in ssam_controller_destroy()  argument
1390 	lockdep_assert_held_write(&ctrl->lock);  in ssam_controller_destroy()
1392 	if (ctrl->state == SSAM_CONTROLLER_UNINITIALIZED)  in ssam_controller_destroy()
1395 	WARN_ON(ctrl->state != SSAM_CONTROLLER_STOPPED &&  in ssam_controller_destroy()
1396 		ctrl->state != SSAM_CONTROLLER_INITIALIZED);  in ssam_controller_destroy()
1407 	ssam_cplt_destroy(&ctrl->cplt);  in ssam_controller_destroy()
1408 	ssh_rtl_destroy(&ctrl->rtl);  in ssam_controller_destroy()
1415 	WRITE_ONCE(ctrl->state, SSAM_CONTROLLER_UNINITIALIZED);  in ssam_controller_destroy()
1431 int ssam_controller_suspend(struct ssam_controller *ctrl)  in ssam_controller_suspend()  argument
1433 	ssam_controller_lock(ctrl);  in ssam_controller_suspend()
1435 	if (ctrl->state != SSAM_CONTROLLER_STARTED) {  in ssam_controller_suspend()
1436 		ssam_controller_unlock(ctrl);  in ssam_controller_suspend()
1440 	ssam_dbg(ctrl, "pm: suspending controller\n");  in ssam_controller_suspend()
1446 	WRITE_ONCE(ctrl->state, SSAM_CONTROLLER_SUSPENDED);  in ssam_controller_suspend()
1448 	ssam_controller_unlock(ctrl);  in ssam_controller_suspend()
1463 int ssam_controller_resume(struct ssam_controller *ctrl)  in ssam_controller_resume()  argument
1465 	ssam_controller_lock(ctrl);  in ssam_controller_resume()
1467 	if (ctrl->state != SSAM_CONTROLLER_SUSPENDED) {  in ssam_controller_resume()
1468 		ssam_controller_unlock(ctrl);  in ssam_controller_resume()
1472 	ssam_dbg(ctrl, "pm: resuming controller\n");  in ssam_controller_resume()
1478 	WRITE_ONCE(ctrl->state, SSAM_CONTROLLER_STARTED);  in ssam_controller_resume()
1480 	ssam_controller_unlock(ctrl);  in ssam_controller_resume()
1508 				struct ssam_controller *ctrl,  in ssam_request_write_data()  argument
1522 	seq = ssh_seq_next(&ctrl->counter.seq);  in ssam_request_write_data()
1523 	rqid = ssh_rqid_next(&ctrl->counter.rqid);  in ssam_request_write_data()
1684 int ssam_request_sync_submit(struct ssam_controller *ctrl,  in ssam_request_sync_submit()  argument
1703 	if (WARN_ON(READ_ONCE(ctrl->state) != SSAM_CONTROLLER_STARTED)) {  in ssam_request_sync_submit()
1708 	status = ssh_rtl_submit(&ctrl->rtl, &rqst->base);  in ssam_request_sync_submit()
1728 int ssam_request_do_sync(struct ssam_controller *ctrl,  in ssam_request_do_sync()  argument
1749 	len = ssam_request_write_data(&buf, ctrl, spec);  in ssam_request_do_sync()
1757 	status = ssam_request_sync_submit(ctrl, rqst);  in ssam_request_do_sync()
1787 int ssam_request_do_sync_with_buffer(struct ssam_controller *ctrl,  in ssam_request_do_sync_with_buffer()  argument
1802 	len = ssam_request_write_data(buf, ctrl, spec);  in ssam_request_do_sync_with_buffer()
1808 	status = ssam_request_sync_submit(ctrl, &rqst);  in ssam_request_do_sync_with_buffer()
1873 static int __ssam_ssh_event_request(struct ssam_controller *ctrl,  in __ssam_ssh_event_request()  argument
1906 	status = ssam_retry(ssam_request_do_sync_onstack, ctrl, &rqst, &result,  in __ssam_ssh_event_request()
1929 static int ssam_ssh_event_enable(struct ssam_controller *ctrl,  in ssam_ssh_event_enable()  argument
1935 	status = __ssam_ssh_event_request(ctrl, reg, reg.cid_enable, id, flags);  in ssam_ssh_event_enable()
1938 		ssam_err(ctrl,  in ssam_ssh_event_enable()
1944 		ssam_err(ctrl,  in ssam_ssh_event_enable()
1970 static int ssam_ssh_event_disable(struct ssam_controller *ctrl,  in ssam_ssh_event_disable()  argument
1976 	status = __ssam_ssh_event_request(ctrl, reg, reg.cid_disable, id, flags);  in ssam_ssh_event_disable()
1979 		ssam_err(ctrl,  in ssam_ssh_event_disable()
1985 		ssam_err(ctrl,  in ssam_ssh_event_disable()
2005 int ssam_get_firmware_version(struct ssam_controller *ctrl, u32 *version)  in ssam_get_firmware_version()  argument
2010 	status = ssam_retry(ssam_ssh_get_firmware_version, ctrl, &__version);  in ssam_get_firmware_version()
2045 int ssam_ctrl_notif_display_off(struct ssam_controller *ctrl)  in ssam_ctrl_notif_display_off()  argument
2050 	ssam_dbg(ctrl, "pm: notifying display off\n");  in ssam_ctrl_notif_display_off()
2052 	status = ssam_retry(ssam_ssh_notif_display_off, ctrl, &response);  in ssam_ctrl_notif_display_off()
2057 		ssam_err(ctrl, "unexpected response from display-off notification: %#04x\n",  in ssam_ctrl_notif_display_off()
2084 int ssam_ctrl_notif_display_on(struct ssam_controller *ctrl)  in ssam_ctrl_notif_display_on()  argument
2089 	ssam_dbg(ctrl, "pm: notifying display on\n");  in ssam_ctrl_notif_display_on()
2091 	status = ssam_retry(ssam_ssh_notif_display_on, ctrl, &response);  in ssam_ctrl_notif_display_on()
2096 		ssam_err(ctrl, "unexpected response from display-on notification: %#04x\n",  in ssam_ctrl_notif_display_on()
2123 int ssam_ctrl_notif_d0_exit(struct ssam_controller *ctrl)  in ssam_ctrl_notif_d0_exit()  argument
2128 	if (!ctrl->caps.d3_closes_handle)  in ssam_ctrl_notif_d0_exit()
2131 	ssam_dbg(ctrl, "pm: notifying D0 exit\n");  in ssam_ctrl_notif_d0_exit()
2133 	status = ssam_retry(ssam_ssh_notif_d0_exit, ctrl, &response);  in ssam_ctrl_notif_d0_exit()
2138 		ssam_err(ctrl, "unexpected response from D0-exit notification: %#04x\n",  in ssam_ctrl_notif_d0_exit()
2165 int ssam_ctrl_notif_d0_entry(struct ssam_controller *ctrl)  in ssam_ctrl_notif_d0_entry()  argument
2170 	if (!ctrl->caps.d3_closes_handle)  in ssam_ctrl_notif_d0_entry()
2173 	ssam_dbg(ctrl, "pm: notifying D0 entry\n");  in ssam_ctrl_notif_d0_entry()
2175 	status = ssam_retry(ssam_ssh_notif_d0_entry, ctrl, &response);  in ssam_ctrl_notif_d0_entry()
2180 		ssam_err(ctrl, "unexpected response from D0-entry notification: %#04x\n",  in ssam_ctrl_notif_d0_entry()
2212 static int ssam_nf_refcount_enable(struct ssam_controller *ctrl,  in ssam_nf_refcount_enable()  argument
2217 	struct ssam_nf *nf = &ctrl->cplt.event.notif;  in ssam_nf_refcount_enable()
2222 	ssam_dbg(ctrl, "enabling event (reg: %#04x, tc: %#04x, iid: %#04x, rc: %d)\n",  in ssam_nf_refcount_enable()
2226 		status = ssam_ssh_event_enable(ctrl, reg, id, flags);  in ssam_nf_refcount_enable()
2233 		ssam_warn(ctrl,  in ssam_nf_refcount_enable()
2276 static int ssam_nf_refcount_disable_free(struct ssam_controller *ctrl,  in ssam_nf_refcount_disable_free()  argument
2281 	struct ssam_nf *nf = &ctrl->cplt.event.notif;  in ssam_nf_refcount_disable_free()
2286 	ssam_dbg(ctrl, "%s event (reg: %#04x, tc: %#04x, iid: %#04x, rc: %d)\n",  in ssam_nf_refcount_disable_free()
2291 		ssam_warn(ctrl,  in ssam_nf_refcount_disable_free()
2298 		status = ssam_ssh_event_disable(ctrl, reg, id, flags);  in ssam_nf_refcount_disable_free()
2327 int ssam_notifier_register(struct ssam_controller *ctrl, struct ssam_event_notifier *n)  in ssam_notifier_register()  argument
2338 	nf = &ctrl->cplt.event.notif;  in ssam_notifier_register()
2361 		status = ssam_nf_refcount_enable(ctrl, entry, n->event.flags);  in ssam_notifier_register()
2395 int __ssam_notifier_unregister(struct ssam_controller *ctrl, struct ssam_event_notifier *n,  in __ssam_notifier_unregister()  argument
2407 	nf = &ctrl->cplt.event.notif;  in __ssam_notifier_unregister()
2434 		status = ssam_nf_refcount_disable_free(ctrl, entry, n->event.flags, disable);  in __ssam_notifier_unregister()
2467 int ssam_controller_event_enable(struct ssam_controller *ctrl,  in ssam_controller_event_enable()  argument
2472 	struct ssam_nf *nf = &ctrl->cplt.event.notif;  in ssam_controller_event_enable()
2487 	status = ssam_nf_refcount_enable(ctrl, entry, flags);  in ssam_controller_event_enable()
2518 int ssam_controller_event_disable(struct ssam_controller *ctrl,  in ssam_controller_event_disable()  argument
2523 	struct ssam_nf *nf = &ctrl->cplt.event.notif;  in ssam_controller_event_disable()
2538 	status = ssam_nf_refcount_disable_free(ctrl, entry, flags, true);  in ssam_controller_event_disable()
2566 int ssam_notifier_disable_registered(struct ssam_controller *ctrl)  in ssam_notifier_disable_registered()  argument
2568 	struct ssam_nf *nf = &ctrl->cplt.event.notif;  in ssam_notifier_disable_registered()
2577 		status = ssam_ssh_event_disable(ctrl, e->key.reg,  in ssam_notifier_disable_registered()
2591 		ssam_ssh_event_enable(ctrl, e->key.reg, e->key.id, e->flags);  in ssam_notifier_disable_registered()
2611 void ssam_notifier_restore_registered(struct ssam_controller *ctrl)  in ssam_notifier_restore_registered()  argument
2613 	struct ssam_nf *nf = &ctrl->cplt.event.notif;  in ssam_notifier_restore_registered()
2623 		ssam_ssh_event_enable(ctrl, e->key.reg, e->key.id, e->flags);  in ssam_notifier_restore_registered()
2635 static bool ssam_notifier_is_empty(struct ssam_controller *ctrl)  in ssam_notifier_is_empty()  argument
2637 	struct ssam_nf *nf = &ctrl->cplt.event.notif;  in ssam_notifier_is_empty()
2656 static void ssam_notifier_unregister_all(struct ssam_controller *ctrl)  in ssam_notifier_unregister_all()  argument
2658 	struct ssam_nf *nf = &ctrl->cplt.event.notif;  in ssam_notifier_unregister_all()
2664 		ssam_ssh_event_disable(ctrl, e->key.reg, e->key.id, e->flags);  in ssam_notifier_unregister_all()
2676 	struct ssam_controller *ctrl = dev_id;  in ssam_irq_handle()  local
2678 	ssam_dbg(ctrl, "pm: wake irq triggered\n");  in ssam_irq_handle()
2736 int ssam_irq_setup(struct ssam_controller *ctrl)  in ssam_irq_setup()  argument
2738 	struct device *dev = ssam_controller_device(ctrl);  in ssam_irq_setup()
2768 				      "ssam_wakeup", ctrl);  in ssam_irq_setup()
2772 	ctrl->irq.num = irq;  in ssam_irq_setup()
2782 void ssam_irq_free(struct ssam_controller *ctrl)  in ssam_irq_free()  argument
2784 	free_irq(ctrl->irq.num, ctrl);  in ssam_irq_free()
2785 	ctrl->irq.num = -1;  in ssam_irq_free()
2802 int ssam_irq_arm_for_wakeup(struct ssam_controller *ctrl)  in ssam_irq_arm_for_wakeup()  argument
2804 	struct device *dev = ssam_controller_device(ctrl);  in ssam_irq_arm_for_wakeup()
2807 	enable_irq(ctrl->irq.num);  in ssam_irq_arm_for_wakeup()
2809 		status = enable_irq_wake(ctrl->irq.num);  in ssam_irq_arm_for_wakeup()
2811 			ssam_err(ctrl, "failed to enable wake IRQ: %d\n", status);  in ssam_irq_arm_for_wakeup()
2812 			disable_irq(ctrl->irq.num);  in ssam_irq_arm_for_wakeup()
2816 		ctrl->irq.wakeup_enabled = true;  in ssam_irq_arm_for_wakeup()
2818 		ctrl->irq.wakeup_enabled = false;  in ssam_irq_arm_for_wakeup()
2835 void ssam_irq_disarm_wakeup(struct ssam_controller *ctrl)  in ssam_irq_disarm_wakeup()  argument
2839 	if (ctrl->irq.wakeup_enabled) {  in ssam_irq_disarm_wakeup()
2840 		status = disable_irq_wake(ctrl->irq.num);  in ssam_irq_disarm_wakeup()
2842 			ssam_err(ctrl, "failed to disable wake IRQ: %d\n", status);  in ssam_irq_disarm_wakeup()
2844 		ctrl->irq.wakeup_enabled = false;  in ssam_irq_disarm_wakeup()
2846 	disable_irq(ctrl->irq.num);  in ssam_irq_disarm_wakeup()