22 static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset)  in __rpm_get_callback()  argument
27 	if (dev->pm_domain)  in __rpm_get_callback()
28 		ops = &dev->pm_domain->ops;  in __rpm_get_callback()
29 	else if (dev->type && dev->type->pm)  in __rpm_get_callback()
30 		ops = dev->type->pm;  in __rpm_get_callback()
31 	else if (dev->class && dev->class->pm)  in __rpm_get_callback()
32 		ops = dev->class->pm;  in __rpm_get_callback()
33 	else if (dev->bus && dev->bus->pm)  in __rpm_get_callback()
34 		ops = dev->bus->pm;  in __rpm_get_callback()
43 	if (!cb && dev->driver && dev->driver->pm)  in __rpm_get_callback()
44 		cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset);  in __rpm_get_callback()
49 #define RPM_GET_CALLBACK(dev, callback) \  argument
50 		__rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback))
52 static int rpm_resume(struct device *dev, int rpmflags);
53 static int rpm_suspend(struct device *dev, int rpmflags);
66 static void update_pm_runtime_accounting(struct device *dev)  in update_pm_runtime_accounting()  argument
70 	if (dev->power.disable_depth > 0)  in update_pm_runtime_accounting()
73 	last = dev->power.accounting_timestamp;  in update_pm_runtime_accounting()
76 	dev->power.accounting_timestamp = now;  in update_pm_runtime_accounting()
88 	if (dev->power.runtime_status == RPM_SUSPENDED)  in update_pm_runtime_accounting()
89 		dev->power.suspended_time += delta;  in update_pm_runtime_accounting()
91 		dev->power.active_time += delta;  in update_pm_runtime_accounting()
94 static void __update_runtime_status(struct device *dev, enum rpm_status status)  in __update_runtime_status()  argument
96 	update_pm_runtime_accounting(dev);  in __update_runtime_status()
97 	trace_rpm_status(dev, status);  in __update_runtime_status()
98 	dev->power.runtime_status = status;  in __update_runtime_status()
101 static u64 rpm_get_accounted_time(struct device *dev, bool suspended)  in rpm_get_accounted_time()  argument
106 	spin_lock_irqsave(&dev->power.lock, flags);  in rpm_get_accounted_time()
108 	update_pm_runtime_accounting(dev);  in rpm_get_accounted_time()
109 	time = suspended ? dev->power.suspended_time : dev->power.active_time;  in rpm_get_accounted_time()
111 	spin_unlock_irqrestore(&dev->power.lock, flags);  in rpm_get_accounted_time()
116 u64 pm_runtime_active_time(struct device *dev)  in pm_runtime_active_time()  argument
118 	return rpm_get_accounted_time(dev, false);  in pm_runtime_active_time()
121 u64 pm_runtime_suspended_time(struct device *dev)  in pm_runtime_suspended_time()  argument
123 	return rpm_get_accounted_time(dev, true);  in pm_runtime_suspended_time()
131 static void pm_runtime_deactivate_timer(struct device *dev)  in pm_runtime_deactivate_timer()  argument
133 	if (dev->power.timer_expires > 0) {  in pm_runtime_deactivate_timer()
134 		hrtimer_try_to_cancel(&dev->power.suspend_timer);  in pm_runtime_deactivate_timer()
135 		dev->power.timer_expires = 0;  in pm_runtime_deactivate_timer()
143 static void pm_runtime_cancel_pending(struct device *dev)  in pm_runtime_cancel_pending()  argument
145 	pm_runtime_deactivate_timer(dev);  in pm_runtime_cancel_pending()
150 	dev->power.request = RPM_REQ_NONE;  in pm_runtime_cancel_pending()
165 u64 pm_runtime_autosuspend_expiration(struct device *dev)  in pm_runtime_autosuspend_expiration()  argument
170 	if (!dev->power.use_autosuspend)  in pm_runtime_autosuspend_expiration()
173 	autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay);  in pm_runtime_autosuspend_expiration()
177 	expires  = READ_ONCE(dev->power.last_busy);  in pm_runtime_autosuspend_expiration()
186 static int dev_memalloc_noio(struct device *dev, void *data)  in dev_memalloc_noio()  argument
188 	return dev->power.memalloc_noio;  in dev_memalloc_noio()
219 void pm_runtime_set_memalloc_noio(struct device *dev, bool enable)  in pm_runtime_set_memalloc_noio()  argument
228 		spin_lock_irq(&dev->power.lock);  in pm_runtime_set_memalloc_noio()
229 		enabled = dev->power.memalloc_noio;  in pm_runtime_set_memalloc_noio()
230 		dev->power.memalloc_noio = enable;  in pm_runtime_set_memalloc_noio()
231 		spin_unlock_irq(&dev->power.lock);  in pm_runtime_set_memalloc_noio()
240 		dev = dev->parent;  in pm_runtime_set_memalloc_noio()
247 		if (!dev || (!enable &&  in pm_runtime_set_memalloc_noio()
248 		    device_for_each_child(dev, NULL, dev_memalloc_noio)))  in pm_runtime_set_memalloc_noio()
259 static int rpm_check_suspend_allowed(struct device *dev)  in rpm_check_suspend_allowed()  argument
263 	if (dev->power.runtime_error)  in rpm_check_suspend_allowed()
265 	else if (dev->power.disable_depth > 0)  in rpm_check_suspend_allowed()
267 	else if (atomic_read(&dev->power.usage_count))  in rpm_check_suspend_allowed()
269 	else if (!dev->power.ignore_children && atomic_read(&dev->power.child_count))  in rpm_check_suspend_allowed()
273 	else if ((dev->power.deferred_resume &&  in rpm_check_suspend_allowed()
274 	    dev->power.runtime_status == RPM_SUSPENDING) ||  in rpm_check_suspend_allowed()
275 	    (dev->power.request_pending && dev->power.request == RPM_REQ_RESUME))  in rpm_check_suspend_allowed()
277 	else if (__dev_pm_qos_resume_latency(dev) == 0)  in rpm_check_suspend_allowed()
279 	else if (dev->power.runtime_status == RPM_SUSPENDED)  in rpm_check_suspend_allowed()
285 static int rpm_get_suppliers(struct device *dev)  in rpm_get_suppliers()  argument
289 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,  in rpm_get_suppliers()
328 static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend)  in __rpm_put_suppliers()  argument
332 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,  in __rpm_put_suppliers()
340 static void rpm_put_suppliers(struct device *dev)  in rpm_put_suppliers()  argument
342 	__rpm_put_suppliers(dev, true);  in rpm_put_suppliers()
345 static void rpm_suspend_suppliers(struct device *dev)  in rpm_suspend_suppliers()  argument
350 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,  in rpm_suspend_suppliers()
362 static int __rpm_callback(int (*cb)(struct device *), struct device *dev)  in __rpm_callback()  argument
363 	__releases(&dev->power.lock) __acquires(&dev->power.lock)  in __rpm_callback()
366 	bool use_links = dev->power.links_count > 0;  in __rpm_callback()
368 	if (dev->power.irq_safe) {  in __rpm_callback()
369 		spin_unlock(&dev->power.lock);  in __rpm_callback()
371 		spin_unlock_irq(&dev->power.lock);  in __rpm_callback()
380 		if (use_links && dev->power.runtime_status == RPM_RESUMING) {  in __rpm_callback()
383 			retval = rpm_get_suppliers(dev);  in __rpm_callback()
385 				rpm_put_suppliers(dev);  in __rpm_callback()
394 		retval = cb(dev);  in __rpm_callback()
396 	if (dev->power.irq_safe) {  in __rpm_callback()
397 		spin_lock(&dev->power.lock);  in __rpm_callback()
407 		    ((dev->power.runtime_status == RPM_SUSPENDING && !retval) ||  in __rpm_callback()
408 		    (dev->power.runtime_status == RPM_RESUMING && retval))) {  in __rpm_callback()
411 			__rpm_put_suppliers(dev, false);  in __rpm_callback()
417 		spin_lock_irq(&dev->power.lock);  in __rpm_callback()
428 static int rpm_callback(int (*cb)(struct device *), struct device *dev)  in rpm_callback()  argument
432 	if (dev->power.memalloc_noio) {  in rpm_callback()
445 		retval = __rpm_callback(cb, dev);  in rpm_callback()
448 		retval = __rpm_callback(cb, dev);  in rpm_callback()
461 		dev->power.runtime_error = retval;  in rpm_callback()
479 static int rpm_idle(struct device *dev, int rpmflags)  in rpm_idle()  argument
484 	trace_rpm_idle(dev, rpmflags);  in rpm_idle()
485 	retval = rpm_check_suspend_allowed(dev);  in rpm_idle()
490 	else if (dev->power.runtime_status != RPM_ACTIVE)  in rpm_idle()
497 	else if (dev->power.request_pending &&  in rpm_idle()
498 	    dev->power.request > RPM_REQ_IDLE)  in rpm_idle()
502 	else if (dev->power.idle_notification)  in rpm_idle()
509 	dev->power.request = RPM_REQ_NONE;  in rpm_idle()
511 	callback = RPM_GET_CALLBACK(dev, runtime_idle);  in rpm_idle()
514 	if (!callback || dev->power.no_callbacks)  in rpm_idle()
519 		dev->power.request = RPM_REQ_IDLE;  in rpm_idle()
520 		if (!dev->power.request_pending) {  in rpm_idle()
521 			dev->power.request_pending = true;  in rpm_idle()
522 			queue_work(pm_wq, &dev->power.work);  in rpm_idle()
524 		trace_rpm_return_int(dev, _THIS_IP_, 0);  in rpm_idle()
528 	dev->power.idle_notification = true;  in rpm_idle()
530 	if (dev->power.irq_safe)  in rpm_idle()
531 		spin_unlock(&dev->power.lock);  in rpm_idle()
533 		spin_unlock_irq(&dev->power.lock);  in rpm_idle()
535 	retval = callback(dev);  in rpm_idle()
537 	if (dev->power.irq_safe)  in rpm_idle()
538 		spin_lock(&dev->power.lock);  in rpm_idle()
540 		spin_lock_irq(&dev->power.lock);  in rpm_idle()
542 	dev->power.idle_notification = false;  in rpm_idle()
543 	wake_up_all(&dev->power.wait_queue);  in rpm_idle()
546 	trace_rpm_return_int(dev, _THIS_IP_, retval);  in rpm_idle()
547 	return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO);  in rpm_idle()
571 static int rpm_suspend(struct device *dev, int rpmflags)  in rpm_suspend()  argument
572 	__releases(&dev->power.lock) __acquires(&dev->power.lock)  in rpm_suspend()
578 	trace_rpm_suspend(dev, rpmflags);  in rpm_suspend()
581 	retval = rpm_check_suspend_allowed(dev);  in rpm_suspend()
586 	if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC))  in rpm_suspend()
593 	if ((rpmflags & RPM_AUTO) && dev->power.runtime_status != RPM_SUSPENDING) {  in rpm_suspend()
594 		u64 expires = pm_runtime_autosuspend_expiration(dev);  in rpm_suspend()
598 			dev->power.request = RPM_REQ_NONE;  in rpm_suspend()
607 			if (!(dev->power.timer_expires &&  in rpm_suspend()
608 			    dev->power.timer_expires <= expires)) {  in rpm_suspend()
613 				u64 slack = (u64)READ_ONCE(dev->power.autosuspend_delay) *  in rpm_suspend()
616 				dev->power.timer_expires = expires;  in rpm_suspend()
617 				hrtimer_start_range_ns(&dev->power.suspend_timer,  in rpm_suspend()
622 			dev->power.timer_autosuspends = 1;  in rpm_suspend()
628 	pm_runtime_cancel_pending(dev);  in rpm_suspend()
630 	if (dev->power.runtime_status == RPM_SUSPENDING) {  in rpm_suspend()
638 		if (dev->power.irq_safe) {  in rpm_suspend()
639 			spin_unlock(&dev->power.lock);  in rpm_suspend()
643 			spin_lock(&dev->power.lock);  in rpm_suspend()
649 			prepare_to_wait(&dev->power.wait_queue, &wait,  in rpm_suspend()
651 			if (dev->power.runtime_status != RPM_SUSPENDING)  in rpm_suspend()
654 			spin_unlock_irq(&dev->power.lock);  in rpm_suspend()
658 			spin_lock_irq(&dev->power.lock);  in rpm_suspend()
660 		finish_wait(&dev->power.wait_queue, &wait);  in rpm_suspend()
664 	if (dev->power.no_callbacks)  in rpm_suspend()
669 		dev->power.request = (rpmflags & RPM_AUTO) ?  in rpm_suspend()
671 		if (!dev->power.request_pending) {  in rpm_suspend()
672 			dev->power.request_pending = true;  in rpm_suspend()
673 			queue_work(pm_wq, &dev->power.work);  in rpm_suspend()
678 	__update_runtime_status(dev, RPM_SUSPENDING);  in rpm_suspend()
680 	callback = RPM_GET_CALLBACK(dev, runtime_suspend);  in rpm_suspend()
682 	dev_pm_enable_wake_irq_check(dev, true);  in rpm_suspend()
683 	retval = rpm_callback(callback, dev);  in rpm_suspend()
687 	dev_pm_enable_wake_irq_complete(dev);  in rpm_suspend()
690 	__update_runtime_status(dev, RPM_SUSPENDED);  in rpm_suspend()
691 	pm_runtime_deactivate_timer(dev);  in rpm_suspend()
693 	if (dev->parent) {  in rpm_suspend()
694 		parent = dev->parent;  in rpm_suspend()
697 	wake_up_all(&dev->power.wait_queue);  in rpm_suspend()
699 	if (dev->power.deferred_resume) {  in rpm_suspend()
700 		dev->power.deferred_resume = false;  in rpm_suspend()
701 		rpm_resume(dev, 0);  in rpm_suspend()
706 	if (dev->power.irq_safe)  in rpm_suspend()
711 		spin_unlock(&dev->power.lock);  in rpm_suspend()
717 		spin_lock(&dev->power.lock);  in rpm_suspend()
720 	if (dev->power.links_count > 0) {  in rpm_suspend()
721 		spin_unlock_irq(&dev->power.lock);  in rpm_suspend()
723 		rpm_suspend_suppliers(dev);  in rpm_suspend()
725 		spin_lock_irq(&dev->power.lock);  in rpm_suspend()
729 	trace_rpm_return_int(dev, _THIS_IP_, retval);  in rpm_suspend()
734 	dev_pm_disable_wake_irq_check(dev, true);  in rpm_suspend()
735 	__update_runtime_status(dev, RPM_ACTIVE);  in rpm_suspend()
736 	dev->power.deferred_resume = false;  in rpm_suspend()
737 	wake_up_all(&dev->power.wait_queue);  in rpm_suspend()
745 	if (!dev->power.runtime_error && (rpmflags & RPM_AUTO) &&  in rpm_suspend()
746 	    pm_runtime_autosuspend_expiration(dev) != 0)  in rpm_suspend()
749 	pm_runtime_cancel_pending(dev);  in rpm_suspend()
771 static int rpm_resume(struct device *dev, int rpmflags)  in rpm_resume()  argument
772 	__releases(&dev->power.lock) __acquires(&dev->power.lock)  in rpm_resume()
778 	trace_rpm_resume(dev, rpmflags);  in rpm_resume()
781 	if (dev->power.runtime_error) {  in rpm_resume()
783 	} else if (dev->power.disable_depth > 0) {  in rpm_resume()
784 		if (dev->power.runtime_status == RPM_ACTIVE &&  in rpm_resume()
785 		    dev->power.last_status == RPM_ACTIVE)  in rpm_resume()
799 	dev->power.request = RPM_REQ_NONE;  in rpm_resume()
800 	if (!dev->power.timer_autosuspends)  in rpm_resume()
801 		pm_runtime_deactivate_timer(dev);  in rpm_resume()
803 	if (dev->power.runtime_status == RPM_ACTIVE) {  in rpm_resume()
808 	if (dev->power.runtime_status == RPM_RESUMING ||  in rpm_resume()
809 	    dev->power.runtime_status == RPM_SUSPENDING) {  in rpm_resume()
813 			if (dev->power.runtime_status == RPM_SUSPENDING) {  in rpm_resume()
814 				dev->power.deferred_resume = true;  in rpm_resume()
823 		if (dev->power.irq_safe) {  in rpm_resume()
824 			spin_unlock(&dev->power.lock);  in rpm_resume()
828 			spin_lock(&dev->power.lock);  in rpm_resume()
834 			prepare_to_wait(&dev->power.wait_queue, &wait,  in rpm_resume()
836 			if (dev->power.runtime_status != RPM_RESUMING &&  in rpm_resume()
837 			    dev->power.runtime_status != RPM_SUSPENDING)  in rpm_resume()
840 			spin_unlock_irq(&dev->power.lock);  in rpm_resume()
844 			spin_lock_irq(&dev->power.lock);  in rpm_resume()
846 		finish_wait(&dev->power.wait_queue, &wait);  in rpm_resume()
855 	if (dev->power.no_callbacks && !parent && dev->parent) {  in rpm_resume()
856 		spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);  in rpm_resume()
857 		if (dev->parent->power.disable_depth > 0 ||  in rpm_resume()
858 		    dev->parent->power.ignore_children ||  in rpm_resume()
859 		    dev->parent->power.runtime_status == RPM_ACTIVE) {  in rpm_resume()
860 			atomic_inc(&dev->parent->power.child_count);  in rpm_resume()
861 			spin_unlock(&dev->parent->power.lock);  in rpm_resume()
865 		spin_unlock(&dev->parent->power.lock);  in rpm_resume()
870 		dev->power.request = RPM_REQ_RESUME;  in rpm_resume()
871 		if (!dev->power.request_pending) {  in rpm_resume()
872 			dev->power.request_pending = true;  in rpm_resume()
873 			queue_work(pm_wq, &dev->power.work);  in rpm_resume()
879 	if (!parent && dev->parent) {  in rpm_resume()
885 		parent = dev->parent;  in rpm_resume()
886 		if (dev->power.irq_safe)  in rpm_resume()
889 		spin_unlock(&dev->power.lock);  in rpm_resume()
906 		spin_lock(&dev->power.lock);  in rpm_resume()
914 	if (dev->power.no_callbacks)  in rpm_resume()
917 	__update_runtime_status(dev, RPM_RESUMING);  in rpm_resume()
919 	callback = RPM_GET_CALLBACK(dev, runtime_resume);  in rpm_resume()
921 	dev_pm_disable_wake_irq_check(dev, false);  in rpm_resume()
922 	retval = rpm_callback(callback, dev);  in rpm_resume()
924 		__update_runtime_status(dev, RPM_SUSPENDED);  in rpm_resume()
925 		pm_runtime_cancel_pending(dev);  in rpm_resume()
926 		dev_pm_enable_wake_irq_check(dev, false);  in rpm_resume()
929 		__update_runtime_status(dev, RPM_ACTIVE);  in rpm_resume()
930 		pm_runtime_mark_last_busy(dev);  in rpm_resume()
934 	wake_up_all(&dev->power.wait_queue);  in rpm_resume()
937 		rpm_idle(dev, RPM_ASYNC);  in rpm_resume()
940 	if (parent && !dev->power.irq_safe) {  in rpm_resume()
941 		spin_unlock_irq(&dev->power.lock);  in rpm_resume()
945 		spin_lock_irq(&dev->power.lock);  in rpm_resume()
948 	trace_rpm_return_int(dev, _THIS_IP_, retval);  in rpm_resume()
962 	struct device *dev = container_of(work, struct device, power.work);  in pm_runtime_work()  local
965 	spin_lock_irq(&dev->power.lock);  in pm_runtime_work()
967 	if (!dev->power.request_pending)  in pm_runtime_work()
970 	req = dev->power.request;  in pm_runtime_work()
971 	dev->power.request = RPM_REQ_NONE;  in pm_runtime_work()
972 	dev->power.request_pending = false;  in pm_runtime_work()
978 		rpm_idle(dev, RPM_NOWAIT);  in pm_runtime_work()
981 		rpm_suspend(dev, RPM_NOWAIT);  in pm_runtime_work()
984 		rpm_suspend(dev, RPM_NOWAIT | RPM_AUTO);  in pm_runtime_work()
987 		rpm_resume(dev, RPM_NOWAIT);  in pm_runtime_work()
992 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_work()
1003 	struct device *dev = container_of(timer, struct device, power.suspend_timer);  in pm_suspend_timer_fn()  local
1007 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_suspend_timer_fn()
1009 	expires = dev->power.timer_expires;  in pm_suspend_timer_fn()
1015 		dev->power.timer_expires = 0;  in pm_suspend_timer_fn()
1016 		rpm_suspend(dev, dev->power.timer_autosuspends ?  in pm_suspend_timer_fn()
1020 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_suspend_timer_fn()
1030 int pm_schedule_suspend(struct device *dev, unsigned int delay)  in pm_schedule_suspend()  argument
1036 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_schedule_suspend()
1039 		retval = rpm_suspend(dev, RPM_ASYNC);  in pm_schedule_suspend()
1043 	retval = rpm_check_suspend_allowed(dev);  in pm_schedule_suspend()
1048 	pm_runtime_cancel_pending(dev);  in pm_schedule_suspend()
1051 	dev->power.timer_expires = expires;  in pm_schedule_suspend()
1052 	dev->power.timer_autosuspends = 0;  in pm_schedule_suspend()
1053 	hrtimer_start(&dev->power.suspend_timer, expires, HRTIMER_MODE_ABS);  in pm_schedule_suspend()
1056 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_schedule_suspend()
1062 static int rpm_drop_usage_count(struct device *dev)  in rpm_drop_usage_count()  argument
1066 	ret = atomic_sub_return(1, &dev->power.usage_count);  in rpm_drop_usage_count()
1076 	atomic_inc(&dev->power.usage_count);  in rpm_drop_usage_count()
1077 	dev_warn(dev, "Runtime PM usage count underflow!\n");  in rpm_drop_usage_count()
1094 int __pm_runtime_idle(struct device *dev, int rpmflags)  in __pm_runtime_idle()  argument
1100 		retval = rpm_drop_usage_count(dev);  in __pm_runtime_idle()
1104 			trace_rpm_usage(dev, rpmflags);  in __pm_runtime_idle()
1109 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);  in __pm_runtime_idle()
1111 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_idle()
1112 	retval = rpm_idle(dev, rpmflags);  in __pm_runtime_idle()
1113 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_idle()
1132 int __pm_runtime_suspend(struct device *dev, int rpmflags)  in __pm_runtime_suspend()  argument
1138 		retval = rpm_drop_usage_count(dev);  in __pm_runtime_suspend()
1142 			trace_rpm_usage(dev, rpmflags);  in __pm_runtime_suspend()
1147 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);  in __pm_runtime_suspend()
1149 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_suspend()
1150 	retval = rpm_suspend(dev, rpmflags);  in __pm_runtime_suspend()
1151 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_suspend()
1168 int __pm_runtime_resume(struct device *dev, int rpmflags)  in __pm_runtime_resume()  argument
1173 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe &&  in __pm_runtime_resume()
1174 			dev->power.runtime_status != RPM_ACTIVE);  in __pm_runtime_resume()
1177 		atomic_inc(&dev->power.usage_count);  in __pm_runtime_resume()
1179 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_resume()
1180 	retval = rpm_resume(dev, rpmflags);  in __pm_runtime_resume()
1181 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_resume()
1209 static int pm_runtime_get_conditional(struct device *dev, bool ign_usage_count)  in pm_runtime_get_conditional()  argument
1214 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_runtime_get_conditional()
1215 	if (dev->power.disable_depth > 0) {  in pm_runtime_get_conditional()
1217 	} else if (dev->power.runtime_status != RPM_ACTIVE) {  in pm_runtime_get_conditional()
1221 		atomic_inc(&dev->power.usage_count);  in pm_runtime_get_conditional()
1223 		retval = atomic_inc_not_zero(&dev->power.usage_count);  in pm_runtime_get_conditional()
1225 	trace_rpm_usage(dev, 0);  in pm_runtime_get_conditional()
1226 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_runtime_get_conditional()
1241 int pm_runtime_get_if_active(struct device *dev)  in pm_runtime_get_if_active()  argument
1243 	return pm_runtime_get_conditional(dev, true);  in pm_runtime_get_if_active()
1257 int pm_runtime_get_if_in_use(struct device *dev)  in pm_runtime_get_if_in_use()  argument
1259 	return pm_runtime_get_conditional(dev, false);  in pm_runtime_get_if_in_use()
1287 int __pm_runtime_set_status(struct device *dev, unsigned int status)  in __pm_runtime_set_status()  argument
1289 	struct device *parent = dev->parent;  in __pm_runtime_set_status()
1297 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_set_status()
1303 	if (dev->power.runtime_error || dev->power.disable_depth)  in __pm_runtime_set_status()
1304 		dev->power.disable_depth++;  in __pm_runtime_set_status()
1308 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_set_status()
1322 		error = rpm_get_suppliers(dev);  in __pm_runtime_set_status()
1329 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_set_status()
1331 	if (dev->power.runtime_status == status || !parent)  in __pm_runtime_set_status()
1348 			dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n",  in __pm_runtime_set_status()
1349 				dev_name(dev),  in __pm_runtime_set_status()
1352 		} else if (dev->power.runtime_status == RPM_SUSPENDED) {  in __pm_runtime_set_status()
1365 	__update_runtime_status(dev, status);  in __pm_runtime_set_status()
1367 		dev->power.runtime_error = 0;  in __pm_runtime_set_status()
1370 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_set_status()
1378 		rpm_put_suppliers(dev);  in __pm_runtime_set_status()
1383 	pm_runtime_enable(dev);  in __pm_runtime_set_status()
1398 static void __pm_runtime_barrier(struct device *dev)  in __pm_runtime_barrier()  argument
1400 	pm_runtime_deactivate_timer(dev);  in __pm_runtime_barrier()
1402 	if (dev->power.request_pending) {  in __pm_runtime_barrier()
1403 		dev->power.request = RPM_REQ_NONE;  in __pm_runtime_barrier()
1404 		spin_unlock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1406 		cancel_work_sync(&dev->power.work);  in __pm_runtime_barrier()
1408 		spin_lock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1409 		dev->power.request_pending = false;  in __pm_runtime_barrier()
1412 	if (dev->power.runtime_status == RPM_SUSPENDING ||  in __pm_runtime_barrier()
1413 	    dev->power.runtime_status == RPM_RESUMING ||  in __pm_runtime_barrier()
1414 	    dev->power.idle_notification) {  in __pm_runtime_barrier()
1419 			prepare_to_wait(&dev->power.wait_queue, &wait,  in __pm_runtime_barrier()
1421 			if (dev->power.runtime_status != RPM_SUSPENDING  in __pm_runtime_barrier()
1422 			    && dev->power.runtime_status != RPM_RESUMING  in __pm_runtime_barrier()
1423 			    && !dev->power.idle_notification)  in __pm_runtime_barrier()
1425 			spin_unlock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1429 			spin_lock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1431 		finish_wait(&dev->power.wait_queue, &wait);  in __pm_runtime_barrier()
1449 int pm_runtime_barrier(struct device *dev)  in pm_runtime_barrier()  argument
1453 	pm_runtime_get_noresume(dev);  in pm_runtime_barrier()
1454 	spin_lock_irq(&dev->power.lock);  in pm_runtime_barrier()
1456 	if (dev->power.request_pending  in pm_runtime_barrier()
1457 	    && dev->power.request == RPM_REQ_RESUME) {  in pm_runtime_barrier()
1458 		rpm_resume(dev, 0);  in pm_runtime_barrier()
1462 	__pm_runtime_barrier(dev);  in pm_runtime_barrier()
1464 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_barrier()
1465 	pm_runtime_put_noidle(dev);  in pm_runtime_barrier()
1471 bool pm_runtime_block_if_disabled(struct device *dev)  in pm_runtime_block_if_disabled()  argument
1475 	spin_lock_irq(&dev->power.lock);  in pm_runtime_block_if_disabled()
1477 	ret = !pm_runtime_enabled(dev);  in pm_runtime_block_if_disabled()
1478 	if (ret && dev->power.last_status == RPM_INVALID)  in pm_runtime_block_if_disabled()
1479 		dev->power.last_status = RPM_BLOCKED;  in pm_runtime_block_if_disabled()
1481 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_block_if_disabled()
1486 void pm_runtime_unblock(struct device *dev)  in pm_runtime_unblock()  argument
1488 	spin_lock_irq(&dev->power.lock);  in pm_runtime_unblock()
1490 	if (dev->power.last_status == RPM_BLOCKED)  in pm_runtime_unblock()
1491 		dev->power.last_status = RPM_INVALID;  in pm_runtime_unblock()
1493 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_unblock()
1496 void __pm_runtime_disable(struct device *dev, bool check_resume)  in __pm_runtime_disable()  argument
1498 	spin_lock_irq(&dev->power.lock);  in __pm_runtime_disable()
1500 	if (dev->power.disable_depth > 0) {  in __pm_runtime_disable()
1501 		dev->power.disable_depth++;  in __pm_runtime_disable()
1510 	if (check_resume && dev->power.request_pending &&  in __pm_runtime_disable()
1511 	    dev->power.request == RPM_REQ_RESUME) {  in __pm_runtime_disable()
1516 		pm_runtime_get_noresume(dev);  in __pm_runtime_disable()
1518 		rpm_resume(dev, 0);  in __pm_runtime_disable()
1520 		pm_runtime_put_noidle(dev);  in __pm_runtime_disable()
1524 	update_pm_runtime_accounting(dev);  in __pm_runtime_disable()
1526 	if (!dev->power.disable_depth++) {  in __pm_runtime_disable()
1527 		__pm_runtime_barrier(dev);  in __pm_runtime_disable()
1528 		dev->power.last_status = dev->power.runtime_status;  in __pm_runtime_disable()
1532 	spin_unlock_irq(&dev->power.lock);  in __pm_runtime_disable()
1540 void pm_runtime_enable(struct device *dev)  in pm_runtime_enable()  argument
1544 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_runtime_enable()
1546 	if (!dev->power.disable_depth) {  in pm_runtime_enable()
1547 		dev_warn(dev, "Unbalanced %s!\n", __func__);  in pm_runtime_enable()
1551 	if (--dev->power.disable_depth > 0)  in pm_runtime_enable()
1554 	if (dev->power.last_status == RPM_BLOCKED) {  in pm_runtime_enable()
1555 		dev_warn(dev, "Attempt to enable runtime PM when it is blocked\n");  in pm_runtime_enable()
1558 	dev->power.last_status = RPM_INVALID;  in pm_runtime_enable()
1559 	dev->power.accounting_timestamp = ktime_get_mono_fast_ns();  in pm_runtime_enable()
1561 	if (dev->power.runtime_status == RPM_SUSPENDED &&  in pm_runtime_enable()
1562 	    !dev->power.ignore_children &&  in pm_runtime_enable()
1563 	    atomic_read(&dev->power.child_count) > 0)  in pm_runtime_enable()
1564 		dev_warn(dev, "Enabling runtime PM for inactive device with active children\n");  in pm_runtime_enable()
1567 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_runtime_enable()
1585 int devm_pm_runtime_enable(struct device *dev)  in devm_pm_runtime_enable()  argument
1587 	pm_runtime_enable(dev);  in devm_pm_runtime_enable()
1589 	return devm_add_action_or_reset(dev, pm_runtime_disable_action, dev);  in devm_pm_runtime_enable()
1601 void pm_runtime_forbid(struct device *dev)  in pm_runtime_forbid()  argument
1603 	spin_lock_irq(&dev->power.lock);  in pm_runtime_forbid()
1604 	if (!dev->power.runtime_auto)  in pm_runtime_forbid()
1607 	dev->power.runtime_auto = false;  in pm_runtime_forbid()
1608 	atomic_inc(&dev->power.usage_count);  in pm_runtime_forbid()
1609 	rpm_resume(dev, 0);  in pm_runtime_forbid()
1612 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_forbid()
1622 void pm_runtime_allow(struct device *dev)  in pm_runtime_allow()  argument
1626 	spin_lock_irq(&dev->power.lock);  in pm_runtime_allow()
1627 	if (dev->power.runtime_auto)  in pm_runtime_allow()
1630 	dev->power.runtime_auto = true;  in pm_runtime_allow()
1631 	ret = rpm_drop_usage_count(dev);  in pm_runtime_allow()
1633 		rpm_idle(dev, RPM_AUTO | RPM_ASYNC);  in pm_runtime_allow()
1635 		trace_rpm_usage(dev, RPM_AUTO | RPM_ASYNC);  in pm_runtime_allow()
1638 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_allow()
1650 void pm_runtime_no_callbacks(struct device *dev)  in pm_runtime_no_callbacks()  argument
1652 	spin_lock_irq(&dev->power.lock);  in pm_runtime_no_callbacks()
1653 	dev->power.no_callbacks = 1;  in pm_runtime_no_callbacks()
1654 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_no_callbacks()
1655 	if (device_is_registered(dev))  in pm_runtime_no_callbacks()
1656 		rpm_sysfs_remove(dev);  in pm_runtime_no_callbacks()
1671 void pm_runtime_irq_safe(struct device *dev)  in pm_runtime_irq_safe()  argument
1673 	if (dev->parent)  in pm_runtime_irq_safe()
1674 		pm_runtime_get_sync(dev->parent);  in pm_runtime_irq_safe()
1676 	spin_lock_irq(&dev->power.lock);  in pm_runtime_irq_safe()
1677 	dev->power.irq_safe = 1;  in pm_runtime_irq_safe()
1678 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_irq_safe()
1693 static void update_autosuspend(struct device *dev, int old_delay, int old_use)  in update_autosuspend()  argument
1695 	int delay = dev->power.autosuspend_delay;  in update_autosuspend()
1698 	if (dev->power.use_autosuspend && delay < 0) {  in update_autosuspend()
1702 			atomic_inc(&dev->power.usage_count);  in update_autosuspend()
1703 			rpm_resume(dev, 0);  in update_autosuspend()
1705 			trace_rpm_usage(dev, 0);  in update_autosuspend()
1714 			atomic_dec(&dev->power.usage_count);  in update_autosuspend()
1717 		rpm_idle(dev, RPM_AUTO);  in update_autosuspend()
1730 void pm_runtime_set_autosuspend_delay(struct device *dev, int delay)  in pm_runtime_set_autosuspend_delay()  argument
1734 	spin_lock_irq(&dev->power.lock);  in pm_runtime_set_autosuspend_delay()
1735 	old_delay = dev->power.autosuspend_delay;  in pm_runtime_set_autosuspend_delay()
1736 	old_use = dev->power.use_autosuspend;  in pm_runtime_set_autosuspend_delay()
1737 	dev->power.autosuspend_delay = delay;  in pm_runtime_set_autosuspend_delay()
1738 	update_autosuspend(dev, old_delay, old_use);  in pm_runtime_set_autosuspend_delay()
1739 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_set_autosuspend_delay()
1751 void __pm_runtime_use_autosuspend(struct device *dev, bool use)  in __pm_runtime_use_autosuspend()  argument
1755 	spin_lock_irq(&dev->power.lock);  in __pm_runtime_use_autosuspend()
1756 	old_delay = dev->power.autosuspend_delay;  in __pm_runtime_use_autosuspend()
1757 	old_use = dev->power.use_autosuspend;  in __pm_runtime_use_autosuspend()
1758 	dev->power.use_autosuspend = use;  in __pm_runtime_use_autosuspend()
1759 	update_autosuspend(dev, old_delay, old_use);  in __pm_runtime_use_autosuspend()
1760 	spin_unlock_irq(&dev->power.lock);  in __pm_runtime_use_autosuspend()
1768 void pm_runtime_init(struct device *dev)  in pm_runtime_init()  argument
1770 	dev->power.runtime_status = RPM_SUSPENDED;  in pm_runtime_init()
1771 	dev->power.last_status = RPM_INVALID;  in pm_runtime_init()
1772 	dev->power.idle_notification = false;  in pm_runtime_init()
1774 	dev->power.disable_depth = 1;  in pm_runtime_init()
1775 	atomic_set(&dev->power.usage_count, 0);  in pm_runtime_init()
1777 	dev->power.runtime_error = 0;  in pm_runtime_init()
1779 	atomic_set(&dev->power.child_count, 0);  in pm_runtime_init()
1780 	pm_suspend_ignore_children(dev, false);  in pm_runtime_init()
1781 	dev->power.runtime_auto = true;  in pm_runtime_init()
1783 	dev->power.request_pending = false;  in pm_runtime_init()
1784 	dev->power.request = RPM_REQ_NONE;  in pm_runtime_init()
1785 	dev->power.deferred_resume = false;  in pm_runtime_init()
1786 	dev->power.needs_force_resume = 0;  in pm_runtime_init()
1787 	INIT_WORK(&dev->power.work, pm_runtime_work);  in pm_runtime_init()
1789 	dev->power.timer_expires = 0;  in pm_runtime_init()
1790 	hrtimer_setup(&dev->power.suspend_timer, pm_suspend_timer_fn, CLOCK_MONOTONIC,  in pm_runtime_init()
1793 	init_waitqueue_head(&dev->power.wait_queue);  in pm_runtime_init()
1800 void pm_runtime_reinit(struct device *dev)  in pm_runtime_reinit()  argument
1802 	if (!pm_runtime_enabled(dev)) {  in pm_runtime_reinit()
1803 		if (dev->power.runtime_status == RPM_ACTIVE)  in pm_runtime_reinit()
1804 			pm_runtime_set_suspended(dev);  in pm_runtime_reinit()
1805 		if (dev->power.irq_safe) {  in pm_runtime_reinit()
1806 			spin_lock_irq(&dev->power.lock);  in pm_runtime_reinit()
1807 			dev->power.irq_safe = 0;  in pm_runtime_reinit()
1808 			spin_unlock_irq(&dev->power.lock);  in pm_runtime_reinit()
1809 			if (dev->parent)  in pm_runtime_reinit()
1810 				pm_runtime_put(dev->parent);  in pm_runtime_reinit()
1819 void pm_runtime_remove(struct device *dev)  in pm_runtime_remove()  argument
1821 	__pm_runtime_disable(dev, false);  in pm_runtime_remove()
1822 	pm_runtime_reinit(dev);  in pm_runtime_remove()
1829 void pm_runtime_get_suppliers(struct device *dev)  in pm_runtime_get_suppliers()  argument
1836 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,  in pm_runtime_get_suppliers()
1850 void pm_runtime_put_suppliers(struct device *dev)  in pm_runtime_put_suppliers()  argument
1857 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,  in pm_runtime_put_suppliers()
1867 void pm_runtime_new_link(struct device *dev)  in pm_runtime_new_link()  argument
1869 	spin_lock_irq(&dev->power.lock);  in pm_runtime_new_link()
1870 	dev->power.links_count++;  in pm_runtime_new_link()
1871 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_new_link()
1874 static void pm_runtime_drop_link_count(struct device *dev)  in pm_runtime_drop_link_count()  argument
1876 	spin_lock_irq(&dev->power.lock);  in pm_runtime_drop_link_count()
1877 	WARN_ON(dev->power.links_count == 0);  in pm_runtime_drop_link_count()
1878 	dev->power.links_count--;  in pm_runtime_drop_link_count()
1879 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_drop_link_count()
1900 bool pm_runtime_need_not_resume(struct device *dev)  in pm_runtime_need_not_resume()  argument
1902 	return atomic_read(&dev->power.usage_count) <= 1 &&  in pm_runtime_need_not_resume()
1903 		(atomic_read(&dev->power.child_count) == 0 ||  in pm_runtime_need_not_resume()
1904 		 dev->power.ignore_children);  in pm_runtime_need_not_resume()
1928 int pm_runtime_force_suspend(struct device *dev)  in pm_runtime_force_suspend()  argument
1933 	pm_runtime_disable(dev);  in pm_runtime_force_suspend()
1934 	if (pm_runtime_status_suspended(dev))  in pm_runtime_force_suspend()
1937 	callback = RPM_GET_CALLBACK(dev, runtime_suspend);  in pm_runtime_force_suspend()
1939 	dev_pm_enable_wake_irq_check(dev, true);  in pm_runtime_force_suspend()
1940 	ret = callback ? callback(dev) : 0;  in pm_runtime_force_suspend()
1944 	dev_pm_enable_wake_irq_complete(dev);  in pm_runtime_force_suspend()
1952 	if (pm_runtime_need_not_resume(dev)) {  in pm_runtime_force_suspend()
1953 		pm_runtime_set_suspended(dev);  in pm_runtime_force_suspend()
1955 		__update_runtime_status(dev, RPM_SUSPENDED);  in pm_runtime_force_suspend()
1956 		dev->power.needs_force_resume = 1;  in pm_runtime_force_suspend()
1962 	dev_pm_disable_wake_irq_check(dev, true);  in pm_runtime_force_suspend()
1963 	pm_runtime_enable(dev);  in pm_runtime_force_suspend()
1980 int pm_runtime_force_resume(struct device *dev)  in pm_runtime_force_resume()  argument
1985 	if (!dev->power.needs_force_resume)  in pm_runtime_force_resume()
1992 	__update_runtime_status(dev, RPM_ACTIVE);  in pm_runtime_force_resume()
1994 	callback = RPM_GET_CALLBACK(dev, runtime_resume);  in pm_runtime_force_resume()
1996 	dev_pm_disable_wake_irq_check(dev, false);  in pm_runtime_force_resume()
1997 	ret = callback ? callback(dev) : 0;  in pm_runtime_force_resume()
1999 		pm_runtime_set_suspended(dev);  in pm_runtime_force_resume()
2000 		dev_pm_enable_wake_irq_check(dev, false);  in pm_runtime_force_resume()
2004 	pm_runtime_mark_last_busy(dev);  in pm_runtime_force_resume()
2006 	dev->power.needs_force_resume = 0;  in pm_runtime_force_resume()
2007 	pm_runtime_enable(dev);  in pm_runtime_force_resume()