551 static inline unsigned int timer_get_idx(struct timer_list *timer)  in timer_get_idx()  argument
553 	return (timer->flags & TIMER_ARRAYMASK) >> TIMER_ARRAYSHIFT;  in timer_get_idx()
556 static inline void timer_set_idx(struct timer_list *timer, unsigned int idx)  in timer_set_idx()  argument
558 	timer->flags = (timer->flags & ~TIMER_ARRAYMASK) |  in timer_set_idx()
622 trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)  in trigger_dyntick_cpu()  argument
631 	if (!is_timers_nohz_active() || timer->flags & TIMER_DEFERRABLE)  in trigger_dyntick_cpu()
643 		WARN_ON_ONCE(!(timer->flags & TIMER_PINNED ||  in trigger_dyntick_cpu()
654 static void enqueue_timer(struct timer_base *base, struct timer_list *timer,  in enqueue_timer()  argument
658 	hlist_add_head(&timer->entry, base->vectors + idx);  in enqueue_timer()
660 	timer_set_idx(timer, idx);  in enqueue_timer()
662 	trace_timer_start(timer, bucket_expiry);  in enqueue_timer()
677 		trigger_dyntick_cpu(base, timer);  in enqueue_timer()
681 static void internal_add_timer(struct timer_base *base, struct timer_list *timer)  in internal_add_timer()  argument
686 	idx = calc_wheel_index(timer->expires, base->clk, &bucket_expiry);  in internal_add_timer()
687 	enqueue_timer(base, timer, idx, bucket_expiry);  in internal_add_timer()
708 		   struct delayed_work, timer, work.func),
710 		   struct kthread_delayed_work, timer, work.func),
715 	struct timer_list *timer = addr;  in timer_debug_hint()  local
719 		if (timer_hints[i].function == timer->function) {  in timer_debug_hint()
726 	return timer->function;  in timer_debug_hint()
731 	struct timer_list *timer = addr;  in timer_is_static_object()  local
733 	return (timer->entry.pprev == NULL &&  in timer_is_static_object()
734 		timer->entry.next == TIMER_ENTRY_STATIC);  in timer_is_static_object()
743 	struct timer_list *timer = addr;  in timer_fixup_init()  local
747 		timer_delete_sync(timer);  in timer_fixup_init()
748 		debug_object_init(timer, &timer_debug_descr);  in timer_fixup_init()
768 	struct timer_list *timer = addr;  in timer_fixup_activate()  local
772 		timer_setup(timer, stub_timer, 0);  in timer_fixup_activate()
789 	struct timer_list *timer = addr;  in timer_fixup_free()  local
793 		timer_delete_sync(timer);  in timer_fixup_free()
794 		debug_object_free(timer, &timer_debug_descr);  in timer_fixup_free()
807 	struct timer_list *timer = addr;  in timer_fixup_assert_init()  local
811 		timer_setup(timer, stub_timer, 0);  in timer_fixup_assert_init()
828 static inline void debug_timer_init(struct timer_list *timer)  in debug_timer_init()  argument
830 	debug_object_init(timer, &timer_debug_descr);  in debug_timer_init()
833 static inline void debug_timer_activate(struct timer_list *timer)  in debug_timer_activate()  argument
835 	debug_object_activate(timer, &timer_debug_descr);  in debug_timer_activate()
838 static inline void debug_timer_deactivate(struct timer_list *timer)  in debug_timer_deactivate()  argument
840 	debug_object_deactivate(timer, &timer_debug_descr);  in debug_timer_deactivate()
843 static inline void debug_timer_assert_init(struct timer_list *timer)  in debug_timer_assert_init()  argument
845 	debug_object_assert_init(timer, &timer_debug_descr);  in debug_timer_assert_init()
848 static void do_init_timer(struct timer_list *timer,
853 void init_timer_on_stack_key(struct timer_list *timer,  in init_timer_on_stack_key()  argument
858 	debug_object_init_on_stack(timer, &timer_debug_descr);  in init_timer_on_stack_key()
859 	do_init_timer(timer, func, flags, name, key);  in init_timer_on_stack_key()
863 void destroy_timer_on_stack(struct timer_list *timer)  in destroy_timer_on_stack()  argument
865 	debug_object_free(timer, &timer_debug_descr);  in destroy_timer_on_stack()
870 static inline void debug_timer_init(struct timer_list *timer) { }  in debug_timer_init()  argument
871 static inline void debug_timer_activate(struct timer_list *timer) { }  in debug_timer_activate()  argument
872 static inline void debug_timer_deactivate(struct timer_list *timer) { }  in debug_timer_deactivate()  argument
873 static inline void debug_timer_assert_init(struct timer_list *timer) { }  in debug_timer_assert_init()  argument
876 static inline void debug_init(struct timer_list *timer)  in debug_init()  argument
878 	debug_timer_init(timer);  in debug_init()
879 	trace_timer_init(timer);  in debug_init()
882 static inline void debug_deactivate(struct timer_list *timer)  in debug_deactivate()  argument
884 	debug_timer_deactivate(timer);  in debug_deactivate()
885 	trace_timer_cancel(timer);  in debug_deactivate()
888 static inline void debug_assert_init(struct timer_list *timer)  in debug_assert_init()  argument
890 	debug_timer_assert_init(timer);  in debug_assert_init()
893 static void do_init_timer(struct timer_list *timer,  in do_init_timer()  argument
898 	timer->entry.pprev = NULL;  in do_init_timer()
899 	timer->function = func;  in do_init_timer()
902 	timer->flags = flags | raw_smp_processor_id();  in do_init_timer()
903 	lockdep_init_map(&timer->lockdep_map, name, key, 0);  in do_init_timer()
918 void init_timer_key(struct timer_list *timer,  in init_timer_key()  argument
922 	debug_init(timer);  in init_timer_key()
923 	do_init_timer(timer, func, flags, name, key);  in init_timer_key()
927 static inline void detach_timer(struct timer_list *timer, bool clear_pending)  in detach_timer()  argument
929 	struct hlist_node *entry = &timer->entry;  in detach_timer()
931 	debug_deactivate(timer);  in detach_timer()
939 static int detach_if_pending(struct timer_list *timer, struct timer_base *base,  in detach_if_pending()  argument
942 	unsigned idx = timer_get_idx(timer);  in detach_if_pending()
944 	if (!timer_pending(timer))  in detach_if_pending()
947 	if (hlist_is_singular_node(&timer->entry, base->vectors + idx)) {  in detach_if_pending()
952 	detach_timer(timer, clear_pending);  in detach_if_pending()
1029 static struct timer_base *lock_timer_base(struct timer_list *timer,  in lock_timer_base()  argument
1031 	__acquires(timer->base->lock)  in lock_timer_base()
1042 		tf = READ_ONCE(timer->flags);  in lock_timer_base()
1047 			if (timer->flags == tf)  in lock_timer_base()
1060 __mod_timer(struct timer_list *timer, unsigned long expires, unsigned int options)  in __mod_timer()  argument
1067 	debug_assert_init(timer);  in __mod_timer()
1074 	if (!(options & MOD_TIMER_NOTPENDING) && timer_pending(timer)) {  in __mod_timer()
1080 		long diff = timer->expires - expires;  in __mod_timer()
1093 		base = lock_timer_base(timer, &flags);  in __mod_timer()
1099 		if (!timer->function)  in __mod_timer()
1104 		if (timer_pending(timer) && (options & MOD_TIMER_REDUCE) &&  in __mod_timer()
1105 		    time_before_eq(timer->expires, expires)) {  in __mod_timer()
1118 		if (idx == timer_get_idx(timer)) {  in __mod_timer()
1120 				timer->expires = expires;  in __mod_timer()
1121 			else if (time_after(timer->expires, expires))  in __mod_timer()
1122 				timer->expires = expires;  in __mod_timer()
1127 		base = lock_timer_base(timer, &flags);  in __mod_timer()
1133 		if (!timer->function)  in __mod_timer()
1139 	ret = detach_if_pending(timer, base, false);  in __mod_timer()
1143 	new_base = get_timer_this_cpu_base(timer->flags);  in __mod_timer()
1153 		if (likely(base->running_timer != timer)) {  in __mod_timer()
1155 			timer->flags |= TIMER_MIGRATING;  in __mod_timer()
1160 			WRITE_ONCE(timer->flags,  in __mod_timer()
1161 				   (timer->flags & ~TIMER_BASEMASK) | base->cpu);  in __mod_timer()
1166 	debug_timer_activate(timer);  in __mod_timer()
1168 	timer->expires = expires;  in __mod_timer()
1176 		enqueue_timer(base, timer, idx, bucket_expiry);  in __mod_timer()
1178 		internal_add_timer(base, timer);  in __mod_timer()
1202 int mod_timer_pending(struct timer_list *timer, unsigned long expires)  in mod_timer_pending()  argument
1204 	return __mod_timer(timer, expires, MOD_TIMER_PENDING_ONLY);  in mod_timer_pending()
1235 int mod_timer(struct timer_list *timer, unsigned long expires)  in mod_timer()  argument
1237 	return __mod_timer(timer, expires, 0);  in mod_timer()
1261 int timer_reduce(struct timer_list *timer, unsigned long expires)  in timer_reduce()  argument
1263 	return __mod_timer(timer, expires, MOD_TIMER_REDUCE);  in timer_reduce()
1287 void add_timer(struct timer_list *timer)  in add_timer()  argument
1289 	if (WARN_ON_ONCE(timer_pending(timer)))  in add_timer()
1291 	__mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);  in add_timer()
1303 void add_timer_local(struct timer_list *timer)  in add_timer_local()  argument
1305 	if (WARN_ON_ONCE(timer_pending(timer)))  in add_timer_local()
1307 	timer->flags |= TIMER_PINNED;  in add_timer_local()
1308 	__mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);  in add_timer_local()
1320 void add_timer_global(struct timer_list *timer)  in add_timer_global()  argument
1322 	if (WARN_ON_ONCE(timer_pending(timer)))  in add_timer_global()
1324 	timer->flags &= ~TIMER_PINNED;  in add_timer_global()
1325 	__mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);  in add_timer_global()
1341 void add_timer_on(struct timer_list *timer, int cpu)  in add_timer_on()  argument
1346 	debug_assert_init(timer);  in add_timer_on()
1348 	if (WARN_ON_ONCE(timer_pending(timer)))  in add_timer_on()
1352 	timer->flags |= TIMER_PINNED;  in add_timer_on()
1354 	new_base = get_timer_cpu_base(timer->flags, cpu);  in add_timer_on()
1361 	base = lock_timer_base(timer, &flags);  in add_timer_on()
1366 	if (!timer->function)  in add_timer_on()
1370 		timer->flags |= TIMER_MIGRATING;  in add_timer_on()
1375 		WRITE_ONCE(timer->flags,  in add_timer_on()
1376 			   (timer->flags & ~TIMER_BASEMASK) | cpu);  in add_timer_on()
1380 	debug_timer_activate(timer);  in add_timer_on()
1381 	internal_add_timer(base, timer);  in add_timer_on()
1402 static int __timer_delete(struct timer_list *timer, bool shutdown)  in __timer_delete()  argument
1408 	debug_assert_init(timer);  in __timer_delete()
1421 	if (timer_pending(timer) || shutdown) {  in __timer_delete()
1422 		base = lock_timer_base(timer, &flags);  in __timer_delete()
1423 		ret = detach_if_pending(timer, base, true);  in __timer_delete()
1425 			timer->function = NULL;  in __timer_delete()
1446 int timer_delete(struct timer_list *timer)  in timer_delete()  argument
1448 	return __timer_delete(timer, false);  in timer_delete()
1467 int timer_shutdown(struct timer_list *timer)  in timer_shutdown()  argument
1469 	return __timer_delete(timer, true);  in timer_shutdown()
1493 static int __try_to_del_timer_sync(struct timer_list *timer, bool shutdown)  in __try_to_del_timer_sync()  argument
1499 	debug_assert_init(timer);  in __try_to_del_timer_sync()
1501 	base = lock_timer_base(timer, &flags);  in __try_to_del_timer_sync()
1503 	if (base->running_timer != timer)  in __try_to_del_timer_sync()
1504 		ret = detach_if_pending(timer, base, true);  in __try_to_del_timer_sync()
1506 		timer->function = NULL;  in __try_to_del_timer_sync()
1529 int try_to_del_timer_sync(struct timer_list *timer)  in try_to_del_timer_sync()  argument
1531 	return __try_to_del_timer_sync(timer, false);  in try_to_del_timer_sync()
1580 static void del_timer_wait_running(struct timer_list *timer)  in del_timer_wait_running()  argument
1584 	tf = READ_ONCE(timer->flags);  in del_timer_wait_running()
1607 static inline void del_timer_wait_running(struct timer_list *timer) { }  in del_timer_wait_running()  argument
1632 static int __timer_delete_sync(struct timer_list *timer, bool shutdown)  in __timer_delete_sync()  argument
1644 	lock_map_acquire(&timer->lockdep_map);  in __timer_delete_sync()
1645 	lock_map_release(&timer->lockdep_map);  in __timer_delete_sync()
1652 	WARN_ON(in_hardirq() && !(timer->flags & TIMER_IRQSAFE));  in __timer_delete_sync()
1658 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(timer->flags & TIMER_IRQSAFE))  in __timer_delete_sync()
1662 		ret = __try_to_del_timer_sync(timer, shutdown);  in __timer_delete_sync()
1665 			del_timer_wait_running(timer);  in __timer_delete_sync()
1715 int timer_delete_sync(struct timer_list *timer)  in timer_delete_sync()  argument
1717 	return __timer_delete_sync(timer, false);  in timer_delete_sync()
1757 int timer_shutdown_sync(struct timer_list *timer)  in timer_shutdown_sync()  argument
1759 	return __timer_delete_sync(timer, true);  in timer_shutdown_sync()
1763 static void call_timer_fn(struct timer_list *timer,  in call_timer_fn()  argument
1779 	lockdep_copy_map(&lockdep_map, &timer->lockdep_map);  in call_timer_fn()
1788 	trace_timer_expire_entry(timer, baseclk);  in call_timer_fn()
1789 	fn(timer);  in call_timer_fn()
1790 	trace_timer_expire_exit(timer);  in call_timer_fn()
1817 		struct timer_list *timer;  in expire_timers()  local
1820 		timer = hlist_entry(head->first, struct timer_list, entry);  in expire_timers()
1822 		base->running_timer = timer;  in expire_timers()
1823 		detach_timer(timer, true);  in expire_timers()
1825 		fn = timer->function;  in expire_timers()
1833 		if (timer->flags & TIMER_IRQSAFE) {  in expire_timers()
1835 			call_timer_fn(timer, fn, baseclk);  in expire_timers()
1840 			call_timer_fn(timer, fn, baseclk);  in expire_timers()
2528 	struct timer_list *timer;  in migrate_timer_list()  local
2532 		timer = hlist_entry(head->first, struct timer_list, entry);  in migrate_timer_list()
2533 		detach_timer(timer, false);  in migrate_timer_list()
2534 		timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;  in migrate_timer_list()
2535 		internal_add_timer(new_base, timer);  in migrate_timer_list()