141 		.cpu_base = &migration_cpu_base,
171 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
175 			raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
219 	return expires >= new_base->cpu_base->expires_next;  in hrtimer_suitable_target()
277 		raw_spin_unlock(&base->cpu_base->lock);  in switch_hrtimer_base()
278 		raw_spin_lock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
282 			raw_spin_unlock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
283 			raw_spin_lock(&base->cpu_base->lock);  in switch_hrtimer_base()
302 	__acquires(&timer->base->cpu_base->lock)  in lock_hrtimer_base()
306 	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
495 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)  in __next_base()  argument
505 	return &cpu_base->clock_base[idx];  in __next_base()
508 #define for_each_active_base(base, cpu_base, active)	\  argument
509 	while ((base = __next_base((cpu_base), &(active))))
511 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_next_event_base()  argument
519 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_next_event_base()
542 				cpu_base->softirq_next_timer = timer;  in __hrtimer_next_event_base()
544 				cpu_base->next_timer = timer;  in __hrtimer_next_event_base()
578 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)  in __hrtimer_get_next_event()  argument
584 	if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {  in __hrtimer_get_next_event()
585 		active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in __hrtimer_get_next_event()
586 		cpu_base->softirq_next_timer = NULL;  in __hrtimer_get_next_event()
587 		expires_next = __hrtimer_next_event_base(cpu_base, NULL,  in __hrtimer_get_next_event()
590 		next_timer = cpu_base->softirq_next_timer;  in __hrtimer_get_next_event()
594 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in __hrtimer_get_next_event()
595 		cpu_base->next_timer = next_timer;  in __hrtimer_get_next_event()
596 		expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,  in __hrtimer_get_next_event()
603 static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)  in hrtimer_update_next_event()  argument
612 	if (!cpu_base->softirq_activated) {  in hrtimer_update_next_event()
613 		soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_next_event()
618 		cpu_base->softirq_expires_next = soft;  in hrtimer_update_next_event()
621 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);  in hrtimer_update_next_event()
627 		cpu_base->next_timer = cpu_base->softirq_next_timer;  in hrtimer_update_next_event()
653 static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)  in hrtimer_hres_active()  argument
656 		cpu_base->hres_active : 0;  in hrtimer_hres_active()
659 static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_reprogram()  argument
663 	cpu_base->expires_next = expires_next;  in __hrtimer_reprogram()
682 	if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)  in __hrtimer_reprogram()
694 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)  in hrtimer_force_reprogram()  argument
698 	expires_next = hrtimer_update_next_event(cpu_base);  in hrtimer_force_reprogram()
700 	if (skip_equal && expires_next == cpu_base->expires_next)  in hrtimer_force_reprogram()
703 	__hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);  in hrtimer_force_reprogram()
812 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_reprogram()  local
833 		struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;  in hrtimer_reprogram()
853 	if (base->cpu_base != cpu_base)  in hrtimer_reprogram()
856 	if (expires >= cpu_base->expires_next)  in hrtimer_reprogram()
863 	if (cpu_base->in_hrtirq)  in hrtimer_reprogram()
866 	cpu_base->next_timer = timer;  in hrtimer_reprogram()
868 	__hrtimer_reprogram(cpu_base, timer, expires);  in hrtimer_reprogram()
871 static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,  in update_needs_ipi()  argument
888 	seq = cpu_base->clock_was_set_seq;  in update_needs_ipi()
889 	hrtimer_update_base(cpu_base);  in update_needs_ipi()
895 	if (seq == cpu_base->clock_was_set_seq)  in update_needs_ipi()
903 	if (cpu_base->in_hrtirq)  in update_needs_ipi()
912 	active &= cpu_base->active_bases;  in update_needs_ipi()
914 	for_each_active_base(base, cpu_base, active) {  in update_needs_ipi()
919 		if (expires < cpu_base->expires_next)  in update_needs_ipi()
925 		if (cpu_base->softirq_activated)  in update_needs_ipi()
927 		if (expires < cpu_base->softirq_expires_next)  in update_needs_ipi()
949 	struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases);  in clock_was_set()  local
953 	if (!hrtimer_hres_active(cpu_base) && !tick_nohz_active)  in clock_was_set()
966 		cpu_base = &per_cpu(hrtimer_bases, cpu);  in clock_was_set()
967 		raw_spin_lock_irqsave(&cpu_base->lock, flags);  in clock_was_set()
969 		if (update_needs_ipi(cpu_base, bases))  in clock_was_set()
972 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in clock_was_set()
1018 	__releases(&timer->base->cpu_base->lock)  in unlock_hrtimer_base()
1020 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
1089 	WARN_ON_ONCE(!base->cpu_base->online);  in enqueue_hrtimer()
1091 	base->cpu_base->active_bases |= 1 << base->index;  in enqueue_hrtimer()
1113 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;  in __remove_hrtimer()  local
1122 		cpu_base->active_bases &= ~(1 << base->index);  in __remove_hrtimer()
1132 	if (reprogram && timer == cpu_base->next_timer)  in __remove_hrtimer()
1133 		hrtimer_force_reprogram(cpu_base, 1);  in __remove_hrtimer()
1157 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);  in remove_hrtimer()
1193 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)  in hrtimer_update_softirq_timer()  argument
1200 	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_softirq_timer()
1214 	hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);  in hrtimer_update_softirq_timer()
1233 	force_local = base->cpu_base == this_cpu_base;  in __hrtimer_start_range_ns()
1234 	force_local &= base->cpu_base->next_timer == timer;  in __hrtimer_start_range_ns()
1286 			struct hrtimer_cpu_base *new_cpu_base = new_base->cpu_base;  in __hrtimer_start_range_ns()
1298 	hrtimer_force_reprogram(new_base->cpu_base, 1);  in __hrtimer_start_range_ns()
1399 static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,  in hrtimer_sync_wait_running()  argument
1402 	if (atomic_read(&cpu_base->timer_waiters)) {  in hrtimer_sync_wait_running()
1403 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_sync_wait_running()
1404 		spin_unlock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1405 		spin_lock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1406 		raw_spin_lock_irq(&cpu_base->lock);  in hrtimer_sync_wait_running()
1459 	atomic_inc(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1460 	spin_lock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1461 	atomic_dec(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1462 	spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1526 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_get_next_event()  local
1530 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1532 	if (!hrtimer_hres_active(cpu_base))  in hrtimer_get_next_event()
1533 		expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_get_next_event()
1535 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1549 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_next_event_without()  local
1553 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1555 	if (hrtimer_hres_active(cpu_base)) {  in hrtimer_next_event_without()
1558 		if (!cpu_base->softirq_activated) {  in hrtimer_next_event_without()
1559 			active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in hrtimer_next_event_without()
1560 			expires = __hrtimer_next_event_base(cpu_base, exclude,  in hrtimer_next_event_without()
1563 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in hrtimer_next_event_without()
1564 		expires = __hrtimer_next_event_base(cpu_base, exclude, active,  in hrtimer_next_event_without()
1568 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1596 	struct hrtimer_cpu_base *cpu_base;  in __hrtimer_setup()  local
1610 	cpu_base = raw_cpu_ptr(&hrtimer_bases);  in __hrtimer_setup()
1624 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_setup()
1718 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,  in __run_hrtimer()  argument
1721 			  unsigned long flags) __must_hold(&cpu_base->lock)  in __run_hrtimer()
1727 	lockdep_assert_held(&cpu_base->lock);  in __run_hrtimer()
1757 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in __run_hrtimer()
1765 	raw_spin_lock_irq(&cpu_base->lock);  in __run_hrtimer()
1793 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,  in __hrtimer_run_queues()  argument
1797 	unsigned int active = cpu_base->active_bases & active_mask;  in __hrtimer_run_queues()
1799 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_run_queues()
1825 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);  in __hrtimer_run_queues()
1827 				hrtimer_sync_wait_running(cpu_base, flags);  in __hrtimer_run_queues()
1834 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_softirq()  local
1838 	hrtimer_cpu_base_lock_expiry(cpu_base);  in hrtimer_run_softirq()
1839 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1841 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_softirq()
1842 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);  in hrtimer_run_softirq()
1844 	cpu_base->softirq_activated = 0;  in hrtimer_run_softirq()
1845 	hrtimer_update_softirq_timer(cpu_base, true);  in hrtimer_run_softirq()
1847 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1848 	hrtimer_cpu_base_unlock_expiry(cpu_base);  in hrtimer_run_softirq()
1859 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_interrupt()  local
1864 	BUG_ON(!cpu_base->hres_active);  in hrtimer_interrupt()
1865 	cpu_base->nr_events++;  in hrtimer_interrupt()
1868 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1869 	entry_time = now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1871 	cpu_base->in_hrtirq = 1;  in hrtimer_interrupt()
1879 	cpu_base->expires_next = KTIME_MAX;  in hrtimer_interrupt()
1881 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_interrupt()
1882 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_interrupt()
1883 		cpu_base->softirq_activated = 1;  in hrtimer_interrupt()
1887 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_interrupt()
1890 	expires_next = hrtimer_update_next_event(cpu_base);  in hrtimer_interrupt()
1895 	cpu_base->expires_next = expires_next;  in hrtimer_interrupt()
1896 	cpu_base->in_hrtirq = 0;  in hrtimer_interrupt()
1897 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1901 		cpu_base->hang_detected = 0;  in hrtimer_interrupt()
1918 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1919 	now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1920 	cpu_base->nr_retries++;  in hrtimer_interrupt()
1929 	cpu_base->nr_hangs++;  in hrtimer_interrupt()
1930 	cpu_base->hang_detected = 1;  in hrtimer_interrupt()
1931 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1934 	if ((unsigned int)delta > cpu_base->max_hang_time)  in hrtimer_interrupt()
1935 		cpu_base->max_hang_time = (unsigned int) delta;  in hrtimer_interrupt()
1954 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_queues()  local
1958 	if (hrtimer_hres_active(cpu_base))  in hrtimer_run_queues()
1973 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_queues()
1974 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_queues()
1976 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_run_queues()
1977 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_run_queues()
1978 		cpu_base->softirq_activated = 1;  in hrtimer_run_queues()
1982 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_run_queues()
1983 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_queues()
2214 	struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);  in hrtimers_prepare_cpu()  local
2218 		struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];  in hrtimers_prepare_cpu()
2220 		clock_b->cpu_base = cpu_base;  in hrtimers_prepare_cpu()
2221 		seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);  in hrtimers_prepare_cpu()
2225 	cpu_base->cpu = cpu;  in hrtimers_prepare_cpu()
2226 	hrtimer_cpu_base_init_expiry_lock(cpu_base);  in hrtimers_prepare_cpu()
2232 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimers_cpu_starting()  local
2235 	cpu_base->active_bases = 0;  in hrtimers_cpu_starting()
2236 	cpu_base->hres_active = 0;  in hrtimers_cpu_starting()
2237 	cpu_base->hang_detected = 0;  in hrtimers_cpu_starting()
2238 	cpu_base->next_timer = NULL;  in hrtimers_cpu_starting()
2239 	cpu_base->softirq_next_timer = NULL;  in hrtimers_cpu_starting()
2240 	cpu_base->expires_next = KTIME_MAX;  in hrtimers_cpu_starting()
2241 	cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimers_cpu_starting()
2242 	cpu_base->online = 1;  in hrtimers_cpu_starting()