426 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_is_expedited()  local
428 	return ULONG_CMP_LT(READ_ONCE(sup->srcu_gp_seq), READ_ONCE(sup->srcu_gp_seq_needed_exp));  in srcu_gp_is_expedited()
649 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_get_delay()  local
654 	if (rcu_seq_state(READ_ONCE(sup->srcu_gp_seq))) {  in srcu_get_delay()
656 		gpstart = READ_ONCE(sup->srcu_gp_start);  in srcu_get_delay()
660 			ASSERT_EXCLUSIVE_WRITER(sup->srcu_n_exp_nodelay);  in srcu_get_delay()
661 			WRITE_ONCE(sup->srcu_n_exp_nodelay, READ_ONCE(sup->srcu_n_exp_nodelay) + 1);  in srcu_get_delay()
662 			if (READ_ONCE(sup->srcu_n_exp_nodelay) > srcu_max_nodelay_phase)  in srcu_get_delay()
680 	struct srcu_usage *sup = ssp->srcu_sup;  in cleanup_srcu_struct()  local
689 	flush_delayed_work(&sup->work);  in cleanup_srcu_struct()
698 	if (WARN_ON(rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)) != SRCU_STATE_IDLE) ||  in cleanup_srcu_struct()
699 	    WARN_ON(rcu_seq_current(&sup->srcu_gp_seq) != sup->srcu_gp_seq_needed) ||  in cleanup_srcu_struct()
702 			__func__, ssp, rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)),  in cleanup_srcu_struct()
703 			rcu_seq_current(&sup->srcu_gp_seq), sup->srcu_gp_seq_needed);  in cleanup_srcu_struct()
709 	kfree(sup->node);  in cleanup_srcu_struct()
710 	sup->node = NULL;  in cleanup_srcu_struct()
711 	sup->srcu_size_state = SRCU_SIZE_SMALL;  in cleanup_srcu_struct()
712 	if (!sup->sda_is_static) {  in cleanup_srcu_struct()
715 		kfree(sup);  in cleanup_srcu_struct()
889 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_end()  local
892 	mutex_lock(&sup->srcu_cb_mutex);  in srcu_gp_end()
895 	spin_lock_irq_rcu_node(sup);  in srcu_gp_end()
896 	idx = rcu_seq_state(sup->srcu_gp_seq);  in srcu_gp_end()
901 	WRITE_ONCE(sup->srcu_last_gp_end, ktime_get_mono_fast_ns());  in srcu_gp_end()
902 	rcu_seq_end(&sup->srcu_gp_seq);  in srcu_gp_end()
903 	gpseq = rcu_seq_current(&sup->srcu_gp_seq);  in srcu_gp_end()
904 	if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq))  in srcu_gp_end()
905 		WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq);  in srcu_gp_end()
906 	spin_unlock_irq_rcu_node(sup);  in srcu_gp_end()
907 	mutex_unlock(&sup->srcu_gp_mutex);  in srcu_gp_end()
911 	ss_state = smp_load_acquire(&sup->srcu_size_state);  in srcu_gp_end()
920 			last_lvl = snp >= sup->level[rcu_num_lvls - 1];  in srcu_gp_end()
952 	mutex_unlock(&sup->srcu_cb_mutex);  in srcu_gp_end()
955 	spin_lock_irq_rcu_node(sup);  in srcu_gp_end()
956 	gpseq = rcu_seq_current(&sup->srcu_gp_seq);  in srcu_gp_end()
958 	    ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) {  in srcu_gp_end()
960 		spin_unlock_irq_rcu_node(sup);  in srcu_gp_end()
963 		spin_unlock_irq_rcu_node(sup);  in srcu_gp_end()
971 			smp_store_release(&sup->srcu_size_state, ss_state + 1);  in srcu_gp_end()
1031 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_funnel_gp_start()  local
1034 	if (smp_load_acquire(&sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_funnel_gp_start()
1042 			if (WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) && snp != snp_leaf)  in srcu_funnel_gp_start()
1069 	if (ULONG_CMP_LT(sup->srcu_gp_seq_needed, s)) {  in srcu_funnel_gp_start()
1074 		smp_store_release(&sup->srcu_gp_seq_needed, s); /*^^^*/  in srcu_funnel_gp_start()
1076 	if (!do_norm && ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, s))  in srcu_funnel_gp_start()
1077 		WRITE_ONCE(sup->srcu_gp_seq_needed_exp, s);  in srcu_funnel_gp_start()
1080 	if (!WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) &&  in srcu_funnel_gp_start()
1081 	    rcu_seq_state(sup->srcu_gp_seq) == SRCU_STATE_IDLE) {  in srcu_funnel_gp_start()
1090 			queue_delayed_work(rcu_gp_wq, &sup->work,  in srcu_funnel_gp_start()
1092 		else if (list_empty(&sup->work.work.entry))  in srcu_funnel_gp_start()
1093 			list_add(&sup->work.work.entry, &srcu_boot_list);  in srcu_funnel_gp_start()
1095 	spin_unlock_irqrestore_rcu_node(sup, flags);  in srcu_funnel_gp_start()
1856 	struct srcu_usage *sup;  in process_srcu()  local
1858 	sup = container_of(work, struct srcu_usage, work.work);  in process_srcu()
1859 	ssp = sup->srcu_ssp;  in process_srcu()
1866 		WRITE_ONCE(sup->reschedule_count, 0);  in process_srcu()
1869 		if (READ_ONCE(sup->reschedule_jiffies) == j) {  in process_srcu()
1870 			ASSERT_EXCLUSIVE_WRITER(sup->reschedule_count);  in process_srcu()
1871 			WRITE_ONCE(sup->reschedule_count, READ_ONCE(sup->reschedule_count) + 1);  in process_srcu()
1872 			if (READ_ONCE(sup->reschedule_count) > srcu_max_nodelay)  in process_srcu()
1875 			WRITE_ONCE(sup->reschedule_count, 1);  in process_srcu()
1876 			WRITE_ONCE(sup->reschedule_jiffies, j);  in process_srcu()
1972 	struct srcu_usage *sup;  in srcu_init()  local
1992 		sup = list_first_entry(&srcu_boot_list, struct srcu_usage,  in srcu_init()
1994 		list_del_init(&sup->work.work.entry);  in srcu_init()
1996 		    sup->srcu_size_state == SRCU_SIZE_SMALL)  in srcu_init()
1997 			sup->srcu_size_state = SRCU_SIZE_ALLOC;  in srcu_init()
1998 		queue_work(rcu_gp_wq, &sup->work.work);  in srcu_init()