137 		sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
138 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
176 	ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags);  in init_srcu_struct_nodes()
177 	if (!ssp->srcu_sup->node)  in init_srcu_struct_nodes()
181 	ssp->srcu_sup->level[0] = &ssp->srcu_sup->node[0];  in init_srcu_struct_nodes()
183 		ssp->srcu_sup->level[i] = ssp->srcu_sup->level[i - 1] + num_rcu_lvl[i - 1];  in init_srcu_struct_nodes()
198 		if (snp == &ssp->srcu_sup->node[0]) {  in init_srcu_struct_nodes()
205 		if (snp == ssp->srcu_sup->level[level + 1])  in init_srcu_struct_nodes()
207 		snp->srcu_parent = ssp->srcu_sup->level[level - 1] +  in init_srcu_struct_nodes()
208 				   (snp - ssp->srcu_sup->level[level]) /  in init_srcu_struct_nodes()
217 	snp_first = ssp->srcu_sup->level[level];  in init_srcu_struct_nodes()
228 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_WAIT_BARRIER);  in init_srcu_struct_nodes()
240 		ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL);  in init_srcu_struct_fields()
241 	if (!ssp->srcu_sup)  in init_srcu_struct_fields()
244 		spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in init_srcu_struct_fields()
245 	ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;  in init_srcu_struct_fields()
246 	ssp->srcu_sup->node = NULL;  in init_srcu_struct_fields()
247 	mutex_init(&ssp->srcu_sup->srcu_cb_mutex);  in init_srcu_struct_fields()
248 	mutex_init(&ssp->srcu_sup->srcu_gp_mutex);  in init_srcu_struct_fields()
249 	ssp->srcu_sup->srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL;  in init_srcu_struct_fields()
250 	ssp->srcu_sup->srcu_barrier_seq = 0;  in init_srcu_struct_fields()
251 	mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);  in init_srcu_struct_fields()
252 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);  in init_srcu_struct_fields()
253 	INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);  in init_srcu_struct_fields()
254 	ssp->srcu_sup->sda_is_static = is_static;  in init_srcu_struct_fields()
262 	ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL;  in init_srcu_struct_fields()
263 	ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();  in init_srcu_struct_fields()
264 	if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {  in init_srcu_struct_fields()
267 		WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);  in init_srcu_struct_fields()
269 	ssp->srcu_sup->srcu_ssp = ssp;  in init_srcu_struct_fields()
270 	smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed,  in init_srcu_struct_fields()
281 		kfree(ssp->srcu_sup);  in init_srcu_struct_fields()
282 		ssp->srcu_sup = NULL;  in init_srcu_struct_fields()
322 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in __srcu_transition_to_big()
323 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC);  in __srcu_transition_to_big()
334 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL)  in srcu_transition_to_big()
336 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
337 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) {  in srcu_transition_to_big()
338 		spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
342 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
353 	if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state)  in spin_lock_irqsave_check_contention()
356 	if (ssp->srcu_sup->srcu_size_jiffies != j) {  in spin_lock_irqsave_check_contention()
357 		ssp->srcu_sup->srcu_size_jiffies = j;  in spin_lock_irqsave_check_contention()
358 		ssp->srcu_sup->srcu_n_lock_retries = 0;  in spin_lock_irqsave_check_contention()
360 	if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim)  in spin_lock_irqsave_check_contention()
377 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_sdp_contention()
379 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_sdp_contention()
391 	if (spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags))  in spin_lock_irqsave_ssp_contention()
393 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_ssp_contention()
410 	if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed))) /*^^^*/  in check_init_srcu_struct()
412 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
413 	if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq_needed)) {  in check_init_srcu_struct()
414 		spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
418 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
426 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_is_expedited()
649 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_get_delay()
651 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in srcu_get_delay()
680 	struct srcu_usage *sup = ssp->srcu_sup;  in cleanup_srcu_struct()
682 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in cleanup_srcu_struct()
684 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in cleanup_srcu_struct()
716 		ssp->srcu_sup = NULL;  in cleanup_srcu_struct()
811 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in srcu_gp_start()
812 	WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed));  in srcu_gp_start()
813 	WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies);  in srcu_gp_start()
814 	WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0);  in srcu_gp_start()
816 	rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
817 	state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
889 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_end()
991 			if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, s)) ||  in srcu_funnel_exp_start()
1004 	if (ULONG_CMP_LT(ssp->srcu_sup->srcu_gp_seq_needed_exp, s))  in srcu_funnel_exp_start()
1005 		WRITE_ONCE(ssp->srcu_sup->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
1006 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_funnel_exp_start()
1031 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_funnel_gp_start()
1107 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in try_check_zero()
1109 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in try_check_zero()
1225 	tlast = READ_ONCE(ssp->srcu_sup->srcu_last_gp_end);  in srcu_should_expedite()
1231 	curseq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcu_should_expedite()
1233 	if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_sup->srcu_gp_seq_needed)))  in srcu_should_expedite()
1236 	if (curseq != rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq))  in srcu_should_expedite()
1270 	ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state);  in srcu_gp_start_if_needed()
1314 	s = rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start_if_needed()
1317 				      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_gp_start_if_needed()
1544 	return rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in get_state_synchronize_srcu()
1592 	    !rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, cookie))  in poll_state_synchronize_srcu()
1612 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier_cb()
1613 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier_cb()
1627 	atomic_inc(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1633 		atomic_dec(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1646 	unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1649 	mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1650 	if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) {  in srcu_barrier()
1652 		mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1655 	rcu_seq_start(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1656 	init_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1659 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 1);  in srcu_barrier()
1662 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_barrier()
1670 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier()
1671 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1672 	wait_for_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1674 	rcu_seq_end(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1675 	mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1688 	return READ_ONCE(ssp->srcu_sup->srcu_gp_seq);  in srcu_batches_completed()
1701 	mutex_lock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1713 	idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */  in srcu_advance_state()
1715 		spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1716 		if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_advance_state()
1717 			WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1718 			spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1719 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1722 		idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1725 		spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1727 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1732 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) {  in srcu_advance_state()
1735 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1739 		spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1740 		rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);  in srcu_advance_state()
1741 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1742 		spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1745 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) {  in srcu_advance_state()
1753 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1756 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1783 			      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_invoke_callbacks()
1832 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1833 	if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_reschedule()
1834 		if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq))) {  in srcu_reschedule()
1838 	} else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) {  in srcu_reschedule()
1842 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1845 		queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay);  in srcu_reschedule()
1862 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in process_srcu()
1864 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in process_srcu()
1886 	*gp_seq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcutorture_get_gp_data()
1908 	int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state);  in srcu_torture_stats_print()
1915 		 tt, tf, rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq), ss_state,  in srcu_torture_stats_print()
2030 		if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) &&  in srcu_module_going()
2031 		    !WARN_ON_ONCE(!ssp->srcu_sup->sda_is_static))  in srcu_module_going()