86 	.gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT,
213 	return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));  in rcu_gp_in_progress()
507 	return READ_ONCE(rcu_state.gp_seq);  in rcu_get_gp_seq()
534 void rcutorture_get_gp_data(int *flags, unsigned long *gp_seq)  in rcutorture_get_gp_data()  argument
537 	*gp_seq = rcu_seq_current(&rcu_state.gp_seq);  in rcutorture_get_gp_data()
544 	return ((READ_ONCE(rcu_state.gp_seq) & 0xffffULL) << 40) |  in rcutorture_gather_gp_seqs()
770 	if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + ULONG_MAX / 4,  in rcu_gpnum_ovf()
771 			 rnp->gp_seq))  in rcu_gpnum_ovf()
773 	if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq))  in rcu_gpnum_ovf()
774 		rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4;  in rcu_gpnum_ovf()
797 		trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));  in rcu_watching_snap_save()
829 		trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));  in rcu_watching_snap_recheck()
857 			(long)rnp->gp_seq, (long)rnp->completedqs);  in rcu_watching_snap_recheck()
921 		    !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&  in rcu_watching_snap_recheck()
924 			rdp->rcu_iw_gp_seq = rnp->gp_seq;  in rcu_watching_snap_recheck()
928 		if (rcu_cpu_stall_cputime && rdp->snap_record.gp_seq != rdp->gp_seq) {  in rcu_watching_snap_recheck()
943 			rsrp->gp_seq = rdp->gp_seq;  in rcu_watching_snap_recheck()
954 	trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),  in trace_rcu_this_gp()
996 		    rcu_seq_started(&rnp->gp_seq, gp_seq_req) ||  in rcu_start_this_gp()
998 		     rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) {  in rcu_start_this_gp()
1004 		if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {  in rcu_start_this_gp()
1033 	trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));  in rcu_start_this_gp()
1055 	needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed);  in rcu_future_gp_cleanup()
1057 		rnp->gp_seq_needed = rnp->gp_seq; /* Avoid counter wrap. */  in rcu_future_gp_cleanup()
1058 	trace_rcu_this_gp(rnp, rdp, rnp->gp_seq,  in rcu_future_gp_cleanup()
1118 	WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));  in rcu_gp_kthread_wake()
1158 	gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);  in rcu_accelerate_cbs()
1187 	c = rcu_seq_snap(&rcu_state.gp_seq);  in rcu_accelerate_cbs_unlocked()
1223 	rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq);  in rcu_advance_cbs()
1237 	if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) || !raw_spin_trylock_rcu_node(rnp))  in rcu_advance_cbs_nowake()
1240 	if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))  in rcu_advance_cbs_nowake()
1272 	if (rdp->gp_seq == rnp->gp_seq)  in __note_gp_changes()
1276 	if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) ||  in __note_gp_changes()
1281 		trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));  in __note_gp_changes()
1290 	if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) ||  in __note_gp_changes()
1297 		trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));  in __note_gp_changes()
1303 	rdp->gp_seq = rnp->gp_seq;  /* Remember new grace-period state. */  in __note_gp_changes()
1321 	if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) &&  in note_gp_changes()
1364 	    !(rcu_seq_ctr(rcu_state.gp_seq) % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))  in rcu_gp_slow()
1828 	rcu_seq_start(&rcu_state.gp_seq);  in rcu_gp_init()
1829 	ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);  in rcu_gp_init()
1830 	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));  in rcu_gp_init()
1927 		WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);  in rcu_gp_init()
1931 		trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,  in rcu_gp_init()
1938 			rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);  in rcu_gp_init()
2043 		trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_fqs_loop()
2066 			trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_fqs_loop()
2074 			trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_fqs_loop()
2085 			trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_fqs_loop()
2139 	new_gp_seq = rcu_state.gp_seq;  in rcu_gp_cleanup()
2146 		WRITE_ONCE(rnp->gp_seq, new_gp_seq);  in rcu_gp_cleanup()
2171 	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));  in rcu_gp_cleanup()
2172 	rcu_seq_end(&rcu_state.gp_seq);  in rcu_gp_cleanup()
2173 	ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);  in rcu_gp_cleanup()
2177 	if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) {  in rcu_gp_cleanup()
2198 		trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq"));  in rcu_gp_cleanup()
2229 			trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_kthread()
2243 			trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_kthread()
2301 		if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) {  in rcu_report_qs_rnp()
2314 		trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,  in rcu_report_qs_rnp()
2324 		rnp->completedqs = rnp->gp_seq;  in rcu_report_qs_rnp()
2370 	rnp->completedqs = rnp->gp_seq;  in rcu_report_unblock_qs_rnp()
2382 	gps = rnp->gp_seq;  in rcu_report_unblock_qs_rnp()
2403 	if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq ||  in rcu_report_qs_rdp()
2437 		rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);  in rcu_report_qs_rdp()
2729 			rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);  in force_qs_rnp()
3332 	rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT);  in synchronize_rcu()
3334 		rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;  in synchronize_rcu()
3400 	rgosp->rgos_norm = rcu_seq_snap(&rcu_state.gp_seq);  in get_state_synchronize_rcu_full()
3426 	needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq));  in start_poll_synchronize_rcu_common()
3443 	unsigned long gp_seq = get_state_synchronize_rcu();  in start_poll_synchronize_rcu()  local
3446 	return gp_seq;  in start_poll_synchronize_rcu()
3552 	    rcu_seq_done_exact(&rnp->gp_seq, rgosp->rgos_norm) ||  in poll_state_synchronize_rcu_full()
3661 	if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq ||  in rcu_pending()
4098 	rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;  in rcu_boot_init_percpu_data()
4100 	rdp->rcu_onl_gp_seq = rcu_state.gp_seq;  in rcu_boot_init_percpu_data()
4215 	rdp->gp_seq = READ_ONCE(rnp->gp_seq);  in rcutree_prepare_cpu()
4216 	rdp->gp_seq_needed = rdp->gp_seq;  in rcutree_prepare_cpu()
4221 	rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;  in rcutree_prepare_cpu()
4222 	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));  in rcutree_prepare_cpu()
4307 	rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);  in rcutree_report_cpu_starting()
4318 		rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);  in rcutree_report_cpu_starting()
4358 	rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);  in rcutree_report_cpu_dead()
4363 		rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);  in rcutree_report_cpu_dead()
4455 	trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),  in rcutree_dying_cpu()
4570 		rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;  in rcu_scheduler_starting()
4619 			rnp->gp_seq = rcu_state.gp_seq;  in rcu_init_one()
4620 			rnp->gp_seq_needed = rcu_state.gp_seq;  in rcu_init_one()
4621 			rnp->completedqs = rcu_state.gp_seq;  in rcu_init_one()