Lines Matching refs:rcu_state
83 static struct rcu_state rcu_state = { variable
84 .level = { &rcu_state.node[0] },
87 .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
88 .barrier_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.barrier_lock),
91 .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
92 .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
94 .srs_cleanup_work = __WORK_INITIALIZER(rcu_state.srs_cleanup_work,
98 .nocb_mutex = __MUTEX_INITIALIZER(rcu_state.nocb_mutex),
213 return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq)); in rcu_gp_in_progress()
507 return READ_ONCE(rcu_state.gp_seq); in rcu_get_gp_seq()
519 return rcu_state.expedited_sequence; in rcu_exp_batches_completed()
528 return &rcu_state.node[0]; in rcu_get_root()
536 *flags = READ_ONCE(rcu_state.gp_flags); in rcutorture_get_gp_data()
537 *gp_seq = rcu_seq_current(&rcu_state.gp_seq); in rcutorture_get_gp_data()
544 return ((READ_ONCE(rcu_state.gp_seq) & 0xffffULL) << 40) | in rcutorture_gather_gp_seqs()
545 ((READ_ONCE(rcu_state.expedited_sequence) & 0xffffffULL) << 16) | in rcutorture_gather_gp_seqs()
546 (READ_ONCE(rcu_state.gp_seq_polled) & 0xffffULL); in rcutorture_gather_gp_seqs()
797 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_watching_snap_save()
829 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_watching_snap_recheck()
881 (time_after(jiffies, rcu_state.gp_start + jtsq * 2) || in rcu_watching_snap_recheck()
882 time_after(jiffies, rcu_state.jiffies_resched) || in rcu_watching_snap_recheck()
883 rcu_state.cbovld)) { in rcu_watching_snap_recheck()
887 } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) { in rcu_watching_snap_recheck()
901 rcu_state.cbovld)) { in rcu_watching_snap_recheck()
914 if (time_after(jiffies, rcu_state.jiffies_resched)) { in rcu_watching_snap_recheck()
954 trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), in trace_rcu_this_gp()
1027 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT); in rcu_start_this_gp()
1028 WRITE_ONCE(rcu_state.gp_req_activity, jiffies); in rcu_start_this_gp()
1029 if (!READ_ONCE(rcu_state.gp_kthread)) { in rcu_start_this_gp()
1033 trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq")); in rcu_start_this_gp()
1112 struct task_struct *t = READ_ONCE(rcu_state.gp_kthread); in rcu_gp_kthread_wake()
1115 !READ_ONCE(rcu_state.gp_flags) || !t) in rcu_gp_kthread_wake()
1117 WRITE_ONCE(rcu_state.gp_wake_time, jiffies); in rcu_gp_kthread_wake()
1118 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq)); in rcu_gp_kthread_wake()
1119 swake_up_one_online(&rcu_state.gp_wq); in rcu_gp_kthread_wake()
1158 gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq); in rcu_accelerate_cbs()
1164 trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB")); in rcu_accelerate_cbs()
1166 trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB")); in rcu_accelerate_cbs()
1187 c = rcu_seq_snap(&rcu_state.gp_seq); in rcu_accelerate_cbs_unlocked()
1281 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend")); in __note_gp_changes()
1297 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart")); in __note_gp_changes()
1364 !(rcu_seq_ctr(rcu_state.gp_seq) % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay))) in rcu_gp_slow()
1411 if (!rcu_seq_state(rcu_state.gp_seq_polled)) in rcu_poll_gp_seq_start()
1412 rcu_seq_start(&rcu_state.gp_seq_polled); in rcu_poll_gp_seq_start()
1415 *snap = rcu_state.gp_seq_polled; in rcu_poll_gp_seq_start()
1429 if (*snap && *snap == rcu_state.gp_seq_polled) { in rcu_poll_gp_seq_end()
1430 rcu_seq_end(&rcu_state.gp_seq_polled); in rcu_poll_gp_seq_end()
1431 rcu_state.gp_seq_polled_snap = 0; in rcu_poll_gp_seq_end()
1432 rcu_state.gp_seq_polled_exp_snap = 0; in rcu_poll_gp_seq_end()
1600 return &(rcu_state.srs_wait_nodes)[0].node <= node && in rcu_sr_is_wait_head()
1601 node <= &(rcu_state.srs_wait_nodes)[SR_NORMAL_GP_WAIT_HEAD_MAX - 1].node; in rcu_sr_is_wait_head()
1610 sr_wn = &(rcu_state.srs_wait_nodes)[i]; in rcu_sr_get_wait_head()
1659 done = smp_load_acquire(&rcu_state.srs_done_tail); in rcu_sr_normal_gp_cleanup_work()
1686 atomic_dec_return_release(&rcu_state.srs_cleanups_pending); in rcu_sr_normal_gp_cleanup_work()
1697 wait_tail = rcu_state.srs_wait_tail; in rcu_sr_normal_gp_cleanup()
1701 rcu_state.srs_wait_tail = NULL; in rcu_sr_normal_gp_cleanup()
1702 ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail); in rcu_sr_normal_gp_cleanup()
1729 !atomic_read_acquire(&rcu_state.srs_cleanups_pending)) { in rcu_sr_normal_gp_cleanup()
1735 ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_done_tail); in rcu_sr_normal_gp_cleanup()
1736 smp_store_release(&rcu_state.srs_done_tail, wait_tail); in rcu_sr_normal_gp_cleanup()
1744 atomic_inc(&rcu_state.srs_cleanups_pending); in rcu_sr_normal_gp_cleanup()
1745 if (!queue_work(sync_wq, &rcu_state.srs_cleanup_work)) in rcu_sr_normal_gp_cleanup()
1746 atomic_dec(&rcu_state.srs_cleanups_pending); in rcu_sr_normal_gp_cleanup()
1759 first = READ_ONCE(rcu_state.srs_next.first); in rcu_sr_normal_gp_init()
1771 llist_add(wait_head, &rcu_state.srs_next); in rcu_sr_normal_gp_init()
1778 WARN_ON_ONCE(rcu_state.srs_wait_tail != NULL); in rcu_sr_normal_gp_init()
1779 rcu_state.srs_wait_tail = wait_head; in rcu_sr_normal_gp_init()
1780 ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail); in rcu_sr_normal_gp_init()
1787 llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next); in rcu_sr_normal_add_req()
1802 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_init()
1804 if (!rcu_state.gp_flags) { in rcu_gp_init()
1809 WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */ in rcu_gp_init()
1828 rcu_seq_start(&rcu_state.gp_seq); in rcu_gp_init()
1829 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq); in rcu_gp_init()
1830 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start")); in rcu_gp_init()
1831 rcu_poll_gp_seq_start(&rcu_state.gp_seq_polled_snap); in rcu_gp_init()
1856 WRITE_ONCE(rcu_state.gp_state, RCU_GP_ONOFF); in rcu_gp_init()
1860 arch_spin_lock(&rcu_state.ofl_lock); in rcu_gp_init()
1866 arch_spin_unlock(&rcu_state.ofl_lock); in rcu_gp_init()
1903 arch_spin_unlock(&rcu_state.ofl_lock); in rcu_gp_init()
1920 WRITE_ONCE(rcu_state.gp_state, RCU_GP_INIT); in rcu_gp_init()
1927 WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq); in rcu_gp_init()
1931 trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq, in rcu_gp_init()
1942 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_init()
1965 *gfp = READ_ONCE(rcu_state.gp_flags); in rcu_gp_fqs_check_wake()
1981 int nr_fqs = READ_ONCE(rcu_state.nr_fqs_jiffies_stall); in rcu_gp_fqs()
1984 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs()
1985 WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1); in rcu_gp_fqs()
1991 WRITE_ONCE(rcu_state.jiffies_stall, in rcu_gp_fqs()
1994 WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, --nr_fqs); in rcu_gp_fqs()
2005 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) { in rcu_gp_fqs()
2007 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags & ~RCU_GP_FLAG_FQS); in rcu_gp_fqs()
2024 if (rcu_state.cbovld) in rcu_gp_fqs_loop()
2028 if (rcu_state.cbovld) { in rcu_gp_fqs_loop()
2033 if (!ret || time_before(jiffies + j, rcu_state.jiffies_force_qs)) { in rcu_gp_fqs_loop()
2034 WRITE_ONCE(rcu_state.jiffies_force_qs, jiffies + j); in rcu_gp_fqs_loop()
2040 WRITE_ONCE(rcu_state.jiffies_kick_kthreads, in rcu_gp_fqs_loop()
2043 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
2045 WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_FQS); in rcu_gp_fqs_loop()
2046 (void)swait_event_idle_timeout_exclusive(rcu_state.gp_wq, in rcu_gp_fqs_loop()
2049 WRITE_ONCE(rcu_state.gp_state, RCU_GP_DOING_FQS); in rcu_gp_fqs_loop()
2064 if (!time_after(rcu_state.jiffies_force_qs, jiffies) || in rcu_gp_fqs_loop()
2066 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
2072 gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0; in rcu_gp_fqs_loop()
2074 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
2077 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs_loop()
2083 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs_loop()
2085 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
2089 if (time_after(jiffies, rcu_state.jiffies_force_qs)) in rcu_gp_fqs_loop()
2092 j = rcu_state.jiffies_force_qs - j; in rcu_gp_fqs_loop()
2112 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_cleanup()
2114 rcu_state.gp_end = jiffies; in rcu_gp_cleanup()
2115 gp_duration = rcu_state.gp_end - rcu_state.gp_start; in rcu_gp_cleanup()
2116 if (gp_duration > rcu_state.gp_max) in rcu_gp_cleanup()
2117 rcu_state.gp_max = gp_duration; in rcu_gp_cleanup()
2127 rcu_poll_gp_seq_end(&rcu_state.gp_seq_polled_snap); in rcu_gp_cleanup()
2139 new_gp_seq = rcu_state.gp_seq; in rcu_gp_cleanup()
2164 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_cleanup()
2171 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end")); in rcu_gp_cleanup()
2172 rcu_seq_end(&rcu_state.gp_seq); in rcu_gp_cleanup()
2173 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq); in rcu_gp_cleanup()
2174 WRITE_ONCE(rcu_state.gp_state, RCU_GP_IDLE); in rcu_gp_cleanup()
2196 WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT); in rcu_gp_cleanup()
2197 WRITE_ONCE(rcu_state.gp_req_activity, jiffies); in rcu_gp_cleanup()
2198 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq")); in rcu_gp_cleanup()
2207 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags & RCU_GP_FLAG_INIT); in rcu_gp_cleanup()
2229 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_kthread()
2231 WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_GPS); in rcu_gp_kthread()
2232 swait_event_idle_exclusive(rcu_state.gp_wq, in rcu_gp_kthread()
2233 READ_ONCE(rcu_state.gp_flags) & in rcu_gp_kthread()
2236 WRITE_ONCE(rcu_state.gp_state, RCU_GP_DONE_GPS); in rcu_gp_kthread()
2241 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_kthread()
2243 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_kthread()
2251 WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANUP); in rcu_gp_kthread()
2253 WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANED); in rcu_gp_kthread()
2271 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_FQS); in rcu_report_qs_rsp()
2314 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq, in rcu_report_qs_rnp()
2507 trace_rcu_batch_start(rcu_state.name, in rcu_do_batch()
2509 trace_rcu_batch_end(rcu_state.name, 0, in rcu_do_batch()
2543 trace_rcu_batch_start(rcu_state.name, in rcu_do_batch()
2563 trace_rcu_invoke_callback(rcu_state.name, rhp); in rcu_do_batch()
2604 trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(), in rcu_do_batch()
2619 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcu_do_batch()
2691 rcu_state.cbovld = rcu_state.cbovldnext; in force_qs_rnp()
2692 rcu_state.cbovldnext = false; in force_qs_rnp()
2699 rcu_state.cbovldnext |= !!rnp->cbovldmask; in force_qs_rnp()
2756 ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) || in rcu_force_quiescent_state()
2769 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) { in rcu_force_quiescent_state()
2773 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_FQS); in rcu_force_quiescent_state()
2956 trace_rcu_callback(rcu_state.name, head, in rcutree_enqueue()
2998 if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap && in call_rcu_core()
3001 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in call_rcu_core()
3232 trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request")); in synchronize_rcu_normal()
3259 trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete")); in synchronize_rcu_normal()
3323 rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap); in synchronize_rcu()
3324 rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap); in synchronize_rcu()
3332 rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT); in synchronize_rcu()
3334 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; in synchronize_rcu()
3368 return rcu_seq_snap(&rcu_state.gp_seq_polled); in get_state_synchronize_rcu()
3400 rgosp->rgos_norm = rcu_seq_snap(&rcu_state.gp_seq); in get_state_synchronize_rcu_full()
3401 rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence); in get_state_synchronize_rcu_full()
3426 needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq)); in start_poll_synchronize_rcu_common()
3505 rcu_seq_done_exact(&rcu_state.gp_seq_polled, oldstate)) { in poll_state_synchronize_rcu()
3554 rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) { in poll_state_synchronize_rcu_full()
3640 time_before(jiffies, READ_ONCE(rcu_state.gp_start) + in rcu_pending()
3675 trace_rcu_barrier(rcu_state.name, s, cpu, in rcu_barrier_trace()
3676 atomic_read(&rcu_state.barrier_cpu_count), done); in rcu_barrier_trace()
3691 unsigned long __maybe_unused s = rcu_state.barrier_sequence; in rcu_barrier_callback()
3694 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) { in rcu_barrier_callback()
3696 complete(&rcu_state.barrier_completion); in rcu_barrier_callback()
3707 unsigned long gseq = READ_ONCE(rcu_state.barrier_sequence); in rcu_barrier_entrain()
3712 lockdep_assert_held(&rcu_state.barrier_lock); in rcu_barrier_entrain()
3715 rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence); in rcu_barrier_entrain()
3728 atomic_inc(&rcu_state.barrier_cpu_count); in rcu_barrier_entrain()
3731 rcu_barrier_trace(TPS("IRQNQ"), -1, rcu_state.barrier_sequence); in rcu_barrier_entrain()
3750 raw_spin_lock(&rcu_state.barrier_lock); in rcu_barrier_handler()
3752 raw_spin_unlock(&rcu_state.barrier_lock); in rcu_barrier_handler()
3769 unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence); in rcu_barrier()
3774 mutex_lock(&rcu_state.barrier_mutex); in rcu_barrier()
3777 if (rcu_seq_done(&rcu_state.barrier_sequence, s)) { in rcu_barrier()
3778 rcu_barrier_trace(TPS("EarlyExit"), -1, rcu_state.barrier_sequence); in rcu_barrier()
3780 mutex_unlock(&rcu_state.barrier_mutex); in rcu_barrier()
3785 raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags); in rcu_barrier()
3786 rcu_seq_start(&rcu_state.barrier_sequence); in rcu_barrier()
3787 gseq = rcu_state.barrier_sequence; in rcu_barrier()
3788 rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence); in rcu_barrier()
3797 init_completion(&rcu_state.barrier_completion); in rcu_barrier()
3798 atomic_set(&rcu_state.barrier_cpu_count, 2); in rcu_barrier()
3799 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags); in rcu_barrier()
3811 raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags); in rcu_barrier()
3814 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags); in rcu_barrier()
3815 rcu_barrier_trace(TPS("NQ"), cpu, rcu_state.barrier_sequence); in rcu_barrier()
3821 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags); in rcu_barrier()
3822 rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu, rcu_state.barrier_sequence); in rcu_barrier()
3825 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags); in rcu_barrier()
3831 rcu_barrier_trace(TPS("OnlineQ"), cpu, rcu_state.barrier_sequence); in rcu_barrier()
3838 if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count)) in rcu_barrier()
3839 complete(&rcu_state.barrier_completion); in rcu_barrier()
3842 wait_for_completion(&rcu_state.barrier_completion); in rcu_barrier()
3845 rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence); in rcu_barrier()
3846 rcu_seq_end(&rcu_state.barrier_sequence); in rcu_barrier()
3847 gseq = rcu_state.barrier_sequence; in rcu_barrier()
3855 mutex_unlock(&rcu_state.barrier_mutex); in rcu_barrier()
3884 unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence); in rcu_barrier_throttled()
3889 if (rcu_seq_done(&rcu_state.barrier_sequence, s)) { in rcu_barrier_throttled()
3994 if (rcu_rdp_cpu_online(rdp) || arch_spin_is_locked(&rcu_state.ofl_lock)) in rcu_lockdep_current_cpu_online()
4007 return !!READ_ONCE(rcu_state.n_online_cpus); in rcu_init_invoked()
4097 rdp->barrier_seq_snap = rcu_state.barrier_sequence; in rcu_boot_init_percpu_data()
4098 rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4100 rdp->rcu_onl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4196 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcutree_prepare_cpu()
4222 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl")); in rcutree_prepare_cpu()
4226 ASSERT_EXCLUSIVE_WRITER(rcu_state.n_online_cpus); in rcutree_prepare_cpu()
4227 WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus + 1); in rcutree_prepare_cpu()
4295 arch_spin_lock(&rcu_state.ofl_lock); in rcutree_report_cpu_starting()
4297 raw_spin_lock(&rcu_state.barrier_lock); in rcutree_report_cpu_starting()
4300 raw_spin_unlock(&rcu_state.barrier_lock); in rcutree_report_cpu_starting()
4304 smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */ in rcutree_report_cpu_starting()
4305 ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus); in rcutree_report_cpu_starting()
4307 rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcutree_report_cpu_starting()
4308 rdp->rcu_onl_gp_state = READ_ONCE(rcu_state.gp_state); in rcutree_report_cpu_starting()
4322 arch_spin_unlock(&rcu_state.ofl_lock); in rcutree_report_cpu_starting()
4356 arch_spin_lock(&rcu_state.ofl_lock); in rcutree_report_cpu_dead()
4358 rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcutree_report_cpu_dead()
4359 rdp->rcu_ofl_gp_state = READ_ONCE(rcu_state.gp_state); in rcutree_report_cpu_dead()
4368 arch_spin_unlock(&rcu_state.ofl_lock); in rcutree_report_cpu_dead()
4389 raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags); in rcutree_migrate_callbacks()
4391 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags); in rcutree_migrate_callbacks()
4406 raw_spin_unlock(&rcu_state.barrier_lock); /* irqs remain disabled. */ in rcutree_migrate_callbacks()
4437 ASSERT_EXCLUSIVE_WRITER(rcu_state.n_online_cpus); in rcutree_dead_cpu()
4438 WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus - 1); in rcutree_dead_cpu()
4455 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), in rcutree_dying_cpu()
4518 t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name); in rcu_spawn_gp_kthread()
4527 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_spawn_gp_kthread()
4528 WRITE_ONCE(rcu_state.gp_req_activity, jiffies); in rcu_spawn_gp_kthread()
4530 smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */ in rcu_spawn_gp_kthread()
4570 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; in rcu_scheduler_starting()
4603 rcu_state.level[i] = in rcu_init_one()
4604 rcu_state.level[i - 1] + num_rcu_lvl[i - 1]; in rcu_init_one()
4611 rnp = rcu_state.level[i]; in rcu_init_one()
4619 rnp->gp_seq = rcu_state.gp_seq; in rcu_init_one()
4620 rnp->gp_seq_needed = rcu_state.gp_seq; in rcu_init_one()
4621 rnp->completedqs = rcu_state.gp_seq; in rcu_init_one()
4635 rnp->parent = rcu_state.level[i - 1] + in rcu_init_one()
4653 init_swait_queue_head(&rcu_state.gp_wq); in rcu_init_one()
4654 init_swait_queue_head(&rcu_state.expedited_wq); in rcu_init_one()