Lines Matching refs:rnp
150 static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
155 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
767 static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_gpnum_ovf() argument
769 raw_lockdep_assert_held_rcu_node(rnp); in rcu_gpnum_ovf()
771 rnp->gp_seq)) in rcu_gpnum_ovf()
773 if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq)) in rcu_gpnum_ovf()
774 rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4; in rcu_gpnum_ovf()
818 struct rcu_node *rnp = rdp->mynode; in rcu_watching_snap_recheck() local
830 rcu_gpnum_ovf(rnp, rdp); in rcu_watching_snap_recheck()
856 __func__, rnp->grplo, rnp->grphi, rnp->level, in rcu_watching_snap_recheck()
857 (long)rnp->gp_seq, (long)rnp->completedqs); in rcu_watching_snap_recheck()
858 for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent) in rcu_watching_snap_recheck()
921 !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq && in rcu_watching_snap_recheck()
922 (rnp->ffmask & rdp->grpmask)) { in rcu_watching_snap_recheck()
924 rdp->rcu_iw_gp_seq = rnp->gp_seq; in rcu_watching_snap_recheck()
951 static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, in trace_rcu_this_gp() argument
954 trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), in trace_rcu_this_gp()
955 gp_seq_req, rnp->level, in trace_rcu_this_gp()
956 rnp->grplo, rnp->grphi, s); in trace_rcu_this_gp()
979 struct rcu_node *rnp; in rcu_start_this_gp() local
992 for (rnp = rnp_start; 1; rnp = rnp->parent) { in rcu_start_this_gp()
993 if (rnp != rnp_start) in rcu_start_this_gp()
994 raw_spin_lock_rcu_node(rnp); in rcu_start_this_gp()
995 if (ULONG_CMP_GE(rnp->gp_seq_needed, gp_seq_req) || in rcu_start_this_gp()
996 rcu_seq_started(&rnp->gp_seq, gp_seq_req) || in rcu_start_this_gp()
997 (rnp != rnp_start && in rcu_start_this_gp()
998 rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) { in rcu_start_this_gp()
999 trace_rcu_this_gp(rnp, rdp, gp_seq_req, in rcu_start_this_gp()
1003 WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req); in rcu_start_this_gp()
1004 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) { in rcu_start_this_gp()
1015 if (rnp != rnp_start && rnp->parent != NULL) in rcu_start_this_gp()
1016 raw_spin_unlock_rcu_node(rnp); in rcu_start_this_gp()
1017 if (!rnp->parent) in rcu_start_this_gp()
1023 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot")); in rcu_start_this_gp()
1026 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot")); in rcu_start_this_gp()
1030 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread")); in rcu_start_this_gp()
1037 if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) { in rcu_start_this_gp()
1038 WRITE_ONCE(rnp_start->gp_seq_needed, rnp->gp_seq_needed); in rcu_start_this_gp()
1039 WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed); in rcu_start_this_gp()
1041 if (rnp != rnp_start) in rcu_start_this_gp()
1042 raw_spin_unlock_rcu_node(rnp); in rcu_start_this_gp()
1050 static bool rcu_future_gp_cleanup(struct rcu_node *rnp) in rcu_future_gp_cleanup() argument
1055 needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed); in rcu_future_gp_cleanup()
1057 rnp->gp_seq_needed = rnp->gp_seq; /* Avoid counter wrap. */ in rcu_future_gp_cleanup()
1058 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq, in rcu_future_gp_cleanup()
1134 static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_accelerate_cbs() argument
1140 raw_lockdep_assert_held_rcu_node(rnp); in rcu_accelerate_cbs()
1160 ret = rcu_start_this_gp(rnp, rdp, gp_seq_req); in rcu_accelerate_cbs()
1180 static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, in rcu_accelerate_cbs_unlocked() argument
1193 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ in rcu_accelerate_cbs_unlocked()
1194 needwake = rcu_accelerate_cbs(rnp, rdp); in rcu_accelerate_cbs_unlocked()
1195 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ in rcu_accelerate_cbs_unlocked()
1210 static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_advance_cbs() argument
1213 raw_lockdep_assert_held_rcu_node(rnp); in rcu_advance_cbs()
1223 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
1226 return rcu_accelerate_cbs(rnp, rdp); in rcu_advance_cbs()
1233 static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp, in rcu_advance_cbs_nowake() argument
1237 if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) || !raw_spin_trylock_rcu_node(rnp)) in rcu_advance_cbs_nowake()
1240 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) in rcu_advance_cbs_nowake()
1241 WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp)); in rcu_advance_cbs_nowake()
1242 raw_spin_unlock_rcu_node(rnp); in rcu_advance_cbs_nowake()
1264 static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) in __note_gp_changes() argument
1270 raw_lockdep_assert_held_rcu_node(rnp); in __note_gp_changes()
1272 if (rdp->gp_seq == rnp->gp_seq) in __note_gp_changes()
1276 if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) || in __note_gp_changes()
1279 ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */ in __note_gp_changes()
1284 ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */ in __note_gp_changes()
1286 rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask); in __note_gp_changes()
1290 if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) || in __note_gp_changes()
1297 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart")); in __note_gp_changes()
1298 need_qs = !!(rnp->qsmask & rdp->grpmask); in __note_gp_changes()
1303 rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */ in __note_gp_changes()
1304 if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap) in __note_gp_changes()
1305 WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed); in __note_gp_changes()
1309 rcu_gpnum_ovf(rnp, rdp); in __note_gp_changes()
1317 struct rcu_node *rnp; in note_gp_changes() local
1320 rnp = rdp->mynode; in note_gp_changes()
1321 if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) && in note_gp_changes()
1323 !raw_spin_trylock_rcu_node(rnp)) { /* irqs already off, so later. */ in note_gp_changes()
1327 needwake = __note_gp_changes(rnp, rdp); in note_gp_changes()
1328 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in note_gp_changes()
1405 struct rcu_node *rnp = rcu_get_root(); in rcu_poll_gp_seq_start() local
1408 raw_lockdep_assert_held_rcu_node(rnp); in rcu_poll_gp_seq_start()
1421 struct rcu_node *rnp = rcu_get_root(); in rcu_poll_gp_seq_end() local
1424 raw_lockdep_assert_held_rcu_node(rnp); in rcu_poll_gp_seq_end()
1443 struct rcu_node *rnp = rcu_get_root(); in rcu_poll_gp_seq_start_unlocked() local
1448 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_poll_gp_seq_start_unlocked()
1452 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_poll_gp_seq_start_unlocked()
1460 struct rcu_node *rnp = rcu_get_root(); in rcu_poll_gp_seq_end_unlocked() local
1465 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_poll_gp_seq_end_unlocked()
1469 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_poll_gp_seq_end_unlocked()
1799 struct rcu_node *rnp = rcu_get_root(); in rcu_gp_init() local
1803 raw_spin_lock_irq_rcu_node(rnp); in rcu_gp_init()
1806 raw_spin_unlock_irq_rcu_node(rnp); in rcu_gp_init()
1816 raw_spin_unlock_irq_rcu_node(rnp); in rcu_gp_init()
1832 raw_spin_unlock_irq_rcu_node(rnp); in rcu_gp_init()
1858 rcu_for_each_leaf_node(rnp) { in rcu_gp_init()
1861 raw_spin_lock_rcu_node(rnp); in rcu_gp_init()
1862 if (rnp->qsmaskinit == rnp->qsmaskinitnext && in rcu_gp_init()
1863 !rnp->wait_blkd_tasks) { in rcu_gp_init()
1865 raw_spin_unlock_rcu_node(rnp); in rcu_gp_init()
1872 oldmask = rnp->qsmaskinit; in rcu_gp_init()
1873 rnp->qsmaskinit = rnp->qsmaskinitnext; in rcu_gp_init()
1876 if (!oldmask != !rnp->qsmaskinit) { in rcu_gp_init()
1878 if (!rnp->wait_blkd_tasks) /* Ever offline? */ in rcu_gp_init()
1879 rcu_init_new_rnp(rnp); in rcu_gp_init()
1880 } else if (rcu_preempt_has_tasks(rnp)) { in rcu_gp_init()
1881 rnp->wait_blkd_tasks = true; /* blocked tasks */ in rcu_gp_init()
1883 rcu_cleanup_dead_rnp(rnp); in rcu_gp_init()
1895 if (rnp->wait_blkd_tasks && in rcu_gp_init()
1896 (!rcu_preempt_has_tasks(rnp) || rnp->qsmaskinit)) { in rcu_gp_init()
1897 rnp->wait_blkd_tasks = false; in rcu_gp_init()
1898 if (!rnp->qsmaskinit) in rcu_gp_init()
1899 rcu_cleanup_dead_rnp(rnp); in rcu_gp_init()
1902 raw_spin_unlock_rcu_node(rnp); in rcu_gp_init()
1921 rcu_for_each_node_breadth_first(rnp) { in rcu_gp_init()
1923 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_gp_init()
1925 rcu_preempt_check_blocked_tasks(rnp); in rcu_gp_init()
1926 rnp->qsmask = rnp->qsmaskinit; in rcu_gp_init()
1927 WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq); in rcu_gp_init()
1928 if (rnp == rdp->mynode) in rcu_gp_init()
1929 (void)__note_gp_changes(rnp, rdp); in rcu_gp_init()
1930 rcu_preempt_boost_start_gp(rnp); in rcu_gp_init()
1931 trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq, in rcu_gp_init()
1932 rnp->level, rnp->grplo, in rcu_gp_init()
1933 rnp->grphi, rnp->qsmask); in rcu_gp_init()
1935 mask = rnp->qsmask & ~rnp->qsmaskinitnext; in rcu_gp_init()
1936 rnp->rcu_gp_init_mask = mask; in rcu_gp_init()
1937 if ((mask || rnp->wait_blkd_tasks) && rcu_is_leaf_node(rnp)) in rcu_gp_init()
1938 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); in rcu_gp_init()
1940 raw_spin_unlock_irq_rcu_node(rnp); in rcu_gp_init()
1958 struct rcu_node *rnp = rcu_get_root(); in rcu_gp_fqs_check_wake() local
1970 if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp)) in rcu_gp_fqs_check_wake()
1982 struct rcu_node *rnp = rcu_get_root(); in rcu_gp_fqs() local
2006 raw_spin_lock_irq_rcu_node(rnp); in rcu_gp_fqs()
2008 raw_spin_unlock_irq_rcu_node(rnp); in rcu_gp_fqs()
2021 struct rcu_node *rnp = rcu_get_root(); in rcu_gp_fqs_loop() local
2060 if (!READ_ONCE(rnp->qsmask) && in rcu_gp_fqs_loop()
2061 !rcu_preempt_blocked_readers_cgp(rnp)) in rcu_gp_fqs_loop()
2109 struct rcu_node *rnp = rcu_get_root(); in rcu_gp_cleanup() local
2113 raw_spin_lock_irq_rcu_node(rnp); in rcu_gp_cleanup()
2128 raw_spin_unlock_irq_rcu_node(rnp); in rcu_gp_cleanup()
2141 rcu_for_each_node_breadth_first(rnp) { in rcu_gp_cleanup()
2142 raw_spin_lock_irq_rcu_node(rnp); in rcu_gp_cleanup()
2143 if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp))) in rcu_gp_cleanup()
2144 dump_blkd_tasks(rnp, 10); in rcu_gp_cleanup()
2145 WARN_ON_ONCE(rnp->qsmask); in rcu_gp_cleanup()
2146 WRITE_ONCE(rnp->gp_seq, new_gp_seq); in rcu_gp_cleanup()
2147 if (!rnp->parent) in rcu_gp_cleanup()
2150 if (rnp == rdp->mynode) in rcu_gp_cleanup()
2151 needgp = __note_gp_changes(rnp, rdp) || needgp; in rcu_gp_cleanup()
2153 needgp = rcu_future_gp_cleanup(rnp) || needgp; in rcu_gp_cleanup()
2155 if (rcu_is_leaf_node(rnp)) in rcu_gp_cleanup()
2156 for_each_leaf_node_cpu_mask(rnp, cpu, rnp->cbovldmask) { in rcu_gp_cleanup()
2158 check_cb_ovld_locked(rdp, rnp); in rcu_gp_cleanup()
2160 sq = rcu_nocb_gp_get(rnp); in rcu_gp_cleanup()
2161 raw_spin_unlock_irq_rcu_node(rnp); in rcu_gp_cleanup()
2167 rnp = rcu_get_root(); in rcu_gp_cleanup()
2168 raw_spin_lock_irq_rcu_node(rnp); /* GP before ->gp_seq update. */ in rcu_gp_cleanup()
2177 if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) { in rcu_gp_cleanup()
2178 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed, in rcu_gp_cleanup()
2184 if ((offloaded || !rcu_accelerate_cbs(rnp, rdp)) && needgp) { in rcu_gp_cleanup()
2209 raw_spin_unlock_irq_rcu_node(rnp); in rcu_gp_cleanup()
2290 static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, in rcu_report_qs_rnp() argument
2292 __releases(rnp->lock) in rcu_report_qs_rnp()
2297 raw_lockdep_assert_held_rcu_node(rnp); in rcu_report_qs_rnp()
2301 if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) { in rcu_report_qs_rnp()
2307 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_report_qs_rnp()
2311 WARN_ON_ONCE(!rcu_is_leaf_node(rnp) && in rcu_report_qs_rnp()
2312 rcu_preempt_blocked_readers_cgp(rnp)); in rcu_report_qs_rnp()
2313 WRITE_ONCE(rnp->qsmask, rnp->qsmask & ~mask); in rcu_report_qs_rnp()
2314 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq, in rcu_report_qs_rnp()
2315 mask, rnp->qsmask, rnp->level, in rcu_report_qs_rnp()
2316 rnp->grplo, rnp->grphi, in rcu_report_qs_rnp()
2317 !!rnp->gp_tasks); in rcu_report_qs_rnp()
2318 if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) { in rcu_report_qs_rnp()
2321 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_report_qs_rnp()
2324 rnp->completedqs = rnp->gp_seq; in rcu_report_qs_rnp()
2325 mask = rnp->grpmask; in rcu_report_qs_rnp()
2326 if (rnp->parent == NULL) { in rcu_report_qs_rnp()
2332 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_report_qs_rnp()
2333 rnp_c = rnp; in rcu_report_qs_rnp()
2334 rnp = rnp->parent; in rcu_report_qs_rnp()
2335 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_report_qs_rnp()
2355 rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags) in rcu_report_unblock_qs_rnp() argument
2356 __releases(rnp->lock) in rcu_report_unblock_qs_rnp()
2362 raw_lockdep_assert_held_rcu_node(rnp); in rcu_report_unblock_qs_rnp()
2364 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)) || in rcu_report_unblock_qs_rnp()
2365 rnp->qsmask != 0) { in rcu_report_unblock_qs_rnp()
2366 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_report_unblock_qs_rnp()
2370 rnp->completedqs = rnp->gp_seq; in rcu_report_unblock_qs_rnp()
2371 rnp_p = rnp->parent; in rcu_report_unblock_qs_rnp()
2382 gps = rnp->gp_seq; in rcu_report_unblock_qs_rnp()
2383 mask = rnp->grpmask; in rcu_report_unblock_qs_rnp()
2384 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ in rcu_report_unblock_qs_rnp()
2398 struct rcu_node *rnp; in rcu_report_qs_rdp() local
2401 rnp = rdp->mynode; in rcu_report_qs_rdp()
2402 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_report_qs_rdp()
2403 if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq || in rcu_report_qs_rdp()
2413 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_report_qs_rdp()
2418 if ((rnp->qsmask & mask) == 0) { in rcu_report_qs_rdp()
2419 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_report_qs_rdp()
2433 WARN_ON_ONCE(rcu_accelerate_cbs(rnp, rdp)); in rcu_report_qs_rdp()
2437 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); in rcu_report_qs_rdp()
2689 struct rcu_node *rnp; in force_qs_rnp() local
2693 rcu_for_each_leaf_node(rnp) { in force_qs_rnp()
2698 raw_spin_lock_irqsave_rcu_node(rnp, flags); in force_qs_rnp()
2699 rcu_state.cbovldnext |= !!rnp->cbovldmask; in force_qs_rnp()
2700 if (rnp->qsmask == 0) { in force_qs_rnp()
2701 if (rcu_preempt_blocked_readers_cgp(rnp)) { in force_qs_rnp()
2707 rcu_initiate_boost(rnp, flags); in force_qs_rnp()
2711 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in force_qs_rnp()
2714 for_each_leaf_node_cpu_mask(rnp, cpu, rnp->qsmask) { in force_qs_rnp()
2729 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); in force_qs_rnp()
2732 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in force_qs_rnp()
2735 for_each_leaf_node_cpu_mask(rnp, cpu, rsmask) in force_qs_rnp()
2748 struct rcu_node *rnp; in rcu_force_quiescent_state() local
2754 rnp = raw_cpu_read(rcu_data.mynode); in rcu_force_quiescent_state()
2755 for (; rnp != NULL; rnp = rnp->parent) { in rcu_force_quiescent_state()
2757 !raw_spin_trylock(&rnp->fqslock); in rcu_force_quiescent_state()
2762 rnp_old = rnp; in rcu_force_quiescent_state()
2792 struct rcu_node *rnp = rdp->mynode; in rcu_core() local
2815 rcu_accelerate_cbs_unlocked(rnp, rdp); in rcu_core()
2819 rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); in rcu_core()
3020 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp) in check_cb_ovld_locked() argument
3022 raw_lockdep_assert_held_rcu_node(rnp); in check_cb_ovld_locked()
3026 WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask | rdp->grpmask); in check_cb_ovld_locked()
3028 WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask); in check_cb_ovld_locked()
3045 struct rcu_node *const rnp = rdp->mynode; in check_cb_ovld() local
3049 !!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask))) in check_cb_ovld()
3051 raw_spin_lock_rcu_node(rnp); in check_cb_ovld()
3052 check_cb_ovld_locked(rdp, rnp); in check_cb_ovld()
3053 raw_spin_unlock_rcu_node(rnp); in check_cb_ovld()
3303 struct rcu_node *rnp; in synchronize_rcu() local
3333 for (rnp = this_cpu_ptr(&rcu_data)->mynode; rnp; rnp = rnp->parent) in synchronize_rcu()
3334 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; in synchronize_rcu()
3414 struct rcu_node *rnp; in start_poll_synchronize_rcu_common() local
3418 rnp = rdp->mynode; in start_poll_synchronize_rcu_common()
3419 raw_spin_lock_rcu_node(rnp); // irqs already disabled. in start_poll_synchronize_rcu_common()
3426 needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq)); in start_poll_synchronize_rcu_common()
3427 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in start_poll_synchronize_rcu_common()
3548 struct rcu_node *rnp = rcu_get_root(); in poll_state_synchronize_rcu_full() local
3552 rcu_seq_done_exact(&rnp->gp_seq, rgosp->rgos_norm) || in poll_state_synchronize_rcu_full()
3625 struct rcu_node *rnp = rdp->mynode; in rcu_pending() local
3661 if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq || in rcu_pending()
3941 static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp) in rcu_rnp_online_cpus() argument
3943 return READ_ONCE(rnp->qsmaskinitnext); in rcu_rnp_online_cpus()
4030 struct rcu_node *rnp = rnp_leaf; in rcu_cleanup_dead_rnp() local
4038 mask = rnp->grpmask; in rcu_cleanup_dead_rnp()
4039 rnp = rnp->parent; in rcu_cleanup_dead_rnp()
4040 if (!rnp) in rcu_cleanup_dead_rnp()
4042 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ in rcu_cleanup_dead_rnp()
4043 rnp->qsmaskinit &= ~mask; in rcu_cleanup_dead_rnp()
4045 WARN_ON_ONCE(rnp->qsmask); in rcu_cleanup_dead_rnp()
4046 if (rnp->qsmaskinit) { in rcu_cleanup_dead_rnp()
4047 raw_spin_unlock_rcu_node(rnp); in rcu_cleanup_dead_rnp()
4051 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ in rcu_cleanup_dead_rnp()
4065 struct rcu_node *rnp = rnp_leaf; in rcu_init_new_rnp() local
4068 WARN_ON_ONCE(rnp->wait_blkd_tasks); in rcu_init_new_rnp()
4070 mask = rnp->grpmask; in rcu_init_new_rnp()
4071 rnp = rnp->parent; in rcu_init_new_rnp()
4072 if (rnp == NULL) in rcu_init_new_rnp()
4074 raw_spin_lock_rcu_node(rnp); /* Interrupts already disabled. */ in rcu_init_new_rnp()
4075 oldmask = rnp->qsmaskinit; in rcu_init_new_rnp()
4076 rnp->qsmaskinit |= mask; in rcu_init_new_rnp()
4077 raw_spin_unlock_rcu_node(rnp); /* Interrupts remain disabled. */ in rcu_init_new_rnp()
4107 static void rcu_thread_affine_rnp(struct task_struct *t, struct rcu_node *rnp) in rcu_thread_affine_rnp() argument
4115 for_each_leaf_node_possible_cpu(rnp, cpu) in rcu_thread_affine_rnp()
4125 static void rcu_spawn_exp_par_gp_kworker(struct rcu_node *rnp) in rcu_spawn_exp_par_gp_kworker() argument
4130 int rnp_index = rnp - rcu_get_root(); in rcu_spawn_exp_par_gp_kworker()
4132 if (rnp->exp_kworker) in rcu_spawn_exp_par_gp_kworker()
4138 rnp->grplo, rnp->grphi); in rcu_spawn_exp_par_gp_kworker()
4141 WRITE_ONCE(rnp->exp_kworker, kworker); in rcu_spawn_exp_par_gp_kworker()
4146 rcu_thread_affine_rnp(kworker->task, rnp); in rcu_spawn_exp_par_gp_kworker()
4166 static void rcu_spawn_rnp_kthreads(struct rcu_node *rnp) in rcu_spawn_rnp_kthreads() argument
4169 mutex_lock(&rnp->kthread_mutex); in rcu_spawn_rnp_kthreads()
4170 rcu_spawn_one_boost_kthread(rnp); in rcu_spawn_rnp_kthreads()
4171 rcu_spawn_exp_par_gp_kworker(rnp); in rcu_spawn_rnp_kthreads()
4172 mutex_unlock(&rnp->kthread_mutex); in rcu_spawn_rnp_kthreads()
4191 struct rcu_node *rnp = rcu_get_root(); in rcutree_prepare_cpu() local
4194 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcutree_prepare_cpu()
4199 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ in rcutree_prepare_cpu()
4213 rnp = rdp->mynode; in rcutree_prepare_cpu()
4214 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ in rcutree_prepare_cpu()
4215 rdp->gp_seq = READ_ONCE(rnp->gp_seq); in rcutree_prepare_cpu()
4223 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcutree_prepare_cpu()
4224 rcu_spawn_rnp_kthreads(rnp); in rcutree_prepare_cpu()
4250 struct rcu_node *rnp; in rcutree_online_cpu() local
4253 rnp = rdp->mynode; in rcutree_online_cpu()
4254 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcutree_online_cpu()
4255 rnp->ffmask |= rdp->grpmask; in rcutree_online_cpu()
4256 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcutree_online_cpu()
4284 struct rcu_node *rnp; in rcutree_report_cpu_starting() local
4293 rnp = rdp->mynode; in rcutree_report_cpu_starting()
4298 raw_spin_lock_rcu_node(rnp); in rcutree_report_cpu_starting()
4299 WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext | mask); in rcutree_report_cpu_starting()
4301 newcpu = !(rnp->expmaskinitnext & mask); in rcutree_report_cpu_starting()
4302 rnp->expmaskinitnext |= mask; in rcutree_report_cpu_starting()
4306 rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */ in rcutree_report_cpu_starting()
4311 if (WARN_ON_ONCE(rnp->qsmask & mask)) { /* RCU waiting on incoming CPU? */ in rcutree_report_cpu_starting()
4318 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); in rcutree_report_cpu_starting()
4320 raw_spin_unlock_rcu_node(rnp); in rcutree_report_cpu_starting()
4342 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ in rcutree_report_cpu_dead() local
4357 raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */ in rcutree_report_cpu_dead()
4360 if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */ in rcutree_report_cpu_dead()
4363 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); in rcutree_report_cpu_dead()
4364 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcutree_report_cpu_dead()
4366 WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext & ~mask); in rcutree_report_cpu_dead()
4367 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcutree_report_cpu_dead()
4452 struct rcu_node *rnp = rdp->mynode; in rcutree_dying_cpu() local
4454 blkd = !!(READ_ONCE(rnp->qsmask) & rdp->grpmask); in rcutree_dying_cpu()
4455 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), in rcutree_dying_cpu()
4468 struct rcu_node *rnp; in rcutree_offline_cpu() local
4471 rnp = rdp->mynode; in rcutree_offline_cpu()
4472 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcutree_offline_cpu()
4473 rnp->ffmask &= ~rdp->grpmask; in rcutree_offline_cpu()
4474 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcutree_offline_cpu()
4512 struct rcu_node *rnp; in rcu_spawn_gp_kthread() local
4525 rnp = rcu_get_root(); in rcu_spawn_gp_kthread()
4526 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_spawn_gp_kthread()
4531 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_spawn_gp_kthread()
4561 struct rcu_node *rnp; in rcu_scheduler_starting() local
4569 rcu_for_each_node_breadth_first(rnp) in rcu_scheduler_starting()
4570 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; in rcu_scheduler_starting()
4592 struct rcu_node *rnp; in rcu_init_one() local
4611 rnp = rcu_state.level[i]; in rcu_init_one()
4612 for (j = 0; j < num_rcu_lvl[i]; j++, rnp++) { in rcu_init_one()
4613 raw_spin_lock_init(&ACCESS_PRIVATE(rnp, lock)); in rcu_init_one()
4614 lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp, lock), in rcu_init_one()
4616 raw_spin_lock_init(&rnp->fqslock); in rcu_init_one()
4617 lockdep_set_class_and_name(&rnp->fqslock, in rcu_init_one()
4619 rnp->gp_seq = rcu_state.gp_seq; in rcu_init_one()
4620 rnp->gp_seq_needed = rcu_state.gp_seq; in rcu_init_one()
4621 rnp->completedqs = rcu_state.gp_seq; in rcu_init_one()
4622 rnp->qsmask = 0; in rcu_init_one()
4623 rnp->qsmaskinit = 0; in rcu_init_one()
4624 rnp->grplo = j * cpustride; in rcu_init_one()
4625 rnp->grphi = (j + 1) * cpustride - 1; in rcu_init_one()
4626 if (rnp->grphi >= nr_cpu_ids) in rcu_init_one()
4627 rnp->grphi = nr_cpu_ids - 1; in rcu_init_one()
4629 rnp->grpnum = 0; in rcu_init_one()
4630 rnp->grpmask = 0; in rcu_init_one()
4631 rnp->parent = NULL; in rcu_init_one()
4633 rnp->grpnum = j % levelspread[i - 1]; in rcu_init_one()
4634 rnp->grpmask = BIT(rnp->grpnum); in rcu_init_one()
4635 rnp->parent = rcu_state.level[i - 1] + in rcu_init_one()
4638 rnp->level = i; in rcu_init_one()
4639 INIT_LIST_HEAD(&rnp->blkd_tasks); in rcu_init_one()
4640 rcu_init_one_nocb(rnp); in rcu_init_one()
4641 init_waitqueue_head(&rnp->exp_wq[0]); in rcu_init_one()
4642 init_waitqueue_head(&rnp->exp_wq[1]); in rcu_init_one()
4643 init_waitqueue_head(&rnp->exp_wq[2]); in rcu_init_one()
4644 init_waitqueue_head(&rnp->exp_wq[3]); in rcu_init_one()
4645 spin_lock_init(&rnp->exp_lock); in rcu_init_one()
4646 mutex_init(&rnp->kthread_mutex); in rcu_init_one()
4647 raw_spin_lock_init(&rnp->exp_poll_lock); in rcu_init_one()
4648 rnp->exp_seq_poll_rq = RCU_GET_STATE_COMPLETED; in rcu_init_one()
4649 INIT_WORK(&rnp->exp_poll_wq, sync_rcu_do_polled_gp); in rcu_init_one()
4655 rnp = rcu_first_leaf_node(); in rcu_init_one()
4657 while (i > rnp->grphi) in rcu_init_one()
4658 rnp++; in rcu_init_one()
4659 per_cpu_ptr(&rcu_data, i)->mynode = rnp; in rcu_init_one()
4788 struct rcu_node *rnp; in rcu_dump_rcu_node_tree() local
4792 rcu_for_each_node_breadth_first(rnp) { in rcu_dump_rcu_node_tree()
4793 if (rnp->level != level) { in rcu_dump_rcu_node_tree()
4796 level = rnp->level; in rcu_dump_rcu_node_tree()
4798 pr_cont("%d:%d ^%d ", rnp->grplo, rnp->grphi, rnp->grpnum); in rcu_dump_rcu_node_tree()