Lines Matching refs:rnp
14 static int rcu_print_task_exp_stall(struct rcu_node *rnp);
15 static void rcu_exp_print_detail_task_stall_rnp(struct rcu_node *rnp);
84 struct rcu_node *rnp; in sync_exp_reset_tree_hotplug() local
96 rcu_for_each_leaf_node(rnp) { in sync_exp_reset_tree_hotplug()
97 raw_spin_lock_irqsave_rcu_node(rnp, flags); in sync_exp_reset_tree_hotplug()
98 if (rnp->expmaskinit == rnp->expmaskinitnext) { in sync_exp_reset_tree_hotplug()
99 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in sync_exp_reset_tree_hotplug()
104 oldmask = rnp->expmaskinit; in sync_exp_reset_tree_hotplug()
105 rnp->expmaskinit = rnp->expmaskinitnext; in sync_exp_reset_tree_hotplug()
106 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in sync_exp_reset_tree_hotplug()
113 mask = rnp->grpmask; in sync_exp_reset_tree_hotplug()
114 rnp_up = rnp->parent; in sync_exp_reset_tree_hotplug()
137 struct rcu_node *rnp; in sync_exp_reset_tree() local
140 rcu_for_each_node_breadth_first(rnp) { in sync_exp_reset_tree()
141 raw_spin_lock_irqsave_rcu_node(rnp, flags); in sync_exp_reset_tree()
142 WARN_ON_ONCE(rnp->expmask); in sync_exp_reset_tree()
143 WRITE_ONCE(rnp->expmask, rnp->expmaskinit); in sync_exp_reset_tree()
144 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in sync_exp_reset_tree()
154 static bool sync_rcu_exp_done(struct rcu_node *rnp) in sync_rcu_exp_done() argument
156 raw_lockdep_assert_held_rcu_node(rnp); in sync_rcu_exp_done()
157 return READ_ONCE(rnp->exp_tasks) == NULL && in sync_rcu_exp_done()
158 READ_ONCE(rnp->expmask) == 0; in sync_rcu_exp_done()
165 static bool sync_rcu_exp_done_unlocked(struct rcu_node *rnp) in sync_rcu_exp_done_unlocked() argument
170 raw_spin_lock_irqsave_rcu_node(rnp, flags); in sync_rcu_exp_done_unlocked()
171 ret = sync_rcu_exp_done(rnp); in sync_rcu_exp_done_unlocked()
172 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in sync_rcu_exp_done_unlocked()
185 static void __rcu_report_exp_rnp(struct rcu_node *rnp, in __rcu_report_exp_rnp() argument
187 __releases(rnp->lock) in __rcu_report_exp_rnp()
191 raw_lockdep_assert_held_rcu_node(rnp); in __rcu_report_exp_rnp()
193 if (!sync_rcu_exp_done(rnp)) { in __rcu_report_exp_rnp()
194 if (!rnp->expmask) in __rcu_report_exp_rnp()
195 rcu_initiate_boost(rnp, flags); in __rcu_report_exp_rnp()
197 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in __rcu_report_exp_rnp()
200 if (rnp->parent == NULL) { in __rcu_report_exp_rnp()
201 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in __rcu_report_exp_rnp()
207 mask = rnp->grpmask; in __rcu_report_exp_rnp()
208 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled */ in __rcu_report_exp_rnp()
209 rnp = rnp->parent; in __rcu_report_exp_rnp()
210 raw_spin_lock_rcu_node(rnp); /* irqs already disabled */ in __rcu_report_exp_rnp()
211 WARN_ON_ONCE(!(rnp->expmask & mask)); in __rcu_report_exp_rnp()
212 WRITE_ONCE(rnp->expmask, rnp->expmask & ~mask); in __rcu_report_exp_rnp()
220 static void __maybe_unused rcu_report_exp_rnp(struct rcu_node *rnp, bool wake) in rcu_report_exp_rnp() argument
224 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_report_exp_rnp()
225 __rcu_report_exp_rnp(rnp, wake, flags); in rcu_report_exp_rnp()
232 static void rcu_report_exp_cpu_mult(struct rcu_node *rnp, unsigned long flags, in rcu_report_exp_cpu_mult() argument
234 __releases(rnp->lock) in rcu_report_exp_cpu_mult()
240 raw_lockdep_assert_held_rcu_node(rnp); in rcu_report_exp_cpu_mult()
241 if (!(rnp->expmask & mask_in)) { in rcu_report_exp_cpu_mult()
242 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_report_exp_cpu_mult()
245 mask = mask_in & rnp->expmask; in rcu_report_exp_cpu_mult()
246 WRITE_ONCE(rnp->expmask, rnp->expmask & ~mask); in rcu_report_exp_cpu_mult()
247 for_each_leaf_node_cpu_mask(rnp, cpu, mask) { in rcu_report_exp_cpu_mult()
254 __rcu_report_exp_rnp(rnp, wake, flags); /* Releases rnp->lock. */ in rcu_report_exp_cpu_mult()
263 struct rcu_node *rnp = rdp->mynode; in rcu_report_exp_rdp() local
265 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_report_exp_rdp()
268 rcu_report_exp_cpu_mult(rnp, flags, rdp->grpmask, true); in rcu_report_exp_rdp()
297 struct rcu_node *rnp = rdp->mynode; in exp_funnel_lock() local
301 if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s) && in exp_funnel_lock()
302 (rnp == rnp_root || in exp_funnel_lock()
314 for (; rnp != NULL; rnp = rnp->parent) { in exp_funnel_lock()
319 spin_lock(&rnp->exp_lock); in exp_funnel_lock()
320 if (ULONG_CMP_GE(rnp->exp_seq_rq, s)) { in exp_funnel_lock()
323 spin_unlock(&rnp->exp_lock); in exp_funnel_lock()
324 trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level, in exp_funnel_lock()
325 rnp->grplo, rnp->grphi, in exp_funnel_lock()
327 wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3], in exp_funnel_lock()
331 WRITE_ONCE(rnp->exp_seq_rq, s); /* Followers can wait on us. */ in exp_funnel_lock()
332 spin_unlock(&rnp->exp_lock); in exp_funnel_lock()
333 trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level, in exp_funnel_lock()
334 rnp->grplo, rnp->grphi, TPS("nxtlvl")); in exp_funnel_lock()
358 struct rcu_node *rnp = container_of(rewp, struct rcu_node, rew); in __sync_rcu_exp_select_node_cpus() local
360 raw_spin_lock_irqsave_rcu_node(rnp, flags); in __sync_rcu_exp_select_node_cpus()
364 for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) { in __sync_rcu_exp_select_node_cpus()
370 !(rnp->qsmaskinitnext & mask)) { in __sync_rcu_exp_select_node_cpus()
394 mask_ofl_ipi = rnp->expmask & ~mask_ofl_test; in __sync_rcu_exp_select_node_cpus()
401 if (rcu_preempt_has_tasks(rnp)) in __sync_rcu_exp_select_node_cpus()
402 WRITE_ONCE(rnp->exp_tasks, rnp->blkd_tasks.next); in __sync_rcu_exp_select_node_cpus()
403 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in __sync_rcu_exp_select_node_cpus()
406 for_each_leaf_node_cpu_mask(rnp, cpu, mask_ofl_ipi) { in __sync_rcu_exp_select_node_cpus()
427 raw_spin_lock_irqsave_rcu_node(rnp, flags); in __sync_rcu_exp_select_node_cpus()
428 if ((rnp->qsmaskinitnext & mask) && in __sync_rcu_exp_select_node_cpus()
429 (rnp->expmask & mask)) { in __sync_rcu_exp_select_node_cpus()
431 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in __sync_rcu_exp_select_node_cpus()
437 if (rnp->expmask & mask) in __sync_rcu_exp_select_node_cpus()
439 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in __sync_rcu_exp_select_node_cpus()
443 raw_spin_lock_irqsave_rcu_node(rnp, flags); in __sync_rcu_exp_select_node_cpus()
444 rcu_report_exp_cpu_mult(rnp, flags, mask_ofl_test, false); in __sync_rcu_exp_select_node_cpus()
463 static inline bool rcu_exp_par_worker_started(struct rcu_node *rnp) in rcu_exp_par_worker_started() argument
465 return !!READ_ONCE(rnp->exp_kworker); in rcu_exp_par_worker_started()
468 static inline void sync_rcu_exp_select_cpus_queue_work(struct rcu_node *rnp) in sync_rcu_exp_select_cpus_queue_work() argument
470 kthread_init_work(&rnp->rew.rew_work, sync_rcu_exp_select_node_cpus); in sync_rcu_exp_select_cpus_queue_work()
476 kthread_queue_work(READ_ONCE(rnp->exp_kworker), &rnp->rew.rew_work); in sync_rcu_exp_select_cpus_queue_work()
479 static inline void sync_rcu_exp_select_cpus_flush_work(struct rcu_node *rnp) in sync_rcu_exp_select_cpus_flush_work() argument
481 kthread_flush_work(&rnp->rew.rew_work); in sync_rcu_exp_select_cpus_flush_work()
507 struct rcu_node *rnp; in sync_rcu_exp_select_cpus() local
514 rcu_for_each_leaf_node(rnp) { in sync_rcu_exp_select_cpus()
515 rnp->exp_need_flush = false; in sync_rcu_exp_select_cpus()
516 if (!READ_ONCE(rnp->expmask)) in sync_rcu_exp_select_cpus()
518 if (!rcu_exp_par_worker_started(rnp) || in sync_rcu_exp_select_cpus()
520 rcu_is_last_leaf_node(rnp)) { in sync_rcu_exp_select_cpus()
522 sync_rcu_exp_select_node_cpus(&rnp->rew.rew_work); in sync_rcu_exp_select_cpus()
525 sync_rcu_exp_select_cpus_queue_work(rnp); in sync_rcu_exp_select_cpus()
526 rnp->exp_need_flush = true; in sync_rcu_exp_select_cpus()
530 rcu_for_each_leaf_node(rnp) in sync_rcu_exp_select_cpus()
531 if (rnp->exp_need_flush) in sync_rcu_exp_select_cpus()
532 sync_rcu_exp_select_cpus_flush_work(rnp); in sync_rcu_exp_select_cpus()
563 struct rcu_node *rnp; in synchronize_rcu_expedited_stall() local
572 rcu_for_each_leaf_node(rnp) { in synchronize_rcu_expedited_stall()
573 ndetected += rcu_print_task_exp_stall(rnp); in synchronize_rcu_expedited_stall()
574 for_each_leaf_node_possible_cpu(rnp, cpu) { in synchronize_rcu_expedited_stall()
577 mask = leaf_node_cpu_bit(rnp, cpu); in synchronize_rcu_expedited_stall()
578 if (!(READ_ONCE(rnp->expmask) & mask)) in synchronize_rcu_expedited_stall()
584 "o."[!!(rdp->grpmask & rnp->expmaskinit)], in synchronize_rcu_expedited_stall()
585 "N."[!!(rdp->grpmask & rnp->expmaskinitnext)], in synchronize_rcu_expedited_stall()
594 rcu_for_each_node_breadth_first(rnp) { in synchronize_rcu_expedited_stall()
595 if (rnp == rnp_root) in synchronize_rcu_expedited_stall()
597 if (sync_rcu_exp_done_unlocked(rnp)) in synchronize_rcu_expedited_stall()
600 rnp->level, rnp->grplo, rnp->grphi, data_race(rnp->expmask), in synchronize_rcu_expedited_stall()
601 ".T"[!!data_race(rnp->exp_tasks)]); in synchronize_rcu_expedited_stall()
605 rcu_for_each_leaf_node(rnp) { in synchronize_rcu_expedited_stall()
606 for_each_leaf_node_possible_cpu(rnp, cpu) { in synchronize_rcu_expedited_stall()
607 mask = leaf_node_cpu_bit(rnp, cpu); in synchronize_rcu_expedited_stall()
608 if (!(READ_ONCE(rnp->expmask) & mask)) in synchronize_rcu_expedited_stall()
612 rcu_exp_print_detail_task_stall_rnp(rnp); in synchronize_rcu_expedited_stall()
628 struct rcu_node *rnp; in synchronize_rcu_expedited_wait() local
637 rcu_for_each_leaf_node(rnp) { in synchronize_rcu_expedited_wait()
638 raw_spin_lock_irqsave_rcu_node(rnp, flags); in synchronize_rcu_expedited_wait()
639 mask = READ_ONCE(rnp->expmask); in synchronize_rcu_expedited_wait()
640 for_each_leaf_node_cpu_mask(rnp, cpu, mask) { in synchronize_rcu_expedited_wait()
648 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in synchronize_rcu_expedited_wait()
685 struct rcu_node *rnp; in rcu_exp_wait_wake() local
696 rcu_for_each_node_breadth_first(rnp) { in rcu_exp_wait_wake()
697 if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) { in rcu_exp_wait_wake()
698 spin_lock(&rnp->exp_lock); in rcu_exp_wait_wake()
700 if (ULONG_CMP_LT(rnp->exp_seq_rq, s)) in rcu_exp_wait_wake()
701 WRITE_ONCE(rnp->exp_seq_rq, s); in rcu_exp_wait_wake()
702 spin_unlock(&rnp->exp_lock); in rcu_exp_wait_wake()
705 wake_up_all(&rnp->exp_wq[rcu_seq_ctr(s) & 0x3]); in rcu_exp_wait_wake()
750 struct rcu_node *rnp = rdp->mynode; in rcu_exp_handler() local
762 if (WARN_ON_ONCE(!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in rcu_exp_handler()
793 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_exp_handler()
794 if (rnp->expmask & rdp->grpmask) { in rcu_exp_handler()
798 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_exp_handler()
816 static int rcu_print_task_exp_stall(struct rcu_node *rnp) in rcu_print_task_exp_stall() argument
822 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_print_task_exp_stall()
823 if (!rnp->exp_tasks) { in rcu_print_task_exp_stall()
824 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_print_task_exp_stall()
827 t = list_entry(rnp->exp_tasks->prev, in rcu_print_task_exp_stall()
829 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { in rcu_print_task_exp_stall()
833 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_print_task_exp_stall()
842 static void rcu_exp_print_detail_task_stall_rnp(struct rcu_node *rnp) in rcu_exp_print_detail_task_stall_rnp() argument
849 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_exp_print_detail_task_stall_rnp()
850 if (!READ_ONCE(rnp->exp_tasks)) { in rcu_exp_print_detail_task_stall_rnp()
851 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_exp_print_detail_task_stall_rnp()
854 t = list_entry(rnp->exp_tasks->prev, in rcu_exp_print_detail_task_stall_rnp()
856 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { in rcu_exp_print_detail_task_stall_rnp()
864 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_exp_print_detail_task_stall_rnp()
873 struct rcu_node *rnp = rdp->mynode; in rcu_exp_handler() local
877 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in rcu_exp_handler()
895 struct rcu_node *rnp; in sync_sched_exp_online_cleanup() local
898 rnp = rdp->mynode; in sync_sched_exp_online_cleanup()
901 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in sync_sched_exp_online_cleanup()
925 static int rcu_print_task_exp_stall(struct rcu_node *rnp) in rcu_print_task_exp_stall() argument
935 static void rcu_exp_print_detail_task_stall_rnp(struct rcu_node *rnp) in rcu_exp_print_detail_task_stall_rnp() argument
965 struct rcu_node *rnp; in synchronize_rcu_expedited() local
1012 rnp = rcu_get_root(); in synchronize_rcu_expedited()
1013 wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3], in synchronize_rcu_expedited()
1029 struct rcu_node *rnp = container_of(wp, struct rcu_node, exp_poll_wq); in sync_rcu_do_polled_gp() local
1032 raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags); in sync_rcu_do_polled_gp()
1033 s = rnp->exp_seq_poll_rq; in sync_rcu_do_polled_gp()
1034 rnp->exp_seq_poll_rq = RCU_GET_STATE_COMPLETED; in sync_rcu_do_polled_gp()
1035 raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags); in sync_rcu_do_polled_gp()
1044 raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags); in sync_rcu_do_polled_gp()
1045 s = rnp->exp_seq_poll_rq; in sync_rcu_do_polled_gp()
1047 rnp->exp_seq_poll_rq = RCU_GET_STATE_COMPLETED; in sync_rcu_do_polled_gp()
1048 raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags); in sync_rcu_do_polled_gp()
1064 struct rcu_node *rnp; in start_poll_synchronize_rcu_expedited() local
1069 rnp = rdp->mynode; in start_poll_synchronize_rcu_expedited()
1071 raw_spin_lock_irqsave(&rnp->exp_poll_lock, flags); in start_poll_synchronize_rcu_expedited()
1074 rnp->exp_seq_poll_rq = s; in start_poll_synchronize_rcu_expedited()
1075 queue_work(rcu_gp_wq, &rnp->exp_poll_wq); in start_poll_synchronize_rcu_expedited()
1079 raw_spin_unlock_irqrestore(&rnp->exp_poll_lock, flags); in start_poll_synchronize_rcu_expedited()