194 	struct rcu_node *rnp;  in rcu_iw_handler()  local
197 	rnp = rdp->mynode;  in rcu_iw_handler()
198 	raw_spin_lock_rcu_node(rnp);  in rcu_iw_handler()
200 		rdp->rcu_iw_gp_seq = rnp->gp_seq;  in rcu_iw_handler()
203 	raw_spin_unlock_rcu_node(rnp);  in rcu_iw_handler()
216 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)  in rcu_print_detail_task_stall_rnp()  argument
221 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_print_detail_task_stall_rnp()
222 	if (!rcu_preempt_blocked_readers_cgp(rnp)) {  in rcu_print_detail_task_stall_rnp()
223 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_print_detail_task_stall_rnp()
226 	t = list_entry(rnp->gp_tasks->prev,  in rcu_print_detail_task_stall_rnp()
228 	list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {  in rcu_print_detail_task_stall_rnp()
236 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_print_detail_task_stall_rnp()
266 static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)  in rcu_print_task_stall()  argument
267 	__releases(rnp->lock)  in rcu_print_task_stall()
276 	if (!rcu_preempt_blocked_readers_cgp(rnp)) {  in rcu_print_task_stall()
277 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_print_task_stall()
281 	       rnp->level, rnp->grplo, rnp->grphi);  in rcu_print_task_stall()
282 	t = list_entry(rnp->gp_tasks->prev,  in rcu_print_task_stall()
284 	list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {  in rcu_print_task_stall()
290 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_print_task_stall()
316 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)  in rcu_print_detail_task_stall_rnp()  argument
324 static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)  in rcu_print_task_stall()  argument
325 	__releases(rnp->lock)  in rcu_print_task_stall()
327 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_print_task_stall()
342 	struct rcu_node *rnp;  in rcu_dump_cpu_stacks()  local
344 	rcu_for_each_leaf_node(rnp) {  in rcu_dump_cpu_stacks()
346 		for_each_leaf_node_possible_cpu(rnp, cpu) {  in rcu_dump_cpu_stacks()
352 			if (!(data_race(rnp->qsmask) & leaf_node_cpu_bit(rnp, cpu)))  in rcu_dump_cpu_stacks()
354 			raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_dump_cpu_stacks()
355 			if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {  in rcu_dump_cpu_stacks()
361 			raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_dump_cpu_stacks()
580 	struct rcu_node *rnp;  in print_other_cpu_stall()  local
599 	rcu_for_each_leaf_node(rnp) {  in print_other_cpu_stall()
600 		raw_spin_lock_irqsave_rcu_node(rnp, flags);  in print_other_cpu_stall()
601 		if (rnp->qsmask != 0) {  in print_other_cpu_stall()
602 			for_each_leaf_node_possible_cpu(rnp, cpu)  in print_other_cpu_stall()
603 				if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {  in print_other_cpu_stall()
608 		ndetected += rcu_print_task_stall(rnp, flags); // Releases rnp->lock.  in print_other_cpu_stall()
622 		rcu_for_each_leaf_node(rnp)  in print_other_cpu_stall()
623 			rcu_print_detail_task_stall_rnp(rnp);  in print_other_cpu_stall()
656 	struct rcu_node *rnp = rcu_get_root();  in print_cpu_stall()  local
690 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in print_cpu_stall()
695 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in print_cpu_stall()
724 	struct rcu_node *rnp;  in check_cpu_stall()  local
773 	rnp = rdp->mynode;  in check_cpu_stall()
775 	self_detected = READ_ONCE(rnp->qsmask) & rdp->grpmask;  in check_cpu_stall()
834 	struct rcu_node *rnp;  in rcu_check_boost_fail()  local
836 	rcu_for_each_leaf_node(rnp) {  in rcu_check_boost_fail()
838 			if (data_race(READ_ONCE(rnp->qsmask))) {  in rcu_check_boost_fail()
841 				if (READ_ONCE(rnp->gp_tasks))  in rcu_check_boost_fail()
847 		raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_check_boost_fail()
848 		if (rnp->gp_tasks)  in rcu_check_boost_fail()
850 		if (!rnp->qsmask) {  in rcu_check_boost_fail()
852 			raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_check_boost_fail()
856 		for_each_leaf_node_possible_cpu(rnp, cpu) {  in rcu_check_boost_fail()
857 			if (rnp->qsmask & (1UL << (cpu - rnp->grplo))) {  in rcu_check_boost_fail()
858 				raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_check_boost_fail()
863 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_check_boost_fail()
883 	struct rcu_node *rnp;  in show_rcu_gp_kthreads()  local
900 	rcu_for_each_node_breadth_first(rnp) {  in show_rcu_gp_kthreads()
901 		if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) &&  in show_rcu_gp_kthreads()
902 		    !data_race(READ_ONCE(rnp->qsmask)) && !data_race(READ_ONCE(rnp->boost_tasks)) &&  in show_rcu_gp_kthreads()
903 		    !data_race(READ_ONCE(rnp->exp_tasks)) && !data_race(READ_ONCE(rnp->gp_tasks)))  in show_rcu_gp_kthreads()
906 			rnp->grplo, rnp->grphi,  in show_rcu_gp_kthreads()
907 			(long)data_race(READ_ONCE(rnp->gp_seq)),  in show_rcu_gp_kthreads()
908 			(long)data_race(READ_ONCE(rnp->gp_seq_needed)),  in show_rcu_gp_kthreads()
909 			data_race(READ_ONCE(rnp->qsmask)),  in show_rcu_gp_kthreads()
910 			".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))],  in show_rcu_gp_kthreads()
911 			".B"[!!data_race(READ_ONCE(rnp->boost_tasks))],  in show_rcu_gp_kthreads()
912 			".E"[!!data_race(READ_ONCE(rnp->exp_tasks))],  in show_rcu_gp_kthreads()
913 			".G"[!!data_race(READ_ONCE(rnp->gp_tasks))],  in show_rcu_gp_kthreads()
914 			data_race(READ_ONCE(rnp->n_boosts)));  in show_rcu_gp_kthreads()
915 		if (!rcu_is_leaf_node(rnp))  in show_rcu_gp_kthreads()
917 		for_each_leaf_node_possible_cpu(rnp, cpu) {  in show_rcu_gp_kthreads()
942 static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp,  in rcu_check_gp_start_stall()  argument
961 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_check_gp_start_stall()
969 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_check_gp_start_stall()
974 	if (rnp_root != rnp)  in rcu_check_gp_start_stall()
983 		if (rnp_root != rnp)  in rcu_check_gp_start_stall()
986 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_check_gp_start_stall()
990 	if (rnp_root != rnp)  in rcu_check_gp_start_stall()
992 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_check_gp_start_stall()