16 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp)  in rcu_rdp_is_offloaded()  argument
30 		  lockdep_is_held(&rdp->nocb_lock) ||  in rcu_rdp_is_offloaded()
33 		   rdp == this_cpu_ptr(&rcu_data)) ||  in rcu_rdp_is_offloaded()
34 		  rcu_current_is_nocb_kthread(rdp)),  in rcu_rdp_is_offloaded()
38 	return rcu_segcblist_is_offloaded(&rdp->cblist);  in rcu_rdp_is_offloaded()
162 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)  in rcu_preempt_ctxt_queue()  argument
167 			 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +  in rcu_preempt_ctxt_queue()
168 			 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);  in rcu_preempt_ctxt_queue()
172 	WARN_ON_ONCE(rdp->mynode != rnp);  in rcu_preempt_ctxt_queue()
176 		     rdp->grpmask);  in rcu_preempt_ctxt_queue()
261 		     !(rnp->qsmask & rdp->grpmask));  in rcu_preempt_ctxt_queue()
263 		     !(rnp->expmask & rdp->grpmask));  in rcu_preempt_ctxt_queue()
274 	if (blkd_state & RCU_EXP_BLKD && rdp->cpu_no_qs.b.exp)  in rcu_preempt_ctxt_queue()
275 		rcu_report_exp_rdp(rdp);  in rcu_preempt_ctxt_queue()
277 		WARN_ON_ONCE(rdp->cpu_no_qs.b.exp);  in rcu_preempt_ctxt_queue()
278 	ASSERT_EXCLUSIVE_WRITER_SCOPED(rdp->cpu_no_qs.b.exp);  in rcu_preempt_ctxt_queue()
327 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_note_context_switch()  local
337 		rnp = rdp->mynode;  in rcu_note_context_switch()
347 		WARN_ON_ONCE(!rcu_rdp_cpu_online(rdp));  in rcu_note_context_switch()
351 				       (rnp->qsmask & rdp->grpmask)  in rcu_note_context_switch()
354 		rcu_preempt_ctxt_queue(rnp, rdp);  in rcu_note_context_switch()
369 	if (rdp->cpu_no_qs.b.exp)  in rcu_note_context_switch()
370 		rcu_report_exp_rdp(rdp);  in rcu_note_context_switch()
485 	struct rcu_data *rdp;  in rcu_preempt_deferred_qs_irqrestore()  local
495 	rdp = this_cpu_ptr(&rcu_data);  in rcu_preempt_deferred_qs_irqrestore()
496 	if (!special.s && !rdp->cpu_no_qs.b.exp) {  in rcu_preempt_deferred_qs_irqrestore()
503 			rdp->cpu_no_qs.b.norm = false;  in rcu_preempt_deferred_qs_irqrestore()
504 			rcu_report_qs_rdp(rdp);  in rcu_preempt_deferred_qs_irqrestore()
517 	if (rdp->cpu_no_qs.b.exp)  in rcu_preempt_deferred_qs_irqrestore()
518 		rcu_report_exp_rdp(rdp);  in rcu_preempt_deferred_qs_irqrestore()
627 	struct rcu_data *rdp;  in rcu_preempt_deferred_qs_handler()  local
629 	rdp = container_of(iwp, struct rcu_data, defer_qs_iw);  in rcu_preempt_deferred_qs_handler()
630 	rdp->defer_qs_iw_pending = false;  in rcu_preempt_deferred_qs_handler()
653 		struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_read_unlock_special()  local
654 		struct rcu_node *rnp = rdp->mynode;  in rcu_read_unlock_special()
657 			   (rdp->grpmask & READ_ONCE(rnp->expmask)) ||  in rcu_read_unlock_special()
659 			   ((rdp->grpmask & READ_ONCE(rnp->qsmask)) || t->rcu_blocked_node)) ||  in rcu_read_unlock_special()
676 			    expboost && !rdp->defer_qs_iw_pending && cpu_online(rdp->cpu)) {  in rcu_read_unlock_special()
681 					rdp->defer_qs_iw = IRQ_WORK_INIT_HARD(  in rcu_read_unlock_special()
684 					init_irq_work(&rdp->defer_qs_iw,  in rcu_read_unlock_special()
686 				rdp->defer_qs_iw_pending = true;  in rcu_read_unlock_special()
687 				irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu);  in rcu_read_unlock_special()
795 	struct rcu_data *rdp;  in dump_blkd_tasks()  local
817 		rdp = per_cpu_ptr(&rcu_data, cpu);  in dump_blkd_tasks()
819 			cpu, ".o"[rcu_rdp_cpu_online(rdp)],  in dump_blkd_tasks()
820 			(long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_state,  in dump_blkd_tasks()
821 			(long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_state);  in dump_blkd_tasks()
834 	struct rcu_data *rdp;  in rcu_read_unlock_strict()  local
847 	rdp = this_cpu_ptr(&rcu_data);  in rcu_read_unlock_strict()
848 	rdp->cpu_no_qs.b.norm = false;  in rcu_read_unlock_strict()
849 	rcu_report_qs_rdp(rdp);  in rcu_read_unlock_strict()
965 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_preempt_deferred_qs()  local
967 	if (READ_ONCE(rdp->cpu_no_qs.b.exp))  in rcu_preempt_deferred_qs()
968 		rcu_report_exp_rdp(rdp);  in rcu_preempt_deferred_qs()
1031 	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_cpu_kthread_setup()  local
1039 	WRITE_ONCE(rdp->rcuc_activity, jiffies);  in rcu_cpu_kthread_setup()
1042 static bool rcu_is_callbacks_nocb_kthread(struct rcu_data *rdp)  in rcu_is_callbacks_nocb_kthread()  argument
1045 	return rdp->nocb_cb_kthread == current;  in rcu_is_callbacks_nocb_kthread()
1055 static bool rcu_is_callbacks_kthread(struct rcu_data *rdp)  in rcu_is_callbacks_kthread()  argument
1057 	return rdp->rcu_cpu_kthread_task == current ||  in rcu_is_callbacks_kthread()
1058 			rcu_is_callbacks_nocb_kthread(rdp);  in rcu_is_callbacks_kthread()