Lines Matching refs:cblist

224 	if (rcu_segcblist_is_enabled(&rdp->cblist))  in rcu_get_n_cbs_cpu()
225 return rcu_segcblist_n_cbs(&rdp->cblist); in rcu_get_n_cbs_cpu()
696 return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) && in rcu_needs_cpu()
1143 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_accelerate_cbs()
1146 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPreAcc")); in rcu_accelerate_cbs()
1159 if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) in rcu_accelerate_cbs()
1163 if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) in rcu_accelerate_cbs()
1168 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPostAcc")); in rcu_accelerate_cbs()
1190 (void)rcu_segcblist_accelerate(&rdp->cblist, c); in rcu_accelerate_cbs_unlocked()
1216 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_advance_cbs()
1223 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
2506 if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { in rcu_do_batch()
2508 rcu_segcblist_n_cbs(&rdp->cblist), 0); in rcu_do_batch()
2510 !rcu_segcblist_empty(&rdp->cblist), in rcu_do_batch()
2529 pending = rcu_segcblist_get_seglen(&rdp->cblist, RCU_DONE_TAIL); in rcu_do_batch()
2544 rcu_segcblist_n_cbs(&rdp->cblist), bl); in rcu_do_batch()
2545 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2547 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2549 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbDequeued")); in rcu_do_batch()
2608 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2609 rcu_segcblist_add_len(&rdp->cblist, -count); in rcu_do_batch()
2612 count = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2627 empty = rcu_segcblist_empty(&rdp->cblist); in rcu_do_batch()
2631 WARN_ON_ONCE(count == 0 && rcu_segcblist_n_segment_cbs(&rdp->cblist) != 0); in rcu_do_batch()
2632 WARN_ON_ONCE(!empty && rcu_segcblist_n_segment_cbs(&rdp->cblist) == 0); in rcu_do_batch()
2812 rcu_segcblist_is_enabled(&rdp->cblist) && !rcu_rdp_is_offloaded(rdp)) { in rcu_core()
2814 if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_core()
2822 if (!rcu_rdp_is_offloaded(rdp) && rcu_segcblist_ready_cbs(&rdp->cblist) && in rcu_core()
2826 if (rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_core()
2955 rcu_segcblist_enqueue(&rdp->cblist, head); in rcutree_enqueue()
2957 rcu_segcblist_n_cbs(&rdp->cblist)); in rcutree_enqueue()
2958 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCBQueued")); in rcutree_enqueue()
2986 if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) > in call_rcu_core()
2999 rcu_segcblist_first_pend_cb(&rdp->cblist) != head) in call_rcu_core()
3002 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in call_rcu_core()
3025 if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) in check_cb_ovld_locked()
3048 ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) == in check_cb_ovld()
3091 if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { in __call_rcu_common()
3097 if (rcu_segcblist_empty(&rdp->cblist)) in __call_rcu_common()
3098 rcu_segcblist_init(&rdp->cblist); in __call_rcu_common()
3651 rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_pending()
3655 if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && in rcu_pending()
3657 !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_pending()
3724 was_alldone = rcu_rdp_is_offloaded(rdp) && !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_barrier_entrain()
3726 wake_nocb = was_alldone && rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_barrier_entrain()
3727 if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) { in rcu_barrier_entrain()
3812 if (!rcu_segcblist_n_cbs(&rdp->cblist)) { in rcu_barrier()
4205 if (!rcu_segcblist_is_enabled(&rdp->cblist)) in rcutree_prepare_cpu()
4206 rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ in rcutree_prepare_cpu()
4390 if (rcu_segcblist_empty(&rdp->cblist)) { in rcutree_migrate_callbacks()
4405 rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); in rcutree_migrate_callbacks()
4408 rcu_segcblist_disable(&rdp->cblist); in rcutree_migrate_callbacks()
4409 WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != !rcu_segcblist_n_cbs(&my_rdp->cblist)); in rcutree_migrate_callbacks()
4422 WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || in rcutree_migrate_callbacks()
4423 !rcu_segcblist_empty(&rdp->cblist), in rcutree_migrate_callbacks()
4425 cpu, rcu_segcblist_n_cbs(&rdp->cblist), in rcutree_migrate_callbacks()
4426 rcu_segcblist_first_cb(&rdp->cblist)); in rcutree_migrate_callbacks()