Lines Matching refs:rtpcp
271 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in cblist_init_generic() local
273 WARN_ON_ONCE(!rtpcp); in cblist_init_generic()
275 raw_spin_lock_init(&ACCESS_PRIVATE(rtpcp, lock)); in cblist_init_generic()
276 if (rcu_segcblist_empty(&rtpcp->cblist)) in cblist_init_generic()
277 rcu_segcblist_init(&rtpcp->cblist); in cblist_init_generic()
278 INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq); in cblist_init_generic()
279 rtpcp->cpu = cpu; in cblist_init_generic()
280 rtpcp->rtpp = rtp; in cblist_init_generic()
281 rtpcp->index = index; in cblist_init_generic()
282 rtp->rtpcp_array[index] = rtpcp; in cblist_init_generic()
284 if (!rtpcp->rtp_blkd_tasks.next) in cblist_init_generic()
285 INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); in cblist_init_generic()
286 if (!rtpcp->rtp_exit_list.next) in cblist_init_generic()
287 INIT_LIST_HEAD(&rtpcp->rtp_exit_list); in cblist_init_generic()
288 rtpcp->barrier_q_head.next = &rtpcp->barrier_q_head; in cblist_init_generic()
319 struct rcu_tasks_percpu *rtpcp = from_timer(rtpcp, tlp, lazy_timer); in call_rcu_tasks_generic_timer() local
321 rtp = rtpcp->rtpp; in call_rcu_tasks_generic_timer()
322 raw_spin_lock_irqsave_rcu_node(rtpcp, flags); in call_rcu_tasks_generic_timer()
323 if (!rcu_segcblist_empty(&rtpcp->cblist) && rtp->lazy_jiffies) { in call_rcu_tasks_generic_timer()
324 if (!rtpcp->urgent_gp) in call_rcu_tasks_generic_timer()
325 rtpcp->urgent_gp = 1; in call_rcu_tasks_generic_timer()
327 mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); in call_rcu_tasks_generic_timer()
329 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in call_rcu_tasks_generic_timer()
338 struct rcu_tasks_percpu *rtpcp = container_of(iwp, struct rcu_tasks_percpu, rtp_irq_work); in call_rcu_tasks_iw_wakeup() local
340 rtp = rtpcp->rtpp; in call_rcu_tasks_iw_wakeup()
355 struct rcu_tasks_percpu *rtpcp; in call_rcu_tasks_generic() local
364 rtpcp = per_cpu_ptr(rtp->rtpcpu, chosen_cpu); in call_rcu_tasks_generic()
365 if (!raw_spin_trylock_rcu_node(rtpcp)) { // irqs already disabled. in call_rcu_tasks_generic()
366 raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. in call_rcu_tasks_generic()
368 if (rtpcp->rtp_jiffies != j) { in call_rcu_tasks_generic()
369 rtpcp->rtp_jiffies = j; in call_rcu_tasks_generic()
370 rtpcp->rtp_n_lock_retries = 0; in call_rcu_tasks_generic()
372 if (rcu_task_cb_adjust && ++rtpcp->rtp_n_lock_retries > rcu_task_contend_lim && in call_rcu_tasks_generic()
377 if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) in call_rcu_tasks_generic()
378 rcu_segcblist_init(&rtpcp->cblist); in call_rcu_tasks_generic()
380 (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); in call_rcu_tasks_generic()
381 if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { in call_rcu_tasks_generic()
383 mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); in call_rcu_tasks_generic()
385 needwake = rcu_segcblist_empty(&rtpcp->cblist); in call_rcu_tasks_generic()
388 rtpcp->urgent_gp = 3; in call_rcu_tasks_generic()
389 rcu_segcblist_enqueue(&rtpcp->cblist, rhp); in call_rcu_tasks_generic()
390 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in call_rcu_tasks_generic()
404 irq_work_queue(&rtpcp->rtp_irq_work); in call_rcu_tasks_generic()
411 struct rcu_tasks_percpu *rtpcp; in rcu_barrier_tasks_generic_cb() local
414 rtpcp = container_of(rhp, struct rcu_tasks_percpu, barrier_q_head); in rcu_barrier_tasks_generic_cb()
415 rtp = rtpcp->rtpp; in rcu_barrier_tasks_generic_cb()
426 struct rcu_tasks_percpu *rtpcp; in rcu_barrier_tasks_generic() local
442 rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in rcu_barrier_tasks_generic()
443 rtpcp->barrier_q_head.func = rcu_barrier_tasks_generic_cb; in rcu_barrier_tasks_generic()
444 raw_spin_lock_irqsave_rcu_node(rtpcp, flags); in rcu_barrier_tasks_generic()
445 if (rcu_segcblist_entrain(&rtpcp->cblist, &rtpcp->barrier_q_head)) in rcu_barrier_tasks_generic()
447 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in rcu_barrier_tasks_generic()
473 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in rcu_tasks_need_gpcb() local
476 if (!rcu_segcblist_n_cbs(&rtpcp->cblist)) in rcu_tasks_need_gpcb()
478 raw_spin_lock_irqsave_rcu_node(rtpcp, flags); in rcu_tasks_need_gpcb()
480 n = rcu_segcblist_n_cbs(&rtpcp->cblist); in rcu_tasks_need_gpcb()
486 rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); in rcu_tasks_need_gpcb()
487 (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); in rcu_tasks_need_gpcb()
488 if (rtpcp->urgent_gp > 0 && rcu_segcblist_pend_cbs(&rtpcp->cblist)) { in rcu_tasks_need_gpcb()
490 rtpcp->urgent_gp--; in rcu_tasks_need_gpcb()
492 } else if (rcu_segcblist_empty(&rtpcp->cblist)) { in rcu_tasks_need_gpcb()
493 rtpcp->urgent_gp = 0; in rcu_tasks_need_gpcb()
495 if (rcu_segcblist_ready_cbs(&rtpcp->cblist)) in rcu_tasks_need_gpcb()
497 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in rcu_tasks_need_gpcb()
528 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in rcu_tasks_need_gpcb() local
530 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rtpcp->cblist)); in rcu_tasks_need_gpcb()
540 static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu *rtpcp) in rcu_tasks_invoke_cbs() argument
550 index = rtpcp->index * 2 + 1; in rcu_tasks_invoke_cbs()
567 if (rcu_segcblist_empty(&rtpcp->cblist)) in rcu_tasks_invoke_cbs()
569 raw_spin_lock_irqsave_rcu_node(rtpcp, flags); in rcu_tasks_invoke_cbs()
570 rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); in rcu_tasks_invoke_cbs()
571 rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl); in rcu_tasks_invoke_cbs()
572 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in rcu_tasks_invoke_cbs()
581 raw_spin_lock_irqsave_rcu_node(rtpcp, flags); in rcu_tasks_invoke_cbs()
582 rcu_segcblist_add_len(&rtpcp->cblist, -len); in rcu_tasks_invoke_cbs()
583 (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); in rcu_tasks_invoke_cbs()
584 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in rcu_tasks_invoke_cbs()
591 struct rcu_tasks_percpu *rtpcp = container_of(wp, struct rcu_tasks_percpu, rtp_work); in rcu_tasks_invoke_cbs_wq() local
593 rtp = rtpcp->rtpp; in rcu_tasks_invoke_cbs_wq()
594 rcu_tasks_invoke_cbs(rtp, rtpcp); in rcu_tasks_invoke_cbs_wq()
638 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in rcu_tasks_kthread() local
640 timer_setup(&rtpcp->lazy_timer, call_rcu_tasks_generic_timer, 0); in rcu_tasks_kthread()
641 rtpcp->urgent_gp = 1; in rcu_tasks_kthread()
730 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in show_rcu_tasks_generic_gp_kthread() local
732 if (!data_race(rcu_segcblist_empty(&rtpcp->cblist))) in show_rcu_tasks_generic_gp_kthread()
734 if (data_race(rtpcp->urgent_gp)) in show_rcu_tasks_generic_gp_kthread()
736 if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)) && data_race(rtpcp->urgent_gp)) in show_rcu_tasks_generic_gp_kthread()
777 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in rcu_tasks_torture_stats_print_generic() local
779 if (cpumask_available(cm) && !rcu_barrier_cb_is_done(&rtpcp->barrier_q_head)) in rcu_tasks_torture_stats_print_generic()
781 n = rcu_segcblist_n_cbs(&rtpcp->cblist); in rcu_tasks_torture_stats_print_generic()
1059 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, cpu); in rcu_tasks_postscan() local
1064 raw_spin_lock_irq_rcu_node(rtpcp); in rcu_tasks_postscan()
1065 list_for_each_entry_safe(t, t1, &rtpcp->rtp_exit_list, rcu_tasks_exit_list) { in rcu_tasks_postscan()
1078 raw_spin_unlock_irq_rcu_node(rtpcp); in rcu_tasks_postscan()
1080 raw_spin_lock_irq_rcu_node(rtpcp); in rcu_tasks_postscan()
1085 raw_spin_unlock_irq_rcu_node(rtpcp); in rcu_tasks_postscan()
1295 struct rcu_tasks_percpu *rtpcp; in exit_tasks_rcu_start() local
1300 rtpcp = this_cpu_ptr(rcu_tasks.rtpcpu); in exit_tasks_rcu_start()
1302 raw_spin_lock_irqsave_rcu_node(rtpcp, flags); in exit_tasks_rcu_start()
1303 WARN_ON_ONCE(!rtpcp->rtp_exit_list.next); in exit_tasks_rcu_start()
1304 list_add(&t->rcu_tasks_exit_list, &rtpcp->rtp_exit_list); in exit_tasks_rcu_start()
1305 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in exit_tasks_rcu_start()
1316 struct rcu_tasks_percpu *rtpcp; in exit_tasks_rcu_finish() local
1320 rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, t->rcu_tasks_exit_cpu); in exit_tasks_rcu_finish()
1321 raw_spin_lock_irqsave_rcu_node(rtpcp, flags); in exit_tasks_rcu_finish()
1323 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in exit_tasks_rcu_finish()
1564 struct rcu_tasks_percpu *rtpcp; in rcu_read_unlock_trace_special() local
1581 rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, t->trc_blkd_cpu); in rcu_read_unlock_trace_special()
1582 raw_spin_lock_irqsave_rcu_node(rtpcp, flags); in rcu_read_unlock_trace_special()
1585 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in rcu_read_unlock_trace_special()
1595 struct rcu_tasks_percpu *rtpcp; in rcu_tasks_trace_qs_blkd() local
1598 rtpcp = this_cpu_ptr(rcu_tasks_trace.rtpcpu); in rcu_tasks_trace_qs_blkd()
1599 raw_spin_lock_rcu_node(rtpcp); // irqs already disabled in rcu_tasks_trace_qs_blkd()
1601 if (!rtpcp->rtp_blkd_tasks.next) in rcu_tasks_trace_qs_blkd()
1602 INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); in rcu_tasks_trace_qs_blkd()
1603 list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks); in rcu_tasks_trace_qs_blkd()
1605 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in rcu_tasks_trace_qs_blkd()
1800 struct rcu_tasks_percpu *rtpcp; in rcu_tasks_trace_pregp_step() local
1837 rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, cpu); in rcu_tasks_trace_pregp_step()
1838 raw_spin_lock_irqsave_rcu_node(rtpcp, flags); in rcu_tasks_trace_pregp_step()
1839 list_splice_init(&rtpcp->rtp_blkd_tasks, &blkd_tasks); in rcu_tasks_trace_pregp_step()
1844 list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks); in rcu_tasks_trace_pregp_step()
1845 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in rcu_tasks_trace_pregp_step()
1848 raw_spin_lock_irqsave_rcu_node(rtpcp, flags); in rcu_tasks_trace_pregp_step()
1850 raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); in rcu_tasks_trace_pregp_step()