Lines Matching refs:cc
160 #define cc_exec_curr(cc, dir) cc->cc_exec_entity[dir].cc_curr argument
161 #define cc_exec_drain(cc, dir) cc->cc_exec_entity[dir].cc_drain argument
162 #define cc_exec_next(cc) cc->cc_next argument
163 #define cc_exec_cancel(cc, dir) cc->cc_exec_entity[dir].cc_cancel argument
164 #define cc_exec_waiting(cc, dir) cc->cc_exec_entity[dir].cc_waiting argument
168 #define CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock) argument
169 #define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock) argument
170 #define CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED) argument
174 static void callout_cpu_init(struct callout_cpu *cc, int cpu);
175 static void softclock_call_cc(struct callout *c, struct callout_cpu *cc,
204 cc_cce_cleanup(struct callout_cpu *cc, int direct) in cc_cce_cleanup() argument
206 cc_exec_curr(cc, direct) = NULL; in cc_cce_cleanup()
207 cc_exec_cancel(cc, direct) = false; in cc_cce_cleanup()
208 cc_exec_waiting(cc, direct) = false; in cc_cce_cleanup()
215 cc_cce_migrating(struct callout_cpu *cc, int direct) in cc_cce_migrating() argument
227 struct callout_cpu *cc; in callout_callwheel_init() local
259 cc = CC_CPU(timeout_cpu); in callout_callwheel_init()
260 cc->cc_callout = malloc(ncallout * sizeof(struct callout), in callout_callwheel_init()
262 callout_cpu_init(cc, timeout_cpu); in callout_callwheel_init()
270 callout_cpu_init(struct callout_cpu *cc, int cpu) in callout_cpu_init() argument
275 mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE); in callout_cpu_init()
276 SLIST_INIT(&cc->cc_callfree); in callout_cpu_init()
277 cc->cc_inited = 1; in callout_cpu_init()
278 cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize, in callout_cpu_init()
281 LIST_INIT(&cc->cc_callwheel[i]); in callout_cpu_init()
282 TAILQ_INIT(&cc->cc_expireq); in callout_cpu_init()
284 cc_cce_cleanup(cc, i); in callout_cpu_init()
285 snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name), in callout_cpu_init()
287 if (cc->cc_callout == NULL) /* Only cpu0 handles timeout(9) */ in callout_cpu_init()
290 c = &cc->cc_callout[i]; in callout_cpu_init()
293 SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle); in callout_cpu_init()
306 struct callout_cpu *cc; in callout_tick() local
315 cc = CC_SELF(); in callout_tick()
316 mtx_lock(&cc->cc_lock); in callout_tick()
317 for (; (cc->cc_softticks - ticks) < 0; cc->cc_softticks++) { in callout_tick()
318 bucket = cc->cc_softticks & callwheelmask; in callout_tick()
319 if (!LIST_EMPTY(&cc->cc_callwheel[bucket])) { in callout_tick()
324 mtx_unlock(&cc->cc_lock); in callout_tick()
330 softclock(cc); in callout_tick()
336 struct callout_cpu *cc; in callout_lock() local
341 cc = CC_CPU(cpu); in callout_lock()
342 CC_LOCK(cc); in callout_lock()
345 CC_UNLOCK(cc); in callout_lock()
347 return (cc); in callout_lock()
351 callout_cc_add(struct callout *c, struct callout_cpu *cc, in callout_cc_add() argument
356 CC_LOCK_ASSERT(cc); in callout_cc_add()
367 LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le); in callout_cc_add()
368 if (cc->cc_bucket == bucket) in callout_cc_add()
369 cc_exec_next(cc) = c; in callout_cc_add()
373 callout_cc_del(struct callout *c, struct callout_cpu *cc) in callout_cc_del() argument
379 SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle); in callout_cc_del()
383 softclock_call_cc(struct callout *c, struct callout_cpu *cc, in softclock_call_cc() argument
424 cc_exec_curr(cc, direct) = c; in softclock_call_cc()
425 cc_exec_cancel(cc, direct) = false; in softclock_call_cc()
426 cc_exec_drain(cc, direct) = NULL; in softclock_call_cc()
427 CC_UNLOCK(cc); in softclock_call_cc()
434 if (cc_exec_cancel(cc, direct)) { in softclock_call_cc()
439 cc_exec_cancel(cc, direct) = true; in softclock_call_cc()
460 KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running", in softclock_call_cc()
484 KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle"); in softclock_call_cc()
489 CC_LOCK(cc); in softclock_call_cc()
490 KASSERT(cc_exec_curr(cc, direct) == c, ("mishandled cc_curr")); in softclock_call_cc()
491 cc_exec_curr(cc, direct) = NULL; in softclock_call_cc()
492 if (cc_exec_drain(cc, direct)) { in softclock_call_cc()
495 drain = cc_exec_drain(cc, direct); in softclock_call_cc()
496 cc_exec_drain(cc, direct) = NULL; in softclock_call_cc()
497 CC_UNLOCK(cc); in softclock_call_cc()
499 CC_LOCK(cc); in softclock_call_cc()
501 if (cc_exec_waiting(cc, direct)) { in softclock_call_cc()
508 if (cc_cce_migrating(cc, direct)) { in softclock_call_cc()
509 cc_cce_cleanup(cc, direct); in softclock_call_cc()
517 cc_exec_waiting(cc, direct) = false; in softclock_call_cc()
518 CC_UNLOCK(cc); in softclock_call_cc()
519 wakeup(&cc_exec_waiting(cc, direct)); in softclock_call_cc()
520 CC_LOCK(cc); in softclock_call_cc()
521 } else if (cc_cce_migrating(cc, direct)) { in softclock_call_cc()
538 callout_cc_del(c, cc); in softclock_call_cc()
561 struct callout_cpu *cc; in softclock() local
568 cc = (struct callout_cpu *)arg; in softclock()
569 CC_LOCK(cc); in softclock()
571 while (cc->cc_softticks != ticks) { in softclock()
576 curticks = cc->cc_softticks; in softclock()
577 cc->cc_softticks++; in softclock()
578 sc = &cc->cc_callwheel[curticks & callwheelmask]; in softclock()
587 cc_exec_next(cc) = in softclock()
589 cc->cc_bucket = callout_get_bucket(curticks); in softclock()
591 softclock_call_cc(c, cc, in softclock()
596 c = cc_exec_next(cc); in softclock()
597 cc_exec_next(cc) = NULL; in softclock()
608 CC_UNLOCK(cc); in softclock()
631 struct callout_cpu *cc;
635 cc = CC_CPU(timeout_cpu);
636 CC_LOCK(cc);
638 new = SLIST_FIRST(&cc->cc_callfree);
642 SLIST_REMOVE_HEAD(&cc->cc_callfree, c_links.sle);
645 CC_UNLOCK(cc);
653 struct callout_cpu *cc;
663 cc = callout_lock(handle.callout);
666 CC_UNLOCK(cc);
696 struct callout_cpu *cc; in callout_reset_tick_on() local
721 cc = callout_lock(c); in callout_reset_tick_on()
732 if (cc_exec_curr(cc, direct) == c) { in callout_reset_tick_on()
738 if (c->c_lock != NULL && !cc_exec_cancel(cc, direct)) in callout_reset_tick_on()
739 cancelled = cc_exec_cancel(cc, direct) = true; in callout_reset_tick_on()
740 if (cc_exec_waiting(cc, direct)) { in callout_reset_tick_on()
748 CC_UNLOCK(cc); in callout_reset_tick_on()
754 if (cc_exec_next(cc) == c) in callout_reset_tick_on()
755 cc_exec_next(cc) = LIST_NEXT(c, c_links.le); in callout_reset_tick_on()
758 TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); in callout_reset_tick_on()
768 callout_cc_add(c, cc, to_ticks, ftn, arg, cpu, flags); in callout_reset_tick_on()
771 CC_UNLOCK(cc); in callout_reset_tick_on()
794 struct callout_cpu *cc, *old_cc; in _callout_stop_safe() local
825 cc = callout_lock(c); in _callout_stop_safe()
854 if (sq_locked != 0 && cc != old_cc) { in _callout_stop_safe()
861 if (cc_exec_curr(cc, direct) == c) { in _callout_stop_safe()
875 while (cc_exec_curr(cc, direct) == c) { in _callout_stop_safe()
894 CC_UNLOCK(cc); in _callout_stop_safe()
896 &cc_exec_waiting(cc, direct)); in _callout_stop_safe()
898 old_cc = cc; in _callout_stop_safe()
908 cc_exec_waiting(cc, direct) = true; in _callout_stop_safe()
910 CC_UNLOCK(cc); in _callout_stop_safe()
912 &cc_exec_waiting(cc, direct), in _callout_stop_safe()
913 &cc->cc_lock.lock_object, "codrain", in _callout_stop_safe()
916 &cc_exec_waiting(cc, direct), in _callout_stop_safe()
923 CC_LOCK(cc); in _callout_stop_safe()
926 !cc_exec_cancel(cc, direct) && (drain == NULL)) { in _callout_stop_safe()
937 cc_exec_cancel(cc, direct) = true; in _callout_stop_safe()
940 KASSERT(!cc_cce_migrating(cc, direct), in _callout_stop_safe()
945 CC_UNLOCK(cc); in _callout_stop_safe()
961 cc_exec_drain(cc, direct) = drain; in _callout_stop_safe()
963 CC_UNLOCK(cc); in _callout_stop_safe()
969 cc_exec_drain(cc, direct) = drain; in _callout_stop_safe()
977 sleepq_release(&cc_exec_waiting(cc, direct)); in _callout_stop_safe()
986 if (cc_exec_curr(cc, direct) != c) in _callout_stop_safe()
988 CC_UNLOCK(cc); in _callout_stop_safe()
999 if (cc_exec_next(cc) == c) in _callout_stop_safe()
1000 cc_exec_next(cc) = LIST_NEXT(c, c_links.le); in _callout_stop_safe()
1003 TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); in _callout_stop_safe()
1006 callout_cc_del(c, cc); in _callout_stop_safe()
1007 CC_UNLOCK(cc); in _callout_stop_safe()
1089 CC_LOCK(cc); in adjust_timeout_calltodo()
1100 CC_UNLOCK(cc); in adjust_timeout_calltodo()
1125 struct callout_cpu *cc; in sysctl_kern_callout_stat() local
1142 cc = CC_CPU(timeout_cpu); in sysctl_kern_callout_stat()
1143 CC_LOCK(cc); in sysctl_kern_callout_stat()
1145 sc = &cc->cc_callwheel[i]; in sysctl_kern_callout_stat()
1162 CC_UNLOCK(cc); in sysctl_kern_callout_stat()