Lines Matching refs:cc
179 #define cc_exec_curr(cc, dir) cc->cc_exec_entity[dir].cc_curr argument
180 #define cc_exec_last_func(cc, dir) cc->cc_exec_entity[dir].cc_last_func argument
181 #define cc_exec_last_arg(cc, dir) cc->cc_exec_entity[dir].cc_last_arg argument
182 #define cc_exec_drain(cc, dir) cc->cc_exec_entity[dir].cc_drain argument
183 #define cc_exec_next(cc) cc->cc_next argument
184 #define cc_exec_cancel(cc, dir) cc->cc_exec_entity[dir].cc_cancel argument
185 #define cc_exec_waiting(cc, dir) cc->cc_exec_entity[dir].cc_waiting argument
187 #define cc_migration_func(cc, dir) cc->cc_exec_entity[dir].ce_migration_func argument
188 #define cc_migration_arg(cc, dir) cc->cc_exec_entity[dir].ce_migration_arg argument
189 #define cc_migration_cpu(cc, dir) cc->cc_exec_entity[dir].ce_migration_cpu argument
190 #define cc_migration_time(cc, dir) cc->cc_exec_entity[dir].ce_migration_time argument
191 #define cc_migration_prec(cc, dir) cc->cc_exec_entity[dir].ce_migration_prec argument
202 #define CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock) argument
203 #define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock) argument
204 #define CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED) argument
208 static void callout_cpu_init(struct callout_cpu *cc, int cpu);
209 static void softclock_call_cc(struct callout *c, struct callout_cpu *cc,
238 cc_cce_cleanup(struct callout_cpu *cc, int direct) in cc_cce_cleanup() argument
241 cc_exec_curr(cc, direct) = NULL; in cc_cce_cleanup()
242 cc_exec_cancel(cc, direct) = false; in cc_cce_cleanup()
243 cc_exec_waiting(cc, direct) = false; in cc_cce_cleanup()
245 cc_migration_cpu(cc, direct) = CPUBLOCK; in cc_cce_cleanup()
246 cc_migration_time(cc, direct) = 0; in cc_cce_cleanup()
247 cc_migration_prec(cc, direct) = 0; in cc_cce_cleanup()
248 cc_migration_func(cc, direct) = NULL; in cc_cce_cleanup()
249 cc_migration_arg(cc, direct) = NULL; in cc_cce_cleanup()
257 cc_cce_migrating(struct callout_cpu *cc, int direct) in cc_cce_migrating() argument
261 return (cc_migration_cpu(cc, direct) != CPUBLOCK); in cc_cce_migrating()
274 struct callout_cpu *cc; in callout_callwheel_init() local
305 cc = CC_CPU(cpu); in callout_callwheel_init()
306 callout_cpu_init(cc, cpu); in callout_callwheel_init()
315 callout_cpu_init(struct callout_cpu *cc, int cpu) in callout_cpu_init() argument
319 mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE); in callout_cpu_init()
320 cc->cc_inited = 1; in callout_cpu_init()
321 cc->cc_callwheel = malloc_domainset(sizeof(struct callout_list) * in callout_cpu_init()
325 LIST_INIT(&cc->cc_callwheel[i]); in callout_cpu_init()
326 TAILQ_INIT(&cc->cc_expireq); in callout_cpu_init()
327 cc->cc_firstevent = SBT_MAX; in callout_cpu_init()
329 cc_cce_cleanup(cc, i); in callout_cpu_init()
331 snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name), in callout_cpu_init()
343 callout_cpu_switch(struct callout *c, struct callout_cpu *cc, int new_cpu) in callout_cpu_switch() argument
347 MPASS(c != NULL && cc != NULL); in callout_cpu_switch()
348 CC_LOCK_ASSERT(cc); in callout_cpu_switch()
357 CC_UNLOCK(cc); in callout_cpu_switch()
372 struct callout_cpu *cc; in start_softclock() local
379 cc = CC_CPU(cpu); in start_softclock()
382 if (swi_add(&ie, name, softclock, cc, SWI_CLOCK, in start_softclock()
383 INTR_MPSAFE, &cc->cc_cookie)) in start_softclock()
419 struct callout_cpu *cc; in callout_process() local
427 cc = CC_SELF(); in callout_process()
428 mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET); in callout_process()
431 firstb = callout_hash(cc->cc_lastscan); in callout_process()
432 cc->cc_lastscan = now; in callout_process()
461 sc = &cc->cc_callwheel[firstb & callwheelmask]; in callout_process()
474 cc_exec_next(cc) = in callout_process()
476 cc->cc_bucket = firstb & callwheelmask; in callout_process()
478 softclock_call_cc(tmp, cc, in callout_process()
483 tmp = cc_exec_next(cc); in callout_process()
484 cc_exec_next(cc) = NULL; in callout_process()
488 TAILQ_INSERT_TAIL(&cc->cc_expireq, in callout_process()
523 cc->cc_firstevent = last; in callout_process()
531 mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET); in callout_process()
536 if (!TAILQ_EMPTY(&cc->cc_expireq)) in callout_process()
537 swi_sched(cc->cc_cookie, 0); in callout_process()
543 struct callout_cpu *cc; in callout_lock() local
555 cc = CC_CPU(cpu); in callout_lock()
556 CC_LOCK(cc); in callout_lock()
559 CC_UNLOCK(cc); in callout_lock()
561 return (cc); in callout_lock()
565 callout_cc_add(struct callout *c, struct callout_cpu *cc, in callout_cc_add() argument
571 CC_LOCK_ASSERT(cc); in callout_cc_add()
572 if (sbt < cc->cc_lastscan) in callout_cc_add()
573 sbt = cc->cc_lastscan; in callout_cc_add()
587 LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le); in callout_cc_add()
588 if (cc->cc_bucket == bucket) in callout_cc_add()
589 cc_exec_next(cc) = c; in callout_cc_add()
598 if (sbt < cc->cc_firstevent) { in callout_cc_add()
599 cc->cc_firstevent = sbt; in callout_cc_add()
605 softclock_call_cc(struct callout *c, struct callout_cpu *cc, in softclock_call_cc() argument
650 cc_exec_curr(cc, direct) = c; in softclock_call_cc()
651 cc_exec_last_func(cc, direct) = c_func; in softclock_call_cc()
652 cc_exec_last_arg(cc, direct) = c_arg; in softclock_call_cc()
653 cc_exec_cancel(cc, direct) = false; in softclock_call_cc()
654 cc_exec_drain(cc, direct) = NULL; in softclock_call_cc()
655 CC_UNLOCK(cc); in softclock_call_cc()
662 if (cc_exec_cancel(cc, direct)) { in softclock_call_cc()
667 cc_exec_cancel(cc, direct) = true; in softclock_call_cc()
688 KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running", in softclock_call_cc()
712 KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle"); in softclock_call_cc()
717 CC_LOCK(cc); in softclock_call_cc()
718 KASSERT(cc_exec_curr(cc, direct) == c, ("mishandled cc_curr")); in softclock_call_cc()
719 cc_exec_curr(cc, direct) = NULL; in softclock_call_cc()
720 if (cc_exec_drain(cc, direct)) { in softclock_call_cc()
721 drain = cc_exec_drain(cc, direct); in softclock_call_cc()
722 cc_exec_drain(cc, direct) = NULL; in softclock_call_cc()
723 CC_UNLOCK(cc); in softclock_call_cc()
725 CC_LOCK(cc); in softclock_call_cc()
727 if (cc_exec_waiting(cc, direct)) { in softclock_call_cc()
734 if (cc_cce_migrating(cc, direct)) { in softclock_call_cc()
735 cc_cce_cleanup(cc, direct); in softclock_call_cc()
743 cc_exec_waiting(cc, direct) = false; in softclock_call_cc()
744 CC_UNLOCK(cc); in softclock_call_cc()
745 wakeup(&cc_exec_waiting(cc, direct)); in softclock_call_cc()
746 CC_LOCK(cc); in softclock_call_cc()
747 } else if (cc_cce_migrating(cc, direct)) { in softclock_call_cc()
753 new_cpu = cc_migration_cpu(cc, direct); in softclock_call_cc()
754 new_time = cc_migration_time(cc, direct); in softclock_call_cc()
755 new_prec = cc_migration_prec(cc, direct); in softclock_call_cc()
756 new_func = cc_migration_func(cc, direct); in softclock_call_cc()
757 new_arg = cc_migration_arg(cc, direct); in softclock_call_cc()
758 cc_cce_cleanup(cc, direct); in softclock_call_cc()
774 new_cc = callout_cpu_switch(c, cc, new_cpu); in softclock_call_cc()
779 CC_LOCK(cc); in softclock_call_cc()
805 struct callout_cpu *cc; in softclock() local
811 cc = (struct callout_cpu *)arg; in softclock()
812 CC_LOCK(cc); in softclock()
813 while ((c = TAILQ_FIRST(&cc->cc_expireq)) != NULL) { in softclock()
814 TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); in softclock()
815 softclock_call_cc(c, cc, in softclock()
830 CC_UNLOCK(cc); in softclock()
897 struct callout_cpu *cc; in callout_reset_sbt_on() local
924 cc = callout_lock(c); in callout_reset_sbt_on()
932 if (cc_exec_curr(cc, direct) == c) { in callout_reset_sbt_on()
938 if (c->c_lock != NULL && !cc_exec_cancel(cc, direct)) in callout_reset_sbt_on()
939 cancelled = cc_exec_cancel(cc, direct) = true; in callout_reset_sbt_on()
940 if (cc_exec_waiting(cc, direct) || cc_exec_drain(cc, direct)) { in callout_reset_sbt_on()
948 CC_UNLOCK(cc); in callout_reset_sbt_on()
960 cc_migration_cpu(cc, direct) = cpu; in callout_reset_sbt_on()
961 cc_migration_time(cc, direct) = to_sbt; in callout_reset_sbt_on()
962 cc_migration_prec(cc, direct) = precision; in callout_reset_sbt_on()
963 cc_migration_func(cc, direct) = ftn; in callout_reset_sbt_on()
964 cc_migration_arg(cc, direct) = arg; in callout_reset_sbt_on()
966 CC_UNLOCK(cc); in callout_reset_sbt_on()
973 if (cc_exec_next(cc) == c) in callout_reset_sbt_on()
974 cc_exec_next(cc) = LIST_NEXT(c, c_links.le); in callout_reset_sbt_on()
977 TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); in callout_reset_sbt_on()
991 if (cc_exec_curr(cc, direct) == c) { in callout_reset_sbt_on()
1008 cc_migration_cpu(cc, direct) = cpu; in callout_reset_sbt_on()
1009 cc_migration_time(cc, direct) = to_sbt; in callout_reset_sbt_on()
1010 cc_migration_prec(cc, direct) = precision; in callout_reset_sbt_on()
1011 cc_migration_func(cc, direct) = ftn; in callout_reset_sbt_on()
1012 cc_migration_arg(cc, direct) = arg; in callout_reset_sbt_on()
1019 CC_UNLOCK(cc); in callout_reset_sbt_on()
1022 cc = callout_cpu_switch(c, cc, cpu); in callout_reset_sbt_on()
1026 callout_cc_add(c, cc, to_sbt, precision, ftn, arg, cpu, flags); in callout_reset_sbt_on()
1030 CC_UNLOCK(cc); in callout_reset_sbt_on()
1053 struct callout_cpu *cc, *old_cc; in _callout_stop_safe() local
1087 cc = callout_lock(c); in _callout_stop_safe()
1116 if (sq_locked != 0 && cc != old_cc) { in _callout_stop_safe()
1118 CC_UNLOCK(cc); in _callout_stop_safe()
1131 if (cc_exec_curr(cc, direct) == c) { in _callout_stop_safe()
1148 if (cc_exec_curr(cc, direct) == c) { in _callout_stop_safe()
1167 CC_UNLOCK(cc); in _callout_stop_safe()
1169 &cc_exec_waiting(cc, direct)); in _callout_stop_safe()
1171 old_cc = cc; in _callout_stop_safe()
1181 cc_exec_waiting(cc, direct) = true; in _callout_stop_safe()
1183 CC_UNLOCK(cc); in _callout_stop_safe()
1185 &cc_exec_waiting(cc, direct), in _callout_stop_safe()
1186 &cc->cc_lock.lock_object, "codrain", in _callout_stop_safe()
1189 &cc_exec_waiting(cc, direct), in _callout_stop_safe()
1200 !cc_exec_cancel(cc, direct) && (drain == NULL)) { in _callout_stop_safe()
1211 cc_exec_cancel(cc, direct) = true; in _callout_stop_safe()
1214 KASSERT(!cc_cce_migrating(cc, direct), in _callout_stop_safe()
1219 cc_migration_cpu(cc, direct) = CPUBLOCK; in _callout_stop_safe()
1220 cc_migration_time(cc, direct) = 0; in _callout_stop_safe()
1221 cc_migration_prec(cc, direct) = 0; in _callout_stop_safe()
1222 cc_migration_func(cc, direct) = NULL; in _callout_stop_safe()
1223 cc_migration_arg(cc, direct) = NULL; in _callout_stop_safe()
1226 CC_UNLOCK(cc); in _callout_stop_safe()
1247 cc_migration_cpu(cc, direct) = CPUBLOCK; in _callout_stop_safe()
1248 cc_migration_time(cc, direct) = 0; in _callout_stop_safe()
1249 cc_migration_prec(cc, direct) = 0; in _callout_stop_safe()
1250 cc_migration_func(cc, direct) = NULL; in _callout_stop_safe()
1251 cc_migration_arg(cc, direct) = NULL; in _callout_stop_safe()
1256 KASSERT(cc_exec_drain(cc, direct) == NULL, in _callout_stop_safe()
1258 cc_exec_drain(cc, direct))); in _callout_stop_safe()
1259 cc_exec_drain(cc, direct) = drain; in _callout_stop_safe()
1261 CC_UNLOCK(cc); in _callout_stop_safe()
1267 KASSERT(cc_exec_drain(cc, direct) == NULL, in _callout_stop_safe()
1269 cc_exec_drain(cc, direct))); in _callout_stop_safe()
1270 cc_exec_drain(cc, direct) = drain; in _callout_stop_safe()
1279 sleepq_release(&cc_exec_waiting(cc, direct)); in _callout_stop_safe()
1288 if (cc_exec_curr(cc, direct) != c) in _callout_stop_safe()
1290 CC_UNLOCK(cc); in _callout_stop_safe()
1301 if (cc_exec_next(cc) == c) in _callout_stop_safe()
1302 cc_exec_next(cc) = LIST_NEXT(c, c_links.le); in _callout_stop_safe()
1305 TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); in _callout_stop_safe()
1308 CC_UNLOCK(cc); in _callout_stop_safe()
1360 struct callout_cpu *cc; in sysctl_kern_callout_stat() local
1378 cc = CC_CPU(cpu); in sysctl_kern_callout_stat()
1379 CC_LOCK(cc); in sysctl_kern_callout_stat()
1381 sc = &cc->cc_callwheel[i]; in sysctl_kern_callout_stat()
1402 CC_UNLOCK(cc); in sysctl_kern_callout_stat()
1484 struct callout_cpu *cc; in _show_last_callout() local
1487 cc = CC_CPU(cpu); in _show_last_callout()
1488 func = cc_exec_last_func(cc, direct); in _show_last_callout()
1489 arg = cc_exec_last_arg(cc, direct); in _show_last_callout()