182 #define	cc_exec_curr(cc, dir)		cc->cc_exec_entity[dir].cc_curr  argument
183 #define	cc_exec_last_func(cc, dir)	cc->cc_exec_entity[dir].cc_last_func  argument
184 #define	cc_exec_last_arg(cc, dir)	cc->cc_exec_entity[dir].cc_last_arg  argument
185 #define	cc_exec_drain(cc, dir)		cc->cc_exec_entity[dir].cc_drain  argument
186 #define	cc_exec_next(cc)		cc->cc_next  argument
187 #define	cc_exec_cancel(cc, dir)		cc->cc_exec_entity[dir].cc_cancel  argument
188 #define	cc_exec_waiting(cc, dir)	cc->cc_exec_entity[dir].cc_waiting  argument
190 #define	cc_migration_func(cc, dir)	cc->cc_exec_entity[dir].ce_migration_func  argument
191 #define	cc_migration_arg(cc, dir)	cc->cc_exec_entity[dir].ce_migration_arg  argument
192 #define	cc_migration_cpu(cc, dir)	cc->cc_exec_entity[dir].ce_migration_cpu  argument
193 #define	cc_migration_time(cc, dir)	cc->cc_exec_entity[dir].ce_migration_time  argument
194 #define	cc_migration_prec(cc, dir)	cc->cc_exec_entity[dir].ce_migration_prec  argument
205 #define	CC_LOCK(cc)	mtx_lock_spin(&(cc)->cc_lock)  argument
206 #define	CC_UNLOCK(cc)	mtx_unlock_spin(&(cc)->cc_lock)  argument
207 #define	CC_LOCK_ASSERT(cc)	mtx_assert(&(cc)->cc_lock, MA_OWNED)  argument
211 static void	callout_cpu_init(struct callout_cpu *cc, int cpu);
212 static void	softclock_call_cc(struct callout *c, struct callout_cpu *cc,
241 cc_cce_cleanup(struct callout_cpu *cc, int direct)  in cc_cce_cleanup()  argument
244 	cc_exec_curr(cc, direct) = NULL;  in cc_cce_cleanup()
245 	cc_exec_cancel(cc, direct) = false;  in cc_cce_cleanup()
246 	cc_exec_waiting(cc, direct) = false;  in cc_cce_cleanup()
248 	cc_migration_cpu(cc, direct) = CPUBLOCK;  in cc_cce_cleanup()
249 	cc_migration_time(cc, direct) = 0;  in cc_cce_cleanup()
250 	cc_migration_prec(cc, direct) = 0;  in cc_cce_cleanup()
251 	cc_migration_func(cc, direct) = NULL;  in cc_cce_cleanup()
252 	cc_migration_arg(cc, direct) = NULL;  in cc_cce_cleanup()
260 cc_cce_migrating(struct callout_cpu *cc, int direct)  in cc_cce_migrating()  argument
264 	return (cc_migration_cpu(cc, direct) != CPUBLOCK);  in cc_cce_migrating()
277 	struct callout_cpu *cc;  in callout_callwheel_init()  local
308 		cc = CC_CPU(cpu);  in callout_callwheel_init()
309 		callout_cpu_init(cc, cpu);  in callout_callwheel_init()
318 callout_cpu_init(struct callout_cpu *cc, int cpu)  in callout_cpu_init()  argument
322 	mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN);  in callout_cpu_init()
323 	cc->cc_callwheel = malloc_domainset(sizeof(struct callout_list) *  in callout_cpu_init()
327 		LIST_INIT(&cc->cc_callwheel[i]);  in callout_cpu_init()
328 	TAILQ_INIT(&cc->cc_expireq);  in callout_cpu_init()
329 	cc->cc_firstevent = SBT_MAX;  in callout_cpu_init()
331 		cc_cce_cleanup(cc, i);  in callout_cpu_init()
333 	snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name),  in callout_cpu_init()
345 callout_cpu_switch(struct callout *c, struct callout_cpu *cc, int new_cpu)  in callout_cpu_switch()  argument
349 	MPASS(c != NULL && cc != NULL);  in callout_cpu_switch()
350 	CC_LOCK_ASSERT(cc);  in callout_cpu_switch()
359 	CC_UNLOCK(cc);  in callout_cpu_switch()
376 	struct callout_cpu *cc;  in start_softclock()  local
382 		cc = CC_CPU(cpu);  in start_softclock()
383 		error = kproc_kthread_add(softclock_thread, cc, &p, &td,  in start_softclock()
388 		CC_LOCK(cc);  in start_softclock()
389 		cc->cc_thread = td;  in start_softclock()
394 		thread_lock_set(td, (struct mtx *)&cc->cc_lock);  in start_softclock()
431 		struct callout_cpu *cc;  in callout_process()  member
436 	struct callout_cpu *cc;  in callout_process()  local
445 	cc = CC_SELF();  in callout_process()
446 	mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET);  in callout_process()
449 	firstb = callout_hash(cc->cc_lastscan);  in callout_process()
450 	cc->cc_lastscan = now;  in callout_process()
479 		sc = &cc->cc_callwheel[firstb & callwheelmask];  in callout_process()
491 					cc_exec_next(cc) = next;  in callout_process()
492 					cc->cc_bucket = firstb & callwheelmask;  in callout_process()
494 					softclock_call_cc(c, cc,  in callout_process()
499 					next = cc_exec_next(cc);  in callout_process()
500 					cc_exec_next(cc) = NULL;  in callout_process()
503 					TAILQ_INSERT_TAIL(&cc->cc_expireq,  in callout_process()
538 	cc->cc_firstevent = last;  in callout_process()
546 	if (!TAILQ_EMPTY(&cc->cc_expireq)) {  in callout_process()
547 		entropy.cc = cc;  in callout_process()
552 		td = cc->cc_thread;  in callout_process()
559 			mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);  in callout_process()
561 		mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);  in callout_process()
567 	struct callout_cpu *cc;  in callout_lock()  local
579 		cc = CC_CPU(cpu);  in callout_lock()
580 		CC_LOCK(cc);  in callout_lock()
583 		CC_UNLOCK(cc);  in callout_lock()
585 	return (cc);  in callout_lock()
589 callout_cc_add(struct callout *c, struct callout_cpu *cc,  in callout_cc_add()  argument
595 	CC_LOCK_ASSERT(cc);  in callout_cc_add()
596 	if (sbt < cc->cc_lastscan)  in callout_cc_add()
597 		sbt = cc->cc_lastscan;  in callout_cc_add()
611 	LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le);  in callout_cc_add()
612 	if (cc->cc_bucket == bucket)  in callout_cc_add()
613 		cc_exec_next(cc) = c;  in callout_cc_add()
622 	if (sbt < cc->cc_firstevent) {  in callout_cc_add()
623 		cc->cc_firstevent = sbt;  in callout_cc_add()
629 softclock_call_cc(struct callout *c, struct callout_cpu *cc,  in softclock_call_cc()  argument
674 	cc_exec_curr(cc, direct) = c;  in softclock_call_cc()
675 	cc_exec_last_func(cc, direct) = c_func;  in softclock_call_cc()
676 	cc_exec_last_arg(cc, direct) = c_arg;  in softclock_call_cc()
677 	cc_exec_cancel(cc, direct) = false;  in softclock_call_cc()
678 	cc_exec_drain(cc, direct) = NULL;  in softclock_call_cc()
679 	CC_UNLOCK(cc);  in softclock_call_cc()
686 		if (cc_exec_cancel(cc, direct)) {  in softclock_call_cc()
691 		cc_exec_cancel(cc, direct) = true;  in softclock_call_cc()
712 	KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running",  in softclock_call_cc()
736 	KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle");  in softclock_call_cc()
741 	CC_LOCK(cc);  in softclock_call_cc()
742 	KASSERT(cc_exec_curr(cc, direct) == c, ("mishandled cc_curr"));  in softclock_call_cc()
743 	cc_exec_curr(cc, direct) = NULL;  in softclock_call_cc()
744 	if (cc_exec_drain(cc, direct)) {  in softclock_call_cc()
745 		drain = cc_exec_drain(cc, direct);  in softclock_call_cc()
746 		cc_exec_drain(cc, direct) = NULL;  in softclock_call_cc()
747 		CC_UNLOCK(cc);  in softclock_call_cc()
749 		CC_LOCK(cc);  in softclock_call_cc()
751 	if (cc_exec_waiting(cc, direct)) {  in softclock_call_cc()
758 		if (cc_cce_migrating(cc, direct)) {  in softclock_call_cc()
759 			cc_cce_cleanup(cc, direct);  in softclock_call_cc()
767 		cc_exec_waiting(cc, direct) = false;  in softclock_call_cc()
768 		CC_UNLOCK(cc);  in softclock_call_cc()
769 		wakeup(&cc_exec_waiting(cc, direct));  in softclock_call_cc()
770 		CC_LOCK(cc);  in softclock_call_cc()
771 	} else if (cc_cce_migrating(cc, direct)) {  in softclock_call_cc()
777 		new_cpu = cc_migration_cpu(cc, direct);  in softclock_call_cc()
778 		new_time = cc_migration_time(cc, direct);  in softclock_call_cc()
779 		new_prec = cc_migration_prec(cc, direct);  in softclock_call_cc()
780 		new_func = cc_migration_func(cc, direct);  in softclock_call_cc()
781 		new_arg = cc_migration_arg(cc, direct);  in softclock_call_cc()
782 		cc_cce_cleanup(cc, direct);  in softclock_call_cc()
798 		new_cc = callout_cpu_switch(c, cc, new_cpu);  in softclock_call_cc()
803 		CC_LOCK(cc);  in softclock_call_cc()
830 	struct callout_cpu *cc;  in softclock_thread()  local
836 	cc = (struct callout_cpu *)arg;  in softclock_thread()
837 	CC_LOCK(cc);  in softclock_thread()
839 		while (TAILQ_EMPTY(&cc->cc_expireq)) {  in softclock_thread()
845 			thread_lock_set(td, (struct mtx *)&cc->cc_lock);  in softclock_thread()
850 			CC_LOCK(cc);  in softclock_thread()
856 		while ((c = TAILQ_FIRST(&cc->cc_expireq)) != NULL) {  in softclock_thread()
857 			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);  in softclock_thread()
858 			softclock_call_cc(c, cc,  in softclock_thread()
940 	struct callout_cpu *cc;  in callout_reset_sbt_on()  local
960 	cc = callout_lock(c);  in callout_reset_sbt_on()
966 	if (cc_exec_curr(cc, direct) == c) {  in callout_reset_sbt_on()
972 		if (c->c_lock != NULL && !cc_exec_cancel(cc, direct))  in callout_reset_sbt_on()
973 			cancelled = cc_exec_cancel(cc, direct) = true;  in callout_reset_sbt_on()
974 		if (cc_exec_waiting(cc, direct) || cc_exec_drain(cc, direct)) {  in callout_reset_sbt_on()
982 			CC_UNLOCK(cc);  in callout_reset_sbt_on()
994 			cc_migration_cpu(cc, direct) = cpu;  in callout_reset_sbt_on()
995 			cc_migration_time(cc, direct) = to_sbt;  in callout_reset_sbt_on()
996 			cc_migration_prec(cc, direct) = precision;  in callout_reset_sbt_on()
997 			cc_migration_func(cc, direct) = ftn;  in callout_reset_sbt_on()
998 			cc_migration_arg(cc, direct) = arg;  in callout_reset_sbt_on()
1000 			CC_UNLOCK(cc);  in callout_reset_sbt_on()
1007 			if (cc_exec_next(cc) == c)  in callout_reset_sbt_on()
1008 				cc_exec_next(cc) = LIST_NEXT(c, c_links.le);  in callout_reset_sbt_on()
1011 			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);  in callout_reset_sbt_on()
1025 		if (cc_exec_curr(cc, direct) == c) {  in callout_reset_sbt_on()
1042 			cc_migration_cpu(cc, direct) = cpu;  in callout_reset_sbt_on()
1043 			cc_migration_time(cc, direct) = to_sbt;  in callout_reset_sbt_on()
1044 			cc_migration_prec(cc, direct) = precision;  in callout_reset_sbt_on()
1045 			cc_migration_func(cc, direct) = ftn;  in callout_reset_sbt_on()
1046 			cc_migration_arg(cc, direct) = arg;  in callout_reset_sbt_on()
1053 			CC_UNLOCK(cc);  in callout_reset_sbt_on()
1056 		cc = callout_cpu_switch(c, cc, cpu);  in callout_reset_sbt_on()
1060 	callout_cc_add(c, cc, to_sbt, precision, ftn, arg, flags);  in callout_reset_sbt_on()
1064 	CC_UNLOCK(cc);  in callout_reset_sbt_on()
1087 	struct callout_cpu *cc, *old_cc;  in _callout_stop_safe()  local
1121 	cc = callout_lock(c);  in _callout_stop_safe()
1150 	if (sq_locked != 0 && cc != old_cc) {  in _callout_stop_safe()
1152 		CC_UNLOCK(cc);  in _callout_stop_safe()
1165 	if (cc_exec_curr(cc, direct) == c) {  in _callout_stop_safe()
1182 			if (cc_exec_curr(cc, direct) == c) {  in _callout_stop_safe()
1201 					CC_UNLOCK(cc);  in _callout_stop_safe()
1203 					    &cc_exec_waiting(cc, direct));  in _callout_stop_safe()
1205 					old_cc = cc;  in _callout_stop_safe()
1215 				cc_exec_waiting(cc, direct) = true;  in _callout_stop_safe()
1217 				CC_UNLOCK(cc);  in _callout_stop_safe()
1219 				    &cc_exec_waiting(cc, direct),  in _callout_stop_safe()
1220 				    &cc->cc_lock.lock_object, "codrain",  in _callout_stop_safe()
1223 				    &cc_exec_waiting(cc, direct),  in _callout_stop_safe()
1234 			   !cc_exec_cancel(cc, direct) && (drain == NULL)) {  in _callout_stop_safe()
1245 			cc_exec_cancel(cc, direct) = true;  in _callout_stop_safe()
1248 			KASSERT(!cc_cce_migrating(cc, direct),  in _callout_stop_safe()
1253 				cc_migration_cpu(cc, direct) = CPUBLOCK;  in _callout_stop_safe()
1254 				cc_migration_time(cc, direct) = 0;  in _callout_stop_safe()
1255 				cc_migration_prec(cc, direct) = 0;  in _callout_stop_safe()
1256 				cc_migration_func(cc, direct) = NULL;  in _callout_stop_safe()
1257 				cc_migration_arg(cc, direct) = NULL;  in _callout_stop_safe()
1260 			CC_UNLOCK(cc);  in _callout_stop_safe()
1281 			cc_migration_cpu(cc, direct) = CPUBLOCK;  in _callout_stop_safe()
1282 			cc_migration_time(cc, direct) = 0;  in _callout_stop_safe()
1283 			cc_migration_prec(cc, direct) = 0;  in _callout_stop_safe()
1284 			cc_migration_func(cc, direct) = NULL;  in _callout_stop_safe()
1285 			cc_migration_arg(cc, direct) = NULL;  in _callout_stop_safe()
1290 				KASSERT(cc_exec_drain(cc, direct) == NULL,  in _callout_stop_safe()
1292 				    cc_exec_drain(cc, direct)));  in _callout_stop_safe()
1293 				cc_exec_drain(cc, direct) = drain;  in _callout_stop_safe()
1295 			CC_UNLOCK(cc);  in _callout_stop_safe()
1301 				KASSERT(cc_exec_drain(cc, direct) == NULL,  in _callout_stop_safe()
1303 				    cc_exec_drain(cc, direct)));  in _callout_stop_safe()
1304 				cc_exec_drain(cc, direct) = drain;  in _callout_stop_safe()
1313 		sleepq_release(&cc_exec_waiting(cc, direct));  in _callout_stop_safe()
1322 		if (cc_exec_curr(cc, direct) != c)  in _callout_stop_safe()
1324 		CC_UNLOCK(cc);  in _callout_stop_safe()
1335 			if (cc_exec_next(cc) == c)  in _callout_stop_safe()
1336 				cc_exec_next(cc) = LIST_NEXT(c, c_links.le);  in _callout_stop_safe()
1339 			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);  in _callout_stop_safe()
1342 	CC_UNLOCK(cc);  in _callout_stop_safe()
1394 	struct callout_cpu *cc;  in sysctl_kern_callout_stat()  local
1412 		cc = CC_CPU(cpu);  in sysctl_kern_callout_stat()
1413 		CC_LOCK(cc);  in sysctl_kern_callout_stat()
1415 			sc = &cc->cc_callwheel[i];  in sysctl_kern_callout_stat()
1436 		CC_UNLOCK(cc);  in sysctl_kern_callout_stat()
1518 	struct callout_cpu *cc;  in _show_last_callout()  local
1521 	cc = CC_CPU(cpu);  in _show_last_callout()
1522 	func = cc_exec_last_func(cc, direct);  in _show_last_callout()
1523 	arg = cc_exec_last_arg(cc, direct);  in _show_last_callout()