179 #define	cc_exec_curr(cc, dir)		cc->cc_exec_entity[dir].cc_curr  argument
180 #define	cc_exec_drain(cc, dir)		cc->cc_exec_entity[dir].cc_drain  argument
181 #define	cc_exec_next(cc)		cc->cc_next  argument
182 #define	cc_exec_cancel(cc, dir)		cc->cc_exec_entity[dir].cc_cancel  argument
183 #define	cc_exec_waiting(cc, dir)	cc->cc_exec_entity[dir].cc_waiting  argument
185 #define	cc_migration_func(cc, dir)	cc->cc_exec_entity[dir].ce_migration_func  argument
186 #define	cc_migration_arg(cc, dir)	cc->cc_exec_entity[dir].ce_migration_arg  argument
187 #define	cc_migration_cpu(cc, dir)	cc->cc_exec_entity[dir].ce_migration_cpu  argument
188 #define	cc_migration_time(cc, dir)	cc->cc_exec_entity[dir].ce_migration_time  argument
189 #define	cc_migration_prec(cc, dir)	cc->cc_exec_entity[dir].ce_migration_prec  argument
200 #define	CC_LOCK(cc)	mtx_lock_spin(&(cc)->cc_lock)  argument
201 #define	CC_UNLOCK(cc)	mtx_unlock_spin(&(cc)->cc_lock)  argument
202 #define	CC_LOCK_ASSERT(cc)	mtx_assert(&(cc)->cc_lock, MA_OWNED)  argument
206 static void	callout_cpu_init(struct callout_cpu *cc, int cpu);
207 static void	softclock_call_cc(struct callout *c, struct callout_cpu *cc,
236 cc_cce_cleanup(struct callout_cpu *cc, int direct)  in cc_cce_cleanup()  argument
239 	cc_exec_curr(cc, direct) = NULL;  in cc_cce_cleanup()
240 	cc_exec_cancel(cc, direct) = false;  in cc_cce_cleanup()
241 	cc_exec_waiting(cc, direct) = false;  in cc_cce_cleanup()
243 	cc_migration_cpu(cc, direct) = CPUBLOCK;  in cc_cce_cleanup()
244 	cc_migration_time(cc, direct) = 0;  in cc_cce_cleanup()
245 	cc_migration_prec(cc, direct) = 0;  in cc_cce_cleanup()
246 	cc_migration_func(cc, direct) = NULL;  in cc_cce_cleanup()
247 	cc_migration_arg(cc, direct) = NULL;  in cc_cce_cleanup()
255 cc_cce_migrating(struct callout_cpu *cc, int direct)  in cc_cce_migrating()  argument
259 	return (cc_migration_cpu(cc, direct) != CPUBLOCK);  in cc_cce_migrating()
272 	struct callout_cpu *cc;  in callout_callwheel_init()  local
304 	cc = CC_CPU(timeout_cpu);  in callout_callwheel_init()
305 	cc->cc_callout = malloc(ncallout * sizeof(struct callout),  in callout_callwheel_init()
307 	callout_cpu_init(cc, timeout_cpu);  in callout_callwheel_init()
315 callout_cpu_init(struct callout_cpu *cc, int cpu)  in callout_cpu_init()  argument
320 	mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE);  in callout_cpu_init()
321 	SLIST_INIT(&cc->cc_callfree);  in callout_cpu_init()
322 	cc->cc_inited = 1;  in callout_cpu_init()
323 	cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize,  in callout_cpu_init()
326 		LIST_INIT(&cc->cc_callwheel[i]);  in callout_cpu_init()
327 	TAILQ_INIT(&cc->cc_expireq);  in callout_cpu_init()
328 	cc->cc_firstevent = SBT_MAX;  in callout_cpu_init()
330 		cc_cce_cleanup(cc, i);  in callout_cpu_init()
331 	snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name),  in callout_cpu_init()
333 	if (cc->cc_callout == NULL)	/* Only BSP handles timeout(9) */  in callout_cpu_init()
336 		c = &cc->cc_callout[i];  in callout_cpu_init()
339 		SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);  in callout_cpu_init()
350 callout_cpu_switch(struct callout *c, struct callout_cpu *cc, int new_cpu)  in callout_cpu_switch()  argument
354 	MPASS(c != NULL && cc != NULL);  in callout_cpu_switch()
355 	CC_LOCK_ASSERT(cc);  in callout_cpu_switch()
364 	CC_UNLOCK(cc);  in callout_cpu_switch()
379 	struct callout_cpu *cc;  in start_softclock()  local
386 	cc = CC_CPU(timeout_cpu);  in start_softclock()
388 	if (swi_add(&clk_intr_event, name, softclock, cc, SWI_CLOCK,  in start_softclock()
389 	    INTR_MPSAFE, &cc->cc_cookie))  in start_softclock()
402 		cc = CC_CPU(cpu);  in start_softclock()
403 		cc->cc_callout = NULL;	/* Only BSP handles timeout(9). */  in start_softclock()
404 		callout_cpu_init(cc, cpu);  in start_softclock()
407 		if (swi_add(&ie, name, softclock, cc, SWI_CLOCK,  in start_softclock()
408 		    INTR_MPSAFE, &cc->cc_cookie))  in start_softclock()
441 	struct callout_cpu *cc;  in callout_process()  local
450 	cc = CC_SELF();  in callout_process()
451 	mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET);  in callout_process()
454 	firstb = callout_hash(cc->cc_lastscan);  in callout_process()
455 	cc->cc_lastscan = now;  in callout_process()
484 		sc = &cc->cc_callwheel[firstb & callwheelmask];  in callout_process()
497 					cc_exec_next(cc) =  in callout_process()
499 					cc->cc_bucket = firstb & callwheelmask;  in callout_process()
501 					softclock_call_cc(tmp, cc,  in callout_process()
506 					tmp = cc_exec_next(cc);  in callout_process()
507 					cc_exec_next(cc) = NULL;  in callout_process()
511 					TAILQ_INSERT_TAIL(&cc->cc_expireq,  in callout_process()
546 	cc->cc_firstevent = last;  in callout_process()
555 	mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);  in callout_process()
560 	if (!TAILQ_EMPTY(&cc->cc_expireq))  in callout_process()
561 		swi_sched(cc->cc_cookie, 0);  in callout_process()
567 	struct callout_cpu *cc;  in callout_lock()  local
579 		cc = CC_CPU(cpu);  in callout_lock()
580 		CC_LOCK(cc);  in callout_lock()
583 		CC_UNLOCK(cc);  in callout_lock()
585 	return (cc);  in callout_lock()
589 callout_cc_add(struct callout *c, struct callout_cpu *cc,  in callout_cc_add()  argument
595 	CC_LOCK_ASSERT(cc);  in callout_cc_add()
596 	if (sbt < cc->cc_lastscan)  in callout_cc_add()
597 		sbt = cc->cc_lastscan;  in callout_cc_add()
611 	LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le);  in callout_cc_add()
612 	if (cc->cc_bucket == bucket)  in callout_cc_add()
613 		cc_exec_next(cc) = c;  in callout_cc_add()
622 	if (sbt < cc->cc_firstevent) {  in callout_cc_add()
623 		cc->cc_firstevent = sbt;  in callout_cc_add()
630 callout_cc_del(struct callout *c, struct callout_cpu *cc)  in callout_cc_del()  argument
636 	SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);  in callout_cc_del()
640 softclock_call_cc(struct callout *c, struct callout_cpu *cc,  in softclock_call_cc()  argument
688 	cc_exec_curr(cc, direct) = c;  in softclock_call_cc()
689 	cc_exec_cancel(cc, direct) = false;  in softclock_call_cc()
690 	cc_exec_drain(cc, direct) = NULL;  in softclock_call_cc()
691 	CC_UNLOCK(cc);  in softclock_call_cc()
698 		if (cc_exec_cancel(cc, direct)) {  in softclock_call_cc()
703 		cc_exec_cancel(cc, direct) = true;  in softclock_call_cc()
724 	KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running",  in softclock_call_cc()
748 	KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle");  in softclock_call_cc()
753 	CC_LOCK(cc);  in softclock_call_cc()
754 	KASSERT(cc_exec_curr(cc, direct) == c, ("mishandled cc_curr"));  in softclock_call_cc()
755 	cc_exec_curr(cc, direct) = NULL;  in softclock_call_cc()
756 	if (cc_exec_drain(cc, direct)) {  in softclock_call_cc()
759 		drain = cc_exec_drain(cc, direct);  in softclock_call_cc()
760 		cc_exec_drain(cc, direct) = NULL;  in softclock_call_cc()
761 		CC_UNLOCK(cc);  in softclock_call_cc()
763 		CC_LOCK(cc);  in softclock_call_cc()
765 	if (cc_exec_waiting(cc, direct)) {  in softclock_call_cc()
772 		if (cc_cce_migrating(cc, direct)) {  in softclock_call_cc()
773 			cc_cce_cleanup(cc, direct);  in softclock_call_cc()
781 		cc_exec_waiting(cc, direct) = false;  in softclock_call_cc()
782 		CC_UNLOCK(cc);  in softclock_call_cc()
783 		wakeup(&cc_exec_waiting(cc, direct));  in softclock_call_cc()
784 		CC_LOCK(cc);  in softclock_call_cc()
785 	} else if (cc_cce_migrating(cc, direct)) {  in softclock_call_cc()
793 		new_cpu = cc_migration_cpu(cc, direct);  in softclock_call_cc()
794 		new_time = cc_migration_time(cc, direct);  in softclock_call_cc()
795 		new_prec = cc_migration_prec(cc, direct);  in softclock_call_cc()
796 		new_func = cc_migration_func(cc, direct);  in softclock_call_cc()
797 		new_arg = cc_migration_arg(cc, direct);  in softclock_call_cc()
798 		cc_cce_cleanup(cc, direct);  in softclock_call_cc()
810 			callout_cc_del(c, cc);  in softclock_call_cc()
815 		new_cc = callout_cpu_switch(c, cc, new_cpu);  in softclock_call_cc()
820 		CC_LOCK(cc);  in softclock_call_cc()
837 		callout_cc_del(c, cc);  in softclock_call_cc()
859 	struct callout_cpu *cc;  in softclock()  local
865 	cc = (struct callout_cpu *)arg;  in softclock()
866 	CC_LOCK(cc);  in softclock()
867 	while ((c = TAILQ_FIRST(&cc->cc_expireq)) != NULL) {  in softclock()
868 		TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);  in softclock()
869 		softclock_call_cc(c, cc,  in softclock()
884 	CC_UNLOCK(cc);  in softclock()
906 	struct callout_cpu *cc;  in timeout()  local
910 	cc = CC_CPU(timeout_cpu);  in timeout()
911 	CC_LOCK(cc);  in timeout()
913 	new = SLIST_FIRST(&cc->cc_callfree);  in timeout()
917 	SLIST_REMOVE_HEAD(&cc->cc_callfree, c_links.sle);  in timeout()
920 	CC_UNLOCK(cc);  in timeout()
928 	struct callout_cpu *cc;  in untimeout()  local
938 	cc = callout_lock(handle.callout);  in untimeout()
941 	CC_UNLOCK(cc);  in untimeout()
1023 	struct callout_cpu *cc;  local
1049 	cc = callout_lock(c);
1060 	if (cc_exec_curr(cc, direct) == c) {
1066 		if (c->c_lock != NULL && !cc_exec_cancel(cc, direct))
1067 			cancelled = cc_exec_cancel(cc, direct) = true;
1068 		if (cc_exec_waiting(cc, direct) || cc_exec_drain(cc, direct)) {
1076 			CC_UNLOCK(cc);
1088 			cc_migration_cpu(cc, direct) = cpu;
1089 			cc_migration_time(cc, direct) = to_sbt;
1090 			cc_migration_prec(cc, direct) = precision;
1091 			cc_migration_func(cc, direct) = ftn;
1092 			cc_migration_arg(cc, direct) = arg;
1094 			CC_UNLOCK(cc);
1101 			if (cc_exec_next(cc) == c)
1102 				cc_exec_next(cc) = LIST_NEXT(c, c_links.le);
1105 			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1119 		if (cc_exec_curr(cc, direct) == c) {
1136 			cc_migration_cpu(cc, direct) = cpu;
1137 			cc_migration_time(cc, direct) = to_sbt;
1138 			cc_migration_prec(cc, direct) = precision;
1139 			cc_migration_func(cc, direct) = ftn;
1140 			cc_migration_arg(cc, direct) = arg;
1147 			CC_UNLOCK(cc);
1150 		cc = callout_cpu_switch(c, cc, cpu);
1154 	callout_cc_add(c, cc, to_sbt, precision, ftn, arg, cpu, flags);
1158 	CC_UNLOCK(cc);
1181 	struct callout_cpu *cc, *old_cc;  local
1212 	cc = callout_lock(c);
1241 	if (sq_locked != 0 && cc != old_cc) {
1243 		CC_UNLOCK(cc);
1256 	if (cc_exec_curr(cc, direct) == c) {
1273 			if (cc_exec_curr(cc, direct) == c) {
1292 					CC_UNLOCK(cc);
1294 					    &cc_exec_waiting(cc, direct));
1296 					old_cc = cc;
1306 				cc_exec_waiting(cc, direct) = true;
1308 				CC_UNLOCK(cc);
1310 				    &cc_exec_waiting(cc, direct),
1311 				    &cc->cc_lock.lock_object, "codrain",
1314 				    &cc_exec_waiting(cc, direct),
1325 			   !cc_exec_cancel(cc, direct) && (drain == NULL)) {
1336 			cc_exec_cancel(cc, direct) = true;
1339 			KASSERT(!cc_cce_migrating(cc, direct),
1344 				cc_migration_cpu(cc, direct) = CPUBLOCK;
1345 				cc_migration_time(cc, direct) = 0;
1346 				cc_migration_prec(cc, direct) = 0;
1347 				cc_migration_func(cc, direct) = NULL;
1348 				cc_migration_arg(cc, direct) = NULL;
1351 			CC_UNLOCK(cc);
1372 			cc_migration_cpu(cc, direct) = CPUBLOCK;
1373 			cc_migration_time(cc, direct) = 0;
1374 			cc_migration_prec(cc, direct) = 0;
1375 			cc_migration_func(cc, direct) = NULL;
1376 			cc_migration_arg(cc, direct) = NULL;
1381 				cc_exec_drain(cc, direct) = drain;
1383 			CC_UNLOCK(cc);
1389 			cc_exec_drain(cc, direct) = drain;
1397 		sleepq_release(&cc_exec_waiting(cc, direct));
1406 		if (cc_exec_curr(cc, direct) != c)
1408 		CC_UNLOCK(cc);
1419 			if (cc_exec_next(cc) == c)
1420 				cc_exec_next(cc) = LIST_NEXT(c, c_links.le);
1423 			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1426 	callout_cc_del(c, cc);
1427 	CC_UNLOCK(cc);
1509 	CC_LOCK(cc);
1520 	CC_UNLOCK(cc);
1545 	struct callout_cpu *cc;  local
1566 		cc = CC_CPU(cpu);
1568 		cc = CC_CPU(timeout_cpu);
1570 		CC_LOCK(cc);
1572 			sc = &cc->cc_callwheel[i];
1593 		CC_UNLOCK(cc);