54 static struct scx_idle_cpus *idle_cpumask(int node)  in idle_cpumask()  argument
56 	return node == NUMA_NO_NODE ? &scx_idle_global_masks : scx_idle_node_masks[node];  in idle_cpumask()
73 	int node = scx_cpu_node_if_enabled(cpu);  in scx_idle_test_and_clear_cpu()  local
74 	struct cpumask *idle_cpus = idle_cpumask(node)->cpu;  in scx_idle_test_and_clear_cpu()
84 		struct cpumask *idle_smts = idle_cpumask(node)->smt;  in scx_idle_test_and_clear_cpu()
109 static s32 pick_idle_cpu_in_node(const struct cpumask *cpus_allowed, int node, u64 flags)  in pick_idle_cpu_in_node()  argument
115 		cpu = cpumask_any_and_distribute(idle_cpumask(node)->smt, cpus_allowed);  in pick_idle_cpu_in_node()
123 	cpu = cpumask_any_and_distribute(idle_cpumask(node)->cpu, cpus_allowed);  in pick_idle_cpu_in_node()
143 static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags)  in pick_idle_cpu_from_online_nodes()  argument
156 	node_clear(node, *unvisited);  in pick_idle_cpu_from_online_nodes()
173 	for_each_node_numadist(node, *unvisited) {  in pick_idle_cpu_from_online_nodes()
174 		cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags);  in pick_idle_cpu_from_online_nodes()
186 s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, int node, u64 flags)  in scx_pick_idle_cpu()  argument
195 	cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags);  in scx_pick_idle_cpu()
204 	if (node == NUMA_NO_NODE || flags & SCX_PICK_IDLE_IN_NODE)  in scx_pick_idle_cpu()
210 	return pick_idle_cpu_from_online_nodes(cpus_allowed, node, flags);  in scx_pick_idle_cpu()
431 	int node = scx_cpu_node_if_enabled(prev_cpu);  in scx_select_cpu_dfl()  local
491 		    (!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) &&  in scx_select_cpu_dfl()
506 		if (cpumask_test_cpu(prev_cpu, idle_cpumask(node)->smt) &&  in scx_select_cpu_dfl()
516 			cpu = pick_idle_cpu_in_node(llc_cpus, node, SCX_PICK_IDLE_CORE);  in scx_select_cpu_dfl()
525 			cpu = pick_idle_cpu_in_node(numa_cpus, node, SCX_PICK_IDLE_CORE);  in scx_select_cpu_dfl()
538 		cpu = scx_pick_idle_cpu(p->cpus_ptr, node, flags | SCX_PICK_IDLE_CORE);  in scx_select_cpu_dfl()
564 		cpu = pick_idle_cpu_in_node(llc_cpus, node, 0);  in scx_select_cpu_dfl()
573 		cpu = pick_idle_cpu_in_node(numa_cpus, node, 0);  in scx_select_cpu_dfl()
586 	cpu = scx_pick_idle_cpu(p->cpus_ptr, node, flags);  in scx_select_cpu_dfl()
599 	int node;  in scx_idle_init_masks()  local
610 	for_each_node(node) {  in scx_idle_init_masks()
611 		scx_idle_node_masks[node] = kzalloc_node(sizeof(**scx_idle_node_masks),  in scx_idle_init_masks()
612 							 GFP_KERNEL, node);  in scx_idle_init_masks()
613 		BUG_ON(!scx_idle_node_masks[node]);  in scx_idle_init_masks()
615 		BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[node]->cpu, GFP_KERNEL, node));  in scx_idle_init_masks()
616 		BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[node]->smt, GFP_KERNEL, node));  in scx_idle_init_masks()
622 	int node = scx_cpu_node_if_enabled(cpu);  in update_builtin_idle()  local
623 	struct cpumask *idle_cpus = idle_cpumask(node)->cpu;  in update_builtin_idle()
630 		struct cpumask *idle_smts = idle_cpumask(node)->smt;  in update_builtin_idle()
700 	int node;  in reset_idle_masks()  local
712 	for_each_node(node) {  in reset_idle_masks()
713 		const struct cpumask *node_mask = cpumask_of_node(node);  in reset_idle_masks()
715 		cpumask_and(idle_cpumask(node)->cpu, cpu_online_mask, node_mask);  in reset_idle_masks()
716 		cpumask_and(idle_cpumask(node)->smt, cpu_online_mask, node_mask);  in reset_idle_masks()
748 static int validate_node(int node)  in validate_node()  argument
756 	if (node == NUMA_NO_NODE)  in validate_node()
760 	if (node < 0 || node >= nr_node_ids) {  in validate_node()
761 		scx_ops_error("invalid node %d", node);  in validate_node()
766 	if (!node_possible(node)) {  in validate_node()
767 		scx_ops_error("unavailable node %d", node);  in validate_node()
771 	return node;  in validate_node()
853 __bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask_node(int node)  in scx_bpf_get_idle_cpumask_node()  argument
855 	node = validate_node(node);  in scx_bpf_get_idle_cpumask_node()
856 	if (node < 0)  in scx_bpf_get_idle_cpumask_node()
860 	return idle_cpumask(node)->cpu;  in scx_bpf_get_idle_cpumask_node()
900 __bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask_node(int node)  in scx_bpf_get_idle_smtmask_node()  argument
902 	node = validate_node(node);  in scx_bpf_get_idle_smtmask_node()
903 	if (node < 0)  in scx_bpf_get_idle_smtmask_node()
908 		return idle_cpumask(node)->smt;  in scx_bpf_get_idle_smtmask_node()
910 		return idle_cpumask(node)->cpu;  in scx_bpf_get_idle_smtmask_node()
1000 					   int node, u64 flags)  in scx_bpf_pick_idle_cpu_node()  argument
1002 	node = validate_node(node);  in scx_bpf_pick_idle_cpu_node()
1003 	if (node < 0)  in scx_bpf_pick_idle_cpu_node()
1004 		return node;  in scx_bpf_pick_idle_cpu_node()
1006 	return scx_pick_idle_cpu(cpus_allowed, node, flags);  in scx_bpf_pick_idle_cpu_node()
1067 					  int node, u64 flags)  in scx_bpf_pick_any_cpu_node()  argument
1071 	node = validate_node(node);  in scx_bpf_pick_any_cpu_node()
1072 	if (node < 0)  in scx_bpf_pick_any_cpu_node()
1073 		return node;  in scx_bpf_pick_any_cpu_node()
1075 	cpu = scx_pick_idle_cpu(cpus_allowed, node, flags);  in scx_bpf_pick_any_cpu_node()
1080 		cpu = cpumask_any_and_distribute(cpumask_of_node(node), cpus_allowed);  in scx_bpf_pick_any_cpu_node()