Lines Matching refs:dsq_id
999 u64 dsq_id; member
1092 static struct scx_dispatch_q *find_user_dsq(u64 dsq_id) in find_user_dsq() argument
1094 return rhashtable_lookup_fast(&dsq_hash, &dsq_id, dsq_hash_params); in find_user_dsq()
2033 static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, in find_dsq_for_dispatch() argument
2038 if (dsq_id == SCX_DSQ_LOCAL) in find_dsq_for_dispatch()
2041 if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { in find_dsq_for_dispatch()
2042 s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; in find_dsq_for_dispatch()
2050 if (dsq_id == SCX_DSQ_GLOBAL) in find_dsq_for_dispatch()
2053 dsq = find_user_dsq(dsq_id); in find_dsq_for_dispatch()
2057 dsq_id, p->comm, p->pid); in find_dsq_for_dispatch()
2065 struct task_struct *p, u64 dsq_id, in mark_direct_dispatch() argument
2090 p->scx.ddsp_dsq_id = dsq_id; in mark_direct_dispatch()
2887 u64 dsq_id, u64 enq_flags) in finish_dispatch() argument
2938 dsq = find_dsq_for_dispatch(this_rq(), dsq_id, p); in finish_dispatch()
2954 finish_dispatch(rq, ent->task, ent->qseq, ent->dsq_id, in flush_dispatch_buf()
4154 static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) in init_dsq() argument
4160 dsq->id = dsq_id; in init_dsq()
4163 static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) in create_dsq() argument
4168 if (dsq_id & SCX_DSQ_FLAG_BUILTIN) in create_dsq()
4175 init_dsq(dsq, dsq_id); in create_dsq()
4197 static void destroy_dsq(u64 dsq_id) in destroy_dsq() argument
4204 dsq = find_user_dsq(dsq_id); in destroy_dsq()
6124 static void scx_dsq_insert_commit(struct task_struct *p, u64 dsq_id, in scx_dsq_insert_commit() argument
6132 mark_direct_dispatch(ddsp_task, p, dsq_id, enq_flags); in scx_dsq_insert_commit()
6144 .dsq_id = dsq_id, in scx_dsq_insert_commit()
6185 __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, in scx_bpf_dsq_insert() argument
6196 scx_dsq_insert_commit(p, dsq_id, enq_flags); in scx_bpf_dsq_insert()
6200 __bpf_kfunc void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, in scx_bpf_dispatch() argument
6204 scx_bpf_dsq_insert(p, dsq_id, slice, enq_flags); in scx_bpf_dispatch()
6228 __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, in scx_bpf_dsq_insert_vtime() argument
6241 scx_dsq_insert_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); in scx_bpf_dsq_insert_vtime()
6245 __bpf_kfunc void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, in scx_bpf_dispatch_vtime() argument
6249 scx_bpf_dsq_insert_vtime(p, dsq_id, slice, vtime, enq_flags); in scx_bpf_dispatch_vtime()
6267 struct task_struct *p, u64 dsq_id, u64 enq_flags) in scx_dsq_move() argument
6321 dst_dsq = find_dsq_for_dispatch(this_rq, dsq_id, p); in scx_dsq_move()
6399 __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id) in scx_bpf_dsq_move_to_local() argument
6409 dsq = find_user_dsq(dsq_id); in scx_bpf_dsq_move_to_local()
6411 scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); in scx_bpf_dsq_move_to_local()
6430 __bpf_kfunc bool scx_bpf_consume(u64 dsq_id) in scx_bpf_consume() argument
6433 return scx_bpf_dsq_move_to_local(dsq_id); in scx_bpf_consume()
6514 struct task_struct *p, u64 dsq_id, in scx_bpf_dsq_move() argument
6518 p, dsq_id, enq_flags); in scx_bpf_dsq_move()
6523 struct task_struct *p, u64 dsq_id, in scx_bpf_dispatch_from_dsq() argument
6527 return scx_bpf_dsq_move(it__iter, p, dsq_id, enq_flags); in scx_bpf_dispatch_from_dsq()
6548 struct task_struct *p, u64 dsq_id, in scx_bpf_dsq_move_vtime() argument
6552 p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); in scx_bpf_dsq_move_vtime()
6557 struct task_struct *p, u64 dsq_id, in scx_bpf_dispatch_vtime_from_dsq() argument
6561 return scx_bpf_dsq_move_vtime(it__iter, p, dsq_id, enq_flags); in scx_bpf_dispatch_vtime_from_dsq()
6667 __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) in scx_bpf_create_dsq() argument
6672 return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); in scx_bpf_create_dsq()
6766 __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id) in scx_bpf_dsq_nr_queued() argument
6773 if (dsq_id == SCX_DSQ_LOCAL) { in scx_bpf_dsq_nr_queued()
6776 } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { in scx_bpf_dsq_nr_queued()
6777 s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; in scx_bpf_dsq_nr_queued()
6784 dsq = find_user_dsq(dsq_id); in scx_bpf_dsq_nr_queued()
6805 __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id) in scx_bpf_destroy_dsq() argument
6807 destroy_dsq(dsq_id); in scx_bpf_destroy_dsq()
6820 __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, in bpf_iter_scx_dsq_new() argument
6839 kit->dsq = find_user_dsq(dsq_id); in bpf_iter_scx_dsq_new()