18 static void dql_check_stall(struct dql *dql, unsigned short stall_thrs)  in dql_check_stall()  argument
27 	if (time_after_eq(now, dql->last_reap + stall_thrs)) {  in dql_check_stall()
35 		hist_head = READ_ONCE(dql->history_head);  in dql_check_stall()
45 		if (time_before(start, dql->last_reap + 1))  in dql_check_stall()
46 			start = dql->last_reap + 1;  in dql_check_stall()
60 				     dql->history))  in dql_check_stall()
68 		if (hist_head != READ_ONCE(dql->history_head))  in dql_check_stall()
71 		dql->stall_cnt++;  in dql_check_stall()
72 		dql->stall_max = max_t(unsigned short, dql->stall_max, now - t);  in dql_check_stall()
74 		trace_dql_stall_detected(dql->stall_thrs, now - t,  in dql_check_stall()
75 					 dql->last_reap, dql->history_head,  in dql_check_stall()
76 					 now, dql->history);  in dql_check_stall()
79 	dql->last_reap = now;  in dql_check_stall()
83 void dql_completed(struct dql *dql, unsigned int count)  in dql_completed()  argument
90 	num_queued = READ_ONCE(dql->num_queued);  in dql_completed()
96 	stall_thrs = READ_ONCE(dql->stall_thrs);  in dql_completed()
99 	BUG_ON(count > num_queued - dql->num_completed);  in dql_completed()
101 	completed = dql->num_completed + count;  in dql_completed()
102 	limit = dql->limit;  in dql_completed()
103 	ovlimit = POSDIFF(num_queued - dql->num_completed, limit);  in dql_completed()
105 	prev_inprogress = dql->prev_num_queued - dql->num_completed;  in dql_completed()
106 	all_prev_completed = AFTER_EQ(completed, dql->prev_num_queued);  in dql_completed()
109 	    (dql->prev_ovlimit && all_prev_completed)) {  in dql_completed()
125 		limit += POSDIFF(completed, dql->prev_num_queued) +  in dql_completed()
126 		     dql->prev_ovlimit;  in dql_completed()
127 		dql->slack_start_time = jiffies;  in dql_completed()
128 		dql->lowest_slack = UINT_MAX;  in dql_completed()
154 		slack = POSDIFF(limit + dql->prev_ovlimit,  in dql_completed()
155 		    2 * (completed - dql->num_completed));  in dql_completed()
156 		slack_last_objs = dql->prev_ovlimit ?  in dql_completed()
157 		    POSDIFF(dql->prev_last_obj_cnt, dql->prev_ovlimit) : 0;  in dql_completed()
161 		if (slack < dql->lowest_slack)  in dql_completed()
162 			dql->lowest_slack = slack;  in dql_completed()
165 			       dql->slack_start_time + dql->slack_hold_time)) {  in dql_completed()
166 			limit = POSDIFF(limit, dql->lowest_slack);  in dql_completed()
167 			dql->slack_start_time = jiffies;  in dql_completed()
168 			dql->lowest_slack = UINT_MAX;  in dql_completed()
173 	limit = clamp(limit, dql->min_limit, dql->max_limit);  in dql_completed()
175 	if (limit != dql->limit) {  in dql_completed()
176 		dql->limit = limit;  in dql_completed()
180 	dql->adj_limit = limit + completed;  in dql_completed()
181 	dql->prev_ovlimit = ovlimit;  in dql_completed()
182 	dql->prev_last_obj_cnt = READ_ONCE(dql->last_obj_cnt);  in dql_completed()
183 	dql->num_completed = completed;  in dql_completed()
184 	dql->prev_num_queued = num_queued;  in dql_completed()
186 	dql_check_stall(dql, stall_thrs);  in dql_completed()
190 void dql_reset(struct dql *dql)  in dql_reset()  argument
193 	dql->limit = dql->min_limit;  in dql_reset()
194 	dql->num_queued = 0;  in dql_reset()
195 	dql->num_completed = 0;  in dql_reset()
196 	dql->last_obj_cnt = 0;  in dql_reset()
197 	dql->prev_num_queued = 0;  in dql_reset()
198 	dql->prev_last_obj_cnt = 0;  in dql_reset()
199 	dql->prev_ovlimit = 0;  in dql_reset()
200 	dql->lowest_slack = UINT_MAX;  in dql_reset()
201 	dql->slack_start_time = jiffies;  in dql_reset()
203 	dql->last_reap = jiffies;  in dql_reset()
204 	dql->history_head = jiffies / BITS_PER_LONG;  in dql_reset()
205 	memset(dql->history, 0, sizeof(dql->history));  in dql_reset()
209 void dql_init(struct dql *dql, unsigned int hold_time)  in dql_init()  argument
211 	dql->max_limit = DQL_MAX_LIMIT;  in dql_init()
212 	dql->min_limit = 0;  in dql_init()
213 	dql->slack_hold_time = hold_time;  in dql_init()
214 	dql->stall_thrs = 0;  in dql_init()
215 	dql_reset(dql);  in dql_init()