54 static int	task_is_running(struct gtaskqueue *queue, struct gtask *gtask);
55 static void	gtaskqueue_drain_locked(struct gtaskqueue *queue, struct gtask *gtask);
132 	struct gtaskqueue *queue;  in _gtaskqueue_create()  local
141 	queue = malloc(sizeof(struct gtaskqueue), M_GTASKQUEUE, mflags | M_ZERO);  in _gtaskqueue_create()
142 	if (!queue) {  in _gtaskqueue_create()
147 	STAILQ_INIT(&queue->tq_queue);  in _gtaskqueue_create()
148 	TAILQ_INIT(&queue->tq_active);  in _gtaskqueue_create()
149 	queue->tq_enqueue = enqueue;  in _gtaskqueue_create()
150 	queue->tq_context = context;  in _gtaskqueue_create()
151 	queue->tq_name = tq_name;  in _gtaskqueue_create()
152 	queue->tq_spin = (mtxflags & MTX_SPIN) != 0;  in _gtaskqueue_create()
153 	queue->tq_flags |= TQ_FLAGS_ACTIVE;  in _gtaskqueue_create()
155 		queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE;  in _gtaskqueue_create()
156 	mtx_init(&queue->tq_mutex, tq_name, NULL, mtxflags);  in _gtaskqueue_create()
158 	return (queue);  in _gtaskqueue_create()
176 gtaskqueue_free(struct gtaskqueue *queue)  in gtaskqueue_free()  argument
179 	TQ_LOCK(queue);  in gtaskqueue_free()
180 	queue->tq_flags &= ~TQ_FLAGS_ACTIVE;  in gtaskqueue_free()
181 	gtaskqueue_terminate(queue->tq_threads, queue);  in gtaskqueue_free()
182 	KASSERT(TAILQ_EMPTY(&queue->tq_active), ("Tasks still running?"));  in gtaskqueue_free()
183 	KASSERT(queue->tq_callouts == 0, ("Armed timeout tasks"));  in gtaskqueue_free()
184 	mtx_destroy(&queue->tq_mutex);  in gtaskqueue_free()
185 	free(queue->tq_threads, M_GTASKQUEUE);  in gtaskqueue_free()
186 	free(queue->tq_name, M_GTASKQUEUE);  in gtaskqueue_free()
187 	free(queue, M_GTASKQUEUE);  in gtaskqueue_free()
196 	struct gtaskqueue *queue = grouptask->gt_taskqueue;  in grouptask_block()  local
200 	if (queue == NULL) {  in grouptask_block()
205 	TQ_LOCK(queue);  in grouptask_block()
207   	gtaskqueue_drain_locked(queue, gtask);  in grouptask_block()
208 	TQ_UNLOCK(queue);  in grouptask_block()
214 	struct gtaskqueue *queue = grouptask->gt_taskqueue;  in grouptask_unblock()  local
218 	if (queue == NULL) {  in grouptask_unblock()
223 	TQ_LOCK(queue);  in grouptask_unblock()
225 	TQ_UNLOCK(queue);  in grouptask_unblock()
229 grouptaskqueue_enqueue(struct gtaskqueue *queue, struct gtask *gtask)  in grouptaskqueue_enqueue()  argument
232 	if (queue == NULL) {  in grouptaskqueue_enqueue()
237 	TQ_LOCK(queue);  in grouptaskqueue_enqueue()
239 		TQ_UNLOCK(queue);  in grouptaskqueue_enqueue()
243 		TQ_UNLOCK(queue);  in grouptaskqueue_enqueue()
246 	STAILQ_INSERT_TAIL(&queue->tq_queue, gtask, ta_link);  in grouptaskqueue_enqueue()
248 	TQ_UNLOCK(queue);  in grouptaskqueue_enqueue()
249 	if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0)  in grouptaskqueue_enqueue()
250 		queue->tq_enqueue(queue->tq_context);  in grouptaskqueue_enqueue()
265 gtaskqueue_drain_tq_queue(struct gtaskqueue *queue)  in gtaskqueue_drain_tq_queue()  argument
269 	if (STAILQ_EMPTY(&queue->tq_queue))  in gtaskqueue_drain_tq_queue()
281 	STAILQ_INSERT_TAIL(&queue->tq_queue, &t_barrier, ta_link);  in gtaskqueue_drain_tq_queue()
289 		TQ_SLEEP(queue, &t_barrier, &queue->tq_mutex, PWAIT, "-", 0);  in gtaskqueue_drain_tq_queue()
298 gtaskqueue_drain_tq_active(struct gtaskqueue *queue)  in gtaskqueue_drain_tq_active()  argument
302 	if (TAILQ_EMPTY(&queue->tq_active))  in gtaskqueue_drain_tq_active()
306 	queue->tq_callouts++;  in gtaskqueue_drain_tq_active()
313 	TAILQ_INSERT_TAIL(&queue->tq_active, &tb_marker, tb_link);  in gtaskqueue_drain_tq_active()
314 	while (TAILQ_FIRST(&queue->tq_active) != &tb_marker)  in gtaskqueue_drain_tq_active()
315 		TQ_SLEEP(queue, &tb_marker, &queue->tq_mutex, PWAIT, "-", 0);  in gtaskqueue_drain_tq_active()
316 	TAILQ_REMOVE(&queue->tq_active, &tb_marker, tb_link);  in gtaskqueue_drain_tq_active()
322 	tb_first = TAILQ_FIRST(&queue->tq_active);  in gtaskqueue_drain_tq_active()
327 	queue->tq_callouts--;  in gtaskqueue_drain_tq_active()
328 	if ((queue->tq_flags & TQ_FLAGS_ACTIVE) == 0)  in gtaskqueue_drain_tq_active()
329 		wakeup_one(queue->tq_threads);  in gtaskqueue_drain_tq_active()
333 gtaskqueue_block(struct gtaskqueue *queue)  in gtaskqueue_block()  argument
336 	TQ_LOCK(queue);  in gtaskqueue_block()
337 	queue->tq_flags |= TQ_FLAGS_BLOCKED;  in gtaskqueue_block()
338 	TQ_UNLOCK(queue);  in gtaskqueue_block()
342 gtaskqueue_unblock(struct gtaskqueue *queue)  in gtaskqueue_unblock()  argument
345 	TQ_LOCK(queue);  in gtaskqueue_unblock()
346 	queue->tq_flags &= ~TQ_FLAGS_BLOCKED;  in gtaskqueue_unblock()
347 	if (!STAILQ_EMPTY(&queue->tq_queue))  in gtaskqueue_unblock()
348 		queue->tq_enqueue(queue->tq_context);  in gtaskqueue_unblock()
349 	TQ_UNLOCK(queue);  in gtaskqueue_unblock()
353 gtaskqueue_run_locked(struct gtaskqueue *queue)  in gtaskqueue_run_locked()  argument
359 	KASSERT(queue != NULL, ("tq is NULL"));  in gtaskqueue_run_locked()
360 	TQ_ASSERT_LOCKED(queue);  in gtaskqueue_run_locked()
363 	while (STAILQ_FIRST(&queue->tq_queue)) {  in gtaskqueue_run_locked()
364 		TAILQ_INSERT_TAIL(&queue->tq_active, &tb, tb_link);  in gtaskqueue_run_locked()
370 		gtask = STAILQ_FIRST(&queue->tq_queue);  in gtaskqueue_run_locked()
372 		STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link);  in gtaskqueue_run_locked()
375 		TQ_UNLOCK(queue);  in gtaskqueue_run_locked()
380 		TQ_LOCK(queue);  in gtaskqueue_run_locked()
384 		TAILQ_REMOVE(&queue->tq_active, &tb, tb_link);  in gtaskqueue_run_locked()
385 		tb_first = TAILQ_FIRST(&queue->tq_active);  in gtaskqueue_run_locked()
393 task_is_running(struct gtaskqueue *queue, struct gtask *gtask)  in task_is_running()  argument
397 	TQ_ASSERT_LOCKED(queue);  in task_is_running()
398 	TAILQ_FOREACH(tb, &queue->tq_active, tb_link) {  in task_is_running()
406 gtaskqueue_cancel_locked(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_cancel_locked()  argument
410 		STAILQ_REMOVE(&queue->tq_queue, gtask, gtask, ta_link);  in gtaskqueue_cancel_locked()
412 	return (task_is_running(queue, gtask) ? EBUSY : 0);  in gtaskqueue_cancel_locked()
416 gtaskqueue_cancel(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_cancel()  argument
420 	TQ_LOCK(queue);  in gtaskqueue_cancel()
421 	error = gtaskqueue_cancel_locked(queue, gtask);  in gtaskqueue_cancel()
422 	TQ_UNLOCK(queue);  in gtaskqueue_cancel()
428 gtaskqueue_drain_locked(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_drain_locked()  argument
430 	while ((gtask->ta_flags & TASK_ENQUEUED) || task_is_running(queue, gtask))  in gtaskqueue_drain_locked()
431 		TQ_SLEEP(queue, gtask, &queue->tq_mutex, PWAIT, "-", 0);  in gtaskqueue_drain_locked()
435 gtaskqueue_drain(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_drain()  argument
438 	if (!queue->tq_spin)  in gtaskqueue_drain()
441 	TQ_LOCK(queue);  in gtaskqueue_drain()
442 	gtaskqueue_drain_locked(queue, gtask);  in gtaskqueue_drain()
443 	TQ_UNLOCK(queue);  in gtaskqueue_drain()
447 gtaskqueue_drain_all(struct gtaskqueue *queue)  in gtaskqueue_drain_all()  argument
450 	if (!queue->tq_spin)  in gtaskqueue_drain_all()
453 	TQ_LOCK(queue);  in gtaskqueue_drain_all()
454 	gtaskqueue_drain_tq_queue(queue);  in gtaskqueue_drain_all()
455 	gtaskqueue_drain_tq_active(queue);  in gtaskqueue_drain_all()
456 	TQ_UNLOCK(queue);  in gtaskqueue_drain_all()