54 static int	task_is_running(struct gtaskqueue *queue, struct gtask *gtask);
55 static void	gtaskqueue_drain_locked(struct gtaskqueue *queue, struct gtask *gtask);
130 	struct gtaskqueue *queue;  in _gtaskqueue_create()  local
139 	queue = malloc(sizeof(struct gtaskqueue), M_GTASKQUEUE, mflags | M_ZERO);  in _gtaskqueue_create()
140 	if (!queue) {  in _gtaskqueue_create()
145 	STAILQ_INIT(&queue->tq_queue);  in _gtaskqueue_create()
146 	LIST_INIT(&queue->tq_active);  in _gtaskqueue_create()
147 	queue->tq_enqueue = enqueue;  in _gtaskqueue_create()
148 	queue->tq_context = context;  in _gtaskqueue_create()
149 	queue->tq_name = tq_name;  in _gtaskqueue_create()
150 	queue->tq_spin = (mtxflags & MTX_SPIN) != 0;  in _gtaskqueue_create()
151 	queue->tq_flags |= TQ_FLAGS_ACTIVE;  in _gtaskqueue_create()
153 		queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE;  in _gtaskqueue_create()
154 	mtx_init(&queue->tq_mutex, tq_name, NULL, mtxflags);  in _gtaskqueue_create()
156 	return (queue);  in _gtaskqueue_create()
173 gtaskqueue_free(struct gtaskqueue *queue)  in gtaskqueue_free()  argument
176 	TQ_LOCK(queue);  in gtaskqueue_free()
177 	queue->tq_flags &= ~TQ_FLAGS_ACTIVE;  in gtaskqueue_free()
178 	gtaskqueue_terminate(queue->tq_threads, queue);  in gtaskqueue_free()
179 	KASSERT(LIST_EMPTY(&queue->tq_active), ("Tasks still running?"));  in gtaskqueue_free()
180 	KASSERT(queue->tq_callouts == 0, ("Armed timeout tasks"));  in gtaskqueue_free()
181 	mtx_destroy(&queue->tq_mutex);  in gtaskqueue_free()
182 	free(queue->tq_threads, M_GTASKQUEUE);  in gtaskqueue_free()
183 	free(queue->tq_name, M_GTASKQUEUE);  in gtaskqueue_free()
184 	free(queue, M_GTASKQUEUE);  in gtaskqueue_free()
193 	struct gtaskqueue *queue = grouptask->gt_taskqueue;  in grouptask_block()  local
197 	if (queue == NULL) {  in grouptask_block()
202 	TQ_LOCK(queue);  in grouptask_block()
204   	gtaskqueue_drain_locked(queue, gtask);  in grouptask_block()
205 	TQ_UNLOCK(queue);  in grouptask_block()
211 	struct gtaskqueue *queue = grouptask->gt_taskqueue;  in grouptask_unblock()  local
215 	if (queue == NULL) {  in grouptask_unblock()
220 	TQ_LOCK(queue);  in grouptask_unblock()
222 	TQ_UNLOCK(queue);  in grouptask_unblock()
226 grouptaskqueue_enqueue(struct gtaskqueue *queue, struct gtask *gtask)  in grouptaskqueue_enqueue()  argument
229 	if (queue == NULL) {  in grouptaskqueue_enqueue()
234 	TQ_LOCK(queue);  in grouptaskqueue_enqueue()
236 		TQ_UNLOCK(queue);  in grouptaskqueue_enqueue()
240 		TQ_UNLOCK(queue);  in grouptaskqueue_enqueue()
243 	STAILQ_INSERT_TAIL(&queue->tq_queue, gtask, ta_link);  in grouptaskqueue_enqueue()
245 	TQ_UNLOCK(queue);  in grouptaskqueue_enqueue()
246 	if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0)  in grouptaskqueue_enqueue()
247 		queue->tq_enqueue(queue->tq_context);  in grouptaskqueue_enqueue()
262 gtaskqueue_drain_tq_queue(struct gtaskqueue *queue)  in gtaskqueue_drain_tq_queue()  argument
266 	if (STAILQ_EMPTY(&queue->tq_queue))  in gtaskqueue_drain_tq_queue()
278 	STAILQ_INSERT_TAIL(&queue->tq_queue, &t_barrier, ta_link);  in gtaskqueue_drain_tq_queue()
286 		TQ_SLEEP(queue, &t_barrier, "gtq_qdrain");  in gtaskqueue_drain_tq_queue()
295 gtaskqueue_drain_tq_active(struct gtaskqueue *queue)  in gtaskqueue_drain_tq_active()  argument
300 	if (LIST_EMPTY(&queue->tq_active))  in gtaskqueue_drain_tq_active()
304 	queue->tq_callouts++;  in gtaskqueue_drain_tq_active()
307 	seq = queue->tq_seq;  in gtaskqueue_drain_tq_active()
309 	LIST_FOREACH(tb, &queue->tq_active, tb_link) {  in gtaskqueue_drain_tq_active()
311 			TQ_SLEEP(queue, tb->tb_running, "gtq_adrain");  in gtaskqueue_drain_tq_active()
317 	queue->tq_callouts--;  in gtaskqueue_drain_tq_active()
318 	if ((queue->tq_flags & TQ_FLAGS_ACTIVE) == 0)  in gtaskqueue_drain_tq_active()
319 		wakeup_one(queue->tq_threads);  in gtaskqueue_drain_tq_active()
323 gtaskqueue_block(struct gtaskqueue *queue)  in gtaskqueue_block()  argument
326 	TQ_LOCK(queue);  in gtaskqueue_block()
327 	queue->tq_flags |= TQ_FLAGS_BLOCKED;  in gtaskqueue_block()
328 	TQ_UNLOCK(queue);  in gtaskqueue_block()
332 gtaskqueue_unblock(struct gtaskqueue *queue)  in gtaskqueue_unblock()  argument
335 	TQ_LOCK(queue);  in gtaskqueue_unblock()
336 	queue->tq_flags &= ~TQ_FLAGS_BLOCKED;  in gtaskqueue_unblock()
337 	if (!STAILQ_EMPTY(&queue->tq_queue))  in gtaskqueue_unblock()
338 		queue->tq_enqueue(queue->tq_context);  in gtaskqueue_unblock()
339 	TQ_UNLOCK(queue);  in gtaskqueue_unblock()
343 gtaskqueue_run_locked(struct gtaskqueue *queue)  in gtaskqueue_run_locked()  argument
350 	KASSERT(queue != NULL, ("tq is NULL"));  in gtaskqueue_run_locked()
351 	TQ_ASSERT_LOCKED(queue);  in gtaskqueue_run_locked()
353 	LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link);  in gtaskqueue_run_locked()
356 	while ((gtask = STAILQ_FIRST(&queue->tq_queue)) != NULL) {  in gtaskqueue_run_locked()
357 		STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link);  in gtaskqueue_run_locked()
360 		tb.tb_seq = ++queue->tq_seq;  in gtaskqueue_run_locked()
361 		TQ_UNLOCK(queue);  in gtaskqueue_run_locked()
373 		TQ_LOCK(queue);  in gtaskqueue_run_locked()
382 task_is_running(struct gtaskqueue *queue, struct gtask *gtask)  in task_is_running()  argument
386 	TQ_ASSERT_LOCKED(queue);  in task_is_running()
387 	LIST_FOREACH(tb, &queue->tq_active, tb_link) {  in task_is_running()
395 gtaskqueue_cancel_locked(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_cancel_locked()  argument
399 		STAILQ_REMOVE(&queue->tq_queue, gtask, gtask, ta_link);  in gtaskqueue_cancel_locked()
401 	return (task_is_running(queue, gtask) ? EBUSY : 0);  in gtaskqueue_cancel_locked()
405 gtaskqueue_cancel(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_cancel()  argument
409 	TQ_LOCK(queue);  in gtaskqueue_cancel()
410 	error = gtaskqueue_cancel_locked(queue, gtask);  in gtaskqueue_cancel()
411 	TQ_UNLOCK(queue);  in gtaskqueue_cancel()
417 gtaskqueue_drain_locked(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_drain_locked()  argument
419 	while ((gtask->ta_flags & TASK_ENQUEUED) || task_is_running(queue, gtask))  in gtaskqueue_drain_locked()
420 		TQ_SLEEP(queue, gtask, "gtq_drain");  in gtaskqueue_drain_locked()
424 gtaskqueue_drain(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_drain()  argument
427 	if (!queue->tq_spin)  in gtaskqueue_drain()
430 	TQ_LOCK(queue);  in gtaskqueue_drain()
431 	gtaskqueue_drain_locked(queue, gtask);  in gtaskqueue_drain()
432 	TQ_UNLOCK(queue);  in gtaskqueue_drain()
436 gtaskqueue_drain_all(struct gtaskqueue *queue)  in gtaskqueue_drain_all()  argument
439 	if (!queue->tq_spin)  in gtaskqueue_drain_all()
442 	TQ_LOCK(queue);  in gtaskqueue_drain_all()
443 	gtaskqueue_drain_tq_queue(queue);  in gtaskqueue_drain_all()
444 	gtaskqueue_drain_tq_active(queue);  in gtaskqueue_drain_all()
445 	TQ_UNLOCK(queue);  in gtaskqueue_drain_all()