Lines Matching refs:wq
258 struct workqueue_struct *wq; /* I: the owning workqueue */ member
526 static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
538 #define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \ argument
540 !lockdep_is_held(&wq->mutex) && \
598 #define for_each_pwq(pwq, wq) \ argument
599 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
600 lockdep_is_held(&(wq->mutex)))
722 unbound_pwq_slot(struct workqueue_struct *wq, int cpu) in unbound_pwq_slot() argument
725 return per_cpu_ptr(wq->cpu_pwq, cpu); in unbound_pwq_slot()
727 return &wq->dfl_pwq; in unbound_pwq_slot()
731 static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu) in unbound_pwq() argument
733 return rcu_dereference_check(*unbound_pwq_slot(wq, cpu), in unbound_pwq()
735 lockdep_is_held(&wq->mutex)); in unbound_pwq()
746 static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq) in unbound_effective_cpumask() argument
748 return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask; in unbound_effective_cpumask()
1552 static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq, in wq_node_nr_active() argument
1555 if (!(wq->flags & WQ_UNBOUND)) in wq_node_nr_active()
1561 return wq->node_nr_active[node]; in wq_node_nr_active()
1573 static void wq_update_node_max_active(struct workqueue_struct *wq, int off_cpu) in wq_update_node_max_active() argument
1575 struct cpumask *effective = unbound_effective_cpumask(wq); in wq_update_node_max_active()
1576 int min_active = READ_ONCE(wq->min_active); in wq_update_node_max_active()
1577 int max_active = READ_ONCE(wq->max_active); in wq_update_node_max_active()
1580 lockdep_assert_held(&wq->mutex); in wq_update_node_max_active()
1595 wq_node_nr_active(wq, node)->max = min_active; in wq_update_node_max_active()
1597 wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active; in wq_update_node_max_active()
1608 wq_node_nr_active(wq, node)->max = in wq_update_node_max_active()
1613 wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active; in wq_update_node_max_active()
1712 struct workqueue_struct *wq = pwq->wq; in pwq_tryinc_nr_active() local
1714 struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node); in pwq_tryinc_nr_active()
1721 obtained = pwq->nr_active < READ_ONCE(wq->max_active); in pwq_tryinc_nr_active()
1824 static void unplug_oldest_pwq(struct workqueue_struct *wq) in unplug_oldest_pwq() argument
1828 lockdep_assert_held(&wq->mutex); in unplug_oldest_pwq()
1831 pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue, in unplug_oldest_pwq()
1933 struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node); in pwq_dec_nr_active()
2013 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush)) in pwq_dec_nr_in_flight()
2014 complete(&pwq->wq->first_flusher->done); in pwq_dec_nr_in_flight()
2195 static bool is_chained_work(struct workqueue_struct *wq) in is_chained_work() argument
2204 return worker && worker->current_pwq->wq == wq; in is_chained_work()
2235 static void __queue_work(int cpu, struct workqueue_struct *wq, in __queue_work() argument
2256 if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) && in __queue_work()
2257 WARN_ONCE(!is_chained_work(wq), "workqueue: cannot queue %ps on wq %s\n", in __queue_work()
2258 work->func, wq->name))) { in __queue_work()
2265 if (wq->flags & WQ_UNBOUND) in __queue_work()
2271 pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu)); in __queue_work()
2284 if (last_pool && last_pool != pool && !(wq->flags & __WQ_ORDERED)) { in __queue_work()
2291 if (worker && worker->current_pwq->wq == wq) { in __queue_work()
2312 if (wq->flags & WQ_UNBOUND) { in __queue_work()
2319 wq->name, cpu); in __queue_work()
2382 bool queue_work_on(int cpu, struct workqueue_struct *wq, in queue_work_on() argument
2392 __queue_work(cpu, wq, work); in queue_work_on()
2450 bool queue_work_node(int node, struct workqueue_struct *wq, in queue_work_node() argument
2465 WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)); in queue_work_node()
2473 __queue_work(cpu, wq, work); in queue_work_node()
2487 __queue_work(dwork->cpu, dwork->wq, &dwork->work); in delayed_work_timer_fn()
2491 static void __queue_delayed_work(int cpu, struct workqueue_struct *wq, in __queue_delayed_work() argument
2497 WARN_ON_ONCE(!wq); in __queue_delayed_work()
2509 __queue_work(cpu, wq, &dwork->work); in __queue_delayed_work()
2514 dwork->wq = wq; in __queue_delayed_work()
2549 bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq, in queue_delayed_work_on() argument
2561 __queue_delayed_work(cpu, wq, dwork, delay); in queue_delayed_work_on()
2588 bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq, in mod_delayed_work_on() argument
2597 __queue_delayed_work(cpu, wq, dwork, delay); in mod_delayed_work_on()
2610 __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); in rcu_work_rcufn()
2624 bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) in queue_rcu_work() argument
2634 rwork->wq = wq; in queue_rcu_work()
2988 struct workqueue_struct *wq = pwq->wq; in send_mayday() local
2992 if (!wq->rescuer) in send_mayday()
3003 list_add_tail(&pwq->mayday_node, &wq->maydays); in send_mayday()
3004 wake_up_process(wq->rescuer->task); in send_mayday()
3177 strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN); in process_one_work()
3187 if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE)) in process_one_work()
3213 lock_map_acquire(pwq->wq->lockdep_map); in process_one_work()
3247 lock_map_release(pwq->wq->lockdep_map); in process_one_work()
3443 struct workqueue_struct *wq = rescuer->rescue_wq; in rescuer_thread() local
3469 while (!list_empty(&wq->maydays)) { in rescuer_thread()
3470 struct pool_workqueue *pwq = list_first_entry(&wq->maydays, in rescuer_thread()
3513 if (wq->rescuer && list_empty(&pwq->mayday_node)) { in rescuer_thread()
3515 list_add_tail(&pwq->mayday_node, &wq->maydays); in rescuer_thread()
3717 WARN_ONCE(worker && ((worker->current_pwq->wq->flags & in check_flush_dependency()
3720 worker->current_pwq->wq->name, worker->current_func, in check_flush_dependency()
3780 (pwq->wq->flags & WQ_BH) ? &bh_key : &thr_key); in insert_wq_barrier()
3844 static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, in flush_workqueue_prep_pwqs() argument
3852 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); in flush_workqueue_prep_pwqs()
3853 atomic_set(&wq->nr_pwqs_to_flush, 1); in flush_workqueue_prep_pwqs()
3864 for_each_pwq(pwq, wq) { in flush_workqueue_prep_pwqs()
3877 atomic_inc(&wq->nr_pwqs_to_flush); in flush_workqueue_prep_pwqs()
3892 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) in flush_workqueue_prep_pwqs()
3893 complete(&wq->first_flusher->done); in flush_workqueue_prep_pwqs()
3898 static void touch_wq_lockdep_map(struct workqueue_struct *wq) in touch_wq_lockdep_map() argument
3901 if (unlikely(!wq->lockdep_map)) in touch_wq_lockdep_map()
3904 if (wq->flags & WQ_BH) in touch_wq_lockdep_map()
3907 lock_map_acquire(wq->lockdep_map); in touch_wq_lockdep_map()
3908 lock_map_release(wq->lockdep_map); in touch_wq_lockdep_map()
3910 if (wq->flags & WQ_BH) in touch_wq_lockdep_map()
3916 struct workqueue_struct *wq) in touch_work_lockdep_map() argument
3919 if (wq->flags & WQ_BH) in touch_work_lockdep_map()
3925 if (wq->flags & WQ_BH) in touch_work_lockdep_map()
3937 void __flush_workqueue(struct workqueue_struct *wq) in __flush_workqueue() argument
3942 .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, (*wq->lockdep_map)), in __flush_workqueue()
3949 touch_wq_lockdep_map(wq); in __flush_workqueue()
3951 mutex_lock(&wq->mutex); in __flush_workqueue()
3956 next_color = work_next_color(wq->work_color); in __flush_workqueue()
3958 if (next_color != wq->flush_color) { in __flush_workqueue()
3964 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow)); in __flush_workqueue()
3965 this_flusher.flush_color = wq->work_color; in __flush_workqueue()
3966 wq->work_color = next_color; in __flush_workqueue()
3968 if (!wq->first_flusher) { in __flush_workqueue()
3970 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); in __flush_workqueue()
3972 wq->first_flusher = &this_flusher; in __flush_workqueue()
3974 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color, in __flush_workqueue()
3975 wq->work_color)) { in __flush_workqueue()
3977 wq->flush_color = next_color; in __flush_workqueue()
3978 wq->first_flusher = NULL; in __flush_workqueue()
3983 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color); in __flush_workqueue()
3984 list_add_tail(&this_flusher.list, &wq->flusher_queue); in __flush_workqueue()
3985 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); in __flush_workqueue()
3993 list_add_tail(&this_flusher.list, &wq->flusher_overflow); in __flush_workqueue()
3996 check_flush_dependency(wq, NULL, false); in __flush_workqueue()
3998 mutex_unlock(&wq->mutex); in __flush_workqueue()
4008 if (READ_ONCE(wq->first_flusher) != &this_flusher) in __flush_workqueue()
4011 mutex_lock(&wq->mutex); in __flush_workqueue()
4014 if (wq->first_flusher != &this_flusher) in __flush_workqueue()
4017 WRITE_ONCE(wq->first_flusher, NULL); in __flush_workqueue()
4020 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); in __flush_workqueue()
4026 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { in __flush_workqueue()
4027 if (next->flush_color != wq->flush_color) in __flush_workqueue()
4033 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) && in __flush_workqueue()
4034 wq->flush_color != work_next_color(wq->work_color)); in __flush_workqueue()
4037 wq->flush_color = work_next_color(wq->flush_color); in __flush_workqueue()
4040 if (!list_empty(&wq->flusher_overflow)) { in __flush_workqueue()
4047 list_for_each_entry(tmp, &wq->flusher_overflow, list) in __flush_workqueue()
4048 tmp->flush_color = wq->work_color; in __flush_workqueue()
4050 wq->work_color = work_next_color(wq->work_color); in __flush_workqueue()
4052 list_splice_tail_init(&wq->flusher_overflow, in __flush_workqueue()
4053 &wq->flusher_queue); in __flush_workqueue()
4054 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); in __flush_workqueue()
4057 if (list_empty(&wq->flusher_queue)) { in __flush_workqueue()
4058 WARN_ON_ONCE(wq->flush_color != wq->work_color); in __flush_workqueue()
4066 WARN_ON_ONCE(wq->flush_color == wq->work_color); in __flush_workqueue()
4067 WARN_ON_ONCE(wq->flush_color != next->flush_color); in __flush_workqueue()
4070 wq->first_flusher = next; in __flush_workqueue()
4072 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1)) in __flush_workqueue()
4079 wq->first_flusher = NULL; in __flush_workqueue()
4083 mutex_unlock(&wq->mutex); in __flush_workqueue()
4098 void drain_workqueue(struct workqueue_struct *wq) in drain_workqueue() argument
4108 mutex_lock(&wq->mutex); in drain_workqueue()
4109 if (!wq->nr_drainers++) in drain_workqueue()
4110 wq->flags |= __WQ_DRAINING; in drain_workqueue()
4111 mutex_unlock(&wq->mutex); in drain_workqueue()
4113 __flush_workqueue(wq); in drain_workqueue()
4115 mutex_lock(&wq->mutex); in drain_workqueue()
4117 for_each_pwq(pwq, wq) { in drain_workqueue()
4130 wq->name, __func__, flush_cnt); in drain_workqueue()
4132 mutex_unlock(&wq->mutex); in drain_workqueue()
4136 if (!--wq->nr_drainers) in drain_workqueue()
4137 wq->flags &= ~__WQ_DRAINING; in drain_workqueue()
4138 mutex_unlock(&wq->mutex); in drain_workqueue()
4148 struct workqueue_struct *wq; in start_flush_work() local
4170 wq = pwq->wq; in start_flush_work()
4171 check_flush_dependency(wq, work, from_cancel); in start_flush_work()
4176 touch_work_lockdep_map(work, wq); in start_flush_work()
4187 if (!from_cancel && (wq->saved_max_active == 1 || wq->rescuer)) in start_flush_work()
4188 touch_wq_lockdep_map(wq); in start_flush_work()
4285 __queue_work(dwork->cpu, dwork->wq, &dwork->work); in flush_delayed_work()
4797 static void wq_init_lockdep(struct workqueue_struct *wq) in wq_init_lockdep() argument
4801 lockdep_register_key(&wq->key); in wq_init_lockdep()
4802 lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name); in wq_init_lockdep()
4804 lock_name = wq->name; in wq_init_lockdep()
4806 wq->lock_name = lock_name; in wq_init_lockdep()
4807 wq->lockdep_map = &wq->__lockdep_map; in wq_init_lockdep()
4808 lockdep_init_map(wq->lockdep_map, lock_name, &wq->key, 0); in wq_init_lockdep()
4811 static void wq_unregister_lockdep(struct workqueue_struct *wq) in wq_unregister_lockdep() argument
4813 if (wq->lockdep_map != &wq->__lockdep_map) in wq_unregister_lockdep()
4816 lockdep_unregister_key(&wq->key); in wq_unregister_lockdep()
4819 static void wq_free_lockdep(struct workqueue_struct *wq) in wq_free_lockdep() argument
4821 if (wq->lockdep_map != &wq->__lockdep_map) in wq_free_lockdep()
4824 if (wq->lock_name != wq->name) in wq_free_lockdep()
4825 kfree(wq->lock_name); in wq_free_lockdep()
4828 static void wq_init_lockdep(struct workqueue_struct *wq) in wq_init_lockdep() argument
4832 static void wq_unregister_lockdep(struct workqueue_struct *wq) in wq_unregister_lockdep() argument
4836 static void wq_free_lockdep(struct workqueue_struct *wq) in wq_free_lockdep() argument
4895 struct workqueue_struct *wq = in rcu_free_wq() local
4898 if (wq->flags & WQ_UNBOUND) in rcu_free_wq()
4899 free_node_nr_active(wq->node_nr_active); in rcu_free_wq()
4901 wq_free_lockdep(wq); in rcu_free_wq()
4902 free_percpu(wq->cpu_pwq); in rcu_free_wq()
4903 free_workqueue_attrs(wq->unbound_attrs); in rcu_free_wq()
4904 kfree(wq); in rcu_free_wq()
5068 struct workqueue_struct *wq = pwq->wq; in pwq_release_workfn() local
5077 mutex_lock(&wq->mutex); in pwq_release_workfn()
5079 is_last = list_empty(&wq->pwqs); in pwq_release_workfn()
5084 if (!is_last && (wq->flags & __WQ_ORDERED)) in pwq_release_workfn()
5085 unplug_oldest_pwq(wq); in pwq_release_workfn()
5087 mutex_unlock(&wq->mutex); in pwq_release_workfn()
5090 if (wq->flags & WQ_UNBOUND) { in pwq_release_workfn()
5098 wq_node_nr_active(pwq->wq, pwq->pool->node); in pwq_release_workfn()
5112 wq_unregister_lockdep(wq); in pwq_release_workfn()
5113 call_rcu(&wq->rcu, rcu_free_wq); in pwq_release_workfn()
5118 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, in init_pwq() argument
5126 pwq->wq = wq; in init_pwq()
5139 struct workqueue_struct *wq = pwq->wq; in link_pwq() local
5141 lockdep_assert_held(&wq->mutex); in link_pwq()
5148 pwq->work_color = wq->work_color; in link_pwq()
5151 list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs); in link_pwq()
5155 static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq, in alloc_unbound_pwq() argument
5173 init_pwq(pwq, wq, pool); in alloc_unbound_pwq()
5216 static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq, in install_unbound_pwq() argument
5219 struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu); in install_unbound_pwq()
5223 lockdep_assert_held(&wq->mutex); in install_unbound_pwq()
5235 struct workqueue_struct *wq; /* target workqueue */ member
5260 apply_wqattrs_prepare(struct workqueue_struct *wq, in apply_wqattrs_prepare() argument
5288 ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); in apply_wqattrs_prepare()
5298 ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs); in apply_wqattrs_prepare()
5316 if ((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)) in apply_wqattrs_prepare()
5319 ctx->wq = wq; in apply_wqattrs_prepare()
5334 mutex_lock(&ctx->wq->mutex); in apply_wqattrs_commit()
5336 copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs); in apply_wqattrs_commit()
5340 ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu, in apply_wqattrs_commit()
5342 ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq); in apply_wqattrs_commit()
5345 wq_update_node_max_active(ctx->wq, -1); in apply_wqattrs_commit()
5348 if (ctx->wq->rescuer) in apply_wqattrs_commit()
5349 set_cpus_allowed_ptr(ctx->wq->rescuer->task, in apply_wqattrs_commit()
5350 unbound_effective_cpumask(ctx->wq)); in apply_wqattrs_commit()
5352 mutex_unlock(&ctx->wq->mutex); in apply_wqattrs_commit()
5355 static int apply_workqueue_attrs_locked(struct workqueue_struct *wq, in apply_workqueue_attrs_locked() argument
5361 if (WARN_ON(!(wq->flags & WQ_UNBOUND))) in apply_workqueue_attrs_locked()
5364 ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask); in apply_workqueue_attrs_locked()
5390 int apply_workqueue_attrs(struct workqueue_struct *wq, in apply_workqueue_attrs() argument
5396 ret = apply_workqueue_attrs_locked(wq, attrs); in apply_workqueue_attrs()
5421 static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) in unbound_wq_update_pwq() argument
5428 if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered) in unbound_wq_update_pwq()
5438 copy_workqueue_attrs(target_attrs, wq->unbound_attrs); in unbound_wq_update_pwq()
5443 if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs)) in unbound_wq_update_pwq()
5447 pwq = alloc_unbound_pwq(wq, target_attrs); in unbound_wq_update_pwq()
5450 wq->name); in unbound_wq_update_pwq()
5455 mutex_lock(&wq->mutex); in unbound_wq_update_pwq()
5456 old_pwq = install_unbound_pwq(wq, cpu, pwq); in unbound_wq_update_pwq()
5460 mutex_lock(&wq->mutex); in unbound_wq_update_pwq()
5461 pwq = unbound_pwq(wq, -1); in unbound_wq_update_pwq()
5465 old_pwq = install_unbound_pwq(wq, cpu, pwq); in unbound_wq_update_pwq()
5467 mutex_unlock(&wq->mutex); in unbound_wq_update_pwq()
5471 static int alloc_and_link_pwqs(struct workqueue_struct *wq) in alloc_and_link_pwqs() argument
5473 bool highpri = wq->flags & WQ_HIGHPRI; in alloc_and_link_pwqs()
5478 wq->cpu_pwq = alloc_percpu(struct pool_workqueue *); in alloc_and_link_pwqs()
5479 if (!wq->cpu_pwq) in alloc_and_link_pwqs()
5482 if (!(wq->flags & WQ_UNBOUND)) { in alloc_and_link_pwqs()
5485 if (wq->flags & WQ_BH) in alloc_and_link_pwqs()
5495 pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu); in alloc_and_link_pwqs()
5502 init_pwq(*pwq_p, wq, pool); in alloc_and_link_pwqs()
5504 mutex_lock(&wq->mutex); in alloc_and_link_pwqs()
5506 mutex_unlock(&wq->mutex); in alloc_and_link_pwqs()
5511 if (wq->flags & __WQ_ORDERED) { in alloc_and_link_pwqs()
5514 ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]); in alloc_and_link_pwqs()
5516 dfl_pwq = rcu_access_pointer(wq->dfl_pwq); in alloc_and_link_pwqs()
5517 WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node || in alloc_and_link_pwqs()
5518 wq->pwqs.prev != &dfl_pwq->pwqs_node), in alloc_and_link_pwqs()
5519 "ordering guarantee broken for workqueue %s\n", wq->name); in alloc_and_link_pwqs()
5521 ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]); in alloc_and_link_pwqs()
5527 if (wq->cpu_pwq) { in alloc_and_link_pwqs()
5529 struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); in alloc_and_link_pwqs()
5534 free_percpu(wq->cpu_pwq); in alloc_and_link_pwqs()
5535 wq->cpu_pwq = NULL; in alloc_and_link_pwqs()
5554 static int init_rescuer(struct workqueue_struct *wq) in init_rescuer() argument
5562 if (!(wq->flags & WQ_MEM_RECLAIM)) in init_rescuer()
5568 wq->name); in init_rescuer()
5572 rescuer->rescue_wq = wq; in init_rescuer()
5579 wq->name, ERR_PTR(ret)); in init_rescuer()
5584 wq->rescuer = rescuer; in init_rescuer()
5585 if (wq->flags & WQ_UNBOUND) in init_rescuer()
5586 kthread_bind_mask(rescuer->task, unbound_effective_cpumask(wq)); in init_rescuer()
5602 static void wq_adjust_max_active(struct workqueue_struct *wq) in wq_adjust_max_active() argument
5607 lockdep_assert_held(&wq->mutex); in wq_adjust_max_active()
5609 if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) { in wq_adjust_max_active()
5613 new_max = wq->saved_max_active; in wq_adjust_max_active()
5614 new_min = wq->saved_min_active; in wq_adjust_max_active()
5617 if (wq->max_active == new_max && wq->min_active == new_min) in wq_adjust_max_active()
5626 WRITE_ONCE(wq->max_active, new_max); in wq_adjust_max_active()
5627 WRITE_ONCE(wq->min_active, new_min); in wq_adjust_max_active()
5629 if (wq->flags & WQ_UNBOUND) in wq_adjust_max_active()
5630 wq_update_node_max_active(wq, -1); in wq_adjust_max_active()
5643 for_each_pwq(pwq, wq) { in wq_adjust_max_active()
5662 struct workqueue_struct *wq; in __alloc_workqueue() local
5679 wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1); in __alloc_workqueue()
5681 wq_size = sizeof(*wq); in __alloc_workqueue()
5683 wq = kzalloc(wq_size, GFP_KERNEL); in __alloc_workqueue()
5684 if (!wq) in __alloc_workqueue()
5688 wq->unbound_attrs = alloc_workqueue_attrs(); in __alloc_workqueue()
5689 if (!wq->unbound_attrs) in __alloc_workqueue()
5693 name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args); in __alloc_workqueue()
5697 wq->name); in __alloc_workqueue()
5707 max_active = wq_clamp_max_active(max_active, flags, wq->name); in __alloc_workqueue()
5711 wq->flags = flags; in __alloc_workqueue()
5712 wq->max_active = max_active; in __alloc_workqueue()
5713 wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE); in __alloc_workqueue()
5714 wq->saved_max_active = wq->max_active; in __alloc_workqueue()
5715 wq->saved_min_active = wq->min_active; in __alloc_workqueue()
5716 mutex_init(&wq->mutex); in __alloc_workqueue()
5717 atomic_set(&wq->nr_pwqs_to_flush, 0); in __alloc_workqueue()
5718 INIT_LIST_HEAD(&wq->pwqs); in __alloc_workqueue()
5719 INIT_LIST_HEAD(&wq->flusher_queue); in __alloc_workqueue()
5720 INIT_LIST_HEAD(&wq->flusher_overflow); in __alloc_workqueue()
5721 INIT_LIST_HEAD(&wq->maydays); in __alloc_workqueue()
5723 INIT_LIST_HEAD(&wq->list); in __alloc_workqueue()
5726 if (alloc_node_nr_active(wq->node_nr_active) < 0) in __alloc_workqueue()
5736 if (alloc_and_link_pwqs(wq) < 0) in __alloc_workqueue()
5739 mutex_lock(&wq->mutex); in __alloc_workqueue()
5740 wq_adjust_max_active(wq); in __alloc_workqueue()
5741 mutex_unlock(&wq->mutex); in __alloc_workqueue()
5743 list_add_tail_rcu(&wq->list, &workqueues); in __alloc_workqueue()
5745 if (wq_online && init_rescuer(wq) < 0) in __alloc_workqueue()
5750 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq)) in __alloc_workqueue()
5753 return wq; in __alloc_workqueue()
5762 if (wq->flags & WQ_UNBOUND) { in __alloc_workqueue()
5764 free_node_nr_active(wq->node_nr_active); in __alloc_workqueue()
5767 free_workqueue_attrs(wq->unbound_attrs); in __alloc_workqueue()
5768 kfree(wq); in __alloc_workqueue()
5773 destroy_workqueue(wq); in __alloc_workqueue()
5782 struct workqueue_struct *wq; in alloc_workqueue() local
5786 wq = __alloc_workqueue(fmt, flags, max_active, args); in alloc_workqueue()
5788 if (!wq) in alloc_workqueue()
5791 wq_init_lockdep(wq); in alloc_workqueue()
5793 return wq; in alloc_workqueue()
5803 struct workqueue_struct *wq; in alloc_workqueue_lockdep_map() local
5807 wq = __alloc_workqueue(fmt, flags, max_active, args); in alloc_workqueue_lockdep_map()
5809 if (!wq) in alloc_workqueue_lockdep_map()
5812 wq->lockdep_map = lockdep_map; in alloc_workqueue_lockdep_map()
5814 return wq; in alloc_workqueue_lockdep_map()
5827 if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1)) in pwq_busy()
5841 void destroy_workqueue(struct workqueue_struct *wq) in destroy_workqueue() argument
5850 workqueue_sysfs_unregister(wq); in destroy_workqueue()
5853 mutex_lock(&wq->mutex); in destroy_workqueue()
5854 wq->flags |= __WQ_DESTROYING; in destroy_workqueue()
5855 mutex_unlock(&wq->mutex); in destroy_workqueue()
5858 drain_workqueue(wq); in destroy_workqueue()
5861 if (wq->rescuer) { in destroy_workqueue()
5862 struct worker *rescuer = wq->rescuer; in destroy_workqueue()
5866 wq->rescuer = NULL; in destroy_workqueue()
5879 mutex_lock(&wq->mutex); in destroy_workqueue()
5880 for_each_pwq(pwq, wq) { in destroy_workqueue()
5884 __func__, wq->name); in destroy_workqueue()
5887 mutex_unlock(&wq->mutex); in destroy_workqueue()
5889 show_one_workqueue(wq); in destroy_workqueue()
5894 mutex_unlock(&wq->mutex); in destroy_workqueue()
5900 list_del_rcu(&wq->list); in destroy_workqueue()
5911 put_pwq_unlocked(unbound_pwq(wq, cpu)); in destroy_workqueue()
5912 RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL); in destroy_workqueue()
5915 put_pwq_unlocked(unbound_pwq(wq, -1)); in destroy_workqueue()
5916 RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL); in destroy_workqueue()
5933 void workqueue_set_max_active(struct workqueue_struct *wq, int max_active) in workqueue_set_max_active() argument
5936 if (WARN_ON(wq->flags & WQ_BH)) in workqueue_set_max_active()
5939 if (WARN_ON(wq->flags & __WQ_ORDERED)) in workqueue_set_max_active()
5942 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name); in workqueue_set_max_active()
5944 mutex_lock(&wq->mutex); in workqueue_set_max_active()
5946 wq->saved_max_active = max_active; in workqueue_set_max_active()
5947 if (wq->flags & WQ_UNBOUND) in workqueue_set_max_active()
5948 wq->saved_min_active = min(wq->saved_min_active, max_active); in workqueue_set_max_active()
5950 wq_adjust_max_active(wq); in workqueue_set_max_active()
5952 mutex_unlock(&wq->mutex); in workqueue_set_max_active()
5970 void workqueue_set_min_active(struct workqueue_struct *wq, int min_active) in workqueue_set_min_active() argument
5973 if (WARN_ON((wq->flags & (WQ_BH | WQ_UNBOUND | __WQ_ORDERED)) != in workqueue_set_min_active()
5977 mutex_lock(&wq->mutex); in workqueue_set_min_active()
5978 wq->saved_min_active = clamp(min_active, 0, wq->saved_max_active); in workqueue_set_min_active()
5979 wq_adjust_max_active(wq); in workqueue_set_min_active()
5980 mutex_unlock(&wq->mutex); in workqueue_set_min_active()
6033 bool workqueue_congested(int cpu, struct workqueue_struct *wq) in workqueue_congested() argument
6044 pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); in workqueue_congested()
6130 struct workqueue_struct *wq = NULL; in print_worker_info() local
6148 copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq)); in print_worker_info()
6149 copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1); in print_worker_info()
6308 void show_one_workqueue(struct workqueue_struct *wq) in show_one_workqueue() argument
6314 for_each_pwq(pwq, wq) { in show_one_workqueue()
6323 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags); in show_one_workqueue()
6325 for_each_pwq(pwq, wq) { in show_one_workqueue()
6404 struct workqueue_struct *wq; in show_all_workqueues() local
6412 list_for_each_entry_rcu(wq, &workqueues, list) in show_all_workqueues()
6413 show_one_workqueue(wq); in show_all_workqueues()
6429 struct workqueue_struct *wq; in show_freezable_workqueues() local
6435 list_for_each_entry_rcu(wq, &workqueues, list) { in show_freezable_workqueues()
6436 if (!(wq->flags & WQ_FREEZABLE)) in show_freezable_workqueues()
6438 show_one_workqueue(wq); in show_freezable_workqueues()
6646 struct workqueue_struct *wq; in workqueue_online_cpu() local
6667 list_for_each_entry(wq, &workqueues, list) { in workqueue_online_cpu()
6668 struct workqueue_attrs *attrs = wq->unbound_attrs; in workqueue_online_cpu()
6675 unbound_wq_update_pwq(wq, tcpu); in workqueue_online_cpu()
6677 mutex_lock(&wq->mutex); in workqueue_online_cpu()
6678 wq_update_node_max_active(wq, -1); in workqueue_online_cpu()
6679 mutex_unlock(&wq->mutex); in workqueue_online_cpu()
6689 struct workqueue_struct *wq; in workqueue_offline_cpu() local
6702 list_for_each_entry(wq, &workqueues, list) { in workqueue_offline_cpu()
6703 struct workqueue_attrs *attrs = wq->unbound_attrs; in workqueue_offline_cpu()
6710 unbound_wq_update_pwq(wq, tcpu); in workqueue_offline_cpu()
6712 mutex_lock(&wq->mutex); in workqueue_offline_cpu()
6713 wq_update_node_max_active(wq, cpu); in workqueue_offline_cpu()
6714 mutex_unlock(&wq->mutex); in workqueue_offline_cpu()
6801 struct workqueue_struct *wq; in freeze_workqueues_begin() local
6808 list_for_each_entry(wq, &workqueues, list) { in freeze_workqueues_begin()
6809 mutex_lock(&wq->mutex); in freeze_workqueues_begin()
6810 wq_adjust_max_active(wq); in freeze_workqueues_begin()
6811 mutex_unlock(&wq->mutex); in freeze_workqueues_begin()
6833 struct workqueue_struct *wq; in freeze_workqueues_busy() local
6840 list_for_each_entry(wq, &workqueues, list) { in freeze_workqueues_busy()
6841 if (!(wq->flags & WQ_FREEZABLE)) in freeze_workqueues_busy()
6848 for_each_pwq(pwq, wq) { in freeze_workqueues_busy()
6874 struct workqueue_struct *wq; in thaw_workqueues() local
6884 list_for_each_entry(wq, &workqueues, list) { in thaw_workqueues()
6885 mutex_lock(&wq->mutex); in thaw_workqueues()
6886 wq_adjust_max_active(wq); in thaw_workqueues()
6887 mutex_unlock(&wq->mutex); in thaw_workqueues()
6899 struct workqueue_struct *wq; in workqueue_apply_unbound_cpumask() local
6904 list_for_each_entry(wq, &workqueues, list) { in workqueue_apply_unbound_cpumask()
6905 if (!(wq->flags & WQ_UNBOUND) || (wq->flags & __WQ_DESTROYING)) in workqueue_apply_unbound_cpumask()
6908 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask); in workqueue_apply_unbound_cpumask()
6981 struct workqueue_struct *wq; in wq_affn_dfl_set() local
6995 list_for_each_entry(wq, &workqueues, list) { in wq_affn_dfl_set()
6997 unbound_wq_update_pwq(wq, cpu); in wq_affn_dfl_set()
7035 struct workqueue_struct *wq; member
7043 return wq_dev->wq; in dev_to_wq()
7049 struct workqueue_struct *wq = dev_to_wq(dev); in per_cpu_show() local
7051 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND)); in per_cpu_show()
7058 struct workqueue_struct *wq = dev_to_wq(dev); in max_active_show() local
7060 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active); in max_active_show()
7067 struct workqueue_struct *wq = dev_to_wq(dev); in max_active_store() local
7073 workqueue_set_max_active(wq, val); in max_active_store()
7088 struct workqueue_struct *wq = dev_to_wq(dev); in wq_nice_show() local
7091 mutex_lock(&wq->mutex); in wq_nice_show()
7092 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice); in wq_nice_show()
7093 mutex_unlock(&wq->mutex); in wq_nice_show()
7099 static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq) in wq_sysfs_prep_attrs() argument
7109 copy_workqueue_attrs(attrs, wq->unbound_attrs); in wq_sysfs_prep_attrs()
7116 struct workqueue_struct *wq = dev_to_wq(dev); in wq_nice_store() local
7122 attrs = wq_sysfs_prep_attrs(wq); in wq_nice_store()
7128 ret = apply_workqueue_attrs_locked(wq, attrs); in wq_nice_store()
7141 struct workqueue_struct *wq = dev_to_wq(dev); in wq_cpumask_show() local
7144 mutex_lock(&wq->mutex); in wq_cpumask_show()
7146 cpumask_pr_args(wq->unbound_attrs->cpumask)); in wq_cpumask_show()
7147 mutex_unlock(&wq->mutex); in wq_cpumask_show()
7155 struct workqueue_struct *wq = dev_to_wq(dev); in wq_cpumask_store() local
7161 attrs = wq_sysfs_prep_attrs(wq); in wq_cpumask_store()
7167 ret = apply_workqueue_attrs_locked(wq, attrs); in wq_cpumask_store()
7178 struct workqueue_struct *wq = dev_to_wq(dev); in wq_affn_scope_show() local
7181 mutex_lock(&wq->mutex); in wq_affn_scope_show()
7182 if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL) in wq_affn_scope_show()
7188 wq_affn_names[wq->unbound_attrs->affn_scope]); in wq_affn_scope_show()
7189 mutex_unlock(&wq->mutex); in wq_affn_scope_show()
7198 struct workqueue_struct *wq = dev_to_wq(dev); in wq_affn_scope_store() local
7207 attrs = wq_sysfs_prep_attrs(wq); in wq_affn_scope_store()
7210 ret = apply_workqueue_attrs_locked(wq, attrs); in wq_affn_scope_store()
7220 struct workqueue_struct *wq = dev_to_wq(dev); in wq_affinity_strict_show() local
7223 wq->unbound_attrs->affn_strict); in wq_affinity_strict_show()
7230 struct workqueue_struct *wq = dev_to_wq(dev); in wq_affinity_strict_store() local
7238 attrs = wq_sysfs_prep_attrs(wq); in wq_affinity_strict_store()
7241 ret = apply_workqueue_attrs_locked(wq, attrs); in wq_affinity_strict_store()
7381 int workqueue_sysfs_register(struct workqueue_struct *wq) in workqueue_sysfs_register() argument
7390 if (WARN_ON(wq->flags & __WQ_ORDERED)) in workqueue_sysfs_register()
7393 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL); in workqueue_sysfs_register()
7397 wq_dev->wq = wq; in workqueue_sysfs_register()
7400 dev_set_name(&wq_dev->dev, "%s", wq->name); in workqueue_sysfs_register()
7411 wq->wq_dev = NULL; in workqueue_sysfs_register()
7415 if (wq->flags & WQ_UNBOUND) { in workqueue_sysfs_register()
7422 wq->wq_dev = NULL; in workqueue_sysfs_register()
7439 static void workqueue_sysfs_unregister(struct workqueue_struct *wq) in workqueue_sysfs_unregister() argument
7441 struct wq_device *wq_dev = wq->wq_dev; in workqueue_sysfs_unregister()
7443 if (!wq->wq_dev) in workqueue_sysfs_unregister()
7446 wq->wq_dev = NULL; in workqueue_sysfs_unregister()
7450 static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { } in workqueue_sysfs_unregister() argument
7890 struct workqueue_struct *wq; in workqueue_init() local
7909 list_for_each_entry(wq, &workqueues, list) { in workqueue_init()
7910 WARN(init_rescuer(wq), in workqueue_init()
7912 wq->name); in workqueue_init()
8012 struct workqueue_struct *wq; in workqueue_init_topology() local
8029 list_for_each_entry(wq, &workqueues, list) { in workqueue_init_topology()
8031 unbound_wq_update_pwq(wq, cpu); in workqueue_init_topology()
8032 if (wq->flags & WQ_UNBOUND) { in workqueue_init_topology()
8033 mutex_lock(&wq->mutex); in workqueue_init_topology()
8034 wq_update_node_max_active(wq, -1); in workqueue_init_topology()
8035 mutex_unlock(&wq->mutex); in workqueue_init_topology()