527 static void show_pwq(struct pool_workqueue *pwq);
598 #define for_each_pwq(pwq, wq)						\  argument
599 	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node,		\
792 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,  in set_work_pwq()  argument
795 	set_work_data(work, (unsigned long)pwq | WORK_STRUCT_PENDING |  in set_work_pwq()
1468 	struct pool_workqueue *pwq = worker->current_pwq;  in wq_worker_tick()  local
1471 	if (!pwq)  in wq_worker_tick()
1474 	pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;  in wq_worker_tick()
1500 	pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;  in wq_worker_tick()
1503 		pwq->stats[PWQ_STAT_CM_WAKEUP]++;  in wq_worker_tick()
1623 static void get_pwq(struct pool_workqueue *pwq)  in get_pwq()  argument
1625 	lockdep_assert_held(&pwq->pool->lock);  in get_pwq()
1626 	WARN_ON_ONCE(pwq->refcnt <= 0);  in get_pwq()
1627 	pwq->refcnt++;  in get_pwq()
1637 static void put_pwq(struct pool_workqueue *pwq)  in put_pwq()  argument
1639 	lockdep_assert_held(&pwq->pool->lock);  in put_pwq()
1640 	if (likely(--pwq->refcnt))  in put_pwq()
1646 	kthread_queue_work(pwq_release_worker, &pwq->release_work);  in put_pwq()
1655 static void put_pwq_unlocked(struct pool_workqueue *pwq)  in put_pwq_unlocked()  argument
1657 	if (pwq) {  in put_pwq_unlocked()
1662 		raw_spin_lock_irq(&pwq->pool->lock);  in put_pwq_unlocked()
1663 		put_pwq(pwq);  in put_pwq_unlocked()
1664 		raw_spin_unlock_irq(&pwq->pool->lock);  in put_pwq_unlocked()
1668 static bool pwq_is_empty(struct pool_workqueue *pwq)  in pwq_is_empty()  argument
1670 	return !pwq->nr_active && list_empty(&pwq->inactive_works);  in pwq_is_empty()
1673 static void __pwq_activate_work(struct pool_workqueue *pwq,  in __pwq_activate_work()  argument
1680 	if (list_empty(&pwq->pool->worklist))  in __pwq_activate_work()
1681 		pwq->pool->watchdog_ts = jiffies;  in __pwq_activate_work()
1682 	move_linked_works(work, &pwq->pool->worklist, NULL);  in __pwq_activate_work()
1710 static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)  in pwq_tryinc_nr_active()  argument
1712 	struct workqueue_struct *wq = pwq->wq;  in pwq_tryinc_nr_active()
1713 	struct worker_pool *pool = pwq->pool;  in pwq_tryinc_nr_active()
1721 		obtained = pwq->nr_active < READ_ONCE(wq->max_active);  in pwq_tryinc_nr_active()
1725 	if (unlikely(pwq->plugged))  in pwq_tryinc_nr_active()
1737 	if (!list_empty(&pwq->pending_node) && likely(!fill))  in pwq_tryinc_nr_active()
1753 	if (list_empty(&pwq->pending_node))  in pwq_tryinc_nr_active()
1754 		list_add_tail(&pwq->pending_node, &nna->pending_pwqs);  in pwq_tryinc_nr_active()
1767 		list_del_init(&pwq->pending_node);  in pwq_tryinc_nr_active()
1773 		pwq->nr_active++;  in pwq_tryinc_nr_active()
1788 static bool pwq_activate_first_inactive(struct pool_workqueue *pwq, bool fill)  in pwq_activate_first_inactive()  argument
1791 		list_first_entry_or_null(&pwq->inactive_works,  in pwq_activate_first_inactive()
1794 	if (work && pwq_tryinc_nr_active(pwq, fill)) {  in pwq_activate_first_inactive()
1795 		__pwq_activate_work(pwq, work);  in pwq_activate_first_inactive()
1826 	struct pool_workqueue *pwq;  in unplug_oldest_pwq()  local
1831 	pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue,  in unplug_oldest_pwq()
1833 	raw_spin_lock_irq(&pwq->pool->lock);  in unplug_oldest_pwq()
1834 	if (pwq->plugged) {  in unplug_oldest_pwq()
1835 		pwq->plugged = false;  in unplug_oldest_pwq()
1836 		if (pwq_activate_first_inactive(pwq, true))  in unplug_oldest_pwq()
1837 			kick_pool(pwq->pool);  in unplug_oldest_pwq()
1839 	raw_spin_unlock_irq(&pwq->pool->lock);  in unplug_oldest_pwq()
1854 	struct pool_workqueue *pwq;  in node_activate_pending_pwq()  local
1861 	pwq = list_first_entry_or_null(&nna->pending_pwqs,  in node_activate_pending_pwq()
1863 	if (!pwq)  in node_activate_pending_pwq()
1872 	if (pwq->pool != locked_pool) {  in node_activate_pending_pwq()
1874 		locked_pool = pwq->pool;  in node_activate_pending_pwq()
1887 	work = list_first_entry_or_null(&pwq->inactive_works,  in node_activate_pending_pwq()
1890 		list_del_init(&pwq->pending_node);  in node_activate_pending_pwq()
1902 		pwq->nr_active++;  in node_activate_pending_pwq()
1903 		__pwq_activate_work(pwq, work);  in node_activate_pending_pwq()
1905 		if (list_empty(&pwq->inactive_works))  in node_activate_pending_pwq()
1906 			list_del_init(&pwq->pending_node);  in node_activate_pending_pwq()
1908 			list_move_tail(&pwq->pending_node, &nna->pending_pwqs);  in node_activate_pending_pwq()
1911 		if (pwq->pool != caller_pool)  in node_activate_pending_pwq()
1912 			kick_pool(pwq->pool);  in node_activate_pending_pwq()
1930 static void pwq_dec_nr_active(struct pool_workqueue *pwq)  in pwq_dec_nr_active()  argument
1932 	struct worker_pool *pool = pwq->pool;  in pwq_dec_nr_active()
1933 	struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);  in pwq_dec_nr_active()
1941 	pwq->nr_active--;  in pwq_dec_nr_active()
1948 		pwq_activate_first_inactive(pwq, false);  in pwq_dec_nr_active()
1989 static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)  in pwq_dec_nr_in_flight()  argument
1994 		pwq_dec_nr_active(pwq);  in pwq_dec_nr_in_flight()
1996 	pwq->nr_in_flight[color]--;  in pwq_dec_nr_in_flight()
1999 	if (likely(pwq->flush_color != color))  in pwq_dec_nr_in_flight()
2003 	if (pwq->nr_in_flight[color])  in pwq_dec_nr_in_flight()
2007 	pwq->flush_color = -1;  in pwq_dec_nr_in_flight()
2013 	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))  in pwq_dec_nr_in_flight()
2014 		complete(&pwq->wq->first_flusher->done);  in pwq_dec_nr_in_flight()
2016 	put_pwq(pwq);  in pwq_dec_nr_in_flight()
2051 	struct pool_workqueue *pwq;  in try_to_grab_pending()  local
2090 	pwq = get_work_pwq(work);  in try_to_grab_pending()
2091 	if (pwq && pwq->pool == pool) {  in try_to_grab_pending()
2111 			move_linked_works(work, &pwq->pool->worklist, NULL);  in try_to_grab_pending()
2123 		pwq_dec_nr_in_flight(pwq, work_data);  in try_to_grab_pending()
2177 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,  in insert_work()  argument
2186 	set_work_pwq(work, pwq, extra_flags);  in insert_work()
2188 	get_pwq(pwq);  in insert_work()
2238 	struct pool_workqueue *pwq;  in __queue_work()  local
2271 	pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu));  in __queue_work()
2272 	pool = pwq->pool;  in __queue_work()
2292 			pwq = worker->current_pwq;  in __queue_work()
2293 			pool = pwq->pool;  in __queue_work()
2311 	if (unlikely(!pwq->refcnt)) {  in __queue_work()
2323 	trace_workqueue_queue_work(req_cpu, pwq, work);  in __queue_work()
2328 	pwq->nr_in_flight[pwq->work_color]++;  in __queue_work()
2329 	work_flags = work_color_to_flags(pwq->work_color);  in __queue_work()
2336 	if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq, false)) {  in __queue_work()
2341 		insert_work(pwq, work, &pool->worklist, work_flags);  in __queue_work()
2345 		insert_work(pwq, work, &pwq->inactive_works, work_flags);  in __queue_work()
2987 	struct pool_workqueue *pwq = get_work_pwq(work);  in send_mayday()  local
2988 	struct workqueue_struct *wq = pwq->wq;  in send_mayday()
2996 	if (list_empty(&pwq->mayday_node)) {  in send_mayday()
3002 		get_pwq(pwq);  in send_mayday()
3003 		list_add_tail(&pwq->mayday_node, &wq->maydays);  in send_mayday()
3005 		pwq->stats[PWQ_STAT_MAYDAY]++;  in send_mayday()
3141 	struct pool_workqueue *pwq = get_work_pwq(work);  in process_one_work()  local
3167 	worker->current_pwq = pwq;  in process_one_work()
3177 	strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);  in process_one_work()
3187 	if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))  in process_one_work()
3206 	pwq->stats[PWQ_STAT_STARTED]++;  in process_one_work()
3213 		lock_map_acquire(pwq->wq->lockdep_map);  in process_one_work()
3244 	pwq->stats[PWQ_STAT_COMPLETED]++;  in process_one_work()
3247 		lock_map_release(pwq->wq->lockdep_map);  in process_one_work()
3293 	pwq_dec_nr_in_flight(pwq, work_data);  in process_one_work()
3470 		struct pool_workqueue *pwq = list_first_entry(&wq->maydays,  in rescuer_thread()  local
3472 		struct worker_pool *pool = pwq->pool;  in rescuer_thread()
3476 		list_del_init(&pwq->mayday_node);  in rescuer_thread()
3490 			if (get_work_pwq(work) == pwq &&  in rescuer_thread()
3492 				pwq->stats[PWQ_STAT_RESCUED]++;  in rescuer_thread()
3507 			if (pwq->nr_active && need_to_create_worker(pool)) {  in rescuer_thread()
3513 				if (wq->rescuer && list_empty(&pwq->mayday_node)) {  in rescuer_thread()
3514 					get_pwq(pwq);  in rescuer_thread()
3515 					list_add_tail(&pwq->mayday_node, &wq->maydays);  in rescuer_thread()
3535 		put_pwq_unlocked(pwq);  in rescuer_thread()
3760 static void insert_wq_barrier(struct pool_workqueue *pwq,  in insert_wq_barrier()  argument
3780 			      (pwq->wq->flags & WQ_BH) ? &bh_key : &thr_key);  in insert_wq_barrier()
3807 	pwq->nr_in_flight[work_color]++;  in insert_wq_barrier()
3810 	insert_work(pwq, &barr->work, head, work_flags);  in insert_wq_barrier()
3848 	struct pool_workqueue *pwq;  in flush_workqueue_prep_pwqs()  local
3864 	for_each_pwq(pwq, wq) {  in flush_workqueue_prep_pwqs()
3865 		if (current_pool != pwq->pool) {  in flush_workqueue_prep_pwqs()
3868 			current_pool = pwq->pool;  in flush_workqueue_prep_pwqs()
3873 			WARN_ON_ONCE(pwq->flush_color != -1);  in flush_workqueue_prep_pwqs()
3875 			if (pwq->nr_in_flight[flush_color]) {  in flush_workqueue_prep_pwqs()
3876 				pwq->flush_color = flush_color;  in flush_workqueue_prep_pwqs()
3883 			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));  in flush_workqueue_prep_pwqs()
3884 			pwq->work_color = work_color;  in flush_workqueue_prep_pwqs()
4101 	struct pool_workqueue *pwq;  in drain_workqueue()  local
4117 	for_each_pwq(pwq, wq) {  in drain_workqueue()
4120 		raw_spin_lock_irq(&pwq->pool->lock);  in drain_workqueue()
4121 		drained = pwq_is_empty(pwq);  in drain_workqueue()
4122 		raw_spin_unlock_irq(&pwq->pool->lock);  in drain_workqueue()
4147 	struct pool_workqueue *pwq;  in start_flush_work()  local
4159 	pwq = get_work_pwq(work);  in start_flush_work()
4160 	if (pwq) {  in start_flush_work()
4161 		if (unlikely(pwq->pool != pool))  in start_flush_work()
4167 		pwq = worker->current_pwq;  in start_flush_work()
4170 	wq = pwq->wq;  in start_flush_work()
4173 	insert_wq_barrier(pwq, barr, work, worker);  in start_flush_work()
5066 	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,  in pwq_release_workfn()  local
5068 	struct workqueue_struct *wq = pwq->wq;  in pwq_release_workfn()
5069 	struct worker_pool *pool = pwq->pool;  in pwq_release_workfn()
5076 	if (!list_empty(&pwq->pwqs_node)) {  in pwq_release_workfn()
5078 		list_del_rcu(&pwq->pwqs_node);  in pwq_release_workfn()
5096 	if (!list_empty(&pwq->pending_node)) {  in pwq_release_workfn()
5098 			wq_node_nr_active(pwq->wq, pwq->pool->node);  in pwq_release_workfn()
5101 		list_del_init(&pwq->pending_node);  in pwq_release_workfn()
5105 	kfree_rcu(pwq, rcu);  in pwq_release_workfn()
5118 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,  in init_pwq()  argument
5121 	BUG_ON((unsigned long)pwq & ~WORK_STRUCT_PWQ_MASK);  in init_pwq()
5123 	memset(pwq, 0, sizeof(*pwq));  in init_pwq()
5125 	pwq->pool = pool;  in init_pwq()
5126 	pwq->wq = wq;  in init_pwq()
5127 	pwq->flush_color = -1;  in init_pwq()
5128 	pwq->refcnt = 1;  in init_pwq()
5129 	INIT_LIST_HEAD(&pwq->inactive_works);  in init_pwq()
5130 	INIT_LIST_HEAD(&pwq->pending_node);  in init_pwq()
5131 	INIT_LIST_HEAD(&pwq->pwqs_node);  in init_pwq()
5132 	INIT_LIST_HEAD(&pwq->mayday_node);  in init_pwq()
5133 	kthread_init_work(&pwq->release_work, pwq_release_workfn);  in init_pwq()
5137 static void link_pwq(struct pool_workqueue *pwq)  in link_pwq()  argument
5139 	struct workqueue_struct *wq = pwq->wq;  in link_pwq()
5144 	if (!list_empty(&pwq->pwqs_node))  in link_pwq()
5148 	pwq->work_color = wq->work_color;  in link_pwq()
5151 	list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs);  in link_pwq()
5159 	struct pool_workqueue *pwq;  in alloc_unbound_pwq()  local
5167 	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);  in alloc_unbound_pwq()
5168 	if (!pwq) {  in alloc_unbound_pwq()
5173 	init_pwq(pwq, wq, pool);  in alloc_unbound_pwq()
5174 	return pwq;  in alloc_unbound_pwq()
5217 					int cpu, struct pool_workqueue *pwq)  in install_unbound_pwq()  argument
5226 	link_pwq(pwq);  in install_unbound_pwq()
5229 	rcu_assign_pointer(*slot, pwq);  in install_unbound_pwq()
5423 	struct pool_workqueue *old_pwq = NULL, *pwq;  in unbound_wq_update_pwq()  local
5447 	pwq = alloc_unbound_pwq(wq, target_attrs);  in unbound_wq_update_pwq()
5448 	if (!pwq) {  in unbound_wq_update_pwq()
5456 	old_pwq = install_unbound_pwq(wq, cpu, pwq);  in unbound_wq_update_pwq()
5461 	pwq = unbound_pwq(wq, -1);  in unbound_wq_update_pwq()
5462 	raw_spin_lock_irq(&pwq->pool->lock);  in unbound_wq_update_pwq()
5463 	get_pwq(pwq);  in unbound_wq_update_pwq()
5464 	raw_spin_unlock_irq(&pwq->pool->lock);  in unbound_wq_update_pwq()
5465 	old_pwq = install_unbound_pwq(wq, cpu, pwq);  in unbound_wq_update_pwq()
5529 			struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);  in alloc_and_link_pwqs()  local
5531 			if (pwq)  in alloc_and_link_pwqs()
5532 				kmem_cache_free(pwq_cache, pwq);  in alloc_and_link_pwqs()
5640 		struct pool_workqueue *pwq;  in wq_adjust_max_active()  local
5643 		for_each_pwq(pwq, wq) {  in wq_adjust_max_active()
5647 			raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);  in wq_adjust_max_active()
5648 			if (pwq_activate_first_inactive(pwq, true)) {  in wq_adjust_max_active()
5650 				kick_pool(pwq->pool);  in wq_adjust_max_active()
5652 			raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);  in wq_adjust_max_active()
5819 static bool pwq_busy(struct pool_workqueue *pwq)  in pwq_busy()  argument
5824 		if (pwq->nr_in_flight[i])  in pwq_busy()
5827 	if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))  in pwq_busy()
5829 	if (!pwq_is_empty(pwq))  in pwq_busy()
5843 	struct pool_workqueue *pwq;  in destroy_workqueue()  local
5880 	for_each_pwq(pwq, wq) {  in destroy_workqueue()
5881 		raw_spin_lock_irq(&pwq->pool->lock);  in destroy_workqueue()
5882 		if (WARN_ON(pwq_busy(pwq))) {  in destroy_workqueue()
5885 			show_pwq(pwq);  in destroy_workqueue()
5886 			raw_spin_unlock_irq(&pwq->pool->lock);  in destroy_workqueue()
5892 		raw_spin_unlock_irq(&pwq->pool->lock);  in destroy_workqueue()
6035 	struct pool_workqueue *pwq;  in workqueue_congested()  local
6044 	pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);  in workqueue_congested()
6045 	ret = !list_empty(&pwq->inactive_works);  in workqueue_congested()
6129 	struct pool_workqueue *pwq = NULL;  in print_worker_info()  local
6147 	copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));  in print_worker_info()
6148 	copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));  in print_worker_info()
6229 static void show_pwq(struct pool_workqueue *pwq)  in show_pwq()  argument
6232 	struct worker_pool *pool = pwq->pool;  in show_pwq()
6242 		pwq->nr_active, pwq->refcnt,  in show_pwq()
6243 		!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");  in show_pwq()
6246 		if (worker->current_pwq == pwq) {  in show_pwq()
6256 			if (worker->current_pwq != pwq)  in show_pwq()
6271 		if (get_work_pwq(work) == pwq) {  in show_pwq()
6281 			if (get_work_pwq(work) != pwq)  in show_pwq()
6291 	if (!list_empty(&pwq->inactive_works)) {  in show_pwq()
6295 		list_for_each_entry(work, &pwq->inactive_works, entry) {  in show_pwq()
6310 	struct pool_workqueue *pwq;  in show_one_workqueue()  local
6314 	for_each_pwq(pwq, wq) {  in show_one_workqueue()
6315 		if (!pwq_is_empty(pwq)) {  in show_one_workqueue()
6325 	for_each_pwq(pwq, wq) {  in show_one_workqueue()
6326 		raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);  in show_one_workqueue()
6327 		if (!pwq_is_empty(pwq)) {  in show_one_workqueue()
6334 			show_pwq(pwq);  in show_one_workqueue()
6337 		raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);  in show_one_workqueue()
6834 	struct pool_workqueue *pwq;  in freeze_workqueues_busy()  local
6848 		for_each_pwq(pwq, wq) {  in freeze_workqueues_busy()
6849 			WARN_ON_ONCE(pwq->nr_active < 0);  in freeze_workqueues_busy()
6850 			if (pwq->nr_active) {  in freeze_workqueues_busy()