1307 					struct per_cpu_pages *pcp,  in free_pcppages_bulk()  argument
1318 	count = min(pcp->count, count);  in free_pcppages_bulk()
1333 			list = &pcp->lists[pindex];  in free_pcppages_bulk()
1349 			pcp->count -= nr_pages;  in free_pcppages_bulk()
2390 int decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp)  in decay_pcp_high()  argument
2395 	high_min = READ_ONCE(pcp->high_min);  in decay_pcp_high()
2396 	batch = READ_ONCE(pcp->batch);  in decay_pcp_high()
2402 	if (pcp->high > high_min) {  in decay_pcp_high()
2403 		pcp->high = max3(pcp->count - (batch << CONFIG_PCP_BATCH_SCALE_MAX),  in decay_pcp_high()
2404 				 pcp->high - (pcp->high >> 3), high_min);  in decay_pcp_high()
2405 		if (pcp->high > high_min)  in decay_pcp_high()
2409 	to_drain = pcp->count - pcp->high;  in decay_pcp_high()
2411 		spin_lock(&pcp->lock);  in decay_pcp_high()
2412 		free_pcppages_bulk(zone, to_drain, pcp, 0);  in decay_pcp_high()
2413 		spin_unlock(&pcp->lock);  in decay_pcp_high()
2426 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)  in drain_zone_pages()  argument
2430 	batch = READ_ONCE(pcp->batch);  in drain_zone_pages()
2431 	to_drain = min(pcp->count, batch);  in drain_zone_pages()
2433 		spin_lock(&pcp->lock);  in drain_zone_pages()
2434 		free_pcppages_bulk(zone, to_drain, pcp, 0);  in drain_zone_pages()
2435 		spin_unlock(&pcp->lock);  in drain_zone_pages()
2445 	struct per_cpu_pages *pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in drain_pages_zone()  local
2449 		spin_lock(&pcp->lock);  in drain_pages_zone()
2450 		count = pcp->count;  in drain_pages_zone()
2453 				pcp->batch << CONFIG_PCP_BATCH_SCALE_MAX);  in drain_pages_zone()
2455 			free_pcppages_bulk(zone, to_drain, pcp, 0);  in drain_pages_zone()
2458 		spin_unlock(&pcp->lock);  in drain_pages_zone()
2525 		struct per_cpu_pages *pcp;  in __drain_all_pages()  local
2536 			pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in __drain_all_pages()
2537 			if (pcp->count)  in __drain_all_pages()
2541 				pcp = per_cpu_ptr(z->per_cpu_pageset, cpu);  in __drain_all_pages()
2542 				if (pcp->count) {  in __drain_all_pages()
2575 static int nr_pcp_free(struct per_cpu_pages *pcp, int batch, int high, bool free_high)  in nr_pcp_free()  argument
2581 		return min(pcp->count, batch << CONFIG_PCP_BATCH_SCALE_MAX);  in nr_pcp_free()
2595 	batch = clamp_t(int, pcp->free_count, min_nr_free, max_nr_free);  in nr_pcp_free()
2600 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone,  in nr_pcp_high()  argument
2605 	high_min = READ_ONCE(pcp->high_min);  in nr_pcp_high()
2606 	high_max = READ_ONCE(pcp->high_max);  in nr_pcp_high()
2607 	high = pcp->high = clamp(pcp->high, high_min, high_max);  in nr_pcp_high()
2613 		pcp->high = max(high - (batch << CONFIG_PCP_BATCH_SCALE_MAX),  in nr_pcp_high()
2623 		int free_count = max_t(int, pcp->free_count, batch);  in nr_pcp_high()
2625 		pcp->high = max(high - free_count, high_min);  in nr_pcp_high()
2626 		return min(batch << 2, pcp->high);  in nr_pcp_high()
2633 		int free_count = max_t(int, pcp->free_count, batch);  in nr_pcp_high()
2635 		pcp->high = max(high - free_count, high_min);  in nr_pcp_high()
2636 		high = max(pcp->count, high_min);  in nr_pcp_high()
2637 	} else if (pcp->count >= high) {  in nr_pcp_high()
2638 		int need_high = pcp->free_count + batch;  in nr_pcp_high()
2641 		if (pcp->high < need_high)  in nr_pcp_high()
2642 			pcp->high = clamp(need_high, high_min, high_max);  in nr_pcp_high()
2649 		struct per_cpu_pages *pcp, struct page *page, int migratetype,  in free_frozen_page_commit()  argument
2661 	pcp->alloc_factor >>= 1;  in free_frozen_page_commit()
2664 	list_add(&page->pcp_list, &pcp->lists[pindex]);  in free_frozen_page_commit()
2665 	pcp->count += 1 << order;  in free_frozen_page_commit()
2667 	batch = READ_ONCE(pcp->batch);  in free_frozen_page_commit()
2675 		free_high = (pcp->free_count >= batch &&  in free_frozen_page_commit()
2676 			     (pcp->flags & PCPF_PREV_FREE_HIGH_ORDER) &&  in free_frozen_page_commit()
2677 			     (!(pcp->flags & PCPF_FREE_HIGH_BATCH) ||  in free_frozen_page_commit()
2678 			      pcp->count >= READ_ONCE(batch)));  in free_frozen_page_commit()
2679 		pcp->flags |= PCPF_PREV_FREE_HIGH_ORDER;  in free_frozen_page_commit()
2680 	} else if (pcp->flags & PCPF_PREV_FREE_HIGH_ORDER) {  in free_frozen_page_commit()
2681 		pcp->flags &= ~PCPF_PREV_FREE_HIGH_ORDER;  in free_frozen_page_commit()
2683 	if (pcp->free_count < (batch << CONFIG_PCP_BATCH_SCALE_MAX))  in free_frozen_page_commit()
2684 		pcp->free_count += (1 << order);  in free_frozen_page_commit()
2693 	high = nr_pcp_high(pcp, zone, batch, free_high);  in free_frozen_page_commit()
2694 	if (pcp->count >= high) {  in free_frozen_page_commit()
2695 		free_pcppages_bulk(zone, nr_pcp_free(pcp, batch, high, free_high),  in free_frozen_page_commit()
2696 				   pcp, pindex);  in free_frozen_page_commit()
2711 	struct per_cpu_pages *pcp;  in __free_frozen_pages()  local
2747 	pcp = pcp_spin_trylock(zone->per_cpu_pageset);  in __free_frozen_pages()
2748 	if (pcp) {  in __free_frozen_pages()
2749 		free_frozen_page_commit(zone, pcp, page, migratetype, order, fpi_flags);  in __free_frozen_pages()
2750 		pcp_spin_unlock(pcp);  in __free_frozen_pages()
2768 	struct per_cpu_pages *pcp = NULL;  in free_unref_folios()  local
2809 			if (pcp) {  in free_unref_folios()
2810 				pcp_spin_unlock(pcp);  in free_unref_folios()
2813 				pcp = NULL;  in free_unref_folios()
2831 			pcp = pcp_spin_trylock(zone->per_cpu_pageset);  in free_unref_folios()
2832 			if (unlikely(!pcp)) {  in free_unref_folios()
2849 		free_frozen_page_commit(zone, pcp, &folio->page, migratetype,  in free_unref_folios()
2853 	if (pcp) {  in free_unref_folios()
2854 		pcp_spin_unlock(pcp);  in free_unref_folios()
3017 static int nr_pcp_alloc(struct per_cpu_pages *pcp, struct zone *zone, int order)  in nr_pcp_alloc()  argument
3022 	base_batch = READ_ONCE(pcp->batch);  in nr_pcp_alloc()
3023 	high_min = READ_ONCE(pcp->high_min);  in nr_pcp_alloc()
3024 	high_max = READ_ONCE(pcp->high_max);  in nr_pcp_alloc()
3025 	high = pcp->high = clamp(pcp->high, high_min, high_max);  in nr_pcp_alloc()
3034 		batch = (base_batch << pcp->alloc_factor);  in nr_pcp_alloc()
3041 		high = pcp->high = min(high + batch, high_max);  in nr_pcp_alloc()
3044 		max_nr_alloc = max(high - pcp->count - base_batch, base_batch);  in nr_pcp_alloc()
3050 		    pcp->alloc_factor < CONFIG_PCP_BATCH_SCALE_MAX)  in nr_pcp_alloc()
3051 			pcp->alloc_factor++;  in nr_pcp_alloc()
3072 			struct per_cpu_pages *pcp,  in __rmqueue_pcplist()  argument
3079 			int batch = nr_pcp_alloc(pcp, zone, order);  in __rmqueue_pcplist()
3086 			pcp->count += alloced << order;  in __rmqueue_pcplist()
3093 		pcp->count -= 1 << order;  in __rmqueue_pcplist()
3104 	struct per_cpu_pages *pcp;  in rmqueue_pcplist()  local
3111 	pcp = pcp_spin_trylock(zone->per_cpu_pageset);  in rmqueue_pcplist()
3112 	if (!pcp) {  in rmqueue_pcplist()
3122 	pcp->free_count >>= 1;  in rmqueue_pcplist()
3123 	list = &pcp->lists[order_to_pindex(migratetype, order)];  in rmqueue_pcplist()
3124 	page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list);  in rmqueue_pcplist()
3125 	pcp_spin_unlock(pcp);  in rmqueue_pcplist()
4792 	struct per_cpu_pages *pcp;  in alloc_pages_bulk_noprof()  local
4884 	pcp = pcp_spin_trylock(zone->per_cpu_pageset);  in alloc_pages_bulk_noprof()
4885 	if (!pcp)  in alloc_pages_bulk_noprof()
4889 	pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)];  in alloc_pages_bulk_noprof()
4899 								pcp, pcp_list);  in alloc_pages_bulk_noprof()
4903 				pcp_spin_unlock(pcp);  in alloc_pages_bulk_noprof()
4915 	pcp_spin_unlock(pcp);  in alloc_pages_bulk_noprof()
5500 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats);
5747 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high_min,  in pageset_update()  argument
5750 	WRITE_ONCE(pcp->batch, batch);  in pageset_update()
5751 	WRITE_ONCE(pcp->high_min, high_min);  in pageset_update()
5752 	WRITE_ONCE(pcp->high_max, high_max);  in pageset_update()
5755 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats)  in per_cpu_pages_init()  argument
5759 	memset(pcp, 0, sizeof(*pcp));  in per_cpu_pages_init()
5762 	spin_lock_init(&pcp->lock);  in per_cpu_pages_init()
5764 		INIT_LIST_HEAD(&pcp->lists[pindex]);  in per_cpu_pages_init()
5772 	pcp->high_min = BOOT_PAGESET_HIGH;  in per_cpu_pages_init()
5773 	pcp->high_max = BOOT_PAGESET_HIGH;  in per_cpu_pages_init()
5774 	pcp->batch = BOOT_PAGESET_BATCH;  in per_cpu_pages_init()
5775 	pcp->free_count = 0;  in per_cpu_pages_init()
5781 	struct per_cpu_pages *pcp;  in __zone_set_pageset_high_and_batch()  local
5785 		pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in __zone_set_pageset_high_and_batch()
5786 		pageset_update(pcp, high_min, high_max, batch);  in __zone_set_pageset_high_and_batch()
5836 		struct per_cpu_pages *pcp;  in setup_zone_pageset()  local
5839 		pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in setup_zone_pageset()
5841 		per_cpu_pages_init(pcp, pzstats);  in setup_zone_pageset()
5860 	struct per_cpu_pages *pcp;  in zone_pcp_update_cacheinfo()  local
5863 	pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in zone_pcp_update_cacheinfo()
5872 	spin_lock(&pcp->lock);  in zone_pcp_update_cacheinfo()
5873 	if ((cci->per_cpu_data_slice_size >> PAGE_SHIFT) > 3 * pcp->batch)  in zone_pcp_update_cacheinfo()
5874 		pcp->flags |= PCPF_FREE_HIGH_BATCH;  in zone_pcp_update_cacheinfo()
5876 		pcp->flags &= ~PCPF_FREE_HIGH_BATCH;  in zone_pcp_update_cacheinfo()
5877 	spin_unlock(&pcp->lock);  in zone_pcp_update_cacheinfo()