1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_PAGE_COUNTER_H 3 #define _LINUX_PAGE_COUNTER_H 4 5 #include <linux/atomic.h> 6 #include <linux/cache.h> 7 #include <linux/limits.h> 8 #include <asm/page.h> 9 10 struct page_counter { 11 /* 12 * Make sure 'usage' does not share cacheline with any other field in 13 * v2. The memcg->memory.usage is a hot member of struct mem_cgroup. 14 */ 15 atomic_long_t usage; 16 unsigned long failcnt; /* v1-only field */ 17 18 CACHELINE_PADDING(_pad1_); 19 20 /* effective memory.min and memory.min usage tracking */ 21 unsigned long emin; 22 atomic_long_t min_usage; 23 atomic_long_t children_min_usage; 24 25 /* effective memory.low and memory.low usage tracking */ 26 unsigned long elow; 27 atomic_long_t low_usage; 28 atomic_long_t children_low_usage; 29 30 unsigned long watermark; 31 /* Latest cg2 reset watermark */ 32 unsigned long local_watermark; 33 34 /* Keep all the read most fields in a separete cacheline. */ 35 CACHELINE_PADDING(_pad2_); 36 37 bool protection_support; 38 bool track_failcnt; 39 unsigned long min; 40 unsigned long low; 41 unsigned long high; 42 unsigned long max; 43 struct page_counter *parent; 44 } ____cacheline_internodealigned_in_smp; 45 46 #if BITS_PER_LONG == 32 47 #define PAGE_COUNTER_MAX LONG_MAX 48 #else 49 #define PAGE_COUNTER_MAX (LONG_MAX / PAGE_SIZE) 50 #endif 51 52 /* 53 * Protection is supported only for the first counter (with id 0). 54 */ 55 static inline void page_counter_init(struct page_counter *counter, 56 struct page_counter *parent, 57 bool protection_support) 58 { 59 counter->usage = (atomic_long_t)ATOMIC_LONG_INIT(0); 60 counter->max = PAGE_COUNTER_MAX; 61 counter->parent = parent; 62 counter->protection_support = protection_support; 63 counter->track_failcnt = false; 64 } 65 66 static inline unsigned long page_counter_read(struct page_counter *counter) 67 { 68 return atomic_long_read(&counter->usage); 69 } 70 71 void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages); 72 void page_counter_charge(struct page_counter *counter, unsigned long nr_pages); 73 bool page_counter_try_charge(struct page_counter *counter, 74 unsigned long nr_pages, 75 struct page_counter **fail); 76 void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages); 77 void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages); 78 void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages); 79 80 static inline void page_counter_set_high(struct page_counter *counter, 81 unsigned long nr_pages) 82 { 83 WRITE_ONCE(counter->high, nr_pages); 84 } 85 86 int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages); 87 int page_counter_memparse(const char *buf, const char *max, 88 unsigned long *nr_pages); 89 90 static inline void page_counter_reset_watermark(struct page_counter *counter) 91 { 92 unsigned long usage = page_counter_read(counter); 93 94 /* 95 * Update local_watermark first, so it's always <= watermark 96 * (modulo CPU/compiler re-ordering) 97 */ 98 counter->local_watermark = usage; 99 counter->watermark = usage; 100 } 101 102 #if IS_ENABLED(CONFIG_MEMCG) || IS_ENABLED(CONFIG_CGROUP_DMEM) 103 void page_counter_calculate_protection(struct page_counter *root, 104 struct page_counter *counter, 105 bool recursive_protection); 106 #else 107 static inline void page_counter_calculate_protection(struct page_counter *root, 108 struct page_counter *counter, 109 bool recursive_protection) {} 110 #endif 111 112 #endif /* _LINUX_PAGE_COUNTER_H */ 113