89 	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */  member
169 	struct mem_cgroup *memcg;  member
362 	return READ_ONCE(objcg->memcg);  in obj_cgroup_memcg()
494 	struct mem_cgroup *memcg;  in get_mem_cgroup_from_objcg()  local
498 	memcg = obj_cgroup_memcg(objcg);  in get_mem_cgroup_from_objcg()
499 	if (unlikely(!css_tryget(&memcg->css)))  in get_mem_cgroup_from_objcg()
503 	return memcg;  in get_mem_cgroup_from_objcg()
526 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)  in mem_cgroup_is_root()  argument
528 	return (memcg == root_mem_cgroup);  in mem_cgroup_is_root()
537 					 struct mem_cgroup *memcg,  in mem_cgroup_protection()  argument
579 	if (root == memcg)  in mem_cgroup_protection()
582 	*min = READ_ONCE(memcg->memory.emin);  in mem_cgroup_protection()
583 	*low = READ_ONCE(memcg->memory.elow);  in mem_cgroup_protection()
587 				     struct mem_cgroup *memcg);
590 					  struct mem_cgroup *memcg)  in mem_cgroup_unprotected()  argument
597 	return mem_cgroup_disabled() || mem_cgroup_is_root(memcg) ||  in mem_cgroup_unprotected()
598 		memcg == target;  in mem_cgroup_unprotected()
602 					struct mem_cgroup *memcg)  in mem_cgroup_below_low()  argument
604 	if (mem_cgroup_unprotected(target, memcg))  in mem_cgroup_below_low()
607 	return READ_ONCE(memcg->memory.elow) >=  in mem_cgroup_below_low()
608 		page_counter_read(&memcg->memory);  in mem_cgroup_below_low()
612 					struct mem_cgroup *memcg)  in mem_cgroup_below_min()  argument
614 	if (mem_cgroup_unprotected(target, memcg))  in mem_cgroup_below_min()
617 	return READ_ONCE(memcg->memory.emin) >=  in mem_cgroup_below_min()
618 		page_counter_read(&memcg->memory);  in mem_cgroup_below_min()
685 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,  in mem_cgroup_lruvec()  argument
696 	if (!memcg)  in mem_cgroup_lruvec()
697 		memcg = root_mem_cgroup;  in mem_cgroup_lruvec()
699 	mz = memcg->nodeinfo[pgdat->node_id];  in mem_cgroup_lruvec()
720 	struct mem_cgroup *memcg = folio_memcg(folio);  in folio_lruvec()  local
722 	VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio);  in folio_lruvec()
723 	return mem_cgroup_lruvec(memcg, folio_pgdat(folio));  in folio_lruvec()
775 static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg)  in mem_cgroup_tryget()  argument
777 	return !memcg || css_tryget(&memcg->css);  in mem_cgroup_tryget()
780 static inline bool mem_cgroup_tryget_online(struct mem_cgroup *memcg)  in mem_cgroup_tryget_online()  argument
782 	return !memcg || css_tryget_online(&memcg->css);  in mem_cgroup_tryget_online()
785 static inline void mem_cgroup_put(struct mem_cgroup *memcg)  in mem_cgroup_put()  argument
787 	if (memcg)  in mem_cgroup_put()
788 		css_put(&memcg->css);  in mem_cgroup_put()
798 void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
801 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)  in mem_cgroup_id()  argument
806 	return memcg->id.id;  in mem_cgroup_id()
811 static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg)  in mem_cgroup_ino()  argument
813 	return memcg ? cgroup_ino(memcg->css.cgroup) : 0;  in mem_cgroup_ino()
832 	return mz->memcg;  in lruvec_memcg()
841 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)  in parent_mem_cgroup()  argument
843 	return mem_cgroup_from_css(memcg->css.parent);  in parent_mem_cgroup()
846 static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,  in mem_cgroup_is_descendant()  argument
849 	if (root == memcg)  in mem_cgroup_is_descendant()
851 	return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);  in mem_cgroup_is_descendant()
855 				   struct mem_cgroup *memcg)  in mm_match_cgroup()  argument
863 		match = mem_cgroup_is_descendant(task_memcg, memcg);  in mm_match_cgroup()
871 static inline bool mem_cgroup_online(struct mem_cgroup *memcg)  in mem_cgroup_online()  argument
875 	return !!(memcg->css.flags & CSS_ONLINE);  in mem_cgroup_online()
893 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg);
895 unsigned long mem_cgroup_size(struct mem_cgroup *memcg);
897 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg,
900 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);
904 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);
906 void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx,
910 static inline void mod_memcg_state(struct mem_cgroup *memcg,  in mod_memcg_state()  argument
916 	__mod_memcg_state(memcg, idx, val);  in mod_memcg_state()
923 	struct mem_cgroup *memcg;  in mod_memcg_page_state()  local
929 	memcg = folio_memcg(page_folio(page));  in mod_memcg_page_state()
930 	if (memcg)  in mod_memcg_page_state()
931 		mod_memcg_state(memcg, idx, val);  in mod_memcg_page_state()
935 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx);
940 void mem_cgroup_flush_stats(struct mem_cgroup *memcg);
941 void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg);
955 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
958 static inline void count_memcg_events(struct mem_cgroup *memcg,  in count_memcg_events()  argument
965 	__count_memcg_events(memcg, idx, count);  in count_memcg_events()
972 	struct mem_cgroup *memcg = folio_memcg(folio);  in count_memcg_folio_events()  local
974 	if (memcg)  in count_memcg_folio_events()
975 		count_memcg_events(memcg, idx, nr);  in count_memcg_folio_events()
981 	struct mem_cgroup *memcg;  in count_memcg_events_mm()  local
987 	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));  in count_memcg_events_mm()
988 	if (likely(memcg))  in count_memcg_events_mm()
989 		count_memcg_events(memcg, idx, count);  in count_memcg_events_mm()
999 static inline void memcg_memory_event(struct mem_cgroup *memcg,  in memcg_memory_event()  argument
1005 	atomic_long_inc(&memcg->memory_events_local[event]);  in memcg_memory_event()
1007 		cgroup_file_notify(&memcg->events_local_file);  in memcg_memory_event()
1010 		atomic_long_inc(&memcg->memory_events[event]);  in memcg_memory_event()
1012 			cgroup_file_notify(&memcg->swap_events_file);  in memcg_memory_event()
1014 			cgroup_file_notify(&memcg->events_file);  in memcg_memory_event()
1020 	} while ((memcg = parent_mem_cgroup(memcg)) &&  in memcg_memory_event()
1021 		 !mem_cgroup_is_root(memcg));  in memcg_memory_event()
1027 	struct mem_cgroup *memcg;  in memcg_memory_event_mm()  local
1033 	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));  in memcg_memory_event_mm()
1034 	if (likely(memcg))  in memcg_memory_event_mm()
1035 		memcg_memory_event(memcg, event);  in memcg_memory_event_mm()
1045 	struct mem_cgroup *memcg;  in cgroup_id_from_mm()  local
1052 	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));  in cgroup_id_from_mm()
1053 	if (!memcg)  in cgroup_id_from_mm()
1054 		memcg = root_mem_cgroup;  in cgroup_id_from_mm()
1055 	id = cgroup_id(memcg->css.cgroup);  in cgroup_id_from_mm()
1099 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)  in mem_cgroup_is_root()  argument
1109 static inline void memcg_memory_event(struct mem_cgroup *memcg,  in memcg_memory_event()  argument
1120 					 struct mem_cgroup *memcg,  in mem_cgroup_protection()  argument
1128 						   struct mem_cgroup *memcg)  in mem_cgroup_calculate_protection()  argument
1133 					  struct mem_cgroup *memcg)  in mem_cgroup_unprotected()  argument
1138 					struct mem_cgroup *memcg)  in mem_cgroup_below_low()  argument
1144 					struct mem_cgroup *memcg)  in mem_cgroup_below_min()  argument
1183 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,  in mem_cgroup_lruvec()  argument
1200 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)  in parent_mem_cgroup()  argument
1206 		struct mem_cgroup *memcg)  in mm_match_cgroup()  argument
1240 static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg)  in mem_cgroup_tryget()  argument
1245 static inline bool mem_cgroup_tryget_online(struct mem_cgroup *memcg)  in mem_cgroup_tryget_online()  argument
1250 static inline void mem_cgroup_put(struct mem_cgroup *memcg)  in mem_cgroup_put()  argument
1292 static inline void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,  in mem_cgroup_scan_tasks()  argument
1297 static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)  in mem_cgroup_id()  argument
1310 static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg)  in mem_cgroup_ino()  argument
1331 static inline bool mem_cgroup_online(struct mem_cgroup *memcg)  in mem_cgroup_online()  argument
1343 static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)  in mem_cgroup_get_max()  argument
1348 static inline unsigned long mem_cgroup_size(struct mem_cgroup *memcg)  in mem_cgroup_size()  argument
1354 mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)  in mem_cgroup_print_oom_context()  argument
1359 mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)  in mem_cgroup_print_oom_meminfo()  argument
1373 static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)  in mem_cgroup_print_oom_group()  argument
1377 static inline void __mod_memcg_state(struct mem_cgroup *memcg,  in __mod_memcg_state()  argument
1383 static inline void mod_memcg_state(struct mem_cgroup *memcg,  in mod_memcg_state()  argument
1394 static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)  in memcg_page_state()  argument
1411 static inline void mem_cgroup_flush_stats(struct mem_cgroup *memcg)  in mem_cgroup_flush_stats()  argument
1415 static inline void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg)  in mem_cgroup_flush_stats_ratelimited()  argument
1435 static inline void count_memcg_events(struct mem_cgroup *memcg,  in count_memcg_events()  argument
1441 static inline void __count_memcg_events(struct mem_cgroup *memcg,  in __count_memcg_events()  argument
1502 	struct mem_cgroup *memcg;  in parent_lruvec()  local
1504 	memcg = lruvec_memcg(lruvec);  in parent_lruvec()
1505 	if (!memcg)  in parent_lruvec()
1507 	memcg = parent_mem_cgroup(memcg);  in parent_lruvec()
1508 	if (!memcg)  in parent_lruvec()
1510 	return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec));  in parent_lruvec()
1578 	struct mem_cgroup *memcg;  in mem_cgroup_track_foreign_dirty()  local
1583 	memcg = folio_memcg(folio);  in mem_cgroup_track_foreign_dirty()
1584 	if (unlikely(memcg && &memcg->css != wb->memcg_css))  in mem_cgroup_track_foreign_dirty()
1617 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages,
1619 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
1625 static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)  in mem_cgroup_under_socket_pressure()  argument
1629 		return !!memcg->tcpmem_pressure;  in mem_cgroup_under_socket_pressure()
1632 		if (time_before(jiffies, READ_ONCE(memcg->socket_pressure)))  in mem_cgroup_under_socket_pressure()
1634 	} while ((memcg = parent_mem_cgroup(memcg)));  in mem_cgroup_under_socket_pressure()
1638 int alloc_shrinker_info(struct mem_cgroup *memcg);
1639 void free_shrinker_info(struct mem_cgroup *memcg);
1640 void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id);
1641 void reparent_shrinker_deferred(struct mem_cgroup *memcg);
1646 static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)  in mem_cgroup_under_socket_pressure()  argument
1651 static inline void set_shrinker_bit(struct mem_cgroup *memcg,  in set_shrinker_bit()  argument
1717 static inline int memcg_kmem_id(struct mem_cgroup *memcg)  in memcg_kmem_id()  argument
1719 	return memcg ? memcg->kmemcg_id : -1;  in memcg_kmem_id()
1728 	struct mem_cgroup *memcg;  in count_objcg_events()  local
1734 	memcg = obj_cgroup_memcg(objcg);  in count_objcg_events()
1735 	count_memcg_events(memcg, idx, count);  in count_objcg_events()
1780 static inline int memcg_kmem_id(struct mem_cgroup *memcg)  in memcg_kmem_id()  argument
1802 bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg);
1816 static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg)  in mem_cgroup_zswap_writeback_enabled()  argument