1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_SCHED_MM_H 3 #define _LINUX_SCHED_MM_H 4 5 #include <linux/kernel.h> 6 #include <linux/atomic.h> 7 #include <linux/sched.h> 8 #include <linux/mm_types.h> 9 #include <linux/gfp.h> 10 11 /* 12 * Routines for handling mm_structs 13 */ 14 extern struct mm_struct * mm_alloc(void); 15 16 /** 17 * mmgrab() - Pin a &struct mm_struct. 18 * @mm: The &struct mm_struct to pin. 19 * 20 * Make sure that @mm will not get freed even after the owning task 21 * exits. This doesn't guarantee that the associated address space 22 * will still exist later on and mmget_not_zero() has to be used before 23 * accessing it. 24 * 25 * This is a preferred way to to pin @mm for a longer/unbounded amount 26 * of time. 27 * 28 * Use mmdrop() to release the reference acquired by mmgrab(). 29 * 30 * See also <Documentation/vm/active_mm.txt> for an in-depth explanation 31 * of &mm_struct.mm_count vs &mm_struct.mm_users. 32 */ 33 static inline void mmgrab(struct mm_struct *mm) 34 { 35 atomic_inc(&mm->mm_count); 36 } 37 38 /* mmdrop drops the mm and the page tables */ 39 extern void __mmdrop(struct mm_struct *); 40 static inline void mmdrop(struct mm_struct *mm) 41 { 42 if (unlikely(atomic_dec_and_test(&mm->mm_count))) 43 __mmdrop(mm); 44 } 45 46 static inline void mmdrop_async_fn(struct work_struct *work) 47 { 48 struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work); 49 __mmdrop(mm); 50 } 51 52 static inline void mmdrop_async(struct mm_struct *mm) 53 { 54 if (unlikely(atomic_dec_and_test(&mm->mm_count))) { 55 INIT_WORK(&mm->async_put_work, mmdrop_async_fn); 56 schedule_work(&mm->async_put_work); 57 } 58 } 59 60 /** 61 * mmget() - Pin the address space associated with a &struct mm_struct. 62 * @mm: The address space to pin. 63 * 64 * Make sure that the address space of the given &struct mm_struct doesn't 65 * go away. This does not protect against parts of the address space being 66 * modified or freed, however. 67 * 68 * Never use this function to pin this address space for an 69 * unbounded/indefinite amount of time. 70 * 71 * Use mmput() to release the reference acquired by mmget(). 72 * 73 * See also <Documentation/vm/active_mm.txt> for an in-depth explanation 74 * of &mm_struct.mm_count vs &mm_struct.mm_users. 75 */ 76 static inline void mmget(struct mm_struct *mm) 77 { 78 atomic_inc(&mm->mm_users); 79 } 80 81 static inline bool mmget_not_zero(struct mm_struct *mm) 82 { 83 return atomic_inc_not_zero(&mm->mm_users); 84 } 85 86 /* mmput gets rid of the mappings and all user-space */ 87 extern void mmput(struct mm_struct *); 88 #ifdef CONFIG_MMU 89 /* same as above but performs the slow path from the async context. Can 90 * be called from the atomic context as well 91 */ 92 void mmput_async(struct mm_struct *); 93 #endif 94 95 /* Grab a reference to a task's mm, if it is not already going away */ 96 extern struct mm_struct *get_task_mm(struct task_struct *task); 97 /* 98 * Grab a reference to a task's mm, if it is not already going away 99 * and ptrace_may_access with the mode parameter passed to it 100 * succeeds. 101 */ 102 extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); 103 /* Remove the current tasks stale references to the old mm_struct */ 104 extern void mm_release(struct task_struct *, struct mm_struct *); 105 106 #ifdef CONFIG_MEMCG 107 extern void mm_update_next_owner(struct mm_struct *mm); 108 #else 109 static inline void mm_update_next_owner(struct mm_struct *mm) 110 { 111 } 112 #endif /* CONFIG_MEMCG */ 113 114 #ifdef CONFIG_MMU 115 extern void arch_pick_mmap_layout(struct mm_struct *mm); 116 extern unsigned long 117 arch_get_unmapped_area(struct file *, unsigned long, unsigned long, 118 unsigned long, unsigned long); 119 extern unsigned long 120 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 121 unsigned long len, unsigned long pgoff, 122 unsigned long flags); 123 #else 124 static inline void arch_pick_mmap_layout(struct mm_struct *mm) {} 125 #endif 126 127 static inline bool in_vfork(struct task_struct *tsk) 128 { 129 bool ret; 130 131 /* 132 * need RCU to access ->real_parent if CLONE_VM was used along with 133 * CLONE_PARENT. 134 * 135 * We check real_parent->mm == tsk->mm because CLONE_VFORK does not 136 * imply CLONE_VM 137 * 138 * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus 139 * ->real_parent is not necessarily the task doing vfork(), so in 140 * theory we can't rely on task_lock() if we want to dereference it. 141 * 142 * And in this case we can't trust the real_parent->mm == tsk->mm 143 * check, it can be false negative. But we do not care, if init or 144 * another oom-unkillable task does this it should blame itself. 145 */ 146 rcu_read_lock(); 147 ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm; 148 rcu_read_unlock(); 149 150 return ret; 151 } 152 153 /* 154 * Applies per-task gfp context to the given allocation flags. 155 * PF_MEMALLOC_NOIO implies GFP_NOIO 156 * PF_MEMALLOC_NOFS implies GFP_NOFS 157 */ 158 static inline gfp_t current_gfp_context(gfp_t flags) 159 { 160 /* 161 * NOIO implies both NOIO and NOFS and it is a weaker context 162 * so always make sure it makes precendence 163 */ 164 if (unlikely(current->flags & PF_MEMALLOC_NOIO)) 165 flags &= ~(__GFP_IO | __GFP_FS); 166 else if (unlikely(current->flags & PF_MEMALLOC_NOFS)) 167 flags &= ~__GFP_FS; 168 return flags; 169 } 170 171 #ifdef CONFIG_LOCKDEP 172 extern void fs_reclaim_acquire(gfp_t gfp_mask); 173 extern void fs_reclaim_release(gfp_t gfp_mask); 174 #else 175 static inline void fs_reclaim_acquire(gfp_t gfp_mask) { } 176 static inline void fs_reclaim_release(gfp_t gfp_mask) { } 177 #endif 178 179 static inline unsigned int memalloc_noio_save(void) 180 { 181 unsigned int flags = current->flags & PF_MEMALLOC_NOIO; 182 current->flags |= PF_MEMALLOC_NOIO; 183 return flags; 184 } 185 186 static inline void memalloc_noio_restore(unsigned int flags) 187 { 188 current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags; 189 } 190 191 static inline unsigned int memalloc_nofs_save(void) 192 { 193 unsigned int flags = current->flags & PF_MEMALLOC_NOFS; 194 current->flags |= PF_MEMALLOC_NOFS; 195 return flags; 196 } 197 198 static inline void memalloc_nofs_restore(unsigned int flags) 199 { 200 current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags; 201 } 202 203 static inline unsigned int memalloc_noreclaim_save(void) 204 { 205 unsigned int flags = current->flags & PF_MEMALLOC; 206 current->flags |= PF_MEMALLOC; 207 return flags; 208 } 209 210 static inline void memalloc_noreclaim_restore(unsigned int flags) 211 { 212 current->flags = (current->flags & ~PF_MEMALLOC) | flags; 213 } 214 215 #ifdef CONFIG_MEMBARRIER 216 enum { 217 MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = (1U << 0), 218 MEMBARRIER_STATE_SWITCH_MM = (1U << 1), 219 }; 220 221 static inline void membarrier_execve(struct task_struct *t) 222 { 223 atomic_set(&t->mm->membarrier_state, 0); 224 } 225 #else 226 static inline void membarrier_execve(struct task_struct *t) 227 { 228 } 229 #endif 230 231 #endif /* _LINUX_SCHED_MM_H */ 232