1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_SCHED_MM_H 3 #define _LINUX_SCHED_MM_H 4 5 #include <linux/kernel.h> 6 #include <linux/atomic.h> 7 #include <linux/sched.h> 8 #include <linux/mm_types.h> 9 #include <linux/gfp.h> 10 #include <linux/sync_core.h> 11 12 /* 13 * Routines for handling mm_structs 14 */ 15 extern struct mm_struct *mm_alloc(void); 16 17 /** 18 * mmgrab() - Pin a &struct mm_struct. 19 * @mm: The &struct mm_struct to pin. 20 * 21 * Make sure that @mm will not get freed even after the owning task 22 * exits. This doesn't guarantee that the associated address space 23 * will still exist later on and mmget_not_zero() has to be used before 24 * accessing it. 25 * 26 * This is a preferred way to to pin @mm for a longer/unbounded amount 27 * of time. 28 * 29 * Use mmdrop() to release the reference acquired by mmgrab(). 30 * 31 * See also <Documentation/vm/active_mm.txt> for an in-depth explanation 32 * of &mm_struct.mm_count vs &mm_struct.mm_users. 33 */ 34 static inline void mmgrab(struct mm_struct *mm) 35 { 36 atomic_inc(&mm->mm_count); 37 } 38 39 extern void __mmdrop(struct mm_struct *mm); 40 41 static inline void mmdrop(struct mm_struct *mm) 42 { 43 /* 44 * The implicit full barrier implied by atomic_dec_and_test() is 45 * required by the membarrier system call before returning to 46 * user-space, after storing to rq->curr. 47 */ 48 if (unlikely(atomic_dec_and_test(&mm->mm_count))) 49 __mmdrop(mm); 50 } 51 52 /** 53 * mmget() - Pin the address space associated with a &struct mm_struct. 54 * @mm: The address space to pin. 55 * 56 * Make sure that the address space of the given &struct mm_struct doesn't 57 * go away. This does not protect against parts of the address space being 58 * modified or freed, however. 59 * 60 * Never use this function to pin this address space for an 61 * unbounded/indefinite amount of time. 62 * 63 * Use mmput() to release the reference acquired by mmget(). 64 * 65 * See also <Documentation/vm/active_mm.txt> for an in-depth explanation 66 * of &mm_struct.mm_count vs &mm_struct.mm_users. 67 */ 68 static inline void mmget(struct mm_struct *mm) 69 { 70 atomic_inc(&mm->mm_users); 71 } 72 73 static inline bool mmget_not_zero(struct mm_struct *mm) 74 { 75 return atomic_inc_not_zero(&mm->mm_users); 76 } 77 78 /* mmput gets rid of the mappings and all user-space */ 79 extern void mmput(struct mm_struct *); 80 #ifdef CONFIG_MMU 81 /* same as above but performs the slow path from the async context. Can 82 * be called from the atomic context as well 83 */ 84 void mmput_async(struct mm_struct *); 85 #endif 86 87 /* Grab a reference to a task's mm, if it is not already going away */ 88 extern struct mm_struct *get_task_mm(struct task_struct *task); 89 /* 90 * Grab a reference to a task's mm, if it is not already going away 91 * and ptrace_may_access with the mode parameter passed to it 92 * succeeds. 93 */ 94 extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); 95 /* Remove the current tasks stale references to the old mm_struct */ 96 extern void mm_release(struct task_struct *, struct mm_struct *); 97 98 #ifdef CONFIG_MEMCG 99 extern void mm_update_next_owner(struct mm_struct *mm); 100 #else 101 static inline void mm_update_next_owner(struct mm_struct *mm) 102 { 103 } 104 #endif /* CONFIG_MEMCG */ 105 106 #ifdef CONFIG_MMU 107 extern void arch_pick_mmap_layout(struct mm_struct *mm, 108 struct rlimit *rlim_stack); 109 extern unsigned long 110 arch_get_unmapped_area(struct file *, unsigned long, unsigned long, 111 unsigned long, unsigned long); 112 extern unsigned long 113 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, 114 unsigned long len, unsigned long pgoff, 115 unsigned long flags); 116 #else 117 static inline void arch_pick_mmap_layout(struct mm_struct *mm, 118 struct rlimit *rlim_stack) {} 119 #endif 120 121 static inline bool in_vfork(struct task_struct *tsk) 122 { 123 bool ret; 124 125 /* 126 * need RCU to access ->real_parent if CLONE_VM was used along with 127 * CLONE_PARENT. 128 * 129 * We check real_parent->mm == tsk->mm because CLONE_VFORK does not 130 * imply CLONE_VM 131 * 132 * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus 133 * ->real_parent is not necessarily the task doing vfork(), so in 134 * theory we can't rely on task_lock() if we want to dereference it. 135 * 136 * And in this case we can't trust the real_parent->mm == tsk->mm 137 * check, it can be false negative. But we do not care, if init or 138 * another oom-unkillable task does this it should blame itself. 139 */ 140 rcu_read_lock(); 141 ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm; 142 rcu_read_unlock(); 143 144 return ret; 145 } 146 147 /* 148 * Applies per-task gfp context to the given allocation flags. 149 * PF_MEMALLOC_NOIO implies GFP_NOIO 150 * PF_MEMALLOC_NOFS implies GFP_NOFS 151 */ 152 static inline gfp_t current_gfp_context(gfp_t flags) 153 { 154 /* 155 * NOIO implies both NOIO and NOFS and it is a weaker context 156 * so always make sure it makes precendence 157 */ 158 if (unlikely(current->flags & PF_MEMALLOC_NOIO)) 159 flags &= ~(__GFP_IO | __GFP_FS); 160 else if (unlikely(current->flags & PF_MEMALLOC_NOFS)) 161 flags &= ~__GFP_FS; 162 return flags; 163 } 164 165 #ifdef CONFIG_LOCKDEP 166 extern void fs_reclaim_acquire(gfp_t gfp_mask); 167 extern void fs_reclaim_release(gfp_t gfp_mask); 168 #else 169 static inline void fs_reclaim_acquire(gfp_t gfp_mask) { } 170 static inline void fs_reclaim_release(gfp_t gfp_mask) { } 171 #endif 172 173 static inline unsigned int memalloc_noio_save(void) 174 { 175 unsigned int flags = current->flags & PF_MEMALLOC_NOIO; 176 current->flags |= PF_MEMALLOC_NOIO; 177 return flags; 178 } 179 180 static inline void memalloc_noio_restore(unsigned int flags) 181 { 182 current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags; 183 } 184 185 static inline unsigned int memalloc_nofs_save(void) 186 { 187 unsigned int flags = current->flags & PF_MEMALLOC_NOFS; 188 current->flags |= PF_MEMALLOC_NOFS; 189 return flags; 190 } 191 192 static inline void memalloc_nofs_restore(unsigned int flags) 193 { 194 current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags; 195 } 196 197 static inline unsigned int memalloc_noreclaim_save(void) 198 { 199 unsigned int flags = current->flags & PF_MEMALLOC; 200 current->flags |= PF_MEMALLOC; 201 return flags; 202 } 203 204 static inline void memalloc_noreclaim_restore(unsigned int flags) 205 { 206 current->flags = (current->flags & ~PF_MEMALLOC) | flags; 207 } 208 209 #ifdef CONFIG_MEMBARRIER 210 enum { 211 MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = (1U << 0), 212 MEMBARRIER_STATE_PRIVATE_EXPEDITED = (1U << 1), 213 MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY = (1U << 2), 214 MEMBARRIER_STATE_GLOBAL_EXPEDITED = (1U << 3), 215 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY = (1U << 4), 216 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE = (1U << 5), 217 }; 218 219 enum { 220 MEMBARRIER_FLAG_SYNC_CORE = (1U << 0), 221 }; 222 223 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS 224 #include <asm/membarrier.h> 225 #endif 226 227 static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) 228 { 229 if (likely(!(atomic_read(&mm->membarrier_state) & 230 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) 231 return; 232 sync_core_before_usermode(); 233 } 234 235 static inline void membarrier_execve(struct task_struct *t) 236 { 237 atomic_set(&t->mm->membarrier_state, 0); 238 } 239 #else 240 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS 241 static inline void membarrier_arch_switch_mm(struct mm_struct *prev, 242 struct mm_struct *next, 243 struct task_struct *tsk) 244 { 245 } 246 #endif 247 static inline void membarrier_execve(struct task_struct *t) 248 { 249 } 250 static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) 251 { 252 } 253 #endif 254 255 #endif /* _LINUX_SCHED_MM_H */ 256