1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * include/linux/userfaultfd_k.h 4 * 5 * Copyright (C) 2015 Red Hat, Inc. 6 * 7 */ 8 9 #ifndef _LINUX_USERFAULTFD_K_H 10 #define _LINUX_USERFAULTFD_K_H 11 12 #ifdef CONFIG_USERFAULTFD 13 14 #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */ 15 16 #include <linux/fcntl.h> 17 #include <linux/mm.h> 18 #include <linux/swap.h> 19 #include <linux/swapops.h> 20 #include <asm-generic/pgtable_uffd.h> 21 #include <linux/hugetlb_inline.h> 22 23 /* The set of all possible UFFD-related VM flags. */ 24 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) 25 26 /* 27 * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining 28 * new flags, since they might collide with O_* ones. We want 29 * to re-use O_* flags that couldn't possibly have a meaning 30 * from userfaultfd, in order to leave a free define-space for 31 * shared O_* flags. 32 */ 33 #define UFFD_CLOEXEC O_CLOEXEC 34 #define UFFD_NONBLOCK O_NONBLOCK 35 36 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) 37 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) 38 39 extern int sysctl_unprivileged_userfaultfd; 40 41 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); 42 43 /* A combined operation mode + behavior flags. */ 44 typedef unsigned int __bitwise uffd_flags_t; 45 46 /* Mutually exclusive modes of operation. */ 47 enum mfill_atomic_mode { 48 MFILL_ATOMIC_COPY, 49 MFILL_ATOMIC_ZEROPAGE, 50 MFILL_ATOMIC_CONTINUE, 51 NR_MFILL_ATOMIC_MODES, 52 }; 53 54 #define MFILL_ATOMIC_MODE_BITS (const_ilog2(NR_MFILL_ATOMIC_MODES - 1) + 1) 55 #define MFILL_ATOMIC_BIT(nr) BIT(MFILL_ATOMIC_MODE_BITS + (nr)) 56 #define MFILL_ATOMIC_FLAG(nr) ((__force uffd_flags_t) MFILL_ATOMIC_BIT(nr)) 57 #define MFILL_ATOMIC_MODE_MASK ((__force uffd_flags_t) (MFILL_ATOMIC_BIT(0) - 1)) 58 59 static inline bool uffd_flags_mode_is(uffd_flags_t flags, enum mfill_atomic_mode expected) 60 { 61 return (flags & MFILL_ATOMIC_MODE_MASK) == ((__force uffd_flags_t) expected); 62 } 63 64 static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_atomic_mode mode) 65 { 66 flags &= ~MFILL_ATOMIC_MODE_MASK; 67 return flags | ((__force uffd_flags_t) mode); 68 } 69 70 /* Flags controlling behavior. These behavior changes are mode-independent. */ 71 #define MFILL_ATOMIC_WP MFILL_ATOMIC_FLAG(0) 72 73 extern int mfill_atomic_install_pte(pmd_t *dst_pmd, 74 struct vm_area_struct *dst_vma, 75 unsigned long dst_addr, struct page *page, 76 bool newly_allocated, uffd_flags_t flags); 77 78 extern ssize_t mfill_atomic_copy(struct mm_struct *dst_mm, unsigned long dst_start, 79 unsigned long src_start, unsigned long len, 80 atomic_t *mmap_changing, uffd_flags_t flags); 81 extern ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, 82 unsigned long dst_start, 83 unsigned long len, 84 atomic_t *mmap_changing); 85 extern ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long dst_start, 86 unsigned long len, atomic_t *mmap_changing); 87 extern int mwriteprotect_range(struct mm_struct *dst_mm, 88 unsigned long start, unsigned long len, 89 bool enable_wp, atomic_t *mmap_changing); 90 extern long uffd_wp_range(struct vm_area_struct *vma, 91 unsigned long start, unsigned long len, bool enable_wp); 92 93 /* mm helpers */ 94 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 95 struct vm_userfaultfd_ctx vm_ctx) 96 { 97 return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx; 98 } 99 100 /* 101 * Never enable huge pmd sharing on some uffd registered vmas: 102 * 103 * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. 104 * 105 * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for 106 * VMAs which share huge pmds. (If you have two mappings to the same 107 * underlying pages, and fault in the non-UFFD-registered one with a write, 108 * with huge pmd sharing this would *also* setup the second UFFD-registered 109 * mapping, and we'd not get minor faults.) 110 */ 111 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) 112 { 113 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 114 } 115 116 /* 117 * Don't do fault around for either WP or MINOR registered uffd range. For 118 * MINOR registered range, fault around will be a total disaster and ptes can 119 * be installed without notifications; for WP it should mostly be fine as long 120 * as the fault around checks for pte_none() before the installation, however 121 * to be super safe we just forbid it. 122 */ 123 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 124 { 125 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 126 } 127 128 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 129 { 130 return vma->vm_flags & VM_UFFD_MISSING; 131 } 132 133 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 134 { 135 return vma->vm_flags & VM_UFFD_WP; 136 } 137 138 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 139 { 140 return vma->vm_flags & VM_UFFD_MINOR; 141 } 142 143 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 144 pte_t pte) 145 { 146 return userfaultfd_wp(vma) && pte_uffd_wp(pte); 147 } 148 149 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 150 pmd_t pmd) 151 { 152 return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); 153 } 154 155 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 156 { 157 return vma->vm_flags & __VM_UFFD_FLAGS; 158 } 159 160 static inline bool vma_can_userfault(struct vm_area_struct *vma, 161 unsigned long vm_flags) 162 { 163 if ((vm_flags & VM_UFFD_MINOR) && 164 (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma))) 165 return false; 166 #ifndef CONFIG_PTE_MARKER_UFFD_WP 167 /* 168 * If user requested uffd-wp but not enabled pte markers for 169 * uffd-wp, then shmem & hugetlbfs are not supported but only 170 * anonymous. 171 */ 172 if ((vm_flags & VM_UFFD_WP) && !vma_is_anonymous(vma)) 173 return false; 174 #endif 175 return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) || 176 vma_is_shmem(vma); 177 } 178 179 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *); 180 extern void dup_userfaultfd_complete(struct list_head *); 181 182 extern void mremap_userfaultfd_prep(struct vm_area_struct *, 183 struct vm_userfaultfd_ctx *); 184 extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *, 185 unsigned long from, unsigned long to, 186 unsigned long len); 187 188 extern bool userfaultfd_remove(struct vm_area_struct *vma, 189 unsigned long start, 190 unsigned long end); 191 192 extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, 193 unsigned long end, struct list_head *uf); 194 extern void userfaultfd_unmap_complete(struct mm_struct *mm, 195 struct list_head *uf); 196 extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma); 197 198 #else /* CONFIG_USERFAULTFD */ 199 200 /* mm helpers */ 201 static inline vm_fault_t handle_userfault(struct vm_fault *vmf, 202 unsigned long reason) 203 { 204 return VM_FAULT_SIGBUS; 205 } 206 207 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 208 struct vm_userfaultfd_ctx vm_ctx) 209 { 210 return true; 211 } 212 213 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 214 { 215 return false; 216 } 217 218 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 219 { 220 return false; 221 } 222 223 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 224 { 225 return false; 226 } 227 228 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 229 pte_t pte) 230 { 231 return false; 232 } 233 234 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 235 pmd_t pmd) 236 { 237 return false; 238 } 239 240 241 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 242 { 243 return false; 244 } 245 246 static inline int dup_userfaultfd(struct vm_area_struct *vma, 247 struct list_head *l) 248 { 249 return 0; 250 } 251 252 static inline void dup_userfaultfd_complete(struct list_head *l) 253 { 254 } 255 256 static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma, 257 struct vm_userfaultfd_ctx *ctx) 258 { 259 } 260 261 static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx, 262 unsigned long from, 263 unsigned long to, 264 unsigned long len) 265 { 266 } 267 268 static inline bool userfaultfd_remove(struct vm_area_struct *vma, 269 unsigned long start, 270 unsigned long end) 271 { 272 return true; 273 } 274 275 static inline int userfaultfd_unmap_prep(struct mm_struct *mm, 276 unsigned long start, unsigned long end, 277 struct list_head *uf) 278 { 279 return 0; 280 } 281 282 static inline void userfaultfd_unmap_complete(struct mm_struct *mm, 283 struct list_head *uf) 284 { 285 } 286 287 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 288 { 289 return false; 290 } 291 292 static inline bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) 293 { 294 return false; 295 } 296 297 #endif /* CONFIG_USERFAULTFD */ 298 299 static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) 300 { 301 /* Only wr-protect mode uses pte markers */ 302 if (!userfaultfd_wp(vma)) 303 return false; 304 305 /* File-based uffd-wp always need markers */ 306 if (!vma_is_anonymous(vma)) 307 return true; 308 309 /* 310 * Anonymous uffd-wp only needs the markers if WP_UNPOPULATED 311 * enabled (to apply markers on zero pages). 312 */ 313 return userfaultfd_wp_unpopulated(vma); 314 } 315 316 static inline bool pte_marker_entry_uffd_wp(swp_entry_t entry) 317 { 318 #ifdef CONFIG_PTE_MARKER_UFFD_WP 319 return is_pte_marker_entry(entry) && 320 (pte_marker_get(entry) & PTE_MARKER_UFFD_WP); 321 #else 322 return false; 323 #endif 324 } 325 326 static inline bool pte_marker_uffd_wp(pte_t pte) 327 { 328 #ifdef CONFIG_PTE_MARKER_UFFD_WP 329 swp_entry_t entry; 330 331 if (!is_swap_pte(pte)) 332 return false; 333 334 entry = pte_to_swp_entry(pte); 335 336 return pte_marker_entry_uffd_wp(entry); 337 #else 338 return false; 339 #endif 340 } 341 342 /* 343 * Returns true if this is a swap pte and was uffd-wp wr-protected in either 344 * forms (pte marker or a normal swap pte), false otherwise. 345 */ 346 static inline bool pte_swp_uffd_wp_any(pte_t pte) 347 { 348 #ifdef CONFIG_PTE_MARKER_UFFD_WP 349 if (!is_swap_pte(pte)) 350 return false; 351 352 if (pte_swp_uffd_wp(pte)) 353 return true; 354 355 if (pte_marker_uffd_wp(pte)) 356 return true; 357 #endif 358 return false; 359 } 360 361 #endif /* _LINUX_USERFAULTFD_K_H */ 362