1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * include/linux/userfaultfd_k.h 4 * 5 * Copyright (C) 2015 Red Hat, Inc. 6 * 7 */ 8 9 #ifndef _LINUX_USERFAULTFD_K_H 10 #define _LINUX_USERFAULTFD_K_H 11 12 #ifdef CONFIG_USERFAULTFD 13 14 #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */ 15 16 #include <linux/fcntl.h> 17 #include <linux/mm.h> 18 #include <linux/swap.h> 19 #include <linux/swapops.h> 20 #include <asm-generic/pgtable_uffd.h> 21 #include <linux/hugetlb_inline.h> 22 23 /* The set of all possible UFFD-related VM flags. */ 24 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) 25 26 /* 27 * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining 28 * new flags, since they might collide with O_* ones. We want 29 * to re-use O_* flags that couldn't possibly have a meaning 30 * from userfaultfd, in order to leave a free define-space for 31 * shared O_* flags. 32 */ 33 #define UFFD_CLOEXEC O_CLOEXEC 34 #define UFFD_NONBLOCK O_NONBLOCK 35 36 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) 37 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) 38 39 extern int sysctl_unprivileged_userfaultfd; 40 41 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); 42 43 /* A combined operation mode + behavior flags. */ 44 typedef unsigned int __bitwise uffd_flags_t; 45 46 /* Mutually exclusive modes of operation. */ 47 enum mfill_atomic_mode { 48 MFILL_ATOMIC_COPY, 49 MFILL_ATOMIC_ZEROPAGE, 50 MFILL_ATOMIC_CONTINUE, 51 NR_MFILL_ATOMIC_MODES, 52 }; 53 54 #define MFILL_ATOMIC_MODE_BITS (const_ilog2(NR_MFILL_ATOMIC_MODES - 1) + 1) 55 #define MFILL_ATOMIC_BIT(nr) BIT(MFILL_ATOMIC_MODE_BITS + (nr)) 56 #define MFILL_ATOMIC_FLAG(nr) ((__force uffd_flags_t) MFILL_ATOMIC_BIT(nr)) 57 #define MFILL_ATOMIC_MODE_MASK ((__force uffd_flags_t) (MFILL_ATOMIC_BIT(0) - 1)) 58 59 static inline bool uffd_flags_mode_is(uffd_flags_t flags, enum mfill_atomic_mode expected) 60 { 61 return (flags & MFILL_ATOMIC_MODE_MASK) == ((__force uffd_flags_t) expected); 62 } 63 64 static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_atomic_mode mode) 65 { 66 flags &= ~MFILL_ATOMIC_MODE_MASK; 67 return flags | ((__force uffd_flags_t) mode); 68 } 69 70 /* Flags controlling behavior. These behavior changes are mode-independent. */ 71 #define MFILL_ATOMIC_WP MFILL_ATOMIC_FLAG(0) 72 73 extern int mfill_atomic_install_pte(pmd_t *dst_pmd, 74 struct vm_area_struct *dst_vma, 75 unsigned long dst_addr, struct page *page, 76 bool newly_allocated, uffd_flags_t flags); 77 78 extern ssize_t mfill_atomic_copy(struct mm_struct *dst_mm, unsigned long dst_start, 79 unsigned long src_start, unsigned long len, 80 atomic_t *mmap_changing, uffd_flags_t flags); 81 extern ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, 82 unsigned long dst_start, 83 unsigned long len, 84 atomic_t *mmap_changing); 85 extern ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long dst_start, 86 unsigned long len, atomic_t *mmap_changing, 87 uffd_flags_t flags); 88 extern int mwriteprotect_range(struct mm_struct *dst_mm, 89 unsigned long start, unsigned long len, 90 bool enable_wp, atomic_t *mmap_changing); 91 extern long uffd_wp_range(struct vm_area_struct *vma, 92 unsigned long start, unsigned long len, bool enable_wp); 93 94 /* mm helpers */ 95 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 96 struct vm_userfaultfd_ctx vm_ctx) 97 { 98 return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx; 99 } 100 101 /* 102 * Never enable huge pmd sharing on some uffd registered vmas: 103 * 104 * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. 105 * 106 * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for 107 * VMAs which share huge pmds. (If you have two mappings to the same 108 * underlying pages, and fault in the non-UFFD-registered one with a write, 109 * with huge pmd sharing this would *also* setup the second UFFD-registered 110 * mapping, and we'd not get minor faults.) 111 */ 112 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) 113 { 114 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 115 } 116 117 /* 118 * Don't do fault around for either WP or MINOR registered uffd range. For 119 * MINOR registered range, fault around will be a total disaster and ptes can 120 * be installed without notifications; for WP it should mostly be fine as long 121 * as the fault around checks for pte_none() before the installation, however 122 * to be super safe we just forbid it. 123 */ 124 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 125 { 126 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 127 } 128 129 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 130 { 131 return vma->vm_flags & VM_UFFD_MISSING; 132 } 133 134 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 135 { 136 return vma->vm_flags & VM_UFFD_WP; 137 } 138 139 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 140 { 141 return vma->vm_flags & VM_UFFD_MINOR; 142 } 143 144 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 145 pte_t pte) 146 { 147 return userfaultfd_wp(vma) && pte_uffd_wp(pte); 148 } 149 150 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 151 pmd_t pmd) 152 { 153 return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); 154 } 155 156 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 157 { 158 return vma->vm_flags & __VM_UFFD_FLAGS; 159 } 160 161 static inline bool vma_can_userfault(struct vm_area_struct *vma, 162 unsigned long vm_flags) 163 { 164 if ((vm_flags & VM_UFFD_MINOR) && 165 (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma))) 166 return false; 167 #ifndef CONFIG_PTE_MARKER_UFFD_WP 168 /* 169 * If user requested uffd-wp but not enabled pte markers for 170 * uffd-wp, then shmem & hugetlbfs are not supported but only 171 * anonymous. 172 */ 173 if ((vm_flags & VM_UFFD_WP) && !vma_is_anonymous(vma)) 174 return false; 175 #endif 176 return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) || 177 vma_is_shmem(vma); 178 } 179 180 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *); 181 extern void dup_userfaultfd_complete(struct list_head *); 182 183 extern void mremap_userfaultfd_prep(struct vm_area_struct *, 184 struct vm_userfaultfd_ctx *); 185 extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *, 186 unsigned long from, unsigned long to, 187 unsigned long len); 188 189 extern bool userfaultfd_remove(struct vm_area_struct *vma, 190 unsigned long start, 191 unsigned long end); 192 193 extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, 194 unsigned long end, struct list_head *uf); 195 extern void userfaultfd_unmap_complete(struct mm_struct *mm, 196 struct list_head *uf); 197 extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma); 198 199 #else /* CONFIG_USERFAULTFD */ 200 201 /* mm helpers */ 202 static inline vm_fault_t handle_userfault(struct vm_fault *vmf, 203 unsigned long reason) 204 { 205 return VM_FAULT_SIGBUS; 206 } 207 208 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 209 struct vm_userfaultfd_ctx vm_ctx) 210 { 211 return true; 212 } 213 214 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 215 { 216 return false; 217 } 218 219 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 220 { 221 return false; 222 } 223 224 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 225 { 226 return false; 227 } 228 229 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 230 pte_t pte) 231 { 232 return false; 233 } 234 235 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 236 pmd_t pmd) 237 { 238 return false; 239 } 240 241 242 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 243 { 244 return false; 245 } 246 247 static inline int dup_userfaultfd(struct vm_area_struct *vma, 248 struct list_head *l) 249 { 250 return 0; 251 } 252 253 static inline void dup_userfaultfd_complete(struct list_head *l) 254 { 255 } 256 257 static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma, 258 struct vm_userfaultfd_ctx *ctx) 259 { 260 } 261 262 static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx, 263 unsigned long from, 264 unsigned long to, 265 unsigned long len) 266 { 267 } 268 269 static inline bool userfaultfd_remove(struct vm_area_struct *vma, 270 unsigned long start, 271 unsigned long end) 272 { 273 return true; 274 } 275 276 static inline int userfaultfd_unmap_prep(struct mm_struct *mm, 277 unsigned long start, unsigned long end, 278 struct list_head *uf) 279 { 280 return 0; 281 } 282 283 static inline void userfaultfd_unmap_complete(struct mm_struct *mm, 284 struct list_head *uf) 285 { 286 } 287 288 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 289 { 290 return false; 291 } 292 293 static inline bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) 294 { 295 return false; 296 } 297 298 #endif /* CONFIG_USERFAULTFD */ 299 300 static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) 301 { 302 /* Only wr-protect mode uses pte markers */ 303 if (!userfaultfd_wp(vma)) 304 return false; 305 306 /* File-based uffd-wp always need markers */ 307 if (!vma_is_anonymous(vma)) 308 return true; 309 310 /* 311 * Anonymous uffd-wp only needs the markers if WP_UNPOPULATED 312 * enabled (to apply markers on zero pages). 313 */ 314 return userfaultfd_wp_unpopulated(vma); 315 } 316 317 static inline bool pte_marker_entry_uffd_wp(swp_entry_t entry) 318 { 319 #ifdef CONFIG_PTE_MARKER_UFFD_WP 320 return is_pte_marker_entry(entry) && 321 (pte_marker_get(entry) & PTE_MARKER_UFFD_WP); 322 #else 323 return false; 324 #endif 325 } 326 327 static inline bool pte_marker_uffd_wp(pte_t pte) 328 { 329 #ifdef CONFIG_PTE_MARKER_UFFD_WP 330 swp_entry_t entry; 331 332 if (!is_swap_pte(pte)) 333 return false; 334 335 entry = pte_to_swp_entry(pte); 336 337 return pte_marker_entry_uffd_wp(entry); 338 #else 339 return false; 340 #endif 341 } 342 343 /* 344 * Returns true if this is a swap pte and was uffd-wp wr-protected in either 345 * forms (pte marker or a normal swap pte), false otherwise. 346 */ 347 static inline bool pte_swp_uffd_wp_any(pte_t pte) 348 { 349 #ifdef CONFIG_PTE_MARKER_UFFD_WP 350 if (!is_swap_pte(pte)) 351 return false; 352 353 if (pte_swp_uffd_wp(pte)) 354 return true; 355 356 if (pte_marker_uffd_wp(pte)) 357 return true; 358 #endif 359 return false; 360 } 361 362 #endif /* _LINUX_USERFAULTFD_K_H */ 363