1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * include/linux/userfaultfd_k.h 4 * 5 * Copyright (C) 2015 Red Hat, Inc. 6 * 7 */ 8 9 #ifndef _LINUX_USERFAULTFD_K_H 10 #define _LINUX_USERFAULTFD_K_H 11 12 #ifdef CONFIG_USERFAULTFD 13 14 #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */ 15 16 #include <linux/fcntl.h> 17 #include <linux/mm.h> 18 #include <linux/swap.h> 19 #include <linux/swapops.h> 20 #include <asm-generic/pgtable_uffd.h> 21 #include <linux/hugetlb_inline.h> 22 23 /* The set of all possible UFFD-related VM flags. */ 24 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) 25 26 /* 27 * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining 28 * new flags, since they might collide with O_* ones. We want 29 * to re-use O_* flags that couldn't possibly have a meaning 30 * from userfaultfd, in order to leave a free define-space for 31 * shared O_* flags. 32 */ 33 #define UFFD_CLOEXEC O_CLOEXEC 34 #define UFFD_NONBLOCK O_NONBLOCK 35 36 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) 37 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) 38 39 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); 40 41 /* A combined operation mode + behavior flags. */ 42 typedef unsigned int __bitwise uffd_flags_t; 43 44 /* Mutually exclusive modes of operation. */ 45 enum mfill_atomic_mode { 46 MFILL_ATOMIC_COPY, 47 MFILL_ATOMIC_ZEROPAGE, 48 MFILL_ATOMIC_CONTINUE, 49 NR_MFILL_ATOMIC_MODES, 50 }; 51 52 #define MFILL_ATOMIC_MODE_BITS (const_ilog2(NR_MFILL_ATOMIC_MODES - 1) + 1) 53 #define MFILL_ATOMIC_BIT(nr) BIT(MFILL_ATOMIC_MODE_BITS + (nr)) 54 #define MFILL_ATOMIC_FLAG(nr) ((__force uffd_flags_t) MFILL_ATOMIC_BIT(nr)) 55 #define MFILL_ATOMIC_MODE_MASK ((__force uffd_flags_t) (MFILL_ATOMIC_BIT(0) - 1)) 56 57 static inline bool uffd_flags_mode_is(uffd_flags_t flags, enum mfill_atomic_mode expected) 58 { 59 return (flags & MFILL_ATOMIC_MODE_MASK) == ((__force uffd_flags_t) expected); 60 } 61 62 static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_atomic_mode mode) 63 { 64 flags &= ~MFILL_ATOMIC_MODE_MASK; 65 return flags | ((__force uffd_flags_t) mode); 66 } 67 68 /* Flags controlling behavior. These behavior changes are mode-independent. */ 69 #define MFILL_ATOMIC_WP MFILL_ATOMIC_FLAG(0) 70 71 extern int mfill_atomic_install_pte(pmd_t *dst_pmd, 72 struct vm_area_struct *dst_vma, 73 unsigned long dst_addr, struct page *page, 74 bool newly_allocated, uffd_flags_t flags); 75 76 extern ssize_t mfill_atomic_copy(struct mm_struct *dst_mm, unsigned long dst_start, 77 unsigned long src_start, unsigned long len, 78 atomic_t *mmap_changing, uffd_flags_t flags); 79 extern ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, 80 unsigned long dst_start, 81 unsigned long len, 82 atomic_t *mmap_changing); 83 extern ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long dst_start, 84 unsigned long len, atomic_t *mmap_changing, 85 uffd_flags_t flags); 86 extern int mwriteprotect_range(struct mm_struct *dst_mm, 87 unsigned long start, unsigned long len, 88 bool enable_wp, atomic_t *mmap_changing); 89 extern long uffd_wp_range(struct vm_area_struct *vma, 90 unsigned long start, unsigned long len, bool enable_wp); 91 92 /* mm helpers */ 93 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 94 struct vm_userfaultfd_ctx vm_ctx) 95 { 96 return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx; 97 } 98 99 /* 100 * Never enable huge pmd sharing on some uffd registered vmas: 101 * 102 * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. 103 * 104 * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for 105 * VMAs which share huge pmds. (If you have two mappings to the same 106 * underlying pages, and fault in the non-UFFD-registered one with a write, 107 * with huge pmd sharing this would *also* setup the second UFFD-registered 108 * mapping, and we'd not get minor faults.) 109 */ 110 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) 111 { 112 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 113 } 114 115 /* 116 * Don't do fault around for either WP or MINOR registered uffd range. For 117 * MINOR registered range, fault around will be a total disaster and ptes can 118 * be installed without notifications; for WP it should mostly be fine as long 119 * as the fault around checks for pte_none() before the installation, however 120 * to be super safe we just forbid it. 121 */ 122 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 123 { 124 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 125 } 126 127 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 128 { 129 return vma->vm_flags & VM_UFFD_MISSING; 130 } 131 132 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 133 { 134 return vma->vm_flags & VM_UFFD_WP; 135 } 136 137 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 138 { 139 return vma->vm_flags & VM_UFFD_MINOR; 140 } 141 142 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 143 pte_t pte) 144 { 145 return userfaultfd_wp(vma) && pte_uffd_wp(pte); 146 } 147 148 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 149 pmd_t pmd) 150 { 151 return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); 152 } 153 154 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 155 { 156 return vma->vm_flags & __VM_UFFD_FLAGS; 157 } 158 159 static inline bool vma_can_userfault(struct vm_area_struct *vma, 160 unsigned long vm_flags) 161 { 162 if ((vm_flags & VM_UFFD_MINOR) && 163 (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma))) 164 return false; 165 #ifndef CONFIG_PTE_MARKER_UFFD_WP 166 /* 167 * If user requested uffd-wp but not enabled pte markers for 168 * uffd-wp, then shmem & hugetlbfs are not supported but only 169 * anonymous. 170 */ 171 if ((vm_flags & VM_UFFD_WP) && !vma_is_anonymous(vma)) 172 return false; 173 #endif 174 return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) || 175 vma_is_shmem(vma); 176 } 177 178 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *); 179 extern void dup_userfaultfd_complete(struct list_head *); 180 181 extern void mremap_userfaultfd_prep(struct vm_area_struct *, 182 struct vm_userfaultfd_ctx *); 183 extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *, 184 unsigned long from, unsigned long to, 185 unsigned long len); 186 187 extern bool userfaultfd_remove(struct vm_area_struct *vma, 188 unsigned long start, 189 unsigned long end); 190 191 extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, 192 unsigned long start, unsigned long end, struct list_head *uf); 193 extern void userfaultfd_unmap_complete(struct mm_struct *mm, 194 struct list_head *uf); 195 extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma); 196 197 #else /* CONFIG_USERFAULTFD */ 198 199 /* mm helpers */ 200 static inline vm_fault_t handle_userfault(struct vm_fault *vmf, 201 unsigned long reason) 202 { 203 return VM_FAULT_SIGBUS; 204 } 205 206 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 207 struct vm_userfaultfd_ctx vm_ctx) 208 { 209 return true; 210 } 211 212 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 213 { 214 return false; 215 } 216 217 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 218 { 219 return false; 220 } 221 222 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 223 { 224 return false; 225 } 226 227 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 228 pte_t pte) 229 { 230 return false; 231 } 232 233 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 234 pmd_t pmd) 235 { 236 return false; 237 } 238 239 240 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 241 { 242 return false; 243 } 244 245 static inline int dup_userfaultfd(struct vm_area_struct *vma, 246 struct list_head *l) 247 { 248 return 0; 249 } 250 251 static inline void dup_userfaultfd_complete(struct list_head *l) 252 { 253 } 254 255 static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma, 256 struct vm_userfaultfd_ctx *ctx) 257 { 258 } 259 260 static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx, 261 unsigned long from, 262 unsigned long to, 263 unsigned long len) 264 { 265 } 266 267 static inline bool userfaultfd_remove(struct vm_area_struct *vma, 268 unsigned long start, 269 unsigned long end) 270 { 271 return true; 272 } 273 274 static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, 275 unsigned long start, unsigned long end, 276 struct list_head *uf) 277 { 278 return 0; 279 } 280 281 static inline void userfaultfd_unmap_complete(struct mm_struct *mm, 282 struct list_head *uf) 283 { 284 } 285 286 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 287 { 288 return false; 289 } 290 291 static inline bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) 292 { 293 return false; 294 } 295 296 #endif /* CONFIG_USERFAULTFD */ 297 298 static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) 299 { 300 /* Only wr-protect mode uses pte markers */ 301 if (!userfaultfd_wp(vma)) 302 return false; 303 304 /* File-based uffd-wp always need markers */ 305 if (!vma_is_anonymous(vma)) 306 return true; 307 308 /* 309 * Anonymous uffd-wp only needs the markers if WP_UNPOPULATED 310 * enabled (to apply markers on zero pages). 311 */ 312 return userfaultfd_wp_unpopulated(vma); 313 } 314 315 static inline bool pte_marker_entry_uffd_wp(swp_entry_t entry) 316 { 317 #ifdef CONFIG_PTE_MARKER_UFFD_WP 318 return is_pte_marker_entry(entry) && 319 (pte_marker_get(entry) & PTE_MARKER_UFFD_WP); 320 #else 321 return false; 322 #endif 323 } 324 325 static inline bool pte_marker_uffd_wp(pte_t pte) 326 { 327 #ifdef CONFIG_PTE_MARKER_UFFD_WP 328 swp_entry_t entry; 329 330 if (!is_swap_pte(pte)) 331 return false; 332 333 entry = pte_to_swp_entry(pte); 334 335 return pte_marker_entry_uffd_wp(entry); 336 #else 337 return false; 338 #endif 339 } 340 341 /* 342 * Returns true if this is a swap pte and was uffd-wp wr-protected in either 343 * forms (pte marker or a normal swap pte), false otherwise. 344 */ 345 static inline bool pte_swp_uffd_wp_any(pte_t pte) 346 { 347 #ifdef CONFIG_PTE_MARKER_UFFD_WP 348 if (!is_swap_pte(pte)) 349 return false; 350 351 if (pte_swp_uffd_wp(pte)) 352 return true; 353 354 if (pte_marker_uffd_wp(pte)) 355 return true; 356 #endif 357 return false; 358 } 359 360 #endif /* _LINUX_USERFAULTFD_K_H */ 361