1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * include/linux/userfaultfd_k.h 4 * 5 * Copyright (C) 2015 Red Hat, Inc. 6 * 7 */ 8 9 #ifndef _LINUX_USERFAULTFD_K_H 10 #define _LINUX_USERFAULTFD_K_H 11 12 #ifdef CONFIG_USERFAULTFD 13 14 #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */ 15 16 #include <linux/fcntl.h> 17 #include <linux/mm.h> 18 #include <linux/swap.h> 19 #include <linux/swapops.h> 20 #include <asm-generic/pgtable_uffd.h> 21 #include <linux/hugetlb_inline.h> 22 23 /* The set of all possible UFFD-related VM flags. */ 24 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) 25 26 /* 27 * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining 28 * new flags, since they might collide with O_* ones. We want 29 * to re-use O_* flags that couldn't possibly have a meaning 30 * from userfaultfd, in order to leave a free define-space for 31 * shared O_* flags. 32 */ 33 #define UFFD_CLOEXEC O_CLOEXEC 34 #define UFFD_NONBLOCK O_NONBLOCK 35 36 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) 37 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) 38 39 extern int sysctl_unprivileged_userfaultfd; 40 41 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); 42 43 /* 44 * The mode of operation for __mcopy_atomic and its helpers. 45 * 46 * This is almost an implementation detail (mcopy_atomic below doesn't take this 47 * as a parameter), but it's exposed here because memory-kind-specific 48 * implementations (e.g. hugetlbfs) need to know the mode of operation. 49 */ 50 enum mcopy_atomic_mode { 51 /* A normal copy_from_user into the destination range. */ 52 MCOPY_ATOMIC_NORMAL, 53 /* Don't copy; map the destination range to the zero page. */ 54 MCOPY_ATOMIC_ZEROPAGE, 55 /* Just install pte(s) with the existing page(s) in the page cache. */ 56 MCOPY_ATOMIC_CONTINUE, 57 }; 58 59 extern int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, 60 struct vm_area_struct *dst_vma, 61 unsigned long dst_addr, struct page *page, 62 bool newly_allocated, bool wp_copy); 63 64 extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, 65 unsigned long src_start, unsigned long len, 66 atomic_t *mmap_changing, __u64 mode); 67 extern ssize_t mfill_zeropage(struct mm_struct *dst_mm, 68 unsigned long dst_start, 69 unsigned long len, 70 atomic_t *mmap_changing); 71 extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start, 72 unsigned long len, atomic_t *mmap_changing); 73 extern int mwriteprotect_range(struct mm_struct *dst_mm, 74 unsigned long start, unsigned long len, 75 bool enable_wp, atomic_t *mmap_changing); 76 extern void uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *vma, 77 unsigned long start, unsigned long len, bool enable_wp); 78 79 /* mm helpers */ 80 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 81 struct vm_userfaultfd_ctx vm_ctx) 82 { 83 return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx; 84 } 85 86 /* 87 * Never enable huge pmd sharing on some uffd registered vmas: 88 * 89 * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. 90 * 91 * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for 92 * VMAs which share huge pmds. (If you have two mappings to the same 93 * underlying pages, and fault in the non-UFFD-registered one with a write, 94 * with huge pmd sharing this would *also* setup the second UFFD-registered 95 * mapping, and we'd not get minor faults.) 96 */ 97 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) 98 { 99 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 100 } 101 102 /* 103 * Don't do fault around for either WP or MINOR registered uffd range. For 104 * MINOR registered range, fault around will be a total disaster and ptes can 105 * be installed without notifications; for WP it should mostly be fine as long 106 * as the fault around checks for pte_none() before the installation, however 107 * to be super safe we just forbid it. 108 */ 109 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 110 { 111 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 112 } 113 114 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 115 { 116 return vma->vm_flags & VM_UFFD_MISSING; 117 } 118 119 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 120 { 121 return vma->vm_flags & VM_UFFD_WP; 122 } 123 124 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 125 { 126 return vma->vm_flags & VM_UFFD_MINOR; 127 } 128 129 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 130 pte_t pte) 131 { 132 return userfaultfd_wp(vma) && pte_uffd_wp(pte); 133 } 134 135 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 136 pmd_t pmd) 137 { 138 return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); 139 } 140 141 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 142 { 143 return vma->vm_flags & __VM_UFFD_FLAGS; 144 } 145 146 static inline bool vma_can_userfault(struct vm_area_struct *vma, 147 unsigned long vm_flags) 148 { 149 if (vm_flags & VM_UFFD_MINOR) 150 return is_vm_hugetlb_page(vma) || vma_is_shmem(vma); 151 152 #ifndef CONFIG_PTE_MARKER_UFFD_WP 153 /* 154 * If user requested uffd-wp but not enabled pte markers for 155 * uffd-wp, then shmem & hugetlbfs are not supported but only 156 * anonymous. 157 */ 158 if ((vm_flags & VM_UFFD_WP) && !vma_is_anonymous(vma)) 159 return false; 160 #endif 161 return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) || 162 vma_is_shmem(vma); 163 } 164 165 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *); 166 extern void dup_userfaultfd_complete(struct list_head *); 167 168 extern void mremap_userfaultfd_prep(struct vm_area_struct *, 169 struct vm_userfaultfd_ctx *); 170 extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *, 171 unsigned long from, unsigned long to, 172 unsigned long len); 173 174 extern bool userfaultfd_remove(struct vm_area_struct *vma, 175 unsigned long start, 176 unsigned long end); 177 178 extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, 179 unsigned long start, unsigned long end, 180 struct list_head *uf); 181 extern void userfaultfd_unmap_complete(struct mm_struct *mm, 182 struct list_head *uf); 183 184 #else /* CONFIG_USERFAULTFD */ 185 186 /* mm helpers */ 187 static inline vm_fault_t handle_userfault(struct vm_fault *vmf, 188 unsigned long reason) 189 { 190 return VM_FAULT_SIGBUS; 191 } 192 193 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 194 struct vm_userfaultfd_ctx vm_ctx) 195 { 196 return true; 197 } 198 199 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 200 { 201 return false; 202 } 203 204 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 205 { 206 return false; 207 } 208 209 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 210 { 211 return false; 212 } 213 214 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 215 pte_t pte) 216 { 217 return false; 218 } 219 220 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 221 pmd_t pmd) 222 { 223 return false; 224 } 225 226 227 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 228 { 229 return false; 230 } 231 232 static inline int dup_userfaultfd(struct vm_area_struct *vma, 233 struct list_head *l) 234 { 235 return 0; 236 } 237 238 static inline void dup_userfaultfd_complete(struct list_head *l) 239 { 240 } 241 242 static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma, 243 struct vm_userfaultfd_ctx *ctx) 244 { 245 } 246 247 static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx, 248 unsigned long from, 249 unsigned long to, 250 unsigned long len) 251 { 252 } 253 254 static inline bool userfaultfd_remove(struct vm_area_struct *vma, 255 unsigned long start, 256 unsigned long end) 257 { 258 return true; 259 } 260 261 static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, 262 unsigned long start, unsigned long end, 263 struct list_head *uf) 264 { 265 return 0; 266 } 267 268 static inline void userfaultfd_unmap_complete(struct mm_struct *mm, 269 struct list_head *uf) 270 { 271 } 272 273 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 274 { 275 return false; 276 } 277 278 #endif /* CONFIG_USERFAULTFD */ 279 280 static inline bool pte_marker_entry_uffd_wp(swp_entry_t entry) 281 { 282 #ifdef CONFIG_PTE_MARKER_UFFD_WP 283 return is_pte_marker_entry(entry) && 284 (pte_marker_get(entry) & PTE_MARKER_UFFD_WP); 285 #else 286 return false; 287 #endif 288 } 289 290 static inline bool pte_marker_uffd_wp(pte_t pte) 291 { 292 #ifdef CONFIG_PTE_MARKER_UFFD_WP 293 swp_entry_t entry; 294 295 if (!is_swap_pte(pte)) 296 return false; 297 298 entry = pte_to_swp_entry(pte); 299 300 return pte_marker_entry_uffd_wp(entry); 301 #else 302 return false; 303 #endif 304 } 305 306 /* 307 * Returns true if this is a swap pte and was uffd-wp wr-protected in either 308 * forms (pte marker or a normal swap pte), false otherwise. 309 */ 310 static inline bool pte_swp_uffd_wp_any(pte_t pte) 311 { 312 #ifdef CONFIG_PTE_MARKER_UFFD_WP 313 if (!is_swap_pte(pte)) 314 return false; 315 316 if (pte_swp_uffd_wp(pte)) 317 return true; 318 319 if (pte_marker_uffd_wp(pte)) 320 return true; 321 #endif 322 return false; 323 } 324 325 #endif /* _LINUX_USERFAULTFD_K_H */ 326