1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * include/linux/userfaultfd_k.h 4 * 5 * Copyright (C) 2015 Red Hat, Inc. 6 * 7 */ 8 9 #ifndef _LINUX_USERFAULTFD_K_H 10 #define _LINUX_USERFAULTFD_K_H 11 12 #ifdef CONFIG_USERFAULTFD 13 14 #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */ 15 16 #include <linux/fcntl.h> 17 #include <linux/mm.h> 18 #include <linux/swap.h> 19 #include <linux/swapops.h> 20 #include <asm-generic/pgtable_uffd.h> 21 #include <linux/hugetlb_inline.h> 22 23 /* The set of all possible UFFD-related VM flags. */ 24 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) 25 26 /* 27 * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining 28 * new flags, since they might collide with O_* ones. We want 29 * to re-use O_* flags that couldn't possibly have a meaning 30 * from userfaultfd, in order to leave a free define-space for 31 * shared O_* flags. 32 */ 33 #define UFFD_CLOEXEC O_CLOEXEC 34 #define UFFD_NONBLOCK O_NONBLOCK 35 36 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) 37 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) 38 39 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); 40 41 /* 42 * The mode of operation for __mcopy_atomic and its helpers. 43 * 44 * This is almost an implementation detail (mcopy_atomic below doesn't take this 45 * as a parameter), but it's exposed here because memory-kind-specific 46 * implementations (e.g. hugetlbfs) need to know the mode of operation. 47 */ 48 enum mcopy_atomic_mode { 49 /* A normal copy_from_user into the destination range. */ 50 MCOPY_ATOMIC_NORMAL, 51 /* Don't copy; map the destination range to the zero page. */ 52 MCOPY_ATOMIC_ZEROPAGE, 53 /* Just install pte(s) with the existing page(s) in the page cache. */ 54 MCOPY_ATOMIC_CONTINUE, 55 }; 56 57 extern int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, 58 struct vm_area_struct *dst_vma, 59 unsigned long dst_addr, struct page *page, 60 bool newly_allocated, bool wp_copy); 61 62 extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, 63 unsigned long src_start, unsigned long len, 64 atomic_t *mmap_changing, __u64 mode); 65 extern ssize_t mfill_zeropage(struct mm_struct *dst_mm, 66 unsigned long dst_start, 67 unsigned long len, 68 atomic_t *mmap_changing); 69 extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start, 70 unsigned long len, atomic_t *mmap_changing); 71 extern int mwriteprotect_range(struct mm_struct *dst_mm, 72 unsigned long start, unsigned long len, 73 bool enable_wp, atomic_t *mmap_changing); 74 extern long uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *vma, 75 unsigned long start, unsigned long len, bool enable_wp); 76 77 /* mm helpers */ 78 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 79 struct vm_userfaultfd_ctx vm_ctx) 80 { 81 return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx; 82 } 83 84 /* 85 * Never enable huge pmd sharing on some uffd registered vmas: 86 * 87 * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. 88 * 89 * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for 90 * VMAs which share huge pmds. (If you have two mappings to the same 91 * underlying pages, and fault in the non-UFFD-registered one with a write, 92 * with huge pmd sharing this would *also* setup the second UFFD-registered 93 * mapping, and we'd not get minor faults.) 94 */ 95 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) 96 { 97 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 98 } 99 100 /* 101 * Don't do fault around for either WP or MINOR registered uffd range. For 102 * MINOR registered range, fault around will be a total disaster and ptes can 103 * be installed without notifications; for WP it should mostly be fine as long 104 * as the fault around checks for pte_none() before the installation, however 105 * to be super safe we just forbid it. 106 */ 107 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 108 { 109 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 110 } 111 112 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 113 { 114 return vma->vm_flags & VM_UFFD_MISSING; 115 } 116 117 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 118 { 119 return vma->vm_flags & VM_UFFD_WP; 120 } 121 122 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 123 { 124 return vma->vm_flags & VM_UFFD_MINOR; 125 } 126 127 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 128 pte_t pte) 129 { 130 return userfaultfd_wp(vma) && pte_uffd_wp(pte); 131 } 132 133 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 134 pmd_t pmd) 135 { 136 return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); 137 } 138 139 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 140 { 141 return vma->vm_flags & __VM_UFFD_FLAGS; 142 } 143 144 static inline bool vma_can_userfault(struct vm_area_struct *vma, 145 unsigned long vm_flags) 146 { 147 if ((vm_flags & VM_UFFD_MINOR) && 148 (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma))) 149 return false; 150 #ifndef CONFIG_PTE_MARKER_UFFD_WP 151 /* 152 * If user requested uffd-wp but not enabled pte markers for 153 * uffd-wp, then shmem & hugetlbfs are not supported but only 154 * anonymous. 155 */ 156 if ((vm_flags & VM_UFFD_WP) && !vma_is_anonymous(vma)) 157 return false; 158 #endif 159 return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) || 160 vma_is_shmem(vma); 161 } 162 163 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *); 164 extern void dup_userfaultfd_complete(struct list_head *); 165 166 extern void mremap_userfaultfd_prep(struct vm_area_struct *, 167 struct vm_userfaultfd_ctx *); 168 extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *, 169 unsigned long from, unsigned long to, 170 unsigned long len); 171 172 extern bool userfaultfd_remove(struct vm_area_struct *vma, 173 unsigned long start, 174 unsigned long end); 175 176 extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, 177 unsigned long end, struct list_head *uf); 178 extern void userfaultfd_unmap_complete(struct mm_struct *mm, 179 struct list_head *uf); 180 181 #else /* CONFIG_USERFAULTFD */ 182 183 /* mm helpers */ 184 static inline vm_fault_t handle_userfault(struct vm_fault *vmf, 185 unsigned long reason) 186 { 187 return VM_FAULT_SIGBUS; 188 } 189 190 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 191 struct vm_userfaultfd_ctx vm_ctx) 192 { 193 return true; 194 } 195 196 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 197 { 198 return false; 199 } 200 201 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 202 { 203 return false; 204 } 205 206 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 207 { 208 return false; 209 } 210 211 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 212 pte_t pte) 213 { 214 return false; 215 } 216 217 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 218 pmd_t pmd) 219 { 220 return false; 221 } 222 223 224 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 225 { 226 return false; 227 } 228 229 static inline int dup_userfaultfd(struct vm_area_struct *vma, 230 struct list_head *l) 231 { 232 return 0; 233 } 234 235 static inline void dup_userfaultfd_complete(struct list_head *l) 236 { 237 } 238 239 static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma, 240 struct vm_userfaultfd_ctx *ctx) 241 { 242 } 243 244 static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx, 245 unsigned long from, 246 unsigned long to, 247 unsigned long len) 248 { 249 } 250 251 static inline bool userfaultfd_remove(struct vm_area_struct *vma, 252 unsigned long start, 253 unsigned long end) 254 { 255 return true; 256 } 257 258 static inline int userfaultfd_unmap_prep(struct mm_struct *mm, 259 unsigned long start, unsigned long end, 260 struct list_head *uf) 261 { 262 return 0; 263 } 264 265 static inline void userfaultfd_unmap_complete(struct mm_struct *mm, 266 struct list_head *uf) 267 { 268 } 269 270 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 271 { 272 return false; 273 } 274 275 #endif /* CONFIG_USERFAULTFD */ 276 277 static inline bool pte_marker_entry_uffd_wp(swp_entry_t entry) 278 { 279 #ifdef CONFIG_PTE_MARKER_UFFD_WP 280 return is_pte_marker_entry(entry) && 281 (pte_marker_get(entry) & PTE_MARKER_UFFD_WP); 282 #else 283 return false; 284 #endif 285 } 286 287 static inline bool pte_marker_uffd_wp(pte_t pte) 288 { 289 #ifdef CONFIG_PTE_MARKER_UFFD_WP 290 swp_entry_t entry; 291 292 if (!is_swap_pte(pte)) 293 return false; 294 295 entry = pte_to_swp_entry(pte); 296 297 return pte_marker_entry_uffd_wp(entry); 298 #else 299 return false; 300 #endif 301 } 302 303 /* 304 * Returns true if this is a swap pte and was uffd-wp wr-protected in either 305 * forms (pte marker or a normal swap pte), false otherwise. 306 */ 307 static inline bool pte_swp_uffd_wp_any(pte_t pte) 308 { 309 #ifdef CONFIG_PTE_MARKER_UFFD_WP 310 if (!is_swap_pte(pte)) 311 return false; 312 313 if (pte_swp_uffd_wp(pte)) 314 return true; 315 316 if (pte_marker_uffd_wp(pte)) 317 return true; 318 #endif 319 return false; 320 } 321 322 #endif /* _LINUX_USERFAULTFD_K_H */ 323