1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * include/linux/userfaultfd_k.h 4 * 5 * Copyright (C) 2015 Red Hat, Inc. 6 * 7 */ 8 9 #ifndef _LINUX_USERFAULTFD_K_H 10 #define _LINUX_USERFAULTFD_K_H 11 12 #ifdef CONFIG_USERFAULTFD 13 14 #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */ 15 16 #include <linux/fcntl.h> 17 #include <linux/mm.h> 18 #include <linux/swap.h> 19 #include <linux/swapops.h> 20 #include <asm-generic/pgtable_uffd.h> 21 22 /* The set of all possible UFFD-related VM flags. */ 23 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) 24 25 /* 26 * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining 27 * new flags, since they might collide with O_* ones. We want 28 * to re-use O_* flags that couldn't possibly have a meaning 29 * from userfaultfd, in order to leave a free define-space for 30 * shared O_* flags. 31 */ 32 #define UFFD_CLOEXEC O_CLOEXEC 33 #define UFFD_NONBLOCK O_NONBLOCK 34 35 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) 36 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) 37 38 extern int sysctl_unprivileged_userfaultfd; 39 40 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); 41 42 /* 43 * The mode of operation for __mcopy_atomic and its helpers. 44 * 45 * This is almost an implementation detail (mcopy_atomic below doesn't take this 46 * as a parameter), but it's exposed here because memory-kind-specific 47 * implementations (e.g. hugetlbfs) need to know the mode of operation. 48 */ 49 enum mcopy_atomic_mode { 50 /* A normal copy_from_user into the destination range. */ 51 MCOPY_ATOMIC_NORMAL, 52 /* Don't copy; map the destination range to the zero page. */ 53 MCOPY_ATOMIC_ZEROPAGE, 54 /* Just install pte(s) with the existing page(s) in the page cache. */ 55 MCOPY_ATOMIC_CONTINUE, 56 }; 57 58 extern int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, 59 struct vm_area_struct *dst_vma, 60 unsigned long dst_addr, struct page *page, 61 bool newly_allocated, bool wp_copy); 62 63 extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, 64 unsigned long src_start, unsigned long len, 65 atomic_t *mmap_changing, __u64 mode); 66 extern ssize_t mfill_zeropage(struct mm_struct *dst_mm, 67 unsigned long dst_start, 68 unsigned long len, 69 atomic_t *mmap_changing); 70 extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start, 71 unsigned long len, atomic_t *mmap_changing); 72 extern int mwriteprotect_range(struct mm_struct *dst_mm, 73 unsigned long start, unsigned long len, 74 bool enable_wp, atomic_t *mmap_changing); 75 76 /* mm helpers */ 77 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 78 struct vm_userfaultfd_ctx vm_ctx) 79 { 80 return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx; 81 } 82 83 /* 84 * Never enable huge pmd sharing on some uffd registered vmas: 85 * 86 * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. 87 * 88 * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for 89 * VMAs which share huge pmds. (If you have two mappings to the same 90 * underlying pages, and fault in the non-UFFD-registered one with a write, 91 * with huge pmd sharing this would *also* setup the second UFFD-registered 92 * mapping, and we'd not get minor faults.) 93 */ 94 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) 95 { 96 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 97 } 98 99 /* 100 * Don't do fault around for either WP or MINOR registered uffd range. For 101 * MINOR registered range, fault around will be a total disaster and ptes can 102 * be installed without notifications; for WP it should mostly be fine as long 103 * as the fault around checks for pte_none() before the installation, however 104 * to be super safe we just forbid it. 105 */ 106 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 107 { 108 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 109 } 110 111 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 112 { 113 return vma->vm_flags & VM_UFFD_MISSING; 114 } 115 116 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 117 { 118 return vma->vm_flags & VM_UFFD_WP; 119 } 120 121 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 122 { 123 return vma->vm_flags & VM_UFFD_MINOR; 124 } 125 126 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 127 pte_t pte) 128 { 129 return userfaultfd_wp(vma) && pte_uffd_wp(pte); 130 } 131 132 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 133 pmd_t pmd) 134 { 135 return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); 136 } 137 138 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 139 { 140 return vma->vm_flags & __VM_UFFD_FLAGS; 141 } 142 143 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *); 144 extern void dup_userfaultfd_complete(struct list_head *); 145 146 extern void mremap_userfaultfd_prep(struct vm_area_struct *, 147 struct vm_userfaultfd_ctx *); 148 extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *, 149 unsigned long from, unsigned long to, 150 unsigned long len); 151 152 extern bool userfaultfd_remove(struct vm_area_struct *vma, 153 unsigned long start, 154 unsigned long end); 155 156 extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, 157 unsigned long start, unsigned long end, 158 struct list_head *uf); 159 extern void userfaultfd_unmap_complete(struct mm_struct *mm, 160 struct list_head *uf); 161 162 #else /* CONFIG_USERFAULTFD */ 163 164 /* mm helpers */ 165 static inline vm_fault_t handle_userfault(struct vm_fault *vmf, 166 unsigned long reason) 167 { 168 return VM_FAULT_SIGBUS; 169 } 170 171 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 172 struct vm_userfaultfd_ctx vm_ctx) 173 { 174 return true; 175 } 176 177 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 178 { 179 return false; 180 } 181 182 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 183 { 184 return false; 185 } 186 187 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 188 { 189 return false; 190 } 191 192 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 193 pte_t pte) 194 { 195 return false; 196 } 197 198 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 199 pmd_t pmd) 200 { 201 return false; 202 } 203 204 205 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 206 { 207 return false; 208 } 209 210 static inline int dup_userfaultfd(struct vm_area_struct *vma, 211 struct list_head *l) 212 { 213 return 0; 214 } 215 216 static inline void dup_userfaultfd_complete(struct list_head *l) 217 { 218 } 219 220 static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma, 221 struct vm_userfaultfd_ctx *ctx) 222 { 223 } 224 225 static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx, 226 unsigned long from, 227 unsigned long to, 228 unsigned long len) 229 { 230 } 231 232 static inline bool userfaultfd_remove(struct vm_area_struct *vma, 233 unsigned long start, 234 unsigned long end) 235 { 236 return true; 237 } 238 239 static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, 240 unsigned long start, unsigned long end, 241 struct list_head *uf) 242 { 243 return 0; 244 } 245 246 static inline void userfaultfd_unmap_complete(struct mm_struct *mm, 247 struct list_head *uf) 248 { 249 } 250 251 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 252 { 253 return false; 254 } 255 256 #endif /* CONFIG_USERFAULTFD */ 257 258 static inline bool pte_marker_entry_uffd_wp(swp_entry_t entry) 259 { 260 #ifdef CONFIG_PTE_MARKER_UFFD_WP 261 return is_pte_marker_entry(entry) && 262 (pte_marker_get(entry) & PTE_MARKER_UFFD_WP); 263 #else 264 return false; 265 #endif 266 } 267 268 static inline bool pte_marker_uffd_wp(pte_t pte) 269 { 270 #ifdef CONFIG_PTE_MARKER_UFFD_WP 271 swp_entry_t entry; 272 273 if (!is_swap_pte(pte)) 274 return false; 275 276 entry = pte_to_swp_entry(pte); 277 278 return pte_marker_entry_uffd_wp(entry); 279 #else 280 return false; 281 #endif 282 } 283 284 /* 285 * Returns true if this is a swap pte and was uffd-wp wr-protected in either 286 * forms (pte marker or a normal swap pte), false otherwise. 287 */ 288 static inline bool pte_swp_uffd_wp_any(pte_t pte) 289 { 290 #ifdef CONFIG_PTE_MARKER_UFFD_WP 291 if (!is_swap_pte(pte)) 292 return false; 293 294 if (pte_swp_uffd_wp(pte)) 295 return true; 296 297 if (pte_marker_uffd_wp(pte)) 298 return true; 299 #endif 300 return false; 301 } 302 303 #endif /* _LINUX_USERFAULTFD_K_H */ 304