1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * include/linux/userfaultfd_k.h 4 * 5 * Copyright (C) 2015 Red Hat, Inc. 6 * 7 */ 8 9 #ifndef _LINUX_USERFAULTFD_K_H 10 #define _LINUX_USERFAULTFD_K_H 11 12 #ifdef CONFIG_USERFAULTFD 13 14 #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */ 15 16 #include <linux/fcntl.h> 17 #include <linux/mm.h> 18 #include <asm-generic/pgtable_uffd.h> 19 20 /* The set of all possible UFFD-related VM flags. */ 21 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) 22 23 /* 24 * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining 25 * new flags, since they might collide with O_* ones. We want 26 * to re-use O_* flags that couldn't possibly have a meaning 27 * from userfaultfd, in order to leave a free define-space for 28 * shared O_* flags. 29 */ 30 #define UFFD_CLOEXEC O_CLOEXEC 31 #define UFFD_NONBLOCK O_NONBLOCK 32 33 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) 34 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) 35 36 extern int sysctl_unprivileged_userfaultfd; 37 38 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); 39 40 /* 41 * The mode of operation for __mcopy_atomic and its helpers. 42 * 43 * This is almost an implementation detail (mcopy_atomic below doesn't take this 44 * as a parameter), but it's exposed here because memory-kind-specific 45 * implementations (e.g. hugetlbfs) need to know the mode of operation. 46 */ 47 enum mcopy_atomic_mode { 48 /* A normal copy_from_user into the destination range. */ 49 MCOPY_ATOMIC_NORMAL, 50 /* Don't copy; map the destination range to the zero page. */ 51 MCOPY_ATOMIC_ZEROPAGE, 52 /* Just install pte(s) with the existing page(s) in the page cache. */ 53 MCOPY_ATOMIC_CONTINUE, 54 }; 55 56 extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, 57 unsigned long src_start, unsigned long len, 58 bool *mmap_changing, __u64 mode); 59 extern ssize_t mfill_zeropage(struct mm_struct *dst_mm, 60 unsigned long dst_start, 61 unsigned long len, 62 bool *mmap_changing); 63 extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start, 64 unsigned long len, bool *mmap_changing); 65 extern int mwriteprotect_range(struct mm_struct *dst_mm, 66 unsigned long start, unsigned long len, 67 bool enable_wp, bool *mmap_changing); 68 69 /* mm helpers */ 70 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 71 struct vm_userfaultfd_ctx vm_ctx) 72 { 73 return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx; 74 } 75 76 /* 77 * Never enable huge pmd sharing on some uffd registered vmas: 78 * 79 * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. 80 * 81 * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for 82 * VMAs which share huge pmds. (If you have two mappings to the same 83 * underlying pages, and fault in the non-UFFD-registered one with a write, 84 * with huge pmd sharing this would *also* setup the second UFFD-registered 85 * mapping, and we'd not get minor faults.) 86 */ 87 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) 88 { 89 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 90 } 91 92 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 93 { 94 return vma->vm_flags & VM_UFFD_MISSING; 95 } 96 97 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 98 { 99 return vma->vm_flags & VM_UFFD_WP; 100 } 101 102 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 103 { 104 return vma->vm_flags & VM_UFFD_MINOR; 105 } 106 107 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 108 pte_t pte) 109 { 110 return userfaultfd_wp(vma) && pte_uffd_wp(pte); 111 } 112 113 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 114 pmd_t pmd) 115 { 116 return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); 117 } 118 119 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 120 { 121 return vma->vm_flags & __VM_UFFD_FLAGS; 122 } 123 124 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *); 125 extern void dup_userfaultfd_complete(struct list_head *); 126 127 extern void mremap_userfaultfd_prep(struct vm_area_struct *, 128 struct vm_userfaultfd_ctx *); 129 extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *, 130 unsigned long from, unsigned long to, 131 unsigned long len); 132 133 extern bool userfaultfd_remove(struct vm_area_struct *vma, 134 unsigned long start, 135 unsigned long end); 136 137 extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, 138 unsigned long start, unsigned long end, 139 struct list_head *uf); 140 extern void userfaultfd_unmap_complete(struct mm_struct *mm, 141 struct list_head *uf); 142 143 #else /* CONFIG_USERFAULTFD */ 144 145 /* mm helpers */ 146 static inline vm_fault_t handle_userfault(struct vm_fault *vmf, 147 unsigned long reason) 148 { 149 return VM_FAULT_SIGBUS; 150 } 151 152 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 153 struct vm_userfaultfd_ctx vm_ctx) 154 { 155 return true; 156 } 157 158 static inline bool userfaultfd_missing(struct vm_area_struct *vma) 159 { 160 return false; 161 } 162 163 static inline bool userfaultfd_wp(struct vm_area_struct *vma) 164 { 165 return false; 166 } 167 168 static inline bool userfaultfd_minor(struct vm_area_struct *vma) 169 { 170 return false; 171 } 172 173 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 174 pte_t pte) 175 { 176 return false; 177 } 178 179 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 180 pmd_t pmd) 181 { 182 return false; 183 } 184 185 186 static inline bool userfaultfd_armed(struct vm_area_struct *vma) 187 { 188 return false; 189 } 190 191 static inline int dup_userfaultfd(struct vm_area_struct *vma, 192 struct list_head *l) 193 { 194 return 0; 195 } 196 197 static inline void dup_userfaultfd_complete(struct list_head *l) 198 { 199 } 200 201 static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma, 202 struct vm_userfaultfd_ctx *ctx) 203 { 204 } 205 206 static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx, 207 unsigned long from, 208 unsigned long to, 209 unsigned long len) 210 { 211 } 212 213 static inline bool userfaultfd_remove(struct vm_area_struct *vma, 214 unsigned long start, 215 unsigned long end) 216 { 217 return true; 218 } 219 220 static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, 221 unsigned long start, unsigned long end, 222 struct list_head *uf) 223 { 224 return 0; 225 } 226 227 static inline void userfaultfd_unmap_complete(struct mm_struct *mm, 228 struct list_head *uf) 229 { 230 } 231 232 #endif /* CONFIG_USERFAULTFD */ 233 234 #endif /* _LINUX_USERFAULTFD_K_H */ 235