1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_MIGRATE_H 3 #define _LINUX_MIGRATE_H 4 5 #include <linux/mm.h> 6 #include <linux/mempolicy.h> 7 #include <linux/migrate_mode.h> 8 #include <linux/hugetlb.h> 9 10 typedef struct page *new_page_t(struct page *page, unsigned long private); 11 typedef void free_page_t(struct page *page, unsigned long private); 12 13 /* 14 * Return values from addresss_space_operations.migratepage(): 15 * - negative errno on page migration failure; 16 * - zero on page migration success; 17 */ 18 #define MIGRATEPAGE_SUCCESS 0 19 20 enum migrate_reason { 21 MR_COMPACTION, 22 MR_MEMORY_FAILURE, 23 MR_MEMORY_HOTPLUG, 24 MR_SYSCALL, /* also applies to cpusets */ 25 MR_MEMPOLICY_MBIND, 26 MR_NUMA_MISPLACED, 27 MR_CONTIG_RANGE, 28 MR_TYPES 29 }; 30 31 /* In mm/debug.c; also keep sync with include/trace/events/migrate.h */ 32 extern const char *migrate_reason_names[MR_TYPES]; 33 34 static inline struct page *new_page_nodemask(struct page *page, 35 int preferred_nid, nodemask_t *nodemask) 36 { 37 gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL; 38 unsigned int order = 0; 39 struct page *new_page = NULL; 40 41 if (PageHuge(page)) 42 return alloc_huge_page_nodemask(page_hstate(compound_head(page)), 43 preferred_nid, nodemask); 44 45 if (PageTransHuge(page)) { 46 gfp_mask |= GFP_TRANSHUGE; 47 order = HPAGE_PMD_ORDER; 48 } 49 50 if (PageHighMem(page) || (zone_idx(page_zone(page)) == ZONE_MOVABLE)) 51 gfp_mask |= __GFP_HIGHMEM; 52 53 new_page = __alloc_pages_nodemask(gfp_mask, order, 54 preferred_nid, nodemask); 55 56 if (new_page && PageTransHuge(new_page)) 57 prep_transhuge_page(new_page); 58 59 return new_page; 60 } 61 62 #ifdef CONFIG_MIGRATION 63 64 extern void putback_movable_pages(struct list_head *l); 65 extern int migrate_page(struct address_space *mapping, 66 struct page *newpage, struct page *page, 67 enum migrate_mode mode); 68 extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free, 69 unsigned long private, enum migrate_mode mode, int reason); 70 extern int isolate_movable_page(struct page *page, isolate_mode_t mode); 71 extern void putback_movable_page(struct page *page); 72 73 extern int migrate_prep(void); 74 extern int migrate_prep_local(void); 75 extern void migrate_page_states(struct page *newpage, struct page *page); 76 extern void migrate_page_copy(struct page *newpage, struct page *page); 77 extern int migrate_huge_page_move_mapping(struct address_space *mapping, 78 struct page *newpage, struct page *page); 79 extern int migrate_page_move_mapping(struct address_space *mapping, 80 struct page *newpage, struct page *page, int extra_count); 81 #else 82 83 static inline void putback_movable_pages(struct list_head *l) {} 84 static inline int migrate_pages(struct list_head *l, new_page_t new, 85 free_page_t free, unsigned long private, enum migrate_mode mode, 86 int reason) 87 { return -ENOSYS; } 88 static inline int isolate_movable_page(struct page *page, isolate_mode_t mode) 89 { return -EBUSY; } 90 91 static inline int migrate_prep(void) { return -ENOSYS; } 92 static inline int migrate_prep_local(void) { return -ENOSYS; } 93 94 static inline void migrate_page_states(struct page *newpage, struct page *page) 95 { 96 } 97 98 static inline void migrate_page_copy(struct page *newpage, 99 struct page *page) {} 100 101 static inline int migrate_huge_page_move_mapping(struct address_space *mapping, 102 struct page *newpage, struct page *page) 103 { 104 return -ENOSYS; 105 } 106 107 #endif /* CONFIG_MIGRATION */ 108 109 #ifdef CONFIG_COMPACTION 110 extern int PageMovable(struct page *page); 111 extern void __SetPageMovable(struct page *page, struct address_space *mapping); 112 extern void __ClearPageMovable(struct page *page); 113 #else 114 static inline int PageMovable(struct page *page) { return 0; }; 115 static inline void __SetPageMovable(struct page *page, 116 struct address_space *mapping) 117 { 118 } 119 static inline void __ClearPageMovable(struct page *page) 120 { 121 } 122 #endif 123 124 #ifdef CONFIG_NUMA_BALANCING 125 extern bool pmd_trans_migrating(pmd_t pmd); 126 extern int migrate_misplaced_page(struct page *page, 127 struct vm_area_struct *vma, int node); 128 #else 129 static inline bool pmd_trans_migrating(pmd_t pmd) 130 { 131 return false; 132 } 133 static inline int migrate_misplaced_page(struct page *page, 134 struct vm_area_struct *vma, int node) 135 { 136 return -EAGAIN; /* can't migrate now */ 137 } 138 #endif /* CONFIG_NUMA_BALANCING */ 139 140 #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE) 141 extern int migrate_misplaced_transhuge_page(struct mm_struct *mm, 142 struct vm_area_struct *vma, 143 pmd_t *pmd, pmd_t entry, 144 unsigned long address, 145 struct page *page, int node); 146 #else 147 static inline int migrate_misplaced_transhuge_page(struct mm_struct *mm, 148 struct vm_area_struct *vma, 149 pmd_t *pmd, pmd_t entry, 150 unsigned long address, 151 struct page *page, int node) 152 { 153 return -EAGAIN; 154 } 155 #endif /* CONFIG_NUMA_BALANCING && CONFIG_TRANSPARENT_HUGEPAGE*/ 156 157 158 #ifdef CONFIG_MIGRATION 159 160 /* 161 * Watch out for PAE architecture, which has an unsigned long, and might not 162 * have enough bits to store all physical address and flags. So far we have 163 * enough room for all our flags. 164 */ 165 #define MIGRATE_PFN_VALID (1UL << 0) 166 #define MIGRATE_PFN_MIGRATE (1UL << 1) 167 #define MIGRATE_PFN_LOCKED (1UL << 2) 168 #define MIGRATE_PFN_WRITE (1UL << 3) 169 #define MIGRATE_PFN_DEVICE (1UL << 4) 170 #define MIGRATE_PFN_ERROR (1UL << 5) 171 #define MIGRATE_PFN_SHIFT 6 172 173 static inline struct page *migrate_pfn_to_page(unsigned long mpfn) 174 { 175 if (!(mpfn & MIGRATE_PFN_VALID)) 176 return NULL; 177 return pfn_to_page(mpfn >> MIGRATE_PFN_SHIFT); 178 } 179 180 static inline unsigned long migrate_pfn(unsigned long pfn) 181 { 182 return (pfn << MIGRATE_PFN_SHIFT) | MIGRATE_PFN_VALID; 183 } 184 185 /* 186 * struct migrate_vma_ops - migrate operation callback 187 * 188 * @alloc_and_copy: alloc destination memory and copy source memory to it 189 * @finalize_and_map: allow caller to map the successfully migrated pages 190 * 191 * 192 * The alloc_and_copy() callback happens once all source pages have been locked, 193 * unmapped and checked (checked whether pinned or not). All pages that can be 194 * migrated will have an entry in the src array set with the pfn value of the 195 * page and with the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set (other 196 * flags might be set but should be ignored by the callback). 197 * 198 * The alloc_and_copy() callback can then allocate destination memory and copy 199 * source memory to it for all those entries (ie with MIGRATE_PFN_VALID and 200 * MIGRATE_PFN_MIGRATE flag set). Once these are allocated and copied, the 201 * callback must update each corresponding entry in the dst array with the pfn 202 * value of the destination page and with the MIGRATE_PFN_VALID and 203 * MIGRATE_PFN_LOCKED flags set (destination pages must have their struct pages 204 * locked, via lock_page()). 205 * 206 * At this point the alloc_and_copy() callback is done and returns. 207 * 208 * Note that the callback does not have to migrate all the pages that are 209 * marked with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration 210 * from device memory to system memory (ie the MIGRATE_PFN_DEVICE flag is also 211 * set in the src array entry). If the device driver cannot migrate a device 212 * page back to system memory, then it must set the corresponding dst array 213 * entry to MIGRATE_PFN_ERROR. This will trigger a SIGBUS if CPU tries to 214 * access any of the virtual addresses originally backed by this page. Because 215 * a SIGBUS is such a severe result for the userspace process, the device 216 * driver should avoid setting MIGRATE_PFN_ERROR unless it is really in an 217 * unrecoverable state. 218 * 219 * For empty entry inside CPU page table (pte_none() or pmd_none() is true) we 220 * do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus 221 * allowing device driver to allocate device memory for those unback virtual 222 * address. For this the device driver simply have to allocate device memory 223 * and properly set the destination entry like for regular migration. Note that 224 * this can still fails and thus inside the device driver must check if the 225 * migration was successful for those entry inside the finalize_and_map() 226 * callback just like for regular migration. 227 * 228 * THE alloc_and_copy() CALLBACK MUST NOT CHANGE ANY OF THE SRC ARRAY ENTRIES 229 * OR BAD THINGS WILL HAPPEN ! 230 * 231 * 232 * The finalize_and_map() callback happens after struct page migration from 233 * source to destination (destination struct pages are the struct pages for the 234 * memory allocated by the alloc_and_copy() callback). Migration can fail, and 235 * thus the finalize_and_map() allows the driver to inspect which pages were 236 * successfully migrated, and which were not. Successfully migrated pages will 237 * have the MIGRATE_PFN_MIGRATE flag set for their src array entry. 238 * 239 * It is safe to update device page table from within the finalize_and_map() 240 * callback because both destination and source page are still locked, and the 241 * mmap_sem is held in read mode (hence no one can unmap the range being 242 * migrated). 243 * 244 * Once callback is done cleaning up things and updating its page table (if it 245 * chose to do so, this is not an obligation) then it returns. At this point, 246 * the HMM core will finish up the final steps, and the migration is complete. 247 * 248 * THE finalize_and_map() CALLBACK MUST NOT CHANGE ANY OF THE SRC OR DST ARRAY 249 * ENTRIES OR BAD THINGS WILL HAPPEN ! 250 */ 251 struct migrate_vma_ops { 252 void (*alloc_and_copy)(struct vm_area_struct *vma, 253 const unsigned long *src, 254 unsigned long *dst, 255 unsigned long start, 256 unsigned long end, 257 void *private); 258 void (*finalize_and_map)(struct vm_area_struct *vma, 259 const unsigned long *src, 260 const unsigned long *dst, 261 unsigned long start, 262 unsigned long end, 263 void *private); 264 }; 265 266 #if defined(CONFIG_MIGRATE_VMA_HELPER) 267 int migrate_vma(const struct migrate_vma_ops *ops, 268 struct vm_area_struct *vma, 269 unsigned long start, 270 unsigned long end, 271 unsigned long *src, 272 unsigned long *dst, 273 void *private); 274 #else 275 static inline int migrate_vma(const struct migrate_vma_ops *ops, 276 struct vm_area_struct *vma, 277 unsigned long start, 278 unsigned long end, 279 unsigned long *src, 280 unsigned long *dst, 281 void *private) 282 { 283 return -EINVAL; 284 } 285 #endif /* IS_ENABLED(CONFIG_MIGRATE_VMA_HELPER) */ 286 287 #endif /* CONFIG_MIGRATION */ 288 289 #endif /* _LINUX_MIGRATE_H */ 290