1 #ifndef _LINUX_HUGE_MM_H 2 #define _LINUX_HUGE_MM_H 3 4 extern int do_huge_pmd_anonymous_page(struct mm_struct *mm, 5 struct vm_area_struct *vma, 6 unsigned long address, pmd_t *pmd, 7 unsigned int flags); 8 extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 9 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 10 struct vm_area_struct *vma); 11 extern void huge_pmd_set_accessed(struct mm_struct *mm, 12 struct vm_area_struct *vma, 13 unsigned long address, pmd_t *pmd, 14 pmd_t orig_pmd, int dirty); 15 extern int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 16 unsigned long address, pmd_t *pmd, 17 pmd_t orig_pmd); 18 extern struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, 19 unsigned long addr, 20 pmd_t *pmd, 21 unsigned int flags); 22 extern int zap_huge_pmd(struct mmu_gather *tlb, 23 struct vm_area_struct *vma, 24 pmd_t *pmd, unsigned long addr); 25 extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 26 unsigned long addr, unsigned long end, 27 unsigned char *vec); 28 extern int move_huge_pmd(struct vm_area_struct *vma, 29 struct vm_area_struct *new_vma, 30 unsigned long old_addr, 31 unsigned long new_addr, unsigned long old_end, 32 pmd_t *old_pmd, pmd_t *new_pmd); 33 extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 34 unsigned long addr, pgprot_t newprot); 35 36 enum transparent_hugepage_flag { 37 TRANSPARENT_HUGEPAGE_FLAG, 38 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 39 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, 40 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 41 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG, 42 #ifdef CONFIG_DEBUG_VM 43 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG, 44 #endif 45 }; 46 47 enum page_check_address_pmd_flag { 48 PAGE_CHECK_ADDRESS_PMD_FLAG, 49 PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG, 50 PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG, 51 }; 52 extern pmd_t *page_check_address_pmd(struct page *page, 53 struct mm_struct *mm, 54 unsigned long address, 55 enum page_check_address_pmd_flag flag); 56 57 #define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT) 58 #define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER) 59 60 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 61 #define HPAGE_PMD_SHIFT HPAGE_SHIFT 62 #define HPAGE_PMD_MASK HPAGE_MASK 63 #define HPAGE_PMD_SIZE HPAGE_SIZE 64 65 extern bool is_vma_temporary_stack(struct vm_area_struct *vma); 66 67 #define transparent_hugepage_enabled(__vma) \ 68 ((transparent_hugepage_flags & \ 69 (1<<TRANSPARENT_HUGEPAGE_FLAG) || \ 70 (transparent_hugepage_flags & \ 71 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) && \ 72 ((__vma)->vm_flags & VM_HUGEPAGE))) && \ 73 !((__vma)->vm_flags & VM_NOHUGEPAGE) && \ 74 !is_vma_temporary_stack(__vma)) 75 #define transparent_hugepage_defrag(__vma) \ 76 ((transparent_hugepage_flags & \ 77 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) || \ 78 (transparent_hugepage_flags & \ 79 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) && \ 80 (__vma)->vm_flags & VM_HUGEPAGE)) 81 #ifdef CONFIG_DEBUG_VM 82 #define transparent_hugepage_debug_cow() \ 83 (transparent_hugepage_flags & \ 84 (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG)) 85 #else /* CONFIG_DEBUG_VM */ 86 #define transparent_hugepage_debug_cow() 0 87 #endif /* CONFIG_DEBUG_VM */ 88 89 extern unsigned long transparent_hugepage_flags; 90 extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 91 pmd_t *dst_pmd, pmd_t *src_pmd, 92 struct vm_area_struct *vma, 93 unsigned long addr, unsigned long end); 94 extern int handle_pte_fault(struct mm_struct *mm, 95 struct vm_area_struct *vma, unsigned long address, 96 pte_t *pte, pmd_t *pmd, unsigned int flags); 97 extern int split_huge_page(struct page *page); 98 extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd); 99 #define split_huge_page_pmd(__mm, __pmd) \ 100 do { \ 101 pmd_t *____pmd = (__pmd); \ 102 if (unlikely(pmd_trans_huge(*____pmd))) \ 103 __split_huge_page_pmd(__mm, ____pmd); \ 104 } while (0) 105 #define wait_split_huge_page(__anon_vma, __pmd) \ 106 do { \ 107 pmd_t *____pmd = (__pmd); \ 108 anon_vma_lock(__anon_vma); \ 109 anon_vma_unlock(__anon_vma); \ 110 BUG_ON(pmd_trans_splitting(*____pmd) || \ 111 pmd_trans_huge(*____pmd)); \ 112 } while (0) 113 #if HPAGE_PMD_ORDER > MAX_ORDER 114 #error "hugepages can't be allocated by the buddy allocator" 115 #endif 116 extern int hugepage_madvise(struct vm_area_struct *vma, 117 unsigned long *vm_flags, int advice); 118 extern void __vma_adjust_trans_huge(struct vm_area_struct *vma, 119 unsigned long start, 120 unsigned long end, 121 long adjust_next); 122 extern int __pmd_trans_huge_lock(pmd_t *pmd, 123 struct vm_area_struct *vma); 124 /* mmap_sem must be held on entry */ 125 static inline int pmd_trans_huge_lock(pmd_t *pmd, 126 struct vm_area_struct *vma) 127 { 128 VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem)); 129 if (pmd_trans_huge(*pmd)) 130 return __pmd_trans_huge_lock(pmd, vma); 131 else 132 return 0; 133 } 134 static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, 135 unsigned long start, 136 unsigned long end, 137 long adjust_next) 138 { 139 if (!vma->anon_vma || vma->vm_ops) 140 return; 141 __vma_adjust_trans_huge(vma, start, end, adjust_next); 142 } 143 static inline int hpage_nr_pages(struct page *page) 144 { 145 if (unlikely(PageTransHuge(page))) 146 return HPAGE_PMD_NR; 147 return 1; 148 } 149 static inline struct page *compound_trans_head(struct page *page) 150 { 151 if (PageTail(page)) { 152 struct page *head; 153 head = page->first_page; 154 smp_rmb(); 155 /* 156 * head may be a dangling pointer. 157 * __split_huge_page_refcount clears PageTail before 158 * overwriting first_page, so if PageTail is still 159 * there it means the head pointer isn't dangling. 160 */ 161 if (PageTail(page)) 162 return head; 163 } 164 return page; 165 } 166 #else /* CONFIG_TRANSPARENT_HUGEPAGE */ 167 #define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; }) 168 #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; }) 169 #define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; }) 170 171 #define hpage_nr_pages(x) 1 172 173 #define transparent_hugepage_enabled(__vma) 0 174 175 #define transparent_hugepage_flags 0UL 176 static inline int split_huge_page(struct page *page) 177 { 178 return 0; 179 } 180 #define split_huge_page_pmd(__mm, __pmd) \ 181 do { } while (0) 182 #define wait_split_huge_page(__anon_vma, __pmd) \ 183 do { } while (0) 184 #define compound_trans_head(page) compound_head(page) 185 static inline int hugepage_madvise(struct vm_area_struct *vma, 186 unsigned long *vm_flags, int advice) 187 { 188 BUG(); 189 return 0; 190 } 191 static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, 192 unsigned long start, 193 unsigned long end, 194 long adjust_next) 195 { 196 } 197 static inline int pmd_trans_huge_lock(pmd_t *pmd, 198 struct vm_area_struct *vma) 199 { 200 return 0; 201 } 202 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 203 204 #endif /* _LINUX_HUGE_MM_H */ 205