Lines Matching refs:vma
107 if (!userfaultfd_wp(vmf->vma)) in vmf_orig_pte_uffd_wp()
347 struct vm_area_struct *vma, unsigned long floor, in free_pgtables() argument
353 unsigned long addr = vma->vm_start; in free_pgtables()
369 vma_start_write(vma); in free_pgtables()
370 unlink_anon_vmas(vma); in free_pgtables()
372 if (is_vm_hugetlb_page(vma)) { in free_pgtables()
373 unlink_file_vma(vma); in free_pgtables()
374 hugetlb_free_pgd_range(tlb, addr, vma->vm_end, in free_pgtables()
378 unlink_file_vma_batch_add(&vb, vma); in free_pgtables()
383 while (next && next->vm_start <= vma->vm_end + PMD_SIZE in free_pgtables()
385 vma = next; in free_pgtables()
390 vma_start_write(vma); in free_pgtables()
391 unlink_anon_vmas(vma); in free_pgtables()
392 unlink_file_vma_batch_add(&vb, vma); in free_pgtables()
395 free_pgd_range(tlb, addr, vma->vm_end, in free_pgtables()
398 vma = next; in free_pgtables()
399 } while (vma); in free_pgtables()
479 static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, in print_bad_pte() argument
482 pgd_t *pgd = pgd_offset(vma->vm_mm, addr); in print_bad_pte()
511 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; in print_bad_pte()
512 index = linear_page_index(vma, addr); in print_bad_pte()
520 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index); in print_bad_pte()
522 vma->vm_file, in print_bad_pte()
523 vma->vm_ops ? vma->vm_ops->fault : NULL, in print_bad_pte()
524 vma->vm_file ? vma->vm_file->f_op->mmap : NULL, in print_bad_pte()
575 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, in vm_normal_page() argument
583 if (vma->vm_ops && vma->vm_ops->find_special_page) in vm_normal_page()
584 return vma->vm_ops->find_special_page(vma, addr); in vm_normal_page()
585 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) in vm_normal_page()
600 print_bad_pte(vma, addr, pte, NULL); in vm_normal_page()
606 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { in vm_normal_page()
607 if (vma->vm_flags & VM_MIXEDMAP) { in vm_normal_page()
615 off = (addr - vma->vm_start) >> PAGE_SHIFT; in vm_normal_page()
616 if (pfn == vma->vm_pgoff + off) in vm_normal_page()
618 if (!is_cow_mapping(vma->vm_flags)) in vm_normal_page()
628 print_bad_pte(vma, addr, pte, NULL); in vm_normal_page()
641 struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, in vm_normal_folio() argument
644 struct page *page = vm_normal_page(vma, addr, pte); in vm_normal_folio()
652 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, in vm_normal_page_pmd() argument
661 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { in vm_normal_page_pmd()
662 if (vma->vm_flags & VM_MIXEDMAP) { in vm_normal_page_pmd()
668 off = (addr - vma->vm_start) >> PAGE_SHIFT; in vm_normal_page_pmd()
669 if (pfn == vma->vm_pgoff + off) in vm_normal_page_pmd()
671 if (!is_cow_mapping(vma->vm_flags)) in vm_normal_page_pmd()
691 struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, in vm_normal_folio_pmd() argument
694 struct page *page = vm_normal_page_pmd(vma, addr, pmd); in vm_normal_folio_pmd()
728 static void restore_exclusive_pte(struct vm_area_struct *vma, in restore_exclusive_pte() argument
736 pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot))); in restore_exclusive_pte()
743 if ((vma->vm_flags & VM_WRITE) && in restore_exclusive_pte()
744 can_change_pte_writable(vma, address, pte)) { in restore_exclusive_pte()
747 pte = pte_mkwrite(pte, vma); in restore_exclusive_pte()
749 set_pte_at(vma->vm_mm, address, ptep, pte); in restore_exclusive_pte()
755 update_mmu_cache(vma, address, ptep); in restore_exclusive_pte()
762 static int try_restore_exclusive_pte(struct vm_area_struct *vma, in try_restore_exclusive_pte() argument
769 restore_exclusive_pte(vma, folio, page, addr, ptep, orig_pte); in try_restore_exclusive_pte()
1049 struct vm_area_struct *vma, unsigned long addr, bool need_zero) in folio_prealloc() argument
1054 new_folio = vma_alloc_zeroed_movable_folio(vma, addr); in folio_prealloc()
1056 new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr); in folio_prealloc()
1464 zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, in zap_install_uffd_wp_if_needed() argument
1472 if (vma_is_anonymous(vma)) in zap_install_uffd_wp_if_needed()
1480 if (pte_install_uffd_wp_if_needed(vma, addr, pte, pteval)) in zap_install_uffd_wp_if_needed()
1492 struct vm_area_struct *vma, struct folio *folio, in zap_present_folio_ptes() argument
1509 if (pte_young(ptent) && likely(vma_has_recency(vma))) in zap_present_folio_ptes()
1518 arch_check_zapped_pte(vma, ptent); in zap_present_folio_ptes()
1520 if (unlikely(userfaultfd_pte_wp(vma, ptent))) in zap_present_folio_ptes()
1521 *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, in zap_present_folio_ptes()
1525 folio_remove_rmap_ptes(folio, page, nr, vma); in zap_present_folio_ptes()
1528 print_bad_pte(vma, addr, ptent, page); in zap_present_folio_ptes()
1543 struct vm_area_struct *vma, pte_t *pte, pte_t ptent, in zap_present_ptes() argument
1554 page = vm_normal_page(vma, addr, ptent); in zap_present_ptes()
1558 arch_check_zapped_pte(vma, ptent); in zap_present_ptes()
1560 if (userfaultfd_pte_wp(vma, ptent)) in zap_present_ptes()
1561 *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, in zap_present_ptes()
1581 zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, in zap_present_ptes()
1586 zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, 1, addr, in zap_present_ptes()
1592 struct vm_area_struct *vma, pte_t *pte, pte_t ptent, in zap_nonpresent_ptes() argument
1614 WARN_ON_ONCE(!vma_is_anonymous(vma)); in zap_nonpresent_ptes()
1616 folio_remove_rmap_pte(folio, page, vma); in zap_nonpresent_ptes()
1637 if (!vma_is_anonymous(vma) && !zap_drop_markers(details)) in zap_nonpresent_ptes()
1655 clear_not_present_full_ptes(vma->vm_mm, addr, pte, nr, tlb->fullmm); in zap_nonpresent_ptes()
1656 *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent); in zap_nonpresent_ptes()
1662 struct vm_area_struct *vma, pte_t *pte, in do_zap_pte_range() argument
1687 nr += zap_present_ptes(tlb, vma, pte, ptent, max_nr, addr, in do_zap_pte_range()
1691 nr += zap_nonpresent_ptes(tlb, vma, pte, ptent, max_nr, addr, in do_zap_pte_range()
1698 struct vm_area_struct *vma, pmd_t *pmd, in zap_pte_range() argument
1731 nr = do_zap_pte_range(tlb, vma, pte, addr, end, details, rss, in zap_pte_range()
1759 tlb_flush_rmaps(tlb, vma); in zap_pte_range()
1790 struct vm_area_struct *vma, pud_t *pud, in zap_pmd_range() argument
1802 __split_huge_pmd(vma, pmd, addr, false, NULL); in zap_pmd_range()
1803 else if (zap_huge_pmd(tlb, vma, pmd, addr)) { in zap_pmd_range()
1823 addr = zap_pte_range(tlb, vma, pmd, addr, next, details); in zap_pmd_range()
1832 struct vm_area_struct *vma, p4d_t *p4d, in zap_pud_range() argument
1845 split_huge_pud(vma, pud, addr); in zap_pud_range()
1846 } else if (zap_huge_pud(tlb, vma, pud, addr)) in zap_pud_range()
1852 next = zap_pmd_range(tlb, vma, pud, addr, next, details); in zap_pud_range()
1861 struct vm_area_struct *vma, pgd_t *pgd, in zap_p4d_range() argument
1873 next = zap_pud_range(tlb, vma, p4d, addr, next, details); in zap_p4d_range()
1880 struct vm_area_struct *vma, in unmap_page_range() argument
1888 tlb_start_vma(tlb, vma); in unmap_page_range()
1889 pgd = pgd_offset(vma->vm_mm, addr); in unmap_page_range()
1894 next = zap_p4d_range(tlb, vma, pgd, addr, next, details); in unmap_page_range()
1896 tlb_end_vma(tlb, vma); in unmap_page_range()
1901 struct vm_area_struct *vma, unsigned long start_addr, in unmap_single_vma() argument
1905 unsigned long start = max(vma->vm_start, start_addr); in unmap_single_vma()
1908 if (start >= vma->vm_end) in unmap_single_vma()
1910 end = min(vma->vm_end, end_addr); in unmap_single_vma()
1911 if (end <= vma->vm_start) in unmap_single_vma()
1914 if (vma->vm_file) in unmap_single_vma()
1915 uprobe_munmap(vma, start, end); in unmap_single_vma()
1917 if (unlikely(vma->vm_flags & VM_PFNMAP)) in unmap_single_vma()
1918 untrack_pfn(vma, 0, 0, mm_wr_locked); in unmap_single_vma()
1921 if (unlikely(is_vm_hugetlb_page(vma))) { in unmap_single_vma()
1933 if (vma->vm_file) { in unmap_single_vma()
1936 __unmap_hugepage_range(tlb, vma, start, end, in unmap_single_vma()
1940 unmap_page_range(tlb, vma, start, end, details); in unmap_single_vma()
1966 struct vm_area_struct *vma, unsigned long start_addr, in unmap_vmas() argument
1977 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm, in unmap_vmas()
1983 hugetlb_zap_begin(vma, &start, &end); in unmap_vmas()
1984 unmap_single_vma(tlb, vma, start, end, &details, in unmap_vmas()
1986 hugetlb_zap_end(vma, &details); in unmap_vmas()
1987 vma = mas_find(mas, tree_end - 1); in unmap_vmas()
1988 } while (vma && likely(!xa_is_zero(vma))); in unmap_vmas()
2001 void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, in zap_page_range_single() argument
2008 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, in zap_page_range_single()
2010 hugetlb_zap_begin(vma, &range.start, &range.end); in zap_page_range_single()
2011 tlb_gather_mmu(&tlb, vma->vm_mm); in zap_page_range_single()
2012 update_hiwater_rss(vma->vm_mm); in zap_page_range_single()
2018 unmap_single_vma(&tlb, vma, address, end, details, false); in zap_page_range_single()
2021 hugetlb_zap_end(vma, details); in zap_page_range_single()
2035 void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, in zap_vma_ptes() argument
2038 if (!range_in_vma(vma, address, address + size) || in zap_vma_ptes()
2039 !(vma->vm_flags & VM_PFNMAP)) in zap_vma_ptes()
2042 zap_page_range_single(vma, address, size, NULL); in zap_vma_ptes()
2078 static bool vm_mixed_zeropage_allowed(struct vm_area_struct *vma) in vm_mixed_zeropage_allowed() argument
2080 VM_WARN_ON_ONCE(vma->vm_flags & VM_PFNMAP); in vm_mixed_zeropage_allowed()
2087 if (mm_forbids_zeropage(vma->vm_mm)) in vm_mixed_zeropage_allowed()
2090 if (is_cow_mapping(vma->vm_flags)) in vm_mixed_zeropage_allowed()
2093 if (!(vma->vm_flags & (VM_WRITE | VM_MAYWRITE))) in vm_mixed_zeropage_allowed()
2104 return vma->vm_ops && vma->vm_ops->pfn_mkwrite && in vm_mixed_zeropage_allowed()
2105 (vma_is_fsdax(vma) || vma->vm_flags & VM_IO); in vm_mixed_zeropage_allowed()
2108 static int validate_page_before_insert(struct vm_area_struct *vma, in validate_page_before_insert() argument
2116 if (!vm_mixed_zeropage_allowed(vma)) in validate_page_before_insert()
2127 static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, in insert_page_into_pte_locked() argument
2143 pteval = maybe_mkwrite(pteval, vma); in insert_page_into_pte_locked()
2145 if (ptep_set_access_flags(vma, addr, pte, pteval, 1)) in insert_page_into_pte_locked()
2146 update_mmu_cache(vma, addr, pte); in insert_page_into_pte_locked()
2159 pteval = maybe_mkwrite(pte_mkdirty(pteval), vma); in insert_page_into_pte_locked()
2161 inc_mm_counter(vma->vm_mm, mm_counter_file(folio)); in insert_page_into_pte_locked()
2162 folio_add_file_rmap_pte(folio, page, vma); in insert_page_into_pte_locked()
2164 set_pte_at(vma->vm_mm, addr, pte, pteval); in insert_page_into_pte_locked()
2168 static int insert_page(struct vm_area_struct *vma, unsigned long addr, in insert_page() argument
2175 retval = validate_page_before_insert(vma, page); in insert_page()
2179 pte = get_locked_pte(vma->vm_mm, addr, &ptl); in insert_page()
2182 retval = insert_page_into_pte_locked(vma, pte, addr, page, prot, in insert_page()
2189 static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, in insert_page_in_batch_locked() argument
2194 err = validate_page_before_insert(vma, page); in insert_page_in_batch_locked()
2197 return insert_page_into_pte_locked(vma, pte, addr, page, prot, false); in insert_page_in_batch_locked()
2203 static int insert_pages(struct vm_area_struct *vma, unsigned long addr, in insert_pages() argument
2209 struct mm_struct *const mm = vma->vm_mm; in insert_pages()
2238 int err = insert_page_in_batch_locked(vma, pte, in insert_pages()
2276 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, in vm_insert_pages() argument
2281 if (addr < vma->vm_start || end_addr >= vma->vm_end) in vm_insert_pages()
2283 if (!(vma->vm_flags & VM_MIXEDMAP)) { in vm_insert_pages()
2284 BUG_ON(mmap_read_trylock(vma->vm_mm)); in vm_insert_pages()
2285 BUG_ON(vma->vm_flags & VM_PFNMAP); in vm_insert_pages()
2286 vm_flags_set(vma, VM_MIXEDMAP); in vm_insert_pages()
2289 return insert_pages(vma, addr, pages, num, vma->vm_page_prot); in vm_insert_pages()
2323 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, in vm_insert_page() argument
2326 if (addr < vma->vm_start || addr >= vma->vm_end) in vm_insert_page()
2328 if (!(vma->vm_flags & VM_MIXEDMAP)) { in vm_insert_page()
2329 BUG_ON(mmap_read_trylock(vma->vm_mm)); in vm_insert_page()
2330 BUG_ON(vma->vm_flags & VM_PFNMAP); in vm_insert_page()
2331 vm_flags_set(vma, VM_MIXEDMAP); in vm_insert_page()
2333 return insert_page(vma, addr, page, vma->vm_page_prot, false); in vm_insert_page()
2350 static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, in __vm_map_pages() argument
2353 unsigned long count = vma_pages(vma); in __vm_map_pages()
2354 unsigned long uaddr = vma->vm_start; in __vm_map_pages()
2366 ret = vm_insert_page(vma, uaddr, pages[offset + i]); in __vm_map_pages()
2393 int vm_map_pages(struct vm_area_struct *vma, struct page **pages, in vm_map_pages() argument
2396 return __vm_map_pages(vma, pages, num, vma->vm_pgoff); in vm_map_pages()
2413 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, in vm_map_pages_zero() argument
2416 return __vm_map_pages(vma, pages, num, 0); in vm_map_pages_zero()
2420 static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, in insert_pfn() argument
2423 struct mm_struct *mm = vma->vm_mm; in insert_pfn()
2448 entry = maybe_mkwrite(pte_mkdirty(entry), vma); in insert_pfn()
2449 if (ptep_set_access_flags(vma, addr, pte, entry, 1)) in insert_pfn()
2450 update_mmu_cache(vma, addr, pte); in insert_pfn()
2463 entry = maybe_mkwrite(pte_mkdirty(entry), vma); in insert_pfn()
2467 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */ in insert_pfn()
2507 vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, in vmf_insert_pfn_prot() argument
2516 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); in vmf_insert_pfn_prot()
2517 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == in vmf_insert_pfn_prot()
2519 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); in vmf_insert_pfn_prot()
2520 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); in vmf_insert_pfn_prot()
2522 if (addr < vma->vm_start || addr >= vma->vm_end) in vmf_insert_pfn_prot()
2528 track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV)); in vmf_insert_pfn_prot()
2530 return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot, in vmf_insert_pfn_prot()
2555 vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, in vmf_insert_pfn() argument
2558 return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot); in vmf_insert_pfn()
2562 static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn, bool mkwrite) in vm_mixed_ok() argument
2565 (mkwrite || !vm_mixed_zeropage_allowed(vma))) in vm_mixed_ok()
2568 if (vma->vm_flags & VM_MIXEDMAP) in vm_mixed_ok()
2579 static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma, in __vm_insert_mixed() argument
2582 pgprot_t pgprot = vma->vm_page_prot; in __vm_insert_mixed()
2585 if (!vm_mixed_ok(vma, pfn, mkwrite)) in __vm_insert_mixed()
2588 if (addr < vma->vm_start || addr >= vma->vm_end) in __vm_insert_mixed()
2591 track_pfn_insert(vma, &pgprot, pfn); in __vm_insert_mixed()
2613 err = insert_page(vma, addr, page, pgprot, mkwrite); in __vm_insert_mixed()
2615 return insert_pfn(vma, addr, pfn, pgprot, mkwrite); in __vm_insert_mixed()
2629 pgprot_t pgprot = vmf->vma->vm_page_prot; in vmf_insert_page_mkwrite()
2633 if (addr < vmf->vma->vm_start || addr >= vmf->vma->vm_end) in vmf_insert_page_mkwrite()
2636 err = insert_page(vmf->vma, addr, page, pgprot, write); in vmf_insert_page_mkwrite()
2646 vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, in vmf_insert_mixed() argument
2649 return __vm_insert_mixed(vma, addr, pfn, false); in vmf_insert_mixed()
2658 vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, in vmf_insert_mixed_mkwrite() argument
2661 return __vm_insert_mixed(vma, addr, pfn, true); in vmf_insert_mixed_mkwrite()
2762 static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr, in remap_pfn_range_internal() argument
2768 struct mm_struct *mm = vma->vm_mm; in remap_pfn_range_internal()
2792 if (is_cow_mapping(vma->vm_flags)) { in remap_pfn_range_internal()
2793 if (addr != vma->vm_start || end != vma->vm_end) in remap_pfn_range_internal()
2795 vma->vm_pgoff = pfn; in remap_pfn_range_internal()
2798 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); in remap_pfn_range_internal()
2803 flush_cache_range(vma, addr, end); in remap_pfn_range_internal()
2819 int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr, in remap_pfn_range_notrack() argument
2822 int error = remap_pfn_range_internal(vma, addr, pfn, size, prot); in remap_pfn_range_notrack()
2832 zap_page_range_single(vma, addr, size, NULL); in remap_pfn_range_notrack()
2848 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, in remap_pfn_range() argument
2853 err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size)); in remap_pfn_range()
2857 err = remap_pfn_range_notrack(vma, addr, pfn, size, prot); in remap_pfn_range()
2859 untrack_pfn(vma, pfn, PAGE_ALIGN(size), true); in remap_pfn_range()
2879 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len) in vm_iomap_memory() argument
2898 if (vma->vm_pgoff > pages) in vm_iomap_memory()
2900 pfn += vma->vm_pgoff; in vm_iomap_memory()
2901 pages -= vma->vm_pgoff; in vm_iomap_memory()
2904 vm_len = vma->vm_end - vma->vm_start; in vm_iomap_memory()
2909 return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); in vm_iomap_memory()
3164 struct vm_area_struct *vma = vmf->vma; in __wp_page_copy_user() local
3165 struct mm_struct *mm = vma->vm_mm; in __wp_page_copy_user()
3169 if (copy_mc_user_highpage(dst, src, addr, vma)) in __wp_page_copy_user()
3199 update_mmu_tlb(vma, addr, vmf->pte); in __wp_page_copy_user()
3205 if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0)) in __wp_page_copy_user()
3206 update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1); in __wp_page_copy_user()
3224 update_mmu_tlb(vma, addr, vmf->pte); in __wp_page_copy_user()
3256 static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma) in __get_fault_gfp_mask() argument
3258 struct file *vm_file = vma->vm_file; in __get_fault_gfp_mask()
3283 if (vmf->vma->vm_file && in do_page_mkwrite()
3284 IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host)) in do_page_mkwrite()
3287 ret = vmf->vma->vm_ops->page_mkwrite(vmf); in do_page_mkwrite()
3311 struct vm_area_struct *vma = vmf->vma; in fault_dirty_shared_page() local
3315 bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite; in fault_dirty_shared_page()
3329 file_update_time(vma->vm_file); in fault_dirty_shared_page()
3365 struct vm_area_struct *vma = vmf->vma; in wp_page_reuse() local
3382 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); in wp_page_reuse()
3384 entry = maybe_mkwrite(pte_mkdirty(entry), vma); in wp_page_reuse()
3385 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1)) in wp_page_reuse()
3386 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); in wp_page_reuse()
3398 struct vm_area_struct *vma = vmf->vma; in vmf_can_call_fault() local
3400 if (vma->vm_ops->map_pages || !(vmf->flags & FAULT_FLAG_VMA_LOCK)) in vmf_can_call_fault()
3402 vma_end_read(vma); in vmf_can_call_fault()
3423 struct vm_area_struct *vma = vmf->vma; in __vmf_anon_prepare() local
3426 if (likely(vma->anon_vma)) in __vmf_anon_prepare()
3429 if (!mmap_read_trylock(vma->vm_mm)) in __vmf_anon_prepare()
3432 if (__anon_vma_prepare(vma)) in __vmf_anon_prepare()
3435 mmap_read_unlock(vma->vm_mm); in __vmf_anon_prepare()
3459 struct vm_area_struct *vma = vmf->vma; in wp_page_copy() local
3460 struct mm_struct *mm = vma->vm_mm; in wp_page_copy()
3478 new_folio = folio_prealloc(mm, vma, vmf->address, pfn_is_zero); in wp_page_copy()
3525 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); in wp_page_copy()
3526 entry = mk_pte(&new_folio->page, vma->vm_page_prot); in wp_page_copy()
3534 entry = maybe_mkwrite(pte_mkdirty(entry), vma); in wp_page_copy()
3544 ptep_clear_flush(vma, vmf->address, vmf->pte); in wp_page_copy()
3545 folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE); in wp_page_copy()
3546 folio_add_lru_vma(new_folio, vma); in wp_page_copy()
3549 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); in wp_page_copy()
3573 folio_remove_rmap_pte(old_folio, vmf->page, vma); in wp_page_copy()
3581 update_mmu_tlb(vma, vmf->address, vmf->pte); in wp_page_copy()
3626 WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED)); in finish_mkwrite_fault()
3627 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address, in finish_mkwrite_fault()
3636 update_mmu_tlb(vmf->vma, vmf->address, vmf->pte); in finish_mkwrite_fault()
3650 struct vm_area_struct *vma = vmf->vma; in wp_pfn_shared() local
3652 if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) { in wp_pfn_shared()
3661 ret = vma->vm_ops->pfn_mkwrite(vmf); in wp_pfn_shared()
3673 struct vm_area_struct *vma = vmf->vma; in wp_page_shared() local
3678 if (vma->vm_ops && vma->vm_ops->page_mkwrite) { in wp_page_shared()
3712 struct vm_area_struct *vma) in __wp_can_reuse_large_anon_folio() argument
3763 VM_WARN_ON_ONCE(folio_mm_id(folio, 0) != vma->vm_mm->mm_id && in __wp_can_reuse_large_anon_folio()
3764 folio_mm_id(folio, 1) != vma->vm_mm->mm_id); in __wp_can_reuse_large_anon_folio()
3778 struct vm_area_struct *vma) in __wp_can_reuse_large_anon_folio() argument
3785 struct vm_area_struct *vma) in wp_can_reuse_anon_folio() argument
3788 return __wp_can_reuse_large_anon_folio(folio, vma); in wp_can_reuse_anon_folio()
3820 folio_move_anon_rmap(folio, vma); in wp_can_reuse_anon_folio()
3851 struct vm_area_struct *vma = vmf->vma; in do_wp_page() local
3856 if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { in do_wp_page()
3857 if (!userfaultfd_wp_async(vma)) { in do_wp_page()
3869 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); in do_wp_page()
3881 if (unlikely(userfaultfd_wp(vmf->vma) && in do_wp_page()
3882 mm_tlb_flush_pending(vmf->vma->vm_mm))) in do_wp_page()
3883 flush_tlb_page(vmf->vma, vmf->address); in do_wp_page()
3886 vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte); in do_wp_page()
3895 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) { in do_wp_page()
3918 (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) { in do_wp_page()
3942 static void unmap_mapping_range_vma(struct vm_area_struct *vma, in unmap_mapping_range_vma() argument
3946 zap_page_range_single(vma, start_addr, end_addr - start_addr, details); in unmap_mapping_range_vma()
3954 struct vm_area_struct *vma; in unmap_mapping_range_tree() local
3957 vma_interval_tree_foreach(vma, root, first_index, last_index) { in unmap_mapping_range_tree()
3958 vba = vma->vm_pgoff; in unmap_mapping_range_tree()
3959 vea = vba + vma_pages(vma) - 1; in unmap_mapping_range_tree()
3963 unmap_mapping_range_vma(vma, in unmap_mapping_range_tree()
3964 ((zba - vba) << PAGE_SHIFT) + vma->vm_start, in unmap_mapping_range_tree()
3965 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start, in unmap_mapping_range_tree()
4076 struct vm_area_struct *vma = vmf->vma; in remove_device_exclusive_entry() local
4097 vma->vm_mm, vmf->address & PAGE_MASK, in remove_device_exclusive_entry()
4101 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, in remove_device_exclusive_entry()
4104 restore_exclusive_pte(vma, folio, vmf->page, vmf->address, in remove_device_exclusive_entry()
4117 struct vm_area_struct *vma, in should_try_to_free_swap() argument
4122 if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) || in should_try_to_free_swap()
4137 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in pte_marker_clear()
4150 pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte); in pte_marker_clear()
4157 if (vma_is_anonymous(vmf->vma)) in do_pte_missing()
4173 if (unlikely(!userfaultfd_wp(vmf->vma))) in pte_marker_handle_uffd_wp()
4208 struct vm_area_struct *vma = vmf->vma; in __alloc_swap_folio() local
4212 folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, vmf->address); in __alloc_swap_folio()
4217 if (mem_cgroup_swapin_charge_folio(folio, vma->vm_mm, in __alloc_swap_folio()
4306 struct vm_area_struct *vma = vmf->vma; in alloc_swap_folio() local
4320 if (unlikely(userfaultfd_armed(vma))) in alloc_swap_folio()
4336 orders = thp_vma_allowable_orders(vma, vma->vm_flags, in alloc_swap_folio()
4338 orders = thp_vma_suitable_orders(vma, vmf->address, orders); in alloc_swap_folio()
4345 pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in alloc_swap_folio()
4365 gfp = vma_thp_gfp_mask(vma); in alloc_swap_folio()
4368 folio = vma_alloc_folio(gfp, order, vma, addr); in alloc_swap_folio()
4370 if (!mem_cgroup_swapin_charge_folio(folio, vma->vm_mm, in alloc_swap_folio()
4402 struct vm_area_struct *vma = vmf->vma; in do_swap_page() local
4425 migration_entry_wait(vma->vm_mm, vmf->pmd, in do_swap_page()
4436 vma_end_read(vma); in do_swap_page()
4442 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_swap_page()
4470 print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL); in do_swap_page()
4481 folio = swap_cache_get_folio(entry, vma, vmf->address); in do_swap_page()
4542 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_swap_page()
4553 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT); in do_swap_page()
4585 folio = ksm_might_need_to_copy(folio, vma, vmf->address); in do_swap_page()
4614 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, in do_swap_page()
4654 if (unlikely(folio_start < max(address & PMD_MASK, vma->vm_start))) in do_swap_page()
4656 if (unlikely(folio_end > pmd_addr_end(address, vma->vm_end))) in do_swap_page()
4734 if (should_try_to_free_swap(folio, vma, vmf->flags)) in do_swap_page()
4737 add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages); in do_swap_page()
4738 add_mm_counter(vma->vm_mm, MM_SWAPENTS, -nr_pages); in do_swap_page()
4739 pte = mk_pte(page, vma->vm_page_prot); in do_swap_page()
4753 if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && in do_swap_page()
4754 !pte_needs_soft_dirty_wp(vma, pte)) { in do_swap_page()
4755 pte = pte_mkwrite(pte, vma); in do_swap_page()
4764 flush_icache_pages(vma, page, nr_pages); in do_swap_page()
4769 folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE); in do_swap_page()
4770 folio_add_lru_vma(folio, vma); in do_swap_page()
4780 folio_add_new_anon_rmap(folio, vma, address, rmap_flags); in do_swap_page()
4782 folio_add_anon_rmap_ptes(folio, page, nr_pages, vma, address, in do_swap_page()
4788 set_ptes(vma->vm_mm, address, ptep, pte, nr_pages); in do_swap_page()
4789 arch_do_swap_page_nr(vma->vm_mm, vma, address, in do_swap_page()
4814 update_mmu_cache_range(vmf, vma, address, ptep, nr_pages); in do_swap_page()
4863 struct vm_area_struct *vma = vmf->vma; in alloc_anon_folio() local
4876 if (unlikely(userfaultfd_armed(vma))) in alloc_anon_folio()
4884 orders = thp_vma_allowable_orders(vma, vma->vm_flags, in alloc_anon_folio()
4886 orders = thp_vma_suitable_orders(vma, vmf->address, orders); in alloc_anon_folio()
4914 gfp = vma_thp_gfp_mask(vma); in alloc_anon_folio()
4917 folio = vma_alloc_folio(gfp, order, vma, addr); in alloc_anon_folio()
4919 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { in alloc_anon_folio()
4943 return folio_prealloc(vma->vm_mm, vma, vmf->address, true); in alloc_anon_folio()
4953 struct vm_area_struct *vma = vmf->vma; in do_anonymous_page() local
4961 if (vma->vm_flags & VM_SHARED) in do_anonymous_page()
4968 if (pte_alloc(vma->vm_mm, vmf->pmd)) in do_anonymous_page()
4973 !mm_forbids_zeropage(vma->vm_mm)) { in do_anonymous_page()
4975 vma->vm_page_prot)); in do_anonymous_page()
4976 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_anonymous_page()
4981 update_mmu_tlb(vma, vmf->address, vmf->pte); in do_anonymous_page()
4984 ret = check_stable_address_space(vma->vm_mm); in do_anonymous_page()
4988 if (userfaultfd_missing(vma)) { in do_anonymous_page()
5016 entry = mk_pte(&folio->page, vma->vm_page_prot); in do_anonymous_page()
5018 if (vma->vm_flags & VM_WRITE) in do_anonymous_page()
5019 entry = pte_mkwrite(pte_mkdirty(entry), vma); in do_anonymous_page()
5021 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); in do_anonymous_page()
5025 update_mmu_tlb(vma, addr, vmf->pte); in do_anonymous_page()
5028 update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages); in do_anonymous_page()
5032 ret = check_stable_address_space(vma->vm_mm); in do_anonymous_page()
5037 if (userfaultfd_missing(vma)) { in do_anonymous_page()
5044 add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages); in do_anonymous_page()
5046 folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); in do_anonymous_page()
5047 folio_add_lru_vma(folio, vma); in do_anonymous_page()
5051 set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr_pages); in do_anonymous_page()
5054 update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr_pages); in do_anonymous_page()
5073 struct vm_area_struct *vma = vmf->vma; in __do_fault() local
5093 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); in __do_fault()
5098 ret = vma->vm_ops->fault(vmf); in __do_fault()
5130 struct vm_area_struct *vma = vmf->vma; in deposit_prealloc_pte() local
5132 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); in deposit_prealloc_pte()
5137 mm_inc_nr_ptes(vma->vm_mm); in deposit_prealloc_pte()
5144 struct vm_area_struct *vma = vmf->vma; in do_set_pmd() local
5156 if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags)) in do_set_pmd()
5159 if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER)) in do_set_pmd()
5180 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); in do_set_pmd()
5185 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_set_pmd()
5189 flush_icache_pages(vma, page, HPAGE_PMD_NR); in do_set_pmd()
5191 entry = mk_huge_pmd(page, vma->vm_page_prot); in do_set_pmd()
5193 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); in do_set_pmd()
5195 add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR); in do_set_pmd()
5196 folio_add_file_rmap_pmd(folio, page, vma); in do_set_pmd()
5204 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); in do_set_pmd()
5206 update_mmu_cache_pmd(vma, haddr, vmf->pmd); in do_set_pmd()
5233 struct vm_area_struct *vma = vmf->vma; in set_pte_range() local
5238 flush_icache_pages(vma, page, nr); in set_pte_range()
5239 entry = mk_pte(page, vma->vm_page_prot); in set_pte_range()
5247 entry = maybe_mkwrite(pte_mkdirty(entry), vma); in set_pte_range()
5251 if (write && !(vma->vm_flags & VM_SHARED)) { in set_pte_range()
5253 folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); in set_pte_range()
5254 folio_add_lru_vma(folio, vma); in set_pte_range()
5256 folio_add_file_rmap_ptes(folio, page, nr, vma); in set_pte_range()
5258 set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr); in set_pte_range()
5261 update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr); in set_pte_range()
5289 struct vm_area_struct *vma = vmf->vma; in finish_fault() local
5294 !(vma->vm_flags & VM_SHARED); in finish_fault()
5312 if (!(vma->vm_flags & VM_SHARED)) { in finish_fault()
5313 ret = check_stable_address_space(vma->vm_mm); in finish_fault()
5326 pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte); in finish_fault()
5327 else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) in finish_fault()
5339 if (!vma_is_anon_shmem(vma) || unlikely(userfaultfd_armed(vma)) || in finish_fault()
5345 pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; in finish_fault()
5354 vma_off + (nr_pages - idx) > vma_pages(vma) || in finish_fault()
5365 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in finish_fault()
5372 update_mmu_tlb(vma, addr, vmf->pte); in finish_fault()
5384 add_mm_counter(vma->vm_mm, type, nr_pages); in finish_fault()
5457 pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; in do_fault_around()
5467 pte_off + vma_pages(vmf->vma) - vma_off) - 1; in do_fault_around()
5470 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); in do_fault_around()
5476 ret = vmf->vma->vm_ops->map_pages(vmf, in do_fault_around()
5488 if (!vmf->vma->vm_ops->map_pages) in should_fault_around()
5491 if (uffd_disable_fault_around(vmf->vma)) in should_fault_around()
5532 struct vm_area_struct *vma = vmf->vma; in do_cow_fault() local
5542 folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false); in do_cow_fault()
5554 if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) { in do_cow_fault()
5574 struct vm_area_struct *vma = vmf->vma; in do_shared_fault() local
5592 if (vma->vm_ops->page_mkwrite) { in do_shared_fault()
5624 struct vm_area_struct *vma = vmf->vma; in do_fault() local
5625 struct mm_struct *vm_mm = vma->vm_mm; in do_fault()
5631 if (!vma->vm_ops->fault) { in do_fault()
5632 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in do_fault()
5653 else if (!(vma->vm_flags & VM_SHARED)) in do_fault()
5670 struct vm_area_struct *vma = vmf->vma; in numa_migrate_check() local
5687 if (folio_maybe_mapped_shared(folio) && (vma->vm_flags & VM_SHARED)) in numa_migrate_check()
5699 vma_set_access_pid_bit(vma); in numa_migrate_check()
5713 static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, in numa_rebuild_single_mapping() argument
5719 old_pte = ptep_modify_prot_start(vma, fault_addr, fault_pte); in numa_rebuild_single_mapping()
5720 pte = pte_modify(old_pte, vma->vm_page_prot); in numa_rebuild_single_mapping()
5723 pte = pte_mkwrite(pte, vma); in numa_rebuild_single_mapping()
5724 ptep_modify_prot_commit(vma, fault_addr, fault_pte, old_pte, pte); in numa_rebuild_single_mapping()
5725 update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1); in numa_rebuild_single_mapping()
5728 static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, in numa_rebuild_large_mapping() argument
5739 start = max3(addr_start, pt_start, vma->vm_start); in numa_rebuild_large_mapping()
5741 vma->vm_end); in numa_rebuild_large_mapping()
5756 ptent = pte_modify(ptent, vma->vm_page_prot); in numa_rebuild_large_mapping()
5759 can_change_pte_writable(vma, addr, ptent)) in numa_rebuild_large_mapping()
5763 numa_rebuild_single_mapping(vmf, vma, addr, start_ptep, writable); in numa_rebuild_large_mapping()
5769 struct vm_area_struct *vma = vmf->vma; in do_numa_page() local
5773 bool pte_write_upgrade = vma_wants_manual_pte_write_upgrade(vma); in do_numa_page()
5792 pte = pte_modify(old_pte, vma->vm_page_prot); in do_numa_page()
5800 can_change_pte_writable(vma, vmf->address, pte)) in do_numa_page()
5803 folio = vm_normal_folio(vma, vmf->address, pte); in do_numa_page()
5814 if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) { in do_numa_page()
5832 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_numa_page()
5846 numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable, in do_numa_page()
5849 numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte, in do_numa_page()
5860 struct vm_area_struct *vma = vmf->vma; in create_huge_pmd() local
5861 if (vma_is_anonymous(vma)) in create_huge_pmd()
5863 if (vma->vm_ops->huge_fault) in create_huge_pmd()
5864 return vma->vm_ops->huge_fault(vmf, PMD_ORDER); in create_huge_pmd()
5871 struct vm_area_struct *vma = vmf->vma; in wp_huge_pmd() local
5875 if (vma_is_anonymous(vma)) { in wp_huge_pmd()
5877 userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) { in wp_huge_pmd()
5878 if (userfaultfd_wp_async(vmf->vma)) in wp_huge_pmd()
5885 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) { in wp_huge_pmd()
5886 if (vma->vm_ops->huge_fault) { in wp_huge_pmd()
5887 ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER); in wp_huge_pmd()
5895 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); in wp_huge_pmd()
5904 struct vm_area_struct *vma = vmf->vma; in create_huge_pud() local
5906 if (vma_is_anonymous(vma)) in create_huge_pud()
5908 if (vma->vm_ops->huge_fault) in create_huge_pud()
5909 return vma->vm_ops->huge_fault(vmf, PUD_ORDER); in create_huge_pud()
5918 struct vm_area_struct *vma = vmf->vma; in wp_huge_pud() local
5922 if (vma_is_anonymous(vma)) in wp_huge_pud()
5924 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) { in wp_huge_pud()
5925 if (vma->vm_ops->huge_fault) { in wp_huge_pud()
5926 ret = vma->vm_ops->huge_fault(vmf, PUD_ORDER); in wp_huge_pud()
5933 __split_huge_pud(vma, vmf->pud, vmf->address); in wp_huge_pud()
5982 vmf->pte = pte_offset_map_rw_nolock(vmf->vma->vm_mm, vmf->pmd, in handle_pte_fault()
6002 if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) in handle_pte_fault()
6008 update_mmu_tlb(vmf->vma, vmf->address, vmf->pte); in handle_pte_fault()
6018 if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry, in handle_pte_fault()
6020 update_mmu_cache_range(vmf, vmf->vma, vmf->address, in handle_pte_fault()
6033 flush_tlb_fix_spurious_fault(vmf->vma, vmf->address, in handle_pte_fault()
6047 static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, in __handle_mm_fault() argument
6051 .vma = vma, in __handle_mm_fault()
6055 .pgoff = linear_page_index(vma, address), in __handle_mm_fault()
6056 .gfp_mask = __get_fault_gfp_mask(vma), in __handle_mm_fault()
6058 struct mm_struct *mm = vma->vm_mm; in __handle_mm_fault()
6059 unsigned long vm_flags = vma->vm_flags; in __handle_mm_fault()
6074 thp_vma_allowable_order(vma, vm_flags, in __handle_mm_fault()
6109 thp_vma_allowable_order(vma, vm_flags, in __handle_mm_fault()
6125 if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) in __handle_mm_fault()
6212 static void lru_gen_enter_fault(struct vm_area_struct *vma) in lru_gen_enter_fault() argument
6215 current->in_lru_fault = vma_has_recency(vma); in lru_gen_enter_fault()
6223 static void lru_gen_enter_fault(struct vm_area_struct *vma) in lru_gen_enter_fault() argument
6232 static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma, in sanitize_fault_flags() argument
6242 if (!is_cow_mapping(vma->vm_flags)) in sanitize_fault_flags()
6246 if (WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE))) in sanitize_fault_flags()
6249 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) && in sanitize_fault_flags()
6250 !is_cow_mapping(vma->vm_flags))) in sanitize_fault_flags()
6274 vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, in handle_mm_fault() argument
6278 struct mm_struct *mm = vma->vm_mm; in handle_mm_fault()
6284 ret = sanitize_fault_flags(vma, &flags); in handle_mm_fault()
6288 if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE, in handle_mm_fault()
6295 is_droppable = !!(vma->vm_flags & VM_DROPPABLE); in handle_mm_fault()
6304 lru_gen_enter_fault(vma); in handle_mm_fault()
6306 if (unlikely(is_vm_hugetlb_page(vma))) in handle_mm_fault()
6307 ret = hugetlb_fault(vma->vm_mm, vma, address, flags); in handle_mm_fault()
6309 ret = __handle_mm_fault(vma, address, flags); in handle_mm_fault()
6404 struct vm_area_struct *vma; in lock_mm_and_find_vma() local
6409 vma = find_vma(mm, addr); in lock_mm_and_find_vma()
6410 if (likely(vma && (vma->vm_start <= addr))) in lock_mm_and_find_vma()
6411 return vma; in lock_mm_and_find_vma()
6417 if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) { in lock_mm_and_find_vma()
6435 vma = find_vma(mm, addr); in lock_mm_and_find_vma()
6436 if (!vma) in lock_mm_and_find_vma()
6438 if (vma->vm_start <= addr) in lock_mm_and_find_vma()
6440 if (!(vma->vm_flags & VM_GROWSDOWN)) in lock_mm_and_find_vma()
6444 if (expand_stack_locked(vma, addr)) in lock_mm_and_find_vma()
6449 return vma; in lock_mm_and_find_vma()
6458 static inline bool __vma_enter_locked(struct vm_area_struct *vma, bool detaching) in __vma_enter_locked() argument
6470 if (!refcount_add_not_zero(VMA_LOCK_OFFSET, &vma->vm_refcnt)) in __vma_enter_locked()
6473 rwsem_acquire(&vma->vmlock_dep_map, 0, 0, _RET_IP_); in __vma_enter_locked()
6474 rcuwait_wait_event(&vma->vm_mm->vma_writer_wait, in __vma_enter_locked()
6475 refcount_read(&vma->vm_refcnt) == tgt_refcnt, in __vma_enter_locked()
6477 lock_acquired(&vma->vmlock_dep_map, _RET_IP_); in __vma_enter_locked()
6482 static inline void __vma_exit_locked(struct vm_area_struct *vma, bool *detached) in __vma_exit_locked() argument
6484 *detached = refcount_sub_and_test(VMA_LOCK_OFFSET, &vma->vm_refcnt); in __vma_exit_locked()
6485 rwsem_release(&vma->vmlock_dep_map, _RET_IP_); in __vma_exit_locked()
6488 void __vma_start_write(struct vm_area_struct *vma, unsigned int mm_lock_seq) in __vma_start_write() argument
6497 locked = __vma_enter_locked(vma, false); in __vma_start_write()
6505 WRITE_ONCE(vma->vm_lock_seq, mm_lock_seq); in __vma_start_write()
6510 __vma_exit_locked(vma, &detached); in __vma_start_write()
6516 void vma_mark_detached(struct vm_area_struct *vma) in vma_mark_detached() argument
6518 vma_assert_write_locked(vma); in vma_mark_detached()
6519 vma_assert_attached(vma); in vma_mark_detached()
6529 if (unlikely(!refcount_dec_and_test(&vma->vm_refcnt))) { in vma_mark_detached()
6531 if (__vma_enter_locked(vma, true)) { in vma_mark_detached()
6534 __vma_exit_locked(vma, &detached); in vma_mark_detached()
6549 struct vm_area_struct *vma; in lock_vma_under_rcu() local
6553 vma = mas_walk(&mas); in lock_vma_under_rcu()
6554 if (!vma) in lock_vma_under_rcu()
6557 vma = vma_start_read(mm, vma); in lock_vma_under_rcu()
6558 if (IS_ERR_OR_NULL(vma)) { in lock_vma_under_rcu()
6560 if (PTR_ERR(vma) == -EAGAIN) { in lock_vma_under_rcu()
6577 if (unlikely(vma->vm_mm != mm || in lock_vma_under_rcu()
6578 address < vma->vm_start || address >= vma->vm_end)) in lock_vma_under_rcu()
6582 return vma; in lock_vma_under_rcu()
6585 vma_end_read(vma); in lock_vma_under_rcu()
6679 static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma) in pfnmap_lockdep_assert() argument
6682 struct file *file = vma->vm_file; in pfnmap_lockdep_assert()
6687 lockdep_is_held(&vma->vm_mm->mmap_lock)); in pfnmap_lockdep_assert()
6689 lockdep_assert(lockdep_is_held(&vma->vm_mm->mmap_lock)); in pfnmap_lockdep_assert()
6726 struct vm_area_struct *vma = args->vma; in follow_pfnmap_start() local
6728 struct mm_struct *mm = vma->vm_mm; in follow_pfnmap_start()
6736 pfnmap_lockdep_assert(vma); in follow_pfnmap_start()
6738 if (unlikely(address < vma->vm_start || address >= vma->vm_end)) in follow_pfnmap_start()
6741 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) in follow_pfnmap_start()
6829 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, in generic_access_phys() argument
6838 struct follow_pfnmap_args args = { .vma = vma, .address = addr }; in generic_access_phys()
6903 struct vm_area_struct *vma = NULL; in __access_remote_vm() local
6905 gup_flags, &vma); in __access_remote_vm()
6909 vma = vma_lookup(mm, addr); in __access_remote_vm()
6910 if (!vma) { in __access_remote_vm()
6911 vma = expand_stack(mm, addr); in __access_remote_vm()
6914 if (!vma) in __access_remote_vm()
6927 if (vma->vm_ops && vma->vm_ops->access) in __access_remote_vm()
6928 bytes = vma->vm_ops->access(vma, addr, buf, in __access_remote_vm()
6941 copy_to_user_page(vma, page, addr, in __access_remote_vm()
6945 copy_from_user_page(vma, page, addr, in __access_remote_vm()
7028 struct vm_area_struct *vma = NULL; in __copy_remote_vm_str() local
7030 page = get_user_page_vma_remote(mm, addr, gup_flags, &vma); in __copy_remote_vm_str()
7064 copy_from_user_page(vma, page, addr, buf, maddr + (PAGE_SIZE - 1), 1); in __copy_remote_vm_str()
7124 struct vm_area_struct *vma; in print_vma_addr() local
7132 vma = vma_lookup(mm, ip); in print_vma_addr()
7133 if (vma && vma->vm_file) { in print_vma_addr()
7134 struct file *f = vma->vm_file; in print_vma_addr()
7135 ip -= vma->vm_start; in print_vma_addr()
7136 ip += vma->vm_pgoff << PAGE_SHIFT; in print_vma_addr()
7138 vma->vm_start, in print_vma_addr()
7139 vma->vm_end - vma->vm_start); in print_vma_addr()
7255 struct vm_area_struct *vma, in copy_user_gigantic_page() argument
7269 addr + i*PAGE_SIZE, vma)) in copy_user_gigantic_page()
7278 struct vm_area_struct *vma; member
7287 if (copy_mc_user_highpage(dst, src, addr, copy_arg->vma)) in copy_subpage()
7293 unsigned long addr_hint, struct vm_area_struct *vma) in copy_user_large_folio() argument
7299 .vma = vma, in copy_user_large_folio()
7303 return copy_user_gigantic_page(dst, src, addr_hint, vma, nr_pages); in copy_user_large_folio()
7368 void vma_pgtable_walk_begin(struct vm_area_struct *vma) in vma_pgtable_walk_begin() argument
7370 if (is_vm_hugetlb_page(vma)) in vma_pgtable_walk_begin()
7371 hugetlb_vma_lock_read(vma); in vma_pgtable_walk_begin()
7374 void vma_pgtable_walk_end(struct vm_area_struct *vma) in vma_pgtable_walk_end() argument
7376 if (is_vm_hugetlb_page(vma)) in vma_pgtable_walk_end()
7377 hugetlb_vma_unlock_read(vma); in vma_pgtable_walk_end()