Lines Matching refs:dst_vma
24 bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) in validate_dst_vma() argument
27 if (dst_end > dst_vma->vm_end) in validate_dst_vma()
35 if (!dst_vma->vm_userfaultfd_ctx.ctx) in validate_dst_vma()
103 struct vm_area_struct *dst_vma; in uffd_mfill_lock() local
105 dst_vma = uffd_lock_vma(dst_mm, dst_start); in uffd_mfill_lock()
106 if (IS_ERR(dst_vma) || validate_dst_vma(dst_vma, dst_start + len)) in uffd_mfill_lock()
107 return dst_vma; in uffd_mfill_lock()
109 vma_end_read(dst_vma); in uffd_mfill_lock()
124 struct vm_area_struct *dst_vma; in uffd_mfill_lock() local
127 dst_vma = find_vma_and_prepare_anon(dst_mm, dst_start); in uffd_mfill_lock()
128 if (IS_ERR(dst_vma)) in uffd_mfill_lock()
131 if (validate_dst_vma(dst_vma, dst_start + len)) in uffd_mfill_lock()
132 return dst_vma; in uffd_mfill_lock()
134 dst_vma = ERR_PTR(-ENOENT); in uffd_mfill_lock()
137 return dst_vma; in uffd_mfill_lock()
147 static bool mfill_file_over_size(struct vm_area_struct *dst_vma, in mfill_file_over_size() argument
153 if (!dst_vma->vm_file) in mfill_file_over_size()
156 inode = dst_vma->vm_file->f_inode; in mfill_file_over_size()
157 offset = linear_page_index(dst_vma, dst_addr); in mfill_file_over_size()
169 struct vm_area_struct *dst_vma, in mfill_atomic_install_pte() argument
174 struct mm_struct *dst_mm = dst_vma->vm_mm; in mfill_atomic_install_pte()
176 bool writable = dst_vma->vm_flags & VM_WRITE; in mfill_atomic_install_pte()
177 bool vm_shared = dst_vma->vm_flags & VM_SHARED; in mfill_atomic_install_pte()
182 _dst_pte = mk_pte(page, dst_vma->vm_page_prot); in mfill_atomic_install_pte()
187 _dst_pte = pte_mkwrite(_dst_pte, dst_vma); in mfill_atomic_install_pte()
196 if (mfill_file_over_size(dst_vma, dst_addr)) { in mfill_atomic_install_pte()
214 folio_add_file_rmap_pte(folio, page, dst_vma); in mfill_atomic_install_pte()
216 folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE); in mfill_atomic_install_pte()
217 folio_add_lru_vma(folio, dst_vma); in mfill_atomic_install_pte()
229 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_install_pte()
238 struct vm_area_struct *dst_vma, in mfill_atomic_pte_copy() argument
250 folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, dst_vma, in mfill_atomic_pte_copy()
299 if (mem_cgroup_charge(folio, dst_vma->vm_mm, GFP_KERNEL)) in mfill_atomic_pte_copy()
302 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte_copy()
314 struct vm_area_struct *dst_vma, in mfill_atomic_pte_zeroed_folio() argument
320 folio = vma_alloc_zeroed_movable_folio(dst_vma, dst_addr); in mfill_atomic_pte_zeroed_folio()
324 if (mem_cgroup_charge(folio, dst_vma->vm_mm, GFP_KERNEL)) in mfill_atomic_pte_zeroed_folio()
334 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte_zeroed_folio()
346 struct vm_area_struct *dst_vma, in mfill_atomic_pte_zeropage() argument
353 if (mm_forbids_zeropage(dst_vma->vm_mm)) in mfill_atomic_pte_zeropage()
354 return mfill_atomic_pte_zeroed_folio(dst_pmd, dst_vma, dst_addr); in mfill_atomic_pte_zeropage()
357 dst_vma->vm_page_prot)); in mfill_atomic_pte_zeropage()
359 dst_pte = pte_offset_map_lock(dst_vma->vm_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_pte_zeropage()
362 if (mfill_file_over_size(dst_vma, dst_addr)) { in mfill_atomic_pte_zeropage()
369 set_pte_at(dst_vma->vm_mm, dst_addr, dst_pte, _dst_pte); in mfill_atomic_pte_zeropage()
371 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_pte_zeropage()
381 struct vm_area_struct *dst_vma, in mfill_atomic_pte_continue() argument
385 struct inode *inode = file_inode(dst_vma->vm_file); in mfill_atomic_pte_continue()
386 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); in mfill_atomic_pte_continue()
408 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte_continue()
425 struct vm_area_struct *dst_vma, in mfill_atomic_pte_poison() argument
430 struct mm_struct *dst_mm = dst_vma->vm_mm; in mfill_atomic_pte_poison()
440 if (mfill_file_over_size(dst_vma, dst_addr)) { in mfill_atomic_pte_poison()
453 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_pte_poison()
490 struct vm_area_struct *dst_vma, in mfill_atomic_hugetlb() argument
496 struct mm_struct *dst_mm = dst_vma->vm_mm; in mfill_atomic_hugetlb()
515 uffd_mfill_unlock(dst_vma); in mfill_atomic_hugetlb()
523 vma_hpagesize = vma_kernel_pagesize(dst_vma); in mfill_atomic_hugetlb()
537 if (!dst_vma) { in mfill_atomic_hugetlb()
538 dst_vma = uffd_mfill_lock(dst_mm, dst_start, len); in mfill_atomic_hugetlb()
539 if (IS_ERR(dst_vma)) { in mfill_atomic_hugetlb()
540 err = PTR_ERR(dst_vma); in mfill_atomic_hugetlb()
545 if (!is_vm_hugetlb_page(dst_vma)) in mfill_atomic_hugetlb()
549 if (vma_hpagesize != vma_kernel_pagesize(dst_vma)) in mfill_atomic_hugetlb()
572 idx = linear_page_index(dst_vma, dst_addr); in mfill_atomic_hugetlb()
573 mapping = dst_vma->vm_file->f_mapping; in mfill_atomic_hugetlb()
576 hugetlb_vma_lock_read(dst_vma); in mfill_atomic_hugetlb()
579 dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize); in mfill_atomic_hugetlb()
581 hugetlb_vma_unlock_read(dst_vma); in mfill_atomic_hugetlb()
589 hugetlb_vma_unlock_read(dst_vma); in mfill_atomic_hugetlb()
594 err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, dst_addr, in mfill_atomic_hugetlb()
597 hugetlb_vma_unlock_read(dst_vma); in mfill_atomic_hugetlb()
604 uffd_mfill_unlock(dst_vma); in mfill_atomic_hugetlb()
614 dst_vma = NULL; in mfill_atomic_hugetlb()
634 uffd_mfill_unlock(dst_vma); in mfill_atomic_hugetlb()
646 struct vm_area_struct *dst_vma,
654 struct vm_area_struct *dst_vma, in mfill_atomic_pte() argument
663 return mfill_atomic_pte_continue(dst_pmd, dst_vma, in mfill_atomic_pte()
666 return mfill_atomic_pte_poison(dst_pmd, dst_vma, in mfill_atomic_pte()
680 if (!(dst_vma->vm_flags & VM_SHARED)) { in mfill_atomic_pte()
682 err = mfill_atomic_pte_copy(dst_pmd, dst_vma, in mfill_atomic_pte()
687 dst_vma, dst_addr); in mfill_atomic_pte()
689 err = shmem_mfill_atomic_pte(dst_pmd, dst_vma, in mfill_atomic_pte()
704 struct vm_area_struct *dst_vma; in mfill_atomic() local
730 dst_vma = uffd_mfill_lock(dst_mm, dst_start, len); in mfill_atomic()
731 if (IS_ERR(dst_vma)) { in mfill_atomic()
732 err = PTR_ERR(dst_vma); in mfill_atomic()
751 if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) && in mfill_atomic()
752 dst_vma->vm_flags & VM_SHARED)) in mfill_atomic()
759 if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP)) in mfill_atomic()
765 if (is_vm_hugetlb_page(dst_vma)) in mfill_atomic()
766 return mfill_atomic_hugetlb(ctx, dst_vma, dst_start, in mfill_atomic()
769 if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma)) in mfill_atomic()
771 if (!vma_is_shmem(dst_vma) && in mfill_atomic()
812 err = mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic()
820 uffd_mfill_unlock(dst_vma); in mfill_atomic()
851 uffd_mfill_unlock(dst_vma); in mfill_atomic()
900 long uffd_wp_range(struct vm_area_struct *dst_vma, in uffd_wp_range() argument
907 VM_WARN_ONCE(start < dst_vma->vm_start || start + len > dst_vma->vm_end, in uffd_wp_range()
920 if (!enable_wp && vma_wants_manual_pte_write_upgrade(dst_vma)) in uffd_wp_range()
922 tlb_gather_mmu(&tlb, dst_vma->vm_mm); in uffd_wp_range()
923 ret = change_protection(&tlb, dst_vma, start, start + len, mm_cp_flags); in uffd_wp_range()
935 struct vm_area_struct *dst_vma; in mwriteprotect_range() local
962 for_each_vma_range(vmi, dst_vma, end) { in mwriteprotect_range()
964 if (!userfaultfd_wp(dst_vma)) { in mwriteprotect_range()
969 if (is_vm_hugetlb_page(dst_vma)) { in mwriteprotect_range()
971 page_mask = vma_kernel_pagesize(dst_vma) - 1; in mwriteprotect_range()
976 _start = max(dst_vma->vm_start, start); in mwriteprotect_range()
977 _end = min(dst_vma->vm_end, end); in mwriteprotect_range()
979 err = uffd_wp_range(dst_vma, _start, _end - _start, enable_wp); in mwriteprotect_range()
1030 struct vm_area_struct *dst_vma, in move_present_pte() argument
1063 folio_move_anon_rmap(src_folio, dst_vma); in move_present_pte()
1064 src_folio->index = linear_page_index(dst_vma, dst_addr); in move_present_pte()
1066 orig_dst_pte = mk_pte(&src_folio->page, dst_vma->vm_page_prot); in move_present_pte()
1073 orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma); in move_present_pte()
1081 static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, in move_swap_pte() argument
1103 folio_move_anon_rmap(src_folio, dst_vma); in move_swap_pte()
1104 src_folio->index = linear_page_index(dst_vma, dst_addr); in move_swap_pte()
1118 struct vm_area_struct *dst_vma, in move_zeropage_pte() argument
1136 dst_vma->vm_page_prot)); in move_zeropage_pte()
1151 struct vm_area_struct *dst_vma, in move_pages_pte() argument
1243 err = move_zeropage_pte(mm, dst_vma, src_vma, in move_pages_pte()
1349 err = move_present_pte(mm, dst_vma, src_vma, in move_pages_pte()
1413 err = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, in move_pages_pte()
1464 struct vm_area_struct *dst_vma) in validate_move_areas() argument
1467 if ((src_vma->vm_flags & VM_ACCESS_FLAGS) != (dst_vma->vm_flags & VM_ACCESS_FLAGS) || in validate_move_areas()
1468 pgprot_val(src_vma->vm_page_prot) != pgprot_val(dst_vma->vm_page_prot)) in validate_move_areas()
1472 if ((src_vma->vm_flags & VM_LOCKED) != (dst_vma->vm_flags & VM_LOCKED)) in validate_move_areas()
1483 if (!vma_move_compatible(src_vma) || !vma_move_compatible(dst_vma)) in validate_move_areas()
1487 if (!dst_vma->vm_userfaultfd_ctx.ctx || in validate_move_areas()
1488 dst_vma->vm_userfaultfd_ctx.ctx != ctx) in validate_move_areas()
1492 if (!vma_is_anonymous(src_vma) || !vma_is_anonymous(dst_vma)) in validate_move_areas()
1592 static void uffd_move_unlock(struct vm_area_struct *dst_vma, in uffd_move_unlock() argument
1596 if (src_vma != dst_vma) in uffd_move_unlock()
1597 vma_end_read(dst_vma); in uffd_move_unlock()
1617 static void uffd_move_unlock(struct vm_area_struct *dst_vma, in uffd_move_unlock() argument
1621 mmap_read_unlock(dst_vma->vm_mm); in uffd_move_unlock()
1705 struct vm_area_struct *src_vma, *dst_vma; in move_pages() local
1722 err = uffd_move_lock(mm, dst_start, src_start, &dst_vma, &src_vma); in move_pages()
1742 if (dst_vma->vm_flags & VM_SHARED) in move_pages()
1744 if (dst_start + len > dst_vma->vm_end) in move_pages()
1747 err = validate_move_areas(ctx, src_vma, dst_vma); in move_pages()
1819 dst_pmdval, dst_vma, src_vma, in move_pages()
1840 dst_vma, src_vma, in move_pages()
1868 uffd_move_unlock(dst_vma, src_vma); in move_pages()