Lines Matching refs:vm_mm

482 	pgd_t *pgd = pgd_offset(vma->vm_mm, addr);  in print_bad_pte()
749 set_pte_at(vma->vm_mm, address, ptep, pte); in restore_exclusive_pte()
937 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); in copy_present_page()
945 struct mm_struct *src_mm = src_vma->vm_mm; in __copy_present_ptes()
961 set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr); in __copy_present_ptes()
1075 struct mm_struct *dst_mm = dst_vma->vm_mm; in copy_pte_range()
1076 struct mm_struct *src_mm = src_vma->vm_mm; in copy_pte_range()
1230 struct mm_struct *dst_mm = dst_vma->vm_mm; in copy_pmd_range()
1231 struct mm_struct *src_mm = src_vma->vm_mm; in copy_pmd_range()
1267 struct mm_struct *dst_mm = dst_vma->vm_mm; in copy_pud_range()
1268 struct mm_struct *src_mm = src_vma->vm_mm; in copy_pud_range()
1304 struct mm_struct *dst_mm = dst_vma->vm_mm; in copy_p4d_range()
1361 struct mm_struct *dst_mm = dst_vma->vm_mm; in copy_page_range()
1362 struct mm_struct *src_mm = src_vma->vm_mm; in copy_page_range()
1655 clear_not_present_full_ptes(vma->vm_mm, addr, pte, nr, tlb->fullmm); in zap_nonpresent_ptes()
1889 pgd = pgd_offset(vma->vm_mm, addr); in unmap_page_range()
1977 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm, in unmap_vmas()
2008 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, in zap_page_range_single()
2011 tlb_gather_mmu(&tlb, vma->vm_mm); in zap_page_range_single()
2012 update_hiwater_rss(vma->vm_mm); in zap_page_range_single()
2087 if (mm_forbids_zeropage(vma->vm_mm)) in vm_mixed_zeropage_allowed()
2161 inc_mm_counter(vma->vm_mm, mm_counter_file(folio)); in insert_page_into_pte_locked()
2164 set_pte_at(vma->vm_mm, addr, pte, pteval); in insert_page_into_pte_locked()
2179 pte = get_locked_pte(vma->vm_mm, addr, &ptl); in insert_page()
2209 struct mm_struct *const mm = vma->vm_mm; in insert_pages()
2284 BUG_ON(mmap_read_trylock(vma->vm_mm)); in vm_insert_pages()
2329 BUG_ON(mmap_read_trylock(vma->vm_mm)); in vm_insert_page()
2423 struct mm_struct *mm = vma->vm_mm; in insert_pfn()
2768 struct mm_struct *mm = vma->vm_mm; in remap_pfn_range_internal()
3165 struct mm_struct *mm = vma->vm_mm; in __wp_page_copy_user()
3429 if (!mmap_read_trylock(vma->vm_mm)) in __vmf_anon_prepare()
3435 mmap_read_unlock(vma->vm_mm); in __vmf_anon_prepare()
3460 struct mm_struct *mm = vma->vm_mm; in wp_page_copy()
3627 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address, in finish_mkwrite_fault()
3763 VM_WARN_ON_ONCE(folio_mm_id(folio, 0) != vma->vm_mm->mm_id && in __wp_can_reuse_large_anon_folio()
3764 folio_mm_id(folio, 1) != vma->vm_mm->mm_id); in __wp_can_reuse_large_anon_folio()
3869 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); in do_wp_page()
3882 mm_tlb_flush_pending(vmf->vma->vm_mm))) in do_wp_page()
4097 vma->vm_mm, vmf->address & PAGE_MASK, in remove_device_exclusive_entry()
4101 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, in remove_device_exclusive_entry()
4137 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in pte_marker_clear()
4150 pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte); in pte_marker_clear()
4217 if (mem_cgroup_swapin_charge_folio(folio, vma->vm_mm, in __alloc_swap_folio()
4345 pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in alloc_swap_folio()
4370 if (!mem_cgroup_swapin_charge_folio(folio, vma->vm_mm, in alloc_swap_folio()
4425 migration_entry_wait(vma->vm_mm, vmf->pmd, in do_swap_page()
4442 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_swap_page()
4542 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_swap_page()
4553 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT); in do_swap_page()
4614 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, in do_swap_page()
4737 add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages); in do_swap_page()
4738 add_mm_counter(vma->vm_mm, MM_SWAPENTS, -nr_pages); in do_swap_page()
4788 set_ptes(vma->vm_mm, address, ptep, pte, nr_pages); in do_swap_page()
4789 arch_do_swap_page_nr(vma->vm_mm, vma, address, in do_swap_page()
4919 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { in alloc_anon_folio()
4943 return folio_prealloc(vma->vm_mm, vma, vmf->address, true); in alloc_anon_folio()
4968 if (pte_alloc(vma->vm_mm, vmf->pmd)) in do_anonymous_page()
4973 !mm_forbids_zeropage(vma->vm_mm)) { in do_anonymous_page()
4976 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_anonymous_page()
4984 ret = check_stable_address_space(vma->vm_mm); in do_anonymous_page()
5021 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); in do_anonymous_page()
5032 ret = check_stable_address_space(vma->vm_mm); in do_anonymous_page()
5044 add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages); in do_anonymous_page()
5051 set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr_pages); in do_anonymous_page()
5093 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); in __do_fault()
5132 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); in deposit_prealloc_pte()
5137 mm_inc_nr_ptes(vma->vm_mm); in deposit_prealloc_pte()
5180 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); in do_set_pmd()
5185 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_set_pmd()
5195 add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR); in do_set_pmd()
5204 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); in do_set_pmd()
5258 set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr); in set_pte_range()
5313 ret = check_stable_address_space(vma->vm_mm); in finish_fault()
5326 pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte); in finish_fault()
5327 else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) in finish_fault()
5365 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in finish_fault()
5384 add_mm_counter(vma->vm_mm, type, nr_pages); in finish_fault()
5470 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); in do_fault_around()
5542 folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false); in do_cow_fault()
5625 struct mm_struct *vm_mm = vma->vm_mm; in do_fault() local
5632 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, in do_fault()
5660 pte_free(vm_mm, vmf->prealloc_pte); in do_fault()
5832 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, in do_numa_page()
5982 vmf->pte = pte_offset_map_rw_nolock(vmf->vma->vm_mm, vmf->pmd, in handle_pte_fault()
6058 struct mm_struct *mm = vma->vm_mm; in __handle_mm_fault()
6278 struct mm_struct *mm = vma->vm_mm; in handle_mm_fault()
6307 ret = hugetlb_fault(vma->vm_mm, vma, address, flags); in handle_mm_fault()
6474 rcuwait_wait_event(&vma->vm_mm->vma_writer_wait, in __vma_enter_locked()
6577 if (unlikely(vma->vm_mm != mm || in lock_vma_under_rcu()
6687 lockdep_is_held(&vma->vm_mm->mmap_lock)); in pfnmap_lockdep_assert()
6689 lockdep_assert(lockdep_is_held(&vma->vm_mm->mmap_lock)); in pfnmap_lockdep_assert()
6728 struct mm_struct *mm = vma->vm_mm; in follow_pfnmap_start()