Home
last modified time | relevance | path

Searched refs:pmdp (Results 1 – 25 of 117) sorted by relevance

12345

/linux-6.15/mm/
H A Dpgtable-generic.c138 pmd_t *pmdp) in pmdp_huge_clear_flush() argument
142 VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) && in pmdp_huge_clear_flush()
143 !pmd_devmap(*pmdp)); in pmdp_huge_clear_flush()
171 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit()
175 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
188 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw()
191 if (pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_withdraw()
199 pmd_t *pmdp) in pmdp_invalidate() argument
202 pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp)); in pmdp_invalidate()
210 pmd_t *pmdp) in pmdp_invalidate_ad() argument
[all …]
/linux-6.15/arch/powerpc/mm/book3s64/
H A Dhash_pgtable.c152 pmd_t *pmdp; in hash__map_kernel_page() local
163 if (!pmdp) in hash__map_kernel_page()
165 ptep = pte_alloc_kernel(pmdp, ea); in hash__map_kernel_page()
198 WARN_ON(!hash__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); in hash__pmd_hugepage_update()
211 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp), in hash__pmd_hugepage_update()
224 pmd_t *pmdp) in hash__pmdp_collapse_flush() argument
229 VM_BUG_ON(pmd_trans_huge(*pmdp)); in hash__pmdp_collapse_flush()
230 VM_BUG_ON(pmd_devmap(*pmdp)); in hash__pmdp_collapse_flush()
232 pmd = *pmdp; in hash__pmdp_collapse_flush()
233 pmd_clear(pmdp); in hash__pmdp_collapse_flush()
[all …]
H A Dpgtable.c61 pmd_t *pmdp, pmd_t entry, int dirty) in pmdp_set_access_flags() argument
65 WARN_ON(!pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); in pmdp_set_access_flags()
68 changed = !pmd_same(*(pmdp), entry); in pmdp_set_access_flags()
102 unsigned long address, pmd_t *pmdp) in pmdp_test_and_clear_young() argument
118 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
126 WARN_ON(pte_hw_valid(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp))); in set_pmd_at()
127 assert_spin_locked(pmd_lockptr(mm, pmdp)); in set_pmd_at()
181 pmd_t *pmdp) in pmdp_invalidate() argument
185 VM_WARN_ON_ONCE(!pmd_present(*pmdp)); in pmdp_invalidate()
207 VM_BUG_ON((pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) && in pmdp_huge_get_and_clear_full()
[all …]
H A Dradix_pgtable.c81 pmd_t *pmdp; in early_map_kernel_page() local
103 ptep = pmdp_ptep(pmdp); in early_map_kernel_page()
133 pmd_t *pmdp; in __map_kernel_page() local
163 if (!pmdp) in __map_kernel_page()
194 pmd_t *pmdp; in radix__change_memory_range() local
214 if (!pmdp) in radix__change_memory_range()
216 if (pmd_leaf(*pmdp)) { in radix__change_memory_range()
1429 WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); in radix__pmd_hugepage_update()
1457 pmd_t *pmdp) in radix__pmdp_collapse_flush() argument
1468 pmd = *pmdp; in radix__pmdp_collapse_flush()
[all …]
H A Dhash_hugepage.c22 pmd_t *pmdp, unsigned long trap, unsigned long flags, in __hash_page_thp() argument
36 pmd_t pmd = READ_ONCE(*pmdp); in __hash_page_thp()
52 } while (!pmd_xchg(pmdp, __pmd(old_pmd), __pmd(new_pmd))); in __hash_page_thp()
77 hpte_slot_array = get_hpte_slot_array(pmdp); in __hash_page_thp()
85 flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K, in __hash_page_thp()
162 *pmdp = __pmd(old_pmd); in __hash_page_thp()
186 *pmdp = __pmd(new_pmd & ~H_PAGE_BUSY); in __hash_page_thp()
/linux-6.15/arch/sparc/mm/
H A Dtlb.c226 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
228 pmd_t orig = *pmdp; in set_pmd_at()
230 *pmdp = pmd; in set_pmd_at()
240 old = *pmdp; in pmdp_establish()
251 pmd_t *pmdp) in pmdp_invalidate() argument
255 VM_WARN_ON_ONCE(!pmd_present(*pmdp)); in pmdp_invalidate()
256 entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID); in pmdp_invalidate()
280 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit()
284 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
295 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw()
[all …]
/linux-6.15/arch/powerpc/mm/nohash/
H A Dbook3e_pgtable.c79 pmd_t *pmdp; in map_kernel_page() local
89 pmdp = pmd_alloc(&init_mm, pudp, ea); in map_kernel_page()
90 if (!pmdp) in map_kernel_page()
92 ptep = pte_alloc_kernel(pmdp, ea); in map_kernel_page()
104 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); in map_kernel_page()
105 pud_populate(&init_mm, pudp, pmdp); in map_kernel_page()
107 pmdp = pmd_offset(pudp, ea); in map_kernel_page()
108 if (!pmd_present(*pmdp)) { in map_kernel_page()
110 pmd_populate_kernel(&init_mm, pmdp, ptep); in map_kernel_page()
112 ptep = pte_offset_kernel(pmdp, ea); in map_kernel_page()
H A D8xx.c55 pmd_t *pmdp = pmd_off_k(va); in __early_map_kernel_hugepage() local
66 ptep = early_pte_alloc_kernel(pmdp, va); in __early_map_kernel_hugepage()
71 if (WARN_ON(!pmd_none(*pmdp) || !pmd_none(*(pmdp + 1)))) in __early_map_kernel_hugepage()
75 pmd_populate_kernel(&init_mm, pmdp, ptep); in __early_map_kernel_hugepage()
78 pmd_populate_kernel(&init_mm, pmdp + 1, ptep); in __early_map_kernel_hugepage()
80 ptep = (pte_t *)pmdp; in __early_map_kernel_hugepage()
84 ptep = pte_offset_kernel(pmdp, va); in __early_map_kernel_hugepage()
86 ptep = (pte_t *)pmdp; in __early_map_kernel_hugepage()
/linux-6.15/arch/arm64/include/asm/
H A Dpgalloc.h24 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument
26 set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); in __pud_populate()
29 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) in pud_populate() argument
34 __pud_populate(pudp, __pa(pmdp), pudval); in pud_populate()
37 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument
98 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep, in __pmd_populate() argument
101 set_pmd(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); in __pmd_populate()
109 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
112 __pmd_populate(pmdp, __pa(ptep), in pmd_populate_kernel()
117 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
[all …]
/linux-6.15/arch/s390/mm/
H A Dpgtable.c391 __pmdp_csp(pmdp); in pmdp_idte_global()
402 old = *pmdp; in pmdp_flush_direct()
420 old = *pmdp; in pmdp_flush_lazy()
426 set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID))); in pmdp_flush_lazy()
478 set_pmd(pmdp, new); in pmdp_xchg_direct()
491 set_pmd(pmdp, new); in pmdp_xchg_lazy()
798 pmd_t *pmdp; in set_guest_storage_key() local
820 if (pmd_leaf(*pmdp)) { in set_guest_storage_key()
908 pmd_t *pmdp; in reset_guest_reference_bit() local
931 if (pmd_leaf(*pmdp)) { in reset_guest_reference_bit()
[all …]
H A Dpageattr.c95 ptep = pte_offset_kernel(pmdp, addr); in walk_pte_level()
154 pmd_t new = *pmdp; in modify_pmd_page()
178 pmd_t *pmdp; in walk_pmd_level() local
181 pmdp = pmd_offset(pudp, addr); in walk_pmd_level()
183 if (pmd_none(*pmdp)) in walk_pmd_level()
186 if (pmd_leaf(*pmdp)) { in walk_pmd_level()
191 rc = split_pmd_page(pmdp, addr); in walk_pmd_level()
202 pmdp++; in walk_pmd_level()
212 pmd_t *pm_dir, *pmdp; in split_pud_page() local
225 pmdp = pm_dir; in split_pud_page()
[all …]
H A Dgmap.c365 pmd_t *pmdp; in __gmap_unlink_by_vmaddr() local
371 if (pmdp) { in __gmap_unlink_by_vmaddr()
865 pmd_t *pmdp; in gmap_pmd_op_walk() local
869 if (!pmdp) in gmap_pmd_op_walk()
874 return pmd_none(*pmdp) ? NULL : pmdp; in gmap_pmd_op_walk()
885 return pmdp; in gmap_pmd_op_walk()
1000 pmd_t *pmdp; in gmap_protect_one() local
1006 if (!pmdp) in gmap_protect_one()
2043 pmd_t *pmdp; in gmap_pmdp_clear() local
2051 if (pmdp) { in gmap_pmdp_clear()
[all …]
/linux-6.15/arch/x86/include/asm/
H A Dpgtable-3level.h48 static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) in native_set_pmd() argument
50 pxx_xchg64(pmd, pmdp, native_pmd_val(pmd)); in native_set_pmd()
74 static inline void native_pmd_clear(pmd_t *pmdp) in native_pmd_clear() argument
76 WRITE_ONCE(pmdp->pmd_low, 0); in native_pmd_clear()
78 WRITE_ONCE(pmdp->pmd_high, 0); in native_pmd_clear()
110 return pxx_xchg64(pmd, pmdp, 0ULL); in native_pmdp_get_and_clear()
126 unsigned long address, pmd_t *pmdp, pmd_t pmd) in pmdp_establish() argument
137 old.pmd_low = xchg(&pmdp->pmd_low, pmd.pmd_low); in pmdp_establish()
138 old.pmd_high = READ_ONCE(pmdp->pmd_high); in pmdp_establish()
139 WRITE_ONCE(pmdp->pmd_high, pmd.pmd_high); in pmdp_establish()
[all …]
/linux-6.15/arch/powerpc/include/asm/nohash/32/
H A Dpgalloc.h17 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument
21 *pmdp = __pmd((unsigned long)pte | _PMD_PRESENT); in pmd_populate_kernel()
23 *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); in pmd_populate_kernel()
26 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument
30 *pmdp = __pmd((unsigned long)pte_page | _PMD_PRESENT); in pmd_populate()
32 *pmdp = __pmd(__pa(pte_page) | _PMD_USER | _PMD_PRESENT); in pmd_populate()
/linux-6.15/arch/arm64/mm/
H A Dmmu.c284 pmd_t *pmdp; in alloc_init_cont_pmd() local
300 pmdp += pmd_index(addr); in alloc_init_cont_pmd()
880 pmd_t *pmdp, pmd; in unmap_hotplug_pmd_range() local
891 pmd_clear(pmdp); in unmap_hotplug_pmd_range()
1018 pmd_clear(pmdp); in free_empty_pte_table()
1027 pmd_t *pmdp, pmd; in free_empty_pmd_table() local
1225 set_pmd(pmdp, new_pmd); in pmd_set_huge()
1247 pmd_clear(pmdp); in pmd_clear_huge()
1264 pmd_clear(pmdp); in pmd_free_pte_page()
1273 pmd_t *pmdp; in pud_free_pmd_page() local
[all …]
H A Dfixmap.c43 static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr) in early_fixmap_init_pte() argument
45 pmd_t pmd = READ_ONCE(*pmdp); in early_fixmap_init_pte()
50 __pmd_populate(pmdp, __pa_symbol(ptep), in early_fixmap_init_pte()
60 pmd_t *pmdp; in early_fixmap_init_pmd() local
66 pmdp = pmd_offset_kimg(pudp, addr); in early_fixmap_init_pmd()
69 early_fixmap_init_pte(pmdp, addr); in early_fixmap_init_pmd()
70 } while (pmdp++, addr = next, addr != end); in early_fixmap_init_pmd()
/linux-6.15/arch/arm/include/asm/
H A Dpgalloc.h107 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte, in __pmd_populate() argument
111 pmdp[0] = __pmd(pmdval); in __pmd_populate()
113 pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); in __pmd_populate()
115 flush_pmd_entry(pmdp); in __pmd_populate()
125 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
130 __pmd_populate(pmdp, __pa(ptep), _PAGE_KERNEL_TABLE); in pmd_populate_kernel()
134 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
144 __pmd_populate(pmdp, page_to_phys(ptep), prot); in pmd_populate()
/linux-6.15/arch/powerpc/mm/
H A Dpgtable.c229 pmd_t *pmdp = pmd_off_k(va); in unmap_kernel_page() local
326 pmd_t *pmdp = pmd_off(mm, addr); in set_huge_pte_at() local
331 *pmdp = __pmd(pmd_val(*pmdp) | _PMD_PAGE_8M); in set_huge_pte_at()
332 *(pmdp + 1) = __pmd(pmd_val(*(pmdp + 1)) | _PMD_PAGE_8M); in set_huge_pte_at()
334 __set_huge_pte_at(pmdp, pte_offset_kernel(pmdp, 0), pte_val(pte)); in set_huge_pte_at()
335 __set_huge_pte_at(pmdp, pte_offset_kernel(pmdp + 1, 0), pte_val(pte) + SZ_4M); in set_huge_pte_at()
436 pmd_t pmd, *pmdp; in __find_linux_pte() local
487 pmdp = pmd_offset(&pud, ea); in __find_linux_pte()
492 pmd = READ_ONCE(*pmdp); in __find_linux_pte()
515 ret_pte = (pte_t *)pmdp; in __find_linux_pte()
[all …]
H A Dpgtable_32.c42 pmd_t *pmdp = pmd_off_k(addr); in early_ioremap_init() local
45 addr += PGDIR_SIZE, ptep += PTRS_PER_PTE, pmdp++) in early_ioremap_init()
46 pmd_populate_kernel(&init_mm, pmdp, ptep); in early_ioremap_init()
57 pte_t __init *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va) in early_pte_alloc_kernel() argument
59 if (pmd_none(*pmdp)) { in early_pte_alloc_kernel()
62 pmd_populate_kernel(&init_mm, pmdp, ptep); in early_pte_alloc_kernel()
64 return pte_offset_kernel(pmdp, va); in early_pte_alloc_kernel()
/linux-6.15/include/linux/
H A Dpgtable.h391 pmd_t *pmdp) in pmdp_test_and_clear_young() argument
393 pmd_t pmd = *pmdp; in pmdp_test_and_clear_young()
404 pmd_t *pmdp) in pmdp_test_and_clear_young() argument
624 return pmdp_get(pmdp); in pmdp_get_lockless()
635 pmd_t *pmdp) in pmdp_huge_get_and_clear() argument
637 pmd_t pmd = *pmdp; in pmdp_huge_get_and_clear()
639 pmd_clear(pmdp); in pmdp_huge_get_and_clear()
834 pmd_t *pmdp);
953 pmd_t *pmdp) in pmdp_collapse_flush() argument
956 return *pmdp; in pmdp_collapse_flush()
[all …]
/linux-6.15/arch/powerpc/mm/kasan/
H A Dinit_book3e_64.c37 pmd_t *pmdp; in kasan_map_kernel_page() local
49 pmdp = memblock_alloc_or_panic(PMD_TABLE_SIZE, PMD_TABLE_SIZE); in kasan_map_kernel_page()
50 memcpy(pmdp, kasan_early_shadow_pmd, PMD_TABLE_SIZE); in kasan_map_kernel_page()
51 pud_populate(&init_mm, pudp, pmdp); in kasan_map_kernel_page()
53 pmdp = pmd_offset(pudp, ea); in kasan_map_kernel_page()
54 if (kasan_pte_table(*pmdp)) { in kasan_map_kernel_page()
57 pmd_populate_kernel(&init_mm, pmdp, ptep); in kasan_map_kernel_page()
59 ptep = pte_offset_kernel(pmdp, ea); in kasan_map_kernel_page()
/linux-6.15/arch/powerpc/include/asm/book3s/64/
H A Dhash-4k.h122 static inline char *get_hpte_slot_array(pmd_t *pmdp) in get_hpte_slot_array() argument
159 unsigned long addr, pmd_t *pmdp,
162 unsigned long address, pmd_t *pmdp);
163 extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
165 extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
167 unsigned long addr, pmd_t *pmdp);
/linux-6.15/arch/arm/mm/
H A Dkasan_init.c43 static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, in kasan_pte_populate() argument
47 pte_t *ptep = pte_offset_kernel(pmdp, addr); in kasan_pte_populate()
97 pmd_t *pmdp = pmd_offset(pudp, addr); in kasan_pmd_populate() local
100 if (pmd_none(*pmdp)) { in kasan_pmd_populate()
114 pmd_populate_kernel(&init_mm, pmdp, p); in kasan_pmd_populate()
115 flush_pmd_entry(pmdp); in kasan_pmd_populate()
119 kasan_pte_populate(pmdp, addr, next, early); in kasan_pmd_populate()
120 } while (pmdp++, addr = next, addr != end); in kasan_pmd_populate()
/linux-6.15/arch/xtensa/include/asm/
H A Dpgalloc.h24 #define pmd_populate_kernel(mm, pmdp, ptep) \ argument
25 (pmd_val(*(pmdp)) = ((unsigned long)ptep))
26 #define pmd_populate(mm, pmdp, page) \ argument
27 (pmd_val(*(pmdp)) = ((unsigned long)page_to_virt(page)))
/linux-6.15/arch/s390/include/asm/
H A Dpgtable.h1032 WRITE_ONCE(*pmdp, pmd); in set_pmd()
1701 csp((unsigned int *)pmdp + 1, pmd_val(*pmdp), in __pmdp_csp()
1723 : "+m" (*pmdp) in __pmdp_idte()
1731 : "+m" (*pmdp) in __pmdp_idte()
1798 pmd_t pmd = *pmdp; in pmdp_test_and_clear_young()
1813 pmd_t *pmdp, pmd_t entry) in set_pmd_at() argument
1815 set_pmd(pmdp, entry); in set_pmd_at()
1835 pmd_t *pmdp, int full) in pmdp_huge_get_and_clear_full() argument
1838 pmd_t pmd = *pmdp; in pmdp_huge_get_and_clear_full()
1867 pmd_t pmd = *pmdp; in pmdp_set_wrprotect()
[all …]

12345