Home
last modified time | relevance | path

Searched refs:pgdp (Results 1 – 25 of 54) sorted by relevance

123

/linux-6.15/arch/riscv/mm/
H A Dkasan_init.c142 static void __init kasan_populate_pgd(pgd_t *pgdp, in kasan_populate_pgd() argument
161 kasan_populate_p4d(pgdp, vaddr, next); in kasan_populate_pgd()
162 } while (pgdp++, vaddr = next, vaddr != end); in kasan_populate_pgd()
190 static void __init kasan_early_clear_p4d(pgd_t *pgdp, in kasan_early_clear_p4d() argument
197 p4dp = (p4d_t *)pgdp; in kasan_early_clear_p4d()
226 pgd_clear(pgdp); in kasan_early_clear_pgd()
230 kasan_early_clear_p4d(pgdp, vaddr, next); in kasan_early_clear_pgd()
231 } while (pgdp++, vaddr = next, vaddr != end); in kasan_early_clear_pgd()
281 p4dp = (p4d_t *)pgdp; in kasan_early_populate_p4d()
318 kasan_early_populate_p4d(pgdp, vaddr, next); in kasan_early_populate_pgd()
[all …]
H A Dfault.c27 pgd_t *pgdp, pgd; in show_pte() local
41 pgdp = pgd_offset(mm, addr); in show_pte()
42 pgd = pgdp_get(pgdp); in show_pte()
47 p4dp = p4d_offset(pgdp, addr); in show_pte()
H A Dpageattr.c180 static int __split_linear_mapping_p4d(pgd_t *pgdp, in __split_linear_mapping_p4d() argument
187 p4dp = p4d_offset(pgdp, vaddr); in __split_linear_mapping_p4d()
238 static int __split_linear_mapping_pgd(pgd_t *pgdp, in __split_linear_mapping_pgd() argument
248 ret = __split_linear_mapping_p4d(pgdp, vaddr, next); in __split_linear_mapping_pgd()
251 } while (pgdp++, vaddr = next, vaddr != end); in __split_linear_mapping_pgd()
/linux-6.15/arch/arm64/mm/
H A Dkasan_init.c105 if (pgd_none(READ_ONCE(*pgdp))) { in kasan_p4d_offset()
109 __pgd_populate(pgdp, p4d_phys, PGD_TYPE_TABLE); in kasan_p4d_offset()
112 return early ? p4d_offset_kimg(pgdp, addr) : p4d_offset(pgdp, addr); in kasan_p4d_offset()
156 static void __init kasan_p4d_populate(pgd_t *pgdp, unsigned long addr, in kasan_p4d_populate() argument
160 p4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early); in kasan_p4d_populate()
172 pgd_t *pgdp; in kasan_pgd_populate() local
174 pgdp = pgd_offset_k(addr); in kasan_pgd_populate()
177 kasan_p4d_populate(pgdp, addr, next, node, early); in kasan_pgd_populate()
178 } while (pgdp++, addr = next, addr != end); in kasan_pgd_populate()
218 pgd_t *pgdp = pgd_offset_k(KASAN_SHADOW_START); in kasan_early_init() local
[all …]
H A Dmmu.c81 WRITE_ONCE(*pgdp, pgd); in set_swapper_pgd()
390 pgd_t pgd = READ_ONCE(*pgdp); in alloc_init_p4d()
950 p4dp = p4d_offset(pgdp, addr); in unmap_hotplug_p4d_range()
964 pgd_t *pgdp, pgd; in unmap_hotplug_range() local
976 pgdp = pgd_offset_k(addr); in unmap_hotplug_range()
977 pgd = READ_ONCE(*pgdp); in unmap_hotplug_range()
1132 p4dp = p4d_offset(pgdp, 0UL); in free_empty_p4d_table()
1138 pgd_clear(pgdp); in free_empty_p4d_table()
1147 pgd_t *pgdp, pgd; in free_empty_tables() local
1151 pgdp = pgd_offset_k(addr); in free_empty_tables()
[all …]
H A Dhugetlbpage.c85 pgd_t *pgdp = pgd_offset(mm, addr); in find_num_contig() local
91 p4dp = p4d_offset(pgdp, addr); in find_num_contig()
250 pgd_t *pgdp; in huge_pte_alloc() local
256 pgdp = pgd_offset(mm, addr); in huge_pte_alloc()
257 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc()
291 pgd_t *pgdp; in huge_pte_offset() local
296 pgdp = pgd_offset(mm, addr); in huge_pte_offset()
297 if (!pgd_present(READ_ONCE(*pgdp))) in huge_pte_offset()
300 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
H A Dpageattr.c321 pgd_t *pgdp; in kernel_page_present() local
328 pgdp = pgd_offset_k(addr); in kernel_page_present()
329 if (pgd_none(READ_ONCE(*pgdp))) in kernel_page_present()
332 p4dp = p4d_offset(pgdp, addr); in kernel_page_present()
H A Dfixmap.c109 pgd_t *pgdp = pgd_offset_k(addr); in early_fixmap_init() local
110 p4d_t *p4dp = p4d_offset_kimg(pgdp, addr); in early_fixmap_init()
/linux-6.15/arch/loongarch/mm/
H A Dkasan_init.c159 if (__pgd_none(early, pgdp_get(pgdp))) { in kasan_p4d_offset()
164 pgd_populate(&init_mm, pgdp, (p4d_t *)__va(p4d_phys)); in kasan_p4d_offset()
167 return p4d_offset(pgdp, addr); in kasan_p4d_offset()
209 static void __init kasan_p4d_populate(pgd_t *pgdp, unsigned long addr, in kasan_p4d_populate() argument
213 p4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early); in kasan_p4d_populate()
225 pgd_t *pgdp; in kasan_pgd_populate() local
227 pgdp = pgd_offset_k(addr); in kasan_pgd_populate()
231 kasan_p4d_populate(pgdp, addr, next, node, early); in kasan_pgd_populate()
232 } while (pgdp++, addr = next, addr != end); in kasan_pgd_populate()
249 static inline void kasan_set_pgd(pgd_t *pgdp, pgd_t pgdval) in kasan_set_pgd() argument
[all …]
/linux-6.15/arch/arm64/include/asm/
H A Dpgalloc.h74 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t p4dp, pgdval_t prot) in __pgd_populate() argument
77 set_pgd(pgdp, __pgd(__phys_to_pgd_val(p4dp) | prot)); in __pgd_populate()
80 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, p4d_t *p4dp) in pgd_populate() argument
85 __pgd_populate(pgdp, __pa(p4dp), pgdval); in pgd_populate()
89 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t p4dp, pgdval_t prot) in __pgd_populate() argument
96 extern void pgd_free(struct mm_struct *mm, pgd_t *pgdp);
H A Dpgtable.h1023 #define pud_set_fixmap_offset(pgdp, addr) ((pud_t *)pgdp) argument
1056 if (in_swapper_pgdir(pgdp)) { in set_pgd()
1057 set_swapper_pgd(pgdp, __pgd(pgd_val(pgd))); in set_pgd()
1061 WRITE_ONCE(*pgdp, pgd); in set_pgd()
1066 static inline void pgd_clear(pgd_t *pgdp) in pgd_clear() argument
1069 set_pgd(pgdp, __pgd(0)); in pgd_clear()
1098 return pgd_to_folded_p4d(pgdp, addr); in p4d_offset_lockless()
1105 return p4d_offset_lockless(pgdp, READ_ONCE(*pgdp), addr); in p4d_offset()
1118 return pgd_to_folded_p4d(pgdp, addr); in p4d_set_fixmap_offset()
1132 return pgd_to_folded_p4d(pgdp, addr); in p4d_offset_kimg()
[all …]
H A Dmmu_context.h145 void __cpu_replace_ttbr1(pgd_t *pgdp, bool cnp);
152 static inline void cpu_replace_ttbr1(pgd_t *pgdp) in cpu_replace_ttbr1() argument
159 __cpu_replace_ttbr1(pgdp, false); in cpu_replace_ttbr1()
/linux-6.15/arch/powerpc/mm/nohash/
H A Dbook3e_pgtable.c76 pgd_t *pgdp; in map_kernel_page() local
84 pgdp = pgd_offset_k(ea); in map_kernel_page()
85 p4dp = p4d_offset(pgdp, ea); in map_kernel_page()
96 pgdp = pgd_offset_k(ea); in map_kernel_page()
97 p4dp = p4d_offset(pgdp, ea); in map_kernel_page()
/linux-6.15/arch/s390/mm/
H A Dhugetlbpage.c209 pgd_t *pgdp; in huge_pte_alloc() local
214 pgdp = pgd_offset(mm, addr); in huge_pte_alloc()
215 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc()
231 pgd_t *pgdp; in huge_pte_offset() local
236 pgdp = pgd_offset(mm, addr); in huge_pte_offset()
237 if (pgd_present(*pgdp)) { in huge_pte_offset()
238 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
H A Dpageattr.c321 pgd_t *pgdp; in change_page_attr() local
323 pgdp = pgd_offset_k(addr); in change_page_attr()
325 if (pgd_none(*pgdp)) in change_page_attr()
328 rc = walk_p4d_level(pgdp, addr, next, flags); in change_page_attr()
332 } while (pgdp++, addr = next, addr < end && !rc); in change_page_attr()
/linux-6.15/arch/arm/mm/
H A Dkasan_init.c127 pgd_t *pgdp; in kasan_pgd_populate() local
131 pgdp = pgd_offset_k(addr); in kasan_pgd_populate()
138 if (!early && pgd_none(*pgdp)) { in kasan_pgd_populate()
146 pgd_populate(&init_mm, pgdp, p); in kasan_pgd_populate()
155 p4dp = p4d_offset(pgdp, addr); in kasan_pgd_populate()
159 } while (pgdp++, addr = next, addr != end); in kasan_pgd_populate()
/linux-6.15/arch/arm64/kernel/pi/
H A Dmap_kernel.c43 u64 pgdp = (u64)init_pg_dir + PAGE_SIZE; in map_kernel() local
81 map_segment(init_pg_dir, &pgdp, va_offset, _stext, _etext, prot, in map_kernel()
83 map_segment(init_pg_dir, &pgdp, va_offset, __start_rodata, in map_kernel()
85 map_segment(init_pg_dir, &pgdp, va_offset, __inittext_begin, in map_kernel()
87 map_segment(init_pg_dir, &pgdp, va_offset, __initdata_begin, in map_kernel()
89 map_segment(init_pg_dir, &pgdp, va_offset, _data, _end, data_prot, in map_kernel()
/linux-6.15/arch/sparc/mm/
H A Dsrmmu.c489 pgd_t *pgdp; in srmmu_mapioaddr() local
497 pgdp = pgd_offset_k(virt_addr); in srmmu_mapioaddr()
528 pgd_t *pgdp; in srmmu_unmapioaddr() local
535 pgdp = pgd_offset_k(virt_addr); in srmmu_unmapioaddr()
675 pgd_t *pgdp; in srmmu_early_allocate_ptable_skeleton() local
682 pgdp = pgd_offset_k(start); in srmmu_early_allocate_ptable_skeleton()
710 pgd_t *pgdp; in srmmu_allocate_ptable_skeleton() local
717 pgdp = pgd_offset_k(start); in srmmu_allocate_ptable_skeleton()
725 pud_set((pud_t *)pgdp, pmdp); in srmmu_allocate_ptable_skeleton()
769 pgd_t *pgdp; in srmmu_inherit_prom_mappings() local
[all …]
H A Dfault_64.c81 pgd_t *pgdp = pgd_offset(current->mm, tpc); in get_user_insn() local
89 if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) in get_user_insn()
91 p4dp = p4d_offset(pgdp, tpc); in get_user_insn()
/linux-6.15/arch/x86/mm/
H A Dpgtable.c235 static void mop_up_one_pmd(struct mm_struct *mm, pgd_t *pgdp) in mop_up_one_pmd() argument
237 pgd_t pgd = *pgdp; in mop_up_one_pmd()
242 pgd_clear(pgdp); in mop_up_one_pmd()
250 static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp) in pgd_mop_up_pmds() argument
255 mop_up_one_pmd(mm, &pgdp[i]); in pgd_mop_up_pmds()
262 pgdp = kernel_to_user_pgdp(pgdp); in pgd_mop_up_pmds()
265 mop_up_one_pmd(mm, &pgdp[i + KERNEL_PGD_BOUNDARY]); in pgd_mop_up_pmds()
/linux-6.15/arch/powerpc/mm/kasan/
H A Dinit_book3e_64.c34 pgd_t *pgdp; in kasan_map_kernel_page() local
40 pgdp = pgd_offset_k(ea); in kasan_map_kernel_page()
41 p4dp = p4d_offset(pgdp, ea); in kasan_map_kernel_page()
/linux-6.15/arch/x86/include/asm/
H A Dparavirt.h493 static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd) in __set_pgd() argument
495 PVOP_VCALL2(mmu.set_pgd, pgdp, native_pgd_val(pgd)); in __set_pgd()
498 #define set_pgd(pgdp, pgdval) do { \ argument
500 __set_pgd(pgdp, pgdval); \
502 set_p4d((p4d_t *)(pgdp), (p4d_t) { (pgdval).pgd }); \
505 #define pgd_clear(pgdp) do { \ argument
507 set_pgd(pgdp, native_make_pgd(0)); \
H A Dpgtable.h76 #define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd) argument
956 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);
963 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgtbl() argument
967 return __pti_set_user_pgtbl(pgdp, pgd); in pti_set_user_pgtbl()
1506 static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp) in kernel_to_user_pgdp() argument
1508 return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); in kernel_to_user_pgdp()
1511 static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp) in user_to_kernel_pgdp() argument
1513 return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); in user_to_kernel_pgdp()
1810 #define set_pgd_safe(pgdp, pgd) \ argument
1812 WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \
[all …]
H A Dpgtable_64.h162 static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) in native_set_pgd() argument
164 WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); in native_set_pgd()
/linux-6.15/arch/powerpc/mm/
H A Dpgtable.c431 pgd_t *pgdp; in __find_linux_pte() local
456 pgdp = pgdir + pgd_index(ea); in __find_linux_pte()
458 p4dp = p4d_offset(pgdp, ea); in __find_linux_pte()
489 pmdp = pmd_offset(pud_offset(p4d_offset(pgdp, ea), ea), ea); in __find_linux_pte()

123