Home
last modified time | relevance | path

Searched refs:ptdesc (Results 1 – 25 of 25) sorted by relevance

/linux-6.15/include/asm-generic/
H A Dpgalloc.h24 if (!ptdesc) in __pte_alloc_one_kernel_noprof()
68 struct ptdesc *ptdesc; in __pte_alloc_one_noprof() local
110 struct ptdesc *ptdesc = page_ptdesc(pte_page); in pte_free() local
132 struct ptdesc *ptdesc; in pmd_alloc_one_noprof() local
152 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); in pmd_free() local
166 struct ptdesc *ptdesc; in __pud_alloc_one_noprof() local
200 struct ptdesc *ptdesc = virt_to_ptdesc(pud); in __pud_free() local
220 struct ptdesc *ptdesc; in __p4d_alloc_one_noprof() local
245 struct ptdesc *ptdesc = virt_to_ptdesc(p4d); in __p4d_free() local
264 struct ptdesc *ptdesc; in __pgd_alloc_noprof() local
[all …]
H A Dtlb.h213 struct ptdesc *ptdesc = (struct ptdesc *)table; in __tlb_remove_table() local
215 pagetable_dtor_free(ptdesc); in __tlb_remove_table()
230 struct ptdesc *ptdesc = (struct ptdesc *)table; in tlb_remove_table() local
232 pagetable_dtor(ptdesc); in tlb_remove_table()
233 tlb_remove_page(tlb, ptdesc_page(ptdesc)); in tlb_remove_table()
496 static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt) in tlb_remove_ptdesc()
/linux-6.15/arch/powerpc/mm/
H A Dpgtable-frag.c21 struct ptdesc *ptdesc; in pte_frag_destroy() local
28 pagetable_dtor(ptdesc); in pte_frag_destroy()
29 pagetable_free(ptdesc); in pte_frag_destroy()
58 struct ptdesc *ptdesc; in __alloc_for_ptecache() local
62 if (!ptdesc) in __alloc_for_ptecache()
70 if (!ptdesc) in __alloc_for_ptecache()
111 struct ptdesc *ptdesc; in pte_free_now() local
113 ptdesc = container_of(head, struct ptdesc, pt_rcu_head); in pte_free_now()
114 pagetable_dtor(ptdesc); in pte_free_now()
115 pagetable_free(ptdesc); in pte_free_now()
[all …]
/linux-6.15/arch/m68k/include/asm/
H A Dmcf_pgalloc.h17 struct ptdesc *ptdesc = pagetable_alloc((GFP_DMA | __GFP_ZERO) & in pte_alloc_one_kernel() local
20 if (!ptdesc) in pte_alloc_one_kernel()
38 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); in __pte_free_tlb() local
40 pagetable_dtor(ptdesc); in __pte_free_tlb()
41 pagetable_free(ptdesc); in __pte_free_tlb()
46 struct ptdesc *ptdesc = pagetable_alloc(GFP_DMA | __GFP_ZERO, 0); in pte_alloc_one() local
49 if (!ptdesc) in pte_alloc_one()
62 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); in pte_free() local
64 pagetable_dtor(ptdesc); in pte_free()
82 struct ptdesc *ptdesc = pagetable_alloc((GFP_DMA | __GFP_NOWARN) & in pgd_alloc() local
[all …]
/linux-6.15/arch/s390/mm/
H A Dpgalloc.c24 if (!ptdesc) in crst_table_alloc()
120 struct ptdesc *ptdesc; in page_table_alloc_pgste() local
124 if (ptdesc) { in page_table_alloc_pgste()
133 void page_table_free_pgste(struct ptdesc *ptdesc) in page_table_free_pgste() argument
142 struct ptdesc *ptdesc; in page_table_alloc() local
146 if (!ptdesc) in page_table_alloc()
161 struct ptdesc *ptdesc = virt_to_ptdesc(table); in page_table_free() local
169 struct ptdesc *ptdesc = container_of(head, struct ptdesc, pt_rcu_head); in pte_free_now() local
176 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); in pte_free_defer() local
212 struct ptdesc *ptdesc; in base_crst_alloc() local
[all …]
H A Dgmap.c1242 struct ptdesc *ptdesc; in gmap_unshadow_pgt() local
1255 ptdesc = page_ptdesc(phys_to_page(pgt)); in gmap_unshadow_pgt()
1256 page_table_free_pgste(ptdesc); in gmap_unshadow_pgt()
1270 struct ptdesc *ptdesc; in __gmap_unshadow_sgt() local
1283 page_table_free_pgste(ptdesc); in __gmap_unshadow_sgt()
1738 static void gmap_pgste_set_pgt_addr(struct ptdesc *ptdesc, unsigned long pgt_addr) in gmap_pgste_set_pgt_addr() argument
1773 struct ptdesc *ptdesc; in gmap_shadow_pgt() local
1779 ptdesc = page_table_alloc_pgste(sg->mm); in gmap_shadow_pgt()
1780 if (!ptdesc) in gmap_shadow_pgt()
1785 gmap_pgste_set_pgt_addr(ptdesc, origin); in gmap_shadow_pgt()
[all …]
/linux-6.15/arch/mips/include/asm/
H A Dpgalloc.h59 struct ptdesc *ptdesc; in pmd_alloc_one() local
61 ptdesc = pagetable_alloc(GFP_KERNEL_ACCOUNT, PMD_TABLE_ORDER); in pmd_alloc_one()
62 if (!ptdesc) in pmd_alloc_one()
65 if (!pagetable_pmd_ctor(ptdesc)) { in pmd_alloc_one()
66 pagetable_free(ptdesc); in pmd_alloc_one()
70 pmd = ptdesc_address(ptdesc); in pmd_alloc_one()
84 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, in pud_alloc_one() local
87 if (!ptdesc) in pud_alloc_one()
89 pagetable_pud_ctor(ptdesc); in pud_alloc_one()
90 pud = ptdesc_address(ptdesc); in pud_alloc_one()
/linux-6.15/arch/arm64/include/asm/
H A Dtlb.h78 struct ptdesc *ptdesc = page_ptdesc(pte); in __pte_free_tlb() local
80 tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
87 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); in __pmd_free_tlb() local
89 tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
97 struct ptdesc *ptdesc = virt_to_ptdesc(pudp); in __pud_free_tlb() local
102 tlb_remove_ptdesc(tlb, ptdesc); in __pud_free_tlb()
110 struct ptdesc *ptdesc = virt_to_ptdesc(p4dp); in __p4d_free_tlb() local
115 tlb_remove_ptdesc(tlb, ptdesc); in __p4d_free_tlb()
/linux-6.15/arch/loongarch/include/asm/
H A Dpgalloc.h66 struct ptdesc *ptdesc; in pmd_alloc_one() local
68 ptdesc = pagetable_alloc(GFP_KERNEL_ACCOUNT, 0); in pmd_alloc_one()
69 if (!ptdesc) in pmd_alloc_one()
72 if (!pagetable_pmd_ctor(ptdesc)) { in pmd_alloc_one()
73 pagetable_free(ptdesc); in pmd_alloc_one()
77 pmd = ptdesc_address(ptdesc); in pmd_alloc_one()
91 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in pud_alloc_one() local
93 if (!ptdesc) in pud_alloc_one()
95 pagetable_pud_ctor(ptdesc); in pud_alloc_one()
96 pud = ptdesc_address(ptdesc); in pud_alloc_one()
/linux-6.15/arch/parisc/include/asm/
H A Dpgalloc.h34 struct ptdesc *ptdesc; in pmd_alloc_one() local
39 ptdesc = pagetable_alloc(gfp, PMD_TABLE_ORDER); in pmd_alloc_one()
40 if (!ptdesc) in pmd_alloc_one()
42 if (!pagetable_pmd_ctor(ptdesc)) { in pmd_alloc_one()
43 pagetable_free(ptdesc); in pmd_alloc_one()
46 return ptdesc_address(ptdesc); in pmd_alloc_one()
/linux-6.15/arch/arm/include/asm/
H A Dtlb.h35 struct ptdesc *ptdesc = page_ptdesc(pte); in __pte_free_tlb() local
46 tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
53 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); in __pmd_free_tlb() local
55 tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
/linux-6.15/arch/powerpc/mm/book3s64/
H A Dpgtable.c417 struct ptdesc *ptdesc; in __alloc_for_pmdcache() local
422 ptdesc = pagetable_alloc(gfp, 0); in __alloc_for_pmdcache()
423 if (!ptdesc) in __alloc_for_pmdcache()
425 if (!pagetable_pmd_ctor(ptdesc)) { in __alloc_for_pmdcache()
426 pagetable_free(ptdesc); in __alloc_for_pmdcache()
432 ret = ptdesc_address(ptdesc); in __alloc_for_pmdcache()
468 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); in pmd_fragment_free() local
470 if (pagetable_is_reserved(ptdesc)) in pmd_fragment_free()
471 return free_reserved_ptdesc(ptdesc); in pmd_fragment_free()
475 pagetable_dtor(ptdesc); in pmd_fragment_free()
[all …]
H A Dmmu_context.c249 struct ptdesc *ptdesc; in pmd_frag_destroy() local
251 ptdesc = virt_to_ptdesc(pmd_frag); in pmd_frag_destroy()
255 if (atomic_sub_and_test(PMD_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { in pmd_frag_destroy()
256 pagetable_dtor(ptdesc); in pmd_frag_destroy()
257 pagetable_free(ptdesc); in pmd_frag_destroy()
/linux-6.15/arch/x86/mm/
H A Dpgtable.c59 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); in pgd_list_add() local
66 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); in pgd_list_del() local
68 list_del(&ptdesc->pt_list); in pgd_list_del()
180 struct ptdesc *ptdesc; in free_pmds() local
186 pagetable_dtor(ptdesc); in free_pmds()
187 pagetable_free(ptdesc); in free_pmds()
204 struct ptdesc *ptdesc = pagetable_alloc(gfp, 0); in preallocate_pmds() local
206 if (!ptdesc) in preallocate_pmds()
208 if (ptdesc && !pagetable_pmd_ctor(ptdesc)) { in preallocate_pmds()
210 ptdesc = NULL; in preallocate_pmds()
[all …]
/linux-6.15/include/linux/
H A Dmm_types.h550 struct ptdesc { struct
605 struct ptdesc *: (struct page *)(pt)))
613 struct page *: (struct ptdesc *)(p)))
616 static inline void ptdesc_pmd_pts_init(struct ptdesc *ptdesc) in ptdesc_pmd_pts_init() argument
618 atomic_set(&ptdesc->pt_share_count, 0); in ptdesc_pmd_pts_init()
621 static inline void ptdesc_pmd_pts_inc(struct ptdesc *ptdesc) in ptdesc_pmd_pts_inc() argument
623 atomic_inc(&ptdesc->pt_share_count); in ptdesc_pmd_pts_inc()
626 static inline void ptdesc_pmd_pts_dec(struct ptdesc *ptdesc) in ptdesc_pmd_pts_dec() argument
628 atomic_dec(&ptdesc->pt_share_count); in ptdesc_pmd_pts_dec()
631 static inline int ptdesc_pmd_pts_count(struct ptdesc *ptdesc) in ptdesc_pmd_pts_count() argument
[all …]
H A Dmm.h3025 bool ptlock_alloc(struct ptdesc *ptdesc);
3026 void ptlock_free(struct ptdesc *ptdesc);
3028 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) in ptlock_ptr() argument
3037 static inline bool ptlock_alloc(struct ptdesc *ptdesc) in ptlock_alloc() argument
3042 static inline void ptlock_free(struct ptdesc *ptdesc) in ptlock_free() argument
3046 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) in ptlock_ptr() argument
3064 static inline bool ptlock_init(struct ptdesc *ptdesc) in ptlock_init() argument
3094 static inline void ptlock_free(struct ptdesc *ptdesc) {} in ptlock_free() argument
3097 static inline void __pagetable_ctor(struct ptdesc *ptdesc) in __pagetable_ctor() argument
3105 static inline void pagetable_dtor(struct ptdesc *ptdesc) in pagetable_dtor() argument
[all …]
/linux-6.15/mm/
H A Dmmu_gather.c325 struct ptdesc *ptdesc; in __tlb_remove_table_one_rcu() local
327 ptdesc = container_of(head, struct ptdesc, pt_rcu_head); in __tlb_remove_table_one_rcu()
328 __tlb_remove_table(ptdesc); in __tlb_remove_table_one_rcu()
333 struct ptdesc *ptdesc; in __tlb_remove_table_one() local
335 ptdesc = table; in __tlb_remove_table_one()
336 call_rcu(&ptdesc->pt_rcu_head, __tlb_remove_table_one_rcu); in __tlb_remove_table_one()
H A Dmemory.c7350 bool ptlock_alloc(struct ptdesc *ptdesc) in ptlock_alloc() argument
7357 ptdesc->ptl = ptl; in ptlock_alloc()
7361 void ptlock_free(struct ptdesc *ptdesc) in ptlock_free() argument
7363 if (ptdesc->ptl) in ptlock_free()
7364 kmem_cache_free(page_ptl_cachep, ptdesc->ptl); in ptlock_free()
/linux-6.15/arch/s390/include/asm/
H A Dpgalloc.h26 struct ptdesc *page_table_alloc_pgste(struct mm_struct *mm);
28 void page_table_free_pgste(struct ptdesc *ptdesc);
/linux-6.15/arch/riscv/mm/
H A Dinit.c443 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in alloc_pte_late() local
445 BUG_ON(!ptdesc || !pagetable_pte_ctor(ptdesc)); in alloc_pte_late()
446 return __pa((pte_t *)ptdesc_address(ptdesc)); in alloc_pte_late()
523 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in alloc_pmd_late() local
525 BUG_ON(!ptdesc || !pagetable_pmd_ctor(ptdesc)); in alloc_pmd_late()
1566 struct ptdesc *ptdesc = page_ptdesc(page); in free_pte_table() local
1576 pagetable_dtor(ptdesc); in free_pte_table()
1580 pagetable_free(ptdesc); in free_pte_table()
1587 struct ptdesc *ptdesc = page_ptdesc(page); in free_pmd_table() local
1598 pagetable_dtor(ptdesc); in free_pmd_table()
[all …]
/linux-6.15/arch/sparc/mm/
H A Dinit_64.c2894 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); in pte_alloc_one() local
2896 if (!ptdesc) in pte_alloc_one()
2898 if (!pagetable_pte_ctor(ptdesc)) { in pte_alloc_one()
2899 pagetable_free(ptdesc); in pte_alloc_one()
2902 return ptdesc_address(ptdesc); in pte_alloc_one()
2912 struct ptdesc *ptdesc = virt_to_ptdesc(pte); in __pte_free() local
2914 pagetable_dtor(ptdesc); in __pte_free()
2915 pagetable_free(ptdesc); in __pte_free()
/linux-6.15/arch/arm64/mm/
H A Dmmu.c486 struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa)); in pgd_pgtable_alloc() local
497 BUG_ON(!pagetable_pte_ctor(ptdesc)); in pgd_pgtable_alloc()
499 BUG_ON(!pagetable_pmd_ctor(ptdesc)); in pgd_pgtable_alloc()
/linux-6.15/arch/arm/mm/
H A Dmmu.c735 void *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, in late_alloc() local
738 if (!ptdesc || !pagetable_pte_ctor(ptdesc)) in late_alloc()
740 return ptdesc_to_virt(ptdesc); in late_alloc()
/linux-6.15/arch/x86/mm/pat/
H A Dset_memory.c416 struct ptdesc *ptdesc, *tmp; in cpa_collapse_large_pages() local
439 list_for_each_entry_safe(ptdesc, tmp, &pgtables, pt_list) { in cpa_collapse_large_pages()
440 list_del(&ptdesc->pt_list); in cpa_collapse_large_pages()
441 __free_page(ptdesc_page(ptdesc)); in cpa_collapse_large_pages()
/linux-6.15/Documentation/mm/
H A Dprocess_addrs.rst609 physical page metadata in the form of a :c:struct:`!struct ptdesc`, acquired by
621 the :c:struct:`!struct ptdesc` associated with the physical PTE page. The lock