1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_PGTABLE_H 3 #define _LINUX_PGTABLE_H 4 5 #include <linux/pfn.h> 6 #include <asm/pgtable.h> 7 8 #ifndef __ASSEMBLY__ 9 #ifdef CONFIG_MMU 10 11 #include <linux/mm_types.h> 12 #include <linux/bug.h> 13 #include <linux/errno.h> 14 #include <asm-generic/pgtable_uffd.h> 15 16 #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ 17 defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS 18 #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED 19 #endif 20 21 /* 22 * On almost all architectures and configurations, 0 can be used as the 23 * upper ceiling to free_pgtables(): on many architectures it has the same 24 * effect as using TASK_SIZE. However, there is one configuration which 25 * must impose a more careful limit, to avoid freeing kernel pgtables. 26 */ 27 #ifndef USER_PGTABLES_CEILING 28 #define USER_PGTABLES_CEILING 0UL 29 #endif 30 31 /* 32 * A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD] 33 * 34 * The pXx_index() functions return the index of the entry in the page 35 * table page which would control the given virtual address 36 * 37 * As these functions may be used by the same code for different levels of 38 * the page table folding, they are always available, regardless of 39 * CONFIG_PGTABLE_LEVELS value. For the folded levels they simply return 0 40 * because in such cases PTRS_PER_PxD equals 1. 41 */ 42 43 static inline unsigned long pte_index(unsigned long address) 44 { 45 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); 46 } 47 48 #ifndef pmd_index 49 static inline unsigned long pmd_index(unsigned long address) 50 { 51 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); 52 } 53 #define pmd_index pmd_index 54 #endif 55 56 #ifndef pud_index 57 static inline unsigned long pud_index(unsigned long address) 58 { 59 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); 60 } 61 #define pud_index pud_index 62 #endif 63 64 #ifndef pgd_index 65 /* Must be a compile-time constant, so implement it as a macro */ 66 #define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) 67 #endif 68 69 #ifndef pte_offset_kernel 70 static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) 71 { 72 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); 73 } 74 #define pte_offset_kernel pte_offset_kernel 75 #endif 76 77 #if defined(CONFIG_HIGHPTE) 78 #define pte_offset_map(dir, address) \ 79 ((pte_t *)kmap_atomic(pmd_page(*(dir))) + \ 80 pte_index((address))) 81 #define pte_unmap(pte) kunmap_atomic((pte)) 82 #else 83 #define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) 84 #define pte_unmap(pte) ((void)(pte)) /* NOP */ 85 #endif 86 87 /* Find an entry in the second-level page table.. */ 88 #ifndef pmd_offset 89 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) 90 { 91 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); 92 } 93 #define pmd_offset pmd_offset 94 #endif 95 96 #ifndef pud_offset 97 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) 98 { 99 return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address); 100 } 101 #define pud_offset pud_offset 102 #endif 103 104 static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) 105 { 106 return (pgd + pgd_index(address)); 107 }; 108 109 /* 110 * a shortcut to get a pgd_t in a given mm 111 */ 112 #ifndef pgd_offset 113 #define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address)) 114 #endif 115 116 /* 117 * a shortcut which implies the use of the kernel's pgd, instead 118 * of a process's 119 */ 120 #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) 121 122 /* 123 * In many cases it is known that a virtual address is mapped at PMD or PTE 124 * level, so instead of traversing all the page table levels, we can get a 125 * pointer to the PMD entry in user or kernel page table or translate a virtual 126 * address to the pointer in the PTE in the kernel page tables with simple 127 * helpers. 128 */ 129 static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va) 130 { 131 return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va); 132 } 133 134 static inline pmd_t *pmd_off_k(unsigned long va) 135 { 136 return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va); 137 } 138 139 static inline pte_t *virt_to_kpte(unsigned long vaddr) 140 { 141 pmd_t *pmd = pmd_off_k(vaddr); 142 143 return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr); 144 } 145 146 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 147 extern int ptep_set_access_flags(struct vm_area_struct *vma, 148 unsigned long address, pte_t *ptep, 149 pte_t entry, int dirty); 150 #endif 151 152 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS 153 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 154 extern int pmdp_set_access_flags(struct vm_area_struct *vma, 155 unsigned long address, pmd_t *pmdp, 156 pmd_t entry, int dirty); 157 extern int pudp_set_access_flags(struct vm_area_struct *vma, 158 unsigned long address, pud_t *pudp, 159 pud_t entry, int dirty); 160 #else 161 static inline int pmdp_set_access_flags(struct vm_area_struct *vma, 162 unsigned long address, pmd_t *pmdp, 163 pmd_t entry, int dirty) 164 { 165 BUILD_BUG(); 166 return 0; 167 } 168 static inline int pudp_set_access_flags(struct vm_area_struct *vma, 169 unsigned long address, pud_t *pudp, 170 pud_t entry, int dirty) 171 { 172 BUILD_BUG(); 173 return 0; 174 } 175 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 176 #endif 177 178 #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG 179 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, 180 unsigned long address, 181 pte_t *ptep) 182 { 183 pte_t pte = *ptep; 184 int r = 1; 185 if (!pte_young(pte)) 186 r = 0; 187 else 188 set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte)); 189 return r; 190 } 191 #endif 192 193 #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG 194 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 195 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 196 unsigned long address, 197 pmd_t *pmdp) 198 { 199 pmd_t pmd = *pmdp; 200 int r = 1; 201 if (!pmd_young(pmd)) 202 r = 0; 203 else 204 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); 205 return r; 206 } 207 #else 208 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 209 unsigned long address, 210 pmd_t *pmdp) 211 { 212 BUILD_BUG(); 213 return 0; 214 } 215 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 216 #endif 217 218 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 219 int ptep_clear_flush_young(struct vm_area_struct *vma, 220 unsigned long address, pte_t *ptep); 221 #endif 222 223 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH 224 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 225 extern int pmdp_clear_flush_young(struct vm_area_struct *vma, 226 unsigned long address, pmd_t *pmdp); 227 #else 228 /* 229 * Despite relevant to THP only, this API is called from generic rmap code 230 * under PageTransHuge(), hence needs a dummy implementation for !THP 231 */ 232 static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, 233 unsigned long address, pmd_t *pmdp) 234 { 235 BUILD_BUG(); 236 return 0; 237 } 238 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 239 #endif 240 241 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR 242 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, 243 unsigned long address, 244 pte_t *ptep) 245 { 246 pte_t pte = *ptep; 247 pte_clear(mm, address, ptep); 248 return pte; 249 } 250 #endif 251 252 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 253 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR 254 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, 255 unsigned long address, 256 pmd_t *pmdp) 257 { 258 pmd_t pmd = *pmdp; 259 pmd_clear(pmdp); 260 return pmd; 261 } 262 #endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */ 263 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR 264 static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, 265 unsigned long address, 266 pud_t *pudp) 267 { 268 pud_t pud = *pudp; 269 270 pud_clear(pudp); 271 return pud; 272 } 273 #endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */ 274 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 275 276 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 277 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL 278 static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, 279 unsigned long address, pmd_t *pmdp, 280 int full) 281 { 282 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 283 } 284 #endif 285 286 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL 287 static inline pud_t pudp_huge_get_and_clear_full(struct mm_struct *mm, 288 unsigned long address, pud_t *pudp, 289 int full) 290 { 291 return pudp_huge_get_and_clear(mm, address, pudp); 292 } 293 #endif 294 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 295 296 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL 297 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, 298 unsigned long address, pte_t *ptep, 299 int full) 300 { 301 pte_t pte; 302 pte = ptep_get_and_clear(mm, address, ptep); 303 return pte; 304 } 305 #endif 306 307 308 /* 309 * If two threads concurrently fault at the same page, the thread that 310 * won the race updates the PTE and its local TLB/Cache. The other thread 311 * gives up, simply does nothing, and continues; on architectures where 312 * software can update TLB, local TLB can be updated here to avoid next page 313 * fault. This function updates TLB only, do nothing with cache or others. 314 * It is the difference with function update_mmu_cache. 315 */ 316 #ifndef __HAVE_ARCH_UPDATE_MMU_TLB 317 static inline void update_mmu_tlb(struct vm_area_struct *vma, 318 unsigned long address, pte_t *ptep) 319 { 320 } 321 #define __HAVE_ARCH_UPDATE_MMU_TLB 322 #endif 323 324 /* 325 * Some architectures may be able to avoid expensive synchronization 326 * primitives when modifications are made to PTE's which are already 327 * not present, or in the process of an address space destruction. 328 */ 329 #ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL 330 static inline void pte_clear_not_present_full(struct mm_struct *mm, 331 unsigned long address, 332 pte_t *ptep, 333 int full) 334 { 335 pte_clear(mm, address, ptep); 336 } 337 #endif 338 339 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH 340 extern pte_t ptep_clear_flush(struct vm_area_struct *vma, 341 unsigned long address, 342 pte_t *ptep); 343 #endif 344 345 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH 346 extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, 347 unsigned long address, 348 pmd_t *pmdp); 349 extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, 350 unsigned long address, 351 pud_t *pudp); 352 #endif 353 354 #ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT 355 struct mm_struct; 356 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) 357 { 358 pte_t old_pte = *ptep; 359 set_pte_at(mm, address, ptep, pte_wrprotect(old_pte)); 360 } 361 #endif 362 363 /* 364 * On some architectures hardware does not set page access bit when accessing 365 * memory page, it is responsibilty of software setting this bit. It brings 366 * out extra page fault penalty to track page access bit. For optimization page 367 * access bit can be set during all page fault flow on these arches. 368 * To be differentiate with macro pte_mkyoung, this macro is used on platforms 369 * where software maintains page access bit. 370 */ 371 #ifndef pte_sw_mkyoung 372 static inline pte_t pte_sw_mkyoung(pte_t pte) 373 { 374 return pte; 375 } 376 #define pte_sw_mkyoung pte_sw_mkyoung 377 #endif 378 379 #ifndef pte_savedwrite 380 #define pte_savedwrite pte_write 381 #endif 382 383 #ifndef pte_mk_savedwrite 384 #define pte_mk_savedwrite pte_mkwrite 385 #endif 386 387 #ifndef pte_clear_savedwrite 388 #define pte_clear_savedwrite pte_wrprotect 389 #endif 390 391 #ifndef pmd_savedwrite 392 #define pmd_savedwrite pmd_write 393 #endif 394 395 #ifndef pmd_mk_savedwrite 396 #define pmd_mk_savedwrite pmd_mkwrite 397 #endif 398 399 #ifndef pmd_clear_savedwrite 400 #define pmd_clear_savedwrite pmd_wrprotect 401 #endif 402 403 #ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT 404 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 405 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 406 unsigned long address, pmd_t *pmdp) 407 { 408 pmd_t old_pmd = *pmdp; 409 set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd)); 410 } 411 #else 412 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 413 unsigned long address, pmd_t *pmdp) 414 { 415 BUILD_BUG(); 416 } 417 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 418 #endif 419 #ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT 420 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 421 static inline void pudp_set_wrprotect(struct mm_struct *mm, 422 unsigned long address, pud_t *pudp) 423 { 424 pud_t old_pud = *pudp; 425 426 set_pud_at(mm, address, pudp, pud_wrprotect(old_pud)); 427 } 428 #else 429 static inline void pudp_set_wrprotect(struct mm_struct *mm, 430 unsigned long address, pud_t *pudp) 431 { 432 BUILD_BUG(); 433 } 434 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 435 #endif 436 437 #ifndef pmdp_collapse_flush 438 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 439 extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 440 unsigned long address, pmd_t *pmdp); 441 #else 442 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 443 unsigned long address, 444 pmd_t *pmdp) 445 { 446 BUILD_BUG(); 447 return *pmdp; 448 } 449 #define pmdp_collapse_flush pmdp_collapse_flush 450 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 451 #endif 452 453 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT 454 extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 455 pgtable_t pgtable); 456 #endif 457 458 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW 459 extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); 460 #endif 461 462 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 463 /* 464 * This is an implementation of pmdp_establish() that is only suitable for an 465 * architecture that doesn't have hardware dirty/accessed bits. In this case we 466 * can't race with CPU which sets these bits and non-atomic aproach is fine. 467 */ 468 static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma, 469 unsigned long address, pmd_t *pmdp, pmd_t pmd) 470 { 471 pmd_t old_pmd = *pmdp; 472 set_pmd_at(vma->vm_mm, address, pmdp, pmd); 473 return old_pmd; 474 } 475 #endif 476 477 #ifndef __HAVE_ARCH_PMDP_INVALIDATE 478 extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, 479 pmd_t *pmdp); 480 #endif 481 482 #ifndef __HAVE_ARCH_PTE_SAME 483 static inline int pte_same(pte_t pte_a, pte_t pte_b) 484 { 485 return pte_val(pte_a) == pte_val(pte_b); 486 } 487 #endif 488 489 #ifndef __HAVE_ARCH_PTE_UNUSED 490 /* 491 * Some architectures provide facilities to virtualization guests 492 * so that they can flag allocated pages as unused. This allows the 493 * host to transparently reclaim unused pages. This function returns 494 * whether the pte's page is unused. 495 */ 496 static inline int pte_unused(pte_t pte) 497 { 498 return 0; 499 } 500 #endif 501 502 #ifndef pte_access_permitted 503 #define pte_access_permitted(pte, write) \ 504 (pte_present(pte) && (!(write) || pte_write(pte))) 505 #endif 506 507 #ifndef pmd_access_permitted 508 #define pmd_access_permitted(pmd, write) \ 509 (pmd_present(pmd) && (!(write) || pmd_write(pmd))) 510 #endif 511 512 #ifndef pud_access_permitted 513 #define pud_access_permitted(pud, write) \ 514 (pud_present(pud) && (!(write) || pud_write(pud))) 515 #endif 516 517 #ifndef p4d_access_permitted 518 #define p4d_access_permitted(p4d, write) \ 519 (p4d_present(p4d) && (!(write) || p4d_write(p4d))) 520 #endif 521 522 #ifndef pgd_access_permitted 523 #define pgd_access_permitted(pgd, write) \ 524 (pgd_present(pgd) && (!(write) || pgd_write(pgd))) 525 #endif 526 527 #ifndef __HAVE_ARCH_PMD_SAME 528 static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) 529 { 530 return pmd_val(pmd_a) == pmd_val(pmd_b); 531 } 532 533 static inline int pud_same(pud_t pud_a, pud_t pud_b) 534 { 535 return pud_val(pud_a) == pud_val(pud_b); 536 } 537 #endif 538 539 #ifndef __HAVE_ARCH_P4D_SAME 540 static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b) 541 { 542 return p4d_val(p4d_a) == p4d_val(p4d_b); 543 } 544 #endif 545 546 #ifndef __HAVE_ARCH_PGD_SAME 547 static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b) 548 { 549 return pgd_val(pgd_a) == pgd_val(pgd_b); 550 } 551 #endif 552 553 /* 554 * Use set_p*_safe(), and elide TLB flushing, when confident that *no* 555 * TLB flush will be required as a result of the "set". For example, use 556 * in scenarios where it is known ahead of time that the routine is 557 * setting non-present entries, or re-setting an existing entry to the 558 * same value. Otherwise, use the typical "set" helpers and flush the 559 * TLB. 560 */ 561 #define set_pte_safe(ptep, pte) \ 562 ({ \ 563 WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \ 564 set_pte(ptep, pte); \ 565 }) 566 567 #define set_pmd_safe(pmdp, pmd) \ 568 ({ \ 569 WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \ 570 set_pmd(pmdp, pmd); \ 571 }) 572 573 #define set_pud_safe(pudp, pud) \ 574 ({ \ 575 WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \ 576 set_pud(pudp, pud); \ 577 }) 578 579 #define set_p4d_safe(p4dp, p4d) \ 580 ({ \ 581 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \ 582 set_p4d(p4dp, p4d); \ 583 }) 584 585 #define set_pgd_safe(pgdp, pgd) \ 586 ({ \ 587 WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \ 588 set_pgd(pgdp, pgd); \ 589 }) 590 591 #ifndef __HAVE_ARCH_DO_SWAP_PAGE 592 /* 593 * Some architectures support metadata associated with a page. When a 594 * page is being swapped out, this metadata must be saved so it can be 595 * restored when the page is swapped back in. SPARC M7 and newer 596 * processors support an ADI (Application Data Integrity) tag for the 597 * page as metadata for the page. arch_do_swap_page() can restore this 598 * metadata when a page is swapped back in. 599 */ 600 static inline void arch_do_swap_page(struct mm_struct *mm, 601 struct vm_area_struct *vma, 602 unsigned long addr, 603 pte_t pte, pte_t oldpte) 604 { 605 606 } 607 #endif 608 609 #ifndef __HAVE_ARCH_UNMAP_ONE 610 /* 611 * Some architectures support metadata associated with a page. When a 612 * page is being swapped out, this metadata must be saved so it can be 613 * restored when the page is swapped back in. SPARC M7 and newer 614 * processors support an ADI (Application Data Integrity) tag for the 615 * page as metadata for the page. arch_unmap_one() can save this 616 * metadata on a swap-out of a page. 617 */ 618 static inline int arch_unmap_one(struct mm_struct *mm, 619 struct vm_area_struct *vma, 620 unsigned long addr, 621 pte_t orig_pte) 622 { 623 return 0; 624 } 625 #endif 626 627 #ifndef __HAVE_ARCH_PGD_OFFSET_GATE 628 #define pgd_offset_gate(mm, addr) pgd_offset(mm, addr) 629 #endif 630 631 #ifndef __HAVE_ARCH_MOVE_PTE 632 #define move_pte(pte, prot, old_addr, new_addr) (pte) 633 #endif 634 635 #ifndef pte_accessible 636 # define pte_accessible(mm, pte) ((void)(pte), 1) 637 #endif 638 639 #ifndef flush_tlb_fix_spurious_fault 640 #define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address) 641 #endif 642 643 #ifndef pgprot_nx 644 #define pgprot_nx(prot) (prot) 645 #endif 646 647 #ifndef pgprot_noncached 648 #define pgprot_noncached(prot) (prot) 649 #endif 650 651 #ifndef pgprot_writecombine 652 #define pgprot_writecombine pgprot_noncached 653 #endif 654 655 #ifndef pgprot_writethrough 656 #define pgprot_writethrough pgprot_noncached 657 #endif 658 659 #ifndef pgprot_device 660 #define pgprot_device pgprot_noncached 661 #endif 662 663 #ifndef pgprot_modify 664 #define pgprot_modify pgprot_modify 665 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 666 { 667 if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot))) 668 newprot = pgprot_noncached(newprot); 669 if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot))) 670 newprot = pgprot_writecombine(newprot); 671 if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot))) 672 newprot = pgprot_device(newprot); 673 return newprot; 674 } 675 #endif 676 677 /* 678 * When walking page tables, get the address of the next boundary, 679 * or the end address of the range if that comes earlier. Although no 680 * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout. 681 */ 682 683 #define pgd_addr_end(addr, end) \ 684 ({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \ 685 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 686 }) 687 688 #ifndef p4d_addr_end 689 #define p4d_addr_end(addr, end) \ 690 ({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \ 691 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 692 }) 693 #endif 694 695 #ifndef pud_addr_end 696 #define pud_addr_end(addr, end) \ 697 ({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \ 698 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 699 }) 700 #endif 701 702 #ifndef pmd_addr_end 703 #define pmd_addr_end(addr, end) \ 704 ({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \ 705 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 706 }) 707 #endif 708 709 /* 710 * When walking page tables, we usually want to skip any p?d_none entries; 711 * and any p?d_bad entries - reporting the error before resetting to none. 712 * Do the tests inline, but report and clear the bad entry in mm/memory.c. 713 */ 714 void pgd_clear_bad(pgd_t *); 715 716 #ifndef __PAGETABLE_P4D_FOLDED 717 void p4d_clear_bad(p4d_t *); 718 #else 719 #define p4d_clear_bad(p4d) do { } while (0) 720 #endif 721 722 #ifndef __PAGETABLE_PUD_FOLDED 723 void pud_clear_bad(pud_t *); 724 #else 725 #define pud_clear_bad(p4d) do { } while (0) 726 #endif 727 728 void pmd_clear_bad(pmd_t *); 729 730 static inline int pgd_none_or_clear_bad(pgd_t *pgd) 731 { 732 if (pgd_none(*pgd)) 733 return 1; 734 if (unlikely(pgd_bad(*pgd))) { 735 pgd_clear_bad(pgd); 736 return 1; 737 } 738 return 0; 739 } 740 741 static inline int p4d_none_or_clear_bad(p4d_t *p4d) 742 { 743 if (p4d_none(*p4d)) 744 return 1; 745 if (unlikely(p4d_bad(*p4d))) { 746 p4d_clear_bad(p4d); 747 return 1; 748 } 749 return 0; 750 } 751 752 static inline int pud_none_or_clear_bad(pud_t *pud) 753 { 754 if (pud_none(*pud)) 755 return 1; 756 if (unlikely(pud_bad(*pud))) { 757 pud_clear_bad(pud); 758 return 1; 759 } 760 return 0; 761 } 762 763 static inline int pmd_none_or_clear_bad(pmd_t *pmd) 764 { 765 if (pmd_none(*pmd)) 766 return 1; 767 if (unlikely(pmd_bad(*pmd))) { 768 pmd_clear_bad(pmd); 769 return 1; 770 } 771 return 0; 772 } 773 774 static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma, 775 unsigned long addr, 776 pte_t *ptep) 777 { 778 /* 779 * Get the current pte state, but zero it out to make it 780 * non-present, preventing the hardware from asynchronously 781 * updating it. 782 */ 783 return ptep_get_and_clear(vma->vm_mm, addr, ptep); 784 } 785 786 static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma, 787 unsigned long addr, 788 pte_t *ptep, pte_t pte) 789 { 790 /* 791 * The pte is non-present, so there's no hardware state to 792 * preserve. 793 */ 794 set_pte_at(vma->vm_mm, addr, ptep, pte); 795 } 796 797 #ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 798 /* 799 * Start a pte protection read-modify-write transaction, which 800 * protects against asynchronous hardware modifications to the pte. 801 * The intention is not to prevent the hardware from making pte 802 * updates, but to prevent any updates it may make from being lost. 803 * 804 * This does not protect against other software modifications of the 805 * pte; the appropriate pte lock must be held over the transation. 806 * 807 * Note that this interface is intended to be batchable, meaning that 808 * ptep_modify_prot_commit may not actually update the pte, but merely 809 * queue the update to be done at some later time. The update must be 810 * actually committed before the pte lock is released, however. 811 */ 812 static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, 813 unsigned long addr, 814 pte_t *ptep) 815 { 816 return __ptep_modify_prot_start(vma, addr, ptep); 817 } 818 819 /* 820 * Commit an update to a pte, leaving any hardware-controlled bits in 821 * the PTE unmodified. 822 */ 823 static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, 824 unsigned long addr, 825 pte_t *ptep, pte_t old_pte, pte_t pte) 826 { 827 __ptep_modify_prot_commit(vma, addr, ptep, pte); 828 } 829 #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */ 830 #endif /* CONFIG_MMU */ 831 832 /* 833 * No-op macros that just return the current protection value. Defined here 834 * because these macros can be used used even if CONFIG_MMU is not defined. 835 */ 836 #ifndef pgprot_encrypted 837 #define pgprot_encrypted(prot) (prot) 838 #endif 839 840 #ifndef pgprot_decrypted 841 #define pgprot_decrypted(prot) (prot) 842 #endif 843 844 /* 845 * A facility to provide lazy MMU batching. This allows PTE updates and 846 * page invalidations to be delayed until a call to leave lazy MMU mode 847 * is issued. Some architectures may benefit from doing this, and it is 848 * beneficial for both shadow and direct mode hypervisors, which may batch 849 * the PTE updates which happen during this window. Note that using this 850 * interface requires that read hazards be removed from the code. A read 851 * hazard could result in the direct mode hypervisor case, since the actual 852 * write to the page tables may not yet have taken place, so reads though 853 * a raw PTE pointer after it has been modified are not guaranteed to be 854 * up to date. This mode can only be entered and left under the protection of 855 * the page table locks for all page tables which may be modified. In the UP 856 * case, this is required so that preemption is disabled, and in the SMP case, 857 * it must synchronize the delayed page table writes properly on other CPUs. 858 */ 859 #ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE 860 #define arch_enter_lazy_mmu_mode() do {} while (0) 861 #define arch_leave_lazy_mmu_mode() do {} while (0) 862 #define arch_flush_lazy_mmu_mode() do {} while (0) 863 #endif 864 865 /* 866 * A facility to provide batching of the reload of page tables and 867 * other process state with the actual context switch code for 868 * paravirtualized guests. By convention, only one of the batched 869 * update (lazy) modes (CPU, MMU) should be active at any given time, 870 * entry should never be nested, and entry and exits should always be 871 * paired. This is for sanity of maintaining and reasoning about the 872 * kernel code. In this case, the exit (end of the context switch) is 873 * in architecture-specific code, and so doesn't need a generic 874 * definition. 875 */ 876 #ifndef __HAVE_ARCH_START_CONTEXT_SWITCH 877 #define arch_start_context_switch(prev) do {} while (0) 878 #endif 879 880 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY 881 #ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION 882 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) 883 { 884 return pmd; 885 } 886 887 static inline int pmd_swp_soft_dirty(pmd_t pmd) 888 { 889 return 0; 890 } 891 892 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) 893 { 894 return pmd; 895 } 896 #endif 897 #else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */ 898 static inline int pte_soft_dirty(pte_t pte) 899 { 900 return 0; 901 } 902 903 static inline int pmd_soft_dirty(pmd_t pmd) 904 { 905 return 0; 906 } 907 908 static inline pte_t pte_mksoft_dirty(pte_t pte) 909 { 910 return pte; 911 } 912 913 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) 914 { 915 return pmd; 916 } 917 918 static inline pte_t pte_clear_soft_dirty(pte_t pte) 919 { 920 return pte; 921 } 922 923 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) 924 { 925 return pmd; 926 } 927 928 static inline pte_t pte_swp_mksoft_dirty(pte_t pte) 929 { 930 return pte; 931 } 932 933 static inline int pte_swp_soft_dirty(pte_t pte) 934 { 935 return 0; 936 } 937 938 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) 939 { 940 return pte; 941 } 942 943 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) 944 { 945 return pmd; 946 } 947 948 static inline int pmd_swp_soft_dirty(pmd_t pmd) 949 { 950 return 0; 951 } 952 953 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) 954 { 955 return pmd; 956 } 957 #endif 958 959 #ifndef __HAVE_PFNMAP_TRACKING 960 /* 961 * Interfaces that can be used by architecture code to keep track of 962 * memory type of pfn mappings specified by the remap_pfn_range, 963 * vmf_insert_pfn. 964 */ 965 966 /* 967 * track_pfn_remap is called when a _new_ pfn mapping is being established 968 * by remap_pfn_range() for physical range indicated by pfn and size. 969 */ 970 static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, 971 unsigned long pfn, unsigned long addr, 972 unsigned long size) 973 { 974 return 0; 975 } 976 977 /* 978 * track_pfn_insert is called when a _new_ single pfn is established 979 * by vmf_insert_pfn(). 980 */ 981 static inline void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, 982 pfn_t pfn) 983 { 984 } 985 986 /* 987 * track_pfn_copy is called when vma that is covering the pfnmap gets 988 * copied through copy_page_range(). 989 */ 990 static inline int track_pfn_copy(struct vm_area_struct *vma) 991 { 992 return 0; 993 } 994 995 /* 996 * untrack_pfn is called while unmapping a pfnmap for a region. 997 * untrack can be called for a specific region indicated by pfn and size or 998 * can be for the entire vma (in which case pfn, size are zero). 999 */ 1000 static inline void untrack_pfn(struct vm_area_struct *vma, 1001 unsigned long pfn, unsigned long size) 1002 { 1003 } 1004 1005 /* 1006 * untrack_pfn_moved is called while mremapping a pfnmap for a new region. 1007 */ 1008 static inline void untrack_pfn_moved(struct vm_area_struct *vma) 1009 { 1010 } 1011 #else 1012 extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, 1013 unsigned long pfn, unsigned long addr, 1014 unsigned long size); 1015 extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, 1016 pfn_t pfn); 1017 extern int track_pfn_copy(struct vm_area_struct *vma); 1018 extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn, 1019 unsigned long size); 1020 extern void untrack_pfn_moved(struct vm_area_struct *vma); 1021 #endif 1022 1023 #ifdef __HAVE_COLOR_ZERO_PAGE 1024 static inline int is_zero_pfn(unsigned long pfn) 1025 { 1026 extern unsigned long zero_pfn; 1027 unsigned long offset_from_zero_pfn = pfn - zero_pfn; 1028 return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT); 1029 } 1030 1031 #define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) 1032 1033 #else 1034 static inline int is_zero_pfn(unsigned long pfn) 1035 { 1036 extern unsigned long zero_pfn; 1037 return pfn == zero_pfn; 1038 } 1039 1040 static inline unsigned long my_zero_pfn(unsigned long addr) 1041 { 1042 extern unsigned long zero_pfn; 1043 return zero_pfn; 1044 } 1045 #endif 1046 1047 #ifdef CONFIG_MMU 1048 1049 #ifndef CONFIG_TRANSPARENT_HUGEPAGE 1050 static inline int pmd_trans_huge(pmd_t pmd) 1051 { 1052 return 0; 1053 } 1054 #ifndef pmd_write 1055 static inline int pmd_write(pmd_t pmd) 1056 { 1057 BUG(); 1058 return 0; 1059 } 1060 #endif /* pmd_write */ 1061 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1062 1063 #ifndef pud_write 1064 static inline int pud_write(pud_t pud) 1065 { 1066 BUG(); 1067 return 0; 1068 } 1069 #endif /* pud_write */ 1070 1071 #if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE) 1072 static inline int pmd_devmap(pmd_t pmd) 1073 { 1074 return 0; 1075 } 1076 static inline int pud_devmap(pud_t pud) 1077 { 1078 return 0; 1079 } 1080 static inline int pgd_devmap(pgd_t pgd) 1081 { 1082 return 0; 1083 } 1084 #endif 1085 1086 #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ 1087 (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1088 !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) 1089 static inline int pud_trans_huge(pud_t pud) 1090 { 1091 return 0; 1092 } 1093 #endif 1094 1095 /* See pmd_none_or_trans_huge_or_clear_bad for discussion. */ 1096 static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud) 1097 { 1098 pud_t pudval = READ_ONCE(*pud); 1099 1100 if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval)) 1101 return 1; 1102 if (unlikely(pud_bad(pudval))) { 1103 pud_clear_bad(pud); 1104 return 1; 1105 } 1106 return 0; 1107 } 1108 1109 /* See pmd_trans_unstable for discussion. */ 1110 static inline int pud_trans_unstable(pud_t *pud) 1111 { 1112 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1113 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 1114 return pud_none_or_trans_huge_or_dev_or_clear_bad(pud); 1115 #else 1116 return 0; 1117 #endif 1118 } 1119 1120 #ifndef pmd_read_atomic 1121 static inline pmd_t pmd_read_atomic(pmd_t *pmdp) 1122 { 1123 /* 1124 * Depend on compiler for an atomic pmd read. NOTE: this is 1125 * only going to work, if the pmdval_t isn't larger than 1126 * an unsigned long. 1127 */ 1128 return *pmdp; 1129 } 1130 #endif 1131 1132 #ifndef arch_needs_pgtable_deposit 1133 #define arch_needs_pgtable_deposit() (false) 1134 #endif 1135 /* 1136 * This function is meant to be used by sites walking pagetables with 1137 * the mmap_lock held in read mode to protect against MADV_DONTNEED and 1138 * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd 1139 * into a null pmd and the transhuge page fault can convert a null pmd 1140 * into an hugepmd or into a regular pmd (if the hugepage allocation 1141 * fails). While holding the mmap_lock in read mode the pmd becomes 1142 * stable and stops changing under us only if it's not null and not a 1143 * transhuge pmd. When those races occurs and this function makes a 1144 * difference vs the standard pmd_none_or_clear_bad, the result is 1145 * undefined so behaving like if the pmd was none is safe (because it 1146 * can return none anyway). The compiler level barrier() is critically 1147 * important to compute the two checks atomically on the same pmdval. 1148 * 1149 * For 32bit kernels with a 64bit large pmd_t this automatically takes 1150 * care of reading the pmd atomically to avoid SMP race conditions 1151 * against pmd_populate() when the mmap_lock is hold for reading by the 1152 * caller (a special atomic read not done by "gcc" as in the generic 1153 * version above, is also needed when THP is disabled because the page 1154 * fault can populate the pmd from under us). 1155 */ 1156 static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd) 1157 { 1158 pmd_t pmdval = pmd_read_atomic(pmd); 1159 /* 1160 * The barrier will stabilize the pmdval in a register or on 1161 * the stack so that it will stop changing under the code. 1162 * 1163 * When CONFIG_TRANSPARENT_HUGEPAGE=y on x86 32bit PAE, 1164 * pmd_read_atomic is allowed to return a not atomic pmdval 1165 * (for example pointing to an hugepage that has never been 1166 * mapped in the pmd). The below checks will only care about 1167 * the low part of the pmd with 32bit PAE x86 anyway, with the 1168 * exception of pmd_none(). So the important thing is that if 1169 * the low part of the pmd is found null, the high part will 1170 * be also null or the pmd_none() check below would be 1171 * confused. 1172 */ 1173 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1174 barrier(); 1175 #endif 1176 /* 1177 * !pmd_present() checks for pmd migration entries 1178 * 1179 * The complete check uses is_pmd_migration_entry() in linux/swapops.h 1180 * But using that requires moving current function and pmd_trans_unstable() 1181 * to linux/swapops.h to resovle dependency, which is too much code move. 1182 * 1183 * !pmd_present() is equivalent to is_pmd_migration_entry() currently, 1184 * because !pmd_present() pages can only be under migration not swapped 1185 * out. 1186 * 1187 * pmd_none() is preseved for future condition checks on pmd migration 1188 * entries and not confusing with this function name, although it is 1189 * redundant with !pmd_present(). 1190 */ 1191 if (pmd_none(pmdval) || pmd_trans_huge(pmdval) || 1192 (IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION) && !pmd_present(pmdval))) 1193 return 1; 1194 if (unlikely(pmd_bad(pmdval))) { 1195 pmd_clear_bad(pmd); 1196 return 1; 1197 } 1198 return 0; 1199 } 1200 1201 /* 1202 * This is a noop if Transparent Hugepage Support is not built into 1203 * the kernel. Otherwise it is equivalent to 1204 * pmd_none_or_trans_huge_or_clear_bad(), and shall only be called in 1205 * places that already verified the pmd is not none and they want to 1206 * walk ptes while holding the mmap sem in read mode (write mode don't 1207 * need this). If THP is not enabled, the pmd can't go away under the 1208 * code even if MADV_DONTNEED runs, but if THP is enabled we need to 1209 * run a pmd_trans_unstable before walking the ptes after 1210 * split_huge_pmd returns (because it may have run when the pmd become 1211 * null, but then a page fault can map in a THP and not a regular page). 1212 */ 1213 static inline int pmd_trans_unstable(pmd_t *pmd) 1214 { 1215 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1216 return pmd_none_or_trans_huge_or_clear_bad(pmd); 1217 #else 1218 return 0; 1219 #endif 1220 } 1221 1222 #ifndef CONFIG_NUMA_BALANCING 1223 /* 1224 * Technically a PTE can be PROTNONE even when not doing NUMA balancing but 1225 * the only case the kernel cares is for NUMA balancing and is only ever set 1226 * when the VMA is accessible. For PROT_NONE VMAs, the PTEs are not marked 1227 * _PAGE_PROTNONE so by by default, implement the helper as "always no". It 1228 * is the responsibility of the caller to distinguish between PROT_NONE 1229 * protections and NUMA hinting fault protections. 1230 */ 1231 static inline int pte_protnone(pte_t pte) 1232 { 1233 return 0; 1234 } 1235 1236 static inline int pmd_protnone(pmd_t pmd) 1237 { 1238 return 0; 1239 } 1240 #endif /* CONFIG_NUMA_BALANCING */ 1241 1242 #endif /* CONFIG_MMU */ 1243 1244 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 1245 1246 #ifndef __PAGETABLE_P4D_FOLDED 1247 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot); 1248 int p4d_clear_huge(p4d_t *p4d); 1249 #else 1250 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 1251 { 1252 return 0; 1253 } 1254 static inline int p4d_clear_huge(p4d_t *p4d) 1255 { 1256 return 0; 1257 } 1258 #endif /* !__PAGETABLE_P4D_FOLDED */ 1259 1260 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot); 1261 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot); 1262 int pud_clear_huge(pud_t *pud); 1263 int pmd_clear_huge(pmd_t *pmd); 1264 int p4d_free_pud_page(p4d_t *p4d, unsigned long addr); 1265 int pud_free_pmd_page(pud_t *pud, unsigned long addr); 1266 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr); 1267 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 1268 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 1269 { 1270 return 0; 1271 } 1272 static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) 1273 { 1274 return 0; 1275 } 1276 static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) 1277 { 1278 return 0; 1279 } 1280 static inline int p4d_clear_huge(p4d_t *p4d) 1281 { 1282 return 0; 1283 } 1284 static inline int pud_clear_huge(pud_t *pud) 1285 { 1286 return 0; 1287 } 1288 static inline int pmd_clear_huge(pmd_t *pmd) 1289 { 1290 return 0; 1291 } 1292 static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) 1293 { 1294 return 0; 1295 } 1296 static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr) 1297 { 1298 return 0; 1299 } 1300 static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) 1301 { 1302 return 0; 1303 } 1304 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 1305 1306 #ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE 1307 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1308 /* 1309 * ARCHes with special requirements for evicting THP backing TLB entries can 1310 * implement this. Otherwise also, it can help optimize normal TLB flush in 1311 * THP regime. stock flush_tlb_range() typically has optimization to nuke the 1312 * entire TLB TLB if flush span is greater than a threshold, which will 1313 * likely be true for a single huge page. Thus a single thp flush will 1314 * invalidate the entire TLB which is not desitable. 1315 * e.g. see arch/arc: flush_pmd_tlb_range 1316 */ 1317 #define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 1318 #define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 1319 #else 1320 #define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG() 1321 #define flush_pud_tlb_range(vma, addr, end) BUILD_BUG() 1322 #endif 1323 #endif 1324 1325 struct file; 1326 int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, 1327 unsigned long size, pgprot_t *vma_prot); 1328 1329 #ifndef CONFIG_X86_ESPFIX64 1330 static inline void init_espfix_bsp(void) { } 1331 #endif 1332 1333 extern void __init pgtable_cache_init(void); 1334 1335 #ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED 1336 static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot) 1337 { 1338 return true; 1339 } 1340 1341 static inline bool arch_has_pfn_modify_check(void) 1342 { 1343 return false; 1344 } 1345 #endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */ 1346 1347 /* 1348 * Architecture PAGE_KERNEL_* fallbacks 1349 * 1350 * Some architectures don't define certain PAGE_KERNEL_* flags. This is either 1351 * because they really don't support them, or the port needs to be updated to 1352 * reflect the required functionality. Below are a set of relatively safe 1353 * fallbacks, as best effort, which we can count on in lieu of the architectures 1354 * not defining them on their own yet. 1355 */ 1356 1357 #ifndef PAGE_KERNEL_RO 1358 # define PAGE_KERNEL_RO PAGE_KERNEL 1359 #endif 1360 1361 #ifndef PAGE_KERNEL_EXEC 1362 # define PAGE_KERNEL_EXEC PAGE_KERNEL 1363 #endif 1364 1365 /* 1366 * Page Table Modification bits for pgtbl_mod_mask. 1367 * 1368 * These are used by the p?d_alloc_track*() set of functions an in the generic 1369 * vmalloc/ioremap code to track at which page-table levels entries have been 1370 * modified. Based on that the code can better decide when vmalloc and ioremap 1371 * mapping changes need to be synchronized to other page-tables in the system. 1372 */ 1373 #define __PGTBL_PGD_MODIFIED 0 1374 #define __PGTBL_P4D_MODIFIED 1 1375 #define __PGTBL_PUD_MODIFIED 2 1376 #define __PGTBL_PMD_MODIFIED 3 1377 #define __PGTBL_PTE_MODIFIED 4 1378 1379 #define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED) 1380 #define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED) 1381 #define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED) 1382 #define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED) 1383 #define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED) 1384 1385 /* Page-Table Modification Mask */ 1386 typedef unsigned int pgtbl_mod_mask; 1387 1388 #endif /* !__ASSEMBLY__ */ 1389 1390 #ifndef io_remap_pfn_range 1391 #define io_remap_pfn_range remap_pfn_range 1392 #endif 1393 1394 #ifndef has_transparent_hugepage 1395 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1396 #define has_transparent_hugepage() 1 1397 #else 1398 #define has_transparent_hugepage() 0 1399 #endif 1400 #endif 1401 1402 /* 1403 * On some architectures it depends on the mm if the p4d/pud or pmd 1404 * layer of the page table hierarchy is folded or not. 1405 */ 1406 #ifndef mm_p4d_folded 1407 #define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED) 1408 #endif 1409 1410 #ifndef mm_pud_folded 1411 #define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED) 1412 #endif 1413 1414 #ifndef mm_pmd_folded 1415 #define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED) 1416 #endif 1417 1418 /* 1419 * p?d_leaf() - true if this entry is a final mapping to a physical address. 1420 * This differs from p?d_huge() by the fact that they are always available (if 1421 * the architecture supports large pages at the appropriate level) even 1422 * if CONFIG_HUGETLB_PAGE is not defined. 1423 * Only meaningful when called on a valid entry. 1424 */ 1425 #ifndef pgd_leaf 1426 #define pgd_leaf(x) 0 1427 #endif 1428 #ifndef p4d_leaf 1429 #define p4d_leaf(x) 0 1430 #endif 1431 #ifndef pud_leaf 1432 #define pud_leaf(x) 0 1433 #endif 1434 #ifndef pmd_leaf 1435 #define pmd_leaf(x) 0 1436 #endif 1437 1438 #endif /* _LINUX_PGTABLE_H */ 1439