1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_PGTABLE_H 3 #define _LINUX_PGTABLE_H 4 5 #include <linux/pfn.h> 6 #include <asm/pgtable.h> 7 8 #ifndef __ASSEMBLY__ 9 #ifdef CONFIG_MMU 10 11 #include <linux/mm_types.h> 12 #include <linux/bug.h> 13 #include <linux/errno.h> 14 #include <asm-generic/pgtable_uffd.h> 15 16 #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ 17 defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS 18 #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED 19 #endif 20 21 /* 22 * On almost all architectures and configurations, 0 can be used as the 23 * upper ceiling to free_pgtables(): on many architectures it has the same 24 * effect as using TASK_SIZE. However, there is one configuration which 25 * must impose a more careful limit, to avoid freeing kernel pgtables. 26 */ 27 #ifndef USER_PGTABLES_CEILING 28 #define USER_PGTABLES_CEILING 0UL 29 #endif 30 31 /* 32 * A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD] 33 * 34 * The pXx_index() functions return the index of the entry in the page 35 * table page which would control the given virtual address 36 * 37 * As these functions may be used by the same code for different levels of 38 * the page table folding, they are always available, regardless of 39 * CONFIG_PGTABLE_LEVELS value. For the folded levels they simply return 0 40 * because in such cases PTRS_PER_PxD equals 1. 41 */ 42 43 static inline unsigned long pte_index(unsigned long address) 44 { 45 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); 46 } 47 48 #ifndef pmd_index 49 static inline unsigned long pmd_index(unsigned long address) 50 { 51 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); 52 } 53 #define pmd_index pmd_index 54 #endif 55 56 #ifndef pud_index 57 static inline unsigned long pud_index(unsigned long address) 58 { 59 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); 60 } 61 #define pud_index pud_index 62 #endif 63 64 #ifndef pgd_index 65 /* Must be a compile-time constant, so implement it as a macro */ 66 #define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) 67 #endif 68 69 #ifndef pte_offset_kernel 70 static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) 71 { 72 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); 73 } 74 #define pte_offset_kernel pte_offset_kernel 75 #endif 76 77 #if defined(CONFIG_HIGHPTE) 78 #define pte_offset_map(dir, address) \ 79 ((pte_t *)kmap_atomic(pmd_page(*(dir))) + \ 80 pte_index((address))) 81 #define pte_unmap(pte) kunmap_atomic((pte)) 82 #else 83 #define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) 84 #define pte_unmap(pte) ((void)(pte)) /* NOP */ 85 #endif 86 87 /* Find an entry in the second-level page table.. */ 88 #ifndef pmd_offset 89 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) 90 { 91 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); 92 } 93 #define pmd_offset pmd_offset 94 #endif 95 96 #ifndef pud_offset 97 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) 98 { 99 return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address); 100 } 101 #define pud_offset pud_offset 102 #endif 103 104 static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) 105 { 106 return (pgd + pgd_index(address)); 107 }; 108 109 /* 110 * a shortcut to get a pgd_t in a given mm 111 */ 112 #ifndef pgd_offset 113 #define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address)) 114 #endif 115 116 /* 117 * a shortcut which implies the use of the kernel's pgd, instead 118 * of a process's 119 */ 120 #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) 121 122 /* 123 * In many cases it is known that a virtual address is mapped at PMD or PTE 124 * level, so instead of traversing all the page table levels, we can get a 125 * pointer to the PMD entry in user or kernel page table or translate a virtual 126 * address to the pointer in the PTE in the kernel page tables with simple 127 * helpers. 128 */ 129 static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va) 130 { 131 return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va); 132 } 133 134 static inline pmd_t *pmd_off_k(unsigned long va) 135 { 136 return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va); 137 } 138 139 static inline pte_t *virt_to_kpte(unsigned long vaddr) 140 { 141 pmd_t *pmd = pmd_off_k(vaddr); 142 143 return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr); 144 } 145 146 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 147 extern int ptep_set_access_flags(struct vm_area_struct *vma, 148 unsigned long address, pte_t *ptep, 149 pte_t entry, int dirty); 150 #endif 151 152 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS 153 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 154 extern int pmdp_set_access_flags(struct vm_area_struct *vma, 155 unsigned long address, pmd_t *pmdp, 156 pmd_t entry, int dirty); 157 extern int pudp_set_access_flags(struct vm_area_struct *vma, 158 unsigned long address, pud_t *pudp, 159 pud_t entry, int dirty); 160 #else 161 static inline int pmdp_set_access_flags(struct vm_area_struct *vma, 162 unsigned long address, pmd_t *pmdp, 163 pmd_t entry, int dirty) 164 { 165 BUILD_BUG(); 166 return 0; 167 } 168 static inline int pudp_set_access_flags(struct vm_area_struct *vma, 169 unsigned long address, pud_t *pudp, 170 pud_t entry, int dirty) 171 { 172 BUILD_BUG(); 173 return 0; 174 } 175 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 176 #endif 177 178 #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG 179 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, 180 unsigned long address, 181 pte_t *ptep) 182 { 183 pte_t pte = *ptep; 184 int r = 1; 185 if (!pte_young(pte)) 186 r = 0; 187 else 188 set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte)); 189 return r; 190 } 191 #endif 192 193 #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG 194 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 195 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 196 unsigned long address, 197 pmd_t *pmdp) 198 { 199 pmd_t pmd = *pmdp; 200 int r = 1; 201 if (!pmd_young(pmd)) 202 r = 0; 203 else 204 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); 205 return r; 206 } 207 #else 208 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 209 unsigned long address, 210 pmd_t *pmdp) 211 { 212 BUILD_BUG(); 213 return 0; 214 } 215 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 216 #endif 217 218 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 219 int ptep_clear_flush_young(struct vm_area_struct *vma, 220 unsigned long address, pte_t *ptep); 221 #endif 222 223 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH 224 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 225 extern int pmdp_clear_flush_young(struct vm_area_struct *vma, 226 unsigned long address, pmd_t *pmdp); 227 #else 228 /* 229 * Despite relevant to THP only, this API is called from generic rmap code 230 * under PageTransHuge(), hence needs a dummy implementation for !THP 231 */ 232 static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, 233 unsigned long address, pmd_t *pmdp) 234 { 235 BUILD_BUG(); 236 return 0; 237 } 238 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 239 #endif 240 241 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR 242 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, 243 unsigned long address, 244 pte_t *ptep) 245 { 246 pte_t pte = *ptep; 247 pte_clear(mm, address, ptep); 248 return pte; 249 } 250 #endif 251 252 #ifndef __HAVE_ARCH_PTEP_GET 253 static inline pte_t ptep_get(pte_t *ptep) 254 { 255 return READ_ONCE(*ptep); 256 } 257 #endif 258 259 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 260 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR 261 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, 262 unsigned long address, 263 pmd_t *pmdp) 264 { 265 pmd_t pmd = *pmdp; 266 pmd_clear(pmdp); 267 return pmd; 268 } 269 #endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */ 270 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR 271 static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, 272 unsigned long address, 273 pud_t *pudp) 274 { 275 pud_t pud = *pudp; 276 277 pud_clear(pudp); 278 return pud; 279 } 280 #endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */ 281 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 282 283 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 284 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL 285 static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, 286 unsigned long address, pmd_t *pmdp, 287 int full) 288 { 289 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 290 } 291 #endif 292 293 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL 294 static inline pud_t pudp_huge_get_and_clear_full(struct mm_struct *mm, 295 unsigned long address, pud_t *pudp, 296 int full) 297 { 298 return pudp_huge_get_and_clear(mm, address, pudp); 299 } 300 #endif 301 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 302 303 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL 304 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, 305 unsigned long address, pte_t *ptep, 306 int full) 307 { 308 pte_t pte; 309 pte = ptep_get_and_clear(mm, address, ptep); 310 return pte; 311 } 312 #endif 313 314 315 /* 316 * If two threads concurrently fault at the same page, the thread that 317 * won the race updates the PTE and its local TLB/Cache. The other thread 318 * gives up, simply does nothing, and continues; on architectures where 319 * software can update TLB, local TLB can be updated here to avoid next page 320 * fault. This function updates TLB only, do nothing with cache or others. 321 * It is the difference with function update_mmu_cache. 322 */ 323 #ifndef __HAVE_ARCH_UPDATE_MMU_TLB 324 static inline void update_mmu_tlb(struct vm_area_struct *vma, 325 unsigned long address, pte_t *ptep) 326 { 327 } 328 #define __HAVE_ARCH_UPDATE_MMU_TLB 329 #endif 330 331 /* 332 * Some architectures may be able to avoid expensive synchronization 333 * primitives when modifications are made to PTE's which are already 334 * not present, or in the process of an address space destruction. 335 */ 336 #ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL 337 static inline void pte_clear_not_present_full(struct mm_struct *mm, 338 unsigned long address, 339 pte_t *ptep, 340 int full) 341 { 342 pte_clear(mm, address, ptep); 343 } 344 #endif 345 346 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH 347 extern pte_t ptep_clear_flush(struct vm_area_struct *vma, 348 unsigned long address, 349 pte_t *ptep); 350 #endif 351 352 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH 353 extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, 354 unsigned long address, 355 pmd_t *pmdp); 356 extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, 357 unsigned long address, 358 pud_t *pudp); 359 #endif 360 361 #ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT 362 struct mm_struct; 363 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) 364 { 365 pte_t old_pte = *ptep; 366 set_pte_at(mm, address, ptep, pte_wrprotect(old_pte)); 367 } 368 #endif 369 370 /* 371 * On some architectures hardware does not set page access bit when accessing 372 * memory page, it is responsibilty of software setting this bit. It brings 373 * out extra page fault penalty to track page access bit. For optimization page 374 * access bit can be set during all page fault flow on these arches. 375 * To be differentiate with macro pte_mkyoung, this macro is used on platforms 376 * where software maintains page access bit. 377 */ 378 #ifndef pte_sw_mkyoung 379 static inline pte_t pte_sw_mkyoung(pte_t pte) 380 { 381 return pte; 382 } 383 #define pte_sw_mkyoung pte_sw_mkyoung 384 #endif 385 386 #ifndef pte_savedwrite 387 #define pte_savedwrite pte_write 388 #endif 389 390 #ifndef pte_mk_savedwrite 391 #define pte_mk_savedwrite pte_mkwrite 392 #endif 393 394 #ifndef pte_clear_savedwrite 395 #define pte_clear_savedwrite pte_wrprotect 396 #endif 397 398 #ifndef pmd_savedwrite 399 #define pmd_savedwrite pmd_write 400 #endif 401 402 #ifndef pmd_mk_savedwrite 403 #define pmd_mk_savedwrite pmd_mkwrite 404 #endif 405 406 #ifndef pmd_clear_savedwrite 407 #define pmd_clear_savedwrite pmd_wrprotect 408 #endif 409 410 #ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT 411 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 412 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 413 unsigned long address, pmd_t *pmdp) 414 { 415 pmd_t old_pmd = *pmdp; 416 set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd)); 417 } 418 #else 419 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 420 unsigned long address, pmd_t *pmdp) 421 { 422 BUILD_BUG(); 423 } 424 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 425 #endif 426 #ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT 427 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 428 static inline void pudp_set_wrprotect(struct mm_struct *mm, 429 unsigned long address, pud_t *pudp) 430 { 431 pud_t old_pud = *pudp; 432 433 set_pud_at(mm, address, pudp, pud_wrprotect(old_pud)); 434 } 435 #else 436 static inline void pudp_set_wrprotect(struct mm_struct *mm, 437 unsigned long address, pud_t *pudp) 438 { 439 BUILD_BUG(); 440 } 441 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 442 #endif 443 444 #ifndef pmdp_collapse_flush 445 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 446 extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 447 unsigned long address, pmd_t *pmdp); 448 #else 449 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 450 unsigned long address, 451 pmd_t *pmdp) 452 { 453 BUILD_BUG(); 454 return *pmdp; 455 } 456 #define pmdp_collapse_flush pmdp_collapse_flush 457 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 458 #endif 459 460 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT 461 extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 462 pgtable_t pgtable); 463 #endif 464 465 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW 466 extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); 467 #endif 468 469 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 470 /* 471 * This is an implementation of pmdp_establish() that is only suitable for an 472 * architecture that doesn't have hardware dirty/accessed bits. In this case we 473 * can't race with CPU which sets these bits and non-atomic aproach is fine. 474 */ 475 static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma, 476 unsigned long address, pmd_t *pmdp, pmd_t pmd) 477 { 478 pmd_t old_pmd = *pmdp; 479 set_pmd_at(vma->vm_mm, address, pmdp, pmd); 480 return old_pmd; 481 } 482 #endif 483 484 #ifndef __HAVE_ARCH_PMDP_INVALIDATE 485 extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, 486 pmd_t *pmdp); 487 #endif 488 489 #ifndef __HAVE_ARCH_PTE_SAME 490 static inline int pte_same(pte_t pte_a, pte_t pte_b) 491 { 492 return pte_val(pte_a) == pte_val(pte_b); 493 } 494 #endif 495 496 #ifndef __HAVE_ARCH_PTE_UNUSED 497 /* 498 * Some architectures provide facilities to virtualization guests 499 * so that they can flag allocated pages as unused. This allows the 500 * host to transparently reclaim unused pages. This function returns 501 * whether the pte's page is unused. 502 */ 503 static inline int pte_unused(pte_t pte) 504 { 505 return 0; 506 } 507 #endif 508 509 #ifndef pte_access_permitted 510 #define pte_access_permitted(pte, write) \ 511 (pte_present(pte) && (!(write) || pte_write(pte))) 512 #endif 513 514 #ifndef pmd_access_permitted 515 #define pmd_access_permitted(pmd, write) \ 516 (pmd_present(pmd) && (!(write) || pmd_write(pmd))) 517 #endif 518 519 #ifndef pud_access_permitted 520 #define pud_access_permitted(pud, write) \ 521 (pud_present(pud) && (!(write) || pud_write(pud))) 522 #endif 523 524 #ifndef p4d_access_permitted 525 #define p4d_access_permitted(p4d, write) \ 526 (p4d_present(p4d) && (!(write) || p4d_write(p4d))) 527 #endif 528 529 #ifndef pgd_access_permitted 530 #define pgd_access_permitted(pgd, write) \ 531 (pgd_present(pgd) && (!(write) || pgd_write(pgd))) 532 #endif 533 534 #ifndef __HAVE_ARCH_PMD_SAME 535 static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) 536 { 537 return pmd_val(pmd_a) == pmd_val(pmd_b); 538 } 539 540 static inline int pud_same(pud_t pud_a, pud_t pud_b) 541 { 542 return pud_val(pud_a) == pud_val(pud_b); 543 } 544 #endif 545 546 #ifndef __HAVE_ARCH_P4D_SAME 547 static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b) 548 { 549 return p4d_val(p4d_a) == p4d_val(p4d_b); 550 } 551 #endif 552 553 #ifndef __HAVE_ARCH_PGD_SAME 554 static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b) 555 { 556 return pgd_val(pgd_a) == pgd_val(pgd_b); 557 } 558 #endif 559 560 /* 561 * Use set_p*_safe(), and elide TLB flushing, when confident that *no* 562 * TLB flush will be required as a result of the "set". For example, use 563 * in scenarios where it is known ahead of time that the routine is 564 * setting non-present entries, or re-setting an existing entry to the 565 * same value. Otherwise, use the typical "set" helpers and flush the 566 * TLB. 567 */ 568 #define set_pte_safe(ptep, pte) \ 569 ({ \ 570 WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \ 571 set_pte(ptep, pte); \ 572 }) 573 574 #define set_pmd_safe(pmdp, pmd) \ 575 ({ \ 576 WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \ 577 set_pmd(pmdp, pmd); \ 578 }) 579 580 #define set_pud_safe(pudp, pud) \ 581 ({ \ 582 WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \ 583 set_pud(pudp, pud); \ 584 }) 585 586 #define set_p4d_safe(p4dp, p4d) \ 587 ({ \ 588 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \ 589 set_p4d(p4dp, p4d); \ 590 }) 591 592 #define set_pgd_safe(pgdp, pgd) \ 593 ({ \ 594 WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \ 595 set_pgd(pgdp, pgd); \ 596 }) 597 598 #ifndef __HAVE_ARCH_DO_SWAP_PAGE 599 /* 600 * Some architectures support metadata associated with a page. When a 601 * page is being swapped out, this metadata must be saved so it can be 602 * restored when the page is swapped back in. SPARC M7 and newer 603 * processors support an ADI (Application Data Integrity) tag for the 604 * page as metadata for the page. arch_do_swap_page() can restore this 605 * metadata when a page is swapped back in. 606 */ 607 static inline void arch_do_swap_page(struct mm_struct *mm, 608 struct vm_area_struct *vma, 609 unsigned long addr, 610 pte_t pte, pte_t oldpte) 611 { 612 613 } 614 #endif 615 616 #ifndef __HAVE_ARCH_UNMAP_ONE 617 /* 618 * Some architectures support metadata associated with a page. When a 619 * page is being swapped out, this metadata must be saved so it can be 620 * restored when the page is swapped back in. SPARC M7 and newer 621 * processors support an ADI (Application Data Integrity) tag for the 622 * page as metadata for the page. arch_unmap_one() can save this 623 * metadata on a swap-out of a page. 624 */ 625 static inline int arch_unmap_one(struct mm_struct *mm, 626 struct vm_area_struct *vma, 627 unsigned long addr, 628 pte_t orig_pte) 629 { 630 return 0; 631 } 632 #endif 633 634 #ifndef __HAVE_ARCH_PGD_OFFSET_GATE 635 #define pgd_offset_gate(mm, addr) pgd_offset(mm, addr) 636 #endif 637 638 #ifndef __HAVE_ARCH_MOVE_PTE 639 #define move_pte(pte, prot, old_addr, new_addr) (pte) 640 #endif 641 642 #ifndef pte_accessible 643 # define pte_accessible(mm, pte) ((void)(pte), 1) 644 #endif 645 646 #ifndef flush_tlb_fix_spurious_fault 647 #define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address) 648 #endif 649 650 #ifndef pgprot_nx 651 #define pgprot_nx(prot) (prot) 652 #endif 653 654 #ifndef pgprot_noncached 655 #define pgprot_noncached(prot) (prot) 656 #endif 657 658 #ifndef pgprot_writecombine 659 #define pgprot_writecombine pgprot_noncached 660 #endif 661 662 #ifndef pgprot_writethrough 663 #define pgprot_writethrough pgprot_noncached 664 #endif 665 666 #ifndef pgprot_device 667 #define pgprot_device pgprot_noncached 668 #endif 669 670 #ifndef pgprot_modify 671 #define pgprot_modify pgprot_modify 672 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 673 { 674 if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot))) 675 newprot = pgprot_noncached(newprot); 676 if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot))) 677 newprot = pgprot_writecombine(newprot); 678 if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot))) 679 newprot = pgprot_device(newprot); 680 return newprot; 681 } 682 #endif 683 684 /* 685 * When walking page tables, get the address of the next boundary, 686 * or the end address of the range if that comes earlier. Although no 687 * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout. 688 */ 689 690 #define pgd_addr_end(addr, end) \ 691 ({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \ 692 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 693 }) 694 695 #ifndef p4d_addr_end 696 #define p4d_addr_end(addr, end) \ 697 ({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \ 698 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 699 }) 700 #endif 701 702 #ifndef pud_addr_end 703 #define pud_addr_end(addr, end) \ 704 ({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \ 705 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 706 }) 707 #endif 708 709 #ifndef pmd_addr_end 710 #define pmd_addr_end(addr, end) \ 711 ({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \ 712 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 713 }) 714 #endif 715 716 /* 717 * When walking page tables, we usually want to skip any p?d_none entries; 718 * and any p?d_bad entries - reporting the error before resetting to none. 719 * Do the tests inline, but report and clear the bad entry in mm/memory.c. 720 */ 721 void pgd_clear_bad(pgd_t *); 722 723 #ifndef __PAGETABLE_P4D_FOLDED 724 void p4d_clear_bad(p4d_t *); 725 #else 726 #define p4d_clear_bad(p4d) do { } while (0) 727 #endif 728 729 #ifndef __PAGETABLE_PUD_FOLDED 730 void pud_clear_bad(pud_t *); 731 #else 732 #define pud_clear_bad(p4d) do { } while (0) 733 #endif 734 735 void pmd_clear_bad(pmd_t *); 736 737 static inline int pgd_none_or_clear_bad(pgd_t *pgd) 738 { 739 if (pgd_none(*pgd)) 740 return 1; 741 if (unlikely(pgd_bad(*pgd))) { 742 pgd_clear_bad(pgd); 743 return 1; 744 } 745 return 0; 746 } 747 748 static inline int p4d_none_or_clear_bad(p4d_t *p4d) 749 { 750 if (p4d_none(*p4d)) 751 return 1; 752 if (unlikely(p4d_bad(*p4d))) { 753 p4d_clear_bad(p4d); 754 return 1; 755 } 756 return 0; 757 } 758 759 static inline int pud_none_or_clear_bad(pud_t *pud) 760 { 761 if (pud_none(*pud)) 762 return 1; 763 if (unlikely(pud_bad(*pud))) { 764 pud_clear_bad(pud); 765 return 1; 766 } 767 return 0; 768 } 769 770 static inline int pmd_none_or_clear_bad(pmd_t *pmd) 771 { 772 if (pmd_none(*pmd)) 773 return 1; 774 if (unlikely(pmd_bad(*pmd))) { 775 pmd_clear_bad(pmd); 776 return 1; 777 } 778 return 0; 779 } 780 781 static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma, 782 unsigned long addr, 783 pte_t *ptep) 784 { 785 /* 786 * Get the current pte state, but zero it out to make it 787 * non-present, preventing the hardware from asynchronously 788 * updating it. 789 */ 790 return ptep_get_and_clear(vma->vm_mm, addr, ptep); 791 } 792 793 static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma, 794 unsigned long addr, 795 pte_t *ptep, pte_t pte) 796 { 797 /* 798 * The pte is non-present, so there's no hardware state to 799 * preserve. 800 */ 801 set_pte_at(vma->vm_mm, addr, ptep, pte); 802 } 803 804 #ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 805 /* 806 * Start a pte protection read-modify-write transaction, which 807 * protects against asynchronous hardware modifications to the pte. 808 * The intention is not to prevent the hardware from making pte 809 * updates, but to prevent any updates it may make from being lost. 810 * 811 * This does not protect against other software modifications of the 812 * pte; the appropriate pte lock must be held over the transation. 813 * 814 * Note that this interface is intended to be batchable, meaning that 815 * ptep_modify_prot_commit may not actually update the pte, but merely 816 * queue the update to be done at some later time. The update must be 817 * actually committed before the pte lock is released, however. 818 */ 819 static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, 820 unsigned long addr, 821 pte_t *ptep) 822 { 823 return __ptep_modify_prot_start(vma, addr, ptep); 824 } 825 826 /* 827 * Commit an update to a pte, leaving any hardware-controlled bits in 828 * the PTE unmodified. 829 */ 830 static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, 831 unsigned long addr, 832 pte_t *ptep, pte_t old_pte, pte_t pte) 833 { 834 __ptep_modify_prot_commit(vma, addr, ptep, pte); 835 } 836 #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */ 837 #endif /* CONFIG_MMU */ 838 839 /* 840 * No-op macros that just return the current protection value. Defined here 841 * because these macros can be used used even if CONFIG_MMU is not defined. 842 */ 843 #ifndef pgprot_encrypted 844 #define pgprot_encrypted(prot) (prot) 845 #endif 846 847 #ifndef pgprot_decrypted 848 #define pgprot_decrypted(prot) (prot) 849 #endif 850 851 /* 852 * A facility to provide lazy MMU batching. This allows PTE updates and 853 * page invalidations to be delayed until a call to leave lazy MMU mode 854 * is issued. Some architectures may benefit from doing this, and it is 855 * beneficial for both shadow and direct mode hypervisors, which may batch 856 * the PTE updates which happen during this window. Note that using this 857 * interface requires that read hazards be removed from the code. A read 858 * hazard could result in the direct mode hypervisor case, since the actual 859 * write to the page tables may not yet have taken place, so reads though 860 * a raw PTE pointer after it has been modified are not guaranteed to be 861 * up to date. This mode can only be entered and left under the protection of 862 * the page table locks for all page tables which may be modified. In the UP 863 * case, this is required so that preemption is disabled, and in the SMP case, 864 * it must synchronize the delayed page table writes properly on other CPUs. 865 */ 866 #ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE 867 #define arch_enter_lazy_mmu_mode() do {} while (0) 868 #define arch_leave_lazy_mmu_mode() do {} while (0) 869 #define arch_flush_lazy_mmu_mode() do {} while (0) 870 #endif 871 872 /* 873 * A facility to provide batching of the reload of page tables and 874 * other process state with the actual context switch code for 875 * paravirtualized guests. By convention, only one of the batched 876 * update (lazy) modes (CPU, MMU) should be active at any given time, 877 * entry should never be nested, and entry and exits should always be 878 * paired. This is for sanity of maintaining and reasoning about the 879 * kernel code. In this case, the exit (end of the context switch) is 880 * in architecture-specific code, and so doesn't need a generic 881 * definition. 882 */ 883 #ifndef __HAVE_ARCH_START_CONTEXT_SWITCH 884 #define arch_start_context_switch(prev) do {} while (0) 885 #endif 886 887 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY 888 #ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION 889 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) 890 { 891 return pmd; 892 } 893 894 static inline int pmd_swp_soft_dirty(pmd_t pmd) 895 { 896 return 0; 897 } 898 899 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) 900 { 901 return pmd; 902 } 903 #endif 904 #else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */ 905 static inline int pte_soft_dirty(pte_t pte) 906 { 907 return 0; 908 } 909 910 static inline int pmd_soft_dirty(pmd_t pmd) 911 { 912 return 0; 913 } 914 915 static inline pte_t pte_mksoft_dirty(pte_t pte) 916 { 917 return pte; 918 } 919 920 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) 921 { 922 return pmd; 923 } 924 925 static inline pte_t pte_clear_soft_dirty(pte_t pte) 926 { 927 return pte; 928 } 929 930 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) 931 { 932 return pmd; 933 } 934 935 static inline pte_t pte_swp_mksoft_dirty(pte_t pte) 936 { 937 return pte; 938 } 939 940 static inline int pte_swp_soft_dirty(pte_t pte) 941 { 942 return 0; 943 } 944 945 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) 946 { 947 return pte; 948 } 949 950 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) 951 { 952 return pmd; 953 } 954 955 static inline int pmd_swp_soft_dirty(pmd_t pmd) 956 { 957 return 0; 958 } 959 960 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) 961 { 962 return pmd; 963 } 964 #endif 965 966 #ifndef __HAVE_PFNMAP_TRACKING 967 /* 968 * Interfaces that can be used by architecture code to keep track of 969 * memory type of pfn mappings specified by the remap_pfn_range, 970 * vmf_insert_pfn. 971 */ 972 973 /* 974 * track_pfn_remap is called when a _new_ pfn mapping is being established 975 * by remap_pfn_range() for physical range indicated by pfn and size. 976 */ 977 static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, 978 unsigned long pfn, unsigned long addr, 979 unsigned long size) 980 { 981 return 0; 982 } 983 984 /* 985 * track_pfn_insert is called when a _new_ single pfn is established 986 * by vmf_insert_pfn(). 987 */ 988 static inline void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, 989 pfn_t pfn) 990 { 991 } 992 993 /* 994 * track_pfn_copy is called when vma that is covering the pfnmap gets 995 * copied through copy_page_range(). 996 */ 997 static inline int track_pfn_copy(struct vm_area_struct *vma) 998 { 999 return 0; 1000 } 1001 1002 /* 1003 * untrack_pfn is called while unmapping a pfnmap for a region. 1004 * untrack can be called for a specific region indicated by pfn and size or 1005 * can be for the entire vma (in which case pfn, size are zero). 1006 */ 1007 static inline void untrack_pfn(struct vm_area_struct *vma, 1008 unsigned long pfn, unsigned long size) 1009 { 1010 } 1011 1012 /* 1013 * untrack_pfn_moved is called while mremapping a pfnmap for a new region. 1014 */ 1015 static inline void untrack_pfn_moved(struct vm_area_struct *vma) 1016 { 1017 } 1018 #else 1019 extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, 1020 unsigned long pfn, unsigned long addr, 1021 unsigned long size); 1022 extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, 1023 pfn_t pfn); 1024 extern int track_pfn_copy(struct vm_area_struct *vma); 1025 extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn, 1026 unsigned long size); 1027 extern void untrack_pfn_moved(struct vm_area_struct *vma); 1028 #endif 1029 1030 #ifdef __HAVE_COLOR_ZERO_PAGE 1031 static inline int is_zero_pfn(unsigned long pfn) 1032 { 1033 extern unsigned long zero_pfn; 1034 unsigned long offset_from_zero_pfn = pfn - zero_pfn; 1035 return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT); 1036 } 1037 1038 #define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) 1039 1040 #else 1041 static inline int is_zero_pfn(unsigned long pfn) 1042 { 1043 extern unsigned long zero_pfn; 1044 return pfn == zero_pfn; 1045 } 1046 1047 static inline unsigned long my_zero_pfn(unsigned long addr) 1048 { 1049 extern unsigned long zero_pfn; 1050 return zero_pfn; 1051 } 1052 #endif 1053 1054 #ifdef CONFIG_MMU 1055 1056 #ifndef CONFIG_TRANSPARENT_HUGEPAGE 1057 static inline int pmd_trans_huge(pmd_t pmd) 1058 { 1059 return 0; 1060 } 1061 #ifndef pmd_write 1062 static inline int pmd_write(pmd_t pmd) 1063 { 1064 BUG(); 1065 return 0; 1066 } 1067 #endif /* pmd_write */ 1068 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1069 1070 #ifndef pud_write 1071 static inline int pud_write(pud_t pud) 1072 { 1073 BUG(); 1074 return 0; 1075 } 1076 #endif /* pud_write */ 1077 1078 #if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE) 1079 static inline int pmd_devmap(pmd_t pmd) 1080 { 1081 return 0; 1082 } 1083 static inline int pud_devmap(pud_t pud) 1084 { 1085 return 0; 1086 } 1087 static inline int pgd_devmap(pgd_t pgd) 1088 { 1089 return 0; 1090 } 1091 #endif 1092 1093 #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ 1094 (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1095 !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) 1096 static inline int pud_trans_huge(pud_t pud) 1097 { 1098 return 0; 1099 } 1100 #endif 1101 1102 /* See pmd_none_or_trans_huge_or_clear_bad for discussion. */ 1103 static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud) 1104 { 1105 pud_t pudval = READ_ONCE(*pud); 1106 1107 if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval)) 1108 return 1; 1109 if (unlikely(pud_bad(pudval))) { 1110 pud_clear_bad(pud); 1111 return 1; 1112 } 1113 return 0; 1114 } 1115 1116 /* See pmd_trans_unstable for discussion. */ 1117 static inline int pud_trans_unstable(pud_t *pud) 1118 { 1119 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1120 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 1121 return pud_none_or_trans_huge_or_dev_or_clear_bad(pud); 1122 #else 1123 return 0; 1124 #endif 1125 } 1126 1127 #ifndef pmd_read_atomic 1128 static inline pmd_t pmd_read_atomic(pmd_t *pmdp) 1129 { 1130 /* 1131 * Depend on compiler for an atomic pmd read. NOTE: this is 1132 * only going to work, if the pmdval_t isn't larger than 1133 * an unsigned long. 1134 */ 1135 return *pmdp; 1136 } 1137 #endif 1138 1139 #ifndef arch_needs_pgtable_deposit 1140 #define arch_needs_pgtable_deposit() (false) 1141 #endif 1142 /* 1143 * This function is meant to be used by sites walking pagetables with 1144 * the mmap_lock held in read mode to protect against MADV_DONTNEED and 1145 * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd 1146 * into a null pmd and the transhuge page fault can convert a null pmd 1147 * into an hugepmd or into a regular pmd (if the hugepage allocation 1148 * fails). While holding the mmap_lock in read mode the pmd becomes 1149 * stable and stops changing under us only if it's not null and not a 1150 * transhuge pmd. When those races occurs and this function makes a 1151 * difference vs the standard pmd_none_or_clear_bad, the result is 1152 * undefined so behaving like if the pmd was none is safe (because it 1153 * can return none anyway). The compiler level barrier() is critically 1154 * important to compute the two checks atomically on the same pmdval. 1155 * 1156 * For 32bit kernels with a 64bit large pmd_t this automatically takes 1157 * care of reading the pmd atomically to avoid SMP race conditions 1158 * against pmd_populate() when the mmap_lock is hold for reading by the 1159 * caller (a special atomic read not done by "gcc" as in the generic 1160 * version above, is also needed when THP is disabled because the page 1161 * fault can populate the pmd from under us). 1162 */ 1163 static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd) 1164 { 1165 pmd_t pmdval = pmd_read_atomic(pmd); 1166 /* 1167 * The barrier will stabilize the pmdval in a register or on 1168 * the stack so that it will stop changing under the code. 1169 * 1170 * When CONFIG_TRANSPARENT_HUGEPAGE=y on x86 32bit PAE, 1171 * pmd_read_atomic is allowed to return a not atomic pmdval 1172 * (for example pointing to an hugepage that has never been 1173 * mapped in the pmd). The below checks will only care about 1174 * the low part of the pmd with 32bit PAE x86 anyway, with the 1175 * exception of pmd_none(). So the important thing is that if 1176 * the low part of the pmd is found null, the high part will 1177 * be also null or the pmd_none() check below would be 1178 * confused. 1179 */ 1180 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1181 barrier(); 1182 #endif 1183 /* 1184 * !pmd_present() checks for pmd migration entries 1185 * 1186 * The complete check uses is_pmd_migration_entry() in linux/swapops.h 1187 * But using that requires moving current function and pmd_trans_unstable() 1188 * to linux/swapops.h to resovle dependency, which is too much code move. 1189 * 1190 * !pmd_present() is equivalent to is_pmd_migration_entry() currently, 1191 * because !pmd_present() pages can only be under migration not swapped 1192 * out. 1193 * 1194 * pmd_none() is preseved for future condition checks on pmd migration 1195 * entries and not confusing with this function name, although it is 1196 * redundant with !pmd_present(). 1197 */ 1198 if (pmd_none(pmdval) || pmd_trans_huge(pmdval) || 1199 (IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION) && !pmd_present(pmdval))) 1200 return 1; 1201 if (unlikely(pmd_bad(pmdval))) { 1202 pmd_clear_bad(pmd); 1203 return 1; 1204 } 1205 return 0; 1206 } 1207 1208 /* 1209 * This is a noop if Transparent Hugepage Support is not built into 1210 * the kernel. Otherwise it is equivalent to 1211 * pmd_none_or_trans_huge_or_clear_bad(), and shall only be called in 1212 * places that already verified the pmd is not none and they want to 1213 * walk ptes while holding the mmap sem in read mode (write mode don't 1214 * need this). If THP is not enabled, the pmd can't go away under the 1215 * code even if MADV_DONTNEED runs, but if THP is enabled we need to 1216 * run a pmd_trans_unstable before walking the ptes after 1217 * split_huge_pmd returns (because it may have run when the pmd become 1218 * null, but then a page fault can map in a THP and not a regular page). 1219 */ 1220 static inline int pmd_trans_unstable(pmd_t *pmd) 1221 { 1222 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1223 return pmd_none_or_trans_huge_or_clear_bad(pmd); 1224 #else 1225 return 0; 1226 #endif 1227 } 1228 1229 #ifndef CONFIG_NUMA_BALANCING 1230 /* 1231 * Technically a PTE can be PROTNONE even when not doing NUMA balancing but 1232 * the only case the kernel cares is for NUMA balancing and is only ever set 1233 * when the VMA is accessible. For PROT_NONE VMAs, the PTEs are not marked 1234 * _PAGE_PROTNONE so by by default, implement the helper as "always no". It 1235 * is the responsibility of the caller to distinguish between PROT_NONE 1236 * protections and NUMA hinting fault protections. 1237 */ 1238 static inline int pte_protnone(pte_t pte) 1239 { 1240 return 0; 1241 } 1242 1243 static inline int pmd_protnone(pmd_t pmd) 1244 { 1245 return 0; 1246 } 1247 #endif /* CONFIG_NUMA_BALANCING */ 1248 1249 #endif /* CONFIG_MMU */ 1250 1251 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 1252 1253 #ifndef __PAGETABLE_P4D_FOLDED 1254 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot); 1255 int p4d_clear_huge(p4d_t *p4d); 1256 #else 1257 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 1258 { 1259 return 0; 1260 } 1261 static inline int p4d_clear_huge(p4d_t *p4d) 1262 { 1263 return 0; 1264 } 1265 #endif /* !__PAGETABLE_P4D_FOLDED */ 1266 1267 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot); 1268 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot); 1269 int pud_clear_huge(pud_t *pud); 1270 int pmd_clear_huge(pmd_t *pmd); 1271 int p4d_free_pud_page(p4d_t *p4d, unsigned long addr); 1272 int pud_free_pmd_page(pud_t *pud, unsigned long addr); 1273 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr); 1274 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 1275 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 1276 { 1277 return 0; 1278 } 1279 static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) 1280 { 1281 return 0; 1282 } 1283 static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) 1284 { 1285 return 0; 1286 } 1287 static inline int p4d_clear_huge(p4d_t *p4d) 1288 { 1289 return 0; 1290 } 1291 static inline int pud_clear_huge(pud_t *pud) 1292 { 1293 return 0; 1294 } 1295 static inline int pmd_clear_huge(pmd_t *pmd) 1296 { 1297 return 0; 1298 } 1299 static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) 1300 { 1301 return 0; 1302 } 1303 static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr) 1304 { 1305 return 0; 1306 } 1307 static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) 1308 { 1309 return 0; 1310 } 1311 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 1312 1313 #ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE 1314 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1315 /* 1316 * ARCHes with special requirements for evicting THP backing TLB entries can 1317 * implement this. Otherwise also, it can help optimize normal TLB flush in 1318 * THP regime. stock flush_tlb_range() typically has optimization to nuke the 1319 * entire TLB TLB if flush span is greater than a threshold, which will 1320 * likely be true for a single huge page. Thus a single thp flush will 1321 * invalidate the entire TLB which is not desitable. 1322 * e.g. see arch/arc: flush_pmd_tlb_range 1323 */ 1324 #define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 1325 #define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 1326 #else 1327 #define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG() 1328 #define flush_pud_tlb_range(vma, addr, end) BUILD_BUG() 1329 #endif 1330 #endif 1331 1332 struct file; 1333 int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, 1334 unsigned long size, pgprot_t *vma_prot); 1335 1336 #ifndef CONFIG_X86_ESPFIX64 1337 static inline void init_espfix_bsp(void) { } 1338 #endif 1339 1340 extern void __init pgtable_cache_init(void); 1341 1342 #ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED 1343 static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot) 1344 { 1345 return true; 1346 } 1347 1348 static inline bool arch_has_pfn_modify_check(void) 1349 { 1350 return false; 1351 } 1352 #endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */ 1353 1354 /* 1355 * Architecture PAGE_KERNEL_* fallbacks 1356 * 1357 * Some architectures don't define certain PAGE_KERNEL_* flags. This is either 1358 * because they really don't support them, or the port needs to be updated to 1359 * reflect the required functionality. Below are a set of relatively safe 1360 * fallbacks, as best effort, which we can count on in lieu of the architectures 1361 * not defining them on their own yet. 1362 */ 1363 1364 #ifndef PAGE_KERNEL_RO 1365 # define PAGE_KERNEL_RO PAGE_KERNEL 1366 #endif 1367 1368 #ifndef PAGE_KERNEL_EXEC 1369 # define PAGE_KERNEL_EXEC PAGE_KERNEL 1370 #endif 1371 1372 /* 1373 * Page Table Modification bits for pgtbl_mod_mask. 1374 * 1375 * These are used by the p?d_alloc_track*() set of functions an in the generic 1376 * vmalloc/ioremap code to track at which page-table levels entries have been 1377 * modified. Based on that the code can better decide when vmalloc and ioremap 1378 * mapping changes need to be synchronized to other page-tables in the system. 1379 */ 1380 #define __PGTBL_PGD_MODIFIED 0 1381 #define __PGTBL_P4D_MODIFIED 1 1382 #define __PGTBL_PUD_MODIFIED 2 1383 #define __PGTBL_PMD_MODIFIED 3 1384 #define __PGTBL_PTE_MODIFIED 4 1385 1386 #define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED) 1387 #define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED) 1388 #define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED) 1389 #define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED) 1390 #define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED) 1391 1392 /* Page-Table Modification Mask */ 1393 typedef unsigned int pgtbl_mod_mask; 1394 1395 #endif /* !__ASSEMBLY__ */ 1396 1397 #ifndef io_remap_pfn_range 1398 #define io_remap_pfn_range remap_pfn_range 1399 #endif 1400 1401 #ifndef has_transparent_hugepage 1402 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1403 #define has_transparent_hugepage() 1 1404 #else 1405 #define has_transparent_hugepage() 0 1406 #endif 1407 #endif 1408 1409 /* 1410 * On some architectures it depends on the mm if the p4d/pud or pmd 1411 * layer of the page table hierarchy is folded or not. 1412 */ 1413 #ifndef mm_p4d_folded 1414 #define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED) 1415 #endif 1416 1417 #ifndef mm_pud_folded 1418 #define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED) 1419 #endif 1420 1421 #ifndef mm_pmd_folded 1422 #define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED) 1423 #endif 1424 1425 /* 1426 * p?d_leaf() - true if this entry is a final mapping to a physical address. 1427 * This differs from p?d_huge() by the fact that they are always available (if 1428 * the architecture supports large pages at the appropriate level) even 1429 * if CONFIG_HUGETLB_PAGE is not defined. 1430 * Only meaningful when called on a valid entry. 1431 */ 1432 #ifndef pgd_leaf 1433 #define pgd_leaf(x) 0 1434 #endif 1435 #ifndef p4d_leaf 1436 #define p4d_leaf(x) 0 1437 #endif 1438 #ifndef pud_leaf 1439 #define pud_leaf(x) 0 1440 #endif 1441 #ifndef pmd_leaf 1442 #define pmd_leaf(x) 0 1443 #endif 1444 1445 #endif /* _LINUX_PGTABLE_H */ 1446