1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_PGTABLE_H 3 #define _LINUX_PGTABLE_H 4 5 #include <linux/pfn.h> 6 #include <asm/pgtable.h> 7 8 #ifndef __ASSEMBLY__ 9 #ifdef CONFIG_MMU 10 11 #include <linux/mm_types.h> 12 #include <linux/bug.h> 13 #include <linux/errno.h> 14 #include <asm-generic/pgtable_uffd.h> 15 16 #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ 17 defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS 18 #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED 19 #endif 20 21 /* 22 * On almost all architectures and configurations, 0 can be used as the 23 * upper ceiling to free_pgtables(): on many architectures it has the same 24 * effect as using TASK_SIZE. However, there is one configuration which 25 * must impose a more careful limit, to avoid freeing kernel pgtables. 26 */ 27 #ifndef USER_PGTABLES_CEILING 28 #define USER_PGTABLES_CEILING 0UL 29 #endif 30 31 /* 32 * This defines the first usable user address. Platforms 33 * can override its value with custom FIRST_USER_ADDRESS 34 * defined in their respective <asm/pgtable.h>. 35 */ 36 #ifndef FIRST_USER_ADDRESS 37 #define FIRST_USER_ADDRESS 0UL 38 #endif 39 40 /* 41 * This defines the generic helper for accessing PMD page 42 * table page. Although platforms can still override this 43 * via their respective <asm/pgtable.h>. 44 */ 45 #ifndef pmd_pgtable 46 #define pmd_pgtable(pmd) pmd_page(pmd) 47 #endif 48 49 /* 50 * A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD] 51 * 52 * The pXx_index() functions return the index of the entry in the page 53 * table page which would control the given virtual address 54 * 55 * As these functions may be used by the same code for different levels of 56 * the page table folding, they are always available, regardless of 57 * CONFIG_PGTABLE_LEVELS value. For the folded levels they simply return 0 58 * because in such cases PTRS_PER_PxD equals 1. 59 */ 60 61 static inline unsigned long pte_index(unsigned long address) 62 { 63 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); 64 } 65 66 #ifndef pmd_index 67 static inline unsigned long pmd_index(unsigned long address) 68 { 69 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); 70 } 71 #define pmd_index pmd_index 72 #endif 73 74 #ifndef pud_index 75 static inline unsigned long pud_index(unsigned long address) 76 { 77 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); 78 } 79 #define pud_index pud_index 80 #endif 81 82 #ifndef pgd_index 83 /* Must be a compile-time constant, so implement it as a macro */ 84 #define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) 85 #endif 86 87 #ifndef pte_offset_kernel 88 static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) 89 { 90 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); 91 } 92 #define pte_offset_kernel pte_offset_kernel 93 #endif 94 95 #if defined(CONFIG_HIGHPTE) 96 #define pte_offset_map(dir, address) \ 97 ((pte_t *)kmap_atomic(pmd_page(*(dir))) + \ 98 pte_index((address))) 99 #define pte_unmap(pte) kunmap_atomic((pte)) 100 #else 101 #define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) 102 #define pte_unmap(pte) ((void)(pte)) /* NOP */ 103 #endif 104 105 /* Find an entry in the second-level page table.. */ 106 #ifndef pmd_offset 107 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) 108 { 109 return pud_pgtable(*pud) + pmd_index(address); 110 } 111 #define pmd_offset pmd_offset 112 #endif 113 114 #ifndef pud_offset 115 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) 116 { 117 return p4d_pgtable(*p4d) + pud_index(address); 118 } 119 #define pud_offset pud_offset 120 #endif 121 122 static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) 123 { 124 return (pgd + pgd_index(address)); 125 }; 126 127 /* 128 * a shortcut to get a pgd_t in a given mm 129 */ 130 #ifndef pgd_offset 131 #define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address)) 132 #endif 133 134 /* 135 * a shortcut which implies the use of the kernel's pgd, instead 136 * of a process's 137 */ 138 #ifndef pgd_offset_k 139 #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) 140 #endif 141 142 /* 143 * In many cases it is known that a virtual address is mapped at PMD or PTE 144 * level, so instead of traversing all the page table levels, we can get a 145 * pointer to the PMD entry in user or kernel page table or translate a virtual 146 * address to the pointer in the PTE in the kernel page tables with simple 147 * helpers. 148 */ 149 static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va) 150 { 151 return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va); 152 } 153 154 static inline pmd_t *pmd_off_k(unsigned long va) 155 { 156 return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va); 157 } 158 159 static inline pte_t *virt_to_kpte(unsigned long vaddr) 160 { 161 pmd_t *pmd = pmd_off_k(vaddr); 162 163 return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr); 164 } 165 166 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 167 extern int ptep_set_access_flags(struct vm_area_struct *vma, 168 unsigned long address, pte_t *ptep, 169 pte_t entry, int dirty); 170 #endif 171 172 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS 173 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 174 extern int pmdp_set_access_flags(struct vm_area_struct *vma, 175 unsigned long address, pmd_t *pmdp, 176 pmd_t entry, int dirty); 177 extern int pudp_set_access_flags(struct vm_area_struct *vma, 178 unsigned long address, pud_t *pudp, 179 pud_t entry, int dirty); 180 #else 181 static inline int pmdp_set_access_flags(struct vm_area_struct *vma, 182 unsigned long address, pmd_t *pmdp, 183 pmd_t entry, int dirty) 184 { 185 BUILD_BUG(); 186 return 0; 187 } 188 static inline int pudp_set_access_flags(struct vm_area_struct *vma, 189 unsigned long address, pud_t *pudp, 190 pud_t entry, int dirty) 191 { 192 BUILD_BUG(); 193 return 0; 194 } 195 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 196 #endif 197 198 #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG 199 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, 200 unsigned long address, 201 pte_t *ptep) 202 { 203 pte_t pte = *ptep; 204 int r = 1; 205 if (!pte_young(pte)) 206 r = 0; 207 else 208 set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte)); 209 return r; 210 } 211 #endif 212 213 #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG 214 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 215 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 216 unsigned long address, 217 pmd_t *pmdp) 218 { 219 pmd_t pmd = *pmdp; 220 int r = 1; 221 if (!pmd_young(pmd)) 222 r = 0; 223 else 224 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); 225 return r; 226 } 227 #else 228 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 229 unsigned long address, 230 pmd_t *pmdp) 231 { 232 BUILD_BUG(); 233 return 0; 234 } 235 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 236 #endif 237 238 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 239 int ptep_clear_flush_young(struct vm_area_struct *vma, 240 unsigned long address, pte_t *ptep); 241 #endif 242 243 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH 244 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 245 extern int pmdp_clear_flush_young(struct vm_area_struct *vma, 246 unsigned long address, pmd_t *pmdp); 247 #else 248 /* 249 * Despite relevant to THP only, this API is called from generic rmap code 250 * under PageTransHuge(), hence needs a dummy implementation for !THP 251 */ 252 static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, 253 unsigned long address, pmd_t *pmdp) 254 { 255 BUILD_BUG(); 256 return 0; 257 } 258 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 259 #endif 260 261 #ifndef __HAVE_ARCH_PTEP_CLEAR 262 static inline void ptep_clear(struct mm_struct *mm, unsigned long addr, 263 pte_t *ptep) 264 { 265 pte_clear(mm, addr, ptep); 266 } 267 #endif 268 269 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR 270 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, 271 unsigned long address, 272 pte_t *ptep) 273 { 274 pte_t pte = *ptep; 275 pte_clear(mm, address, ptep); 276 return pte; 277 } 278 #endif 279 280 #ifndef __HAVE_ARCH_PTEP_GET 281 static inline pte_t ptep_get(pte_t *ptep) 282 { 283 return READ_ONCE(*ptep); 284 } 285 #endif 286 287 #ifdef CONFIG_GUP_GET_PTE_LOW_HIGH 288 /* 289 * WARNING: only to be used in the get_user_pages_fast() implementation. 290 * 291 * With get_user_pages_fast(), we walk down the pagetables without taking any 292 * locks. For this we would like to load the pointers atomically, but sometimes 293 * that is not possible (e.g. without expensive cmpxchg8b on x86_32 PAE). What 294 * we do have is the guarantee that a PTE will only either go from not present 295 * to present, or present to not present or both -- it will not switch to a 296 * completely different present page without a TLB flush in between; something 297 * that we are blocking by holding interrupts off. 298 * 299 * Setting ptes from not present to present goes: 300 * 301 * ptep->pte_high = h; 302 * smp_wmb(); 303 * ptep->pte_low = l; 304 * 305 * And present to not present goes: 306 * 307 * ptep->pte_low = 0; 308 * smp_wmb(); 309 * ptep->pte_high = 0; 310 * 311 * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'. 312 * We load pte_high *after* loading pte_low, which ensures we don't see an older 313 * value of pte_high. *Then* we recheck pte_low, which ensures that we haven't 314 * picked up a changed pte high. We might have gotten rubbish values from 315 * pte_low and pte_high, but we are guaranteed that pte_low will not have the 316 * present bit set *unless* it is 'l'. Because get_user_pages_fast() only 317 * operates on present ptes we're safe. 318 */ 319 static inline pte_t ptep_get_lockless(pte_t *ptep) 320 { 321 pte_t pte; 322 323 do { 324 pte.pte_low = ptep->pte_low; 325 smp_rmb(); 326 pte.pte_high = ptep->pte_high; 327 smp_rmb(); 328 } while (unlikely(pte.pte_low != ptep->pte_low)); 329 330 return pte; 331 } 332 #else /* CONFIG_GUP_GET_PTE_LOW_HIGH */ 333 /* 334 * We require that the PTE can be read atomically. 335 */ 336 static inline pte_t ptep_get_lockless(pte_t *ptep) 337 { 338 return ptep_get(ptep); 339 } 340 #endif /* CONFIG_GUP_GET_PTE_LOW_HIGH */ 341 342 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 343 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR 344 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, 345 unsigned long address, 346 pmd_t *pmdp) 347 { 348 pmd_t pmd = *pmdp; 349 pmd_clear(pmdp); 350 return pmd; 351 } 352 #endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */ 353 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR 354 static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, 355 unsigned long address, 356 pud_t *pudp) 357 { 358 pud_t pud = *pudp; 359 360 pud_clear(pudp); 361 return pud; 362 } 363 #endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */ 364 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 365 366 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 367 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL 368 static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, 369 unsigned long address, pmd_t *pmdp, 370 int full) 371 { 372 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 373 } 374 #endif 375 376 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL 377 static inline pud_t pudp_huge_get_and_clear_full(struct mm_struct *mm, 378 unsigned long address, pud_t *pudp, 379 int full) 380 { 381 return pudp_huge_get_and_clear(mm, address, pudp); 382 } 383 #endif 384 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 385 386 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL 387 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, 388 unsigned long address, pte_t *ptep, 389 int full) 390 { 391 pte_t pte; 392 pte = ptep_get_and_clear(mm, address, ptep); 393 return pte; 394 } 395 #endif 396 397 398 /* 399 * If two threads concurrently fault at the same page, the thread that 400 * won the race updates the PTE and its local TLB/Cache. The other thread 401 * gives up, simply does nothing, and continues; on architectures where 402 * software can update TLB, local TLB can be updated here to avoid next page 403 * fault. This function updates TLB only, do nothing with cache or others. 404 * It is the difference with function update_mmu_cache. 405 */ 406 #ifndef __HAVE_ARCH_UPDATE_MMU_TLB 407 static inline void update_mmu_tlb(struct vm_area_struct *vma, 408 unsigned long address, pte_t *ptep) 409 { 410 } 411 #define __HAVE_ARCH_UPDATE_MMU_TLB 412 #endif 413 414 /* 415 * Some architectures may be able to avoid expensive synchronization 416 * primitives when modifications are made to PTE's which are already 417 * not present, or in the process of an address space destruction. 418 */ 419 #ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL 420 static inline void pte_clear_not_present_full(struct mm_struct *mm, 421 unsigned long address, 422 pte_t *ptep, 423 int full) 424 { 425 pte_clear(mm, address, ptep); 426 } 427 #endif 428 429 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH 430 extern pte_t ptep_clear_flush(struct vm_area_struct *vma, 431 unsigned long address, 432 pte_t *ptep); 433 #endif 434 435 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH 436 extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, 437 unsigned long address, 438 pmd_t *pmdp); 439 extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, 440 unsigned long address, 441 pud_t *pudp); 442 #endif 443 444 #ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT 445 struct mm_struct; 446 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) 447 { 448 pte_t old_pte = *ptep; 449 set_pte_at(mm, address, ptep, pte_wrprotect(old_pte)); 450 } 451 #endif 452 453 /* 454 * On some architectures hardware does not set page access bit when accessing 455 * memory page, it is responsibility of software setting this bit. It brings 456 * out extra page fault penalty to track page access bit. For optimization page 457 * access bit can be set during all page fault flow on these arches. 458 * To be differentiate with macro pte_mkyoung, this macro is used on platforms 459 * where software maintains page access bit. 460 */ 461 #ifndef pte_sw_mkyoung 462 static inline pte_t pte_sw_mkyoung(pte_t pte) 463 { 464 return pte; 465 } 466 #define pte_sw_mkyoung pte_sw_mkyoung 467 #endif 468 469 #ifndef pte_savedwrite 470 #define pte_savedwrite pte_write 471 #endif 472 473 #ifndef pte_mk_savedwrite 474 #define pte_mk_savedwrite pte_mkwrite 475 #endif 476 477 #ifndef pte_clear_savedwrite 478 #define pte_clear_savedwrite pte_wrprotect 479 #endif 480 481 #ifndef pmd_savedwrite 482 #define pmd_savedwrite pmd_write 483 #endif 484 485 #ifndef pmd_mk_savedwrite 486 #define pmd_mk_savedwrite pmd_mkwrite 487 #endif 488 489 #ifndef pmd_clear_savedwrite 490 #define pmd_clear_savedwrite pmd_wrprotect 491 #endif 492 493 #ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT 494 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 495 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 496 unsigned long address, pmd_t *pmdp) 497 { 498 pmd_t old_pmd = *pmdp; 499 set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd)); 500 } 501 #else 502 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 503 unsigned long address, pmd_t *pmdp) 504 { 505 BUILD_BUG(); 506 } 507 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 508 #endif 509 #ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT 510 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 511 static inline void pudp_set_wrprotect(struct mm_struct *mm, 512 unsigned long address, pud_t *pudp) 513 { 514 pud_t old_pud = *pudp; 515 516 set_pud_at(mm, address, pudp, pud_wrprotect(old_pud)); 517 } 518 #else 519 static inline void pudp_set_wrprotect(struct mm_struct *mm, 520 unsigned long address, pud_t *pudp) 521 { 522 BUILD_BUG(); 523 } 524 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 525 #endif 526 527 #ifndef pmdp_collapse_flush 528 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 529 extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 530 unsigned long address, pmd_t *pmdp); 531 #else 532 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 533 unsigned long address, 534 pmd_t *pmdp) 535 { 536 BUILD_BUG(); 537 return *pmdp; 538 } 539 #define pmdp_collapse_flush pmdp_collapse_flush 540 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 541 #endif 542 543 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT 544 extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 545 pgtable_t pgtable); 546 #endif 547 548 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW 549 extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); 550 #endif 551 552 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 553 /* 554 * This is an implementation of pmdp_establish() that is only suitable for an 555 * architecture that doesn't have hardware dirty/accessed bits. In this case we 556 * can't race with CPU which sets these bits and non-atomic approach is fine. 557 */ 558 static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma, 559 unsigned long address, pmd_t *pmdp, pmd_t pmd) 560 { 561 pmd_t old_pmd = *pmdp; 562 set_pmd_at(vma->vm_mm, address, pmdp, pmd); 563 return old_pmd; 564 } 565 #endif 566 567 #ifndef __HAVE_ARCH_PMDP_INVALIDATE 568 extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, 569 pmd_t *pmdp); 570 #endif 571 572 #ifndef __HAVE_ARCH_PTE_SAME 573 static inline int pte_same(pte_t pte_a, pte_t pte_b) 574 { 575 return pte_val(pte_a) == pte_val(pte_b); 576 } 577 #endif 578 579 #ifndef __HAVE_ARCH_PTE_UNUSED 580 /* 581 * Some architectures provide facilities to virtualization guests 582 * so that they can flag allocated pages as unused. This allows the 583 * host to transparently reclaim unused pages. This function returns 584 * whether the pte's page is unused. 585 */ 586 static inline int pte_unused(pte_t pte) 587 { 588 return 0; 589 } 590 #endif 591 592 #ifndef pte_access_permitted 593 #define pte_access_permitted(pte, write) \ 594 (pte_present(pte) && (!(write) || pte_write(pte))) 595 #endif 596 597 #ifndef pmd_access_permitted 598 #define pmd_access_permitted(pmd, write) \ 599 (pmd_present(pmd) && (!(write) || pmd_write(pmd))) 600 #endif 601 602 #ifndef pud_access_permitted 603 #define pud_access_permitted(pud, write) \ 604 (pud_present(pud) && (!(write) || pud_write(pud))) 605 #endif 606 607 #ifndef p4d_access_permitted 608 #define p4d_access_permitted(p4d, write) \ 609 (p4d_present(p4d) && (!(write) || p4d_write(p4d))) 610 #endif 611 612 #ifndef pgd_access_permitted 613 #define pgd_access_permitted(pgd, write) \ 614 (pgd_present(pgd) && (!(write) || pgd_write(pgd))) 615 #endif 616 617 #ifndef __HAVE_ARCH_PMD_SAME 618 static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) 619 { 620 return pmd_val(pmd_a) == pmd_val(pmd_b); 621 } 622 623 static inline int pud_same(pud_t pud_a, pud_t pud_b) 624 { 625 return pud_val(pud_a) == pud_val(pud_b); 626 } 627 #endif 628 629 #ifndef __HAVE_ARCH_P4D_SAME 630 static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b) 631 { 632 return p4d_val(p4d_a) == p4d_val(p4d_b); 633 } 634 #endif 635 636 #ifndef __HAVE_ARCH_PGD_SAME 637 static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b) 638 { 639 return pgd_val(pgd_a) == pgd_val(pgd_b); 640 } 641 #endif 642 643 /* 644 * Use set_p*_safe(), and elide TLB flushing, when confident that *no* 645 * TLB flush will be required as a result of the "set". For example, use 646 * in scenarios where it is known ahead of time that the routine is 647 * setting non-present entries, or re-setting an existing entry to the 648 * same value. Otherwise, use the typical "set" helpers and flush the 649 * TLB. 650 */ 651 #define set_pte_safe(ptep, pte) \ 652 ({ \ 653 WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \ 654 set_pte(ptep, pte); \ 655 }) 656 657 #define set_pmd_safe(pmdp, pmd) \ 658 ({ \ 659 WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \ 660 set_pmd(pmdp, pmd); \ 661 }) 662 663 #define set_pud_safe(pudp, pud) \ 664 ({ \ 665 WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \ 666 set_pud(pudp, pud); \ 667 }) 668 669 #define set_p4d_safe(p4dp, p4d) \ 670 ({ \ 671 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \ 672 set_p4d(p4dp, p4d); \ 673 }) 674 675 #define set_pgd_safe(pgdp, pgd) \ 676 ({ \ 677 WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \ 678 set_pgd(pgdp, pgd); \ 679 }) 680 681 #ifndef __HAVE_ARCH_DO_SWAP_PAGE 682 /* 683 * Some architectures support metadata associated with a page. When a 684 * page is being swapped out, this metadata must be saved so it can be 685 * restored when the page is swapped back in. SPARC M7 and newer 686 * processors support an ADI (Application Data Integrity) tag for the 687 * page as metadata for the page. arch_do_swap_page() can restore this 688 * metadata when a page is swapped back in. 689 */ 690 static inline void arch_do_swap_page(struct mm_struct *mm, 691 struct vm_area_struct *vma, 692 unsigned long addr, 693 pte_t pte, pte_t oldpte) 694 { 695 696 } 697 #endif 698 699 #ifndef __HAVE_ARCH_UNMAP_ONE 700 /* 701 * Some architectures support metadata associated with a page. When a 702 * page is being swapped out, this metadata must be saved so it can be 703 * restored when the page is swapped back in. SPARC M7 and newer 704 * processors support an ADI (Application Data Integrity) tag for the 705 * page as metadata for the page. arch_unmap_one() can save this 706 * metadata on a swap-out of a page. 707 */ 708 static inline int arch_unmap_one(struct mm_struct *mm, 709 struct vm_area_struct *vma, 710 unsigned long addr, 711 pte_t orig_pte) 712 { 713 return 0; 714 } 715 #endif 716 717 /* 718 * Allow architectures to preserve additional metadata associated with 719 * swapped-out pages. The corresponding __HAVE_ARCH_SWAP_* macros and function 720 * prototypes must be defined in the arch-specific asm/pgtable.h file. 721 */ 722 #ifndef __HAVE_ARCH_PREPARE_TO_SWAP 723 static inline int arch_prepare_to_swap(struct page *page) 724 { 725 return 0; 726 } 727 #endif 728 729 #ifndef __HAVE_ARCH_SWAP_INVALIDATE 730 static inline void arch_swap_invalidate_page(int type, pgoff_t offset) 731 { 732 } 733 734 static inline void arch_swap_invalidate_area(int type) 735 { 736 } 737 #endif 738 739 #ifndef __HAVE_ARCH_SWAP_RESTORE 740 static inline void arch_swap_restore(swp_entry_t entry, struct page *page) 741 { 742 } 743 #endif 744 745 #ifndef __HAVE_ARCH_PGD_OFFSET_GATE 746 #define pgd_offset_gate(mm, addr) pgd_offset(mm, addr) 747 #endif 748 749 #ifndef __HAVE_ARCH_MOVE_PTE 750 #define move_pte(pte, prot, old_addr, new_addr) (pte) 751 #endif 752 753 #ifndef pte_accessible 754 # define pte_accessible(mm, pte) ((void)(pte), 1) 755 #endif 756 757 #ifndef flush_tlb_fix_spurious_fault 758 #define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address) 759 #endif 760 761 /* 762 * When walking page tables, get the address of the next boundary, 763 * or the end address of the range if that comes earlier. Although no 764 * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout. 765 */ 766 767 #define pgd_addr_end(addr, end) \ 768 ({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \ 769 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 770 }) 771 772 #ifndef p4d_addr_end 773 #define p4d_addr_end(addr, end) \ 774 ({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \ 775 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 776 }) 777 #endif 778 779 #ifndef pud_addr_end 780 #define pud_addr_end(addr, end) \ 781 ({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \ 782 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 783 }) 784 #endif 785 786 #ifndef pmd_addr_end 787 #define pmd_addr_end(addr, end) \ 788 ({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \ 789 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 790 }) 791 #endif 792 793 /* 794 * When walking page tables, we usually want to skip any p?d_none entries; 795 * and any p?d_bad entries - reporting the error before resetting to none. 796 * Do the tests inline, but report and clear the bad entry in mm/memory.c. 797 */ 798 void pgd_clear_bad(pgd_t *); 799 800 #ifndef __PAGETABLE_P4D_FOLDED 801 void p4d_clear_bad(p4d_t *); 802 #else 803 #define p4d_clear_bad(p4d) do { } while (0) 804 #endif 805 806 #ifndef __PAGETABLE_PUD_FOLDED 807 void pud_clear_bad(pud_t *); 808 #else 809 #define pud_clear_bad(p4d) do { } while (0) 810 #endif 811 812 void pmd_clear_bad(pmd_t *); 813 814 static inline int pgd_none_or_clear_bad(pgd_t *pgd) 815 { 816 if (pgd_none(*pgd)) 817 return 1; 818 if (unlikely(pgd_bad(*pgd))) { 819 pgd_clear_bad(pgd); 820 return 1; 821 } 822 return 0; 823 } 824 825 static inline int p4d_none_or_clear_bad(p4d_t *p4d) 826 { 827 if (p4d_none(*p4d)) 828 return 1; 829 if (unlikely(p4d_bad(*p4d))) { 830 p4d_clear_bad(p4d); 831 return 1; 832 } 833 return 0; 834 } 835 836 static inline int pud_none_or_clear_bad(pud_t *pud) 837 { 838 if (pud_none(*pud)) 839 return 1; 840 if (unlikely(pud_bad(*pud))) { 841 pud_clear_bad(pud); 842 return 1; 843 } 844 return 0; 845 } 846 847 static inline int pmd_none_or_clear_bad(pmd_t *pmd) 848 { 849 if (pmd_none(*pmd)) 850 return 1; 851 if (unlikely(pmd_bad(*pmd))) { 852 pmd_clear_bad(pmd); 853 return 1; 854 } 855 return 0; 856 } 857 858 static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma, 859 unsigned long addr, 860 pte_t *ptep) 861 { 862 /* 863 * Get the current pte state, but zero it out to make it 864 * non-present, preventing the hardware from asynchronously 865 * updating it. 866 */ 867 return ptep_get_and_clear(vma->vm_mm, addr, ptep); 868 } 869 870 static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma, 871 unsigned long addr, 872 pte_t *ptep, pte_t pte) 873 { 874 /* 875 * The pte is non-present, so there's no hardware state to 876 * preserve. 877 */ 878 set_pte_at(vma->vm_mm, addr, ptep, pte); 879 } 880 881 #ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 882 /* 883 * Start a pte protection read-modify-write transaction, which 884 * protects against asynchronous hardware modifications to the pte. 885 * The intention is not to prevent the hardware from making pte 886 * updates, but to prevent any updates it may make from being lost. 887 * 888 * This does not protect against other software modifications of the 889 * pte; the appropriate pte lock must be held over the transaction. 890 * 891 * Note that this interface is intended to be batchable, meaning that 892 * ptep_modify_prot_commit may not actually update the pte, but merely 893 * queue the update to be done at some later time. The update must be 894 * actually committed before the pte lock is released, however. 895 */ 896 static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, 897 unsigned long addr, 898 pte_t *ptep) 899 { 900 return __ptep_modify_prot_start(vma, addr, ptep); 901 } 902 903 /* 904 * Commit an update to a pte, leaving any hardware-controlled bits in 905 * the PTE unmodified. 906 */ 907 static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, 908 unsigned long addr, 909 pte_t *ptep, pte_t old_pte, pte_t pte) 910 { 911 __ptep_modify_prot_commit(vma, addr, ptep, pte); 912 } 913 #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */ 914 #endif /* CONFIG_MMU */ 915 916 /* 917 * No-op macros that just return the current protection value. Defined here 918 * because these macros can be used even if CONFIG_MMU is not defined. 919 */ 920 921 #ifndef pgprot_nx 922 #define pgprot_nx(prot) (prot) 923 #endif 924 925 #ifndef pgprot_noncached 926 #define pgprot_noncached(prot) (prot) 927 #endif 928 929 #ifndef pgprot_writecombine 930 #define pgprot_writecombine pgprot_noncached 931 #endif 932 933 #ifndef pgprot_writethrough 934 #define pgprot_writethrough pgprot_noncached 935 #endif 936 937 #ifndef pgprot_device 938 #define pgprot_device pgprot_noncached 939 #endif 940 941 #ifndef pgprot_mhp 942 #define pgprot_mhp(prot) (prot) 943 #endif 944 945 #ifdef CONFIG_MMU 946 #ifndef pgprot_modify 947 #define pgprot_modify pgprot_modify 948 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 949 { 950 if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot))) 951 newprot = pgprot_noncached(newprot); 952 if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot))) 953 newprot = pgprot_writecombine(newprot); 954 if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot))) 955 newprot = pgprot_device(newprot); 956 return newprot; 957 } 958 #endif 959 #endif /* CONFIG_MMU */ 960 961 #ifndef pgprot_encrypted 962 #define pgprot_encrypted(prot) (prot) 963 #endif 964 965 #ifndef pgprot_decrypted 966 #define pgprot_decrypted(prot) (prot) 967 #endif 968 969 /* 970 * A facility to provide lazy MMU batching. This allows PTE updates and 971 * page invalidations to be delayed until a call to leave lazy MMU mode 972 * is issued. Some architectures may benefit from doing this, and it is 973 * beneficial for both shadow and direct mode hypervisors, which may batch 974 * the PTE updates which happen during this window. Note that using this 975 * interface requires that read hazards be removed from the code. A read 976 * hazard could result in the direct mode hypervisor case, since the actual 977 * write to the page tables may not yet have taken place, so reads though 978 * a raw PTE pointer after it has been modified are not guaranteed to be 979 * up to date. This mode can only be entered and left under the protection of 980 * the page table locks for all page tables which may be modified. In the UP 981 * case, this is required so that preemption is disabled, and in the SMP case, 982 * it must synchronize the delayed page table writes properly on other CPUs. 983 */ 984 #ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE 985 #define arch_enter_lazy_mmu_mode() do {} while (0) 986 #define arch_leave_lazy_mmu_mode() do {} while (0) 987 #define arch_flush_lazy_mmu_mode() do {} while (0) 988 #endif 989 990 /* 991 * A facility to provide batching of the reload of page tables and 992 * other process state with the actual context switch code for 993 * paravirtualized guests. By convention, only one of the batched 994 * update (lazy) modes (CPU, MMU) should be active at any given time, 995 * entry should never be nested, and entry and exits should always be 996 * paired. This is for sanity of maintaining and reasoning about the 997 * kernel code. In this case, the exit (end of the context switch) is 998 * in architecture-specific code, and so doesn't need a generic 999 * definition. 1000 */ 1001 #ifndef __HAVE_ARCH_START_CONTEXT_SWITCH 1002 #define arch_start_context_switch(prev) do {} while (0) 1003 #endif 1004 1005 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY 1006 #ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION 1007 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) 1008 { 1009 return pmd; 1010 } 1011 1012 static inline int pmd_swp_soft_dirty(pmd_t pmd) 1013 { 1014 return 0; 1015 } 1016 1017 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) 1018 { 1019 return pmd; 1020 } 1021 #endif 1022 #else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */ 1023 static inline int pte_soft_dirty(pte_t pte) 1024 { 1025 return 0; 1026 } 1027 1028 static inline int pmd_soft_dirty(pmd_t pmd) 1029 { 1030 return 0; 1031 } 1032 1033 static inline pte_t pte_mksoft_dirty(pte_t pte) 1034 { 1035 return pte; 1036 } 1037 1038 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) 1039 { 1040 return pmd; 1041 } 1042 1043 static inline pte_t pte_clear_soft_dirty(pte_t pte) 1044 { 1045 return pte; 1046 } 1047 1048 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) 1049 { 1050 return pmd; 1051 } 1052 1053 static inline pte_t pte_swp_mksoft_dirty(pte_t pte) 1054 { 1055 return pte; 1056 } 1057 1058 static inline int pte_swp_soft_dirty(pte_t pte) 1059 { 1060 return 0; 1061 } 1062 1063 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) 1064 { 1065 return pte; 1066 } 1067 1068 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) 1069 { 1070 return pmd; 1071 } 1072 1073 static inline int pmd_swp_soft_dirty(pmd_t pmd) 1074 { 1075 return 0; 1076 } 1077 1078 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) 1079 { 1080 return pmd; 1081 } 1082 #endif 1083 1084 #ifndef __HAVE_PFNMAP_TRACKING 1085 /* 1086 * Interfaces that can be used by architecture code to keep track of 1087 * memory type of pfn mappings specified by the remap_pfn_range, 1088 * vmf_insert_pfn. 1089 */ 1090 1091 /* 1092 * track_pfn_remap is called when a _new_ pfn mapping is being established 1093 * by remap_pfn_range() for physical range indicated by pfn and size. 1094 */ 1095 static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, 1096 unsigned long pfn, unsigned long addr, 1097 unsigned long size) 1098 { 1099 return 0; 1100 } 1101 1102 /* 1103 * track_pfn_insert is called when a _new_ single pfn is established 1104 * by vmf_insert_pfn(). 1105 */ 1106 static inline void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, 1107 pfn_t pfn) 1108 { 1109 } 1110 1111 /* 1112 * track_pfn_copy is called when vma that is covering the pfnmap gets 1113 * copied through copy_page_range(). 1114 */ 1115 static inline int track_pfn_copy(struct vm_area_struct *vma) 1116 { 1117 return 0; 1118 } 1119 1120 /* 1121 * untrack_pfn is called while unmapping a pfnmap for a region. 1122 * untrack can be called for a specific region indicated by pfn and size or 1123 * can be for the entire vma (in which case pfn, size are zero). 1124 */ 1125 static inline void untrack_pfn(struct vm_area_struct *vma, 1126 unsigned long pfn, unsigned long size) 1127 { 1128 } 1129 1130 /* 1131 * untrack_pfn_moved is called while mremapping a pfnmap for a new region. 1132 */ 1133 static inline void untrack_pfn_moved(struct vm_area_struct *vma) 1134 { 1135 } 1136 #else 1137 extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, 1138 unsigned long pfn, unsigned long addr, 1139 unsigned long size); 1140 extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, 1141 pfn_t pfn); 1142 extern int track_pfn_copy(struct vm_area_struct *vma); 1143 extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn, 1144 unsigned long size); 1145 extern void untrack_pfn_moved(struct vm_area_struct *vma); 1146 #endif 1147 1148 #ifdef CONFIG_MMU 1149 #ifdef __HAVE_COLOR_ZERO_PAGE 1150 static inline int is_zero_pfn(unsigned long pfn) 1151 { 1152 extern unsigned long zero_pfn; 1153 unsigned long offset_from_zero_pfn = pfn - zero_pfn; 1154 return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT); 1155 } 1156 1157 #define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) 1158 1159 #else 1160 static inline int is_zero_pfn(unsigned long pfn) 1161 { 1162 extern unsigned long zero_pfn; 1163 return pfn == zero_pfn; 1164 } 1165 1166 static inline unsigned long my_zero_pfn(unsigned long addr) 1167 { 1168 extern unsigned long zero_pfn; 1169 return zero_pfn; 1170 } 1171 #endif 1172 #else 1173 static inline int is_zero_pfn(unsigned long pfn) 1174 { 1175 return 0; 1176 } 1177 1178 static inline unsigned long my_zero_pfn(unsigned long addr) 1179 { 1180 return 0; 1181 } 1182 #endif /* CONFIG_MMU */ 1183 1184 #ifdef CONFIG_MMU 1185 1186 #ifndef CONFIG_TRANSPARENT_HUGEPAGE 1187 static inline int pmd_trans_huge(pmd_t pmd) 1188 { 1189 return 0; 1190 } 1191 #ifndef pmd_write 1192 static inline int pmd_write(pmd_t pmd) 1193 { 1194 BUG(); 1195 return 0; 1196 } 1197 #endif /* pmd_write */ 1198 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1199 1200 #ifndef pud_write 1201 static inline int pud_write(pud_t pud) 1202 { 1203 BUG(); 1204 return 0; 1205 } 1206 #endif /* pud_write */ 1207 1208 #if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE) 1209 static inline int pmd_devmap(pmd_t pmd) 1210 { 1211 return 0; 1212 } 1213 static inline int pud_devmap(pud_t pud) 1214 { 1215 return 0; 1216 } 1217 static inline int pgd_devmap(pgd_t pgd) 1218 { 1219 return 0; 1220 } 1221 #endif 1222 1223 #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ 1224 (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1225 !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) 1226 static inline int pud_trans_huge(pud_t pud) 1227 { 1228 return 0; 1229 } 1230 #endif 1231 1232 /* See pmd_none_or_trans_huge_or_clear_bad for discussion. */ 1233 static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud) 1234 { 1235 pud_t pudval = READ_ONCE(*pud); 1236 1237 if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval)) 1238 return 1; 1239 if (unlikely(pud_bad(pudval))) { 1240 pud_clear_bad(pud); 1241 return 1; 1242 } 1243 return 0; 1244 } 1245 1246 /* See pmd_trans_unstable for discussion. */ 1247 static inline int pud_trans_unstable(pud_t *pud) 1248 { 1249 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1250 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 1251 return pud_none_or_trans_huge_or_dev_or_clear_bad(pud); 1252 #else 1253 return 0; 1254 #endif 1255 } 1256 1257 #ifndef pmd_read_atomic 1258 static inline pmd_t pmd_read_atomic(pmd_t *pmdp) 1259 { 1260 /* 1261 * Depend on compiler for an atomic pmd read. NOTE: this is 1262 * only going to work, if the pmdval_t isn't larger than 1263 * an unsigned long. 1264 */ 1265 return *pmdp; 1266 } 1267 #endif 1268 1269 #ifndef arch_needs_pgtable_deposit 1270 #define arch_needs_pgtable_deposit() (false) 1271 #endif 1272 /* 1273 * This function is meant to be used by sites walking pagetables with 1274 * the mmap_lock held in read mode to protect against MADV_DONTNEED and 1275 * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd 1276 * into a null pmd and the transhuge page fault can convert a null pmd 1277 * into an hugepmd or into a regular pmd (if the hugepage allocation 1278 * fails). While holding the mmap_lock in read mode the pmd becomes 1279 * stable and stops changing under us only if it's not null and not a 1280 * transhuge pmd. When those races occurs and this function makes a 1281 * difference vs the standard pmd_none_or_clear_bad, the result is 1282 * undefined so behaving like if the pmd was none is safe (because it 1283 * can return none anyway). The compiler level barrier() is critically 1284 * important to compute the two checks atomically on the same pmdval. 1285 * 1286 * For 32bit kernels with a 64bit large pmd_t this automatically takes 1287 * care of reading the pmd atomically to avoid SMP race conditions 1288 * against pmd_populate() when the mmap_lock is hold for reading by the 1289 * caller (a special atomic read not done by "gcc" as in the generic 1290 * version above, is also needed when THP is disabled because the page 1291 * fault can populate the pmd from under us). 1292 */ 1293 static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd) 1294 { 1295 pmd_t pmdval = pmd_read_atomic(pmd); 1296 /* 1297 * The barrier will stabilize the pmdval in a register or on 1298 * the stack so that it will stop changing under the code. 1299 * 1300 * When CONFIG_TRANSPARENT_HUGEPAGE=y on x86 32bit PAE, 1301 * pmd_read_atomic is allowed to return a not atomic pmdval 1302 * (for example pointing to an hugepage that has never been 1303 * mapped in the pmd). The below checks will only care about 1304 * the low part of the pmd with 32bit PAE x86 anyway, with the 1305 * exception of pmd_none(). So the important thing is that if 1306 * the low part of the pmd is found null, the high part will 1307 * be also null or the pmd_none() check below would be 1308 * confused. 1309 */ 1310 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1311 barrier(); 1312 #endif 1313 /* 1314 * !pmd_present() checks for pmd migration entries 1315 * 1316 * The complete check uses is_pmd_migration_entry() in linux/swapops.h 1317 * But using that requires moving current function and pmd_trans_unstable() 1318 * to linux/swapops.h to resolve dependency, which is too much code move. 1319 * 1320 * !pmd_present() is equivalent to is_pmd_migration_entry() currently, 1321 * because !pmd_present() pages can only be under migration not swapped 1322 * out. 1323 * 1324 * pmd_none() is preserved for future condition checks on pmd migration 1325 * entries and not confusing with this function name, although it is 1326 * redundant with !pmd_present(). 1327 */ 1328 if (pmd_none(pmdval) || pmd_trans_huge(pmdval) || 1329 (IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION) && !pmd_present(pmdval))) 1330 return 1; 1331 if (unlikely(pmd_bad(pmdval))) { 1332 pmd_clear_bad(pmd); 1333 return 1; 1334 } 1335 return 0; 1336 } 1337 1338 /* 1339 * This is a noop if Transparent Hugepage Support is not built into 1340 * the kernel. Otherwise it is equivalent to 1341 * pmd_none_or_trans_huge_or_clear_bad(), and shall only be called in 1342 * places that already verified the pmd is not none and they want to 1343 * walk ptes while holding the mmap sem in read mode (write mode don't 1344 * need this). If THP is not enabled, the pmd can't go away under the 1345 * code even if MADV_DONTNEED runs, but if THP is enabled we need to 1346 * run a pmd_trans_unstable before walking the ptes after 1347 * split_huge_pmd returns (because it may have run when the pmd become 1348 * null, but then a page fault can map in a THP and not a regular page). 1349 */ 1350 static inline int pmd_trans_unstable(pmd_t *pmd) 1351 { 1352 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1353 return pmd_none_or_trans_huge_or_clear_bad(pmd); 1354 #else 1355 return 0; 1356 #endif 1357 } 1358 1359 /* 1360 * the ordering of these checks is important for pmds with _page_devmap set. 1361 * if we check pmd_trans_unstable() first we will trip the bad_pmd() check 1362 * inside of pmd_none_or_trans_huge_or_clear_bad(). this will end up correctly 1363 * returning 1 but not before it spams dmesg with the pmd_clear_bad() output. 1364 */ 1365 static inline int pmd_devmap_trans_unstable(pmd_t *pmd) 1366 { 1367 return pmd_devmap(*pmd) || pmd_trans_unstable(pmd); 1368 } 1369 1370 #ifndef CONFIG_NUMA_BALANCING 1371 /* 1372 * Technically a PTE can be PROTNONE even when not doing NUMA balancing but 1373 * the only case the kernel cares is for NUMA balancing and is only ever set 1374 * when the VMA is accessible. For PROT_NONE VMAs, the PTEs are not marked 1375 * _PAGE_PROTNONE so by default, implement the helper as "always no". It 1376 * is the responsibility of the caller to distinguish between PROT_NONE 1377 * protections and NUMA hinting fault protections. 1378 */ 1379 static inline int pte_protnone(pte_t pte) 1380 { 1381 return 0; 1382 } 1383 1384 static inline int pmd_protnone(pmd_t pmd) 1385 { 1386 return 0; 1387 } 1388 #endif /* CONFIG_NUMA_BALANCING */ 1389 1390 #endif /* CONFIG_MMU */ 1391 1392 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 1393 1394 #ifndef __PAGETABLE_P4D_FOLDED 1395 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot); 1396 int p4d_clear_huge(p4d_t *p4d); 1397 #else 1398 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 1399 { 1400 return 0; 1401 } 1402 static inline int p4d_clear_huge(p4d_t *p4d) 1403 { 1404 return 0; 1405 } 1406 #endif /* !__PAGETABLE_P4D_FOLDED */ 1407 1408 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot); 1409 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot); 1410 int pud_clear_huge(pud_t *pud); 1411 int pmd_clear_huge(pmd_t *pmd); 1412 int p4d_free_pud_page(p4d_t *p4d, unsigned long addr); 1413 int pud_free_pmd_page(pud_t *pud, unsigned long addr); 1414 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr); 1415 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 1416 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 1417 { 1418 return 0; 1419 } 1420 static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) 1421 { 1422 return 0; 1423 } 1424 static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) 1425 { 1426 return 0; 1427 } 1428 static inline int p4d_clear_huge(p4d_t *p4d) 1429 { 1430 return 0; 1431 } 1432 static inline int pud_clear_huge(pud_t *pud) 1433 { 1434 return 0; 1435 } 1436 static inline int pmd_clear_huge(pmd_t *pmd) 1437 { 1438 return 0; 1439 } 1440 static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) 1441 { 1442 return 0; 1443 } 1444 static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr) 1445 { 1446 return 0; 1447 } 1448 static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) 1449 { 1450 return 0; 1451 } 1452 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 1453 1454 #ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE 1455 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1456 /* 1457 * ARCHes with special requirements for evicting THP backing TLB entries can 1458 * implement this. Otherwise also, it can help optimize normal TLB flush in 1459 * THP regime. Stock flush_tlb_range() typically has optimization to nuke the 1460 * entire TLB if flush span is greater than a threshold, which will 1461 * likely be true for a single huge page. Thus a single THP flush will 1462 * invalidate the entire TLB which is not desirable. 1463 * e.g. see arch/arc: flush_pmd_tlb_range 1464 */ 1465 #define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 1466 #define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 1467 #else 1468 #define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG() 1469 #define flush_pud_tlb_range(vma, addr, end) BUILD_BUG() 1470 #endif 1471 #endif 1472 1473 struct file; 1474 int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, 1475 unsigned long size, pgprot_t *vma_prot); 1476 1477 #ifndef CONFIG_X86_ESPFIX64 1478 static inline void init_espfix_bsp(void) { } 1479 #endif 1480 1481 extern void __init pgtable_cache_init(void); 1482 1483 #ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED 1484 static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot) 1485 { 1486 return true; 1487 } 1488 1489 static inline bool arch_has_pfn_modify_check(void) 1490 { 1491 return false; 1492 } 1493 #endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */ 1494 1495 /* 1496 * Architecture PAGE_KERNEL_* fallbacks 1497 * 1498 * Some architectures don't define certain PAGE_KERNEL_* flags. This is either 1499 * because they really don't support them, or the port needs to be updated to 1500 * reflect the required functionality. Below are a set of relatively safe 1501 * fallbacks, as best effort, which we can count on in lieu of the architectures 1502 * not defining them on their own yet. 1503 */ 1504 1505 #ifndef PAGE_KERNEL_RO 1506 # define PAGE_KERNEL_RO PAGE_KERNEL 1507 #endif 1508 1509 #ifndef PAGE_KERNEL_EXEC 1510 # define PAGE_KERNEL_EXEC PAGE_KERNEL 1511 #endif 1512 1513 /* 1514 * Page Table Modification bits for pgtbl_mod_mask. 1515 * 1516 * These are used by the p?d_alloc_track*() set of functions an in the generic 1517 * vmalloc/ioremap code to track at which page-table levels entries have been 1518 * modified. Based on that the code can better decide when vmalloc and ioremap 1519 * mapping changes need to be synchronized to other page-tables in the system. 1520 */ 1521 #define __PGTBL_PGD_MODIFIED 0 1522 #define __PGTBL_P4D_MODIFIED 1 1523 #define __PGTBL_PUD_MODIFIED 2 1524 #define __PGTBL_PMD_MODIFIED 3 1525 #define __PGTBL_PTE_MODIFIED 4 1526 1527 #define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED) 1528 #define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED) 1529 #define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED) 1530 #define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED) 1531 #define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED) 1532 1533 /* Page-Table Modification Mask */ 1534 typedef unsigned int pgtbl_mod_mask; 1535 1536 #endif /* !__ASSEMBLY__ */ 1537 1538 #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) 1539 #ifdef CONFIG_PHYS_ADDR_T_64BIT 1540 /* 1541 * ZSMALLOC needs to know the highest PFN on 32-bit architectures 1542 * with physical address space extension, but falls back to 1543 * BITS_PER_LONG otherwise. 1544 */ 1545 #error Missing MAX_POSSIBLE_PHYSMEM_BITS definition 1546 #else 1547 #define MAX_POSSIBLE_PHYSMEM_BITS 32 1548 #endif 1549 #endif 1550 1551 #ifndef has_transparent_hugepage 1552 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1553 #define has_transparent_hugepage() 1 1554 #else 1555 #define has_transparent_hugepage() 0 1556 #endif 1557 #endif 1558 1559 /* 1560 * On some architectures it depends on the mm if the p4d/pud or pmd 1561 * layer of the page table hierarchy is folded or not. 1562 */ 1563 #ifndef mm_p4d_folded 1564 #define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED) 1565 #endif 1566 1567 #ifndef mm_pud_folded 1568 #define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED) 1569 #endif 1570 1571 #ifndef mm_pmd_folded 1572 #define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED) 1573 #endif 1574 1575 #ifndef p4d_offset_lockless 1576 #define p4d_offset_lockless(pgdp, pgd, address) p4d_offset(&(pgd), address) 1577 #endif 1578 #ifndef pud_offset_lockless 1579 #define pud_offset_lockless(p4dp, p4d, address) pud_offset(&(p4d), address) 1580 #endif 1581 #ifndef pmd_offset_lockless 1582 #define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address) 1583 #endif 1584 1585 /* 1586 * p?d_leaf() - true if this entry is a final mapping to a physical address. 1587 * This differs from p?d_huge() by the fact that they are always available (if 1588 * the architecture supports large pages at the appropriate level) even 1589 * if CONFIG_HUGETLB_PAGE is not defined. 1590 * Only meaningful when called on a valid entry. 1591 */ 1592 #ifndef pgd_leaf 1593 #define pgd_leaf(x) 0 1594 #endif 1595 #ifndef p4d_leaf 1596 #define p4d_leaf(x) 0 1597 #endif 1598 #ifndef pud_leaf 1599 #define pud_leaf(x) 0 1600 #endif 1601 #ifndef pmd_leaf 1602 #define pmd_leaf(x) 0 1603 #endif 1604 1605 #ifndef pgd_leaf_size 1606 #define pgd_leaf_size(x) (1ULL << PGDIR_SHIFT) 1607 #endif 1608 #ifndef p4d_leaf_size 1609 #define p4d_leaf_size(x) P4D_SIZE 1610 #endif 1611 #ifndef pud_leaf_size 1612 #define pud_leaf_size(x) PUD_SIZE 1613 #endif 1614 #ifndef pmd_leaf_size 1615 #define pmd_leaf_size(x) PMD_SIZE 1616 #endif 1617 #ifndef pte_leaf_size 1618 #define pte_leaf_size(x) PAGE_SIZE 1619 #endif 1620 1621 /* 1622 * Some architectures have MMUs that are configurable or selectable at boot 1623 * time. These lead to variable PTRS_PER_x. For statically allocated arrays it 1624 * helps to have a static maximum value. 1625 */ 1626 1627 #ifndef MAX_PTRS_PER_PTE 1628 #define MAX_PTRS_PER_PTE PTRS_PER_PTE 1629 #endif 1630 1631 #ifndef MAX_PTRS_PER_PMD 1632 #define MAX_PTRS_PER_PMD PTRS_PER_PMD 1633 #endif 1634 1635 #ifndef MAX_PTRS_PER_PUD 1636 #define MAX_PTRS_PER_PUD PTRS_PER_PUD 1637 #endif 1638 1639 #ifndef MAX_PTRS_PER_P4D 1640 #define MAX_PTRS_PER_P4D PTRS_PER_P4D 1641 #endif 1642 1643 #endif /* _LINUX_PGTABLE_H */ 1644