59 #define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0)  argument
99 #define __pte_clear(mm, addr, ptep) \  argument
100 				__set_pte(ptep, __pte(0))
318 static inline void __set_pte_nosync(pte_t *ptep, pte_t pte)  in __set_pte_nosync()  argument
320 	WRITE_ONCE(*ptep, pte);  in __set_pte_nosync()
323 static inline void __set_pte(pte_t *ptep, pte_t pte)  in __set_pte()  argument
325 	__set_pte_nosync(ptep, pte);  in __set_pte()
337 static inline pte_t __ptep_get(pte_t *ptep)  in __ptep_get()  argument
339 	return READ_ONCE(*ptep);  in __ptep_get()
361 static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep,  in __check_safe_pte_update()  argument
369 	old_pte = __ptep_get(ptep);  in __check_safe_pte_update()
428 			      pte_t *ptep, pte_t pte, unsigned int nr)  in __set_ptes()  argument
430 	page_table_check_ptes_set(mm, ptep, pte, nr);  in __set_ptes()
434 		__check_safe_pte_update(mm, ptep, pte);  in __set_ptes()
435 		__set_pte(ptep, pte);  in __set_ptes()
438 		ptep++;  in __set_ptes()
654 				pte_t *ptep, pte_t pte, unsigned int nr)  in __set_pte_at()  argument
657 	__check_safe_pte_update(mm, ptep, pte);  in __set_pte_at()
658 	__set_pte(ptep, pte);  in __set_pte_at()
1211 				 unsigned long address, pte_t *ptep,
1257 					      pte_t *ptep)  in __ptep_test_and_clear_young()  argument
1261 	pte = __ptep_get(ptep);  in __ptep_test_and_clear_young()
1265 		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),  in __ptep_test_and_clear_young()
1273 					 unsigned long address, pte_t *ptep)  in __ptep_clear_flush_young()  argument
1275 	int young = __ptep_test_and_clear_young(vma, address, ptep);  in __ptep_clear_flush_young()
1305 				       unsigned long address, pte_t *ptep)  in __ptep_get_and_clear()  argument
1307 	pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0));  in __ptep_get_and_clear()
1315 				pte_t *ptep, unsigned int nr, int full)  in __clear_full_ptes()  argument
1318 		__ptep_get_and_clear(mm, addr, ptep);  in __clear_full_ptes()
1321 		ptep++;  in __clear_full_ptes()
1327 				unsigned long addr, pte_t *ptep,  in __get_and_clear_full_ptes()  argument
1332 	pte = __ptep_get_and_clear(mm, addr, ptep);  in __get_and_clear_full_ptes()
1334 		ptep++;  in __get_and_clear_full_ptes()
1336 		tmp_pte = __ptep_get_and_clear(mm, addr, ptep);  in __get_and_clear_full_ptes()
1359 					unsigned long address, pte_t *ptep,  in ___ptep_set_wrprotect()  argument
1367 		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),  in ___ptep_set_wrprotect()
1377 					unsigned long address, pte_t *ptep)  in __ptep_set_wrprotect()  argument
1379 	___ptep_set_wrprotect(mm, address, ptep, __ptep_get(ptep));  in __ptep_set_wrprotect()
1383 				pte_t *ptep, unsigned int nr)  in __wrprotect_ptes()  argument
1387 	for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++)  in __wrprotect_ptes()
1388 		__ptep_set_wrprotect(mm, address, ptep);  in __wrprotect_ptes()
1392 					   unsigned long addr, pte_t *ptep,  in __clear_young_dirty_pte()  argument
1405 		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),  in __clear_young_dirty_pte()
1411 					    unsigned long addr, pte_t *ptep,  in __clear_young_dirty_ptes()  argument
1417 		pte = __ptep_get(ptep);  in __clear_young_dirty_ptes()
1420 			__set_pte(ptep, pte_mkclean(pte_mkold(pte)));  in __clear_young_dirty_ptes()
1422 			__clear_young_dirty_pte(vma, addr, ptep, pte, flags);  in __clear_young_dirty_ptes()
1426 		ptep++;  in __clear_young_dirty_ptes()
1509 		struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,  in update_mmu_cache_range()  argument
1519 #define update_mmu_cache(vma, addr, ptep) \  argument
1520 	update_mmu_cache_range(NULL, vma, addr, ptep, 1)
1556 				    unsigned long addr, pte_t *ptep);
1560 				    unsigned long addr, pte_t *ptep,
1571 				pte_t *ptep, pte_t pte);
1573 				pte_t *ptep, pte_t pte);
1574 extern pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte);
1577 				pte_t *ptep, pte_t pte, unsigned int nr);
1579 				pte_t *ptep, unsigned int nr, int full);
1581 				unsigned long addr, pte_t *ptep,
1584 				unsigned long addr, pte_t *ptep);
1586 				unsigned long addr, pte_t *ptep);
1588 				pte_t *ptep, unsigned int nr);
1590 				unsigned long addr, pte_t *ptep,
1593 				unsigned long addr, pte_t *ptep,
1597 				unsigned long addr, pte_t *ptep, pte_t pte)  in contpte_try_fold()  argument
1615 			__contpte_try_fold(mm, addr, ptep, pte);  in contpte_try_fold()
1620 				unsigned long addr, pte_t *ptep, pte_t pte)  in contpte_try_unfold()  argument
1623 		__contpte_try_unfold(mm, addr, ptep, pte);  in contpte_try_unfold()
1627 static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte)  in pte_batch_hint()  argument
1632 	return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1));  in pte_batch_hint()
1654 static inline pte_t ptep_get(pte_t *ptep)  in ptep_get()  argument
1656 	pte_t pte = __ptep_get(ptep);  in ptep_get()
1661 	return contpte_ptep_get(ptep, pte);  in ptep_get()
1665 static inline pte_t ptep_get_lockless(pte_t *ptep)  in ptep_get_lockless()  argument
1667 	pte_t pte = __ptep_get(ptep);  in ptep_get_lockless()
1672 	return contpte_ptep_get_lockless(ptep);  in ptep_get_lockless()
1675 static inline void set_pte(pte_t *ptep, pte_t pte)  in set_pte()  argument
1684 	pte_t orig_pte = __ptep_get(ptep);  in set_pte()
1687 	__set_pte(ptep, pte_mknoncont(pte));  in set_pte()
1692 				pte_t *ptep, pte_t pte, unsigned int nr)  in set_ptes()  argument
1697 		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));  in set_ptes()
1698 		__set_ptes(mm, addr, ptep, pte, 1);  in set_ptes()
1699 		contpte_try_fold(mm, addr, ptep, pte);  in set_ptes()
1701 		contpte_set_ptes(mm, addr, ptep, pte, nr);  in set_ptes()
1706 				unsigned long addr, pte_t *ptep)  in pte_clear()  argument
1708 	contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));  in pte_clear()
1709 	__pte_clear(mm, addr, ptep);  in pte_clear()
1714 				pte_t *ptep, unsigned int nr, int full)  in clear_full_ptes()  argument
1717 		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));  in clear_full_ptes()
1718 		__clear_full_ptes(mm, addr, ptep, nr, full);  in clear_full_ptes()
1720 		contpte_clear_full_ptes(mm, addr, ptep, nr, full);  in clear_full_ptes()
1726 				unsigned long addr, pte_t *ptep,  in get_and_clear_full_ptes()  argument
1732 		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));  in get_and_clear_full_ptes()
1733 		pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full);  in get_and_clear_full_ptes()
1735 		pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full);  in get_and_clear_full_ptes()
1743 				unsigned long addr, pte_t *ptep)  in ptep_get_and_clear()  argument
1745 	contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));  in ptep_get_and_clear()
1746 	return __ptep_get_and_clear(mm, addr, ptep);  in ptep_get_and_clear()
1751 				unsigned long addr, pte_t *ptep)  in ptep_test_and_clear_young()  argument
1753 	pte_t orig_pte = __ptep_get(ptep);  in ptep_test_and_clear_young()
1756 		return __ptep_test_and_clear_young(vma, addr, ptep);  in ptep_test_and_clear_young()
1758 	return contpte_ptep_test_and_clear_young(vma, addr, ptep);  in ptep_test_and_clear_young()
1763 				unsigned long addr, pte_t *ptep)  in ptep_clear_flush_young()  argument
1765 	pte_t orig_pte = __ptep_get(ptep);  in ptep_clear_flush_young()
1768 		return __ptep_clear_flush_young(vma, addr, ptep);  in ptep_clear_flush_young()
1770 	return contpte_ptep_clear_flush_young(vma, addr, ptep);  in ptep_clear_flush_young()
1775 				unsigned long addr, pte_t *ptep, unsigned int nr)  in wrprotect_ptes()  argument
1785 		pte_t orig_pte = __ptep_get(ptep);  in wrprotect_ptes()
1788 			__contpte_try_unfold(mm, addr, ptep, orig_pte);  in wrprotect_ptes()
1791 		___ptep_set_wrprotect(mm, addr, ptep, orig_pte);  in wrprotect_ptes()
1793 		contpte_wrprotect_ptes(mm, addr, ptep, nr);  in wrprotect_ptes()
1799 				unsigned long addr, pte_t *ptep)  in ptep_set_wrprotect()  argument
1801 	wrprotect_ptes(mm, addr, ptep, 1);  in ptep_set_wrprotect()
1806 				unsigned long addr, pte_t *ptep,  in ptep_set_access_flags()  argument
1809 	pte_t orig_pte = __ptep_get(ptep);  in ptep_set_access_flags()
1814 		return __ptep_set_access_flags(vma, addr, ptep, entry, dirty);  in ptep_set_access_flags()
1816 	return contpte_ptep_set_access_flags(vma, addr, ptep, entry, dirty);  in ptep_set_access_flags()
1821 					  unsigned long addr, pte_t *ptep,  in clear_young_dirty_ptes()  argument
1824 	if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))  in clear_young_dirty_ptes()
1825 		__clear_young_dirty_ptes(vma, addr, ptep, nr, flags);  in clear_young_dirty_ptes()
1827 		contpte_clear_young_dirty_ptes(vma, addr, ptep, nr, flags);  in clear_young_dirty_ptes()