79 void num_poisoned_pages_inc(unsigned long pfn)  in num_poisoned_pages_inc()  argument
82 	memblk_nr_poison_inc(pfn);  in num_poisoned_pages_inc()
85 void num_poisoned_pages_sub(unsigned long pfn, long i)  in num_poisoned_pages_sub()  argument
88 	if (pfn != -1UL)  in num_poisoned_pages_sub()
89 		memblk_nr_poison_sub(pfn, i);  in num_poisoned_pages_sub()
350 static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags)  in kill_proc()  argument
357 			pfn, t->comm, task_pid_nr(t));  in kill_proc()
527 		unsigned long pfn, int flags)  in kill_procs()  argument
535 				       pfn, tk->tsk->comm, task_pid_nr(tk->tsk));  in kill_procs()
546 			else if (kill_proc(tk, pfn, flags) < 0)  in kill_procs()
548 				       pfn, tk->tsk->comm, task_pid_nr(tk->tsk));  in kill_procs()
745 	unsigned long pfn;  member
758 	unsigned long pfn = 0;  in check_hwpoisoned_entry()  local
761 		pfn = pte_pfn(pte);  in check_hwpoisoned_entry()
766 			pfn = swp_offset_pfn(swp);  in check_hwpoisoned_entry()
769 	if (!pfn || pfn != poisoned_pfn)  in check_hwpoisoned_entry()
781 	unsigned long pfn;  in check_hwpoisoned_pmd_entry()  local
786 	pfn = pmd_pfn(pmd);  in check_hwpoisoned_pmd_entry()
787 	if (pfn <= hwp->pfn && hwp->pfn < pfn + HPAGE_PMD_NR) {  in check_hwpoisoned_pmd_entry()
788 		hwpoison_vaddr = addr + ((hwp->pfn - pfn) << PAGE_SHIFT);  in check_hwpoisoned_pmd_entry()
824 					     hwp->pfn, &hwp->tk);  in hwpoison_pte_range()
844 				      hwp->pfn, &hwp->tk);  in hwpoison_hugetlb_range()
869 static int kill_accessing_process(struct task_struct *p, unsigned long pfn,  in kill_accessing_process()  argument
874 		.pfn = pfn,  in kill_accessing_process()
891 		kill_proc(&priv.tk, pfn, flags);  in kill_accessing_process()
980 static int truncate_error_folio(struct folio *folio, unsigned long pfn,  in truncate_error_folio()  argument
989 			pr_info("%#lx: Failed to punch page: %d\n", pfn, err);  in truncate_error_folio()
991 			pr_info("%#lx: failed to release buffers\n", pfn);  in truncate_error_folio()
1002 			pr_info("%#lx: Failed to invalidate\n",	pfn);  in truncate_error_folio()
1296 static void update_per_node_mf_stats(unsigned long pfn,  in update_per_node_mf_stats()  argument
1302 	nid = pfn_to_nid(pfn);  in update_per_node_mf_stats()
1304 		WARN_ONCE(1, "Memory failure: pfn=%#lx, invalid nid=%d", pfn, nid);  in update_per_node_mf_stats()
1333 static int action_result(unsigned long pfn, enum mf_action_page_type type,  in action_result()  argument
1336 	trace_memory_failure_event(pfn, type, result);  in action_result()
1338 	num_poisoned_pages_inc(pfn);  in action_result()
1340 	update_per_node_mf_stats(pfn, result);  in action_result()
1343 		pfn, action_page_types[type], action_name[result]);  in action_result()
1349 			unsigned long pfn)  in page_action()  argument
1361 	return action_result(pfn, ps->type, result);  in page_action()
1564 int unmap_poisoned_folio(struct folio *folio, unsigned long pfn, bool must_kill)  in unmap_poisoned_folio()  argument
1570 		pr_err("%#lx: keeping poisoned page in swap cache\n", pfn);  in unmap_poisoned_folio()
1588 				pfn);  in unmap_poisoned_folio()
1621 		unsigned long pfn, int flags)  in hwpoison_user_mappings()  argument
1652 	unmap_success = !unmap_poisoned_folio(folio, pfn, flags & MF_MUST_KILL);  in hwpoison_user_mappings()
1655 		       pfn, folio_mapcount(folio));  in hwpoison_user_mappings()
1676 	kill_procs(&tokill, forcekill, pfn, flags);  in hwpoison_user_mappings()
1681 static int identify_page_state(unsigned long pfn, struct page *p,  in identify_page_state()  argument
1701 	return page_action(ps, p, pfn);  in identify_page_state()
1723 static void unmap_and_kill(struct list_head *to_kill, unsigned long pfn,  in unmap_and_kill()  argument
1745 	kill_procs(to_kill, flags & MF_MUST_KILL, pfn, flags);  in unmap_and_kill()
1757 static int mf_generic_kill_procs(unsigned long long pfn, int flags,  in mf_generic_kill_procs()  argument
1760 	struct folio *folio = pfn_folio(pfn);  in mf_generic_kill_procs()
1809 	unmap_and_kill(&to_kill, pfn, folio->mapping, folio->index, flags);  in mf_generic_kill_procs()
2011 int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,  in __get_huge_page_for_hwpoison()  argument
2014 	struct page *page = pfn_to_page(pfn);  in __get_huge_page_for_hwpoison()
2064 static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb)  in try_memory_failure_hugetlb()  argument
2067 	struct page *p = pfn_to_page(pfn);  in try_memory_failure_hugetlb()
2074 	res = get_huge_page_for_hwpoison(pfn, flags, &migratable_cleared);  in try_memory_failure_hugetlb()
2079 		pr_err("%#lx: already hardware poisoned\n", pfn);  in try_memory_failure_hugetlb()
2083 			action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED);  in try_memory_failure_hugetlb()
2091 		return action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED);  in try_memory_failure_hugetlb()
2119 		return action_result(pfn, MF_MSG_FREE_HUGE, res);  in try_memory_failure_hugetlb()
2124 	if (!hwpoison_user_mappings(folio, p, pfn, flags)) {  in try_memory_failure_hugetlb()
2126 		return action_result(pfn, MF_MSG_UNMAP_FAILED, MF_FAILED);  in try_memory_failure_hugetlb()
2129 	return identify_page_state(pfn, p, page_flags);  in try_memory_failure_hugetlb()
2133 static inline int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb)  in try_memory_failure_hugetlb()  argument
2145 static void put_ref_page(unsigned long pfn, int flags)  in put_ref_page()  argument
2150 	put_page(pfn_to_page(pfn));  in put_ref_page()
2153 static int memory_failure_dev_pagemap(unsigned long pfn, int flags,  in memory_failure_dev_pagemap()  argument
2159 	if (!pgmap_pfn_valid(pgmap, pfn))  in memory_failure_dev_pagemap()
2167 		rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags);  in memory_failure_dev_pagemap()
2176 	rc = mf_generic_kill_procs(pfn, flags, pgmap);  in memory_failure_dev_pagemap()
2181 		action_result(pfn, MF_MSG_DAX, rc ? MF_FAILED : MF_RECOVERED);  in memory_failure_dev_pagemap()
2192 static void kill_procs_now(struct page *p, unsigned long pfn, int flags,  in kill_procs_now()  argument
2198 	kill_procs(&tokill, true, pfn, flags);  in kill_procs_now()
2226 int memory_failure(unsigned long pfn, int flags)  in memory_failure()  argument
2237 		panic("Memory failure on page %lx", pfn);  in memory_failure()
2244 	p = pfn_to_online_page(pfn);  in memory_failure()
2246 		res = arch_memory_failure(pfn, flags);  in memory_failure()
2250 		if (pfn_valid(pfn)) {  in memory_failure()
2251 			pgmap = get_dev_pagemap(pfn, NULL);  in memory_failure()
2252 			put_ref_page(pfn, flags);  in memory_failure()
2254 				res = memory_failure_dev_pagemap(pfn, flags,  in memory_failure()
2259 		pr_err("%#lx: memory outside kernel control\n", pfn);  in memory_failure()
2265 	res = try_memory_failure_hugetlb(pfn, flags, &hugetlb);  in memory_failure()
2270 		pr_err("%#lx: already hardware poisoned\n", pfn);  in memory_failure()
2273 			res = kill_accessing_process(current, pfn, flags);  in memory_failure()
2276 		action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED);  in memory_failure()
2307 				res = action_result(pfn, MF_MSG_BUDDY, res);  in memory_failure()
2309 				res = action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED);  in memory_failure()
2313 			res = action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED);  in memory_failure()
2348 			kill_procs_now(p, pfn, flags, folio);  in memory_failure()
2350 			action_result(pfn, MF_MSG_UNSPLIT_THP, MF_FAILED);  in memory_failure()
2403 	if (!hwpoison_user_mappings(folio, p, pfn, flags)) {  in memory_failure()
2404 		res = action_result(pfn, MF_MSG_UNMAP_FAILED, MF_FAILED);  in memory_failure()
2413 		res = action_result(pfn, MF_MSG_TRUNCATED_LRU, MF_IGNORED);  in memory_failure()
2418 	res = identify_page_state(pfn, p, page_flags);  in memory_failure()
2433 	unsigned long pfn;  member
2462 void memory_failure_queue(unsigned long pfn, int flags)  in memory_failure_queue()  argument
2468 		.pfn =		pfn,  in memory_failure_queue()
2481 		       pfn);  in memory_failure_queue()
2500 			soft_offline_page(entry.pfn, entry.flags);  in memory_failure_work_func()
2502 			memory_failure(entry.pfn, entry.flags);  in memory_failure_work_func()
2539 #define unpoison_pr_info(fmt, pfn, rs)			\  argument
2542 		pr_info(fmt, pfn);			\
2557 int unpoison_memory(unsigned long pfn)  in unpoison_memory()  argument
2567 	if (!pfn_valid(pfn))  in unpoison_memory()
2570 	p = pfn_to_page(pfn);  in unpoison_memory()
2577 				 pfn, &unpoison_rs);  in unpoison_memory()
2584 				 pfn, &unpoison_rs);  in unpoison_memory()
2591 				 pfn, &unpoison_rs);  in unpoison_memory()
2597 				 pfn, &unpoison_rs);  in unpoison_memory()
2607 				 pfn, &unpoison_rs);  in unpoison_memory()
2613 				 pfn, &unpoison_rs);  in unpoison_memory()
2632 					 pfn, &unpoison_rs);  in unpoison_memory()
2655 			num_poisoned_pages_sub(pfn, 1);  in unpoison_memory()
2674 	unsigned long pfn = page_to_pfn(page);  in soft_offline_in_use_page()  local
2688 			pr_info("%#lx: thp split failed\n", pfn);  in soft_offline_in_use_page()
2700 		pr_info("%#lx: page already poisoned\n", pfn);  in soft_offline_in_use_page()
2713 		pr_info("%#lx: invalidated\n", pfn);  in soft_offline_in_use_page()
2742 				pfn, msg_page[huge], ret, &page->flags);  in soft_offline_in_use_page()
2748 			pfn, msg_page[huge], page_count(page), &page->flags);  in soft_offline_in_use_page()
2779 int soft_offline_page(unsigned long pfn, int flags)  in soft_offline_page()  argument
2785 	if (!pfn_valid(pfn)) {  in soft_offline_page()
2791 	page = pfn_to_online_page(pfn);  in soft_offline_page()
2793 		put_ref_page(pfn, flags);  in soft_offline_page()
2799 		put_ref_page(pfn, flags);  in soft_offline_page()
2806 		pr_info("%#lx: page already poisoned\n", pfn);  in soft_offline_page()
2807 		put_ref_page(pfn, flags);  in soft_offline_page()