1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * relocate_kernel.S - put the kernel image in place to boot
4 * Copyright (C) 2002-2005 Eric Biederman  <[email protected]>
5 */
6
7#include <linux/linkage.h>
8#include <linux/stringify.h>
9#include <asm/alternative.h>
10#include <asm/page_types.h>
11#include <asm/kexec.h>
12#include <asm/processor-flags.h>
13#include <asm/pgtable_types.h>
14#include <asm/nospec-branch.h>
15#include <asm/unwind_hints.h>
16
17/*
18 * Must be relocatable PIC code callable as a C function, in particular
19 * there must be a plain RET and not jump to return thunk.
20 */
21
22#define PTR(x) (x << 3)
23#define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY)
24
25/*
26 * The .text.relocate_kernel and .data.relocate_kernel sections are copied
27 * into the control page, and the remainder of the page is used as the stack.
28 */
29
30	.section .data.relocate_kernel,"a";
31/* Minimal CPU state */
32SYM_DATA_LOCAL(saved_rsp, .quad 0)
33SYM_DATA_LOCAL(saved_cr0, .quad 0)
34SYM_DATA_LOCAL(saved_cr3, .quad 0)
35SYM_DATA_LOCAL(saved_cr4, .quad 0)
36	/* other data */
37SYM_DATA(kexec_va_control_page, .quad 0)
38SYM_DATA(kexec_pa_table_page, .quad 0)
39SYM_DATA(kexec_pa_swap_page, .quad 0)
40SYM_DATA_LOCAL(pa_backup_pages_map, .quad 0)
41
42	.section .text.relocate_kernel,"ax";
43	.code64
44SYM_CODE_START_NOALIGN(relocate_kernel)
45	UNWIND_HINT_END_OF_STACK
46	ANNOTATE_NOENDBR
47	/*
48	 * %rdi indirection_page
49	 * %rsi pa_control_page
50	 * %rdx start address
51	 * %rcx preserve_context
52	 * %r8  host_mem_enc_active
53	 */
54
55	/* Save the CPU context, used for jumping back */
56	pushq %rbx
57	pushq %rbp
58	pushq %r12
59	pushq %r13
60	pushq %r14
61	pushq %r15
62	pushf
63
64	/* zero out flags, and disable interrupts */
65	pushq $0
66	popfq
67
68	/* Switch to the identity mapped page tables */
69	movq	%cr3, %rax
70	movq	kexec_pa_table_page(%rip), %r9
71	movq	%r9, %cr3
72
73	/* Save %rsp and CRs. */
74	movq    %rsp, saved_rsp(%rip)
75	movq	%rax, saved_cr3(%rip)
76	movq	%cr0, %rax
77	movq	%rax, saved_cr0(%rip)
78	/* Leave CR4 in %r13 to enable the right paging mode later. */
79	movq	%cr4, %r13
80	movq	%r13, saved_cr4(%rip)
81
82	/* Save SME active flag */
83	movq	%r8, %r12
84
85	/* save indirection list for jumping back */
86	movq	%rdi, pa_backup_pages_map(%rip)
87
88	/* Save the preserve_context to %r11 as swap_pages clobbers %rcx. */
89	movq	%rcx, %r11
90
91	/* Physical address of control page */
92	movq    %rsi, %r8
93
94	/* setup a new stack at the end of the physical control page */
95	lea	PAGE_SIZE(%r8), %rsp
96
97	/* jump to identity mapped page */
98	addq	$(identity_mapped - relocate_kernel), %r8
99	pushq	%r8
100	ANNOTATE_UNRET_SAFE
101	ret
102	int3
103SYM_CODE_END(relocate_kernel)
104
105SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
106	UNWIND_HINT_END_OF_STACK
107	/*
108	 * %rdi	indirection page
109	 * %rdx start address
110	 * %r11 preserve_context
111	 * %r12 host_mem_enc_active
112	 * %r13 original CR4 when relocate_kernel() was invoked
113	 */
114
115	/* set return address to 0 if not preserving context */
116	pushq	$0
117	/* store the start address on the stack */
118	pushq   %rdx
119
120	/*
121	 * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP
122	 * below.
123	 */
124	movq	%cr4, %rax
125	andq	$~(X86_CR4_CET), %rax
126	movq	%rax, %cr4
127
128	/*
129	 * Set cr0 to a known state:
130	 *  - Paging enabled
131	 *  - Alignment check disabled
132	 *  - Write protect disabled
133	 *  - No task switch
134	 *  - Don't do FP software emulation.
135	 *  - Protected mode enabled
136	 */
137	movq	%cr0, %rax
138	andq	$~(X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %rax
139	orl	$(X86_CR0_PG | X86_CR0_PE), %eax
140	movq	%rax, %cr0
141
142	/*
143	 * Set cr4 to a known state:
144	 *  - physical address extension enabled
145	 *  - 5-level paging, if it was enabled before
146	 *  - Machine check exception on TDX guest, if it was enabled before.
147	 *    Clearing MCE might not be allowed in TDX guests, depending on setup.
148	 *
149	 * Use R13 that contains the original CR4 value, read in relocate_kernel().
150	 * PAE is always set in the original CR4.
151	 */
152	andl	$(X86_CR4_PAE | X86_CR4_LA57), %r13d
153	ALTERNATIVE "", __stringify(orl $X86_CR4_MCE, %r13d), X86_FEATURE_TDX_GUEST
154	movq	%r13, %cr4
155
156	/* Flush the TLB (needed?) */
157	movq	%r9, %cr3
158
159	/*
160	 * If SME is active, there could be old encrypted cache line
161	 * entries that will conflict with the now unencrypted memory
162	 * used by kexec. Flush the caches before copying the kernel.
163	 */
164	testq	%r12, %r12
165	jz .Lsme_off
166	wbinvd
167.Lsme_off:
168
169	call	swap_pages
170
171	/*
172	 * To be certain of avoiding problems with self-modifying code
173	 * I need to execute a serializing instruction here.
174	 * So I flush the TLB by reloading %cr3 here, it's handy,
175	 * and not processor dependent.
176	 */
177	movq	%cr3, %rax
178	movq	%rax, %cr3
179
180	testq	%r11, %r11	/* preserve_context */
181	jnz .Lrelocate
182
183	/*
184	 * set all of the registers to known values
185	 * leave %rsp alone
186	 */
187
188	xorl	%eax, %eax
189	xorl	%ebx, %ebx
190	xorl    %ecx, %ecx
191	xorl    %edx, %edx
192	xorl    %esi, %esi
193	xorl    %edi, %edi
194	xorl    %ebp, %ebp
195	xorl	%r8d, %r8d
196	xorl	%r9d, %r9d
197	xorl	%r10d, %r10d
198	xorl	%r11d, %r11d
199	xorl	%r12d, %r12d
200	xorl	%r13d, %r13d
201	xorl	%r14d, %r14d
202	xorl	%r15d, %r15d
203
204	ANNOTATE_UNRET_SAFE
205	ret
206	int3
207
208.Lrelocate:
209	popq	%rdx
210	leaq	PAGE_SIZE(%r10), %rsp
211	ANNOTATE_RETPOLINE_SAFE
212	call	*%rdx
213
214	/* get the re-entry point of the peer system */
215	movq	0(%rsp), %rbp
216	leaq	relocate_kernel(%rip), %r8
217	movq	kexec_pa_swap_page(%rip), %r10
218	movq	pa_backup_pages_map(%rip), %rdi
219	movq	kexec_pa_table_page(%rip), %rax
220	movq	%rax, %cr3
221	lea	PAGE_SIZE(%r8), %rsp
222	call	swap_pages
223	movq	kexec_va_control_page(%rip), %rax
224	addq	$(virtual_mapped - relocate_kernel), %rax
225	pushq	%rax
226	ANNOTATE_UNRET_SAFE
227	ret
228	int3
229SYM_CODE_END(identity_mapped)
230
231SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
232	UNWIND_HINT_END_OF_STACK
233	ANNOTATE_NOENDBR // RET target, above
234	movq	saved_rsp(%rip), %rsp
235	movq	saved_cr4(%rip), %rax
236	movq	%rax, %cr4
237	movq	saved_cr3(%rip), %rax
238	movq	saved_cr0(%rip), %r8
239	movq	%rax, %cr3
240	movq	%r8, %cr0
241
242#ifdef CONFIG_KEXEC_JUMP
243	/* Saved in save_processor_state. */
244	movq    $saved_context, %rax
245	lgdt    saved_context_gdt_desc(%rax)
246#endif
247
248	movq	%rbp, %rax
249
250	popf
251	popq	%r15
252	popq	%r14
253	popq	%r13
254	popq	%r12
255	popq	%rbp
256	popq	%rbx
257	ANNOTATE_UNRET_SAFE
258	ret
259	int3
260SYM_CODE_END(virtual_mapped)
261
262	/* Do the copies */
263SYM_CODE_START_LOCAL_NOALIGN(swap_pages)
264	UNWIND_HINT_END_OF_STACK
265	movq	%rdi, %rcx	/* Put the indirection_page in %rcx */
266	xorl	%edi, %edi
267	xorl	%esi, %esi
268	jmp	.Lstart		/* Should start with an indirection record */
269
270.Lloop:	/* top, read another word for the indirection page */
271
272	movq	(%rbx), %rcx
273	addq	$8,	%rbx
274.Lstart:
275	testb	$0x1,	%cl   /* is it a destination page? */
276	jz	.Lnotdest
277	movq	%rcx,	%rdi
278	andq	$0xfffffffffffff000, %rdi
279	jmp	.Lloop
280.Lnotdest:
281	testb	$0x2,	%cl   /* is it an indirection page? */
282	jz	.Lnotind
283	movq	%rcx,   %rbx
284	andq	$0xfffffffffffff000, %rbx
285	jmp	.Lloop
286.Lnotind:
287	testb	$0x4,	%cl   /* is it the done indicator? */
288	jz	.Lnotdone
289	jmp	.Ldone
290.Lnotdone:
291	testb	$0x8,	%cl   /* is it the source indicator? */
292	jz	.Lloop	      /* Ignore it otherwise */
293	movq	%rcx,   %rsi  /* For ever source page do a copy */
294	andq	$0xfffffffffffff000, %rsi
295
296	movq	%rdi, %rdx    /* Save destination page to %rdx */
297	movq	%rsi, %rax    /* Save source page to %rax */
298
299	testq	%r11, %r11    /* Only actually swap for ::preserve_context */
300	jz	.Lnoswap
301
302	/* copy source page to swap page */
303	movq	%r10, %rdi
304	movl	$512, %ecx
305	rep ; movsq
306
307	/* copy destination page to source page */
308	movq	%rax, %rdi
309	movq	%rdx, %rsi
310	movl	$512, %ecx
311	rep ; movsq
312
313	/* copy swap page to destination page */
314	movq	%rdx, %rdi
315	movq	%r10, %rsi
316.Lnoswap:
317	movl	$512, %ecx
318	rep ; movsq
319
320	lea	PAGE_SIZE(%rax), %rsi
321	jmp	.Lloop
322.Ldone:
323	ANNOTATE_UNRET_SAFE
324	ret
325	int3
326SYM_CODE_END(swap_pages)
327