xref: /freebsd-14.2/sys/arm64/arm64/exception.S (revision 8b9f8a7e) 
1/*-
2 * Copyright (c) 2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28#include <machine/asm.h>
29#include <machine/armreg.h>
30#include "assym.inc"
31
32	.text
33
34/*
35 * This is limited to 28 instructions as it's placed in the exception vector
36 * slot that is 32 instructions long. We need one for the branch, and three
37 * for the prologue.
38 */
39.macro	save_registers_head el
40.if \el == 1
41	mov	x18, sp
42	stp	x0,  x1,  [sp, #(TF_X - TF_SIZE - 128)]!
43.else
44	stp	x0,  x1,  [sp, #(TF_X - TF_SIZE)]!
45.endif
46	stp	x2,  x3,  [sp, #(2  * 8)]
47	stp	x4,  x5,  [sp, #(4  * 8)]
48	stp	x6,  x7,  [sp, #(6  * 8)]
49	stp	x8,  x9,  [sp, #(8  * 8)]
50	stp	x10, x11, [sp, #(10 * 8)]
51	stp	x12, x13, [sp, #(12 * 8)]
52	stp	x14, x15, [sp, #(14 * 8)]
53	stp	x16, x17, [sp, #(16 * 8)]
54	stp	x18, x19, [sp, #(18 * 8)]
55	stp	x20, x21, [sp, #(20 * 8)]
56	stp	x22, x23, [sp, #(22 * 8)]
57	stp	x24, x25, [sp, #(24 * 8)]
58	stp	x26, x27, [sp, #(26 * 8)]
59	stp	x28, x29, [sp, #(28 * 8)]
60.if \el == 0
61	mrs	x18, sp_el0
62.endif
63	mrs	x10, elr_el1
64	mrs	x11, spsr_el1
65	mrs	x12, esr_el1
66	mrs	x13, far_el1
67	stp	x18,  lr, [sp, #(TF_SP - TF_X)]!
68	stp	x10, x11, [sp, #(TF_ELR)]
69	stp	x12, x13, [sp, #(TF_ESR)]
70	mrs	x18, tpidr_el1
71.endm
72
73.macro	save_registers el
74	add	x29, sp, #(TF_SIZE)
75.if \el == 0
76#if defined(PERTHREAD_SSP)
77	/* Load the SSP canary to sp_el0 */
78	ldr	x1, [x18, #(PC_CURTHREAD)]
79	add	x1, x1, #(TD_MD_CANARY)
80	msr	sp_el0, x1
81#endif
82
83	/* Apply the SSBD (CVE-2018-3639) workaround if needed */
84	ldr	x1, [x18, #PC_SSBD]
85	cbz	x1, 1f
86	mov	w0, #1
87	blr	x1
881:
89
90	ldr	x0, [x18, #PC_CURTHREAD]
91	bl	ptrauth_exit_el0
92
93	ldr	x0, [x18, #(PC_CURTHREAD)]
94	bl	dbg_monitor_enter
95
96	/* Unmask debug and SError exceptions */
97	msr	daifclr, #(DAIF_D | DAIF_A)
98.else
99	/*
100	 * Unmask debug and SError exceptions.
101	 * For EL1, debug exceptions are conditionally unmasked in
102	 * do_el1h_sync().
103	 */
104	msr	daifclr, #(DAIF_A)
105.endif
106.endm
107
108.macro	restore_registers el
109	/*
110	 * Mask all exceptions, x18 may change in the interrupt exception
111	 * handler.
112	 */
113	msr	daifset, #(DAIF_ALL)
114.if \el == 0
115	ldr	x0, [x18, #PC_CURTHREAD]
116	mov	x1, sp
117	bl	dbg_monitor_exit
118
119	ldr	x0, [x18, #PC_CURTHREAD]
120	bl	ptrauth_enter_el0
121
122	/* Remove the SSBD (CVE-2018-3639) workaround if needed */
123	ldr	x1, [x18, #PC_SSBD]
124	cbz	x1, 1f
125	mov	w0, #0
126	blr	x1
1271:
128.endif
129	ldp	x18,  lr, [sp, #(TF_SP)]
130	ldp	x10, x11, [sp, #(TF_ELR)]
131.if \el == 0
132	msr	sp_el0, x18
133.endif
134	msr	spsr_el1, x11
135	msr	elr_el1, x10
136	ldp	x0,  x1,  [sp, #(TF_X + 0  * 8)]
137	ldp	x2,  x3,  [sp, #(TF_X + 2  * 8)]
138	ldp	x4,  x5,  [sp, #(TF_X + 4  * 8)]
139	ldp	x6,  x7,  [sp, #(TF_X + 6  * 8)]
140	ldp	x8,  x9,  [sp, #(TF_X + 8  * 8)]
141	ldp	x10, x11, [sp, #(TF_X + 10 * 8)]
142	ldp	x12, x13, [sp, #(TF_X + 12 * 8)]
143	ldp	x14, x15, [sp, #(TF_X + 14 * 8)]
144	ldp	x16, x17, [sp, #(TF_X + 16 * 8)]
145.if \el == 0
146	/*
147	 * We only restore the callee saved registers when returning to
148	 * userland as they may have been updated by a system call or signal.
149	 */
150	ldp	x18, x19, [sp, #(TF_X + 18 * 8)]
151	ldp	x20, x21, [sp, #(TF_X + 20 * 8)]
152	ldp	x22, x23, [sp, #(TF_X + 22 * 8)]
153	ldp	x24, x25, [sp, #(TF_X + 24 * 8)]
154	ldp	x26, x27, [sp, #(TF_X + 26 * 8)]
155	ldp	x28, x29, [sp, #(TF_X + 28 * 8)]
156.else
157	ldr	     x29, [sp, #(TF_X + 29 * 8)]
158.endif
159.if \el == 0
160	add	sp, sp, #(TF_SIZE)
161.else
162	mov	sp, x18
163	mrs	x18, tpidr_el1
164.endif
165.endm
166
167.macro	do_ast
168	mrs	x19, daif
169	/* Make sure the IRQs are enabled before calling ast() */
170	bic	x19, x19, #PSR_I
1711:
172	/*
173	 * Mask interrupts while checking the ast pending flag
174	 */
175	msr	daifset, #(DAIF_INTR)
176
177	/* Read the current thread AST mask */
178	ldr	x1, [x18, #PC_CURTHREAD]	/* Load curthread */
179	ldr	w1, [x1, #(TD_AST)]
180
181	/* Check if we have a non-zero AST mask */
182	cbz	w1, 2f
183
184	/* Restore interrupts */
185	msr	daif, x19
186
187	/* handle the ast */
188	mov	x0, sp
189	bl	_C_LABEL(ast)
190
191	/* Re-check for new ast scheduled */
192	b	1b
1932:
194.endm
195
196ENTRY(handle_el1h_sync)
197	save_registers 1
198	ldr	x0, [x18, #PC_CURTHREAD]
199	mov	x1, sp
200	bl	do_el1h_sync
201	restore_registers 1
202	ERET
203END(handle_el1h_sync)
204
205ENTRY(handle_el1h_irq)
206	save_registers 1
207	mov	x0, sp
208	bl	intr_irq_handler
209	restore_registers 1
210	ERET
211END(handle_el1h_irq)
212
213ENTRY(handle_el0_sync)
214	save_registers 0
215	ldr	x0, [x18, #PC_CURTHREAD]
216	mov	x1, sp
217	mov	x22, x0
218	str	x1, [x0, #TD_FRAME]
219	bl	do_el0_sync
220	do_ast
221	str	xzr, [x22, #TD_FRAME]
222	restore_registers 0
223	ERET
224END(handle_el0_sync)
225
226ENTRY(handle_el0_irq)
227	save_registers 0
228	mov	x0, sp
229	bl	intr_irq_handler
230	do_ast
231	restore_registers 0
232	ERET
233END(handle_el0_irq)
234
235ENTRY(handle_serror)
236	save_registers 0
237	mov	x0, sp
2381:	bl	do_serror
239	b	1b
240END(handle_serror)
241
242ENTRY(handle_empty_exception)
243	save_registers 0
244	mov	x0, sp
2451:	bl	unhandled_exception
246	b	1b
247END(handle_empty_exception)
248
249.macro	vector	name, el
250	.align 7
251	save_registers_head \el
252	b	handle_\name
253	dsb	sy
254	isb
255	/* Break instruction to ensure we aren't executing code here. */
256	brk	0x42
257.endm
258
259.macro	vempty el
260	vector	empty_exception \el
261.endm
262
263	.align 11
264	.globl exception_vectors
265exception_vectors:
266	vempty 1		/* Synchronous EL1t */
267	vempty 1		/* IRQ EL1t */
268	vempty 1		/* FIQ EL1t */
269	vempty 1		/* Error EL1t */
270
271	vector el1h_sync 1	/* Synchronous EL1h */
272	vector el1h_irq 1	/* IRQ EL1h */
273	vempty 1		/* FIQ EL1h */
274	vector serror 1		/* Error EL1h */
275
276	vector el0_sync 0	/* Synchronous 64-bit EL0 */
277	vector el0_irq 0	/* IRQ 64-bit EL0 */
278	vempty 0		/* FIQ 64-bit EL0 */
279	vector serror 0		/* Error 64-bit EL0 */
280
281	vector el0_sync 0	/* Synchronous 32-bit EL0 */
282	vector el0_irq 0	/* IRQ 32-bit EL0 */
283	vempty 0		/* FIQ 32-bit EL0 */
284	vector serror 0		/* Error 32-bit EL0 */
285
286