xref: /freebsd-14.2/sys/arm64/arm64/locore.S (revision a194eabe)
1/*-
2 * Copyright (c) 2012-2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include "assym.inc"
28#include "opt_kstack_pages.h"
29#include <sys/syscall.h>
30#include <machine/asm.h>
31#include <machine/armreg.h>
32#include <machine/hypervisor.h>
33#include <machine/param.h>
34#include <machine/pte.h>
35#include <machine/vm.h>
36#include <machine/vmparam.h>
37
38#define	VIRT_BITS	48
39
40#if PAGE_SIZE == PAGE_SIZE_16K
41/*
42 * The number of level 3 tables to create. 32 will allow for 1G of address
43 * space, the same as a single level 2 page with 4k pages.
44 */
45#define	L3_PAGE_COUNT	32
46#endif
47
48/*
49 * The size of our bootstrap stack.
50 */
51#define	BOOT_STACK_SIZE	(KSTACK_PAGES * PAGE_SIZE)
52
53	.globl	kernbase
54	.set	kernbase, KERNBASE
55
56/*
57 * We assume:
58 *  MMU      on with an identity map, or off
59 *  D-Cache: off
60 *  I-Cache: on or off
61 *  We are loaded at a 2MiB aligned address
62 */
63
64ENTRY(_start)
65	/* Enter the kernel exception level */
66	bl	enter_kernel_el
67
68	/* Set the context id */
69	msr	contextidr_el1, xzr
70
71	/* Get the virt -> phys offset */
72	bl	get_load_phys_addr
73
74	/*
75	 * At this point:
76	 * x28 = Our physical load address
77	 */
78
79	/* Create the page tables */
80	bl	create_pagetables
81
82	/*
83	 * At this point:
84	 * x27 = TTBR0 table
85	 * x26 = Kernel L1 table
86	 * x24 = TTBR1 table
87	 */
88
89	/* Enable the mmu */
90	bl	start_mmu
91
92	/* Load the new ttbr0 pagetable */
93	adrp	x27, pagetable_l0_ttbr0
94	add	x27, x27, :lo12:pagetable_l0_ttbr0
95
96	/* Jump to the virtual address space */
97	ldr	x15, .Lvirtdone
98	br	x15
99
100virtdone:
101	BTI_J
102
103	/* Set up the stack */
104	adrp	x25, initstack_end
105	add	x25, x25, :lo12:initstack_end
106	sub	sp, x25, #PCB_SIZE
107
108	/* Zero the BSS */
109	ldr	x15, .Lbss
110	ldr	x14, .Lend
1111:
112	stp	xzr, xzr, [x15], #16
113	cmp	x15, x14
114	b.lo	1b
115
116#if defined(PERTHREAD_SSP)
117	/* Set sp_el0 to the boot canary for early per-thread SSP to work */
118	adrp	x15, boot_canary
119	add	x15, x15, :lo12:boot_canary
120	msr	sp_el0, x15
121#endif
122
123	/* Backup the module pointer */
124	mov	x1, x0
125
126	sub	sp, sp, #BOOTPARAMS_SIZE
127	mov	x0, sp
128
129	str	x1,  [x0, #BP_MODULEP]
130	adrp	x25, initstack
131	add	x25, x25, :lo12:initstack
132	str	x25, [x0, #BP_KERN_STACK]
133	str	x27, [x0, #BP_KERN_TTBR0]
134	str	x23, [x0, #BP_BOOT_EL]
135	str	x4,  [x0, #BP_HCR_EL2]
136
137#ifdef KASAN
138	/* Save bootparams */
139	mov	x19, x0
140
141	/* Bootstrap an early shadow map for the boot stack. */
142	ldr	x0, [x0, #BP_KERN_STACK]
143	ldr	x1, =BOOT_STACK_SIZE
144	bl	kasan_init_early
145
146	/* Restore bootparams */
147	mov	x0, x19
148#endif
149
150	/* trace back starts here */
151	mov	fp, #0
152	/* Branch to C code */
153	bl	initarm
154	/* We are done with the boot params */
155	add	sp, sp, #BOOTPARAMS_SIZE
156
157	/*
158	 * Enable pointer authentication in the kernel. We set the keys for
159	 * thread0 in initarm so have to wait until it returns to enable it.
160	 * If we were to enable it in initarm then any authentication when
161	 * returning would fail as it was called with pointer authentication
162	 * disabled.
163	 */
164	bl	ptrauth_start
165
166	bl	mi_startup
167
168	/* We should not get here */
169	brk	0
170
171	.align 3
172.Lvirtdone:
173	.quad	virtdone
174.Lbss:
175	.quad	__bss_start
176.Lend:
177	.quad	__bss_end
178END(_start)
179
180#ifdef SMP
181/*
182 * mpentry(unsigned long)
183 *
184 * Called by a core when it is being brought online.
185 * The data in x0 is passed straight to init_secondary.
186 */
187ENTRY(mpentry)
188	/* Disable interrupts */
189	msr	daifset, #DAIF_INTR
190
191	/* Enter the kernel exception level */
192	bl	enter_kernel_el
193
194	/* Set the context id */
195	msr	contextidr_el1, xzr
196
197	/* Load the kernel page table */
198	adrp	x24, pagetable_l0_ttbr1
199	add	x24, x24, :lo12:pagetable_l0_ttbr1
200	/* Load the identity page table */
201	adrp	x27, pagetable_l0_ttbr0_bootstrap
202	add	x27, x27, :lo12:pagetable_l0_ttbr0_bootstrap
203
204	/* Enable the mmu */
205	bl	start_mmu
206
207	/* Load the new ttbr0 pagetable */
208	adrp	x27, pagetable_l0_ttbr0
209	add	x27, x27, :lo12:pagetable_l0_ttbr0
210
211	/* Jump to the virtual address space */
212	ldr	x15, =mp_virtdone
213	br	x15
214
215mp_virtdone:
216	BTI_J
217
218	/* Start using the AP boot stack */
219	adrp	x4, bootstack
220	ldr	x4, [x4, :lo12:bootstack]
221	mov	sp, x4
222
223#if defined(PERTHREAD_SSP)
224	/* Set sp_el0 to the boot canary for early per-thread SSP to work */
225	adrp	x15, boot_canary
226	add	x15, x15, :lo12:boot_canary
227	msr	sp_el0, x15
228#endif
229
230	/* Load the kernel ttbr0 pagetable */
231	msr	ttbr0_el1, x27
232	isb
233
234	/* Invalidate the TLB */
235	tlbi	vmalle1
236	dsb	sy
237	isb
238
239	/*
240	 * Initialize the per-CPU pointer before calling into C code, for the
241	 * benefit of kernel sanitizers.
242	 */
243	adrp	x18, bootpcpu
244	ldr	x18, [x18, :lo12:bootpcpu]
245	msr	tpidr_el1, x18
246
247	b	init_secondary
248END(mpentry)
249#endif
250
251/*
252 * If we are started in EL2, configure the required hypervisor
253 * registers and drop to EL1.
254 */
255LENTRY(enter_kernel_el)
256#define	INIT_SCTLR_EL1	(SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_EIS | \
257    SCTLR_TSCXT | SCTLR_EOS)
258	mrs	x23, CurrentEL
259	lsr	x23, x23, #2
260	cmp	x23, #0x2
261	b.eq	1f
262
263	ldr	x2, =INIT_SCTLR_EL1
264	msr	sctlr_el1, x2
265	/* SCTLR_EOS is set so eret is a context synchronizing event so we
266	 * need an isb here to ensure it's observed by later instructions,
267	 * but don't need it in the eret below.
268	 */
269	isb
270
271	/* Ensure SPSR_EL1 and pstate are in sync. The only wat to set the
272	 * latter is to set the former and return from an exception with eret.
273	 */
274	mov	x2, #(PSR_DAIF | PSR_M_EL1h)
275	msr	spsr_el1, x2
276	msr	elr_el1, lr
277	eret
278
2791:
280	dsb	sy
281	/*
282	 * Set just the reserved bits in sctlr_el2. This will disable the
283	 * MMU which may have broken the kernel if we enter the kernel in
284	 * EL2, e.g. when using VHE.
285	 */
286	ldr	x2, =(SCTLR_EL2_RES1 | SCTLR_EL2_EIS | SCTLR_EL2_EOS)
287	msr	sctlr_el2, x2
288	isb
289
290	/* Configure the Hypervisor */
291	ldr	x2, =(HCR_RW | HCR_APK | HCR_API)
292	msr	hcr_el2, x2
293
294	/* Stash value of HCR_EL2 for later */
295	isb
296	mrs	x4, hcr_el2
297
298	/* Load the Virtualization Process ID Register */
299	mrs	x2, midr_el1
300	msr	vpidr_el2, x2
301
302	/* Load the Virtualization Multiprocess ID Register */
303	mrs	x2, mpidr_el1
304	msr	vmpidr_el2, x2
305
306	/* Set the initial sctlr_el1 */
307	ldr	x2, =INIT_SCTLR_EL1
308	msr	sctlr_el1, x2
309
310	/*
311	 * On some hardware, e.g., Apple M1, we can't clear E2H, so make sure we
312	 * don't trap to EL2 for SIMD register usage to have at least a
313	 * minimally usable system.
314	 */
315	tst	x4, #HCR_E2H
316	mov	x3, #CPTR_RES1	/* HCR_E2H == 0 */
317	mov	x5, #CPTR_FPEN	/* HCR_E2H == 1 */
318	csel	x2, x3, x5, eq
319	msr	cptr_el2, x2
320
321	/* Don't trap to EL2 for CP15 traps */
322	msr	hstr_el2, xzr
323
324	/* Enable access to the physical timers at EL1 */
325	tst	x4, #HCR_E2H
326	ldr	x3, =(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN)
327	ldr	x5, =(CNTHCTL_E2H_EL1PCTEN | CNTHCTL_E2H_EL1PTEN)
328	csel	x2, x3, x5, eq
329	msr	cnthctl_el2, x2
330
331	/* Set the counter offset to a known value */
332	msr	cntvoff_el2, xzr
333
334	/* Hypervisor trap functions */
335	adrp	x2, hyp_stub_vectors
336	add	x2, x2, :lo12:hyp_stub_vectors
337	msr	vbar_el2, x2
338
339	/* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */
340	msr	vttbr_el2, xzr
341
342	mov	x2, #(PSR_DAIF | PSR_M_EL1h)
343	msr	spsr_el2, x2
344
345	/* Configure GICv3 CPU interface */
346	mrs	x2, id_aa64pfr0_el1
347	/* Extract GIC bits from the register */
348	ubfx	x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS
349	/* GIC[3:0] != 0000 - GIC CPU interface via special regs. supported */
350	cbz	x2, 2f
351
352	mrs	x2, icc_sre_el2
353	orr	x2, x2, #ICC_SRE_EL2_EN	/* Enable access from insecure EL1 */
354	orr	x2, x2, #ICC_SRE_EL2_SRE	/* Enable system registers */
355	msr	icc_sre_el2, x2
3562:
357
358	/* Set the address to return to our return address */
359	msr	elr_el2, x30
360	isb
361
362	eret
363#undef INIT_SCTLR_EL1
364LEND(enter_kernel_el)
365
366/*
367 * Get the physical address the kernel was loaded at.
368 */
369LENTRY(get_load_phys_addr)
370	/* Load the offset of get_load_phys_addr from KERNBASE */
371	ldr	x28, =(get_load_phys_addr - KERNBASE)
372	/* Load the physical address of get_load_phys_addr */
373	adr	x29, get_load_phys_addr
374	/* Find the physical address of KERNBASE, i.e. our load address */
375	sub	x28, x29, x28
376	ret
377LEND(get_load_phys_addr)
378
379/*
380 * This builds the page tables containing the identity map, and the kernel
381 * virtual map.
382 *
383 * It relys on:
384 *  We were loaded to an address that is on a 2MiB boundary
385 *  All the memory must not cross a 1GiB boundaty
386 *  x28 contains the physical address we were loaded from
387 *
388 * TODO: This is out of date.
389 *  There are at least 5 pages before that address for the page tables
390 *   The pages used are:
391 *    - The Kernel L2 table
392 *    - The Kernel L1 table
393 *    - The Kernel L0 table             (TTBR1)
394 *    - The identity (PA = VA) L1 table
395 *    - The identity (PA = VA) L0 table (TTBR0)
396 */
397LENTRY(create_pagetables)
398	/* Save the Link register */
399	mov	x5, x30
400
401	/* Clean the page table */
402	adrp	x6, pagetable
403	add	x6, x6, :lo12:pagetable
404	mov	x26, x6
405	adrp	x27, pagetable_end
406	add	x27, x27, :lo12:pagetable_end
4071:
408	stp	xzr, xzr, [x6], #16
409	stp	xzr, xzr, [x6], #16
410	stp	xzr, xzr, [x6], #16
411	stp	xzr, xzr, [x6], #16
412	cmp	x6, x27
413	b.lo	1b
414
415	/*
416	 * Build the TTBR1 maps.
417	 */
418
419	/* Find the size of the kernel */
420	mov	x6, #(KERNBASE)
421
422#if defined(LINUX_BOOT_ABI)
423	/* X19 is used as 'map FDT data' flag */
424	mov	x19, xzr
425
426	/* No modules or FDT pointer ? */
427	cbz	x0, booti_no_fdt
428
429	/*
430	 * Test if x0 points to modules descriptor(virtual address) or
431	 * to FDT (physical address)
432	 */
433	cmp	x0, x6		/* x6 is #(KERNBASE) */
434	b.lo	booti_fdt
435#endif
436
437	/* Booted with modules pointer */
438	/* Find modulep - begin */
439	sub	x8, x0, x6
440	/*
441	 * Add space for the module data. When PAGE_SIZE is 4k this will
442	 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is
443	 * larger it will be at least as large as we use smaller level 3
444	 * pages.
445	 */
446	ldr	x7, =((6 * 1024 * 1024) - 1)
447	add	x8, x8, x7
448	b	common
449
450#if defined(LINUX_BOOT_ABI)
451booti_fdt:
452	/* Booted by U-Boot booti with FDT data */
453	/* Set 'map FDT data' flag */
454	mov	x19, #1
455
456booti_no_fdt:
457	/* Booted by U-Boot booti without FTD data */
458	/* Find the end - begin */
459	ldr     x7, .Lend
460	sub     x8, x7, x6
461
462	/*
463	 * Add one 2MiB page for copy of FDT data (maximum FDT size),
464	 * one for metadata and round up
465	 */
466	ldr	x7, =(3 * L2_SIZE - 1)
467	add	x8, x8, x7
468#endif
469
470common:
471#if PAGE_SIZE != PAGE_SIZE_4K
472	/*
473	 * Create L3 pages. The kernel will be loaded at a 2M aligned
474	 * address, however L2 blocks are too large when the page size is
475	 * not 4k to map the kernel with such an aligned address. However,
476	 * when the page size is larger than 4k, L2 blocks are too large to
477	 * map the kernel with such an alignment.
478	 */
479
480	/* Get the number of l3 pages to allocate, rounded down */
481	lsr	x10, x8, #(L3_SHIFT)
482
483	/* Create the kernel space L2 table */
484	mov	x6, x26
485	mov	x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
486	mov	x8, #(KERNBASE)
487	mov	x9, x28
488	bl	build_l3_page_pagetable
489
490	/* Move to the l2 table */
491	ldr	x9, =(PAGE_SIZE * L3_PAGE_COUNT)
492	add	x26, x26, x9
493
494	/* Link the l2 -> l3 table */
495	mov	x9, x6
496	mov	x6, x26
497	bl	link_l2_pagetable
498#else
499	/* Get the number of l2 pages to allocate, rounded down */
500	lsr	x10, x8, #(L2_SHIFT)
501
502	/* Create the kernel space L2 table */
503	mov	x6, x26
504	mov	x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
505	mov	x8, #(KERNBASE)
506	mov	x9, x28
507	bl	build_l2_block_pagetable
508#endif
509
510	/* Move to the l1 table */
511	add	x26, x26, #PAGE_SIZE
512
513	/* Link the l1 -> l2 table */
514	mov	x9, x6
515	mov	x6, x26
516	bl	link_l1_pagetable
517
518	/* Move to the l0 table */
519	add	x24, x26, #PAGE_SIZE
520
521	/* Link the l0 -> l1 table */
522	mov	x9, x6
523	mov	x6, x24
524	mov	x10, #1
525	bl	link_l0_pagetable
526
527	/*
528	 * Build the TTBR0 maps.  As TTBR0 maps, they must specify ATTR_S1_nG.
529	 * They are only needed early on, so the VA = PA map is uncached.
530	 */
531	add	x27, x24, #PAGE_SIZE
532
533	mov	x6, x27		/* The initial page table */
534
535	/* Create the VA = PA map */
536	mov	x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
537	adrp	x16, _start
538	and	x16, x16, #(~L2_OFFSET)
539	mov	x9, x16		/* PA start */
540	mov	x8, x16		/* VA start (== PA start) */
541	mov	x10, #1
542	bl	build_l2_block_pagetable
543
544#if defined(SOCDEV_PA)
545	/* Create a table for the UART */
546	mov	x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE))
547	ldr	x9, =(L2_SIZE)
548	add	x16, x16, x9	/* VA start */
549	mov	x8, x16
550
551	/* Store the socdev virtual address */
552	add	x17, x8, #(SOCDEV_PA & L2_OFFSET)
553	adrp	x9, socdev_va
554	str	x17, [x9, :lo12:socdev_va]
555
556	mov	x9, #(SOCDEV_PA & ~L2_OFFSET)	/* PA start */
557	mov	x10, #1
558	bl	build_l2_block_pagetable
559#endif
560
561#if defined(LINUX_BOOT_ABI)
562	/* Map FDT data ? */
563	cbz	x19, 1f
564
565	/* Create the mapping for FDT data (2 MiB max) */
566	mov	x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
567	ldr	x9, =(L2_SIZE)
568	add	x16, x16, x9	/* VA start */
569	mov	x8, x16
570	mov	x9, x0			/* PA start */
571	/* Update the module pointer to point at the allocated memory */
572	and	x0, x0, #(L2_OFFSET)	/* Keep the lower bits */
573	add	x0, x0, x8		/* Add the aligned virtual address */
574
575	mov	x10, #1
576	bl	build_l2_block_pagetable
577
5781:
579#endif
580
581	/* Move to the l1 table */
582	add	x27, x27, #PAGE_SIZE
583
584	/* Link the l1 -> l2 table */
585	mov	x9, x6
586	mov	x6, x27
587	bl	link_l1_pagetable
588
589	/* Move to the l0 table */
590	add	x27, x27, #PAGE_SIZE
591
592	/* Link the l0 -> l1 table */
593	mov	x9, x6
594	mov	x6, x27
595	mov	x10, #1
596	bl	link_l0_pagetable
597
598	/* Restore the Link register */
599	mov	x30, x5
600	ret
601LEND(create_pagetables)
602
603/*
604 * Builds an L0 -> L1 table descriptor
605 *
606 *  x6  = L0 table
607 *  x8  = Virtual Address
608 *  x9  = L1 PA (trashed)
609 *  x10 = Entry count (trashed)
610 *  x11, x12 and x13 are trashed
611 */
612LENTRY(link_l0_pagetable)
613	/*
614	 * Link an L0 -> L1 table entry.
615	 */
616	/* Find the table index */
617	lsr	x11, x8, #L0_SHIFT
618	and	x11, x11, #L0_ADDR_MASK
619
620	/* Build the L0 block entry */
621	mov	x12, #L0_TABLE
622	orr	x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0)
623
624	/* Only use the output address bits */
625	lsr	x9, x9, #PAGE_SHIFT
6261:	orr	x13, x12, x9, lsl #PAGE_SHIFT
627
628	/* Store the entry */
629	str	x13, [x6, x11, lsl #3]
630
631	sub	x10, x10, #1
632	add	x11, x11, #1
633	add	x9, x9, #1
634	cbnz	x10, 1b
635
636	ret
637LEND(link_l0_pagetable)
638
639/*
640 * Builds an L1 -> L2 table descriptor
641 *
642 *  x6  = L1 table
643 *  x8  = Virtual Address
644 *  x9  = L2 PA (trashed)
645 *  x11, x12 and x13 are trashed
646 */
647LENTRY(link_l1_pagetable)
648	/*
649	 * Link an L1 -> L2 table entry.
650	 */
651	/* Find the table index */
652	lsr	x11, x8, #L1_SHIFT
653	and	x11, x11, #Ln_ADDR_MASK
654
655	/* Build the L1 block entry */
656	mov	x12, #L1_TABLE
657
658	/* Only use the output address bits */
659	lsr	x9, x9, #PAGE_SHIFT
660	orr	x13, x12, x9, lsl #PAGE_SHIFT
661
662	/* Store the entry */
663	str	x13, [x6, x11, lsl #3]
664
665	ret
666LEND(link_l1_pagetable)
667
668/*
669 * Builds count 2 MiB page table entry
670 *  x6  = L2 table
671 *  x7  = Block attributes
672 *  x8  = VA start
673 *  x9  = PA start (trashed)
674 *  x10 = Entry count (trashed)
675 *  x11, x12 and x13 are trashed
676 */
677LENTRY(build_l2_block_pagetable)
678	/*
679	 * Build the L2 table entry.
680	 */
681	/* Find the table index */
682	lsr	x11, x8, #L2_SHIFT
683	and	x11, x11, #Ln_ADDR_MASK
684
685	/* Build the L2 block entry */
686	orr	x12, x7, #L2_BLOCK
687	orr	x12, x12, #(ATTR_DEFAULT)
688	orr	x12, x12, #(ATTR_S1_UXN)
689
690	/* Only use the output address bits */
691	lsr	x9, x9, #L2_SHIFT
692
693	/* Set the physical address for this virtual address */
6941:	orr	x13, x12, x9, lsl #L2_SHIFT
695
696	/* Store the entry */
697	str	x13, [x6, x11, lsl #3]
698
699	sub	x10, x10, #1
700	add	x11, x11, #1
701	add	x9, x9, #1
702	cbnz	x10, 1b
703
704	ret
705LEND(build_l2_block_pagetable)
706
707#if PAGE_SIZE != PAGE_SIZE_4K
708/*
709 * Builds an L2 -> L3 table descriptor
710 *
711 *  x6  = L2 table
712 *  x8  = Virtual Address
713 *  x9  = L3 PA (trashed)
714 *  x11, x12 and x13 are trashed
715 */
716LENTRY(link_l2_pagetable)
717	/*
718	 * Link an L2 -> L3 table entry.
719	 */
720	/* Find the table index */
721	lsr	x11, x8, #L2_SHIFT
722	and	x11, x11, #Ln_ADDR_MASK
723
724	/* Build the L1 block entry */
725	mov	x12, #L2_TABLE
726
727	/* Only use the output address bits */
728	lsr	x9, x9, #PAGE_SHIFT
729	orr	x13, x12, x9, lsl #PAGE_SHIFT
730
731	/* Store the entry */
732	str	x13, [x6, x11, lsl #3]
733
734	ret
735LEND(link_l2_pagetable)
736
737/*
738 * Builds count level 3 page table entries
739 *  x6  = L3 table
740 *  x7  = Block attributes
741 *  x8  = VA start
742 *  x9  = PA start (trashed)
743 *  x10 = Entry count (trashed)
744 *  x11, x12 and x13 are trashed
745 */
746LENTRY(build_l3_page_pagetable)
747	/*
748	 * Build the L3 table entry.
749	 */
750	/* Find the table index */
751	lsr	x11, x8, #L3_SHIFT
752	and	x11, x11, #Ln_ADDR_MASK
753
754	/* Build the L3 page entry */
755	orr	x12, x7, #L3_PAGE
756	orr	x12, x12, #(ATTR_DEFAULT)
757	orr	x12, x12, #(ATTR_S1_UXN)
758
759	/* Only use the output address bits */
760	lsr	x9, x9, #L3_SHIFT
761
762	/* Set the physical address for this virtual address */
7631:	orr	x13, x12, x9, lsl #L3_SHIFT
764
765	/* Store the entry */
766	str	x13, [x6, x11, lsl #3]
767
768	sub	x10, x10, #1
769	add	x11, x11, #1
770	add	x9, x9, #1
771	cbnz	x10, 1b
772
773	ret
774LEND(build_l3_page_pagetable)
775#endif
776
777LENTRY(start_mmu)
778	dsb	sy
779
780	/* Load the exception vectors */
781	ldr	x2, =exception_vectors
782	msr	vbar_el1, x2
783
784	/* Load ttbr0 and ttbr1 */
785	msr	ttbr0_el1, x27
786	msr	ttbr1_el1, x24
787	isb
788
789	/* Clear the Monitor Debug System control register */
790	msr	mdscr_el1, xzr
791
792	/* Invalidate the TLB */
793	tlbi	vmalle1is
794	dsb	ish
795	isb
796
797	ldr	x2, mair
798	msr	mair_el1, x2
799
800	/*
801	 * Setup TCR according to the PARange and ASIDBits fields
802	 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the
803	 * ID_AA64MMFR1_EL1.  More precisely, set TCR_EL1.AS
804	 * to 1 only if the ASIDBits field equals 0b0010.
805	 */
806	ldr	x2, tcr
807	mrs	x3, id_aa64mmfr0_el1
808
809	/* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */
810	bfi	x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH)
811	and	x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK)
812
813	/* Check if the HW supports 16 bit ASIDS */
814	cmp	x3, #(ID_AA64MMFR0_ASIDBits_16)
815	/* If so x3 == 1, else x3 == 0 */
816	cset	x3, eq
817	/* Set TCR.AS with x3 */
818	bfi	x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH)
819
820	/*
821	 * Check if the HW supports access flag and dirty state updates,
822	 * and set TCR_EL1.HA and TCR_EL1.HD accordingly.
823	 */
824	mrs	x3, id_aa64mmfr1_el1
825	and	x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK)
826	cmp	x3, #1
827	b.ne	1f
828	orr 	x2, x2, #(TCR_HA)
829	b	2f
8301:
831	cmp	x3, #2
832	b.ne	2f
833	orr 	x2, x2, #(TCR_HA | TCR_HD)
8342:
835	msr	tcr_el1, x2
836
837	/*
838	 * Setup SCTLR.
839	 */
840	ldr	x2, sctlr_set
841	ldr	x3, sctlr_clear
842	mrs	x1, sctlr_el1
843	bic	x1, x1, x3	/* Clear the required bits */
844	orr	x1, x1, x2	/* Set the required bits */
845	msr	sctlr_el1, x1
846	isb
847
848	ret
849
850	.align 3
851mair:
852	.quad	MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \
853		MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE)   |	\
854		MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK)    |	\
855		MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) |	\
856		MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE)
857tcr:
858#if PAGE_SIZE == PAGE_SIZE_4K
859#define	TCR_TG	(TCR_TG1_4K | TCR_TG0_4K)
860#elif PAGE_SIZE == PAGE_SIZE_16K
861#define	TCR_TG	(TCR_TG1_16K | TCR_TG0_16K)
862#else
863#error Unsupported page size
864#endif
865
866	.quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \
867	    TCR_CACHE_ATTRS | TCR_SMP_ATTRS)
868sctlr_set:
869	/* Bits to set */
870	.quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \
871	    SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \
872	    SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \
873	    SCTLR_M | SCTLR_CP15BEN | SCTLR_BT1 | SCTLR_BT0)
874sctlr_clear:
875	/* Bits to clear */
876	.quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \
877	    SCTLR_ITD | SCTLR_A)
878LEND(start_mmu)
879
880ENTRY(abort)
881	b abort
882END(abort)
883
884.bss
885	.align	PAGE_SHIFT
886initstack:
887	.space	BOOT_STACK_SIZE
888initstack_end:
889
890	.section .init_pagetable, "aw", %nobits
891	.align PAGE_SHIFT
892	/*
893	 * 6 initial tables (in the following order):
894	 *           L2 for kernel (High addresses)
895	 *           L1 for kernel
896	 *           L0 for kernel
897	 *           L1 bootstrap for user   (Low addresses)
898	 *           L0 bootstrap for user
899	 *           L0 for user
900	 */
901	.globl pagetable_l0_ttbr1
902pagetable:
903#if PAGE_SIZE != PAGE_SIZE_4K
904	.space	(PAGE_SIZE * L3_PAGE_COUNT)
905pagetable_l2_ttbr1:
906#endif
907	.space	PAGE_SIZE
908pagetable_l1_ttbr1:
909	.space	PAGE_SIZE
910pagetable_l0_ttbr1:
911	.space	PAGE_SIZE
912pagetable_l2_ttbr0_bootstrap:
913	.space	PAGE_SIZE
914pagetable_l1_ttbr0_bootstrap:
915	.space	PAGE_SIZE
916pagetable_l0_ttbr0_bootstrap:
917	.space	PAGE_SIZE
918pagetable_l0_ttbr0:
919	.space	PAGE_SIZE
920pagetable_end:
921
922el2_pagetable:
923	.space	PAGE_SIZE
924
925	.section .rodata, "a", %progbits
926	.globl	aarch32_sigcode
927	.align 2
928aarch32_sigcode:
929	.word 0xe1a0000d	// mov r0, sp
930	.word 0xe2800040	// add r0, r0, #SIGF_UC
931	.word 0xe59f700c	// ldr r7, [pc, #12]
932	.word 0xef000000	// swi #0
933	.word 0xe59f7008	// ldr r7, [pc, #8]
934	.word 0xef000000	// swi #0
935	.word 0xeafffffa	// b . - 16
936	.word SYS_sigreturn
937	.word SYS_exit
938	.align	3
939	.size aarch32_sigcode, . - aarch32_sigcode
940aarch32_esigcode:
941	.data
942	.global sz_aarch32_sigcode
943sz_aarch32_sigcode:
944	.quad aarch32_esigcode - aarch32_sigcode
945