1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include "assym.inc" 28#include "opt_kstack_pages.h" 29#include <sys/syscall.h> 30#include <machine/asm.h> 31#include <machine/armreg.h> 32#include <machine/hypervisor.h> 33#include <machine/param.h> 34#include <machine/pte.h> 35#include <machine/vm.h> 36#include <machine/vmparam.h> 37 38#define VIRT_BITS 48 39 40#if PAGE_SIZE == PAGE_SIZE_16K 41/* 42 * The number of level 3 tables to create. 32 will allow for 1G of address 43 * space, the same as a single level 2 page with 4k pages. 44 */ 45#define L3_PAGE_COUNT 32 46#endif 47 48/* 49 * The size of our bootstrap stack. 50 */ 51#define BOOT_STACK_SIZE (KSTACK_PAGES * PAGE_SIZE) 52 53 .globl kernbase 54 .set kernbase, KERNBASE 55 56/* 57 * We assume: 58 * MMU on with an identity map, or off 59 * D-Cache: off 60 * I-Cache: on or off 61 * We are loaded at a 2MiB aligned address 62 */ 63 64ENTRY(_start) 65 /* Enter the kernel exception level */ 66 bl enter_kernel_el 67 68 /* Set the context id */ 69 msr contextidr_el1, xzr 70 71 /* Get the virt -> phys offset */ 72 bl get_load_phys_addr 73 74 /* 75 * At this point: 76 * x28 = Our physical load address 77 */ 78 79 /* Create the page tables */ 80 bl create_pagetables 81 82 /* 83 * At this point: 84 * x27 = TTBR0 table 85 * x26 = Kernel L1 table 86 * x24 = TTBR1 table 87 */ 88 89 /* Enable the mmu */ 90 bl start_mmu 91 92 /* Load the new ttbr0 pagetable */ 93 adrp x27, pagetable_l0_ttbr0 94 add x27, x27, :lo12:pagetable_l0_ttbr0 95 96 /* Jump to the virtual address space */ 97 ldr x15, .Lvirtdone 98 br x15 99 100virtdone: 101 BTI_J 102 103 /* Set up the stack */ 104 adrp x25, initstack_end 105 add x25, x25, :lo12:initstack_end 106 sub sp, x25, #PCB_SIZE 107 108 /* Zero the BSS */ 109 ldr x15, .Lbss 110 ldr x14, .Lend 1111: 112 stp xzr, xzr, [x15], #16 113 cmp x15, x14 114 b.lo 1b 115 116#if defined(PERTHREAD_SSP) 117 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 118 adrp x15, boot_canary 119 add x15, x15, :lo12:boot_canary 120 msr sp_el0, x15 121#endif 122 123 /* Backup the module pointer */ 124 mov x1, x0 125 126 sub sp, sp, #BOOTPARAMS_SIZE 127 mov x0, sp 128 129 str x1, [x0, #BP_MODULEP] 130 adrp x25, initstack 131 add x25, x25, :lo12:initstack 132 str x25, [x0, #BP_KERN_STACK] 133 str x27, [x0, #BP_KERN_TTBR0] 134 str x23, [x0, #BP_BOOT_EL] 135 str x4, [x0, #BP_HCR_EL2] 136 137#ifdef KASAN 138 /* Save bootparams */ 139 mov x19, x0 140 141 /* Bootstrap an early shadow map for the boot stack. */ 142 ldr x0, [x0, #BP_KERN_STACK] 143 ldr x1, =BOOT_STACK_SIZE 144 bl kasan_init_early 145 146 /* Restore bootparams */ 147 mov x0, x19 148#endif 149 150 /* trace back starts here */ 151 mov fp, #0 152 /* Branch to C code */ 153 bl initarm 154 /* We are done with the boot params */ 155 add sp, sp, #BOOTPARAMS_SIZE 156 157 /* 158 * Enable pointer authentication in the kernel. We set the keys for 159 * thread0 in initarm so have to wait until it returns to enable it. 160 * If we were to enable it in initarm then any authentication when 161 * returning would fail as it was called with pointer authentication 162 * disabled. 163 */ 164 bl ptrauth_start 165 166 bl mi_startup 167 168 /* We should not get here */ 169 brk 0 170 171 .align 3 172.Lvirtdone: 173 .quad virtdone 174.Lbss: 175 .quad __bss_start 176.Lend: 177 .quad __bss_end 178END(_start) 179 180#ifdef SMP 181/* 182 * mpentry(unsigned long) 183 * 184 * Called by a core when it is being brought online. 185 * The data in x0 is passed straight to init_secondary. 186 */ 187ENTRY(mpentry) 188 /* Disable interrupts */ 189 msr daifset, #DAIF_INTR 190 191 /* Enter the kernel exception level */ 192 bl enter_kernel_el 193 194 /* Set the context id */ 195 msr contextidr_el1, xzr 196 197 /* Load the kernel page table */ 198 adrp x24, pagetable_l0_ttbr1 199 add x24, x24, :lo12:pagetable_l0_ttbr1 200 /* Load the identity page table */ 201 adrp x27, pagetable_l0_ttbr0_bootstrap 202 add x27, x27, :lo12:pagetable_l0_ttbr0_bootstrap 203 204 /* Enable the mmu */ 205 bl start_mmu 206 207 /* Load the new ttbr0 pagetable */ 208 adrp x27, pagetable_l0_ttbr0 209 add x27, x27, :lo12:pagetable_l0_ttbr0 210 211 /* Jump to the virtual address space */ 212 ldr x15, =mp_virtdone 213 br x15 214 215mp_virtdone: 216 BTI_J 217 218 /* Start using the AP boot stack */ 219 adrp x4, bootstack 220 ldr x4, [x4, :lo12:bootstack] 221 mov sp, x4 222 223#if defined(PERTHREAD_SSP) 224 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 225 adrp x15, boot_canary 226 add x15, x15, :lo12:boot_canary 227 msr sp_el0, x15 228#endif 229 230 /* Load the kernel ttbr0 pagetable */ 231 msr ttbr0_el1, x27 232 isb 233 234 /* Invalidate the TLB */ 235 tlbi vmalle1 236 dsb sy 237 isb 238 239 /* 240 * Initialize the per-CPU pointer before calling into C code, for the 241 * benefit of kernel sanitizers. 242 */ 243 adrp x18, bootpcpu 244 ldr x18, [x18, :lo12:bootpcpu] 245 msr tpidr_el1, x18 246 247 b init_secondary 248END(mpentry) 249#endif 250 251/* 252 * If we are started in EL2, configure the required hypervisor 253 * registers and drop to EL1. 254 */ 255LENTRY(enter_kernel_el) 256#define INIT_SCTLR_EL1 (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_EIS | \ 257 SCTLR_TSCXT | SCTLR_EOS) 258 mrs x23, CurrentEL 259 lsr x23, x23, #2 260 cmp x23, #0x2 261 b.eq 1f 262 263 ldr x2, =INIT_SCTLR_EL1 264 msr sctlr_el1, x2 265 /* SCTLR_EOS is set so eret is a context synchronizing event so we 266 * need an isb here to ensure it's observed by later instructions, 267 * but don't need it in the eret below. 268 */ 269 isb 270 271 /* Ensure SPSR_EL1 and pstate are in sync. The only wat to set the 272 * latter is to set the former and return from an exception with eret. 273 */ 274 mov x2, #(PSR_DAIF | PSR_M_EL1h) 275 msr spsr_el1, x2 276 msr elr_el1, lr 277 eret 278 2791: 280 dsb sy 281 /* 282 * Set just the reserved bits in sctlr_el2. This will disable the 283 * MMU which may have broken the kernel if we enter the kernel in 284 * EL2, e.g. when using VHE. 285 */ 286 ldr x2, =(SCTLR_EL2_RES1 | SCTLR_EL2_EIS | SCTLR_EL2_EOS) 287 msr sctlr_el2, x2 288 isb 289 290 /* Configure the Hypervisor */ 291 ldr x2, =(HCR_RW | HCR_APK | HCR_API) 292 msr hcr_el2, x2 293 294 /* Stash value of HCR_EL2 for later */ 295 isb 296 mrs x4, hcr_el2 297 298 /* Load the Virtualization Process ID Register */ 299 mrs x2, midr_el1 300 msr vpidr_el2, x2 301 302 /* Load the Virtualization Multiprocess ID Register */ 303 mrs x2, mpidr_el1 304 msr vmpidr_el2, x2 305 306 /* Set the initial sctlr_el1 */ 307 ldr x2, =INIT_SCTLR_EL1 308 msr sctlr_el1, x2 309 310 /* 311 * On some hardware, e.g., Apple M1, we can't clear E2H, so make sure we 312 * don't trap to EL2 for SIMD register usage to have at least a 313 * minimally usable system. 314 */ 315 tst x4, #HCR_E2H 316 mov x3, #CPTR_RES1 /* HCR_E2H == 0 */ 317 mov x5, #CPTR_FPEN /* HCR_E2H == 1 */ 318 csel x2, x3, x5, eq 319 msr cptr_el2, x2 320 321 /* Don't trap to EL2 for CP15 traps */ 322 msr hstr_el2, xzr 323 324 /* Enable access to the physical timers at EL1 */ 325 tst x4, #HCR_E2H 326 ldr x3, =(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 327 ldr x5, =(CNTHCTL_E2H_EL1PCTEN | CNTHCTL_E2H_EL1PTEN) 328 csel x2, x3, x5, eq 329 msr cnthctl_el2, x2 330 331 /* Set the counter offset to a known value */ 332 msr cntvoff_el2, xzr 333 334 /* Hypervisor trap functions */ 335 adrp x2, hyp_stub_vectors 336 add x2, x2, :lo12:hyp_stub_vectors 337 msr vbar_el2, x2 338 339 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ 340 msr vttbr_el2, xzr 341 342 mov x2, #(PSR_DAIF | PSR_M_EL1h) 343 msr spsr_el2, x2 344 345 /* Configure GICv3 CPU interface */ 346 mrs x2, id_aa64pfr0_el1 347 /* Extract GIC bits from the register */ 348 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 349 /* GIC[3:0] != 0000 - GIC CPU interface via special regs. supported */ 350 cbz x2, 2f 351 352 mrs x2, icc_sre_el2 353 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 354 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 355 msr icc_sre_el2, x2 3562: 357 358 /* Set the address to return to our return address */ 359 msr elr_el2, x30 360 isb 361 362 eret 363#undef INIT_SCTLR_EL1 364LEND(enter_kernel_el) 365 366/* 367 * Get the physical address the kernel was loaded at. 368 */ 369LENTRY(get_load_phys_addr) 370 /* Load the offset of get_load_phys_addr from KERNBASE */ 371 ldr x28, =(get_load_phys_addr - KERNBASE) 372 /* Load the physical address of get_load_phys_addr */ 373 adr x29, get_load_phys_addr 374 /* Find the physical address of KERNBASE, i.e. our load address */ 375 sub x28, x29, x28 376 ret 377LEND(get_load_phys_addr) 378 379/* 380 * This builds the page tables containing the identity map, and the kernel 381 * virtual map. 382 * 383 * It relys on: 384 * We were loaded to an address that is on a 2MiB boundary 385 * All the memory must not cross a 1GiB boundaty 386 * x28 contains the physical address we were loaded from 387 * 388 * TODO: This is out of date. 389 * There are at least 5 pages before that address for the page tables 390 * The pages used are: 391 * - The Kernel L2 table 392 * - The Kernel L1 table 393 * - The Kernel L0 table (TTBR1) 394 * - The identity (PA = VA) L1 table 395 * - The identity (PA = VA) L0 table (TTBR0) 396 */ 397LENTRY(create_pagetables) 398 /* Save the Link register */ 399 mov x5, x30 400 401 /* Clean the page table */ 402 adrp x6, pagetable 403 add x6, x6, :lo12:pagetable 404 mov x26, x6 405 adrp x27, pagetable_end 406 add x27, x27, :lo12:pagetable_end 4071: 408 stp xzr, xzr, [x6], #16 409 stp xzr, xzr, [x6], #16 410 stp xzr, xzr, [x6], #16 411 stp xzr, xzr, [x6], #16 412 cmp x6, x27 413 b.lo 1b 414 415 /* 416 * Build the TTBR1 maps. 417 */ 418 419 /* Find the size of the kernel */ 420 mov x6, #(KERNBASE) 421 422#if defined(LINUX_BOOT_ABI) 423 /* X19 is used as 'map FDT data' flag */ 424 mov x19, xzr 425 426 /* No modules or FDT pointer ? */ 427 cbz x0, booti_no_fdt 428 429 /* 430 * Test if x0 points to modules descriptor(virtual address) or 431 * to FDT (physical address) 432 */ 433 cmp x0, x6 /* x6 is #(KERNBASE) */ 434 b.lo booti_fdt 435#endif 436 437 /* Booted with modules pointer */ 438 /* Find modulep - begin */ 439 sub x8, x0, x6 440 /* 441 * Add space for the module data. When PAGE_SIZE is 4k this will 442 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 443 * larger it will be at least as large as we use smaller level 3 444 * pages. 445 */ 446 ldr x7, =((6 * 1024 * 1024) - 1) 447 add x8, x8, x7 448 b common 449 450#if defined(LINUX_BOOT_ABI) 451booti_fdt: 452 /* Booted by U-Boot booti with FDT data */ 453 /* Set 'map FDT data' flag */ 454 mov x19, #1 455 456booti_no_fdt: 457 /* Booted by U-Boot booti without FTD data */ 458 /* Find the end - begin */ 459 ldr x7, .Lend 460 sub x8, x7, x6 461 462 /* 463 * Add one 2MiB page for copy of FDT data (maximum FDT size), 464 * one for metadata and round up 465 */ 466 ldr x7, =(3 * L2_SIZE - 1) 467 add x8, x8, x7 468#endif 469 470common: 471#if PAGE_SIZE != PAGE_SIZE_4K 472 /* 473 * Create L3 pages. The kernel will be loaded at a 2M aligned 474 * address, however L2 blocks are too large when the page size is 475 * not 4k to map the kernel with such an aligned address. However, 476 * when the page size is larger than 4k, L2 blocks are too large to 477 * map the kernel with such an alignment. 478 */ 479 480 /* Get the number of l3 pages to allocate, rounded down */ 481 lsr x10, x8, #(L3_SHIFT) 482 483 /* Create the kernel space L2 table */ 484 mov x6, x26 485 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 486 mov x8, #(KERNBASE) 487 mov x9, x28 488 bl build_l3_page_pagetable 489 490 /* Move to the l2 table */ 491 ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT) 492 add x26, x26, x9 493 494 /* Link the l2 -> l3 table */ 495 mov x9, x6 496 mov x6, x26 497 bl link_l2_pagetable 498#else 499 /* Get the number of l2 pages to allocate, rounded down */ 500 lsr x10, x8, #(L2_SHIFT) 501 502 /* Create the kernel space L2 table */ 503 mov x6, x26 504 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 505 mov x8, #(KERNBASE) 506 mov x9, x28 507 bl build_l2_block_pagetable 508#endif 509 510 /* Move to the l1 table */ 511 add x26, x26, #PAGE_SIZE 512 513 /* Link the l1 -> l2 table */ 514 mov x9, x6 515 mov x6, x26 516 bl link_l1_pagetable 517 518 /* Move to the l0 table */ 519 add x24, x26, #PAGE_SIZE 520 521 /* Link the l0 -> l1 table */ 522 mov x9, x6 523 mov x6, x24 524 mov x10, #1 525 bl link_l0_pagetable 526 527 /* 528 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 529 * They are only needed early on, so the VA = PA map is uncached. 530 */ 531 add x27, x24, #PAGE_SIZE 532 533 mov x6, x27 /* The initial page table */ 534 535 /* Create the VA = PA map */ 536 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 537 adrp x16, _start 538 and x16, x16, #(~L2_OFFSET) 539 mov x9, x16 /* PA start */ 540 mov x8, x16 /* VA start (== PA start) */ 541 mov x10, #1 542 bl build_l2_block_pagetable 543 544#if defined(SOCDEV_PA) 545 /* Create a table for the UART */ 546 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 547 ldr x9, =(L2_SIZE) 548 add x16, x16, x9 /* VA start */ 549 mov x8, x16 550 551 /* Store the socdev virtual address */ 552 add x17, x8, #(SOCDEV_PA & L2_OFFSET) 553 adrp x9, socdev_va 554 str x17, [x9, :lo12:socdev_va] 555 556 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ 557 mov x10, #1 558 bl build_l2_block_pagetable 559#endif 560 561#if defined(LINUX_BOOT_ABI) 562 /* Map FDT data ? */ 563 cbz x19, 1f 564 565 /* Create the mapping for FDT data (2 MiB max) */ 566 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 567 ldr x9, =(L2_SIZE) 568 add x16, x16, x9 /* VA start */ 569 mov x8, x16 570 mov x9, x0 /* PA start */ 571 /* Update the module pointer to point at the allocated memory */ 572 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 573 add x0, x0, x8 /* Add the aligned virtual address */ 574 575 mov x10, #1 576 bl build_l2_block_pagetable 577 5781: 579#endif 580 581 /* Move to the l1 table */ 582 add x27, x27, #PAGE_SIZE 583 584 /* Link the l1 -> l2 table */ 585 mov x9, x6 586 mov x6, x27 587 bl link_l1_pagetable 588 589 /* Move to the l0 table */ 590 add x27, x27, #PAGE_SIZE 591 592 /* Link the l0 -> l1 table */ 593 mov x9, x6 594 mov x6, x27 595 mov x10, #1 596 bl link_l0_pagetable 597 598 /* Restore the Link register */ 599 mov x30, x5 600 ret 601LEND(create_pagetables) 602 603/* 604 * Builds an L0 -> L1 table descriptor 605 * 606 * x6 = L0 table 607 * x8 = Virtual Address 608 * x9 = L1 PA (trashed) 609 * x10 = Entry count (trashed) 610 * x11, x12 and x13 are trashed 611 */ 612LENTRY(link_l0_pagetable) 613 /* 614 * Link an L0 -> L1 table entry. 615 */ 616 /* Find the table index */ 617 lsr x11, x8, #L0_SHIFT 618 and x11, x11, #L0_ADDR_MASK 619 620 /* Build the L0 block entry */ 621 mov x12, #L0_TABLE 622 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 623 624 /* Only use the output address bits */ 625 lsr x9, x9, #PAGE_SHIFT 6261: orr x13, x12, x9, lsl #PAGE_SHIFT 627 628 /* Store the entry */ 629 str x13, [x6, x11, lsl #3] 630 631 sub x10, x10, #1 632 add x11, x11, #1 633 add x9, x9, #1 634 cbnz x10, 1b 635 636 ret 637LEND(link_l0_pagetable) 638 639/* 640 * Builds an L1 -> L2 table descriptor 641 * 642 * x6 = L1 table 643 * x8 = Virtual Address 644 * x9 = L2 PA (trashed) 645 * x11, x12 and x13 are trashed 646 */ 647LENTRY(link_l1_pagetable) 648 /* 649 * Link an L1 -> L2 table entry. 650 */ 651 /* Find the table index */ 652 lsr x11, x8, #L1_SHIFT 653 and x11, x11, #Ln_ADDR_MASK 654 655 /* Build the L1 block entry */ 656 mov x12, #L1_TABLE 657 658 /* Only use the output address bits */ 659 lsr x9, x9, #PAGE_SHIFT 660 orr x13, x12, x9, lsl #PAGE_SHIFT 661 662 /* Store the entry */ 663 str x13, [x6, x11, lsl #3] 664 665 ret 666LEND(link_l1_pagetable) 667 668/* 669 * Builds count 2 MiB page table entry 670 * x6 = L2 table 671 * x7 = Block attributes 672 * x8 = VA start 673 * x9 = PA start (trashed) 674 * x10 = Entry count (trashed) 675 * x11, x12 and x13 are trashed 676 */ 677LENTRY(build_l2_block_pagetable) 678 /* 679 * Build the L2 table entry. 680 */ 681 /* Find the table index */ 682 lsr x11, x8, #L2_SHIFT 683 and x11, x11, #Ln_ADDR_MASK 684 685 /* Build the L2 block entry */ 686 orr x12, x7, #L2_BLOCK 687 orr x12, x12, #(ATTR_DEFAULT) 688 orr x12, x12, #(ATTR_S1_UXN) 689 690 /* Only use the output address bits */ 691 lsr x9, x9, #L2_SHIFT 692 693 /* Set the physical address for this virtual address */ 6941: orr x13, x12, x9, lsl #L2_SHIFT 695 696 /* Store the entry */ 697 str x13, [x6, x11, lsl #3] 698 699 sub x10, x10, #1 700 add x11, x11, #1 701 add x9, x9, #1 702 cbnz x10, 1b 703 704 ret 705LEND(build_l2_block_pagetable) 706 707#if PAGE_SIZE != PAGE_SIZE_4K 708/* 709 * Builds an L2 -> L3 table descriptor 710 * 711 * x6 = L2 table 712 * x8 = Virtual Address 713 * x9 = L3 PA (trashed) 714 * x11, x12 and x13 are trashed 715 */ 716LENTRY(link_l2_pagetable) 717 /* 718 * Link an L2 -> L3 table entry. 719 */ 720 /* Find the table index */ 721 lsr x11, x8, #L2_SHIFT 722 and x11, x11, #Ln_ADDR_MASK 723 724 /* Build the L1 block entry */ 725 mov x12, #L2_TABLE 726 727 /* Only use the output address bits */ 728 lsr x9, x9, #PAGE_SHIFT 729 orr x13, x12, x9, lsl #PAGE_SHIFT 730 731 /* Store the entry */ 732 str x13, [x6, x11, lsl #3] 733 734 ret 735LEND(link_l2_pagetable) 736 737/* 738 * Builds count level 3 page table entries 739 * x6 = L3 table 740 * x7 = Block attributes 741 * x8 = VA start 742 * x9 = PA start (trashed) 743 * x10 = Entry count (trashed) 744 * x11, x12 and x13 are trashed 745 */ 746LENTRY(build_l3_page_pagetable) 747 /* 748 * Build the L3 table entry. 749 */ 750 /* Find the table index */ 751 lsr x11, x8, #L3_SHIFT 752 and x11, x11, #Ln_ADDR_MASK 753 754 /* Build the L3 page entry */ 755 orr x12, x7, #L3_PAGE 756 orr x12, x12, #(ATTR_DEFAULT) 757 orr x12, x12, #(ATTR_S1_UXN) 758 759 /* Only use the output address bits */ 760 lsr x9, x9, #L3_SHIFT 761 762 /* Set the physical address for this virtual address */ 7631: orr x13, x12, x9, lsl #L3_SHIFT 764 765 /* Store the entry */ 766 str x13, [x6, x11, lsl #3] 767 768 sub x10, x10, #1 769 add x11, x11, #1 770 add x9, x9, #1 771 cbnz x10, 1b 772 773 ret 774LEND(build_l3_page_pagetable) 775#endif 776 777LENTRY(start_mmu) 778 dsb sy 779 780 /* Load the exception vectors */ 781 ldr x2, =exception_vectors 782 msr vbar_el1, x2 783 784 /* Load ttbr0 and ttbr1 */ 785 msr ttbr0_el1, x27 786 msr ttbr1_el1, x24 787 isb 788 789 /* Clear the Monitor Debug System control register */ 790 msr mdscr_el1, xzr 791 792 /* Invalidate the TLB */ 793 tlbi vmalle1is 794 dsb ish 795 isb 796 797 ldr x2, mair 798 msr mair_el1, x2 799 800 /* 801 * Setup TCR according to the PARange and ASIDBits fields 802 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 803 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 804 * to 1 only if the ASIDBits field equals 0b0010. 805 */ 806 ldr x2, tcr 807 mrs x3, id_aa64mmfr0_el1 808 809 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 810 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 811 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 812 813 /* Check if the HW supports 16 bit ASIDS */ 814 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 815 /* If so x3 == 1, else x3 == 0 */ 816 cset x3, eq 817 /* Set TCR.AS with x3 */ 818 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 819 820 /* 821 * Check if the HW supports access flag and dirty state updates, 822 * and set TCR_EL1.HA and TCR_EL1.HD accordingly. 823 */ 824 mrs x3, id_aa64mmfr1_el1 825 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK) 826 cmp x3, #1 827 b.ne 1f 828 orr x2, x2, #(TCR_HA) 829 b 2f 8301: 831 cmp x3, #2 832 b.ne 2f 833 orr x2, x2, #(TCR_HA | TCR_HD) 8342: 835 msr tcr_el1, x2 836 837 /* 838 * Setup SCTLR. 839 */ 840 ldr x2, sctlr_set 841 ldr x3, sctlr_clear 842 mrs x1, sctlr_el1 843 bic x1, x1, x3 /* Clear the required bits */ 844 orr x1, x1, x2 /* Set the required bits */ 845 msr sctlr_el1, x1 846 isb 847 848 ret 849 850 .align 3 851mair: 852 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 853 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 854 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 855 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 856 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) 857tcr: 858#if PAGE_SIZE == PAGE_SIZE_4K 859#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 860#elif PAGE_SIZE == PAGE_SIZE_16K 861#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 862#else 863#error Unsupported page size 864#endif 865 866 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 867 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 868sctlr_set: 869 /* Bits to set */ 870 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 871 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 872 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 873 SCTLR_M | SCTLR_CP15BEN | SCTLR_BT1 | SCTLR_BT0) 874sctlr_clear: 875 /* Bits to clear */ 876 .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 877 SCTLR_ITD | SCTLR_A) 878LEND(start_mmu) 879 880ENTRY(abort) 881 b abort 882END(abort) 883 884.bss 885 .align PAGE_SHIFT 886initstack: 887 .space BOOT_STACK_SIZE 888initstack_end: 889 890 .section .init_pagetable, "aw", %nobits 891 .align PAGE_SHIFT 892 /* 893 * 6 initial tables (in the following order): 894 * L2 for kernel (High addresses) 895 * L1 for kernel 896 * L0 for kernel 897 * L1 bootstrap for user (Low addresses) 898 * L0 bootstrap for user 899 * L0 for user 900 */ 901 .globl pagetable_l0_ttbr1 902pagetable: 903#if PAGE_SIZE != PAGE_SIZE_4K 904 .space (PAGE_SIZE * L3_PAGE_COUNT) 905pagetable_l2_ttbr1: 906#endif 907 .space PAGE_SIZE 908pagetable_l1_ttbr1: 909 .space PAGE_SIZE 910pagetable_l0_ttbr1: 911 .space PAGE_SIZE 912pagetable_l2_ttbr0_bootstrap: 913 .space PAGE_SIZE 914pagetable_l1_ttbr0_bootstrap: 915 .space PAGE_SIZE 916pagetable_l0_ttbr0_bootstrap: 917 .space PAGE_SIZE 918pagetable_l0_ttbr0: 919 .space PAGE_SIZE 920pagetable_end: 921 922el2_pagetable: 923 .space PAGE_SIZE 924 925 .section .rodata, "a", %progbits 926 .globl aarch32_sigcode 927 .align 2 928aarch32_sigcode: 929 .word 0xe1a0000d // mov r0, sp 930 .word 0xe2800040 // add r0, r0, #SIGF_UC 931 .word 0xe59f700c // ldr r7, [pc, #12] 932 .word 0xef000000 // swi #0 933 .word 0xe59f7008 // ldr r7, [pc, #8] 934 .word 0xef000000 // swi #0 935 .word 0xeafffffa // b . - 16 936 .word SYS_sigreturn 937 .word SYS_exit 938 .align 3 939 .size aarch32_sigcode, . - aarch32_sigcode 940aarch32_esigcode: 941 .data 942 .global sz_aarch32_sigcode 943sz_aarch32_sigcode: 944 .quad aarch32_esigcode - aarch32_sigcode 945