1/*- 2 * Copyright 2004-2014 Olivier Houchard <[email protected]> 3 * Copyright 2012-2014 Ian Lepore <[email protected]> 4 * Copyright 2013-2014 Andrew Turner <[email protected]> 5 * Copyright 2014 Svatopluk Kraus <[email protected]> 6 * Copyright 2014 Michal Meloun <[email protected]> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31#include "assym.inc" 32#include <sys/syscall.h> 33#include <machine/asm.h> 34#include <machine/asmacros.h> 35#include <machine/armreg.h> 36#include <machine/sysreg.h> 37#include <machine/pte.h> 38/* We map 64MB of kernel unless overridden in assym.inc by the kernel option. */ 39#ifndef LOCORE_MAP_MB 40#define LOCORE_MAP_MB 64 41#endif 42 43#if __ARM_ARCH >= 7 44#if defined(__ARM_ARCH_7VE__) || defined(__clang__) 45/* 46 * HYP support is in bintuils >= 2.21 and gcc >= 4.9 defines __ARM_ARCH_7VE__ 47 * when enabled. llvm >= 3.6 supports it too. 48 */ 49.arch_extension virt 50#endif 51#endif /* __ARM_ARCH >= 7 */ 52 53/* A small statically-allocated stack used only during initarm() and AP startup. */ 54#define INIT_ARM_STACK_SIZE 2048 55 56 .text 57 .align 2 58 59 .globl kernbase 60 .set kernbase,KERNVIRTADDR 61 62#if __ARM_ARCH >= 7 63#define HANDLE_HYP \ 64 /* Leave HYP mode */ ;\ 65 mrs r0, cpsr ;\ 66 and r0, r0, #(PSR_MODE) /* Mode is in the low 5 bits of CPSR */ ;\ 67 teq r0, #(PSR_HYP32_MODE) /* Hyp Mode? */ ;\ 68 bne 1f ;\ 69 /* Install Hypervisor Stub Exception Vector */ ;\ 70 bl hypervisor_stub_vect_install ;\ 71 mov r0, 0 ;\ 72 adr r1, hypmode_enabled ;\ 73 str r0, [r1] ;\ 74 /* Ensure that IRQ, FIQ and Aborts will be disabled after eret */ ;\ 75 mrs r0, cpsr ;\ 76 bic r0, r0, #(PSR_MODE) ;\ 77 orr r0, r0, #(PSR_SVC32_MODE) ;\ 78 orr r0, r0, #(PSR_I | PSR_F | PSR_A) ;\ 79 msr spsr_cxsf, r0 ;\ 80 /* Exit hypervisor mode */ ;\ 81 adr lr, 2f ;\ 82 MSR_ELR_HYP(14) ;\ 83 ERET ;\ 841: ;\ 85 mov r0, -1 ;\ 86 adr r1, hypmode_enabled ;\ 87 str r0, [r1] ;\ 882: 89#else 90#define HANDLE_HYP 91#endif /* __ARM_ARCH >= 7 */ 92 93/* 94 * On entry for FreeBSD boot ABI: 95 * r0 - metadata pointer or 0 (boothowto on AT91's boot2) 96 * r1 - if (r0 == 0) then metadata pointer 97 * On entry for Linux boot ABI: 98 * r0 - 0 99 * r1 - machine type (passed as arg2 to initarm) 100 * r2 - Pointer to a tagged list or dtb image (phys addr) (passed as arg1 initarm) 101 * 102 * For both types of boot we gather up the args, put them in a struct arm_boot_params 103 * structure and pass that to initarm. 104 */ 105 .globl btext 106btext: 107ASENTRY_NP(_start) 108 STOP_UNWINDING /* Can't unwind into the bootloader! */ 109 110 /* Make sure interrupts are disabled. */ 111 cpsid ifa 112 113 mov r8, r0 /* 0 or boot mode from boot2 */ 114 mov r9, r1 /* Save Machine type */ 115 mov r10, r2 /* Save meta data */ 116 mov r11, r3 /* Future expansion */ 117 118 # If HYP-MODE is active, install an exception vector stub 119 HANDLE_HYP 120 121 /* 122 * Check whether data cache is enabled. If it is, then we know 123 * current tags are valid (not power-on garbage values) and there 124 * might be dirty lines that need cleaning. Disable cache to prevent 125 * new lines being allocated, then call wbinv_poc_all to clean it. 126 */ 127 mrc CP15_SCTLR(r7) 128 tst r7, #CPU_CONTROL_DC_ENABLE 129 blne dcache_wbinv_poc_all 130 131 /* ! Do not write to memory between wbinv and disabling cache ! */ 132 133 /* 134 * Now there are no dirty lines, but there may still be lines marked 135 * valid. Disable all caches and the MMU, and invalidate everything 136 * before setting up new page tables and re-enabling the mmu. 137 */ 1381: 139 bic r7, #CPU_CONTROL_DC_ENABLE 140 bic r7, #CPU_CONTROL_AFLT_ENABLE 141 bic r7, #CPU_CONTROL_MMU_ENABLE 142 bic r7, #CPU_CONTROL_IC_ENABLE 143 bic r7, #CPU_CONTROL_BPRD_ENABLE 144 bic r7, #CPU_CONTROL_SW_ENABLE 145 orr r7, #CPU_CONTROL_UNAL_ENABLE 146 orr r7, #CPU_CONTROL_VECRELOC 147 mcr CP15_SCTLR(r7) 148 DSB 149 ISB 150 bl dcache_inv_poc_all 151 mcr CP15_ICIALLU 152 DSB 153 ISB 154 155 /* 156 * Build page table from scratch. 157 */ 158 159 /* 160 * Figure out the physical address we're loaded at by assuming this 161 * entry point code is in the first L1 section and so if we clear the 162 * offset bits of the pc that will give us the section-aligned load 163 * address, which remains in r5 throughout all the following code. 164 */ 165 ldr r2, =(L1_S_OFFSET) 166 bic r5, pc, r2 167 168 /* Find the delta between VA and PA, result stays in r0 throughout. */ 169 adr r0, Lpagetable 170 bl translate_va_to_pa 171 172 /* 173 * First map the entire 4GB address space as VA=PA. It's mapped as 174 * normal (cached) memory because it's for things like accessing the 175 * parameters passed in from the bootloader, which might be at any 176 * physical address, different for every platform. 177 */ 178 mov r1, #0 179 mov r2, #0 180 mov r3, #4096 181 bl build_pagetables 182 183 /* 184 * Next we map the kernel starting at the physical load address, mapped 185 * to the VA the kernel is linked for. The default size we map is 64MiB 186 * but it can be overridden with a kernel option. 187 */ 188 mov r1, r5 189 ldr r2, =(KERNVIRTADDR) 190 ldr r3, =(LOCORE_MAP_MB) 191 bl build_pagetables 192 193 /* Create a device mapping for early_printf if specified. */ 194#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 195 ldr r1, =SOCDEV_PA 196 ldr r2, =SOCDEV_VA 197 mov r3, #1 198 bl build_device_pagetables 199#endif 200 bl init_mmu 201 202 /* Transition the PC from physical to virtual addressing. */ 203 ldr pc, =1f 2041: 205 206 /* Setup stack, clear BSS */ 207 ldr r1, =.Lstart 208 ldmia r1, {r1, r2, sp} /* Set initial stack and */ 209 add sp, sp, #INIT_ARM_STACK_SIZE 210 sub r2, r2, r1 /* get zero init data */ 211 mov r3, #0 2122: 213 str r3, [r1], #0x0004 /* get zero init data */ 214 subs r2, r2, #4 215 bgt 2b 216 217 mov r1, #28 /* loader info size is 28 bytes also second arg */ 218 subs sp, sp, r1 /* allocate arm_boot_params struct on stack */ 219 mov r0, sp /* loader info pointer is first arg */ 220 bic sp, sp, #7 /* align stack to 8 bytes */ 221 str r1, [r0] /* Store length of loader info */ 222 str r8, [r0, #4] /* Store r0 from boot loader */ 223 str r9, [r0, #8] /* Store r1 from boot loader */ 224 str r10, [r0, #12] /* store r2 from boot loader */ 225 str r11, [r0, #16] /* store r3 from boot loader */ 226 str r5, [r0, #20] /* store the physical address */ 227 adr r4, Lpagetable /* load the pagetable address */ 228 ldr r5, [r4, #4] 229 str r5, [r0, #24] /* store the pagetable address */ 230 mov fp, #0 /* trace back starts here */ 231 bl _C_LABEL(initarm) /* Off we go */ 232 233 /* init arm will return the new stack pointer. */ 234 mov sp, r0 235 236 bl _C_LABEL(mi_startup) /* call mi_startup()! */ 237 238 ldr r0, =.Lmainreturned 239 b _C_LABEL(panic) 240 /* NOTREACHED */ 241END(_start) 242 243#define VA_TO_PA_POINTER(name, table) \ 244name: ;\ 245 .word . ;\ 246 .word table 247 248/* 249 * Returns the physical address of a magic va to pa pointer. 250 * r0 - The pagetable data pointer. This must be built using the 251 * VA_TO_PA_POINTER macro. 252 * e.g. 253 * VA_TO_PA_POINTER(Lpagetable, pagetable) 254 * ... 255 * adr r0, Lpagetable 256 * bl translate_va_to_pa 257 * r0 will now contain the physical address of pagetable 258 * r1, r2 - Trashed 259 */ 260translate_va_to_pa: 261 ldr r1, [r0] 262 sub r2, r1, r0 263 /* At this point: r2 = VA - PA */ 264 265 /* 266 * Find the physical address of the table. After these two 267 * instructions: 268 * r1 = va(pagetable) 269 * 270 * r0 = va(pagetable) - (VA - PA) 271 * = va(pagetable) - VA + PA 272 * = pa(pagetable) 273 */ 274 ldr r1, [r0, #4] 275 sub r0, r1, r2 276 mov pc, lr 277 278/* 279 * Init MMU 280 * r0 - the table base address 281 */ 282 283ASENTRY_NP(init_mmu) 284 285 /* Setup TLB and MMU registers */ 286 mcr CP15_TTBR0(r0) /* Set TTB */ 287 mov r0, #0 288 mcr CP15_CONTEXTIDR(r0) /* Set ASID to 0 */ 289 290 /* Set the Domain Access register */ 291 mov r0, #DOMAIN_CLIENT /* Only domain #0 is used */ 292 mcr CP15_DACR(r0) 293 294 /* 295 * Ensure that LPAE is disabled and that TTBR0 is used for translation, 296 * use a 16KB translation table 297 */ 298 mov r0, #0 299 mcr CP15_TTBCR(r0) 300 301 /* 302 * Set TEX remap registers 303 * - All is set to uncacheable memory 304 */ 305 ldr r0, =0xAAAAA 306 mcr CP15_PRRR(r0) 307 mov r0, #0 308 mcr CP15_NMRR(r0) 309 mcr CP15_TLBIALL /* Flush TLB */ 310 DSB 311 ISB 312 313 /* Enable MMU */ 314 mrc CP15_SCTLR(r0) 315 orr r0, r0, #CPU_CONTROL_MMU_ENABLE 316 orr r0, r0, #CPU_CONTROL_V6_EXTPAGE 317 orr r0, r0, #CPU_CONTROL_TR_ENABLE 318 orr r0, r0, #CPU_CONTROL_AF_ENABLE 319 mcr CP15_SCTLR(r0) 320 DSB 321 ISB 322 mcr CP15_TLBIALL /* Flush TLB */ 323 mcr CP15_BPIALL /* Flush Branch predictor */ 324 DSB 325 ISB 326 327 mov pc, lr 328END(init_mmu) 329 330 331/* 332 * Init SMP coherent mode, enable caching and switch to final MMU table. 333 * Called with disabled caches 334 * r0 - The table base address 335 * r1 - clear bits for aux register 336 * r2 - set bits for aux register 337 */ 338ASENTRY_NP(reinit_mmu) 339 push {r4-r11, lr} 340 mov r4, r0 341 mov r5, r1 342 mov r6, r2 343 344 /* !! Be very paranoid here !! */ 345 /* !! We cannot write single bit here !! */ 346 347#if 0 /* XXX writeback shouldn't be necessary */ 348 /* Write back and invalidate all integrated caches */ 349 bl dcache_wbinv_poc_all 350#else 351 bl dcache_inv_pou_all 352#endif 353 mcr CP15_ICIALLU 354 DSB 355 ISB 356 357 /* Set auxiliary register */ 358 mrc CP15_ACTLR(r7) 359 bic r8, r7, r5 /* Mask bits */ 360 eor r8, r8, r6 /* Set bits */ 361 teq r7, r8 362 mcrne CP15_ACTLR(r8) 363 DSB 364 ISB 365 366 /* Enable caches. */ 367 mrc CP15_SCTLR(r7) 368 orr r7, #CPU_CONTROL_DC_ENABLE 369 orr r7, #CPU_CONTROL_IC_ENABLE 370 orr r7, #CPU_CONTROL_BPRD_ENABLE 371 mcr CP15_SCTLR(r7) 372 DSB 373 374 mcr CP15_TTBR0(r4) /* Set new TTB */ 375 DSB 376 ISB 377 378 mcr CP15_TLBIALL /* Flush TLB */ 379 mcr CP15_BPIALL /* Flush Branch predictor */ 380 DSB 381 ISB 382 383#if 0 /* XXX writeback shouldn't be necessary */ 384 /* Write back and invalidate all integrated caches */ 385 bl dcache_wbinv_poc_all 386#else 387 bl dcache_inv_pou_all 388#endif 389 mcr CP15_ICIALLU 390 DSB 391 ISB 392 393 pop {r4-r11, pc} 394END(reinit_mmu) 395 396 397/* 398 * Builds the page table 399 * r0 - The table base address 400 * r1 - The physical address (trashed) 401 * r2 - The virtual address (trashed) 402 * r3 - The number of 1MiB sections 403 * r4 - Trashed 404 * 405 * Addresses must be 1MiB aligned 406 */ 407build_device_pagetables: 408 ldr r4, =PTE1_V|PTE1_A|PTE1_AP_KRW|TEX1_CLASS_0 409 b 1f 410build_pagetables: 411 /* Set the required page attributed */ 412 ldr r4, =PTE1_V|PTE1_A|PTE1_AP_KRW|TEX1_CLASS_0 4131: 414 orr r1, r4 415 416 /* Move the virtual address to the correct bit location */ 417 lsr r2, #(PTE1_SHIFT - 2) 418 419 mov r4, r3 4202: 421 str r1, [r0, r2] 422 add r2, r2, #4 423 add r1, r1, #(PTE1_SIZE) 424 adds r4, r4, #-1 425 bhi 2b 426 427 mov pc, lr 428 429VA_TO_PA_POINTER(Lpagetable, boot_pt1) 430 431 .global _C_LABEL(hypmode_enabled) 432_C_LABEL(hypmode_enabled): 433 .word 0 434 435.Lstart: 436 .word _edata /* Note that these three items are */ 437 .word _ebss /* loaded with a single ldmia and */ 438 .word svcstk /* must remain in order together. */ 439 440.Lmainreturned: 441 .asciz "main() returned" 442 .align 2 443 444 .bss 445svcstk: 446 .space INIT_ARM_STACK_SIZE * MAXCPU 447 448/* 449 * Memory for the initial pagetable. We are unable to place this in 450 * the bss as this will be cleared after the table is loaded. 451 */ 452 .section ".init_pagetable", "aw", %nobits 453 .align 14 /* 16KiB aligned */ 454 .globl boot_pt1 455boot_pt1: 456 .space L1_TABLE_SIZE 457 458 .text 459 .align 2 460 461#if defined(SMP) 462 463ASENTRY_NP(mpentry) 464 /* Make sure interrupts are disabled. */ 465 cpsid ifa 466 467 HANDLE_HYP 468 469 /* Setup core, disable all caches. */ 470 mrc CP15_SCTLR(r0) 471 bic r0, #CPU_CONTROL_MMU_ENABLE 472 bic r0, #CPU_CONTROL_AFLT_ENABLE 473 bic r0, #CPU_CONTROL_DC_ENABLE 474 bic r0, #CPU_CONTROL_IC_ENABLE 475 bic r0, #CPU_CONTROL_BPRD_ENABLE 476 bic r0, #CPU_CONTROL_SW_ENABLE 477 orr r0, #CPU_CONTROL_UNAL_ENABLE 478 orr r0, #CPU_CONTROL_VECRELOC 479 mcr CP15_SCTLR(r0) 480 DSB 481 ISB 482 483 /* Invalidate L1 cache I+D cache */ 484 bl dcache_inv_pou_all 485 mcr CP15_ICIALLU 486 DSB 487 ISB 488 489 /* Find the delta between VA and PA */ 490 adr r0, Lpagetable 491 bl translate_va_to_pa 492 493 bl init_mmu 494 495 adr r1, .Lstart+8 /* Get initstack pointer from */ 496 ldr sp, [r1] /* startup data. */ 497 mrc CP15_MPIDR(r0) /* Get processor id number. */ 498 and r0, r0, #0x0f 499 mov r1, #INIT_ARM_STACK_SIZE 500 mul r2, r1, r0 /* Point sp to initstack */ 501 add sp, sp, r2 /* area for this processor. */ 502 503 /* Switch to virtual addresses. */ 504 ldr pc, =1f 5051: 506 mov fp, #0 /* trace back starts here */ 507 bl _C_LABEL(init_secondary)/* Off we go, cpu id in r0. */ 508 509 adr r0, .Lmpreturned 510 b _C_LABEL(panic) 511 /* NOTREACHED */ 512END(mpentry) 513 514.Lmpreturned: 515 .asciz "init_secondary() returned" 516 .align 2 517#endif 518 519ENTRY_NP(cpu_halt) 520 521 /* XXX re-implement !!! */ 522 cpsid ifa 523 bl dcache_wbinv_poc_all 524 525 ldr r4, .Lcpu_reset_address 526 ldr r4, [r4] 527 teq r4, #0 528 movne pc, r4 5291: 530 WFI 531 b 1b 532 533 /* 534 * _cpu_reset_address contains the address to branch to, to complete 535 * the cpu reset after turning the MMU off 536 * This variable is provided by the hardware specific code 537 */ 538.Lcpu_reset_address: 539 .word _C_LABEL(cpu_reset_address) 540END(cpu_halt) 541 542 543/* 544 * setjump + longjmp 545 */ 546ENTRY(setjmp) 547 stmia r0, {r4-r14} 548 mov r0, #0x00000000 549 RET 550END(setjmp) 551 552ENTRY(longjmp) 553 ldmia r0, {r4-r14} 554 mov r0, #0x00000001 555 RET 556END(longjmp) 557 558 .data 559 .global _C_LABEL(esym) 560_C_LABEL(esym): .word _C_LABEL(end) 561 562ENTRY_NP(abort) 563 b _C_LABEL(abort) 564END(abort) 565 566ENTRY_NP(sigcode) 567 mov r0, sp 568 add r0, r0, #SIGF_UC 569 570 /* 571 * Call the sigreturn system call. 572 * 573 * We have to load r7 manually rather than using 574 * "ldr r7, =SYS_sigreturn" to ensure the value of szsigcode is 575 * correct. Using the alternative places esigcode at the address 576 * of the data rather than the address one past the data. 577 */ 578 579 ldr r7, [pc, #12] /* Load SYS_sigreturn */ 580 swi SYS_sigreturn 581 582 /* Well if that failed we better exit quick ! */ 583 584 ldr r7, [pc, #8] /* Load SYS_exit */ 585 swi SYS_exit 586 587 /* Branch back to retry SYS_sigreturn */ 588 b . - 16 589END(sigcode) 590 .word SYS_sigreturn 591 .word SYS_exit 592 593 .align 2 594 .global _C_LABEL(esigcode) 595 _C_LABEL(esigcode): 596 597 .data 598 .global szsigcode 599szsigcode: 600 .long esigcode-sigcode 601 602/* End of locore.S */ 603