1/*- 2 * Copyright (c) 2003 Peter Wemm <[email protected]> 3 * All rights reserved. 4 * 5 * Copyright (c) 2020, 2021 The FreeBSD Foundation 6 * 7 * Portions of this software were developed by 8 * Konstantin Belousov <[email protected]> under sponsorship from 9 * the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33#include <machine/asmacros.h> 34#include <machine/psl.h> 35#include <machine/pmap.h> 36#include <machine/specialreg.h> 37 38#include "assym.inc" 39 40/* 41 * Compiled KERNBASE location 42 */ 43 .globl kernbase, kernload, dmapbase, dmapend 44 .set kernbase,KERNBASE 45 .set kernload,KERNLOAD 46 .set dmapbase,DMAP_MIN_ADDRESS 47 .set dmapend,DMAP_MAX_ADDRESS 48 49#define BOOTSTACK_SIZE (PAGE_SIZE * KSTACK_PAGES) 50 51 .text 52/********************************************************************** 53 * 54 * This is where the loader trampoline start us, set the ball rolling... 55 * 56 * We are called with the stack looking like this: 57 * 0(%rsp) = 32 bit return address (cannot be used) 58 * 4(%rsp) = 32 bit modulep 59 * 8(%rsp) = 32 bit kernend 60 * 61 * We are already in long mode, on a 64 bit %cs and running at KERNBASE. 62 */ 63ENTRY(btext) 64 65 /* Don't trust what the loader gives for rflags. */ 66 pushq $PSL_KERNEL 67 popfq 68 69 /* Get onto a stack that we can trust - there is no going back now. */ 70 movq %rsp, %rbp 71 movq $bootstack,%rsp 72 73#ifdef KASAN 74 /* Bootstrap a shadow map for the boot stack. */ 75 movq $bootstack, %rdi 76 subq $BOOTSTACK_SIZE, %rdi 77 movq $BOOTSTACK_SIZE, %rsi 78 call kasan_init_early 79#endif 80 81 /* Grab metadata pointers from the loader. */ 82 movl 4(%rbp),%edi /* modulep (arg 1) */ 83 movl 8(%rbp),%esi /* kernend (arg 2) */ 84 xorq %rbp, %rbp 85 86 call hammer_time /* set up cpu for unix operation */ 87 movq %rax,%rsp /* set up kstack for mi_startup() */ 88 call mi_startup /* autoconfiguration, mountroot etc */ 890: hlt 90 jmp 0b 91 92/* 93 * void la57_trampoline(%rdi pml5) 94 * 95 * Entered in 4-level paging long mode on AP, hopefully returns alive in 96 * 5-level paging mode. The parameter is a pointer to a 5-level page 97 * table root. The passed 5-level page table, and the current 4-level page 98 * table, both must map the trampoline code page 1:1 physical, below 4G. 99 * The trampoline must be PIC because it is copied from kernel text into 100 * this page. 101 * 102 * The current paging level cannot be changed while paging is enabled, and 103 * paging cannot be disabled while in long mode. As consequence, code 104 * switches into the compat mode, then disables paging to descend into 105 * protected mode. There, the paging level bit CR4.LA57 can be changed, 106 * and code directly jumps back into long mode. 107 * 108 * Falling into the protected mode requires single-purpose GDT entries, 109 * which are provided by the private GDT. It is the caller's responsibility 110 * to 111 * - restore the GDT and %gsbase after the call 112 * - reset IDT back to long mode. 113 */ 114ENTRY(la57_trampoline) 115 movq %rsp,lst(%rip) /* save registers into memeory */ 116 movq %rbx,lst+8(%rip) /* upper halves are not saved .. */ 117 movq %rbp,lst+0x10(%rip) /* by 64->32->64 switch */ 118 movq %cr4,%rax 119 orq $CR4_LA57,%rax /* 5-lvl %cr4 */ 120 movq %rax,lst+0x18(%rip) 121 leaq la57_trampoline_end(%rip),%rsp /* priv stack */ 122 123 movq %cr0,%rbp 124 lgdtq la57_trampoline_gdt_desc(%rip) 125 126 pushq $(2<<3) 127 leaq l1(%rip),%rax 128 leaq l2(%rip),%rbx 129 130 pushq %rax 131 lretq 132 .code32 133 134l1: movl $(3<<3),%eax 135 movl %eax,%ss /* 32bit paged, priv gdt and stack */ 136 137 movl %cr4,%eax 138 andl $~(CR4_PGE | CR4_PCIDE),%eax /* clear sensitive paging ctrls */ 139 movl %eax,%cr4 140 141 movl %ebp,%eax 142 andl $~CR0_PG,%eax /* protected mode */ 143 movl %eax,%cr0 144 145 movl $MSR_EFER,%ecx /* disable long mode bit */ 146 rdmsr /* to safer tweaking LA57 */ 147 andl $~EFER_LME,%eax 148 wrmsr 149 150 movl %cr4,%eax /* finally safe to switch bit */ 151 orl $CR4_LA57,%eax 152 movl %eax,%cr4 153 154 movl %edi,%cr3 /* and load the 5-level pgtable root */ 155 156 rdmsr 157 orl $EFER_LME,%eax 158 wrmsr /* prepare for ... */ 159 160 movl %ebp,%cr0 /* and jump back directly into long */ 161 jmp 1f /* mode from protected by enabling pg */ 162 1631: pushl $(1<<3) /* reload %cs */ 164 pushl %ebx 165 lretl 166 .code64 167 168l2: movq lst(%rip),%rsp /* back on C stack */ 169 movq lst+8(%rip),%rbx 170 movq lst+0x10(%rip),%rbp 171 movq lst+0x18(%rip),%rax 172 movq %rax,%cr4 /* re-enable paging controls */ 173 retq /* back to C */ 174 .p2align 4,0 175lst: .quad 0,0,0,0 176ENTRY(la57_trampoline_gdt_desc) 177 .word la57_trampoline_end - la57_trampoline_gdt 178 .long 0, 0 /* filled by pmap_bootstrap_la57 */ 179 .p2align 4,0 180ENTRY(la57_trampoline_gdt) 181 .long 0x00000000 /* null desc */ 182 .long 0x00000000 183 .long 0x00000000 /* 64bit code */ 184 .long 0x00209800 185 .long 0x0000ffff /* 32bit code */ 186 .long 0x00cf9b00 187 .long 0x0000ffff /* universal data */ 188 .long 0x00cf9300 189 .dcb.l 16,0 190ENTRY(la57_trampoline_end) 191 192 .bss 193 .p2align PAGE_SHIFT 194 .globl bootstack 195 .space BOOTSTACK_SIZE /* space for bootstack - temporary stack */ 196bootstack: 197