1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (c) 2003 Peter Wemm.
5 * Copyright (c) 1992 Terrence R. Lambert.
6 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * William Jolitz.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 * must display the following acknowledgement:
22 * This product includes software developed by the University of
23 * California, Berkeley and its contributors.
24 * 4. Neither the name of the University nor the names of its contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 *
40 * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91
41 */
42
43 #include <sys/cdefs.h>
44 #include "opt_atpic.h"
45 #include "opt_cpu.h"
46 #include "opt_ddb.h"
47 #include "opt_inet.h"
48 #include "opt_isa.h"
49 #include "opt_kstack_pages.h"
50 #include "opt_maxmem.h"
51 #include "opt_pci.h"
52 #include "opt_platform.h"
53 #include "opt_sched.h"
54
55 #include <sys/param.h>
56 #include <sys/proc.h>
57 #include <sys/systm.h>
58 #include <sys/asan.h>
59 #include <sys/bio.h>
60 #include <sys/buf.h>
61 #include <sys/bus.h>
62 #include <sys/callout.h>
63 #include <sys/cons.h>
64 #include <sys/cpu.h>
65 #include <sys/csan.h>
66 #include <sys/efi.h>
67 #include <sys/eventhandler.h>
68 #include <sys/exec.h>
69 #include <sys/imgact.h>
70 #include <sys/kdb.h>
71 #include <sys/kernel.h>
72 #include <sys/ktr.h>
73 #include <sys/linker.h>
74 #include <sys/lock.h>
75 #include <sys/malloc.h>
76 #include <sys/memrange.h>
77 #include <sys/msan.h>
78 #include <sys/msgbuf.h>
79 #include <sys/mutex.h>
80 #include <sys/pcpu.h>
81 #include <sys/ptrace.h>
82 #include <sys/reboot.h>
83 #include <sys/reg.h>
84 #include <sys/rwlock.h>
85 #include <sys/sched.h>
86 #include <sys/signalvar.h>
87 #ifdef SMP
88 #include <sys/smp.h>
89 #endif
90 #include <sys/syscallsubr.h>
91 #include <sys/sysctl.h>
92 #include <sys/sysent.h>
93 #include <sys/sysproto.h>
94 #include <sys/ucontext.h>
95 #include <sys/vmmeter.h>
96
97 #include <vm/vm.h>
98 #include <vm/vm_param.h>
99 #include <vm/vm_extern.h>
100 #include <vm/vm_kern.h>
101 #include <vm/vm_page.h>
102 #include <vm/vm_map.h>
103 #include <vm/vm_object.h>
104 #include <vm/vm_pager.h>
105 #include <vm/vm_phys.h>
106 #include <vm/vm_dumpset.h>
107
108 #ifdef DDB
109 #ifndef KDB
110 #error KDB must be enabled in order for DDB to work!
111 #endif
112 #include <ddb/ddb.h>
113 #include <ddb/db_sym.h>
114 #endif
115
116 #include <net/netisr.h>
117
118 #include <dev/smbios/smbios.h>
119
120 #include <machine/clock.h>
121 #include <machine/cpu.h>
122 #include <machine/cputypes.h>
123 #include <machine/frame.h>
124 #include <machine/intr_machdep.h>
125 #include <x86/mca.h>
126 #include <machine/md_var.h>
127 #include <machine/metadata.h>
128 #include <machine/pc/bios.h>
129 #include <machine/pcb.h>
130 #include <machine/proc.h>
131 #include <machine/sigframe.h>
132 #include <machine/specialreg.h>
133 #include <machine/trap.h>
134 #include <machine/tss.h>
135 #include <x86/ucode.h>
136 #include <x86/ifunc.h>
137 #ifdef SMP
138 #include <machine/smp.h>
139 #endif
140 #ifdef FDT
141 #include <x86/fdt.h>
142 #endif
143
144 #ifdef DEV_ATPIC
145 #include <x86/isa/icu.h>
146 #else
147 #include <x86/apicvar.h>
148 #endif
149
150 #include <isa/isareg.h>
151 #include <isa/rtc.h>
152 #include <x86/init.h>
153
154 /* Sanity check for __curthread() */
155 CTASSERT(offsetof(struct pcpu, pc_curthread) == 0);
156
157 /*
158 * The PTI trampoline stack needs enough space for a hardware trapframe and a
159 * couple of scratch registers, as well as the trapframe left behind after an
160 * iret fault.
161 */
162 CTASSERT(PC_PTI_STACK_SZ * sizeof(register_t) >= 2 * sizeof(struct pti_frame) -
163 offsetof(struct pti_frame, pti_rip));
164
165 extern u_int64_t hammer_time(u_int64_t, u_int64_t);
166
167 static void cpu_startup(void *);
168 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
169
170 /* Probe 8254 PIT and TSC. */
171 static void native_clock_source_init(void);
172
173 /* Preload data parse function */
174 static caddr_t native_parse_preload_data(u_int64_t);
175
176 /* Native function to fetch and parse the e820 map */
177 static void native_parse_memmap(caddr_t, vm_paddr_t *, int *);
178
179 /* Default init_ops implementation. */
180 struct init_ops init_ops = {
181 .parse_preload_data = native_parse_preload_data,
182 .early_clock_source_init = native_clock_source_init,
183 .early_delay = i8254_delay,
184 .parse_memmap = native_parse_memmap,
185 };
186
187 /*
188 * Physical address of the EFI System Table. Stashed from the metadata hints
189 * passed into the kernel and used by the EFI code to call runtime services.
190 */
191 vm_paddr_t efi_systbl_phys;
192
193 /* Intel ICH registers */
194 #define ICH_PMBASE 0x400
195 #define ICH_SMI_EN ICH_PMBASE + 0x30
196
197 int _udatasel, _ucodesel, _ucode32sel, _ufssel, _ugssel;
198
199 int cold = 1;
200
201 long Maxmem = 0;
202 long realmem = 0;
203 int late_console = 1;
204
205 struct kva_md_info kmi;
206
207 struct region_descriptor r_idt;
208
209 struct pcpu *__pcpu;
210 struct pcpu temp_bsp_pcpu;
211
212 struct mtx icu_lock;
213
214 struct mem_range_softc mem_range_softc;
215
216 struct mtx dt_lock; /* lock for GDT and LDT */
217
218 void (*vmm_resume_p)(void);
219
220 bool efi_boot;
221
222 static void
cpu_startup(void * dummy)223 cpu_startup(void *dummy)
224 {
225 uintmax_t memsize;
226 char *sysenv;
227
228 /*
229 * On MacBooks, we need to disallow the legacy USB circuit to
230 * generate an SMI# because this can cause several problems,
231 * namely: incorrect CPU frequency detection and failure to
232 * start the APs.
233 * We do this by disabling a bit in the SMI_EN (SMI Control and
234 * Enable register) of the Intel ICH LPC Interface Bridge.
235 */
236 sysenv = kern_getenv("smbios.system.product");
237 if (sysenv != NULL) {
238 if (strncmp(sysenv, "MacBook1,1", 10) == 0 ||
239 strncmp(sysenv, "MacBook3,1", 10) == 0 ||
240 strncmp(sysenv, "MacBook4,1", 10) == 0 ||
241 strncmp(sysenv, "MacBookPro1,1", 13) == 0 ||
242 strncmp(sysenv, "MacBookPro1,2", 13) == 0 ||
243 strncmp(sysenv, "MacBookPro3,1", 13) == 0 ||
244 strncmp(sysenv, "MacBookPro4,1", 13) == 0 ||
245 strncmp(sysenv, "Macmini1,1", 10) == 0) {
246 if (bootverbose)
247 printf("Disabling LEGACY_USB_EN bit on "
248 "Intel ICH.\n");
249 outl(ICH_SMI_EN, inl(ICH_SMI_EN) & ~0x8);
250 }
251 freeenv(sysenv);
252 }
253
254 /*
255 * Good {morning,afternoon,evening,night}.
256 */
257 startrtclock();
258 printcpuinfo();
259
260 /*
261 * Display physical memory if SMBIOS reports reasonable amount.
262 */
263 memsize = 0;
264 sysenv = kern_getenv("smbios.memory.enabled");
265 if (sysenv != NULL) {
266 memsize = (uintmax_t)strtoul(sysenv, (char **)NULL, 10) << 10;
267 freeenv(sysenv);
268 }
269 if (memsize < ptoa((uintmax_t)vm_free_count()))
270 memsize = ptoa((uintmax_t)Maxmem);
271 printf("real memory = %ju (%ju MB)\n", memsize, memsize >> 20);
272 realmem = atop(memsize);
273
274 /*
275 * Display any holes after the first chunk of extended memory.
276 */
277 if (bootverbose) {
278 int indx;
279
280 printf("Physical memory chunk(s):\n");
281 for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) {
282 vm_paddr_t size;
283
284 size = phys_avail[indx + 1] - phys_avail[indx];
285 printf(
286 "0x%016jx - 0x%016jx, %ju bytes (%ju pages)\n",
287 (uintmax_t)phys_avail[indx],
288 (uintmax_t)phys_avail[indx + 1] - 1,
289 (uintmax_t)size, (uintmax_t)size / PAGE_SIZE);
290 }
291 }
292
293 vm_ksubmap_init(&kmi);
294
295 printf("avail memory = %ju (%ju MB)\n",
296 ptoa((uintmax_t)vm_free_count()),
297 ptoa((uintmax_t)vm_free_count()) / 1048576);
298 #ifdef DEV_PCI
299 if (bootverbose && intel_graphics_stolen_base != 0)
300 printf("intel stolen mem: base %#jx size %ju MB\n",
301 (uintmax_t)intel_graphics_stolen_base,
302 (uintmax_t)intel_graphics_stolen_size / 1024 / 1024);
303 #endif
304
305 /*
306 * Set up buffers, so they can be used to read disk labels.
307 */
308 bufinit();
309 vm_pager_bufferinit();
310
311 cpu_setregs();
312 }
313
314 static void
late_ifunc_resolve(void * dummy __unused)315 late_ifunc_resolve(void *dummy __unused)
316 {
317 link_elf_late_ireloc();
318 }
319 SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL);
320
321
322 void
cpu_setregs(void)323 cpu_setregs(void)
324 {
325 register_t cr0;
326
327 TSENTER();
328 cr0 = rcr0();
329 cr0 |= CR0_MP | CR0_NE | CR0_TS | CR0_WP | CR0_AM;
330 TSENTER2("load_cr0");
331 load_cr0(cr0);
332 TSEXIT2("load_cr0");
333 TSEXIT();
334 }
335
336 /*
337 * Initialize amd64 and configure to run kernel
338 */
339
340 /*
341 * Initialize segments & interrupt table
342 */
343 static struct gate_descriptor idt0[NIDT];
344 struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */
345
346 static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(16);
347 static char mce0_stack[MCE_STACK_SIZE] __aligned(16);
348 static char nmi0_stack[NMI_STACK_SIZE] __aligned(16);
349 static char dbg0_stack[DBG_STACK_SIZE] __aligned(16);
350 CTASSERT(sizeof(struct nmi_pcpu) == 16);
351
352 /*
353 * Software prototypes -- in more palatable form.
354 *
355 * Keep GUFS32, GUGS32, GUCODE32 and GUDATA at the same
356 * slots as corresponding segments for i386 kernel.
357 */
358 struct soft_segment_descriptor gdt_segs[] = {
359 [GNULL_SEL] = { /* 0 Null Descriptor */
360 .ssd_base = 0x0,
361 .ssd_limit = 0x0,
362 .ssd_type = 0,
363 .ssd_dpl = 0,
364 .ssd_p = 0,
365 .ssd_long = 0,
366 .ssd_def32 = 0,
367 .ssd_gran = 0 },
368 [GNULL2_SEL] = { /* 1 Null Descriptor */
369 .ssd_base = 0x0,
370 .ssd_limit = 0x0,
371 .ssd_type = 0,
372 .ssd_dpl = 0,
373 .ssd_p = 0,
374 .ssd_long = 0,
375 .ssd_def32 = 0,
376 .ssd_gran = 0 },
377 [GUFS32_SEL] = { /* 2 32 bit %gs Descriptor for user */
378 .ssd_base = 0x0,
379 .ssd_limit = 0xfffff,
380 .ssd_type = SDT_MEMRWA,
381 .ssd_dpl = SEL_UPL,
382 .ssd_p = 1,
383 .ssd_long = 0,
384 .ssd_def32 = 1,
385 .ssd_gran = 1 },
386 [GUGS32_SEL] = { /* 3 32 bit %fs Descriptor for user */
387 .ssd_base = 0x0,
388 .ssd_limit = 0xfffff,
389 .ssd_type = SDT_MEMRWA,
390 .ssd_dpl = SEL_UPL,
391 .ssd_p = 1,
392 .ssd_long = 0,
393 .ssd_def32 = 1,
394 .ssd_gran = 1 },
395 [GCODE_SEL] = { /* 4 Code Descriptor for kernel */
396 .ssd_base = 0x0,
397 .ssd_limit = 0xfffff,
398 .ssd_type = SDT_MEMERA,
399 .ssd_dpl = SEL_KPL,
400 .ssd_p = 1,
401 .ssd_long = 1,
402 .ssd_def32 = 0,
403 .ssd_gran = 1 },
404 [GDATA_SEL] = { /* 5 Data Descriptor for kernel */
405 .ssd_base = 0x0,
406 .ssd_limit = 0xfffff,
407 .ssd_type = SDT_MEMRWA,
408 .ssd_dpl = SEL_KPL,
409 .ssd_p = 1,
410 .ssd_long = 1,
411 .ssd_def32 = 0,
412 .ssd_gran = 1 },
413 [GUCODE32_SEL] = { /* 6 32 bit Code Descriptor for user */
414 .ssd_base = 0x0,
415 .ssd_limit = 0xfffff,
416 .ssd_type = SDT_MEMERA,
417 .ssd_dpl = SEL_UPL,
418 .ssd_p = 1,
419 .ssd_long = 0,
420 .ssd_def32 = 1,
421 .ssd_gran = 1 },
422 [GUDATA_SEL] = { /* 7 32/64 bit Data Descriptor for user */
423 .ssd_base = 0x0,
424 .ssd_limit = 0xfffff,
425 .ssd_type = SDT_MEMRWA,
426 .ssd_dpl = SEL_UPL,
427 .ssd_p = 1,
428 .ssd_long = 0,
429 .ssd_def32 = 1,
430 .ssd_gran = 1 },
431 [GUCODE_SEL] = { /* 8 64 bit Code Descriptor for user */
432 .ssd_base = 0x0,
433 .ssd_limit = 0xfffff,
434 .ssd_type = SDT_MEMERA,
435 .ssd_dpl = SEL_UPL,
436 .ssd_p = 1,
437 .ssd_long = 1,
438 .ssd_def32 = 0,
439 .ssd_gran = 1 },
440 [GPROC0_SEL] = { /* 9 Proc 0 TSS Descriptor */
441 .ssd_base = 0x0,
442 .ssd_limit = sizeof(struct amd64tss) + IOPERM_BITMAP_SIZE - 1,
443 .ssd_type = SDT_SYSTSS,
444 .ssd_dpl = SEL_KPL,
445 .ssd_p = 1,
446 .ssd_long = 0,
447 .ssd_def32 = 0,
448 .ssd_gran = 0 },
449 [GPROC0_SEL + 1] = { /* 10 Proc 0 TSS descriptor, double size */
450 .ssd_base = 0x0,
451 .ssd_limit = 0x0,
452 .ssd_type = 0,
453 .ssd_dpl = 0,
454 .ssd_p = 0,
455 .ssd_long = 0,
456 .ssd_def32 = 0,
457 .ssd_gran = 0 },
458 [GUSERLDT_SEL] = { /* 11 LDT Descriptor */
459 .ssd_base = 0x0,
460 .ssd_limit = 0x0,
461 .ssd_type = 0,
462 .ssd_dpl = 0,
463 .ssd_p = 0,
464 .ssd_long = 0,
465 .ssd_def32 = 0,
466 .ssd_gran = 0 },
467 [GUSERLDT_SEL + 1] = { /* 12 LDT Descriptor, double size */
468 .ssd_base = 0x0,
469 .ssd_limit = 0x0,
470 .ssd_type = 0,
471 .ssd_dpl = 0,
472 .ssd_p = 0,
473 .ssd_long = 0,
474 .ssd_def32 = 0,
475 .ssd_gran = 0 },
476 };
477 _Static_assert(nitems(gdt_segs) == NGDT, "Stale NGDT");
478
479 void
setidt(int idx,inthand_t * func,int typ,int dpl,int ist)480 setidt(int idx, inthand_t *func, int typ, int dpl, int ist)
481 {
482 struct gate_descriptor *ip;
483
484 ip = idt + idx;
485 ip->gd_looffset = (uintptr_t)func;
486 ip->gd_selector = GSEL(GCODE_SEL, SEL_KPL);
487 ip->gd_ist = ist;
488 ip->gd_xx = 0;
489 ip->gd_type = typ;
490 ip->gd_dpl = dpl;
491 ip->gd_p = 1;
492 ip->gd_hioffset = ((uintptr_t)func)>>16 ;
493 }
494
495 extern inthand_t
496 IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
497 IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),
498 IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),
499 IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),
500 IDTVEC(xmm), IDTVEC(dblfault),
501 IDTVEC(div_pti), IDTVEC(bpt_pti),
502 IDTVEC(ofl_pti), IDTVEC(bnd_pti), IDTVEC(ill_pti), IDTVEC(dna_pti),
503 IDTVEC(fpusegm_pti), IDTVEC(tss_pti), IDTVEC(missing_pti),
504 IDTVEC(stk_pti), IDTVEC(prot_pti), IDTVEC(page_pti),
505 IDTVEC(rsvd_pti), IDTVEC(fpu_pti), IDTVEC(align_pti),
506 IDTVEC(xmm_pti),
507 #ifdef KDTRACE_HOOKS
508 IDTVEC(dtrace_ret), IDTVEC(dtrace_ret_pti),
509 #endif
510 #ifdef XENHVM
511 IDTVEC(xen_intr_upcall), IDTVEC(xen_intr_upcall_pti),
512 #endif
513 IDTVEC(fast_syscall), IDTVEC(fast_syscall32),
514 IDTVEC(fast_syscall_pti);
515
516 #ifdef DDB
517 /*
518 * Display the index and function name of any IDT entries that don't use
519 * the default 'rsvd' entry point.
520 */
DB_SHOW_COMMAND_FLAGS(idt,db_show_idt,DB_CMD_MEMSAFE)521 DB_SHOW_COMMAND_FLAGS(idt, db_show_idt, DB_CMD_MEMSAFE)
522 {
523 struct gate_descriptor *ip;
524 int idx;
525 uintptr_t func;
526
527 ip = idt;
528 for (idx = 0; idx < NIDT && !db_pager_quit; idx++) {
529 func = ((long)ip->gd_hioffset << 16 | ip->gd_looffset);
530 if (func != (uintptr_t)&IDTVEC(rsvd)) {
531 db_printf("%3d\t", idx);
532 db_printsym(func, DB_STGY_PROC);
533 db_printf("\n");
534 }
535 ip++;
536 }
537 }
538
539 /* Show privileged registers. */
DB_SHOW_COMMAND_FLAGS(sysregs,db_show_sysregs,DB_CMD_MEMSAFE)540 DB_SHOW_COMMAND_FLAGS(sysregs, db_show_sysregs, DB_CMD_MEMSAFE)
541 {
542 struct {
543 uint16_t limit;
544 uint64_t base;
545 } __packed idtr, gdtr;
546 uint16_t ldt, tr;
547
548 __asm __volatile("sidt %0" : "=m" (idtr));
549 db_printf("idtr\t0x%016lx/%04x\n",
550 (u_long)idtr.base, (u_int)idtr.limit);
551 __asm __volatile("sgdt %0" : "=m" (gdtr));
552 db_printf("gdtr\t0x%016lx/%04x\n",
553 (u_long)gdtr.base, (u_int)gdtr.limit);
554 __asm __volatile("sldt %0" : "=r" (ldt));
555 db_printf("ldtr\t0x%04x\n", ldt);
556 __asm __volatile("str %0" : "=r" (tr));
557 db_printf("tr\t0x%04x\n", tr);
558 db_printf("cr0\t0x%016lx\n", rcr0());
559 db_printf("cr2\t0x%016lx\n", rcr2());
560 db_printf("cr3\t0x%016lx\n", rcr3());
561 db_printf("cr4\t0x%016lx\n", rcr4());
562 if (rcr4() & CR4_XSAVE)
563 db_printf("xcr0\t0x%016lx\n", rxcr(0));
564 db_printf("EFER\t0x%016lx\n", rdmsr(MSR_EFER));
565 if (cpu_feature2 & (CPUID2_VMX | CPUID2_SMX))
566 db_printf("FEATURES_CTL\t%016lx\n",
567 rdmsr(MSR_IA32_FEATURE_CONTROL));
568 db_printf("DEBUG_CTL\t0x%016lx\n", rdmsr(MSR_DEBUGCTLMSR));
569 db_printf("PAT\t0x%016lx\n", rdmsr(MSR_PAT));
570 db_printf("GSBASE\t0x%016lx\n", rdmsr(MSR_GSBASE));
571 }
572
DB_SHOW_COMMAND_FLAGS(dbregs,db_show_dbregs,DB_CMD_MEMSAFE)573 DB_SHOW_COMMAND_FLAGS(dbregs, db_show_dbregs, DB_CMD_MEMSAFE)
574 {
575
576 db_printf("dr0\t0x%016lx\n", rdr0());
577 db_printf("dr1\t0x%016lx\n", rdr1());
578 db_printf("dr2\t0x%016lx\n", rdr2());
579 db_printf("dr3\t0x%016lx\n", rdr3());
580 db_printf("dr6\t0x%016lx\n", rdr6());
581 db_printf("dr7\t0x%016lx\n", rdr7());
582 }
583 #endif
584
585 void
sdtossd(struct user_segment_descriptor * sd,struct soft_segment_descriptor * ssd)586 sdtossd(struct user_segment_descriptor *sd, struct soft_segment_descriptor *ssd)
587 {
588
589 ssd->ssd_base = (sd->sd_hibase << 24) | sd->sd_lobase;
590 ssd->ssd_limit = (sd->sd_hilimit << 16) | sd->sd_lolimit;
591 ssd->ssd_type = sd->sd_type;
592 ssd->ssd_dpl = sd->sd_dpl;
593 ssd->ssd_p = sd->sd_p;
594 ssd->ssd_long = sd->sd_long;
595 ssd->ssd_def32 = sd->sd_def32;
596 ssd->ssd_gran = sd->sd_gran;
597 }
598
599 void
ssdtosd(struct soft_segment_descriptor * ssd,struct user_segment_descriptor * sd)600 ssdtosd(struct soft_segment_descriptor *ssd, struct user_segment_descriptor *sd)
601 {
602
603 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
604 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xff;
605 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
606 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
607 sd->sd_type = ssd->ssd_type;
608 sd->sd_dpl = ssd->ssd_dpl;
609 sd->sd_p = ssd->ssd_p;
610 sd->sd_long = ssd->ssd_long;
611 sd->sd_def32 = ssd->ssd_def32;
612 sd->sd_gran = ssd->ssd_gran;
613 }
614
615 void
ssdtosyssd(struct soft_segment_descriptor * ssd,struct system_segment_descriptor * sd)616 ssdtosyssd(struct soft_segment_descriptor *ssd, struct system_segment_descriptor *sd)
617 {
618
619 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
620 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xfffffffffful;
621 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
622 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
623 sd->sd_type = ssd->ssd_type;
624 sd->sd_dpl = ssd->ssd_dpl;
625 sd->sd_p = ssd->ssd_p;
626 sd->sd_gran = ssd->ssd_gran;
627 }
628
629 u_int basemem;
630
631 static int
add_physmap_entry(uint64_t base,uint64_t length,vm_paddr_t * physmap,int * physmap_idxp)632 add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap,
633 int *physmap_idxp)
634 {
635 int i, insert_idx, physmap_idx;
636
637 physmap_idx = *physmap_idxp;
638
639 if (length == 0)
640 return (1);
641
642 /*
643 * Find insertion point while checking for overlap. Start off by
644 * assuming the new entry will be added to the end.
645 *
646 * NB: physmap_idx points to the next free slot.
647 */
648 insert_idx = physmap_idx;
649 for (i = 0; i <= physmap_idx; i += 2) {
650 if (base < physmap[i + 1]) {
651 if (base + length <= physmap[i]) {
652 insert_idx = i;
653 break;
654 }
655 if (boothowto & RB_VERBOSE)
656 printf(
657 "Overlapping memory regions, ignoring second region\n");
658 return (1);
659 }
660 }
661
662 /* See if we can prepend to the next entry. */
663 if (insert_idx <= physmap_idx && base + length == physmap[insert_idx]) {
664 physmap[insert_idx] = base;
665 return (1);
666 }
667
668 /* See if we can append to the previous entry. */
669 if (insert_idx > 0 && base == physmap[insert_idx - 1]) {
670 physmap[insert_idx - 1] += length;
671 return (1);
672 }
673
674 physmap_idx += 2;
675 *physmap_idxp = physmap_idx;
676 if (physmap_idx == PHYS_AVAIL_ENTRIES) {
677 printf(
678 "Too many segments in the physical address map, giving up\n");
679 return (0);
680 }
681
682 /*
683 * Move the last 'N' entries down to make room for the new
684 * entry if needed.
685 */
686 for (i = (physmap_idx - 2); i > insert_idx; i -= 2) {
687 physmap[i] = physmap[i - 2];
688 physmap[i + 1] = physmap[i - 1];
689 }
690
691 /* Insert the new entry. */
692 physmap[insert_idx] = base;
693 physmap[insert_idx + 1] = base + length;
694 return (1);
695 }
696
697 void
bios_add_smap_entries(struct bios_smap * smapbase,u_int32_t smapsize,vm_paddr_t * physmap,int * physmap_idx)698 bios_add_smap_entries(struct bios_smap *smapbase, u_int32_t smapsize,
699 vm_paddr_t *physmap, int *physmap_idx)
700 {
701 struct bios_smap *smap, *smapend;
702
703 smapend = (struct bios_smap *)((uintptr_t)smapbase + smapsize);
704
705 for (smap = smapbase; smap < smapend; smap++) {
706 if (boothowto & RB_VERBOSE)
707 printf("SMAP type=%02x base=%016lx len=%016lx\n",
708 smap->type, smap->base, smap->length);
709
710 if (smap->type != SMAP_TYPE_MEMORY)
711 continue;
712
713 if (!add_physmap_entry(smap->base, smap->length, physmap,
714 physmap_idx))
715 break;
716 }
717 }
718
719 static void
add_efi_map_entries(struct efi_map_header * efihdr,vm_paddr_t * physmap,int * physmap_idx)720 add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap,
721 int *physmap_idx)
722 {
723 struct efi_md *map, *p;
724 const char *type;
725 size_t efisz;
726 int ndesc, i;
727
728 static const char *types[] = {
729 "Reserved",
730 "LoaderCode",
731 "LoaderData",
732 "BootServicesCode",
733 "BootServicesData",
734 "RuntimeServicesCode",
735 "RuntimeServicesData",
736 "ConventionalMemory",
737 "UnusableMemory",
738 "ACPIReclaimMemory",
739 "ACPIMemoryNVS",
740 "MemoryMappedIO",
741 "MemoryMappedIOPortSpace",
742 "PalCode",
743 "PersistentMemory"
744 };
745
746 /*
747 * Memory map data provided by UEFI via the GetMemoryMap
748 * Boot Services API.
749 */
750 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf;
751 map = (struct efi_md *)((uint8_t *)efihdr + efisz);
752
753 if (efihdr->descriptor_size == 0)
754 return;
755 ndesc = efihdr->memory_size / efihdr->descriptor_size;
756
757 if (boothowto & RB_VERBOSE)
758 printf("%23s %12s %12s %8s %4s\n",
759 "Type", "Physical", "Virtual", "#Pages", "Attr");
760
761 for (i = 0, p = map; i < ndesc; i++,
762 p = efi_next_descriptor(p, efihdr->descriptor_size)) {
763 if (boothowto & RB_VERBOSE) {
764 if (p->md_type < nitems(types))
765 type = types[p->md_type];
766 else
767 type = "<INVALID>";
768 printf("%23s %012lx %012lx %08lx ", type, p->md_phys,
769 p->md_virt, p->md_pages);
770 if (p->md_attr & EFI_MD_ATTR_UC)
771 printf("UC ");
772 if (p->md_attr & EFI_MD_ATTR_WC)
773 printf("WC ");
774 if (p->md_attr & EFI_MD_ATTR_WT)
775 printf("WT ");
776 if (p->md_attr & EFI_MD_ATTR_WB)
777 printf("WB ");
778 if (p->md_attr & EFI_MD_ATTR_UCE)
779 printf("UCE ");
780 if (p->md_attr & EFI_MD_ATTR_WP)
781 printf("WP ");
782 if (p->md_attr & EFI_MD_ATTR_RP)
783 printf("RP ");
784 if (p->md_attr & EFI_MD_ATTR_XP)
785 printf("XP ");
786 if (p->md_attr & EFI_MD_ATTR_NV)
787 printf("NV ");
788 if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE)
789 printf("MORE_RELIABLE ");
790 if (p->md_attr & EFI_MD_ATTR_RO)
791 printf("RO ");
792 if (p->md_attr & EFI_MD_ATTR_RT)
793 printf("RUNTIME");
794 printf("\n");
795 }
796
797 switch (p->md_type) {
798 case EFI_MD_TYPE_CODE:
799 case EFI_MD_TYPE_DATA:
800 case EFI_MD_TYPE_BS_CODE:
801 case EFI_MD_TYPE_BS_DATA:
802 case EFI_MD_TYPE_FREE:
803 /*
804 * We're allowed to use any entry with these types.
805 */
806 break;
807 default:
808 continue;
809 }
810
811 if (!add_physmap_entry(p->md_phys, p->md_pages * EFI_PAGE_SIZE,
812 physmap, physmap_idx))
813 break;
814 }
815 }
816
817 static void
native_parse_memmap(caddr_t kmdp,vm_paddr_t * physmap,int * physmap_idx)818 native_parse_memmap(caddr_t kmdp, vm_paddr_t *physmap, int *physmap_idx)
819 {
820 struct bios_smap *smap;
821 struct efi_map_header *efihdr;
822 u_int32_t size;
823
824 /*
825 * Memory map from INT 15:E820.
826 *
827 * subr_module.c says:
828 * "Consumer may safely assume that size value precedes data."
829 * ie: an int32_t immediately precedes smap.
830 */
831
832 efihdr = (struct efi_map_header *)preload_search_info(kmdp,
833 MODINFO_METADATA | MODINFOMD_EFI_MAP);
834 smap = (struct bios_smap *)preload_search_info(kmdp,
835 MODINFO_METADATA | MODINFOMD_SMAP);
836 if (efihdr == NULL && smap == NULL)
837 panic("No BIOS smap or EFI map info from loader!");
838
839 if (efihdr != NULL) {
840 add_efi_map_entries(efihdr, physmap, physmap_idx);
841 strlcpy(bootmethod, "UEFI", sizeof(bootmethod));
842 } else {
843 size = *((u_int32_t *)smap - 1);
844 bios_add_smap_entries(smap, size, physmap, physmap_idx);
845 strlcpy(bootmethod, "BIOS", sizeof(bootmethod));
846 }
847 }
848
849 #define PAGES_PER_GB (1024 * 1024 * 1024 / PAGE_SIZE)
850
851 /*
852 * Populate the (physmap) array with base/bound pairs describing the
853 * available physical memory in the system, then test this memory and
854 * build the phys_avail array describing the actually-available memory.
855 *
856 * Total memory size may be set by the kernel environment variable
857 * hw.physmem or the compile-time define MAXMEM.
858 *
859 * XXX first should be vm_paddr_t.
860 */
861 static void
getmemsize(caddr_t kmdp,u_int64_t first)862 getmemsize(caddr_t kmdp, u_int64_t first)
863 {
864 int i, physmap_idx, pa_indx, da_indx;
865 vm_paddr_t pa, physmap[PHYS_AVAIL_ENTRIES];
866 u_long physmem_start, physmem_tunable, memtest;
867 pt_entry_t *pte;
868 quad_t dcons_addr, dcons_size;
869 int page_counter;
870
871 TSENTER();
872 /*
873 * Tell the physical memory allocator about pages used to store
874 * the kernel and preloaded data. See kmem_bootstrap_free().
875 */
876 vm_phys_early_add_seg((vm_paddr_t)kernphys, trunc_page(first));
877
878 bzero(physmap, sizeof(physmap));
879 physmap_idx = 0;
880
881 init_ops.parse_memmap(kmdp, physmap, &physmap_idx);
882 physmap_idx -= 2;
883
884 /*
885 * Find the 'base memory' segment for SMP
886 */
887 basemem = 0;
888 for (i = 0; i <= physmap_idx; i += 2) {
889 if (physmap[i] <= 0xA0000) {
890 basemem = physmap[i + 1] / 1024;
891 break;
892 }
893 }
894 if (basemem == 0 || basemem > 640) {
895 if (bootverbose)
896 printf(
897 "Memory map doesn't contain a basemem segment, faking it");
898 basemem = 640;
899 }
900
901 /*
902 * Maxmem isn't the "maximum memory", it's one larger than the
903 * highest page of the physical address space. It should be
904 * called something like "Maxphyspage". We may adjust this
905 * based on ``hw.physmem'' and the results of the memory test.
906 */
907 Maxmem = atop(physmap[physmap_idx + 1]);
908
909 #ifdef MAXMEM
910 Maxmem = MAXMEM / 4;
911 #endif
912
913 if (TUNABLE_ULONG_FETCH("hw.physmem", &physmem_tunable))
914 Maxmem = atop(physmem_tunable);
915
916 /*
917 * The boot memory test is disabled by default, as it takes a
918 * significant amount of time on large-memory systems, and is
919 * unfriendly to virtual machines as it unnecessarily touches all
920 * pages.
921 *
922 * A general name is used as the code may be extended to support
923 * additional tests beyond the current "page present" test.
924 */
925 memtest = 0;
926 TUNABLE_ULONG_FETCH("hw.memtest.tests", &memtest);
927
928 /*
929 * Don't allow MAXMEM or hw.physmem to extend the amount of memory
930 * in the system.
931 */
932 if (Maxmem > atop(physmap[physmap_idx + 1]))
933 Maxmem = atop(physmap[physmap_idx + 1]);
934
935 if (atop(physmap[physmap_idx + 1]) != Maxmem &&
936 (boothowto & RB_VERBOSE))
937 printf("Physical memory use set to %ldK\n", Maxmem * 4);
938
939 /* call pmap initialization to make new kernel address space */
940 pmap_bootstrap(&first);
941
942 /*
943 * Size up each available chunk of physical memory.
944 *
945 * XXX Some BIOSes corrupt low 64KB between suspend and resume.
946 * By default, mask off the first 16 pages unless we appear to be
947 * running in a VM.
948 */
949 physmem_start = (vm_guest > VM_GUEST_NO ? 1 : 16) << PAGE_SHIFT;
950 TUNABLE_ULONG_FETCH("hw.physmem.start", &physmem_start);
951 if (physmap[0] < physmem_start) {
952 if (physmem_start < PAGE_SIZE)
953 physmap[0] = PAGE_SIZE;
954 else if (physmem_start >= physmap[1])
955 physmap[0] = round_page(physmap[1] - PAGE_SIZE);
956 else
957 physmap[0] = round_page(physmem_start);
958 }
959 pa_indx = 0;
960 da_indx = 1;
961 phys_avail[pa_indx++] = physmap[0];
962 phys_avail[pa_indx] = physmap[0];
963 dump_avail[da_indx] = physmap[0];
964 pte = CMAP1;
965
966 /*
967 * Get dcons buffer address
968 */
969 if (getenv_quad("dcons.addr", &dcons_addr) == 0 ||
970 getenv_quad("dcons.size", &dcons_size) == 0)
971 dcons_addr = 0;
972
973 /*
974 * physmap is in bytes, so when converting to page boundaries,
975 * round up the start address and round down the end address.
976 */
977 page_counter = 0;
978 if (memtest != 0)
979 printf("Testing system memory");
980 for (i = 0; i <= physmap_idx; i += 2) {
981 vm_paddr_t end;
982
983 end = ptoa((vm_paddr_t)Maxmem);
984 if (physmap[i + 1] < end)
985 end = trunc_page(physmap[i + 1]);
986 for (pa = round_page(physmap[i]); pa < end; pa += PAGE_SIZE) {
987 int tmp, page_bad, full;
988 int *ptr = (int *)CADDR1;
989
990 full = FALSE;
991 /*
992 * block out kernel memory as not available.
993 */
994 if (pa >= (vm_paddr_t)kernphys && pa < first)
995 goto do_dump_avail;
996
997 /*
998 * block out dcons buffer
999 */
1000 if (dcons_addr > 0
1001 && pa >= trunc_page(dcons_addr)
1002 && pa < dcons_addr + dcons_size)
1003 goto do_dump_avail;
1004
1005 page_bad = FALSE;
1006 if (memtest == 0)
1007 goto skip_memtest;
1008
1009 /*
1010 * Print a "." every GB to show we're making
1011 * progress.
1012 */
1013 page_counter++;
1014 if ((page_counter % PAGES_PER_GB) == 0)
1015 printf(".");
1016
1017 /*
1018 * map page into kernel: valid, read/write,non-cacheable
1019 */
1020 *pte = pa | PG_V | PG_RW | PG_NC_PWT | PG_NC_PCD;
1021 invltlb();
1022
1023 tmp = *(int *)ptr;
1024 /*
1025 * Test for alternating 1's and 0's
1026 */
1027 *(volatile int *)ptr = 0xaaaaaaaa;
1028 if (*(volatile int *)ptr != 0xaaaaaaaa)
1029 page_bad = TRUE;
1030 /*
1031 * Test for alternating 0's and 1's
1032 */
1033 *(volatile int *)ptr = 0x55555555;
1034 if (*(volatile int *)ptr != 0x55555555)
1035 page_bad = TRUE;
1036 /*
1037 * Test for all 1's
1038 */
1039 *(volatile int *)ptr = 0xffffffff;
1040 if (*(volatile int *)ptr != 0xffffffff)
1041 page_bad = TRUE;
1042 /*
1043 * Test for all 0's
1044 */
1045 *(volatile int *)ptr = 0x0;
1046 if (*(volatile int *)ptr != 0x0)
1047 page_bad = TRUE;
1048 /*
1049 * Restore original value.
1050 */
1051 *(int *)ptr = tmp;
1052
1053 skip_memtest:
1054 /*
1055 * Adjust array of valid/good pages.
1056 */
1057 if (page_bad == TRUE)
1058 continue;
1059 /*
1060 * If this good page is a continuation of the
1061 * previous set of good pages, then just increase
1062 * the end pointer. Otherwise start a new chunk.
1063 * Note that "end" points one higher than end,
1064 * making the range >= start and < end.
1065 * If we're also doing a speculative memory
1066 * test and we at or past the end, bump up Maxmem
1067 * so that we keep going. The first bad page
1068 * will terminate the loop.
1069 */
1070 if (phys_avail[pa_indx] == pa) {
1071 phys_avail[pa_indx] += PAGE_SIZE;
1072 } else {
1073 pa_indx++;
1074 if (pa_indx == PHYS_AVAIL_ENTRIES) {
1075 printf(
1076 "Too many holes in the physical address space, giving up\n");
1077 pa_indx--;
1078 full = TRUE;
1079 goto do_dump_avail;
1080 }
1081 phys_avail[pa_indx++] = pa; /* start */
1082 phys_avail[pa_indx] = pa + PAGE_SIZE; /* end */
1083 }
1084 physmem++;
1085 do_dump_avail:
1086 if (dump_avail[da_indx] == pa) {
1087 dump_avail[da_indx] += PAGE_SIZE;
1088 } else {
1089 da_indx++;
1090 if (da_indx == PHYS_AVAIL_ENTRIES) {
1091 da_indx--;
1092 goto do_next;
1093 }
1094 dump_avail[da_indx++] = pa; /* start */
1095 dump_avail[da_indx] = pa + PAGE_SIZE; /* end */
1096 }
1097 do_next:
1098 if (full)
1099 break;
1100 }
1101 }
1102 *pte = 0;
1103 invltlb();
1104 if (memtest != 0)
1105 printf("\n");
1106
1107 /*
1108 * XXX
1109 * The last chunk must contain at least one page plus the message
1110 * buffer to avoid complicating other code (message buffer address
1111 * calculation, etc.).
1112 */
1113 while (phys_avail[pa_indx - 1] + PAGE_SIZE +
1114 round_page(msgbufsize) >= phys_avail[pa_indx]) {
1115 physmem -= atop(phys_avail[pa_indx] - phys_avail[pa_indx - 1]);
1116 phys_avail[pa_indx--] = 0;
1117 phys_avail[pa_indx--] = 0;
1118 }
1119
1120 Maxmem = atop(phys_avail[pa_indx]);
1121
1122 /* Trim off space for the message buffer. */
1123 phys_avail[pa_indx] -= round_page(msgbufsize);
1124
1125 /* Map the message buffer. */
1126 msgbufp = (struct msgbuf *)PHYS_TO_DMAP(phys_avail[pa_indx]);
1127 TSEXIT();
1128 }
1129
1130 static caddr_t
native_parse_preload_data(u_int64_t modulep)1131 native_parse_preload_data(u_int64_t modulep)
1132 {
1133 caddr_t kmdp;
1134 char *envp;
1135 #ifdef DDB
1136 vm_offset_t ksym_start;
1137 vm_offset_t ksym_end;
1138 #endif
1139
1140 preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);
1141 preload_bootstrap_relocate(KERNBASE);
1142 kmdp = preload_search_by_type("elf kernel");
1143 if (kmdp == NULL)
1144 kmdp = preload_search_by_type("elf64 kernel");
1145 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
1146 envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *);
1147 if (envp != NULL)
1148 envp += KERNBASE;
1149 init_static_kenv(envp, 0);
1150 #ifdef DDB
1151 ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
1152 ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
1153 db_fetch_ksymtab(ksym_start, ksym_end, 0);
1154 #endif
1155 efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t);
1156
1157 return (kmdp);
1158 }
1159
1160 static void
native_clock_source_init(void)1161 native_clock_source_init(void)
1162 {
1163 i8254_init();
1164 }
1165
1166 static void
amd64_kdb_init(void)1167 amd64_kdb_init(void)
1168 {
1169 kdb_init();
1170 #ifdef KDB
1171 if (boothowto & RB_KDB)
1172 kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger");
1173 #endif
1174 }
1175
1176 /* Set up the fast syscall stuff */
1177 void
amd64_conf_fast_syscall(void)1178 amd64_conf_fast_syscall(void)
1179 {
1180 uint64_t msr;
1181
1182 msr = rdmsr(MSR_EFER) | EFER_SCE;
1183 wrmsr(MSR_EFER, msr);
1184 wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
1185 (u_int64_t)IDTVEC(fast_syscall));
1186 wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
1187 msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
1188 ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
1189 wrmsr(MSR_STAR, msr);
1190 wrmsr(MSR_SF_MASK, PSL_NT | PSL_T | PSL_I | PSL_C | PSL_D | PSL_AC);
1191 }
1192
1193 void
amd64_bsp_pcpu_init1(struct pcpu * pc)1194 amd64_bsp_pcpu_init1(struct pcpu *pc)
1195 {
1196 struct user_segment_descriptor *gdt;
1197
1198 PCPU_SET(prvspace, pc);
1199 gdt = *PCPU_PTR(gdt);
1200 PCPU_SET(curthread, &thread0);
1201 PCPU_SET(tssp, PCPU_PTR(common_tss));
1202 PCPU_SET(tss, (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
1203 PCPU_SET(ldt, (struct system_segment_descriptor *)&gdt[GUSERLDT_SEL]);
1204 PCPU_SET(fs32p, &gdt[GUFS32_SEL]);
1205 PCPU_SET(gs32p, &gdt[GUGS32_SEL]);
1206 PCPU_SET(ucr3_load_mask, PMAP_UCR3_NOMASK);
1207 PCPU_SET(smp_tlb_gen, 1);
1208 }
1209
1210 void
amd64_bsp_pcpu_init2(uint64_t rsp0)1211 amd64_bsp_pcpu_init2(uint64_t rsp0)
1212 {
1213
1214 PCPU_SET(rsp0, rsp0);
1215 PCPU_SET(pti_rsp0, ((vm_offset_t)PCPU_PTR(pti_stack) +
1216 PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful);
1217 PCPU_SET(curpcb, thread0.td_pcb);
1218 }
1219
1220 void
amd64_bsp_ist_init(struct pcpu * pc)1221 amd64_bsp_ist_init(struct pcpu *pc)
1222 {
1223 struct nmi_pcpu *np;
1224 struct amd64tss *tssp;
1225
1226 tssp = &pc->pc_common_tss;
1227
1228 /* doublefault stack space, runs on ist1 */
1229 np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(dblfault_stack)]) - 1;
1230 np->np_pcpu = (register_t)pc;
1231 tssp->tss_ist1 = (long)np;
1232
1233 /*
1234 * NMI stack, runs on ist2. The pcpu pointer is stored just
1235 * above the start of the ist2 stack.
1236 */
1237 np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
1238 np->np_pcpu = (register_t)pc;
1239 tssp->tss_ist2 = (long)np;
1240
1241 /*
1242 * MC# stack, runs on ist3. The pcpu pointer is stored just
1243 * above the start of the ist3 stack.
1244 */
1245 np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
1246 np->np_pcpu = (register_t)pc;
1247 tssp->tss_ist3 = (long)np;
1248
1249 /*
1250 * DB# stack, runs on ist4.
1251 */
1252 np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
1253 np->np_pcpu = (register_t)pc;
1254 tssp->tss_ist4 = (long)np;
1255 }
1256
1257 /*
1258 * Calculate the kernel load address by inspecting page table created by loader.
1259 * The assumptions:
1260 * - kernel is mapped at KERNBASE, backed by contiguous phys memory
1261 * aligned at 2M, below 4G (the latter is important for AP startup)
1262 * - there is a 2M hole at KERNBASE (KERNSTART = KERNBASE + 2M)
1263 * - kernel is mapped with 2M superpages
1264 * - all participating memory, i.e. kernel, modules, metadata,
1265 * page table is accessible by pre-created 1:1 mapping
1266 * (right now loader creates 1:1 mapping for lower 4G, and all
1267 * memory is from there)
1268 * - there is a usable memory block right after the end of the
1269 * mapped kernel and all modules/metadata, pointed to by
1270 * physfree, for early allocations
1271 */
1272 vm_paddr_t __nosanitizeaddress __nosanitizememory
amd64_loadaddr(void)1273 amd64_loadaddr(void)
1274 {
1275 pml4_entry_t *pml4e;
1276 pdp_entry_t *pdpe;
1277 pd_entry_t *pde;
1278 uint64_t cr3;
1279
1280 cr3 = rcr3();
1281 pml4e = (pml4_entry_t *)cr3 + pmap_pml4e_index(KERNSTART);
1282 pdpe = (pdp_entry_t *)(*pml4e & PG_FRAME) + pmap_pdpe_index(KERNSTART);
1283 pde = (pd_entry_t *)(*pdpe & PG_FRAME) + pmap_pde_index(KERNSTART);
1284 return (*pde & PG_FRAME);
1285 }
1286
1287 u_int64_t
hammer_time(u_int64_t modulep,u_int64_t physfree)1288 hammer_time(u_int64_t modulep, u_int64_t physfree)
1289 {
1290 caddr_t kmdp;
1291 int gsel_tss, x;
1292 struct pcpu *pc;
1293 uint64_t rsp0;
1294 char *env;
1295 struct user_segment_descriptor *gdt;
1296 struct region_descriptor r_gdt;
1297 size_t kstack0_sz;
1298
1299 TSRAW(&thread0, TS_ENTER, __func__, NULL);
1300
1301 kernphys = amd64_loadaddr();
1302
1303 physfree += kernphys;
1304
1305 kmdp = init_ops.parse_preload_data(modulep);
1306
1307 efi_boot = preload_search_info(kmdp, MODINFO_METADATA |
1308 MODINFOMD_EFI_MAP) != NULL;
1309
1310 if (!efi_boot) {
1311 /* Tell the bios to warmboot next time */
1312 atomic_store_short((u_short *)0x472, 0x1234);
1313 }
1314
1315 physfree += ucode_load_bsp(physfree - kernphys + KERNSTART);
1316 physfree = roundup2(physfree, PAGE_SIZE);
1317
1318 identify_cpu1();
1319 identify_hypervisor();
1320 identify_hypervisor_smbios();
1321 identify_cpu_fixup_bsp();
1322 identify_cpu2();
1323 initializecpucache();
1324
1325 /*
1326 * Check for pti, pcid, and invpcid before ifuncs are
1327 * resolved, to correctly select the implementation for
1328 * pmap_activate_sw_mode().
1329 */
1330 pti = pti_get_default();
1331 TUNABLE_INT_FETCH("vm.pmap.pti", &pti);
1332 TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled);
1333 if ((cpu_feature2 & CPUID2_PCID) != 0 && pmap_pcid_enabled) {
1334 invpcid_works = (cpu_stdext_feature &
1335 CPUID_STDEXT_INVPCID) != 0;
1336 } else {
1337 pmap_pcid_enabled = 0;
1338 }
1339
1340 /*
1341 * Now we can do small core initialization, after the PCID
1342 * CPU features and user knobs are evaluated.
1343 */
1344 TUNABLE_INT_FETCH("vm.pmap.pcid_invlpg_workaround",
1345 &pmap_pcid_invlpg_workaround_uena);
1346 cpu_init_small_core();
1347
1348 if ((cpu_feature2 & CPUID2_XSAVE) != 0) {
1349 use_xsave = 1;
1350 TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
1351 }
1352
1353 link_elf_ireloc(kmdp);
1354
1355 /*
1356 * This may be done better later if it gets more high level
1357 * components in it. If so just link td->td_proc here.
1358 */
1359 proc_linkup0(&proc0, &thread0);
1360
1361 /* Init basic tunables, hz etc */
1362 init_param1();
1363
1364 thread0.td_kstack = physfree - kernphys + KERNSTART;
1365 thread0.td_kstack_pages = kstack_pages;
1366 kstack0_sz = thread0.td_kstack_pages * PAGE_SIZE;
1367 bzero((void *)thread0.td_kstack, kstack0_sz);
1368 physfree += kstack0_sz;
1369
1370 /*
1371 * Initialize enough of thread0 for delayed invalidation to
1372 * work very early. Rely on thread0.td_base_pri
1373 * zero-initialization, it is reset to PVM at proc0_init().
1374 */
1375 pmap_thread_init_invl_gen(&thread0);
1376
1377 pc = &temp_bsp_pcpu;
1378 pcpu_init(pc, 0, sizeof(struct pcpu));
1379 gdt = &temp_bsp_pcpu.pc_gdt[0];
1380
1381 /*
1382 * make gdt memory segments
1383 */
1384 for (x = 0; x < NGDT; x++) {
1385 if (x != GPROC0_SEL && x != (GPROC0_SEL + 1) &&
1386 x != GUSERLDT_SEL && x != (GUSERLDT_SEL) + 1)
1387 ssdtosd(&gdt_segs[x], &gdt[x]);
1388 }
1389 gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&pc->pc_common_tss;
1390 ssdtosyssd(&gdt_segs[GPROC0_SEL],
1391 (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
1392
1393 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
1394 r_gdt.rd_base = (long)gdt;
1395 lgdt(&r_gdt);
1396
1397 wrmsr(MSR_FSBASE, 0); /* User value */
1398 wrmsr(MSR_GSBASE, (u_int64_t)pc);
1399 wrmsr(MSR_KGSBASE, 0); /* User value while in the kernel */
1400
1401 dpcpu_init((void *)(physfree - kernphys + KERNSTART), 0);
1402 physfree += DPCPU_SIZE;
1403 amd64_bsp_pcpu_init1(pc);
1404 /* Non-late cninit() and printf() can be moved up to here. */
1405
1406 /*
1407 * Initialize mutexes.
1408 *
1409 * icu_lock: in order to allow an interrupt to occur in a critical
1410 * section, to set pcpu->ipending (etc...) properly, we
1411 * must be able to get the icu lock, so it can't be
1412 * under witness.
1413 */
1414 mutex_init();
1415 mtx_init(&icu_lock, "icu", NULL, MTX_SPIN | MTX_NOWITNESS);
1416 mtx_init(&dt_lock, "descriptor tables", NULL, MTX_DEF);
1417
1418 /* exceptions */
1419 for (x = 0; x < NIDT; x++)
1420 setidt(x, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_SYSIGT,
1421 SEL_KPL, 0);
1422 setidt(IDT_DE, pti ? &IDTVEC(div_pti) : &IDTVEC(div), SDT_SYSIGT,
1423 SEL_KPL, 0);
1424 setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 4);
1425 setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2);
1426 setidt(IDT_BP, pti ? &IDTVEC(bpt_pti) : &IDTVEC(bpt), SDT_SYSIGT,
1427 SEL_UPL, 0);
1428 setidt(IDT_OF, pti ? &IDTVEC(ofl_pti) : &IDTVEC(ofl), SDT_SYSIGT,
1429 SEL_UPL, 0);
1430 setidt(IDT_BR, pti ? &IDTVEC(bnd_pti) : &IDTVEC(bnd), SDT_SYSIGT,
1431 SEL_KPL, 0);
1432 setidt(IDT_UD, pti ? &IDTVEC(ill_pti) : &IDTVEC(ill), SDT_SYSIGT,
1433 SEL_KPL, 0);
1434 setidt(IDT_NM, pti ? &IDTVEC(dna_pti) : &IDTVEC(dna), SDT_SYSIGT,
1435 SEL_KPL, 0);
1436 setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1);
1437 setidt(IDT_FPUGP, pti ? &IDTVEC(fpusegm_pti) : &IDTVEC(fpusegm),
1438 SDT_SYSIGT, SEL_KPL, 0);
1439 setidt(IDT_TS, pti ? &IDTVEC(tss_pti) : &IDTVEC(tss), SDT_SYSIGT,
1440 SEL_KPL, 0);
1441 setidt(IDT_NP, pti ? &IDTVEC(missing_pti) : &IDTVEC(missing),
1442 SDT_SYSIGT, SEL_KPL, 0);
1443 setidt(IDT_SS, pti ? &IDTVEC(stk_pti) : &IDTVEC(stk), SDT_SYSIGT,
1444 SEL_KPL, 0);
1445 setidt(IDT_GP, pti ? &IDTVEC(prot_pti) : &IDTVEC(prot), SDT_SYSIGT,
1446 SEL_KPL, 0);
1447 setidt(IDT_PF, pti ? &IDTVEC(page_pti) : &IDTVEC(page), SDT_SYSIGT,
1448 SEL_KPL, 0);
1449 setidt(IDT_MF, pti ? &IDTVEC(fpu_pti) : &IDTVEC(fpu), SDT_SYSIGT,
1450 SEL_KPL, 0);
1451 setidt(IDT_AC, pti ? &IDTVEC(align_pti) : &IDTVEC(align), SDT_SYSIGT,
1452 SEL_KPL, 0);
1453 setidt(IDT_MC, &IDTVEC(mchk), SDT_SYSIGT, SEL_KPL, 3);
1454 setidt(IDT_XF, pti ? &IDTVEC(xmm_pti) : &IDTVEC(xmm), SDT_SYSIGT,
1455 SEL_KPL, 0);
1456 #ifdef KDTRACE_HOOKS
1457 setidt(IDT_DTRACE_RET, pti ? &IDTVEC(dtrace_ret_pti) :
1458 &IDTVEC(dtrace_ret), SDT_SYSIGT, SEL_UPL, 0);
1459 #endif
1460 #ifdef XENHVM
1461 setidt(IDT_EVTCHN, pti ? &IDTVEC(xen_intr_upcall_pti) :
1462 &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_KPL, 0);
1463 #endif
1464 r_idt.rd_limit = sizeof(idt0) - 1;
1465 r_idt.rd_base = (long) idt;
1466 lidt(&r_idt);
1467
1468 TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
1469 TUNABLE_INT_FETCH("machdep.mitigations.ibrs.disable", &hw_ibrs_disable);
1470
1471 TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
1472 TUNABLE_INT_FETCH("machdep.mitigations.ssb.disable", &hw_ssb_disable);
1473
1474 TUNABLE_INT_FETCH("machdep.syscall_ret_flush_l1d",
1475 &syscall_ret_l1d_flush_mode);
1476
1477 TUNABLE_INT_FETCH("hw.mds_disable", &hw_mds_disable);
1478 TUNABLE_INT_FETCH("machdep.mitigations.mds.disable", &hw_mds_disable);
1479
1480 TUNABLE_INT_FETCH("machdep.mitigations.taa.enable", &x86_taa_enable);
1481
1482 TUNABLE_INT_FETCH("machdep.mitigations.rngds.enable",
1483 &x86_rngds_mitg_enable);
1484
1485 TUNABLE_INT_FETCH("machdep.mitigations.zenbleed.enable",
1486 &zenbleed_enable);
1487 zenbleed_sanitize_enable();
1488
1489 finishidentcpu(); /* Final stage of CPU initialization */
1490
1491 invlpgb_works = (amd_extended_feature_extensions &
1492 AMDFEID_INVLPGB) != 0;
1493 TUNABLE_INT_FETCH("vm.pmap.invlpgb_works", &invlpgb_works);
1494 if (invlpgb_works)
1495 invlpgb_maxcnt = cpu_procinfo3 & AMDID_INVLPGB_MAXCNT;
1496
1497 /*
1498 * Initialize the clock before the console so that console
1499 * initialization can use DELAY().
1500 */
1501 clock_init();
1502
1503 initializecpu(); /* Initialize CPU registers */
1504
1505 amd64_bsp_ist_init(pc);
1506
1507 /* Set the IO permission bitmap (empty due to tss seg limit) */
1508 pc->pc_common_tss.tss_iobase = sizeof(struct amd64tss) +
1509 IOPERM_BITMAP_SIZE;
1510
1511 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
1512 ltr(gsel_tss);
1513
1514 amd64_conf_fast_syscall();
1515
1516 /*
1517 * We initialize the PCB pointer early so that exception
1518 * handlers will work. Also set up td_critnest to short-cut
1519 * the page fault handler.
1520 */
1521 cpu_max_ext_state_size = sizeof(struct savefpu);
1522 set_top_of_stack_td(&thread0);
1523 thread0.td_pcb = get_pcb_td(&thread0);
1524 thread0.td_critnest = 1;
1525
1526 /*
1527 * The console and kdb should be initialized even earlier than here,
1528 * but some console drivers don't work until after getmemsize().
1529 * Default to late console initialization to support these drivers.
1530 * This loses mainly printf()s in getmemsize() and early debugging.
1531 */
1532 TUNABLE_INT_FETCH("debug.late_console", &late_console);
1533 if (!late_console) {
1534 cninit();
1535 amd64_kdb_init();
1536 }
1537
1538 getmemsize(kmdp, physfree);
1539 init_param2(physmem);
1540
1541 /* now running on new page tables, configured,and u/iom is accessible */
1542
1543 #ifdef DEV_PCI
1544 /* This call might adjust phys_avail[]. */
1545 pci_early_quirks();
1546 #endif
1547
1548 if (late_console)
1549 cninit();
1550
1551 /*
1552 * Dump the boot metadata. We have to wait for cninit() since console
1553 * output is required. If it's grossly incorrect the kernel will never
1554 * make it this far.
1555 */
1556 if (getenv_is_true("debug.dump_modinfo_at_boot"))
1557 preload_dump();
1558
1559 #ifdef DEV_ISA
1560 #ifdef DEV_ATPIC
1561 elcr_probe();
1562 atpic_startup();
1563 #else
1564 /* Reset and mask the atpics and leave them shut down. */
1565 atpic_reset();
1566
1567 /*
1568 * Point the ICU spurious interrupt vectors at the APIC spurious
1569 * interrupt handler.
1570 */
1571 setidt(IDT_IO_INTS + 7, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
1572 setidt(IDT_IO_INTS + 15, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
1573 #endif
1574 #else
1575 #error "have you forgotten the isa device?"
1576 #endif
1577
1578 if (late_console)
1579 amd64_kdb_init();
1580
1581 msgbufinit(msgbufp, msgbufsize);
1582 fpuinit();
1583
1584 /* make an initial tss so cpu can get interrupt stack on syscall! */
1585 rsp0 = thread0.td_md.md_stack_base;
1586 /* Ensure the stack is aligned to 16 bytes */
1587 rsp0 &= ~0xFul;
1588 PCPU_PTR(common_tss)->tss_rsp0 = rsp0;
1589 amd64_bsp_pcpu_init2(rsp0);
1590
1591 /* transfer to user mode */
1592
1593 _ucodesel = GSEL(GUCODE_SEL, SEL_UPL);
1594 _udatasel = GSEL(GUDATA_SEL, SEL_UPL);
1595 _ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL);
1596 _ufssel = GSEL(GUFS32_SEL, SEL_UPL);
1597 _ugssel = GSEL(GUGS32_SEL, SEL_UPL);
1598
1599 load_ds(_udatasel);
1600 load_es(_udatasel);
1601 load_fs(_ufssel);
1602
1603 /* setup proc 0's pcb */
1604 thread0.td_pcb->pcb_flags = 0;
1605
1606 env = kern_getenv("kernelname");
1607 if (env != NULL)
1608 strlcpy(kernelname, env, sizeof(kernelname));
1609
1610 kcsan_cpu_init(0);
1611
1612 #ifdef FDT
1613 x86_init_fdt();
1614 #endif
1615 thread0.td_critnest = 0;
1616
1617 kasan_init();
1618 kmsan_init();
1619
1620 TSEXIT();
1621
1622 /* Location of kernel stack for locore */
1623 return (thread0.td_md.md_stack_base);
1624 }
1625
1626 void
cpu_pcpu_init(struct pcpu * pcpu,int cpuid,size_t size)1627 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
1628 {
1629
1630 pcpu->pc_acpi_id = 0xffffffff;
1631 }
1632
1633 static int
smap_sysctl_handler(SYSCTL_HANDLER_ARGS)1634 smap_sysctl_handler(SYSCTL_HANDLER_ARGS)
1635 {
1636 struct bios_smap *smapbase;
1637 struct bios_smap_xattr smap;
1638 caddr_t kmdp;
1639 uint32_t *smapattr;
1640 int count, error, i;
1641
1642 /* Retrieve the system memory map from the loader. */
1643 kmdp = preload_search_by_type("elf kernel");
1644 if (kmdp == NULL)
1645 kmdp = preload_search_by_type("elf64 kernel");
1646 smapbase = (struct bios_smap *)preload_search_info(kmdp,
1647 MODINFO_METADATA | MODINFOMD_SMAP);
1648 if (smapbase == NULL)
1649 return (0);
1650 smapattr = (uint32_t *)preload_search_info(kmdp,
1651 MODINFO_METADATA | MODINFOMD_SMAP_XATTR);
1652 count = *((uint32_t *)smapbase - 1) / sizeof(*smapbase);
1653 error = 0;
1654 for (i = 0; i < count; i++) {
1655 smap.base = smapbase[i].base;
1656 smap.length = smapbase[i].length;
1657 smap.type = smapbase[i].type;
1658 if (smapattr != NULL)
1659 smap.xattr = smapattr[i];
1660 else
1661 smap.xattr = 0;
1662 error = SYSCTL_OUT(req, &smap, sizeof(smap));
1663 }
1664 return (error);
1665 }
1666 SYSCTL_PROC(_machdep, OID_AUTO, smap,
1667 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
1668 smap_sysctl_handler, "S,bios_smap_xattr",
1669 "Raw BIOS SMAP data");
1670
1671 static int
efi_map_sysctl_handler(SYSCTL_HANDLER_ARGS)1672 efi_map_sysctl_handler(SYSCTL_HANDLER_ARGS)
1673 {
1674 struct efi_map_header *efihdr;
1675 caddr_t kmdp;
1676 uint32_t efisize;
1677
1678 kmdp = preload_search_by_type("elf kernel");
1679 if (kmdp == NULL)
1680 kmdp = preload_search_by_type("elf64 kernel");
1681 efihdr = (struct efi_map_header *)preload_search_info(kmdp,
1682 MODINFO_METADATA | MODINFOMD_EFI_MAP);
1683 if (efihdr == NULL)
1684 return (0);
1685 efisize = *((uint32_t *)efihdr - 1);
1686 return (SYSCTL_OUT(req, efihdr, efisize));
1687 }
1688 SYSCTL_PROC(_machdep, OID_AUTO, efi_map,
1689 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
1690 efi_map_sysctl_handler, "S,efi_map_header",
1691 "Raw EFI Memory Map");
1692
1693 void
spinlock_enter(void)1694 spinlock_enter(void)
1695 {
1696 struct thread *td;
1697 register_t flags;
1698
1699 td = curthread;
1700 if (td->td_md.md_spinlock_count == 0) {
1701 flags = intr_disable();
1702 td->td_md.md_spinlock_count = 1;
1703 td->td_md.md_saved_flags = flags;
1704 critical_enter();
1705 } else
1706 td->td_md.md_spinlock_count++;
1707 }
1708
1709 void
spinlock_exit(void)1710 spinlock_exit(void)
1711 {
1712 struct thread *td;
1713 register_t flags;
1714
1715 td = curthread;
1716 flags = td->td_md.md_saved_flags;
1717 td->td_md.md_spinlock_count--;
1718 if (td->td_md.md_spinlock_count == 0) {
1719 critical_exit();
1720 intr_restore(flags);
1721 }
1722 }
1723
1724 /*
1725 * Construct a PCB from a trapframe. This is called from kdb_trap() where
1726 * we want to start a backtrace from the function that caused us to enter
1727 * the debugger. We have the context in the trapframe, but base the trace
1728 * on the PCB. The PCB doesn't have to be perfect, as long as it contains
1729 * enough for a backtrace.
1730 */
1731 void
makectx(struct trapframe * tf,struct pcb * pcb)1732 makectx(struct trapframe *tf, struct pcb *pcb)
1733 {
1734
1735 pcb->pcb_r12 = tf->tf_r12;
1736 pcb->pcb_r13 = tf->tf_r13;
1737 pcb->pcb_r14 = tf->tf_r14;
1738 pcb->pcb_r15 = tf->tf_r15;
1739 pcb->pcb_rbp = tf->tf_rbp;
1740 pcb->pcb_rbx = tf->tf_rbx;
1741 pcb->pcb_rip = tf->tf_rip;
1742 pcb->pcb_rsp = tf->tf_rsp;
1743 }
1744
1745 /*
1746 * The pcb_flags is only modified by current thread, or by other threads
1747 * when current thread is stopped. However, current thread may change it
1748 * from the interrupt context in cpu_switch(), or in the trap handler.
1749 * When we read-modify-write pcb_flags from C sources, compiler may generate
1750 * code that is not atomic regarding the interrupt handler. If a trap or
1751 * interrupt happens and any flag is modified from the handler, it can be
1752 * clobbered with the cached value later. Therefore, we implement setting
1753 * and clearing flags with single-instruction functions, which do not race
1754 * with possible modification of the flags from the trap or interrupt context,
1755 * because traps and interrupts are executed only on instruction boundary.
1756 */
1757 void
set_pcb_flags_raw(struct pcb * pcb,const u_int flags)1758 set_pcb_flags_raw(struct pcb *pcb, const u_int flags)
1759 {
1760
1761 __asm __volatile("orl %1,%0"
1762 : "=m" (pcb->pcb_flags) : "ir" (flags), "m" (pcb->pcb_flags)
1763 : "cc", "memory");
1764
1765 }
1766
1767 /*
1768 * The support for RDFSBASE, WRFSBASE and similar instructions for %gs
1769 * base requires that kernel saves MSR_FSBASE and MSR_{K,}GSBASE into
1770 * pcb if user space modified the bases. We must save on the context
1771 * switch or if the return to usermode happens through the doreti.
1772 *
1773 * Tracking of both events is performed by the pcb flag PCB_FULL_IRET,
1774 * which have a consequence that the base MSRs must be saved each time
1775 * the PCB_FULL_IRET flag is set. We disable interrupts to sync with
1776 * context switches.
1777 */
1778 static void
set_pcb_flags_fsgsbase(struct pcb * pcb,const u_int flags)1779 set_pcb_flags_fsgsbase(struct pcb *pcb, const u_int flags)
1780 {
1781 register_t r;
1782
1783 if (curpcb == pcb &&
1784 (flags & PCB_FULL_IRET) != 0 &&
1785 (pcb->pcb_flags & PCB_FULL_IRET) == 0) {
1786 r = intr_disable();
1787 if ((pcb->pcb_flags & PCB_FULL_IRET) == 0) {
1788 if (rfs() == _ufssel)
1789 pcb->pcb_fsbase = rdfsbase();
1790 if (rgs() == _ugssel)
1791 pcb->pcb_gsbase = rdmsr(MSR_KGSBASE);
1792 }
1793 set_pcb_flags_raw(pcb, flags);
1794 intr_restore(r);
1795 } else {
1796 set_pcb_flags_raw(pcb, flags);
1797 }
1798 }
1799
1800 DEFINE_IFUNC(, void, set_pcb_flags, (struct pcb *, const u_int))
1801 {
1802
1803 return ((cpu_stdext_feature & CPUID_STDEXT_FSGSBASE) != 0 ?
1804 set_pcb_flags_fsgsbase : set_pcb_flags_raw);
1805 }
1806
1807 void
clear_pcb_flags(struct pcb * pcb,const u_int flags)1808 clear_pcb_flags(struct pcb *pcb, const u_int flags)
1809 {
1810
1811 __asm __volatile("andl %1,%0"
1812 : "=m" (pcb->pcb_flags) : "ir" (~flags), "m" (pcb->pcb_flags)
1813 : "cc", "memory");
1814 }
1815
1816 #ifdef KDB
1817
1818 /*
1819 * Provide inb() and outb() as functions. They are normally only available as
1820 * inline functions, thus cannot be called from the debugger.
1821 */
1822
1823 /* silence compiler warnings */
1824 u_char inb_(u_short);
1825 void outb_(u_short, u_char);
1826
1827 u_char
inb_(u_short port)1828 inb_(u_short port)
1829 {
1830 return inb(port);
1831 }
1832
1833 void
outb_(u_short port,u_char data)1834 outb_(u_short port, u_char data)
1835 {
1836 outb(port, data);
1837 }
1838
1839 #endif /* KDB */
1840
1841 #undef memset
1842 #undef memmove
1843 #undef memcpy
1844
1845 void *memset_std(void *buf, int c, size_t len);
1846 void *memset_erms(void *buf, int c, size_t len);
1847 void *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
1848 size_t len);
1849 void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
1850 size_t len);
1851 void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
1852 size_t len);
1853 void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
1854 size_t len);
1855
1856 #ifdef KCSAN
1857 /*
1858 * These fail to build as ifuncs when used with KCSAN.
1859 */
1860 void *
memset(void * buf,int c,size_t len)1861 memset(void *buf, int c, size_t len)
1862 {
1863
1864 return (memset_std(buf, c, len));
1865 }
1866
1867 void *
memmove(void * _Nonnull dst,const void * _Nonnull src,size_t len)1868 memmove(void * _Nonnull dst, const void * _Nonnull src, size_t len)
1869 {
1870
1871 return (memmove_std(dst, src, len));
1872 }
1873
1874 void *
memcpy(void * _Nonnull dst,const void * _Nonnull src,size_t len)1875 memcpy(void * _Nonnull dst, const void * _Nonnull src, size_t len)
1876 {
1877
1878 return (memcpy_std(dst, src, len));
1879 }
1880 #else
1881 DEFINE_IFUNC(, void *, memset, (void *, int, size_t))
1882 {
1883
1884 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
1885 memset_erms : memset_std);
1886 }
1887
1888 DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull,
1889 size_t))
1890 {
1891
1892 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
1893 memmove_erms : memmove_std);
1894 }
1895
1896 DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,size_t))
1897 {
1898
1899 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
1900 memcpy_erms : memcpy_std);
1901 }
1902 #endif
1903
1904 void pagezero_std(void *addr);
1905 void pagezero_erms(void *addr);
1906 DEFINE_IFUNC(, void , pagezero, (void *))
1907 {
1908
1909 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
1910 pagezero_erms : pagezero_std);
1911 }
1912