1 /*-
2 * Copyright (c) 2004 Marcel Moolenaar
3 * Copyright (c) 2001 Doug Rabson
4 * Copyright (c) 2016 The FreeBSD Foundation
5 * All rights reserved.
6 *
7 * Portions of this software were developed by Konstantin Belousov
8 * under sponsorship from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 #include <sys/param.h>
34 #include <sys/efi.h>
35 #include <sys/kernel.h>
36 #include <sys/linker.h>
37 #include <sys/lock.h>
38 #include <sys/module.h>
39 #include <sys/mutex.h>
40 #include <sys/clock.h>
41 #include <sys/proc.h>
42 #include <sys/rwlock.h>
43 #include <sys/sched.h>
44 #include <sys/sysctl.h>
45 #include <sys/systm.h>
46 #include <sys/vmmeter.h>
47 #include <isa/rtc.h>
48 #include <machine/efi.h>
49 #include <machine/md_var.h>
50 #include <machine/vmparam.h>
51 #include <vm/vm.h>
52 #include <vm/pmap.h>
53 #include <vm/vm_extern.h>
54 #include <vm/vm_map.h>
55 #include <vm/vm_object.h>
56 #include <vm/vm_page.h>
57 #include <vm/vm_pager.h>
58
59 static pml5_entry_t *efi_pml5;
60 static pml4_entry_t *efi_pml4;
61 static vm_object_t obj_1t1_pt;
62 static vm_page_t efi_pmltop_page;
63 static vm_pindex_t efi_1t1_idx;
64
65 void
efi_destroy_1t1_map(void)66 efi_destroy_1t1_map(void)
67 {
68 vm_page_t m;
69
70 if (obj_1t1_pt != NULL) {
71 VM_OBJECT_RLOCK(obj_1t1_pt);
72 TAILQ_FOREACH(m, &obj_1t1_pt->memq, listq)
73 m->ref_count = VPRC_OBJREF;
74 vm_wire_sub(obj_1t1_pt->resident_page_count);
75 VM_OBJECT_RUNLOCK(obj_1t1_pt);
76 vm_object_deallocate(obj_1t1_pt);
77 }
78
79 obj_1t1_pt = NULL;
80 efi_pml4 = NULL;
81 efi_pml5 = NULL;
82 efi_pmltop_page = NULL;
83 }
84
85 /*
86 * Map a physical address from EFI runtime space into KVA space. Returns 0 to
87 * indicate a failed mapping so that the caller may handle error.
88 */
89 vm_offset_t
efi_phys_to_kva(vm_paddr_t paddr)90 efi_phys_to_kva(vm_paddr_t paddr)
91 {
92
93 if (paddr >= dmaplimit)
94 return (0);
95 return (PHYS_TO_DMAP(paddr));
96 }
97
98 static vm_page_t
efi_1t1_page(void)99 efi_1t1_page(void)
100 {
101
102 return (vm_page_grab(obj_1t1_pt, efi_1t1_idx++, VM_ALLOC_NOBUSY |
103 VM_ALLOC_WIRED | VM_ALLOC_ZERO));
104 }
105
106 static pt_entry_t *
efi_1t1_pte(vm_offset_t va)107 efi_1t1_pte(vm_offset_t va)
108 {
109 pml5_entry_t *pml5e;
110 pml4_entry_t *pml4e;
111 pdp_entry_t *pdpe;
112 pd_entry_t *pde;
113 pt_entry_t *pte;
114 vm_page_t m;
115 vm_pindex_t pml5_idx, pml4_idx, pdp_idx, pd_idx;
116 vm_paddr_t mphys;
117
118 pml4_idx = pmap_pml4e_index(va);
119 if (la57) {
120 pml5_idx = pmap_pml5e_index(va);
121 pml5e = &efi_pml5[pml5_idx];
122 if (*pml5e == 0) {
123 m = efi_1t1_page();
124 mphys = VM_PAGE_TO_PHYS(m);
125 *pml5e = mphys | X86_PG_RW | X86_PG_V;
126 } else {
127 mphys = *pml5e & PG_FRAME;
128 }
129 pml4e = (pml4_entry_t *)PHYS_TO_DMAP(mphys);
130 pml4e = &pml4e[pml4_idx];
131 } else {
132 pml4e = &efi_pml4[pml4_idx];
133 }
134
135 if (*pml4e == 0) {
136 m = efi_1t1_page();
137 mphys = VM_PAGE_TO_PHYS(m);
138 *pml4e = mphys | X86_PG_RW | X86_PG_V;
139 } else {
140 mphys = *pml4e & PG_FRAME;
141 }
142
143 pdpe = (pdp_entry_t *)PHYS_TO_DMAP(mphys);
144 pdp_idx = pmap_pdpe_index(va);
145 pdpe += pdp_idx;
146 if (*pdpe == 0) {
147 m = efi_1t1_page();
148 mphys = VM_PAGE_TO_PHYS(m);
149 *pdpe = mphys | X86_PG_RW | X86_PG_V;
150 } else {
151 mphys = *pdpe & PG_FRAME;
152 }
153
154 pde = (pd_entry_t *)PHYS_TO_DMAP(mphys);
155 pd_idx = pmap_pde_index(va);
156 pde += pd_idx;
157 if (*pde == 0) {
158 m = efi_1t1_page();
159 mphys = VM_PAGE_TO_PHYS(m);
160 *pde = mphys | X86_PG_RW | X86_PG_V;
161 } else {
162 mphys = *pde & PG_FRAME;
163 }
164
165 pte = (pt_entry_t *)PHYS_TO_DMAP(mphys);
166 pte += pmap_pte_index(va);
167 KASSERT(*pte == 0, ("va %#jx *pt %#jx", va, *pte));
168
169 return (pte);
170 }
171
172 bool
efi_create_1t1_map(struct efi_md * map,int ndesc,int descsz)173 efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz)
174 {
175 struct efi_md *p;
176 pt_entry_t *pte;
177 void *pml;
178 vm_page_t m;
179 vm_offset_t va;
180 uint64_t idx;
181 int bits, i, mode;
182
183 obj_1t1_pt = vm_pager_allocate(OBJT_PHYS, NULL, ptoa(1 +
184 NPML4EPG + NPML4EPG * NPDPEPG + NPML4EPG * NPDPEPG * NPDEPG),
185 VM_PROT_ALL, 0, NULL);
186 efi_1t1_idx = 0;
187 VM_OBJECT_WLOCK(obj_1t1_pt);
188 efi_pmltop_page = efi_1t1_page();
189 VM_OBJECT_WUNLOCK(obj_1t1_pt);
190 pml = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(efi_pmltop_page));
191 if (la57) {
192 efi_pml5 = pml;
193 pmap_pinit_pml5(efi_pmltop_page);
194 } else {
195 efi_pml4 = pml;
196 pmap_pinit_pml4(efi_pmltop_page);
197 }
198
199 for (i = 0, p = map; i < ndesc; i++, p = efi_next_descriptor(p,
200 descsz)) {
201 if ((p->md_attr & EFI_MD_ATTR_RT) == 0)
202 continue;
203 if (p->md_virt != 0 && p->md_virt != p->md_phys) {
204 if (bootverbose)
205 printf("EFI Runtime entry %d is mapped\n", i);
206 goto fail;
207 }
208 if ((p->md_phys & EFI_PAGE_MASK) != 0) {
209 if (bootverbose)
210 printf("EFI Runtime entry %d is not aligned\n",
211 i);
212 goto fail;
213 }
214 if (p->md_phys + p->md_pages * EFI_PAGE_SIZE < p->md_phys ||
215 p->md_phys + p->md_pages * EFI_PAGE_SIZE >=
216 VM_MAXUSER_ADDRESS) {
217 printf("EFI Runtime entry %d is not in mappable for RT:"
218 "base %#016jx %#jx pages\n",
219 i, (uintmax_t)p->md_phys,
220 (uintmax_t)p->md_pages);
221 goto fail;
222 }
223 if ((p->md_attr & EFI_MD_ATTR_WB) != 0)
224 mode = VM_MEMATTR_WRITE_BACK;
225 else if ((p->md_attr & EFI_MD_ATTR_WT) != 0)
226 mode = VM_MEMATTR_WRITE_THROUGH;
227 else if ((p->md_attr & EFI_MD_ATTR_WC) != 0)
228 mode = VM_MEMATTR_WRITE_COMBINING;
229 else if ((p->md_attr & EFI_MD_ATTR_WP) != 0)
230 mode = VM_MEMATTR_WRITE_PROTECTED;
231 else if ((p->md_attr & EFI_MD_ATTR_UC) != 0)
232 mode = VM_MEMATTR_UNCACHEABLE;
233 else {
234 if (bootverbose)
235 printf("EFI Runtime entry %d mapping "
236 "attributes unsupported\n", i);
237 mode = VM_MEMATTR_UNCACHEABLE;
238 }
239 bits = pmap_cache_bits(kernel_pmap, mode, FALSE) | X86_PG_RW |
240 X86_PG_V;
241 VM_OBJECT_WLOCK(obj_1t1_pt);
242 for (va = p->md_phys, idx = 0; idx < p->md_pages; idx++,
243 va += PAGE_SIZE) {
244 pte = efi_1t1_pte(va);
245 pte_store(pte, va | bits);
246
247 m = PHYS_TO_VM_PAGE(va);
248 if (m != NULL && VM_PAGE_TO_PHYS(m) == 0) {
249 vm_page_init_page(m, va, -1);
250 m->order = VM_NFREEORDER + 1; /* invalid */
251 m->pool = VM_NFREEPOOL + 1; /* invalid */
252 pmap_page_set_memattr_noflush(m, mode);
253 }
254 }
255 VM_OBJECT_WUNLOCK(obj_1t1_pt);
256 }
257
258 return (true);
259
260 fail:
261 efi_destroy_1t1_map();
262 return (false);
263 }
264
265 /*
266 * Create an environment for the EFI runtime code call. The most
267 * important part is creating the required 1:1 physical->virtual
268 * mappings for the runtime segments. To do that, we manually create
269 * page table which unmap userspace but gives correct kernel mapping.
270 * The 1:1 mappings for runtime segments usually occupy low 4G of the
271 * physical address map.
272 *
273 * The 1:1 mappings were chosen over the SetVirtualAddressMap() EFI RT
274 * service, because there are some BIOSes which fail to correctly
275 * relocate itself on the call, requiring both 1:1 and virtual
276 * mapping. As result, we must provide 1:1 mapping anyway, so no
277 * reason to bother with the virtual map, and no need to add a
278 * complexity into loader.
279 *
280 * There is no need to disable interrupts around the change of %cr3,
281 * the kernel mappings are correct, while we only grabbed the
282 * userspace portion of VA. Interrupts handlers must not access
283 * userspace. Having interrupts enabled fixes the issue with
284 * firmware/SMM long operation, which would negatively affect IPIs,
285 * esp. TLB shootdown requests.
286 */
287 int
efi_arch_enter(void)288 efi_arch_enter(void)
289 {
290 pmap_t curpmap;
291 uint64_t cr3;
292
293 curpmap = PCPU_GET(curpmap);
294 PMAP_LOCK_ASSERT(curpmap, MA_OWNED);
295 curthread->td_md.md_efirt_dis_pf = vm_fault_disable_pagefaults();
296
297 /*
298 * IPI TLB shootdown handler invltlb_pcid_handler() reloads
299 * %cr3 from the curpmap->pm_cr3, which would disable runtime
300 * segments mappings. Block the handler's action by setting
301 * curpmap to impossible value. See also comment in
302 * pmap.c:pmap_activate_sw().
303 */
304 if (pmap_pcid_enabled && !invpcid_works)
305 PCPU_SET(curpmap, NULL);
306
307 cr3 = VM_PAGE_TO_PHYS(efi_pmltop_page);
308 if (pmap_pcid_enabled)
309 cr3 |= pmap_get_pcid(curpmap);
310 load_cr3(cr3);
311 /*
312 * If PCID is enabled, the clear CR3_PCID_SAVE bit in the loaded %cr3
313 * causes TLB invalidation.
314 */
315 if (!pmap_pcid_enabled)
316 invltlb();
317 return (0);
318 }
319
320 void
efi_arch_leave(void)321 efi_arch_leave(void)
322 {
323 pmap_t curpmap;
324 uint64_t cr3;
325
326 curpmap = &curproc->p_vmspace->vm_pmap;
327 cr3 = curpmap->pm_cr3;
328 if (pmap_pcid_enabled) {
329 cr3 |= pmap_get_pcid(curpmap);
330 if (!invpcid_works)
331 PCPU_SET(curpmap, curpmap);
332 }
333 load_cr3(cr3);
334 if (!pmap_pcid_enabled)
335 invltlb();
336 vm_fault_enable_pagefaults(curthread->td_md.md_efirt_dis_pf);
337 }
338
339 /* XXX debug stuff */
340 static int
efi_time_sysctl_handler(SYSCTL_HANDLER_ARGS)341 efi_time_sysctl_handler(SYSCTL_HANDLER_ARGS)
342 {
343 struct efi_tm tm;
344 int error, val;
345
346 val = 0;
347 error = sysctl_handle_int(oidp, &val, 0, req);
348 if (error != 0 || req->newptr == NULL)
349 return (error);
350 error = efi_get_time(&tm);
351 if (error == 0) {
352 uprintf("EFI reports: Year %d Month %d Day %d Hour %d Min %d "
353 "Sec %d\n", tm.tm_year, tm.tm_mon, tm.tm_mday, tm.tm_hour,
354 tm.tm_min, tm.tm_sec);
355 }
356 return (error);
357 }
358
359 SYSCTL_PROC(_debug, OID_AUTO, efi_time,
360 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
361 efi_time_sysctl_handler, "I",
362 "");
363