1 /*
2 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <stdint.h>
30 #include <sys/param.h>
31 #include <mach/mach_types.h>
32 #include <mach/vm_param.h>
33 #include <IOKit/IOHibernatePrivate.h>
34 #include <IOKit/IOLib.h>
35 #include <pexpert/boot.h>
36 #include <libkern/libkern.h>
37
38 #include "IOHibernateInternal.h"
39
40 #include <machine/pal_hibernate.h>
41
42 /*
43 * This code is linked into the kernel but part of the "__HIB" section, which means
44 * its used by code running in the special context of restoring the kernel text and data
45 * from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
46 * it calls or references needs to be careful to only touch memory also in the "__HIB" section.
47 */
48
49 #define HIB_ROUND_PAGE(x) (((x) + PAGE_MASK) & ~PAGE_MASK)
50
51 uint32_t gIOHibernateState;
52
53 uint32_t gIOHibernateDebugFlags;
54
55 static IOHibernateImageHeader _hibernateHeader;
56 IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
57
58 ppnum_t gIOHibernateHandoffPages[64];
59 const uint32_t gIOHibernateHandoffPageCount = sizeof(gIOHibernateHandoffPages)
60 / sizeof(gIOHibernateHandoffPages[0]);
61
62 #if CONFIG_DEBUG
63 #if defined(__arm64__)
64 extern void serial_hibernation_init(void);
65 #endif /* defined(__arm64__) */
66 void hibprintf(const char *fmt, ...);
67 #else
68 #define hibprintf(x...)
69 #endif
70
71
72 #if CONFIG_SLEEP
73 #if defined(__i386__) || defined(__x86_64__)
74 extern void acpi_wake_prot_entry(void);
75 #endif
76 #endif
77
78 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
79
80 #if defined(__i386__) || defined(__x86_64__)
81 #include <i386/proc_reg.h>
82 #else
83
84 static inline uint64_t
rdtsc64(void)85 rdtsc64(void)
86 {
87 return 0;
88 }
89
90 #endif /* defined(__i386__) || defined(__x86_64__) */
91
92 #if defined(__i386__) || defined(__x86_64__)
93
94 #define DBGLOG 1
95
96 #include <architecture/i386/pio.h>
97
98 /* standard port addresses */
99 enum {
100 COM1_PORT_ADDR = 0x3f8,
101 COM2_PORT_ADDR = 0x2f8
102 };
103
104 /* UART register offsets */
105 enum {
106 UART_RBR = 0, /* receive buffer Register (R) */
107 UART_THR = 0, /* transmit holding register (W) */
108 UART_DLL = 0, /* DLAB = 1, divisor latch (LSB) */
109 UART_IER = 1, /* interrupt enable register */
110 UART_DLM = 1, /* DLAB = 1, divisor latch (MSB) */
111 UART_IIR = 2, /* interrupt ident register (R) */
112 UART_FCR = 2, /* fifo control register (W) */
113 UART_LCR = 3, /* line control register */
114 UART_MCR = 4, /* modem control register */
115 UART_LSR = 5, /* line status register */
116 UART_MSR = 6, /* modem status register */
117 UART_SCR = 7 /* scratch register */
118 };
119
120 enum {
121 UART_LCR_8BITS = 0x03,
122 UART_LCR_DLAB = 0x80
123 };
124
125 enum {
126 UART_MCR_DTR = 0x01,
127 UART_MCR_RTS = 0x02,
128 UART_MCR_OUT1 = 0x04,
129 UART_MCR_OUT2 = 0x08,
130 UART_MCR_LOOP = 0x10
131 };
132
133 enum {
134 UART_LSR_DR = 0x01,
135 UART_LSR_OE = 0x02,
136 UART_LSR_PE = 0x04,
137 UART_LSR_FE = 0x08,
138 UART_LSR_THRE = 0x20
139 };
140
141 static void
hib_uart_putc(char c)142 hib_uart_putc(char c)
143 {
144 while (!(inb(COM1_PORT_ADDR + UART_LSR) & UART_LSR_THRE)) {
145 }
146 outb(COM1_PORT_ADDR + UART_THR, c);
147 }
148
149 static int
debug_probe(void)150 debug_probe( void )
151 {
152 /* Verify that the Scratch Register is accessible */
153 outb(COM1_PORT_ADDR + UART_SCR, 0x5a);
154 if (inb(COM1_PORT_ADDR + UART_SCR) != 0x5a) {
155 return false;
156 }
157 outb(COM1_PORT_ADDR + UART_SCR, 0xa5);
158 if (inb(COM1_PORT_ADDR + UART_SCR) != 0xa5) {
159 return false;
160 }
161 hib_uart_putc('\n');
162 return true;
163 }
164
165 #elif defined(__arm64__)
166
167 #define DBGLOG 1
168
169 static void
hib_uart_putc(char c)170 hib_uart_putc(char c)
171 {
172 uart_putc(c);
173 }
174
175 static int
debug_probe(void)176 debug_probe( void )
177 {
178 // todo
179 return false;
180 }
181
182 #endif /* defined(__arm64__) */
183
184 #if defined(__i386__) || defined(__x86_64__) || defined(__arm64__)
185
186 static void
uart_putstring(const char * str)187 uart_putstring(const char *str)
188 {
189 while (*str) {
190 hib_uart_putc(*str++);
191 }
192 }
193
194 static void
uart_putdec(uint64_t num)195 uart_putdec(uint64_t num)
196 {
197 bool leading = true;
198 for (uint64_t pos = 10000000000000000000ull; pos != 0; pos /= 10) {
199 char c = (char) (num / pos);
200 if (c) {
201 leading = false;
202 num -= c * pos;
203 } else if (leading && (pos != 1)) {
204 continue;
205 }
206 hib_uart_putc(c + '0');
207 }
208 }
209
210 static void
uart_puthex(uint64_t num)211 uart_puthex(uint64_t num)
212 {
213 int bit;
214 char c;
215 bool leading = true;
216
217 for (bit = 60; bit >= 0; bit -= 4) {
218 c = 0xf & (num >> bit);
219 if (c) {
220 leading = false;
221 } else if (leading && bit) {
222 continue;
223 }
224 if (c <= 9) {
225 c += '0';
226 } else {
227 c += 'a' - 10;
228 }
229 hib_uart_putc(c);
230 }
231 }
232
233 static void
debug_code(uint32_t code,uint64_t value)234 debug_code(uint32_t code, uint64_t value)
235 {
236 int bit;
237 char c;
238
239 if (!(kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags)) {
240 return;
241 }
242
243 for (bit = 24; bit >= 0; bit -= 8) {
244 c = 0xFF & (code >> bit);
245 if (c) {
246 hib_uart_putc(c);
247 }
248 }
249 hib_uart_putc('=');
250 uart_puthex(value);
251 hib_uart_putc('\n');
252 hib_uart_putc('\r');
253 }
254
255 #endif /* defined(__i386__) || defined(__x86_64__) || defined(__arm64__) */
256
257 #if !defined(DBGLOG)
258 #define debug_probe() (false)
259 #define debug_code(c, v) {}
260 #endif
261
262 enum{
263 kIOHibernateRestoreCodeImageStart = 'imgS',
264 kIOHibernateRestoreCodeImageEnd = 'imgE',
265 kIOHibernateRestoreCodePageIndexStart = 'pgiS',
266 kIOHibernateRestoreCodePageIndexEnd = 'pgiE',
267 kIOHibernateRestoreCodeMapStart = 'mapS',
268 kIOHibernateRestoreCodeMapEnd = 'mapE',
269 kIOHibernateRestoreCodeWakeMapSize = 'wkms',
270 kIOHibernateRestoreCodeConflictPage = 'cfpg',
271 kIOHibernateRestoreCodeConflictSource = 'cfsr',
272 kIOHibernateRestoreCodeNoMemory = 'nomm',
273 kIOHibernateRestoreCodeTag = 'tag ',
274 kIOHibernateRestoreCodeSignature = 'sign',
275 kIOHibernateRestoreCodeMapVirt = 'mapV',
276 kIOHibernateRestoreCodeHandoffPages = 'hand',
277 kIOHibernateRestoreCodeHandoffCount = 'hndc',
278 };
279
280 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
281
282
283 void
__hib_assert(const char * file,int line,const char * expression)284 __hib_assert(const char *file, int line, const char *expression)
285 {
286 uart_putstring(file);
287 hib_uart_putc(':');
288 uart_putdec(line);
289 uart_putstring(" Assertion failed: ");
290 uart_putstring(expression);
291 hib_uart_putc('\n');
292 #if defined(__i386__) || defined(__x86_64__)
293 outb(0xcf9, 6);
294 #endif /* defined(__i386__) || defined(__x86_64__) */
295 while (true) {
296 }
297 }
298
299 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
300
301 uint32_t
hibernate_sum_page(uint8_t * buf,uint32_t ppnum)302 hibernate_sum_page(uint8_t *buf, uint32_t ppnum)
303 {
304 return ((uint32_t *)buf)[((PAGE_SIZE >> 2) - 1) & ppnum];
305 }
306
307 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
308
309 static hibernate_bitmap_t *
hibernate_page_bitmap(hibernate_page_list_t * list,uint32_t page)310 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
311 {
312 uint32_t bank;
313 hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
314
315 for (bank = 0; bank < list->bank_count; bank++) {
316 if ((page >= bitmap->first_page) && (page <= bitmap->last_page)) {
317 break;
318 }
319 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
320 }
321 if (bank == list->bank_count) {
322 bitmap = NULL;
323 }
324
325 return bitmap;
326 }
327
328 hibernate_bitmap_t *
hibernate_page_bitmap_pin(hibernate_page_list_t * list,uint32_t * pPage)329 hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
330 {
331 uint32_t bank, page = *pPage;
332 hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
333
334 for (bank = 0; bank < list->bank_count; bank++) {
335 if (page <= bitmap->first_page) {
336 *pPage = bitmap->first_page;
337 break;
338 }
339 if (page <= bitmap->last_page) {
340 break;
341 }
342 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
343 }
344 if (bank == list->bank_count) {
345 bitmap = NULL;
346 }
347
348 return bitmap;
349 }
350
351 void
hibernate_page_bitset(hibernate_page_list_t * list,boolean_t set,uint32_t page)352 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
353 {
354 hibernate_bitmap_t * bitmap;
355
356 bitmap = hibernate_page_bitmap(list, page);
357 if (bitmap) {
358 page -= bitmap->first_page;
359 if (set) {
360 bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
361 }
362 //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
363 else {
364 bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
365 }
366 //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
367 }
368 }
369
370 boolean_t
hibernate_page_bittst(hibernate_page_list_t * list,uint32_t page)371 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
372 {
373 boolean_t result = TRUE;
374 hibernate_bitmap_t * bitmap;
375
376 bitmap = hibernate_page_bitmap(list, page);
377 if (bitmap) {
378 page -= bitmap->first_page;
379 result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
380 }
381 return result;
382 }
383
384 // count bits clear or set (set == TRUE) starting at page.
385 uint32_t
hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap,uint32_t set,uint32_t page)386 hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
387 {
388 uint32_t index, bit, bits;
389 uint32_t count;
390
391 count = 0;
392
393 index = (page - bitmap->first_page) >> 5;
394 bit = (page - bitmap->first_page) & 31;
395
396 bits = bitmap->bitmap[index];
397 if (set) {
398 bits = ~bits;
399 }
400 bits = (bits << bit);
401 if (bits) {
402 count += __builtin_clz(bits);
403 } else {
404 count += 32 - bit;
405 while (++index < bitmap->bitmapwords) {
406 bits = bitmap->bitmap[index];
407 if (set) {
408 bits = ~bits;
409 }
410 if (bits) {
411 count += __builtin_clz(bits);
412 break;
413 }
414 count += 32;
415 }
416 }
417
418 if ((page + count) > (bitmap->last_page + 1)) {
419 count = (bitmap->last_page + 1) - page;
420 }
421
422 return count;
423 }
424
425 ppnum_t
hibernate_page_list_grab(hibernate_page_list_t * list,uint32_t * pNextFree)426 hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
427 {
428 uint32_t nextFree = *pNextFree;
429 uint32_t nextFreeInBank;
430 hibernate_bitmap_t * bitmap;
431
432 nextFreeInBank = nextFree + 1;
433 while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank))) {
434 nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
435 if (nextFreeInBank <= bitmap->last_page) {
436 *pNextFree = nextFreeInBank;
437 break;
438 }
439 }
440
441 if (!bitmap) {
442 debug_code(kIOHibernateRestoreCodeNoMemory, nextFree);
443 HIB_ASSERT(0);
444 }
445
446 return nextFree;
447 }
448
449 #pragma mark -
450 #pragma mark hibernate_scratch
451
452 void
hibernate_scratch_init(hibernate_scratch_t * scratch,hibernate_page_list_t * map,uint32_t * nextFree)453 hibernate_scratch_init(hibernate_scratch_t * scratch, hibernate_page_list_t * map, uint32_t * nextFree)
454 {
455 // initialize "scratch" so we can start writing into it
456 __nosan_bzero(scratch, sizeof(*scratch));
457 scratch->map = map;
458 scratch->nextFree = nextFree;
459 scratch->headPage = hibernate_page_list_grab(scratch->map, scratch->nextFree);
460 scratch->curPage = (uint8_t *)pal_hib_map(SCRATCH_AREA, ptoa_64(scratch->headPage));
461 }
462
463 void
hibernate_scratch_start_read(hibernate_scratch_t * scratch)464 hibernate_scratch_start_read(hibernate_scratch_t * scratch)
465 {
466 // re-initialize "scratch" so we can start reading from it it
467 hibernate_scratch_t result;
468 __nosan_bzero(&result, sizeof(result));
469 result.headPage = scratch->headPage;
470 result.curPage = (uint8_t *)pal_hib_map(SCRATCH_AREA, ptoa_64(result.headPage));
471 result.totalLength = scratch->curPos;
472 *scratch = result;
473 }
474
475 static void
hibernate_scratch_io(hibernate_scratch_t * scratch,void * buffer,size_t size,bool write)476 hibernate_scratch_io(hibernate_scratch_t * scratch, void * buffer, size_t size, bool write)
477 {
478 // copy data to or from "scratch" based on the value of "write"
479 if (!write) {
480 // check that we are in bounds
481 HIB_ASSERT(scratch->curPos + size <= scratch->totalLength);
482 }
483 while (size) {
484 // if we got to the end of a page (leaving room for our chain pointer), advance to the next page
485 if (scratch->curPagePos == PAGE_SIZE - sizeof(ppnum_t)) {
486 ppnum_t *nextPage = (ppnum_t *)(scratch->curPage + scratch->curPagePos);
487 if (write) {
488 // allocate the next page and store the page number
489 *nextPage = hibernate_page_list_grab(scratch->map, scratch->nextFree);
490 }
491 scratch->curPage = (uint8_t *)pal_hib_map(SCRATCH_AREA, ptoa_64(*nextPage));
492 scratch->curPagePos = 0;
493 }
494 size_t curPageRemaining = PAGE_SIZE - sizeof(ppnum_t) - scratch->curPagePos;
495 size_t toCopy = MIN(size, curPageRemaining);
496 if (write) {
497 // copy from "buffer" into "scratch"
498 __nosan_memcpy(scratch->curPage + scratch->curPagePos, buffer, toCopy);
499 } else {
500 // copy from "scratch" into "buffer"
501 __nosan_memcpy(buffer, scratch->curPage + scratch->curPagePos, toCopy);
502 }
503 scratch->curPos += toCopy;
504 scratch->curPagePos += toCopy;
505 buffer = (void *)((uintptr_t)buffer + toCopy);
506 size -= toCopy;
507 }
508 }
509
510 void
hibernate_scratch_write(hibernate_scratch_t * scratch,const void * buffer,size_t size)511 hibernate_scratch_write(hibernate_scratch_t * scratch, const void * buffer, size_t size)
512 {
513 hibernate_scratch_io(scratch, (void *)(uintptr_t)buffer, size, true);
514 }
515
516 void
hibernate_scratch_read(hibernate_scratch_t * scratch,void * buffer,size_t size)517 hibernate_scratch_read(hibernate_scratch_t * scratch, void * buffer, size_t size)
518 {
519 hibernate_scratch_io(scratch, buffer, size, false);
520 }
521
522 #pragma mark -
523
524 static uint32_t
store_one_page(uint32_t procFlags,uint32_t * src,uint32_t compressedSize,uint8_t * scratch,uint32_t ppnum)525 store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
526 uint8_t * scratch, uint32_t ppnum)
527 {
528 uint64_t dst = ptoa_64(ppnum);
529
530 if (compressedSize != PAGE_SIZE) {
531 dst = pal_hib_map(DEST_COPY_AREA, dst);
532 if (compressedSize != 4) {
533 pal_hib_decompress_page(src, (void *)dst, scratch, compressedSize);
534 } else {
535 size_t i;
536 uint32_t s, *d;
537
538 s = *src;
539 d = (uint32_t *)(uintptr_t)dst;
540 if (!s) {
541 __nosan_bzero((void *) dst, PAGE_SIZE);
542 } else {
543 for (i = 0; i < (PAGE_SIZE / sizeof(int32_t)); i++) {
544 *d++ = s;
545 }
546 }
547 }
548 } else {
549 dst = hibernate_restore_phys_page((uint64_t) (uintptr_t) src, dst, PAGE_SIZE, procFlags);
550 }
551
552 return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum);
553 }
554
555 void
hibernate_reserve_restore_pages(uint64_t headerPhys,IOHibernateImageHeader * header,hibernate_page_list_t * map)556 hibernate_reserve_restore_pages(uint64_t headerPhys, IOHibernateImageHeader *header, hibernate_page_list_t * map)
557 {
558 uint32_t lastImagePage = atop_64_ppnum(HIB_ROUND_PAGE(headerPhys + header->image1Size));
559 uint32_t handoffPages = header->handoffPages;
560 uint32_t handoffPageCount = header->handoffPageCount;
561 uint32_t ppnum;
562
563 // knock all the image pages to be used out of free map
564 for (ppnum = atop_64_ppnum(headerPhys); ppnum <= lastImagePage; ppnum++) {
565 hibernate_page_bitset(map, FALSE, ppnum);
566 }
567 // knock all the handoff pages to be used out of free map
568 for (ppnum = handoffPages; ppnum < (handoffPages + handoffPageCount); ppnum++) {
569 hibernate_page_bitset(map, FALSE, ppnum);
570 }
571 }
572
573 long
hibernate_kernel_entrypoint(uint32_t p1,uint32_t p2,uint32_t p3,uint32_t p4)574 hibernate_kernel_entrypoint(uint32_t p1,
575 uint32_t p2, uint32_t p3, uint32_t p4)
576 {
577 uint64_t headerPhys;
578 uint64_t mapPhys;
579 uint64_t srcPhys;
580 uint64_t imageReadPhys;
581 uint64_t pageIndexPhys;
582 uint32_t * pageIndexSource;
583 hibernate_page_list_t * map;
584 pal_hib_restore_stage_t stage;
585 uint32_t count;
586 uint32_t ppnum;
587 uint32_t page;
588 uint32_t conflictCount;
589 uint32_t compressedSize;
590 uint32_t uncompressedPages;
591 uint32_t * src;
592 uint32_t sum;
593 uint32_t pageSum;
594 uint32_t nextFree;
595 uint32_t lastImagePage;
596 uint32_t lastMapPage;
597 uint32_t lastPageIndexPage;
598 uint32_t handoffPages;
599 uint32_t handoffPageCount;
600 uint8_t * wkdmScratch;
601 hibernate_scratch_t conflictList = {};
602 pal_hib_ctx_t palHibCtx;
603
604 uint64_t timeStart;
605 timeStart = rdtsc64();
606
607 #if defined(__arm64__)
608 serial_hibernation_init();
609 #endif /* defined(__arm64__) */
610
611 #if !defined(__arm64__)
612 static_assert(sizeof(IOHibernateImageHeader) == 512);
613 #endif /* !defined(__arm64__) */
614
615 headerPhys = ptoa_64(p1);
616
617 if ((kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags) && !debug_probe()) {
618 gIOHibernateDebugFlags &= ~kIOHibernateDebugRestoreLogs;
619 }
620
621 debug_code(kIOHibernateRestoreCodeImageStart, headerPhys);
622
623 __nosan_memcpy(gIOHibernateCurrentHeader,
624 (void *) pal_hib_map(IMAGE_AREA, headerPhys),
625 sizeof(IOHibernateImageHeader));
626
627 debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature);
628
629 mapPhys = headerPhys
630 + (offsetof(IOHibernateImageHeader, fileExtentMap)
631 + gIOHibernateCurrentHeader->fileExtentMapSize
632 + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount)
633 + gIOHibernateCurrentHeader->previewSize);
634
635 map = (hibernate_page_list_t *) pal_hib_map(BITMAP_AREA, mapPhys);
636
637
638 // make the rest of the image is safe for atop()
639 uint64_t imageEnd;
640 if (os_add_overflow(headerPhys, gIOHibernateCurrentHeader->image1Size, &imageEnd) || (imageEnd > IO_MAX_PAGE_ADDR)) {
641 HIB_ASSERT(0);
642 }
643
644 lastImagePage = atop_64_ppnum(HIB_ROUND_PAGE(headerPhys + gIOHibernateCurrentHeader->image1Size));
645 lastMapPage = atop_64_ppnum(HIB_ROUND_PAGE(mapPhys + gIOHibernateCurrentHeader->bitmapSize));
646
647 handoffPages = gIOHibernateCurrentHeader->handoffPages;
648 handoffPageCount = gIOHibernateCurrentHeader->handoffPageCount;
649
650 debug_code(kIOHibernateRestoreCodeImageEnd, ptoa_64(lastImagePage));
651 debug_code(kIOHibernateRestoreCodeMapStart, mapPhys);
652 debug_code(kIOHibernateRestoreCodeMapEnd, ptoa_64(lastMapPage));
653
654 debug_code(kIOHibernateRestoreCodeMapVirt, (uintptr_t) map);
655 debug_code(kIOHibernateRestoreCodeHandoffPages, ptoa_64(handoffPages));
656 debug_code(kIOHibernateRestoreCodeHandoffCount, handoffPageCount);
657
658 #if defined(__arm64__)
659 // on arm64 we've already done this in pal_hib_resume_tramp
660 #else /* !defined(__arm64__) */
661 hibernate_reserve_restore_pages(headerPhys, gIOHibernateCurrentHeader, map);
662 #endif /* !defined(__arm64__) */
663
664 nextFree = 0;
665 hibernate_page_list_grab(map, &nextFree);
666
667 pal_hib_resume_init(&palHibCtx, map, &nextFree);
668
669 // allocate scratch space for wkdm
670 wkdmScratch = (uint8_t *)pal_hib_map(WKDM_AREA, ptoa_64(hibernate_page_list_grab(map, &nextFree)));
671
672 sum = gIOHibernateCurrentHeader->actualRestore1Sum;
673 gIOHibernateCurrentHeader->diag[0] = atop_64_ppnum(headerPhys);
674 gIOHibernateCurrentHeader->diag[1] = sum;
675 gIOHibernateCurrentHeader->trampolineTime = 0;
676
677 uncompressedPages = 0;
678 conflictCount = 0;
679
680 compressedSize = PAGE_SIZE;
681 stage = pal_hib_restore_stage_handoff_data;
682 count = 0;
683 srcPhys = 0;
684
685 if (gIOHibernateCurrentHeader->previewSize) {
686 pageIndexPhys = headerPhys
687 + (offsetof(IOHibernateImageHeader, fileExtentMap)
688 + gIOHibernateCurrentHeader->fileExtentMapSize
689 + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount));
690 imageReadPhys = (pageIndexPhys + gIOHibernateCurrentHeader->previewPageListSize);
691 lastPageIndexPage = atop_64_ppnum(HIB_ROUND_PAGE(imageReadPhys));
692 pageIndexSource = (uint32_t *) pal_hib_map(IMAGE2_AREA, pageIndexPhys);
693 } else {
694 pageIndexPhys = 0;
695 lastPageIndexPage = 0;
696 imageReadPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
697 }
698
699 debug_code(kIOHibernateRestoreCodePageIndexStart, pageIndexPhys);
700 debug_code(kIOHibernateRestoreCodePageIndexEnd, ptoa_64(lastPageIndexPage));
701
702 while (1) {
703 switch (stage) {
704 case pal_hib_restore_stage_handoff_data:
705 // copy handoff data
706 count = srcPhys ? 0 : handoffPageCount;
707 if (!count) {
708 break;
709 }
710 if (count > gIOHibernateHandoffPageCount) {
711 count = gIOHibernateHandoffPageCount;
712 }
713 srcPhys = ptoa_64(handoffPages);
714 break;
715
716 case pal_hib_restore_stage_preview_pages:
717 // copy pageIndexSource pages == preview image data
718 if (!srcPhys) {
719 if (!pageIndexPhys) {
720 break;
721 }
722 srcPhys = imageReadPhys;
723 }
724 ppnum = pageIndexSource[0];
725 count = pageIndexSource[1];
726 pageIndexSource += 2;
727 pageIndexPhys += 2 * sizeof(pageIndexSource[0]);
728 imageReadPhys = srcPhys;
729 break;
730
731 case pal_hib_restore_stage_dram_pages:
732 // copy pages
733 if (!srcPhys) {
734 srcPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
735 }
736 src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
737 ppnum = src[0];
738 count = src[1];
739 srcPhys += 2 * sizeof(*src);
740 imageReadPhys = srcPhys;
741 break;
742 }
743
744
745 if (!count) {
746 if (stage == pal_hib_restore_stage_dram_pages) {
747 break;
748 }
749 stage--;
750 srcPhys = 0;
751 continue;
752 }
753
754 for (page = 0; page < count; page++, ppnum++) {
755 uint32_t tag;
756 int conflicts;
757
758 src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
759
760 if (stage == pal_hib_restore_stage_handoff_data) {
761 ppnum = gIOHibernateHandoffPages[page];
762 } else if (stage == pal_hib_restore_stage_dram_pages) {
763 tag = *src++;
764 HIB_ASSERT((tag & ~kIOHibernateTagLength) == kIOHibernateTagSignature);
765 // debug_code(kIOHibernateRestoreCodeTag, (uintptr_t) tag);
766 srcPhys += sizeof(*src);
767 compressedSize = kIOHibernateTagLength & tag;
768 HIB_ASSERT(compressedSize <= PAGE_SIZE);
769 }
770
771 conflicts = (ppnum >= atop_64_ppnum(mapPhys)) && (ppnum <= lastMapPage);
772
773 conflicts |= ((ppnum >= atop_64_ppnum(imageReadPhys)) && (ppnum <= lastImagePage));
774
775 if (stage >= pal_hib_restore_stage_handoff_data) {
776 conflicts |= ((ppnum >= atop_64_ppnum(srcPhys)) && (ppnum <= (handoffPages + handoffPageCount - 1)));
777 }
778
779 if (stage >= pal_hib_restore_stage_preview_pages) {
780 conflicts |= ((ppnum >= atop_64_ppnum(pageIndexPhys)) && (ppnum <= lastPageIndexPage));
781 }
782
783 if (!conflicts) {
784 pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
785 src, compressedSize, wkdmScratch, ppnum);
786 if (stage != pal_hib_restore_stage_handoff_data) {
787 sum += pageSum;
788 }
789 uncompressedPages++;
790 } else {
791 // debug_code(kIOHibernateRestoreCodeConflictPage, ppnum);
792 // debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);
793 conflictCount++;
794 if (!conflictList.headPage) {
795 hibernate_scratch_init(&conflictList, map, &nextFree);
796 }
797 hibernate_scratch_write(&conflictList, &ppnum, sizeof(ppnum));
798 hibernate_scratch_write(&conflictList, &compressedSize, sizeof(compressedSize));
799 hibernate_scratch_write(&conflictList, &stage, sizeof(stage));
800 hibernate_scratch_write(&conflictList, src, compressedSize);
801 }
802 srcPhys += ((compressedSize + 3) & ~3);
803 src += ((compressedSize + 3) >> 2);
804 pal_hib_restored_page(&palHibCtx, stage, ppnum);
805 }
806 }
807
808 /* src points to the last page restored, so we need to skip over that */
809 pal_hib_restore_pal_state(src);
810
811 // -- copy back conflicts
812
813 if (conflictCount) {
814 src = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, ptoa_64(hibernate_page_list_grab(map, &nextFree)));
815 hibernate_scratch_start_read(&conflictList);
816 for (uint32_t i = 0; i < conflictCount; i++) {
817 hibernate_scratch_read(&conflictList, &ppnum, sizeof(ppnum));
818 hibernate_scratch_read(&conflictList, &compressedSize, sizeof(compressedSize));
819 hibernate_scratch_read(&conflictList, &stage, sizeof(stage));
820 HIB_ASSERT(compressedSize <= PAGE_SIZE);
821 hibernate_scratch_read(&conflictList, src, compressedSize);
822 pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
823 src, compressedSize, wkdmScratch, ppnum);
824 if (stage != pal_hib_restore_stage_handoff_data) {
825 sum += pageSum;
826 }
827 uncompressedPages++;
828 }
829 }
830
831 pal_hib_patchup(&palHibCtx);
832
833 // -- image has been destroyed...
834
835 gIOHibernateCurrentHeader->actualImage1Sum = sum;
836 gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
837 gIOHibernateCurrentHeader->conflictCount = conflictCount;
838 gIOHibernateCurrentHeader->nextFree = nextFree;
839
840 gIOHibernateState = kIOHibernateStateWakingFromHibernate;
841
842 gIOHibernateCurrentHeader->trampolineTime = ((uint32_t) (((rdtsc64() - timeStart)) >> 8));
843
844 // debug_code('done', 0);
845
846 #if CONFIG_SLEEP
847 #if defined(__i386__) || defined(__x86_64__)
848 typedef void (*ResetProc)(void);
849 ResetProc proc;
850 proc = HIB_ENTRYPOINT;
851 // flush caches
852 __asm__("wbinvd");
853 proc();
854 return -1;
855 #elif defined(__arm64__)
856 // return control to hibernate_machine_entrypoint
857 return 0;
858 #else
859 // implement me
860 #endif
861 #endif
862 }
863
864 #if CONFIG_DEBUG
865 /* standalone printf implementation */
866 /*-
867 * Copyright (c) 1986, 1988, 1991, 1993
868 * The Regents of the University of California. All rights reserved.
869 * (c) UNIX System Laboratories, Inc.
870 * All or some portions of this file are derived from material licensed
871 * to the University of California by American Telephone and Telegraph
872 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
873 * the permission of UNIX System Laboratories, Inc.
874 *
875 * Redistribution and use in source and binary forms, with or without
876 * modification, are permitted provided that the following conditions
877 * are met:
878 * 1. Redistributions of source code must retain the above copyright
879 * notice, this list of conditions and the following disclaimer.
880 * 2. Redistributions in binary form must reproduce the above copyright
881 * notice, this list of conditions and the following disclaimer in the
882 * documentation and/or other materials provided with the distribution.
883 * 4. Neither the name of the University nor the names of its contributors
884 * may be used to endorse or promote products derived from this software
885 * without specific prior written permission.
886 *
887 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
888 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
889 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
890 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
891 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
892 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
893 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
894 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
895 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
896 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
897 * SUCH DAMAGE.
898 *
899 * @(#)subr_prf.c 8.3 (Berkeley) 1/21/94
900 */
901
902 typedef long ptrdiff_t;
903 char const hibhex2ascii_data[] = "0123456789abcdefghijklmnopqrstuvwxyz";
904 #define hibhex2ascii(hex) (hibhex2ascii_data[hex])
905 #define toupper(c) ((c) - 0x20 * (((c) >= 'a') && ((c) <= 'z')))
906 static size_t
hibstrlen(const char * s)907 hibstrlen(const char *s)
908 {
909 size_t l = 0;
910 while (*s++) {
911 l++;
912 }
913 return l;
914 }
915
916 /* Max number conversion buffer length: a u_quad_t in base 2, plus NUL byte. */
917 #define MAXNBUF (sizeof(intmax_t) * NBBY + 1)
918
919 /*
920 * Put a NUL-terminated ASCII number (base <= 36) in a buffer in reverse
921 * order; return an optional length and a pointer to the last character
922 * written in the buffer (i.e., the first character of the string).
923 * The buffer pointed to by `nbuf' must have length >= MAXNBUF.
924 */
925 static char *
ksprintn(char * nbuf,uintmax_t num,int base,int * lenp,int upper)926 ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper)
927 {
928 char *p, c;
929
930 /* Truncate so we don't call umoddi3, which isn't in __HIB */
931 #if !defined(__LP64__)
932 uint32_t num2 = (uint32_t) num;
933 #else
934 uintmax_t num2 = num;
935 #endif
936
937 p = nbuf;
938 *p = '\0';
939 do {
940 c = hibhex2ascii(num2 % base);
941 *++p = upper ? toupper(c) : c;
942 } while (num2 /= base);
943 if (lenp) {
944 *lenp = (int)(p - nbuf);
945 }
946 return p;
947 }
948
949 /*
950 * Scaled down version of printf(3).
951 *
952 * Two additional formats:
953 *
954 * The format %b is supported to decode error registers.
955 * Its usage is:
956 *
957 * printf("reg=%b\n", regval, "<base><arg>*");
958 *
959 * where <base> is the output base expressed as a control character, e.g.
960 * \10 gives octal; \20 gives hex. Each arg is a sequence of characters,
961 * the first of which gives the bit number to be inspected (origin 1), and
962 * the next characters (up to a control character, i.e. a character <= 32),
963 * give the name of the register. Thus:
964 *
965 * kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE");
966 *
967 * would produce output:
968 *
969 * reg=3<BITTWO,BITONE>
970 *
971 * XXX: %D -- Hexdump, takes pointer and separator string:
972 * ("%6D", ptr, ":") -> XX:XX:XX:XX:XX:XX
973 * ("%*D", len, ptr, " " -> XX XX XX XX ...
974 */
975 static int
hibkvprintf(char const * fmt,void (* func)(int,void *),void * arg,int radix,va_list ap)976 hibkvprintf(char const *fmt, void (*func)(int, void*), void *arg, int radix, va_list ap)
977 {
978 #define PCHAR(c) {int cc=(c); if (func) (*func)(cc,arg); else *d++ = (char)cc; retval++; }
979 char nbuf[MAXNBUF];
980 char *d;
981 const char *p, *percent, *q;
982 u_char *up;
983 int ch, n;
984 uintmax_t num;
985 int base, lflag, qflag, tmp, width, ladjust, sharpflag, neg, sign, dot;
986 int cflag, hflag, jflag, tflag, zflag;
987 int dwidth, upper;
988 char padc;
989 int stop = 0, retval = 0;
990
991 num = 0;
992 if (!func) {
993 d = (char *) arg;
994 } else {
995 d = NULL;
996 }
997
998 if (fmt == NULL) {
999 fmt = "(fmt null)\n";
1000 }
1001
1002 if (radix < 2 || radix > 36) {
1003 radix = 10;
1004 }
1005
1006 for (;;) {
1007 padc = ' ';
1008 width = 0;
1009 while ((ch = (u_char) * fmt++) != '%' || stop) {
1010 if (ch == '\0') {
1011 return retval;
1012 }
1013 PCHAR(ch);
1014 }
1015 percent = fmt - 1;
1016 qflag = 0; lflag = 0; ladjust = 0; sharpflag = 0; neg = 0;
1017 sign = 0; dot = 0; dwidth = 0; upper = 0;
1018 cflag = 0; hflag = 0; jflag = 0; tflag = 0; zflag = 0;
1019 reswitch: switch (ch = (u_char) * fmt++) {
1020 case '.':
1021 dot = 1;
1022 goto reswitch;
1023 case '#':
1024 sharpflag = 1;
1025 goto reswitch;
1026 case '+':
1027 sign = 1;
1028 goto reswitch;
1029 case '-':
1030 ladjust = 1;
1031 goto reswitch;
1032 case '%':
1033 PCHAR(ch);
1034 break;
1035 case '*':
1036 if (!dot) {
1037 width = va_arg(ap, int);
1038 if (width < 0) {
1039 ladjust = !ladjust;
1040 width = -width;
1041 }
1042 } else {
1043 dwidth = va_arg(ap, int);
1044 }
1045 goto reswitch;
1046 case '0':
1047 if (!dot) {
1048 padc = '0';
1049 goto reswitch;
1050 }
1051 case '1': case '2': case '3': case '4':
1052 case '5': case '6': case '7': case '8': case '9':
1053 for (n = 0;; ++fmt) {
1054 n = n * 10 + ch - '0';
1055 ch = *fmt;
1056 if (ch < '0' || ch > '9') {
1057 break;
1058 }
1059 }
1060 if (dot) {
1061 dwidth = n;
1062 } else {
1063 width = n;
1064 }
1065 goto reswitch;
1066 case 'b':
1067 num = (u_int)va_arg(ap, int);
1068 p = va_arg(ap, char *);
1069 for (q = ksprintn(nbuf, num, *p++, NULL, 0); *q;) {
1070 PCHAR(*q--);
1071 }
1072
1073 if (num == 0) {
1074 break;
1075 }
1076
1077 for (tmp = 0; *p;) {
1078 n = *p++;
1079 if (num & (1 << (n - 1))) {
1080 PCHAR(tmp ? ',' : '<');
1081 for (; (n = *p) > ' '; ++p) {
1082 PCHAR(n);
1083 }
1084 tmp = 1;
1085 } else {
1086 for (; *p > ' '; ++p) {
1087 continue;
1088 }
1089 }
1090 }
1091 if (tmp) {
1092 PCHAR('>');
1093 }
1094 break;
1095 case 'c':
1096 PCHAR(va_arg(ap, int));
1097 break;
1098 case 'D':
1099 up = va_arg(ap, u_char *);
1100 p = va_arg(ap, char *);
1101 if (!width) {
1102 width = 16;
1103 }
1104 while (width--) {
1105 PCHAR(hibhex2ascii(*up >> 4));
1106 PCHAR(hibhex2ascii(*up & 0x0f));
1107 up++;
1108 if (width) {
1109 for (q = p; *q; q++) {
1110 PCHAR(*q);
1111 }
1112 }
1113 }
1114 break;
1115 case 'd':
1116 case 'i':
1117 base = 10;
1118 sign = 1;
1119 goto handle_sign;
1120 case 'h':
1121 if (hflag) {
1122 hflag = 0;
1123 cflag = 1;
1124 } else {
1125 hflag = 1;
1126 }
1127 goto reswitch;
1128 case 'j':
1129 jflag = 1;
1130 goto reswitch;
1131 case 'l':
1132 if (lflag) {
1133 lflag = 0;
1134 qflag = 1;
1135 } else {
1136 lflag = 1;
1137 }
1138 goto reswitch;
1139 case 'n':
1140 if (jflag) {
1141 *(va_arg(ap, intmax_t *)) = retval;
1142 } else if (qflag) {
1143 *(va_arg(ap, quad_t *)) = retval;
1144 } else if (lflag) {
1145 *(va_arg(ap, long *)) = retval;
1146 } else if (zflag) {
1147 *(va_arg(ap, size_t *)) = retval;
1148 } else if (hflag) {
1149 *(va_arg(ap, short *)) = (short)retval;
1150 } else if (cflag) {
1151 *(va_arg(ap, char *)) = (char)retval;
1152 } else {
1153 *(va_arg(ap, int *)) = retval;
1154 }
1155 break;
1156 case 'o':
1157 base = 8;
1158 goto handle_nosign;
1159 case 'p':
1160 base = 16;
1161 sharpflag = (width == 0);
1162 sign = 0;
1163 num = (uintptr_t)va_arg(ap, void *);
1164 goto number;
1165 case 'q':
1166 qflag = 1;
1167 goto reswitch;
1168 case 'r':
1169 base = radix;
1170 if (sign) {
1171 goto handle_sign;
1172 }
1173 goto handle_nosign;
1174 case 's':
1175 p = va_arg(ap, char *);
1176 if (p == NULL) {
1177 p = "(null)";
1178 }
1179 if (!dot) {
1180 n = (typeof(n))hibstrlen(p);
1181 } else {
1182 for (n = 0; n < dwidth && p[n]; n++) {
1183 continue;
1184 }
1185 }
1186
1187 width -= n;
1188
1189 if (!ladjust && width > 0) {
1190 while (width--) {
1191 PCHAR(padc);
1192 }
1193 }
1194 while (n--) {
1195 PCHAR(*p++);
1196 }
1197 if (ladjust && width > 0) {
1198 while (width--) {
1199 PCHAR(padc);
1200 }
1201 }
1202 break;
1203 case 't':
1204 tflag = 1;
1205 goto reswitch;
1206 case 'u':
1207 base = 10;
1208 goto handle_nosign;
1209 case 'X':
1210 upper = 1;
1211 case 'x':
1212 base = 16;
1213 goto handle_nosign;
1214 case 'y':
1215 base = 16;
1216 sign = 1;
1217 goto handle_sign;
1218 case 'z':
1219 zflag = 1;
1220 goto reswitch;
1221 handle_nosign:
1222 sign = 0;
1223 if (jflag) {
1224 num = va_arg(ap, uintmax_t);
1225 } else if (qflag) {
1226 num = va_arg(ap, u_quad_t);
1227 } else if (tflag) {
1228 num = va_arg(ap, ptrdiff_t);
1229 } else if (lflag) {
1230 num = va_arg(ap, u_long);
1231 } else if (zflag) {
1232 num = va_arg(ap, size_t);
1233 } else if (hflag) {
1234 num = (u_short)va_arg(ap, int);
1235 } else if (cflag) {
1236 num = (u_char)va_arg(ap, int);
1237 } else {
1238 num = va_arg(ap, u_int);
1239 }
1240 goto number;
1241 handle_sign:
1242 if (jflag) {
1243 num = va_arg(ap, intmax_t);
1244 } else if (qflag) {
1245 num = va_arg(ap, quad_t);
1246 } else if (tflag) {
1247 num = va_arg(ap, ptrdiff_t);
1248 } else if (lflag) {
1249 num = va_arg(ap, long);
1250 } else if (zflag) {
1251 num = va_arg(ap, ssize_t);
1252 } else if (hflag) {
1253 num = (short)va_arg(ap, int);
1254 } else if (cflag) {
1255 num = (char)va_arg(ap, int);
1256 } else {
1257 num = va_arg(ap, int);
1258 }
1259 number:
1260 if (sign && (intmax_t)num < 0) {
1261 neg = 1;
1262 num = -(intmax_t)num;
1263 }
1264 p = ksprintn(nbuf, num, base, &tmp, upper);
1265 if (sharpflag && num != 0) {
1266 if (base == 8) {
1267 tmp++;
1268 } else if (base == 16) {
1269 tmp += 2;
1270 }
1271 }
1272 if (neg) {
1273 tmp++;
1274 }
1275
1276 if (!ladjust && padc != '0' && width
1277 && (width -= tmp) > 0) {
1278 while (width--) {
1279 PCHAR(padc);
1280 }
1281 }
1282 if (neg) {
1283 PCHAR('-');
1284 }
1285 if (sharpflag && num != 0) {
1286 if (base == 8) {
1287 PCHAR('0');
1288 } else if (base == 16) {
1289 PCHAR('0');
1290 PCHAR('x');
1291 }
1292 }
1293 if (!ladjust && width && (width -= tmp) > 0) {
1294 while (width--) {
1295 PCHAR(padc);
1296 }
1297 }
1298
1299 while (*p) {
1300 PCHAR(*p--);
1301 }
1302
1303 if (ladjust && width && (width -= tmp) > 0) {
1304 while (width--) {
1305 PCHAR(padc);
1306 }
1307 }
1308
1309 break;
1310 default:
1311 while (percent < fmt) {
1312 PCHAR(*percent++);
1313 }
1314 /*
1315 * Since we ignore a formatting argument it is no
1316 * longer safe to obey the remaining formatting
1317 * arguments as the arguments will no longer match
1318 * the format specs.
1319 */
1320 stop = 1;
1321 break;
1322 }
1323 }
1324 #undef PCHAR
1325 }
1326
1327
1328 static void
putchar(int c,void * arg)1329 putchar(int c, void *arg)
1330 {
1331 (void)arg;
1332 hib_uart_putc((char)c);
1333 }
1334
1335 void
hibprintf(const char * fmt,...)1336 hibprintf(const char *fmt, ...)
1337 {
1338 /* http://www.pagetable.com/?p=298 */
1339 va_list ap;
1340
1341 va_start(ap, fmt);
1342 hibkvprintf(fmt, putchar, NULL, 10, ap);
1343 va_end(ap);
1344 }
1345 #endif /* CONFIG_DEBUG */
1346
1347 #if __arm64__ && HIBERNATE_TRAP_HANDLER
1348 void
hibernate_trap(__unused arm_context_t * context,__unused uint64_t trap_addr)1349 hibernate_trap(__unused arm_context_t *context, __unused uint64_t trap_addr)
1350 __attribute__((optnone))
1351 {
1352 // enable logging
1353 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
1354
1355 // dump some interesting registers
1356 for (int i = 0; i < 29; i++) {
1357 debug_code(' r00' + (i / 10 * 256) + (i % 10), context->ss.ss_64.x[i]);
1358 }
1359 debug_code(' fp', context->ss.ss_64.fp);
1360 debug_code(' lr', context->ss.ss_64.lr);
1361 debug_code(' sp', context->ss.ss_64.sp);
1362 debug_code(' pc', context->ss.ss_64.pc);
1363 debug_code('cpsr', context->ss.ss_64.cpsr);
1364 debug_code(' far', context->ss.ss_64.far);
1365 debug_code(' esr', context->ss.ss_64.esr);
1366
1367 // dump the trap_addr
1368 debug_code('trap', trap_addr);
1369
1370 // dump the kernel slide
1371 debug_code('slid', _hibernateHeader.kernVirtSlide);
1372
1373 // loop forever
1374 while (true) {
1375 ;
1376 }
1377 }
1378 #endif /* __arm64__ && HIBERNATE_TRAP_HANDLER */
1379