1 /*
2  * Copyright (c) 2006-2019 Apple Inc. All rights reserved.
3  *
4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5  *
6  * This file contains Original Code and/or Modifications of Original Code
7  * as defined in and that are subject to the Apple Public Source License
8  * Version 2.0 (the 'License'). You may not use this file except in
9  * compliance with the License. The rights granted to you under the License
10  * may not be used to create, or enable the creation or redistribution of,
11  * unlawful or unlicensed copies of an Apple operating system, or to
12  * circumvent, violate, or enable the circumvention or violation of, any
13  * terms of an Apple operating system software license agreement.
14  *
15  * Please obtain a copy of the License at
16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
17  *
18  * The Original Code and all software distributed under the License are
19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23  * Please see the License for the specific language governing rights and
24  * limitations under the License.
25  *
26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27  */
28 
29 #include <sys/uio.h>
30 #include <sys/conf.h>
31 
32 #include <IOKit/IOLib.h>
33 #include <IOKit/IOBSD.h>
34 #include <IOKit/IOService.h>
35 #include <IOKit/IOPlatformExpert.h>
36 #include <IOKit/IOPolledInterface.h>
37 #include <IOKit/IOHibernatePrivate.h>
38 #include <IOKit/IOBufferMemoryDescriptor.h>
39 #include <IOKit/AppleKeyStoreInterface.h>
40 #include <libkern/c++/OSSharedPtr.h>
41 #include "IOKitKernelInternal.h"
42 
43 #if defined(__arm64__)
44 #include <pexpert/arm64/board_config.h>
45 #endif /* defined(__arm64__) */
46 
47 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
48 
49 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
50 
51 OSMetaClassDefineReservedUsedX86(IOPolledInterface, 0);
52 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
53 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
54 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
55 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
56 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
57 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
58 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
59 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
60 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
61 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
62 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
63 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
64 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
65 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
66 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
67 
68 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
69 
70 #ifndef kIOMediaPreferredBlockSizeKey
71 #define kIOMediaPreferredBlockSizeKey   "Preferred Block Size"
72 #endif
73 
74 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
75 
76 class IOPolledFilePollers : public OSObject
77 {
78 	OSDeclareDefaultStructors(IOPolledFilePollers);
79 
80 public:
81 	IOService                * media;
82 	OSArray                  * pollers;
83 	IOBufferMemoryDescriptor * ioBuffer;
84 	bool                 abortable;
85 	bool                 io;
86 	IOReturn             ioStatus;
87 	uint32_t             openCount;
88 
89 	static IOPolledFilePollers * copyPollers(IOService * media);
90 };
91 
92 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
93 
94 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
95 
96 IOPolledFilePollers *
97 IOPolledFilePollers::copyPollers(IOService * media)
98 {
99 	IOPolledFilePollers * vars;
100 	IOReturn              err;
101 	IOService       * service;
102 	OSObject        * obj;
103 	IORegistryEntry * next;
104 	IORegistryEntry * child;
105 
106 	if ((obj = media->copyProperty(kIOPolledInterfaceStackKey))) {
107 		IOPolledFilePollers * ioPFPObj = OSDynamicCast(IOPolledFilePollers, obj);
108 		if (!ioPFPObj) {
109 			OSSafeReleaseNULL(obj);
110 		}
111 		return ioPFPObj;
112 	}
113 
114 	do{
115 		vars = OSTypeAlloc(IOPolledFilePollers);
116 		vars->init();
117 
118 		vars->pollers = OSArray::withCapacity(4);
119 		if (!vars->pollers) {
120 			err = kIOReturnNoMemory;
121 			break;
122 		}
123 
124 		next = vars->media = media;
125 		do{
126 			IOPolledInterface * poller;
127 			OSObject *          obj;
128 
129 			obj = next->getProperty(kIOPolledInterfaceSupportKey);
130 			if (kOSBooleanFalse == obj) {
131 				vars->pollers->flushCollection();
132 				break;
133 			} else if ((poller = OSDynamicCast(IOPolledInterface, obj))) {
134 				vars->pollers->setObject(poller);
135 			}
136 
137 			if ((service = OSDynamicCast(IOService, next))
138 			    && service->getDeviceMemory()
139 			    && !vars->pollers->getCount()) {
140 				break;
141 			}
142 
143 			child = next;
144 		}while ((next = child->getParentEntry(gIOServicePlane))
145 		    && child->isParent(next, gIOServicePlane, true));
146 
147 		if (!vars->pollers->getCount()) {
148 			err = kIOReturnUnsupported;
149 			break;
150 		}
151 	}while (false);
152 
153 	media->setProperty(kIOPolledInterfaceStackKey, vars);
154 
155 	return vars;
156 }
157 
158 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
159 
160 static IOReturn
161 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
162 
163 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
164 
165 static IOReturn
166 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
167 {
168 	IOReturn            err = kIOReturnError;
169 	int32_t             idx;
170 	IOPolledInterface * poller;
171 
172 	for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
173 		poller = (IOPolledInterface *) vars->pollers->getObject(idx);
174 		err = poller->probe(vars->media);
175 		if (err) {
176 			HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
177 			break;
178 		}
179 	}
180 
181 	return err;
182 }
183 
184 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
185 
186 IOReturn
187 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
188 {
189 	IOPolledFilePollers      * vars = filevars->pollers;
190 	IOBufferMemoryDescriptor * ioBuffer;
191 	IOPolledInterface        * poller;
192 	IOService                * next;
193 	IOReturn                   err = kIOReturnError;
194 	int32_t                    idx;
195 
196 	vars->abortable = abortable;
197 	ioBuffer = NULL;
198 
199 	if (kIOPolledAfterSleepState == state) {
200 		vars->ioStatus = 0;
201 		vars->io = false;
202 	}
203 	(void) IOPolledFilePollersIODone(vars, false);
204 
205 	if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state)) {
206 		ioBuffer = vars->ioBuffer;
207 		if (!ioBuffer) {
208 			vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
209 			    kDefaultIONumBuffers * kDefaultIOSize, page_size);
210 			if (!ioBuffer) {
211 				return kIOReturnNoMemory;
212 			}
213 		}
214 	}
215 
216 	for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
217 		poller = (IOPolledInterface *) vars->pollers->getObject(idx);
218 		err = poller->open(state, ioBuffer);
219 		if (kIOReturnSuccess != err) {
220 			HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
221 			break;
222 		}
223 	}
224 	if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state)) {
225 		next = vars->media;
226 		while (next) {
227 			next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
228 			next = next->getProvider();
229 		}
230 	}
231 
232 	return err;
233 }
234 
235 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
236 
237 IOReturn
238 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
239 {
240 	IOPolledFilePollers * vars = filevars->pollers;
241 	IOPolledInterface * poller;
242 	IORegistryEntry *   next;
243 	IOReturn            err;
244 	int32_t             idx;
245 
246 	(void) IOPolledFilePollersIODone(vars, false);
247 
248 	if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
249 		vars->openCount--;
250 	}
251 
252 	for (idx = 0, err = kIOReturnSuccess;
253 	    (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
254 	    idx++) {
255 		err = poller->close(state);
256 		if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state)) {
257 			err = poller->close(kIOPolledBeforeSleepState);
258 		}
259 		if (err) {
260 			HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
261 		}
262 	}
263 
264 	if (kIOPolledPostflightState == state) {
265 		next = vars->media;
266 		while (next) {
267 			next->removeProperty(kIOPolledInterfaceActiveKey);
268 			next = next->getParentEntry(gIOServicePlane);
269 		}
270 	}
271 
272 	if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
273 		do{
274 			if (vars->openCount) {
275 				break;
276 			}
277 			if (vars->ioBuffer) {
278 				vars->ioBuffer->release();
279 				vars->ioBuffer = NULL;
280 			}
281 		}while (false);
282 	}
283 
284 	return err;
285 }
286 
287 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
288 
289 IOReturn
290 IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize)
291 {
292 	return kIOReturnUnsupported;
293 }
294 
295 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
296 
297 IOReturn
298 IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,
299     const uint8_t * key, size_t keySize)
300 {
301 	IOReturn              ret = kIOReturnUnsupported;
302 	IOReturn              err;
303 	int32_t               idx;
304 	IOPolledFilePollers * vars = filevars->pollers;
305 	IOPolledInterface   * poller;
306 
307 	for (idx = 0;
308 	    (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
309 	    idx++) {
310 		poller = (IOPolledInterface *) vars->pollers->getObject(idx);
311 		err = poller->setEncryptionKey(key, keySize);
312 		if (kIOReturnSuccess == err) {
313 			ret = err;
314 		}
315 	}
316 
317 	return ret;
318 }
319 
320 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
321 
322 IOMemoryDescriptor *
323 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
324 {
325 	return vars->pollers->ioBuffer;
326 }
327 
328 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
329 
330 static void
331 IOPolledIOComplete(void *   target,
332     void *   parameter,
333     IOReturn status,
334     UInt64   actualByteCount)
335 {
336 	IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
337 
338 	vars->ioStatus = status;
339 }
340 
341 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
342 
343 static IOReturn
344 IOStartPolledIO(IOPolledFilePollers * vars,
345     uint32_t operation, uint32_t bufferOffset,
346     uint64_t deviceOffset, uint64_t length)
347 {
348 	IOReturn            err;
349 	IOPolledInterface * poller;
350 	IOPolledCompletion  completion;
351 
352 	err = vars->ioStatus;
353 	if (kIOReturnSuccess != err) {
354 		return err;
355 	}
356 
357 	completion.target    = NULL;
358 	completion.action    = &IOPolledIOComplete;
359 	completion.parameter = vars;
360 
361 	vars->ioStatus = -1;
362 
363 	poller = (IOPolledInterface *) vars->pollers->getObject(0);
364 	err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
365 	if (err) {
366 		if (kernel_debugger_entry_count) {
367 			HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
368 		} else {
369 			HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
370 			    vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
371 		}
372 	}
373 	return err;
374 }
375 
376 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
377 
378 static IOReturn
379 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
380 {
381 	IOReturn            err = kIOReturnSuccess;
382 	int32_t             idx = 0;
383 	IOPolledInterface * poller;
384 	AbsoluteTime        deadline;
385 
386 	if (!vars->io) {
387 		return kIOReturnSuccess;
388 	}
389 
390 	abortable &= vars->abortable;
391 
392 	clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
393 
394 	while (-1 == vars->ioStatus) {
395 		for (idx = 0;
396 		    (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
397 		    idx++) {
398 			IOReturn newErr;
399 			newErr = poller->checkForWork();
400 			if ((newErr == kIOReturnAborted) && !abortable) {
401 				newErr = kIOReturnSuccess;
402 			}
403 			if (kIOReturnSuccess == err) {
404 				err = newErr;
405 			}
406 		}
407 		if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline))) {
408 			HIBLOG("IOPolledInterface::forced timeout\n");
409 			vars->ioStatus = kIOReturnTimeout;
410 		}
411 	}
412 	vars->io = false;
413 
414 #if HIBERNATION
415 	if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) {
416 		err = kIOReturnAborted;
417 		HIBLOG("IOPolledInterface::checkForWork sw abort\n");
418 	}
419 #endif
420 
421 	if (err) {
422 		HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
423 	} else {
424 		err = vars->ioStatus;
425 		if (kIOReturnSuccess != err) {
426 			HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
427 		}
428 	}
429 
430 	return err;
431 }
432 
433 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
434 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
435 
436 struct _OpenFileContext {
437 	OSData * extents;
438 	uint64_t size;
439 };
440 
441 static void
442 file_extent_callback(void * ref, uint64_t start, uint64_t length)
443 {
444 	_OpenFileContext * ctx = (_OpenFileContext *) ref;
445 	IOPolledFileExtent extent;
446 
447 	extent.start  = start;
448 	extent.length = length;
449 	ctx->extents->appendValue(extent);
450 	ctx->size += length;
451 }
452 
453 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
454 
455 static IOService *
456 IOCopyMediaForDev(dev_t device)
457 {
458 	OSDictionary * matching;
459 	OSNumber *     num;
460 	OSIterator *   iter;
461 	IOService *    result = NULL;
462 
463 	matching = IOService::serviceMatching("IOMedia");
464 	if (!matching) {
465 		return NULL;
466 	}
467 	do{
468 		num = OSNumber::withNumber(major(device), 32);
469 		if (!num) {
470 			break;
471 		}
472 		matching->setObject(kIOBSDMajorKey, num);
473 		num->release();
474 		num = OSNumber::withNumber(minor(device), 32);
475 		if (!num) {
476 			break;
477 		}
478 		matching->setObject(kIOBSDMinorKey, num);
479 		num->release();
480 		if (!num) {
481 			break;
482 		}
483 		iter = IOService::getMatchingServices(matching);
484 		if (iter) {
485 			result = (IOService *) iter->getNextObject();
486 			result->retain();
487 			iter->release();
488 		}
489 	}while (false);
490 	matching->release();
491 
492 	return result;
493 }
494 
495 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
496 
497 #if defined(__i386__) || defined(__x86_64__)
498 #define APFSMEDIA_GETHIBERKEY         "getHiberKey"
499 
500 static IOReturn
501 IOGetVolumeCryptKey(dev_t block_dev,
502     LIBKERN_RETURNS_RETAINED OSString ** pKeyUUID,
503     uint8_t * volumeCryptKey,
504     size_t * keySize)
505 {
506 	IOReturn         err;
507 	IOService *      part;
508 	OSString *       keyUUID = NULL;
509 	OSString *       keyStoreUUID = NULL;
510 	uuid_t           volumeKeyUUID;
511 	aks_volume_key_t vek;
512 	size_t           callerKeySize;
513 
514 	static IOService * sKeyStore;
515 
516 	part = IOCopyMediaForDev(block_dev);
517 	if (!part) {
518 		return kIOReturnNotFound;
519 	}
520 
521 	callerKeySize = *keySize;
522 	// Try APFS first
523 	{
524 		uuid_t volUuid = {0};
525 		err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
526 		if (kIOReturnBadArgument == err) {
527 			// apfs fails on buffer size >32
528 			*keySize = 32;
529 			err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
530 		}
531 		if (err != kIOReturnSuccess) {
532 			*keySize = 0;
533 		} else {
534 			// No need to create uuid string if it's not requested
535 			if (pKeyUUID) {
536 				uuid_string_t volUuidStr;
537 				uuid_unparse(volUuid, volUuidStr);
538 				*pKeyUUID = OSString::withCString(volUuidStr);
539 			}
540 
541 			part->release();
542 			return kIOReturnSuccess;
543 		}
544 	}
545 
546 	// Then old CS path
547 	err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
548 	    (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
549 	if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) {
550 //        IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
551 
552 		if (!sKeyStore) {
553 			sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
554 		}
555 		if (sKeyStore) {
556 			err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
557 		} else {
558 			err = kIOReturnNoResources;
559 		}
560 		if (kIOReturnSuccess == err) {
561 			err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
562 		}
563 		if (kIOReturnSuccess != err) {
564 			IOLog("volume key err 0x%x\n", err);
565 		} else {
566 			if (vek.key.keybytecount <= callerKeySize) {
567 				*keySize = vek.key.keybytecount;
568 			}
569 			bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize);
570 		}
571 		bzero(&vek, sizeof(vek));
572 
573 		if (pKeyUUID) {
574 			// Create a copy because the caller would release it
575 			*pKeyUUID = OSString::withString(keyUUID);
576 		}
577 	}
578 
579 	part->release();
580 	return err;
581 }
582 #endif /* defined(__i386__) || defined(__x86_64__) */
583 
584 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
585 
586 #if defined(__arm64__)
587 static IOReturn
588 IOGetHibernationCryptKey(uint8_t * hibernationKey,
589     size_t * keySize,
590     uint32_t *swSeed
591     )
592 {
593 	return kIOReturnNotFound;
594 }
595 #endif /* defined(__arm64__) */
596 
597 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
598 
599 IOReturn
600 IOPolledFileOpen(const char * filename,
601     uint32_t flags,
602     uint64_t setFileSize, uint64_t fsFreeSize,
603     void * write_file_addr, size_t write_file_len,
604     IOPolledFileIOVars ** fileVars,
605     OSData ** imagePath,
606     uint8_t * volumeCryptKey, size_t * keySize)
607 {
608 	IOReturn             err = kIOReturnSuccess;
609 	IOPolledFileIOVars * vars;
610 	_OpenFileContext     ctx;
611 	OSData *             extentsData = NULL;
612 	OSNumber *           num;
613 	IOService *          part = NULL;
614 	dev_t                block_dev;
615 	dev_t                image_dev;
616 	AbsoluteTime         startTime, endTime;
617 	uint64_t             nsec;
618 
619 	vars = IOMallocType(IOPolledFileIOVars);
620 	vars->allocated = true;
621 
622 	do{
623 		extentsData = OSData::withCapacity(32);
624 		ctx.extents = extentsData;
625 		ctx.size    = 0;
626 		clock_get_uptime(&startTime);
627 
628 		vars->fileRef = kern_open_file_for_direct_io(filename,
629 		    flags,
630 		    &file_extent_callback, &ctx,
631 		    setFileSize,
632 		    fsFreeSize,
633 		    // write file:
634 		    0, write_file_addr, write_file_len,
635 		    // results
636 		    &block_dev,
637 		    &image_dev,
638 		    &vars->block0,
639 		    &vars->maxiobytes,
640 		    &vars->flags);
641 #if 0
642 		uint32_t msDelay = (131071 & random());
643 		HIBLOG("sleep %d\n", msDelay);
644 		IOSleep(msDelay);
645 #endif
646 		clock_get_uptime(&endTime);
647 		SUB_ABSOLUTETIME(&endTime, &startTime);
648 		absolutetime_to_nanoseconds(endTime, &nsec);
649 
650 		if (!vars->fileRef) {
651 			err = kIOReturnNoSpace;
652 		}
653 
654 		HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
655 		if (kIOReturnSuccess != err) {
656 			break;
657 		}
658 
659 		HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
660 		    (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
661 		    vars->maxiobytes, kIOPolledFileSSD & vars->flags);
662 		assert(!vars->block0);
663 		if (extentsData->getLength() < sizeof(IOPolledFileExtent)) {
664 			err = kIOReturnNoSpace;
665 			break;
666 		}
667 
668 		vars->fileSize = ctx.size;
669 		vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
670 
671 		part = IOCopyMediaForDev(image_dev);
672 		if (!part) {
673 			err = kIOReturnNotFound;
674 			break;
675 		}
676 
677 		if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) {
678 			break;
679 		}
680 
681 		if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) {
682 			vars->blockSize = num->unsigned32BitValue();
683 		}
684 		if (vars->blockSize < 4096) {
685 			vars->blockSize = 4096;
686 		}
687 
688 		HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
689 		    major(image_dev), minor(image_dev), (long)vars->blockSize,
690 		    vars->pollers->pollers->getCount());
691 
692 		OSString * keyUUID = NULL;
693 #if defined(__i386__) || defined(__x86_64__)
694 		if (volumeCryptKey) {
695 			err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
696 		}
697 #elif defined(__arm64__)
698 		uint32_t swSeed = 0;
699 		if (volumeCryptKey) {
700 			if (flags & kIOPolledFileHibernate) {
701 				err = IOGetHibernationCryptKey(volumeCryptKey, keySize, &swSeed);
702 				if (kIOReturnSuccess != err) {
703 					HIBLOG("error 0x%x from IOGetHibernationCryptKey\n", err);
704 					break;
705 				}
706 			} else {
707 				*keySize = 0;
708 			}
709 		}
710 #else
711 		if (volumeCryptKey) {
712 			HIBLOG("IOPolledFileOpen: unable to get volumeCryptKey\n");
713 			err = kIOReturnNotFound;
714 			break;
715 		}
716 #endif
717 
718 		*fileVars    = vars;
719 		vars->fileExtents = extentsData;
720 
721 		// make imagePath
722 		OSData * data = NULL;
723 		if (imagePath) {
724 #if defined(__i386__) || defined(__x86_64__)
725 			char str2[24 + sizeof(uuid_string_t) + 2];
726 
727 			if (keyUUID) {
728 				snprintf(str2, sizeof(str2), "%qx:%s",
729 				    vars->extentMap[0].start, keyUUID->getCStringNoCopy());
730 			} else {
731 				snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
732 			}
733 
734 			err = IOService::getPlatform()->callPlatformFunction(
735 				gIOCreateEFIDevicePathSymbol, false,
736 				(void *) part, (void *) str2,
737 				(void *) (uintptr_t) true, (void *) &data);
738 #elif defined(__arm64__)
739 			char str2[26];
740 			snprintf(str2, sizeof(str2), "%qx:%x", vars->extentMap[0].start, swSeed);
741 			data = OSData::withBytes(str2, (unsigned int) strlen(str2));
742 			err = kIOReturnSuccess;
743 #else
744 			err = kIOReturnNotFound;
745 #endif
746 			if (kIOReturnSuccess != err) {
747 				HIBLOG("error 0x%x getting path\n", err);
748 				OSSafeReleaseNULL(keyUUID);
749 				break;
750 			}
751 			*imagePath = data;
752 		}
753 
754 		// Release key UUID if we have one
755 		OSSafeReleaseNULL(keyUUID);
756 	}while (false);
757 
758 	if (kIOReturnSuccess != err) {
759 		HIBLOG("error 0x%x opening polled file\n", err);
760 		IOPolledFileClose(&vars, 0, NULL, 0, 0, 0, false);
761 		if (extentsData) {
762 			extentsData->release();
763 		}
764 	}
765 
766 	if (part) {
767 		part->release();
768 	}
769 
770 	return err;
771 }
772 
773 IOReturn
774 IOPolledFileOpen(const char * filename,
775     uint32_t flags,
776     uint64_t setFileSize, uint64_t fsFreeSize,
777     void * write_file_addr, size_t write_file_len,
778     IOPolledFileIOVars ** fileVars,
779     OSSharedPtr<OSData>& imagePath,
780     uint8_t * volumeCryptKey, size_t * keySize)
781 {
782 	OSData* imagePathRaw = NULL;
783 	IOReturn result = IOPolledFileOpen(filename, flags, setFileSize, fsFreeSize, write_file_addr, write_file_len,
784 	    fileVars, &imagePathRaw, volumeCryptKey, keySize);
785 	imagePath.reset(imagePathRaw, OSNoRetain);
786 	return result;
787 }
788 
789 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
790 
791 IOReturn
792 IOPolledFileClose(IOPolledFileIOVars ** pVars,
793     off_t write_offset, void * addr, size_t write_length,
794     off_t discard_offset, off_t discard_end, bool unlink)
795 {
796 	IOPolledFileIOVars * vars;
797 
798 	vars = *pVars;
799 	if (!vars) {
800 		return kIOReturnSuccess;
801 	}
802 
803 	if (vars->fileRef) {
804 		kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
805 		    discard_offset, discard_end, unlink);
806 		vars->fileRef = NULL;
807 	}
808 	if (vars->fileExtents) {
809 		vars->fileExtents->release();
810 		vars->fileExtents = NULL;
811 	}
812 	if (vars->pollers) {
813 		vars->pollers->release();
814 		vars->pollers = NULL;
815 	}
816 
817 	if (vars->allocated) {
818 		IOFreeType(vars, IOPolledFileIOVars);
819 	} else {
820 		bzero(vars, sizeof(IOPolledFileIOVars));
821 	}
822 	*pVars = NULL;
823 
824 	return kIOReturnSuccess;
825 }
826 
827 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
828 
829 IOReturn
830 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
831     uint32_t openState)
832 {
833 	IOReturn err;
834 
835 	err = kIOReturnSuccess;
836 	do{
837 		if (!vars->pollers->openCount) {
838 			err = IOPolledFilePollersProbe(vars->pollers);
839 			if (kIOReturnSuccess != err) {
840 				break;
841 			}
842 		}
843 		err = IOPolledFilePollersOpen(vars, openState, false);
844 		if (kIOReturnSuccess != err) {
845 			break;
846 		}
847 		if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState)) {
848 			vars->pollers->openCount++;
849 		}
850 		vars->pollers->io  = false;
851 		vars->buffer       = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
852 		vars->bufferHalf   = 0;
853 		vars->bufferOffset = 0;
854 		assert(vars->pollers->ioBuffer->getLength() <= UINT_MAX);
855 		vars->bufferSize   = (typeof(vars->bufferSize))(vars->pollers->ioBuffer->getLength() >> 1);
856 
857 		if (vars->maxiobytes < vars->bufferSize) {
858 			vars->bufferSize = (typeof(vars->bufferSize))vars->maxiobytes;
859 		}
860 	}while (false);
861 
862 	if (kIOReturnSuccess != err) {
863 		HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
864 	}
865 
866 	return err;
867 }
868 
869 
870 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
871 
872 IOReturn
873 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
874 {
875 	IOPolledFileExtent * extentMap;
876 
877 	extentMap = vars->extentMap;
878 
879 	vars->position = position;
880 
881 	if (position > vars->fileSize) {
882 		HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize);
883 		return kIOReturnNoSpace;
884 	}
885 
886 	while (position >= extentMap->length) {
887 		position -= extentMap->length;
888 		extentMap++;
889 	}
890 
891 	vars->currentExtent   = extentMap;
892 	vars->extentRemaining = extentMap->length - position;
893 	vars->extentPosition  = vars->position - position;
894 
895 	if (vars->bufferSize <= vars->extentRemaining) {
896 		vars->bufferLimit = vars->bufferSize;
897 	} else {
898 		vars->bufferLimit = ((uint32_t) vars->extentRemaining);
899 	}
900 
901 	return kIOReturnSuccess;
902 }
903 
904 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
905 
906 IOReturn
907 IOPolledFileWrite(IOPolledFileIOVars * vars,
908     const uint8_t * bytes, IOByteCount size,
909     IOPolledFileCryptVars * cryptvars)
910 {
911 	IOReturn    err = kIOReturnSuccess;
912 	IOByteCount copy, original_size = size;
913 	bool        flush = false;
914 
915 	do{
916 		if (!bytes && !size) {
917 			// seek to end of block & flush
918 			size = vars->position & (vars->blockSize - 1);
919 			if (size) {
920 				size = vars->blockSize - size;
921 			}
922 			flush = true;
923 		}
924 
925 		copy = vars->bufferLimit - vars->bufferOffset;
926 		if (copy > size) {
927 			copy = size;
928 		} else {
929 			flush = true;
930 		}
931 
932 		if (bytes) {
933 #if KASAN
934 			/* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
935 			 * avoid triggering global redzone sanitizer violations when accessing
936 			 * interstices between 'C' structures
937 			 */
938 			__nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
939 #else
940 			bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
941 #endif
942 			bytes += copy;
943 		} else {
944 			bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
945 		}
946 
947 		size -= copy;
948 		vars->bufferOffset += copy;
949 		vars->position += copy;
950 
951 		if (flush && vars->bufferOffset) {
952 			uint64_t offset = (vars->position - vars->bufferOffset
953 			    - vars->extentPosition + vars->currentExtent->start);
954 			uint32_t length = (vars->bufferOffset);
955 
956 #if CRYPTO
957 			if (cryptvars && vars->encryptStart
958 			    && (vars->position > vars->encryptStart)
959 			    && ((vars->position - length) < vars->encryptEnd)) {
960 				AbsoluteTime startTime, endTime;
961 
962 				uint64_t encryptLen, encryptStart;
963 				encryptLen = vars->position - vars->encryptStart;
964 				if (encryptLen > length) {
965 					encryptLen = length;
966 				}
967 				encryptStart = length - encryptLen;
968 				if (vars->position > vars->encryptEnd) {
969 					encryptLen -= (vars->position - vars->encryptEnd);
970 				}
971 
972 				clock_get_uptime(&startTime);
973 
974 				assert(encryptLen <= UINT_MAX);
975 				// encrypt the buffer
976 				aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
977 				    &cryptvars->aes_iv[0],
978 				    (unsigned int) (encryptLen / AES_BLOCK_SIZE),
979 				    vars->buffer + vars->bufferHalf + encryptStart,
980 				    &cryptvars->ctx.encrypt);
981 
982 				clock_get_uptime(&endTime);
983 				ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
984 				SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
985 				vars->cryptBytes += encryptLen;
986 
987 				// save initial vector for following encrypts
988 				bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
989 				    &cryptvars->aes_iv[0],
990 				    AES_BLOCK_SIZE);
991 			}
992 #endif /* CRYPTO */
993 
994 			err = IOPolledFilePollersIODone(vars->pollers, true);
995 			if (kIOReturnSuccess != err) {
996 				break;
997 			}
998 
999 			if (vars->position & (vars->blockSize - 1)) {
1000 				HIBLOG("misaligned file pos %qx\n", vars->position);
1001 			}
1002 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1003 
1004 			err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
1005 			if (kIOReturnSuccess != err) {
1006 				HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
1007 				    vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
1008 				break;
1009 			}
1010 			vars->pollers->io = true;
1011 
1012 			vars->extentRemaining -= vars->bufferOffset;
1013 			if (!vars->extentRemaining) {
1014 				vars->currentExtent++;
1015 				vars->extentRemaining = vars->currentExtent->length;
1016 				vars->extentPosition  = vars->position;
1017 			}
1018 
1019 			vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1020 			vars->bufferOffset = 0;
1021 			if (vars->bufferSize <= vars->extentRemaining) {
1022 				vars->bufferLimit = vars->bufferSize;
1023 			} else {
1024 				vars->bufferLimit =  ((uint32_t) vars->extentRemaining);
1025 			}
1026 
1027 			if (!vars->extentRemaining) {
1028 				err = kIOReturnOverrun;
1029 				break;
1030 			}
1031 
1032 			flush = false;
1033 		}
1034 	}while (size);
1035 
1036 	return err;
1037 }
1038 
1039 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1040 
1041 IOReturn
1042 IOPolledFileFlush(IOPolledFileIOVars * vars)
1043 {
1044 	// Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
1045 	IOReturn err = kIOReturnSuccess;
1046 
1047 	err = IOPolledFilePollersIODone(vars->pollers, true);
1048 	if (kIOReturnSuccess != err) {
1049 		return err;
1050 	}
1051 
1052 	err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0);
1053 	if (kIOReturnSuccess != err) {
1054 		HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
1055 		    vars, vars->pollers, err);
1056 		return err;
1057 	}
1058 	vars->pollers->io = true;
1059 
1060 	return err;
1061 }
1062 
1063 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1064 
1065 IOReturn
1066 IOPolledFileRead(IOPolledFileIOVars * vars,
1067     uint8_t * bytes, IOByteCount size,
1068     IOPolledFileCryptVars * cryptvars)
1069 {
1070 	IOReturn    err = kIOReturnSuccess;
1071 	IOByteCount copy;
1072 
1073 //    bytesWritten += size;
1074 
1075 	do{
1076 		copy = vars->bufferLimit - vars->bufferOffset;
1077 		if (copy > size) {
1078 			copy = size;
1079 		}
1080 
1081 		if (bytes) {
1082 #if KASAN
1083 			__nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1084 #else
1085 			bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1086 #endif
1087 			bytes += copy;
1088 		}
1089 		size -= copy;
1090 		vars->bufferOffset += copy;
1091 //	vars->position += copy;
1092 
1093 		if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) {
1094 			if (!vars->pollers->io) {
1095 				cryptvars = NULL;
1096 			}
1097 			err = IOPolledFilePollersIODone(vars->pollers, true);
1098 			if (kIOReturnSuccess != err) {
1099 				break;
1100 			}
1101 
1102 			if (vars->position & (vars->blockSize - 1)) {
1103 				HIBLOG("misaligned file pos %qx\n", vars->position);
1104 			}
1105 
1106 			vars->position        += vars->lastRead;
1107 			vars->extentRemaining -= vars->lastRead;
1108 			vars->bufferLimit      = vars->lastRead;
1109 
1110 			if (!vars->extentRemaining) {
1111 				vars->currentExtent++;
1112 				vars->extentRemaining = vars->currentExtent->length;
1113 				vars->extentPosition  = vars->position;
1114 				if (!vars->extentRemaining) {
1115 					err = kIOReturnOverrun;
1116 					break;
1117 				}
1118 			}
1119 
1120 			uint32_t length;
1121 			uint32_t lastReadLength = vars->lastRead;
1122 			uint64_t offset = (vars->position
1123 			    - vars->extentPosition + vars->currentExtent->start);
1124 			if (vars->extentRemaining <= vars->bufferSize) {
1125 				length =  ((uint32_t) vars->extentRemaining);
1126 			} else {
1127 				length = vars->bufferSize;
1128 			}
1129 			if ((length + vars->position) > vars->readEnd) {
1130 				length = ((uint32_t) (vars->readEnd - vars->position));
1131 			}
1132 
1133 			vars->lastRead = length;
1134 			if (length) {
1135 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1136 				err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
1137 				if (kIOReturnSuccess != err) {
1138 					break;
1139 				}
1140 				vars->pollers->io = true;
1141 			}
1142 
1143 			vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1144 			vars->bufferOffset = 0;
1145 
1146 #if CRYPTO
1147 			if (cryptvars) {
1148 				uint8_t thisVector[AES_BLOCK_SIZE];
1149 				AbsoluteTime startTime, endTime;
1150 
1151 				// save initial vector for following decrypts
1152 				bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1153 				bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1154 				    &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1155 
1156 				// decrypt the buffer
1157 				clock_get_uptime(&startTime);
1158 
1159 				assert(lastReadLength <= UINT_MAX);
1160 				aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1161 				    &thisVector[0],
1162 				    (unsigned int) (lastReadLength / AES_BLOCK_SIZE),
1163 				    vars->buffer + vars->bufferHalf,
1164 				    &cryptvars->ctx.decrypt);
1165 
1166 				clock_get_uptime(&endTime);
1167 				ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1168 				SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1169 				vars->cryptBytes += lastReadLength;
1170 			}
1171 #endif /* CRYPTO */
1172 		}
1173 	}while (size);
1174 
1175 	return err;
1176 }
1177 
1178 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1179