1 /*
2 * Copyright (c) 1998-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <IOKit/IOService.h>
30 #include <IOKit/IOInterruptEventSource.h>
31 #include <IOKit/IOTimerEventSource.h>
32 #include <IOKit/IOMapper.h>
33 #include "../Kernel/IOServicePrivate.h"
34
35 #include <Exclaves/Exclaves.h>
36
37 #if CONFIG_EXCLAVES
38 #include <mach/exclaves.h>
39 #include <Exclaves/IOService.tightbeam.h>
40
41 #define EXLOG(x...) do { \
42 if (kIOLogExclaves & gIOKitDebug) \
43 IOLog(x); \
44 } while (false)
45
46 /* Global IOExclaveProxyState lookup table */
47
48 OSDictionary *gExclaveProxyStates;
49 IORecursiveLock *gExclaveProxyStateLock;
50 const OSSymbol *gDARTMapperFunctionSetActive;
51
52 /* IOExclaveProxyState */
53
54 class IOExclaveWorkLoopAperture {
55 public:
56 IOWorkLoop *workLoop;
57 void
closeGate()58 closeGate()
59 {
60 workLoop->closeGate();
61 }
62 void
openGate()63 openGate()
64 {
65 workLoop->openGate();
66 }
67 };
68
69 #endif /* CONFIG_EXCLAVES */
70
71 struct IOService::IOExclaveProxyState {
72 IOService *service;
73 uint64_t mach_endpoint;
74 #if CONFIG_EXCLAVES
75 tb_endpoint_t tb_endpoint;
76 ioservice_ioserviceconcrete client;
77 // ExclaveDriverKit related state
78 uint64_t edk_mach_endpoint;
79 tb_endpoint_t edk_tb_endpoint;
80 ioservice_ioserviceprivate edk_client;
81 OSDictionary *exclave_interrupts;
82 IOLock *exclave_interrupts_lock;
83 OSDictionary *exclave_timers;
84 IOLock *exclave_timers_lock;
85 uint32_t nextExclaveTimerId;
86
87 // TODO: implement properly once ExclaveAperture removed
88 IOExclaveWorkLoopAperture *ewla;
89
90 IOLock * exclaveAsyncNotificationEventSourcesLock;
91 OSArray *exclaveAsyncNotificationEventSources;
92
93 // ANE specific upcalls
94 ANEUpcallSetPowerStateHandler aneSetPowerStateUpcallHandler;
95 ANEUpcallWorkHandler aneWorkSubmitUpcallHandler;
96 ANEUpcallWorkHandler aneWorkBeginUpcallHandler;
97 ANEUpcallWorkHandler aneWorkEndUpcallHandler;
98 #endif /* CONFIG_EXCLAVES */
99 };
100
101 #if CONFIG_EXCLAVES
102 class IOExclaveProxyStateWrapper : public OSObject {
103 OSDeclareFinalStructors(IOExclaveProxyStateWrapper);
104 public:
105 IOService::IOExclaveProxyState *proxyState;
106 };
107 OSDefineMetaClassAndFinalStructors(IOExclaveProxyStateWrapper, OSObject);
108
109 extern "C" kern_return_t
110 exclaves_driver_service_lookup(const char *service_name, uint64_t *endpoint);
111 #endif /* CONFIG_EXCLAVES */
112
113 bool
exclaveStart(IOService * provider,IOExclaveProxyState ** pRef)114 IOService::exclaveStart(IOService * provider, IOExclaveProxyState ** pRef)
115 {
116 IOExclaveProxyState * ref;
117
118 ref = NULL;
119 #if CONFIG_EXCLAVES
120 int err = 1;
121 do {
122 char key[16];
123 uint64_t serviceID;
124 uint64_t mach_endpoint = 0;
125 uint64_t edk_mach_endpoint = 0;
126 tb_error_t tberr;
127 tb_endpoint_t tb_endpoint;
128 tb_endpoint_t edk_tb_endpoint;
129 ioservice_ioserviceconcrete client;
130 ioservice_ioserviceprivate edk_client;
131 OSObject * prop;
132 OSData * data;
133 IOWorkLoop * wl;
134 IOExclaveProxyStateWrapper * wrapper;
135 bool result;
136
137 // exit early if Exclaves are not available
138 if (exclaves_get_status() == EXCLAVES_STATUS_NOT_SUPPORTED) {
139 break;
140 }
141
142 prop = provider->copyProperty("exclave-endpoint");
143 if ((data = OSDynamicCast(OSData, prop))) {
144 mach_endpoint = ((uint32_t *)data->getBytesNoCopy())[0];
145 }
146 OSSafeReleaseNULL(prop);
147
148 prop = provider->copyProperty("exclave-service");
149 if ((data = OSDynamicCast(OSData, prop))) {
150 const char *exclave_service = (const char *) data->getBytesNoCopy();
151 if (exclave_service[data->getLength() - 1] != '\0') {
152 IOLog("%s: %s-0x%qx exclave-service property is invalid\n", __func__, provider->getName(), provider->getRegistryEntryID());
153 OSSafeReleaseNULL(prop);
154 break;
155 }
156 if (exclaves_driver_service_lookup(exclave_service, &mach_endpoint) != KERN_SUCCESS) {
157 IOLog("%s: %s-0x%qx could not find exclave-service %s\n", __func__, provider->getName(), provider->getRegistryEntryID(), exclave_service);
158 OSSafeReleaseNULL(prop);
159 break;
160 }
161 }
162 OSSafeReleaseNULL(prop);
163
164 prop = provider->copyProperty("exclave-edk-endpoint");
165 if ((data = OSDynamicCast(OSData, prop))) {
166 edk_mach_endpoint = ((uint32_t *)data->getBytesNoCopy())[0];
167 }
168 OSSafeReleaseNULL(prop);
169
170 prop = provider->copyProperty("exclave-edk-service");
171 if ((data = OSDynamicCast(OSData, prop))) {
172 const char *exclave_edk_service = (const char *) data->getBytesNoCopy();
173 if (exclave_edk_service[data->getLength() - 1] != '\0') {
174 IOLog("%s: %s-0x%qx exclave-edk-service property is invalid\n", __func__, provider->getName(), provider->getRegistryEntryID());
175 OSSafeReleaseNULL(prop);
176 break;
177 }
178 if (exclaves_driver_service_lookup(exclave_edk_service, &edk_mach_endpoint) != KERN_SUCCESS) {
179 IOLog("%s: %s-0x%qx could not find exclave-edk-service %s\n", __func__, provider->getName(), provider->getRegistryEntryID(), exclave_edk_service);
180 OSSafeReleaseNULL(prop);
181 break;
182 }
183 }
184 OSSafeReleaseNULL(prop);
185
186 // Initialize IOServiceConcrete endpoint
187 tb_endpoint = tb_endpoint_create_with_value(TB_TRANSPORT_TYPE_XNU, mach_endpoint, TB_ENDPOINT_OPTIONS_NONE);
188 assert(NULL != tb_endpoint);
189 if (NULL == tb_endpoint) {
190 break;
191 }
192 tberr = ioservice_ioserviceconcrete_init(&client, tb_endpoint);
193 assert(TB_ERROR_SUCCESS == tberr);
194 if (TB_ERROR_SUCCESS != tberr) {
195 break;
196 }
197
198 // Initialize IOServicePrivate endpoint
199 edk_tb_endpoint = tb_endpoint_create_with_value(TB_TRANSPORT_TYPE_XNU, edk_mach_endpoint, TB_ENDPOINT_OPTIONS_NONE);
200 assert(NULL != edk_tb_endpoint);
201 if (NULL == edk_tb_endpoint) {
202 printf("%s: ERROR: Failed to create endpoint\n", __func__);
203 break;
204 }
205 tberr = ioservice_ioserviceprivate_init(&edk_client, edk_tb_endpoint);
206 assert(TB_ERROR_SUCCESS == tberr);
207 if (TB_ERROR_SUCCESS != tberr) {
208 printf("%s: ERROR: Failed to init IOServicePrivate\n", __func__);
209 break;
210 }
211
212 ref = IONewZero(IOExclaveProxyState, 1);
213 if (!ref) {
214 break;
215 }
216 ref->service = this;
217 ref->mach_endpoint = mach_endpoint;
218 ref->tb_endpoint = tb_endpoint;
219 ref->client = client;
220 ref->edk_mach_endpoint = edk_mach_endpoint;
221 ref->edk_tb_endpoint = edk_tb_endpoint;
222 ref->edk_client = edk_client;
223 ref->exclave_interrupts = OSDictionary::withCapacity(1);
224 ref->exclave_interrupts_lock = IOLockAlloc();
225 ref->exclave_timers = OSDictionary::withCapacity(1);
226 ref->exclave_timers_lock = IOLockAlloc();
227 ref->exclaveAsyncNotificationEventSourcesLock = IOLockAlloc();
228
229 // TODO: remove once workloop aperture workaround removed
230 wl = getWorkLoop();
231 if (!wl) {
232 printf("%s ERROR: getWorkLoop failed\n", __func__);
233 break;
234 }
235 ref->ewla = IONew(IOExclaveWorkLoopAperture, 1);
236 if (!ref->ewla) {
237 printf("%s ERROR: exclaveWorkLoopAperture init failed\n", __func__);
238 break;
239 }
240 ref->ewla->workLoop = wl;
241
242 // Add proxy state to global lookup table
243 serviceID = getRegistryEntryID();
244 snprintf(key, sizeof(key), "%llu", serviceID);
245 wrapper = OSTypeAlloc(IOExclaveProxyStateWrapper);
246 wrapper->proxyState = ref;
247 IORecursiveLockLock(gExclaveProxyStateLock);
248 gExclaveProxyStates->setObject(key, wrapper);
249 IORecursiveLockUnlock(gExclaveProxyStateLock);
250
251 // Start() called after lookup table registration in case upcalls are made during exclave start().
252 // Use registry ID as exclave's upcall identifer
253 tberr = ioservice_ioserviceprivate_startprivate(&edk_client, serviceID, &result);
254 if (TB_ERROR_SUCCESS != tberr || !result) {
255 printf("%s ERROR: Failed StartPrivate\n", __func__);
256 // Deregister from lookup table if start() fails
257 IORecursiveLockLock(gExclaveProxyStateLock);
258 gExclaveProxyStates->removeObject(key);
259 IORecursiveLockUnlock(gExclaveProxyStateLock);
260 wrapper->release();
261 break;
262 }
263
264 err = 0;
265 } while (false);
266
267 if (err) {
268 if (ref) {
269 OSSafeReleaseNULL(ref->exclave_interrupts);
270 if (ref->exclave_interrupts_lock) {
271 IOLockFree(ref->exclave_interrupts_lock);
272 ref->exclave_interrupts_lock = NULL;
273 }
274 OSSafeReleaseNULL(ref->exclave_timers);
275 if (ref->exclave_timers_lock) {
276 IOLockFree(ref->exclave_timers_lock);
277 ref->exclave_timers_lock = NULL;
278 }
279 if (ref->exclaveAsyncNotificationEventSourcesLock) {
280 IOLockFree(ref->exclaveAsyncNotificationEventSourcesLock);
281 ref->exclaveAsyncNotificationEventSourcesLock = NULL;
282 }
283 if (ref->ewla) {
284 IODelete(ref->ewla, IOExclaveWorkLoopAperture, 1);
285 ref->ewla = NULL;
286 }
287 IODelete(ref, IOExclaveProxyState, 1);
288 ref = NULL;
289 }
290 }
291 #endif /* CONFIG_EXCLAVES */
292
293 if (!ref) {
294 return false;
295 }
296
297 *pRef = ref;
298 return true;
299 }
300
301 uint64_t
exclaveEndpoint(IOExclaveProxyState * pRef)302 IOService::exclaveEndpoint(IOExclaveProxyState * pRef)
303 {
304 return pRef->mach_endpoint;
305 }
306
307 bool
start(IOService * provider)308 IOExclaveProxy::start(IOService * provider)
309 {
310 bool ok;
311
312 ok = exclaveStart(provider, &exclaveState);
313
314 return ok;
315 }
316
317 /* Exclave upcall handlers */
318
319 #if CONFIG_EXCLAVES
320
321 static IOService::IOExclaveProxyState *
getProxyStateFromRegistryID(uint64_t id)322 getProxyStateFromRegistryID(uint64_t id)
323 {
324 OSObject *obj = NULL;
325 IOExclaveProxyStateWrapper *wrapper = NULL;
326 char key[15];
327
328 snprintf(key, sizeof(key), "%llu", id);
329 IORecursiveLockLock(gExclaveProxyStateLock);
330 obj = gExclaveProxyStates->getObject(key);
331 IORecursiveLockUnlock(gExclaveProxyStateLock);
332 if (!obj) {
333 printf("%s ERROR: failed to find proxy state\n", __func__);
334 return NULL;
335 }
336
337 wrapper = OSDynamicCast(IOExclaveProxyStateWrapper, obj);
338 if (!wrapper) {
339 printf("%s ERROR: failed to cast IOExclaveProxyStateWrapper\n", __func__);
340 return NULL;
341 }
342
343 if (!wrapper->proxyState) {
344 printf("%s ERROR: IOExclaveProxyStateWrapper contains NULL proxy state\n", __func__);
345 return NULL;
346 }
347
348 return wrapper->proxyState;
349 }
350
351 bool
IOExclaveInterruptUpcallHandler(uint64_t id,IOExclaveInterruptUpcallArgs * args)352 IOExclaveInterruptUpcallHandler(uint64_t id, IOExclaveInterruptUpcallArgs *args)
353 {
354 assert(args);
355 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
356 if (!ref || !args) {
357 return false;
358 }
359 ref->service->retain();
360
361 bool res;
362 switch (args->type) {
363 case kIOExclaveInterruptUpcallTypeRegister:
364 // Register interrupt
365 res = ref->service->exclaveRegisterInterrupt(ref, args->index, args->data.register_args.test_irq);
366 break;
367 case kIOExclaveInterruptUpcallTypeRemove:
368 // Remove interrupt
369 res = ref->service->exclaveRemoveInterrupt(ref, args->index);
370 break;
371 case kIOExclaveInterruptUpcallTypeEnable:
372 // Enable/disable interrupt
373 res = ref->service->exclaveEnableInterrupt(ref, args->index, args->data.enable_args.enable);
374 break;
375 default:
376 res = false;
377 printf("%s ERROR: invalid upcall type\n", __func__);
378 }
379
380 if (!res) {
381 printf("%s ERROR: upcall handler type %d failed\n", __func__, args->type);
382 ref->service->release();
383 return false;
384 }
385
386 ref->service->release();
387 return true;
388 }
389
390 bool
IOExclaveTimerUpcallHandler(uint64_t id,IOExclaveTimerUpcallArgs * args)391 IOExclaveTimerUpcallHandler(uint64_t id, IOExclaveTimerUpcallArgs *args)
392 {
393 assert(args);
394 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
395 if (!ref || !args) {
396 return false;
397 }
398 ref->service->retain();
399
400 bool res;
401 uint32_t timer_id = args->timer_id;
402 switch (args->type) {
403 case kIOExclaveTimerUpcallTypeRegister:
404 // Register timer
405 res = ref->service->exclaveRegisterTimer(ref, &args->timer_id);
406 break;
407 case kIOExclaveTimerUpcallTypeRemove:
408 // Remove timer
409 res = ref->service->exclaveRemoveTimer(ref, timer_id);
410 break;
411 case kIOExclaveTimerUpcallTypeEnable:
412 {
413 // Enable/disable timer
414 bool enable = args->data.enable_args.enable;
415 res = ref->service->exclaveEnableTimer(ref, timer_id, enable);
416 break;
417 }
418 case kIOExclaveTimerUpcallTypeSetTimeout:
419 {
420 // Set timeout
421 uint32_t options = args->data.set_timeout_args.clock_continuous ? kIOTimeOptionsContinuous : 0;
422 AbsoluteTime duration = args->data.set_timeout_args.duration;
423 kern_return_t *kr = &args->data.set_timeout_args.kr;
424 res = ref->service->exclaveTimerSetTimeout(ref, timer_id, options, duration, 0, kr);
425 break;
426 }
427 case kIOExclaveTimerUpcallTypeCancelTimeout:
428 // Cancel timeout
429 res = ref->service->exclaveTimerCancelTimeout(ref, timer_id);
430 break;
431 default:
432 res = false;
433 printf("%s ERROR: invalid upcall type\n", __func__);
434 }
435
436 if (!res) {
437 printf("%s ERROR: upcall handler type %d failed\n", __func__, args->type);
438 ref->service->release();
439 return false;
440 }
441
442 ref->service->release();
443 return true;
444 }
445
446 bool
IOExclaveLockWorkloop(uint64_t id,bool lock)447 IOExclaveLockWorkloop(uint64_t id, bool lock)
448 {
449 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
450 if (!ref) {
451 return false;
452 }
453
454 // Lock or unlock workloop
455 if (lock) {
456 ref->ewla->closeGate();
457 EXLOG("%s locked workloop\n", __func__);
458 } else {
459 ref->ewla->openGate();
460 EXLOG("%s unlocked workloop\n", __func__);
461 }
462 return true;
463 }
464
465 static void
getExclaveInterruptKey(int index,char * key,size_t size)466 getExclaveInterruptKey(int index, char *key, size_t size)
467 {
468 snprintf(key, size, "%d", index);
469 }
470
471 static IOInterruptEventSource *
copyExclaveInterruptEventSource(IOService::IOExclaveProxyState * pRef,int index)472 copyExclaveInterruptEventSource(IOService::IOExclaveProxyState * pRef, int index)
473 {
474 OSObject *obj;
475 IOInterruptEventSource *ies;
476 char irqKey[5];
477
478 if (!pRef) {
479 return NULL;
480 }
481
482 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
483 IOLockAssert(pRef->exclave_interrupts_lock, kIOLockAssertOwned);
484 obj = pRef->exclave_interrupts->getObject(irqKey);
485 if (!obj) {
486 return NULL;
487 }
488
489 ies = OSDynamicCast(IOInterruptEventSource, obj);
490 if (ies) {
491 ies->retain();
492 }
493 return ies;
494 }
495
496 // TODO: Remove after testing
497 void
IOExclaveTestSignalInterrupt(thread_call_param_t arg0,__unused thread_call_param_t arg1)498 IOExclaveTestSignalInterrupt(thread_call_param_t arg0, __unused thread_call_param_t arg1)
499 {
500 EXLOG("%s called\n", __func__);
501
502 // Unpackage params
503 struct IOExclaveTestSignalInterruptParam *params = (struct IOExclaveTestSignalInterruptParam *) arg0;
504 if (params->id == -1 || params->index == -1) {
505 printf("%s: ERROR: id and irq index not initialized\n", __func__);
506 return;
507 }
508
509 uint64_t id = params->id;
510 int index = (int) params->index;
511
512 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
513 if (!ref) {
514 return;
515 }
516 ref->service->retain();
517
518 // Get interrupt
519 char irqKey[5];
520 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
521 OSObject *obj2 = ref->exclave_interrupts->getObject(irqKey);
522 if (!obj2) {
523 printf("%s: ERROR: failed to get ies\n", __func__);
524 ref->service->release();
525 return;
526 }
527
528 IOInterruptEventSource *ies = OSDynamicCast(IOInterruptEventSource, obj2);
529 if (!ies) {
530 printf("%s: ERROR: failed to cast ies\n", __func__);
531 ref->service->release();
532 return;
533 }
534
535 // Signal interrupt
536 ies->interruptOccurred(NULL, NULL, 1);
537
538 ref->service->release();
539 }
540
541 bool
IOExclaveAsyncNotificationUpcallHandler(uint64_t id,struct IOExclaveAsyncNotificationUpcallArgs * args)542 IOExclaveAsyncNotificationUpcallHandler(uint64_t id, struct IOExclaveAsyncNotificationUpcallArgs *args)
543 {
544 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
545 bool ret = false;
546 if (!ref) {
547 return false;
548 }
549
550 switch (args->type) {
551 case AsyncNotificationUpcallTypeSignal:
552 ret = ref->service->exclaveAsyncNotificationSignal(ref, args->notificationID) == kIOReturnSuccess;
553 break;
554 default:
555 ret = false;
556 break;
557 }
558 return ret;
559 }
560
561 bool
IOExclaveMapperOperationUpcallHandler(uint64_t id,IOExclaveMapperOperationUpcallArgs * args)562 IOExclaveMapperOperationUpcallHandler(uint64_t id, IOExclaveMapperOperationUpcallArgs *args)
563 {
564 assert(args);
565 IOService *provider = NULL;
566 bool res = false;
567 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
568 if (!ref) {
569 return false;
570 }
571 provider = ref->service->getProvider();
572
573 IOMapper *mapper = IOMapper::copyMapperForDeviceWithIndex(provider, (unsigned int)(args->mapperIndex));
574 if (!mapper) {
575 goto finish;
576 }
577
578 switch (args->type) {
579 case MapperActivate:
580 res = kIOReturnSuccess == mapper->callPlatformFunction(gDARTMapperFunctionSetActive, false, (void *)(true), (void *)(false), NULL, NULL);
581 break;
582 case MapperDeactivate:
583 res = kIOReturnSuccess == mapper->callPlatformFunction(gDARTMapperFunctionSetActive, false, (void *)(false), (void *)(false), NULL, NULL);
584 break;
585 default:
586 break;
587 }
588
589 finish:
590 return res;
591 }
592
593 bool
IOExclaveANEUpcallHandler(uint64_t id,struct IOExclaveANEUpcallArgs * args,bool * result)594 IOExclaveANEUpcallHandler(uint64_t id, struct IOExclaveANEUpcallArgs *args, bool *result)
595 {
596 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
597 bool ret = false;
598 bool _result = false;
599 if (!ref || !args) {
600 return false;
601 }
602
603 switch (args->type) {
604 case kIOExclaveANEUpcallTypeSetPowerState:
605 if (ref->aneSetPowerStateUpcallHandler) {
606 _result = (ref->aneSetPowerStateUpcallHandler)(
607 args->setpowerstate_args.desired_state
608 );
609 ret = true;
610 } else {
611 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
612 }
613 break;
614 case kIOExclaveANEUpcallTypeWorkSubmit:
615 if (ref->aneWorkSubmitUpcallHandler) {
616 _result = (ref->aneWorkSubmitUpcallHandler)(
617 args->work_args.arg0,
618 args->work_args.arg1,
619 args->work_args.arg2
620 );
621 ret = true;
622 } else {
623 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
624 }
625 break;
626 case kIOExclaveANEUpcallTypeWorkBegin:
627 if (ref->aneWorkBeginUpcallHandler) {
628 _result = (ref->aneWorkBeginUpcallHandler)(
629 args->work_args.arg0,
630 args->work_args.arg1,
631 args->work_args.arg2
632 );
633 ret = true;
634 } else {
635 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
636 }
637 break;
638 case kIOExclaveANEUpcallTypeWorkEnd:
639 if (ref->aneWorkEndUpcallHandler) {
640 _result = (ref->aneWorkEndUpcallHandler)(
641 args->work_args.arg0,
642 args->work_args.arg1,
643 args->work_args.arg2
644 );
645 ret = true;
646 } else {
647 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
648 }
649 break;
650 default:
651 ret = false;
652 break;
653 }
654
655 if (result) {
656 *result = _result;
657 }
658
659 return ret;
660 }
661
662 /* IOService exclave methods */
663
664 #endif /* CONFIG_EXCLAVES */
665
666 bool
exclaveRegisterInterrupt(IOExclaveProxyState * pRef,int index,bool noProvider=false)667 IOService::exclaveRegisterInterrupt(IOExclaveProxyState * pRef, int index, bool noProvider = false)
668 {
669 #if CONFIG_EXCLAVES
670 IOInterruptEventSource *ies = NULL;
671 IOInterruptEventSource::Action action;
672 IOWorkLoop *wl;
673 char irqKey[5];
674
675 assert(getWorkLoop());
676
677 if (!pRef) {
678 return false;
679 }
680
681 action = OSMemberFunctionCast(IOInterruptEventSource::Action,
682 this, &IOService::exclaveInterruptOccurred);
683 ies = IOInterruptEventSource::interruptEventSource(this, action, noProvider ? nullptr : getProvider(), index);
684 if (!ies) {
685 return false;
686 }
687
688 wl = getWorkLoop();
689 if (!wl) {
690 ies->release();
691 return false;
692 }
693 if (wl->addEventSource(ies) != kIOReturnSuccess) {
694 ies->release();
695 return false;
696 }
697
698 // Register IOIES in exclave proxy state
699 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
700 IOLockLock(pRef->exclave_interrupts_lock);
701 pRef->exclave_interrupts->setObject(irqKey, ies);
702 IOLockUnlock(pRef->exclave_interrupts_lock);
703 OSSafeReleaseNULL(ies);
704
705 EXLOG("%s: IRQ %d register success!\n", __func__, index);
706 return true;
707 #else /* CONFIG_EXCLAVES */
708 return false;
709 #endif /* CONFIG_EXCLAVES */
710 }
711
712 bool
exclaveRemoveInterrupt(IOExclaveProxyState * pRef,int index)713 IOService::exclaveRemoveInterrupt(IOExclaveProxyState * pRef, int index)
714 {
715 #if CONFIG_EXCLAVES
716 IOInterruptEventSource *ies;
717 IOWorkLoop *wl;
718 char irqKey[5];
719
720 assert(getWorkLoop());
721
722 if (!pRef) {
723 return false;
724 }
725 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
726
727 IOLockLock(pRef->exclave_interrupts_lock);
728
729 ies = copyExclaveInterruptEventSource(pRef, index);
730
731 if (!ies) {
732 IOLockUnlock(pRef->exclave_interrupts_lock);
733 OSSafeReleaseNULL(ies);
734 return false;
735 }
736 pRef->exclave_interrupts->removeObject(irqKey);
737 IOLockUnlock(pRef->exclave_interrupts_lock);
738
739 wl = getWorkLoop();
740 if (!wl) {
741 OSSafeReleaseNULL(ies);
742 return false;
743 }
744
745 wl->removeEventSource(ies);
746 OSSafeReleaseNULL(ies);
747
748 EXLOG("%s: IRQ %d removed successfully\n", __func__, index);
749 return true;
750 #else /* CONFIG_EXCLAVES */
751 return false;
752 #endif /* CONFIG_EXCLAVES */
753 }
754
755 bool
exclaveEnableInterrupt(IOExclaveProxyState * pRef,int index,bool enable)756 IOService::exclaveEnableInterrupt(IOExclaveProxyState * pRef, int index, bool enable)
757 {
758 #if CONFIG_EXCLAVES
759 IOInterruptEventSource *ies;
760
761 assert(getWorkLoop());
762
763 if (!pRef) {
764 return false;
765 }
766
767 IOLockLock(pRef->exclave_interrupts_lock);
768 ies = copyExclaveInterruptEventSource(pRef, index);
769 if (!ies) {
770 IOLockUnlock(pRef->exclave_interrupts_lock);
771 OSSafeReleaseNULL(ies);
772 return false;
773 }
774
775 if (enable) {
776 ies->enable();
777 } else {
778 ies->disable();
779 }
780 IOLockUnlock(pRef->exclave_interrupts_lock);
781 OSSafeReleaseNULL(ies);
782
783 EXLOG("%s: IRQ %s success!\n", __func__, enable ? "enable" : "disable");
784 return true;
785 #else /* CONFIG_EXCLAVES */
786 return false;
787 #endif /* CONFIG_EXCLAVES */
788 }
789
790
791 void
exclaveInterruptOccurred(IOInterruptEventSource * eventSource,int count)792 IOService::exclaveInterruptOccurred(IOInterruptEventSource *eventSource, int count)
793 {
794 #if CONFIG_EXCLAVES
795 tb_error_t tberr;
796 IOService::IOExclaveProxyState *ref;
797
798 if (!eventSource) {
799 printf("%s ERROR: IOInterruptEventSource is null\n", __func__);
800 return;
801 }
802
803 EXLOG("%s id 0x%llx (irq index %d)\n", __func__, getRegistryEntryID(), eventSource->getIntIndex());
804
805 ref = getProxyStateFromRegistryID(getRegistryEntryID());
806 if (!ref) {
807 printf("%s ERROR: failed to get IOExclaveProxyState\n", __func__);
808 return;
809 }
810
811 tberr = ioservice_ioserviceprivate_interruptoccurredprivate(&ref->edk_client, eventSource->getIntIndex(), count);
812 assert(TB_ERROR_SUCCESS == tberr);
813 if (TB_ERROR_SUCCESS != tberr) {
814 printf("%s ERROR: tightbeam call failed\n", __func__);
815 return;
816 }
817 #endif /* CONFIG_EXCLAVES */
818 }
819
820 #if CONFIG_EXCLAVES
821 static void
getExclaveTimerKey(uint32_t timer_id,char * key,size_t size)822 getExclaveTimerKey(uint32_t timer_id, char *key, size_t size)
823 {
824 snprintf(key, size, "%d", timer_id);
825 }
826
827 static IOTimerEventSource *
copyExclaveTimerEventSource(IOService::IOExclaveProxyState * pRef,uint32_t timer_id)828 copyExclaveTimerEventSource(IOService::IOExclaveProxyState * pRef, uint32_t timer_id)
829 {
830 OSObject *obj;
831 IOTimerEventSource *tes;
832 char timerKey[5];
833
834 if (!pRef) {
835 return NULL;
836 }
837
838 getExclaveTimerKey(timer_id, timerKey, sizeof(timerKey));
839 IOLockAssert(pRef->exclave_timers_lock, kIOLockAssertOwned);
840 obj = pRef->exclave_timers->getObject(timerKey);
841 if (!obj) {
842 return NULL;
843 }
844
845 tes = OSDynamicCast(IOTimerEventSource, obj);
846 if (tes) {
847 tes->retain();
848 }
849 return tes;
850 }
851 #endif /* CONFIG_EXCLAVES */
852
853 bool
exclaveRegisterTimer(IOExclaveProxyState * pRef,uint32_t * timer_id)854 IOService::exclaveRegisterTimer(IOExclaveProxyState * pRef, uint32_t *timer_id)
855 {
856 #if CONFIG_EXCLAVES
857 IOTimerEventSource *tes = NULL;
858 IOTimerEventSource::Action action;
859 IOWorkLoop *wl;
860 char timerKey[5];
861
862 assert(getWorkLoop());
863
864 if (!pRef || !timer_id) {
865 return false;
866 }
867
868 action = OSMemberFunctionCast(IOTimerEventSource::Action,
869 this, &IOService::exclaveTimerFired);
870 tes = IOTimerEventSource::timerEventSource(this, action);
871 if (!tes) {
872 return false;
873 }
874
875 wl = getWorkLoop();
876 if (!wl) {
877 tes->release();
878 return false;
879 }
880 if (wl->addEventSource(tes) != kIOReturnSuccess) {
881 tes->release();
882 return false;
883 }
884
885 // Register IOTES in exclave proxy state
886 IOLockLock(pRef->exclave_timers_lock);
887 *timer_id = pRef->nextExclaveTimerId++;
888 getExclaveTimerKey(*timer_id, timerKey, sizeof(timerKey));
889 pRef->exclave_timers->setObject(timerKey, tes);
890 IOLockUnlock(pRef->exclave_timers_lock);
891 OSSafeReleaseNULL(tes);
892
893 EXLOG("%s: timer %u register success!\n", __func__, *timer_id);
894 return true;
895 #else /* CONFIG_EXCLAVES */
896 return false;
897 #endif /* CONFIG_EXCLAVES */
898 }
899
900 bool
exclaveRemoveTimer(IOExclaveProxyState * pRef,uint32_t timer_id)901 IOService::exclaveRemoveTimer(IOExclaveProxyState * pRef, uint32_t timer_id)
902 {
903 #if CONFIG_EXCLAVES
904 IOTimerEventSource *tes;
905 IOWorkLoop *wl;
906 char timerKey[5];
907
908 assert(getWorkLoop());
909
910 if (!pRef) {
911 return false;
912 }
913
914 getExclaveTimerKey(timer_id, timerKey, sizeof(timerKey));
915
916 IOLockLock(pRef->exclave_timers_lock);
917 tes = copyExclaveTimerEventSource(pRef, timer_id);
918 if (!tes) {
919 IOLockUnlock(pRef->exclave_timers_lock);
920 OSSafeReleaseNULL(tes);
921 return false;
922 }
923 pRef->exclave_timers->removeObject(timerKey);
924 IOLockUnlock(pRef->exclave_timers_lock);
925
926 wl = getWorkLoop();
927 if (!wl) {
928 OSSafeReleaseNULL(tes);
929 return false;
930 }
931
932 wl->removeEventSource(tes);
933 OSSafeReleaseNULL(tes);
934
935 EXLOG("%s: timer %u removed successfully\n", __func__, timer_id);
936 return true;
937 #else /* CONFIG_EXCLAVES */
938 return false;
939 #endif /* CONFIG_EXCLAVES */
940 }
941
942 bool
exclaveEnableTimer(IOExclaveProxyState * pRef,uint32_t timer_id,bool enable)943 IOService::exclaveEnableTimer(IOExclaveProxyState * pRef, uint32_t timer_id, bool enable)
944 {
945 #if CONFIG_EXCLAVES
946 IOTimerEventSource *tes;
947
948 assert(getWorkLoop());
949
950 if (!pRef) {
951 return false;
952 }
953
954 IOLockLock(pRef->exclave_timers_lock);
955 tes = copyExclaveTimerEventSource(pRef, timer_id);
956 if (!tes) {
957 IOLockUnlock(pRef->exclave_timers_lock);
958 OSSafeReleaseNULL(tes);
959 return false;
960 }
961
962 if (enable) {
963 tes->enable();
964 } else {
965 tes->disable();
966 }
967 IOLockUnlock(pRef->exclave_timers_lock);
968 OSSafeReleaseNULL(tes);
969
970 EXLOG("%s: timer %u %s success\n", __func__, timer_id, enable ? "enable" : "disable");
971 return true;
972 #else /* CONFIG_EXCLAVES */
973 return false;
974 #endif /* CONFIG_EXCLAVES */
975 }
976
977 bool
exclaveTimerSetTimeout(IOExclaveProxyState * pRef,uint32_t timer_id,uint32_t options,AbsoluteTime interval,AbsoluteTime leeway,kern_return_t * kr)978 IOService::exclaveTimerSetTimeout(IOExclaveProxyState * pRef, uint32_t timer_id, uint32_t options, AbsoluteTime interval, AbsoluteTime leeway, kern_return_t *kr)
979 {
980 #if CONFIG_EXCLAVES
981 IOTimerEventSource *tes;
982
983 assert(getWorkLoop());
984
985 if (!pRef || !kr) {
986 return false;
987 }
988
989 IOLockLock(pRef->exclave_timers_lock);
990 tes = copyExclaveTimerEventSource(pRef, timer_id);
991 if (!tes) {
992 IOLockUnlock(pRef->exclave_timers_lock);
993 OSSafeReleaseNULL(tes);
994 return false;
995 }
996
997 *kr = tes->setTimeout(options, interval, leeway);
998 IOLockUnlock(pRef->exclave_timers_lock);
999 OSSafeReleaseNULL(tes);
1000
1001 EXLOG("%s: timer %u setTimeout completed (kr %d)\n", __func__, timer_id, *kr);
1002 return true;
1003 #else /* CONFIG_EXCLAVES */
1004 return false;
1005 #endif /* CONFIG_EXCLAVES */
1006 }
1007
1008 bool
exclaveTimerCancelTimeout(IOExclaveProxyState * pRef,uint32_t timer_id)1009 IOService::exclaveTimerCancelTimeout(IOExclaveProxyState * pRef, uint32_t timer_id)
1010 {
1011 #if CONFIG_EXCLAVES
1012 IOTimerEventSource *tes;
1013
1014 assert(getWorkLoop());
1015
1016 if (!pRef) {
1017 return false;
1018 }
1019
1020 IOLockLock(pRef->exclave_timers_lock);
1021 tes = copyExclaveTimerEventSource(pRef, timer_id);
1022 if (!tes) {
1023 IOLockUnlock(pRef->exclave_timers_lock);
1024 OSSafeReleaseNULL(tes);
1025 return false;
1026 }
1027
1028 tes->cancelTimeout();
1029 IOLockUnlock(pRef->exclave_timers_lock);
1030 OSSafeReleaseNULL(tes);
1031 EXLOG("%s: timer %u setTimeout success\n", __func__, timer_id);
1032 return true;
1033 #else /* CONFIG_EXCLAVES */
1034 return false;
1035 #endif /* CONFIG_EXCLAVES */
1036 }
1037
1038 void
exclaveTimerFired(IOTimerEventSource * eventSource)1039 IOService::exclaveTimerFired(IOTimerEventSource *eventSource)
1040 {
1041 #if CONFIG_EXCLAVES
1042 tb_error_t tberr;
1043 IOService::IOExclaveProxyState *ref;
1044 __block bool found = false;
1045 __block uint32_t timer_id;
1046
1047 if (!eventSource) {
1048 printf("%s ERROR: IOTimerEventSource is null\n", __func__);
1049 return;
1050 }
1051
1052 ref = getProxyStateFromRegistryID(getRegistryEntryID());
1053 if (!ref) {
1054 printf("%s ERROR: failed to get IOExclaveProxyState\n", __func__);
1055 return;
1056 }
1057
1058 // Find timer ID
1059 IOLockLock(ref->exclave_timers_lock);
1060 ref->exclave_timers->iterateObjects(^bool (const OSSymbol * id, OSObject * obj)
1061 {
1062 if (obj == eventSource) {
1063 found = true;
1064 const char *key = id->getCStringNoCopy();
1065 timer_id = (uint32_t) strtol(key, NULL, 0);
1066 return true;
1067 }
1068 return false;
1069 });
1070 IOLockUnlock(ref->exclave_timers_lock);
1071
1072 if (!found) {
1073 printf("%s ERROR: Could not find timer ID\n", __func__);
1074 return;
1075 }
1076
1077 EXLOG("%s id 0x%llx (timer_id %u)\n", __func__, getRegistryEntryID(), timer_id);
1078 tberr = ioservice_ioserviceprivate_timerfiredprivate(&ref->edk_client, timer_id);
1079 assert(TB_ERROR_SUCCESS == tberr);
1080 if (TB_ERROR_SUCCESS != tberr) {
1081 printf("%s ERROR: tightbeam call failed\n", __func__);
1082 return;
1083 }
1084 #endif /* CONFIG_EXCLAVES */
1085 }
1086
1087
1088 kern_return_t
exclaveAsyncNotificationRegister(IOExclaveProxyState * pRef,IOInterruptEventSource * notification,uint32_t * notificationID)1089 IOService::exclaveAsyncNotificationRegister(IOExclaveProxyState * pRef, IOInterruptEventSource *notification, uint32_t *notificationID)
1090 {
1091 #if CONFIG_EXCLAVES
1092 kern_return_t ret;
1093 if (notification == NULL) {
1094 return kIOReturnBadArgument;
1095 }
1096
1097 IOLockLock(pRef->exclaveAsyncNotificationEventSourcesLock);
1098 if (!pRef->exclaveAsyncNotificationEventSources) {
1099 pRef->exclaveAsyncNotificationEventSources = OSArray::withCapacity(1);
1100 }
1101 if (pRef->exclaveAsyncNotificationEventSources) {
1102 *notificationID = (uint32_t) pRef->exclaveAsyncNotificationEventSources->getCount();
1103 pRef->exclaveAsyncNotificationEventSources->setObject(notification);
1104 ret = kIOReturnSuccess;
1105 } else {
1106 ret = kIOReturnNoMemory;
1107 }
1108 IOLockUnlock(pRef->exclaveAsyncNotificationEventSourcesLock);
1109 return ret;
1110 #else
1111 #pragma unused(pRef, notification, notificationID)
1112 return kIOReturnUnsupported;
1113 #endif /* CONFIG_EXCLAVES*/
1114 }
1115
1116 kern_return_t
exclaveAsyncNotificationSignal(IOExclaveProxyState * pRef,uint32_t notificationID)1117 IOService::exclaveAsyncNotificationSignal(IOExclaveProxyState * pRef, uint32_t notificationID)
1118 {
1119 #if CONFIG_EXCLAVES
1120 kern_return_t ret;
1121 IOInterruptEventSource *event;
1122
1123 IOLockLock(pRef->exclaveAsyncNotificationEventSourcesLock);
1124 if (pRef->exclaveAsyncNotificationEventSources && (event = OSDynamicCast(IOInterruptEventSource, pRef->exclaveAsyncNotificationEventSources->getObject((unsigned int)notificationID)))) {
1125 event->interruptOccurred(NULL, NULL, 0);
1126 ret = kIOReturnSuccess;
1127 } else {
1128 ret = kIOReturnError;
1129 }
1130 IOLockUnlock(pRef->exclaveAsyncNotificationEventSourcesLock);
1131 return ret;
1132 #else
1133 #pragma unused(pRef, notificationID)
1134 return kIOReturnUnsupported;
1135 #endif /* CONFIG_EXCLAVES */
1136 }
1137
1138 #if CONFIG_EXCLAVES
1139
1140 void
exclaves_wait_for_cpu_init()1141 exclaves_wait_for_cpu_init()
1142 {
1143 OSDictionary *match_dict = IOService::resourceMatching(gIOAllCPUInitializedKey);
1144 IOService *match = IOService::waitForMatchingService(match_dict);
1145 match_dict->release();
1146 match->release();
1147 }
1148
1149 /* ANE Upcalls */
1150
1151 static kern_return_t
exclaveRegisterANEUpcallHelper(IOService::IOExclaveProxyState * pRef,IOExclaveANEUpcallType type,IOWorkLoop * wl,void * block)1152 exclaveRegisterANEUpcallHelper(IOService::IOExclaveProxyState * pRef, IOExclaveANEUpcallType type, IOWorkLoop *wl, void *block)
1153 {
1154 void __block * _block = block;
1155
1156 if (!_block) {
1157 return kIOReturnBadArgument;
1158 }
1159
1160 if (!pRef) {
1161 Block_release(_block);
1162 return kIOReturnBadArgument;
1163 }
1164
1165 if (wl != NULL) {
1166 return wl->runActionBlock(^{
1167 switch (type) {
1168 case kIOExclaveANEUpcallTypeSetPowerState:
1169 if (pRef->aneSetPowerStateUpcallHandler) {
1170 Block_release(pRef->aneSetPowerStateUpcallHandler);
1171 }
1172 pRef->aneSetPowerStateUpcallHandler = (ANEUpcallSetPowerStateHandler) _block;
1173 break;
1174 case kIOExclaveANEUpcallTypeWorkSubmit:
1175 if (pRef->aneWorkSubmitUpcallHandler) {
1176 Block_release(pRef->aneWorkSubmitUpcallHandler);
1177 }
1178 pRef->aneWorkSubmitUpcallHandler = (ANEUpcallWorkHandler) _block;
1179 break;
1180 case kIOExclaveANEUpcallTypeWorkBegin:
1181 if (pRef->aneWorkBeginUpcallHandler) {
1182 Block_release(pRef->aneWorkBeginUpcallHandler);
1183 }
1184 pRef->aneWorkBeginUpcallHandler = (ANEUpcallWorkHandler) _block;
1185 break;
1186 case kIOExclaveANEUpcallTypeWorkEnd:
1187 if (pRef->aneWorkEndUpcallHandler) {
1188 Block_release(pRef->aneWorkEndUpcallHandler);
1189 }
1190 pRef->aneWorkEndUpcallHandler = (ANEUpcallWorkHandler) _block;
1191 break;
1192 default:
1193 Block_release(_block);
1194 return kIOReturnBadArgument;
1195 }
1196 return kIOReturnSuccess;
1197 });
1198 } else {
1199 Block_release(_block);
1200 return kIOReturnError;
1201 }
1202 }
1203
1204 #endif /* CONFIG_EXCLAVES */
1205
1206 kern_return_t
exclaveRegisterANEUpcallSetPowerState(IOExclaveProxyState * pRef,ANEUpcallSetPowerStateHandler handler)1207 IOService::exclaveRegisterANEUpcallSetPowerState(IOExclaveProxyState * pRef, ANEUpcallSetPowerStateHandler handler)
1208 {
1209 #if CONFIG_EXCLAVES
1210 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeSetPowerState, getWorkLoop(), Block_copy(handler));
1211 #else
1212 #pragma unused(pRef, handler)
1213 return kIOReturnUnsupported;
1214 #endif /* CONFIG_EXCLAVES*/
1215 }
1216
1217 kern_return_t
exclaveRegisterANEUpcallWorkSubmit(IOExclaveProxyState * pRef,ANEUpcallWorkHandler handler)1218 IOService::exclaveRegisterANEUpcallWorkSubmit(IOExclaveProxyState * pRef, ANEUpcallWorkHandler handler)
1219 {
1220 #if CONFIG_EXCLAVES
1221 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeWorkSubmit, getWorkLoop(), Block_copy(handler));
1222 #else
1223 #pragma unused(pRef, handler)
1224 return kIOReturnUnsupported;
1225 #endif /* CONFIG_EXCLAVES*/
1226 }
1227
1228 kern_return_t
exclaveRegisterANEUpcallWorkBegin(IOExclaveProxyState * pRef,ANEUpcallWorkHandler handler)1229 IOService::exclaveRegisterANEUpcallWorkBegin(IOExclaveProxyState * pRef, ANEUpcallWorkHandler handler)
1230 {
1231 #if CONFIG_EXCLAVES
1232 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeWorkBegin, getWorkLoop(), Block_copy(handler));
1233 #else
1234 #pragma unused(pRef, handler)
1235 return kIOReturnUnsupported;
1236 #endif /* CONFIG_EXCLAVES*/
1237 }
1238
1239 kern_return_t
exclaveRegisterANEUpcallWorkEnd(IOExclaveProxyState * pRef,ANEUpcallWorkHandler handler)1240 IOService::exclaveRegisterANEUpcallWorkEnd(IOExclaveProxyState * pRef, ANEUpcallWorkHandler handler)
1241 {
1242 #if CONFIG_EXCLAVES
1243 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeWorkEnd, getWorkLoop(), Block_copy(handler));
1244 #else
1245 #pragma unused(pRef, handler)
1246 return kIOReturnUnsupported;
1247 #endif /* CONFIG_EXCLAVES*/
1248 }
1249