1 /*
2 * Copyright (c) 1998-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <IOKit/system.h>
30 #include <IOKit/IOService.h>
31 #include <libkern/OSDebug.h>
32 #include <libkern/c++/OSAllocation.h>
33 #include <libkern/c++/OSContainers.h>
34 #include <libkern/c++/OSKext.h>
35 #include <libkern/c++/OSUnserialize.h>
36 #include <libkern/c++/OSKext.h>
37 #include <libkern/c++/OSSharedPtr.h>
38 #include <libkern/Block.h>
39 #include <IOKit/IOCatalogue.h>
40 #include <IOKit/IOCommand.h>
41 #include <IOKit/IODeviceTreeSupport.h>
42 #include <IOKit/IODeviceMemory.h>
43 #include <IOKit/IOInterrupts.h>
44 #include <IOKit/IOInterruptController.h>
45 #include <IOKit/IOPlatformExpert.h>
46 #include <IOKit/IOMessage.h>
47 #include <IOKit/IOLib.h>
48 #include <IOKit/IOKitKeysPrivate.h>
49 #include <IOKit/IOBSD.h>
50 #include <IOKit/IOUserClient.h>
51 #include <IOKit/IOUserServer.h>
52 #include <IOKit/IOWorkLoop.h>
53 #include <IOKit/IOTimeStamp.h>
54 #include <IOKit/IOHibernatePrivate.h>
55 #include <IOKit/IOInterruptAccountingPrivate.h>
56 #include <IOKit/IOKernelReporters.h>
57 #include <IOKit/AppleKeyStoreInterface.h>
58 #include <IOKit/pwr_mgt/RootDomain.h>
59 #include <IOKit/IOCPU.h>
60 #include <Exclaves/Exclaves.h>
61 #include <kern/cs_blobs.h>
62 #include <mach/sync_policy.h>
63 #include <mach/thread_info.h>
64 #include <IOKit/assert.h>
65 #include <sys/errno.h>
66 #include <sys/kdebug.h>
67 #include <string.h>
68
69 #include <machine/pal_routines.h>
70
71 #define LOG kprintf
72 //#define LOG IOLog
73 #define MATCH_DEBUG 0
74 #define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
75
76 // disabled since lockForArbitration() can be held externally
77 #define DEBUG_NOTIFIER_LOCKED 0
78
79 enum{
80 kIOUserServerCheckInTimeoutSecs = 120ULL
81 };
82
83 #include "IOServicePrivate.h"
84 #include "IOKitKernelInternal.h"
85
86 // take lockForArbitration before LOCKNOTIFY
87
88 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
89
90 #define super IORegistryEntry
91
92 OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
93
94 OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
95 OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
96
97 OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
98
99 OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
100
101 OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
102
103 OSDefineMetaClassAndStructors(IOResources, IOService)
104 OSDefineMetaClassAndStructors(IOUserResources, IOService)
105 OSDefineMetaClassAndStructors(IOExclaveProxy, IOService)
106
107 OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
108
109 OSDefineMetaClassAndStructors(_IOServiceStateNotification, IOService)
110
111 OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
112
113 OSDefineMetaClassAndStructors(IOServiceCompatibility, IOService)
114
115 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
116
117 static IOPlatformExpert * gIOPlatform;
118 static class IOPMrootDomain * gIOPMRootDomain;
119 const IORegistryPlane * gIOServicePlane;
120 const IORegistryPlane * gIOPowerPlane;
121 const OSSymbol * gIODeviceMemoryKey;
122 const OSSymbol * gIOInterruptControllersKey;
123 const OSSymbol * gIOInterruptSpecifiersKey;
124
125 const OSSymbol * gIOResourcesKey;
126 const OSSymbol * gIOUserResourcesKey;
127 const OSSymbol * gIOResourceMatchKey;
128 const OSSymbol * gIOResourceMatchedKey;
129 const OSSymbol * gIOResourceIOKitKey;
130
131 const OSSymbol * gIOProviderClassKey;
132 const OSSymbol * gIONameMatchKey;
133 const OSSymbol * gIONameMatchedKey;
134 const OSSymbol * gIOPropertyMatchKey;
135 const OSSymbol * gIOPropertyExistsMatchKey;
136 const OSSymbol * gIOLocationMatchKey;
137 const OSSymbol * gIOParentMatchKey;
138 const OSSymbol * gIOPathMatchKey;
139 const OSSymbol * gIOMatchCategoryKey;
140 const OSSymbol * gIODefaultMatchCategoryKey;
141 const OSSymbol * gIOMatchedAtBootKey;
142 const OSSymbol * gIOMatchedServiceCountKey;
143 const OSSymbol * gIOMatchedPersonalityKey;
144 const OSSymbol * gIORematchPersonalityKey;
145 const OSSymbol * gIORematchCountKey;
146 const OSSymbol * gIODEXTMatchCountKey;
147 const OSSymbol * gIOSupportedPropertiesKey;
148 const OSSymbol * gIOUserServicePropertiesKey;
149 #if defined(XNU_TARGET_OS_OSX)
150 const OSSymbol * gIOServiceLegacyMatchingRegistryIDKey;
151 #endif /* defined(XNU_TARGET_OS_OSX) */
152
153 const OSSymbol * gIOCompatibilityMatchKey;
154 const OSSymbol * gIOCompatibilityPropertiesKey;
155 const OSSymbol * gIOPathKey;
156
157 const OSSymbol * gIOMapperIDKey;
158 const OSSymbol * gIOUserClientClassKey;
159
160 const OSSymbol * gIOUserClassKey;
161 const OSSymbol * gIOUserClassesKey;
162 const OSSymbol * gIOUserServerClassKey;
163 const OSSymbol * gIOUserServerNameKey;
164 const OSSymbol * gIOUserServerTagKey;
165 const OSSymbol * gIOUserUserClientKey;
166 const OSSymbol * gIOUserServerOneProcessKey;
167 const OSSymbol * gIOUserServerPreserveUserspaceRebootKey;
168
169 const OSSymbol * gIOKitDebugKey;
170
171 const OSSymbol * gIOCommandPoolSizeKey;
172
173 const OSSymbol * gIOConsoleLockedKey;
174 const OSSymbol * gIOConsoleUsersKey;
175 const OSSymbol * gIOConsoleSessionUIDKey;
176 const OSSymbol * gIOConsoleSessionAuditIDKey;
177 const OSSymbol * gIOConsoleUsersSeedKey;
178 const OSSymbol * gIOConsoleSessionOnConsoleKey;
179 const OSSymbol * gIOConsoleSessionLoginDoneKey;
180 const OSSymbol * gIOConsoleSessionSecureInputPIDKey;
181 const OSSymbol * gIOConsoleSessionScreenLockedTimeKey;
182 const OSSymbol * gIOConsoleSessionScreenIsLockedKey;
183 clock_sec_t gIOConsoleLockTime;
184 static bool gIOConsoleLoggedIn;
185 #if HIBERNATION
186 static OSBoolean * gIOConsoleBooterLockState;
187 static uint32_t gIOScreenLockState;
188 #endif
189 static IORegistryEntry * gIOChosenEntry;
190
191 static int gIOResourceGenerationCount;
192
193 const OSSymbol * gIOServiceKey;
194 const OSSymbol * gIOPublishNotification;
195 const OSSymbol * gIOFirstPublishNotification;
196 const OSSymbol * gIOMatchedNotification;
197 const OSSymbol * gIOFirstMatchNotification;
198 const OSSymbol * gIOTerminatedNotification;
199 const OSSymbol * gIOWillTerminateNotification;
200
201 const OSSymbol * gIOUserClientEntitlementsKey;
202 const OSSymbol * gIOServiceDEXTEntitlementsKey;
203 const OSSymbol * gIODriverKitEntitlementKey;
204 const OSSymbol * gIODriverKitUserClientEntitlementsKey;
205 const OSSymbol * gIODriverKitUserClientEntitlementAllowAnyKey;
206 const OSSymbol * gIODriverKitRequiredEntitlementsKey;
207 const OSSymbol * gIODriverKitTestDriverEntitlementKey;
208 const OSSymbol * gIODriverKitUserClientEntitlementCommunicatesWithDriversKey;
209 const OSSymbol * gIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey;
210 const OSSymbol * gIOMatchDeferKey;
211 const OSSymbol * gIOServiceMatchDeferredKey;
212 const OSSymbol * gIOServiceNotificationUserKey;
213
214 const OSSymbol * gIOExclaveAssignedKey;
215 const OSSymbol * gIOExclaveProxyKey;
216
217 const OSSymbol * gIOPrimaryDriverTerminateOptionsKey;
218 const OSSymbol * gIOMediaKey;
219 const OSSymbol * gIOBlockStorageDriverKey;
220 static const OSSymbol * gPhysicalInterconnectKey;
221 static const OSSymbol * gVirtualInterfaceKey;
222
223 const OSSymbol * gIOAllCPUInitializedKey;
224
225 const OSSymbol * gIOGeneralInterest;
226 const OSSymbol * gIOBusyInterest;
227 const OSSymbol * gIOAppPowerStateInterest;
228 const OSSymbol * gIOPriorityPowerStateInterest;
229 const OSSymbol * gIOConsoleSecurityInterest;
230
231 const OSSymbol * gIOBSDKey;
232 const OSSymbol * gIOBSDNameKey;
233 const OSSymbol * gIOBSDMajorKey;
234 const OSSymbol * gIOBSDMinorKey;
235 const OSSymbol * gIOBSDUnitKey;
236
237 const OSSymbol * gAKSGetKey;
238 #if defined(__i386__) || defined(__x86_64__)
239 const OSSymbol * gIOCreateEFIDevicePathSymbol;
240 #endif
241
242 static OSDictionary * gNotifications;
243 static IORecursiveLock * gNotificationLock;
244
245 static IOService * gIOResources;
246 static IOService * gIOUserResources;
247 static IOService * gIOServiceRoot;
248
249 static OSOrderedSet * gJobs;
250 static semaphore_port_t gJobsSemaphore;
251 static IOLock * gJobsLock;
252 static int gOutstandingJobs;
253 static int gNumConfigThreads;
254 static int gHighNumConfigThreads;
255 static int gMaxConfigThreads = kMaxConfigThreads;
256 static int gNumWaitingThreads;
257 static IOLock * gIOServiceBusyLock;
258 bool gCPUsRunning;
259 bool gIOKitWillTerminate;
260 bool gInUserspaceReboot;
261
262 #define kIOServiceRootMediaParentInvalid ((IOService *) -1UL)
263 #if NO_KEXTD
264 static bool gIOServiceHideIOMedia = false;
265 static IOService * gIOServiceRootMediaParent = NULL;
266 #else /* NO_KEXTD */
267 static bool gIOServiceHideIOMedia = true;
268 static IOService * gIOServiceRootMediaParent = kIOServiceRootMediaParentInvalid;
269 #endif /* !NO_KEXTD */
270
271 static thread_t gIOTerminateThread;
272 static thread_t gIOTerminateWorkerThread;
273 static UInt32 gIOTerminateWork;
274 static OSArray * gIOTerminatePhase2List;
275 static OSArray * gIOStopList;
276 static OSArray * gIOStopProviderList;
277 static OSArray * gIOFinalizeList;
278
279 #if !NO_KEXTD
280 static OSArray * gIOMatchDeferList;
281 #endif
282
283 static SInt32 gIOConsoleUsersSeed;
284 static OSData * gIOConsoleUsersSeedValue;
285
286 extern const OSSymbol * gIODTPHandleKey;
287
288 const OSSymbol * gIOPlatformFunctionHandlerSet;
289
290
291 static IOLock * gIOConsoleUsersLock;
292 static thread_call_t gIOConsoleLockCallout;
293 static IONotifier * gIOServiceNullNotifier;
294
295 static SECURITY_READ_ONLY_LATE(uint32_t) gIODextRelaunchMax;
296
297 #if DEVELOPMENT || DEBUG
298 uint64_t driverkit_checkin_timed_out = 0;
299 #endif
300
301 IORecursiveLock * gDriverKitLaunchLock;
302 OSSet * gDriverKitLaunches;
303 const OSSymbol * gIOAssociatedServicesKey;
304
305 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
306
307 #define LOCKREADNOTIFY() \
308 IORecursiveLockLock( gNotificationLock )
309 #define LOCKWRITENOTIFY() \
310 IORecursiveLockLock( gNotificationLock )
311 #define LOCKWRITE2READNOTIFY()
312 #define UNLOCKNOTIFY() \
313 IORecursiveLockUnlock( gNotificationLock )
314 #define SLEEPNOTIFY(event) \
315 IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
316 #define SLEEPNOTIFYTO(event, deadline) \
317 IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
318 #define WAKEUPNOTIFY(event) \
319 IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
320
321 #define randomDelay() \
322 int del = read_processor_clock(); \
323 del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff; \
324 IOSleep( del );
325
326 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
327
328 #define queue_element(entry, element, type, field) do { \
329 vm_address_t __ele = (vm_address_t) (entry); \
330 __ele -= -4 + ((size_t)(&((type) 4)->field)); \
331 (element) = (type) __ele; \
332 } while(0)
333
334 #define iterqueue(que, elt) \
335 for (queue_entry_t elt = queue_first(que); \
336 !queue_end(que, elt); \
337 elt = queue_next(elt))
338
339 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
340
341 struct IOInterruptAccountingReporter {
342 IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
343 IOInterruptAccountingData * statistics; /* The live statistics values, if any */
344 };
345
346 struct ArbitrationLockQueueElement {
347 queue_chain_t link;
348 IOThread thread;
349 IOService * service;
350 unsigned count;
351 bool required;
352 bool aborted;
353 };
354
355 static queue_head_t gArbitrationLockQueueActive;
356 static queue_head_t gArbitrationLockQueueWaiting;
357 static queue_head_t gArbitrationLockQueueFree;
358 static IOLock * gArbitrationLockQueueLock;
359
360 bool
isInactive(void) const361 IOService::isInactive( void ) const
362 {
363 return 0 != (kIOServiceInactiveState & getState());
364 }
365
366 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
367
368 // Only used by the intel implementation of
369 // IOService::requireMaxBusStall(UInt32 ns)
370 // IOService::requireMaxInterruptDelay(uint32_t ns)
371 struct CpuDelayEntry {
372 IOService * fService;
373 UInt32 fMaxDelay;
374 UInt32 fDelayType;
375 };
376
377 enum {
378 kCpuDelayBusStall,
379 #if defined(__x86_64__)
380 kCpuDelayInterrupt,
381 #endif /* defined(__x86_64__) */
382 kCpuNumDelayTypes
383 };
384
385 static OSData *sCpuDelayData = OSData::withCapacity(8 * sizeof(CpuDelayEntry));
386 static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
387 static OSArray *sCpuLatencyHandlers[kCpuNumDelayTypes];
388 const OSSymbol *sCPULatencyFunctionName[kCpuNumDelayTypes];
389 static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
390 static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
391 static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
392
393 static void
394 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
395 static IOReturn
396 setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
397
398 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
399
400 IOCoreAnalyticsSendEventProc gIOCoreAnalyticsSendEventProc;
401
402 kern_return_t
IOSetCoreAnalyticsSendEventProc(IOCoreAnalyticsSendEventProc proc)403 IOSetCoreAnalyticsSendEventProc(IOCoreAnalyticsSendEventProc proc)
404 {
405 if (gIOCoreAnalyticsSendEventProc) {
406 return kIOReturnNotPermitted;
407 }
408 gIOCoreAnalyticsSendEventProc = proc;
409
410 return kIOReturnSuccess;
411 }
412
413 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
414
415
416 static IOMessage sSystemPower;
417 extern "C" bool restore_boot;
418
419 namespace IOServicePH
420 {
421 IONotifier * fRootNotifier;
422 OSArray * fUserServers;
423 OSArray * fUserServersWait;
424 OSArray * fMatchingWork;
425 OSArray * fMatchingDelayed;
426 IOService * fSystemPowerAckTo;
427 uint32_t fSystemPowerAckRef;
428 uint8_t fSystemOff;
429 uint8_t fUserServerOff;
430 uint8_t fWaitingUserServers;
431 thread_call_t fUserServerAckTimer;
432
433 void lock();
434 void unlock();
435
436 void init(IOPMrootDomain * root);
437
438 IOReturn systemPowerChange(
439 void * target,
440 void * refCon,
441 UInt32 messageType, IOService * service,
442 void * messageArgument, vm_size_t argSize);
443
444 bool matchingStart(IOService * service);
445 void matchingEnd(IOService * service);
446 void userServerAckTimerExpired(void *, void *);
447 };
448
449 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
450
451 void
initialize(void)452 IOService::initialize( void )
453 {
454 kern_return_t err;
455
456 gIOServicePlane = IORegistryEntry::makePlane( kIOServicePlane );
457 gIOPowerPlane = IORegistryEntry::makePlane( kIOPowerPlane );
458
459 gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
460 gIONameMatchKey = OSSymbol::withCStringNoCopy( kIONameMatchKey );
461 gIONameMatchedKey = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
462 gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
463 gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
464 gIOPathMatchKey = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
465 gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
466 gIOParentMatchKey = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
467
468 gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
469 gIODefaultMatchCategoryKey = OSSymbol::withCStringNoCopy(
470 kIODefaultMatchCategoryKey );
471 gIOMatchedAtBootKey = OSSymbol::withCStringNoCopy(
472 kIOMatchedAtBootKey );
473
474 gIOMatchedServiceCountKey = OSSymbol::withCStringNoCopy(
475 kIOMatchedServiceCountKey );
476 gIOMatchedPersonalityKey = OSSymbol::withCStringNoCopy(
477 kIOMatchedPersonalityKey );
478 gIORematchPersonalityKey = OSSymbol::withCStringNoCopy(
479 kIORematchPersonalityKey );
480 gIORematchCountKey = OSSymbol::withCStringNoCopy(
481 kIORematchCountKey );
482 gIODEXTMatchCountKey = OSSymbol::withCStringNoCopy(
483 kIODEXTMatchCountKey );
484
485 #if defined(XNU_TARGET_OS_OSX)
486 gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy(
487 kIOServiceLegacyMatchingRegistryIDKey );
488 #endif /* defined(XNU_TARGET_OS_OSX) */
489
490 if (!PE_parse_boot_argn("dextrelaunch", &gIODextRelaunchMax, sizeof(gIODextRelaunchMax))) {
491 if (restore_boot) {
492 // Limit dext relaunches in the restore environment
493 gIODextRelaunchMax = 10;
494 } else {
495 gIODextRelaunchMax = 1000;
496 }
497 }
498 PE_parse_boot_argn("iocthreads", &gMaxConfigThreads, sizeof(gMaxConfigThreads));
499
500 gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
501
502 gIOUserClassKey = OSSymbol::withCStringNoCopy(kIOUserClassKey);
503 gIOUserClassesKey = OSSymbol::withCStringNoCopy(kIOUserClassesKey);
504
505 gIOUserServerClassKey = OSSymbol::withCStringNoCopy(kIOUserServerClassKey);
506 gIOUserServerNameKey = OSSymbol::withCStringNoCopy(kIOUserServerNameKey);
507 gIOUserServerTagKey = OSSymbol::withCStringNoCopy(kIOUserServerTagKey);
508 gIOUserUserClientKey = OSSymbol::withCStringNoCopy(kIOUserUserClientKey);
509
510 gIOUserServerOneProcessKey = OSSymbol::withCStringNoCopy(kIOUserServerOneProcessKey);
511
512 gIOUserServerPreserveUserspaceRebootKey = OSSymbol::withCStringNoCopy(kIOUserServerPreserveUserspaceRebootKey);
513
514 gIOResourcesKey = OSSymbol::withCStringNoCopy( kIOResourcesClass );
515 gIOResourceMatchKey = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
516 gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
517 gIOResourceIOKitKey = OSSymbol::withCStringNoCopy("IOKit");
518
519 gIODeviceMemoryKey = OSSymbol::withCStringNoCopy( "IODeviceMemory" );
520 gIOInterruptControllersKey
521 = OSSymbol::withCStringNoCopy("IOInterruptControllers");
522 gIOInterruptSpecifiersKey
523 = OSSymbol::withCStringNoCopy("IOInterruptSpecifiers");
524
525 gIOCompatibilityMatchKey = OSSymbol::withCStringNoCopy(kIOCompatibilityMatchKey);
526 gIOCompatibilityPropertiesKey = OSSymbol::withCStringNoCopy(kIOCompatibilityPropertiesKey);
527 gIOPathKey = OSSymbol::withCStringNoCopy(kIOPathKey);
528 gIOSupportedPropertiesKey = OSSymbol::withCStringNoCopy(kIOSupportedPropertiesKey);
529 gIOUserServicePropertiesKey = OSSymbol::withCStringNoCopy(kIOUserServicePropertiesKey);
530
531 gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
532
533 gIOKitDebugKey = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
534
535 gIOCommandPoolSizeKey = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
536
537 gIOGeneralInterest = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
538 gIOBusyInterest = OSSymbol::withCStringNoCopy( kIOBusyInterest );
539 gIOAppPowerStateInterest = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
540 gIOPriorityPowerStateInterest = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
541 gIOConsoleSecurityInterest = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
542
543 gIOBSDKey = OSSymbol::withCStringNoCopy(kIOBSDKey);
544 gIOBSDNameKey = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
545 gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
546 gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
547 gIOBSDUnitKey = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
548
549 gNotifications = OSDictionary::withCapacity( 1 );
550 gIOPublishNotification = OSSymbol::withCStringNoCopy(
551 kIOPublishNotification );
552 gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
553 kIOFirstPublishNotification );
554 gIOMatchedNotification = OSSymbol::withCStringNoCopy(
555 kIOMatchedNotification );
556 gIOFirstMatchNotification = OSSymbol::withCStringNoCopy(
557 kIOFirstMatchNotification );
558 gIOTerminatedNotification = OSSymbol::withCStringNoCopy(
559 kIOTerminatedNotification );
560 gIOWillTerminateNotification = OSSymbol::withCStringNoCopy(
561 kIOWillTerminateNotification );
562 gIOServiceKey = OSSymbol::withCStringNoCopy( kIOServiceClass);
563
564
565 gIOConsoleLockedKey = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
566 gIOConsoleUsersKey = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
567 gIOConsoleSessionUIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
568 gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
569
570 gIOConsoleUsersSeedKey = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
571 gIOConsoleSessionOnConsoleKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
572 gIOConsoleSessionLoginDoneKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
573 gIOConsoleSessionSecureInputPIDKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
574 gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
575 gIOConsoleSessionScreenIsLockedKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
576
577 gIOConsoleUsersSeedValue = OSData::withValueNoCopy(gIOConsoleUsersSeed);
578
579 gIOUserClientEntitlementsKey = OSSymbol::withCStringNoCopy( kIOUserClientEntitlementsKey );
580 gIOServiceDEXTEntitlementsKey = OSSymbol::withCStringNoCopy( kIOServiceDEXTEntitlementsKey );
581 gIODriverKitEntitlementKey = OSSymbol::withCStringNoCopy( kIODriverKitEntitlementKey );
582 gIODriverKitUserClientEntitlementsKey = OSSymbol::withCStringNoCopy( kIODriverKitUserClientEntitlementsKey );
583 #if XNU_TARGET_OS_OSX
584 gIODriverKitUserClientEntitlementAllowAnyKey = OSSymbol::withCStringNoCopy( kIODriverKitUserClientEntitlementAllowAnyKey );
585 #else
586 gIODriverKitUserClientEntitlementAllowAnyKey = NULL;
587 #endif
588 gIODriverKitRequiredEntitlementsKey = OSSymbol::withCStringNoCopy( kIODriverKitRequiredEntitlementsKey );
589 gIODriverKitTestDriverEntitlementKey = OSSymbol::withCStringNoCopy( kIODriverKitTestDriverEntitlementKey );
590 gIODriverKitUserClientEntitlementCommunicatesWithDriversKey = OSSymbol::withCStringNoCopy(kIODriverKitUserClientEntitlementCommunicatesWithDriversKey);
591 gIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey = OSSymbol::withCStringNoCopy(kIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey);
592
593 gIOMatchDeferKey = OSSymbol::withCStringNoCopy( kIOMatchDeferKey );
594 gIOServiceMatchDeferredKey = OSSymbol::withCStringNoCopy( kIOServiceMatchDeferredKey );
595 gIOServiceNotificationUserKey = OSSymbol::withCStringNoCopy( kIOServiceNotificationUserKey );
596 gIOExclaveAssignedKey = OSSymbol::withCStringNoCopy(kIOExclaveAssignedKey);
597 gIOExclaveProxyKey = OSSymbol::withCStringNoCopy(kIOExclaveProxyKey);
598
599 gIOPrimaryDriverTerminateOptionsKey = OSSymbol::withCStringNoCopy(kIOPrimaryDriverTerminateOptionsKey);
600 gIOMediaKey = OSSymbol::withCStringNoCopy("IOMedia");
601 gIOBlockStorageDriverKey = OSSymbol::withCStringNoCopy("IOBlockStorageDriver");
602 gPhysicalInterconnectKey = OSSymbol::withCStringNoCopy("Physical Interconnect");
603 gVirtualInterfaceKey = OSSymbol::withCStringNoCopy("Virtual Interface");
604
605 gIOAllCPUInitializedKey = OSSymbol::withCStringNoCopy( kIOAllCPUInitializedKey );
606
607 gIOPlatformFunctionHandlerSet = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
608 sCPULatencyFunctionName[kCpuDelayBusStall] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
609 #if defined(__x86_64__)
610 sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
611 #endif /* defined(__x86_64__) */
612 uint32_t idx;
613 for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
614 sCPULatencySet[idx] = OSNumber::withNumber(UINT_MAX, 32);
615 sCPULatencyHolder[idx] = OSNumber::withNumber(0ULL, 64);
616 assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
617 }
618
619 #if defined(__x86_64__)
620 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
621 #endif /* defined(__x86_64__) */
622
623 gNotificationLock = IORecursiveLockAlloc();
624
625 gAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
626
627 #if CONFIG_EXCLAVES
628 gExclaveProxyStates = OSDictionary::withCapacity( 1 );
629
630 gExclaveProxyStateLock = IORecursiveLockAlloc();
631 assert( gExclaveProxyStates && gExclaveProxyStateLock);
632 #endif /* CONFIG_EXCLAVES */
633
634 assert( gIOServicePlane && gIODeviceMemoryKey
635 && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
636 && gIOResourcesKey && gNotifications && gNotificationLock
637 && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
638 && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
639 && gIOPublishNotification && gIOMatchedNotification
640 && gIOTerminatedNotification && gIOServiceKey
641 && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
642 && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
643 && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
644
645 gJobsLock = IOLockAlloc();
646 gJobs = OSOrderedSet::withCapacity( 10 );
647
648 gIOServiceBusyLock = IOLockAlloc();
649
650 gIOConsoleUsersLock = IOLockAlloc();
651
652 err = semaphore_create(kernel_task, &gJobsSemaphore, SYNC_POLICY_FIFO, 0);
653
654 gIOConsoleLockCallout = thread_call_allocate(&IOService::consoleLockTimer, NULL);
655
656 IORegistryEntry::getRegistryRoot()->setProperty(gIOConsoleLockedKey, kOSBooleanTrue);
657
658 assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
659 && gIOConsoleLockCallout && (err == KERN_SUCCESS));
660
661 gIOResources = IOResources::resources();
662 gIOUserResources = IOUserResources::resources();
663 assert( gIOResources && gIOUserResources );
664
665 gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
666 assert(gIOServiceNullNotifier);
667
668 gArbitrationLockQueueLock = IOLockAlloc();
669 queue_init(&gArbitrationLockQueueActive);
670 queue_init(&gArbitrationLockQueueWaiting);
671 queue_init(&gArbitrationLockQueueFree);
672
673 assert( gArbitrationLockQueueLock );
674
675 allocPMInitLock();
676
677 gIOTerminatePhase2List = OSArray::withCapacity( 2 );
678 gIOStopList = OSArray::withCapacity( 16 );
679 gIOStopProviderList = OSArray::withCapacity( 16 );
680 gIOFinalizeList = OSArray::withCapacity( 16 );
681 #if !NO_KEXTD
682 if (OSKext::iokitDaemonAvailable()) {
683 gIOMatchDeferList = OSArray::withCapacity( 16 );
684 } else {
685 gIOMatchDeferList = NULL;
686 }
687 #endif
688 assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
689
690 gDriverKitLaunches = OSSet::withCapacity(0);
691 gDriverKitLaunchLock = IORecursiveLockAlloc();
692 gIOAssociatedServicesKey = OSSymbol::withCStringNoCopy( "IOAssociatedServices" );
693 #if CONFIG_EXCLAVES
694 gDARTMapperFunctionSetActive = OSSymbol::withCStringNoCopy("setActive");
695 #endif /* CONFIG_EXCLAVES */
696
697 // worker thread that is responsible for terminating / cleaning up threads
698 kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread);
699 assert(gIOTerminateWorkerThread);
700 thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread");
701 }
702
703 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
704
705 #if defined(__x86_64__)
706 extern "C" {
707 const char *getCpuDelayBusStallHolderName(void);
708 const char *
getCpuDelayBusStallHolderName(void)709 getCpuDelayBusStallHolderName(void)
710 {
711 return sCPULatencyHolderName[kCpuDelayBusStall];
712 }
713
714 const char *getCpuInterruptDelayHolderName(void);
715 const char *
getCpuInterruptDelayHolderName(void)716 getCpuInterruptDelayHolderName(void)
717 {
718 return sCPULatencyHolderName[kCpuDelayInterrupt];
719 }
720 }
721 #endif /* defined(__x86_64__) */
722
723
724
725 #if IOMATCHDEBUG
726 static UInt64
getDebugFlags(OSDictionary * props)727 getDebugFlags( OSDictionary * props )
728 {
729 OSNumber * debugProp;
730 UInt64 debugFlags;
731
732 debugProp = OSDynamicCast( OSNumber,
733 props->getObject( gIOKitDebugKey ));
734 if (debugProp) {
735 debugFlags = debugProp->unsigned64BitValue();
736 } else {
737 debugFlags = gIOKitDebug;
738 }
739
740 return debugFlags;
741 }
742
743 static UInt64
getDebugFlags(IOService * inst)744 getDebugFlags( IOService * inst )
745 {
746 OSObject * prop;
747 OSNumber * debugProp;
748 UInt64 debugFlags;
749
750 prop = inst->copyProperty(gIOKitDebugKey);
751 debugProp = OSDynamicCast(OSNumber, prop);
752 if (debugProp) {
753 debugFlags = debugProp->unsigned64BitValue();
754 } else {
755 debugFlags = gIOKitDebug;
756 }
757
758 OSSafeReleaseNULL(prop);
759
760 return debugFlags;
761 }
762 #endif
763
764 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
765
766 // Probe a matched service and return an instance to be started.
767 // The default score is from the property table, & may be altered
768 // during probe to change the start order.
769
770 IOService *
probe(IOService * provider,SInt32 * score)771 IOService::probe( IOService * provider,
772 SInt32 * score )
773 {
774 return this;
775 }
776
777 bool
start(IOService * provider)778 IOService::start( IOService * provider )
779 {
780 return true;
781 }
782
783 void
stop(IOService * provider)784 IOService::stop( IOService * provider )
785 {
786 if (reserved->uvars && reserved->uvars->started && reserved->uvars->userServer) {
787 reserved->uvars->userServer->serviceStop(this, provider);
788 }
789 }
790
791 bool
init(OSDictionary * dictionary)792 IOService::init( OSDictionary * dictionary )
793 {
794 bool ret;
795
796 ret = super::init(dictionary);
797 if (!ret) {
798 return false;
799 }
800 if (reserved) {
801 return true;
802 }
803
804 reserved = IOMallocType(ExpansionData);
805 IOLockInlineInit(&reserved->interruptStatisticsLock);
806 return true;
807 }
808
809 bool
init(IORegistryEntry * from,const IORegistryPlane * inPlane)810 IOService::init( IORegistryEntry * from,
811 const IORegistryPlane * inPlane )
812 {
813 bool ret;
814
815 ret = super::init(from, inPlane);
816 if (!ret) {
817 return false;
818 }
819 if (reserved) {
820 return true;
821 }
822
823 reserved = IOMallocType(ExpansionData);
824 IOLockInlineInit(&reserved->interruptStatisticsLock);
825
826 return true;
827 }
828
829 void
free(void)830 IOService::free( void )
831 {
832 IOInterruptSourcePrivate *sourcesPrivate = NULL;
833 int i = 0;
834 requireMaxBusStall(0);
835 #if defined(__x86_64__)
836 requireMaxInterruptDelay(0);
837 #endif /* defined(__x86_64__) */
838 if (getPropertyTable()) {
839 unregisterAllInterest();
840 }
841 PMfree();
842
843 if (reserved) {
844 if (reserved->interruptStatisticsArray) {
845 for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
846 if (reserved->interruptStatisticsArray[i].reporter) {
847 reserved->interruptStatisticsArray[i].reporter->release();
848 }
849 }
850
851 IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
852 }
853
854 if (reserved->uvars && reserved->uvars->userServer) {
855 reserved->uvars->userServer->serviceFree(this);
856 }
857 sourcesPrivate = reserved->interruptSourcesPrivate;
858 IOLockInlineDestroy(&reserved->interruptStatisticsLock);
859 IOFreeType(reserved, ExpansionData);
860 }
861
862 if (_numInterruptSources && _interruptSources) {
863 assert(sourcesPrivate);
864 for (i = 0; i < _numInterruptSources; i++) {
865 void * block = sourcesPrivate[i].vectorBlock;
866 if (block) {
867 Block_release(block);
868 }
869 }
870 IODelete(_interruptSources, IOInterruptSource, _numInterruptSources);
871 _interruptSources = NULL;
872 IODelete(sourcesPrivate, IOInterruptSourcePrivate, _numInterruptSources);
873 }
874
875 super::free();
876 }
877
878 /*
879 * Attach in service plane
880 */
881 bool
attach(IOService * provider)882 IOService::attach( IOService * provider )
883 {
884 bool ok;
885 uint32_t count;
886 AbsoluteTime deadline;
887 int waitResult = THREAD_AWAKENED;
888 bool wait, computeDeadline = true;
889
890 if (provider) {
891 if (gIOKitDebug & kIOLogAttach) {
892 LOG( "%s::attach(%s)\n", getName(),
893 provider->getName());
894 }
895
896 ok = false;
897 do{
898 wait = false;
899 provider->lockForArbitration();
900 if (provider->__state[0] & kIOServiceInactiveState) {
901 ok = false;
902 } else {
903 count = provider->getChildCount(gIOServicePlane);
904 wait = (count > (kIOServiceBusyMax - 4));
905 if (!wait) {
906 ok = attachToParent(provider, gIOServicePlane);
907 } else {
908 IOLog("stalling for detach from %s\n", provider->getName());
909 IOLockLock( gIOServiceBusyLock );
910 provider->__state[1] |= kIOServiceWaitDetachState;
911 }
912 }
913 provider->unlockForArbitration();
914 if (wait) {
915 if (computeDeadline) {
916 clock_interval_to_deadline(15, kSecondScale, &deadline);
917 computeDeadline = false;
918 }
919 assert_wait_deadline((event_t)&provider->__provider, THREAD_UNINT, deadline);
920 IOLockUnlock( gIOServiceBusyLock );
921 waitResult = thread_block(THREAD_CONTINUE_NULL);
922 wait = (waitResult != THREAD_TIMED_OUT);
923 }
924 }while (wait);
925 } else {
926 gIOServiceRoot = this;
927 ok = attachToParent( getRegistryRoot(), gIOServicePlane);
928 }
929
930 if (ok && !__provider) {
931 (void) getProvider();
932 }
933
934 return ok;
935 }
936
937 IOService *
getServiceRoot(void)938 IOService::getServiceRoot( void )
939 {
940 return gIOServiceRoot;
941 }
942
943 void
detach(IOService * provider)944 IOService::detach( IOService * provider )
945 {
946 IOService * newProvider = NULL;
947 SInt32 busy;
948 bool adjParent;
949
950 if (gIOKitDebug & kIOLogAttach) {
951 LOG("%s::detach(%s)\n", getName(), provider->getName());
952 }
953
954 #if !NO_KEXTD
955 IOLockLock(gJobsLock);
956 if (gIOMatchDeferList) {
957 auto idx = gIOMatchDeferList->getNextIndexOfObject(this, 0);
958 if (-1U != idx) {
959 gIOMatchDeferList->removeObject(idx);
960 }
961 }
962 if (IOServicePH::fMatchingDelayed) {
963 auto idx = IOServicePH::fMatchingDelayed->getNextIndexOfObject(this, 0);
964 if (-1U != idx) {
965 IOServicePH::fMatchingDelayed->removeObject(idx);
966 }
967 }
968 IOLockUnlock(gJobsLock);
969 #endif /* NO_KEXTD */
970
971 lockForArbitration();
972
973 uint64_t regID1 = provider->getRegistryEntryID();
974 uint64_t regID2 = getRegistryEntryID();
975 IOServiceTrace(
976 IOSERVICE_DETACH,
977 (uintptr_t) regID1,
978 (uintptr_t) (regID1 >> 32),
979 (uintptr_t) regID2,
980 (uintptr_t) (regID2 >> 32));
981
982 adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
983 && (provider == getProvider()));
984
985 detachFromParent( provider, gIOServicePlane );
986
987 if (busy) {
988 newProvider = getProvider();
989 if (busy && (__state[1] & kIOServiceTermPhase3State) && (NULL == newProvider)) {
990 _adjustBusy( -busy );
991 }
992 }
993
994 if (kIOServiceInactiveState & __state[0]) {
995 getMetaClass()->removeInstance(this);
996 IORemoveServicePlatformActions(this);
997 }
998
999 unlockForArbitration();
1000
1001 if (newProvider && adjParent) {
1002 newProvider->lockForArbitration();
1003 newProvider->_adjustBusy(1);
1004 newProvider->unlockForArbitration();
1005 }
1006
1007 // check for last client detach from a terminated service
1008 if (provider->lockForArbitration( true )) {
1009 if (kIOServiceStartState & __state[1]) {
1010 provider->scheduleTerminatePhase2();
1011 }
1012 if (adjParent) {
1013 provider->_adjustBusy( -1 );
1014 }
1015 if ((provider->__state[1] & kIOServiceTermPhase3State)
1016 && (NULL == provider->getClient())) {
1017 provider->scheduleFinalize(false);
1018 }
1019
1020 IOLockLock( gIOServiceBusyLock );
1021 if (kIOServiceWaitDetachState & provider->__state[1]) {
1022 provider->__state[1] &= ~kIOServiceWaitDetachState;
1023 thread_wakeup(&provider->__provider);
1024 }
1025 IOLockUnlock( gIOServiceBusyLock );
1026
1027 provider->unlockForArbitration();
1028 }
1029
1030 if (kIOServiceRematchOnDetach & __state[1]) {
1031 provider->registerService();
1032 }
1033 }
1034
1035 /*
1036 * Register instance - publish it for matching
1037 */
1038
1039 void
registerService(IOOptionBits options)1040 IOService::registerService( IOOptionBits options )
1041 {
1042 OSDataAllocation<char> pathBuf;
1043 const char * path;
1044 const char * skip;
1045 int len;
1046 enum { kMaxPathLen = 256 };
1047 enum { kMaxChars = 63 };
1048
1049 IORegistryEntry * parent = this;
1050 IORegistryEntry * root = getRegistryRoot();
1051 while (parent && (parent != root)) {
1052 parent = parent->getParentEntry( gIOServicePlane);
1053 }
1054
1055 if (parent != root) {
1056 IOLog("%s: not registry member at registerService()\n", getName());
1057 return;
1058 }
1059
1060 // Allow the Platform Expert to adjust this node.
1061 if (gIOPlatform && (!gIOPlatform->platformAdjustService(this))) {
1062 return;
1063 }
1064
1065 IOInstallServicePlatformActions(this);
1066 IOInstallServiceSleepPlatformActions(this);
1067
1068 if ((this != gIOResources)
1069 && (kIOLogRegister & gIOKitDebug)) {
1070 pathBuf = OSDataAllocation<char>( kMaxPathLen, OSAllocateMemory );
1071
1072 IOLog( "Registering: " );
1073
1074 len = kMaxPathLen;
1075 if (pathBuf && getPath( pathBuf.data(), &len, gIOServicePlane)) {
1076 path = pathBuf.data();
1077 if (len > kMaxChars) {
1078 IOLog("..");
1079 len -= kMaxChars;
1080 path += len;
1081 if ((skip = strchr( path, '/'))) {
1082 path = skip;
1083 }
1084 }
1085 } else {
1086 path = getName();
1087 }
1088
1089 IOLog( "%s\n", path );
1090 }
1091
1092 startMatching( options );
1093 }
1094
1095 void
startMatching(IOOptionBits options)1096 IOService::startMatching( IOOptionBits options )
1097 {
1098 IOService * provider;
1099 UInt32 prevBusy = 0;
1100 bool needConfig;
1101 bool needWake = false;
1102 bool sync;
1103 bool waitAgain;
1104 bool releaseAssertion = false;
1105
1106 if (options & kIOServiceDextRequirePowerForMatching) {
1107 bool ok = gIOPMRootDomain->acquireDriverKitMatchingAssertion() == kIOReturnSuccess;
1108 if (!ok) {
1109 panic("%s: Failed to acquire power assertion for matching", getName());
1110 }
1111 releaseAssertion = true;
1112 }
1113
1114 lockForArbitration();
1115
1116 sync = (options & kIOServiceSynchronous)
1117 || ((provider = getProvider())
1118 && (provider->__state[1] & kIOServiceSynchronousState));
1119
1120 if (options & kIOServiceAsynchronous) {
1121 sync = false;
1122 }
1123
1124 needConfig = (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning)))
1125 && (0 == (__state[0] & kIOServiceInactiveState));
1126
1127 __state[1] |= kIOServiceNeedConfigState;
1128
1129 // __state[0] &= ~kIOServiceInactiveState;
1130
1131 // if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
1132 // OSKernelStackRemaining(), getName());
1133
1134 if (needConfig) {
1135 needWake = (0 != (kIOServiceSyncPubState & __state[1]));
1136 }
1137
1138 if (sync) {
1139 __state[1] |= kIOServiceSynchronousState;
1140 } else {
1141 __state[1] &= ~kIOServiceSynchronousState;
1142 }
1143
1144 if (needConfig) {
1145 prevBusy = _adjustBusy( 1 );
1146 }
1147
1148 unlockForArbitration();
1149
1150 if (needConfig) {
1151 if (needWake) {
1152 IOLockLock( gIOServiceBusyLock );
1153 thread_wakeup((event_t) this /*&__state[1]*/ );
1154 IOLockUnlock( gIOServiceBusyLock );
1155 } else if (!sync || (kIOServiceAsynchronous & options)) {
1156 // assertion will be released when matching job is complete
1157 releaseAssertion = false;
1158 _IOServiceJob::startJob( this, kMatchNubJob, options );
1159 } else {
1160 do {
1161 if ((__state[1] & kIOServiceNeedConfigState)) {
1162 doServiceMatch( options );
1163 }
1164
1165 lockForArbitration();
1166 IOLockLock( gIOServiceBusyLock );
1167
1168 waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
1169 && (0 == (__state[0] & kIOServiceInactiveState)));
1170
1171 if (waitAgain) {
1172 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
1173 } else {
1174 __state[1] &= ~kIOServiceSyncPubState;
1175 }
1176
1177 unlockForArbitration();
1178
1179 if (waitAgain) {
1180 assert_wait((event_t) this /*&__state[1]*/, THREAD_UNINT);
1181 }
1182
1183 IOLockUnlock( gIOServiceBusyLock );
1184 if (waitAgain) {
1185 thread_block(THREAD_CONTINUE_NULL);
1186 }
1187 } while (waitAgain);
1188 }
1189 }
1190
1191 if (releaseAssertion) {
1192 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
1193 }
1194 }
1195
1196
1197 void
startDeferredMatches(void)1198 IOService::startDeferredMatches(void)
1199 {
1200 #if !NO_KEXTD
1201 OSArray * array;
1202
1203 IOLockLock(gJobsLock);
1204 array = gIOMatchDeferList;
1205 gIOMatchDeferList = NULL;
1206 IOLockUnlock(gJobsLock);
1207
1208 if (array) {
1209 IOLog("deferred rematching count %d\n", array->getCount());
1210 array->iterateObjects(^bool (OSObject * obj)
1211 {
1212 ((IOService *)obj)->startMatching(kIOServiceAsynchronous);
1213 return false;
1214 });
1215 array->release();
1216 }
1217 #endif /* !NO_KEXTD */
1218 }
1219
1220 void
iokitDaemonLaunched(void)1221 IOService::iokitDaemonLaunched(void)
1222 {
1223 #if !NO_KEXTD
1224 if (!OSKext::iokitDaemonAvailable()) {
1225 panic(kIOKitDaemonName " is unavailable in this environment, but it was launched");
1226 }
1227 IOServiceTrace(IOSERVICE_KEXTD_READY, 0, 0, 0, 0);
1228 startDeferredMatches();
1229 getServiceRoot()->adjustBusy(-1);
1230 IOService::publishUserResource(gIOResourceIOKitKey);
1231 #endif /* !NO_KEXTD */
1232 }
1233
1234 /*
1235 * Possibly called with IORWLock from IOCatalog held.
1236 * This means that no calls to OSKext that could take
1237 * sKextLock can be performed from this function.
1238 */
1239 IOReturn
catalogNewDrivers(OSOrderedSet * newTables)1240 IOService::catalogNewDrivers( OSOrderedSet * newTables )
1241 {
1242 OSDictionary * table;
1243 OSSet * set;
1244 OSSet * allSet = NULL;
1245 IOService * service;
1246 #if IOMATCHDEBUG
1247 SInt32 count = 0;
1248 #endif
1249
1250 newTables->retain();
1251
1252 while ((table = (OSDictionary *) newTables->getFirstObject())) {
1253 LOCKWRITENOTIFY();
1254 set = (OSSet *) copyExistingServices( table,
1255 kIOServiceRegisteredState,
1256 kIOServiceExistingSet);
1257 UNLOCKNOTIFY();
1258 if (set) {
1259 #if IOMATCHDEBUG
1260 count += set->getCount();
1261 #endif
1262 if (allSet) {
1263 allSet->merge((const OSSet *) set);
1264 set->release();
1265 } else {
1266 allSet = set;
1267 }
1268 }
1269
1270 #if IOMATCHDEBUG
1271 if (getDebugFlags( table ) & kIOLogMatch) {
1272 LOG("Matching service count = %ld\n", (long)count);
1273 }
1274 #endif
1275 newTables->removeObject(table);
1276 }
1277
1278 if (allSet) {
1279 while ((service = (IOService *) allSet->getAnyObject())) {
1280 service->startMatching(kIOServiceAsynchronous);
1281 allSet->removeObject(service);
1282 }
1283 allSet->release();
1284 }
1285
1286 newTables->release();
1287
1288 return kIOReturnSuccess;
1289 }
1290
1291 _IOServiceJob *
startJob(IOService * nub,int type,IOOptionBits options)1292 _IOServiceJob::startJob( IOService * nub, int type,
1293 IOOptionBits options )
1294 {
1295 _IOServiceJob * job;
1296
1297 job = new _IOServiceJob;
1298 if (job && !job->init()) {
1299 job->release();
1300 job = NULL;
1301 }
1302
1303 if (job) {
1304 job->type = type;
1305 job->nub = nub;
1306 job->options = options;
1307 nub->retain(); // thread will release()
1308 pingConfig( job );
1309 }
1310
1311 return job;
1312 }
1313
1314 /*
1315 * Called on a registered service to see if it matches
1316 * a property table.
1317 */
1318
1319 bool
matchPropertyTable(OSDictionary * table,SInt32 * score)1320 IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
1321 {
1322 return matchPropertyTable(table);
1323 }
1324
1325 bool
matchPropertyTable(OSDictionary * table)1326 IOService::matchPropertyTable( OSDictionary * table )
1327 {
1328 return true;
1329 }
1330
1331 /*
1332 * Called on a matched service to allocate resources
1333 * before first driver is attached.
1334 */
1335
1336 IOReturn
getResources(void)1337 IOService::getResources( void )
1338 {
1339 return kIOReturnSuccess;
1340 }
1341
1342 /*
1343 * Client/provider accessors
1344 */
1345
1346 IOService *
getProvider(void) const1347 IOService::getProvider( void ) const
1348 {
1349 IOService * self = (IOService *) this;
1350 IOService * parent;
1351 SInt32 generation;
1352
1353 generation = getRegistryEntryParentGenerationCount();
1354 if (__providerGeneration == generation) {
1355 return __provider;
1356 }
1357
1358 parent = (IOService *) getParentEntry( gIOServicePlane);
1359 if (parent == IORegistryEntry::getRegistryRoot()) {
1360 /* root is not an IOService */
1361 parent = NULL;
1362 }
1363
1364 self->__provider = parent;
1365 OSMemoryBarrier();
1366 // save the count from before call to getParentEntry()
1367 self->__providerGeneration = generation;
1368
1369 return parent;
1370 }
1371
1372 IOWorkLoop *
getWorkLoop() const1373 IOService::getWorkLoop() const
1374 {
1375 IOService *provider = getProvider();
1376
1377 if (provider) {
1378 return provider->getWorkLoop();
1379 } else {
1380 return NULL;
1381 }
1382 }
1383
1384 OSIterator *
getProviderIterator(void) const1385 IOService::getProviderIterator( void ) const
1386 {
1387 return getParentIterator( gIOServicePlane);
1388 }
1389
1390 IOService *
getClient(void) const1391 IOService::getClient( void ) const
1392 {
1393 return (IOService *) getChildEntry( gIOServicePlane);
1394 }
1395
1396 OSIterator *
getClientIterator(void) const1397 IOService::getClientIterator( void ) const
1398 {
1399 return getChildIterator( gIOServicePlane);
1400 }
1401
1402 OSIterator *
iterator(OSIterator * _iter,const IOService * client,const IOService * provider)1403 _IOOpenServiceIterator::iterator( OSIterator * _iter,
1404 const IOService * client,
1405 const IOService * provider )
1406 {
1407 _IOOpenServiceIterator * inst;
1408
1409 if (!_iter) {
1410 return NULL;
1411 }
1412
1413 inst = new _IOOpenServiceIterator;
1414
1415 if (inst && !inst->init()) {
1416 inst->release();
1417 inst = NULL;
1418 }
1419 if (inst) {
1420 inst->iter = _iter;
1421 inst->client = client;
1422 inst->provider = provider;
1423 } else {
1424 OSSafeReleaseNULL(_iter);
1425 }
1426
1427 return inst;
1428 }
1429
1430 void
free()1431 _IOOpenServiceIterator::free()
1432 {
1433 iter->release();
1434 if (last) {
1435 last->unlockForArbitration();
1436 }
1437 OSIterator::free();
1438 }
1439
1440 OSObject *
getNextObject()1441 _IOOpenServiceIterator::getNextObject()
1442 {
1443 IOService * next;
1444
1445 if (last) {
1446 last->unlockForArbitration();
1447 }
1448
1449 while ((next = (IOService *) iter->getNextObject())) {
1450 next->lockForArbitration();
1451 if ((client && (next->isOpen( client )))
1452 || (provider && (provider->isOpen( next )))) {
1453 break;
1454 }
1455 next->unlockForArbitration();
1456 }
1457
1458 last = next;
1459
1460 return next;
1461 }
1462
1463 bool
isValid()1464 _IOOpenServiceIterator::isValid()
1465 {
1466 return iter->isValid();
1467 }
1468
1469 void
reset()1470 _IOOpenServiceIterator::reset()
1471 {
1472 if (last) {
1473 last->unlockForArbitration();
1474 last = NULL;
1475 }
1476 iter->reset();
1477 }
1478
1479 OSIterator *
getOpenProviderIterator(void) const1480 IOService::getOpenProviderIterator( void ) const
1481 {
1482 return _IOOpenServiceIterator::iterator( getProviderIterator(), this, NULL );
1483 }
1484
1485 OSIterator *
getOpenClientIterator(void) const1486 IOService::getOpenClientIterator( void ) const
1487 {
1488 return _IOOpenServiceIterator::iterator( getClientIterator(), NULL, this );
1489 }
1490
1491
1492 IOReturn
callPlatformFunction(const OSSymbol * functionName,bool waitForFunction,void * param1,void * param2,void * param3,void * param4)1493 IOService::callPlatformFunction( const OSSymbol * functionName,
1494 bool waitForFunction,
1495 void *param1, void *param2,
1496 void *param3, void *param4 )
1497 {
1498 IOReturn result = kIOReturnUnsupported;
1499 IOService *provider;
1500
1501 if (functionName == gIOPlatformQuiesceActionKey ||
1502 functionName == gIOPlatformActiveActionKey ||
1503 functionName == gIOPlatformPanicActionKey) {
1504 /*
1505 * Services which register for IOPlatformQuiesceAction / IOPlatformActiveAction / IOPlatformPanicAction
1506 * must consume that event themselves, without passing it up to super/IOService.
1507 */
1508 if (gEnforcePlatformActionSafety) {
1509 panic("Class %s passed the %s action to IOService",
1510 getMetaClass()->getClassName(), functionName->getCStringNoCopy());
1511 }
1512 }
1513
1514 if (gIOPlatformFunctionHandlerSet == functionName) {
1515 const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1516 IOService * target = (IOService *) param2;
1517 bool enable = (param3 != NULL);
1518
1519 if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName) {
1520 result = setLatencyHandler(kCpuDelayBusStall, target, enable);
1521 }
1522 #if defined(__x86_64__)
1523 else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1) {
1524 result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1525 }
1526 #endif /* defined(__x86_64__) */
1527 }
1528
1529 if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1530 result = provider->callPlatformFunction(functionName, waitForFunction,
1531 param1, param2, param3, param4);
1532 }
1533
1534 return result;
1535 }
1536
1537 IOReturn
callPlatformFunction(const char * functionName,bool waitForFunction,void * param1,void * param2,void * param3,void * param4)1538 IOService::callPlatformFunction( const char * functionName,
1539 bool waitForFunction,
1540 void *param1, void *param2,
1541 void *param3, void *param4 )
1542 {
1543 IOReturn result = kIOReturnNoMemory;
1544 const OSSymbol *functionSymbol = OSSymbol::withCString(functionName);
1545
1546 if (functionSymbol != NULL) {
1547 result = callPlatformFunction(functionSymbol, waitForFunction,
1548 param1, param2, param3, param4);
1549 functionSymbol->release();
1550 }
1551
1552 return result;
1553 }
1554
1555
1556 /*
1557 * Accessors for global services
1558 */
1559
1560 IOPlatformExpert *
getPlatform(void)1561 IOService::getPlatform( void )
1562 {
1563 return gIOPlatform;
1564 }
1565
1566 class IOPMrootDomain *
getPMRootDomain(void)1567 IOService::getPMRootDomain( void )
1568 {
1569 return gIOPMRootDomain;
1570 }
1571
1572 IOService *
getResourceService(void)1573 IOService::getResourceService( void )
1574 {
1575 return gIOResources;
1576 }
1577
1578 IOService * gIOSystemStateNotificationService;
1579
1580 IOService *
getSystemStateNotificationService(void)1581 IOService::getSystemStateNotificationService(void)
1582 {
1583 return gIOSystemStateNotificationService;
1584 }
1585
1586 void
setPlatform(IOPlatformExpert * platform)1587 IOService::setPlatform( IOPlatformExpert * platform)
1588 {
1589 gIOPlatform = platform;
1590 gIOResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1591
1592 gIOUserResources->attachToParent( gIOServiceRoot, gIOServicePlane );
1593
1594 #if DEVELOPMENT || DEBUG
1595 // Test object that will be terminated for dext to match
1596 {
1597 IOService * ios;
1598 ios = OSTypeAlloc(IOService);
1599 ios->init();
1600 ios->attach(gIOUserResources);
1601 ios->setProperty(gIOMatchCategoryKey->getCStringNoCopy(), "com.apple.iokit.test");
1602 ios->setProperty(gIOModuleIdentifierKey->getCStringNoCopy(), "com.apple.kpi.iokit");
1603 ios->setProperty(gIOMatchedAtBootKey, kOSBooleanTrue);
1604 ios->setProperty(gIOPrimaryDriverTerminateOptionsKey, kOSBooleanTrue);
1605 ios->release();
1606 }
1607 #endif
1608
1609 gIOSystemStateNotificationService = IOSystemStateNotification::initialize();
1610 gIOSystemStateNotificationService->attachToParent(platform, gIOServicePlane);
1611 gIOSystemStateNotificationService->registerService();
1612
1613 static const char * keys[kCpuNumDelayTypes] = {
1614 kIOPlatformMaxBusDelay,
1615 #if defined(__x86_64__)
1616 kIOPlatformMaxInterruptDelay
1617 #endif /* defined(__x86_64__) */
1618 };
1619 const OSObject * objs[2];
1620 OSArray * array;
1621 uint32_t idx;
1622
1623 for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
1624 objs[0] = sCPULatencySet[idx];
1625 objs[1] = sCPULatencyHolder[idx];
1626 array = OSArray::withObjects(objs, 2);
1627 if (!array) {
1628 break;
1629 }
1630 platform->setProperty(keys[idx], array);
1631 array->release();
1632 }
1633 }
1634
1635 void
setPMRootDomain(class IOPMrootDomain * rootDomain)1636 IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1637 {
1638 gIOPMRootDomain = rootDomain;
1639 }
1640
1641 void
publishPMRootDomain(void)1642 IOService::publishPMRootDomain( void )
1643 {
1644 assert(getPMRootDomain() != NULL);
1645 publishResource(gIOResourceIOKitKey);
1646 #if NO_KEXTD
1647 // Publish IOUserResources now since there is no IOKit daemon.
1648 publishUserResource(gIOResourceIOKitKey);
1649 #endif
1650 IOServicePH::init(getPMRootDomain());
1651 }
1652
1653 /*
1654 * Stacking change
1655 */
1656
1657 bool
lockForArbitration(bool isSuccessRequired)1658 IOService::lockForArbitration( bool isSuccessRequired )
1659 {
1660 bool found;
1661 bool success;
1662 ArbitrationLockQueueElement * element;
1663 ArbitrationLockQueueElement * owner;
1664 ArbitrationLockQueueElement * active;
1665 ArbitrationLockQueueElement * waiting;
1666
1667 enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1668
1669 // lock global access
1670 IOTakeLock( gArbitrationLockQueueLock );
1671
1672 // obtain an unused queue element
1673 if (!queue_empty( &gArbitrationLockQueueFree )) {
1674 queue_remove_first( &gArbitrationLockQueueFree,
1675 element,
1676 ArbitrationLockQueueElement *,
1677 link );
1678 } else {
1679 element = IOMallocType(ArbitrationLockQueueElement);
1680 assert( element );
1681 }
1682
1683 // prepare the queue element
1684 element->thread = IOThreadSelf();
1685 element->service = this;
1686 element->count = 1;
1687 element->required = isSuccessRequired;
1688 element->aborted = false;
1689
1690 // determine whether this object is already locked (ie. on active queue)
1691 found = false;
1692 queue_iterate( &gArbitrationLockQueueActive,
1693 owner,
1694 ArbitrationLockQueueElement *,
1695 link )
1696 {
1697 if (owner->service == element->service) {
1698 found = true;
1699 break;
1700 }
1701 }
1702
1703 if (found) { // this object is already locked
1704 active = owner;
1705 // determine whether it is the same or a different thread trying to lock
1706 if (active->thread != element->thread) { // it is a different thread
1707 ArbitrationLockQueueElement * victim = NULL;
1708
1709 // before placing this new thread on the waiting queue, we look for
1710 // a deadlock cycle...
1711
1712 while (1) {
1713 // determine whether the active thread holding the object we
1714 // want is waiting for another object to be unlocked
1715 found = false;
1716 queue_iterate( &gArbitrationLockQueueWaiting,
1717 waiting,
1718 ArbitrationLockQueueElement *,
1719 link )
1720 {
1721 if (waiting->thread == active->thread) {
1722 assert( false == waiting->aborted );
1723 found = true;
1724 break;
1725 }
1726 }
1727
1728 if (found) { // yes, active thread waiting for another object
1729 // this may be a candidate for rejection if the required
1730 // flag is not set, should we detect a deadlock later on
1731 if (false == waiting->required) {
1732 victim = waiting;
1733 }
1734
1735 // find the thread that is holding this other object, that
1736 // is blocking the active thread from proceeding (fun :-)
1737 found = false;
1738 queue_iterate( &gArbitrationLockQueueActive,
1739 active, // (reuse active queue element)
1740 ArbitrationLockQueueElement *,
1741 link )
1742 {
1743 if (active->service == waiting->service) {
1744 found = true;
1745 break;
1746 }
1747 }
1748
1749 // someone must be holding it or it wouldn't be waiting
1750 assert( found );
1751
1752 if (active->thread == element->thread) {
1753 // doh, it's waiting for the thread that originated
1754 // this whole lock (ie. current thread) -> deadlock
1755 if (false == element->required) { // willing to fail?
1756 // the originating thread doesn't have the required
1757 // flag, so it can fail
1758 success = false; // (fail originating lock request)
1759 break; // (out of while)
1760 } else { // originating thread is not willing to fail
1761 // see if we came across a waiting thread that did
1762 // not have the 'required' flag set: we'll fail it
1763 if (victim) {
1764 // we do have a willing victim, fail it's lock
1765 victim->aborted = true;
1766
1767 // take the victim off the waiting queue
1768 queue_remove( &gArbitrationLockQueueWaiting,
1769 victim,
1770 ArbitrationLockQueueElement *,
1771 link );
1772
1773 // wake the victim
1774 wakeup_thread_with_inheritor(&victim->service->__machPortHoldDestroy, // event
1775 THREAD_AWAKENED, LCK_WAKE_DEFAULT, victim->thread);
1776
1777 // allow this thread to proceed (ie. wait)
1778 success = true; // (put request on wait queue)
1779 break; // (out of while)
1780 } else {
1781 // all the waiting threads we came across in
1782 // finding this loop had the 'required' flag
1783 // set, so we've got a deadlock we can't avoid
1784 panic("I/O Kit: Unrecoverable deadlock.");
1785 }
1786 }
1787 } else {
1788 // repeat while loop, redefining active thread to be the
1789 // thread holding "this other object" (see above), and
1790 // looking for threads waiting on it; note the active
1791 // variable points to "this other object" already... so
1792 // there nothing to do in this else clause.
1793 }
1794 } else { // no, active thread is not waiting for another object
1795 success = true; // (put request on wait queue)
1796 break; // (out of while)
1797 }
1798 } // while forever
1799
1800 if (success) { // put the request on the waiting queue?
1801 kern_return_t wait_result;
1802
1803 // place this thread on the waiting queue and put it to sleep;
1804 // we place it at the tail of the queue...
1805 queue_enter( &gArbitrationLockQueueWaiting,
1806 element,
1807 ArbitrationLockQueueElement *,
1808 link );
1809
1810 // declare that this thread will wait for a given event
1811 restart_sleep:
1812 // unlock global access
1813 // & put thread to sleep, waiting for our event to fire...
1814 wait_result = IOLockSleepWithInheritor(gArbitrationLockQueueLock,
1815 LCK_SLEEP_UNLOCK,
1816 &element->service->__machPortHoldDestroy, // event
1817 owner->thread,
1818 element->required ? THREAD_UNINT : THREAD_INTERRUPTIBLE, TIMEOUT_WAIT_FOREVER);
1819
1820 // ...and we've been woken up; we might be in one of two states:
1821 // (a) we've been aborted and our queue element is not on
1822 // any of the three queues, but is floating around
1823 // (b) we're allowed to proceed with the lock and we have
1824 // already been moved from the waiting queue to the
1825 // active queue.
1826 // ...plus a 3rd state, should the thread have been interrupted:
1827 // (c) we're still on the waiting queue
1828
1829 // determine whether we were interrupted out of our sleep
1830 if (THREAD_INTERRUPTED == wait_result) {
1831 // re-lock global access
1832 IOTakeLock( gArbitrationLockQueueLock );
1833
1834 // determine whether we're still on the waiting queue
1835 found = false;
1836 queue_iterate( &gArbitrationLockQueueWaiting,
1837 waiting, // (reuse waiting queue element)
1838 ArbitrationLockQueueElement *,
1839 link )
1840 {
1841 if (waiting == element) {
1842 found = true;
1843 break;
1844 }
1845 }
1846
1847 if (found) { // yes, we're still on the waiting queue
1848 // determine whether we're willing to fail
1849 if (false == element->required) {
1850 // mark us as aborted
1851 element->aborted = true;
1852
1853 // take us off the waiting queue
1854 queue_remove( &gArbitrationLockQueueWaiting,
1855 element,
1856 ArbitrationLockQueueElement *,
1857 link );
1858 } else { // we are not willing to fail
1859 // ignore interruption, go back to sleep
1860 goto restart_sleep;
1861 }
1862 }
1863
1864 // unlock global access
1865 IOUnlock( gArbitrationLockQueueLock );
1866
1867 // proceed as though this were a normal wake up
1868 wait_result = THREAD_AWAKENED;
1869 }
1870
1871 assert( THREAD_AWAKENED == wait_result );
1872
1873 // determine whether we've been aborted while we were asleep
1874 if (element->aborted) {
1875 assert( false == element->required );
1876 // re-lock global access
1877 IOTakeLock( gArbitrationLockQueueLock );
1878
1879 action = kPutOnFreeQueue;
1880 success = false;
1881 } else { // we weren't aborted, so we must be ready to go :-)
1882 // we've already been moved from waiting to active queue
1883 return true;
1884 }
1885 } else { // the lock request is to be failed
1886 // return unused queue element to queue
1887 action = kPutOnFreeQueue;
1888 }
1889 } else { // it is the same thread, recursive access is allowed
1890 // add one level of recursion
1891 active->count++;
1892
1893 // return unused queue element to queue
1894 action = kPutOnFreeQueue;
1895 success = true;
1896 }
1897 } else { // this object is not already locked, so let this thread through
1898 action = kPutOnActiveQueue;
1899 success = true;
1900 }
1901
1902 // put the new element on a queue
1903 if (kPutOnActiveQueue == action) {
1904 queue_enter( &gArbitrationLockQueueActive,
1905 element,
1906 ArbitrationLockQueueElement *,
1907 link );
1908 } else if (kPutOnFreeQueue == action) {
1909 queue_enter( &gArbitrationLockQueueFree,
1910 element,
1911 ArbitrationLockQueueElement *,
1912 link );
1913 } else {
1914 assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1915 }
1916
1917 // unlock global access
1918 IOUnlock( gArbitrationLockQueueLock );
1919
1920 return success;
1921 }
1922
1923 void
unlockForArbitration(void)1924 IOService::unlockForArbitration( void )
1925 {
1926 bool found;
1927 ArbitrationLockQueueElement * element;
1928
1929 // lock global access
1930 IOTakeLock( gArbitrationLockQueueLock );
1931
1932 // find the lock element for this object (ie. on active queue)
1933 found = false;
1934 queue_iterate( &gArbitrationLockQueueActive,
1935 element,
1936 ArbitrationLockQueueElement *,
1937 link )
1938 {
1939 if (element->service == this) {
1940 found = true;
1941 break;
1942 }
1943 }
1944
1945 assert( found );
1946
1947 // determine whether the lock has been taken recursively
1948 if (element->count > 1) {
1949 // undo one level of recursion
1950 element->count--;
1951 } else {
1952 // remove it from the active queue
1953 queue_remove( &gArbitrationLockQueueActive,
1954 element,
1955 ArbitrationLockQueueElement *,
1956 link );
1957
1958 // put it on the free queue
1959 queue_enter( &gArbitrationLockQueueFree,
1960 element,
1961 ArbitrationLockQueueElement *,
1962 link );
1963
1964 // determine whether a thread is waiting for object
1965 thread_t woken;
1966 kern_return_t kr =
1967 wakeup_one_with_inheritor(&__machPortHoldDestroy, // event
1968 THREAD_AWAKENED, LCK_WAKE_DEFAULT, &woken);
1969
1970 if (KERN_SUCCESS == kr) {
1971 found = false;
1972 queue_iterate( &gArbitrationLockQueueWaiting,
1973 element,
1974 ArbitrationLockQueueElement *,
1975 link )
1976 {
1977 if (element->thread == woken) {
1978 found = true;
1979 break;
1980 }
1981 }
1982 assert(found); // we found an interested thread on waiting queue
1983 // remove it from the waiting queue
1984 queue_remove( &gArbitrationLockQueueWaiting,
1985 element,
1986 ArbitrationLockQueueElement *,
1987 link );
1988
1989 // put it on the active queue
1990 queue_enter( &gArbitrationLockQueueActive,
1991 element,
1992 ArbitrationLockQueueElement *,
1993 link );
1994
1995 thread_deallocate(woken);
1996 }
1997 }
1998
1999 // unlock global access
2000 IOUnlock( gArbitrationLockQueueLock );
2001 }
2002
2003 uint32_t
isLockedForArbitration(IOService * service)2004 IOService::isLockedForArbitration(IOService * service)
2005 {
2006 #if DEBUG_NOTIFIER_LOCKED
2007 uint32_t count;
2008 ArbitrationLockQueueElement * active;
2009
2010 // lock global access
2011 IOLockLock(gArbitrationLockQueueLock);
2012
2013 // determine whether this object is already locked (ie. on active queue)
2014 count = 0;
2015 queue_iterate(&gArbitrationLockQueueActive,
2016 active,
2017 ArbitrationLockQueueElement *,
2018 link)
2019 {
2020 if ((active->thread == IOThreadSelf())
2021 && (!service || (active->service == service))) {
2022 count += 0x10000;
2023 count += active->count;
2024 }
2025 }
2026
2027 IOLockUnlock(gArbitrationLockQueueLock);
2028
2029 return count;
2030
2031 #else /* DEBUG_NOTIFIER_LOCKED */
2032
2033 return 0;
2034
2035 #endif /* DEBUG_NOTIFIER_LOCKED */
2036 }
2037
2038 void
setMachPortHoldDestroy(bool holdDestroy)2039 IOService::setMachPortHoldDestroy(bool holdDestroy)
2040 {
2041 __machPortHoldDestroy = holdDestroy;
2042 }
2043
2044 bool
machPortHoldDestroy()2045 IOService::machPortHoldDestroy()
2046 {
2047 return __machPortHoldDestroy;
2048 }
2049
2050 void
applyToProviders(IOServiceApplierFunction applier,void * context)2051 IOService::applyToProviders( IOServiceApplierFunction applier,
2052 void * context )
2053 {
2054 applyToParents((IORegistryEntryApplierFunction) applier,
2055 context, gIOServicePlane );
2056 }
2057
2058 void
applyToClients(IOServiceApplierFunction applier,void * context)2059 IOService::applyToClients( IOServiceApplierFunction applier,
2060 void * context )
2061 {
2062 applyToChildren((IORegistryEntryApplierFunction) applier,
2063 context, gIOServicePlane );
2064 }
2065
2066
2067 static void
IOServiceApplierToBlock(IOService * next,void * context)2068 IOServiceApplierToBlock(IOService * next, void * context)
2069 {
2070 IOServiceApplierBlock block = (IOServiceApplierBlock) context;
2071 block(next);
2072 }
2073
2074 void
applyToProviders(IOServiceApplierBlock applier)2075 IOService::applyToProviders(IOServiceApplierBlock applier)
2076 {
2077 applyToProviders(&IOServiceApplierToBlock, applier);
2078 }
2079
2080 void
applyToClients(IOServiceApplierBlock applier)2081 IOService::applyToClients(IOServiceApplierBlock applier)
2082 {
2083 applyToClients(&IOServiceApplierToBlock, applier);
2084 }
2085
2086 /*
2087 * Client messages
2088 */
2089
2090
2091 // send a message to a client or interested party of this service
2092 IOReturn
messageClient(UInt32 type,OSObject * client,void * argument,vm_size_t argSize)2093 IOService::messageClient( UInt32 type, OSObject * client,
2094 void * argument, vm_size_t argSize )
2095 {
2096 IOReturn ret;
2097 IOService * service;
2098 _IOServiceInterestNotifier * notify;
2099
2100 if ((service = OSDynamicCast( IOService, client))) {
2101 ret = service->message( type, this, argument );
2102 } else if ((notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
2103 _IOServiceNotifierInvocation invocation;
2104 bool willNotify;
2105
2106 invocation.thread = current_thread();
2107
2108 LOCKWRITENOTIFY();
2109 willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
2110
2111 if (willNotify) {
2112 queue_enter( ¬ify->handlerInvocations, &invocation,
2113 _IOServiceNotifierInvocation *, link );
2114 }
2115 UNLOCKNOTIFY();
2116
2117 if (willNotify) {
2118 ret = (*notify->handler)( notify->target, notify->ref,
2119 type, this, argument, argSize );
2120
2121 LOCKWRITENOTIFY();
2122 queue_remove( ¬ify->handlerInvocations, &invocation,
2123 _IOServiceNotifierInvocation *, link );
2124 if (kIOServiceNotifyWaiter & notify->state) {
2125 notify->state &= ~kIOServiceNotifyWaiter;
2126 WAKEUPNOTIFY( notify );
2127 }
2128 UNLOCKNOTIFY();
2129 } else {
2130 ret = kIOReturnSuccess;
2131 }
2132 } else {
2133 ret = kIOReturnBadArgument;
2134 }
2135
2136 return ret;
2137 }
2138
2139 static void
applyToInterestNotifiers(const IORegistryEntry * target,const OSSymbol * typeOfInterest,OSObjectApplierFunction applier,void * context)2140 applyToInterestNotifiers(const IORegistryEntry *target,
2141 const OSSymbol * typeOfInterest,
2142 OSObjectApplierFunction applier,
2143 void * context )
2144 {
2145 OSArray * copyArray = NULL;
2146 OSObject * prop;
2147
2148 LOCKREADNOTIFY();
2149
2150 prop = target->copyProperty(typeOfInterest);
2151 IOCommand *notifyList = OSDynamicCast(IOCommand, prop);
2152
2153 if (notifyList) {
2154 copyArray = OSArray::withCapacity(1);
2155
2156 // iterate over queue, entry is set to each element in the list
2157 iterqueue(¬ifyList->fCommandChain, entry) {
2158 _IOServiceInterestNotifier * notify;
2159
2160 queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
2161 copyArray->setObject(notify);
2162 }
2163 }
2164 UNLOCKNOTIFY();
2165
2166 if (copyArray) {
2167 unsigned int index;
2168 OSObject * next;
2169
2170 for (index = 0; (next = copyArray->getObject( index )); index++) {
2171 (*applier)(next, context);
2172 }
2173 copyArray->release();
2174 }
2175
2176 OSSafeReleaseNULL(prop);
2177 }
2178
2179 void
applyToInterested(const OSSymbol * typeOfInterest,OSObjectApplierFunction applier,void * context)2180 IOService::applyToInterested( const OSSymbol * typeOfInterest,
2181 OSObjectApplierFunction applier,
2182 void * context )
2183 {
2184 if (gIOGeneralInterest == typeOfInterest) {
2185 applyToClients((IOServiceApplierFunction) applier, context );
2186 }
2187 applyToInterestNotifiers(this, typeOfInterest, applier, context);
2188 }
2189
2190 struct MessageClientsContext {
2191 IOService * service;
2192 UInt32 type;
2193 void * argument;
2194 vm_size_t argSize;
2195 IOReturn ret;
2196 };
2197
2198 static void
messageClientsApplier(OSObject * object,void * ctx)2199 messageClientsApplier( OSObject * object, void * ctx )
2200 {
2201 IOReturn ret;
2202 MessageClientsContext * context = (MessageClientsContext *) ctx;
2203
2204 ret = context->service->messageClient( context->type,
2205 object, context->argument, context->argSize );
2206 if (kIOReturnSuccess != ret) {
2207 context->ret = ret;
2208 }
2209 }
2210
2211 // send a message to all clients
2212 IOReturn
messageClients(UInt32 type,void * argument,vm_size_t argSize)2213 IOService::messageClients( UInt32 type,
2214 void * argument, vm_size_t argSize )
2215 {
2216 MessageClientsContext context;
2217
2218 context.service = this;
2219 context.type = type;
2220 context.argument = argument;
2221 context.argSize = argSize;
2222 context.ret = kIOReturnSuccess;
2223
2224 applyToInterested( gIOGeneralInterest,
2225 &messageClientsApplier, &context );
2226
2227 return context.ret;
2228 }
2229
2230 IOReturn
acknowledgeNotification(IONotificationRef notification,IOOptionBits response)2231 IOService::acknowledgeNotification( IONotificationRef notification,
2232 IOOptionBits response )
2233 {
2234 return kIOReturnUnsupported;
2235 }
2236
2237 IONotifier *
registerInterest(const OSSymbol * typeOfInterest,IOServiceInterestHandler handler,void * target,void * ref)2238 IOService::registerInterest( const OSSymbol * typeOfInterest,
2239 IOServiceInterestHandler handler, void * target, void * ref )
2240 {
2241 _IOServiceInterestNotifier * notify = NULL;
2242 IOReturn rc = kIOReturnError;
2243
2244 notify = new _IOServiceInterestNotifier;
2245 if (!notify) {
2246 return NULL;
2247 }
2248
2249 if (notify->init()) {
2250 rc = registerInterestForNotifier(notify, typeOfInterest,
2251 handler, target, ref);
2252 }
2253
2254 if (rc != kIOReturnSuccess) {
2255 notify->release();
2256 notify = NULL;
2257 }
2258
2259 return notify;
2260 }
2261
2262
2263
2264 static IOReturn
IOServiceInterestHandlerToBlock(void * target __unused,void * refCon,UInt32 messageType,IOService * provider,void * messageArgument,vm_size_t argSize)2265 IOServiceInterestHandlerToBlock( void * target __unused, void * refCon,
2266 UInt32 messageType, IOService * provider,
2267 void * messageArgument, vm_size_t argSize )
2268 {
2269 return ((IOServiceInterestHandlerBlock) refCon)(messageType, provider, messageArgument, argSize);
2270 }
2271
2272 IONotifier *
registerInterest(const OSSymbol * typeOfInterest,IOServiceInterestHandlerBlock handler)2273 IOService::registerInterest(const OSSymbol * typeOfInterest,
2274 IOServiceInterestHandlerBlock handler)
2275 {
2276 IONotifier * notify;
2277 void * block;
2278
2279 block = Block_copy(handler);
2280 if (!block) {
2281 return NULL;
2282 }
2283
2284 notify = registerInterest(typeOfInterest, &IOServiceInterestHandlerToBlock, NULL, block);
2285
2286 if (!notify) {
2287 Block_release(block);
2288 }
2289
2290 return notify;
2291 }
2292
2293 IOReturn
registerInterestForNotifier(IONotifier * svcNotify,const OSSymbol * typeOfInterest,IOServiceInterestHandler handler,void * target,void * ref)2294 IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
2295 IOServiceInterestHandler handler, void * target, void * ref )
2296 {
2297 IOReturn rc = kIOReturnSuccess;
2298 _IOServiceInterestNotifier *notify = NULL;
2299
2300
2301 if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)) || !handler) {
2302 return kIOReturnBadArgument;
2303 }
2304
2305 notify->handler = handler;
2306 notify->target = target;
2307 notify->ref = ref;
2308
2309 if ((typeOfInterest != gIOGeneralInterest)
2310 && (typeOfInterest != gIOBusyInterest)
2311 && (typeOfInterest != gIOAppPowerStateInterest)
2312 && (typeOfInterest != gIOConsoleSecurityInterest)
2313 && (typeOfInterest != gIOPriorityPowerStateInterest)) {
2314 return kIOReturnBadArgument;
2315 }
2316
2317 lockForArbitration();
2318 if (0 == (__state[0] & kIOServiceInactiveState)) {
2319 notify->state = kIOServiceNotifyEnable;
2320
2321 ////// queue
2322
2323 LOCKWRITENOTIFY();
2324
2325 // Get the head of the notifier linked list
2326 IOCommand * notifyList;
2327 OSObject * obj = copyProperty( typeOfInterest );
2328 if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
2329 notifyList = OSTypeAlloc(IOCommand);
2330 if (notifyList) {
2331 notifyList->init();
2332 bool ok = setProperty( typeOfInterest, notifyList);
2333 notifyList->release();
2334 if (!ok) {
2335 notifyList = NULL;
2336 }
2337 }
2338 }
2339 if (obj) {
2340 obj->release();
2341 }
2342
2343 if (notifyList) {
2344 enqueue(¬ifyList->fCommandChain, ¬ify->chain);
2345 notify->retain(); // ref'ed while in list
2346 }
2347
2348 UNLOCKNOTIFY();
2349 } else {
2350 rc = kIOReturnNotReady;
2351 }
2352 unlockForArbitration();
2353
2354 return rc;
2355 }
2356
2357 static void
cleanInterestList(OSObject * head)2358 cleanInterestList( OSObject * head )
2359 {
2360 IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
2361 if (!notifyHead) {
2362 return;
2363 }
2364
2365 LOCKWRITENOTIFY();
2366 while (queue_entry_t entry = dequeue(¬ifyHead->fCommandChain)) {
2367 queue_next(entry) = queue_prev(entry) = NULL;
2368
2369 _IOServiceInterestNotifier * notify;
2370
2371 queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
2372 notify->release();
2373 }
2374 UNLOCKNOTIFY();
2375 }
2376
2377 void
unregisterAllInterest(void)2378 IOService::unregisterAllInterest( void )
2379 {
2380 OSObject * prop;
2381
2382 prop = copyProperty(gIOGeneralInterest);
2383 cleanInterestList(prop);
2384 OSSafeReleaseNULL(prop);
2385
2386 prop = copyProperty(gIOBusyInterest);
2387 cleanInterestList(prop);
2388 OSSafeReleaseNULL(prop);
2389
2390 prop = copyProperty(gIOAppPowerStateInterest);
2391 cleanInterestList(prop);
2392 OSSafeReleaseNULL(prop);
2393
2394 prop = copyProperty(gIOPriorityPowerStateInterest);
2395 cleanInterestList(prop);
2396 OSSafeReleaseNULL(prop);
2397
2398 prop = copyProperty(gIOConsoleSecurityInterest);
2399 cleanInterestList(prop);
2400 OSSafeReleaseNULL(prop);
2401 }
2402
2403 /*
2404 * _IOServiceInterestNotifier
2405 */
2406
2407 // wait for all threads, other than the current one,
2408 // to exit the handler
2409
2410 void
wait()2411 _IOServiceInterestNotifier::wait()
2412 {
2413 _IOServiceNotifierInvocation * next;
2414 bool doWait;
2415
2416 do {
2417 doWait = false;
2418 queue_iterate( &handlerInvocations, next,
2419 _IOServiceNotifierInvocation *, link) {
2420 if (next->thread != current_thread()) {
2421 doWait = true;
2422 break;
2423 }
2424 }
2425 if (doWait) {
2426 state |= kIOServiceNotifyWaiter;
2427 SLEEPNOTIFY(this);
2428 }
2429 } while (doWait);
2430 }
2431
2432 void
free()2433 _IOServiceInterestNotifier::free()
2434 {
2435 assert( queue_empty( &handlerInvocations ));
2436
2437 if (handler == &IOServiceInterestHandlerToBlock) {
2438 Block_release(ref);
2439 }
2440
2441 OSObject::free();
2442 }
2443
2444 void
remove()2445 _IOServiceInterestNotifier::remove()
2446 {
2447 LOCKWRITENOTIFY();
2448
2449 if (queue_next( &chain )) {
2450 remqueue(&chain);
2451 queue_next( &chain) = queue_prev( &chain) = NULL;
2452 release();
2453 }
2454
2455 state &= ~kIOServiceNotifyEnable;
2456
2457 wait();
2458
2459 UNLOCKNOTIFY();
2460
2461 release();
2462 }
2463
2464 bool
disable()2465 _IOServiceInterestNotifier::disable()
2466 {
2467 bool ret;
2468
2469 LOCKWRITENOTIFY();
2470
2471 ret = (0 != (kIOServiceNotifyEnable & state));
2472 state &= ~kIOServiceNotifyEnable;
2473 if (ret) {
2474 wait();
2475 }
2476
2477 UNLOCKNOTIFY();
2478
2479 return ret;
2480 }
2481
2482 void
enable(bool was)2483 _IOServiceInterestNotifier::enable( bool was )
2484 {
2485 LOCKWRITENOTIFY();
2486 if (was) {
2487 state |= kIOServiceNotifyEnable;
2488 } else {
2489 state &= ~kIOServiceNotifyEnable;
2490 }
2491 UNLOCKNOTIFY();
2492 }
2493
2494 bool
init()2495 _IOServiceInterestNotifier::init()
2496 {
2497 queue_init( &handlerInvocations );
2498 return OSObject::init();
2499 }
2500 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2501
2502 /*
2503 * Termination
2504 */
2505
2506 #define tailQ(o) setObject(o)
2507 #define headQ(o) setObject(0, o)
2508 #define TLOG(fmt, args...) { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
2509
2510 static void
_workLoopAction(IOWorkLoop::Action action,IOService * service,void * p0=NULL,void * p1=NULL,void * p2=NULL,void * p3=NULL)2511 _workLoopAction( IOWorkLoop::Action action,
2512 IOService * service,
2513 void * p0 = NULL, void * p1 = NULL,
2514 void * p2 = NULL, void * p3 = NULL )
2515 {
2516 IOWorkLoop * wl;
2517
2518 if ((wl = service->getWorkLoop())) {
2519 wl->retain();
2520 wl->runAction( action, service, p0, p1, p2, p3 );
2521 wl->release();
2522 } else {
2523 (*action)( service, p0, p1, p2, p3 );
2524 }
2525 }
2526
2527 bool
requestTerminate(IOService * provider,IOOptionBits options)2528 IOService::requestTerminate( IOService * provider, IOOptionBits options )
2529 {
2530 bool ok;
2531
2532 // if its our only provider
2533 ok = isParent( provider, gIOServicePlane, true);
2534
2535 // -- compat
2536 if (ok) {
2537 provider->terminateClient( this, options | kIOServiceRecursing );
2538 ok = (0 != (kIOServiceInactiveState & __state[0]));
2539 }
2540 // --
2541
2542 return ok;
2543 }
2544
2545 bool
terminatePhase1(IOOptionBits options)2546 IOService::terminatePhase1( IOOptionBits options )
2547 {
2548 IOService * victim;
2549 IOService * client;
2550 IOService * rematchProvider;
2551 OSIterator * iter;
2552 OSArray * makeInactive;
2553 OSArray * waitingInactive;
2554 IOOptionBits callerOptions;
2555 int waitResult = THREAD_AWAKENED;
2556 bool wait;
2557 bool ok;
2558 bool didInactive;
2559 bool startPhase2 = false;
2560
2561 TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
2562
2563 callerOptions = options;
2564 rematchProvider = NULL;
2565 uint64_t regID = getRegistryEntryID();
2566 IOServiceTrace(
2567 IOSERVICE_TERMINATE_PHASE1,
2568 (uintptr_t) regID,
2569 (uintptr_t) (regID >> 32),
2570 (uintptr_t) this,
2571 (uintptr_t) options);
2572
2573 // -- compat
2574 if (options & kIOServiceRecursing) {
2575 lockForArbitration();
2576 if (0 == (kIOServiceInactiveState & __state[0])) {
2577 __state[0] |= kIOServiceInactiveState;
2578 __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
2579 }
2580 unlockForArbitration();
2581
2582 return true;
2583 }
2584 // --
2585
2586 makeInactive = OSArray::withCapacity( 16 );
2587 waitingInactive = OSArray::withCapacity( 16 );
2588 if (!makeInactive || !waitingInactive) {
2589 OSSafeReleaseNULL(makeInactive);
2590 OSSafeReleaseNULL(waitingInactive);
2591 return false;
2592 }
2593
2594 victim = this;
2595 victim->retain();
2596
2597 while (victim) {
2598 didInactive = victim->lockForArbitration( true );
2599 if (didInactive) {
2600 uint64_t regID1 = victim->getRegistryEntryID();
2601 IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
2602 (uintptr_t) regID1,
2603 (uintptr_t) (regID1 >> 32),
2604 (uintptr_t) victim->__state[1],
2605 (uintptr_t) 0);
2606
2607 enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2608 didInactive = (kRP1 == (victim->__state[1] & kRP1))
2609 || (0 == (victim->__state[0] & kIOServiceInactiveState));
2610
2611 if (!didInactive) {
2612 // a multiply attached IOService can be visited twice
2613 if (-1U == waitingInactive->getNextIndexOfObject(victim, 0)) {
2614 do{
2615 IOLockLock(gIOServiceBusyLock);
2616 wait = (victim->__state[1] & kIOServiceTermPhase1State);
2617 if (wait) {
2618 TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2619 getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2620 victim->__state[1] |= kIOServiceTerm1WaiterState;
2621 victim->unlockForArbitration();
2622 assert_wait((event_t)&victim->__state[1], THREAD_UNINT);
2623 }
2624 IOLockUnlock(gIOServiceBusyLock);
2625 if (wait) {
2626 waitResult = thread_block(THREAD_CONTINUE_NULL);
2627 TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2628 getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2629 victim->lockForArbitration();
2630 }
2631 }while (wait && (waitResult != THREAD_TIMED_OUT));
2632 }
2633 } else {
2634 victim->__state[0] |= kIOServiceInactiveState;
2635 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2636 | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2637 victim->__state[1] &= ~kIOServiceRecursing;
2638 victim->__state[1] |= kIOServiceTermPhase1State;
2639 waitingInactive->headQ(victim);
2640 if (victim == this) {
2641 if (kIOServiceTerminateNeedWillTerminate & options) {
2642 victim->__state[1] |= kIOServiceNeedWillTerminate;
2643 }
2644 }
2645 victim->_adjustBusy( 1 );
2646
2647 if ((options & kIOServiceTerminateWithRematch) && (victim == this)) {
2648 if ((options & kIOServiceTerminateWithRematchCurrentDext)) {
2649 OSObject * obj;
2650 OSObject * rematchProps;
2651 OSNumber * num;
2652 uint32_t count;
2653
2654 rematchProvider = getProvider();
2655 if (rematchProvider) {
2656 obj = rematchProvider->copyProperty(gIORematchCountKey);
2657 num = OSDynamicCast(OSNumber, obj);
2658 count = 0;
2659 if (num) {
2660 count = num->unsigned32BitValue();
2661 count++;
2662 }
2663 num = OSNumber::withNumber(count, 32);
2664 rematchProvider->setProperty(gIORematchCountKey, num);
2665 rematchProps = copyProperty(gIOMatchedPersonalityKey);
2666 rematchProvider->setProperty(gIORematchPersonalityKey, rematchProps);
2667 OSSafeReleaseNULL(num);
2668 OSSafeReleaseNULL(rematchProps);
2669 OSSafeReleaseNULL(obj);
2670 }
2671 }
2672 victim->__state[1] |= kIOServiceRematchOnDetach;
2673 }
2674 }
2675 victim->unlockForArbitration();
2676 }
2677 if (victim == this) {
2678 options &= ~(kIOServiceTerminateWithRematch | kIOServiceTerminateWithRematchCurrentDext);
2679 startPhase2 = didInactive;
2680 }
2681 if (didInactive) {
2682 OSArray * notifiers;
2683 notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff);
2684 victim->invokeNotifiers(¬ifiers);
2685
2686 IOUserClient::destroyUserReferences( victim );
2687
2688 iter = victim->getClientIterator();
2689 if (iter) {
2690 while ((client = (IOService *) iter->getNextObject())) {
2691 TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2692 client->getName(), client->getRegistryEntryID(),
2693 victim->getName(), victim->getRegistryEntryID(), (long long)options);
2694 ok = client->requestTerminate( victim, options );
2695 TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2696 client->getName(), client->getRegistryEntryID(),
2697 victim->getName(), victim->getRegistryEntryID(), ok);
2698
2699 uint64_t regID1 = client->getRegistryEntryID();
2700 uint64_t regID2 = victim->getRegistryEntryID();
2701 IOServiceTrace(
2702 (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2703 : IOSERVICE_TERMINATE_REQUEST_FAIL),
2704 (uintptr_t) regID1,
2705 (uintptr_t) (regID1 >> 32),
2706 (uintptr_t) regID2,
2707 (uintptr_t) (regID2 >> 32));
2708
2709 if (ok) {
2710 makeInactive->setObject( client );
2711 }
2712 }
2713 iter->release();
2714 }
2715 }
2716 victim->release();
2717 victim = (IOService *) makeInactive->getObject(0);
2718 if (victim) {
2719 victim->retain();
2720 makeInactive->removeObject(0);
2721 }
2722 }
2723
2724 makeInactive->release();
2725
2726 while ((victim = (IOService *) waitingInactive->getObject(0))) {
2727 victim->retain();
2728 waitingInactive->removeObject(0);
2729
2730 victim->lockForArbitration();
2731 victim->__state[1] &= ~kIOServiceTermPhase1State;
2732 if (kIOServiceTerm1WaiterState & victim->__state[1]) {
2733 victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2734 TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2735 IOLockLock( gIOServiceBusyLock );
2736 thread_wakeup((event_t) &victim->__state[1]);
2737 IOLockUnlock( gIOServiceBusyLock );
2738 }
2739 victim->unlockForArbitration();
2740 victim->release();
2741 }
2742
2743 waitingInactive->release();
2744
2745 if (startPhase2) {
2746 retain();
2747 lockForArbitration();
2748 scheduleTerminatePhase2(options);
2749 unlockForArbitration();
2750 release();
2751 }
2752
2753 if (rematchProvider) {
2754 DKLOG(DKS " rematching after dext crash\n", DKN(rematchProvider));
2755 }
2756
2757 return true;
2758 }
2759
2760 void
setTerminateDefer(IOService * provider,bool defer)2761 IOService::setTerminateDefer(IOService * provider, bool defer)
2762 {
2763 lockForArbitration();
2764 if (defer) {
2765 __state[1] |= kIOServiceStartState;
2766 } else {
2767 __state[1] &= ~kIOServiceStartState;
2768 }
2769 unlockForArbitration();
2770
2771 if (provider && !defer) {
2772 provider->lockForArbitration();
2773 provider->scheduleTerminatePhase2();
2774 provider->unlockForArbitration();
2775 }
2776 }
2777
2778 // Must call this while holding gJobsLock
2779 void
waitToBecomeTerminateThread(void)2780 IOService::waitToBecomeTerminateThread(void)
2781 {
2782 IOLockAssert(gJobsLock, kIOLockAssertOwned);
2783 bool wait;
2784 do {
2785 wait = (gIOTerminateThread != THREAD_NULL);
2786 if (wait) {
2787 IOLockSleepWithInheritor(
2788 gJobsLock,
2789 LCK_SLEEP_DEFAULT,
2790 &gIOTerminateThread,
2791 gIOTerminateThread,
2792 THREAD_UNINT,
2793 TIMEOUT_WAIT_FOREVER);
2794 }
2795 } while (wait);
2796 gIOTerminateThread = current_thread();
2797 }
2798
2799 // call with lockForArbitration
2800 void
scheduleTerminatePhase2(IOOptionBits options)2801 IOService::scheduleTerminatePhase2( IOOptionBits options )
2802 {
2803 AbsoluteTime deadline;
2804 uint64_t regID1;
2805 int waitResult = THREAD_AWAKENED;
2806 bool wait = false, haveDeadline = false;
2807
2808 if (!(__state[0] & kIOServiceInactiveState)) {
2809 return;
2810 }
2811
2812 regID1 = getRegistryEntryID();
2813 IOServiceTrace(
2814 IOSERVICE_TERM_SCHED_PHASE2,
2815 (uintptr_t) regID1,
2816 (uintptr_t) (regID1 >> 32),
2817 (uintptr_t) __state[1],
2818 (uintptr_t) options);
2819
2820 if (__state[1] & kIOServiceTermPhase1State) {
2821 return;
2822 }
2823
2824 retain();
2825 unlockForArbitration();
2826 options |= kIOServiceRequired;
2827 IOLockLock( gJobsLock );
2828
2829 if ((options & kIOServiceSynchronous)
2830 && (current_thread() != gIOTerminateThread)) {
2831 waitToBecomeTerminateThread();
2832 gIOTerminatePhase2List->setObject( this );
2833 gIOTerminateWork++;
2834
2835 do {
2836 while (gIOTerminateWork) {
2837 terminateWorker( options );
2838 }
2839 wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2840 if (wait) {
2841 /* wait for the victim to go non-busy */
2842 if (!haveDeadline) {
2843 clock_interval_to_deadline( 15, kSecondScale, &deadline );
2844 haveDeadline = true;
2845 }
2846 /* let others do work while we wait */
2847 gIOTerminateThread = NULL;
2848 IOLockWakeupAllWithInheritor(gJobsLock, &gIOTerminateThread);
2849 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
2850 deadline, THREAD_UNINT );
2851 if (__improbable(waitResult == THREAD_TIMED_OUT)) {
2852 IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous): THREAD_TIMED_OUT. "
2853 "Attempting to auto-resolve your deadlock. PLEASE FIX!\n", getName(), getRegistryEntryID());
2854 }
2855 waitToBecomeTerminateThread();
2856 }
2857 } while (gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2858
2859 gIOTerminateThread = NULL;
2860 IOLockWakeupAllWithInheritor(gJobsLock, &gIOTerminateThread);
2861 } else {
2862 // ! kIOServiceSynchronous
2863
2864 gIOTerminatePhase2List->setObject( this );
2865 if (0 == gIOTerminateWork++) {
2866 assert(gIOTerminateWorkerThread);
2867 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2868 }
2869 }
2870
2871 IOLockUnlock( gJobsLock );
2872 lockForArbitration();
2873 release();
2874 }
2875
2876 __attribute__((__noreturn__))
2877 void
terminateThread(void * arg,wait_result_t waitResult)2878 IOService::terminateThread( void * arg, wait_result_t waitResult )
2879 {
2880 // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
2881 IOLockLock(gJobsLock);
2882 while (true) {
2883 if (gIOTerminateThread != gIOTerminateWorkerThread) {
2884 waitToBecomeTerminateThread();
2885 }
2886
2887 while (gIOTerminateWork) {
2888 terminateWorker((IOOptionBits)(uintptr_t)arg );
2889 }
2890
2891 gIOTerminateThread = NULL;
2892 IOLockWakeupAllWithInheritor(gJobsLock, &gIOTerminateThread);
2893 IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT);
2894 }
2895 }
2896
2897 void
scheduleStop(IOService * provider)2898 IOService::scheduleStop( IOService * provider )
2899 {
2900 uint64_t regID1 = getRegistryEntryID();
2901 uint64_t regID2 = provider->getRegistryEntryID();
2902
2903 TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2904 IOServiceTrace(
2905 IOSERVICE_TERMINATE_SCHEDULE_STOP,
2906 (uintptr_t) regID1,
2907 (uintptr_t) (regID1 >> 32),
2908 (uintptr_t) regID2,
2909 (uintptr_t) (regID2 >> 32));
2910
2911 IOLockLock( gJobsLock );
2912 gIOStopList->tailQ( this );
2913 gIOStopProviderList->tailQ( provider );
2914
2915 if (0 == gIOTerminateWork++) {
2916 assert(gIOTerminateWorkerThread);
2917 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2918 }
2919
2920 IOLockUnlock( gJobsLock );
2921 }
2922
2923 void
scheduleFinalize(bool now)2924 IOService::scheduleFinalize(bool now)
2925 {
2926 uint64_t regID1 = getRegistryEntryID();
2927
2928 TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2929 IOServiceTrace(
2930 IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2931 (uintptr_t) regID1,
2932 (uintptr_t) (regID1 >> 32),
2933 0, 0);
2934
2935 if (now || IOUserClient::finalizeUserReferences(this)) {
2936 IOLockLock( gJobsLock );
2937 gIOFinalizeList->tailQ(this);
2938 if (0 == gIOTerminateWork++) {
2939 assert(gIOTerminateWorkerThread);
2940 IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
2941 }
2942 IOLockUnlock( gJobsLock );
2943 }
2944 }
2945
2946 bool
willTerminate(IOService * provider,IOOptionBits options)2947 IOService::willTerminate( IOService * provider, IOOptionBits options )
2948 {
2949 if (reserved->uvars) {
2950 IOUserServer::serviceWillTerminate(this, provider, options);
2951 }
2952 return true;
2953 }
2954
2955 bool
didTerminate(IOService * provider,IOOptionBits options,bool * defer)2956 IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2957 {
2958 if (reserved->uvars) {
2959 IOUserServer::serviceDidTerminate(this, provider, options, defer);
2960 }
2961
2962 if (false == *defer) {
2963 if (lockForArbitration( true )) {
2964 if (false == provider->handleIsOpen( this )) {
2965 scheduleStop( provider );
2966 }
2967 // -- compat
2968 else {
2969 message( kIOMessageServiceIsRequestingClose, provider, (void *)(uintptr_t) options );
2970 if (false == provider->handleIsOpen( this )) {
2971 scheduleStop( provider );
2972 }
2973 }
2974 // --
2975 unlockForArbitration();
2976 }
2977 }
2978
2979 return true;
2980 }
2981
2982 void
actionWillTerminate(IOService * victim,IOOptionBits options,OSArray * doPhase2List,bool user,void * unused3 __unused)2983 IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2984 OSArray * doPhase2List,
2985 bool user,
2986 void *unused3 __unused)
2987 {
2988 OSIterator * iter;
2989 IOService * client;
2990 bool ok;
2991 uint64_t regID1, regID2 = victim->getRegistryEntryID();
2992
2993 iter = victim->getClientIterator();
2994 if (iter) {
2995 while ((client = (IOService *) iter->getNextObject())) {
2996 if (user != (NULL != client->reserved->uvars)) {
2997 continue;
2998 }
2999 regID1 = client->getRegistryEntryID();
3000 TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
3001 client->getName(), regID1,
3002 victim->getName(), regID2, (long long)options);
3003 IOServiceTrace(
3004 IOSERVICE_TERMINATE_WILL,
3005 (uintptr_t) regID1,
3006 (uintptr_t) (regID1 >> 32),
3007 (uintptr_t) regID2,
3008 (uintptr_t) (regID2 >> 32));
3009
3010 ok = client->willTerminate( victim, options );
3011 doPhase2List->tailQ( client );
3012 }
3013 iter->release();
3014 }
3015 }
3016
3017 void
actionDidTerminate(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3018 IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
3019 void *unused1 __unused, void *unused2 __unused,
3020 void *unused3 __unused )
3021 {
3022 OSIterator * iter;
3023 IOService * client;
3024 bool defer;
3025 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3026
3027 victim->messageClients( kIOMessageServiceIsTerminated, (void *)(uintptr_t) options );
3028
3029 iter = victim->getClientIterator();
3030 if (iter) {
3031 while ((client = (IOService *) iter->getNextObject())) {
3032 regID1 = client->getRegistryEntryID();
3033 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
3034 client->getName(), regID1,
3035 victim->getName(), regID2, (long long)options);
3036 defer = false;
3037 client->didTerminate( victim, options, &defer );
3038
3039 IOServiceTrace(
3040 (defer ? IOSERVICE_TERMINATE_DID_DEFER
3041 : IOSERVICE_TERMINATE_DID),
3042 (uintptr_t) regID1,
3043 (uintptr_t) (regID1 >> 32),
3044 (uintptr_t) regID2,
3045 (uintptr_t) (regID2 >> 32));
3046
3047 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
3048 client->getName(), regID1,
3049 victim->getName(), regID2, defer);
3050 }
3051 iter->release();
3052 }
3053 }
3054
3055
3056 void
actionWillStop(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3057 IOService::actionWillStop( IOService * victim, IOOptionBits options,
3058 void *unused1 __unused, void *unused2 __unused,
3059 void *unused3 __unused )
3060 {
3061 OSIterator * iter;
3062 IOService * provider;
3063 bool ok;
3064 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3065
3066 iter = victim->getProviderIterator();
3067 if (iter) {
3068 while ((provider = (IOService *) iter->getNextObject())) {
3069 regID1 = provider->getRegistryEntryID();
3070 TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
3071 victim->getName(), regID2,
3072 provider->getName(), regID1, (long long)options);
3073 IOServiceTrace(
3074 IOSERVICE_TERMINATE_WILL,
3075 (uintptr_t) regID2,
3076 (uintptr_t) (regID2 >> 32),
3077 (uintptr_t) regID1,
3078 (uintptr_t) (regID1 >> 32));
3079
3080 ok = victim->willTerminate( provider, options );
3081 }
3082 iter->release();
3083 }
3084 }
3085
3086 void
actionDidStop(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3087 IOService::actionDidStop( IOService * victim, IOOptionBits options,
3088 void *unused1 __unused, void *unused2 __unused,
3089 void *unused3 __unused )
3090 {
3091 OSIterator * iter;
3092 IOService * provider;
3093 bool defer = false;
3094 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3095
3096 iter = victim->getProviderIterator();
3097 if (iter) {
3098 while ((provider = (IOService *) iter->getNextObject())) {
3099 regID1 = provider->getRegistryEntryID();
3100 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
3101 victim->getName(), regID2,
3102 provider->getName(), regID1, (long long)options);
3103 victim->didTerminate( provider, options, &defer );
3104
3105 IOServiceTrace(
3106 (defer ? IOSERVICE_TERMINATE_DID_DEFER
3107 : IOSERVICE_TERMINATE_DID),
3108 (uintptr_t) regID2,
3109 (uintptr_t) (regID2 >> 32),
3110 (uintptr_t) regID1,
3111 (uintptr_t) (regID1 >> 32));
3112
3113 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
3114 victim->getName(), regID2,
3115 provider->getName(), regID1, defer);
3116 }
3117 iter->release();
3118 }
3119 }
3120
3121
3122 void
actionFinalize(IOService * victim,IOOptionBits options,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3123 IOService::actionFinalize( IOService * victim, IOOptionBits options,
3124 void *unused1 __unused, void *unused2 __unused,
3125 void *unused3 __unused )
3126 {
3127 uint64_t regID1 = victim->getRegistryEntryID();
3128 TLOG("%s[0x%qx]::finalize(%08llx)\n", victim->getName(), regID1, (long long)options);
3129 IOServiceTrace(
3130 IOSERVICE_TERMINATE_FINALIZE,
3131 (uintptr_t) regID1,
3132 (uintptr_t) (regID1 >> 32),
3133 0, 0);
3134
3135 victim->finalize( options );
3136 }
3137
3138 void
actionStop(IOService * provider,IOService * client,void * unused1 __unused,void * unused2 __unused,void * unused3 __unused)3139 IOService::actionStop( IOService * provider, IOService * client,
3140 void *unused1 __unused, void *unused2 __unused,
3141 void *unused3 __unused )
3142 {
3143 uint64_t regID1 = provider->getRegistryEntryID();
3144 uint64_t regID2 = client->getRegistryEntryID();
3145
3146 TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3147 IOServiceTrace(
3148 IOSERVICE_TERMINATE_STOP,
3149 (uintptr_t) regID1,
3150 (uintptr_t) (regID1 >> 32),
3151 (uintptr_t) regID2,
3152 (uintptr_t) (regID2 >> 32));
3153
3154 client->stop( provider );
3155 if (provider->isOpen( client )) {
3156 provider->close( client );
3157 }
3158
3159 TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3160 client->detach( provider );
3161 }
3162
3163 #pragma clang diagnostic push
3164 #pragma clang diagnostic ignored "-Wcast-function-type"
3165
3166 void
terminateWorker(IOOptionBits options)3167 IOService::terminateWorker( IOOptionBits options )
3168 {
3169 OSArray * doPhase2List;
3170 OSArray * didPhase2List;
3171 OSSet * freeList;
3172 OSIterator * iter;
3173 UInt32 workDone;
3174 IOService * victim;
3175 IOService * client;
3176 IOService * provider;
3177 unsigned int idx;
3178 bool moreToDo;
3179 bool doPhase2;
3180 bool doPhase3;
3181
3182 options |= kIOServiceRequired;
3183
3184 doPhase2List = OSArray::withCapacity( 16 );
3185 didPhase2List = OSArray::withCapacity( 16 );
3186 freeList = OSSet::withCapacity( 16 );
3187 if ((NULL == doPhase2List) || (NULL == didPhase2List) || (NULL == freeList)) {
3188 OSSafeReleaseNULL(doPhase2List);
3189 OSSafeReleaseNULL(didPhase2List);
3190 OSSafeReleaseNULL(freeList);
3191 return;
3192 }
3193
3194 do {
3195 workDone = gIOTerminateWork;
3196
3197 while ((victim = (IOService *) gIOTerminatePhase2List->getObject(0))) {
3198 victim->retain();
3199 gIOTerminatePhase2List->removeObject(0);
3200 IOLockUnlock( gJobsLock );
3201
3202 uint64_t regID1 = victim->getRegistryEntryID();
3203 IOServiceTrace(
3204 IOSERVICE_TERM_START_PHASE2,
3205 (uintptr_t) regID1,
3206 (uintptr_t) (regID1 >> 32),
3207 (uintptr_t) 0,
3208 (uintptr_t) 0);
3209
3210 while (victim) {
3211 doPhase2 = victim->lockForArbitration( true );
3212 if (doPhase2) {
3213 doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
3214 if (doPhase2) {
3215 uint64_t regID1 = victim->getRegistryEntryID();
3216 IOServiceTrace(
3217 IOSERVICE_TERM_TRY_PHASE2,
3218 (uintptr_t) regID1,
3219 (uintptr_t) (regID1 >> 32),
3220 (uintptr_t) victim->__state[1],
3221 (uintptr_t) 0);
3222
3223 doPhase2 = (0 == (victim->__state[1] &
3224 (kIOServiceTermPhase1State
3225 | kIOServiceTermPhase2State
3226 | kIOServiceConfigState)));
3227
3228 if (doPhase2 && (iter = victim->getClientIterator())) {
3229 while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
3230 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
3231 if (!doPhase2) {
3232 uint64_t regID1 = client->getRegistryEntryID();
3233 IOServiceTrace(
3234 IOSERVICE_TERM_UC_DEFER,
3235 (uintptr_t) regID1,
3236 (uintptr_t) (regID1 >> 32),
3237 (uintptr_t) client->__state[1],
3238 (uintptr_t) 0);
3239 TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
3240 victim->getName(), victim->getRegistryEntryID(),
3241 client->getName(), client->getRegistryEntryID());
3242 }
3243 }
3244 iter->release();
3245 }
3246 if (doPhase2) {
3247 victim->__state[1] |= kIOServiceTermPhase2State;
3248 }
3249 }
3250 victim->unlockForArbitration();
3251 }
3252 if (doPhase2) {
3253 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3254 if (NULL == victim->reserved->uvars) {
3255 _workLoopAction((IOWorkLoop::Action) &actionWillStop,
3256 victim, (void *)(uintptr_t) options);
3257 } else {
3258 actionWillStop(victim, options, NULL, NULL, NULL);
3259 }
3260 }
3261
3262 OSArray * notifiers;
3263 notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff);
3264 victim->invokeNotifiers(¬ifiers);
3265
3266 _workLoopAction((IOWorkLoop::Action) &actionWillTerminate,
3267 victim,
3268 (void *)(uintptr_t) options,
3269 (void *)(uintptr_t) doPhase2List,
3270 (void *)(uintptr_t) false);
3271
3272 actionWillTerminate(
3273 victim, options, doPhase2List, true, NULL);
3274
3275 didPhase2List->headQ( victim );
3276 }
3277 victim->release();
3278 victim = (IOService *) doPhase2List->getObject(0);
3279 if (victim) {
3280 victim->retain();
3281 doPhase2List->removeObject(0);
3282 }
3283 }
3284
3285 while ((victim = (IOService *) didPhase2List->getObject(0))) {
3286 bool scheduleFinalize = false;
3287 if (victim->lockForArbitration( true )) {
3288 victim->__state[1] |= kIOServiceTermPhase3State;
3289 scheduleFinalize = (NULL == victim->getClient());
3290 victim->unlockForArbitration();
3291 }
3292 _workLoopAction((IOWorkLoop::Action) &actionDidTerminate,
3293 victim, (void *)(uintptr_t) options );
3294 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3295 _workLoopAction((IOWorkLoop::Action) &actionDidStop,
3296 victim, (void *)(uintptr_t) options, NULL );
3297 }
3298 // no clients - will go to finalize
3299 if (scheduleFinalize) {
3300 victim->scheduleFinalize(false);
3301 }
3302 didPhase2List->removeObject(0);
3303 }
3304 IOLockLock( gJobsLock );
3305 }
3306
3307 // phase 3
3308 do {
3309 doPhase3 = false;
3310 // finalize leaves
3311 while ((victim = (IOService *) gIOFinalizeList->getObject(0))) {
3312 bool sendFinal = false;
3313 IOLockUnlock( gJobsLock );
3314 if (victim->lockForArbitration(true)) {
3315 sendFinal = (0 == (victim->__state[1] & kIOServiceFinalized));
3316 if (sendFinal) {
3317 victim->__state[1] |= kIOServiceFinalized;
3318 }
3319 victim->unlockForArbitration();
3320 }
3321 if (sendFinal) {
3322 _workLoopAction((IOWorkLoop::Action) &actionFinalize,
3323 victim, (void *)(uintptr_t) options );
3324 }
3325 IOLockLock( gJobsLock );
3326 // hold off free
3327 freeList->setObject( victim );
3328 // safe if finalize list is append only
3329 gIOFinalizeList->removeObject(0);
3330 }
3331
3332 for (idx = 0;
3333 (!doPhase3) && (client = (IOService *) gIOStopList->getObject(idx));) {
3334 provider = (IOService *) gIOStopProviderList->getObject(idx);
3335 assert( provider );
3336
3337 uint64_t regID1 = provider->getRegistryEntryID();
3338 uint64_t regID2 = client->getRegistryEntryID();
3339
3340 if (!provider->isChild( client, gIOServicePlane )) {
3341 // may be multiply queued - nop it
3342 TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3343 IOServiceTrace(
3344 IOSERVICE_TERMINATE_STOP_NOP,
3345 (uintptr_t) regID1,
3346 (uintptr_t) (regID1 >> 32),
3347 (uintptr_t) regID2,
3348 (uintptr_t) (regID2 >> 32));
3349 } else {
3350 // a terminated client is not ready for stop if it has clients, skip it
3351 bool deferStop = (0 != (kIOServiceInactiveState & client->__state[0]));
3352 IOLockUnlock( gJobsLock );
3353 if (deferStop && client->lockForArbitration(true)) {
3354 deferStop = (0 == (client->__state[1] & kIOServiceFinalized));
3355 //deferStop = (!deferStop && (0 != client->getClient()));
3356 //deferStop = (0 != client->getClient());
3357 client->unlockForArbitration();
3358 if (deferStop) {
3359 TLOG("%s[0x%qx]::defer stop()\n", client->getName(), regID2);
3360 IOServiceTrace(IOSERVICE_TERMINATE_STOP_DEFER,
3361 (uintptr_t) regID1,
3362 (uintptr_t) (regID1 >> 32),
3363 (uintptr_t) regID2,
3364 (uintptr_t) (regID2 >> 32));
3365
3366 idx++;
3367 IOLockLock( gJobsLock );
3368 continue;
3369 }
3370 }
3371 _workLoopAction((IOWorkLoop::Action) &actionStop,
3372 provider, (void *) client );
3373 IOLockLock( gJobsLock );
3374 // check the finalize list now
3375 doPhase3 = true;
3376 }
3377 // hold off free
3378 freeList->setObject( client );
3379 freeList->setObject( provider );
3380
3381 // safe if stop list is append only
3382 gIOStopList->removeObject( idx );
3383 gIOStopProviderList->removeObject( idx );
3384 idx = 0;
3385 }
3386 } while (doPhase3);
3387
3388 gIOTerminateWork -= workDone;
3389 moreToDo = (gIOTerminateWork != 0);
3390
3391 if (!moreToDo) {
3392 TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
3393 IOServiceTrace(
3394 IOSERVICE_TERMINATE_DONE,
3395 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
3396 }
3397 } while (moreToDo);
3398
3399 IOLockUnlock( gJobsLock );
3400
3401 freeList->release();
3402 doPhase2List->release();
3403 didPhase2List->release();
3404
3405 IOLockLock( gJobsLock );
3406 }
3407
3408 #pragma clang diagnostic pop
3409
3410 bool
finalize(IOOptionBits options)3411 IOService::finalize( IOOptionBits options )
3412 {
3413 OSIterator * iter;
3414 IOService * provider;
3415 uint64_t regID1, regID2 = getRegistryEntryID();
3416
3417 iter = getProviderIterator();
3418 assert( iter );
3419
3420 if (iter) {
3421 while ((provider = (IOService *) iter->getNextObject())) {
3422 // -- compat
3423 if (0 == (__state[1] & kIOServiceTermPhase3State)) {
3424 /* we come down here on programmatic terminate */
3425
3426 regID1 = provider->getRegistryEntryID();
3427 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
3428 IOServiceTrace(
3429 IOSERVICE_TERMINATE_STOP,
3430 (uintptr_t) regID1,
3431 (uintptr_t) (regID1 >> 32),
3432 (uintptr_t) regID2,
3433 (uintptr_t) (regID2 >> 32));
3434
3435 stop( provider );
3436 if (provider->isOpen( this )) {
3437 provider->close( this );
3438 }
3439 detach( provider );
3440 } else {
3441 //--
3442 if (provider->lockForArbitration( true )) {
3443 if (0 == (provider->__state[1] & kIOServiceTermPhase3State)) {
3444 scheduleStop( provider );
3445 }
3446 provider->unlockForArbitration();
3447 }
3448 }
3449 }
3450 iter->release();
3451 }
3452
3453 return true;
3454 }
3455
3456 #undef tailQ
3457 #undef headQ
3458
3459 /*
3460 * Terminate
3461 */
3462
3463 void
doServiceTerminate(IOOptionBits options)3464 IOService::doServiceTerminate( IOOptionBits options )
3465 {
3466 }
3467
3468 // a method in case someone needs to override it
3469 bool
terminateClient(IOService * client,IOOptionBits options)3470 IOService::terminateClient( IOService * client, IOOptionBits options )
3471 {
3472 bool ok;
3473
3474 if (client->isParent( this, gIOServicePlane, true)) {
3475 // we are the clients only provider
3476 ok = client->terminate( options );
3477 } else {
3478 ok = true;
3479 }
3480
3481 return ok;
3482 }
3483
3484 bool
terminate(IOOptionBits options)3485 IOService::terminate( IOOptionBits options )
3486 {
3487 options |= kIOServiceTerminate;
3488
3489 return terminatePhase1( options );
3490 }
3491
3492 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3493
3494 /*
3495 * Open & close
3496 */
3497
3498 struct ServiceOpenMessageContext {
3499 IOService * service;
3500 UInt32 type;
3501 IOService * excludeClient;
3502 IOOptionBits options;
3503 };
3504
3505 static void
serviceOpenMessageApplier(OSObject * object,void * ctx)3506 serviceOpenMessageApplier( OSObject * object, void * ctx )
3507 {
3508 ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
3509
3510 if (object != context->excludeClient) {
3511 context->service->messageClient( context->type, object, (void *)(uintptr_t) context->options );
3512 }
3513 }
3514
3515 bool
open(IOService * forClient,IOOptionBits options,void * arg)3516 IOService::open( IOService * forClient,
3517 IOOptionBits options,
3518 void * arg )
3519 {
3520 bool ok;
3521 kern_return_t ret = kIOReturnSuccess;
3522 ServiceOpenMessageContext context;
3523
3524 context.service = this;
3525 context.type = kIOMessageServiceIsAttemptingOpen;
3526 context.excludeClient = forClient;
3527 context.options = options;
3528
3529 applyToInterested( gIOGeneralInterest,
3530 &serviceOpenMessageApplier, &context );
3531
3532 if (false == lockForArbitration(false)) {
3533 return false;
3534 }
3535
3536 ok = (0 == (__state[0] & kIOServiceInactiveState));
3537
3538 if (ok && forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3539 ret = forClient->reserved->uvars->userServer->serviceOpen(this, forClient);
3540 if (ret != kIOReturnSuccess) {
3541 ok = false;
3542 }
3543 }
3544
3545 if (ok) {
3546 ok = handleOpen( forClient, options, arg );
3547
3548 if (!ok && forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3549 forClient->reserved->uvars->userServer->serviceClose(this, forClient);
3550 }
3551 }
3552
3553 unlockForArbitration();
3554
3555 return ok;
3556 }
3557
3558 void
close(IOService * forClient,IOOptionBits options)3559 IOService::close( IOService * forClient,
3560 IOOptionBits options )
3561 {
3562 bool wasClosed;
3563 bool last = false;
3564
3565 lockForArbitration();
3566
3567 wasClosed = handleIsOpen( forClient );
3568 if (wasClosed) {
3569 handleClose( forClient, options );
3570 last = (__state[1] & kIOServiceTermPhase3State);
3571
3572 if (forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3573 forClient->reserved->uvars->userServer->serviceClose(this, forClient);
3574 }
3575 }
3576
3577 unlockForArbitration();
3578
3579 if (last) {
3580 forClient->scheduleStop( this );
3581 } else if (wasClosed) {
3582 ServiceOpenMessageContext context;
3583
3584 context.service = this;
3585 context.type = kIOMessageServiceWasClosed;
3586 context.excludeClient = forClient;
3587 context.options = options;
3588
3589 applyToInterested( gIOGeneralInterest,
3590 &serviceOpenMessageApplier, &context );
3591 }
3592 }
3593
3594 bool
isOpen(const IOService * forClient) const3595 IOService::isOpen( const IOService * forClient ) const
3596 {
3597 IOService * self = (IOService *) this;
3598 bool ok;
3599
3600 self->lockForArbitration();
3601
3602 ok = handleIsOpen( forClient );
3603
3604 self->unlockForArbitration();
3605
3606 return ok;
3607 }
3608
3609 bool
handleOpen(IOService * forClient,IOOptionBits options,void * arg)3610 IOService::handleOpen( IOService * forClient,
3611 IOOptionBits options,
3612 void * arg )
3613 {
3614 bool ok;
3615
3616 ok = (NULL == __owner);
3617 if (ok) {
3618 __owner = forClient;
3619 } else if (options & kIOServiceSeize) {
3620 ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
3621 __owner, (void *)(uintptr_t) options ));
3622 if (ok && (NULL == __owner)) {
3623 __owner = forClient;
3624 } else {
3625 ok = false;
3626 }
3627 }
3628 return ok;
3629 }
3630
3631 void
handleClose(IOService * forClient,IOOptionBits options)3632 IOService::handleClose( IOService * forClient,
3633 IOOptionBits options )
3634 {
3635 if (__owner == forClient) {
3636 __owner = NULL;
3637 }
3638 }
3639
3640 bool
handleIsOpen(const IOService * forClient) const3641 IOService::handleIsOpen( const IOService * forClient ) const
3642 {
3643 if (forClient) {
3644 return __owner == forClient;
3645 } else {
3646 return __owner != forClient;
3647 }
3648 }
3649
3650 /*
3651 * Probing & starting
3652 */
3653 static SInt32
IONotifyOrdering(const OSMetaClassBase * inObj1,const OSMetaClassBase * inObj2,void * ref)3654 IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3655 {
3656 const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
3657 const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
3658 SInt32 val1;
3659 SInt32 val2;
3660
3661 val1 = 0;
3662 val2 = 0;
3663 if (obj1) {
3664 val1 = obj1->priority;
3665 }
3666 if (obj2) {
3667 val2 = obj2->priority;
3668 }
3669 if (val1 > val2) {
3670 return 1;
3671 }
3672 if (val1 < val2) {
3673 return -1;
3674 }
3675 return 0;
3676 }
3677
3678 static SInt32
IOServiceObjectOrder(const OSObject * entry,void * ref)3679 IOServiceObjectOrder( const OSObject * entry, void * ref)
3680 {
3681 OSDictionary * dict;
3682 IOService * service;
3683 _IOServiceNotifier * notify;
3684 OSSymbol * key = (OSSymbol *) ref;
3685 OSNumber * offset;
3686 OSObject * prop;
3687 SInt32 result;
3688
3689 prop = NULL;
3690 result = kIODefaultProbeScore;
3691 if ((dict = OSDynamicCast( OSDictionary, entry))) {
3692 offset = OSDynamicCast(OSNumber, dict->getObject( key ));
3693 } else if ((notify = OSDynamicCast( _IOServiceNotifier, entry))) {
3694 return notify->priority;
3695 } else if ((service = OSDynamicCast( IOService, entry))) {
3696 prop = service->copyProperty(key);
3697 offset = OSDynamicCast(OSNumber, prop);
3698 } else {
3699 assert( false );
3700 offset = NULL;
3701 }
3702
3703 if (offset) {
3704 result = offset->unsigned32BitValue();
3705 }
3706
3707 OSSafeReleaseNULL(prop);
3708
3709 return result;
3710 }
3711
3712 __attribute__((no_sanitize("signed-integer-overflow"))) SInt32
IOServiceOrdering(const OSMetaClassBase * inObj1,const OSMetaClassBase * inObj2,void * ref)3713 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3714 {
3715 const OSObject * obj1 = (const OSObject *) inObj1;
3716 const OSObject * obj2 = (const OSObject *) inObj2;
3717 SInt32 val1;
3718 SInt32 val2;
3719
3720 val1 = 0;
3721 val2 = 0;
3722
3723 if (obj1) {
3724 val1 = IOServiceObjectOrder( obj1, ref );
3725 }
3726
3727 if (obj2) {
3728 val2 = IOServiceObjectOrder( obj2, ref );
3729 }
3730
3731 return val1 - val2;
3732 }
3733
3734 IOService *
copyClientWithCategory(const OSSymbol * category)3735 IOService::copyClientWithCategory( const OSSymbol * category )
3736 {
3737 IOService * service = NULL;
3738 OSIterator * iter;
3739 const OSSymbol * nextCat;
3740
3741 iter = getClientIterator();
3742 if (iter) {
3743 while ((service = (IOService *) iter->getNextObject())) {
3744 if (kIOServiceInactiveState & service->__state[0]) {
3745 if (!(kIOServiceRematchOnDetach & service->__state[1])) {
3746 continue;
3747 }
3748 }
3749 nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
3750 service->getProperty( gIOMatchCategoryKey ));
3751 if (category == nextCat) {
3752 service->retain();
3753 break;
3754 }
3755 }
3756 iter->release();
3757 }
3758 return service;
3759 }
3760
3761 IOService *
getClientWithCategory(const OSSymbol * category)3762 IOService::getClientWithCategory( const OSSymbol * category )
3763 {
3764 IOService *
3765 service = copyClientWithCategory(category);
3766 if (service) {
3767 service->release();
3768 }
3769 return service;
3770 }
3771
3772 bool
invokeNotifier(_IOServiceNotifier * notify)3773 IOService::invokeNotifier( _IOServiceNotifier * notify )
3774 {
3775 _IOServiceNotifierInvocation invocation;
3776 bool willNotify;
3777 bool ret = true;
3778 invocation.thread = current_thread();
3779
3780 #if DEBUG_NOTIFIER_LOCKED
3781 uint32_t count;
3782 if ((count = isLockedForArbitration(0))) {
3783 IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3784 panic("[%s, 0x%x]", notify->type->getCStringNoCopy(), count);
3785 }
3786 #endif /* DEBUG_NOTIFIER_LOCKED */
3787
3788 LOCKWRITENOTIFY();
3789 willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3790
3791 if (willNotify) {
3792 queue_enter( ¬ify->handlerInvocations, &invocation,
3793 _IOServiceNotifierInvocation *, link );
3794 }
3795 UNLOCKNOTIFY();
3796
3797 if (willNotify) {
3798 ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3799
3800 LOCKWRITENOTIFY();
3801 queue_remove( ¬ify->handlerInvocations, &invocation,
3802 _IOServiceNotifierInvocation *, link );
3803 if (kIOServiceNotifyWaiter & notify->state) {
3804 notify->state &= ~kIOServiceNotifyWaiter;
3805 WAKEUPNOTIFY( notify );
3806 }
3807 UNLOCKNOTIFY();
3808 }
3809
3810 return ret;
3811 }
3812
3813 bool
invokeNotifiers(OSArray * willSend[])3814 IOService::invokeNotifiers(OSArray * willSend[])
3815 {
3816 OSArray * array;
3817 _IOServiceNotifier * notify;
3818 bool ret = true;
3819
3820 array = *willSend;
3821 if (!array) {
3822 return true;
3823 }
3824 *willSend = NULL;
3825
3826 for (unsigned int idx = 0;
3827 (notify = (_IOServiceNotifier *) array->getObject(idx));
3828 idx++) {
3829 ret &= invokeNotifier(notify);
3830 }
3831 array->release();
3832
3833 return ret;
3834 }
3835
3836
3837 TUNABLE(bool, iokit_print_verbose_match_logs, "iokit_print_verbose_match_logs", false);
3838
3839 /*
3840 * Alloc and probe matching classes,
3841 * called on the provider instance
3842 */
3843
3844 void
probeCandidates(OSOrderedSet * matches)3845 IOService::probeCandidates( OSOrderedSet * matches )
3846 {
3847 OSDictionary * match = NULL;
3848 OSSymbol * symbol;
3849 IOService * inst;
3850 IOService * newInst;
3851 OSDictionary * props;
3852 SInt32 score;
3853 OSNumber * newPri;
3854 OSOrderedSet * familyMatches = NULL;
3855 OSOrderedSet * startList;
3856 OSSet * kexts = NULL;
3857 OSObject * kextRef;
3858
3859 OSDictionary * startDict = NULL;
3860 const OSSymbol * category;
3861 OSIterator * iter;
3862 _IOServiceNotifier * notify;
3863 OSObject * nextMatch = NULL;
3864 bool started;
3865 bool needReloc = false;
3866 bool matchDeferred = false;
3867 #if IOMATCHDEBUG
3868 SInt64 debugFlags;
3869 #endif
3870 IOService * client = NULL;
3871 OSObject * prop1;
3872 OSObject * rematchCountProp;
3873 OSDictionary * rematchPersonality;
3874 OSNumber * num;
3875 uint32_t count;
3876 uint32_t dextCount;
3877 bool isDext;
3878 bool categoryConsumed;
3879
3880 rematchCountProp = NULL;
3881 count = 0;
3882 prop1 = copyProperty(gIORematchPersonalityKey);
3883 rematchPersonality = OSDynamicCast(OSDictionary, prop1);
3884 if (rematchPersonality) {
3885 rematchCountProp = copyProperty(gIORematchCountKey);
3886 num = OSDynamicCast(OSNumber, rematchCountProp);
3887 if (num) {
3888 count = num->unsigned32BitValue();
3889 }
3890 removeProperty(gIORematchPersonalityKey);
3891 }
3892 dextCount = 0;
3893
3894 assert( matches );
3895 while (!needReloc
3896 && (nextMatch = matches->getFirstObject())) {
3897 nextMatch->retain();
3898 matches->removeObject(nextMatch);
3899
3900 if ((notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3901 if (0 == (__state[0] & kIOServiceInactiveState)) {
3902 invokeNotifier( notify );
3903 }
3904 nextMatch->release();
3905 nextMatch = NULL;
3906 continue;
3907 } else if (!(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3908 nextMatch->release();
3909 nextMatch = NULL;
3910 continue;
3911 }
3912
3913 props = NULL;
3914 #if IOMATCHDEBUG
3915 debugFlags = getDebugFlags( match );
3916 #endif
3917
3918 bool newIsBoot = false;
3919 bool existingIsBoot = false;
3920 bool isReplacementCandidate = false;
3921
3922 do {
3923 client = NULL;
3924 isDext = (NULL != match->getObject(gIOUserServerNameKey));
3925 if (isDext && !(kIODKEnable & gIODKDebug)) {
3926 continue;
3927 }
3928 if (isDext && !OSKext::iokitDaemonAvailable()) {
3929 continue;
3930 }
3931 if (isDext && !gIODextRelaunchMax && rematchCountProp) {
3932 continue;
3933 }
3934 if (isDext && isInactive()) {
3935 continue;
3936 }
3937 newIsBoot = gIOCatalogue->personalityIsBoot(match);
3938
3939 category = OSDynamicCast( OSSymbol,
3940 match->getObject( gIOMatchCategoryKey ));
3941 if (NULL == category) {
3942 category = gIODefaultMatchCategoryKey;
3943 }
3944 client = copyClientWithCategory(category);
3945
3946 categoryConsumed = (client != NULL);
3947 if (categoryConsumed) {
3948 #if IOMATCHDEBUG
3949 if ((debugFlags & kIOLogMatch) && (this != gIOResources)) {
3950 LOG("%s: match category %s exists\n", getName(),
3951 category->getCStringNoCopy());
3952 }
3953 #endif
3954 existingIsBoot = client->propertyExists(gIOMatchedAtBootKey);
3955 isReplacementCandidate = existingIsBoot && !newIsBoot;
3956 if (!isDext && !isReplacementCandidate) {
3957 break;
3958 }
3959 }
3960
3961 // create a copy now in case its modified during matching
3962 props = OSDictionary::withDictionary(match, match->getCount());
3963 if (NULL == props) {
3964 break;
3965 }
3966 props->setCapacityIncrement(1);
3967
3968 // check the nub matches
3969 if (false == matchPassive(props, kIOServiceChangesOK | kIOServiceClassDone)) {
3970 break;
3971 }
3972 if (isReplacementCandidate) {
3973 if (canTerminateForReplacement(client)) {
3974 client->terminate(kIOServiceTerminateNeedWillTerminate | kIOServiceTerminateWithRematch);
3975 break;
3976 }
3977 }
3978
3979 if (isDext || isReplacementCandidate) {
3980 if (isDext) {
3981 dextCount++;
3982 }
3983 if (categoryConsumed) {
3984 break;
3985 }
3986 }
3987 if (rematchPersonality) {
3988 bool personalityMatch = match->isEqualTo(rematchPersonality);
3989 if (count > gIODextRelaunchMax) {
3990 personalityMatch = !personalityMatch;
3991 }
3992 if (!personalityMatch) {
3993 break;
3994 }
3995 }
3996
3997 // Check to see if driver reloc has been loaded.
3998 needReloc = (false == gIOCatalogue->isModuleLoaded( match, &kextRef ));
3999 if (needReloc) {
4000 #if IOMATCHDEBUG
4001 if (debugFlags & kIOLogCatalogue) {
4002 LOG("%s: stalling for module\n", getName());
4003 }
4004 #endif
4005 // If reloc hasn't been loaded, exit;
4006 // reprobing will occur after reloc has been loaded.
4007 break;
4008 }
4009 if (kextRef) {
4010 if (NULL == kexts) {
4011 kexts = OSSet::withCapacity(1);
4012 }
4013 if (kexts) {
4014 kexts->setObject(kextRef);
4015 kextRef->release();
4016 }
4017 }
4018 if (newIsBoot) {
4019 props->setObject(gIOMatchedAtBootKey, kOSBooleanTrue);
4020 }
4021 if (isDext) {
4022 // copy saved for rematchng
4023 props->setObject(gIOMatchedPersonalityKey, match);
4024 }
4025 // reorder on family matchPropertyTable score.
4026 if (NULL == familyMatches) {
4027 familyMatches = OSOrderedSet::withCapacity( 1,
4028 IOServiceOrdering, (void *) gIOProbeScoreKey );
4029 }
4030 if (familyMatches) {
4031 familyMatches->setObject( props );
4032 }
4033 } while (false);
4034
4035 OSSafeReleaseNULL(client);
4036 OSSafeReleaseNULL(nextMatch);
4037 OSSafeReleaseNULL(props);
4038 }
4039 OSSafeReleaseNULL(matches);
4040 OSSafeReleaseNULL(rematchCountProp);
4041
4042 if (familyMatches) {
4043 while (!needReloc
4044 && (props = (OSDictionary *) familyMatches->getFirstObject())) {
4045 props->retain();
4046 familyMatches->removeObject( props );
4047
4048 inst = NULL;
4049 newInst = NULL;
4050 #if IOMATCHDEBUG
4051 debugFlags = getDebugFlags( props );
4052 #endif
4053 do {
4054 symbol = OSDynamicCast( OSSymbol,
4055 props->getObject( gIOClassKey));
4056 if (!symbol) {
4057 continue;
4058 }
4059
4060 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props));
4061
4062 // alloc the driver instance
4063 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
4064
4065 if (!inst || !OSDynamicCast(IOService, inst)) {
4066 IOLog("Couldn't alloc class \"%s\"\n",
4067 symbol->getCStringNoCopy());
4068 continue;
4069 }
4070
4071 // init driver instance
4072 if (!(inst->init( props ))) {
4073 #if IOMATCHDEBUG
4074 if (debugFlags & kIOLogStart) {
4075 IOLog("%s::init fails\n", symbol->getCStringNoCopy());
4076 }
4077 #endif
4078 continue;
4079 }
4080 if (__state[1] & kIOServiceSynchronousState) {
4081 inst->__state[1] |= kIOServiceSynchronousState;
4082 }
4083
4084 // give the driver the default match category if not specified
4085 category = OSDynamicCast( OSSymbol,
4086 props->getObject( gIOMatchCategoryKey ));
4087 if (NULL == category) {
4088 category = gIODefaultMatchCategoryKey;
4089 }
4090 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
4091 // attach driver instance
4092 if (!(inst->attach( this ))) {
4093 continue;
4094 }
4095
4096 // pass in score from property table
4097 score = familyMatches->orderObject( props );
4098
4099 // & probe the new driver instance
4100 #if IOMATCHDEBUG
4101 if (debugFlags & kIOLogProbe) {
4102 LOG("%s::probe(%s)\n",
4103 inst->getMetaClass()->getClassName(), getName());
4104 }
4105 #endif
4106 newInst = inst->probe( this, &score );
4107 inst->detach( this );
4108 if (NULL == newInst) {
4109 #if IOMATCHDEBUG
4110 if (debugFlags & kIOLogProbe) {
4111 IOLog("%s::probe fails\n", symbol->getCStringNoCopy());
4112 }
4113 #endif
4114 continue;
4115 }
4116
4117 // save the score
4118 newPri = OSNumber::withNumber( score, 32 );
4119 if (newPri) {
4120 newInst->setProperty( gIOProbeScoreKey, newPri );
4121 newPri->release();
4122 }
4123
4124 // add to start list for the match category
4125 if (NULL == startDict) {
4126 startDict = OSDictionary::withCapacity( 1 );
4127 }
4128 assert( startDict );
4129 startList = (OSOrderedSet *)
4130 startDict->getObject( category );
4131 if (NULL == startList) {
4132 startList = OSOrderedSet::withCapacity( 1,
4133 IOServiceOrdering, (void *) gIOProbeScoreKey );
4134 if (startDict && startList) {
4135 startDict->setObject( category, startList );
4136 startList->release();
4137 }
4138 }
4139 assert( startList );
4140 if (startList) {
4141 startList->setObject( newInst );
4142 }
4143 } while (false);
4144
4145 props->release();
4146 if (inst) {
4147 inst->release();
4148 }
4149 }
4150 familyMatches->release();
4151 familyMatches = NULL;
4152 }
4153
4154 if ((debugFlags & kIOLogMatch) && iokit_print_verbose_match_logs && startDict != NULL) {
4155 IOLog("%s(0x%qx): %u categories\n", getName(), getRegistryEntryID(), startList->getCount());
4156 startDict->iterateObjects(^(const OSSymbol *key, OSObject *value) {
4157 OSOrderedSet *startList = OSDynamicCast(OSOrderedSet, value);
4158 if (startList) {
4159 IOLog("%s(0x%qx): category %s, %u matches\n", getName(), getRegistryEntryID(), key->getCStringNoCopy(), startList->getCount());
4160 startList->iterateObjects(^(OSObject *obj) {
4161 IOService *match = OSDynamicCast(IOService, obj);
4162 OSNumber *probeScore = OSDynamicCast(OSNumber, match->getProperty(gIOProbeScoreKey));
4163
4164 if (match && probeScore) {
4165 IOLog("%s(0x%qx): category %s: matched %s, probe score %qd\n", getName(), getRegistryEntryID(), key->getCStringNoCopy(), match->getName(), probeScore->unsigned64BitValue());
4166 }
4167 return false;
4168 });
4169 }
4170 return false;
4171 });
4172 }
4173
4174 // start the best (until success) of each category
4175
4176 iter = OSCollectionIterator::withCollection( startDict );
4177 assert(startDict || !iter);
4178 if (iter) {
4179 while ((category = (const OSSymbol *) iter->getNextObject())) {
4180 startList = (OSOrderedSet *) startDict->getObject( category );
4181 assert( startList );
4182 if (!startList) {
4183 continue;
4184 }
4185 started = false;
4186 while (true // (!started)
4187 && !matchDeferred
4188 && (inst = (IOService *)startList->getFirstObject())) {
4189 inst->retain();
4190 startList->removeObject(inst);
4191 #if IOMATCHDEBUG
4192 debugFlags = getDebugFlags( inst );
4193
4194 if (debugFlags & kIOLogStart) {
4195 if (started) {
4196 LOG( "match category exists, skipping " );
4197 }
4198 LOG( "%s::start(%s) <%d>\n", inst->getName(),
4199 getName(), inst->getRetainCount());
4200 }
4201 #endif
4202 if (false == started) {
4203 #if !NO_KEXTD
4204 IOLockLock(gJobsLock);
4205 matchDeferred = (gIOMatchDeferList
4206 && kOSBooleanTrue == inst->getProperty(gIOMatchDeferKey));
4207 if (matchDeferred && (-1U == gIOMatchDeferList->getNextIndexOfObject(this, 0))) {
4208 gIOMatchDeferList->setObject(this);
4209 }
4210 if (matchDeferred) {
4211 symbol = OSDynamicCast(OSSymbol, inst->getProperty(gIOClassKey));
4212 IOLog("%s(0x%qx): matching deferred by %s%s\n",
4213 getName(), getRegistryEntryID(),
4214 symbol ? symbol->getCStringNoCopy() : "",
4215 gInUserspaceReboot ? " in userspace reboot" : "");
4216 // rematching will occur after the IOKit daemon loads all plists
4217 }
4218 IOLockUnlock(gJobsLock);
4219 #endif
4220 if (!matchDeferred) {
4221 /* TODO
4222 * If a dext fails to start because an upgrade happened
4223 * concurrently, then the matching process has to restart
4224 */
4225 started = startCandidate( inst );
4226 #if IOMATCHDEBUG
4227 if ((debugFlags & kIOLogStart) && (false == started)) {
4228 LOG( "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
4229 inst->getRetainCount());
4230 }
4231 #endif
4232 if (!started && inst->propertyExists(gIOServiceMatchDeferredKey)) {
4233 matchDeferred = true;
4234 }
4235 }
4236 }
4237 inst->release();
4238 }
4239 }
4240 iter->release();
4241 }
4242
4243 OSSafeReleaseNULL(prop1);
4244
4245 if (dextCount) {
4246 num = OSNumber::withNumber(dextCount, 32);
4247 setProperty(gIODEXTMatchCountKey, num);
4248 OSSafeReleaseNULL(num);
4249 } else if (rematchPersonality) {
4250 removeProperty(gIODEXTMatchCountKey);
4251 }
4252
4253 // now that instances are created, drop the refs on any kexts allowing unload
4254 if (kexts) {
4255 OSKext::dropMatchingReferences(kexts);
4256 OSSafeReleaseNULL(kexts);
4257 }
4258
4259 // adjust the busy count by +1 if matching is stalled for a module,
4260 // or -1 if a previously stalled matching is complete.
4261 lockForArbitration();
4262 SInt32 adjBusy = 0;
4263 uint64_t regID = getRegistryEntryID();
4264
4265 if (needReloc) {
4266 adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
4267 if (adjBusy) {
4268 IOServiceTrace(
4269 IOSERVICE_MODULESTALL,
4270 (uintptr_t) regID,
4271 (uintptr_t) (regID >> 32),
4272 (uintptr_t) this,
4273 0);
4274
4275 __state[1] |= kIOServiceModuleStallState;
4276 }
4277 } else if (__state[1] & kIOServiceModuleStallState) {
4278 IOServiceTrace(
4279 IOSERVICE_MODULEUNSTALL,
4280 (uintptr_t) regID,
4281 (uintptr_t) (regID >> 32),
4282 (uintptr_t) this,
4283 0);
4284
4285 __state[1] &= ~kIOServiceModuleStallState;
4286 adjBusy = -1;
4287 }
4288 if (adjBusy) {
4289 _adjustBusy( adjBusy );
4290 }
4291 unlockForArbitration();
4292
4293 if (startDict) {
4294 startDict->release();
4295 }
4296 }
4297
4298 /*
4299 * Wait for a IOUserServer to check in
4300 */
4301
4302 static
4303 __attribute__((noinline, not_tail_called))
4304 IOUserServer *
__WAITING_FOR_USER_SERVER__(IOUserServerCheckInToken * token)4305 __WAITING_FOR_USER_SERVER__(IOUserServerCheckInToken * token)
4306 {
4307 IOUserServer * result = NULL;
4308 IOService * server = NULL;
4309 const OSSymbol * serverName = token->copyServerName();
4310 OSNumber * serverTag = token->copyServerTag();
4311 OSDictionary * matching = IOService::serviceMatching(gIOUserServerClassKey);
4312
4313 if (!matching || !serverName || !serverTag) {
4314 goto finish;
4315 }
4316 IOService::propertyMatching(gIOUserServerNameKey, serverName, matching);
4317 if (!(kIODKDisableDextTag & gIODKDebug)) {
4318 IOService::propertyMatching(gIOUserServerTagKey, serverTag, matching);
4319 }
4320
4321 server = IOService::waitForMatchingServiceWithToken(matching, kIOUserServerCheckInTimeoutSecs * NSEC_PER_SEC, token);
4322 result = OSDynamicCast(IOUserServer, server);
4323 if (!result) {
4324 OSSafeReleaseNULL(server);
4325 token->cancel();
4326 }
4327
4328 finish:
4329 OSSafeReleaseNULL(matching);
4330 OSSafeReleaseNULL(serverName);
4331 OSSafeReleaseNULL(serverTag);
4332
4333 return result;
4334 }
4335
4336 void
willShutdown()4337 IOService::willShutdown()
4338 {
4339 gIOKitWillTerminate = true;
4340 #if !NO_KEXTD
4341 IOUserServerCheckInToken::cancelAll();
4342 #endif
4343 OSKext::willShutdown();
4344 }
4345
4346 void
userSpaceWillReboot()4347 IOService::userSpaceWillReboot()
4348 {
4349 IOLockLock(gJobsLock);
4350 #if !NO_KEXTD
4351 IOService * provider;
4352 IOService * service;
4353 OSIterator * iter;
4354
4355 // Recreate the defer list if it does not exist
4356 if (!gIOMatchDeferList && OSKext::iokitDaemonAvailable()) {
4357 gIOMatchDeferList = OSArray::withCapacity( 16 );
4358 }
4359
4360 if (gIOMatchDeferList) {
4361 iter = IORegistryIterator::iterateOver(gIOServicePlane, kIORegistryIterateRecursively);
4362 if (iter) {
4363 do {
4364 iter->reset();
4365 while ((service = (IOService *)iter->getNextObject())) {
4366 /* Rematch providers of services that will be terminated on userspace reboot, after the userspace reboot
4367 * is complete. This normally happens automatically as the IOKit daemon sends personalities to the kernel
4368 * which triggers rematching. But if this doesn't happen (for example, if a feature flag is turned off),
4369 * then these services will never get rematched.
4370 */
4371 if (service->propertyHasValue(gIOMatchDeferKey, kOSBooleanTrue) || service->hasUserServer()) {
4372 provider = service->getProvider();
4373 IOLog("deferring %s-%llx (provider of %s-%llx) matching after userspace reboot\n",
4374 provider->getName(), provider->getRegistryEntryID(), service->getName(), service->getRegistryEntryID());
4375 gIOMatchDeferList->setObject(provider);
4376 }
4377 }
4378 } while (!service && !iter->isValid());
4379
4380 OSSafeReleaseNULL(iter);
4381 }
4382 }
4383 #endif
4384 gInUserspaceReboot = true;
4385 IOLockUnlock(gJobsLock);
4386 }
4387
4388 void
userSpaceDidReboot()4389 IOService::userSpaceDidReboot()
4390 {
4391 IOLockLock(gJobsLock);
4392 gInUserspaceReboot = false;
4393 IOLockUnlock(gJobsLock);
4394 }
4395
4396 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4397
4398 void
init(IOPMrootDomain * root)4399 IOServicePH::init(IOPMrootDomain * root)
4400 {
4401 fUserServers = OSArray::withCapacity(4);
4402 fMatchingWork = OSArray::withCapacity(4);
4403
4404 assert(fUserServers && fMatchingWork);
4405
4406 fRootNotifier = root->registerInterest(
4407 gIOPriorityPowerStateInterest, &IOServicePH::systemPowerChange, NULL, NULL);
4408
4409 assert(fRootNotifier);
4410
4411 fUserServerAckTimer = thread_call_allocate(&IOServicePH::userServerAckTimerExpired, (thread_call_param_t)NULL);
4412 }
4413
4414 void
lock()4415 IOServicePH::lock()
4416 {
4417 IOLockLock(gJobsLock);
4418 }
4419
4420 void
unlock()4421 IOServicePH::unlock()
4422 {
4423 IOLockUnlock(gJobsLock);
4424 }
4425
4426 void
serverAdd(IOUserServer * server)4427 IOServicePH::serverAdd(IOUserServer * server)
4428 {
4429 uint32_t idx;
4430
4431 lock();
4432 idx = fUserServers->getNextIndexOfObject(server, 0);
4433 if (idx == -1U) {
4434 fUserServers->setObject(server);
4435 }
4436 unlock();
4437 }
4438
4439 void
serverRemove(IOUserServer * server)4440 IOServicePH::serverRemove(IOUserServer * server)
4441 {
4442 uint32_t idx;
4443
4444 lock();
4445 idx = fUserServers->getNextIndexOfObject(server, 0);
4446 if (idx != -1U) {
4447 fUserServers->removeObject(idx);
4448 }
4449
4450 if (fWaitingUserServers) {
4451 fWaitingUserServers = false;
4452 IOLockWakeup(gJobsLock, &fWaitingUserServers, /* one-thread */ false);
4453 }
4454
4455 unlock();
4456 }
4457
4458 void
serverAck(IOUserServer * server)4459 IOServicePH::serverAck(IOUserServer * server)
4460 {
4461 uint32_t idx;
4462 IOService * ackTo;
4463 uint32_t ackToRef;
4464
4465 ackTo = NULL;
4466 lock();
4467 if (server && fUserServersWait) {
4468 idx = fUserServersWait->getNextIndexOfObject(server, 0);
4469 if (idx != -1U) {
4470 fUserServersWait->removeObject(idx);
4471 if (0 == fUserServersWait->getCount()) {
4472 OSSafeReleaseNULL(fUserServersWait);
4473 }
4474 }
4475 }
4476 if (!fUserServersWait && !fMatchingWork->getCount()) {
4477 ackTo = fSystemPowerAckTo;
4478 ackToRef = fSystemPowerAckRef;
4479 fSystemPowerAckTo = NULL;
4480 if (ackTo) {
4481 thread_call_cancel(fUserServerAckTimer);
4482 }
4483 }
4484 if (fUserServersWait && fUserServersWait->getCount() > 0 && fMatchingWork && fMatchingWork->getCount() > 0) {
4485 DKLOG("Waiting for %u user servers, %u matching work\n", fUserServersWait->getCount(), fMatchingWork->getCount());
4486 }
4487 unlock();
4488
4489 if (ackTo) {
4490 DKLOG("allowPowerChange\n");
4491 ackTo->allowPowerChange((uintptr_t) ackToRef);
4492 }
4493 }
4494
4495 bool
matchingStart(IOService * service)4496 IOServicePH::matchingStart(IOService * service)
4497 {
4498 uint32_t idx;
4499 bool assertionActive = gIOPMRootDomain->acquireDriverKitMatchingAssertion() == kIOReturnSuccess;
4500
4501 lock();
4502 bool matchNow = !fSystemOff && assertionActive;
4503 if (matchNow) {
4504 idx = fMatchingWork->getNextIndexOfObject(service, 0);
4505 if (idx == -1U) {
4506 fMatchingWork->setObject(service);
4507 }
4508 } else {
4509 // Delay matching if system is transitioning to sleep
4510 if (!fMatchingDelayed) {
4511 fMatchingDelayed = OSArray::withObjects((const OSObject **) &service, 1, 1);
4512 } else {
4513 idx = fMatchingDelayed->getNextIndexOfObject(service, 0);
4514 if (idx == -1U) {
4515 fMatchingDelayed->setObject(service);
4516 }
4517 }
4518 }
4519 unlock();
4520 if (!matchNow && assertionActive) {
4521 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
4522 }
4523
4524 return matchNow;
4525 }
4526
4527 void
matchingEnd(IOService * service)4528 IOServicePH::matchingEnd(IOService * service)
4529 {
4530 uint32_t idx;
4531 OSArray * notifyServers;
4532 OSArray * deferredMatches;
4533
4534 notifyServers = NULL;
4535 deferredMatches = NULL;
4536
4537 if (service) {
4538 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
4539 }
4540
4541 lock();
4542
4543 if (service) {
4544 idx = fMatchingWork->getNextIndexOfObject(service, 0);
4545 if (idx != -1U) {
4546 fMatchingWork->removeObject(idx);
4547 }
4548 }
4549
4550
4551 if ((fUserServerOff != fSystemOff) && fUserServers->getCount()) {
4552 if (fSystemOff) {
4553 if (0 == fMatchingWork->getCount()) {
4554 fUserServersWait = OSArray::withArray(fUserServers);
4555 notifyServers = OSArray::withArray(fUserServers);
4556 fUserServerOff = fSystemOff;
4557 }
4558 } else {
4559 notifyServers = OSArray::withArray(fUserServers);
4560 fUserServerOff = fSystemOff;
4561 }
4562 }
4563
4564 if (!fSystemOff && fMatchingDelayed) {
4565 deferredMatches = fMatchingDelayed;
4566 fMatchingDelayed = NULL;
4567 }
4568
4569 unlock();
4570
4571 if (notifyServers) {
4572 uint32_t sleepType = 0;
4573 uint32_t standbyTimer = 0;
4574 bool hibernate = false;
4575 if (fSystemOff && IOService::getPMRootDomain()->getSystemSleepType(&sleepType, &standbyTimer) == kIOReturnSuccess) {
4576 hibernate = (sleepType == kIOPMSleepTypeHibernate);
4577 }
4578 notifyServers->iterateObjects(^bool (OSObject * obj) {
4579 IOUserServer * us;
4580 us = (typeof(us))obj;
4581 us->systemPower(fSystemOff, hibernate);
4582 return false;
4583 });
4584 OSSafeReleaseNULL(notifyServers);
4585 }
4586
4587 if (deferredMatches) {
4588 DKLOG("sleep deferred rematching count %d\n", deferredMatches->getCount());
4589 deferredMatches->iterateObjects(^bool (OSObject * obj)
4590 {
4591 ((IOService *)obj)->startMatching(kIOServiceAsynchronous);
4592 return false;
4593 });
4594 deferredMatches->release();
4595 }
4596
4597 serverAck(NULL);
4598 }
4599
4600 TUNABLE(uint32_t, dk_shutdown_timeout_ms, "dk_shutdown_timeout_ms", 5000);
4601 TUNABLE(bool, dk_panic_on_shutdown_hang, "dk_panic_on_shutdown_hang", false);
4602 TUNABLE(bool, dk_panic_on_setpowerstate_hang, "dk_panic_on_setpowerstate_hang", false);
4603
4604 void
userServerAckTimerExpired(void *,void *)4605 IOServicePH::userServerAckTimerExpired(void *, void *)
4606 {
4607 OSArray * userServers = NULL;
4608 lock();
4609 if (fSystemPowerAckTo) {
4610 DKLOG("ack timer expired\n");
4611 if (dk_panic_on_setpowerstate_hang) {
4612 panic("DK ack timer expired after %u ms", dk_shutdown_timeout_ms);
4613 }
4614 userServers = fUserServersWait;
4615 fUserServersWait = NULL;
4616 }
4617 unlock();
4618
4619 if (userServers != NULL) {
4620 userServers->iterateObjects(^bool (OSObject *obj) {
4621 IOUserServer * us = OSDynamicCast(IOUserServer, obj);
4622 if (us) {
4623 DKLOG(DKS " power state transition failed\n", DKN(us));
4624 us->kill("Power Management Failed");
4625 }
4626 return false;
4627 });
4628 OSSafeReleaseNULL(userServers);
4629 }
4630
4631 serverAck(NULL);
4632 }
4633
4634 void
systemHalt(int howto)4635 IOServicePH::systemHalt(int howto)
4636 {
4637 OSArray * notifyServers;
4638 uint64_t deadline;
4639
4640 lock();
4641 notifyServers = OSArray::withArray(fUserServers);
4642 unlock();
4643
4644 if (notifyServers) {
4645 notifyServers->iterateObjects(^bool (OSObject * obj) {
4646 IOUserServer * us;
4647 us = (typeof(us))obj;
4648 us->systemHalt(howto);
4649 return false;
4650 });
4651 OSSafeReleaseNULL(notifyServers);
4652 }
4653
4654 lock();
4655 clock_interval_to_deadline(dk_shutdown_timeout_ms, kMillisecondScale, &deadline);
4656 while (0 < fUserServers->getCount()) {
4657 fWaitingUserServers = true;
4658 __assert_only int waitResult =
4659 IOLockSleepDeadline(gJobsLock, &fWaitingUserServers, deadline, THREAD_UNINT);
4660 assert((THREAD_AWAKENED == waitResult) || (THREAD_TIMED_OUT == waitResult));
4661 if (THREAD_TIMED_OUT == waitResult) {
4662 IOUserServer::beginLeakingObjects();
4663 #if DEVELOPMENT || DEBUG
4664 if (dk_panic_on_shutdown_hang) {
4665 panic("Shutdown timed out waiting for DK drivers to stop");
4666 }
4667 #endif /* DEVELOPMENT || DEBUG */
4668 break;
4669 }
4670 }
4671 unlock();
4672 }
4673
4674 bool
serverSlept(void)4675 IOServicePH::serverSlept(void)
4676 {
4677 bool ret;
4678
4679 lock();
4680 ret = (kIOMessageSystemWillSleep == sSystemPower)
4681 || (kIOMessageSystemWillPowerOff == sSystemPower)
4682 || (kIOMessageSystemWillRestart == sSystemPower);
4683 unlock();
4684
4685 return ret;
4686 }
4687
4688 TUNABLE(uint32_t, dk_power_state_timeout_ms, "dk_power_state_timeout_ms", 30000);
4689
4690 IOReturn
systemPowerChange(void * target,void * refCon,UInt32 messageType,IOService * service,void * messageArgument,vm_size_t argSize)4691 IOServicePH::systemPowerChange(
4692 void * target,
4693 void * refCon,
4694 UInt32 messageType, IOService * service,
4695 void * messageArgument, vm_size_t argSize)
4696 {
4697 IOReturn ret;
4698 IOUserServer * us;
4699 IOPMSystemCapabilityChangeParameters * params;
4700 AbsoluteTime deadline;
4701
4702 us = NULL;
4703
4704 switch (messageType) {
4705 case kIOMessageSystemCapabilityChange:
4706
4707 params = (typeof params)messageArgument;
4708
4709 if (kIODKLogPM & gIODKDebug) {
4710 IOLog("IOServicePH::kIOMessageSystemCapabilityChange: %s%s 0x%x->0x%x\n",
4711 params->changeFlags & kIOPMSystemCapabilityWillChange ? "will" : "",
4712 params->changeFlags & kIOPMSystemCapabilityDidChange ? "did" : "",
4713 params->fromCapabilities,
4714 params->toCapabilities);
4715 }
4716
4717 if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4718 (params->fromCapabilities & kIOPMSystemCapabilityCPU) &&
4719 ((params->toCapabilities & kIOPMSystemCapabilityCPU) == 0)) {
4720 lock();
4721 DKLOG("arming ack timer, %u ms\n", dk_power_state_timeout_ms);
4722 clock_interval_to_deadline(dk_power_state_timeout_ms, kMillisecondScale, &deadline);
4723 fSystemOff = true;
4724 fSystemPowerAckRef = params->notifyRef;
4725 fSystemPowerAckTo = service;
4726 thread_call_enter_delayed(fUserServerAckTimer, deadline);
4727 unlock();
4728
4729 matchingEnd(NULL);
4730
4731 params->maxWaitForReply = dk_power_state_timeout_ms * 2 * 1000;
4732 ret = kIOReturnSuccess;
4733 } else if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4734 ((params->fromCapabilities & kIOPMSystemCapabilityCPU) == 0) &&
4735 (params->toCapabilities & kIOPMSystemCapabilityCPU)) {
4736 lock();
4737 fSystemOff = false;
4738 unlock();
4739
4740 matchingEnd(NULL);
4741
4742 params->maxWaitForReply = 0;
4743 ret = kIOReturnSuccess;
4744 } else {
4745 params->maxWaitForReply = 0;
4746 ret = kIOReturnSuccess;
4747 }
4748 break;
4749
4750 default:
4751 ret = kIOReturnUnsupported;
4752 break;
4753 }
4754
4755 return ret;
4756 }
4757
4758 bool
checkPMReady(void)4759 IOServicePH::checkPMReady(void)
4760 {
4761 bool __block ready = true;
4762
4763 lock();
4764 fUserServers->iterateObjects(^bool (OSObject *obj) {
4765 IOUserServer * us = OSDynamicCast(IOUserServer, obj);
4766 if (us) {
4767 if (!us->checkPMReady()) {
4768 ready = false;
4769 return true;
4770 }
4771 }
4772 return false;
4773 });
4774 unlock();
4775
4776 return ready;
4777 }
4778
4779 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4780
4781 /*
4782 * Start a previously attached & probed instance,
4783 * called on exporting object instance
4784 */
4785
4786 bool
startCandidate(IOService * service)4787 IOService::startCandidate( IOService * service )
4788 {
4789 bool ok;
4790 OSObject * obj;
4791 OSObject * prop;
4792 IOUserServer * userServer;
4793 bool ph;
4794
4795 userServer = NULL;
4796 obj = service->copyProperty(gIOUserServerNameKey);
4797
4798 if (obj && (this == gIOResources)) {
4799 ok = false;
4800 } else {
4801 ok = service->attach( this );
4802 }
4803 if (!ok) {
4804 OSSafeReleaseNULL(obj);
4805 return false;
4806 }
4807
4808 if ((this != gIOResources) && (this != gIOUserResources)) {
4809 // stall for any nub resources
4810 checkResources();
4811 // stall for any driver resources
4812 service->checkResources();
4813 }
4814 ph = false;
4815 {
4816 OSString * bundleID;
4817 OSString * serverName;
4818 OSString * str;
4819 const OSSymbol * sym;
4820 OSNumber * serverTag;
4821 uint64_t entryID;
4822 IOUserServerCheckInToken * token;
4823 OSData * serverDUI;
4824
4825 if ((serverName = OSDynamicCast(OSString, obj))) {
4826 obj = service->copyProperty(gIOModuleIdentifierKey);
4827 bundleID = OSDynamicCast(OSString, obj);
4828 entryID = service->getRegistryEntryID();
4829 serverTag = OSNumber::withNumber(entryID, 64);
4830 token = NULL;
4831
4832 if (kIODKDisableDextLaunch & gIODKDebug) {
4833 DKLOG(DKS " dext launches are disabled \n", DKN(service));
4834 service->detach(this);
4835 OSSafeReleaseNULL(serverName);
4836 OSSafeReleaseNULL(obj);
4837 OSSafeReleaseNULL(serverTag);
4838 return false;
4839 }
4840
4841 if (gIOKitWillTerminate) {
4842 DKLOG("%s disabled in shutdown\n", serverName->getCStringNoCopy());
4843 service->detach(this);
4844 OSSafeReleaseNULL(serverName);
4845 OSSafeReleaseNULL(obj);
4846 OSSafeReleaseNULL(serverTag);
4847 return false;
4848 }
4849
4850 ph = IOServicePH::matchingStart(this);
4851 if (!ph) {
4852 DKLOG("%s deferred in sleep\n", serverName->getCStringNoCopy());
4853 service->setProperty(gIOServiceMatchDeferredKey, kOSBooleanTrue);
4854 service->detach(this);
4855 OSSafeReleaseNULL(serverName);
4856 OSSafeReleaseNULL(obj);
4857 OSSafeReleaseNULL(serverTag);
4858 return false;
4859 }
4860
4861 prop = service->copyProperty(gIOUserClassKey);
4862 str = OSDynamicCast(OSString, prop);
4863 if (str) {
4864 service->setName(str);
4865 }
4866 OSSafeReleaseNULL(prop);
4867
4868 sym = OSSymbol::withString(serverName);
4869 bool reuse = service->propertyExists(gIOUserServerOneProcessKey);
4870 bool reuseRequired = false;
4871 if (propertyExists(gIOUserServerNameKey)) {
4872 reuseRequired = reuse && service->propertyHasValue(gIOUserServerNameKey, getProperty(gIOUserServerNameKey));
4873 }
4874 serverDUI = OSDynamicCast(OSData, service->getProperty(kOSBundleDextUniqueIdentifierKey));
4875 userServer = IOUserServer::launchUserServer(bundleID, sym, serverTag, reuse, &token, serverDUI);
4876 OSSafeReleaseNULL(sym);
4877 OSSafeReleaseNULL(serverTag);
4878 OSSafeReleaseNULL(serverName);
4879 if (userServer) {
4880 DKLOG(DKS " using existing server " DKS "\n", DKN(service), DKN(userServer));
4881 } else if (!reuseRequired && token != NULL) {
4882 const OSSymbol * tokenServerName = token->copyServerName();
4883 OSNumber * tokenServerTag = token->copyServerTag();
4884 assert(tokenServerName && tokenServerTag);
4885 DKLOG(DKS " waiting for server %s-%llx\n", DKN(service), tokenServerName->getCStringNoCopy(), tokenServerTag->unsigned64BitValue());
4886 userServer = __WAITING_FOR_USER_SERVER__(token);
4887 OSSafeReleaseNULL(tokenServerName);
4888 OSSafeReleaseNULL(tokenServerTag);
4889 } else if (reuseRequired) {
4890 DKLOG(DKS " failed to reuse server\n", DKN(service));
4891 } else {
4892 DKLOG(DKS " failed to launch server\n", DKN(service));
4893 }
4894
4895
4896 if (!userServer) {
4897 service->detach(this);
4898 IOServicePH::matchingEnd(this);
4899 OSSafeReleaseNULL(obj);
4900
4901 if (token != NULL) {
4902 DKLOG(DKS " user server timeout\n", DKN(service));
4903 #if DEVELOPMENT || DEBUG
4904 driverkit_checkin_timed_out = mach_absolute_time();
4905 #endif
4906 }
4907
4908 OSSafeReleaseNULL(token);
4909 return false;
4910 }
4911
4912 if (token && !(kIODKDisableCheckInTokenVerification & gIODKDebug)) {
4913 if (!userServer->serviceMatchesCheckInToken(token)) {
4914 OSSafeReleaseNULL(token);
4915 service->detach(this);
4916 IOServicePH::matchingEnd(this);
4917 OSSafeReleaseNULL(obj);
4918 userServer->exit("Check In Token verification failed");
4919 userServer->release();
4920 return false;
4921 }
4922 }
4923 OSSafeReleaseNULL(token);
4924 OSSafeReleaseNULL(obj);
4925
4926 if (!(kIODKDisableEntitlementChecking & gIODKDebug)) {
4927 if (!userServer->checkEntitlements(this, service)) {
4928 service->detach(this);
4929 IOServicePH::matchingEnd(this);
4930 userServer->exit("Entitlements check failed");
4931 userServer->release();
4932 return false;
4933 }
4934 }
4935 #if !XNU_TARGET_OS_OSX && !(DEVELOPMENT || DEBUG)
4936 // Prevent third party drivers from matching IOUserResources/IOResources, except when signed for development
4937 if (!userServer->isPlatformDriver() &&
4938 userServer->getCSValidationCategory() != CS_VALIDATION_CATEGORY_DEVELOPMENT
4939 && (this == gIOUserResources || this == gIOResources)) {
4940 service->detach(this);
4941 IOServicePH::matchingEnd(this);
4942 userServer->exit("Third party driver may only match real hardware");
4943 userServer->release();
4944 return false;
4945 }
4946 #endif /* !XNU_TARGET_OS_OSX && !(DEVELOPMENT || DEBUG) */
4947
4948 userServer->serviceAttach(service, this);
4949 } else {
4950 OSSafeReleaseNULL(obj);
4951 }
4952 }
4953
4954 AbsoluteTime startTime;
4955 AbsoluteTime endTime;
4956 UInt64 nano;
4957 bool recordTime = (kIOLogStart & gIOKitDebug) != 0;
4958
4959 if (recordTime) {
4960 clock_get_uptime(&startTime);
4961 }
4962
4963 ok = service->start(this);
4964
4965 if (recordTime) {
4966 clock_get_uptime(&endTime);
4967
4968 if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0) {
4969 SUB_ABSOLUTETIME(&endTime, &startTime);
4970 absolutetime_to_nanoseconds(endTime, &nano);
4971 if (nano > 500000000ULL) {
4972 IOLog("%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
4973 }
4974 }
4975 }
4976 if (userServer) {
4977 userServer->serviceStarted(service, this, ok);
4978 userServer->release();
4979 }
4980
4981 if (ok) {
4982 IOInstallServiceSleepPlatformActions(service);
4983 #if 00
4984 if (!strcmp("XHC1", getName())) {
4985 service->setProperty(gIOPrimaryDriverTerminateOptionsKey, kOSBooleanTrue);
4986 }
4987 #endif
4988 }
4989
4990 if (!ok) {
4991 service->detach( this );
4992 }
4993
4994 if (ph) {
4995 IOServicePH::matchingEnd(this);
4996 }
4997
4998 return ok;
4999 }
5000
5001 void
publishResource(const char * key,OSObject * value)5002 IOService::publishResource( const char * key, OSObject * value )
5003 {
5004 const OSSymbol * sym;
5005
5006 if ((sym = OSSymbol::withCString( key))) {
5007 publishResource( sym, value);
5008 sym->release();
5009 }
5010 }
5011
5012 void
publishResource(const OSSymbol * key,OSObject * value)5013 IOService::publishResource( const OSSymbol * key, OSObject * value )
5014 {
5015 if (NULL == value) {
5016 value = (OSObject *) gIOServiceKey;
5017 }
5018
5019 gIOResources->setProperty( key, value);
5020
5021 if (IORecursiveLockHaveLock( gNotificationLock)) {
5022 return;
5023 }
5024
5025 gIOResourceGenerationCount++;
5026 gIOResources->registerService();
5027 }
5028
5029 void
publishUserResource(const OSSymbol * key,OSObject * value)5030 IOService::publishUserResource( const OSSymbol * key, OSObject * value )
5031 {
5032 if (NULL == value) {
5033 value = (OSObject *) gIOServiceKey;
5034 }
5035
5036 gIOUserResources->setProperty( key, value);
5037
5038 if (IORecursiveLockHaveLock( gNotificationLock)) {
5039 return;
5040 }
5041
5042 gIOResourceGenerationCount++;
5043 gIOUserResources->registerService();
5044 }
5045
5046 bool
addNeededResource(const char * key)5047 IOService::addNeededResource( const char * key )
5048 {
5049 OSObject * resourcesProp;
5050 OSSet * set;
5051 OSString * newKey;
5052 bool ret;
5053
5054 resourcesProp = copyProperty( gIOResourceMatchKey );
5055 if (!resourcesProp) {
5056 return false;
5057 }
5058
5059 newKey = OSString::withCString( key );
5060 if (!newKey) {
5061 resourcesProp->release();
5062 return false;
5063 }
5064
5065 set = OSDynamicCast( OSSet, resourcesProp );
5066 if (!set) {
5067 set = OSSet::withCapacity( 1 );
5068 set->setObject( resourcesProp );
5069 } else {
5070 set->retain();
5071 }
5072
5073 set->setObject( newKey );
5074 newKey->release();
5075 ret = setProperty( gIOResourceMatchKey, set );
5076 set->release();
5077 resourcesProp->release();
5078
5079 return ret;
5080 }
5081
5082 bool
checkResource(OSObject * matching)5083 IOService::checkResource( OSObject * matching )
5084 {
5085 OSString * str;
5086 OSDictionary * table;
5087
5088 if ((str = OSDynamicCast( OSString, matching ))) {
5089 if (gIOResources->getProperty( str )) {
5090 return true;
5091 }
5092 }
5093
5094 if (str) {
5095 table = resourceMatching( str );
5096 } else if ((table = OSDynamicCast( OSDictionary, matching ))) {
5097 table->retain();
5098 } else {
5099 IOLog("%s: Can't match using: %s\n", getName(),
5100 matching->getMetaClass()->getClassName());
5101 /* false would stall forever */
5102 return true;
5103 }
5104
5105 if (gIOKitDebug & kIOLogConfig) {
5106 LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName());
5107 }
5108
5109 waitForService( table );
5110
5111 if (gIOKitDebug & kIOLogConfig) {
5112 LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
5113 }
5114
5115 return true;
5116 }
5117
5118 bool
checkResources(void)5119 IOService::checkResources( void )
5120 {
5121 OSObject * resourcesProp;
5122 OSSet * set;
5123 OSObject * obj;
5124 OSIterator * iter;
5125 bool ok;
5126
5127 resourcesProp = copyProperty( gIOResourceMatchKey );
5128 if (NULL == resourcesProp) {
5129 return true;
5130 }
5131
5132 if ((set = OSDynamicCast( OSSet, resourcesProp ))) {
5133 iter = OSCollectionIterator::withCollection( set );
5134 ok = (NULL != iter);
5135 while (ok && (obj = iter->getNextObject())) {
5136 ok = checkResource( obj );
5137 }
5138 if (iter) {
5139 iter->release();
5140 }
5141 } else {
5142 ok = checkResource( resourcesProp );
5143 }
5144
5145 OSSafeReleaseNULL(resourcesProp);
5146
5147 return ok;
5148 }
5149
5150
5151 void
configThread(const char * name)5152 _IOConfigThread::configThread( const char * name )
5153 {
5154 _IOConfigThread * inst;
5155
5156 do {
5157 if (!(inst = new _IOConfigThread)) {
5158 continue;
5159 }
5160 if (!inst->init()) {
5161 continue;
5162 }
5163 thread_t thread;
5164 if (KERN_SUCCESS != kernel_thread_start(&_IOConfigThread::main, inst, &thread)) {
5165 continue;
5166 }
5167
5168 char threadName[MAXTHREADNAMESIZE];
5169 snprintf(threadName, sizeof(threadName), "IOConfigThread_'%s'", name);
5170 thread_set_thread_name(thread, threadName);
5171 thread_deallocate(thread);
5172
5173 return;
5174 } while (false);
5175
5176 if (inst) {
5177 inst->release();
5178 }
5179
5180 return;
5181 }
5182
5183 /*
5184 * To support driver replacement of boot matched drivers later in boot, drivers can
5185 * opt-in to be being terminated if a non-boot driver matches their provider, by
5186 * setting the gIOPrimaryDriverTerminateOptionsKey property. The driver providing the
5187 * root disk media may not be terminated.
5188 * IOMedia objects are hidden from user space until all drivers are available, but any
5189 * associated with the root disk must be published immediately.
5190 */
5191
5192 struct FindRootMediaContext {
5193 OSArray * services;
5194 IOService * parent;
5195 };
5196
5197 bool
hasParent(IOService * parent)5198 IOService::hasParent(IOService * parent)
5199 {
5200 IOService * service;
5201
5202 for (service = this;
5203 service && (service != parent);
5204 service = service->getProvider()) {
5205 }
5206
5207 return service != NULL;
5208 }
5209
5210 bool
publishHiddenMediaApplier(const OSObject * entry,void * context)5211 IOService::publishHiddenMediaApplier(const OSObject * entry, void * context)
5212 {
5213 FindRootMediaContext * ctx = (typeof(ctx))context;
5214 IOService * service = (typeof(service))entry;
5215
5216 do {
5217 if (ctx->parent && !service->hasParent(ctx->parent)) {
5218 break;
5219 }
5220 if (ctx->services) {
5221 ctx->services->setObject(service);
5222 } else {
5223 ctx->services = OSArray::withObjects((const OSObject **) &service, 1);
5224 assert(ctx->services);
5225 }
5226 } while (false);
5227
5228 return false;
5229 }
5230
5231 // publish to user space any hidden IOMedia under the 'parent' object, or all
5232 // if 'parent' is NULL
5233
5234 void
publishHiddenMedia(IOService * parent)5235 IOService::publishHiddenMedia(IOService * parent)
5236 {
5237 const OSMetaClass * iomediaClass;
5238 bool wasHiding;
5239
5240 iomediaClass = OSMetaClass::getMetaClassWithName(gIOMediaKey);
5241 assert(iomediaClass);
5242
5243 LOCKWRITENOTIFY();
5244 wasHiding = gIOServiceHideIOMedia;
5245 if (wasHiding && !parent) {
5246 gIOServiceHideIOMedia = false;
5247 }
5248 UNLOCKNOTIFY();
5249
5250 FindRootMediaContext ctx = { .services = NULL, .parent = parent };
5251
5252 if (wasHiding) {
5253 iomediaClass->applyToInstances(publishHiddenMediaApplier, &ctx);
5254 }
5255 if (ctx.services) {
5256 unsigned int idx, notiIdx;
5257 IOService * service;
5258 OSArray * notifiers[3] = {};
5259
5260 for (idx = 0; (service = (IOService *) ctx.services->getObject(idx)); idx++) {
5261 service->lockForArbitration(true);
5262 if (!(kIOServiceUserInvisibleMatchState & service->__state[0])) {
5263 service->unlockForArbitration();
5264 continue;
5265 }
5266 service->__state[0] &= ~kIOServiceUserInvisibleMatchState;
5267 service->__state[1] |= kIOServiceUserUnhidden;
5268 notifiers[0] = service->copyNotifiers(gIOFirstPublishNotification, 0, 0xffffffff);
5269 if (kIOServiceMatchedState & service->__state[0]) {
5270 notifiers[1] = service->copyNotifiers(gIOMatchedNotification, 0, 0xffffffff);
5271 }
5272 if (kIOServiceFirstMatchState & service->__state[0]) {
5273 notifiers[2] = service->copyNotifiers(gIOFirstMatchNotification, 0, 0xffffffff);
5274 }
5275 service->unlockForArbitration();
5276 for (notiIdx = 0; notiIdx < 3; notiIdx++) {
5277 service->invokeNotifiers(¬ifiers[notiIdx]);
5278 }
5279 }
5280 OSSafeReleaseNULL(ctx.services);
5281 }
5282 }
5283
5284 // Find the block storage driver providing the root disk, or NULL if not booting from
5285 // a block device
5286
5287 void
setRootMedia(IOService * root)5288 IOService::setRootMedia(IOService * root)
5289 {
5290 const OSMetaClass * ioblockstoragedriverClass;
5291 bool unhide;
5292
5293 ioblockstoragedriverClass = OSMetaClass::getMetaClassWithName(gIOBlockStorageDriverKey);
5294 assert(ioblockstoragedriverClass);
5295
5296 while (root) {
5297 if (root->metaCast(ioblockstoragedriverClass)) {
5298 break;
5299 }
5300 root = root->getProvider();
5301 }
5302
5303 LOCKWRITENOTIFY();
5304 unhide = (kIOServiceRootMediaParentInvalid == gIOServiceRootMediaParent);
5305 if (unhide) {
5306 gIOServiceRootMediaParent = root;
5307 }
5308 UNLOCKNOTIFY();
5309
5310 if (unhide) {
5311 publishHiddenMedia(root);
5312 }
5313 }
5314
5315 // Check if the driver may be terminated when a later driver could be used instead
5316
5317 bool
canTerminateForReplacement(IOService * client)5318 IOService::canTerminateForReplacement(IOService * client)
5319 {
5320 IOService * parent;
5321
5322 assert(kIOServiceRootMediaParentInvalid != gIOServiceRootMediaParent);
5323
5324 if (!client->propertyExists(gIOPrimaryDriverTerminateOptionsKey)) {
5325 return false;
5326 }
5327 if (!gIOServiceRootMediaParent) {
5328 return false;
5329 }
5330 parent = gIOServiceRootMediaParent;
5331 while (parent && (parent != client)) {
5332 parent = parent->getProvider();
5333 }
5334 if (parent) {
5335 IOLog("Can't replace primary matched driver on root media %s-0x%qx\n",
5336 client->getName(), client->getRegistryEntryID());
5337 return false;
5338 }
5339 return true;
5340 }
5341
5342 void
doServiceMatch(IOOptionBits options)5343 IOService::doServiceMatch( IOOptionBits options )
5344 {
5345 _IOServiceNotifier * notify;
5346 OSIterator * iter;
5347 OSOrderedSet * matches;
5348 OSArray * resourceKeys = NULL;
5349 SInt32 catalogGeneration;
5350 bool keepGuessing = true;
5351 bool reRegistered = true;
5352 bool didRegister;
5353 OSArray * notifiers[2] = {NULL};
5354
5355 // job->nub->deliverNotification( gIOPublishNotification,
5356 // kIOServiceRegisteredState, 0xffffffff );
5357
5358 while (keepGuessing) {
5359 matches = gIOCatalogue->findDrivers( this, &catalogGeneration );
5360 // the matches list should always be created by findDrivers()
5361 if (matches) {
5362 lockForArbitration();
5363 if (0 == (__state[0] & kIOServiceFirstPublishState)) {
5364 getMetaClass()->addInstance(this);
5365 notifiers[0] = copyNotifiers(gIOFirstPublishNotification,
5366 kIOServiceFirstPublishState, 0xffffffff );
5367 }
5368 LOCKREADNOTIFY();
5369 __state[1] &= ~kIOServiceNeedConfigState;
5370 __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning;
5371 didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
5372 __state[0] |= kIOServiceRegisteredState;
5373
5374 if (gIOServiceHideIOMedia
5375 && metaCast(gIOMediaKey)
5376 && !(kIOServiceUserUnhidden & __state[1])
5377 && gIOServiceRootMediaParent
5378 && !hasParent(gIOServiceRootMediaParent)
5379 && propertyExists(gIOPrimaryDriverTerminateOptionsKey, gIOServicePlane)) {
5380 __state[0] |= kIOServiceUserInvisibleMatchState;
5381 }
5382
5383 keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
5384 if (reRegistered && keepGuessing) {
5385 iter = OSCollectionIterator::withCollection((OSOrderedSet *)
5386 gNotifications->getObject( gIOPublishNotification ));
5387 if (iter) {
5388 while ((notify = (_IOServiceNotifier *)
5389 iter->getNextObject())) {
5390 if (matchPassive(notify->matching, 0)
5391 && (kIOServiceNotifyEnable & notify->state)) {
5392 matches->setObject( notify );
5393 }
5394 }
5395 iter->release();
5396 }
5397 }
5398
5399 UNLOCKNOTIFY();
5400 unlockForArbitration();
5401 invokeNotifiers(¬ifiers[0]);
5402 if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources())) {
5403 if ((this == gIOResources) || (this == gIOUserResources)) {
5404 if (resourceKeys) {
5405 resourceKeys->release();
5406 }
5407 resourceKeys = copyPropertyKeys();
5408 }
5409 probeCandidates( matches );
5410 } else {
5411 matches->release();
5412 }
5413 }
5414
5415 lockForArbitration();
5416 reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
5417 keepGuessing =
5418 (reRegistered || (catalogGeneration !=
5419 gIOCatalogue->getGenerationCount()))
5420 && (0 == (__state[0] & kIOServiceInactiveState));
5421
5422 if (keepGuessing) {
5423 unlockForArbitration();
5424 }
5425 }
5426
5427 if ((0 == (__state[0] & kIOServiceInactiveState))
5428 && (0 == (__state[1] & kIOServiceModuleStallState))) {
5429 if (resourceKeys) {
5430 setProperty(gIOResourceMatchedKey, resourceKeys);
5431 }
5432
5433 notifiers[0] = copyNotifiers(gIOMatchedNotification,
5434 kIOServiceMatchedState, 0xffffffff);
5435 if (0 == (__state[0] & kIOServiceFirstMatchState)) {
5436 notifiers[1] = copyNotifiers(gIOFirstMatchNotification,
5437 kIOServiceFirstMatchState, 0xffffffff);
5438 }
5439 }
5440
5441 __state[1] &= ~kIOServiceConfigRunning;
5442 unlockForArbitration();
5443
5444 if (resourceKeys) {
5445 resourceKeys->release();
5446 }
5447
5448 invokeNotifiers(¬ifiers[0]);
5449 invokeNotifiers(¬ifiers[1]);
5450
5451 lockForArbitration();
5452 __state[1] &= ~kIOServiceConfigState;
5453 scheduleTerminatePhase2();
5454
5455 _adjustBusy(-1, /*unlock*/ true);
5456 // does unlockForArbitration();
5457 }
5458
5459 UInt32
_adjustBusy(SInt32 delta)5460 IOService::_adjustBusy(SInt32 delta)
5461 {
5462 return _adjustBusy(delta, false);
5463 }
5464
5465 UInt32
_adjustBusy(SInt32 delta,bool unlock)5466 IOService::_adjustBusy(SInt32 delta, bool unlock)
5467 {
5468 IOService * next;
5469 IOService * nextProvider;
5470 UInt32 count;
5471 UInt32 result;
5472 bool wasQuiet, nowQuiet, needWake;
5473
5474 next = this;
5475 result = __state[1] & kIOServiceBusyStateMask;
5476
5477 if (delta) {
5478 do {
5479 if (next != this) {
5480 next->lockForArbitration();
5481 }
5482 count = next->__state[1] & kIOServiceBusyStateMask;
5483 wasQuiet = (0 == count);
5484 if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count))) {
5485 OSReportWithBacktrace("%s: bad busy count (%d,%d)\n", next->getName(), (uint32_t)count, (int)delta);
5486 } else {
5487 count += delta;
5488 }
5489 next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
5490 nowQuiet = (0 == count);
5491 needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
5492
5493 if (needWake) {
5494 next->__state[1] &= ~kIOServiceBusyWaiterState;
5495 IOLockLock( gIOServiceBusyLock );
5496 thread_wakeup((event_t) next);
5497 IOLockUnlock( gIOServiceBusyLock );
5498 }
5499 if (wasQuiet || nowQuiet) {
5500 nextProvider = next->getProvider();
5501 if (nextProvider) {
5502 nextProvider->retain();
5503 }
5504 }
5505 if ((next != this) || unlock) {
5506 next->unlockForArbitration();
5507 }
5508
5509 if ((wasQuiet || nowQuiet)) {
5510 uint64_t regID = next->getRegistryEntryID();
5511 IOServiceTrace(
5512 ((wasQuiet /*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
5513 (uintptr_t) regID,
5514 (uintptr_t) (regID >> 32),
5515 (uintptr_t) next,
5516 0);
5517
5518 if (wasQuiet) {
5519 next->__timeBusy = mach_absolute_time();
5520 } else {
5521 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
5522 next->__timeBusy = 0;
5523 }
5524
5525 MessageClientsContext context;
5526
5527 context.service = next;
5528 context.type = kIOMessageServiceBusyStateChange;
5529 context.argument = (void *) wasQuiet; /*nowBusy*/
5530 context.argSize = 0;
5531
5532 applyToInterestNotifiers( next, gIOBusyInterest,
5533 &messageClientsApplier, &context );
5534
5535 #if !NO_KEXTD
5536 if (nowQuiet && (next == gIOServiceRoot)) {
5537 if (gIOServiceHideIOMedia) {
5538 publishHiddenMedia(NULL);
5539 }
5540
5541 OSKext::considerUnloads();
5542 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
5543 }
5544 #endif
5545 }
5546
5547 delta = nowQuiet ? -1 : +1;
5548 if (next != this) {
5549 next->release();
5550 }
5551 next = nextProvider;
5552 nextProvider = NULL;
5553 } while ((wasQuiet || nowQuiet) && next);
5554 }
5555
5556 return result;
5557 }
5558
5559 void
adjustBusy(SInt32 delta)5560 IOService::adjustBusy( SInt32 delta )
5561 {
5562 lockForArbitration();
5563 _adjustBusy( delta );
5564 unlockForArbitration();
5565 }
5566
5567 uint64_t
getAccumulatedBusyTime(void)5568 IOService::getAccumulatedBusyTime( void )
5569 {
5570 uint64_t accumBusy = __accumBusy;
5571 uint64_t timeBusy = __timeBusy;
5572 uint64_t nano;
5573
5574 do{
5575 accumBusy = __accumBusy;
5576 timeBusy = __timeBusy;
5577 if (timeBusy) {
5578 accumBusy += mach_absolute_time() - timeBusy;
5579 }
5580 }while (timeBusy != __timeBusy);
5581
5582 absolutetime_to_nanoseconds(*(AbsoluteTime *)&accumBusy, &nano);
5583
5584 return nano;
5585 }
5586
5587 UInt32
getBusyState(void)5588 IOService::getBusyState( void )
5589 {
5590 return __state[1] & kIOServiceBusyStateMask;
5591 }
5592
5593 IOReturn
waitForState(UInt32 mask,UInt32 value,mach_timespec_t * timeout)5594 IOService::waitForState( UInt32 mask, UInt32 value,
5595 mach_timespec_t * timeout )
5596 {
5597 panic("waitForState");
5598 return kIOReturnUnsupported;
5599 }
5600
5601 IOReturn
waitForState(UInt32 mask,UInt32 value,uint64_t timeout)5602 IOService::waitForState( UInt32 mask, UInt32 value,
5603 uint64_t timeout )
5604 {
5605 bool wait, done;
5606 int waitResult = THREAD_AWAKENED;
5607 bool computeDeadline = true;
5608 AbsoluteTime abstime;
5609
5610 do {
5611 lockForArbitration();
5612 IOLockLock( gIOServiceBusyLock );
5613 done = (value == (__state[1] & mask));
5614 wait = (waitResult != THREAD_TIMED_OUT) && !done;
5615 if (wait) {
5616 __state[1] |= kIOServiceBusyWaiterState;
5617 unlockForArbitration();
5618 if (timeout != UINT64_MAX) {
5619 if (computeDeadline) {
5620 AbsoluteTime nsinterval;
5621 nanoseconds_to_absolutetime(timeout, &nsinterval );
5622 clock_absolutetime_interval_to_deadline(nsinterval, &abstime);
5623 computeDeadline = false;
5624 }
5625 assert_wait_deadline((event_t)this, THREAD_UNINT, __OSAbsoluteTime(abstime));
5626 } else {
5627 assert_wait((event_t)this, THREAD_UNINT );
5628 }
5629 } else {
5630 unlockForArbitration();
5631 }
5632 IOLockUnlock( gIOServiceBusyLock );
5633 if (wait) {
5634 waitResult = thread_block(THREAD_CONTINUE_NULL);
5635 }
5636 } while (wait);
5637
5638 if (done) {
5639 return kIOReturnSuccess;
5640 } else {
5641 return kIOReturnTimeout;
5642 }
5643 }
5644
5645 IOReturn
waitQuietWithOptions(uint64_t timeout,IOOptionBits options)5646 IOService::waitQuietWithOptions( uint64_t timeout, IOOptionBits options )
5647 {
5648 IOReturn ret;
5649 uint32_t loops, timeoutExtensions;
5650 char * busyEntriesString = NULL;
5651 char * panicString = NULL;
5652 size_t busyEntriesStringLen;
5653 size_t panicStringLen;
5654 uint64_t time;
5655 uint64_t nano;
5656 bool pendingRequests;
5657 bool registryRootBusy;
5658 bool multipleEntries;
5659 bool dopanic = false;
5660
5661 enum { kIOServiceBusyTimeoutExtensionsMax = 8 };
5662 #if KASAN
5663 /*
5664 * On kasan kernels, everything takes longer, so double the number of
5665 * timeout extensions. This should help with issues like 41259215
5666 * where WindowServer was timing out waiting for kextd to get all the
5667 * kasan kexts loaded and started.
5668 *
5669 * On legacy/x86 systems give a bit more time since we may be
5670 * booting from a HDD.
5671 */
5672 enum { kTimeoutExtensions = 8 };
5673 #define WITH_IOWAITQUIET_EXTENSIONS 1
5674 #elif defined(__x86_64__)
5675 enum { kTimeoutExtensions = 4 };
5676 #define WITH_IOWAITQUIET_EXTENSIONS 1
5677 #elif defined(XNU_TARGET_OS_OSX)
5678 enum { kTimeoutExtensions = 1 };
5679 #define WITH_IOWAITQUIET_EXTENSIONS 1
5680 #else
5681 enum { kTimeoutExtensions = 1 };
5682 #define WITH_IOWAITQUIET_EXTENSIONS 0
5683 #endif
5684
5685 timeoutExtensions = kTimeoutExtensions;
5686 time = mach_absolute_time();
5687 pendingRequests = false;
5688 for (loops = 0; loops < timeoutExtensions; loops++) {
5689 ret = waitForState( kIOServiceBusyStateMask, 0, timeout );
5690
5691 if (loops && (kIOReturnSuccess == ret)) {
5692 time = mach_absolute_time() - time;
5693 absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano);
5694 IOLog("busy extended ok[%d], (%llds, %llds)\n",
5695 loops, timeout / 1000000000ULL, nano / 1000000000ULL);
5696 break;
5697 } else if (kIOReturnTimeout != ret) {
5698 break;
5699 } else if (timeout < (41ull * NSEC_PER_SEC)) {
5700 break;
5701 }
5702
5703 {
5704 IORegistryIterator * iter;
5705 OSOrderedSet * set;
5706 OSOrderedSet * leaves;
5707 IOService * next;
5708 IOService * nextParent;
5709 char * s;
5710 size_t l;
5711 size_t busyEntriesStringRemaining;
5712
5713 busyEntriesStringLen = 256;
5714 panicStringLen = 256;
5715 if (!busyEntriesString) {
5716 busyEntriesString = IONewZeroData(char, busyEntriesStringLen);
5717 assert(busyEntriesString != NULL);
5718 }
5719 if (!panicString) {
5720 panicString = IONewZeroData(char, panicStringLen);
5721 assert(panicString != NULL);
5722 }
5723
5724 set = NULL;
5725 pendingRequests = OSKext::pendingIOKitDaemonRequests();
5726 iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively);
5727 leaves = OSOrderedSet::withCapacity(4);
5728 if (iter) {
5729 set = iter->iterateAll();
5730 }
5731 if (leaves && set) {
5732 busyEntriesString[0] = panicString[0] = 0;
5733 set->setObject(this);
5734 while ((next = (IOService *) set->getLastObject())) {
5735 if (next->getBusyState()) {
5736 if (kIOServiceModuleStallState & next->__state[1]) {
5737 pendingRequests = true;
5738 }
5739 #if defined(XNU_TARGET_OS_OSX)
5740 OSObject * prop;
5741 if ((prop = next->copyProperty(kIOServiceBusyTimeoutExtensionsKey, gIOServicePlane))) {
5742 OSNumber * num;
5743 uint32_t value;
5744 if ((num = OSDynamicCast(OSNumber, prop))) {
5745 value = num->unsigned32BitValue();
5746 if (value
5747 && (value <= kIOServiceBusyTimeoutExtensionsMax)
5748 && (value > timeoutExtensions)) {
5749 timeoutExtensions = value;
5750 }
5751 }
5752 OSSafeReleaseNULL(prop);
5753 }
5754 #endif /* defined(XNU_TARGET_OS_OSX) */
5755 leaves->setObject(next);
5756 nextParent = next;
5757 while ((nextParent = nextParent->getProvider())) {
5758 set->removeObject(nextParent);
5759 leaves->removeObject(nextParent);
5760 }
5761 }
5762 set->removeObject(next);
5763 }
5764 registryRootBusy = leaves->getCount() == 1 && leaves->getObject(0) == getServiceRoot();
5765 multipleEntries = leaves->getCount() > 1;
5766 s = busyEntriesString;
5767 busyEntriesStringRemaining = busyEntriesStringLen;
5768
5769 if (registryRootBusy) {
5770 snprintf(s, busyEntriesStringRemaining, "registry root held busy, " kIOKitDaemonName " %s checked in", OSKext::iokitDaemonActive() ? "has" : "has not");
5771 } else {
5772 while ((next = (IOService *) leaves->getLastObject())) {
5773 l = snprintf(s, busyEntriesStringRemaining, "%s'%s' (%x,%x)", ((s == busyEntriesString) ? "" : ", "), next->getName(), (uint32_t)next->__state[0], (uint32_t)next->__state[1]);
5774 if (l >= busyEntriesStringRemaining) {
5775 break;
5776 }
5777 s += l;
5778 busyEntriesStringRemaining -= l;
5779 leaves->removeObject(next);
5780 }
5781 }
5782 }
5783 OSSafeReleaseNULL(leaves);
5784 OSSafeReleaseNULL(set);
5785 OSSafeReleaseNULL(iter);
5786 }
5787
5788 dopanic = (kIOWaitQuietPanics & gIOKitDebug) && (options & kIOWaitQuietPanicOnFailure) && !gIOKitWillTerminate;
5789 #if WITH_IOWAITQUIET_EXTENSIONS
5790 dopanic = (dopanic && (loops >= (timeoutExtensions - 1)));
5791 #endif
5792 assert(panicString != NULL);
5793 if (multipleEntries) {
5794 snprintf(panicString, panicStringLen,
5795 "%s[%d], (%llds): multiple entries holding the registry busy, IOKit termination queue depth %u: %s",
5796 pendingRequests ? "IOKit Daemon (" kIOKitDaemonName ") stall" : "busy timeout",
5797 loops, timeout / 1000000000ULL,
5798 (uint32_t)gIOTerminateWork,
5799 busyEntriesString ? busyEntriesString : "");
5800 } else {
5801 snprintf(panicString, panicStringLen,
5802 "%s[%d], (%llds): %s",
5803 pendingRequests ? "IOKit Daemon (" kIOKitDaemonName ") stall" : "busy timeout",
5804 loops, timeout / 1000000000ULL,
5805 busyEntriesString ? busyEntriesString : "");
5806 }
5807
5808 IOLog("%s\n", panicString);
5809 if (dopanic) {
5810 panic("%s", panicString);
5811 } else if (!loops) {
5812 getPMRootDomain()->startSpinDump(1);
5813 }
5814 }
5815
5816 if (busyEntriesString) {
5817 IODeleteData(busyEntriesString, char, busyEntriesStringLen);
5818 }
5819 if (panicString) {
5820 IODeleteData(panicString, char, panicStringLen);
5821 }
5822
5823 return ret;
5824 }
5825
5826 IOReturn
waitQuiet(uint64_t timeout)5827 IOService::waitQuiet( uint64_t timeout )
5828 {
5829 return waitQuietWithOptions(timeout);
5830 }
5831
5832 IOReturn
waitQuiet(mach_timespec_t * timeout)5833 IOService::waitQuiet( mach_timespec_t * timeout )
5834 {
5835 uint64_t timeoutNS;
5836
5837 if (timeout) {
5838 timeoutNS = timeout->tv_sec;
5839 timeoutNS *= kSecondScale;
5840 timeoutNS += timeout->tv_nsec;
5841 } else {
5842 timeoutNS = UINT64_MAX;
5843 }
5844
5845 return waitQuiet(timeoutNS);
5846 }
5847
5848 bool
serializeProperties(OSSerialize * s) const5849 IOService::serializeProperties( OSSerialize * s ) const
5850 {
5851 #if 0
5852 ((IOService *)this)->setProperty(((IOService *)this)->__state,
5853 sizeof(__state), "__state");
5854 #endif
5855 return super::serializeProperties(s);
5856 }
5857
5858 void
resetRematchProperties()5859 IOService::resetRematchProperties()
5860 {
5861 removeProperty(gIORematchCountKey);
5862 removeProperty(gIORematchPersonalityKey);
5863 }
5864
5865
5866 void
main(void * arg,wait_result_t result)5867 _IOConfigThread::main(void * arg, wait_result_t result)
5868 {
5869 _IOConfigThread * self = (_IOConfigThread *) arg;
5870 _IOServiceJob * job;
5871 IOService * nub;
5872 bool alive = true;
5873 kern_return_t kr;
5874 thread_precedence_policy_data_t precedence = { -1 };
5875
5876 kr = thread_policy_set(current_thread(),
5877 THREAD_PRECEDENCE_POLICY,
5878 (thread_policy_t) &precedence,
5879 THREAD_PRECEDENCE_POLICY_COUNT);
5880 if (KERN_SUCCESS != kr) {
5881 IOLog("thread_policy_set(%d)\n", kr);
5882 }
5883
5884 do {
5885 // randomDelay();
5886
5887 semaphore_wait( gJobsSemaphore );
5888
5889 IOTakeLock( gJobsLock );
5890 job = (_IOServiceJob *) gJobs->getFirstObject();
5891 job->retain();
5892 gJobs->removeObject(job);
5893 if (job) {
5894 gOutstandingJobs--;
5895 // gNumConfigThreads--; // we're out of service
5896 gNumWaitingThreads--; // we're out of service
5897 }
5898 IOUnlock( gJobsLock );
5899
5900 if (job) {
5901 nub = job->nub;
5902
5903 if (gIOKitDebug & kIOLogConfig) {
5904 LOG("config(%p): starting on %s, %d\n",
5905 IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
5906 }
5907
5908 switch (job->type) {
5909 case kMatchNubJob:
5910 nub->doServiceMatch( job->options );
5911 break;
5912
5913 default:
5914 LOG("config(%p): strange type (%d)\n",
5915 IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type );
5916 break;
5917 }
5918
5919 if (job->options & kIOServiceDextRequirePowerForMatching) {
5920 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
5921 }
5922
5923 OSSafeReleaseNULL(nub);
5924 OSSafeReleaseNULL(job);
5925
5926 IOTakeLock( gJobsLock );
5927 alive = (gOutstandingJobs > gNumWaitingThreads);
5928 if (alive) {
5929 gNumWaitingThreads++; // back in service
5930 }
5931 // gNumConfigThreads++;
5932 else {
5933 if (0 == --gNumConfigThreads) {
5934 // IOLog("MATCH IDLE\n");
5935 IOLockWakeup( gJobsLock, (event_t) &gNumConfigThreads, /* one-thread */ false );
5936 }
5937 }
5938 IOUnlock( gJobsLock );
5939 }
5940 } while (alive);
5941
5942 if (gIOKitDebug & kIOLogConfig) {
5943 LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
5944 }
5945
5946 self->release();
5947 }
5948
5949 IOReturn
waitMatchIdle(UInt32 msToWait)5950 IOService::waitMatchIdle( UInt32 msToWait )
5951 {
5952 bool wait;
5953 int waitResult = THREAD_AWAKENED;
5954 bool computeDeadline = true;
5955 AbsoluteTime deadline;
5956
5957 IOLockLock( gJobsLock );
5958 do {
5959 wait = (0 != gNumConfigThreads);
5960 if (wait) {
5961 if (msToWait) {
5962 if (computeDeadline) {
5963 clock_interval_to_deadline(
5964 msToWait, kMillisecondScale, &deadline );
5965 computeDeadline = false;
5966 }
5967 waitResult = IOLockSleepDeadline( gJobsLock, &gNumConfigThreads,
5968 deadline, THREAD_UNINT );
5969 } else {
5970 waitResult = IOLockSleep( gJobsLock, &gNumConfigThreads,
5971 THREAD_UNINT );
5972 }
5973 }
5974 } while (wait && (waitResult != THREAD_TIMED_OUT));
5975 IOLockUnlock( gJobsLock );
5976
5977 if (waitResult == THREAD_TIMED_OUT) {
5978 return kIOReturnTimeout;
5979 } else {
5980 return kIOReturnSuccess;
5981 }
5982 }
5983
5984 void
cpusRunning(void)5985 IOService::cpusRunning(void)
5986 {
5987 gCPUsRunning = true;
5988 }
5989
5990 void
pingConfig(_IOServiceJob * job)5991 _IOServiceJob::pingConfig( _IOServiceJob * job )
5992 {
5993 int count;
5994 bool create;
5995 IOService * nub;
5996
5997 assert( job );
5998 nub = job->nub;
5999
6000 IOTakeLock( gJobsLock );
6001
6002 gOutstandingJobs++;
6003 if (nub == gIOResources) {
6004 gJobs->setFirstObject( job );
6005 } else {
6006 gJobs->setLastObject( job );
6007 }
6008
6009 count = gNumWaitingThreads;
6010 // if( gNumConfigThreads) count++;// assume we're called from a config thread
6011
6012 create = ((gOutstandingJobs > count)
6013 && ((gNumConfigThreads < gMaxConfigThreads)
6014 || (nub == gIOResources)
6015 || !gCPUsRunning));
6016 if (create) {
6017 gNumConfigThreads++;
6018 gNumWaitingThreads++;
6019 if (gNumConfigThreads > gHighNumConfigThreads) {
6020 gHighNumConfigThreads = gNumConfigThreads;
6021 }
6022 }
6023
6024 IOUnlock( gJobsLock );
6025
6026 job->release();
6027
6028 if (create) {
6029 if (gIOKitDebug & kIOLogConfig) {
6030 LOG("config(%d): creating\n", gNumConfigThreads - 1);
6031 }
6032 _IOConfigThread::configThread(nub->getName());
6033 }
6034
6035 semaphore_signal( gJobsSemaphore );
6036 }
6037
6038 struct IOServiceMatchContext {
6039 OSDictionary * table;
6040 OSObject * result;
6041 uint32_t options;
6042 uint32_t state;
6043 uint32_t count;
6044 uint32_t done;
6045 };
6046
6047 bool
instanceMatch(const OSObject * entry,void * context)6048 IOService::instanceMatch(const OSObject * entry, void * context)
6049 {
6050 IOServiceMatchContext * ctx = (typeof(ctx))context;
6051 IOService * service = (typeof(service))entry;
6052 OSDictionary * table = ctx->table;
6053 uint32_t options = ctx->options;
6054 uint32_t state = ctx->state;
6055 uint32_t done;
6056 bool match;
6057
6058 done = 0;
6059 do{
6060 match = ((state == (state & service->__state[0]))
6061 && (0 == (service->__state[0] & kIOServiceInactiveState)));
6062 if (!match) {
6063 break;
6064 }
6065
6066 match = service->matchInternal(table, options, &done);
6067 if (match) {
6068 ctx->count += table->getCount();
6069 ctx->done += done;
6070 }
6071 }while (false);
6072 if (!match) {
6073 return false;
6074 }
6075
6076 if ((kIONotifyOnce & options) && (ctx->done == ctx->count)) {
6077 service->retain();
6078 ctx->result = service;
6079 return true;
6080 } else if (!ctx->result) {
6081 ctx->result = OSSet::withObjects((const OSObject **) &service, 1, 1);
6082 } else {
6083 ((OSSet *)ctx->result)->setObject(service);
6084 }
6085 return false;
6086 }
6087
6088 // internal - call with gNotificationLock
6089 OSObject *
copyExistingServices(OSDictionary * matching,IOOptionBits inState,IOOptionBits options)6090 IOService::copyExistingServices( OSDictionary * matching,
6091 IOOptionBits inState, IOOptionBits options )
6092 {
6093 OSObject * current = NULL;
6094 OSIterator * iter;
6095 IOService * service;
6096 OSObject * obj;
6097 OSString * str;
6098
6099 if (!matching) {
6100 return NULL;
6101 }
6102
6103 #if MATCH_DEBUG
6104 OSSerialize * s = OSSerialize::withCapacity(128);
6105 matching->serialize(s);
6106 #endif
6107
6108 if ((obj = matching->getObject(gIOProviderClassKey))
6109 && gIOResourcesKey
6110 && gIOResourcesKey->isEqualTo(obj)
6111 && (service = gIOResources)) {
6112 if ((inState == (service->__state[0] & inState))
6113 && (0 == (service->__state[0] & kIOServiceInactiveState))
6114 && service->matchPassive(matching, options)) {
6115 if (options & kIONotifyOnce) {
6116 service->retain();
6117 current = service;
6118 } else {
6119 current = OSSet::withObjects((const OSObject **) &service, 1, 1 );
6120 }
6121 }
6122 } else {
6123 IOServiceMatchContext ctx;
6124
6125 options |= kIOServiceClassDone;
6126 ctx.table = matching;
6127 ctx.state = inState;
6128 ctx.count = 0;
6129 ctx.done = 0;
6130 ctx.options = options;
6131 ctx.result = NULL;
6132
6133 if ((str = OSDynamicCast(OSString, obj))) {
6134 const OSSymbol * sym = OSSymbol::withString(str);
6135 OSMetaClass::applyToInstancesOfClassName(sym, instanceMatch, &ctx);
6136 sym->release();
6137 } else {
6138 IOService::gMetaClass.applyToInstances(instanceMatch, &ctx);
6139 }
6140
6141 if (((!(options & kIONotifyOnce) || !ctx.result))
6142 && matching->getObject(gIOCompatibilityMatchKey)) {
6143 IOServiceCompatibility::gMetaClass.applyToInstances(instanceMatch, &ctx);
6144 }
6145
6146 current = ctx.result;
6147 options |= kIOServiceInternalDone;
6148 if (current && (ctx.done != ctx.count)) {
6149 OSSet * source = OSDynamicCast(OSSet, current);
6150 current = NULL;
6151 while ((service = (IOService *) source->getAnyObject())) {
6152 if (service->matchPassive(matching, options)) {
6153 if (options & kIONotifyOnce) {
6154 service->retain();
6155 current = service;
6156 break;
6157 }
6158 if (current) {
6159 ((OSSet *)current)->setObject( service );
6160 } else {
6161 current = OSSet::withObjects(
6162 (const OSObject **) &service, 1, 1 );
6163 }
6164 }
6165 source->removeObject(service);
6166 }
6167 source->release();
6168 }
6169 }
6170
6171 #if MATCH_DEBUG
6172 {
6173 OSObject * _current = 0;
6174
6175 iter = IORegistryIterator::iterateOver( gIOServicePlane,
6176 kIORegistryIterateRecursively );
6177 if (iter) {
6178 do {
6179 iter->reset();
6180 while ((service = (IOService *) iter->getNextObject())) {
6181 if ((inState == (service->__state[0] & inState))
6182 && (0 == (service->__state[0] & kIOServiceInactiveState))
6183 && service->matchPassive(matching, 0)) {
6184 if (options & kIONotifyOnce) {
6185 service->retain();
6186 _current = service;
6187 break;
6188 }
6189 if (_current) {
6190 ((OSSet *)_current)->setObject( service );
6191 } else {
6192 _current = OSSet::withObjects(
6193 (const OSObject **) &service, 1, 1 );
6194 }
6195 }
6196 }
6197 } while (!service && !iter->isValid());
6198 iter->release();
6199 }
6200
6201 if (((current != 0) != (_current != 0))
6202 || (current && _current && !current->isEqualTo(_current))) {
6203 OSSerialize * s1 = OSSerialize::withCapacity(128);
6204 OSSerialize * s2 = OSSerialize::withCapacity(128);
6205 current->serialize(s1);
6206 _current->serialize(s2);
6207 kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current),
6208 IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text());
6209 s1->release();
6210 s2->release();
6211 }
6212
6213 if (_current) {
6214 _current->release();
6215 }
6216 }
6217
6218 s->release();
6219 #endif
6220
6221 if (current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
6222 iter = OSCollectionIterator::withCollection((OSSet *)current );
6223 current->release();
6224 current = iter;
6225 }
6226
6227 return current;
6228 }
6229
6230 // public version
6231 OSIterator *
getMatchingServices(OSDictionary * matching)6232 IOService::getMatchingServices( OSDictionary * matching )
6233 {
6234 OSIterator * iter;
6235
6236 // is a lock even needed?
6237 LOCKWRITENOTIFY();
6238
6239 iter = (OSIterator *) copyExistingServices( matching,
6240 kIOServiceMatchedState );
6241
6242 UNLOCKNOTIFY();
6243
6244 return iter;
6245 }
6246
6247 IOService *
copyMatchingService(OSDictionary * matching)6248 IOService::copyMatchingService( OSDictionary * matching )
6249 {
6250 IOService * service;
6251
6252 // is a lock even needed?
6253 LOCKWRITENOTIFY();
6254
6255 service = (IOService *) copyExistingServices( matching,
6256 kIOServiceMatchedState, kIONotifyOnce );
6257
6258 UNLOCKNOTIFY();
6259
6260 return service;
6261 }
6262
6263 struct _IOServiceMatchingNotificationHandlerRef {
6264 IOServiceNotificationHandler handler;
6265 void * ref;
6266 };
6267
6268 static bool
_IOServiceMatchingNotificationHandler(void * target,void * refCon,IOService * newService,IONotifier * notifier)6269 _IOServiceMatchingNotificationHandler( void * target, void * refCon,
6270 IOService * newService,
6271 IONotifier * notifier )
6272 {
6273 return (*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService);
6274 }
6275
6276 // internal - call with gNotificationLock
6277 IONotifier *
setNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority)6278 IOService::setNotification(
6279 const OSSymbol * type, OSDictionary * matching,
6280 IOServiceMatchingNotificationHandler handler, void * target, void * ref,
6281 SInt32 priority )
6282 {
6283 _IOServiceNotifier * notify = NULL;
6284 OSOrderedSet * set;
6285
6286 if (!matching) {
6287 return NULL;
6288 }
6289
6290 notify = new _IOServiceNotifier;
6291 if (notify && !notify->init()) {
6292 notify->release();
6293 notify = NULL;
6294 }
6295
6296 if (notify) {
6297 notify->handler = handler;
6298 notify->target = target;
6299 notify->type = type;
6300 notify->matching = matching;
6301 matching->retain();
6302 if (handler == &_IOServiceMatchingNotificationHandler) {
6303 notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
6304 notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
6305 } else {
6306 notify->ref = ref;
6307 }
6308 notify->priority = priority;
6309 notify->state = kIOServiceNotifyEnable;
6310 queue_init( ¬ify->handlerInvocations );
6311
6312 ////// queue
6313
6314 if (NULL == (set = (OSOrderedSet *) gNotifications->getObject( type ))) {
6315 set = OSOrderedSet::withCapacity( 1,
6316 IONotifyOrdering, NULL );
6317 if (set) {
6318 gNotifications->setObject( type, set );
6319 set->release();
6320 }
6321 }
6322 notify->whence = set;
6323 if (set) {
6324 set->setObject( notify );
6325 }
6326 }
6327
6328 return notify;
6329 }
6330
6331 // internal - call with gNotificationLock
6332 IONotifier *
doInstallNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority,OSIterator ** existing)6333 IOService::doInstallNotification(
6334 const OSSymbol * type, OSDictionary * matching,
6335 IOServiceMatchingNotificationHandler handler,
6336 void * target, void * ref,
6337 SInt32 priority, OSIterator ** existing )
6338 {
6339 OSIterator * exist;
6340 IONotifier * notify;
6341 IOOptionBits inState;
6342
6343 if (!matching) {
6344 return NULL;
6345 }
6346
6347 if (type == gIOPublishNotification) {
6348 inState = kIOServiceRegisteredState;
6349 } else if (type == gIOFirstPublishNotification) {
6350 inState = kIOServiceFirstPublishState;
6351 } else if (type == gIOMatchedNotification) {
6352 inState = kIOServiceMatchedState;
6353 } else if (type == gIOFirstMatchNotification) {
6354 inState = kIOServiceFirstMatchState;
6355 } else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification)) {
6356 inState = 0;
6357 } else {
6358 return NULL;
6359 }
6360
6361 notify = setNotification( type, matching, handler, target, ref, priority );
6362
6363 if (inState) {
6364 // get the current set
6365 exist = (OSIterator *) copyExistingServices( matching, inState );
6366 } else {
6367 exist = NULL;
6368 }
6369
6370 *existing = exist;
6371
6372 return notify;
6373 }
6374
6375 #if !defined(__LP64__)
6376 IONotifier *
installNotification(const OSSymbol * type,OSDictionary * matching,IOServiceNotificationHandler handler,void * target,void * refCon,SInt32 priority,OSIterator ** existing)6377 IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
6378 IOServiceNotificationHandler handler,
6379 void * target, void * refCon,
6380 SInt32 priority, OSIterator ** existing )
6381 {
6382 IONotifier * result;
6383 _IOServiceMatchingNotificationHandlerRef ref;
6384 ref.handler = handler;
6385 ref.ref = refCon;
6386
6387 result = (_IOServiceNotifier *) installNotification( type, matching,
6388 &_IOServiceMatchingNotificationHandler,
6389 target, &ref, priority, existing );
6390 if (result) {
6391 matching->release();
6392 }
6393
6394 return result;
6395 }
6396
6397 #endif /* !defined(__LP64__) */
6398
6399
6400 IONotifier *
installNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority,OSIterator ** existing)6401 IOService::installNotification(
6402 const OSSymbol * type, OSDictionary * matching,
6403 IOServiceMatchingNotificationHandler handler,
6404 void * target, void * ref,
6405 SInt32 priority, OSIterator ** existing )
6406 {
6407 IONotifier * notify;
6408
6409 LOCKWRITENOTIFY();
6410
6411 notify = doInstallNotification( type, matching, handler, target, ref,
6412 priority, existing );
6413
6414 // in case handler remove()s
6415 if (notify) {
6416 notify->retain();
6417 }
6418
6419 UNLOCKNOTIFY();
6420
6421 return notify;
6422 }
6423
6424 IONotifier *
addNotification(const OSSymbol * type,OSDictionary * matching,IOServiceNotificationHandler handler,void * target,void * refCon,SInt32 priority)6425 IOService::addNotification(
6426 const OSSymbol * type, OSDictionary * matching,
6427 IOServiceNotificationHandler handler,
6428 void * target, void * refCon,
6429 SInt32 priority )
6430 {
6431 IONotifier * result;
6432 _IOServiceMatchingNotificationHandlerRef ref;
6433
6434 ref.handler = handler;
6435 ref.ref = refCon;
6436
6437 result = addMatchingNotification(type, matching, &_IOServiceMatchingNotificationHandler,
6438 target, &ref, priority);
6439
6440 if (result) {
6441 matching->release();
6442 }
6443
6444 return result;
6445 }
6446
6447 IONotifier *
addMatchingNotification(const OSSymbol * type,OSDictionary * matching,IOServiceMatchingNotificationHandler handler,void * target,void * ref,SInt32 priority)6448 IOService::addMatchingNotification(
6449 const OSSymbol * type, OSDictionary * matching,
6450 IOServiceMatchingNotificationHandler handler,
6451 void * target, void * ref,
6452 SInt32 priority )
6453 {
6454 OSIterator * existing = NULL;
6455 IONotifier * ret;
6456 _IOServiceNotifier * notify;
6457 IOService * next;
6458
6459 ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
6460 handler, target, ref, priority, &existing );
6461 if (!ret) {
6462 OSSafeReleaseNULL(existing);
6463 return NULL;
6464 }
6465
6466 // send notifications for existing set
6467 if (existing) {
6468 while ((next = (IOService *) existing->getNextObject())) {
6469 if (0 == (next->__state[0] & kIOServiceInactiveState)) {
6470 next->invokeNotifier( notify );
6471 }
6472 }
6473 existing->release();
6474 }
6475
6476 LOCKWRITENOTIFY();
6477 bool removed = (NULL == notify->whence);
6478 notify->release();
6479 if (removed) {
6480 ret = gIOServiceNullNotifier;
6481 }
6482 UNLOCKNOTIFY();
6483
6484 return ret;
6485 }
6486
6487 static bool
IOServiceMatchingNotificationHandlerToBlock(void * target __unused,void * refCon,IOService * newService,IONotifier * notifier)6488 IOServiceMatchingNotificationHandlerToBlock( void * target __unused, void * refCon,
6489 IOService * newService,
6490 IONotifier * notifier )
6491 {
6492 return ((IOServiceMatchingNotificationHandlerBlock) refCon)(newService, notifier);
6493 }
6494
6495 IONotifier *
addMatchingNotification(const OSSymbol * type,OSDictionary * matching,SInt32 priority,IOServiceMatchingNotificationHandlerBlock handler)6496 IOService::addMatchingNotification(
6497 const OSSymbol * type, OSDictionary * matching,
6498 SInt32 priority,
6499 IOServiceMatchingNotificationHandlerBlock handler)
6500 {
6501 IONotifier * notify;
6502 void * block;
6503
6504 block = Block_copy(handler);
6505 if (!block) {
6506 return NULL;
6507 }
6508
6509 notify = addMatchingNotification(type, matching,
6510 &IOServiceMatchingNotificationHandlerToBlock, NULL, block, priority);
6511
6512 if (!notify) {
6513 Block_release(block);
6514 }
6515
6516 return notify;
6517 }
6518
6519 struct IOUserServerCancellationHandlerArgs {
6520 IOService ** ref;
6521 bool canceled;
6522 };
6523
6524 void
userServerCheckInTokenCancellationHandler(__unused IOUserServerCheckInToken * token,void * ref)6525 IOService::userServerCheckInTokenCancellationHandler(
6526 __unused IOUserServerCheckInToken *token,
6527 void *ref)
6528 {
6529 IOUserServerCancellationHandlerArgs * args = (typeof(args))ref;
6530 LOCKWRITENOTIFY();
6531 WAKEUPNOTIFY(args->ref);
6532 args->canceled = true;
6533 UNLOCKNOTIFY();
6534 }
6535
6536 bool
syncNotificationHandler(void *,void * ref,IOService * newService,IONotifier * notifier)6537 IOService::syncNotificationHandler(
6538 void * /* target */, void * ref,
6539 IOService * newService,
6540 IONotifier * notifier )
6541 {
6542 LOCKWRITENOTIFY();
6543 if (!*((IOService **) ref)) {
6544 newService->retain();
6545 (*(IOService **) ref) = newService;
6546 WAKEUPNOTIFY(ref);
6547 }
6548 UNLOCKNOTIFY();
6549
6550 return false;
6551 }
6552
6553 IOService *
waitForMatchingServiceWithToken(OSDictionary * matching,uint64_t timeout,IOUserServerCheckInToken * checkInToken)6554 IOService::waitForMatchingServiceWithToken( OSDictionary * matching,
6555 uint64_t timeout,
6556 IOUserServerCheckInToken * checkInToken)
6557 {
6558 IONotifier * notify = NULL;
6559 // priority doesn't help us much since we need a thread wakeup
6560 SInt32 priority = 0;
6561 IOService * result;
6562 IOUserServerCancellationHandlerArgs cancelArgs;
6563 _IOUserServerCheckInCancellationHandler * cancellationHandler = NULL;
6564
6565 if (!matching) {
6566 return NULL;
6567 }
6568
6569 result = NULL;
6570 cancelArgs.ref = &result;
6571 cancelArgs.canceled = false;
6572
6573 #if DEBUG || DEVELOPMENT
6574 char currentName[MAXTHREADNAMESIZE];
6575 char newName[MAXTHREADNAMESIZE];
6576 OSObject * obj;
6577 OSString * str;
6578 OSDictionary * dict;
6579
6580 currentName[0] = '\0';
6581 if (thread_has_thread_name(current_thread())) {
6582 dict = matching;
6583 obj = matching->getObject(gIOPropertyMatchKey);
6584 if ((dict = OSDynamicCast(OSDictionary, obj))) {
6585 OSObject * result __block = NULL;
6586 dict->iterateObjects(^bool (const OSSymbol * sym, OSObject * value) {
6587 result = __DECONST(OSObject *, sym);
6588 return true;
6589 });
6590 obj = result;
6591 }
6592 if (!obj) {
6593 obj = matching->getObject(gIOResourceMatchKey);
6594 }
6595 if (!obj) {
6596 obj = matching->getObject(gIONameMatchKey);
6597 }
6598 if (!obj) {
6599 obj = matching->getObject(gIOProviderClassKey);
6600 }
6601 if ((str = OSDynamicCast(OSString, obj))) {
6602 thread_get_thread_name(current_thread(), currentName);
6603 snprintf(newName, sizeof(newName), "Waiting_'%s'", str->getCStringNoCopy());
6604 thread_set_thread_name(current_thread(), newName);
6605 }
6606 }
6607 #endif /* DEBUG || DEVELOPMENT */
6608
6609 if (checkInToken) {
6610 cancellationHandler = checkInToken->setCancellationHandler(&IOService::userServerCheckInTokenCancellationHandler,
6611 &cancelArgs);
6612 }
6613
6614 LOCKWRITENOTIFY();
6615 do{
6616 if (cancelArgs.canceled) {
6617 // token was already canceled, no need to wait or find services
6618 break;
6619 }
6620 result = (IOService *) copyExistingServices( matching,
6621 kIOServiceMatchedState, kIONotifyOnce );
6622 if (result) {
6623 break;
6624 }
6625 notify = IOService::setNotification( gIOMatchedNotification, matching,
6626 &IOService::syncNotificationHandler, (void *) NULL,
6627 &result, priority );
6628 if (!notify) {
6629 break;
6630 }
6631 if (UINT64_MAX != timeout) {
6632 AbsoluteTime deadline;
6633 nanoseconds_to_absolutetime(timeout, &deadline);
6634 clock_absolutetime_interval_to_deadline(deadline, &deadline);
6635 SLEEPNOTIFYTO(&result, deadline);
6636 } else {
6637 SLEEPNOTIFY(&result);
6638 }
6639 }while (false);
6640
6641 UNLOCKNOTIFY();
6642
6643 if (checkInToken && cancellationHandler) {
6644 checkInToken->removeCancellationHandler(cancellationHandler);
6645 }
6646
6647 #if DEBUG || DEVELOPMENT
6648 if (currentName[0]) {
6649 thread_set_thread_name(current_thread(), currentName);
6650 }
6651 #endif /* DEBUG || DEVELOPMENT */
6652
6653 if (notify) {
6654 notify->remove(); // dequeues
6655 }
6656
6657 OSSafeReleaseNULL(cancellationHandler);
6658
6659 return result;
6660 }
6661
6662 IOService *
waitForMatchingService(OSDictionary * matching,uint64_t timeout)6663 IOService::waitForMatchingService( OSDictionary * matching,
6664 uint64_t timeout)
6665 {
6666 return IOService::waitForMatchingServiceWithToken(matching, timeout, NULL);
6667 }
6668
6669 IOService *
waitForService(OSDictionary * matching,mach_timespec_t * timeout)6670 IOService::waitForService( OSDictionary * matching,
6671 mach_timespec_t * timeout )
6672 {
6673 IOService * result;
6674 uint64_t timeoutNS;
6675
6676 if (timeout) {
6677 timeoutNS = timeout->tv_sec;
6678 timeoutNS *= kSecondScale;
6679 timeoutNS += timeout->tv_nsec;
6680 } else {
6681 timeoutNS = UINT64_MAX;
6682 }
6683
6684 result = waitForMatchingService(matching, timeoutNS);
6685
6686 matching->release();
6687 if (result) {
6688 result->release();
6689 }
6690
6691 return result;
6692 }
6693
6694 __dead2
6695 void
deliverNotification(const OSSymbol * type,IOOptionBits orNewState,IOOptionBits andNewState)6696 IOService::deliverNotification( const OSSymbol * type,
6697 IOOptionBits orNewState, IOOptionBits andNewState )
6698 {
6699 panic("deliverNotification");
6700 }
6701
6702 OSArray *
copyNotifiers(const OSSymbol * type,IOOptionBits orNewState,IOOptionBits andNewState)6703 IOService::copyNotifiers(const OSSymbol * type,
6704 IOOptionBits orNewState, IOOptionBits andNewState )
6705 {
6706 _IOServiceNotifier * notify;
6707 OSIterator * iter;
6708 OSArray * willSend = NULL;
6709
6710 lockForArbitration();
6711
6712 if ((0 == (__state[0] & kIOServiceInactiveState))
6713 || (type == gIOTerminatedNotification)
6714 || (type == gIOWillTerminateNotification)) {
6715 LOCKREADNOTIFY();
6716
6717 iter = OSCollectionIterator::withCollection((OSOrderedSet *)
6718 gNotifications->getObject( type ));
6719
6720 if (iter) {
6721 while ((notify = (_IOServiceNotifier *) iter->getNextObject())) {
6722 if (matchPassive(notify->matching, 0)
6723 && (kIOServiceNotifyEnable & notify->state)) {
6724 if (NULL == willSend) {
6725 willSend = OSArray::withCapacity(8);
6726 }
6727 if (willSend) {
6728 willSend->setObject( notify );
6729 }
6730 }
6731 }
6732 iter->release();
6733 }
6734 __state[0] = (__state[0] | orNewState) & andNewState;
6735 UNLOCKNOTIFY();
6736 }
6737
6738 unlockForArbitration();
6739
6740 return willSend;
6741 }
6742
6743 IOOptionBits
getState(void) const6744 IOService::getState( void ) const
6745 {
6746 return __state[0];
6747 }
6748
6749 /*
6750 * Helpers to make matching objects for simple cases
6751 */
6752
6753 OSDictionary *
serviceMatching(const OSString * name,OSDictionary * table)6754 IOService::serviceMatching( const OSString * name,
6755 OSDictionary * table )
6756 {
6757 const OSString * str;
6758
6759 str = OSSymbol::withString(name);
6760 if (!str) {
6761 return NULL;
6762 }
6763
6764 if (!table) {
6765 table = OSDictionary::withCapacity( 2 );
6766 }
6767 if (table) {
6768 table->setObject(gIOProviderClassKey, (OSObject *)str );
6769 }
6770 str->release();
6771
6772 return table;
6773 }
6774
6775
6776 OSSharedPtr<OSDictionary>
serviceMatching(const OSString * name,OSSharedPtr<OSDictionary> table)6777 IOService::serviceMatching( const OSString * name,
6778 OSSharedPtr<OSDictionary> table)
6779 {
6780 OSDictionary * result = serviceMatching(name, table.get());
6781 if (table) {
6782 return OSSharedPtr<OSDictionary>(result, OSRetain);
6783 } else {
6784 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6785 }
6786 }
6787
6788
6789 OSDictionary *
serviceMatching(const char * name,OSDictionary * table)6790 IOService::serviceMatching( const char * name,
6791 OSDictionary * table )
6792 {
6793 const OSString * str;
6794
6795 str = OSSymbol::withCString( name );
6796 if (!str) {
6797 return NULL;
6798 }
6799
6800 table = serviceMatching( str, table );
6801 str->release();
6802 return table;
6803 }
6804
6805
6806 OSSharedPtr<OSDictionary>
serviceMatching(const char * className,OSSharedPtr<OSDictionary> table)6807 IOService::serviceMatching( const char * className,
6808 OSSharedPtr<OSDictionary> table)
6809 {
6810 OSDictionary * result = serviceMatching(className, table.get());
6811 if (table) {
6812 return OSSharedPtr<OSDictionary>(result, OSRetain);
6813 } else {
6814 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6815 }
6816 }
6817
6818
6819 OSDictionary *
nameMatching(const OSString * name,OSDictionary * table)6820 IOService::nameMatching( const OSString * name,
6821 OSDictionary * table )
6822 {
6823 if (!table) {
6824 table = OSDictionary::withCapacity( 2 );
6825 }
6826 if (table) {
6827 table->setObject( gIONameMatchKey, (OSObject *)name );
6828 }
6829
6830 return table;
6831 }
6832
6833
6834 OSSharedPtr<OSDictionary>
nameMatching(const OSString * name,OSSharedPtr<OSDictionary> table)6835 IOService::nameMatching( const OSString * name,
6836 OSSharedPtr<OSDictionary> table)
6837 {
6838 OSDictionary * result = nameMatching(name, table.get());
6839 if (table) {
6840 return OSSharedPtr<OSDictionary>(result, OSRetain);
6841 } else {
6842 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6843 }
6844 }
6845
6846
6847 OSDictionary *
nameMatching(const char * name,OSDictionary * table)6848 IOService::nameMatching( const char * name,
6849 OSDictionary * table )
6850 {
6851 const OSString * str;
6852
6853 str = OSSymbol::withCString( name );
6854 if (!str) {
6855 return NULL;
6856 }
6857
6858 table = nameMatching( str, table );
6859 str->release();
6860 return table;
6861 }
6862
6863
6864 OSSharedPtr<OSDictionary>
nameMatching(const char * name,OSSharedPtr<OSDictionary> table)6865 IOService::nameMatching( const char * name,
6866 OSSharedPtr<OSDictionary> table)
6867 {
6868 OSDictionary * result = nameMatching(name, table.get());
6869 if (table) {
6870 return OSSharedPtr<OSDictionary>(result, OSRetain);
6871 } else {
6872 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6873 }
6874 }
6875
6876
6877 OSDictionary *
resourceMatching(const OSString * str,OSDictionary * table)6878 IOService::resourceMatching( const OSString * str,
6879 OSDictionary * table )
6880 {
6881 table = serviceMatching( gIOResourcesKey, table );
6882 if (table) {
6883 table->setObject( gIOResourceMatchKey, (OSObject *) str );
6884 }
6885
6886 return table;
6887 }
6888
6889
6890 OSSharedPtr<OSDictionary>
resourceMatching(const OSString * str,OSSharedPtr<OSDictionary> table)6891 IOService::resourceMatching( const OSString * str,
6892 OSSharedPtr<OSDictionary> table)
6893 {
6894 OSDictionary * result = resourceMatching(str, table.get());
6895 if (table) {
6896 return OSSharedPtr<OSDictionary>(result, OSRetain);
6897 } else {
6898 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6899 }
6900 }
6901
6902
6903 OSDictionary *
resourceMatching(const char * name,OSDictionary * table)6904 IOService::resourceMatching( const char * name,
6905 OSDictionary * table )
6906 {
6907 const OSSymbol * str;
6908
6909 str = OSSymbol::withCString( name );
6910 if (!str) {
6911 return NULL;
6912 }
6913
6914 table = resourceMatching( str, table );
6915 str->release();
6916
6917 return table;
6918 }
6919
6920
6921 OSSharedPtr<OSDictionary>
resourceMatching(const char * name,OSSharedPtr<OSDictionary> table)6922 IOService::resourceMatching( const char * name,
6923 OSSharedPtr<OSDictionary> table)
6924 {
6925 OSDictionary * result = resourceMatching(name, table.get());
6926 if (table) {
6927 return OSSharedPtr<OSDictionary>(result, OSRetain);
6928 } else {
6929 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6930 }
6931 }
6932
6933
6934 OSDictionary *
propertyMatching(const OSSymbol * key,const OSObject * value,OSDictionary * table)6935 IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
6936 OSDictionary * table )
6937 {
6938 OSDictionary * properties;
6939
6940 properties = OSDictionary::withCapacity( 2 );
6941 if (!properties) {
6942 return NULL;
6943 }
6944 properties->setObject( key, value );
6945
6946 if (!table) {
6947 table = OSDictionary::withCapacity( 2 );
6948 }
6949 if (table) {
6950 table->setObject( gIOPropertyMatchKey, properties );
6951 }
6952
6953 properties->release();
6954
6955 return table;
6956 }
6957
6958
6959 OSSharedPtr<OSDictionary>
propertyMatching(const OSSymbol * key,const OSObject * value,OSSharedPtr<OSDictionary> table)6960 IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
6961 OSSharedPtr<OSDictionary> table)
6962 {
6963 OSDictionary * result = propertyMatching(key, value, table.get());
6964 if (table) {
6965 return OSSharedPtr<OSDictionary>(result, OSRetain);
6966 } else {
6967 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6968 }
6969 }
6970
6971
6972 OSDictionary *
registryEntryIDMatching(uint64_t entryID,OSDictionary * table)6973 IOService::registryEntryIDMatching( uint64_t entryID,
6974 OSDictionary * table )
6975 {
6976 OSNumber * num;
6977
6978 num = OSNumber::withNumber( entryID, 64 );
6979 if (!num) {
6980 return NULL;
6981 }
6982
6983 if (!table) {
6984 table = OSDictionary::withCapacity( 2 );
6985 }
6986 if (table) {
6987 table->setObject( gIORegistryEntryIDKey, num );
6988 }
6989
6990 if (num) {
6991 num->release();
6992 }
6993
6994 return table;
6995 }
6996
6997
6998 OSSharedPtr<OSDictionary>
registryEntryIDMatching(uint64_t entryID,OSSharedPtr<OSDictionary> table)6999 IOService::registryEntryIDMatching( uint64_t entryID,
7000 OSSharedPtr<OSDictionary> table)
7001 {
7002 OSDictionary * result = registryEntryIDMatching(entryID, table.get());
7003 if (table) {
7004 return OSSharedPtr<OSDictionary>(result, OSRetain);
7005 } else {
7006 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
7007 }
7008 }
7009
7010
7011
7012 /*
7013 * _IOServiceNotifier
7014 */
7015
7016 // wait for all threads, other than the current one,
7017 // to exit the handler
7018
7019 void
wait()7020 _IOServiceNotifier::wait()
7021 {
7022 _IOServiceNotifierInvocation * next;
7023 bool doWait;
7024
7025 do {
7026 doWait = false;
7027 queue_iterate( &handlerInvocations, next,
7028 _IOServiceNotifierInvocation *, link) {
7029 if (next->thread != current_thread()) {
7030 doWait = true;
7031 break;
7032 }
7033 }
7034 if (doWait) {
7035 state |= kIOServiceNotifyWaiter;
7036 SLEEPNOTIFY(this);
7037 }
7038 } while (doWait);
7039 }
7040
7041 void
free()7042 _IOServiceNotifier::free()
7043 {
7044 assert( queue_empty( &handlerInvocations ));
7045
7046 if (handler == &IOServiceMatchingNotificationHandlerToBlock) {
7047 Block_release(ref);
7048 }
7049
7050 OSObject::free();
7051 }
7052
7053 void
remove()7054 _IOServiceNotifier::remove()
7055 {
7056 LOCKWRITENOTIFY();
7057
7058 if (whence) {
7059 whence->removeObject((OSObject *) this );
7060 whence = NULL;
7061 }
7062 if (matching) {
7063 matching->release();
7064 matching = NULL;
7065 }
7066
7067 state &= ~kIOServiceNotifyEnable;
7068
7069 wait();
7070
7071 UNLOCKNOTIFY();
7072
7073 release();
7074 }
7075
7076 bool
disable()7077 _IOServiceNotifier::disable()
7078 {
7079 bool ret;
7080
7081 LOCKWRITENOTIFY();
7082
7083 ret = (0 != (kIOServiceNotifyEnable & state));
7084 state &= ~kIOServiceNotifyEnable;
7085 if (ret) {
7086 wait();
7087 }
7088
7089 UNLOCKNOTIFY();
7090
7091 return ret;
7092 }
7093
7094 void
enable(bool was)7095 _IOServiceNotifier::enable( bool was )
7096 {
7097 LOCKWRITENOTIFY();
7098 if (was) {
7099 state |= kIOServiceNotifyEnable;
7100 } else {
7101 state &= ~kIOServiceNotifyEnable;
7102 }
7103 UNLOCKNOTIFY();
7104 }
7105
7106
7107 /*
7108 * _IOServiceNullNotifier
7109 */
7110
7111 void
taggedRetain(const void * tag) const7112 _IOServiceNullNotifier::taggedRetain(const void *tag) const
7113 {
7114 }
7115 void
taggedRelease(const void * tag,const int when) const7116 _IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const
7117 {
7118 }
7119 void
free()7120 _IOServiceNullNotifier::free()
7121 {
7122 }
7123 void
wait()7124 _IOServiceNullNotifier::wait()
7125 {
7126 }
7127 void
remove()7128 _IOServiceNullNotifier::remove()
7129 {
7130 }
7131 void
enable(bool was)7132 _IOServiceNullNotifier::enable(bool was)
7133 {
7134 }
7135 bool
disable()7136 _IOServiceNullNotifier::disable()
7137 {
7138 return false;
7139 }
7140
7141 /*
7142 * IOResources
7143 */
7144
7145 IOService *
resources(void)7146 IOResources::resources( void )
7147 {
7148 IOResources * inst;
7149
7150 inst = new IOResources;
7151 if (inst && !inst->init()) {
7152 inst->release();
7153 inst = NULL;
7154 }
7155
7156 return inst;
7157 }
7158
7159 bool
init(OSDictionary * dictionary)7160 IOResources::init( OSDictionary * dictionary )
7161 {
7162 // Do super init first
7163 if (!IOService::init()) {
7164 return false;
7165 }
7166
7167 // Allow PAL layer to publish a value
7168 const char *property_name;
7169 int property_value;
7170
7171 pal_get_resource_property( &property_name, &property_value );
7172
7173 if (property_name) {
7174 OSNumber *num;
7175 const OSSymbol * sym;
7176
7177 if ((num = OSNumber::withNumber(property_value, 32)) != NULL) {
7178 if ((sym = OSSymbol::withCString( property_name)) != NULL) {
7179 this->setProperty( sym, num );
7180 sym->release();
7181 }
7182 num->release();
7183 }
7184 }
7185
7186 return true;
7187 }
7188
7189 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,IOUserClient ** handler)7190 IOResources::newUserClient(task_t owningTask, void * securityID,
7191 UInt32 type, OSDictionary * properties,
7192 IOUserClient ** handler)
7193 {
7194 return kIOReturnUnsupported;
7195 }
7196
7197 IOWorkLoop *
getWorkLoop() const7198 IOResources::getWorkLoop() const
7199 {
7200 // If we are the resource root
7201 // then use the platform's workloop
7202 if (this == (IOResources *) gIOResources) {
7203 return getPlatform()->getWorkLoop();
7204 } else {
7205 return IOService::getWorkLoop();
7206 }
7207 }
7208
7209 static bool
IOResourcesMatchPropertyTable(IOService * resources,OSDictionary * table)7210 IOResourcesMatchPropertyTable(IOService * resources, OSDictionary * table)
7211 {
7212 OSObject * prop;
7213 bool __block ok = true;
7214
7215 prop = table->getObject( gIOResourceMatchKey );
7216 if (prop) {
7217 prop->iterateObjects(^bool (OSObject * obj)
7218 {
7219 OSString *
7220 str = OSDynamicCast(OSString, obj);
7221 ok = (NULL != resources->getProperty(str));
7222 return !ok;
7223 });
7224 } else if ((prop = table->getObject(gIOResourceMatchedKey))) {
7225 OSObject * obj;
7226 OSArray * keys;
7227
7228 obj = resources->copyProperty(gIOResourceMatchedKey);
7229 keys = OSDynamicCast(OSArray, obj);
7230 ok = false;
7231 if (keys) {
7232 // assuming OSSymbol
7233 ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
7234 }
7235 OSSafeReleaseNULL(obj);
7236 }
7237
7238 return ok;
7239 }
7240
7241 bool
matchPropertyTable(OSDictionary * table)7242 IOResources::matchPropertyTable( OSDictionary * table )
7243 {
7244 return IOResourcesMatchPropertyTable(this, table);
7245 }
7246
7247 /*
7248 * IOUserResources
7249 */
7250
7251 IOService *
resources(void)7252 IOUserResources::resources( void )
7253 {
7254 IOUserResources * inst;
7255
7256 inst = OSTypeAlloc(IOUserResources);
7257 if (inst && !inst->init()) {
7258 inst->release();
7259 inst = NULL;
7260 }
7261
7262 return inst;
7263 }
7264
7265 bool
init(OSDictionary * dictionary)7266 IOUserResources::init( OSDictionary * dictionary )
7267 {
7268 // Do super init first
7269 if (!IOService::init()) {
7270 return false;
7271 }
7272 return true;
7273 }
7274
7275 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,IOUserClient ** handler)7276 IOUserResources::newUserClient(task_t owningTask, void * securityID,
7277 UInt32 type, OSDictionary * properties,
7278 IOUserClient ** handler)
7279 {
7280 return kIOReturnUnsupported;
7281 }
7282
7283 IOWorkLoop *
getWorkLoop() const7284 IOUserResources::getWorkLoop() const
7285 {
7286 return getPlatform()->getWorkLoop();
7287 }
7288
7289 bool
matchPropertyTable(OSDictionary * table)7290 IOUserResources::matchPropertyTable( OSDictionary * table )
7291 {
7292 return IOResourcesMatchPropertyTable(this, table);
7293 }
7294
7295 // --
7296
7297 void
consoleLockTimer(thread_call_param_t p0,thread_call_param_t p1)7298 IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
7299 {
7300 IOService::updateConsoleUsers(NULL, 0);
7301 }
7302
7303 void
updateConsoleUsers(OSArray * consoleUsers,IOMessage systemMessage,bool afterUserspaceReboot)7304 IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage, bool afterUserspaceReboot)
7305 {
7306 IORegistryEntry * regEntry;
7307 OSObject * locked = kOSBooleanFalse;
7308 uint32_t idx;
7309 bool publish;
7310 OSDictionary * user;
7311 clock_sec_t now = 0;
7312 clock_usec_t microsecs;
7313
7314 regEntry = IORegistryEntry::getRegistryRoot();
7315
7316 if (!gIOChosenEntry) {
7317 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
7318 }
7319
7320 IOLockLock(gIOConsoleUsersLock);
7321
7322 if (systemMessage) {
7323 sSystemPower = systemMessage;
7324 #if HIBERNATION
7325 if (kIOMessageSystemHasPoweredOn == systemMessage) {
7326 uint32_t lockState = IOHibernateWasScreenLocked();
7327 switch (lockState) {
7328 case 0:
7329 break;
7330 case kIOScreenLockLocked:
7331 case kIOScreenLockFileVaultDialog:
7332 gIOConsoleBooterLockState = kOSBooleanTrue;
7333 break;
7334 case kIOScreenLockNoLock:
7335 gIOConsoleBooterLockState = NULL;
7336 break;
7337 case kIOScreenLockUnlocked:
7338 default:
7339 gIOConsoleBooterLockState = kOSBooleanFalse;
7340 break;
7341 }
7342 }
7343 #endif /* HIBERNATION */
7344 }
7345
7346 if (consoleUsers) {
7347 OSNumber * num = NULL;
7348 bool loginLocked = true;
7349
7350 gIOConsoleLoggedIn = false;
7351 for (idx = 0;
7352 (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
7353 idx++) {
7354 gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(gIOConsoleSessionOnConsoleKey))
7355 && (kOSBooleanTrue == user->getObject(gIOConsoleSessionLoginDoneKey)));
7356
7357 loginLocked &= (kOSBooleanTrue == user->getObject(gIOConsoleSessionScreenIsLockedKey));
7358 if (!num) {
7359 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
7360 }
7361 }
7362 #if HIBERNATION
7363 if (!loginLocked || afterUserspaceReboot) {
7364 gIOConsoleBooterLockState = NULL;
7365 }
7366 IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
7367 (num != NULL), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
7368 gIOConsoleLoggedIn, loginLocked);
7369 #endif /* HIBERNATION */
7370 gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
7371 }
7372
7373 if (!gIOConsoleLoggedIn
7374 || (kIOMessageSystemWillSleep == sSystemPower)
7375 || (kIOMessageSystemPagingOff == sSystemPower)) {
7376 if (afterUserspaceReboot) {
7377 // set "locked" to false after a user space reboot
7378 // because the reboot happens directly after a user
7379 // logs into the machine via fvunlock mode.
7380 locked = kOSBooleanFalse;
7381 } else {
7382 locked = kOSBooleanTrue;
7383 }
7384 }
7385 #if HIBERNATION
7386 else if (gIOConsoleBooterLockState) {
7387 locked = gIOConsoleBooterLockState;
7388 }
7389 #endif /* HIBERNATION */
7390 else if (gIOConsoleLockTime) {
7391 clock_get_calendar_microtime(&now, µsecs);
7392 if (gIOConsoleLockTime > now) {
7393 AbsoluteTime deadline;
7394 clock_sec_t interval;
7395 uint32_t interval32;
7396
7397 interval = (gIOConsoleLockTime - now);
7398 interval32 = (uint32_t) interval;
7399 if (interval32 != interval) {
7400 interval32 = UINT_MAX;
7401 }
7402 clock_interval_to_deadline(interval32, kSecondScale, &deadline);
7403 thread_call_enter_delayed(gIOConsoleLockCallout, deadline);
7404 } else {
7405 locked = kOSBooleanTrue;
7406 }
7407 }
7408
7409 publish = (consoleUsers || (locked != regEntry->getProperty(gIOConsoleLockedKey)));
7410 if (publish) {
7411 regEntry->setProperty(gIOConsoleLockedKey, locked);
7412 if (consoleUsers) {
7413 regEntry->setProperty(gIOConsoleUsersKey, consoleUsers);
7414 }
7415 OSIncrementAtomic( &gIOConsoleUsersSeed );
7416 }
7417
7418 #if HIBERNATION
7419 if (gIOChosenEntry) {
7420 if (locked == kOSBooleanTrue) {
7421 gIOScreenLockState = kIOScreenLockLocked;
7422 } else if (gIOConsoleLockTime) {
7423 gIOScreenLockState = kIOScreenLockUnlocked;
7424 } else {
7425 gIOScreenLockState = kIOScreenLockNoLock;
7426 }
7427 gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
7428
7429 IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
7430 gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != NULL), now, systemMessage);
7431 }
7432 #endif /* HIBERNATION */
7433
7434 IOLockUnlock(gIOConsoleUsersLock);
7435
7436 if (publish) {
7437 publishResource( gIOConsoleUsersSeedKey, gIOConsoleUsersSeedValue );
7438
7439 MessageClientsContext context;
7440
7441 context.service = getServiceRoot();
7442 context.type = kIOMessageConsoleSecurityChange;
7443 context.argument = (void *) regEntry;
7444 context.argSize = 0;
7445
7446 applyToInterestNotifiers(getServiceRoot(), gIOConsoleSecurityInterest,
7447 &messageClientsApplier, &context );
7448 }
7449 }
7450
7451 IOReturn
setProperties(OSObject * properties)7452 IOResources::setProperties( OSObject * properties )
7453 {
7454 IOReturn err;
7455 const OSSymbol * key;
7456 OSDictionary * dict;
7457 OSCollectionIterator * iter;
7458
7459 if (!IOCurrentTaskHasEntitlement(kIOResourcesSetPropertyKey)) {
7460 err = IOUserClient::clientHasPrivilege(current_task(), kIOClientPrivilegeAdministrator);
7461 if (kIOReturnSuccess != err) {
7462 return err;
7463 }
7464 }
7465
7466 dict = OSDynamicCast(OSDictionary, properties);
7467 if (NULL == dict) {
7468 return kIOReturnBadArgument;
7469 }
7470
7471 iter = OSCollectionIterator::withCollection( dict);
7472 if (NULL == iter) {
7473 return kIOReturnBadArgument;
7474 }
7475
7476 while ((key = OSDynamicCast(OSSymbol, iter->getNextObject()))) {
7477 if (gIOConsoleUsersKey == key) {
7478 do{
7479 OSArray * consoleUsers;
7480 consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
7481 if (!consoleUsers) {
7482 continue;
7483 }
7484 IOService::updateConsoleUsers(consoleUsers, 0);
7485 }while (false);
7486 }
7487
7488 publishResource( key, dict->getObject(key));
7489 }
7490
7491 iter->release();
7492
7493 return kIOReturnSuccess;
7494 }
7495
7496 /*
7497 * Helpers for matching dictionaries.
7498 * Keys existing in matching are checked in properties.
7499 * Keys may be a string or OSCollection of IOStrings
7500 */
7501
7502 bool
compareProperty(OSDictionary * matching,const char * key)7503 IOService::compareProperty( OSDictionary * matching,
7504 const char * key )
7505 {
7506 OSObject * value;
7507 OSObject * prop;
7508 bool ok;
7509
7510 value = matching->getObject( key );
7511 if (value) {
7512 prop = copyProperty(key);
7513 ok = value->isEqualTo(prop);
7514 if (prop) {
7515 prop->release();
7516 }
7517 } else {
7518 ok = true;
7519 }
7520
7521 return ok;
7522 }
7523
7524
7525 bool
compareProperty(OSDictionary * matching,const OSString * key)7526 IOService::compareProperty( OSDictionary * matching,
7527 const OSString * key )
7528 {
7529 OSObject * value;
7530 OSObject * prop;
7531 bool ok;
7532
7533 value = matching->getObject( key );
7534 if (value) {
7535 prop = copyProperty(key);
7536 ok = value->isEqualTo(prop);
7537 if (prop) {
7538 prop->release();
7539 }
7540 } else {
7541 ok = true;
7542 }
7543
7544 return ok;
7545 }
7546
7547 #ifndef __clang_analyzer__
7548 // Implementation of this function is hidden from the static analyzer.
7549 // The analyzer was worried about this function's confusing contract over
7550 // the 'keys' parameter. The contract is to either release it or not release it
7551 // depending on whether 'matching' is non-null. Such contracts are discouraged
7552 // but changing it now would break compatibility.
7553 bool
compareProperties(OSDictionary * matching,OSCollection * keys)7554 IOService::compareProperties( OSDictionary * matching,
7555 OSCollection * keys )
7556 {
7557 OSCollectionIterator * iter;
7558 const OSString * key;
7559 bool ok = true;
7560
7561 if (!matching || !keys) {
7562 return false;
7563 }
7564
7565 iter = OSCollectionIterator::withCollection( keys );
7566
7567 if (iter) {
7568 while (ok && (key = OSDynamicCast( OSString, iter->getNextObject()))) {
7569 ok = compareProperty( matching, key );
7570 }
7571
7572 iter->release();
7573 }
7574 keys->release(); // !! consume a ref !!
7575
7576 return ok;
7577 }
7578 #endif // __clang_analyzer__
7579
7580 /* Helper to add a location matching dict to the table */
7581
7582 OSDictionary *
addLocation(OSDictionary * table)7583 IOService::addLocation( OSDictionary * table )
7584 {
7585 OSDictionary * dict;
7586
7587 if (!table) {
7588 return NULL;
7589 }
7590
7591 dict = OSDictionary::withCapacity( 1 );
7592 if (dict) {
7593 bool ok = table->setObject( gIOLocationMatchKey, dict );
7594 dict->release();
7595 if (!ok) {
7596 dict = NULL;
7597 }
7598 }
7599
7600 return dict;
7601 }
7602
7603 /*
7604 * Go looking for a provider to match a location dict.
7605 */
7606
7607 IOService *
matchLocation(IOService *)7608 IOService::matchLocation( IOService * /* client */ )
7609 {
7610 IOService * parent;
7611
7612 parent = getProvider();
7613
7614 if (parent) {
7615 parent = parent->matchLocation( this );
7616 }
7617
7618 return parent;
7619 }
7620
7621 OSDictionary *
_copyPropertiesForMatching(void)7622 IOService::_copyPropertiesForMatching(void)
7623 {
7624 OSDictionary * matchProps;
7625
7626 matchProps = dictionaryWithProperties();
7627 if (matchProps) {
7628 // merge will check the OSDynamicCast
7629 matchProps->merge((const OSDictionary *)matchProps->getObject(gIOUserServicePropertiesKey));
7630 }
7631 return matchProps;
7632 }
7633
7634 bool
matchInternal(OSDictionary * table,uint32_t options,uint32_t * did)7635 IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
7636 {
7637 OSString * matched;
7638 OSObject * obj;
7639 OSString * str;
7640 OSDictionary * matchProps;
7641 IORegistryEntry * entry;
7642 OSNumber * num;
7643 bool match = true;
7644 bool changesOK = (0 != (kIOServiceChangesOK & options));
7645 uint32_t count;
7646 uint32_t done;
7647
7648 do{
7649 count = table->getCount();
7650 done = 0;
7651 matchProps = NULL;
7652 bool isUser;
7653
7654 isUser = (NULL != table->getObject(gIOServiceNotificationUserKey));
7655 if (isUser) {
7656 done++;
7657 match = (0 == (kIOServiceUserInvisibleMatchState & __state[0]));
7658 if ((!match) || (done == count)) {
7659 break;
7660 }
7661 }
7662
7663 if (propertyExists(gIOExclaveAssignedKey)) {
7664 if (!table->getObject(gIOExclaveProxyKey) && !isUser) {
7665 match = false;
7666 break;
7667 }
7668 } else if (table->getObject(gIOExclaveProxyKey)) {
7669 match = false;
7670 break;
7671 }
7672
7673 if (table->getObject(gIOCompatibilityMatchKey)) {
7674 done++;
7675 obj = copyProperty(gIOCompatibilityPropertiesKey);
7676 matchProps = OSDynamicCast(OSDictionary, obj);
7677 if (!matchProps) {
7678 OSSafeReleaseNULL(obj);
7679 }
7680 }
7681
7682 str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
7683 if (str) {
7684 done++;
7685 if (matchProps && (obj = matchProps->getObject(gIOClassKey))) {
7686 match = str->isEqualTo(obj);
7687 } else {
7688 match = ((kIOServiceClassDone & options) || (NULL != metaCast(str)));
7689 }
7690
7691 #if MATCH_DEBUG
7692 match = (0 != metaCast( str ));
7693 if ((kIOServiceClassDone & options) && !match) {
7694 panic("classDone");
7695 }
7696 #endif
7697 if ((!match) || (done == count)) {
7698 break;
7699 }
7700 }
7701
7702 obj = table->getObject( gIONameMatchKey );
7703 if (obj) {
7704 done++;
7705 match = compareNames( obj, changesOK ? &matched : NULL );
7706 if (!match) {
7707 break;
7708 }
7709 if (changesOK && matched) {
7710 // leave a hint as to which name matched
7711 table->setObject( gIONameMatchedKey, matched );
7712 matched->release();
7713 }
7714 if (done == count) {
7715 break;
7716 }
7717 }
7718
7719 str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
7720 if (str) {
7721 const OSSymbol * sym;
7722 done++;
7723 match = false;
7724 sym = copyLocation();
7725 if (sym) {
7726 match = sym->isEqualTo( str );
7727 sym->release();
7728 }
7729 if ((!match) || (done == count)) {
7730 break;
7731 }
7732 }
7733
7734 obj = table->getObject( gIOPropertyMatchKey );
7735 if (obj) {
7736 OSDictionary * nextDict;
7737 OSIterator * iter;
7738 done++;
7739 match = false;
7740 if (!matchProps) {
7741 matchProps = _copyPropertiesForMatching();
7742 }
7743 if (matchProps) {
7744 nextDict = OSDynamicCast( OSDictionary, obj);
7745 if (nextDict) {
7746 iter = NULL;
7747 } else {
7748 iter = OSCollectionIterator::withCollection(
7749 OSDynamicCast(OSCollection, obj));
7750 }
7751
7752 while (nextDict
7753 || (iter && (NULL != (nextDict = OSDynamicCast(OSDictionary,
7754 iter->getNextObject()))))) {
7755 match = matchProps->isEqualTo( nextDict, nextDict);
7756 if (match) {
7757 break;
7758 }
7759 nextDict = NULL;
7760 }
7761 if (iter) {
7762 iter->release();
7763 }
7764 }
7765 if ((!match) || (done == count)) {
7766 break;
7767 }
7768 }
7769
7770 obj = table->getObject( gIOPropertyExistsMatchKey );
7771 if (obj) {
7772 OSString * nextKey;
7773 OSIterator * iter;
7774 done++;
7775 match = false;
7776 if (!matchProps) {
7777 matchProps = _copyPropertiesForMatching();
7778 }
7779 if (matchProps) {
7780 nextKey = OSDynamicCast( OSString, obj);
7781 if (nextKey) {
7782 iter = NULL;
7783 } else {
7784 iter = OSCollectionIterator::withCollection(
7785 OSDynamicCast(OSCollection, obj));
7786 }
7787
7788 while (nextKey
7789 || (iter && (NULL != (nextKey = OSDynamicCast(OSString,
7790 iter->getNextObject()))))) {
7791 match = (NULL != matchProps->getObject(nextKey));
7792 if (match) {
7793 break;
7794 }
7795 nextKey = NULL;
7796 }
7797 if (iter) {
7798 iter->release();
7799 }
7800 }
7801 if ((!match) || (done == count)) {
7802 break;
7803 }
7804 }
7805
7806 str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
7807 if (str) {
7808 done++;
7809 entry = IORegistryEntry::fromPath( str->getCStringNoCopy());
7810 match = (this == entry);
7811 if (entry) {
7812 entry->release();
7813 }
7814 if (!match && matchProps && (obj = matchProps->getObject(gIOPathKey))) {
7815 match = str->isEqualTo(obj);
7816 }
7817 if ((!match) || (done == count)) {
7818 break;
7819 }
7820 }
7821
7822 num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
7823 if (num) {
7824 done++;
7825 match = (getRegistryEntryID() == num->unsigned64BitValue());
7826 if ((!match) || (done == count)) {
7827 break;
7828 }
7829 }
7830
7831 num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
7832 if (num) {
7833 OSIterator * iter;
7834 IOService * service = NULL;
7835 UInt32 serviceCount = 0;
7836
7837 done++;
7838 iter = getClientIterator();
7839 if (iter) {
7840 while ((service = (IOService *) iter->getNextObject())) {
7841 if (kIOServiceInactiveState & service->__state[0]) {
7842 continue;
7843 }
7844 if (NULL == service->getProperty( gIOMatchCategoryKey )) {
7845 continue;
7846 }
7847 ++serviceCount;
7848 }
7849 iter->release();
7850 }
7851 match = (serviceCount == num->unsigned32BitValue());
7852 if ((!match) || (done == count)) {
7853 break;
7854 }
7855 }
7856
7857 #define propMatch(key) \
7858 obj = table->getObject(key); \
7859 if (obj) \
7860 { \
7861 OSObject * prop; \
7862 done++; \
7863 prop = copyProperty(key); \
7864 match = obj->isEqualTo(prop); \
7865 if (prop) prop->release(); \
7866 if ((!match) || (done == count)) break; \
7867 }
7868 propMatch(gIOBSDNameKey)
7869 propMatch(gIOBSDMajorKey)
7870 propMatch(gIOBSDMinorKey)
7871 propMatch(gIOBSDUnitKey)
7872 #undef propMatch
7873 }while (false);
7874
7875 OSSafeReleaseNULL(matchProps);
7876
7877 if (did) {
7878 *did = done;
7879 }
7880 return match;
7881 }
7882
7883 bool
passiveMatch(OSDictionary * table,bool changesOK)7884 IOService::passiveMatch( OSDictionary * table, bool changesOK )
7885 {
7886 return matchPassive(table, changesOK ? kIOServiceChangesOK : 0);
7887 }
7888
7889 bool
matchPassive(OSDictionary * table,uint32_t options)7890 IOService::matchPassive(OSDictionary * table, uint32_t options)
7891 {
7892 IOService * where;
7893 OSDictionary * nextTable;
7894 SInt32 score;
7895 OSNumber * newPri;
7896 bool match = true;
7897 bool matchParent = false;
7898 uint32_t count;
7899 uint32_t done;
7900
7901 assert( table );
7902
7903 #if defined(XNU_TARGET_OS_OSX)
7904 OSArray* aliasServiceRegIds = NULL;
7905 IOService* foundAlternateService = NULL;
7906 #endif /* defined(XNU_TARGET_OS_OSX) */
7907
7908 #if MATCH_DEBUG
7909 OSDictionary * root = table;
7910 #endif
7911
7912 where = this;
7913 do{
7914 do{
7915 count = table->getCount();
7916 if (!(kIOServiceInternalDone & options)) {
7917 match = where->matchInternal(table, options, &done);
7918 // don't call family if we've done all the entries in the table
7919 if ((!match) || (done == count)) {
7920 break;
7921 }
7922 }
7923
7924 // pass in score from property table
7925 score = IOServiceObjectOrder( table, (void *) gIOProbeScoreKey);
7926
7927 // do family specific matching
7928 match = where->matchPropertyTable( table, &score );
7929
7930 if (!match) {
7931 #if IOMATCHDEBUG
7932 if (kIOLogMatch & getDebugFlags( table )) {
7933 LOG("%s: family specific matching fails\n", where->getName());
7934 }
7935 #endif
7936 break;
7937 }
7938
7939 if (kIOServiceChangesOK & options) {
7940 // save the score
7941 newPri = OSNumber::withNumber( score, 32 );
7942 if (newPri) {
7943 table->setObject( gIOProbeScoreKey, newPri );
7944 newPri->release();
7945 }
7946 }
7947
7948 options = 0;
7949 matchParent = false;
7950
7951 nextTable = OSDynamicCast(OSDictionary,
7952 table->getObject( gIOParentMatchKey ));
7953 if (nextTable) {
7954 // look for a matching entry anywhere up to root
7955 match = false;
7956 matchParent = true;
7957 table = nextTable;
7958 break;
7959 }
7960
7961 table = OSDynamicCast(OSDictionary,
7962 table->getObject( gIOLocationMatchKey ));
7963 if (table) {
7964 // look for a matching entry at matchLocation()
7965 match = false;
7966 where = where->getProvider();
7967 if (where && (where = where->matchLocation(where))) {
7968 continue;
7969 }
7970 }
7971 break;
7972 }while (true);
7973
7974 if (match == true) {
7975 break;
7976 }
7977
7978 if (matchParent == true) {
7979 #if defined(XNU_TARGET_OS_OSX)
7980 // check if service has an alias to search its other "parents" if a parent match isn't found
7981 OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey);
7982 OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop);
7983 if (alternateRegistryID != NULL) {
7984 if (aliasServiceRegIds == NULL) {
7985 aliasServiceRegIds = OSArray::withCapacity(sizeof(alternateRegistryID));
7986 }
7987 aliasServiceRegIds->setObject(alternateRegistryID);
7988 }
7989 OSSafeReleaseNULL(prop);
7990 #endif /* defined(XNU_TARGET_OS_OSX) */
7991 } else {
7992 break;
7993 }
7994
7995 where = where->getProvider();
7996 #if defined(XNU_TARGET_OS_OSX)
7997 if (where == NULL) {
7998 // there were no matching parent services, check to see if there are aliased services that have a matching parent
7999 if (aliasServiceRegIds != NULL) {
8000 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
8001 if (numAliasedServices != 0) {
8002 OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
8003 if (alternateRegistryID != NULL) {
8004 OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(alternateRegistryID->unsigned64BitValue());
8005 aliasServiceRegIds->removeObject(numAliasedServices - 1);
8006 if (alternateMatchingDict != NULL) {
8007 OSSafeReleaseNULL(foundAlternateService);
8008 foundAlternateService = IOService::copyMatchingService(alternateMatchingDict);
8009 alternateMatchingDict->release();
8010 if (foundAlternateService != NULL) {
8011 where = foundAlternateService;
8012 }
8013 }
8014 }
8015 }
8016 }
8017 }
8018 #endif /* defined(XNU_TARGET_OS_OSX) */
8019 }while (where != NULL);
8020
8021 #if defined(XNU_TARGET_OS_OSX)
8022 OSSafeReleaseNULL(foundAlternateService);
8023 OSSafeReleaseNULL(aliasServiceRegIds);
8024 #endif /* defined(XNU_TARGET_OS_OSX) */
8025
8026 #if MATCH_DEBUG
8027 if (where != this) {
8028 OSSerialize * s = OSSerialize::withCapacity(128);
8029 root->serialize(s);
8030 kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
8031 s->release();
8032 }
8033 #endif
8034
8035 return match;
8036 }
8037
8038
8039 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,IOUserClient ** handler)8040 IOService::newUserClient( task_t owningTask, void * securityID,
8041 UInt32 type, OSDictionary * properties,
8042 IOUserClient ** handler )
8043 {
8044 const OSSymbol *userClientClass = NULL;
8045 IOUserClient *client;
8046 OSObject *prop;
8047 OSObject *temp;
8048
8049 if (reserved && reserved->uvars && reserved->uvars->userServer) {
8050 return reserved->uvars->userServer->serviceNewUserClient(this, owningTask, securityID, type, properties, handler);
8051 }
8052
8053 if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler )) {
8054 return kIOReturnSuccess;
8055 }
8056
8057 // First try my own properties for a user client class name
8058 prop = copyProperty(gIOUserClientClassKey);
8059 if (prop) {
8060 if (OSDynamicCast(OSSymbol, prop)) {
8061 userClientClass = (const OSSymbol *) prop;
8062 prop = NULL;
8063 } else if (OSDynamicCast(OSString, prop)) {
8064 userClientClass = OSSymbol::withString((OSString *) prop);
8065 OSSafeReleaseNULL(prop);
8066 if (userClientClass) {
8067 setProperty(gIOUserClientClassKey,
8068 (OSObject *) userClientClass);
8069 }
8070 } else {
8071 OSSafeReleaseNULL(prop);
8072 }
8073 }
8074
8075 // Didn't find one so lets just bomb out now without further ado.
8076 if (!userClientClass) {
8077 return kIOReturnUnsupported;
8078 }
8079
8080 // This reference is consumed by the IOServiceOpen call
8081 temp = OSMetaClass::allocClassWithName(userClientClass);
8082 OSSafeReleaseNULL(userClientClass);
8083 if (!temp) {
8084 return kIOReturnNoMemory;
8085 }
8086
8087 if (OSDynamicCast(IOUserClient, temp)) {
8088 client = (IOUserClient *) temp;
8089 } else {
8090 temp->release();
8091 return kIOReturnUnsupported;
8092 }
8093
8094 if (!client->initWithTask(owningTask, securityID, type, properties)) {
8095 client->release();
8096 return kIOReturnBadArgument;
8097 }
8098
8099 if (!client->attach(this)) {
8100 client->release();
8101 return kIOReturnUnsupported;
8102 }
8103
8104 if (!client->start(this)) {
8105 client->detach(this);
8106 client->release();
8107 return kIOReturnUnsupported;
8108 }
8109
8110 *handler = client;
8111 return kIOReturnSuccess;
8112 }
8113
8114 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSDictionary * properties,OSSharedPtr<IOUserClient> & handler)8115 IOService::newUserClient( task_t owningTask, void * securityID,
8116 UInt32 type, OSDictionary * properties,
8117 OSSharedPtr<IOUserClient>& handler )
8118 {
8119 IOUserClient* handlerRaw = NULL;
8120 IOReturn result = newUserClient(owningTask, securityID, type, properties, &handlerRaw);
8121 handler.reset(handlerRaw, OSNoRetain);
8122 return result;
8123 }
8124
8125 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,IOUserClient ** handler)8126 IOService::newUserClient( task_t owningTask, void * securityID,
8127 UInt32 type, IOUserClient ** handler )
8128 {
8129 return kIOReturnUnsupported;
8130 }
8131
8132 IOReturn
newUserClient(task_t owningTask,void * securityID,UInt32 type,OSSharedPtr<IOUserClient> & handler)8133 IOService::newUserClient( task_t owningTask, void * securityID,
8134 UInt32 type, OSSharedPtr<IOUserClient>& handler )
8135 {
8136 IOUserClient* handlerRaw = nullptr;
8137 IOReturn result = IOService::newUserClient(owningTask, securityID, type, &handlerRaw);
8138 handler.reset(handlerRaw, OSNoRetain);
8139 return result;
8140 }
8141
8142
8143 IOReturn
requestProbe(IOOptionBits options)8144 IOService::requestProbe( IOOptionBits options )
8145 {
8146 return kIOReturnUnsupported;
8147 }
8148
8149 bool
hasUserServer() const8150 IOService::hasUserServer() const
8151 {
8152 return reserved && reserved->uvars && reserved->uvars->userServer;
8153 }
8154
8155 /*
8156 * Convert an IOReturn to text. Subclasses which add additional
8157 * IOReturn's should override this method and call
8158 * super::stringFromReturn if the desired value is not found.
8159 */
8160
8161 const char *
stringFromReturn(IOReturn rtn)8162 IOService::stringFromReturn( IOReturn rtn )
8163 {
8164 static const IONamedValue IOReturn_values[] = {
8165 {kIOReturnSuccess, "success" },
8166 {kIOReturnError, "general error" },
8167 {kIOReturnNoMemory, "memory allocation error" },
8168 {kIOReturnNoResources, "resource shortage" },
8169 {kIOReturnIPCError, "Mach IPC failure" },
8170 {kIOReturnNoDevice, "no such device" },
8171 {kIOReturnNotPrivileged, "privilege violation" },
8172 {kIOReturnBadArgument, "invalid argument" },
8173 {kIOReturnLockedRead, "device is read locked" },
8174 {kIOReturnLockedWrite, "device is write locked" },
8175 {kIOReturnExclusiveAccess, "device is exclusive access" },
8176 {kIOReturnBadMessageID, "bad IPC message ID" },
8177 {kIOReturnUnsupported, "unsupported function" },
8178 {kIOReturnVMError, "virtual memory error" },
8179 {kIOReturnInternalError, "internal driver error" },
8180 {kIOReturnIOError, "I/O error" },
8181 {kIOReturnCannotLock, "cannot acquire lock" },
8182 {kIOReturnNotOpen, "device is not open" },
8183 {kIOReturnNotReadable, "device is not readable" },
8184 {kIOReturnNotWritable, "device is not writeable" },
8185 {kIOReturnNotAligned, "alignment error" },
8186 {kIOReturnBadMedia, "media error" },
8187 {kIOReturnStillOpen, "device is still open" },
8188 {kIOReturnRLDError, "rld failure" },
8189 {kIOReturnDMAError, "DMA failure" },
8190 {kIOReturnBusy, "device is busy" },
8191 {kIOReturnTimeout, "I/O timeout" },
8192 {kIOReturnOffline, "device is offline" },
8193 {kIOReturnNotReady, "device is not ready" },
8194 {kIOReturnNotAttached, "device/channel is not attached" },
8195 {kIOReturnNoChannels, "no DMA channels available" },
8196 {kIOReturnNoSpace, "no space for data" },
8197 {kIOReturnPortExists, "device port already exists" },
8198 {kIOReturnCannotWire, "cannot wire physical memory" },
8199 {kIOReturnNoInterrupt, "no interrupt attached" },
8200 {kIOReturnNoFrames, "no DMA frames enqueued" },
8201 {kIOReturnMessageTooLarge, "message is too large" },
8202 {kIOReturnNotPermitted, "operation is not permitted" },
8203 {kIOReturnNoPower, "device is without power" },
8204 {kIOReturnNoMedia, "media is not present" },
8205 {kIOReturnUnformattedMedia, "media is not formatted" },
8206 {kIOReturnUnsupportedMode, "unsupported mode" },
8207 {kIOReturnUnderrun, "data underrun" },
8208 {kIOReturnOverrun, "data overrun" },
8209 {kIOReturnDeviceError, "device error" },
8210 {kIOReturnNoCompletion, "no completion routine" },
8211 {kIOReturnAborted, "operation was aborted" },
8212 {kIOReturnNoBandwidth, "bus bandwidth would be exceeded" },
8213 {kIOReturnNotResponding, "device is not responding" },
8214 {kIOReturnInvalid, "unanticipated driver error" },
8215 {0, NULL }
8216 };
8217
8218 return IOFindNameForValue(rtn, IOReturn_values);
8219 }
8220
8221 /*
8222 * Convert an IOReturn to an errno.
8223 */
8224 int
errnoFromReturn(IOReturn rtn)8225 IOService::errnoFromReturn( IOReturn rtn )
8226 {
8227 if (unix_err(err_get_code(rtn)) == rtn) {
8228 return err_get_code(rtn);
8229 }
8230
8231 switch (rtn) {
8232 // (obvious match)
8233 case kIOReturnSuccess:
8234 return 0;
8235 case kIOReturnNoMemory:
8236 return ENOMEM;
8237 case kIOReturnNoDevice:
8238 return ENXIO;
8239 case kIOReturnVMError:
8240 return EFAULT;
8241 case kIOReturnNotPermitted:
8242 return EPERM;
8243 case kIOReturnNotPrivileged:
8244 return EACCES;
8245 case kIOReturnIOError:
8246 return EIO;
8247 case kIOReturnNotWritable:
8248 return EROFS;
8249 case kIOReturnBadArgument:
8250 return EINVAL;
8251 case kIOReturnUnsupported:
8252 return ENOTSUP;
8253 case kIOReturnBusy:
8254 return EBUSY;
8255 case kIOReturnNoPower:
8256 return EPWROFF;
8257 case kIOReturnDeviceError:
8258 return EDEVERR;
8259 case kIOReturnTimeout:
8260 return ETIMEDOUT;
8261 case kIOReturnMessageTooLarge:
8262 return EMSGSIZE;
8263 case kIOReturnNoSpace:
8264 return ENOSPC;
8265 case kIOReturnCannotLock:
8266 return ENOLCK;
8267
8268 // (best match)
8269 case kIOReturnBadMessageID:
8270 case kIOReturnNoCompletion:
8271 case kIOReturnNotAligned:
8272 return EINVAL;
8273 case kIOReturnNotReady:
8274 return EBUSY;
8275 case kIOReturnRLDError:
8276 return EBADMACHO;
8277 case kIOReturnPortExists:
8278 case kIOReturnStillOpen:
8279 return EEXIST;
8280 case kIOReturnExclusiveAccess:
8281 case kIOReturnLockedRead:
8282 case kIOReturnLockedWrite:
8283 case kIOReturnNotOpen:
8284 case kIOReturnNotReadable:
8285 return EACCES;
8286 case kIOReturnCannotWire:
8287 case kIOReturnNoResources:
8288 return ENOMEM;
8289 case kIOReturnAborted:
8290 case kIOReturnOffline:
8291 case kIOReturnNotResponding:
8292 return EBUSY;
8293 case kIOReturnBadMedia:
8294 case kIOReturnNoMedia:
8295 case kIOReturnNotAttached:
8296 case kIOReturnUnformattedMedia:
8297 return ENXIO; // (media error)
8298 case kIOReturnDMAError:
8299 case kIOReturnOverrun:
8300 case kIOReturnUnderrun:
8301 return EIO; // (transfer error)
8302 case kIOReturnNoBandwidth:
8303 case kIOReturnNoChannels:
8304 case kIOReturnNoFrames:
8305 case kIOReturnNoInterrupt:
8306 return EIO; // (hardware error)
8307 case kIOReturnError:
8308 case kIOReturnInternalError:
8309 case kIOReturnInvalid:
8310 return EIO; // (generic error)
8311 case kIOReturnIPCError:
8312 return EIO; // (ipc error)
8313 default:
8314 return EIO; // (all other errors)
8315 }
8316 }
8317
8318 IOReturn
message(UInt32 type,IOService * provider,void * argument)8319 IOService::message( UInt32 type, IOService * provider,
8320 void * argument )
8321 {
8322 /*
8323 * Generic entry point for calls from the provider. A return value of
8324 * kIOReturnSuccess indicates that the message was received, and where
8325 * applicable, that it was successful.
8326 */
8327
8328 return kIOReturnUnsupported;
8329 }
8330
8331 /*
8332 * Device memory
8333 */
8334
8335 IOItemCount
getDeviceMemoryCount(void)8336 IOService::getDeviceMemoryCount( void )
8337 {
8338 OSArray * array;
8339 IOItemCount count;
8340
8341 array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8342 if (array) {
8343 count = array->getCount();
8344 } else {
8345 count = 0;
8346 }
8347
8348 return count;
8349 }
8350
8351 IODeviceMemory *
getDeviceMemoryWithIndex(unsigned int index)8352 IOService::getDeviceMemoryWithIndex( unsigned int index )
8353 {
8354 OSArray * array;
8355 IODeviceMemory * range;
8356
8357 array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8358 if (array) {
8359 range = (IODeviceMemory *) array->getObject( index );
8360 } else {
8361 range = NULL;
8362 }
8363
8364 return range;
8365 }
8366
8367 IOMemoryMap *
mapDeviceMemoryWithIndex(unsigned int index,IOOptionBits options)8368 IOService::mapDeviceMemoryWithIndex( unsigned int index,
8369 IOOptionBits options )
8370 {
8371 IODeviceMemory * range;
8372 IOMemoryMap * map;
8373
8374 range = getDeviceMemoryWithIndex( index );
8375 if (range) {
8376 map = range->map( options );
8377 } else {
8378 map = NULL;
8379 }
8380
8381 return map;
8382 }
8383
8384 OSArray *
getDeviceMemory(void)8385 IOService::getDeviceMemory( void )
8386 {
8387 return OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8388 }
8389
8390
8391 void
setDeviceMemory(OSArray * array)8392 IOService::setDeviceMemory( OSArray * array )
8393 {
8394 setProperty( gIODeviceMemoryKey, array);
8395 }
8396
8397 static void
requireMaxCpuDelay(IOService * service,UInt32 ns,UInt32 delayType)8398 requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
8399 {
8400 static const UInt kNoReplace = -1U; // Must be an illegal index
8401 UInt replace = kNoReplace;
8402 bool setCpuDelay = false;
8403
8404 IORecursiveLockLock(sCpuDelayLock);
8405
8406 UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
8407 __typed_allocators_ignore_push
8408 CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
8409 __typed_allocators_ignore_pop
8410 IOService * holder = NULL;
8411
8412 if (ns) {
8413 const CpuDelayEntry ne = {service, ns, delayType};
8414 holder = service;
8415 // Set maximum delay.
8416 for (UInt i = 0; i < count; i++) {
8417 IOService *thisService = entries[i].fService;
8418 bool sameType = (delayType == entries[i].fDelayType);
8419 if ((service == thisService) && sameType) {
8420 replace = i;
8421 } else if (!thisService) {
8422 if (kNoReplace == replace) {
8423 replace = i;
8424 }
8425 } else if (sameType) {
8426 const UInt32 thisMax = entries[i].fMaxDelay;
8427 if (thisMax < ns) {
8428 ns = thisMax;
8429 holder = thisService;
8430 }
8431 }
8432 }
8433
8434 setCpuDelay = true;
8435 if (kNoReplace == replace) {
8436 __typed_allocators_ignore_push
8437 sCpuDelayData->appendBytes(&ne, sizeof(ne));
8438 __typed_allocators_ignore_pop
8439 } else {
8440 entries[replace] = ne;
8441 }
8442 } else {
8443 ns = -1U; // Set to max unsigned, i.e. no restriction
8444
8445 for (UInt i = 0; i < count; i++) {
8446 // Clear a maximum delay.
8447 IOService *thisService = entries[i].fService;
8448 if (thisService && (delayType == entries[i].fDelayType)) {
8449 UInt32 thisMax = entries[i].fMaxDelay;
8450 if (service == thisService) {
8451 replace = i;
8452 } else if (thisMax < ns) {
8453 ns = thisMax;
8454 holder = thisService;
8455 }
8456 }
8457 }
8458
8459 // Check if entry found
8460 if (kNoReplace != replace) {
8461 entries[replace].fService = NULL; // Null the entry
8462 setCpuDelay = true;
8463 }
8464 }
8465
8466 if (setCpuDelay) {
8467 if (holder && debug_boot_arg) {
8468 strlcpy(sCPULatencyHolderName[delayType], holder->getName(), sizeof(sCPULatencyHolderName[delayType]));
8469 }
8470
8471 // Must be safe to call from locked context
8472 if (delayType == kCpuDelayBusStall) {
8473 #if defined(__x86_64__)
8474 ml_set_maxbusdelay(ns);
8475 #endif /* defined(__x86_64__) */
8476 }
8477 #if defined(__x86_64__)
8478 else if (delayType == kCpuDelayInterrupt) {
8479 ml_set_maxintdelay(ns);
8480 }
8481 #endif /* defined(__x86_64__) */
8482 sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
8483 sCPULatencySet[delayType]->setValue(ns);
8484
8485 OSArray * handlers = sCpuLatencyHandlers[delayType];
8486 IOService * target;
8487 if (handlers) {
8488 for (unsigned int idx = 0;
8489 (target = (IOService *) handlers->getObject(idx));
8490 idx++) {
8491 target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
8492 (void *) (uintptr_t) ns, holder,
8493 NULL, NULL);
8494 }
8495 }
8496 }
8497
8498 IORecursiveLockUnlock(sCpuDelayLock);
8499 }
8500
8501 static IOReturn
setLatencyHandler(UInt32 delayType,IOService * target,bool enable)8502 setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
8503 {
8504 IOReturn result = kIOReturnNotFound;
8505 OSArray * array;
8506 unsigned int idx;
8507
8508 IORecursiveLockLock(sCpuDelayLock);
8509
8510 do{
8511 if (enable && !sCpuLatencyHandlers[delayType]) {
8512 sCpuLatencyHandlers[delayType] = OSArray::withCapacity(4);
8513 }
8514 array = sCpuLatencyHandlers[delayType];
8515 if (!array) {
8516 break;
8517 }
8518 idx = array->getNextIndexOfObject(target, 0);
8519 if (!enable) {
8520 if (-1U != idx) {
8521 array->removeObject(idx);
8522 result = kIOReturnSuccess;
8523 }
8524 } else {
8525 if (-1U != idx) {
8526 result = kIOReturnExclusiveAccess;
8527 break;
8528 }
8529 array->setObject(target);
8530
8531 UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
8532 __typed_allocators_ignore_push
8533 CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
8534 __typed_allocators_ignore_pop
8535 UInt32 ns = -1U; // Set to max unsigned, i.e. no restriction
8536 IOService * holder = NULL;
8537
8538 for (UInt i = 0; i < count; i++) {
8539 if (entries[i].fService
8540 && (delayType == entries[i].fDelayType)
8541 && (entries[i].fMaxDelay < ns)) {
8542 ns = entries[i].fMaxDelay;
8543 holder = entries[i].fService;
8544 }
8545 }
8546 target->callPlatformFunction(sCPULatencyFunctionName[delayType], false,
8547 (void *) (uintptr_t) ns, holder,
8548 NULL, NULL);
8549 result = kIOReturnSuccess;
8550 }
8551 }while (false);
8552
8553 IORecursiveLockUnlock(sCpuDelayLock);
8554
8555 return result;
8556 }
8557
8558 IOReturn
requireMaxBusStall(UInt32 ns)8559 IOService::requireMaxBusStall(UInt32 ns)
8560 {
8561 #if !defined(__x86_64__)
8562 switch (ns) {
8563 case kIOMaxBusStall40usec:
8564 case kIOMaxBusStall30usec:
8565 case kIOMaxBusStall25usec:
8566 case kIOMaxBusStall20usec:
8567 case kIOMaxBusStall10usec:
8568 case kIOMaxBusStall5usec:
8569 case kIOMaxBusStallNone:
8570 break;
8571 default:
8572 return kIOReturnBadArgument;
8573 }
8574 #endif /* !defined(__x86_64__) */
8575 requireMaxCpuDelay(this, ns, kCpuDelayBusStall);
8576 return kIOReturnSuccess;
8577 }
8578
8579 IOReturn
requireMaxInterruptDelay(uint32_t ns)8580 IOService::requireMaxInterruptDelay(uint32_t ns)
8581 {
8582 #if defined(__x86_64__)
8583 requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
8584 return kIOReturnSuccess;
8585 #else /* defined(__x86_64__) */
8586 return kIOReturnUnsupported;
8587 #endif /* defined(__x86_64__) */
8588 }
8589
8590 /*
8591 * Device interrupts
8592 */
8593
8594 IOReturn
resolveInterrupt(IOService * nub,int source)8595 IOService::resolveInterrupt(IOService *nub, int source)
8596 {
8597 IOInterruptController *interruptController;
8598 OSArray *array;
8599 OSData *data;
8600 OSSymbol *interruptControllerName;
8601 unsigned int numSources;
8602 IOInterruptSource *interruptSources;
8603 IOInterruptSourcePrivate *interruptSourcesPrivate;
8604
8605 // Get the parents list from the nub.
8606 array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
8607 if (array == NULL) {
8608 return kIOReturnNoResources;
8609 }
8610
8611 // Allocate space for the IOInterruptSources if needed... then return early.
8612 if (nub->_interruptSources == NULL) {
8613 numSources = array->getCount();
8614 interruptSources = IONewZero(IOInterruptSource, numSources);
8615 interruptSourcesPrivate = IONewZero(IOInterruptSourcePrivate, numSources);
8616
8617 if (interruptSources == NULL || interruptSourcesPrivate == NULL) {
8618 IODelete(interruptSources, IOInterruptSource, numSources);
8619 IODelete(interruptSourcesPrivate, IOInterruptSourcePrivate, numSources);
8620 return kIOReturnNoMemory;
8621 }
8622
8623 nub->_numInterruptSources = numSources;
8624 nub->_interruptSources = interruptSources;
8625 nub->reserved->interruptSourcesPrivate = interruptSourcesPrivate;
8626 return kIOReturnSuccess;
8627 }
8628
8629 interruptControllerName = OSDynamicCast(OSSymbol, array->getObject(source));
8630 if (interruptControllerName == NULL) {
8631 return kIOReturnNoResources;
8632 }
8633
8634 interruptController = getPlatform()->lookUpInterruptController(interruptControllerName);
8635 if (interruptController == NULL) {
8636 return kIOReturnNoResources;
8637 }
8638
8639 // Get the interrupt numbers from the nub.
8640 array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
8641 if (array == NULL) {
8642 return kIOReturnNoResources;
8643 }
8644 data = OSDynamicCast(OSData, array->getObject(source));
8645 if (data == NULL) {
8646 return kIOReturnNoResources;
8647 }
8648
8649 // Set the interruptController and interruptSource in the nub's table.
8650 interruptSources = nub->_interruptSources;
8651 interruptSources[source].interruptController = interruptController;
8652 interruptSources[source].vectorData = data;
8653
8654 return kIOReturnSuccess;
8655 }
8656
8657 IOReturn
lookupInterrupt(int source,bool resolve,IOInterruptController ** interruptController)8658 IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
8659 {
8660 IOReturn ret;
8661
8662 /* Make sure the _interruptSources are set */
8663 if (_interruptSources == NULL) {
8664 ret = resolveInterrupt(this, source);
8665 if (ret != kIOReturnSuccess) {
8666 return ret;
8667 }
8668 }
8669
8670 /* Make sure the local source number is valid */
8671 if ((source < 0) || (source >= _numInterruptSources)) {
8672 return kIOReturnNoInterrupt;
8673 }
8674
8675 /* Look up the contoller for the local source */
8676 *interruptController = _interruptSources[source].interruptController;
8677
8678 if (*interruptController == NULL) {
8679 if (!resolve) {
8680 return kIOReturnNoInterrupt;
8681 }
8682
8683 /* Try to resolve the interrupt */
8684 ret = resolveInterrupt(this, source);
8685 if (ret != kIOReturnSuccess) {
8686 return ret;
8687 }
8688
8689 *interruptController = _interruptSources[source].interruptController;
8690 }
8691
8692 return kIOReturnSuccess;
8693 }
8694
8695 IOReturn
registerInterrupt(int source,OSObject * target,IOInterruptAction handler,void * refCon)8696 IOService::registerInterrupt(int source, OSObject *target,
8697 IOInterruptAction handler,
8698 void *refCon)
8699 {
8700 IOInterruptController *interruptController;
8701 IOReturn ret;
8702
8703 ret = lookupInterrupt(source, true, &interruptController);
8704 if (ret != kIOReturnSuccess) {
8705 return ret;
8706 }
8707
8708 /* Register the source */
8709 return interruptController->registerInterrupt(this, source, target,
8710 (IOInterruptHandler)handler,
8711 refCon);
8712 }
8713
8714 static void
IOServiceInterruptActionToBlock(OSObject * target,void * refCon,IOService * nub,int source)8715 IOServiceInterruptActionToBlock( OSObject * target, void * refCon,
8716 IOService * nub, int source )
8717 {
8718 ((IOInterruptActionBlock)(refCon))(nub, source);
8719 }
8720
8721 IOReturn
registerInterruptBlock(int source,OSObject * target,IOInterruptActionBlock handler)8722 IOService::registerInterruptBlock(int source, OSObject *target,
8723 IOInterruptActionBlock handler)
8724 {
8725 IOReturn ret;
8726 void * block;
8727
8728 block = Block_copy(handler);
8729 if (!block) {
8730 return kIOReturnNoMemory;
8731 }
8732
8733 ret = registerInterrupt(source, target, &IOServiceInterruptActionToBlock, block);
8734 if (kIOReturnSuccess != ret) {
8735 Block_release(block);
8736 return ret;
8737 }
8738
8739 reserved->interruptSourcesPrivate[source].vectorBlock = block;
8740
8741 return ret;
8742 }
8743
8744 IOReturn
unregisterInterrupt(int source)8745 IOService::unregisterInterrupt(int source)
8746 {
8747 IOReturn ret;
8748 IOInterruptController *interruptController;
8749 IOInterruptSourcePrivate *priv;
8750 void *block;
8751
8752 ret = lookupInterrupt(source, false, &interruptController);
8753 if (ret != kIOReturnSuccess) {
8754 return ret;
8755 }
8756
8757 /* Unregister the source */
8758 priv = &reserved->interruptSourcesPrivate[source];
8759 block = priv->vectorBlock;
8760 ret = interruptController->unregisterInterrupt(this, source);
8761 if ((kIOReturnSuccess == ret) && (block = priv->vectorBlock)) {
8762 priv->vectorBlock = NULL;
8763 Block_release(block);
8764 }
8765
8766 return ret;
8767 }
8768
8769 void
unregisterAllInterrupts(void)8770 IOService::unregisterAllInterrupts(void)
8771 {
8772 for (int source = 0; source < _numInterruptSources; source++) {
8773 (void) unregisterInterrupt(source);
8774 }
8775 }
8776
8777 IOReturn
addInterruptStatistics(IOInterruptAccountingData * statistics,int source)8778 IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
8779 {
8780 IOReportLegend * legend = NULL;
8781 IOInterruptAccountingData * oldValue = NULL;
8782 IOInterruptAccountingReporter * newArray = NULL;
8783 char subgroupName[64];
8784 int newArraySize = 0;
8785 int i = 0;
8786
8787 if (source < 0) {
8788 return kIOReturnBadArgument;
8789 }
8790
8791 /*
8792 * We support statistics on a maximum of 256 interrupts per nub; if a nub
8793 * has more than 256 interrupt specifiers associated with it, and tries
8794 * to register a high interrupt index with interrupt accounting, panic.
8795 * Having more than 256 interrupts associated with a single nub is
8796 * probably a sign that something fishy is going on.
8797 */
8798 if (source > IA_INDEX_MAX) {
8799 panic("addInterruptStatistics called for an excessively large index (%d)", source);
8800 }
8801
8802 /*
8803 * TODO: This is ugly (wrapping a lock around an allocation). I'm only
8804 * leaving it as is because the likelihood of contention where we are
8805 * actually growing the array is minimal (we would realistically need
8806 * to be starting a driver for the first time, with an IOReporting
8807 * client already in place). Nonetheless, cleanup that can be done
8808 * to adhere to best practices; it'll make the code more complicated,
8809 * unfortunately.
8810 */
8811 IOLockLock(&reserved->interruptStatisticsLock);
8812
8813 /*
8814 * Lazily allocate the statistics array.
8815 */
8816 if (!reserved->interruptStatisticsArray) {
8817 reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
8818 assert(reserved->interruptStatisticsArray);
8819 reserved->interruptStatisticsArrayCount = 1;
8820 bzero(reserved->interruptStatisticsArray, sizeof(*reserved->interruptStatisticsArray));
8821 }
8822
8823 if (source >= reserved->interruptStatisticsArrayCount) {
8824 /*
8825 * We're still within the range of supported indices, but we are out
8826 * of space in the current array. Do a nasty realloc (because
8827 * IORealloc isn't a thing) here. We'll double the size with each
8828 * reallocation.
8829 *
8830 * Yes, the "next power of 2" could be more efficient; but this will
8831 * be invoked incredibly rarely. Who cares.
8832 */
8833 newArraySize = (reserved->interruptStatisticsArrayCount << 1);
8834
8835 while (newArraySize <= source) {
8836 newArraySize = (newArraySize << 1);
8837 }
8838 newArray = IONew(IOInterruptAccountingReporter, newArraySize);
8839
8840 assert(newArray);
8841
8842 /*
8843 * TODO: This even zeroes the memory it is about to overwrite.
8844 * Shameful; fix it. Not particularly high impact, however.
8845 */
8846 bzero(newArray, newArraySize * sizeof(*newArray));
8847 memcpy(newArray, reserved->interruptStatisticsArray, reserved->interruptStatisticsArrayCount * sizeof(*newArray));
8848 IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
8849 reserved->interruptStatisticsArray = newArray;
8850 reserved->interruptStatisticsArrayCount = newArraySize;
8851 }
8852
8853 if (!reserved->interruptStatisticsArray[source].reporter) {
8854 /*
8855 * We don't have a reporter associated with this index yet, so we
8856 * need to create one.
8857 */
8858 /*
8859 * TODO: Some statistics do in fact have common units (time); should this be
8860 * split into separate reporters to communicate this?
8861 */
8862 reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(this, kIOReportCategoryPower, kIOReportUnitNone);
8863
8864 /*
8865 * Each statistic is given an identifier based on the interrupt index (which
8866 * should be unique relative to any single nub) and the statistic involved.
8867 * We should now have a sane (small and positive) index, so start
8868 * constructing the channels for statistics.
8869 */
8870 for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
8871 /*
8872 * TODO: Currently, this does not add channels for disabled statistics.
8873 * Will this be confusing for clients? If so, we should just add the
8874 * channels; we can avoid updating the channels even if they exist.
8875 */
8876 if (IA_GET_STATISTIC_ENABLED(i)) {
8877 reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), kInterruptAccountingStatisticNameArray[i]);
8878 }
8879 }
8880
8881 /*
8882 * We now need to add the legend for this reporter to the registry.
8883 */
8884 OSObject * prop = copyProperty(kIOReportLegendKey);
8885 legend = IOReportLegend::with(OSDynamicCast(OSArray, prop));
8886 OSSafeReleaseNULL(prop);
8887
8888 /*
8889 * Note that while we compose the subgroup name, we do not need to
8890 * manage its lifecycle (the reporter will handle this).
8891 */
8892 snprintf(subgroupName, sizeof(subgroupName), "%s %d", getName(), source);
8893 subgroupName[sizeof(subgroupName) - 1] = 0;
8894 legend->addReporterLegend(reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subgroupName);
8895 setProperty(kIOReportLegendKey, legend->getLegend());
8896 legend->release();
8897
8898 /*
8899 * TODO: Is this a good idea? Probably not; my assumption is it opts
8900 * all entities who register interrupts into public disclosure of all
8901 * IOReporting channels. Unfortunately, this appears to be as fine
8902 * grain as it gets.
8903 */
8904 setProperty(kIOReportLegendPublicKey, true);
8905 }
8906
8907 /*
8908 * Don't stomp existing entries. If we are about to, panic; this
8909 * probably means we failed to tear down our old interrupt source
8910 * correctly.
8911 */
8912 oldValue = reserved->interruptStatisticsArray[source].statistics;
8913
8914 if (oldValue) {
8915 panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
8916 }
8917
8918 reserved->interruptStatisticsArray[source].statistics = statistics;
8919
8920 /*
8921 * Inherit the reporter values for each statistic. The target may
8922 * be torn down as part of the runtime of the service (especially
8923 * for sleep/wake), so we inherit in order to avoid having values
8924 * reset for no apparent reason. Our statistics are ultimately
8925 * tied to the index and the sevice, not to an individual target,
8926 * so we should maintain them accordingly.
8927 */
8928 interruptAccountingDataInheritChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
8929
8930 IOLockUnlock(&reserved->interruptStatisticsLock);
8931
8932 return kIOReturnSuccess;
8933 }
8934
8935 IOReturn
removeInterruptStatistics(int source)8936 IOService::removeInterruptStatistics(int source)
8937 {
8938 IOInterruptAccountingData * value = NULL;
8939
8940 if (source < 0) {
8941 return kIOReturnBadArgument;
8942 }
8943
8944 IOLockLock(&reserved->interruptStatisticsLock);
8945
8946 /*
8947 * We dynamically grow the statistics array, so an excessively
8948 * large index value has NEVER been registered. This either
8949 * means our cap on the array size is too small (unlikely), or
8950 * that we have been passed a corrupt index (this must be passed
8951 * the plain index into the interrupt specifier list).
8952 */
8953 if (source >= reserved->interruptStatisticsArrayCount) {
8954 panic("removeInterruptStatistics called for index %d, which was never registered", source);
8955 }
8956
8957 assert(reserved->interruptStatisticsArray);
8958
8959 /*
8960 * If there is no existing entry, we are most likely trying to
8961 * free an interrupt owner twice, or we have corrupted the
8962 * index value.
8963 */
8964 value = reserved->interruptStatisticsArray[source].statistics;
8965
8966 if (!value) {
8967 panic("removeInterruptStatistics called for empty index %d", source);
8968 }
8969
8970 /*
8971 * We update the statistics, so that any delta with the reporter
8972 * state is not lost.
8973 */
8974 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[source].statistics, reserved->interruptStatisticsArray[source].reporter);
8975 reserved->interruptStatisticsArray[source].statistics = NULL;
8976 IOLockUnlock(&reserved->interruptStatisticsLock);
8977
8978 return kIOReturnSuccess;
8979 }
8980
8981 IOReturn
getInterruptType(int source,int * interruptType)8982 IOService::getInterruptType(int source, int *interruptType)
8983 {
8984 IOInterruptController *interruptController;
8985 IOReturn ret;
8986
8987 ret = lookupInterrupt(source, true, &interruptController);
8988 if (ret != kIOReturnSuccess) {
8989 return ret;
8990 }
8991
8992 /* Return the type */
8993 return interruptController->getInterruptType(this, source, interruptType);
8994 }
8995
8996 IOReturn
enableInterrupt(int source)8997 IOService::enableInterrupt(int source)
8998 {
8999 IOInterruptController *interruptController;
9000 IOReturn ret;
9001
9002 ret = lookupInterrupt(source, false, &interruptController);
9003 if (ret != kIOReturnSuccess) {
9004 return ret;
9005 }
9006
9007 /* Enable the source */
9008 return interruptController->enableInterrupt(this, source);
9009 }
9010
9011 IOReturn
disableInterrupt(int source)9012 IOService::disableInterrupt(int source)
9013 {
9014 IOInterruptController *interruptController;
9015 IOReturn ret;
9016
9017 ret = lookupInterrupt(source, false, &interruptController);
9018 if (ret != kIOReturnSuccess) {
9019 return ret;
9020 }
9021
9022 /* Disable the source */
9023 return interruptController->disableInterrupt(this, source);
9024 }
9025
9026 IOReturn
causeInterrupt(int source)9027 IOService::causeInterrupt(int source)
9028 {
9029 IOInterruptController *interruptController;
9030 IOReturn ret;
9031
9032 ret = lookupInterrupt(source, false, &interruptController);
9033 if (ret != kIOReturnSuccess) {
9034 return ret;
9035 }
9036
9037 /* Cause an interrupt for the source */
9038 return interruptController->causeInterrupt(this, source);
9039 }
9040
9041 IOReturn
configureReport(IOReportChannelList * channelList,IOReportConfigureAction action,void * result,void * destination)9042 IOService::configureReport(IOReportChannelList *channelList,
9043 IOReportConfigureAction action,
9044 void *result,
9045 void *destination)
9046 {
9047 unsigned cnt;
9048
9049 for (cnt = 0; cnt < channelList->nchannels; cnt++) {
9050 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
9051 if (pwrMgt) {
9052 configurePowerStatesReport(action, result);
9053 } else {
9054 return kIOReturnUnsupported;
9055 }
9056 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
9057 if (pwrMgt) {
9058 configureSimplePowerReport(action, result);
9059 } else {
9060 return kIOReturnUnsupported;
9061 }
9062 }
9063 }
9064
9065 IOLockLock(&reserved->interruptStatisticsLock);
9066
9067 /* The array count is signed (because the interrupt indices are signed), hence the cast */
9068 for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
9069 if (reserved->interruptStatisticsArray[cnt].reporter) {
9070 /*
9071 * If the reporter is currently associated with the statistics
9072 * for an event source, we may need to update the reporter.
9073 */
9074 if (reserved->interruptStatisticsArray[cnt].statistics) {
9075 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
9076 }
9077
9078 reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
9079 }
9080 }
9081
9082 IOLockUnlock(&reserved->interruptStatisticsLock);
9083
9084 if (hasUserServer()) {
9085 return _ConfigureReport(channelList, action, result, destination);
9086 } else {
9087 return kIOReturnSuccess;
9088 }
9089 }
9090
9091 IOReturn
updateReport(IOReportChannelList * channelList,IOReportUpdateAction action,void * result,void * destination)9092 IOService::updateReport(IOReportChannelList *channelList,
9093 IOReportUpdateAction action,
9094 void *result,
9095 void *destination)
9096 {
9097 unsigned cnt;
9098
9099 for (cnt = 0; cnt < channelList->nchannels; cnt++) {
9100 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
9101 if (pwrMgt) {
9102 updatePowerStatesReport(action, result, destination);
9103 } else {
9104 return kIOReturnUnsupported;
9105 }
9106 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
9107 if (pwrMgt) {
9108 updateSimplePowerReport(action, result, destination);
9109 } else {
9110 return kIOReturnUnsupported;
9111 }
9112 }
9113 }
9114
9115 IOLockLock(&reserved->interruptStatisticsLock);
9116
9117 /* The array count is signed (because the interrupt indices are signed), hence the cast */
9118 for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
9119 if (reserved->interruptStatisticsArray[cnt].reporter) {
9120 /*
9121 * If the reporter is currently associated with the statistics
9122 * for an event source, we need to update the reporter.
9123 */
9124 if (reserved->interruptStatisticsArray[cnt].statistics) {
9125 interruptAccountingDataUpdateChannels(reserved->interruptStatisticsArray[cnt].statistics, reserved->interruptStatisticsArray[cnt].reporter);
9126 }
9127
9128 reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
9129 }
9130 }
9131
9132 IOLockUnlock(&reserved->interruptStatisticsLock);
9133
9134
9135 if (hasUserServer()) {
9136 return _UpdateReport(channelList, action, result, destination);
9137 } else {
9138 return kIOReturnSuccess;
9139 }
9140 }
9141
9142 uint64_t
getAuthorizationID(void)9143 IOService::getAuthorizationID( void )
9144 {
9145 return reserved->authorizationID;
9146 }
9147
9148 IOReturn
setAuthorizationID(uint64_t authorizationID)9149 IOService::setAuthorizationID( uint64_t authorizationID )
9150 {
9151 OSObject * entitlement;
9152 IOReturn status;
9153
9154 entitlement = IOUserClient::copyClientEntitlement( current_task(), "com.apple.private.iokit.IOServiceSetAuthorizationID" );
9155
9156 if (entitlement) {
9157 if (entitlement == kOSBooleanTrue) {
9158 reserved->authorizationID = authorizationID;
9159
9160 status = kIOReturnSuccess;
9161 } else {
9162 status = kIOReturnNotPrivileged;
9163 }
9164
9165 entitlement->release();
9166 } else {
9167 status = kIOReturnNotPrivileged;
9168 }
9169
9170 return status;
9171 }
9172
9173 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
9174
9175 #if __LP64__
9176 OSMetaClassDefineReservedUsedX86(IOService, 0);
9177 OSMetaClassDefineReservedUsedX86(IOService, 1);
9178 OSMetaClassDefineReservedUnused(IOService, 2);
9179 OSMetaClassDefineReservedUnused(IOService, 3);
9180 OSMetaClassDefineReservedUnused(IOService, 4);
9181 OSMetaClassDefineReservedUnused(IOService, 5);
9182 OSMetaClassDefineReservedUnused(IOService, 6);
9183 OSMetaClassDefineReservedUnused(IOService, 7);
9184 #else
9185 OSMetaClassDefineReservedUsedX86(IOService, 0);
9186 OSMetaClassDefineReservedUsedX86(IOService, 1);
9187 OSMetaClassDefineReservedUsedX86(IOService, 2);
9188 OSMetaClassDefineReservedUsedX86(IOService, 3);
9189 OSMetaClassDefineReservedUsedX86(IOService, 4);
9190 OSMetaClassDefineReservedUsedX86(IOService, 5);
9191 OSMetaClassDefineReservedUsedX86(IOService, 6);
9192 OSMetaClassDefineReservedUsedX86(IOService, 7);
9193 #endif
9194 OSMetaClassDefineReservedUnused(IOService, 8);
9195 OSMetaClassDefineReservedUnused(IOService, 9);
9196 OSMetaClassDefineReservedUnused(IOService, 10);
9197 OSMetaClassDefineReservedUnused(IOService, 11);
9198 OSMetaClassDefineReservedUnused(IOService, 12);
9199 OSMetaClassDefineReservedUnused(IOService, 13);
9200 OSMetaClassDefineReservedUnused(IOService, 14);
9201 OSMetaClassDefineReservedUnused(IOService, 15);
9202 OSMetaClassDefineReservedUnused(IOService, 16);
9203 OSMetaClassDefineReservedUnused(IOService, 17);
9204 OSMetaClassDefineReservedUnused(IOService, 18);
9205 OSMetaClassDefineReservedUnused(IOService, 19);
9206 OSMetaClassDefineReservedUnused(IOService, 20);
9207 OSMetaClassDefineReservedUnused(IOService, 21);
9208 OSMetaClassDefineReservedUnused(IOService, 22);
9209 OSMetaClassDefineReservedUnused(IOService, 23);
9210 OSMetaClassDefineReservedUnused(IOService, 24);
9211 OSMetaClassDefineReservedUnused(IOService, 25);
9212 OSMetaClassDefineReservedUnused(IOService, 26);
9213 OSMetaClassDefineReservedUnused(IOService, 27);
9214 OSMetaClassDefineReservedUnused(IOService, 28);
9215 OSMetaClassDefineReservedUnused(IOService, 29);
9216 OSMetaClassDefineReservedUnused(IOService, 30);
9217 OSMetaClassDefineReservedUnused(IOService, 31);
9218 OSMetaClassDefineReservedUnused(IOService, 32);
9219 OSMetaClassDefineReservedUnused(IOService, 33);
9220 OSMetaClassDefineReservedUnused(IOService, 34);
9221 OSMetaClassDefineReservedUnused(IOService, 35);
9222 OSMetaClassDefineReservedUnused(IOService, 36);
9223 OSMetaClassDefineReservedUnused(IOService, 37);
9224 OSMetaClassDefineReservedUnused(IOService, 38);
9225 OSMetaClassDefineReservedUnused(IOService, 39);
9226 OSMetaClassDefineReservedUnused(IOService, 40);
9227 OSMetaClassDefineReservedUnused(IOService, 41);
9228 OSMetaClassDefineReservedUnused(IOService, 42);
9229 OSMetaClassDefineReservedUnused(IOService, 43);
9230 OSMetaClassDefineReservedUnused(IOService, 44);
9231 OSMetaClassDefineReservedUnused(IOService, 45);
9232 OSMetaClassDefineReservedUnused(IOService, 46);
9233 OSMetaClassDefineReservedUnused(IOService, 47);
9234