1 /* 2 * Copyright (c) 2017 Apple Inc. All rights reserved. 3 */ 4 5 #pragma once 6 7 #ifdef KERNEL_PRIVATE 8 #ifdef __cplusplus 9 10 #include <IOKit/IOService.h> 11 #include <stdatomic.h> 12 #include <kern/bits.h> 13 #include <libkern/c++/OSPtr.h> 14 15 struct thread_group; 16 17 enum{ 18 kIOPerfControlClientWorkUntracked = 0, 19 }; 20 21 /*! 22 * @class IOPerfControlClient : public OSObject 23 * @abstract Class which implements an interface allowing device drivers to participate in performance control. 24 * @discussion TODO 25 */ 26 class IOPerfControlClient final : public OSObject 27 { 28 OSDeclareDefaultStructors(IOPerfControlClient); 29 30 protected: 31 virtual bool init(IOService *driver, uint64_t maxWorkCapacity); 32 virtual void free() APPLE_KEXT_OVERRIDE; 33 34 public: 35 /*! 36 * @function copyClient 37 * @abstract Return a retained reference to a client object, to be released by the driver. It may be 38 * shared with other drivers in the system. 39 * @param driver The device driver that will be using this interface. 40 * @param maxWorkCapacity The maximum number of concurrent work items supported by the device driver. 41 * @returns An instance of IOPerfControlClient. 42 */ 43 static IOPerfControlClient *copyClient(IOService *driver, uint64_t maxWorkCapacity); 44 45 /*! 46 * @function registerDevice 47 * @abstract Inform the system that work will be dispatched to a device in the future. 48 * @discussion The system will do some one-time setup work associated with the device, and may block the 49 * current thread during the setup. Devices should not be passed to work workSubmit, workSubmitAndBegin, 50 * workBegin, or workEnd until they have been successfully registered. The unregistration process happens 51 * automatically when the device object is deallocated. 52 * @param device The device object. Some platforms require device to be a specific subclass of IOService. 53 * @returns kIOReturnSuccess or an IOReturn error code 54 */ 55 virtual IOReturn registerDevice(IOService *driver, IOService *device); 56 57 /*! 58 * @function unregisterDevice 59 * @abstract Inform the system that work will be no longer be dispatched to a device in the future. 60 * @discussion This call is optional as the unregistration process happens automatically when the device 61 * object is deallocated. This call may block the current thread and/or acquire locks. It should not be 62 * called until after all submitted work has been ended using workEnd. 63 * @param device The device object. Some platforms require device to be a specific subclass of IOService. 64 */ 65 virtual void unregisterDevice(IOService *driver, IOService *device); 66 67 /*! 68 * @struct WorkSubmitArgs 69 * @discussion Drivers may submit additional device-specific arguments related to the submission of a work item 70 * by passing a struct with WorkSubmitArgs as its first member. Note: Drivers are responsible for publishing 71 * a header file describing these arguments. 72 */ 73 struct WorkSubmitArgs { 74 uint32_t version; 75 uint32_t size; 76 uint64_t submit_time; 77 uint64_t reserved[4]; 78 void *driver_data; 79 }; 80 81 /*! 82 * @function workSubmit 83 * @abstract Tell the performance controller that work was submitted. 84 * @param device The device that will execute the work. Some platforms require device to be a 85 * specific subclass of IOService. 86 * @param args Optional device-specific arguments related to the submission of this work item. 87 * @returns A token representing this work item, which must be passed to workEnd when the work is finished 88 * unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks 89 * and a degradation of system performance. 90 */ 91 virtual uint64_t workSubmit(IOService *device, WorkSubmitArgs *args = nullptr); 92 93 /*! 94 * @struct WorkBeginArgs 95 * @discussion Drivers may submit additional device-specific arguments related to the start of a work item 96 * by passing a struct with WorkBeginArgs as its first member. Note: Drivers are responsible for publishing 97 * a header file describing these arguments. 98 */ 99 struct WorkBeginArgs { 100 uint32_t version; 101 uint32_t size; 102 uint64_t begin_time; 103 uint64_t reserved[4]; 104 void *driver_data; 105 }; 106 107 /*! 108 * @function workSubmitAndBegin 109 * @abstract Tell the performance controller that work was submitted and immediately began executing. 110 * @param device The device that is executing the work. Some platforms require device to be a 111 * specific subclass of IOService. 112 * @param submitArgs Optional device-specific arguments related to the submission of this work item. 113 * @param beginArgs Optional device-specific arguments related to the start of this work item. 114 * @returns A token representing this work item, which must be passed to workEnd when the work is finished 115 * unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks 116 * and a degradation of system performance. 117 */ 118 virtual uint64_t workSubmitAndBegin(IOService *device, WorkSubmitArgs *submitArgs = nullptr, 119 WorkBeginArgs *beginArgs = nullptr); 120 121 /*! 122 * @function workBegin 123 * @abstract Tell the performance controller that previously submitted work began executing. 124 * @param device The device that is executing the work. Some platforms require device to be a 125 * specific subclass of IOService. 126 * @param args Optional device-specific arguments related to the start of this work item. 127 */ 128 virtual void workBegin(IOService *device, uint64_t token, WorkBeginArgs *args = nullptr); 129 130 /*! 131 * @struct WorkEndArgs 132 * @discussion Drivers may submit additional device-specific arguments related to the end of a work item 133 * by passing a struct with WorkEndArgs as its first member. Note: Drivers are responsible for publishing 134 * a header file describing these arguments. 135 */ 136 struct WorkEndArgs { 137 uint32_t version; 138 uint32_t size; 139 uint64_t end_time; 140 uint64_t reserved[4]; 141 void *driver_data; 142 }; 143 144 /*! 145 * @function workEnd 146 * @abstract Tell the performance controller that previously started work finished executing. 147 * @param device The device that executed the work. Some platforms require device to be a 148 * specific subclass of IOService. 149 * @param args Optional device-specific arguments related to the end of this work item. 150 * @param done Optional Set to false if the work has not yet completed. Drivers are then responsible for 151 * calling workBegin when the work resumes and workEnd with done set to True when it has completed. A workEnd() call 152 * without a corresponding workBegin() call is a way to cancel a work item and return token to IOPerfControl. 153 */ 154 virtual void workEnd(IOService *device, uint64_t token, WorkEndArgs *args = nullptr, bool done = true); 155 156 /*! 157 * @function copyWorkContext 158 * @abstract Return a retained reference to an opaque OSObject, to be released by the driver. This object can 159 * be used by IOPerfControl to track a work item. This may perform dynamic memory allocation. 160 * @returns A pointer to an OSObject 161 */ 162 OSPtr<OSObject> copyWorkContext(); 163 164 /*! 165 * @function workSubmitAndBeginWithContext 166 * @abstract Tell the performance controller that work was submitted and immediately began executing 167 * @param device The device that is executing the work. Some platforms require device to be a 168 * specific subclass of IOService. 169 * @param context An OSObject returned by copyWorkContext(). The context object will be used by IOPerfControl to track 170 * this work item. 171 * @param submitArgs Optional device-specific arguments related to the submission of this work item. 172 * @param beginArgs Optional device-specific arguments related to the start of this work item. 173 * @returns true if IOPerfControl is tracking this work item, else false. 174 * @note The workEndWithContext() call is optional if the corresponding workSubmitWithContext() call returned false. 175 */ 176 bool workSubmitAndBeginWithContext(IOService *device, OSObject *context, WorkSubmitArgs *submitArgs = nullptr, 177 WorkBeginArgs *beginArgs = nullptr); 178 179 /*! 180 * @function workSubmitWithContext 181 * @abstract Tell the performance controller that work was submitted. 182 * @param device The device that will execute the work. Some platforms require device to be a 183 * specific subclass of IOService. 184 * @param context An OSObject returned by copyWorkContext(). The context object will be used by IOPerfControl to track 185 * this work item. 186 * @param args Optional device-specific arguments related to the submission of this work item. 187 * @returns true if IOPerfControl is tracking this work item, else false. 188 */ 189 bool workSubmitWithContext(IOService *device, OSObject *context, WorkSubmitArgs *args = nullptr); 190 191 /*! 192 * @function workBeginWithContext 193 * @abstract Tell the performance controller that previously submitted work began executing. 194 * @param device The device that is executing the work. Some platforms require device to be a 195 * specific subclass of IOService. 196 * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext(). 197 * @param args Optional device-specific arguments related to the start of this work item. 198 * @note The workBeginWithContext() and workEndWithContext() calls are optional if the corresponding workSubmitWithContext() call returned false. 199 */ 200 void workBeginWithContext(IOService *device, OSObject *context, WorkBeginArgs *args = nullptr); 201 202 /*! 203 * @function workEndWithContext 204 * @abstract Tell the performance controller that previously started work finished executing. 205 * @param device The device that executed the work. Some platforms require device to be a 206 * specific subclass of IOService. 207 * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext(). 208 * @param args Optional device-specific arguments related to the end of this work item. 209 * @param done Optional Set to false if the work has not yet completed. Drivers are then responsible for 210 * calling workBegin when the work resumes and workEnd with done set to True when it has completed. 211 * @note The workEndWithContext() call is optional if the corresponding workSubmitWithContext() call returned false. A workEndWithContext() 212 * call without a corresponding workBeginWithContext() call is a way to cancel a work item. 213 */ 214 void workEndWithContext(IOService *device, OSObject *context, WorkEndArgs *args = nullptr, bool done = true); 215 216 /*! 217 * @struct WorkUpdateArgs 218 * @discussion Drivers may submit additional device-specific arguments related to a work item by passing a 219 * struct with WorkUpdateArgs as its first member. Note: Drivers are responsible for publishing 220 * a header file describing these arguments. 221 */ 222 struct WorkUpdateArgs { 223 uint32_t version; 224 uint32_t size; 225 uint64_t update_time; 226 uint64_t reserved[4]; 227 void *driver_data; 228 }; 229 230 /*! 231 * @function workUpdateWithContext 232 * @abstract Provide and receive additional information from the performance controller. If this call is 233 * made at all, it should be between workSubmit and workEnd. The purpose and implementation of this call are 234 * device specific, and may do nothing on some devices. 235 * @param device The device that submitted the work. Some platforms require device to be a 236 * specific subclass of IOService. 237 * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext(). 238 * @param args Optional device-specific arguments. 239 */ 240 void workUpdateWithContext(IOService *device, OSObject *context, WorkUpdateArgs *args = nullptr); 241 242 /* 243 * Callers should always use the CURRENT version so that the kernel can detect both older 244 * and newer structure layouts. New callbacks should always be added at the end of the 245 * structure, and xnu should expect existing source recompiled against newer headers 246 * to pass NULL for unimplemented callbacks. 247 */ 248 249 #define PERFCONTROL_INTERFACE_VERSION_NONE (0) /* no interface */ 250 #define PERFCONTROL_INTERFACE_VERSION_1 (1) /* up-to workEnd */ 251 #define PERFCONTROL_INTERFACE_VERSION_2 (2) /* up-to workUpdate */ 252 #define PERFCONTROL_INTERFACE_VERSION_3 (3) /* up-to (un)registerDriverDevice */ 253 #define PERFCONTROL_INTERFACE_VERSION_4 (4) /* up-to workEndWithResources */ 254 #define PERFCONTROL_INTERFACE_VERSION_CURRENT PERFCONTROL_INTERFACE_VERSION_4 255 256 /*! 257 * @struct PerfControllerInterface 258 * @discussion Function pointers necessary to register a performance controller. Not for general driver use. 259 */ 260 struct PerfControllerInterface { 261 enum struct PerfDeviceID : uint32_t{ 262 kInvalid = 0, 263 kCPU = 0, 264 kANE = 0x4, 265 kGPU, 266 kMSR, 267 kStorage, 268 }; 269 270 struct DriverState { 271 uint32_t has_target_thread_group : 1; 272 uint32_t has_device_info : 1; 273 uint32_t reserved : 30; 274 275 uint64_t target_thread_group_id; 276 void *target_thread_group_data; 277 278 PerfDeviceID device_type; 279 uint32_t instance_id; 280 bool resource_accounting; 281 }; 282 283 struct WorkState { 284 uint64_t thread_group_id; 285 void *thread_group_data; 286 void *work_data; 287 uint32_t work_data_size; 288 uint32_t started : 1; 289 uint32_t reserved : 31; 290 const DriverState* driver_state; 291 }; 292 293 struct ResourceAccounting { 294 uint64_t mach_time_delta; 295 uint64_t energy_nj_delta; 296 }; 297 298 using RegisterDeviceFunction = IOReturn (*)(IOService *); 299 using RegisterDriverDeviceFunction = IOReturn (*)(IOService *, IOService *, DriverState *); 300 using WorkCanSubmitFunction = bool (*)(IOService *, WorkState *, WorkSubmitArgs *); 301 using WorkSubmitFunction = void (*)(IOService *, uint64_t, WorkState *, WorkSubmitArgs *); 302 using WorkBeginFunction = void (*)(IOService *, uint64_t, WorkState *, WorkBeginArgs *); 303 using WorkEndFunction = void (*)(IOService *, uint64_t, WorkState *, WorkEndArgs *, bool); 304 using WorkEndWithResourcesFunction = void (*)(IOService *, uint64_t, WorkState *, WorkEndArgs *, ResourceAccounting *, bool); 305 using WorkUpdateFunction = void (*)(IOService *, uint64_t, WorkState *, WorkUpdateArgs *); 306 307 uint64_t version; 308 RegisterDeviceFunction registerDevice; 309 RegisterDeviceFunction unregisterDevice; 310 WorkCanSubmitFunction workCanSubmit; 311 WorkSubmitFunction workSubmit; 312 WorkBeginFunction workBegin; 313 WorkEndFunction workEnd; 314 WorkUpdateFunction workUpdate; 315 RegisterDriverDeviceFunction registerDriverDevice; 316 RegisterDriverDeviceFunction unregisterDriverDevice; 317 WorkEndWithResourcesFunction workEndWithResources; 318 }; 319 320 struct IOPerfControlClientShared { 321 atomic_uint_fast8_t maxDriverIndex; 322 PerfControllerInterface interface; 323 IOLock *interfaceLock; 324 OSSet *deviceRegistrationList; 325 }; 326 327 struct IOPerfControlClientData { 328 struct thread_group *target_thread_group; 329 PerfControllerInterface::DriverState driverState; 330 IOService* device; 331 }; 332 /*! 333 * @function registerPerformanceController 334 * @abstract Register a performance controller to receive callbacks. Not for general driver use. 335 * @param interface Struct containing callback functions implemented by the performance controller. 336 * @returns kIOReturnSuccess or kIOReturnError if the interface was already registered. 337 */ 338 virtual IOReturn registerPerformanceController(PerfControllerInterface *interface); 339 340 /*! 341 * @function getClientData 342 * @abstract Not for general driver use. Only used by registerPerformanceController(). Allows performanceController to register existing IOPerfControlClient. 343 * @returns IOPerfControlData associated with a IOPerfControlClient 344 */ 345 IOPerfControlClientData * getClientData()346 getClientData() 347 { 348 return &clientData; 349 } 350 351 private: 352 struct WorkTableEntry { 353 struct thread_group *thread_group; 354 coalition_t coal; 355 bool started; 356 uint8_t perfcontrol_data[32]; 357 }; 358 359 static constexpr size_t kMaxWorkTableNumEntries = 1024; 360 static constexpr size_t kWorkTableIndexBits = 24; 361 static constexpr size_t kWorkTableMaxSize = (1 << kWorkTableIndexBits) - 1; // - 1 since 362 // kIOPerfControlClientWorkUntracked takes number 0 363 static constexpr size_t kWorkTableIndexMask = (const size_t)mask(kWorkTableIndexBits); 364 365 uint64_t allocateToken(thread_group *thread_group); 366 void deallocateToken(uint64_t token); 367 WorkTableEntry *getEntryForToken(uint64_t token); 368 void markEntryStarted(uint64_t token, bool started); 369 inline uint64_t tokenToGlobalUniqueToken(uint64_t token); 370 void accountResources(coalition_t coal, PerfControllerInterface::PerfDeviceID device_type, PerfControllerInterface::ResourceAccounting *resources); 371 372 uint8_t driverIndex; 373 IOPerfControlClientShared *shared; 374 WorkTableEntry *workTable; 375 size_t workTableLength; 376 size_t workTableNextIndex; 377 IOSimpleLock *workTableLock; 378 379 IOPerfControlClientData clientData; 380 }; 381 382 #endif /* __cplusplus */ 383 #endif /* KERNEL_PRIVATE */ 384