1 /* 2 * Copyright (c) 1998-2016 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #define IOKIT_ENABLE_SHARED_PTR 30 31 #include <IOKit/IOLib.h> 32 #include <IOKit/IOMapper.h> 33 #include <IOKit/IODMACommand.h> 34 #include <libkern/c++/OSData.h> 35 #include <libkern/OSDebug.h> 36 #include <mach_debug/zone_info.h> 37 #include "IOKitKernelInternal.h" 38 39 __BEGIN_DECLS 40 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); 41 __END_DECLS 42 43 #define super IOService 44 OSDefineMetaClassAndAbstractStructors(IOMapper, IOService); 45 46 OSMetaClassDefineReservedUnused(IOMapper, 0); 47 OSMetaClassDefineReservedUnused(IOMapper, 1); 48 OSMetaClassDefineReservedUnused(IOMapper, 2); 49 OSMetaClassDefineReservedUnused(IOMapper, 3); 50 OSMetaClassDefineReservedUnused(IOMapper, 4); 51 OSMetaClassDefineReservedUnused(IOMapper, 5); 52 OSMetaClassDefineReservedUnused(IOMapper, 6); 53 OSMetaClassDefineReservedUnused(IOMapper, 7); 54 OSMetaClassDefineReservedUnused(IOMapper, 8); 55 OSMetaClassDefineReservedUnused(IOMapper, 9); 56 OSMetaClassDefineReservedUnused(IOMapper, 10); 57 OSMetaClassDefineReservedUnused(IOMapper, 11); 58 OSMetaClassDefineReservedUnused(IOMapper, 12); 59 OSMetaClassDefineReservedUnused(IOMapper, 13); 60 OSMetaClassDefineReservedUnused(IOMapper, 14); 61 OSMetaClassDefineReservedUnused(IOMapper, 15); 62 63 IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown; 64 65 class IOMapperLock { 66 IOLock *fWaitLock; 67 public: 68 IOMapperLock() 69 { 70 fWaitLock = IOLockAlloc(); 71 } 72 ~IOMapperLock() 73 { 74 IOLockFree(fWaitLock); 75 } 76 77 void 78 lock() 79 { 80 IOLockLock(fWaitLock); 81 } 82 void 83 unlock() 84 { 85 IOLockUnlock(fWaitLock); 86 } 87 void 88 sleep(void *event) 89 { 90 IOLockSleep(fWaitLock, event, THREAD_UNINT); 91 } 92 void 93 wakeup(void *event) 94 { 95 IOLockWakeup(fWaitLock, event, false); 96 } 97 }; 98 99 static IOMapperLock sMapperLock; 100 101 bool 102 IOMapper::start(IOService *provider) 103 { 104 OSSharedPtr<OSObject> obj; 105 if (!super::start(provider)) { 106 return false; 107 } 108 109 if (!initHardware(provider)) { 110 return false; 111 } 112 113 uint64_t pageSize = getPageSize(); 114 assert(pageSize <= UINT_MAX); 115 fPageSize = (uint32_t) pageSize; 116 117 if (fIsSystem) { 118 sMapperLock.lock(); 119 IOMapper::gSystem = this; 120 sMapperLock.wakeup(&IOMapper::gSystem); 121 sMapperLock.unlock(); 122 } 123 124 if (provider) { 125 obj = provider->copyProperty("iommu-id"); 126 if (!obj) { 127 obj = provider->copyProperty("AAPL,phandle"); 128 } 129 if (obj) { 130 setProperty(gIOMapperIDKey, obj.get()); 131 } 132 } 133 return true; 134 } 135 136 void 137 IOMapper::free() 138 { 139 super::free(); 140 } 141 142 void 143 IOMapper::setMapperRequired(bool hasMapper) 144 { 145 if (hasMapper) { 146 IOMapper::gSystem = (IOMapper *) kHasMapper; 147 } else { 148 sMapperLock.lock(); 149 IOMapper::gSystem = (IOMapper *) kNoMapper; 150 sMapperLock.unlock(); 151 sMapperLock.wakeup(&IOMapper::gSystem); 152 } 153 } 154 155 void 156 IOMapper::waitForSystemMapper() 157 { 158 sMapperLock.lock(); 159 while ((uintptr_t) IOMapper::gSystem & kWaitMask) { 160 OSReportWithBacktrace("waitForSystemMapper"); 161 sMapperLock.sleep(&IOMapper::gSystem); 162 } 163 sMapperLock.unlock(); 164 } 165 166 OSSharedPtr<IOMapper> 167 IOMapper::copyMapperForDevice(IOService * device) 168 { 169 return copyMapperForDeviceWithIndex(device, 0); 170 } 171 172 OSSharedPtr<IOMapper> 173 IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned int index) 174 { 175 OSSharedPtr<OSData> data; 176 OSSharedPtr<OSObject> obj; 177 OSSharedPtr<IOMapper> mapper; 178 OSSharedPtr<OSDictionary> matching; 179 180 obj = device->copyProperty("iommu-parent"); 181 if (!obj) { 182 return NULL; 183 } 184 185 if ((mapper = OSDynamicPtrCast<IOMapper>(obj))) { 186 goto found; 187 } 188 189 if ((data = OSDynamicPtrCast<OSData>(obj))) { 190 if (index >= data->getLength() / sizeof(UInt32)) { 191 goto found; 192 } 193 194 data = OSData::withBytesNoCopy((UInt32 *)data->getBytesNoCopy() + index, sizeof(UInt32)); 195 if (!data) { 196 goto found; 197 } 198 199 matching = IOService::propertyMatching(gIOMapperIDKey, data.get()); 200 } else { 201 matching = IOService::propertyMatching(gIOMapperIDKey, obj.get()); 202 } 203 204 if (matching) { 205 mapper = OSDynamicPtrCast<IOMapper>(IOService::waitForMatchingService(matching.get())); 206 } 207 208 found: 209 if (mapper) { 210 if (!mapper->fAllocName) { 211 char name[MACH_ZONE_NAME_MAX_LEN]; 212 char kmodname[KMOD_MAX_NAME]; 213 vm_tag_t tag; 214 uint32_t kmodid; 215 216 tag = IOMemoryTag(kernel_map); 217 if (!(kmodid = vm_tag_get_kext(tag, &kmodname[0], KMOD_MAX_NAME))) { 218 snprintf(kmodname, sizeof(kmodname), "%d", tag); 219 } 220 snprintf(name, sizeof(name), "%s.DMA.%s", kmodname, device->getName()); 221 mapper->fAllocName = kern_allocation_name_allocate(name, 16); 222 } 223 } 224 225 return mapper; 226 } 227 228 __BEGIN_DECLS 229 230 // These are C accessors to the system mapper for non-IOKit clients 231 ppnum_t 232 IOMapperIOVMAlloc(unsigned pages) 233 { 234 IOReturn ret; 235 uint64_t dmaAddress, dmaLength; 236 237 IOMapper::checkForSystemMapper(); 238 239 ret = kIOReturnUnsupported; 240 if (IOMapper::gSystem) { 241 ret = IOMapper::gSystem->iovmMapMemory( 242 NULL, 0, ptoa_64(pages), 243 (kIODMAMapReadAccess | kIODMAMapWriteAccess), 244 NULL, NULL, NULL, 245 &dmaAddress, &dmaLength); 246 } 247 248 if (kIOReturnSuccess == ret) { 249 uint64_t dmaAddressPage64; 250 dmaAddressPage64 = atop_64(dmaAddress); 251 if (dmaAddressPage64 > UINT_MAX) { 252 return 0; 253 } 254 return (ppnum_t) atop_64(dmaAddress); 255 } 256 return 0; 257 } 258 259 void 260 IOMapperIOVMFree(ppnum_t addr, unsigned pages) 261 { 262 if (IOMapper::gSystem) { 263 IOMapper::gSystem->iovmUnmapMemory(NULL, NULL, ptoa_64(addr), ptoa_64(pages)); 264 } 265 } 266 267 ppnum_t 268 IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page) 269 { 270 if (!IOMapper::gSystem) { 271 return page; 272 } 273 if (!addr) { 274 panic("!addr"); 275 } 276 IOMapper::gSystem->iovmInsert((kIODMAMapReadAccess | kIODMAMapWriteAccess), 277 ptoa_64(addr), ptoa_64(offset), ptoa_64(page), ptoa_64(1)); 278 return addr + offset; 279 } 280 281 ///////////////////////////////////////////////////////////////////////////// 282 // 283 // 284 // IOLib.h APIs 285 // 286 // 287 ///////////////////////////////////////////////////////////////////////////// 288 289 #include <machine/machine_routines.h> 290 291 UInt8 292 IOMappedRead8(IOPhysicalAddress address) 293 { 294 IOMapper::checkForSystemMapper(); 295 296 if (IOMapper::gSystem) { 297 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 298 return (UInt8) ml_phys_read_byte_64(addr); 299 } else { 300 return (UInt8) ml_phys_read_byte((vm_offset_t) address); 301 } 302 } 303 304 UInt16 305 IOMappedRead16(IOPhysicalAddress address) 306 { 307 IOMapper::checkForSystemMapper(); 308 309 if (IOMapper::gSystem) { 310 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 311 return (UInt16) ml_phys_read_half_64(addr); 312 } else { 313 return (UInt16) ml_phys_read_half((vm_offset_t) address); 314 } 315 } 316 317 UInt32 318 IOMappedRead32(IOPhysicalAddress address) 319 { 320 IOMapper::checkForSystemMapper(); 321 322 if (IOMapper::gSystem) { 323 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 324 return (UInt32) ml_phys_read_word_64(addr); 325 } else { 326 return (UInt32) ml_phys_read_word((vm_offset_t) address); 327 } 328 } 329 330 UInt64 331 IOMappedRead64(IOPhysicalAddress address) 332 { 333 IOMapper::checkForSystemMapper(); 334 335 if (IOMapper::gSystem) { 336 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 337 return (UInt64) ml_phys_read_double_64(addr); 338 } else { 339 return (UInt64) ml_phys_read_double((vm_offset_t) address); 340 } 341 } 342 343 void 344 IOMappedWrite8(IOPhysicalAddress address, UInt8 value) 345 { 346 IOMapper::checkForSystemMapper(); 347 348 if (IOMapper::gSystem) { 349 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 350 ml_phys_write_byte_64(addr, value); 351 } else { 352 ml_phys_write_byte((vm_offset_t) address, value); 353 } 354 } 355 356 void 357 IOMappedWrite16(IOPhysicalAddress address, UInt16 value) 358 { 359 IOMapper::checkForSystemMapper(); 360 361 if (IOMapper::gSystem) { 362 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 363 ml_phys_write_half_64(addr, value); 364 } else { 365 ml_phys_write_half((vm_offset_t) address, value); 366 } 367 } 368 369 void 370 IOMappedWrite32(IOPhysicalAddress address, UInt32 value) 371 { 372 IOMapper::checkForSystemMapper(); 373 374 if (IOMapper::gSystem) { 375 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 376 ml_phys_write_word_64(addr, value); 377 } else { 378 ml_phys_write_word((vm_offset_t) address, value); 379 } 380 } 381 382 void 383 IOMappedWrite64(IOPhysicalAddress address, UInt64 value) 384 { 385 IOMapper::checkForSystemMapper(); 386 387 if (IOMapper::gSystem) { 388 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 389 ml_phys_write_double_64(addr, value); 390 } else { 391 ml_phys_write_double((vm_offset_t) address, value); 392 } 393 } 394 395 __END_DECLS 396