1 /* 2 * Copyright (c) 1998-2016 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #define IOKIT_ENABLE_SHARED_PTR 30 31 #include <IOKit/IOLib.h> 32 #include <IOKit/IOMapper.h> 33 #include <IOKit/IODMACommand.h> 34 #include <libkern/c++/OSData.h> 35 #include <libkern/OSDebug.h> 36 #include <mach_debug/zone_info.h> 37 #include <vm/vm_iokit.h> 38 #include "IOKitKernelInternal.h" 39 40 __BEGIN_DECLS 41 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); 42 __END_DECLS 43 44 #define super IOService 45 OSDefineMetaClassAndAbstractStructors(IOMapper, IOService); 46 47 OSMetaClassDefineReservedUnused(IOMapper, 0); 48 OSMetaClassDefineReservedUnused(IOMapper, 1); 49 OSMetaClassDefineReservedUnused(IOMapper, 2); 50 OSMetaClassDefineReservedUnused(IOMapper, 3); 51 OSMetaClassDefineReservedUnused(IOMapper, 4); 52 OSMetaClassDefineReservedUnused(IOMapper, 5); 53 OSMetaClassDefineReservedUnused(IOMapper, 6); 54 OSMetaClassDefineReservedUnused(IOMapper, 7); 55 OSMetaClassDefineReservedUnused(IOMapper, 8); 56 OSMetaClassDefineReservedUnused(IOMapper, 9); 57 OSMetaClassDefineReservedUnused(IOMapper, 10); 58 OSMetaClassDefineReservedUnused(IOMapper, 11); 59 OSMetaClassDefineReservedUnused(IOMapper, 12); 60 OSMetaClassDefineReservedUnused(IOMapper, 13); 61 OSMetaClassDefineReservedUnused(IOMapper, 14); 62 OSMetaClassDefineReservedUnused(IOMapper, 15); 63 64 IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown; 65 66 class IOMapperLock { 67 IOLock *fWaitLock; 68 public: 69 IOMapperLock() 70 { 71 fWaitLock = IOLockAlloc(); 72 } 73 ~IOMapperLock() 74 { 75 IOLockFree(fWaitLock); 76 } 77 78 void 79 lock() 80 { 81 IOLockLock(fWaitLock); 82 } 83 void 84 unlock() 85 { 86 IOLockUnlock(fWaitLock); 87 } 88 void 89 sleep(void *event) 90 { 91 IOLockSleep(fWaitLock, event, THREAD_UNINT); 92 } 93 void 94 wakeup(void *event) 95 { 96 IOLockWakeup(fWaitLock, event, false); 97 } 98 }; 99 100 static IOMapperLock sMapperLock; 101 102 bool 103 IOMapper::start(IOService *provider) 104 { 105 OSSharedPtr<OSObject> obj; 106 if (!super::start(provider)) { 107 return false; 108 } 109 110 if (!initHardware(provider)) { 111 return false; 112 } 113 114 uint64_t pageSize = getPageSize(); 115 assert(pageSize <= UINT_MAX); 116 fPageSize = (uint32_t) pageSize; 117 118 if (fIsSystem) { 119 sMapperLock.lock(); 120 IOMapper::gSystem = this; 121 sMapperLock.wakeup(&IOMapper::gSystem); 122 sMapperLock.unlock(); 123 } 124 125 if (provider) { 126 obj = provider->copyProperty("iommu-id"); 127 if (!obj) { 128 obj = provider->copyProperty("AAPL,phandle"); 129 } 130 if (obj) { 131 setProperty(gIOMapperIDKey, obj.get()); 132 } 133 } 134 return true; 135 } 136 137 void 138 IOMapper::free() 139 { 140 super::free(); 141 } 142 143 void 144 IOMapper::setMapperRequired(bool hasMapper) 145 { 146 if (hasMapper) { 147 IOMapper::gSystem = (IOMapper *) kHasMapper; 148 } else { 149 sMapperLock.lock(); 150 IOMapper::gSystem = (IOMapper *) kNoMapper; 151 sMapperLock.unlock(); 152 sMapperLock.wakeup(&IOMapper::gSystem); 153 } 154 } 155 156 void 157 IOMapper::waitForSystemMapper() 158 { 159 sMapperLock.lock(); 160 while ((uintptr_t) IOMapper::gSystem & kWaitMask) { 161 OSReportWithBacktrace("waitForSystemMapper"); 162 sMapperLock.sleep(&IOMapper::gSystem); 163 } 164 sMapperLock.unlock(); 165 } 166 167 OSSharedPtr<IOMapper> 168 IOMapper::copyMapperForDevice(IOService * device) 169 { 170 return copyMapperForDeviceWithIndex(device, 0); 171 } 172 173 OSSharedPtr<IOMapper> 174 IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned int index) 175 { 176 OSSharedPtr<OSData> data; 177 OSSharedPtr<OSObject> obj; 178 OSSharedPtr<IOMapper> mapper; 179 OSSharedPtr<OSDictionary> matching; 180 181 obj = device->copyProperty("iommu-parent"); 182 if (!obj) { 183 return NULL; 184 } 185 186 if ((mapper = OSDynamicPtrCast<IOMapper>(obj))) { 187 goto found; 188 } 189 190 if ((data = OSDynamicPtrCast<OSData>(obj))) { 191 if (index >= data->getLength() / sizeof(UInt32)) { 192 goto found; 193 } 194 195 data = OSData::withValueNoCopy(*((UInt32 *)data->getBytesNoCopy() + index)); 196 if (!data) { 197 goto found; 198 } 199 200 matching = IOService::propertyMatching(gIOMapperIDKey, data.get()); 201 } else { 202 matching = IOService::propertyMatching(gIOMapperIDKey, obj.get()); 203 } 204 205 if (matching) { 206 mapper = OSDynamicPtrCast<IOMapper>(IOService::waitForMatchingService(matching.get())); 207 } 208 209 found: 210 if (mapper) { 211 if (!mapper->fAllocName) { 212 char name[MACH_ZONE_NAME_MAX_LEN]; 213 char kmodname[KMOD_MAX_NAME]; 214 vm_tag_t tag; 215 uint32_t kmodid; 216 217 tag = IOMemoryTag(kernel_map); 218 if (!(kmodid = vm_tag_get_kext(tag, &kmodname[0], KMOD_MAX_NAME))) { 219 snprintf(kmodname, sizeof(kmodname), "%d", tag); 220 } 221 snprintf(name, sizeof(name), "%s.DMA.%s", kmodname, device->getName()); 222 mapper->fAllocName = kern_allocation_name_allocate(name, 16); 223 } 224 } 225 226 return mapper; 227 } 228 229 __BEGIN_DECLS 230 231 // These are C accessors to the system mapper for non-IOKit clients 232 ppnum_t 233 IOMapperIOVMAlloc(unsigned pages) 234 { 235 IOReturn ret; 236 uint64_t dmaAddress, dmaLength; 237 238 IOMapper::checkForSystemMapper(); 239 240 ret = kIOReturnUnsupported; 241 if (IOMapper::gSystem) { 242 ret = IOMapper::gSystem->iovmMapMemory( 243 NULL, 0, ptoa_64(pages), 244 (kIODMAMapReadAccess | kIODMAMapWriteAccess), 245 NULL, NULL, NULL, 246 &dmaAddress, &dmaLength); 247 } 248 249 if (kIOReturnSuccess == ret) { 250 uint64_t dmaAddressPage64; 251 dmaAddressPage64 = atop_64(dmaAddress); 252 if (dmaAddressPage64 > UINT_MAX) { 253 return 0; 254 } 255 return (ppnum_t) atop_64(dmaAddress); 256 } 257 return 0; 258 } 259 260 void 261 IOMapperIOVMFree(ppnum_t addr, unsigned pages) 262 { 263 if (IOMapper::gSystem) { 264 IOMapper::gSystem->iovmUnmapMemory(NULL, NULL, ptoa_64(addr), ptoa_64(pages)); 265 } 266 } 267 268 ppnum_t 269 IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page) 270 { 271 if (!IOMapper::gSystem) { 272 return page; 273 } 274 if (!addr) { 275 panic("!addr"); 276 } 277 IOMapper::gSystem->iovmInsert((kIODMAMapReadAccess | kIODMAMapWriteAccess), 278 ptoa_64(addr), ptoa_64(offset), ptoa_64(page), ptoa_64(1)); 279 return addr + offset; 280 } 281 282 ///////////////////////////////////////////////////////////////////////////// 283 // 284 // 285 // IOLib.h APIs 286 // 287 // 288 ///////////////////////////////////////////////////////////////////////////// 289 290 #include <machine/machine_routines.h> 291 292 UInt8 293 IOMappedRead8(IOPhysicalAddress address) 294 { 295 IOMapper::checkForSystemMapper(); 296 297 if (IOMapper::gSystem) { 298 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 299 return (UInt8) ml_phys_read_byte_64(addr); 300 } else { 301 return (UInt8) ml_phys_read_byte((vm_offset_t) address); 302 } 303 } 304 305 UInt16 306 IOMappedRead16(IOPhysicalAddress address) 307 { 308 IOMapper::checkForSystemMapper(); 309 310 if (IOMapper::gSystem) { 311 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 312 return (UInt16) ml_phys_read_half_64(addr); 313 } else { 314 return (UInt16) ml_phys_read_half((vm_offset_t) address); 315 } 316 } 317 318 UInt32 319 IOMappedRead32(IOPhysicalAddress address) 320 { 321 IOMapper::checkForSystemMapper(); 322 323 if (IOMapper::gSystem) { 324 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 325 return (UInt32) ml_phys_read_word_64(addr); 326 } else { 327 return (UInt32) ml_phys_read_word((vm_offset_t) address); 328 } 329 } 330 331 UInt64 332 IOMappedRead64(IOPhysicalAddress address) 333 { 334 IOMapper::checkForSystemMapper(); 335 336 if (IOMapper::gSystem) { 337 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 338 return (UInt64) ml_phys_read_double_64(addr); 339 } else { 340 return (UInt64) ml_phys_read_double((vm_offset_t) address); 341 } 342 } 343 344 void 345 IOMappedWrite8(IOPhysicalAddress address, UInt8 value) 346 { 347 IOMapper::checkForSystemMapper(); 348 349 if (IOMapper::gSystem) { 350 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 351 ml_phys_write_byte_64(addr, value); 352 } else { 353 ml_phys_write_byte((vm_offset_t) address, value); 354 } 355 } 356 357 void 358 IOMappedWrite16(IOPhysicalAddress address, UInt16 value) 359 { 360 IOMapper::checkForSystemMapper(); 361 362 if (IOMapper::gSystem) { 363 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 364 ml_phys_write_half_64(addr, value); 365 } else { 366 ml_phys_write_half((vm_offset_t) address, value); 367 } 368 } 369 370 void 371 IOMappedWrite32(IOPhysicalAddress address, UInt32 value) 372 { 373 IOMapper::checkForSystemMapper(); 374 375 if (IOMapper::gSystem) { 376 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 377 ml_phys_write_word_64(addr, value); 378 } else { 379 ml_phys_write_word((vm_offset_t) address, value); 380 } 381 } 382 383 void 384 IOMappedWrite64(IOPhysicalAddress address, UInt64 value) 385 { 386 IOMapper::checkForSystemMapper(); 387 388 if (IOMapper::gSystem) { 389 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address); 390 ml_phys_write_double_64(addr, value); 391 } else { 392 ml_phys_write_double((vm_offset_t) address, value); 393 } 394 } 395 396 __END_DECLS 397