1 /*
2 * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #define IOKIT_ENABLE_SHARED_PTR
30
31 #include <IOKit/IOLib.h>
32 #include <IOKit/IOMapper.h>
33 #include <IOKit/IODMACommand.h>
34 #include <libkern/c++/OSData.h>
35 #include <libkern/OSDebug.h>
36 #include <mach_debug/zone_info.h>
37 #include <vm/vm_iokit.h>
38 #include "IOKitKernelInternal.h"
39
40 __BEGIN_DECLS
41 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
42 __END_DECLS
43
44 #define super IOService
45 OSDefineMetaClassAndAbstractStructors(IOMapper, IOService);
46
47 OSMetaClassDefineReservedUnused(IOMapper, 0);
48 OSMetaClassDefineReservedUnused(IOMapper, 1);
49 OSMetaClassDefineReservedUnused(IOMapper, 2);
50 OSMetaClassDefineReservedUnused(IOMapper, 3);
51 OSMetaClassDefineReservedUnused(IOMapper, 4);
52 OSMetaClassDefineReservedUnused(IOMapper, 5);
53 OSMetaClassDefineReservedUnused(IOMapper, 6);
54 OSMetaClassDefineReservedUnused(IOMapper, 7);
55 OSMetaClassDefineReservedUnused(IOMapper, 8);
56 OSMetaClassDefineReservedUnused(IOMapper, 9);
57 OSMetaClassDefineReservedUnused(IOMapper, 10);
58 OSMetaClassDefineReservedUnused(IOMapper, 11);
59 OSMetaClassDefineReservedUnused(IOMapper, 12);
60 OSMetaClassDefineReservedUnused(IOMapper, 13);
61 OSMetaClassDefineReservedUnused(IOMapper, 14);
62 OSMetaClassDefineReservedUnused(IOMapper, 15);
63
64 IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown;
65
66 class IOMapperLock {
67 IOLock *fWaitLock;
68 public:
IOMapperLock()69 IOMapperLock()
70 {
71 fWaitLock = IOLockAlloc();
72 }
~IOMapperLock()73 ~IOMapperLock()
74 {
75 IOLockFree(fWaitLock);
76 }
77
78 void
lock()79 lock()
80 {
81 IOLockLock(fWaitLock);
82 }
83 void
unlock()84 unlock()
85 {
86 IOLockUnlock(fWaitLock);
87 }
88 void
sleep(void * event)89 sleep(void *event)
90 {
91 IOLockSleep(fWaitLock, event, THREAD_UNINT);
92 }
93 void
wakeup(void * event)94 wakeup(void *event)
95 {
96 IOLockWakeup(fWaitLock, event, false);
97 }
98 };
99
100 static IOMapperLock sMapperLock;
101
102 bool
start(IOService * provider)103 IOMapper::start(IOService *provider)
104 {
105 OSSharedPtr<OSObject> obj;
106 if (!super::start(provider)) {
107 return false;
108 }
109
110 if (!initHardware(provider)) {
111 return false;
112 }
113
114 uint64_t pageSize = getPageSize();
115 assert(pageSize <= UINT_MAX);
116 fPageSize = (uint32_t) pageSize;
117
118 if (fIsSystem) {
119 sMapperLock.lock();
120 IOMapper::gSystem = this;
121 sMapperLock.wakeup(&IOMapper::gSystem);
122 sMapperLock.unlock();
123 }
124
125 if (provider) {
126 obj = provider->copyProperty("iommu-id");
127 if (!obj) {
128 obj = provider->copyProperty("AAPL,phandle");
129 }
130 if (obj) {
131 setProperty(gIOMapperIDKey, obj.get());
132 }
133 }
134 return true;
135 }
136
137 void
free()138 IOMapper::free()
139 {
140 super::free();
141 }
142
143 void
setMapperRequired(bool hasMapper)144 IOMapper::setMapperRequired(bool hasMapper)
145 {
146 if (hasMapper) {
147 IOMapper::gSystem = (IOMapper *) kHasMapper;
148 } else {
149 sMapperLock.lock();
150 IOMapper::gSystem = (IOMapper *) kNoMapper;
151 sMapperLock.unlock();
152 sMapperLock.wakeup(&IOMapper::gSystem);
153 }
154 }
155
156 void
waitForSystemMapper()157 IOMapper::waitForSystemMapper()
158 {
159 sMapperLock.lock();
160 while ((uintptr_t) IOMapper::gSystem & kWaitMask) {
161 OSReportWithBacktrace("waitForSystemMapper");
162 sMapperLock.sleep(&IOMapper::gSystem);
163 }
164 sMapperLock.unlock();
165 }
166
167 OSSharedPtr<IOMapper>
copyMapperForDevice(IOService * device)168 IOMapper::copyMapperForDevice(IOService * device)
169 {
170 return copyMapperForDeviceWithIndex(device, 0);
171 }
172
173 OSSharedPtr<IOMapper>
copyMapperForDeviceWithIndex(IOService * device,unsigned int index)174 IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned int index)
175 {
176 OSSharedPtr<OSData> data;
177 OSSharedPtr<OSObject> obj;
178 OSSharedPtr<IOMapper> mapper;
179 OSSharedPtr<OSDictionary> matching;
180
181 obj = device->copyProperty("iommu-parent");
182 if (!obj) {
183 return NULL;
184 }
185
186 if ((mapper = OSDynamicPtrCast<IOMapper>(obj))) {
187 goto found;
188 }
189
190 if ((data = OSDynamicPtrCast<OSData>(obj))) {
191 if (index >= data->getLength() / sizeof(UInt32)) {
192 goto found;
193 }
194
195 data = OSData::withValueNoCopy(*((UInt32 *)data->getBytesNoCopy() + index));
196 if (!data) {
197 goto found;
198 }
199
200 matching = IOService::propertyMatching(gIOMapperIDKey, data.get());
201 } else {
202 matching = IOService::propertyMatching(gIOMapperIDKey, obj.get());
203 }
204
205 if (matching) {
206 mapper = OSDynamicPtrCast<IOMapper>(IOService::waitForMatchingService(matching.get()));
207 }
208
209 found:
210 if (mapper) {
211 if (!mapper->fAllocName) {
212 char name[MACH_ZONE_NAME_MAX_LEN];
213 char kmodname[KMOD_MAX_NAME];
214 vm_tag_t tag;
215 uint32_t kmodid;
216
217 tag = IOMemoryTag(kernel_map);
218 if (!(kmodid = vm_tag_get_kext(tag, &kmodname[0], KMOD_MAX_NAME))) {
219 snprintf(kmodname, sizeof(kmodname), "%d", tag);
220 }
221 snprintf(name, sizeof(name), "%s.DMA.%s", kmodname, device->getName());
222 mapper->fAllocName = kern_allocation_name_allocate(name, 16);
223 }
224 }
225
226 return mapper;
227 }
228
229 __BEGIN_DECLS
230
231 // These are C accessors to the system mapper for non-IOKit clients
232 ppnum_t
IOMapperIOVMAlloc(unsigned pages)233 IOMapperIOVMAlloc(unsigned pages)
234 {
235 IOReturn ret;
236 uint64_t dmaAddress, dmaLength;
237
238 IOMapper::checkForSystemMapper();
239
240 ret = kIOReturnUnsupported;
241 if (IOMapper::gSystem) {
242 ret = IOMapper::gSystem->iovmMapMemory(
243 NULL, 0, ptoa_64(pages),
244 (kIODMAMapReadAccess | kIODMAMapWriteAccess),
245 NULL, NULL, NULL,
246 &dmaAddress, &dmaLength);
247 }
248
249 if (kIOReturnSuccess == ret) {
250 uint64_t dmaAddressPage64;
251 dmaAddressPage64 = atop_64(dmaAddress);
252 if (dmaAddressPage64 > UINT_MAX) {
253 return 0;
254 }
255 return (ppnum_t) atop_64(dmaAddress);
256 }
257 return 0;
258 }
259
260 void
IOMapperIOVMFree(ppnum_t addr,unsigned pages)261 IOMapperIOVMFree(ppnum_t addr, unsigned pages)
262 {
263 if (IOMapper::gSystem) {
264 IOMapper::gSystem->iovmUnmapMemory(NULL, NULL, ptoa_64(addr), ptoa_64(pages));
265 }
266 }
267
268 ppnum_t
IOMapperInsertPage(ppnum_t addr,unsigned offset,ppnum_t page)269 IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page)
270 {
271 if (!IOMapper::gSystem) {
272 return page;
273 }
274 if (!addr) {
275 panic("!addr");
276 }
277 IOMapper::gSystem->iovmInsert((kIODMAMapReadAccess | kIODMAMapWriteAccess),
278 ptoa_64(addr), ptoa_64(offset), ptoa_64(page), ptoa_64(1));
279 return addr + offset;
280 }
281
282 /////////////////////////////////////////////////////////////////////////////
283 //
284 //
285 // IOLib.h APIs
286 //
287 //
288 /////////////////////////////////////////////////////////////////////////////
289
290 #include <machine/machine_routines.h>
291
292 UInt8
IOMappedRead8(IOPhysicalAddress address)293 IOMappedRead8(IOPhysicalAddress address)
294 {
295 IOMapper::checkForSystemMapper();
296
297 if (IOMapper::gSystem) {
298 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
299 return (UInt8) ml_phys_read_byte_64(addr);
300 } else {
301 return (UInt8) ml_phys_read_byte((vm_offset_t) address);
302 }
303 }
304
305 UInt16
IOMappedRead16(IOPhysicalAddress address)306 IOMappedRead16(IOPhysicalAddress address)
307 {
308 IOMapper::checkForSystemMapper();
309
310 if (IOMapper::gSystem) {
311 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
312 return (UInt16) ml_phys_read_half_64(addr);
313 } else {
314 return (UInt16) ml_phys_read_half((vm_offset_t) address);
315 }
316 }
317
318 UInt32
IOMappedRead32(IOPhysicalAddress address)319 IOMappedRead32(IOPhysicalAddress address)
320 {
321 IOMapper::checkForSystemMapper();
322
323 if (IOMapper::gSystem) {
324 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
325 return (UInt32) ml_phys_read_word_64(addr);
326 } else {
327 return (UInt32) ml_phys_read_word((vm_offset_t) address);
328 }
329 }
330
331 UInt64
IOMappedRead64(IOPhysicalAddress address)332 IOMappedRead64(IOPhysicalAddress address)
333 {
334 IOMapper::checkForSystemMapper();
335
336 if (IOMapper::gSystem) {
337 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
338 return (UInt64) ml_phys_read_double_64(addr);
339 } else {
340 return (UInt64) ml_phys_read_double((vm_offset_t) address);
341 }
342 }
343
344 void
IOMappedWrite8(IOPhysicalAddress address,UInt8 value)345 IOMappedWrite8(IOPhysicalAddress address, UInt8 value)
346 {
347 IOMapper::checkForSystemMapper();
348
349 if (IOMapper::gSystem) {
350 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
351 ml_phys_write_byte_64(addr, value);
352 } else {
353 ml_phys_write_byte((vm_offset_t) address, value);
354 }
355 }
356
357 void
IOMappedWrite16(IOPhysicalAddress address,UInt16 value)358 IOMappedWrite16(IOPhysicalAddress address, UInt16 value)
359 {
360 IOMapper::checkForSystemMapper();
361
362 if (IOMapper::gSystem) {
363 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
364 ml_phys_write_half_64(addr, value);
365 } else {
366 ml_phys_write_half((vm_offset_t) address, value);
367 }
368 }
369
370 void
IOMappedWrite32(IOPhysicalAddress address,UInt32 value)371 IOMappedWrite32(IOPhysicalAddress address, UInt32 value)
372 {
373 IOMapper::checkForSystemMapper();
374
375 if (IOMapper::gSystem) {
376 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
377 ml_phys_write_word_64(addr, value);
378 } else {
379 ml_phys_write_word((vm_offset_t) address, value);
380 }
381 }
382
383 void
IOMappedWrite64(IOPhysicalAddress address,UInt64 value)384 IOMappedWrite64(IOPhysicalAddress address, UInt64 value)
385 {
386 IOMapper::checkForSystemMapper();
387
388 if (IOMapper::gSystem) {
389 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
390 ml_phys_write_double_64(addr, value);
391 } else {
392 ml_phys_write_double((vm_offset_t) address, value);
393 }
394 }
395
396 __END_DECLS
397