1 /* 2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 #define IOKIT_ENABLE_SHARED_PTR 29 30 #define _IOMEMORYDESCRIPTOR_INTERNAL_ 31 32 #include <IOKit/assert.h> 33 #include <IOKit/system.h> 34 35 #include <IOKit/IOLib.h> 36 #include <IOKit/IOMapper.h> 37 #include <IOKit/IOBufferMemoryDescriptor.h> 38 #include <libkern/OSDebug.h> 39 #include <mach/mach_vm.h> 40 41 #include "IOKitKernelInternal.h" 42 43 #ifdef IOALLOCDEBUG 44 #include <libkern/c++/OSCPPDebug.h> 45 #endif 46 #include <IOKit/IOStatisticsPrivate.h> 47 48 #if IOKITSTATS 49 #define IOStatisticsAlloc(type, size) \ 50 do { \ 51 IOStatistics::countAlloc(type, size); \ 52 } while (0) 53 #else 54 #define IOStatisticsAlloc(type, size) 55 #endif /* IOKITSTATS */ 56 57 58 __BEGIN_DECLS 59 void ipc_port_release_send(ipc_port_t port); 60 #include <vm/pmap.h> 61 62 KALLOC_HEAP_DEFINE(KHEAP_IOBMD_CONTROL, "IOBMD_control", KHEAP_ID_KT_VAR); 63 __END_DECLS 64 65 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 66 67 enum{ 68 kInternalFlagPhysical = 0x00000001, 69 kInternalFlagPageSized = 0x00000002, 70 kInternalFlagPageAllocated = 0x00000004, 71 kInternalFlagInit = 0x00000008, 72 kInternalFlagHasPointers = 0x00000010, 73 kInternalFlagGuardPages = 0x00000020, 74 }; 75 76 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 77 78 #define super IOGeneralMemoryDescriptor 79 OSDefineMetaClassAndStructorsWithZone(IOBufferMemoryDescriptor, 80 IOGeneralMemoryDescriptor, ZC_ZFREE_CLEARMEM); 81 82 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 83 84 #if defined(__x86_64__) 85 static uintptr_t 86 IOBMDPageProc(kalloc_heap_t kheap, iopa_t * a) 87 { 88 kern_return_t kr; 89 vm_address_t vmaddr = 0; 90 kma_flags_t kma_flags = KMA_ZERO; 91 92 if (kheap == KHEAP_DATA_BUFFERS) { 93 kma_flags = (kma_flags_t) (kma_flags | KMA_DATA); 94 } 95 kr = kmem_alloc(kernel_map, &vmaddr, page_size, 96 kma_flags, VM_KERN_MEMORY_IOKIT); 97 98 if (KERN_SUCCESS != kr) { 99 vmaddr = 0; 100 } 101 102 return (uintptr_t) vmaddr; 103 } 104 #endif /* defined(__x86_64__) */ 105 106 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 107 108 #ifndef __LP64__ 109 bool 110 IOBufferMemoryDescriptor::initWithOptions( 111 IOOptionBits options, 112 vm_size_t capacity, 113 vm_offset_t alignment, 114 task_t inTask) 115 { 116 mach_vm_address_t physicalMask = 0; 117 return initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask); 118 } 119 #endif /* !__LP64__ */ 120 121 OSSharedPtr<IOBufferMemoryDescriptor> 122 IOBufferMemoryDescriptor::withCopy( 123 task_t inTask, 124 IOOptionBits options, 125 vm_map_t sourceMap, 126 mach_vm_address_t source, 127 mach_vm_size_t size) 128 { 129 OSSharedPtr<IOBufferMemoryDescriptor> inst; 130 kern_return_t err; 131 vm_map_copy_t copy; 132 vm_map_address_t address; 133 134 copy = NULL; 135 do { 136 err = kIOReturnNoMemory; 137 inst = OSMakeShared<IOBufferMemoryDescriptor>(); 138 if (!inst) { 139 break; 140 } 141 inst->_ranges.v64 = IOMallocType(IOAddressRange); 142 143 err = vm_map_copyin(sourceMap, source, size, 144 false /* src_destroy */, ©); 145 if (KERN_SUCCESS != err) { 146 break; 147 } 148 149 err = vm_map_copyout(get_task_map(inTask), &address, copy); 150 if (KERN_SUCCESS != err) { 151 break; 152 } 153 copy = NULL; 154 155 inst->_ranges.v64->address = address; 156 inst->_ranges.v64->length = size; 157 158 if (!inst->initWithPhysicalMask(inTask, options, size, page_size, 0)) { 159 err = kIOReturnError; 160 } 161 } while (false); 162 163 if (KERN_SUCCESS == err) { 164 return inst; 165 } 166 167 if (copy) { 168 vm_map_copy_discard(copy); 169 } 170 171 return nullptr; 172 } 173 174 175 bool 176 IOBufferMemoryDescriptor::initWithPhysicalMask( 177 task_t inTask, 178 IOOptionBits options, 179 mach_vm_size_t capacity, 180 mach_vm_address_t alignment, 181 mach_vm_address_t physicalMask) 182 { 183 task_t mapTask = NULL; 184 kalloc_heap_t kheap = KHEAP_DATA_BUFFERS; 185 mach_vm_address_t highestMask = 0; 186 IOOptionBits iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference; 187 IODMAMapSpecification mapSpec; 188 bool mapped = false; 189 bool withCopy = false; 190 bool mappedOrShared = false; 191 192 if (!capacity) { 193 return false; 194 } 195 196 /* 197 * The IOKit constructor requests the allocator for zeroed memory 198 * so the members of the class do not need to be explicitly zeroed. 199 */ 200 _options = options; 201 _capacity = capacity; 202 203 if (!_ranges.v64) { 204 _ranges.v64 = IOMallocType(IOAddressRange); 205 _ranges.v64->address = 0; 206 _ranges.v64->length = 0; 207 } else { 208 if (!_ranges.v64->address) { 209 return false; 210 } 211 if (!(kIOMemoryPageable & options)) { 212 return false; 213 } 214 if (!inTask) { 215 return false; 216 } 217 _buffer = (void *) _ranges.v64->address; 218 withCopy = true; 219 } 220 221 /* 222 * Set kalloc_heap to KHEAP_IOBMD_CONTROL if allocation contains pointers 223 */ 224 if (kInternalFlagHasPointers & _internalFlags) { 225 kheap = KHEAP_IOBMD_CONTROL; 226 } 227 228 // make sure super::free doesn't dealloc _ranges before super::init 229 _flags = kIOMemoryAsReference; 230 231 // Grab IOMD bits from the Buffer MD options 232 iomdOptions |= (options & kIOBufferDescriptorMemoryFlags); 233 234 if (!(kIOMemoryMapperNone & options)) { 235 IOMapper::checkForSystemMapper(); 236 mapped = (NULL != IOMapper::gSystem); 237 } 238 239 if (physicalMask && (alignment <= 1)) { 240 alignment = ((physicalMask ^ (-1ULL)) & (physicalMask - 1)); 241 highestMask = (physicalMask | alignment); 242 alignment++; 243 if (alignment < page_size) { 244 alignment = page_size; 245 } 246 } 247 248 if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size)) { 249 alignment = page_size; 250 } 251 252 if (alignment >= page_size) { 253 if (round_page_overflow(capacity, &capacity)) { 254 return false; 255 } 256 } 257 258 if (alignment > page_size) { 259 options |= kIOMemoryPhysicallyContiguous; 260 } 261 262 _alignment = alignment; 263 264 if ((capacity + alignment) < _capacity) { 265 return false; 266 } 267 268 if ((inTask != kernel_task) && !(options & kIOMemoryPageable)) { 269 return false; 270 } 271 272 bzero(&mapSpec, sizeof(mapSpec)); 273 mapSpec.alignment = _alignment; 274 mapSpec.numAddressBits = 64; 275 if (highestMask && mapped) { 276 if (highestMask <= 0xFFFFFFFF) { 277 mapSpec.numAddressBits = (uint8_t)(32 - __builtin_clz((unsigned int) highestMask)); 278 } else { 279 mapSpec.numAddressBits = (uint8_t)(64 - __builtin_clz((unsigned int) (highestMask >> 32))); 280 } 281 highestMask = 0; 282 } 283 284 // set memory entry cache mode, pageable, purgeable 285 iomdOptions |= ((options & kIOMapCacheMask) >> kIOMapCacheShift) << kIOMemoryBufferCacheShift; 286 if (options & kIOMemoryPageable) { 287 if (_internalFlags & kInternalFlagGuardPages) { 288 printf("IOBMD: Unsupported use of guard pages with pageable memory.\n"); 289 return false; 290 } 291 iomdOptions |= kIOMemoryBufferPageable; 292 if (options & kIOMemoryPurgeable) { 293 iomdOptions |= kIOMemoryBufferPurgeable; 294 } 295 } else { 296 // Buffer shouldn't auto prepare they should be prepared explicitly 297 // But it never was enforced so what are you going to do? 298 iomdOptions |= kIOMemoryAutoPrepare; 299 300 /* Allocate a wired-down buffer inside kernel space. */ 301 302 bool contig = (0 != (options & kIOMemoryHostPhysicallyContiguous)); 303 304 if (!contig && (0 != (options & kIOMemoryPhysicallyContiguous))) { 305 contig |= (!mapped); 306 contig |= (0 != (kIOMemoryMapperNone & options)); 307 #if 0 308 // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now 309 contig |= true; 310 #endif 311 } 312 313 mappedOrShared = (mapped || (0 != (kIOMemorySharingTypeMask & options))); 314 if (contig || highestMask || (alignment > page_size)) { 315 if (_internalFlags & kInternalFlagGuardPages) { 316 printf("IOBMD: Unsupported use of guard pages with physical mask or contiguous memory.\n"); 317 return false; 318 } 319 _internalFlags |= kInternalFlagPhysical; 320 if (highestMask) { 321 _internalFlags |= kInternalFlagPageSized; 322 if (round_page_overflow(capacity, &capacity)) { 323 return false; 324 } 325 } 326 _buffer = (void *) IOKernelAllocateWithPhysicalRestrict(kheap, 327 capacity, highestMask, alignment, contig); 328 } else if (_internalFlags & kInternalFlagGuardPages) { 329 vm_offset_t address = 0; 330 kern_return_t kr; 331 uintptr_t alignMask; 332 kma_flags_t kma_flags = (kma_flags_t) (KMA_GUARD_FIRST | 333 KMA_GUARD_LAST | KMA_ZERO); 334 335 if (((uint32_t) alignment) != alignment) { 336 return false; 337 } 338 if (kheap == KHEAP_DATA_BUFFERS) { 339 kma_flags = (kma_flags_t) (kma_flags | KMA_DATA); 340 } 341 342 alignMask = (1UL << log2up((uint32_t) alignment)) - 1; 343 kr = kernel_memory_allocate(kernel_map, &address, 344 capacity + page_size * 2, alignMask, kma_flags, 345 IOMemoryTag(kernel_map)); 346 if (kr != KERN_SUCCESS || address == 0) { 347 return false; 348 } 349 #if IOALLOCDEBUG 350 OSAddAtomicLong(capacity, &debug_iomalloc_size); 351 #endif 352 IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity); 353 _buffer = (void *)(address + page_size); 354 #if defined(__x86_64__) 355 } else if (mappedOrShared 356 && (capacity + alignment) <= (page_size - gIOPageAllocChunkBytes)) { 357 _internalFlags |= kInternalFlagPageAllocated; 358 _buffer = (void *) iopa_alloc(&gIOBMDPageAllocator, 359 &IOBMDPageProc, kheap, capacity, alignment); 360 if (_buffer) { 361 bzero(_buffer, capacity); 362 IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity); 363 #if IOALLOCDEBUG 364 OSAddAtomicLong(capacity, &debug_iomalloc_size); 365 #endif 366 } 367 #endif /* defined(__x86_64__) */ 368 } else if (alignment > 1) { 369 /* BEGIN IGNORE CODESTYLE */ 370 __typed_allocators_ignore_push 371 _buffer = IOMallocAligned_internal(kheap, capacity, alignment, 372 Z_ZERO_VM_TAG_BT_BIT); 373 } else { 374 _buffer = IOMalloc_internal(kheap, capacity, Z_ZERO_VM_TAG_BT_BIT); 375 __typed_allocators_ignore_pop 376 /* END IGNORE CODESTYLE */ 377 } 378 if (!_buffer) { 379 return false; 380 } 381 } 382 383 if ((options & (kIOMemoryPageable | kIOMapCacheMask))) { 384 vm_size_t size = round_page(capacity); 385 386 // initWithOptions will create memory entry 387 if (!withCopy) { 388 iomdOptions |= kIOMemoryPersistent; 389 } 390 391 if (options & kIOMemoryPageable) { 392 #if IOALLOCDEBUG 393 OSAddAtomicLong(size, &debug_iomallocpageable_size); 394 #endif 395 if (!withCopy) { 396 mapTask = inTask; 397 } 398 if (NULL == inTask) { 399 inTask = kernel_task; 400 } 401 } else if (options & kIOMapCacheMask) { 402 // Prefetch each page to put entries into the pmap 403 volatile UInt8 * startAddr = (UInt8 *)_buffer; 404 volatile UInt8 * endAddr = (UInt8 *)_buffer + capacity; 405 406 while (startAddr < endAddr) { 407 UInt8 dummyVar = *startAddr; 408 (void) dummyVar; 409 startAddr += page_size; 410 } 411 } 412 } 413 414 _ranges.v64->address = (mach_vm_address_t) pgz_decode(_buffer, _capacity); 415 _ranges.v64->length = _capacity; 416 417 if (!super::initWithOptions(_ranges.v64, 1, 0, 418 inTask, iomdOptions, /* System mapper */ NULL)) { 419 return false; 420 } 421 422 _internalFlags |= kInternalFlagInit; 423 #if IOTRACKING 424 if (!(options & kIOMemoryPageable)) { 425 trackingAccumSize(capacity); 426 } 427 #endif /* IOTRACKING */ 428 429 // give any system mapper the allocation params 430 if (kIOReturnSuccess != dmaCommandOperation(kIOMDAddDMAMapSpec, 431 &mapSpec, sizeof(mapSpec))) { 432 return false; 433 } 434 435 if (mapTask) { 436 if (!reserved) { 437 reserved = IOMallocType(ExpansionData); 438 if (!reserved) { 439 return false; 440 } 441 } 442 reserved->map = createMappingInTask(mapTask, 0, 443 kIOMapAnywhere | (options & kIOMapPrefault) | (options & kIOMapCacheMask), 0, 0).detach(); 444 if (!reserved->map) { 445 _buffer = NULL; 446 return false; 447 } 448 release(); // map took a retain on this 449 reserved->map->retain(); 450 removeMapping(reserved->map); 451 mach_vm_address_t buffer = reserved->map->getAddress(); 452 _buffer = (void *) buffer; 453 if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions)) { 454 _ranges.v64->address = buffer; 455 } 456 } 457 458 setLength(_capacity); 459 460 return true; 461 } 462 463 bool 464 IOBufferMemoryDescriptor::initControlWithPhysicalMask( 465 task_t inTask, 466 IOOptionBits options, 467 mach_vm_size_t capacity, 468 mach_vm_address_t alignment, 469 mach_vm_address_t physicalMask) 470 { 471 _internalFlags = kInternalFlagHasPointers; 472 return initWithPhysicalMask(inTask, options, capacity, alignment, 473 physicalMask); 474 } 475 476 bool 477 IOBufferMemoryDescriptor::initWithGuardPages( 478 task_t inTask, 479 IOOptionBits options, 480 mach_vm_size_t capacity) 481 { 482 mach_vm_size_t roundedCapacity; 483 484 _internalFlags = kInternalFlagGuardPages; 485 486 if (round_page_overflow(capacity, &roundedCapacity)) { 487 return false; 488 } 489 490 return initWithPhysicalMask(inTask, options, roundedCapacity, page_size, 491 (mach_vm_address_t)0); 492 } 493 494 OSSharedPtr<IOBufferMemoryDescriptor> 495 IOBufferMemoryDescriptor::inTaskWithOptions( 496 task_t inTask, 497 IOOptionBits options, 498 vm_size_t capacity, 499 vm_offset_t alignment) 500 { 501 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>(); 502 503 if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) { 504 me.reset(); 505 } 506 return me; 507 } 508 509 OSSharedPtr<IOBufferMemoryDescriptor> 510 IOBufferMemoryDescriptor::inTaskWithOptions( 511 task_t inTask, 512 IOOptionBits options, 513 vm_size_t capacity, 514 vm_offset_t alignment, 515 uint32_t kernTag, 516 uint32_t userTag) 517 { 518 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>(); 519 520 if (me) { 521 me->setVMTags(kernTag, userTag); 522 523 if (!me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) { 524 me.reset(); 525 } 526 } 527 return me; 528 } 529 530 OSSharedPtr<IOBufferMemoryDescriptor> 531 IOBufferMemoryDescriptor::inTaskWithPhysicalMask( 532 task_t inTask, 533 IOOptionBits options, 534 mach_vm_size_t capacity, 535 mach_vm_address_t physicalMask) 536 { 537 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>(); 538 539 if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask)) { 540 me.reset(); 541 } 542 return me; 543 } 544 545 OSSharedPtr<IOBufferMemoryDescriptor> 546 IOBufferMemoryDescriptor::inTaskWithGuardPages( 547 task_t inTask, 548 IOOptionBits options, 549 mach_vm_size_t capacity) 550 { 551 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>(); 552 553 if (me && !me->initWithGuardPages(inTask, options, capacity)) { 554 me.reset(); 555 } 556 return me; 557 } 558 559 #ifndef __LP64__ 560 bool 561 IOBufferMemoryDescriptor::initWithOptions( 562 IOOptionBits options, 563 vm_size_t capacity, 564 vm_offset_t alignment) 565 { 566 return initWithPhysicalMask(kernel_task, options, capacity, alignment, (mach_vm_address_t)0); 567 } 568 #endif /* !__LP64__ */ 569 570 OSSharedPtr<IOBufferMemoryDescriptor> 571 IOBufferMemoryDescriptor::withOptions( 572 IOOptionBits options, 573 vm_size_t capacity, 574 vm_offset_t alignment) 575 { 576 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>(); 577 578 if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) { 579 me.reset(); 580 } 581 return me; 582 } 583 584 585 /* 586 * withCapacity: 587 * 588 * Returns a new IOBufferMemoryDescriptor with a buffer large enough to 589 * hold capacity bytes. The descriptor's length is initially set to the capacity. 590 */ 591 OSSharedPtr<IOBufferMemoryDescriptor> 592 IOBufferMemoryDescriptor::withCapacity(vm_size_t inCapacity, 593 IODirection inDirection, 594 bool inContiguous) 595 { 596 return IOBufferMemoryDescriptor::withOptions( 597 inDirection | kIOMemoryUnshared 598 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0), 599 inCapacity, inContiguous ? inCapacity : 1 ); 600 } 601 602 #ifndef __LP64__ 603 /* 604 * initWithBytes: 605 * 606 * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied). 607 * The descriptor's length and capacity are set to the input buffer's size. 608 */ 609 bool 610 IOBufferMemoryDescriptor::initWithBytes(const void * inBytes, 611 vm_size_t inLength, 612 IODirection inDirection, 613 bool inContiguous) 614 { 615 if (!initWithPhysicalMask(kernel_task, inDirection | kIOMemoryUnshared 616 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0), 617 inLength, inLength, (mach_vm_address_t)0)) { 618 return false; 619 } 620 621 // start out with no data 622 setLength(0); 623 624 if (!appendBytes(inBytes, inLength)) { 625 return false; 626 } 627 628 return true; 629 } 630 #endif /* !__LP64__ */ 631 632 /* 633 * withBytes: 634 * 635 * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied). 636 * The descriptor's length and capacity are set to the input buffer's size. 637 */ 638 OSSharedPtr<IOBufferMemoryDescriptor> 639 IOBufferMemoryDescriptor::withBytes(const void * inBytes, 640 vm_size_t inLength, 641 IODirection inDirection, 642 bool inContiguous) 643 { 644 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>(); 645 mach_vm_address_t alignment; 646 647 alignment = (inLength <= page_size) ? inLength : page_size; 648 if (me && !me->initWithPhysicalMask( 649 kernel_task, inDirection | kIOMemoryUnshared 650 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0), 651 inLength, alignment, 0 )) { 652 me.reset(); 653 } 654 655 if (me) { 656 // start out with no data 657 me->setLength(0); 658 659 if (!me->appendBytes(inBytes, inLength)) { 660 me.reset(); 661 } 662 } 663 return me; 664 } 665 666 /* 667 * free: 668 * 669 * Free resources 670 */ 671 void 672 IOBufferMemoryDescriptor::free() 673 { 674 // Cache all of the relevant information on the stack for use 675 // after we call super::free()! 676 IOOptionBits flags = _flags; 677 IOOptionBits internalFlags = _internalFlags; 678 IOOptionBits options = _options; 679 vm_size_t size = _capacity; 680 void * buffer = _buffer; 681 IOMemoryMap * map = NULL; 682 IOAddressRange * range = _ranges.v64; 683 vm_offset_t alignment = _alignment; 684 kalloc_heap_t kheap = KHEAP_DATA_BUFFERS; 685 vm_size_t rsize; 686 687 if (alignment >= page_size) { 688 if (!round_page_overflow(size, &rsize)) { 689 size = rsize; 690 } 691 } 692 693 if (reserved) { 694 map = reserved->map; 695 IOFreeType(reserved, ExpansionData); 696 if (map) { 697 map->release(); 698 } 699 } 700 701 if ((options & kIOMemoryPageable) 702 || (kInternalFlagPageSized & internalFlags)) { 703 if (!round_page_overflow(size, &rsize)) { 704 size = rsize; 705 } 706 } 707 708 if (internalFlags & kInternalFlagHasPointers) { 709 kheap = KHEAP_IOBMD_CONTROL; 710 } 711 712 #if IOTRACKING 713 if (!(options & kIOMemoryPageable) 714 && buffer 715 && (kInternalFlagInit & _internalFlags)) { 716 trackingAccumSize(-size); 717 } 718 #endif /* IOTRACKING */ 719 720 /* super::free may unwire - deallocate buffer afterwards */ 721 super::free(); 722 723 if (options & kIOMemoryPageable) { 724 #if IOALLOCDEBUG 725 OSAddAtomicLong(-size, &debug_iomallocpageable_size); 726 #endif 727 } else if (buffer) { 728 if (kInternalFlagPhysical & internalFlags) { 729 IOKernelFreePhysical(kheap, (mach_vm_address_t) buffer, size); 730 } else if (kInternalFlagPageAllocated & internalFlags) { 731 #if defined(__x86_64__) 732 uintptr_t page; 733 page = iopa_free(&gIOBMDPageAllocator, (uintptr_t) buffer, size); 734 if (page) { 735 kmem_free(kernel_map, page, page_size); 736 } 737 #if IOALLOCDEBUG 738 OSAddAtomicLong(-size, &debug_iomalloc_size); 739 #endif 740 IOStatisticsAlloc(kIOStatisticsFreeAligned, size); 741 #else /* !defined(__x86_64__) */ 742 /* should be unreachable */ 743 panic("Attempting to free IOBMD with page allocated flag"); 744 #endif /* defined(__x86_64__) */ 745 } else if (kInternalFlagGuardPages & internalFlags) { 746 vm_offset_t allocation = (vm_offset_t)buffer - page_size; 747 kmem_free(kernel_map, allocation, size + page_size * 2); 748 #if IOALLOCDEBUG 749 OSAddAtomicLong(-size, &debug_iomalloc_size); 750 #endif 751 IOStatisticsAlloc(kIOStatisticsFreeAligned, size); 752 } else if (alignment > 1) { 753 /* BEGIN IGNORE CODESTYLE */ 754 __typed_allocators_ignore_push 755 IOFreeAligned_internal(kheap, buffer, size); 756 } else { 757 IOFree_internal(kheap, buffer, size); 758 __typed_allocators_ignore_pop 759 /* END IGNORE CODESTYLE */ 760 } 761 } 762 if (range && (kIOMemoryAsReference & flags)) { 763 IOFreeType(range, IOAddressRange); 764 } 765 } 766 767 /* 768 * getCapacity: 769 * 770 * Get the buffer capacity 771 */ 772 vm_size_t 773 IOBufferMemoryDescriptor::getCapacity() const 774 { 775 return _capacity; 776 } 777 778 /* 779 * setLength: 780 * 781 * Change the buffer length of the memory descriptor. When a new buffer 782 * is created, the initial length of the buffer is set to be the same as 783 * the capacity. The length can be adjusted via setLength for a shorter 784 * transfer (there is no need to create more buffer descriptors when you 785 * can reuse an existing one, even for different transfer sizes). Note 786 * that the specified length must not exceed the capacity of the buffer. 787 */ 788 void 789 IOBufferMemoryDescriptor::setLength(vm_size_t length) 790 { 791 assert(length <= _capacity); 792 if (length > _capacity) { 793 return; 794 } 795 796 _length = length; 797 _ranges.v64->length = length; 798 } 799 800 /* 801 * setDirection: 802 * 803 * Change the direction of the transfer. This method allows one to redirect 804 * the descriptor's transfer direction. This eliminates the need to destroy 805 * and create new buffers when different transfer directions are needed. 806 */ 807 void 808 IOBufferMemoryDescriptor::setDirection(IODirection direction) 809 { 810 _flags = (_flags & ~kIOMemoryDirectionMask) | direction; 811 #ifndef __LP64__ 812 _direction = (IODirection) (_flags & kIOMemoryDirectionMask); 813 #endif /* !__LP64__ */ 814 } 815 816 /* 817 * appendBytes: 818 * 819 * Add some data to the end of the buffer. This method automatically 820 * maintains the memory descriptor buffer length. Note that appendBytes 821 * will not copy past the end of the memory descriptor's current capacity. 822 */ 823 bool 824 IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength) 825 { 826 vm_size_t actualBytesToCopy = min(withLength, _capacity - _length); 827 IOByteCount offset; 828 829 assert(_length <= _capacity); 830 831 offset = _length; 832 _length += actualBytesToCopy; 833 _ranges.v64->length += actualBytesToCopy; 834 835 if (_task == kernel_task) { 836 bcopy(/* from */ bytes, (void *)(_ranges.v64->address + offset), 837 actualBytesToCopy); 838 } else { 839 writeBytes(offset, bytes, actualBytesToCopy); 840 } 841 842 return true; 843 } 844 845 /* 846 * getBytesNoCopy: 847 * 848 * Return the virtual address of the beginning of the buffer 849 */ 850 void * 851 IOBufferMemoryDescriptor::getBytesNoCopy() 852 { 853 if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask)) { 854 return _buffer; 855 } else { 856 return (void *)_ranges.v64->address; 857 } 858 } 859 860 861 /* 862 * getBytesNoCopy: 863 * 864 * Return the virtual address of an offset from the beginning of the buffer 865 */ 866 void * 867 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength) 868 { 869 IOVirtualAddress address; 870 871 if ((start + withLength) < start) { 872 return NULL; 873 } 874 875 if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask)) { 876 address = (IOVirtualAddress) _buffer; 877 } else { 878 address = _ranges.v64->address; 879 } 880 881 if (start < _length && (start + withLength) <= _length) { 882 return (void *)(address + start); 883 } 884 return NULL; 885 } 886 887 #ifndef __LP64__ 888 void * 889 IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset, 890 IOByteCount * lengthOfSegment) 891 { 892 void * bytes = getBytesNoCopy(offset, 0); 893 894 if (bytes && lengthOfSegment) { 895 *lengthOfSegment = _length - offset; 896 } 897 898 return bytes; 899 } 900 #endif /* !__LP64__ */ 901 902 #ifdef __LP64__ 903 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 0); 904 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1); 905 #else /* !__LP64__ */ 906 OSMetaClassDefineReservedUsedX86(IOBufferMemoryDescriptor, 0); 907 OSMetaClassDefineReservedUsedX86(IOBufferMemoryDescriptor, 1); 908 #endif /* !__LP64__ */ 909 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2); 910 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3); 911 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4); 912 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 5); 913 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 6); 914 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 7); 915 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 8); 916 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 9); 917 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 10); 918 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 11); 919 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 12); 920 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 13); 921 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 14); 922 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 15); 923