Lines Matching refs:state
79 #define SHOULD_COPY_DIR(state, direction) (true) argument
507 IODMACommandInternal * state = target->reserved; in segmentOp() local
509 …if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperAllocValid ||… in segmentOp()
521 if (!state->fMisaligned) { in segmentOp()
522 mask = (segmentIndex ? target->fAlignMaskInternalSegments : state->fSourceAlignMask); in segmentOp()
523 state->fMisaligned |= (0 != (mask & address)); in segmentOp()
524 if (state->fMisaligned) { in segmentOp()
528 if (!state->fMisaligned) { in segmentOp()
530 state->fMisaligned |= (0 != (mask & length)); in segmentOp()
531 if (state->fMisaligned) { in segmentOp()
536 if (state->fMisaligned && (kWalkPreflight & op)) { in segmentOp()
540 if (!state->fDoubleBuffer) { in segmentOp()
562 state->fCopyPageCount += numPages; in segmentOp()
567 lastPage = state->fCopyNext; in segmentOp()
575 lastPage = state->fCopyNext; in segmentOp()
588 if (!state->fDoubleBuffer) { in segmentOp()
616 state->fCopyNext = lastPage; in segmentOp()
633 IODMACommandInternal * state = fInternalState; in walkAll() local
640 state->fMisaligned = false; in walkAll()
641 state->fDoubleBuffer = false; in walkAll()
642 state->fPrepared = false; in walkAll()
643 state->fCopyNext = NULL; in walkAll()
644 state->fCopyPageAlloc = NULL; in walkAll()
645 state->fCopyPageCount = 0; in walkAll()
646 state->fNextRemapPage = NULL; in walkAll()
647 state->fCopyMD = NULL; in walkAll()
652 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); in walkAll()
657 state->fDoubleBuffer = (state->fMisaligned || state->fForceDoubleBuffer); in walkAll()
658 state->fForceDoubleBuffer = false; in walkAll()
659 if (state->fDoubleBuffer) { in walkAll()
660 …state->fCopyPageCount = (typeof(state->fCopyPageCount))(atop_64(round_page(state->fPreparedLength)… in walkAll()
663 if (state->fCopyPageCount) { in walkAll()
666 DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount); in walkAll()
668 if (!fMapper && !state->fDoubleBuffer) { in walkAll()
675 kr = vm_page_alloc_list(state->fCopyPageCount, in walkAll()
678 DEBG("vm_page_alloc_list(%d) failed (%d)\n", state->fCopyPageCount, kr); in walkAll()
684 state->fCopyPageAlloc = mapBase; in walkAll()
685 state->fCopyNext = state->fCopyPageAlloc; in walkAll()
688 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); in walkAll()
689 state->fPrepared = true; in walkAll()
693 state->fCopyMD = createCopyBuffer(fMDSummary.fDirection, state->fPreparedLength); in walkAll()
695 if (state->fCopyMD) { in walkAll()
697 state->fPrepared = true; in walkAll()
706 if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op)) { in walkAll()
707 if (state->fCopyPageCount) { in walkAll()
708 DEBG("sync fCopyPageCount %d\n", state->fCopyPageCount); in walkAll()
710 if (state->fCopyPageAlloc) { in walkAll()
711 state->fCopyNext = state->fCopyPageAlloc; in walkAll()
714 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); in walkAll()
715 } else if (state->fCopyMD) { in walkAll()
724 bytes = poMD->writeBytes(state->fPreparedOffset, in walkAll()
725 state->fCopyMD->getBytesNoCopy(), in walkAll()
726 state->fPreparedLength); in walkAll()
728 bytes = poMD->readBytes(state->fPreparedOffset, in walkAll()
729 state->fCopyMD->getBytesNoCopy(), in walkAll()
730 state->fPreparedLength); in walkAll()
733 ret = (bytes == state->fPreparedLength) ? kIOReturnSuccess : kIOReturnUnderrun; in walkAll()
742 if (state->fCopyPageAlloc) { in walkAll()
743 vm_page_free_list(state->fCopyPageAlloc, FALSE); in walkAll()
744 state->fCopyPageAlloc = NULL; in walkAll()
745 state->fCopyPageCount = 0; in walkAll()
747 if (state->fCopyMD) { in walkAll()
748 state->fCopyMD.reset(); in walkAll()
751 state->fPrepared = false; in walkAll()
838 IODMACommandInternal * state = fInternalState; in prepare() local
856 if ((state->fPreparedOffset != offset) in prepare()
857 || (state->fPreparedLength != length)) { in prepare()
865 if (atop_64(state->fPreparedLength) > UINT_MAX) { in prepare()
868 state->fPreparedOffset = offset; in prepare()
869 state->fPreparedLength = length; in prepare()
871 state->fMisaligned = false; in prepare()
872 state->fDoubleBuffer = false; in prepare()
873 state->fPrepared = false; in prepare()
874 state->fCopyNext = NULL; in prepare()
875 state->fCopyPageAlloc = NULL; in prepare()
876 state->fCopyPageCount = 0; in prepare()
877 state->fNextRemapPage = NULL; in prepare()
878 state->fCopyMD = NULL; in prepare()
879 state->fLocalMapperAlloc = 0; in prepare()
880 state->fLocalMapperAllocValid = false; in prepare()
881 state->fLocalMapperAllocLength = 0; in prepare()
883 state->fSourceAlignMask = fAlignMask; in prepare()
885 state->fSourceAlignMask &= page_mask; in prepare()
888 state->fCursor = state->fIterateOnly in prepare()
889 || (!state->fCheckAddressing in prepare()
890 && (!state->fSourceAlignMask in prepare()
891 …|| ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask)… in prepare()
893 if (!state->fCursor) { in prepare()
902 if (state->fCopyMD) { in prepare()
903 state->fCopyMD->performOperation(kIOMemoryIncoherentIOStore, 0, length); in prepare()
914 mapArgs.fMapSpec.device = state->fDevice; in prepare()
917 mapArgs.fLength = state->fPreparedLength; in prepare()
918 OSSharedPtr<IOMemoryDescriptor> md = state->fCopyMD; in prepare()
923 mapArgs.fOffset = state->fPreparedOffset; in prepare()
953 length = state->fPreparedLength; in prepare()
957 walkArgs->fOffset = state->fPreparedOffset + index; in prepare()
978 state->fMapSegments = IONewZeroData(IODMACommandMapSegment, segCount); in prepare()
979 if (!state->fMapSegments) { in prepare()
983 state->fMapSegmentsCount = segCount; in prepare()
1000 walkArgs->fOffset = state->fPreparedOffset + index; in prepare()
1023 assert(segCount < state->fMapSegmentsCount); in prepare()
1024 state->fMapSegments[segCount].fDMAOffset = state->fPreparedOffset + index; in prepare()
1025 state->fMapSegments[segCount].fMapOffset = insertOffset; in prepare()
1026 state->fMapSegments[segCount].fPageOffset = align; in prepare()
1035 state->fLocalMapperAllocBase = (mapArgs.fAlloc & ~mapperPageMask); in prepare()
1038 IOLog("IODMACommand fMapSegmentsCount %d\n", state->fMapSegmentsCount); in prepare()
1044 state->fLocalMapperAlloc = mapArgs.fAlloc; in prepare()
1045 state->fLocalMapperAllocValid = true; in prepare()
1046 state->fLocalMapperAllocLength = mapArgs.fAllocLength; in prepare()
1050 state->fPrepared = true; in prepare()
1059 IODMACommandInternal * state = fInternalState; in complete() local
1068 copyMD = state->fCopyMD; in complete()
1072 copyMD->performOperation(kIOMemoryIncoherentIOFlush, 0, state->fPreparedLength); in complete()
1075 … md->performOperation(kIOMemoryIncoherentIOFlush, state->fPreparedOffset, state->fPreparedLength); in complete()
1079 if (!state->fCursor) { in complete()
1087 if (state->fLocalMapperAllocValid) { in complete()
1092 mapArgs.fAlloc = state->fLocalMapperAlloc; in complete()
1093 mapArgs.fAllocLength = state->fLocalMapperAllocLength; in complete()
1099 mapArgs.fOffset = state->fPreparedOffset; in complete()
1104 state->fLocalMapperAlloc = 0; in complete()
1105 state->fLocalMapperAllocValid = false; in complete()
1106 state->fLocalMapperAllocLength = 0; in complete()
1107 if (state->fMapSegments) { in complete()
1108 IODeleteData(state->fMapSegments, IODMACommandMapSegment, state->fMapSegmentsCount); in complete()
1109 state->fMapSegments = NULL; in complete()
1110 state->fMapSegmentsCount = 0; in complete()
1114 state->fPrepared = false; in complete()
1123 IODMACommandInternal * state = fInternalState; in getPreparedOffsetAndLength() local
1129 *offset = state->fPreparedOffset; in getPreparedOffsetAndLength()
1132 *length = state->fPreparedLength; in getPreparedOffsetAndLength()
1141 IODMACommandInternal * state = fInternalState; in synchronize() local
1155 if (state->fDoubleBuffer) { in synchronize()
1159 state->fCursor = false; in synchronize()
1160 state->fForceDoubleBuffer = true; in synchronize()
1161 …ret = prepare(state->fPreparedOffset, state->fPreparedLength, false /* flushCache */, true /* sync… in synchronize()
1164 } else if (state->fCursor) { in synchronize()
1236 IODMACommandInternal * state = fInternalState; in transfer() local
1245 if (offset >= state->fPreparedLength) { in transfer()
1248 length = min(length, state->fPreparedLength - offset); in transfer()
1300 IOMDDMAWalkSegmentArgs *state = in genIOVMSegments() local
1310 …if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNe… in genIOVMSegments()
1311 state->fOffset = 0; in genIOVMSegments()
1312 internalState->fIOVMAddrValid = state->fIOVMAddr = 0; in genIOVMSegments()
1318 state->fMapped = true; in genIOVMSegments()
1319 state->fMappedBase = internalState->fLocalMapperAlloc; in genIOVMSegments()
1321 state->fMapped = false; in genIOVMSegments()
1339 while (internalState->fIOVMAddrValid || (state->fOffset < memLength)) { in genIOVMSegments()
1344 state->fOffset = offset; in genIOVMSegments()
1345 state->fLength = memLength - offset; in genIOVMSegments()
1352 state->fIOVMAddr = internalState->fLocalMapperAlloc + offset - internalState->fPreparedOffset; in genIOVMSegments()
1404 state->fIOVMAddr = address; in genIOVMSegments()
1405 state->fLength = length; in genIOVMSegments()
1409 internalState->fNextOffset = state->fOffset + length; in genIOVMSegments()
1433 for (checkOffset = 0; checkOffset < state->fLength;) { in genIOVMSegments()
1437 mapperPhys = fMapper->mapToPhysicalAddress(state->fIOVMAddr + checkOffset); in genIOVMSegments()
1442 state->fIOVMAddr + checkOffset, mapperPhys, phys, state->fLength); in genIOVMSegments()
1450 assert(state->fLength); in genIOVMSegments()
1451 if (curSegValid && ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr)) { in genIOVMSegments()
1452 UInt64 length = state->fLength; in genIOVMSegments()
1455 internalState->fIOVMAddrValid = state->fIOVMAddr = 0; in genIOVMSegments()
1458 internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end in genIOVMSegments()
1466 UInt64 length = state->fLength; in genIOVMSegments()
1468 curSeg.fIOVMAddr = state->fIOVMAddr; in genIOVMSegments()
1471 internalState->fIOVMAddrValid = state->fIOVMAddr = 0; in genIOVMSegments()
1487 state->fIOVMAddr = newLength + curSeg.fIOVMAddr; in genIOVMSegments()
1490 state->fLength = remain; in genIOVMSegments()
1521 state->fIOVMAddr = addr + newLength; in genIOVMSegments()
1524 state->fLength = remain; in genIOVMSegments()
1538 state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; in genIOVMSegments()
1548 state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; in genIOVMSegments()
1554 reduce = (state->fIOVMAddr & fAlignMaskInternalSegments); in genIOVMSegments()
1558 state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; in genIOVMSegments()
1566 state->fLength = leftover; in genIOVMSegments()
1595 internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end in genIOVMSegments()
1627 state->fOffset = offset; in genIOVMSegments()