47   std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,  in allocateBuffer()
56       alignment = createIndexConstant(rewriter, loc, *alignmentAttr);  in allocateBuffer()
62       alignment = getSizeInBytes(loc, memRefType.getElementType(), rewriter);  in allocateBuffer()
67       sizeBytes = rewriter.create<LLVM::AddOp>(loc, sizeBytes, alignment);  in allocateBuffer()
74     auto results = createLLVMCall(rewriter, loc, allocFuncOp, {sizeBytes},  in allocateBuffer()
77         rewriter.create<LLVM::BitcastOp>(loc, elementPtrType, results[0]);  in allocateBuffer()
83           rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), allocatedPtr);  in allocateBuffer()
85           createAligned(rewriter, loc, allocatedInt, alignment);  in allocateBuffer()
87           rewriter.create<LLVM::IntToPtrOp>(loc, elementPtrType, alignmentInt);  in allocateBuffer()
155   std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,  in allocateBuffer()
162     Value allocAlignment = createIndexConstant(rewriter, loc, alignment);  in allocateBuffer()
167       sizeBytes = createAligned(rewriter, loc, sizeBytes, allocAlignment);  in allocateBuffer()
172         createLLVMCall(rewriter, loc, allocFuncOp, {allocAlignment, sizeBytes},  in allocateBuffer()
175         rewriter.create<LLVM::BitcastOp>(loc, elementPtrType, results[0]);  in allocateBuffer()
198   std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,  in allocateBuffer()
207     auto allocatedElementPtr = rewriter.create<LLVM::AllocaOp>(  in allocateBuffer()
220                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
221     OpBuilder::InsertionGuard guard(rewriter);  in matchAndRewrite()
226     auto *currentBlock = rewriter.getInsertionBlock();  in matchAndRewrite()
228         rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());  in matchAndRewrite()
233       continueBlock = rewriter.createBlock(  in matchAndRewrite()
237       rewriter.create<LLVM::BrOp>(loc, ValueRange(), remainingOpsBlock);  in matchAndRewrite()
243     rewriter.inlineRegionBefore(allocaScopeOp.getBodyRegion(), continueBlock);  in matchAndRewrite()
246     rewriter.setInsertionPointToEnd(currentBlock);  in matchAndRewrite()
248         rewriter.create<LLVM::StackSaveOp>(loc, getVoidPtrType());  in matchAndRewrite()
249     rewriter.create<LLVM::BrOp>(loc, ValueRange(), beforeBody);  in matchAndRewrite()
253     rewriter.setInsertionPointToEnd(afterBody);  in matchAndRewrite()
256     auto branchOp = rewriter.replaceOpWithNewOp<LLVM::BrOp>(  in matchAndRewrite()
260     rewriter.setInsertionPoint(branchOp);  in matchAndRewrite()
261     rewriter.create<LLVM::StackRestoreOp>(loc, stackSaveOp);  in matchAndRewrite()
264     rewriter.replaceOp(allocaScopeOp, continueBlock->getArguments());  in matchAndRewrite()
277                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
283     Value ptr = memRefDescriptor.alignedPtr(rewriter, memref.getLoc());  in matchAndRewrite()
295     Value zero = createIndexAttrConstant(rewriter, loc, intPtrType, 0);  in matchAndRewrite()
297         createIndexAttrConstant(rewriter, loc, intPtrType, alignment - 1);  in matchAndRewrite()
298     Value ptrValue = rewriter.create<LLVM::PtrToIntOp>(loc, intPtrType, ptr);  in matchAndRewrite()
299     rewriter.create<LLVM::AssumeOp>(  in matchAndRewrite()
300         loc, rewriter.create<LLVM::ICmpOp>(  in matchAndRewrite()
302                  rewriter.create<LLVM::AndOp>(loc, ptrValue, mask), zero));  in matchAndRewrite()
304     rewriter.eraseOp(op);  in matchAndRewrite()
329                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
333     Value casted = rewriter.create<LLVM::BitcastOp>(  in matchAndRewrite()
335         memref.allocatedPtr(rewriter, op.getLoc()));  in matchAndRewrite()
336     rewriter.replaceOpWithNewOp<LLVM::CallOp>(  in matchAndRewrite()
349                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
352       rewriter.replaceOp(  in matchAndRewrite()
354                      operandType, dimOp, adaptor.getOperands(), rewriter)});  in matchAndRewrite()
359       rewriter.replaceOp(  in matchAndRewrite()
361                                             adaptor.getOperands(), rewriter)});  in matchAndRewrite()
370                                     ConversionPatternRewriter &rewriter) const {  in extractSizeOfUnrankedMemRef()
382     Value underlyingRankedDesc = unrankedDesc.memRefDescPtr(rewriter, loc);  in extractSizeOfUnrankedMemRef()
383     Value scalarMemRefDescPtr = rewriter.create<LLVM::BitcastOp>(  in extractSizeOfUnrankedMemRef()
392     Value two = rewriter.create<LLVM::ConstantOp>(  in extractSizeOfUnrankedMemRef()
393         loc, typeConverter->convertType(rewriter.getI32Type()),  in extractSizeOfUnrankedMemRef()
394         rewriter.getI32IntegerAttr(2));  in extractSizeOfUnrankedMemRef()
395     Value offsetPtr = rewriter.create<LLVM::GEPOp>(  in extractSizeOfUnrankedMemRef()
397         ValueRange({createIndexConstant(rewriter, loc, 0), two}));  in extractSizeOfUnrankedMemRef()
401     Value idxPlusOne = rewriter.create<LLVM::AddOp>(  in extractSizeOfUnrankedMemRef()
402         loc, createIndexConstant(rewriter, loc, 1), adaptor.getIndex());  in extractSizeOfUnrankedMemRef()
403     Value sizePtr = rewriter.create<LLVM::GEPOp>(loc, indexPtrTy, offsetPtr,  in extractSizeOfUnrankedMemRef()
405     return rewriter.create<LLVM::LoadOp>(loc, sizePtr);  in extractSizeOfUnrankedMemRef()
423                                   ConversionPatternRewriter &rewriter) const {  in extractSizeOfRankedMemRef()
433         return descriptor.size(rewriter, loc, i);  in extractSizeOfRankedMemRef()
437       return createIndexConstant(rewriter, loc, dimSize);  in extractSizeOfRankedMemRef()
442     return memrefDescriptor.size(rewriter, loc, index, rank);  in extractSizeOfRankedMemRef()
494                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
499     auto *initBlock = rewriter.getInsertionBlock();  in matchAndRewrite()
500     auto *loopBlock = rewriter.createBlock(  in matchAndRewrite()
503     auto *endBlock = rewriter.createBlock(  in matchAndRewrite()
511     rewriter.setInsertionPointToEnd(initBlock);  in matchAndRewrite()
514                                         adaptor.getIndices(), rewriter);  in matchAndRewrite()
515     Value init = rewriter.create<LLVM::LoadOp>(loc, dataPtr);  in matchAndRewrite()
516     rewriter.create<LLVM::BrOp>(loc, init, loopBlock);  in matchAndRewrite()
519     rewriter.setInsertionPointToStart(loopBlock);  in matchAndRewrite()
527       Operation *clone = rewriter.clone(nestedOp, mapping);  in matchAndRewrite()
536     auto boolType = IntegerType::get(rewriter.getContext(), 1);  in matchAndRewrite()
537     auto pairType = LLVM::LLVMStructType::getLiteral(rewriter.getContext(),  in matchAndRewrite()
539     auto cmpxchg = rewriter.create<LLVM::AtomicCmpXchgOp>(  in matchAndRewrite()
543     Value newLoaded = rewriter.create<LLVM::ExtractValueOp>(  in matchAndRewrite()
544         loc, valueType, cmpxchg, rewriter.getI64ArrayAttr({0}));  in matchAndRewrite()
545     Value ok = rewriter.create<LLVM::ExtractValueOp>(  in matchAndRewrite()
546         loc, boolType, cmpxchg, rewriter.getI64ArrayAttr({1}));  in matchAndRewrite()
549     rewriter.create<LLVM::CondBrOp>(loc, ok, endBlock, ArrayRef<Value>(),  in matchAndRewrite()
552     rewriter.setInsertionPointToEnd(endBlock);  in matchAndRewrite()
554                  std::next(opsToMoveEnd), rewriter);  in matchAndRewrite()
557     rewriter.replaceOp(atomicOp, {newLoaded});  in matchAndRewrite()
566                     ConversionPatternRewriter &rewriter) const {  in moveOpsRange()
571       rewriter.clone(*it, mapping);  in moveOpsRange()
575       rewriter.eraseOp(it);  in moveOpsRange()
602                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
625     auto newGlobal = rewriter.replaceOpWithNewOp<LLVM::GlobalOp>(  in matchAndRewrite()
631       rewriter.setInsertionPointToStart(blk);  in matchAndRewrite()
633           rewriter.create<LLVM::UndefOp>(global.getLoc(), arrayTy)};  in matchAndRewrite()
634       rewriter.create<LLVM::ReturnOp>(global.getLoc(), undef);  in matchAndRewrite()
650   std::tuple<Value, Value> allocateBuffer(ConversionPatternRewriter &rewriter,  in allocateBuffer()
658     auto addressOf = rewriter.create<LLVM::AddressOfOp>(  in allocateBuffer()
669                     createIndexConstant(rewriter, loc, 0));  in allocateBuffer()
671         rewriter.create<LLVM::GEPOp>(loc, elementPtrType, addressOf, operands);  in allocateBuffer()
678         createIndexAttrConstant(rewriter, op->getLoc(), intPtrType, 0xdeadbeef);  in allocateBuffer()
680         rewriter.create<LLVM::IntToPtrOp>(loc, elementPtrType, deadBeefConst);  in allocateBuffer()
695                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
700                              adaptor.getIndices(), rewriter);  in matchAndRewrite()
701     rewriter.replaceOpWithNewOp<LLVM::LoadOp>(loadOp, dataPtr);  in matchAndRewrite()
713                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
717                                          adaptor.getIndices(), rewriter);  in matchAndRewrite()
718     rewriter.replaceOpWithNewOp<LLVM::StoreOp>(op, adaptor.getValue(), dataPtr);  in matchAndRewrite()
730                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
735                                          adaptor.getIndices(), rewriter);  in matchAndRewrite()
738     auto llvmI32Type = typeConverter->convertType(rewriter.getIntegerType(32));  in matchAndRewrite()
739     auto isWrite = rewriter.create<LLVM::ConstantOp>(  in matchAndRewrite()
740         loc, llvmI32Type, rewriter.getI32IntegerAttr(prefetchOp.getIsWrite()));  in matchAndRewrite()
741     auto localityHint = rewriter.create<LLVM::ConstantOp>(  in matchAndRewrite()
743         rewriter.getI32IntegerAttr(prefetchOp.getLocalityHint()));  in matchAndRewrite()
744     auto isData = rewriter.create<LLVM::ConstantOp>(  in matchAndRewrite()
746         rewriter.getI32IntegerAttr(prefetchOp.getIsDataCache()));  in matchAndRewrite()
748     rewriter.replaceOpWithNewOp<LLVM::Prefetch>(prefetchOp, dataPtr, isWrite,  in matchAndRewrite()
759                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
764       rewriter.replaceOp(op, {desc.rank(rewriter, loc)});  in matchAndRewrite()
768       rewriter.replaceOp(  in matchAndRewrite()
769           op, {createIndexConstant(rewriter, loc, rankedMemRefType.getRank())});  in matchAndRewrite()
804                ConversionPatternRewriter &rewriter) const override {  in rewrite()
812       return rewriter.replaceOp(memRefCastOp, {adaptor.getSource()});  in rewrite()
823           loc, adaptor.getSource(), rewriter);  in rewrite()
826           rewriter.create<LLVM::BitcastOp>(loc, getVoidPtrType(), ptr)  in rewrite()
829       auto rankVal = rewriter.create<LLVM::ConstantOp>(  in rewrite()
830           loc, getIndexType(), rewriter.getIndexAttr(rank));  in rewrite()
833           UnrankedMemRefDescriptor::undef(rewriter, loc, targetStructType);  in rewrite()
835       memRefDesc.setRank(rewriter, loc, rankVal);  in rewrite()
837       memRefDesc.setMemRefDescPtr(rewriter, loc, voidPtr);  in rewrite()
838       rewriter.replaceOp(memRefCastOp, (Value)memRefDesc);  in rewrite()
846       auto ptr = memRefDesc.memRefDescPtr(rewriter, loc);  in rewrite()
849           rewriter  in rewrite()
854       auto loadOp = rewriter.create<LLVM::LoadOp>(loc, castPtr);  in rewrite()
855       rewriter.replaceOp(memRefCastOp, loadOp.getResult());  in rewrite()
872                           ConversionPatternRewriter &rewriter) const {  in lowerToMemCopyIntrinsic()
879     Value numElements = rewriter.create<LLVM::ConstantOp>(  in lowerToMemCopyIntrinsic()
880         loc, getIndexType(), rewriter.getIndexAttr(1));  in lowerToMemCopyIntrinsic()
882       auto size = srcDesc.size(rewriter, loc, pos);  in lowerToMemCopyIntrinsic()
883       numElements = rewriter.create<LLVM::MulOp>(loc, numElements, size);  in lowerToMemCopyIntrinsic()
887     auto sizeInBytes = getSizeInBytes(loc, srcType.getElementType(), rewriter);  in lowerToMemCopyIntrinsic()
890         rewriter.create<LLVM::MulOp>(loc, numElements, sizeInBytes);  in lowerToMemCopyIntrinsic()
892     Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc);  in lowerToMemCopyIntrinsic()
893     Value srcOffset = srcDesc.offset(rewriter, loc);  in lowerToMemCopyIntrinsic()
894     Value srcPtr = rewriter.create<LLVM::GEPOp>(loc, srcBasePtr.getType(),  in lowerToMemCopyIntrinsic()
897     Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc);  in lowerToMemCopyIntrinsic()
898     Value targetOffset = targetDesc.offset(rewriter, loc);  in lowerToMemCopyIntrinsic()
899     Value targetPtr = rewriter.create<LLVM::GEPOp>(loc, targetBasePtr.getType(),  in lowerToMemCopyIntrinsic()
901     Value isVolatile = rewriter.create<LLVM::ConstantOp>(  in lowerToMemCopyIntrinsic()
902         loc, typeConverter->convertType(rewriter.getI1Type()),  in lowerToMemCopyIntrinsic()
903         rewriter.getBoolAttr(false));  in lowerToMemCopyIntrinsic()
904     rewriter.create<LLVM::MemcpyOp>(loc, targetPtr, srcPtr, totalSize,  in lowerToMemCopyIntrinsic()
906     rewriter.eraseOp(op);  in lowerToMemCopyIntrinsic()
913                              ConversionPatternRewriter &rewriter) const {  in lowerToMemCopyFunctionCall()
920       auto rank = rewriter.create<LLVM::ConstantOp>(  in lowerToMemCopyFunctionCall()
921           loc, getIndexType(), rewriter.getIndexAttr(type.getRank()));  in lowerToMemCopyFunctionCall()
924           typeConverter->promoteOneMemRefDescriptor(loc, ranked, rewriter);  in lowerToMemCopyFunctionCall()
926           rewriter.create<LLVM::BitcastOp>(loc, getVoidPtrType(), ptr)  in lowerToMemCopyFunctionCall()
930       return UnrankedMemRefDescriptor::pack(rewriter, loc, *typeConverter,  in lowerToMemCopyFunctionCall()
943     auto one = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),  in lowerToMemCopyFunctionCall()
944                                                  rewriter.getIndexAttr(1));  in lowerToMemCopyFunctionCall()
948           rewriter.create<LLVM::AllocaOp>(loc, ptrType, ValueRange{one});  in lowerToMemCopyFunctionCall()
949       rewriter.create<LLVM::StoreOp>(loc, desc, allocated);  in lowerToMemCopyFunctionCall()
958     auto elemSize = rewriter.create<LLVM::ConstantOp>(  in lowerToMemCopyFunctionCall()
959         loc, getIndexType(), rewriter.getIndexAttr(typeSize));  in lowerToMemCopyFunctionCall()
962     rewriter.create<LLVM::CallOp>(loc, copyFn,  in lowerToMemCopyFunctionCall()
964     rewriter.eraseOp(op);  in lowerToMemCopyFunctionCall()
971                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
987       return lowerToMemCopyIntrinsic(op, adaptor, rewriter);  in matchAndRewrite()
989     return lowerToMemCopyFunctionCall(op, adaptor, rewriter);  in matchAndRewrite()
997                                      ConversionPatternRewriter &rewriter,  in extractPointersAndOffset()  argument
1006     *allocatedPtr = desc.allocatedPtr(rewriter, loc);  in extractPointersAndOffset()
1007     *alignedPtr = desc.alignedPtr(rewriter, loc);  in extractPointersAndOffset()
1009       *offset = desc.offset(rewriter, loc);  in extractPointersAndOffset()
1023   Value underlyingDescPtr = unrankedDesc.memRefDescPtr(rewriter, loc);  in extractPointersAndOffset()
1026       rewriter, loc, underlyingDescPtr, elementPtrPtrType);  in extractPointersAndOffset()
1028       rewriter, loc, typeConverter, underlyingDescPtr, elementPtrPtrType);  in extractPointersAndOffset()
1031         rewriter, loc, typeConverter, underlyingDescPtr, elementPtrPtrType);  in extractPointersAndOffset()
1042                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
1046     if (failed(convertSourceMemRefToDescriptor(rewriter, srcType, castOp,  in matchAndRewrite()
1049     rewriter.replaceOp(castOp, {descriptor});  in matchAndRewrite()
1055       ConversionPatternRewriter &rewriter, Type srcType,  in convertSourceMemRefToDescriptor()
1067     auto desc = MemRefDescriptor::undef(rewriter, loc, llvmTargetDescriptorTy);  in convertSourceMemRefToDescriptor()
1071     extractPointersAndOffset(loc, rewriter, *getTypeConverter(),  in convertSourceMemRefToDescriptor()
1074     desc.setAllocatedPtr(rewriter, loc, allocatedPtr);  in convertSourceMemRefToDescriptor()
1075     desc.setAlignedPtr(rewriter, loc, alignedPtr);  in convertSourceMemRefToDescriptor()
1079       desc.setOffset(rewriter, loc, adaptor.getOffsets()[0]);  in convertSourceMemRefToDescriptor()
1081       desc.setConstantOffset(rewriter, loc, castOp.getStaticOffset(0));  in convertSourceMemRefToDescriptor()
1088         desc.setSize(rewriter, loc, i, adaptor.getSizes()[dynSizeId++]);  in convertSourceMemRefToDescriptor()
1090         desc.setConstantSize(rewriter, loc, i, castOp.getStaticSize(i));  in convertSourceMemRefToDescriptor()
1093         desc.setStride(rewriter, loc, i, adaptor.getStrides()[dynStrideId++]);  in convertSourceMemRefToDescriptor()
1095         desc.setConstantStride(rewriter, loc, i, castOp.getStaticStride(i));  in convertSourceMemRefToDescriptor()
1108                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
1112     if (failed(convertSourceMemRefToDescriptor(rewriter, srcType, reshapeOp,  in matchAndRewrite()
1115     rewriter.replaceOp(reshapeOp, {descriptor});  in matchAndRewrite()
1121   convertSourceMemRefToDescriptor(ConversionPatternRewriter &rewriter,  in convertSourceMemRefToDescriptor()
1138           MemRefDescriptor::undef(rewriter, loc, llvmTargetDescriptorTy);  in convertSourceMemRefToDescriptor()
1142       extractPointersAndOffset(loc, rewriter, *getTypeConverter(),  in convertSourceMemRefToDescriptor()
1145       desc.setAllocatedPtr(rewriter, loc, allocatedPtr);  in convertSourceMemRefToDescriptor()
1146       desc.setAlignedPtr(rewriter, loc, alignedPtr);  in convertSourceMemRefToDescriptor()
1152         return rewriter.notifyMatchFailure(  in convertSourceMemRefToDescriptor()
1156         return rewriter.notifyMatchFailure(reshapeOp,  in convertSourceMemRefToDescriptor()
1159       desc.setConstantOffset(rewriter, loc, offset);  in convertSourceMemRefToDescriptor()
1170           stride = createIndexConstant(rewriter, loc, strides[i]);  in convertSourceMemRefToDescriptor()
1175           stride = createIndexConstant(rewriter, loc, 1);  in convertSourceMemRefToDescriptor()
1183           dimSize = createIndexConstant(rewriter, loc, size);  in convertSourceMemRefToDescriptor()
1186           Value index = createIndexConstant(rewriter, loc, i);  in convertSourceMemRefToDescriptor()
1187           dimSize = rewriter.create<memref::LoadOp>(loc, shapeOp, index);  in convertSourceMemRefToDescriptor()
1191                 rewriter, loc, indexType, dimSize);  in convertSourceMemRefToDescriptor()
1195         desc.setSize(rewriter, loc, i, dimSize);  in convertSourceMemRefToDescriptor()
1196         desc.setStride(rewriter, loc, i, stride);  in convertSourceMemRefToDescriptor()
1199         stride = rewriter.create<LLVM::MulOp>(loc, stride, dimSize);  in convertSourceMemRefToDescriptor()
1209     Value resultRank = shapeDesc.size(rewriter, loc, 0);  in convertSourceMemRefToDescriptor()
1220         rewriter, loc, typeConverter->convertType(targetType));  in convertSourceMemRefToDescriptor()
1221     targetDesc.setRank(rewriter, loc, resultRank);  in convertSourceMemRefToDescriptor()
1223     UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),  in convertSourceMemRefToDescriptor()
1225     Value underlyingDescPtr = rewriter.create<LLVM::AllocaOp>(  in convertSourceMemRefToDescriptor()
1227     targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr);  in convertSourceMemRefToDescriptor()
1231     extractPointersAndOffset(loc, rewriter, *getTypeConverter(),  in convertSourceMemRefToDescriptor()
1239     UnrankedMemRefDescriptor::setAllocatedPtr(rewriter, loc, underlyingDescPtr,  in convertSourceMemRefToDescriptor()
1241     UnrankedMemRefDescriptor::setAlignedPtr(rewriter, loc, *getTypeConverter(),  in convertSourceMemRefToDescriptor()
1244     UnrankedMemRefDescriptor::setOffset(rewriter, loc, *getTypeConverter(),  in convertSourceMemRefToDescriptor()
1251         rewriter, loc, *getTypeConverter(), underlyingDescPtr,  in convertSourceMemRefToDescriptor()
1254         rewriter, loc, *getTypeConverter(), targetSizesBase, resultRank);  in convertSourceMemRefToDescriptor()
1255     Value shapeOperandPtr = shapeDesc.alignedPtr(rewriter, loc);  in convertSourceMemRefToDescriptor()
1256     Value oneIndex = createIndexConstant(rewriter, loc, 1);  in convertSourceMemRefToDescriptor()
1258         rewriter.create<LLVM::SubOp>(loc, resultRank, oneIndex);  in convertSourceMemRefToDescriptor()
1260     Block *initBlock = rewriter.getInsertionBlock();  in convertSourceMemRefToDescriptor()
1262     Block::iterator remainingOpsIt = std::next(rewriter.getInsertionPoint());  in convertSourceMemRefToDescriptor()
1264     Block *condBlock = rewriter.createBlock(initBlock->getParent(), {},  in convertSourceMemRefToDescriptor()
1268     Block *remainingBlock = rewriter.splitBlock(initBlock, remainingOpsIt);  in convertSourceMemRefToDescriptor()
1269     rewriter.mergeBlocks(remainingBlock, condBlock, ValueRange());  in convertSourceMemRefToDescriptor()
1271     rewriter.setInsertionPointToEnd(initBlock);  in convertSourceMemRefToDescriptor()
1272     rewriter.create<LLVM::BrOp>(loc, ValueRange({resultRankMinusOne, oneIndex}),  in convertSourceMemRefToDescriptor()
1274     rewriter.setInsertionPointToStart(condBlock);  in convertSourceMemRefToDescriptor()
1278     Value zeroIndex = createIndexConstant(rewriter, loc, 0);  in convertSourceMemRefToDescriptor()
1279     Value pred = rewriter.create<LLVM::ICmpOp>(  in convertSourceMemRefToDescriptor()
1280         loc, IntegerType::get(rewriter.getContext(), 1),  in convertSourceMemRefToDescriptor()
1284         rewriter.splitBlock(condBlock, rewriter.getInsertionPoint());  in convertSourceMemRefToDescriptor()
1285     rewriter.setInsertionPointToStart(bodyBlock);  in convertSourceMemRefToDescriptor()
1289     Value sizeLoadGep = rewriter.create<LLVM::GEPOp>(  in convertSourceMemRefToDescriptor()
1291     Value size = rewriter.create<LLVM::LoadOp>(loc, sizeLoadGep);  in convertSourceMemRefToDescriptor()
1292     UnrankedMemRefDescriptor::setSize(rewriter, loc, *getTypeConverter(),  in convertSourceMemRefToDescriptor()
1296     UnrankedMemRefDescriptor::setStride(rewriter, loc, *getTypeConverter(),  in convertSourceMemRefToDescriptor()
1298     Value nextStride = rewriter.create<LLVM::MulOp>(loc, strideArg, size);  in convertSourceMemRefToDescriptor()
1301     Value decrement = rewriter.create<LLVM::SubOp>(loc, indexArg, oneIndex);  in convertSourceMemRefToDescriptor()
1302     rewriter.create<LLVM::BrOp>(loc, ValueRange({decrement, nextStride}),  in convertSourceMemRefToDescriptor()
1306         rewriter.splitBlock(bodyBlock, rewriter.getInsertionPoint());  in convertSourceMemRefToDescriptor()
1309     rewriter.setInsertionPointToEnd(condBlock);  in convertSourceMemRefToDescriptor()
1310     rewriter.create<LLVM::CondBrOp>(loc, pred, bodyBlock, llvm::None, remainder,  in convertSourceMemRefToDescriptor()
1314     rewriter.setInsertionPointToStart(remainder);  in convertSourceMemRefToDescriptor()
1471     ConversionPatternRewriter &rewriter, Location loc, Operation *op,  in fillInStridesForCollapsedMemDescriptor()  argument
1478     rewriter.setInsertionPoint(op);  in fillInStridesForCollapsedMemDescriptor()
1484       dstDesc.setStride(rewriter, loc, dstIndex,  in fillInStridesForCollapsedMemDescriptor()
1485                         srcDesc.stride(rewriter, loc, ref.back()));  in fillInStridesForCollapsedMemDescriptor()
1516       OpBuilder::InsertionGuard guard(rewriter);  in fillInStridesForCollapsedMemDescriptor()
1517       Block *initBlock = rewriter.getInsertionBlock();  in fillInStridesForCollapsedMemDescriptor()
1519           rewriter.splitBlock(initBlock, rewriter.getInsertionPoint());  in fillInStridesForCollapsedMemDescriptor()
1521       rewriter.setInsertionPointToStart(continueBlock);  in fillInStridesForCollapsedMemDescriptor()
1522       dstDesc.setStride(rewriter, loc, dstIndex, continueBlock->getArgument(0));  in fillInStridesForCollapsedMemDescriptor()
1529         rewriter.setInsertionPointToEnd(curEntryBlock);  in fillInStridesForCollapsedMemDescriptor()
1530         Value srcStride = srcDesc.stride(rewriter, loc, srcIndex);  in fillInStridesForCollapsedMemDescriptor()
1532           rewriter.create<LLVM::BrOp>(loc, srcStride, continueBlock);  in fillInStridesForCollapsedMemDescriptor()
1535         Value one = rewriter.create<LLVM::ConstantOp>(  in fillInStridesForCollapsedMemDescriptor()
1536             loc, typeConverter->convertType(rewriter.getI64Type()),  in fillInStridesForCollapsedMemDescriptor()
1537             rewriter.getI32IntegerAttr(1));  in fillInStridesForCollapsedMemDescriptor()
1538         Value predNeOne = rewriter.create<LLVM::ICmpOp>(  in fillInStridesForCollapsedMemDescriptor()
1539             loc, LLVM::ICmpPredicate::ne, srcDesc.size(rewriter, loc, srcIndex),  in fillInStridesForCollapsedMemDescriptor()
1542           OpBuilder::InsertionGuard guard(rewriter);  in fillInStridesForCollapsedMemDescriptor()
1543           nextEntryBlock = rewriter.createBlock(  in fillInStridesForCollapsedMemDescriptor()
1546         rewriter.create<LLVM::CondBrOp>(loc, predNeOne, continueBlock,  in fillInStridesForCollapsedMemDescriptor()
1579                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
1586       return rewriter.notifyMatchFailure(  in matchAndRewrite()
1593         rewriter, loc, this->typeConverter->convertType(dstType));  in matchAndRewrite()
1594     dstDesc.setAllocatedPtr(rewriter, loc, srcDesc.allocatedPtr(rewriter, loc));  in matchAndRewrite()
1595     dstDesc.setAlignedPtr(rewriter, loc, srcDesc.alignedPtr(rewriter, loc));  in matchAndRewrite()
1596     dstDesc.setOffset(rewriter, loc, srcDesc.offset(rewriter, loc));  in matchAndRewrite()
1601         this->typeConverter->convertType(rewriter.getIndexType());  in matchAndRewrite()
1603         rewriter, loc, llvmIndexType, reshapeOp.getReassociationIndices(),  in matchAndRewrite()
1606       dstDesc.setSize(rewriter, loc, en.index(), en.value());  in matchAndRewrite()
1610         dstDesc.setConstantStride(rewriter, loc, en.index(), en.value());  in matchAndRewrite()
1613       Value c1 = rewriter.create<LLVM::ConstantOp>(loc, llvmIndexType,  in matchAndRewrite()
1614                                                    rewriter.getIndexAttr(1));  in matchAndRewrite()
1618         dstDesc.setStride(rewriter, loc, dimIndex, stride);  in matchAndRewrite()
1619         stride = rewriter.create<LLVM::MulOp>(loc, dstShape[dimIndex], stride);  in matchAndRewrite()
1625           rewriter, loc, reshapeOp, this->typeConverter, srcType, dstType,  in matchAndRewrite()
1628     rewriter.replaceOp(reshapeOp, {dstDesc});  in matchAndRewrite()
1643                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
1678     auto targetMemRef = MemRefDescriptor::undef(rewriter, loc, targetDescTy);  in matchAndRewrite()
1681     Value extracted = sourceMemRef.allocatedPtr(rewriter, loc);  in matchAndRewrite()
1682     Value bitcastPtr = rewriter.create<LLVM::BitcastOp>(  in matchAndRewrite()
1687     targetMemRef.setAllocatedPtr(rewriter, loc, bitcastPtr);  in matchAndRewrite()
1690     extracted = sourceMemRef.alignedPtr(rewriter, loc);  in matchAndRewrite()
1691     bitcastPtr = rewriter.create<LLVM::BitcastOp>(  in matchAndRewrite()
1696     targetMemRef.setAlignedPtr(rewriter, loc, bitcastPtr);  in matchAndRewrite()
1705       strideValues.push_back(sourceMemRef.stride(rewriter, loc, i));  in matchAndRewrite()
1708     auto llvmIndexType = typeConverter->convertType(rewriter.getIndexType());  in matchAndRewrite()
1710       targetMemRef.setConstantOffset(rewriter, loc, offset);  in matchAndRewrite()
1712       Value baseOffset = sourceMemRef.offset(rewriter, loc);  in matchAndRewrite()
1723                 : rewriter.create<LLVM::ConstantOp>(  in matchAndRewrite()
1725                       rewriter.getI64IntegerAttr(subViewOp.getStaticOffset(i)));  in matchAndRewrite()
1726         Value mul = rewriter.create<LLVM::MulOp>(loc, offset, strideValues[i]);  in matchAndRewrite()
1727         baseOffset = rewriter.create<LLVM::AddOp>(loc, baseOffset, mul);  in matchAndRewrite()
1729       targetMemRef.setOffset(rewriter, loc, baseOffset);  in matchAndRewrite()
1755           size = rewriter.create<LLVM::ConstantOp>(  in matchAndRewrite()
1756               loc, llvmIndexType, rewriter.getI64IntegerAttr(staticSize));  in matchAndRewrite()
1758           Value pos = rewriter.create<LLVM::ConstantOp>(  in matchAndRewrite()
1759               loc, llvmIndexType, rewriter.getI64IntegerAttr(i));  in matchAndRewrite()
1761               rewriter.create<memref::DimOp>(loc, subViewOp.getSource(), pos);  in matchAndRewrite()
1762           auto cast = rewriter.create<UnrealizedConversionCastOp>(  in matchAndRewrite()
1766         stride = rewriter.create<LLVM::ConstantOp>(  in matchAndRewrite()
1767             loc, llvmIndexType, rewriter.getI64IntegerAttr(1));  in matchAndRewrite()
1773                 : rewriter.create<LLVM::ConstantOp>(  in matchAndRewrite()
1775                       rewriter.getI64IntegerAttr(subViewOp.getStaticSize(i)));  in matchAndRewrite()
1777           stride = rewriter.create<LLVM::ConstantOp>(  in matchAndRewrite()
1778               loc, llvmIndexType, rewriter.getI64IntegerAttr(strides[i]));  in matchAndRewrite()
1783                   : rewriter.create<LLVM::ConstantOp>(  in matchAndRewrite()
1785                         rewriter.getI64IntegerAttr(  in matchAndRewrite()
1787           stride = rewriter.create<LLVM::MulOp>(loc, stride, strideValues[i]);  in matchAndRewrite()
1790       targetMemRef.setSize(rewriter, loc, j, size);  in matchAndRewrite()
1791       targetMemRef.setStride(rewriter, loc, j, stride);  in matchAndRewrite()
1795     rewriter.replaceOp(subViewOp, {targetMemRef});  in matchAndRewrite()
1813                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
1819       return rewriter.replaceOp(transposeOp, {viewMemRef}), success();  in matchAndRewrite()
1822         rewriter, loc, typeConverter->convertType(transposeOp.getShapedType()));  in matchAndRewrite()
1826     targetMemRef.setAllocatedPtr(rewriter, loc,  in matchAndRewrite()
1827                                  viewMemRef.allocatedPtr(rewriter, loc));  in matchAndRewrite()
1828     targetMemRef.setAlignedPtr(rewriter, loc,  in matchAndRewrite()
1829                                viewMemRef.alignedPtr(rewriter, loc));  in matchAndRewrite()
1832     targetMemRef.setOffset(rewriter, loc, viewMemRef.offset(rewriter, loc));  in matchAndRewrite()
1839       targetMemRef.setSize(rewriter, loc, targetPos,  in matchAndRewrite()
1840                            viewMemRef.size(rewriter, loc, sourcePos));  in matchAndRewrite()
1841       targetMemRef.setStride(rewriter, loc, targetPos,  in matchAndRewrite()
1842                              viewMemRef.stride(rewriter, loc, sourcePos));  in matchAndRewrite()
1845     rewriter.replaceOp(transposeOp, {targetMemRef});  in matchAndRewrite()
1860   Value getSize(ConversionPatternRewriter &rewriter, Location loc,  in getSize()
1865       return createIndexConstant(rewriter, loc, shape[idx]);  in getSize()
1876   Value getStride(ConversionPatternRewriter &rewriter, Location loc,  in getStride()
1881       return createIndexConstant(rewriter, loc, strides[idx]);  in getStride()
1884                  ? rewriter.create<LLVM::MulOp>(loc, runningStride, nextSize)  in getStride()
1887     return createIndexConstant(rewriter, loc, 1);  in getStride()
1892                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
1920     auto targetMemRef = MemRefDescriptor::undef(rewriter, loc, targetDescTy);  in matchAndRewrite()
1923     Value allocatedPtr = sourceMemRef.allocatedPtr(rewriter, loc);  in matchAndRewrite()
1925     Value bitcastPtr = rewriter.create<LLVM::BitcastOp>(  in matchAndRewrite()
1930     targetMemRef.setAllocatedPtr(rewriter, loc, bitcastPtr);  in matchAndRewrite()
1933     Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc);  in matchAndRewrite()
1934     alignedPtr = rewriter.create<LLVM::GEPOp>(  in matchAndRewrite()
1936     bitcastPtr = rewriter.create<LLVM::BitcastOp>(  in matchAndRewrite()
1941     targetMemRef.setAlignedPtr(rewriter, loc, bitcastPtr);  in matchAndRewrite()
1946     targetMemRef.setOffset(rewriter, loc,  in matchAndRewrite()
1947                            createIndexConstant(rewriter, loc, offset));  in matchAndRewrite()
1951       return rewriter.replaceOp(viewOp, {targetMemRef}), success();  in matchAndRewrite()
1957       Value size = getSize(rewriter, loc, viewMemRefType.getShape(),  in matchAndRewrite()
1959       targetMemRef.setSize(rewriter, loc, i, size);  in matchAndRewrite()
1961       stride = getStride(rewriter, loc, strides, nextSize, stride, i);  in matchAndRewrite()
1962       targetMemRef.setStride(rewriter, loc, i, stride);  in matchAndRewrite()
1966     rewriter.replaceOp(viewOp, {targetMemRef});  in matchAndRewrite()
2009                   ConversionPatternRewriter &rewriter) const override {  in matchAndRewrite()
2019                              adaptor.getIndices(), rewriter);  in matchAndRewrite()
2020     rewriter.replaceOpWithNewOp<LLVM::AtomicRMWOp>(  in matchAndRewrite()