84   Value getNumElements(ConversionPatternRewriter &rewriter, Location loc,  in getNumElements()  argument
88                      rewriter, loc, type.getNumElements())  in getNumElements()
91                : rewriter.create<LLVM::MulOp>(loc,  in getNumElements()
92                                               desc.stride(rewriter, loc, 0),  in getNumElements()
93                                               desc.size(rewriter, loc, 0));  in getNumElements()
205                   ConversionPatternRewriter &rewriter) const override;
219                   ConversionPatternRewriter &rewriter) const override;
233                   ConversionPatternRewriter &rewriter) const override;
245                   ConversionPatternRewriter &rewriter) const override;
259                   ConversionPatternRewriter &rewriter) const override;
273                   ConversionPatternRewriter &rewriter) const override;
305                   ConversionPatternRewriter &rewriter) const override;
314                                 PatternRewriter &rewriter) const override {  in matchAndRewrite()
317     rewriter.eraseOp(op);  in matchAndRewrite()
333                   ConversionPatternRewriter &rewriter) const override;
347                   ConversionPatternRewriter &rewriter) const override;
362                   ConversionPatternRewriter &rewriter) const override;
401                                      ConversionPatternRewriter &rewriter) {  in areAllLLVMTypes()  argument
405     return rewriter.notifyMatchFailure(  in areAllLLVMTypes()
411 isAsyncWithOneDependency(ConversionPatternRewriter &rewriter,  in isAsyncWithOneDependency()  argument
414     return rewriter.notifyMatchFailure(  in isAsyncWithOneDependency()
418     return rewriter.notifyMatchFailure(op, "Can convert only async version.");  in isAsyncWithOneDependency()
425     ConversionPatternRewriter &rewriter) const {  in matchAndRewrite()
427   if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)))  in matchAndRewrite()
434   auto elementSize = getSizeInBytes(loc, elementType, rewriter);  in matchAndRewrite()
437       loc, op->getOperands(), adaptor.getOperands(), rewriter);  in matchAndRewrite()
439   hostRegisterCallBuilder.create(loc, rewriter, arguments);  in matchAndRewrite()
441   rewriter.eraseOp(op);  in matchAndRewrite()
447     ConversionPatternRewriter &rewriter) const {  in matchAndRewrite()
450   if (failed(areAllLLVMTypes(allocOp, adaptor.getOperands(), rewriter)) ||  in matchAndRewrite()
452       failed(isAsyncWithOneDependency(rewriter, allocOp)))  in matchAndRewrite()
462   getMemRefDescriptorSizes(loc, memRefType, adaptor.dynamicSizes(), rewriter,  in matchAndRewrite()
470       allocCallBuilder.create(loc, rewriter, {sizeBytes, stream}).getResult(0);  in matchAndRewrite()
472       rewriter.create<LLVM::BitcastOp>(loc, elementPtrType, allocatedPtr);  in matchAndRewrite()
479       loc, memRefType, allocatedPtr, alignedPtr, shape, strides, rewriter);  in matchAndRewrite()
481   rewriter.replaceOp(allocOp, {memRefDescriptor, stream});  in matchAndRewrite()
488     ConversionPatternRewriter &rewriter) const {  in matchAndRewrite()
489   if (failed(areAllLLVMTypes(deallocOp, adaptor.getOperands(), rewriter)) ||  in matchAndRewrite()
490       failed(isAsyncWithOneDependency(rewriter, deallocOp)))  in matchAndRewrite()
496       MemRefDescriptor(adaptor.memref()).allocatedPtr(rewriter, loc);  in matchAndRewrite()
497   auto casted = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, pointer);  in matchAndRewrite()
499   deallocCallBuilder.create(loc, rewriter, {casted, stream});  in matchAndRewrite()
501   rewriter.replaceOp(deallocOp, {stream});  in matchAndRewrite()
515     ConversionPatternRewriter &rewriter) const {  in matchAndRewrite()
517     return rewriter.notifyMatchFailure(yieldOp, "no gpu async token operand");  in matchAndRewrite()
527     auto event = eventCreateCallBuilder.create(loc, rewriter, {}).getResult(0);  in matchAndRewrite()
528     eventRecordCallBuilder.create(loc, rewriter, {event, stream});  in matchAndRewrite()
533     streamDestroyCallBuilder.create(loc, rewriter, {stream});  in matchAndRewrite()
535   rewriter.updateRootInPlace(yieldOp,  in matchAndRewrite()
554     ConversionPatternRewriter &rewriter) const {  in matchAndRewrite()
556     return rewriter.notifyMatchFailure(waitOp, "Cannot convert async op.");  in matchAndRewrite()
563       streamSynchronizeCallBuilder.create(loc, rewriter, {operand});  in matchAndRewrite()
564       streamDestroyCallBuilder.create(loc, rewriter, {operand});  in matchAndRewrite()
568       eventSynchronizeCallBuilder.create(loc, rewriter, {operand});  in matchAndRewrite()
569       eventDestroyCallBuilder.create(loc, rewriter, {operand});  in matchAndRewrite()
573   rewriter.eraseOp(waitOp);  in matchAndRewrite()
584     ConversionPatternRewriter &rewriter) const {  in matchAndRewrite()
586     return rewriter.notifyMatchFailure(waitOp, "Can only convert async op.");  in matchAndRewrite()
590   auto insertionPoint = rewriter.saveInsertionPoint();  in matchAndRewrite()
599       rewriter.setInsertionPointAfter(defOp);  in matchAndRewrite()
601           eventCreateCallBuilder.create(loc, rewriter, {}).getResult(0);  in matchAndRewrite()
602       eventRecordCallBuilder.create(loc, rewriter, {event, operand});  in matchAndRewrite()
610   rewriter.restoreInsertionPoint(insertionPoint);  in matchAndRewrite()
611   auto stream = streamCreateCallBuilder.create(loc, rewriter, {}).getResult(0);  in matchAndRewrite()
613     streamWaitEventCallBuilder.create(loc, rewriter, {stream, event});  in matchAndRewrite()
615     eventDestroyCallBuilder.create(loc, rewriter, {event});  in matchAndRewrite()
616   rewriter.replaceOp(waitOp, {stream});  in matchAndRewrite()
717     ConversionPatternRewriter &rewriter) const {  in matchAndRewrite()
718   if (failed(areAllLLVMTypes(launchOp, adaptor.getOperands(), rewriter)))  in matchAndRewrite()
722     return rewriter.notifyMatchFailure(  in matchAndRewrite()
729     return rewriter.notifyMatchFailure(  in matchAndRewrite()
751       LLVM::createGlobalString(loc, rewriter, nameBuffer.str(),  in matchAndRewrite()
754   auto module = moduleLoadCallBuilder.create(loc, rewriter, data);  in matchAndRewrite()
759       launchOp.getKernelName().getValue(), loc, rewriter);  in matchAndRewrite()
761       loc, rewriter, {module.getResult(0), kernelName});  in matchAndRewrite()
762   auto zero = rewriter.create<LLVM::ConstantOp>(loc, llvmInt32Type,  in matchAndRewrite()
763                                                 rewriter.getI32IntegerAttr(0));  in matchAndRewrite()
766           ? streamCreateCallBuilder.create(loc, rewriter, {}).getResult(0)  in matchAndRewrite()
769   auto kernelParams = generateParamsArray(launchOp, adaptor, rewriter);  in matchAndRewrite()
770   auto nullpointer = rewriter.create<LLVM::NullOp>(loc, llvmPointerPointerType);  in matchAndRewrite()
775       loc, rewriter,  in matchAndRewrite()
783     rewriter.replaceOp(launchOp, {stream});  in matchAndRewrite()
788     streamSynchronizeCallBuilder.create(loc, rewriter, stream);  in matchAndRewrite()
789     streamDestroyCallBuilder.create(loc, rewriter, stream);  in matchAndRewrite()
790     rewriter.eraseOp(launchOp);  in matchAndRewrite()
792   moduleUnloadCallBuilder.create(loc, rewriter, module.getResult(0));  in matchAndRewrite()
799     ConversionPatternRewriter &rewriter) const {  in matchAndRewrite()
802   if (failed(areAllLLVMTypes(memcpyOp, adaptor.getOperands(), rewriter)) ||  in matchAndRewrite()
804       failed(isAsyncWithOneDependency(rewriter, memcpyOp)))  in matchAndRewrite()
810   Value numElements = getNumElements(rewriter, loc, memRefType, srcDesc);  in matchAndRewrite()
813   Value nullPtr = rewriter.create<LLVM::NullOp>(loc, elementPtrType);  in matchAndRewrite()
814   Value gepPtr = rewriter.create<LLVM::GEPOp>(loc, elementPtrType, nullPtr,  in matchAndRewrite()
817       rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gepPtr);  in matchAndRewrite()
819   auto src = rewriter.create<LLVM::BitcastOp>(  in matchAndRewrite()
820       loc, llvmPointerType, srcDesc.alignedPtr(rewriter, loc));  in matchAndRewrite()
821   auto dst = rewriter.create<LLVM::BitcastOp>(  in matchAndRewrite()
823       MemRefDescriptor(adaptor.dst()).alignedPtr(rewriter, loc));  in matchAndRewrite()
826   memcpyCallBuilder.create(loc, rewriter, {dst, src, sizeBytes, stream});  in matchAndRewrite()
828   rewriter.replaceOp(memcpyOp, {stream});  in matchAndRewrite()
835     ConversionPatternRewriter &rewriter) const {  in matchAndRewrite()
838   if (failed(areAllLLVMTypes(memsetOp, adaptor.getOperands(), rewriter)) ||  in matchAndRewrite()
840       failed(isAsyncWithOneDependency(rewriter, memsetOp)))  in matchAndRewrite()
847     return rewriter.notifyMatchFailure(memsetOp,  in matchAndRewrite()
852   Value numElements = getNumElements(rewriter, loc, memRefType, dstDesc);  in matchAndRewrite()
855       rewriter.create<LLVM::BitcastOp>(loc, llvmInt32Type, adaptor.value());  in matchAndRewrite()
856   auto dst = rewriter.create<LLVM::BitcastOp>(  in matchAndRewrite()
857       loc, llvmPointerType, dstDesc.alignedPtr(rewriter, loc));  in matchAndRewrite()
860   memsetCallBuilder.create(loc, rewriter, {dst, value, numElements, stream});  in matchAndRewrite()
862   rewriter.replaceOp(memsetOp, {stream});  in matchAndRewrite()
868     ConversionPatternRewriter &rewriter) const {  in matchAndRewrite()
870   setDefaultDeviceCallBuilder.create(loc, rewriter, {adaptor.devIndex()});  in matchAndRewrite()
871   rewriter.replaceOp(op, {});  in matchAndRewrite()