Lines Matching refs:isl
167 static isl::union_set getUnrollIsolatedSetOptions(isl::ctx Ctx) { in getUnrollIsolatedSetOptions()
168 isl::space Space = isl::space(Ctx, 0, 0, 1); in getUnrollIsolatedSetOptions()
169 isl::map UnrollIsolatedSetOption = isl::map::universe(Space); in getUnrollIsolatedSetOptions()
170 isl::id DimInId = isl::id::alloc(Ctx, "isolate", nullptr); in getUnrollIsolatedSetOptions()
171 isl::id DimOutId = isl::id::alloc(Ctx, "unroll", nullptr); in getUnrollIsolatedSetOptions()
173 UnrollIsolatedSetOption.set_tuple_id(isl::dim::in, DimInId); in getUnrollIsolatedSetOptions()
175 UnrollIsolatedSetOption.set_tuple_id(isl::dim::out, DimOutId); in getUnrollIsolatedSetOptions()
189 static isl::map permuteDimensions(isl::map Map, isl::dim DimType, in permuteDimensions()
195 isl::id DimId; in permuteDimensions()
198 auto FreeDim = DimType == isl::dim::in ? isl::dim::out : isl::dim::in; in permuteDimensions()
199 isl::id FreeDimId; in permuteDimensions()
227 static bool isMatMulOperandAcc(isl::set Domain, isl::map AccMap, int &FirstPos, in isMatMulOperandAcc()
229 isl::space Space = AccMap.get_space(); in isMatMulOperandAcc()
230 isl::map Universe = isl::map::universe(Space); in isMatMulOperandAcc()
232 if (unsignedFromIslSize(Space.dim(isl::dim::out)) != 2) in isMatMulOperandAcc()
246 Universe.equate(isl::dim::in, FirstDims[i], isl::dim::out, 0) in isMatMulOperandAcc()
247 .equate(isl::dim::in, SecondDims[i], isl::dim::out, 1); in isMatMulOperandAcc()
287 isl::set StmtDomain = MemAccess->getStatement()->getDomain(); in isMatMulNonScalarReadAccess()
316 static bool containsOnlyMatrMultAcc(isl::map PartialSchedule, in containsOnlyMatrMultAcc()
318 auto InputDimId = PartialSchedule.get_tuple_id(isl::dim::in); in containsOnlyMatrMultAcc()
325 permuteDimensions(PartialSchedule, isl::dim::out, MMI.i, OutDimNum - 1); in containsOnlyMatrMultAcc()
327 permuteDimensions(PartialSchedule, isl::dim::out, MMI.j, OutDimNum - 1); in containsOnlyMatrMultAcc()
329 permuteDimensions(PartialSchedule, isl::dim::out, MMI.k, OutDimNum - 1); in containsOnlyMatrMultAcc()
357 static bool containsOnlyMatMulDep(isl::map Schedule, const Dependences *D, in containsOnlyMatMulDep()
359 isl::union_map Dep = D->getDependences(Dependences::TYPE_RAW); in containsOnlyMatMulDep()
360 isl::union_map Red = D->getDependences(Dependences::TYPE_RED); in containsOnlyMatMulDep()
366 int DeltasDimNum = unsignedFromIslSize(Deltas.dim(isl::dim::set)); in containsOnlyMatMulDep()
368 auto Val = Deltas.plain_get_val_if_fixed(isl::dim::set, i); in containsOnlyMatMulDep()
401 static bool containsMatrMult(isl::map PartialSchedule, const Dependences *D, in containsMatrMult()
403 auto InputDimsId = PartialSchedule.get_tuple_id(isl::dim::in); in containsMatrMult()
440 static isl::schedule_node permuteBandNodeDimensions(isl::schedule_node Node, in permuteBandNodeDimensions()
447 isl::manage(isl_schedule_node_band_get_partial_schedule(Node.get())); in permuteBandNodeDimensions()
454 Node = isl::manage(isl_schedule_node_delete(Node.release())); in permuteBandNodeDimensions()
458 static isl::schedule_node
459 createMicroKernel(isl::schedule_node Node, in createMicroKernel()
477 static isl::schedule_node
478 createMacroKernel(isl::schedule_node Node, in createMacroKernel()
684 static isl::map getMatMulAccRel(isl::map MapOldIndVar, unsigned FirstDim, in getMatMulAccRel()
686 auto AccessRelSpace = isl::space(MapOldIndVar.ctx(), 0, 9, 3); in getMatMulAccRel()
687 auto AccessRel = isl::map::universe(AccessRelSpace); in getMatMulAccRel()
688 AccessRel = AccessRel.equate(isl::dim::in, FirstDim, isl::dim::out, 0); in getMatMulAccRel()
689 AccessRel = AccessRel.equate(isl::dim::in, 5, isl::dim::out, 1); in getMatMulAccRel()
690 AccessRel = AccessRel.equate(isl::dim::in, SecondDim, isl::dim::out, 2); in getMatMulAccRel()
694 static isl::schedule_node createExtensionNode(isl::schedule_node Node, in createExtensionNode()
695 isl::map ExtensionMap) { in createExtensionNode()
696 auto Extension = isl::union_map(ExtensionMap); in createExtensionNode()
697 auto NewNode = isl::schedule_node::from_extension(Extension); in createExtensionNode()
701 static isl::schedule_node optimizePackedB(isl::schedule_node Node, in optimizePackedB()
702 ScopStmt *Stmt, isl::map MapOldIndVar, in optimizePackedB()
707 isl::set Domain = Stmt->getDomain(); in optimizePackedB()
718 isl::map AccRelB = MMI.B->getLatestAccessRelation(); in optimizePackedB()
719 isl::map AccRelPackedB = getMatMulAccRel(MapOldIndVar, 3, 7); in optimizePackedB()
721 AccRelPackedB.set_tuple_id(isl::dim::out, PackedB->getBasePtrId()); in optimizePackedB()
730 isl::map ExtMap = MapOldIndVar.project_out(isl::dim::out, 2, Dim - 2); in optimizePackedB()
732 ExtMap = ExtMap.fix_si(isl::dim::out, MMI.i, 0); in optimizePackedB()
734 ExtMap = ExtMap.set_tuple_id(isl::dim::out, CopyStmt->getDomainId()); in optimizePackedB()
738 static isl::schedule_node optimizePackedA(isl::schedule_node Node, ScopStmt *, in optimizePackedA()
739 isl::map MapOldIndVar, in optimizePackedA()
743 isl::id InputDimsId = MapOldIndVar.get_tuple_id(isl::dim::in); in optimizePackedA()
745 isl::set Domain = Stmt->getDomain(); in optimizePackedA()
746 isl::id DomainId = Domain.get_tuple_id(); in optimizePackedA()
757 isl::map AccRelA = MMI.A->getLatestAccessRelation(); in optimizePackedA()
758 isl::map AccRelPackedA = getMatMulAccRel(MapOldIndVar, 4, 6); in optimizePackedA()
760 AccRelPackedA.set_tuple_id(isl::dim::out, PackedA->getBasePtrId()); in optimizePackedA()
762 isl::map PackedATranslator = AccRelPackedA.apply_domain(AccRelA); in optimizePackedA()
769 isl::set ScatterDomain = MapOldIndVar.intersect_domain(Domain).range(); in optimizePackedA()
771 isl::map OuterDomainMap = in optimizePackedA()
772 makeIdentityMap(ScatterDomain, true).project_out(isl::dim::out, 3, 6); in optimizePackedA()
774 isl::map CopyFrom = MapOldIndVar.reverse().apply_range(AccRelA); in optimizePackedA()
776 isl::map DomainTranslator = OuterDomainMap.range_product(CopyFrom); in optimizePackedA()
778 isl::set CopyDomain = DomainTranslator.range(); in optimizePackedA()
784 isl::map CopyTo = CopyFrom.apply_range(PackedATranslator); in optimizePackedA()
793 isl::map ExtScatterCopy = makeIdentityMap(CopyStmt->getDomain(), true); in optimizePackedA()
794 ExtScatterCopy = ExtScatterCopy.project_out(isl::dim::in, 3, 2); in optimizePackedA()
830 static isl::schedule_node
831 optimizeDataLayoutMatrMulPattern(isl::schedule_node Node, isl::map MapOldIndVar, in optimizeDataLayoutMatrMulPattern()
835 isl::id InputDimsId = MapOldIndVar.get_tuple_id(isl::dim::in); in optimizeDataLayoutMatrMulPattern()
839 Node = isl::manage(isl_schedule_node_band_split(Node.release(), 2)); in optimizeDataLayoutMatrMulPattern()
864 static isl::map
865 getInductionVariablesSubstitution(isl::schedule_node Node, in getInductionVariablesSubstitution()
870 auto MapOldIndVar = isl::map::from_union_map(UnMapOldIndVar); in getInductionVariablesSubstitution()
873 return MapOldIndVar.project_out(isl::dim::out, 0, Dim - 9); in getInductionVariablesSubstitution()
891 static isl::schedule_node
892 isolateAndUnrollMatMulInnerLoops(isl::schedule_node Node, in isolateAndUnrollMatMulInnerLoops()
894 isl::schedule_node Child = Node.child(0); in isolateAndUnrollMatMulInnerLoops()
895 isl::union_map UnMapOldIndVar = Child.get_prefix_schedule_relation(); in isolateAndUnrollMatMulInnerLoops()
896 isl::set Prefix = isl::map::from_union_map(UnMapOldIndVar).range(); in isolateAndUnrollMatMulInnerLoops()
899 Prefix = Prefix.project_out(isl::dim::set, Dims - 1, 1); in isolateAndUnrollMatMulInnerLoops()
903 isl::union_set IsolateOption = in isolateAndUnrollMatMulInnerLoops()
904 getIsolateOptions(Prefix.add_dims(isl::dim::set, 3), 3); in isolateAndUnrollMatMulInnerLoops()
905 isl::ctx Ctx = Node.ctx(); in isolateAndUnrollMatMulInnerLoops()
908 Node = Node.as<isl::schedule_node_band>().set_ast_build_options(Options); in isolateAndUnrollMatMulInnerLoops()
912 Node = Node.as<isl::schedule_node_band>().set_ast_build_options(Options); in isolateAndUnrollMatMulInnerLoops()
921 static isl::schedule_node markLoopVectorizerDisabled(isl::schedule_node Node) { in markLoopVectorizerDisabled()
922 auto Id = isl::id::alloc(Node.ctx(), "Loop Vectorizer Disabled", nullptr); in markLoopVectorizerDisabled()
934 static isl::schedule_node
935 getBandNodeWithOriginDimOrder(isl::schedule_node Node) { in getBandNodeWithOriginDimOrder()
942 (unsignedFromIslSize(isl::set(Domain).tuple_dim()) != in getBandNodeWithOriginDimOrder()
943 unsignedFromIslSize(Node.as<isl::schedule_node_band>().n_member()))) in getBandNodeWithOriginDimOrder()
945 Node = isl::manage(isl_schedule_node_delete(Node.copy())); in getBandNodeWithOriginDimOrder()
948 isl::multi_union_pw_aff(PartialSchedulePwAff); in getBandNodeWithOriginDimOrder()
950 PartialScheduleMultiPwAff.reset_tuple_id(isl::dim::set); in getBandNodeWithOriginDimOrder()
954 static isl::schedule_node optimizeMatMulPattern(isl::schedule_node Node, in optimizeMatMulPattern()
1006 static bool isMatrMultPattern(isl::schedule_node Node, const Dependences *D, in isMatrMultPattern()
1008 auto PartialSchedule = isl::manage( in isMatrMultPattern()
1018 auto NewPartialSchedule = isl::map::from_union_map(PartialSchedule); in isMatrMultPattern()
1026 isl::schedule_node
1027 polly::tryOptimizeMatMulPattern(isl::schedule_node Node, in tryOptimizeMatMulPattern()