440 static isl::schedule_node permuteBandNodeDimensions(isl::schedule_node Node,  in permuteBandNodeDimensions()  argument
443   assert(isl_schedule_node_get_type(Node.get()) == isl_schedule_node_band &&  in permuteBandNodeDimensions()
444          (unsigned)isl_schedule_node_band_n_member(Node.get()) >  in permuteBandNodeDimensions()
447       isl::manage(isl_schedule_node_band_get_partial_schedule(Node.get()));  in permuteBandNodeDimensions()
454   Node = isl::manage(isl_schedule_node_delete(Node.release()));  in permuteBandNodeDimensions()
455   return Node.insert_partial_schedule(PartialSchedule);  in permuteBandNodeDimensions()
459 createMicroKernel(isl::schedule_node Node,  in createMicroKernel()  argument
461   Node = applyRegisterTiling(Node, {MicroKernelParams.Mr, MicroKernelParams.Nr},  in createMicroKernel()
463   Node = Node.parent().parent();  in createMicroKernel()
464   return permuteBandNodeDimensions(Node, 0, 1).child(0).child(0);  in createMicroKernel()
478 createMacroKernel(isl::schedule_node Node,  in createMacroKernel()  argument
480   assert(isl_schedule_node_get_type(Node.get()) == isl_schedule_node_band);  in createMacroKernel()
483     return Node;  in createMacroKernel()
484   int DimOutNum = isl_schedule_node_band_n_member(Node.get());  in createMacroKernel()
489   Node = tileNode(Node, "1st level tiling", TileSizes, 1);  in createMacroKernel()
490   Node = Node.parent().parent();  in createMacroKernel()
491   Node = permuteBandNodeDimensions(Node, DimOutNum - 2, DimOutNum - 1);  in createMacroKernel()
492   Node = permuteBandNodeDimensions(Node, DimOutNum - 3, DimOutNum - 1);  in createMacroKernel()
494   return Node.child(0).child(0);  in createMacroKernel()
694 static isl::schedule_node createExtensionNode(isl::schedule_node Node,  in createExtensionNode()  argument
698   return Node.graft_before(NewNode);  in createExtensionNode()
701 static isl::schedule_node optimizePackedB(isl::schedule_node Node,  in optimizePackedB()  argument
735   return createExtensionNode(Node, ExtMap);  in optimizePackedB()
738 static isl::schedule_node optimizePackedA(isl::schedule_node Node, ScopStmt *,  in optimizePackedA()  argument
795   return createExtensionNode(Node, ExtScatterCopy);  in optimizePackedA()
831 optimizeDataLayoutMatrMulPattern(isl::schedule_node Node, isl::map MapOldIndVar,  in optimizeDataLayoutMatrMulPattern()  argument
838   Node = Node.parent().parent().parent().parent().parent().parent();  in optimizeDataLayoutMatrMulPattern()
839   Node = isl::manage(isl_schedule_node_band_split(Node.release(), 2));  in optimizeDataLayoutMatrMulPattern()
841   Node = Node.child(0);  in optimizeDataLayoutMatrMulPattern()
842   Node =  in optimizeDataLayoutMatrMulPattern()
843       optimizePackedB(Node, Stmt, MapOldIndVar, MicroParams, MacroParams, MMI);  in optimizeDataLayoutMatrMulPattern()
845   Node = Node.child(0);  in optimizeDataLayoutMatrMulPattern()
846   Node =  in optimizeDataLayoutMatrMulPattern()
847       optimizePackedA(Node, Stmt, MapOldIndVar, MicroParams, MacroParams, MMI);  in optimizeDataLayoutMatrMulPattern()
849   return Node.child(0).child(0).child(0).child(0).child(0);  in optimizeDataLayoutMatrMulPattern()
865 getInductionVariablesSubstitution(isl::schedule_node Node,  in getInductionVariablesSubstitution()  argument
868   auto Child = Node.child(0);  in getInductionVariablesSubstitution()
892 isolateAndUnrollMatMulInnerLoops(isl::schedule_node Node,  in isolateAndUnrollMatMulInnerLoops()  argument
894   isl::schedule_node Child = Node.child(0);  in isolateAndUnrollMatMulInnerLoops()
905   isl::ctx Ctx = Node.ctx();  in isolateAndUnrollMatMulInnerLoops()
908   Node = Node.as<isl::schedule_node_band>().set_ast_build_options(Options);  in isolateAndUnrollMatMulInnerLoops()
909   Node = Node.parent().parent().parent();  in isolateAndUnrollMatMulInnerLoops()
912   Node = Node.as<isl::schedule_node_band>().set_ast_build_options(Options);  in isolateAndUnrollMatMulInnerLoops()
913   Node = Node.child(0).child(0).child(0);  in isolateAndUnrollMatMulInnerLoops()
914   return Node;  in isolateAndUnrollMatMulInnerLoops()
921 static isl::schedule_node markLoopVectorizerDisabled(isl::schedule_node Node) {  in markLoopVectorizerDisabled()  argument
922   auto Id = isl::id::alloc(Node.ctx(), "Loop Vectorizer Disabled", nullptr);  in markLoopVectorizerDisabled()
923   return Node.insert_mark(Id).child(0);  in markLoopVectorizerDisabled()
935 getBandNodeWithOriginDimOrder(isl::schedule_node Node) {  in getBandNodeWithOriginDimOrder()  argument
936   assert(isl_schedule_node_get_type(Node.get()) == isl_schedule_node_band);  in getBandNodeWithOriginDimOrder()
937   if (isl_schedule_node_get_type(Node.child(0).get()) != isl_schedule_node_leaf)  in getBandNodeWithOriginDimOrder()
938     return Node;  in getBandNodeWithOriginDimOrder()
939   auto Domain = Node.get_universe_domain();  in getBandNodeWithOriginDimOrder()
941   if (Node.get_schedule_depth().release() != 0 ||  in getBandNodeWithOriginDimOrder()
943        unsignedFromIslSize(Node.as<isl::schedule_node_band>().n_member())))  in getBandNodeWithOriginDimOrder()
944     return Node;  in getBandNodeWithOriginDimOrder()
945   Node = isl::manage(isl_schedule_node_delete(Node.copy()));  in getBandNodeWithOriginDimOrder()
951   return Node.insert_partial_schedule(PartialScheduleMultiPwAff);  in getBandNodeWithOriginDimOrder()
954 static isl::schedule_node optimizeMatMulPattern(isl::schedule_node Node,  in optimizeMatMulPattern()  argument
958   int DimOutNum = isl_schedule_node_band_n_member(Node.get());  in optimizeMatMulPattern()
962   Node = getBandNodeWithOriginDimOrder(Node);  in optimizeMatMulPattern()
963   Node = permuteBandNodeDimensions(Node, MMI.i, DimOutNum - 3);  in optimizeMatMulPattern()
966   Node = permuteBandNodeDimensions(Node, NewJ, DimOutNum - 2);  in optimizeMatMulPattern()
968   Node = permuteBandNodeDimensions(Node, NewK, DimOutNum - 1);  in optimizeMatMulPattern()
971   Node = createMacroKernel(Node, MacroKernelParams);  in optimizeMatMulPattern()
972   Node = createMicroKernel(Node, MicroKernelParams);  in optimizeMatMulPattern()
975     return Node;  in optimizeMatMulPattern()
976   auto MapOldIndVar = getInductionVariablesSubstitution(Node, MicroKernelParams,  in optimizeMatMulPattern()
979     return Node;  in optimizeMatMulPattern()
980   Node = markLoopVectorizerDisabled(Node.parent()).child(0);  in optimizeMatMulPattern()
981   Node = isolateAndUnrollMatMulInnerLoops(Node, MicroKernelParams);  in optimizeMatMulPattern()
982   return optimizeDataLayoutMatrMulPattern(Node, MapOldIndVar, MicroKernelParams,  in optimizeMatMulPattern()
1006 static bool isMatrMultPattern(isl::schedule_node Node, const Dependences *D,  in isMatrMultPattern()  argument
1009       isl_schedule_node_band_get_partial_schedule_union_map(Node.get()));  in isMatrMultPattern()
1010   Node = Node.child(0);  in isMatrMultPattern()
1011   auto LeafType = isl_schedule_node_get_type(Node.get());  in isMatrMultPattern()
1012   Node = Node.parent();  in isMatrMultPattern()
1014       isl_schedule_node_band_n_member(Node.get()) < 3 ||  in isMatrMultPattern()
1015       Node.get_schedule_depth().release() != 0 ||  in isMatrMultPattern()
1027 polly::tryOptimizeMatMulPattern(isl::schedule_node Node,  in tryOptimizeMatMulPattern()  argument
1031   if (isMatrMultPattern(Node, D, MMI)) {  in tryOptimizeMatMulPattern()
1033     return optimizeMatMulPattern(Node, TTI, MMI);  in tryOptimizeMatMulPattern()