Lines Matching refs:state
905 VectorizationState &state, in computeMemoryOpIndices() argument
910 auto afOp = state.builder.create<AffineApplyOp>(op->getLoc(), singleResMap, in computeMemoryOpIndices()
949 VectorizationState &state) { in vectorizeConstant() argument
954 auto vecTy = getVectorType(scalarTy, state.strategy); in vectorizeConstant()
957 OpBuilder::InsertionGuard guard(state.builder); in vectorizeConstant()
958 Operation *parentOp = state.builder.getInsertionBlock()->getParentOp(); in vectorizeConstant()
960 while (parentOp && !state.vecLoopToVecDim.count(parentOp)) in vectorizeConstant()
962 assert(parentOp && state.vecLoopToVecDim.count(parentOp) && in vectorizeConstant()
965 state.builder.setInsertionPointToStart(vecForOp.getBody()); in vectorizeConstant()
967 state.builder.create<arith::ConstantOp>(constOp.getLoc(), vecAttr); in vectorizeConstant()
970 state.registerOpVectorReplacement(constOp, newConstOp); in vectorizeConstant()
979 VectorizationState &state) { in createInitialVector() argument
985 reductionKind, scalarTy, state.builder, oldOperand.getLoc()); in createInitialVector()
986 auto vecTy = getVectorType(scalarTy, state.strategy); in createInitialVector()
989 state.builder.create<arith::ConstantOp>(oldOperand.getLoc(), vecAttr); in createInitialVector()
1001 static Value createMask(AffineForOp vecForOp, VectorizationState &state) { in createMask() argument
1002 assert(state.strategy->vectorSizes.size() == 1 && in createMask()
1004 assert(vecForOp.getStep() == state.strategy->vectorSizes[0] && in createMask()
1009 if (Value mask = state.vecLoopToMask.lookup(vecForOp)) in createMask()
1021 OpBuilder::InsertionGuard guard(state.builder); in createMask()
1022 state.builder.setInsertionPointToStart(vecForOp.getBody()); in createMask()
1041 ub = state.builder.create<AffineApplyOp>(loc, vecForOp.getUpperBoundMap(), in createMask()
1044 ub = state.builder.create<AffineMinOp>(loc, vecForOp.getUpperBoundMap(), in createMask()
1048 state.builder.getAffineDimExpr(0) - state.builder.getAffineDimExpr(1); in createMask()
1050 makeComposedAffineApply(state.builder, loc, AffineMap::get(2, 0, subExpr), in createMask()
1056 Type maskTy = VectorType::get(state.strategy->vectorSizes, in createMask()
1057 state.builder.getIntegerType(1)); in createMask()
1059 state.builder.create<vector::CreateMaskOp>(loc, maskTy, itersLeft); in createMask()
1065 state.vecLoopToMask[vecForOp] = mask; in createMask()
1091 VectorizationState &state) { in vectorizeUniform() argument
1092 OpBuilder::InsertionGuard guard(state.builder); in vectorizeUniform()
1094 state.valueScalarReplacement.lookupOrDefault(uniformVal); in vectorizeUniform()
1095 state.builder.setInsertionPointAfterValue(uniformScalarRepl); in vectorizeUniform()
1097 auto vectorTy = getVectorType(uniformVal.getType(), state.strategy); in vectorizeUniform()
1098 auto bcastOp = state.builder.create<BroadcastOp>(uniformVal.getLoc(), in vectorizeUniform()
1100 state.registerValueVectorReplacement(uniformVal, bcastOp); in vectorizeUniform()
1121 static Value vectorizeOperand(Value operand, VectorizationState &state) { in vectorizeOperand() argument
1124 if (Value vecRepl = state.valueVectorReplacement.lookupOrNull(operand)) { in vectorizeOperand()
1137 auto vecConstant = vectorizeConstant(constOp, state); in vectorizeOperand()
1143 if (isUniformDefinition(operand, state.strategy)) { in vectorizeOperand()
1144 Operation *vecUniform = vectorizeUniform(operand, state); in vectorizeOperand()
1167 VectorizationState &state) { in vectorizeAffineLoad() argument
1170 auto vectorType = VectorType::get(state.strategy->vectorSizes, elementType); in vectorizeAffineLoad()
1174 state.getScalarValueReplacementsFor(loadOp.getMapOperands(), mapOperands); in vectorizeAffineLoad()
1180 state.builder.getMultiDimIdentityMap(memRefType.getRank())) in vectorizeAffineLoad()
1181 computeMemoryOpIndices(loadOp, loadOp.getAffineMap(), mapOperands, state, in vectorizeAffineLoad()
1187 auto permutationMap = makePermutationMap(state.builder.getInsertionBlock(), in vectorizeAffineLoad()
1188 indices, state.vecLoopToVecDim); in vectorizeAffineLoad()
1196 auto transfer = state.builder.create<vector::TransferReadOp>( in vectorizeAffineLoad()
1200 state.registerOpVectorReplacement(loadOp, transfer); in vectorizeAffineLoad()
1211 VectorizationState &state) { in vectorizeAffineStore() argument
1213 Value vectorValue = vectorizeOperand(storeOp.getValueToStore(), state); in vectorizeAffineStore()
1219 state.getScalarValueReplacementsFor(storeOp.getMapOperands(), mapOperands); in vectorizeAffineStore()
1225 state.builder.getMultiDimIdentityMap(memRefType.getRank())) in vectorizeAffineStore()
1226 computeMemoryOpIndices(storeOp, storeOp.getAffineMap(), mapOperands, state, in vectorizeAffineStore()
1232 auto permutationMap = makePermutationMap(state.builder.getInsertionBlock(), in vectorizeAffineStore()
1233 indices, state.vecLoopToVecDim); in vectorizeAffineStore()
1239 auto transfer = state.builder.create<vector::TransferWriteOp>( in vectorizeAffineStore()
1245 state.registerOpVectorReplacement(storeOp, transfer); in vectorizeAffineStore()
1252 Value value, VectorizationState &state) { in isNeutralElementConst() argument
1257 state.builder, value.getLoc()); in isNeutralElementConst()
1270 VectorizationState &state) { in vectorizeAffineForOp() argument
1271 const VectorizationStrategy &strategy = *state.strategy; in vectorizeAffineForOp()
1312 vecIterOperands.push_back(vectorizeOperand(operand, state)); in vectorizeAffineForOp()
1319 std::get<0>(redAndOperand).kind, std::get<1>(redAndOperand), state)); in vectorizeAffineForOp()
1323 auto vecForOp = state.builder.create<AffineForOp>( in vectorizeAffineForOp()
1345 state.registerOpVectorReplacement(forOp, vecForOp); in vectorizeAffineForOp()
1346 state.registerValueScalarReplacement(forOp.getInductionVar(), in vectorizeAffineForOp()
1350 state.registerBlockArgVectorReplacement(std::get<0>(iterTuple), in vectorizeAffineForOp()
1357 getVectorReductionOp(reductions[i].kind, state.builder, in vectorizeAffineForOp()
1365 if (!isNeutralElementConst(reductions[i].kind, origInit, state)) in vectorizeAffineForOp()
1367 arith::getReductionOp(reductions[i].kind, state.builder, in vectorizeAffineForOp()
1369 state.registerLoopResultScalarReplacement(forOp.getResult(i), finalRes); in vectorizeAffineForOp()
1374 state.vecLoopToVecDim[vecForOp] = loopToVecDimIt->second; in vectorizeAffineForOp()
1378 state.builder.setInsertionPointToStart(vecForOp.getBody()); in vectorizeAffineForOp()
1383 createMask(vecForOp, state); in vectorizeAffineForOp()
1391 static Operation *widenOp(Operation *op, VectorizationState &state) { in widenOp() argument
1395 VectorType::get(state.strategy->vectorSizes, result.getType())); in widenOp()
1399 Value vecOperand = vectorizeOperand(operand, state); in widenOp()
1413 state.builder.create(op->getLoc(), op->getName().getIdentifier(), in widenOp()
1415 state.registerOpVectorReplacement(op, vecOp); in widenOp()
1424 VectorizationState &state) { in vectorizeAffineYieldOp() argument
1425 Operation *newYieldOp = widenOp(yieldOp, state); in vectorizeAffineYieldOp()
1426 Operation *newParentOp = state.builder.getInsertionBlock()->getParentOp(); in vectorizeAffineYieldOp()
1437 if (Value mask = state.vecLoopToMask.lookup(newParentOp)) { in vectorizeAffineYieldOp()
1438 state.builder.setInsertionPoint(newYieldOp); in vectorizeAffineYieldOp()
1447 state.builder.setInsertionPoint(combinerOps.back()); in vectorizeAffineYieldOp()
1448 Value maskedReducedVal = state.builder.create<arith::SelectOp>( in vectorizeAffineYieldOp()
1458 state.builder.setInsertionPointAfter(newParentOp); in vectorizeAffineYieldOp()
1472 VectorizationState &state) { in vectorizeOneOperation() argument
1480 return vectorizeAffineLoad(loadOp, state); in vectorizeOneOperation()
1482 return vectorizeAffineStore(storeOp, state); in vectorizeOneOperation()
1484 return vectorizeAffineForOp(forOp, state); in vectorizeOneOperation()
1486 return vectorizeAffineYieldOp(yieldOp, state); in vectorizeOneOperation()
1488 return vectorizeConstant(constant, state); in vectorizeOneOperation()
1494 return widenOp(op, state); in vectorizeOneOperation()
1534 VectorizationState state(rootLoop.getContext()); in vectorizeLoopNest() local
1535 state.builder.setInsertionPointAfter(rootLoop); in vectorizeLoopNest()
1536 state.strategy = &strategy; in vectorizeLoopNest()
1560 Operation *vectorOp = vectorizeOneOperation(op, state); in vectorizeLoopNest()
1575 auto vecRootLoopIt = state.opVectorReplacement.find(rootLoop); in vectorizeLoopNest()
1576 if (vecRootLoopIt != state.opVectorReplacement.end()) in vectorizeLoopNest()
1584 for (auto resPair : state.loopResultScalarReplacement) in vectorizeLoopNest()
1587 assert(state.opVectorReplacement.count(rootLoop) == 1 && in vectorizeLoopNest()
1591 << *state.opVectorReplacement[rootLoop]); in vectorizeLoopNest()
1594 state.finishVectorizationPattern(rootLoop); in vectorizeLoopNest()