1 //===-- RISCVTargetTransformInfo.cpp - RISC-V specific TTI ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "RISCVTargetTransformInfo.h"
10 #include "MCTargetDesc/RISCVMatInt.h"
11 #include "llvm/Analysis/TargetTransformInfo.h"
12 #include "llvm/CodeGen/BasicTTIImpl.h"
13 #include "llvm/CodeGen/TargetLowering.h"
14 using namespace llvm;
15 
16 #define DEBUG_TYPE "riscvtti"
17 
18 static cl::opt<unsigned> RVVRegisterWidthLMUL(
19     "riscv-v-register-bit-width-lmul",
20     cl::desc(
21         "The LMUL to use for getRegisterBitWidth queries. Affects LMUL used "
22         "by autovectorized code. Fractional LMULs are not supported."),
23     cl::init(1), cl::Hidden);
24 
25 InstructionCost RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
26                                             TTI::TargetCostKind CostKind) {
27   assert(Ty->isIntegerTy() &&
28          "getIntImmCost can only estimate cost of materialising integers");
29 
30   // We have a Zero register, so 0 is always free.
31   if (Imm == 0)
32     return TTI::TCC_Free;
33 
34   // Otherwise, we check how many instructions it will take to materialise.
35   const DataLayout &DL = getDataLayout();
36   return RISCVMatInt::getIntMatCost(Imm, DL.getTypeSizeInBits(Ty),
37                                     getST()->getFeatureBits());
38 }
39 
40 InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
41                                                 const APInt &Imm, Type *Ty,
42                                                 TTI::TargetCostKind CostKind,
43                                                 Instruction *Inst) {
44   assert(Ty->isIntegerTy() &&
45          "getIntImmCost can only estimate cost of materialising integers");
46 
47   // We have a Zero register, so 0 is always free.
48   if (Imm == 0)
49     return TTI::TCC_Free;
50 
51   // Some instructions in RISC-V can take a 12-bit immediate. Some of these are
52   // commutative, in others the immediate comes from a specific argument index.
53   bool Takes12BitImm = false;
54   unsigned ImmArgIdx = ~0U;
55 
56   switch (Opcode) {
57   case Instruction::GetElementPtr:
58     // Never hoist any arguments to a GetElementPtr. CodeGenPrepare will
59     // split up large offsets in GEP into better parts than ConstantHoisting
60     // can.
61     return TTI::TCC_Free;
62   case Instruction::And:
63     // zext.h
64     if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb())
65       return TTI::TCC_Free;
66     // zext.w
67     if (Imm == UINT64_C(0xffffffff) && ST->hasStdExtZbb())
68       return TTI::TCC_Free;
69     LLVM_FALLTHROUGH;
70   case Instruction::Add:
71   case Instruction::Or:
72   case Instruction::Xor:
73   case Instruction::Mul:
74     Takes12BitImm = true;
75     break;
76   case Instruction::Sub:
77   case Instruction::Shl:
78   case Instruction::LShr:
79   case Instruction::AShr:
80     Takes12BitImm = true;
81     ImmArgIdx = 1;
82     break;
83   default:
84     break;
85   }
86 
87   if (Takes12BitImm) {
88     // Check immediate is the correct argument...
89     if (Instruction::isCommutative(Opcode) || Idx == ImmArgIdx) {
90       // ... and fits into the 12-bit immediate.
91       if (Imm.getMinSignedBits() <= 64 &&
92           getTLI()->isLegalAddImmediate(Imm.getSExtValue())) {
93         return TTI::TCC_Free;
94       }
95     }
96 
97     // Otherwise, use the full materialisation cost.
98     return getIntImmCost(Imm, Ty, CostKind);
99   }
100 
101   // By default, prevent hoisting.
102   return TTI::TCC_Free;
103 }
104 
105 InstructionCost
106 RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
107                                   const APInt &Imm, Type *Ty,
108                                   TTI::TargetCostKind CostKind) {
109   // Prevent hoisting in unknown cases.
110   return TTI::TCC_Free;
111 }
112 
113 TargetTransformInfo::PopcntSupportKind
114 RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) {
115   assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
116   return ST->hasStdExtZbb() ? TTI::PSK_FastHardware : TTI::PSK_Software;
117 }
118 
119 bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
120   // Currently, the ExpandReductions pass can't expand scalable-vector
121   // reductions, but we still request expansion as RVV doesn't support certain
122   // reductions and the SelectionDAG can't legalize them either.
123   switch (II->getIntrinsicID()) {
124   default:
125     return false;
126   // These reductions have no equivalent in RVV
127   case Intrinsic::vector_reduce_mul:
128   case Intrinsic::vector_reduce_fmul:
129     return true;
130   }
131 }
132 
133 Optional<unsigned> RISCVTTIImpl::getMaxVScale() const {
134   // There is no assumption of the maximum vector length in V specification.
135   // We use the value specified by users as the maximum vector length.
136   // This function will use the assumed maximum vector length to get the
137   // maximum vscale for LoopVectorizer.
138   // If users do not specify the maximum vector length, we have no way to
139   // know whether the LoopVectorizer is safe to do or not.
140   // We only consider to use single vector register (LMUL = 1) to vectorize.
141   unsigned MaxVectorSizeInBits = ST->getMaxRVVVectorSizeInBits();
142   if (ST->hasVInstructions() && MaxVectorSizeInBits != 0)
143     return MaxVectorSizeInBits / RISCV::RVVBitsPerBlock;
144   return BaseT::getMaxVScale();
145 }
146 
147 TypeSize
148 RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
149   unsigned LMUL = PowerOf2Floor(
150       std::max<unsigned>(std::min<unsigned>(RVVRegisterWidthLMUL, 8), 1));
151   switch (K) {
152   case TargetTransformInfo::RGK_Scalar:
153     return TypeSize::getFixed(ST->getXLen());
154   case TargetTransformInfo::RGK_FixedWidthVector:
155     return TypeSize::getFixed(
156         ST->hasVInstructions() ? LMUL * ST->getMinRVVVectorSizeInBits() : 0);
157   case TargetTransformInfo::RGK_ScalableVector:
158     return TypeSize::getScalable(
159         ST->hasVInstructions() ? LMUL * RISCV::RVVBitsPerBlock : 0);
160   }
161 
162   llvm_unreachable("Unsupported register kind");
163 }
164 
165 InstructionCost RISCVTTIImpl::getSpliceCost(VectorType *Tp, int Index) {
166   std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
167 
168   unsigned Cost = 2; // vslidedown+vslideup.
169   // TODO: LMUL should increase cost.
170   // TODO: Multiplying by LT.first implies this legalizes into multiple copies
171   // of similar code, but I think we expand through memory.
172   return Cost * LT.first;
173 }
174 
175 InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
176                                              VectorType *Tp, ArrayRef<int> Mask,
177                                              int Index, VectorType *SubTp,
178                                              ArrayRef<Value *> Args) {
179   if (Kind == TTI::SK_Splice && isa<ScalableVectorType>(Tp))
180     return getSpliceCost(Tp, Index);
181   return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp);
182 }
183 
184 InstructionCost
185 RISCVTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
186                                     unsigned AddressSpace,
187                                     TTI::TargetCostKind CostKind) {
188   if (!isa<ScalableVectorType>(Src))
189     return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
190                                         CostKind);
191 
192   return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
193 }
194 
195 InstructionCost RISCVTTIImpl::getGatherScatterOpCost(
196     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
197     Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
198   if (CostKind != TTI::TCK_RecipThroughput)
199     return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
200                                          Alignment, CostKind, I);
201 
202   if ((Opcode == Instruction::Load &&
203        !isLegalMaskedGather(DataTy, Align(Alignment))) ||
204       (Opcode == Instruction::Store &&
205        !isLegalMaskedScatter(DataTy, Align(Alignment))))
206     return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
207                                          Alignment, CostKind, I);
208 
209   // FIXME: Only supporting fixed vectors for now.
210   if (!isa<FixedVectorType>(DataTy))
211     return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
212                                          Alignment, CostKind, I);
213 
214   auto *VTy = cast<FixedVectorType>(DataTy);
215   unsigned NumLoads = VTy->getNumElements();
216   InstructionCost MemOpCost =
217       getMemoryOpCost(Opcode, VTy->getElementType(), Alignment, 0, CostKind, I);
218   return NumLoads * MemOpCost;
219 }
220 
221 InstructionCost
222 RISCVTTIImpl::getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy,
223                                      bool IsUnsigned,
224                                      TTI::TargetCostKind CostKind) {
225   // FIXME: Only supporting fixed vectors for now.
226   if (!isa<FixedVectorType>(Ty))
227     return BaseT::getMinMaxReductionCost(Ty, CondTy, IsUnsigned, CostKind);
228 
229   if (!ST->useRVVForFixedLengthVectors())
230     return BaseT::getMinMaxReductionCost(Ty, CondTy, IsUnsigned, CostKind);
231 
232   // Skip if scalar size of Ty is bigger than ELEN.
233   if (Ty->getScalarSizeInBits() > ST->getMaxELENForFixedLengthVectors())
234     return BaseT::getMinMaxReductionCost(Ty, CondTy, IsUnsigned, CostKind);
235 
236   // IR Reduction is composed by two vmv and one rvv reduction instruction.
237   InstructionCost BaseCost = 2;
238   unsigned VL = cast<FixedVectorType>(Ty)->getNumElements();
239   std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
240   return (LT.first - 1) + BaseCost + Log2_32_Ceil(VL);
241 }
242 
243 InstructionCost
244 RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *VTy,
245                                          Optional<FastMathFlags> FMF,
246                                          TTI::TargetCostKind CostKind) {
247   // FIXME: Only supporting fixed vectors for now.
248   if (!isa<FixedVectorType>(VTy))
249     return BaseT::getArithmeticReductionCost(Opcode, VTy, FMF, CostKind);
250 
251   // FIXME: Do not support i1 and/or reduction now.
252   if (VTy->getElementType()->isIntegerTy(1))
253     return BaseT::getArithmeticReductionCost(Opcode, VTy, FMF, CostKind);
254 
255   if (!ST->useRVVForFixedLengthVectors())
256     return BaseT::getArithmeticReductionCost(Opcode, VTy, FMF, CostKind);
257 
258   // Skip if scalar size of VTy is bigger than ELEN.
259   if (VTy->getScalarSizeInBits() > ST->getMaxELENForFixedLengthVectors())
260     return BaseT::getArithmeticReductionCost(Opcode, VTy, FMF, CostKind);
261 
262   int ISD = TLI->InstructionOpcodeToISD(Opcode);
263   assert(ISD && "Invalid opcode");
264 
265   if (ISD != ISD::ADD && ISD != ISD::OR && ISD != ISD::XOR && ISD != ISD::AND &&
266       ISD != ISD::FADD)
267     return BaseT::getArithmeticReductionCost(Opcode, VTy, FMF, CostKind);
268 
269   // IR Reduction is composed by two vmv and one rvv reduction instruction.
270   InstructionCost BaseCost = 2;
271   unsigned VL = cast<FixedVectorType>(VTy)->getNumElements();
272   std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, VTy);
273 
274   if (TTI::requiresOrderedReduction(FMF))
275     return (LT.first - 1) + BaseCost + VL;
276   return (LT.first - 1) + BaseCost + Log2_32_Ceil(VL);
277 }
278 
279 void RISCVTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
280                                            TTI::UnrollingPreferences &UP,
281                                            OptimizationRemarkEmitter *ORE) {
282   // TODO: More tuning on benchmarks and metrics with changes as needed
283   //       would apply to all settings below to enable performance.
284 
285   // Support explicit targets enabled for SiFive with the unrolling preferences
286   // below
287   bool UseDefaultPreferences = true;
288   if (ST->getProcFamily() == RISCVSubtarget::SiFive7)
289     UseDefaultPreferences = false;
290 
291   if (UseDefaultPreferences)
292     return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
293 
294   // Enable Upper bound unrolling universally, not dependant upon the conditions
295   // below.
296   UP.UpperBound = true;
297 
298   // Disable loop unrolling for Oz and Os.
299   UP.OptSizeThreshold = 0;
300   UP.PartialOptSizeThreshold = 0;
301   if (L->getHeader()->getParent()->hasOptSize())
302     return;
303 
304   SmallVector<BasicBlock *, 4> ExitingBlocks;
305   L->getExitingBlocks(ExitingBlocks);
306   LLVM_DEBUG(dbgs() << "Loop has:\n"
307                     << "Blocks: " << L->getNumBlocks() << "\n"
308                     << "Exit blocks: " << ExitingBlocks.size() << "\n");
309 
310   // Only allow another exit other than the latch. This acts as an early exit
311   // as it mirrors the profitability calculation of the runtime unroller.
312   if (ExitingBlocks.size() > 2)
313     return;
314 
315   // Limit the CFG of the loop body for targets with a branch predictor.
316   // Allowing 4 blocks permits if-then-else diamonds in the body.
317   if (L->getNumBlocks() > 4)
318     return;
319 
320   // Don't unroll vectorized loops, including the remainder loop
321   if (getBooleanLoopAttribute(L, "llvm.loop.isvectorized"))
322     return;
323 
324   // Scan the loop: don't unroll loops with calls as this could prevent
325   // inlining.
326   InstructionCost Cost = 0;
327   for (auto *BB : L->getBlocks()) {
328     for (auto &I : *BB) {
329       // Initial setting - Don't unroll loops containing vectorized
330       // instructions.
331       if (I.getType()->isVectorTy())
332         return;
333 
334       if (isa<CallInst>(I) || isa<InvokeInst>(I)) {
335         if (const Function *F = cast<CallBase>(I).getCalledFunction()) {
336           if (!isLoweredToCall(F))
337             continue;
338         }
339         return;
340       }
341 
342       SmallVector<const Value *> Operands(I.operand_values());
343       Cost +=
344           getUserCost(&I, Operands, TargetTransformInfo::TCK_SizeAndLatency);
345     }
346   }
347 
348   LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
349 
350   UP.Partial = true;
351   UP.Runtime = true;
352   UP.UnrollRemainder = true;
353   UP.UnrollAndJam = true;
354   UP.UnrollAndJamInnerLoopThreshold = 60;
355 
356   // Force unrolling small loops can be very useful because of the branch
357   // taken cost of the backedge.
358   if (Cost < 12)
359     UP.Force = true;
360 }
361 
362 void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
363                                          TTI::PeelingPreferences &PP) {
364   BaseT::getPeelingPreferences(L, SE, PP);
365 }
366 
367 InstructionCost RISCVTTIImpl::getRegUsageForType(Type *Ty) {
368   TypeSize Size = Ty->getPrimitiveSizeInBits();
369   if (Ty->isVectorTy()) {
370     if (Size.isScalable() && ST->hasVInstructions())
371       return divideCeil(Size.getKnownMinValue(), RISCV::RVVBitsPerBlock);
372 
373     if (ST->useRVVForFixedLengthVectors())
374       return divideCeil(Size, ST->getMinRVVVectorSizeInBits());
375   }
376 
377   return BaseT::getRegUsageForType(Ty);
378 }
379