1 //===-- RISCVTargetTransformInfo.cpp - RISC-V specific TTI ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "RISCVTargetTransformInfo.h"
10 #include "MCTargetDesc/RISCVMatInt.h"
11 #include "llvm/Analysis/TargetTransformInfo.h"
12 #include "llvm/CodeGen/BasicTTIImpl.h"
13 #include "llvm/CodeGen/TargetLowering.h"
14 #include <cmath>
15 using namespace llvm;
16 
17 #define DEBUG_TYPE "riscvtti"
18 
19 static cl::opt<unsigned> RVVRegisterWidthLMUL(
20     "riscv-v-register-bit-width-lmul",
21     cl::desc(
22         "The LMUL to use for getRegisterBitWidth queries. Affects LMUL used "
23         "by autovectorized code. Fractional LMULs are not supported."),
24     cl::init(1), cl::Hidden);
25 
26 InstructionCost RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
27                                             TTI::TargetCostKind CostKind) {
28   assert(Ty->isIntegerTy() &&
29          "getIntImmCost can only estimate cost of materialising integers");
30 
31   // We have a Zero register, so 0 is always free.
32   if (Imm == 0)
33     return TTI::TCC_Free;
34 
35   // Otherwise, we check how many instructions it will take to materialise.
36   const DataLayout &DL = getDataLayout();
37   return RISCVMatInt::getIntMatCost(Imm, DL.getTypeSizeInBits(Ty),
38                                     getST()->getFeatureBits());
39 }
40 
41 InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
42                                                 const APInt &Imm, Type *Ty,
43                                                 TTI::TargetCostKind CostKind,
44                                                 Instruction *Inst) {
45   assert(Ty->isIntegerTy() &&
46          "getIntImmCost can only estimate cost of materialising integers");
47 
48   // We have a Zero register, so 0 is always free.
49   if (Imm == 0)
50     return TTI::TCC_Free;
51 
52   // Some instructions in RISC-V can take a 12-bit immediate. Some of these are
53   // commutative, in others the immediate comes from a specific argument index.
54   bool Takes12BitImm = false;
55   unsigned ImmArgIdx = ~0U;
56 
57   switch (Opcode) {
58   case Instruction::GetElementPtr:
59     // Never hoist any arguments to a GetElementPtr. CodeGenPrepare will
60     // split up large offsets in GEP into better parts than ConstantHoisting
61     // can.
62     return TTI::TCC_Free;
63   case Instruction::And:
64     // zext.h
65     if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb())
66       return TTI::TCC_Free;
67     // zext.w
68     if (Imm == UINT64_C(0xffffffff) && ST->hasStdExtZbb())
69       return TTI::TCC_Free;
70     LLVM_FALLTHROUGH;
71   case Instruction::Add:
72   case Instruction::Or:
73   case Instruction::Xor:
74   case Instruction::Mul:
75     Takes12BitImm = true;
76     break;
77   case Instruction::Sub:
78   case Instruction::Shl:
79   case Instruction::LShr:
80   case Instruction::AShr:
81     Takes12BitImm = true;
82     ImmArgIdx = 1;
83     break;
84   default:
85     break;
86   }
87 
88   if (Takes12BitImm) {
89     // Check immediate is the correct argument...
90     if (Instruction::isCommutative(Opcode) || Idx == ImmArgIdx) {
91       // ... and fits into the 12-bit immediate.
92       if (Imm.getMinSignedBits() <= 64 &&
93           getTLI()->isLegalAddImmediate(Imm.getSExtValue())) {
94         return TTI::TCC_Free;
95       }
96     }
97 
98     // Otherwise, use the full materialisation cost.
99     return getIntImmCost(Imm, Ty, CostKind);
100   }
101 
102   // By default, prevent hoisting.
103   return TTI::TCC_Free;
104 }
105 
106 InstructionCost
107 RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
108                                   const APInt &Imm, Type *Ty,
109                                   TTI::TargetCostKind CostKind) {
110   // Prevent hoisting in unknown cases.
111   return TTI::TCC_Free;
112 }
113 
114 TargetTransformInfo::PopcntSupportKind
115 RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) {
116   assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
117   return ST->hasStdExtZbb() ? TTI::PSK_FastHardware : TTI::PSK_Software;
118 }
119 
120 bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
121   // Currently, the ExpandReductions pass can't expand scalable-vector
122   // reductions, but we still request expansion as RVV doesn't support certain
123   // reductions and the SelectionDAG can't legalize them either.
124   switch (II->getIntrinsicID()) {
125   default:
126     return false;
127   // These reductions have no equivalent in RVV
128   case Intrinsic::vector_reduce_mul:
129   case Intrinsic::vector_reduce_fmul:
130     return true;
131   }
132 }
133 
134 Optional<unsigned> RISCVTTIImpl::getMaxVScale() const {
135   // There is no assumption of the maximum vector length in V specification.
136   // We use the value specified by users as the maximum vector length.
137   // This function will use the assumed maximum vector length to get the
138   // maximum vscale for LoopVectorizer.
139   // If users do not specify the maximum vector length, we have no way to
140   // know whether the LoopVectorizer is safe to do or not.
141   // We only consider to use single vector register (LMUL = 1) to vectorize.
142   unsigned MaxVectorSizeInBits = ST->getMaxRVVVectorSizeInBits();
143   if (ST->hasVInstructions() && MaxVectorSizeInBits != 0)
144     return MaxVectorSizeInBits / RISCV::RVVBitsPerBlock;
145   return BaseT::getMaxVScale();
146 }
147 
148 TypeSize
149 RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
150   unsigned LMUL = PowerOf2Floor(
151       std::max<unsigned>(std::min<unsigned>(RVVRegisterWidthLMUL, 8), 1));
152   switch (K) {
153   case TargetTransformInfo::RGK_Scalar:
154     return TypeSize::getFixed(ST->getXLen());
155   case TargetTransformInfo::RGK_FixedWidthVector:
156     return TypeSize::getFixed(
157         ST->hasVInstructions() ? LMUL * ST->getMinRVVVectorSizeInBits() : 0);
158   case TargetTransformInfo::RGK_ScalableVector:
159     return TypeSize::getScalable(
160         ST->hasVInstructions() ? LMUL * RISCV::RVVBitsPerBlock : 0);
161   }
162 
163   llvm_unreachable("Unsupported register kind");
164 }
165 
166 InstructionCost RISCVTTIImpl::getSpliceCost(VectorType *Tp, int Index) {
167   std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
168 
169   unsigned Cost = 2; // vslidedown+vslideup.
170   // TODO: LMUL should increase cost.
171   // TODO: Multiplying by LT.first implies this legalizes into multiple copies
172   // of similar code, but I think we expand through memory.
173   return Cost * LT.first;
174 }
175 
176 InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
177                                              VectorType *Tp, ArrayRef<int> Mask,
178                                              int Index, VectorType *SubTp) {
179   if (Kind == TTI::SK_Splice && isa<ScalableVectorType>(Tp))
180     return getSpliceCost(Tp, Index);
181   return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp);
182 }
183 
184 InstructionCost
185 RISCVTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
186                                     unsigned AddressSpace,
187                                     TTI::TargetCostKind CostKind) {
188   if (!isa<ScalableVectorType>(Src))
189     return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
190                                         CostKind);
191 
192   return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
193 }
194 
195 InstructionCost RISCVTTIImpl::getGatherScatterOpCost(
196     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
197     Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
198   if (CostKind != TTI::TCK_RecipThroughput)
199     return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
200                                          Alignment, CostKind, I);
201 
202   if ((Opcode == Instruction::Load &&
203        !isLegalMaskedGather(DataTy, Align(Alignment))) ||
204       (Opcode == Instruction::Store &&
205        !isLegalMaskedScatter(DataTy, Align(Alignment))))
206     return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
207                                          Alignment, CostKind, I);
208 
209   // FIXME: Only supporting fixed vectors for now.
210   if (!isa<FixedVectorType>(DataTy))
211     return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
212                                          Alignment, CostKind, I);
213 
214   auto *VTy = cast<FixedVectorType>(DataTy);
215   unsigned NumLoads = VTy->getNumElements();
216   InstructionCost MemOpCost =
217       getMemoryOpCost(Opcode, VTy->getElementType(), Alignment, 0, CostKind, I);
218   return NumLoads * MemOpCost;
219 }
220 
221 InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
222                                                Type *Src,
223                                                TTI::CastContextHint CCH,
224                                                TTI::TargetCostKind CostKind,
225                                                const Instruction *I) {
226   if (isa<VectorType>(Dst) && isa<VectorType>(Src)) {
227     // FIXME: Need to compute legalizing cost for illegal types.
228     if (!isTypeLegal(Src) || !isTypeLegal(Dst))
229       return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
230 
231     // Skip if element size of Dst or Src is bigger than ELEN.
232     if (Src->getScalarSizeInBits() > ST->getMaxELENForFixedLengthVectors() ||
233         Dst->getScalarSizeInBits() > ST->getMaxELENForFixedLengthVectors())
234       return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
235 
236     int ISD = TLI->InstructionOpcodeToISD(Opcode);
237     assert(ISD && "Invalid opcode");
238 
239     // FIXME: Need to consider vsetvli and lmul.
240     int PowDiff = (int)Log2_32(Dst->getScalarSizeInBits()) -
241                   (int)Log2_32(Src->getScalarSizeInBits());
242     switch (ISD) {
243     case ISD::SIGN_EXTEND:
244     case ISD::ZERO_EXTEND:
245       return 1;
246     case ISD::TRUNCATE:
247     case ISD::FP_EXTEND:
248     case ISD::FP_ROUND:
249       // Counts of narrow/widen instructions.
250       return std::abs(PowDiff);
251     case ISD::FP_TO_SINT:
252     case ISD::FP_TO_UINT:
253     case ISD::SINT_TO_FP:
254     case ISD::UINT_TO_FP:
255       if (std::abs(PowDiff) <= 1)
256         return 1;
257       // Backend could lower (v[sz]ext i8 to double) to vfcvt(v[sz]ext.f8 i8),
258       // so it only need two conversion.
259       if (Src->isIntOrIntVectorTy())
260         return 2;
261       // Counts of narrow/widen instructions.
262       return std::abs(PowDiff);
263     }
264   }
265   return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
266 }
267 
268 InstructionCost
269 RISCVTTIImpl::getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy,
270                                      bool IsUnsigned,
271                                      TTI::TargetCostKind CostKind) {
272   // FIXME: Only supporting fixed vectors for now.
273   if (!isa<FixedVectorType>(Ty))
274     return BaseT::getMinMaxReductionCost(Ty, CondTy, IsUnsigned, CostKind);
275 
276   if (!ST->useRVVForFixedLengthVectors())
277     return BaseT::getMinMaxReductionCost(Ty, CondTy, IsUnsigned, CostKind);
278 
279   // Skip if scalar size of Ty is bigger than ELEN.
280   if (Ty->getScalarSizeInBits() > ST->getMaxELENForFixedLengthVectors())
281     return BaseT::getMinMaxReductionCost(Ty, CondTy, IsUnsigned, CostKind);
282 
283   // IR Reduction is composed by two vmv and one rvv reduction instruction.
284   InstructionCost BaseCost = 2;
285   unsigned VL = cast<FixedVectorType>(Ty)->getNumElements();
286   std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
287   return (LT.first - 1) + BaseCost + Log2_32_Ceil(VL);
288 }
289 
290 InstructionCost
291 RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *VTy,
292                                          Optional<FastMathFlags> FMF,
293                                          TTI::TargetCostKind CostKind) {
294   // FIXME: Only supporting fixed vectors for now.
295   if (!isa<FixedVectorType>(VTy))
296     return BaseT::getArithmeticReductionCost(Opcode, VTy, FMF, CostKind);
297 
298   // FIXME: Do not support i1 and/or reduction now.
299   if (VTy->getElementType()->isIntegerTy(1))
300     return BaseT::getArithmeticReductionCost(Opcode, VTy, FMF, CostKind);
301 
302   if (!ST->useRVVForFixedLengthVectors())
303     return BaseT::getArithmeticReductionCost(Opcode, VTy, FMF, CostKind);
304 
305   // Skip if scalar size of VTy is bigger than ELEN.
306   if (VTy->getScalarSizeInBits() > ST->getMaxELENForFixedLengthVectors())
307     return BaseT::getArithmeticReductionCost(Opcode, VTy, FMF, CostKind);
308 
309   int ISD = TLI->InstructionOpcodeToISD(Opcode);
310   assert(ISD && "Invalid opcode");
311 
312   if (ISD != ISD::ADD && ISD != ISD::OR && ISD != ISD::XOR && ISD != ISD::AND &&
313       ISD != ISD::FADD)
314     return BaseT::getArithmeticReductionCost(Opcode, VTy, FMF, CostKind);
315 
316   // IR Reduction is composed by two vmv and one rvv reduction instruction.
317   InstructionCost BaseCost = 2;
318   unsigned VL = cast<FixedVectorType>(VTy)->getNumElements();
319   std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, VTy);
320 
321   if (TTI::requiresOrderedReduction(FMF))
322     return (LT.first - 1) + BaseCost + VL;
323   return (LT.first - 1) + BaseCost + Log2_32_Ceil(VL);
324 }
325 
326 void RISCVTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
327                                            TTI::UnrollingPreferences &UP,
328                                            OptimizationRemarkEmitter *ORE) {
329   // TODO: More tuning on benchmarks and metrics with changes as needed
330   //       would apply to all settings below to enable performance.
331 
332   // Support explicit targets enabled for SiFive with the unrolling preferences
333   // below
334   bool UseDefaultPreferences = true;
335   if (ST->getProcFamily() == RISCVSubtarget::SiFive7)
336     UseDefaultPreferences = false;
337 
338   if (UseDefaultPreferences)
339     return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
340 
341   // Enable Upper bound unrolling universally, not dependant upon the conditions
342   // below.
343   UP.UpperBound = true;
344 
345   // Disable loop unrolling for Oz and Os.
346   UP.OptSizeThreshold = 0;
347   UP.PartialOptSizeThreshold = 0;
348   if (L->getHeader()->getParent()->hasOptSize())
349     return;
350 
351   SmallVector<BasicBlock *, 4> ExitingBlocks;
352   L->getExitingBlocks(ExitingBlocks);
353   LLVM_DEBUG(dbgs() << "Loop has:\n"
354                     << "Blocks: " << L->getNumBlocks() << "\n"
355                     << "Exit blocks: " << ExitingBlocks.size() << "\n");
356 
357   // Only allow another exit other than the latch. This acts as an early exit
358   // as it mirrors the profitability calculation of the runtime unroller.
359   if (ExitingBlocks.size() > 2)
360     return;
361 
362   // Limit the CFG of the loop body for targets with a branch predictor.
363   // Allowing 4 blocks permits if-then-else diamonds in the body.
364   if (L->getNumBlocks() > 4)
365     return;
366 
367   // Don't unroll vectorized loops, including the remainder loop
368   if (getBooleanLoopAttribute(L, "llvm.loop.isvectorized"))
369     return;
370 
371   // Scan the loop: don't unroll loops with calls as this could prevent
372   // inlining.
373   InstructionCost Cost = 0;
374   for (auto *BB : L->getBlocks()) {
375     for (auto &I : *BB) {
376       // Initial setting - Don't unroll loops containing vectorized
377       // instructions.
378       if (I.getType()->isVectorTy())
379         return;
380 
381       if (isa<CallInst>(I) || isa<InvokeInst>(I)) {
382         if (const Function *F = cast<CallBase>(I).getCalledFunction()) {
383           if (!isLoweredToCall(F))
384             continue;
385         }
386         return;
387       }
388 
389       SmallVector<const Value *> Operands(I.operand_values());
390       Cost +=
391           getUserCost(&I, Operands, TargetTransformInfo::TCK_SizeAndLatency);
392     }
393   }
394 
395   LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
396 
397   UP.Partial = true;
398   UP.Runtime = true;
399   UP.UnrollRemainder = true;
400   UP.UnrollAndJam = true;
401   UP.UnrollAndJamInnerLoopThreshold = 60;
402 
403   // Force unrolling small loops can be very useful because of the branch
404   // taken cost of the backedge.
405   if (Cost < 12)
406     UP.Force = true;
407 }
408 
409 void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
410                                          TTI::PeelingPreferences &PP) {
411   BaseT::getPeelingPreferences(L, SE, PP);
412 }
413 
414 InstructionCost RISCVTTIImpl::getRegUsageForType(Type *Ty) {
415   TypeSize Size = Ty->getPrimitiveSizeInBits();
416   if (Ty->isVectorTy()) {
417     if (Size.isScalable() && ST->hasVInstructions())
418       return divideCeil(Size.getKnownMinValue(), RISCV::RVVBitsPerBlock);
419 
420     if (ST->useRVVForFixedLengthVectors())
421       return divideCeil(Size, ST->getMinRVVVectorSizeInBits());
422   }
423 
424   return BaseT::getRegUsageForType(Ty);
425 }
426