1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that RISCV uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "RISCVISelLowering.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCV.h"
17 #include "RISCVMachineFunctionInfo.h"
18 #include "RISCVRegisterInfo.h"
19 #include "RISCVSubtarget.h"
20 #include "RISCVTargetMachine.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/MemoryLocation.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineJumpTableInfo.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
30 #include "llvm/CodeGen/ValueTypes.h"
31 #include "llvm/IR/DiagnosticInfo.h"
32 #include "llvm/IR/DiagnosticPrinter.h"
33 #include "llvm/IR/IRBuilder.h"
34 #include "llvm/IR/IntrinsicsRISCV.h"
35 #include "llvm/IR/PatternMatch.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/KnownBits.h"
39 #include "llvm/Support/MathExtras.h"
40 #include "llvm/Support/raw_ostream.h"
41 
42 using namespace llvm;
43 
44 #define DEBUG_TYPE "riscv-lower"
45 
46 STATISTIC(NumTailCalls, "Number of tail calls");
47 
48 RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
49                                          const RISCVSubtarget &STI)
50     : TargetLowering(TM), Subtarget(STI) {
51 
52   if (Subtarget.isRV32E())
53     report_fatal_error("Codegen not yet implemented for RV32E");
54 
55   RISCVABI::ABI ABI = Subtarget.getTargetABI();
56   assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
57 
58   if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
59       !Subtarget.hasStdExtF()) {
60     errs() << "Hard-float 'f' ABI can't be used for a target that "
61                 "doesn't support the F instruction set extension (ignoring "
62                           "target-abi)\n";
63     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
64   } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
65              !Subtarget.hasStdExtD()) {
66     errs() << "Hard-float 'd' ABI can't be used for a target that "
67               "doesn't support the D instruction set extension (ignoring "
68               "target-abi)\n";
69     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
70   }
71 
72   switch (ABI) {
73   default:
74     report_fatal_error("Don't know how to lower this ABI");
75   case RISCVABI::ABI_ILP32:
76   case RISCVABI::ABI_ILP32F:
77   case RISCVABI::ABI_ILP32D:
78   case RISCVABI::ABI_LP64:
79   case RISCVABI::ABI_LP64F:
80   case RISCVABI::ABI_LP64D:
81     break;
82   }
83 
84   MVT XLenVT = Subtarget.getXLenVT();
85 
86   // Set up the register classes.
87   addRegisterClass(XLenVT, &RISCV::GPRRegClass);
88 
89   if (Subtarget.hasStdExtZfh())
90     addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
91   if (Subtarget.hasStdExtF())
92     addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
93   if (Subtarget.hasStdExtD())
94     addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
95 
96   static const MVT::SimpleValueType BoolVecVTs[] = {
97       MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,
98       MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
99   static const MVT::SimpleValueType IntVecVTs[] = {
100       MVT::nxv1i8,  MVT::nxv2i8,   MVT::nxv4i8,   MVT::nxv8i8,  MVT::nxv16i8,
101       MVT::nxv32i8, MVT::nxv64i8,  MVT::nxv1i16,  MVT::nxv2i16, MVT::nxv4i16,
102       MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
103       MVT::nxv4i32, MVT::nxv8i32,  MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
104       MVT::nxv4i64, MVT::nxv8i64};
105   static const MVT::SimpleValueType F16VecVTs[] = {
106       MVT::nxv1f16, MVT::nxv2f16,  MVT::nxv4f16,
107       MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
108   static const MVT::SimpleValueType F32VecVTs[] = {
109       MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
110   static const MVT::SimpleValueType F64VecVTs[] = {
111       MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
112 
113   if (Subtarget.hasVInstructions()) {
114     auto addRegClassForRVV = [this](MVT VT) {
115       unsigned Size = VT.getSizeInBits().getKnownMinValue();
116       assert(Size <= 512 && isPowerOf2_32(Size));
117       const TargetRegisterClass *RC;
118       if (Size <= 64)
119         RC = &RISCV::VRRegClass;
120       else if (Size == 128)
121         RC = &RISCV::VRM2RegClass;
122       else if (Size == 256)
123         RC = &RISCV::VRM4RegClass;
124       else
125         RC = &RISCV::VRM8RegClass;
126 
127       addRegisterClass(VT, RC);
128     };
129 
130     for (MVT VT : BoolVecVTs)
131       addRegClassForRVV(VT);
132     for (MVT VT : IntVecVTs) {
133       if (VT.getVectorElementType() == MVT::i64 &&
134           !Subtarget.hasVInstructionsI64())
135         continue;
136       addRegClassForRVV(VT);
137     }
138 
139     if (Subtarget.hasVInstructionsF16())
140       for (MVT VT : F16VecVTs)
141         addRegClassForRVV(VT);
142 
143     if (Subtarget.hasVInstructionsF32())
144       for (MVT VT : F32VecVTs)
145         addRegClassForRVV(VT);
146 
147     if (Subtarget.hasVInstructionsF64())
148       for (MVT VT : F64VecVTs)
149         addRegClassForRVV(VT);
150 
151     if (Subtarget.useRVVForFixedLengthVectors()) {
152       auto addRegClassForFixedVectors = [this](MVT VT) {
153         MVT ContainerVT = getContainerForFixedLengthVector(VT);
154         unsigned RCID = getRegClassIDForVecVT(ContainerVT);
155         const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
156         addRegisterClass(VT, TRI.getRegClass(RCID));
157       };
158       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
159         if (useRVVForFixedLengthVectorVT(VT))
160           addRegClassForFixedVectors(VT);
161 
162       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
163         if (useRVVForFixedLengthVectorVT(VT))
164           addRegClassForFixedVectors(VT);
165     }
166   }
167 
168   // Compute derived properties from the register classes.
169   computeRegisterProperties(STI.getRegisterInfo());
170 
171   setStackPointerRegisterToSaveRestore(RISCV::X2);
172 
173   for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
174     setLoadExtAction(N, XLenVT, MVT::i1, Promote);
175 
176   // TODO: add all necessary setOperationAction calls.
177   setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
178 
179   setOperationAction(ISD::BR_JT, MVT::Other, Expand);
180   setOperationAction(ISD::BR_CC, XLenVT, Expand);
181   setOperationAction(ISD::BRCOND, MVT::Other, Custom);
182   setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
183 
184   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
185   setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
186 
187   setOperationAction(ISD::VASTART, MVT::Other, Custom);
188   setOperationAction(ISD::VAARG, MVT::Other, Expand);
189   setOperationAction(ISD::VACOPY, MVT::Other, Expand);
190   setOperationAction(ISD::VAEND, MVT::Other, Expand);
191 
192   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
193   if (!Subtarget.hasStdExtZbb()) {
194     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
195     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
196   }
197 
198   if (Subtarget.is64Bit()) {
199     setOperationAction(ISD::ADD, MVT::i32, Custom);
200     setOperationAction(ISD::SUB, MVT::i32, Custom);
201     setOperationAction(ISD::SHL, MVT::i32, Custom);
202     setOperationAction(ISD::SRA, MVT::i32, Custom);
203     setOperationAction(ISD::SRL, MVT::i32, Custom);
204 
205     setOperationAction(ISD::UADDO, MVT::i32, Custom);
206     setOperationAction(ISD::USUBO, MVT::i32, Custom);
207     setOperationAction(ISD::UADDSAT, MVT::i32, Custom);
208     setOperationAction(ISD::USUBSAT, MVT::i32, Custom);
209   } else {
210     setLibcallName(RTLIB::SHL_I128, nullptr);
211     setLibcallName(RTLIB::SRL_I128, nullptr);
212     setLibcallName(RTLIB::SRA_I128, nullptr);
213     setLibcallName(RTLIB::MUL_I128, nullptr);
214     setLibcallName(RTLIB::MULO_I64, nullptr);
215   }
216 
217   if (!Subtarget.hasStdExtM()) {
218     setOperationAction(ISD::MUL, XLenVT, Expand);
219     setOperationAction(ISD::MULHS, XLenVT, Expand);
220     setOperationAction(ISD::MULHU, XLenVT, Expand);
221     setOperationAction(ISD::SDIV, XLenVT, Expand);
222     setOperationAction(ISD::UDIV, XLenVT, Expand);
223     setOperationAction(ISD::SREM, XLenVT, Expand);
224     setOperationAction(ISD::UREM, XLenVT, Expand);
225   } else {
226     if (Subtarget.is64Bit()) {
227       setOperationAction(ISD::MUL, MVT::i32, Custom);
228       setOperationAction(ISD::MUL, MVT::i128, Custom);
229 
230       setOperationAction(ISD::SDIV, MVT::i8, Custom);
231       setOperationAction(ISD::UDIV, MVT::i8, Custom);
232       setOperationAction(ISD::UREM, MVT::i8, Custom);
233       setOperationAction(ISD::SDIV, MVT::i16, Custom);
234       setOperationAction(ISD::UDIV, MVT::i16, Custom);
235       setOperationAction(ISD::UREM, MVT::i16, Custom);
236       setOperationAction(ISD::SDIV, MVT::i32, Custom);
237       setOperationAction(ISD::UDIV, MVT::i32, Custom);
238       setOperationAction(ISD::UREM, MVT::i32, Custom);
239     } else {
240       setOperationAction(ISD::MUL, MVT::i64, Custom);
241     }
242   }
243 
244   setOperationAction(ISD::SDIVREM, XLenVT, Expand);
245   setOperationAction(ISD::UDIVREM, XLenVT, Expand);
246   setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
247   setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
248 
249   setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
250   setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
251   setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
252 
253   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
254     if (Subtarget.is64Bit()) {
255       setOperationAction(ISD::ROTL, MVT::i32, Custom);
256       setOperationAction(ISD::ROTR, MVT::i32, Custom);
257     }
258   } else {
259     setOperationAction(ISD::ROTL, XLenVT, Expand);
260     setOperationAction(ISD::ROTR, XLenVT, Expand);
261   }
262 
263   if (Subtarget.hasStdExtZbp()) {
264     // Custom lower bswap/bitreverse so we can convert them to GREVI to enable
265     // more combining.
266     setOperationAction(ISD::BITREVERSE, XLenVT,   Custom);
267     setOperationAction(ISD::BSWAP,      XLenVT,   Custom);
268     setOperationAction(ISD::BITREVERSE, MVT::i8,  Custom);
269     // BSWAP i8 doesn't exist.
270     setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
271     setOperationAction(ISD::BSWAP,      MVT::i16, Custom);
272 
273     if (Subtarget.is64Bit()) {
274       setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
275       setOperationAction(ISD::BSWAP,      MVT::i32, Custom);
276     }
277   } else {
278     // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
279     // pattern match it directly in isel.
280     setOperationAction(ISD::BSWAP, XLenVT,
281                        Subtarget.hasStdExtZbb() ? Legal : Expand);
282   }
283 
284   if (Subtarget.hasStdExtZbb()) {
285     setOperationAction(ISD::SMIN, XLenVT, Legal);
286     setOperationAction(ISD::SMAX, XLenVT, Legal);
287     setOperationAction(ISD::UMIN, XLenVT, Legal);
288     setOperationAction(ISD::UMAX, XLenVT, Legal);
289 
290     if (Subtarget.is64Bit()) {
291       setOperationAction(ISD::CTTZ, MVT::i32, Custom);
292       setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom);
293       setOperationAction(ISD::CTLZ, MVT::i32, Custom);
294       setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom);
295     }
296   } else {
297     setOperationAction(ISD::CTTZ, XLenVT, Expand);
298     setOperationAction(ISD::CTLZ, XLenVT, Expand);
299     setOperationAction(ISD::CTPOP, XLenVT, Expand);
300   }
301 
302   if (Subtarget.hasStdExtZbt()) {
303     setOperationAction(ISD::FSHL, XLenVT, Custom);
304     setOperationAction(ISD::FSHR, XLenVT, Custom);
305     setOperationAction(ISD::SELECT, XLenVT, Legal);
306 
307     if (Subtarget.is64Bit()) {
308       setOperationAction(ISD::FSHL, MVT::i32, Custom);
309       setOperationAction(ISD::FSHR, MVT::i32, Custom);
310     }
311   } else {
312     setOperationAction(ISD::SELECT, XLenVT, Custom);
313   }
314 
315   static const ISD::CondCode FPCCToExpand[] = {
316       ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
317       ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
318       ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};
319 
320   static const ISD::NodeType FPOpToExpand[] = {
321       ISD::FSIN, ISD::FCOS,       ISD::FSINCOS,   ISD::FPOW,
322       ISD::FREM, ISD::FP16_TO_FP, ISD::FP_TO_FP16};
323 
324   if (Subtarget.hasStdExtZfh())
325     setOperationAction(ISD::BITCAST, MVT::i16, Custom);
326 
327   if (Subtarget.hasStdExtZfh()) {
328     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
329     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
330     setOperationAction(ISD::LRINT, MVT::f16, Legal);
331     setOperationAction(ISD::LLRINT, MVT::f16, Legal);
332     setOperationAction(ISD::LROUND, MVT::f16, Legal);
333     setOperationAction(ISD::LLROUND, MVT::f16, Legal);
334     setOperationAction(ISD::STRICT_LRINT, MVT::f16, Legal);
335     setOperationAction(ISD::STRICT_LLRINT, MVT::f16, Legal);
336     setOperationAction(ISD::STRICT_LROUND, MVT::f16, Legal);
337     setOperationAction(ISD::STRICT_LLROUND, MVT::f16, Legal);
338     setOperationAction(ISD::STRICT_FADD, MVT::f16, Legal);
339     setOperationAction(ISD::STRICT_FMA, MVT::f16, Legal);
340     setOperationAction(ISD::STRICT_FSUB, MVT::f16, Legal);
341     setOperationAction(ISD::STRICT_FMUL, MVT::f16, Legal);
342     setOperationAction(ISD::STRICT_FDIV, MVT::f16, Legal);
343     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
344     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
345     setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Legal);
346     setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Legal);
347     setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Legal);
348     for (auto CC : FPCCToExpand)
349       setCondCodeAction(CC, MVT::f16, Expand);
350     setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
351     setOperationAction(ISD::SELECT, MVT::f16, Custom);
352     setOperationAction(ISD::BR_CC, MVT::f16, Expand);
353 
354     setOperationAction(ISD::FREM,       MVT::f16, Promote);
355     setOperationAction(ISD::FCEIL,      MVT::f16, Promote);
356     setOperationAction(ISD::FFLOOR,     MVT::f16, Promote);
357     setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote);
358     setOperationAction(ISD::FRINT,      MVT::f16, Promote);
359     setOperationAction(ISD::FROUND,     MVT::f16, Promote);
360     setOperationAction(ISD::FROUNDEVEN, MVT::f16, Promote);
361     setOperationAction(ISD::FTRUNC,     MVT::f16, Promote);
362     setOperationAction(ISD::FPOW,       MVT::f16, Promote);
363     setOperationAction(ISD::FPOWI,      MVT::f16, Promote);
364     setOperationAction(ISD::FCOS,       MVT::f16, Promote);
365     setOperationAction(ISD::FSIN,       MVT::f16, Promote);
366     setOperationAction(ISD::FSINCOS,    MVT::f16, Promote);
367     setOperationAction(ISD::FEXP,       MVT::f16, Promote);
368     setOperationAction(ISD::FEXP2,      MVT::f16, Promote);
369     setOperationAction(ISD::FLOG,       MVT::f16, Promote);
370     setOperationAction(ISD::FLOG2,      MVT::f16, Promote);
371     setOperationAction(ISD::FLOG10,     MVT::f16, Promote);
372 
373     // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
374     // complete support for all operations in LegalizeDAG.
375 
376     // We need to custom promote this.
377     if (Subtarget.is64Bit())
378       setOperationAction(ISD::FPOWI, MVT::i32, Custom);
379   }
380 
381   if (Subtarget.hasStdExtF()) {
382     setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
383     setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
384     setOperationAction(ISD::LRINT, MVT::f32, Legal);
385     setOperationAction(ISD::LLRINT, MVT::f32, Legal);
386     setOperationAction(ISD::LROUND, MVT::f32, Legal);
387     setOperationAction(ISD::LLROUND, MVT::f32, Legal);
388     setOperationAction(ISD::STRICT_LRINT, MVT::f32, Legal);
389     setOperationAction(ISD::STRICT_LLRINT, MVT::f32, Legal);
390     setOperationAction(ISD::STRICT_LROUND, MVT::f32, Legal);
391     setOperationAction(ISD::STRICT_LLROUND, MVT::f32, Legal);
392     setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
393     setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal);
394     setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
395     setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
396     setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
397     setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
398     setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);
399     setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
400     for (auto CC : FPCCToExpand)
401       setCondCodeAction(CC, MVT::f32, Expand);
402     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
403     setOperationAction(ISD::SELECT, MVT::f32, Custom);
404     setOperationAction(ISD::BR_CC, MVT::f32, Expand);
405     for (auto Op : FPOpToExpand)
406       setOperationAction(Op, MVT::f32, Expand);
407     setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
408     setTruncStoreAction(MVT::f32, MVT::f16, Expand);
409   }
410 
411   if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
412     setOperationAction(ISD::BITCAST, MVT::i32, Custom);
413 
414   if (Subtarget.hasStdExtD()) {
415     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
416     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
417     setOperationAction(ISD::LRINT, MVT::f64, Legal);
418     setOperationAction(ISD::LLRINT, MVT::f64, Legal);
419     setOperationAction(ISD::LROUND, MVT::f64, Legal);
420     setOperationAction(ISD::LLROUND, MVT::f64, Legal);
421     setOperationAction(ISD::STRICT_LRINT, MVT::f64, Legal);
422     setOperationAction(ISD::STRICT_LLRINT, MVT::f64, Legal);
423     setOperationAction(ISD::STRICT_LROUND, MVT::f64, Legal);
424     setOperationAction(ISD::STRICT_LLROUND, MVT::f64, Legal);
425     setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal);
426     setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
427     setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
428     setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
429     setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
430     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
431     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
432     setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
433     setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal);
434     setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal);
435     for (auto CC : FPCCToExpand)
436       setCondCodeAction(CC, MVT::f64, Expand);
437     setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
438     setOperationAction(ISD::SELECT, MVT::f64, Custom);
439     setOperationAction(ISD::BR_CC, MVT::f64, Expand);
440     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
441     setTruncStoreAction(MVT::f64, MVT::f32, Expand);
442     for (auto Op : FPOpToExpand)
443       setOperationAction(Op, MVT::f64, Expand);
444     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
445     setTruncStoreAction(MVT::f64, MVT::f16, Expand);
446   }
447 
448   if (Subtarget.is64Bit()) {
449     setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
450     setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
451     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
452     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
453   }
454 
455   if (Subtarget.hasStdExtF()) {
456     setOperationAction(ISD::FP_TO_UINT_SAT, XLenVT, Custom);
457     setOperationAction(ISD::FP_TO_SINT_SAT, XLenVT, Custom);
458 
459     setOperationAction(ISD::STRICT_FP_TO_UINT, XLenVT, Legal);
460     setOperationAction(ISD::STRICT_FP_TO_SINT, XLenVT, Legal);
461     setOperationAction(ISD::STRICT_UINT_TO_FP, XLenVT, Legal);
462     setOperationAction(ISD::STRICT_SINT_TO_FP, XLenVT, Legal);
463 
464     setOperationAction(ISD::FLT_ROUNDS_, XLenVT, Custom);
465     setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
466   }
467 
468   setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
469   setOperationAction(ISD::BlockAddress, XLenVT, Custom);
470   setOperationAction(ISD::ConstantPool, XLenVT, Custom);
471   setOperationAction(ISD::JumpTable, XLenVT, Custom);
472 
473   setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
474 
475   // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
476   // Unfortunately this can't be determined just from the ISA naming string.
477   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
478                      Subtarget.is64Bit() ? Legal : Custom);
479 
480   setOperationAction(ISD::TRAP, MVT::Other, Legal);
481   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
482   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
483   if (Subtarget.is64Bit())
484     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
485 
486   if (Subtarget.hasStdExtA()) {
487     setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
488     setMinCmpXchgSizeInBits(32);
489   } else {
490     setMaxAtomicSizeInBitsSupported(0);
491   }
492 
493   setBooleanContents(ZeroOrOneBooleanContent);
494 
495   if (Subtarget.hasVInstructions()) {
496     setBooleanVectorContents(ZeroOrOneBooleanContent);
497 
498     setOperationAction(ISD::VSCALE, XLenVT, Custom);
499 
500     // RVV intrinsics may have illegal operands.
501     // We also need to custom legalize vmv.x.s.
502     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
503     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
504     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
505     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom);
506     if (Subtarget.is64Bit()) {
507       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);
508     } else {
509       setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
510       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
511     }
512 
513     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
514     setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
515 
516     static const unsigned IntegerVPOps[] = {
517         ISD::VP_ADD,         ISD::VP_SUB,         ISD::VP_MUL,
518         ISD::VP_SDIV,        ISD::VP_UDIV,        ISD::VP_SREM,
519         ISD::VP_UREM,        ISD::VP_AND,         ISD::VP_OR,
520         ISD::VP_XOR,         ISD::VP_ASHR,        ISD::VP_LSHR,
521         ISD::VP_SHL,         ISD::VP_REDUCE_ADD,  ISD::VP_REDUCE_AND,
522         ISD::VP_REDUCE_OR,   ISD::VP_REDUCE_XOR,  ISD::VP_REDUCE_SMAX,
523         ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
524         ISD::VP_SELECT};
525 
526     static const unsigned FloatingPointVPOps[] = {
527         ISD::VP_FADD,        ISD::VP_FSUB,        ISD::VP_FMUL,
528         ISD::VP_FDIV,        ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
529         ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SELECT};
530 
531     if (!Subtarget.is64Bit()) {
532       // We must custom-lower certain vXi64 operations on RV32 due to the vector
533       // element type being illegal.
534       setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::i64, Custom);
535       setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::i64, Custom);
536 
537       setOperationAction(ISD::VECREDUCE_ADD, MVT::i64, Custom);
538       setOperationAction(ISD::VECREDUCE_AND, MVT::i64, Custom);
539       setOperationAction(ISD::VECREDUCE_OR, MVT::i64, Custom);
540       setOperationAction(ISD::VECREDUCE_XOR, MVT::i64, Custom);
541       setOperationAction(ISD::VECREDUCE_SMAX, MVT::i64, Custom);
542       setOperationAction(ISD::VECREDUCE_SMIN, MVT::i64, Custom);
543       setOperationAction(ISD::VECREDUCE_UMAX, MVT::i64, Custom);
544       setOperationAction(ISD::VECREDUCE_UMIN, MVT::i64, Custom);
545 
546       setOperationAction(ISD::VP_REDUCE_ADD, MVT::i64, Custom);
547       setOperationAction(ISD::VP_REDUCE_AND, MVT::i64, Custom);
548       setOperationAction(ISD::VP_REDUCE_OR, MVT::i64, Custom);
549       setOperationAction(ISD::VP_REDUCE_XOR, MVT::i64, Custom);
550       setOperationAction(ISD::VP_REDUCE_SMAX, MVT::i64, Custom);
551       setOperationAction(ISD::VP_REDUCE_SMIN, MVT::i64, Custom);
552       setOperationAction(ISD::VP_REDUCE_UMAX, MVT::i64, Custom);
553       setOperationAction(ISD::VP_REDUCE_UMIN, MVT::i64, Custom);
554     }
555 
556     for (MVT VT : BoolVecVTs) {
557       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
558 
559       // Mask VTs are custom-expanded into a series of standard nodes
560       setOperationAction(ISD::TRUNCATE, VT, Custom);
561       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
562       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
563       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
564 
565       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
566       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
567 
568       setOperationAction(ISD::SELECT, VT, Custom);
569       setOperationAction(ISD::SELECT_CC, VT, Expand);
570       setOperationAction(ISD::VSELECT, VT, Expand);
571       setOperationAction(ISD::VP_SELECT, VT, Expand);
572 
573       setOperationAction(ISD::VP_AND, VT, Custom);
574       setOperationAction(ISD::VP_OR, VT, Custom);
575       setOperationAction(ISD::VP_XOR, VT, Custom);
576 
577       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
578       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
579       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
580 
581       setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
582       setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
583       setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
584 
585       // RVV has native int->float & float->int conversions where the
586       // element type sizes are within one power-of-two of each other. Any
587       // wider distances between type sizes have to be lowered as sequences
588       // which progressively narrow the gap in stages.
589       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
590       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
591       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
592       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
593 
594       // Expand all extending loads to types larger than this, and truncating
595       // stores from types larger than this.
596       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
597         setTruncStoreAction(OtherVT, VT, Expand);
598         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
599         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
600         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
601       }
602     }
603 
604     for (MVT VT : IntVecVTs) {
605       if (VT.getVectorElementType() == MVT::i64 &&
606           !Subtarget.hasVInstructionsI64())
607         continue;
608 
609       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
610       setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
611 
612       // Vectors implement MULHS/MULHU.
613       setOperationAction(ISD::SMUL_LOHI, VT, Expand);
614       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
615 
616       setOperationAction(ISD::SMIN, VT, Legal);
617       setOperationAction(ISD::SMAX, VT, Legal);
618       setOperationAction(ISD::UMIN, VT, Legal);
619       setOperationAction(ISD::UMAX, VT, Legal);
620 
621       setOperationAction(ISD::ROTL, VT, Expand);
622       setOperationAction(ISD::ROTR, VT, Expand);
623 
624       setOperationAction(ISD::CTTZ, VT, Expand);
625       setOperationAction(ISD::CTLZ, VT, Expand);
626       setOperationAction(ISD::CTPOP, VT, Expand);
627 
628       setOperationAction(ISD::BSWAP, VT, Expand);
629 
630       // Custom-lower extensions and truncations from/to mask types.
631       setOperationAction(ISD::ANY_EXTEND, VT, Custom);
632       setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
633       setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
634 
635       // RVV has native int->float & float->int conversions where the
636       // element type sizes are within one power-of-two of each other. Any
637       // wider distances between type sizes have to be lowered as sequences
638       // which progressively narrow the gap in stages.
639       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
640       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
641       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
642       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
643 
644       setOperationAction(ISD::SADDSAT, VT, Legal);
645       setOperationAction(ISD::UADDSAT, VT, Legal);
646       setOperationAction(ISD::SSUBSAT, VT, Legal);
647       setOperationAction(ISD::USUBSAT, VT, Legal);
648 
649       // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
650       // nodes which truncate by one power of two at a time.
651       setOperationAction(ISD::TRUNCATE, VT, Custom);
652 
653       // Custom-lower insert/extract operations to simplify patterns.
654       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
655       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
656 
657       // Custom-lower reduction operations to set up the corresponding custom
658       // nodes' operands.
659       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
660       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
661       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
662       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
663       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
664       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
665       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
666       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
667 
668       for (unsigned VPOpc : IntegerVPOps)
669         setOperationAction(VPOpc, VT, Custom);
670 
671       setOperationAction(ISD::LOAD, VT, Custom);
672       setOperationAction(ISD::STORE, VT, Custom);
673 
674       setOperationAction(ISD::MLOAD, VT, Custom);
675       setOperationAction(ISD::MSTORE, VT, Custom);
676       setOperationAction(ISD::MGATHER, VT, Custom);
677       setOperationAction(ISD::MSCATTER, VT, Custom);
678 
679       setOperationAction(ISD::VP_LOAD, VT, Custom);
680       setOperationAction(ISD::VP_STORE, VT, Custom);
681       setOperationAction(ISD::VP_GATHER, VT, Custom);
682       setOperationAction(ISD::VP_SCATTER, VT, Custom);
683 
684       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
685       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
686       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
687 
688       setOperationAction(ISD::SELECT, VT, Custom);
689       setOperationAction(ISD::SELECT_CC, VT, Expand);
690 
691       setOperationAction(ISD::STEP_VECTOR, VT, Custom);
692       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
693 
694       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
695         setTruncStoreAction(VT, OtherVT, Expand);
696         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
697         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
698         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
699       }
700 
701       // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
702       // type that can represent the value exactly.
703       if (VT.getVectorElementType() != MVT::i64) {
704         MVT FloatEltVT =
705             VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
706         EVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
707         if (isTypeLegal(FloatVT)) {
708           setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
709           setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
710         }
711       }
712     }
713 
714     // Expand various CCs to best match the RVV ISA, which natively supports UNE
715     // but no other unordered comparisons, and supports all ordered comparisons
716     // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
717     // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
718     // and we pattern-match those back to the "original", swapping operands once
719     // more. This way we catch both operations and both "vf" and "fv" forms with
720     // fewer patterns.
721     static const ISD::CondCode VFPCCToExpand[] = {
722         ISD::SETO,   ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
723         ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
724         ISD::SETGT,  ISD::SETOGT, ISD::SETGE,  ISD::SETOGE,
725     };
726 
727     // Sets common operation actions on RVV floating-point vector types.
728     const auto SetCommonVFPActions = [&](MVT VT) {
729       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
730       // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
731       // sizes are within one power-of-two of each other. Therefore conversions
732       // between vXf16 and vXf64 must be lowered as sequences which convert via
733       // vXf32.
734       setOperationAction(ISD::FP_ROUND, VT, Custom);
735       setOperationAction(ISD::FP_EXTEND, VT, Custom);
736       // Custom-lower insert/extract operations to simplify patterns.
737       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
738       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
739       // Expand various condition codes (explained above).
740       for (auto CC : VFPCCToExpand)
741         setCondCodeAction(CC, VT, Expand);
742 
743       setOperationAction(ISD::FMINNUM, VT, Legal);
744       setOperationAction(ISD::FMAXNUM, VT, Legal);
745 
746       setOperationAction(ISD::FTRUNC, VT, Custom);
747       setOperationAction(ISD::FCEIL, VT, Custom);
748       setOperationAction(ISD::FFLOOR, VT, Custom);
749 
750       setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
751       setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
752       setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
753       setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
754 
755       setOperationAction(ISD::FCOPYSIGN, VT, Legal);
756 
757       setOperationAction(ISD::LOAD, VT, Custom);
758       setOperationAction(ISD::STORE, VT, Custom);
759 
760       setOperationAction(ISD::MLOAD, VT, Custom);
761       setOperationAction(ISD::MSTORE, VT, Custom);
762       setOperationAction(ISD::MGATHER, VT, Custom);
763       setOperationAction(ISD::MSCATTER, VT, Custom);
764 
765       setOperationAction(ISD::VP_LOAD, VT, Custom);
766       setOperationAction(ISD::VP_STORE, VT, Custom);
767       setOperationAction(ISD::VP_GATHER, VT, Custom);
768       setOperationAction(ISD::VP_SCATTER, VT, Custom);
769 
770       setOperationAction(ISD::SELECT, VT, Custom);
771       setOperationAction(ISD::SELECT_CC, VT, Expand);
772 
773       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
774       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
775       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
776 
777       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
778 
779       for (unsigned VPOpc : FloatingPointVPOps)
780         setOperationAction(VPOpc, VT, Custom);
781     };
782 
783     // Sets common extload/truncstore actions on RVV floating-point vector
784     // types.
785     const auto SetCommonVFPExtLoadTruncStoreActions =
786         [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
787           for (auto SmallVT : SmallerVTs) {
788             setTruncStoreAction(VT, SmallVT, Expand);
789             setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
790           }
791         };
792 
793     if (Subtarget.hasVInstructionsF16())
794       for (MVT VT : F16VecVTs)
795         SetCommonVFPActions(VT);
796 
797     for (MVT VT : F32VecVTs) {
798       if (Subtarget.hasVInstructionsF32())
799         SetCommonVFPActions(VT);
800       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
801     }
802 
803     for (MVT VT : F64VecVTs) {
804       if (Subtarget.hasVInstructionsF64())
805         SetCommonVFPActions(VT);
806       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
807       SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
808     }
809 
810     if (Subtarget.useRVVForFixedLengthVectors()) {
811       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
812         if (!useRVVForFixedLengthVectorVT(VT))
813           continue;
814 
815         // By default everything must be expanded.
816         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
817           setOperationAction(Op, VT, Expand);
818         for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
819           setTruncStoreAction(VT, OtherVT, Expand);
820           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
821           setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
822           setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
823         }
824 
825         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
826         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
827         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
828 
829         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
830         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
831 
832         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
833         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
834 
835         setOperationAction(ISD::LOAD, VT, Custom);
836         setOperationAction(ISD::STORE, VT, Custom);
837 
838         setOperationAction(ISD::SETCC, VT, Custom);
839 
840         setOperationAction(ISD::SELECT, VT, Custom);
841 
842         setOperationAction(ISD::TRUNCATE, VT, Custom);
843 
844         setOperationAction(ISD::BITCAST, VT, Custom);
845 
846         setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
847         setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
848         setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
849 
850         setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
851         setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
852         setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
853 
854         setOperationAction(ISD::SINT_TO_FP, VT, Custom);
855         setOperationAction(ISD::UINT_TO_FP, VT, Custom);
856         setOperationAction(ISD::FP_TO_SINT, VT, Custom);
857         setOperationAction(ISD::FP_TO_UINT, VT, Custom);
858 
859         // Operations below are different for between masks and other vectors.
860         if (VT.getVectorElementType() == MVT::i1) {
861           setOperationAction(ISD::VP_AND, VT, Custom);
862           setOperationAction(ISD::VP_OR, VT, Custom);
863           setOperationAction(ISD::VP_XOR, VT, Custom);
864           setOperationAction(ISD::AND, VT, Custom);
865           setOperationAction(ISD::OR, VT, Custom);
866           setOperationAction(ISD::XOR, VT, Custom);
867           continue;
868         }
869 
870         // Use SPLAT_VECTOR to prevent type legalization from destroying the
871         // splats when type legalizing i64 scalar on RV32.
872         // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
873         // improvements first.
874         if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
875           setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
876           setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
877         }
878 
879         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
880         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
881 
882         setOperationAction(ISD::MLOAD, VT, Custom);
883         setOperationAction(ISD::MSTORE, VT, Custom);
884         setOperationAction(ISD::MGATHER, VT, Custom);
885         setOperationAction(ISD::MSCATTER, VT, Custom);
886 
887         setOperationAction(ISD::VP_LOAD, VT, Custom);
888         setOperationAction(ISD::VP_STORE, VT, Custom);
889         setOperationAction(ISD::VP_GATHER, VT, Custom);
890         setOperationAction(ISD::VP_SCATTER, VT, Custom);
891 
892         setOperationAction(ISD::ADD, VT, Custom);
893         setOperationAction(ISD::MUL, VT, Custom);
894         setOperationAction(ISD::SUB, VT, Custom);
895         setOperationAction(ISD::AND, VT, Custom);
896         setOperationAction(ISD::OR, VT, Custom);
897         setOperationAction(ISD::XOR, VT, Custom);
898         setOperationAction(ISD::SDIV, VT, Custom);
899         setOperationAction(ISD::SREM, VT, Custom);
900         setOperationAction(ISD::UDIV, VT, Custom);
901         setOperationAction(ISD::UREM, VT, Custom);
902         setOperationAction(ISD::SHL, VT, Custom);
903         setOperationAction(ISD::SRA, VT, Custom);
904         setOperationAction(ISD::SRL, VT, Custom);
905 
906         setOperationAction(ISD::SMIN, VT, Custom);
907         setOperationAction(ISD::SMAX, VT, Custom);
908         setOperationAction(ISD::UMIN, VT, Custom);
909         setOperationAction(ISD::UMAX, VT, Custom);
910         setOperationAction(ISD::ABS,  VT, Custom);
911 
912         setOperationAction(ISD::MULHS, VT, Custom);
913         setOperationAction(ISD::MULHU, VT, Custom);
914 
915         setOperationAction(ISD::SADDSAT, VT, Custom);
916         setOperationAction(ISD::UADDSAT, VT, Custom);
917         setOperationAction(ISD::SSUBSAT, VT, Custom);
918         setOperationAction(ISD::USUBSAT, VT, Custom);
919 
920         setOperationAction(ISD::VSELECT, VT, Custom);
921         setOperationAction(ISD::SELECT_CC, VT, Expand);
922 
923         setOperationAction(ISD::ANY_EXTEND, VT, Custom);
924         setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
925         setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
926 
927         // Custom-lower reduction operations to set up the corresponding custom
928         // nodes' operands.
929         setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
930         setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
931         setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
932         setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
933         setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
934 
935         for (unsigned VPOpc : IntegerVPOps)
936           setOperationAction(VPOpc, VT, Custom);
937 
938         // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
939         // type that can represent the value exactly.
940         if (VT.getVectorElementType() != MVT::i64) {
941           MVT FloatEltVT =
942               VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
943           EVT FloatVT =
944               MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
945           if (isTypeLegal(FloatVT)) {
946             setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
947             setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
948           }
949         }
950       }
951 
952       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
953         if (!useRVVForFixedLengthVectorVT(VT))
954           continue;
955 
956         // By default everything must be expanded.
957         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
958           setOperationAction(Op, VT, Expand);
959         for (MVT OtherVT : MVT::fp_fixedlen_vector_valuetypes()) {
960           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
961           setTruncStoreAction(VT, OtherVT, Expand);
962         }
963 
964         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
965         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
966         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
967 
968         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
969         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
970         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
971         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
972         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
973 
974         setOperationAction(ISD::LOAD, VT, Custom);
975         setOperationAction(ISD::STORE, VT, Custom);
976         setOperationAction(ISD::MLOAD, VT, Custom);
977         setOperationAction(ISD::MSTORE, VT, Custom);
978         setOperationAction(ISD::MGATHER, VT, Custom);
979         setOperationAction(ISD::MSCATTER, VT, Custom);
980 
981         setOperationAction(ISD::VP_LOAD, VT, Custom);
982         setOperationAction(ISD::VP_STORE, VT, Custom);
983         setOperationAction(ISD::VP_GATHER, VT, Custom);
984         setOperationAction(ISD::VP_SCATTER, VT, Custom);
985 
986         setOperationAction(ISD::FADD, VT, Custom);
987         setOperationAction(ISD::FSUB, VT, Custom);
988         setOperationAction(ISD::FMUL, VT, Custom);
989         setOperationAction(ISD::FDIV, VT, Custom);
990         setOperationAction(ISD::FNEG, VT, Custom);
991         setOperationAction(ISD::FABS, VT, Custom);
992         setOperationAction(ISD::FCOPYSIGN, VT, Custom);
993         setOperationAction(ISD::FSQRT, VT, Custom);
994         setOperationAction(ISD::FMA, VT, Custom);
995         setOperationAction(ISD::FMINNUM, VT, Custom);
996         setOperationAction(ISD::FMAXNUM, VT, Custom);
997 
998         setOperationAction(ISD::FP_ROUND, VT, Custom);
999         setOperationAction(ISD::FP_EXTEND, VT, Custom);
1000 
1001         setOperationAction(ISD::FTRUNC, VT, Custom);
1002         setOperationAction(ISD::FCEIL, VT, Custom);
1003         setOperationAction(ISD::FFLOOR, VT, Custom);
1004 
1005         for (auto CC : VFPCCToExpand)
1006           setCondCodeAction(CC, VT, Expand);
1007 
1008         setOperationAction(ISD::VSELECT, VT, Custom);
1009         setOperationAction(ISD::SELECT, VT, Custom);
1010         setOperationAction(ISD::SELECT_CC, VT, Expand);
1011 
1012         setOperationAction(ISD::BITCAST, VT, Custom);
1013 
1014         setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1015         setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1016         setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1017         setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1018 
1019         for (unsigned VPOpc : FloatingPointVPOps)
1020           setOperationAction(VPOpc, VT, Custom);
1021       }
1022 
1023       // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1024       setOperationAction(ISD::BITCAST, MVT::i8, Custom);
1025       setOperationAction(ISD::BITCAST, MVT::i16, Custom);
1026       setOperationAction(ISD::BITCAST, MVT::i32, Custom);
1027       setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1028       setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1029       setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1030       setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1031     }
1032   }
1033 
1034   // Function alignments.
1035   const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
1036   setMinFunctionAlignment(FunctionAlignment);
1037   setPrefFunctionAlignment(FunctionAlignment);
1038 
1039   setMinimumJumpTableEntries(5);
1040 
1041   // Jumps are expensive, compared to logic
1042   setJumpIsExpensive();
1043 
1044   setTargetDAGCombine(ISD::ADD);
1045   setTargetDAGCombine(ISD::SUB);
1046   setTargetDAGCombine(ISD::AND);
1047   setTargetDAGCombine(ISD::OR);
1048   setTargetDAGCombine(ISD::XOR);
1049   setTargetDAGCombine(ISD::ANY_EXTEND);
1050   if (Subtarget.hasStdExtF()) {
1051     setTargetDAGCombine(ISD::ZERO_EXTEND);
1052     setTargetDAGCombine(ISD::FP_TO_SINT);
1053     setTargetDAGCombine(ISD::FP_TO_UINT);
1054   }
1055   if (Subtarget.hasVInstructions()) {
1056     setTargetDAGCombine(ISD::FCOPYSIGN);
1057     setTargetDAGCombine(ISD::MGATHER);
1058     setTargetDAGCombine(ISD::MSCATTER);
1059     setTargetDAGCombine(ISD::VP_GATHER);
1060     setTargetDAGCombine(ISD::VP_SCATTER);
1061     setTargetDAGCombine(ISD::SRA);
1062     setTargetDAGCombine(ISD::SRL);
1063     setTargetDAGCombine(ISD::SHL);
1064     setTargetDAGCombine(ISD::STORE);
1065   }
1066 }
1067 
1068 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
1069                                             LLVMContext &Context,
1070                                             EVT VT) const {
1071   if (!VT.isVector())
1072     return getPointerTy(DL);
1073   if (Subtarget.hasVInstructions() &&
1074       (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
1075     return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
1076   return VT.changeVectorElementTypeToInteger();
1077 }
1078 
1079 MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
1080   return Subtarget.getXLenVT();
1081 }
1082 
1083 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1084                                              const CallInst &I,
1085                                              MachineFunction &MF,
1086                                              unsigned Intrinsic) const {
1087   auto &DL = I.getModule()->getDataLayout();
1088   switch (Intrinsic) {
1089   default:
1090     return false;
1091   case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1092   case Intrinsic::riscv_masked_atomicrmw_add_i32:
1093   case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1094   case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1095   case Intrinsic::riscv_masked_atomicrmw_max_i32:
1096   case Intrinsic::riscv_masked_atomicrmw_min_i32:
1097   case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1098   case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1099   case Intrinsic::riscv_masked_cmpxchg_i32: {
1100     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
1101     Info.opc = ISD::INTRINSIC_W_CHAIN;
1102     Info.memVT = MVT::getVT(PtrTy->getElementType());
1103     Info.ptrVal = I.getArgOperand(0);
1104     Info.offset = 0;
1105     Info.align = Align(4);
1106     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1107                  MachineMemOperand::MOVolatile;
1108     return true;
1109   }
1110   case Intrinsic::riscv_masked_strided_load:
1111     Info.opc = ISD::INTRINSIC_W_CHAIN;
1112     Info.ptrVal = I.getArgOperand(1);
1113     Info.memVT = getValueType(DL, I.getType()->getScalarType());
1114     Info.align = Align(DL.getTypeSizeInBits(I.getType()->getScalarType()) / 8);
1115     Info.size = MemoryLocation::UnknownSize;
1116     Info.flags |= MachineMemOperand::MOLoad;
1117     return true;
1118   case Intrinsic::riscv_masked_strided_store:
1119     Info.opc = ISD::INTRINSIC_VOID;
1120     Info.ptrVal = I.getArgOperand(1);
1121     Info.memVT =
1122         getValueType(DL, I.getArgOperand(0)->getType()->getScalarType());
1123     Info.align = Align(
1124         DL.getTypeSizeInBits(I.getArgOperand(0)->getType()->getScalarType()) /
1125         8);
1126     Info.size = MemoryLocation::UnknownSize;
1127     Info.flags |= MachineMemOperand::MOStore;
1128     return true;
1129   }
1130 }
1131 
1132 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1133                                                 const AddrMode &AM, Type *Ty,
1134                                                 unsigned AS,
1135                                                 Instruction *I) const {
1136   // No global is ever allowed as a base.
1137   if (AM.BaseGV)
1138     return false;
1139 
1140   // Require a 12-bit signed offset.
1141   if (!isInt<12>(AM.BaseOffs))
1142     return false;
1143 
1144   switch (AM.Scale) {
1145   case 0: // "r+i" or just "i", depending on HasBaseReg.
1146     break;
1147   case 1:
1148     if (!AM.HasBaseReg) // allow "r+i".
1149       break;
1150     return false; // disallow "r+r" or "r+r+i".
1151   default:
1152     return false;
1153   }
1154 
1155   return true;
1156 }
1157 
1158 bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1159   return isInt<12>(Imm);
1160 }
1161 
1162 bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1163   return isInt<12>(Imm);
1164 }
1165 
1166 // On RV32, 64-bit integers are split into their high and low parts and held
1167 // in two different registers, so the trunc is free since the low register can
1168 // just be used.
1169 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
1170   if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
1171     return false;
1172   unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1173   unsigned DestBits = DstTy->getPrimitiveSizeInBits();
1174   return (SrcBits == 64 && DestBits == 32);
1175 }
1176 
1177 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
1178   if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
1179       !SrcVT.isInteger() || !DstVT.isInteger())
1180     return false;
1181   unsigned SrcBits = SrcVT.getSizeInBits();
1182   unsigned DestBits = DstVT.getSizeInBits();
1183   return (SrcBits == 64 && DestBits == 32);
1184 }
1185 
1186 bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
1187   // Zexts are free if they can be combined with a load.
1188   // Don't advertise i32->i64 zextload as being free for RV64. It interacts
1189   // poorly with type legalization of compares preferring sext.
1190   if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
1191     EVT MemVT = LD->getMemoryVT();
1192     if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1193         (LD->getExtensionType() == ISD::NON_EXTLOAD ||
1194          LD->getExtensionType() == ISD::ZEXTLOAD))
1195       return true;
1196   }
1197 
1198   return TargetLowering::isZExtFree(Val, VT2);
1199 }
1200 
1201 bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
1202   return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1203 }
1204 
1205 bool RISCVTargetLowering::isCheapToSpeculateCttz() const {
1206   return Subtarget.hasStdExtZbb();
1207 }
1208 
1209 bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
1210   return Subtarget.hasStdExtZbb();
1211 }
1212 
1213 bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
1214   EVT VT = Y.getValueType();
1215 
1216   // FIXME: Support vectors once we have tests.
1217   if (VT.isVector())
1218     return false;
1219 
1220   return Subtarget.hasStdExtZbb() && !isa<ConstantSDNode>(Y);
1221 }
1222 
1223 /// Check if sinking \p I's operands to I's basic block is profitable, because
1224 /// the operands can be folded into a target instruction, e.g.
1225 /// splats of scalars can fold into vector instructions.
1226 bool RISCVTargetLowering::shouldSinkOperands(
1227     Instruction *I, SmallVectorImpl<Use *> &Ops) const {
1228   using namespace llvm::PatternMatch;
1229 
1230   if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
1231     return false;
1232 
1233   auto IsSinker = [&](Instruction *I, int Operand) {
1234     switch (I->getOpcode()) {
1235     case Instruction::Add:
1236     case Instruction::Sub:
1237     case Instruction::Mul:
1238     case Instruction::And:
1239     case Instruction::Or:
1240     case Instruction::Xor:
1241     case Instruction::FAdd:
1242     case Instruction::FSub:
1243     case Instruction::FMul:
1244     case Instruction::FDiv:
1245     case Instruction::ICmp:
1246     case Instruction::FCmp:
1247       return true;
1248     case Instruction::Shl:
1249     case Instruction::LShr:
1250     case Instruction::AShr:
1251     case Instruction::UDiv:
1252     case Instruction::SDiv:
1253     case Instruction::URem:
1254     case Instruction::SRem:
1255       return Operand == 1;
1256     case Instruction::Call:
1257       if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1258         switch (II->getIntrinsicID()) {
1259         case Intrinsic::fma:
1260           return Operand == 0 || Operand == 1;
1261         default:
1262           return false;
1263         }
1264       }
1265       return false;
1266     default:
1267       return false;
1268     }
1269   };
1270 
1271   for (auto OpIdx : enumerate(I->operands())) {
1272     if (!IsSinker(I, OpIdx.index()))
1273       continue;
1274 
1275     Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
1276     // Make sure we are not already sinking this operand
1277     if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
1278       continue;
1279 
1280     // We are looking for a splat that can be sunk.
1281     if (!match(Op, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
1282                              m_Undef(), m_ZeroMask())))
1283       continue;
1284 
1285     // All uses of the shuffle should be sunk to avoid duplicating it across gpr
1286     // and vector registers
1287     for (Use &U : Op->uses()) {
1288       Instruction *Insn = cast<Instruction>(U.getUser());
1289       if (!IsSinker(Insn, U.getOperandNo()))
1290         return false;
1291     }
1292 
1293     Ops.push_back(&Op->getOperandUse(0));
1294     Ops.push_back(&OpIdx.value());
1295   }
1296   return true;
1297 }
1298 
1299 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1300                                        bool ForCodeSize) const {
1301   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1302   if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
1303     return false;
1304   if (VT == MVT::f32 && !Subtarget.hasStdExtF())
1305     return false;
1306   if (VT == MVT::f64 && !Subtarget.hasStdExtD())
1307     return false;
1308   return Imm.isZero();
1309 }
1310 
1311 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
1312   return (VT == MVT::f16 && Subtarget.hasStdExtZfh()) ||
1313          (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
1314          (VT == MVT::f64 && Subtarget.hasStdExtD());
1315 }
1316 
1317 MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1318                                                       CallingConv::ID CC,
1319                                                       EVT VT) const {
1320   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1321   // We might still end up using a GPR but that will be decided based on ABI.
1322   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1323   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1324     return MVT::f32;
1325 
1326   return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1327 }
1328 
1329 unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1330                                                            CallingConv::ID CC,
1331                                                            EVT VT) const {
1332   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1333   // We might still end up using a GPR but that will be decided based on ABI.
1334   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1335   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1336     return 1;
1337 
1338   return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1339 }
1340 
1341 // Changes the condition code and swaps operands if necessary, so the SetCC
1342 // operation matches one of the comparisons supported directly by branches
1343 // in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
1344 // with 1/-1.
1345 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1346                                     ISD::CondCode &CC, SelectionDAG &DAG) {
1347   // Convert X > -1 to X >= 0.
1348   if (CC == ISD::SETGT && isAllOnesConstant(RHS)) {
1349     RHS = DAG.getConstant(0, DL, RHS.getValueType());
1350     CC = ISD::SETGE;
1351     return;
1352   }
1353   // Convert X < 1 to 0 >= X.
1354   if (CC == ISD::SETLT && isOneConstant(RHS)) {
1355     RHS = LHS;
1356     LHS = DAG.getConstant(0, DL, RHS.getValueType());
1357     CC = ISD::SETGE;
1358     return;
1359   }
1360 
1361   switch (CC) {
1362   default:
1363     break;
1364   case ISD::SETGT:
1365   case ISD::SETLE:
1366   case ISD::SETUGT:
1367   case ISD::SETULE:
1368     CC = ISD::getSetCCSwappedOperands(CC);
1369     std::swap(LHS, RHS);
1370     break;
1371   }
1372 }
1373 
1374 RISCVII::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
1375   assert(VT.isScalableVector() && "Expecting a scalable vector type");
1376   unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
1377   if (VT.getVectorElementType() == MVT::i1)
1378     KnownSize *= 8;
1379 
1380   switch (KnownSize) {
1381   default:
1382     llvm_unreachable("Invalid LMUL.");
1383   case 8:
1384     return RISCVII::VLMUL::LMUL_F8;
1385   case 16:
1386     return RISCVII::VLMUL::LMUL_F4;
1387   case 32:
1388     return RISCVII::VLMUL::LMUL_F2;
1389   case 64:
1390     return RISCVII::VLMUL::LMUL_1;
1391   case 128:
1392     return RISCVII::VLMUL::LMUL_2;
1393   case 256:
1394     return RISCVII::VLMUL::LMUL_4;
1395   case 512:
1396     return RISCVII::VLMUL::LMUL_8;
1397   }
1398 }
1399 
1400 unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVII::VLMUL LMul) {
1401   switch (LMul) {
1402   default:
1403     llvm_unreachable("Invalid LMUL.");
1404   case RISCVII::VLMUL::LMUL_F8:
1405   case RISCVII::VLMUL::LMUL_F4:
1406   case RISCVII::VLMUL::LMUL_F2:
1407   case RISCVII::VLMUL::LMUL_1:
1408     return RISCV::VRRegClassID;
1409   case RISCVII::VLMUL::LMUL_2:
1410     return RISCV::VRM2RegClassID;
1411   case RISCVII::VLMUL::LMUL_4:
1412     return RISCV::VRM4RegClassID;
1413   case RISCVII::VLMUL::LMUL_8:
1414     return RISCV::VRM8RegClassID;
1415   }
1416 }
1417 
1418 unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
1419   RISCVII::VLMUL LMUL = getLMUL(VT);
1420   if (LMUL == RISCVII::VLMUL::LMUL_F8 ||
1421       LMUL == RISCVII::VLMUL::LMUL_F4 ||
1422       LMUL == RISCVII::VLMUL::LMUL_F2 ||
1423       LMUL == RISCVII::VLMUL::LMUL_1) {
1424     static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
1425                   "Unexpected subreg numbering");
1426     return RISCV::sub_vrm1_0 + Index;
1427   }
1428   if (LMUL == RISCVII::VLMUL::LMUL_2) {
1429     static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
1430                   "Unexpected subreg numbering");
1431     return RISCV::sub_vrm2_0 + Index;
1432   }
1433   if (LMUL == RISCVII::VLMUL::LMUL_4) {
1434     static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
1435                   "Unexpected subreg numbering");
1436     return RISCV::sub_vrm4_0 + Index;
1437   }
1438   llvm_unreachable("Invalid vector type.");
1439 }
1440 
1441 unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
1442   if (VT.getVectorElementType() == MVT::i1)
1443     return RISCV::VRRegClassID;
1444   return getRegClassIDForLMUL(getLMUL(VT));
1445 }
1446 
1447 // Attempt to decompose a subvector insert/extract between VecVT and
1448 // SubVecVT via subregister indices. Returns the subregister index that
1449 // can perform the subvector insert/extract with the given element index, as
1450 // well as the index corresponding to any leftover subvectors that must be
1451 // further inserted/extracted within the register class for SubVecVT.
1452 std::pair<unsigned, unsigned>
1453 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1454     MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
1455     const RISCVRegisterInfo *TRI) {
1456   static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
1457                  RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
1458                  RISCV::VRM2RegClassID > RISCV::VRRegClassID),
1459                 "Register classes not ordered");
1460   unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
1461   unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
1462   // Try to compose a subregister index that takes us from the incoming
1463   // LMUL>1 register class down to the outgoing one. At each step we half
1464   // the LMUL:
1465   //   nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
1466   // Note that this is not guaranteed to find a subregister index, such as
1467   // when we are extracting from one VR type to another.
1468   unsigned SubRegIdx = RISCV::NoSubRegister;
1469   for (const unsigned RCID :
1470        {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
1471     if (VecRegClassID > RCID && SubRegClassID <= RCID) {
1472       VecVT = VecVT.getHalfNumVectorElementsVT();
1473       bool IsHi =
1474           InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
1475       SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
1476                                             getSubregIndexByMVT(VecVT, IsHi));
1477       if (IsHi)
1478         InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
1479     }
1480   return {SubRegIdx, InsertExtractIdx};
1481 }
1482 
1483 // Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
1484 // stores for those types.
1485 bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
1486   return !Subtarget.useRVVForFixedLengthVectors() ||
1487          (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
1488 }
1489 
1490 bool RISCVTargetLowering::isLegalElementTypeForRVV(Type *ScalarTy) const {
1491   if (ScalarTy->isPointerTy())
1492     return true;
1493 
1494   if (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
1495       ScalarTy->isIntegerTy(32))
1496     return true;
1497 
1498   if (ScalarTy->isIntegerTy(64))
1499     return Subtarget.hasVInstructionsI64();
1500 
1501   if (ScalarTy->isHalfTy())
1502     return Subtarget.hasVInstructionsF16();
1503   if (ScalarTy->isFloatTy())
1504     return Subtarget.hasVInstructionsF32();
1505   if (ScalarTy->isDoubleTy())
1506     return Subtarget.hasVInstructionsF64();
1507 
1508   return false;
1509 }
1510 
1511 static bool useRVVForFixedLengthVectorVT(MVT VT,
1512                                          const RISCVSubtarget &Subtarget) {
1513   assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
1514   if (!Subtarget.useRVVForFixedLengthVectors())
1515     return false;
1516 
1517   // We only support a set of vector types with a consistent maximum fixed size
1518   // across all supported vector element types to avoid legalization issues.
1519   // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
1520   // fixed-length vector type we support is 1024 bytes.
1521   if (VT.getFixedSizeInBits() > 1024 * 8)
1522     return false;
1523 
1524   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1525 
1526   MVT EltVT = VT.getVectorElementType();
1527 
1528   // Don't use RVV for vectors we cannot scalarize if required.
1529   switch (EltVT.SimpleTy) {
1530   // i1 is supported but has different rules.
1531   default:
1532     return false;
1533   case MVT::i1:
1534     // Masks can only use a single register.
1535     if (VT.getVectorNumElements() > MinVLen)
1536       return false;
1537     MinVLen /= 8;
1538     break;
1539   case MVT::i8:
1540   case MVT::i16:
1541   case MVT::i32:
1542     break;
1543   case MVT::i64:
1544     if (!Subtarget.hasVInstructionsI64())
1545       return false;
1546     break;
1547   case MVT::f16:
1548     if (!Subtarget.hasVInstructionsF16())
1549       return false;
1550     break;
1551   case MVT::f32:
1552     if (!Subtarget.hasVInstructionsF32())
1553       return false;
1554     break;
1555   case MVT::f64:
1556     if (!Subtarget.hasVInstructionsF64())
1557       return false;
1558     break;
1559   }
1560 
1561   // Reject elements larger than ELEN.
1562   if (EltVT.getSizeInBits() > Subtarget.getMaxELENForFixedLengthVectors())
1563     return false;
1564 
1565   unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
1566   // Don't use RVV for types that don't fit.
1567   if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
1568     return false;
1569 
1570   // TODO: Perhaps an artificial restriction, but worth having whilst getting
1571   // the base fixed length RVV support in place.
1572   if (!VT.isPow2VectorType())
1573     return false;
1574 
1575   return true;
1576 }
1577 
1578 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
1579   return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
1580 }
1581 
1582 // Return the largest legal scalable vector type that matches VT's element type.
1583 static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
1584                                             const RISCVSubtarget &Subtarget) {
1585   // This may be called before legal types are setup.
1586   assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
1587           useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
1588          "Expected legal fixed length vector!");
1589 
1590   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1591   unsigned MaxELen = Subtarget.getMaxELENForFixedLengthVectors();
1592 
1593   MVT EltVT = VT.getVectorElementType();
1594   switch (EltVT.SimpleTy) {
1595   default:
1596     llvm_unreachable("unexpected element type for RVV container");
1597   case MVT::i1:
1598   case MVT::i8:
1599   case MVT::i16:
1600   case MVT::i32:
1601   case MVT::i64:
1602   case MVT::f16:
1603   case MVT::f32:
1604   case MVT::f64: {
1605     // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
1606     // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
1607     // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
1608     unsigned NumElts =
1609         (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
1610     NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
1611     assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
1612     return MVT::getScalableVectorVT(EltVT, NumElts);
1613   }
1614   }
1615 }
1616 
1617 static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
1618                                             const RISCVSubtarget &Subtarget) {
1619   return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
1620                                           Subtarget);
1621 }
1622 
1623 MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
1624   return ::getContainerForFixedLengthVector(*this, VT, getSubtarget());
1625 }
1626 
1627 // Grow V to consume an entire RVV register.
1628 static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1629                                        const RISCVSubtarget &Subtarget) {
1630   assert(VT.isScalableVector() &&
1631          "Expected to convert into a scalable vector!");
1632   assert(V.getValueType().isFixedLengthVector() &&
1633          "Expected a fixed length vector operand!");
1634   SDLoc DL(V);
1635   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1636   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
1637 }
1638 
1639 // Shrink V so it's just big enough to maintain a VT's worth of data.
1640 static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1641                                          const RISCVSubtarget &Subtarget) {
1642   assert(VT.isFixedLengthVector() &&
1643          "Expected to convert into a fixed length vector!");
1644   assert(V.getValueType().isScalableVector() &&
1645          "Expected a scalable vector operand!");
1646   SDLoc DL(V);
1647   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1648   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
1649 }
1650 
1651 // Gets the two common "VL" operands: an all-ones mask and the vector length.
1652 // VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
1653 // the vector type that it is contained in.
1654 static std::pair<SDValue, SDValue>
1655 getDefaultVLOps(MVT VecVT, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
1656                 const RISCVSubtarget &Subtarget) {
1657   assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
1658   MVT XLenVT = Subtarget.getXLenVT();
1659   SDValue VL = VecVT.isFixedLengthVector()
1660                    ? DAG.getConstant(VecVT.getVectorNumElements(), DL, XLenVT)
1661                    : DAG.getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT);
1662   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
1663   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
1664   return {Mask, VL};
1665 }
1666 
1667 // As above but assuming the given type is a scalable vector type.
1668 static std::pair<SDValue, SDValue>
1669 getDefaultScalableVLOps(MVT VecVT, SDLoc DL, SelectionDAG &DAG,
1670                         const RISCVSubtarget &Subtarget) {
1671   assert(VecVT.isScalableVector() && "Expecting a scalable vector");
1672   return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
1673 }
1674 
1675 // The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
1676 // of either is (currently) supported. This can get us into an infinite loop
1677 // where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
1678 // as a ..., etc.
1679 // Until either (or both) of these can reliably lower any node, reporting that
1680 // we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
1681 // the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
1682 // which is not desirable.
1683 bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
1684     EVT VT, unsigned DefinedValues) const {
1685   return false;
1686 }
1687 
1688 bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
1689   // Only splats are currently supported.
1690   if (ShuffleVectorSDNode::isSplatMask(M.data(), VT))
1691     return true;
1692 
1693   return false;
1694 }
1695 
1696 static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
1697                                   const RISCVSubtarget &Subtarget) {
1698   // RISCV FP-to-int conversions saturate to the destination register size, but
1699   // don't produce 0 for nan. We can use a conversion instruction and fix the
1700   // nan case with a compare and a select.
1701   SDValue Src = Op.getOperand(0);
1702 
1703   EVT DstVT = Op.getValueType();
1704   EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1705 
1706   bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
1707   unsigned Opc;
1708   if (SatVT == DstVT)
1709     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
1710   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
1711     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
1712   else
1713     return SDValue();
1714   // FIXME: Support other SatVTs by clamping before or after the conversion.
1715 
1716   SDLoc DL(Op);
1717   SDValue FpToInt = DAG.getNode(
1718       Opc, DL, DstVT, Src,
1719       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
1720 
1721   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
1722   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
1723 }
1724 
1725 // Expand vector FTRUNC, FCEIL, and FFLOOR by converting to the integer domain
1726 // and back. Taking care to avoid converting values that are nan or already
1727 // correct.
1728 // TODO: Floor and ceil could be shorter by changing rounding mode, but we don't
1729 // have FRM dependencies modeled yet.
1730 static SDValue lowerFTRUNC_FCEIL_FFLOOR(SDValue Op, SelectionDAG &DAG) {
1731   MVT VT = Op.getSimpleValueType();
1732   assert(VT.isVector() && "Unexpected type");
1733 
1734   SDLoc DL(Op);
1735 
1736   // Freeze the source since we are increasing the number of uses.
1737   SDValue Src = DAG.getNode(ISD::FREEZE, DL, VT, Op.getOperand(0));
1738 
1739   // Truncate to integer and convert back to FP.
1740   MVT IntVT = VT.changeVectorElementTypeToInteger();
1741   SDValue Truncated = DAG.getNode(ISD::FP_TO_SINT, DL, IntVT, Src);
1742   Truncated = DAG.getNode(ISD::SINT_TO_FP, DL, VT, Truncated);
1743 
1744   MVT SetccVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
1745 
1746   if (Op.getOpcode() == ISD::FCEIL) {
1747     // If the truncated value is the greater than or equal to the original
1748     // value, we've computed the ceil. Otherwise, we went the wrong way and
1749     // need to increase by 1.
1750     // FIXME: This should use a masked operation. Handle here or in isel?
1751     SDValue Adjust = DAG.getNode(ISD::FADD, DL, VT, Truncated,
1752                                  DAG.getConstantFP(1.0, DL, VT));
1753     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOLT);
1754     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1755   } else if (Op.getOpcode() == ISD::FFLOOR) {
1756     // If the truncated value is the less than or equal to the original value,
1757     // we've computed the floor. Otherwise, we went the wrong way and need to
1758     // decrease by 1.
1759     // FIXME: This should use a masked operation. Handle here or in isel?
1760     SDValue Adjust = DAG.getNode(ISD::FSUB, DL, VT, Truncated,
1761                                  DAG.getConstantFP(1.0, DL, VT));
1762     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOGT);
1763     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1764   }
1765 
1766   // Restore the original sign so that -0.0 is preserved.
1767   Truncated = DAG.getNode(ISD::FCOPYSIGN, DL, VT, Truncated, Src);
1768 
1769   // Determine the largest integer that can be represented exactly. This and
1770   // values larger than it don't have any fractional bits so don't need to
1771   // be converted.
1772   const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
1773   unsigned Precision = APFloat::semanticsPrecision(FltSem);
1774   APFloat MaxVal = APFloat(FltSem);
1775   MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
1776                           /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
1777   SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
1778 
1779   // If abs(Src) was larger than MaxVal or nan, keep it.
1780   SDValue Abs = DAG.getNode(ISD::FABS, DL, VT, Src);
1781   SDValue Setcc = DAG.getSetCC(DL, SetccVT, Abs, MaxValNode, ISD::SETOLT);
1782   return DAG.getSelect(DL, VT, Setcc, Truncated, Src);
1783 }
1784 
1785 static SDValue lowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG,
1786                                  const RISCVSubtarget &Subtarget) {
1787   MVT VT = Op.getSimpleValueType();
1788   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1789 
1790   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1791 
1792   SDLoc DL(Op);
1793   SDValue Mask, VL;
1794   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1795 
1796   unsigned Opc =
1797       VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
1798   SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, Op.getOperand(0), VL);
1799   return convertFromScalableVector(VT, Splat, DAG, Subtarget);
1800 }
1801 
1802 struct VIDSequence {
1803   int64_t StepNumerator;
1804   unsigned StepDenominator;
1805   int64_t Addend;
1806 };
1807 
1808 // Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
1809 // to the (non-zero) step S and start value X. This can be then lowered as the
1810 // RVV sequence (VID * S) + X, for example.
1811 // The step S is represented as an integer numerator divided by a positive
1812 // denominator. Note that the implementation currently only identifies
1813 // sequences in which either the numerator is +/- 1 or the denominator is 1. It
1814 // cannot detect 2/3, for example.
1815 // Note that this method will also match potentially unappealing index
1816 // sequences, like <i32 0, i32 50939494>, however it is left to the caller to
1817 // determine whether this is worth generating code for.
1818 static Optional<VIDSequence> isSimpleVIDSequence(SDValue Op) {
1819   unsigned NumElts = Op.getNumOperands();
1820   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
1821   if (!Op.getValueType().isInteger())
1822     return None;
1823 
1824   Optional<unsigned> SeqStepDenom;
1825   Optional<int64_t> SeqStepNum, SeqAddend;
1826   Optional<std::pair<uint64_t, unsigned>> PrevElt;
1827   unsigned EltSizeInBits = Op.getValueType().getScalarSizeInBits();
1828   for (unsigned Idx = 0; Idx < NumElts; Idx++) {
1829     // Assume undef elements match the sequence; we just have to be careful
1830     // when interpolating across them.
1831     if (Op.getOperand(Idx).isUndef())
1832       continue;
1833     // The BUILD_VECTOR must be all constants.
1834     if (!isa<ConstantSDNode>(Op.getOperand(Idx)))
1835       return None;
1836 
1837     uint64_t Val = Op.getConstantOperandVal(Idx) &
1838                    maskTrailingOnes<uint64_t>(EltSizeInBits);
1839 
1840     if (PrevElt) {
1841       // Calculate the step since the last non-undef element, and ensure
1842       // it's consistent across the entire sequence.
1843       unsigned IdxDiff = Idx - PrevElt->second;
1844       int64_t ValDiff = SignExtend64(Val - PrevElt->first, EltSizeInBits);
1845 
1846       // A zero-value value difference means that we're somewhere in the middle
1847       // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
1848       // step change before evaluating the sequence.
1849       if (ValDiff != 0) {
1850         int64_t Remainder = ValDiff % IdxDiff;
1851         // Normalize the step if it's greater than 1.
1852         if (Remainder != ValDiff) {
1853           // The difference must cleanly divide the element span.
1854           if (Remainder != 0)
1855             return None;
1856           ValDiff /= IdxDiff;
1857           IdxDiff = 1;
1858         }
1859 
1860         if (!SeqStepNum)
1861           SeqStepNum = ValDiff;
1862         else if (ValDiff != SeqStepNum)
1863           return None;
1864 
1865         if (!SeqStepDenom)
1866           SeqStepDenom = IdxDiff;
1867         else if (IdxDiff != *SeqStepDenom)
1868           return None;
1869       }
1870     }
1871 
1872     // Record and/or check any addend.
1873     if (SeqStepNum && SeqStepDenom) {
1874       uint64_t ExpectedVal =
1875           (int64_t)(Idx * (uint64_t)*SeqStepNum) / *SeqStepDenom;
1876       int64_t Addend = SignExtend64(Val - ExpectedVal, EltSizeInBits);
1877       if (!SeqAddend)
1878         SeqAddend = Addend;
1879       else if (SeqAddend != Addend)
1880         return None;
1881     }
1882 
1883     // Record this non-undef element for later.
1884     if (!PrevElt || PrevElt->first != Val)
1885       PrevElt = std::make_pair(Val, Idx);
1886   }
1887   // We need to have logged both a step and an addend for this to count as
1888   // a legal index sequence.
1889   if (!SeqStepNum || !SeqStepDenom || !SeqAddend)
1890     return None;
1891 
1892   return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
1893 }
1894 
1895 static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
1896                                  const RISCVSubtarget &Subtarget) {
1897   MVT VT = Op.getSimpleValueType();
1898   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1899 
1900   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1901 
1902   SDLoc DL(Op);
1903   SDValue Mask, VL;
1904   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1905 
1906   MVT XLenVT = Subtarget.getXLenVT();
1907   unsigned NumElts = Op.getNumOperands();
1908 
1909   if (VT.getVectorElementType() == MVT::i1) {
1910     if (ISD::isBuildVectorAllZeros(Op.getNode())) {
1911       SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
1912       return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
1913     }
1914 
1915     if (ISD::isBuildVectorAllOnes(Op.getNode())) {
1916       SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
1917       return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
1918     }
1919 
1920     // Lower constant mask BUILD_VECTORs via an integer vector type, in
1921     // scalar integer chunks whose bit-width depends on the number of mask
1922     // bits and XLEN.
1923     // First, determine the most appropriate scalar integer type to use. This
1924     // is at most XLenVT, but may be shrunk to a smaller vector element type
1925     // according to the size of the final vector - use i8 chunks rather than
1926     // XLenVT if we're producing a v8i1. This results in more consistent
1927     // codegen across RV32 and RV64.
1928     unsigned NumViaIntegerBits =
1929         std::min(std::max(NumElts, 8u), Subtarget.getXLen());
1930     NumViaIntegerBits = std::min(NumViaIntegerBits,
1931                                  Subtarget.getMaxELENForFixedLengthVectors());
1932     if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
1933       // If we have to use more than one INSERT_VECTOR_ELT then this
1934       // optimization is likely to increase code size; avoid peforming it in
1935       // such a case. We can use a load from a constant pool in this case.
1936       if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
1937         return SDValue();
1938       // Now we can create our integer vector type. Note that it may be larger
1939       // than the resulting mask type: v4i1 would use v1i8 as its integer type.
1940       MVT IntegerViaVecVT =
1941           MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
1942                            divideCeil(NumElts, NumViaIntegerBits));
1943 
1944       uint64_t Bits = 0;
1945       unsigned BitPos = 0, IntegerEltIdx = 0;
1946       SDValue Vec = DAG.getUNDEF(IntegerViaVecVT);
1947 
1948       for (unsigned I = 0; I < NumElts; I++, BitPos++) {
1949         // Once we accumulate enough bits to fill our scalar type, insert into
1950         // our vector and clear our accumulated data.
1951         if (I != 0 && I % NumViaIntegerBits == 0) {
1952           if (NumViaIntegerBits <= 32)
1953             Bits = SignExtend64(Bits, 32);
1954           SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
1955           Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec,
1956                             Elt, DAG.getConstant(IntegerEltIdx, DL, XLenVT));
1957           Bits = 0;
1958           BitPos = 0;
1959           IntegerEltIdx++;
1960         }
1961         SDValue V = Op.getOperand(I);
1962         bool BitValue = !V.isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
1963         Bits |= ((uint64_t)BitValue << BitPos);
1964       }
1965 
1966       // Insert the (remaining) scalar value into position in our integer
1967       // vector type.
1968       if (NumViaIntegerBits <= 32)
1969         Bits = SignExtend64(Bits, 32);
1970       SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
1971       Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, Elt,
1972                         DAG.getConstant(IntegerEltIdx, DL, XLenVT));
1973 
1974       if (NumElts < NumViaIntegerBits) {
1975         // If we're producing a smaller vector than our minimum legal integer
1976         // type, bitcast to the equivalent (known-legal) mask type, and extract
1977         // our final mask.
1978         assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
1979         Vec = DAG.getBitcast(MVT::v8i1, Vec);
1980         Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
1981                           DAG.getConstant(0, DL, XLenVT));
1982       } else {
1983         // Else we must have produced an integer type with the same size as the
1984         // mask type; bitcast for the final result.
1985         assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
1986         Vec = DAG.getBitcast(VT, Vec);
1987       }
1988 
1989       return Vec;
1990     }
1991 
1992     // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
1993     // vector type, we have a legal equivalently-sized i8 type, so we can use
1994     // that.
1995     MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
1996     SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
1997 
1998     SDValue WideVec;
1999     if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2000       // For a splat, perform a scalar truncate before creating the wider
2001       // vector.
2002       assert(Splat.getValueType() == XLenVT &&
2003              "Unexpected type for i1 splat value");
2004       Splat = DAG.getNode(ISD::AND, DL, XLenVT, Splat,
2005                           DAG.getConstant(1, DL, XLenVT));
2006       WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
2007     } else {
2008       SmallVector<SDValue, 8> Ops(Op->op_values());
2009       WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
2010       SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
2011       WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
2012     }
2013 
2014     return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
2015   }
2016 
2017   if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2018     unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
2019                                         : RISCVISD::VMV_V_X_VL;
2020     Splat = DAG.getNode(Opc, DL, ContainerVT, Splat, VL);
2021     return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2022   }
2023 
2024   // Try and match index sequences, which we can lower to the vid instruction
2025   // with optional modifications. An all-undef vector is matched by
2026   // getSplatValue, above.
2027   if (auto SimpleVID = isSimpleVIDSequence(Op)) {
2028     int64_t StepNumerator = SimpleVID->StepNumerator;
2029     unsigned StepDenominator = SimpleVID->StepDenominator;
2030     int64_t Addend = SimpleVID->Addend;
2031 
2032     assert(StepNumerator != 0 && "Invalid step");
2033     bool Negate = false;
2034     int64_t SplatStepVal = StepNumerator;
2035     unsigned StepOpcode = ISD::MUL;
2036     if (StepNumerator != 1) {
2037       if (isPowerOf2_64(std::abs(StepNumerator))) {
2038         Negate = StepNumerator < 0;
2039         StepOpcode = ISD::SHL;
2040         SplatStepVal = Log2_64(std::abs(StepNumerator));
2041       }
2042     }
2043 
2044     // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
2045     // threshold since it's the immediate value many RVV instructions accept.
2046     // There is no vmul.vi instruction so ensure multiply constant can fit in
2047     // a single addi instruction.
2048     if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
2049          (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
2050         isPowerOf2_32(StepDenominator) && isInt<5>(Addend)) {
2051       SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
2052       // Convert right out of the scalable type so we can use standard ISD
2053       // nodes for the rest of the computation. If we used scalable types with
2054       // these, we'd lose the fixed-length vector info and generate worse
2055       // vsetvli code.
2056       VID = convertFromScalableVector(VT, VID, DAG, Subtarget);
2057       if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
2058           (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
2059         SDValue SplatStep = DAG.getSplatVector(
2060             VT, DL, DAG.getConstant(SplatStepVal, DL, XLenVT));
2061         VID = DAG.getNode(StepOpcode, DL, VT, VID, SplatStep);
2062       }
2063       if (StepDenominator != 1) {
2064         SDValue SplatStep = DAG.getSplatVector(
2065             VT, DL, DAG.getConstant(Log2_64(StepDenominator), DL, XLenVT));
2066         VID = DAG.getNode(ISD::SRL, DL, VT, VID, SplatStep);
2067       }
2068       if (Addend != 0 || Negate) {
2069         SDValue SplatAddend =
2070             DAG.getSplatVector(VT, DL, DAG.getConstant(Addend, DL, XLenVT));
2071         VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VT, SplatAddend, VID);
2072       }
2073       return VID;
2074     }
2075   }
2076 
2077   // Attempt to detect "hidden" splats, which only reveal themselves as splats
2078   // when re-interpreted as a vector with a larger element type. For example,
2079   //   v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
2080   // could be instead splat as
2081   //   v2i32 = build_vector i32 0x00010000, i32 0x00010000
2082   // TODO: This optimization could also work on non-constant splats, but it
2083   // would require bit-manipulation instructions to construct the splat value.
2084   SmallVector<SDValue> Sequence;
2085   unsigned EltBitSize = VT.getScalarSizeInBits();
2086   const auto *BV = cast<BuildVectorSDNode>(Op);
2087   if (VT.isInteger() && EltBitSize < 64 &&
2088       ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
2089       BV->getRepeatedSequence(Sequence) &&
2090       (Sequence.size() * EltBitSize) <= 64) {
2091     unsigned SeqLen = Sequence.size();
2092     MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
2093     MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, NumElts / SeqLen);
2094     assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
2095             ViaIntVT == MVT::i64) &&
2096            "Unexpected sequence type");
2097 
2098     unsigned EltIdx = 0;
2099     uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
2100     uint64_t SplatValue = 0;
2101     // Construct the amalgamated value which can be splatted as this larger
2102     // vector type.
2103     for (const auto &SeqV : Sequence) {
2104       if (!SeqV.isUndef())
2105         SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
2106                        << (EltIdx * EltBitSize));
2107       EltIdx++;
2108     }
2109 
2110     // On RV64, sign-extend from 32 to 64 bits where possible in order to
2111     // achieve better constant materializion.
2112     if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
2113       SplatValue = SignExtend64(SplatValue, 32);
2114 
2115     // Since we can't introduce illegal i64 types at this stage, we can only
2116     // perform an i64 splat on RV32 if it is its own sign-extended value. That
2117     // way we can use RVV instructions to splat.
2118     assert((ViaIntVT.bitsLE(XLenVT) ||
2119             (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
2120            "Unexpected bitcast sequence");
2121     if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
2122       SDValue ViaVL =
2123           DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
2124       MVT ViaContainerVT =
2125           getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget);
2126       SDValue Splat =
2127           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
2128                       DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
2129       Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
2130       return DAG.getBitcast(VT, Splat);
2131     }
2132   }
2133 
2134   // Try and optimize BUILD_VECTORs with "dominant values" - these are values
2135   // which constitute a large proportion of the elements. In such cases we can
2136   // splat a vector with the dominant element and make up the shortfall with
2137   // INSERT_VECTOR_ELTs.
2138   // Note that this includes vectors of 2 elements by association. The
2139   // upper-most element is the "dominant" one, allowing us to use a splat to
2140   // "insert" the upper element, and an insert of the lower element at position
2141   // 0, which improves codegen.
2142   SDValue DominantValue;
2143   unsigned MostCommonCount = 0;
2144   DenseMap<SDValue, unsigned> ValueCounts;
2145   unsigned NumUndefElts =
2146       count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
2147 
2148   // Track the number of scalar loads we know we'd be inserting, estimated as
2149   // any non-zero floating-point constant. Other kinds of element are either
2150   // already in registers or are materialized on demand. The threshold at which
2151   // a vector load is more desirable than several scalar materializion and
2152   // vector-insertion instructions is not known.
2153   unsigned NumScalarLoads = 0;
2154 
2155   for (SDValue V : Op->op_values()) {
2156     if (V.isUndef())
2157       continue;
2158 
2159     ValueCounts.insert(std::make_pair(V, 0));
2160     unsigned &Count = ValueCounts[V];
2161 
2162     if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
2163       NumScalarLoads += !CFP->isExactlyValue(+0.0);
2164 
2165     // Is this value dominant? In case of a tie, prefer the highest element as
2166     // it's cheaper to insert near the beginning of a vector than it is at the
2167     // end.
2168     if (++Count >= MostCommonCount) {
2169       DominantValue = V;
2170       MostCommonCount = Count;
2171     }
2172   }
2173 
2174   assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
2175   unsigned NumDefElts = NumElts - NumUndefElts;
2176   unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
2177 
2178   // Don't perform this optimization when optimizing for size, since
2179   // materializing elements and inserting them tends to cause code bloat.
2180   if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
2181       ((MostCommonCount > DominantValueCountThreshold) ||
2182        (ValueCounts.size() <= Log2_32(NumDefElts)))) {
2183     // Start by splatting the most common element.
2184     SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
2185 
2186     DenseSet<SDValue> Processed{DominantValue};
2187     MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
2188     for (const auto &OpIdx : enumerate(Op->ops())) {
2189       const SDValue &V = OpIdx.value();
2190       if (V.isUndef() || !Processed.insert(V).second)
2191         continue;
2192       if (ValueCounts[V] == 1) {
2193         Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
2194                           DAG.getConstant(OpIdx.index(), DL, XLenVT));
2195       } else {
2196         // Blend in all instances of this value using a VSELECT, using a
2197         // mask where each bit signals whether that element is the one
2198         // we're after.
2199         SmallVector<SDValue> Ops;
2200         transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
2201           return DAG.getConstant(V == V1, DL, XLenVT);
2202         });
2203         Vec = DAG.getNode(ISD::VSELECT, DL, VT,
2204                           DAG.getBuildVector(SelMaskTy, DL, Ops),
2205                           DAG.getSplatBuildVector(VT, DL, V), Vec);
2206       }
2207     }
2208 
2209     return Vec;
2210   }
2211 
2212   return SDValue();
2213 }
2214 
2215 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Lo,
2216                                    SDValue Hi, SDValue VL, SelectionDAG &DAG) {
2217   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
2218     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
2219     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
2220     // If Hi constant is all the same sign bit as Lo, lower this as a custom
2221     // node in order to try and match RVV vector/scalar instructions.
2222     if ((LoC >> 31) == HiC)
2223       return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Lo, VL);
2224 
2225     // If vl is equal to VLMax and Hi constant is equal to Lo, we could use
2226     // vmv.v.x whose EEW = 32 to lower it.
2227     auto *Const = dyn_cast<ConstantSDNode>(VL);
2228     if (LoC == HiC && Const && Const->getSExtValue() == RISCV::VLMaxSentinel) {
2229       MVT InterVT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
2230       // TODO: if vl <= min(VLMAX), we can also do this. But we could not
2231       // access the subtarget here now.
2232       auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT, Lo, VL);
2233       return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
2234     }
2235   }
2236 
2237   // Fall back to a stack store and stride x0 vector load.
2238   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Lo, Hi, VL);
2239 }
2240 
2241 // Called by type legalization to handle splat of i64 on RV32.
2242 // FIXME: We can optimize this when the type has sign or zero bits in one
2243 // of the halves.
2244 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Scalar,
2245                                    SDValue VL, SelectionDAG &DAG) {
2246   assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
2247   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2248                            DAG.getConstant(0, DL, MVT::i32));
2249   SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2250                            DAG.getConstant(1, DL, MVT::i32));
2251   return splatPartsI64WithVL(DL, VT, Lo, Hi, VL, DAG);
2252 }
2253 
2254 // This function lowers a splat of a scalar operand Splat with the vector
2255 // length VL. It ensures the final sequence is type legal, which is useful when
2256 // lowering a splat after type legalization.
2257 static SDValue lowerScalarSplat(SDValue Scalar, SDValue VL, MVT VT, SDLoc DL,
2258                                 SelectionDAG &DAG,
2259                                 const RISCVSubtarget &Subtarget) {
2260   if (VT.isFloatingPoint()) {
2261     // If VL is 1, we could use vfmv.s.f.
2262     if (isOneConstant(VL))
2263       return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT),
2264                          Scalar, VL);
2265     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Scalar, VL);
2266   }
2267 
2268   MVT XLenVT = Subtarget.getXLenVT();
2269 
2270   // Simplest case is that the operand needs to be promoted to XLenVT.
2271   if (Scalar.getValueType().bitsLE(XLenVT)) {
2272     // If the operand is a constant, sign extend to increase our chances
2273     // of being able to use a .vi instruction. ANY_EXTEND would become a
2274     // a zero extend and the simm5 check in isel would fail.
2275     // FIXME: Should we ignore the upper bits in isel instead?
2276     unsigned ExtOpc =
2277         isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
2278     Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
2279     ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Scalar);
2280     // If VL is 1 and the scalar value won't benefit from immediate, we could
2281     // use vmv.s.x.
2282     if (isOneConstant(VL) &&
2283         (!Const || isNullConstant(Scalar) || !isInt<5>(Const->getSExtValue())))
2284       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
2285                          VL);
2286     return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Scalar, VL);
2287   }
2288 
2289   assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
2290          "Unexpected scalar for splat lowering!");
2291 
2292   if (isOneConstant(VL) && isNullConstant(Scalar))
2293     return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT),
2294                        DAG.getConstant(0, DL, XLenVT), VL);
2295 
2296   // Otherwise use the more complicated splatting algorithm.
2297   return splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
2298 }
2299 
2300 // Is the mask a slidedown that shifts in undefs.
2301 static int matchShuffleAsSlideDown(ArrayRef<int> Mask) {
2302   int Size = Mask.size();
2303 
2304   // Elements shifted in should be undef.
2305   auto CheckUndefs = [&](int Shift) {
2306     for (int i = Size - Shift; i != Size; ++i)
2307       if (Mask[i] >= 0)
2308         return false;
2309     return true;
2310   };
2311 
2312   // Elements should be shifted or undef.
2313   auto MatchShift = [&](int Shift) {
2314     for (int i = 0; i != Size - Shift; ++i)
2315        if (Mask[i] >= 0 && Mask[i] != Shift + i)
2316          return false;
2317     return true;
2318   };
2319 
2320   // Try all possible shifts.
2321   for (int Shift = 1; Shift != Size; ++Shift)
2322     if (CheckUndefs(Shift) && MatchShift(Shift))
2323       return Shift;
2324 
2325   // No match.
2326   return -1;
2327 }
2328 
2329 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
2330                                    const RISCVSubtarget &Subtarget) {
2331   SDValue V1 = Op.getOperand(0);
2332   SDValue V2 = Op.getOperand(1);
2333   SDLoc DL(Op);
2334   MVT XLenVT = Subtarget.getXLenVT();
2335   MVT VT = Op.getSimpleValueType();
2336   unsigned NumElts = VT.getVectorNumElements();
2337   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
2338 
2339   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2340 
2341   SDValue TrueMask, VL;
2342   std::tie(TrueMask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2343 
2344   if (SVN->isSplat()) {
2345     const int Lane = SVN->getSplatIndex();
2346     if (Lane >= 0) {
2347       MVT SVT = VT.getVectorElementType();
2348 
2349       // Turn splatted vector load into a strided load with an X0 stride.
2350       SDValue V = V1;
2351       // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
2352       // with undef.
2353       // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
2354       int Offset = Lane;
2355       if (V.getOpcode() == ISD::CONCAT_VECTORS) {
2356         int OpElements =
2357             V.getOperand(0).getSimpleValueType().getVectorNumElements();
2358         V = V.getOperand(Offset / OpElements);
2359         Offset %= OpElements;
2360       }
2361 
2362       // We need to ensure the load isn't atomic or volatile.
2363       if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
2364         auto *Ld = cast<LoadSDNode>(V);
2365         Offset *= SVT.getStoreSize();
2366         SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(),
2367                                                    TypeSize::Fixed(Offset), DL);
2368 
2369         // If this is SEW=64 on RV32, use a strided load with a stride of x0.
2370         if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
2371           SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
2372           SDValue IntID =
2373               DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
2374           SDValue Ops[] = {Ld->getChain(), IntID, NewAddr,
2375                            DAG.getRegister(RISCV::X0, XLenVT), VL};
2376           SDValue NewLoad = DAG.getMemIntrinsicNode(
2377               ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
2378               DAG.getMachineFunction().getMachineMemOperand(
2379                   Ld->getMemOperand(), Offset, SVT.getStoreSize()));
2380           DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
2381           return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
2382         }
2383 
2384         // Otherwise use a scalar load and splat. This will give the best
2385         // opportunity to fold a splat into the operation. ISel can turn it into
2386         // the x0 strided load if we aren't able to fold away the select.
2387         if (SVT.isFloatingPoint())
2388           V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
2389                           Ld->getPointerInfo().getWithOffset(Offset),
2390                           Ld->getOriginalAlign(),
2391                           Ld->getMemOperand()->getFlags());
2392         else
2393           V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
2394                              Ld->getPointerInfo().getWithOffset(Offset), SVT,
2395                              Ld->getOriginalAlign(),
2396                              Ld->getMemOperand()->getFlags());
2397         DAG.makeEquivalentMemoryOrdering(Ld, V);
2398 
2399         unsigned Opc =
2400             VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
2401         SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, V, VL);
2402         return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2403       }
2404 
2405       V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2406       assert(Lane < (int)NumElts && "Unexpected lane!");
2407       SDValue Gather =
2408           DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
2409                       DAG.getConstant(Lane, DL, XLenVT), TrueMask, VL);
2410       return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2411     }
2412   }
2413 
2414   // Try to match as a slidedown.
2415   int SlideAmt = matchShuffleAsSlideDown(SVN->getMask());
2416   if (SlideAmt >= 0) {
2417     // TODO: Should we reduce the VL to account for the upper undef elements?
2418     // Requires additional vsetvlis, but might be faster to execute.
2419     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2420     SDValue SlideDown =
2421         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
2422                     DAG.getUNDEF(ContainerVT), V1,
2423                     DAG.getConstant(SlideAmt, DL, XLenVT),
2424                     TrueMask, VL);
2425     return convertFromScalableVector(VT, SlideDown, DAG, Subtarget);
2426   }
2427 
2428   // Detect shuffles which can be re-expressed as vector selects; these are
2429   // shuffles in which each element in the destination is taken from an element
2430   // at the corresponding index in either source vectors.
2431   bool IsSelect = all_of(enumerate(SVN->getMask()), [&](const auto &MaskIdx) {
2432     int MaskIndex = MaskIdx.value();
2433     return MaskIndex < 0 || MaskIdx.index() == (unsigned)MaskIndex % NumElts;
2434   });
2435 
2436   assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
2437 
2438   SmallVector<SDValue> MaskVals;
2439   // As a backup, shuffles can be lowered via a vrgather instruction, possibly
2440   // merged with a second vrgather.
2441   SmallVector<SDValue> GatherIndicesLHS, GatherIndicesRHS;
2442 
2443   // By default we preserve the original operand order, and use a mask to
2444   // select LHS as true and RHS as false. However, since RVV vector selects may
2445   // feature splats but only on the LHS, we may choose to invert our mask and
2446   // instead select between RHS and LHS.
2447   bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
2448   bool InvertMask = IsSelect == SwapOps;
2449 
2450   // Keep a track of which non-undef indices are used by each LHS/RHS shuffle
2451   // half.
2452   DenseMap<int, unsigned> LHSIndexCounts, RHSIndexCounts;
2453 
2454   // Now construct the mask that will be used by the vselect or blended
2455   // vrgather operation. For vrgathers, construct the appropriate indices into
2456   // each vector.
2457   for (int MaskIndex : SVN->getMask()) {
2458     bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ InvertMask;
2459     MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
2460     if (!IsSelect) {
2461       bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
2462       GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
2463                                      ? DAG.getConstant(MaskIndex, DL, XLenVT)
2464                                      : DAG.getUNDEF(XLenVT));
2465       GatherIndicesRHS.push_back(
2466           IsLHSOrUndefIndex ? DAG.getUNDEF(XLenVT)
2467                             : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT));
2468       if (IsLHSOrUndefIndex && MaskIndex >= 0)
2469         ++LHSIndexCounts[MaskIndex];
2470       if (!IsLHSOrUndefIndex)
2471         ++RHSIndexCounts[MaskIndex - NumElts];
2472     }
2473   }
2474 
2475   if (SwapOps) {
2476     std::swap(V1, V2);
2477     std::swap(GatherIndicesLHS, GatherIndicesRHS);
2478   }
2479 
2480   assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
2481   MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
2482   SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
2483 
2484   if (IsSelect)
2485     return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
2486 
2487   if (VT.getScalarSizeInBits() == 8 && VT.getVectorNumElements() > 256) {
2488     // On such a large vector we're unable to use i8 as the index type.
2489     // FIXME: We could promote the index to i16 and use vrgatherei16, but that
2490     // may involve vector splitting if we're already at LMUL=8, or our
2491     // user-supplied maximum fixed-length LMUL.
2492     return SDValue();
2493   }
2494 
2495   unsigned GatherVXOpc = RISCVISD::VRGATHER_VX_VL;
2496   unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
2497   MVT IndexVT = VT.changeTypeToInteger();
2498   // Since we can't introduce illegal index types at this stage, use i16 and
2499   // vrgatherei16 if the corresponding index type for plain vrgather is greater
2500   // than XLenVT.
2501   if (IndexVT.getScalarType().bitsGT(XLenVT)) {
2502     GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
2503     IndexVT = IndexVT.changeVectorElementType(MVT::i16);
2504   }
2505 
2506   MVT IndexContainerVT =
2507       ContainerVT.changeVectorElementType(IndexVT.getScalarType());
2508 
2509   SDValue Gather;
2510   // TODO: This doesn't trigger for i64 vectors on RV32, since there we
2511   // encounter a bitcasted BUILD_VECTOR with low/high i32 values.
2512   if (SDValue SplatValue = DAG.getSplatValue(V1, /*LegalTypes*/ true)) {
2513     Gather = lowerScalarSplat(SplatValue, VL, ContainerVT, DL, DAG, Subtarget);
2514   } else {
2515     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2516     // If only one index is used, we can use a "splat" vrgather.
2517     // TODO: We can splat the most-common index and fix-up any stragglers, if
2518     // that's beneficial.
2519     if (LHSIndexCounts.size() == 1) {
2520       int SplatIndex = LHSIndexCounts.begin()->getFirst();
2521       Gather =
2522           DAG.getNode(GatherVXOpc, DL, ContainerVT, V1,
2523                       DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2524     } else {
2525       SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
2526       LHSIndices =
2527           convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
2528 
2529       Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
2530                            TrueMask, VL);
2531     }
2532   }
2533 
2534   // If a second vector operand is used by this shuffle, blend it in with an
2535   // additional vrgather.
2536   if (!V2.isUndef()) {
2537     V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
2538     // If only one index is used, we can use a "splat" vrgather.
2539     // TODO: We can splat the most-common index and fix-up any stragglers, if
2540     // that's beneficial.
2541     if (RHSIndexCounts.size() == 1) {
2542       int SplatIndex = RHSIndexCounts.begin()->getFirst();
2543       V2 = DAG.getNode(GatherVXOpc, DL, ContainerVT, V2,
2544                        DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2545     } else {
2546       SDValue RHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesRHS);
2547       RHSIndices =
2548           convertToScalableVector(IndexContainerVT, RHSIndices, DAG, Subtarget);
2549       V2 = DAG.getNode(GatherVVOpc, DL, ContainerVT, V2, RHSIndices, TrueMask,
2550                        VL);
2551     }
2552 
2553     MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
2554     SelectMask =
2555         convertToScalableVector(MaskContainerVT, SelectMask, DAG, Subtarget);
2556 
2557     Gather = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, SelectMask, V2,
2558                          Gather, VL);
2559   }
2560 
2561   return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2562 }
2563 
2564 static SDValue getRVVFPExtendOrRound(SDValue Op, MVT VT, MVT ContainerVT,
2565                                      SDLoc DL, SelectionDAG &DAG,
2566                                      const RISCVSubtarget &Subtarget) {
2567   if (VT.isScalableVector())
2568     return DAG.getFPExtendOrRound(Op, DL, VT);
2569   assert(VT.isFixedLengthVector() &&
2570          "Unexpected value type for RVV FP extend/round lowering");
2571   SDValue Mask, VL;
2572   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2573   unsigned RVVOpc = ContainerVT.bitsGT(Op.getSimpleValueType())
2574                         ? RISCVISD::FP_EXTEND_VL
2575                         : RISCVISD::FP_ROUND_VL;
2576   return DAG.getNode(RVVOpc, DL, ContainerVT, Op, Mask, VL);
2577 }
2578 
2579 // Lower CTLZ_ZERO_UNDEF or CTTZ_ZERO_UNDEF by converting to FP and extracting
2580 // the exponent.
2581 static SDValue lowerCTLZ_CTTZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
2582   MVT VT = Op.getSimpleValueType();
2583   unsigned EltSize = VT.getScalarSizeInBits();
2584   SDValue Src = Op.getOperand(0);
2585   SDLoc DL(Op);
2586 
2587   // We need a FP type that can represent the value.
2588   // TODO: Use f16 for i8 when possible?
2589   MVT FloatEltVT = EltSize == 32 ? MVT::f64 : MVT::f32;
2590   MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
2591 
2592   // Legal types should have been checked in the RISCVTargetLowering
2593   // constructor.
2594   // TODO: Splitting may make sense in some cases.
2595   assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
2596          "Expected legal float type!");
2597 
2598   // For CTTZ_ZERO_UNDEF, we need to extract the lowest set bit using X & -X.
2599   // The trailing zero count is equal to log2 of this single bit value.
2600   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
2601     SDValue Neg =
2602         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
2603     Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
2604   }
2605 
2606   // We have a legal FP type, convert to it.
2607   SDValue FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
2608   // Bitcast to integer and shift the exponent to the LSB.
2609   EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
2610   SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
2611   unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
2612   SDValue Shift = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
2613                               DAG.getConstant(ShiftAmt, DL, IntVT));
2614   // Truncate back to original type to allow vnsrl.
2615   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, Shift);
2616   // The exponent contains log2 of the value in biased form.
2617   unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
2618 
2619   // For trailing zeros, we just need to subtract the bias.
2620   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF)
2621     return DAG.getNode(ISD::SUB, DL, VT, Trunc,
2622                        DAG.getConstant(ExponentBias, DL, VT));
2623 
2624   // For leading zeros, we need to remove the bias and convert from log2 to
2625   // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
2626   unsigned Adjust = ExponentBias + (EltSize - 1);
2627   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Trunc);
2628 }
2629 
2630 // While RVV has alignment restrictions, we should always be able to load as a
2631 // legal equivalently-sized byte-typed vector instead. This method is
2632 // responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
2633 // the load is already correctly-aligned, it returns SDValue().
2634 SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
2635                                                     SelectionDAG &DAG) const {
2636   auto *Load = cast<LoadSDNode>(Op);
2637   assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
2638 
2639   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2640                                      Load->getMemoryVT(),
2641                                      *Load->getMemOperand()))
2642     return SDValue();
2643 
2644   SDLoc DL(Op);
2645   MVT VT = Op.getSimpleValueType();
2646   unsigned EltSizeBits = VT.getScalarSizeInBits();
2647   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2648          "Unexpected unaligned RVV load type");
2649   MVT NewVT =
2650       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2651   assert(NewVT.isValid() &&
2652          "Expecting equally-sized RVV vector types to be legal");
2653   SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
2654                           Load->getPointerInfo(), Load->getOriginalAlign(),
2655                           Load->getMemOperand()->getFlags());
2656   return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
2657 }
2658 
2659 // While RVV has alignment restrictions, we should always be able to store as a
2660 // legal equivalently-sized byte-typed vector instead. This method is
2661 // responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
2662 // returns SDValue() if the store is already correctly aligned.
2663 SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
2664                                                      SelectionDAG &DAG) const {
2665   auto *Store = cast<StoreSDNode>(Op);
2666   assert(Store && Store->getValue().getValueType().isVector() &&
2667          "Expected vector store");
2668 
2669   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2670                                      Store->getMemoryVT(),
2671                                      *Store->getMemOperand()))
2672     return SDValue();
2673 
2674   SDLoc DL(Op);
2675   SDValue StoredVal = Store->getValue();
2676   MVT VT = StoredVal.getSimpleValueType();
2677   unsigned EltSizeBits = VT.getScalarSizeInBits();
2678   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2679          "Unexpected unaligned RVV store type");
2680   MVT NewVT =
2681       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2682   assert(NewVT.isValid() &&
2683          "Expecting equally-sized RVV vector types to be legal");
2684   StoredVal = DAG.getBitcast(NewVT, StoredVal);
2685   return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
2686                       Store->getPointerInfo(), Store->getOriginalAlign(),
2687                       Store->getMemOperand()->getFlags());
2688 }
2689 
2690 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
2691                                             SelectionDAG &DAG) const {
2692   switch (Op.getOpcode()) {
2693   default:
2694     report_fatal_error("unimplemented operand");
2695   case ISD::GlobalAddress:
2696     return lowerGlobalAddress(Op, DAG);
2697   case ISD::BlockAddress:
2698     return lowerBlockAddress(Op, DAG);
2699   case ISD::ConstantPool:
2700     return lowerConstantPool(Op, DAG);
2701   case ISD::JumpTable:
2702     return lowerJumpTable(Op, DAG);
2703   case ISD::GlobalTLSAddress:
2704     return lowerGlobalTLSAddress(Op, DAG);
2705   case ISD::SELECT:
2706     return lowerSELECT(Op, DAG);
2707   case ISD::BRCOND:
2708     return lowerBRCOND(Op, DAG);
2709   case ISD::VASTART:
2710     return lowerVASTART(Op, DAG);
2711   case ISD::FRAMEADDR:
2712     return lowerFRAMEADDR(Op, DAG);
2713   case ISD::RETURNADDR:
2714     return lowerRETURNADDR(Op, DAG);
2715   case ISD::SHL_PARTS:
2716     return lowerShiftLeftParts(Op, DAG);
2717   case ISD::SRA_PARTS:
2718     return lowerShiftRightParts(Op, DAG, true);
2719   case ISD::SRL_PARTS:
2720     return lowerShiftRightParts(Op, DAG, false);
2721   case ISD::BITCAST: {
2722     SDLoc DL(Op);
2723     EVT VT = Op.getValueType();
2724     SDValue Op0 = Op.getOperand(0);
2725     EVT Op0VT = Op0.getValueType();
2726     MVT XLenVT = Subtarget.getXLenVT();
2727     if (VT.isFixedLengthVector()) {
2728       // We can handle fixed length vector bitcasts with a simple replacement
2729       // in isel.
2730       if (Op0VT.isFixedLengthVector())
2731         return Op;
2732       // When bitcasting from scalar to fixed-length vector, insert the scalar
2733       // into a one-element vector of the result type, and perform a vector
2734       // bitcast.
2735       if (!Op0VT.isVector()) {
2736         EVT BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
2737         if (!isTypeLegal(BVT))
2738           return SDValue();
2739         return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
2740                                               DAG.getUNDEF(BVT), Op0,
2741                                               DAG.getConstant(0, DL, XLenVT)));
2742       }
2743       return SDValue();
2744     }
2745     // Custom-legalize bitcasts from fixed-length vector types to scalar types
2746     // thus: bitcast the vector to a one-element vector type whose element type
2747     // is the same as the result type, and extract the first element.
2748     if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
2749       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
2750       if (!isTypeLegal(BVT))
2751         return SDValue();
2752       SDValue BVec = DAG.getBitcast(BVT, Op0);
2753       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
2754                          DAG.getConstant(0, DL, XLenVT));
2755     }
2756     if (VT == MVT::f16 && Op0VT == MVT::i16 && Subtarget.hasStdExtZfh()) {
2757       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
2758       SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
2759       return FPConv;
2760     }
2761     if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
2762         Subtarget.hasStdExtF()) {
2763       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
2764       SDValue FPConv =
2765           DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
2766       return FPConv;
2767     }
2768     return SDValue();
2769   }
2770   case ISD::INTRINSIC_WO_CHAIN:
2771     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
2772   case ISD::INTRINSIC_W_CHAIN:
2773     return LowerINTRINSIC_W_CHAIN(Op, DAG);
2774   case ISD::INTRINSIC_VOID:
2775     return LowerINTRINSIC_VOID(Op, DAG);
2776   case ISD::BSWAP:
2777   case ISD::BITREVERSE: {
2778     // Convert BSWAP/BITREVERSE to GREVI to enable GREVI combinining.
2779     assert(Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
2780     MVT VT = Op.getSimpleValueType();
2781     SDLoc DL(Op);
2782     // Start with the maximum immediate value which is the bitwidth - 1.
2783     unsigned Imm = VT.getSizeInBits() - 1;
2784     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
2785     if (Op.getOpcode() == ISD::BSWAP)
2786       Imm &= ~0x7U;
2787     return DAG.getNode(RISCVISD::GREV, DL, VT, Op.getOperand(0),
2788                        DAG.getConstant(Imm, DL, VT));
2789   }
2790   case ISD::FSHL:
2791   case ISD::FSHR: {
2792     MVT VT = Op.getSimpleValueType();
2793     assert(VT == Subtarget.getXLenVT() && "Unexpected custom legalization");
2794     SDLoc DL(Op);
2795     // FSL/FSR take a log2(XLen)+1 bit shift amount but XLenVT FSHL/FSHR only
2796     // use log(XLen) bits. Mask the shift amount accordingly to prevent
2797     // accidentally setting the extra bit.
2798     unsigned ShAmtWidth = Subtarget.getXLen() - 1;
2799     SDValue ShAmt = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(2),
2800                                 DAG.getConstant(ShAmtWidth, DL, VT));
2801     // fshl and fshr concatenate their operands in the same order. fsr and fsl
2802     // instruction use different orders. fshl will return its first operand for
2803     // shift of zero, fshr will return its second operand. fsl and fsr both
2804     // return rs1 so the ISD nodes need to have different operand orders.
2805     // Shift amount is in rs2.
2806     SDValue Op0 = Op.getOperand(0);
2807     SDValue Op1 = Op.getOperand(1);
2808     unsigned Opc = RISCVISD::FSL;
2809     if (Op.getOpcode() == ISD::FSHR) {
2810       std::swap(Op0, Op1);
2811       Opc = RISCVISD::FSR;
2812     }
2813     return DAG.getNode(Opc, DL, VT, Op0, Op1, ShAmt);
2814   }
2815   case ISD::TRUNCATE: {
2816     SDLoc DL(Op);
2817     MVT VT = Op.getSimpleValueType();
2818     // Only custom-lower vector truncates
2819     if (!VT.isVector())
2820       return Op;
2821 
2822     // Truncates to mask types are handled differently
2823     if (VT.getVectorElementType() == MVT::i1)
2824       return lowerVectorMaskTrunc(Op, DAG);
2825 
2826     // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
2827     // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
2828     // truncate by one power of two at a time.
2829     MVT DstEltVT = VT.getVectorElementType();
2830 
2831     SDValue Src = Op.getOperand(0);
2832     MVT SrcVT = Src.getSimpleValueType();
2833     MVT SrcEltVT = SrcVT.getVectorElementType();
2834 
2835     assert(DstEltVT.bitsLT(SrcEltVT) &&
2836            isPowerOf2_64(DstEltVT.getSizeInBits()) &&
2837            isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
2838            "Unexpected vector truncate lowering");
2839 
2840     MVT ContainerVT = SrcVT;
2841     if (SrcVT.isFixedLengthVector()) {
2842       ContainerVT = getContainerForFixedLengthVector(SrcVT);
2843       Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
2844     }
2845 
2846     SDValue Result = Src;
2847     SDValue Mask, VL;
2848     std::tie(Mask, VL) =
2849         getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
2850     LLVMContext &Context = *DAG.getContext();
2851     const ElementCount Count = ContainerVT.getVectorElementCount();
2852     do {
2853       SrcEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
2854       EVT ResultVT = EVT::getVectorVT(Context, SrcEltVT, Count);
2855       Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
2856                            Mask, VL);
2857     } while (SrcEltVT != DstEltVT);
2858 
2859     if (SrcVT.isFixedLengthVector())
2860       Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
2861 
2862     return Result;
2863   }
2864   case ISD::ANY_EXTEND:
2865   case ISD::ZERO_EXTEND:
2866     if (Op.getOperand(0).getValueType().isVector() &&
2867         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
2868       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
2869     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
2870   case ISD::SIGN_EXTEND:
2871     if (Op.getOperand(0).getValueType().isVector() &&
2872         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
2873       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
2874     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
2875   case ISD::SPLAT_VECTOR_PARTS:
2876     return lowerSPLAT_VECTOR_PARTS(Op, DAG);
2877   case ISD::INSERT_VECTOR_ELT:
2878     return lowerINSERT_VECTOR_ELT(Op, DAG);
2879   case ISD::EXTRACT_VECTOR_ELT:
2880     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
2881   case ISD::VSCALE: {
2882     MVT VT = Op.getSimpleValueType();
2883     SDLoc DL(Op);
2884     SDValue VLENB = DAG.getNode(RISCVISD::READ_VLENB, DL, VT);
2885     // We define our scalable vector types for lmul=1 to use a 64 bit known
2886     // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
2887     // vscale as VLENB / 8.
2888     static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
2889     if (isa<ConstantSDNode>(Op.getOperand(0))) {
2890       // We assume VLENB is a multiple of 8. We manually choose the best shift
2891       // here because SimplifyDemandedBits isn't always able to simplify it.
2892       uint64_t Val = Op.getConstantOperandVal(0);
2893       if (isPowerOf2_64(Val)) {
2894         uint64_t Log2 = Log2_64(Val);
2895         if (Log2 < 3)
2896           return DAG.getNode(ISD::SRL, DL, VT, VLENB,
2897                              DAG.getConstant(3 - Log2, DL, VT));
2898         if (Log2 > 3)
2899           return DAG.getNode(ISD::SHL, DL, VT, VLENB,
2900                              DAG.getConstant(Log2 - 3, DL, VT));
2901         return VLENB;
2902       }
2903       // If the multiplier is a multiple of 8, scale it down to avoid needing
2904       // to shift the VLENB value.
2905       if ((Val % 8) == 0)
2906         return DAG.getNode(ISD::MUL, DL, VT, VLENB,
2907                            DAG.getConstant(Val / 8, DL, VT));
2908     }
2909 
2910     SDValue VScale = DAG.getNode(ISD::SRL, DL, VT, VLENB,
2911                                  DAG.getConstant(3, DL, VT));
2912     return DAG.getNode(ISD::MUL, DL, VT, VScale, Op.getOperand(0));
2913   }
2914   case ISD::FPOWI: {
2915     // Custom promote f16 powi with illegal i32 integer type on RV64. Once
2916     // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
2917     if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
2918         Op.getOperand(1).getValueType() == MVT::i32) {
2919       SDLoc DL(Op);
2920       SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
2921       SDValue Powi =
2922           DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
2923       return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
2924                          DAG.getIntPtrConstant(0, DL));
2925     }
2926     return SDValue();
2927   }
2928   case ISD::FP_EXTEND: {
2929     // RVV can only do fp_extend to types double the size as the source. We
2930     // custom-lower f16->f64 extensions to two hops of ISD::FP_EXTEND, going
2931     // via f32.
2932     SDLoc DL(Op);
2933     MVT VT = Op.getSimpleValueType();
2934     SDValue Src = Op.getOperand(0);
2935     MVT SrcVT = Src.getSimpleValueType();
2936 
2937     // Prepare any fixed-length vector operands.
2938     MVT ContainerVT = VT;
2939     if (SrcVT.isFixedLengthVector()) {
2940       ContainerVT = getContainerForFixedLengthVector(VT);
2941       MVT SrcContainerVT =
2942           ContainerVT.changeVectorElementType(SrcVT.getVectorElementType());
2943       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
2944     }
2945 
2946     if (!VT.isVector() || VT.getVectorElementType() != MVT::f64 ||
2947         SrcVT.getVectorElementType() != MVT::f16) {
2948       // For scalable vectors, we only need to close the gap between
2949       // vXf16->vXf64.
2950       if (!VT.isFixedLengthVector())
2951         return Op;
2952       // For fixed-length vectors, lower the FP_EXTEND to a custom "VL" version.
2953       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
2954       return convertFromScalableVector(VT, Src, DAG, Subtarget);
2955     }
2956 
2957     MVT InterVT = VT.changeVectorElementType(MVT::f32);
2958     MVT InterContainerVT = ContainerVT.changeVectorElementType(MVT::f32);
2959     SDValue IntermediateExtend = getRVVFPExtendOrRound(
2960         Src, InterVT, InterContainerVT, DL, DAG, Subtarget);
2961 
2962     SDValue Extend = getRVVFPExtendOrRound(IntermediateExtend, VT, ContainerVT,
2963                                            DL, DAG, Subtarget);
2964     if (VT.isFixedLengthVector())
2965       return convertFromScalableVector(VT, Extend, DAG, Subtarget);
2966     return Extend;
2967   }
2968   case ISD::FP_ROUND: {
2969     // RVV can only do fp_round to types half the size as the source. We
2970     // custom-lower f64->f16 rounds via RVV's round-to-odd float
2971     // conversion instruction.
2972     SDLoc DL(Op);
2973     MVT VT = Op.getSimpleValueType();
2974     SDValue Src = Op.getOperand(0);
2975     MVT SrcVT = Src.getSimpleValueType();
2976 
2977     // Prepare any fixed-length vector operands.
2978     MVT ContainerVT = VT;
2979     if (VT.isFixedLengthVector()) {
2980       MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
2981       ContainerVT =
2982           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
2983       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
2984     }
2985 
2986     if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
2987         SrcVT.getVectorElementType() != MVT::f64) {
2988       // For scalable vectors, we only need to close the gap between
2989       // vXf64<->vXf16.
2990       if (!VT.isFixedLengthVector())
2991         return Op;
2992       // For fixed-length vectors, lower the FP_ROUND to a custom "VL" version.
2993       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
2994       return convertFromScalableVector(VT, Src, DAG, Subtarget);
2995     }
2996 
2997     SDValue Mask, VL;
2998     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2999 
3000     MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
3001     SDValue IntermediateRound =
3002         DAG.getNode(RISCVISD::VFNCVT_ROD_VL, DL, InterVT, Src, Mask, VL);
3003     SDValue Round = getRVVFPExtendOrRound(IntermediateRound, VT, ContainerVT,
3004                                           DL, DAG, Subtarget);
3005 
3006     if (VT.isFixedLengthVector())
3007       return convertFromScalableVector(VT, Round, DAG, Subtarget);
3008     return Round;
3009   }
3010   case ISD::FP_TO_SINT:
3011   case ISD::FP_TO_UINT:
3012   case ISD::SINT_TO_FP:
3013   case ISD::UINT_TO_FP: {
3014     // RVV can only do fp<->int conversions to types half/double the size as
3015     // the source. We custom-lower any conversions that do two hops into
3016     // sequences.
3017     MVT VT = Op.getSimpleValueType();
3018     if (!VT.isVector())
3019       return Op;
3020     SDLoc DL(Op);
3021     SDValue Src = Op.getOperand(0);
3022     MVT EltVT = VT.getVectorElementType();
3023     MVT SrcVT = Src.getSimpleValueType();
3024     MVT SrcEltVT = SrcVT.getVectorElementType();
3025     unsigned EltSize = EltVT.getSizeInBits();
3026     unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3027     assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
3028            "Unexpected vector element types");
3029 
3030     bool IsInt2FP = SrcEltVT.isInteger();
3031     // Widening conversions
3032     if (EltSize > SrcEltSize && (EltSize / SrcEltSize >= 4)) {
3033       if (IsInt2FP) {
3034         // Do a regular integer sign/zero extension then convert to float.
3035         MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltVT.getSizeInBits()),
3036                                       VT.getVectorElementCount());
3037         unsigned ExtOpcode = Op.getOpcode() == ISD::UINT_TO_FP
3038                                  ? ISD::ZERO_EXTEND
3039                                  : ISD::SIGN_EXTEND;
3040         SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
3041         return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
3042       }
3043       // FP2Int
3044       assert(SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering");
3045       // Do one doubling fp_extend then complete the operation by converting
3046       // to int.
3047       MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3048       SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
3049       return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
3050     }
3051 
3052     // Narrowing conversions
3053     if (SrcEltSize > EltSize && (SrcEltSize / EltSize >= 4)) {
3054       if (IsInt2FP) {
3055         // One narrowing int_to_fp, then an fp_round.
3056         assert(EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering");
3057         MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3058         SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
3059         return DAG.getFPExtendOrRound(Int2FP, DL, VT);
3060       }
3061       // FP2Int
3062       // One narrowing fp_to_int, then truncate the integer. If the float isn't
3063       // representable by the integer, the result is poison.
3064       MVT IVecVT =
3065           MVT::getVectorVT(MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2),
3066                            VT.getVectorElementCount());
3067       SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
3068       return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
3069     }
3070 
3071     // Scalable vectors can exit here. Patterns will handle equally-sized
3072     // conversions halving/doubling ones.
3073     if (!VT.isFixedLengthVector())
3074       return Op;
3075 
3076     // For fixed-length vectors we lower to a custom "VL" node.
3077     unsigned RVVOpc = 0;
3078     switch (Op.getOpcode()) {
3079     default:
3080       llvm_unreachable("Impossible opcode");
3081     case ISD::FP_TO_SINT:
3082       RVVOpc = RISCVISD::FP_TO_SINT_VL;
3083       break;
3084     case ISD::FP_TO_UINT:
3085       RVVOpc = RISCVISD::FP_TO_UINT_VL;
3086       break;
3087     case ISD::SINT_TO_FP:
3088       RVVOpc = RISCVISD::SINT_TO_FP_VL;
3089       break;
3090     case ISD::UINT_TO_FP:
3091       RVVOpc = RISCVISD::UINT_TO_FP_VL;
3092       break;
3093     }
3094 
3095     MVT ContainerVT, SrcContainerVT;
3096     // Derive the reference container type from the larger vector type.
3097     if (SrcEltSize > EltSize) {
3098       SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3099       ContainerVT =
3100           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3101     } else {
3102       ContainerVT = getContainerForFixedLengthVector(VT);
3103       SrcContainerVT = ContainerVT.changeVectorElementType(SrcEltVT);
3104     }
3105 
3106     SDValue Mask, VL;
3107     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3108 
3109     Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3110     Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
3111     return convertFromScalableVector(VT, Src, DAG, Subtarget);
3112   }
3113   case ISD::FP_TO_SINT_SAT:
3114   case ISD::FP_TO_UINT_SAT:
3115     return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
3116   case ISD::FTRUNC:
3117   case ISD::FCEIL:
3118   case ISD::FFLOOR:
3119     return lowerFTRUNC_FCEIL_FFLOOR(Op, DAG);
3120   case ISD::VECREDUCE_ADD:
3121   case ISD::VECREDUCE_UMAX:
3122   case ISD::VECREDUCE_SMAX:
3123   case ISD::VECREDUCE_UMIN:
3124   case ISD::VECREDUCE_SMIN:
3125     return lowerVECREDUCE(Op, DAG);
3126   case ISD::VECREDUCE_AND:
3127   case ISD::VECREDUCE_OR:
3128   case ISD::VECREDUCE_XOR:
3129     if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3130       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
3131     return lowerVECREDUCE(Op, DAG);
3132   case ISD::VECREDUCE_FADD:
3133   case ISD::VECREDUCE_SEQ_FADD:
3134   case ISD::VECREDUCE_FMIN:
3135   case ISD::VECREDUCE_FMAX:
3136     return lowerFPVECREDUCE(Op, DAG);
3137   case ISD::VP_REDUCE_ADD:
3138   case ISD::VP_REDUCE_UMAX:
3139   case ISD::VP_REDUCE_SMAX:
3140   case ISD::VP_REDUCE_UMIN:
3141   case ISD::VP_REDUCE_SMIN:
3142   case ISD::VP_REDUCE_FADD:
3143   case ISD::VP_REDUCE_SEQ_FADD:
3144   case ISD::VP_REDUCE_FMIN:
3145   case ISD::VP_REDUCE_FMAX:
3146     return lowerVPREDUCE(Op, DAG);
3147   case ISD::VP_REDUCE_AND:
3148   case ISD::VP_REDUCE_OR:
3149   case ISD::VP_REDUCE_XOR:
3150     if (Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
3151       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
3152     return lowerVPREDUCE(Op, DAG);
3153   case ISD::INSERT_SUBVECTOR:
3154     return lowerINSERT_SUBVECTOR(Op, DAG);
3155   case ISD::EXTRACT_SUBVECTOR:
3156     return lowerEXTRACT_SUBVECTOR(Op, DAG);
3157   case ISD::STEP_VECTOR:
3158     return lowerSTEP_VECTOR(Op, DAG);
3159   case ISD::VECTOR_REVERSE:
3160     return lowerVECTOR_REVERSE(Op, DAG);
3161   case ISD::BUILD_VECTOR:
3162     return lowerBUILD_VECTOR(Op, DAG, Subtarget);
3163   case ISD::SPLAT_VECTOR:
3164     if (Op.getValueType().getVectorElementType() == MVT::i1)
3165       return lowerVectorMaskSplat(Op, DAG);
3166     return lowerSPLAT_VECTOR(Op, DAG, Subtarget);
3167   case ISD::VECTOR_SHUFFLE:
3168     return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
3169   case ISD::CONCAT_VECTORS: {
3170     // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
3171     // better than going through the stack, as the default expansion does.
3172     SDLoc DL(Op);
3173     MVT VT = Op.getSimpleValueType();
3174     unsigned NumOpElts =
3175         Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
3176     SDValue Vec = DAG.getUNDEF(VT);
3177     for (const auto &OpIdx : enumerate(Op->ops())) {
3178       SDValue SubVec = OpIdx.value();
3179       // Don't insert undef subvectors.
3180       if (SubVec.isUndef())
3181         continue;
3182       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec,
3183                         DAG.getIntPtrConstant(OpIdx.index() * NumOpElts, DL));
3184     }
3185     return Vec;
3186   }
3187   case ISD::LOAD:
3188     if (auto V = expandUnalignedRVVLoad(Op, DAG))
3189       return V;
3190     if (Op.getValueType().isFixedLengthVector())
3191       return lowerFixedLengthVectorLoadToRVV(Op, DAG);
3192     return Op;
3193   case ISD::STORE:
3194     if (auto V = expandUnalignedRVVStore(Op, DAG))
3195       return V;
3196     if (Op.getOperand(1).getValueType().isFixedLengthVector())
3197       return lowerFixedLengthVectorStoreToRVV(Op, DAG);
3198     return Op;
3199   case ISD::MLOAD:
3200   case ISD::VP_LOAD:
3201     return lowerMaskedLoad(Op, DAG);
3202   case ISD::MSTORE:
3203   case ISD::VP_STORE:
3204     return lowerMaskedStore(Op, DAG);
3205   case ISD::SETCC:
3206     return lowerFixedLengthVectorSetccToRVV(Op, DAG);
3207   case ISD::ADD:
3208     return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);
3209   case ISD::SUB:
3210     return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL);
3211   case ISD::MUL:
3212     return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL);
3213   case ISD::MULHS:
3214     return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL);
3215   case ISD::MULHU:
3216     return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL);
3217   case ISD::AND:
3218     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMAND_VL,
3219                                               RISCVISD::AND_VL);
3220   case ISD::OR:
3221     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMOR_VL,
3222                                               RISCVISD::OR_VL);
3223   case ISD::XOR:
3224     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMXOR_VL,
3225                                               RISCVISD::XOR_VL);
3226   case ISD::SDIV:
3227     return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL);
3228   case ISD::SREM:
3229     return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL);
3230   case ISD::UDIV:
3231     return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL);
3232   case ISD::UREM:
3233     return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL);
3234   case ISD::SHL:
3235   case ISD::SRA:
3236   case ISD::SRL:
3237     if (Op.getSimpleValueType().isFixedLengthVector())
3238       return lowerFixedLengthVectorShiftToRVV(Op, DAG);
3239     // This can be called for an i32 shift amount that needs to be promoted.
3240     assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
3241            "Unexpected custom legalisation");
3242     return SDValue();
3243   case ISD::SADDSAT:
3244     return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL);
3245   case ISD::UADDSAT:
3246     return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL);
3247   case ISD::SSUBSAT:
3248     return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL);
3249   case ISD::USUBSAT:
3250     return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL);
3251   case ISD::FADD:
3252     return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL);
3253   case ISD::FSUB:
3254     return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL);
3255   case ISD::FMUL:
3256     return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL);
3257   case ISD::FDIV:
3258     return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL);
3259   case ISD::FNEG:
3260     return lowerToScalableOp(Op, DAG, RISCVISD::FNEG_VL);
3261   case ISD::FABS:
3262     return lowerToScalableOp(Op, DAG, RISCVISD::FABS_VL);
3263   case ISD::FSQRT:
3264     return lowerToScalableOp(Op, DAG, RISCVISD::FSQRT_VL);
3265   case ISD::FMA:
3266     return lowerToScalableOp(Op, DAG, RISCVISD::FMA_VL);
3267   case ISD::SMIN:
3268     return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL);
3269   case ISD::SMAX:
3270     return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL);
3271   case ISD::UMIN:
3272     return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL);
3273   case ISD::UMAX:
3274     return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL);
3275   case ISD::FMINNUM:
3276     return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL);
3277   case ISD::FMAXNUM:
3278     return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL);
3279   case ISD::ABS:
3280     return lowerABS(Op, DAG);
3281   case ISD::CTLZ_ZERO_UNDEF:
3282   case ISD::CTTZ_ZERO_UNDEF:
3283     return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
3284   case ISD::VSELECT:
3285     return lowerFixedLengthVectorSelectToRVV(Op, DAG);
3286   case ISD::FCOPYSIGN:
3287     return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG);
3288   case ISD::MGATHER:
3289   case ISD::VP_GATHER:
3290     return lowerMaskedGather(Op, DAG);
3291   case ISD::MSCATTER:
3292   case ISD::VP_SCATTER:
3293     return lowerMaskedScatter(Op, DAG);
3294   case ISD::FLT_ROUNDS_:
3295     return lowerGET_ROUNDING(Op, DAG);
3296   case ISD::SET_ROUNDING:
3297     return lowerSET_ROUNDING(Op, DAG);
3298   case ISD::VP_SELECT:
3299     return lowerVPOp(Op, DAG, RISCVISD::VSELECT_VL);
3300   case ISD::VP_ADD:
3301     return lowerVPOp(Op, DAG, RISCVISD::ADD_VL);
3302   case ISD::VP_SUB:
3303     return lowerVPOp(Op, DAG, RISCVISD::SUB_VL);
3304   case ISD::VP_MUL:
3305     return lowerVPOp(Op, DAG, RISCVISD::MUL_VL);
3306   case ISD::VP_SDIV:
3307     return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL);
3308   case ISD::VP_UDIV:
3309     return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL);
3310   case ISD::VP_SREM:
3311     return lowerVPOp(Op, DAG, RISCVISD::SREM_VL);
3312   case ISD::VP_UREM:
3313     return lowerVPOp(Op, DAG, RISCVISD::UREM_VL);
3314   case ISD::VP_AND:
3315     return lowerLogicVPOp(Op, DAG, RISCVISD::VMAND_VL, RISCVISD::AND_VL);
3316   case ISD::VP_OR:
3317     return lowerLogicVPOp(Op, DAG, RISCVISD::VMOR_VL, RISCVISD::OR_VL);
3318   case ISD::VP_XOR:
3319     return lowerLogicVPOp(Op, DAG, RISCVISD::VMXOR_VL, RISCVISD::XOR_VL);
3320   case ISD::VP_ASHR:
3321     return lowerVPOp(Op, DAG, RISCVISD::SRA_VL);
3322   case ISD::VP_LSHR:
3323     return lowerVPOp(Op, DAG, RISCVISD::SRL_VL);
3324   case ISD::VP_SHL:
3325     return lowerVPOp(Op, DAG, RISCVISD::SHL_VL);
3326   case ISD::VP_FADD:
3327     return lowerVPOp(Op, DAG, RISCVISD::FADD_VL);
3328   case ISD::VP_FSUB:
3329     return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL);
3330   case ISD::VP_FMUL:
3331     return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL);
3332   case ISD::VP_FDIV:
3333     return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL);
3334   }
3335 }
3336 
3337 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
3338                              SelectionDAG &DAG, unsigned Flags) {
3339   return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
3340 }
3341 
3342 static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
3343                              SelectionDAG &DAG, unsigned Flags) {
3344   return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
3345                                    Flags);
3346 }
3347 
3348 static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
3349                              SelectionDAG &DAG, unsigned Flags) {
3350   return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
3351                                    N->getOffset(), Flags);
3352 }
3353 
3354 static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
3355                              SelectionDAG &DAG, unsigned Flags) {
3356   return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
3357 }
3358 
3359 template <class NodeTy>
3360 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
3361                                      bool IsLocal) const {
3362   SDLoc DL(N);
3363   EVT Ty = getPointerTy(DAG.getDataLayout());
3364 
3365   if (isPositionIndependent()) {
3366     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3367     if (IsLocal)
3368       // Use PC-relative addressing to access the symbol. This generates the
3369       // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
3370       // %pcrel_lo(auipc)).
3371       return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3372 
3373     // Use PC-relative addressing to access the GOT for this symbol, then load
3374     // the address from the GOT. This generates the pattern (PseudoLA sym),
3375     // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
3376     return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
3377   }
3378 
3379   switch (getTargetMachine().getCodeModel()) {
3380   default:
3381     report_fatal_error("Unsupported code model for lowering");
3382   case CodeModel::Small: {
3383     // Generate a sequence for accessing addresses within the first 2 GiB of
3384     // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
3385     SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
3386     SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
3387     SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3388     return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
3389   }
3390   case CodeModel::Medium: {
3391     // Generate a sequence for accessing addresses within any 2GiB range within
3392     // the address space. This generates the pattern (PseudoLLA sym), which
3393     // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
3394     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3395     return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3396   }
3397   }
3398 }
3399 
3400 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
3401                                                 SelectionDAG &DAG) const {
3402   SDLoc DL(Op);
3403   EVT Ty = Op.getValueType();
3404   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3405   int64_t Offset = N->getOffset();
3406   MVT XLenVT = Subtarget.getXLenVT();
3407 
3408   const GlobalValue *GV = N->getGlobal();
3409   bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
3410   SDValue Addr = getAddr(N, DAG, IsLocal);
3411 
3412   // In order to maximise the opportunity for common subexpression elimination,
3413   // emit a separate ADD node for the global address offset instead of folding
3414   // it in the global address node. Later peephole optimisations may choose to
3415   // fold it back in when profitable.
3416   if (Offset != 0)
3417     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3418                        DAG.getConstant(Offset, DL, XLenVT));
3419   return Addr;
3420 }
3421 
3422 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
3423                                                SelectionDAG &DAG) const {
3424   BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
3425 
3426   return getAddr(N, DAG);
3427 }
3428 
3429 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
3430                                                SelectionDAG &DAG) const {
3431   ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
3432 
3433   return getAddr(N, DAG);
3434 }
3435 
3436 SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
3437                                             SelectionDAG &DAG) const {
3438   JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
3439 
3440   return getAddr(N, DAG);
3441 }
3442 
3443 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
3444                                               SelectionDAG &DAG,
3445                                               bool UseGOT) const {
3446   SDLoc DL(N);
3447   EVT Ty = getPointerTy(DAG.getDataLayout());
3448   const GlobalValue *GV = N->getGlobal();
3449   MVT XLenVT = Subtarget.getXLenVT();
3450 
3451   if (UseGOT) {
3452     // Use PC-relative addressing to access the GOT for this TLS symbol, then
3453     // load the address from the GOT and add the thread pointer. This generates
3454     // the pattern (PseudoLA_TLS_IE sym), which expands to
3455     // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
3456     SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3457     SDValue Load =
3458         SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
3459 
3460     // Add the thread pointer.
3461     SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3462     return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
3463   }
3464 
3465   // Generate a sequence for accessing the address relative to the thread
3466   // pointer, with the appropriate adjustment for the thread pointer offset.
3467   // This generates the pattern
3468   // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
3469   SDValue AddrHi =
3470       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
3471   SDValue AddrAdd =
3472       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
3473   SDValue AddrLo =
3474       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
3475 
3476   SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3477   SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3478   SDValue MNAdd = SDValue(
3479       DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
3480       0);
3481   return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
3482 }
3483 
3484 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
3485                                                SelectionDAG &DAG) const {
3486   SDLoc DL(N);
3487   EVT Ty = getPointerTy(DAG.getDataLayout());
3488   IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
3489   const GlobalValue *GV = N->getGlobal();
3490 
3491   // Use a PC-relative addressing mode to access the global dynamic GOT address.
3492   // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
3493   // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
3494   SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3495   SDValue Load =
3496       SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
3497 
3498   // Prepare argument list to generate call.
3499   ArgListTy Args;
3500   ArgListEntry Entry;
3501   Entry.Node = Load;
3502   Entry.Ty = CallTy;
3503   Args.push_back(Entry);
3504 
3505   // Setup call to __tls_get_addr.
3506   TargetLowering::CallLoweringInfo CLI(DAG);
3507   CLI.setDebugLoc(DL)
3508       .setChain(DAG.getEntryNode())
3509       .setLibCallee(CallingConv::C, CallTy,
3510                     DAG.getExternalSymbol("__tls_get_addr", Ty),
3511                     std::move(Args));
3512 
3513   return LowerCallTo(CLI).first;
3514 }
3515 
3516 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
3517                                                    SelectionDAG &DAG) const {
3518   SDLoc DL(Op);
3519   EVT Ty = Op.getValueType();
3520   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3521   int64_t Offset = N->getOffset();
3522   MVT XLenVT = Subtarget.getXLenVT();
3523 
3524   TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());
3525 
3526   if (DAG.getMachineFunction().getFunction().getCallingConv() ==
3527       CallingConv::GHC)
3528     report_fatal_error("In GHC calling convention TLS is not supported");
3529 
3530   SDValue Addr;
3531   switch (Model) {
3532   case TLSModel::LocalExec:
3533     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
3534     break;
3535   case TLSModel::InitialExec:
3536     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
3537     break;
3538   case TLSModel::LocalDynamic:
3539   case TLSModel::GeneralDynamic:
3540     Addr = getDynamicTLSAddr(N, DAG);
3541     break;
3542   }
3543 
3544   // In order to maximise the opportunity for common subexpression elimination,
3545   // emit a separate ADD node for the global address offset instead of folding
3546   // it in the global address node. Later peephole optimisations may choose to
3547   // fold it back in when profitable.
3548   if (Offset != 0)
3549     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3550                        DAG.getConstant(Offset, DL, XLenVT));
3551   return Addr;
3552 }
3553 
3554 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
3555   SDValue CondV = Op.getOperand(0);
3556   SDValue TrueV = Op.getOperand(1);
3557   SDValue FalseV = Op.getOperand(2);
3558   SDLoc DL(Op);
3559   MVT VT = Op.getSimpleValueType();
3560   MVT XLenVT = Subtarget.getXLenVT();
3561 
3562   // Lower vector SELECTs to VSELECTs by splatting the condition.
3563   if (VT.isVector()) {
3564     MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
3565     SDValue CondSplat = VT.isScalableVector()
3566                             ? DAG.getSplatVector(SplatCondVT, DL, CondV)
3567                             : DAG.getSplatBuildVector(SplatCondVT, DL, CondV);
3568     return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
3569   }
3570 
3571   // If the result type is XLenVT and CondV is the output of a SETCC node
3572   // which also operated on XLenVT inputs, then merge the SETCC node into the
3573   // lowered RISCVISD::SELECT_CC to take advantage of the integer
3574   // compare+branch instructions. i.e.:
3575   // (select (setcc lhs, rhs, cc), truev, falsev)
3576   // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
3577   if (VT == XLenVT && CondV.getOpcode() == ISD::SETCC &&
3578       CondV.getOperand(0).getSimpleValueType() == XLenVT) {
3579     SDValue LHS = CondV.getOperand(0);
3580     SDValue RHS = CondV.getOperand(1);
3581     const auto *CC = cast<CondCodeSDNode>(CondV.getOperand(2));
3582     ISD::CondCode CCVal = CC->get();
3583 
3584     // Special case for a select of 2 constants that have a diffence of 1.
3585     // Normally this is done by DAGCombine, but if the select is introduced by
3586     // type legalization or op legalization, we miss it. Restricting to SETLT
3587     // case for now because that is what signed saturating add/sub need.
3588     // FIXME: We don't need the condition to be SETLT or even a SETCC,
3589     // but we would probably want to swap the true/false values if the condition
3590     // is SETGE/SETLE to avoid an XORI.
3591     if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
3592         CCVal == ISD::SETLT) {
3593       const APInt &TrueVal = cast<ConstantSDNode>(TrueV)->getAPIntValue();
3594       const APInt &FalseVal = cast<ConstantSDNode>(FalseV)->getAPIntValue();
3595       if (TrueVal - 1 == FalseVal)
3596         return DAG.getNode(ISD::ADD, DL, Op.getValueType(), CondV, FalseV);
3597       if (TrueVal + 1 == FalseVal)
3598         return DAG.getNode(ISD::SUB, DL, Op.getValueType(), FalseV, CondV);
3599     }
3600 
3601     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3602 
3603     SDValue TargetCC = DAG.getCondCode(CCVal);
3604     SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
3605     return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3606   }
3607 
3608   // Otherwise:
3609   // (select condv, truev, falsev)
3610   // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
3611   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
3612   SDValue SetNE = DAG.getCondCode(ISD::SETNE);
3613 
3614   SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
3615 
3616   return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3617 }
3618 
3619 SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
3620   SDValue CondV = Op.getOperand(1);
3621   SDLoc DL(Op);
3622   MVT XLenVT = Subtarget.getXLenVT();
3623 
3624   if (CondV.getOpcode() == ISD::SETCC &&
3625       CondV.getOperand(0).getValueType() == XLenVT) {
3626     SDValue LHS = CondV.getOperand(0);
3627     SDValue RHS = CondV.getOperand(1);
3628     ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
3629 
3630     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3631 
3632     SDValue TargetCC = DAG.getCondCode(CCVal);
3633     return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3634                        LHS, RHS, TargetCC, Op.getOperand(2));
3635   }
3636 
3637   return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3638                      CondV, DAG.getConstant(0, DL, XLenVT),
3639                      DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
3640 }
3641 
3642 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3643   MachineFunction &MF = DAG.getMachineFunction();
3644   RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
3645 
3646   SDLoc DL(Op);
3647   SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
3648                                  getPointerTy(MF.getDataLayout()));
3649 
3650   // vastart just stores the address of the VarArgsFrameIndex slot into the
3651   // memory location argument.
3652   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3653   return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
3654                       MachinePointerInfo(SV));
3655 }
3656 
3657 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
3658                                             SelectionDAG &DAG) const {
3659   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3660   MachineFunction &MF = DAG.getMachineFunction();
3661   MachineFrameInfo &MFI = MF.getFrameInfo();
3662   MFI.setFrameAddressIsTaken(true);
3663   Register FrameReg = RI.getFrameRegister(MF);
3664   int XLenInBytes = Subtarget.getXLen() / 8;
3665 
3666   EVT VT = Op.getValueType();
3667   SDLoc DL(Op);
3668   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
3669   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3670   while (Depth--) {
3671     int Offset = -(XLenInBytes * 2);
3672     SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
3673                               DAG.getIntPtrConstant(Offset, DL));
3674     FrameAddr =
3675         DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
3676   }
3677   return FrameAddr;
3678 }
3679 
3680 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
3681                                              SelectionDAG &DAG) const {
3682   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3683   MachineFunction &MF = DAG.getMachineFunction();
3684   MachineFrameInfo &MFI = MF.getFrameInfo();
3685   MFI.setReturnAddressIsTaken(true);
3686   MVT XLenVT = Subtarget.getXLenVT();
3687   int XLenInBytes = Subtarget.getXLen() / 8;
3688 
3689   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
3690     return SDValue();
3691 
3692   EVT VT = Op.getValueType();
3693   SDLoc DL(Op);
3694   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3695   if (Depth) {
3696     int Off = -XLenInBytes;
3697     SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
3698     SDValue Offset = DAG.getConstant(Off, DL, VT);
3699     return DAG.getLoad(VT, DL, DAG.getEntryNode(),
3700                        DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
3701                        MachinePointerInfo());
3702   }
3703 
3704   // Return the value of the return address register, marking it an implicit
3705   // live-in.
3706   Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
3707   return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
3708 }
3709 
3710 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
3711                                                  SelectionDAG &DAG) const {
3712   SDLoc DL(Op);
3713   SDValue Lo = Op.getOperand(0);
3714   SDValue Hi = Op.getOperand(1);
3715   SDValue Shamt = Op.getOperand(2);
3716   EVT VT = Lo.getValueType();
3717 
3718   // if Shamt-XLEN < 0: // Shamt < XLEN
3719   //   Lo = Lo << Shamt
3720   //   Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
3721   // else:
3722   //   Lo = 0
3723   //   Hi = Lo << (Shamt-XLEN)
3724 
3725   SDValue Zero = DAG.getConstant(0, DL, VT);
3726   SDValue One = DAG.getConstant(1, DL, VT);
3727   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3728   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3729   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3730   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3731 
3732   SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
3733   SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
3734   SDValue ShiftRightLo =
3735       DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
3736   SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
3737   SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
3738   SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
3739 
3740   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3741 
3742   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
3743   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3744 
3745   SDValue Parts[2] = {Lo, Hi};
3746   return DAG.getMergeValues(Parts, DL);
3747 }
3748 
3749 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
3750                                                   bool IsSRA) const {
3751   SDLoc DL(Op);
3752   SDValue Lo = Op.getOperand(0);
3753   SDValue Hi = Op.getOperand(1);
3754   SDValue Shamt = Op.getOperand(2);
3755   EVT VT = Lo.getValueType();
3756 
3757   // SRA expansion:
3758   //   if Shamt-XLEN < 0: // Shamt < XLEN
3759   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3760   //     Hi = Hi >>s Shamt
3761   //   else:
3762   //     Lo = Hi >>s (Shamt-XLEN);
3763   //     Hi = Hi >>s (XLEN-1)
3764   //
3765   // SRL expansion:
3766   //   if Shamt-XLEN < 0: // Shamt < XLEN
3767   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3768   //     Hi = Hi >>u Shamt
3769   //   else:
3770   //     Lo = Hi >>u (Shamt-XLEN);
3771   //     Hi = 0;
3772 
3773   unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
3774 
3775   SDValue Zero = DAG.getConstant(0, DL, VT);
3776   SDValue One = DAG.getConstant(1, DL, VT);
3777   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3778   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3779   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3780   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3781 
3782   SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
3783   SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
3784   SDValue ShiftLeftHi =
3785       DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
3786   SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
3787   SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
3788   SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
3789   SDValue HiFalse =
3790       IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
3791 
3792   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3793 
3794   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
3795   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3796 
3797   SDValue Parts[2] = {Lo, Hi};
3798   return DAG.getMergeValues(Parts, DL);
3799 }
3800 
3801 // Lower splats of i1 types to SETCC. For each mask vector type, we have a
3802 // legal equivalently-sized i8 type, so we can use that as a go-between.
3803 SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
3804                                                   SelectionDAG &DAG) const {
3805   SDLoc DL(Op);
3806   MVT VT = Op.getSimpleValueType();
3807   SDValue SplatVal = Op.getOperand(0);
3808   // All-zeros or all-ones splats are handled specially.
3809   if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) {
3810     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
3811     return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
3812   }
3813   if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) {
3814     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
3815     return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
3816   }
3817   MVT XLenVT = Subtarget.getXLenVT();
3818   assert(SplatVal.getValueType() == XLenVT &&
3819          "Unexpected type for i1 splat value");
3820   MVT InterVT = VT.changeVectorElementType(MVT::i8);
3821   SplatVal = DAG.getNode(ISD::AND, DL, XLenVT, SplatVal,
3822                          DAG.getConstant(1, DL, XLenVT));
3823   SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
3824   SDValue Zero = DAG.getConstant(0, DL, InterVT);
3825   return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
3826 }
3827 
3828 // Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
3829 // illegal (currently only vXi64 RV32).
3830 // FIXME: We could also catch non-constant sign-extended i32 values and lower
3831 // them to SPLAT_VECTOR_I64
3832 SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
3833                                                      SelectionDAG &DAG) const {
3834   SDLoc DL(Op);
3835   MVT VecVT = Op.getSimpleValueType();
3836   assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
3837          "Unexpected SPLAT_VECTOR_PARTS lowering");
3838 
3839   assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
3840   SDValue Lo = Op.getOperand(0);
3841   SDValue Hi = Op.getOperand(1);
3842 
3843   if (VecVT.isFixedLengthVector()) {
3844     MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
3845     SDLoc DL(Op);
3846     SDValue Mask, VL;
3847     std::tie(Mask, VL) =
3848         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
3849 
3850     SDValue Res = splatPartsI64WithVL(DL, ContainerVT, Lo, Hi, VL, DAG);
3851     return convertFromScalableVector(VecVT, Res, DAG, Subtarget);
3852   }
3853 
3854   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
3855     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
3856     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
3857     // If Hi constant is all the same sign bit as Lo, lower this as a custom
3858     // node in order to try and match RVV vector/scalar instructions.
3859     if ((LoC >> 31) == HiC)
3860       return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
3861   }
3862 
3863   // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
3864   if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
3865       isa<ConstantSDNode>(Hi.getOperand(1)) &&
3866       Hi.getConstantOperandVal(1) == 31)
3867     return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
3868 
3869   // Fall back to use a stack store and stride x0 vector load. Use X0 as VL.
3870   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VecVT, Lo, Hi,
3871                      DAG.getTargetConstant(RISCV::VLMaxSentinel, DL, MVT::i64));
3872 }
3873 
3874 // Custom-lower extensions from mask vectors by using a vselect either with 1
3875 // for zero/any-extension or -1 for sign-extension:
3876 //   (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
3877 // Note that any-extension is lowered identically to zero-extension.
3878 SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
3879                                                 int64_t ExtTrueVal) const {
3880   SDLoc DL(Op);
3881   MVT VecVT = Op.getSimpleValueType();
3882   SDValue Src = Op.getOperand(0);
3883   // Only custom-lower extensions from mask types
3884   assert(Src.getValueType().isVector() &&
3885          Src.getValueType().getVectorElementType() == MVT::i1);
3886 
3887   MVT XLenVT = Subtarget.getXLenVT();
3888   SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
3889   SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
3890 
3891   if (VecVT.isScalableVector()) {
3892     // Be careful not to introduce illegal scalar types at this stage, and be
3893     // careful also about splatting constants as on RV32, vXi64 SPLAT_VECTOR is
3894     // illegal and must be expanded. Since we know that the constants are
3895     // sign-extended 32-bit values, we use SPLAT_VECTOR_I64 directly.
3896     bool IsRV32E64 =
3897         !Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64;
3898 
3899     if (!IsRV32E64) {
3900       SplatZero = DAG.getSplatVector(VecVT, DL, SplatZero);
3901       SplatTrueVal = DAG.getSplatVector(VecVT, DL, SplatTrueVal);
3902     } else {
3903       SplatZero = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatZero);
3904       SplatTrueVal =
3905           DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatTrueVal);
3906     }
3907 
3908     return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
3909   }
3910 
3911   MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
3912   MVT I1ContainerVT =
3913       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
3914 
3915   SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);
3916 
3917   SDValue Mask, VL;
3918   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
3919 
3920   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero, VL);
3921   SplatTrueVal =
3922       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatTrueVal, VL);
3923   SDValue Select = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC,
3924                                SplatTrueVal, SplatZero, VL);
3925 
3926   return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
3927 }
3928 
3929 SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
3930     SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const {
3931   MVT ExtVT = Op.getSimpleValueType();
3932   // Only custom-lower extensions from fixed-length vector types.
3933   if (!ExtVT.isFixedLengthVector())
3934     return Op;
3935   MVT VT = Op.getOperand(0).getSimpleValueType();
3936   // Grab the canonical container type for the extended type. Infer the smaller
3937   // type from that to ensure the same number of vector elements, as we know
3938   // the LMUL will be sufficient to hold the smaller type.
3939   MVT ContainerExtVT = getContainerForFixedLengthVector(ExtVT);
3940   // Get the extended container type manually to ensure the same number of
3941   // vector elements between source and dest.
3942   MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
3943                                      ContainerExtVT.getVectorElementCount());
3944 
3945   SDValue Op1 =
3946       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
3947 
3948   SDLoc DL(Op);
3949   SDValue Mask, VL;
3950   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3951 
3952   SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);
3953 
3954   return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget);
3955 }
3956 
3957 // Custom-lower truncations from vectors to mask vectors by using a mask and a
3958 // setcc operation:
3959 //   (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
3960 SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
3961                                                   SelectionDAG &DAG) const {
3962   SDLoc DL(Op);
3963   EVT MaskVT = Op.getValueType();
3964   // Only expect to custom-lower truncations to mask types
3965   assert(MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 &&
3966          "Unexpected type for vector mask lowering");
3967   SDValue Src = Op.getOperand(0);
3968   MVT VecVT = Src.getSimpleValueType();
3969 
3970   // If this is a fixed vector, we need to convert it to a scalable vector.
3971   MVT ContainerVT = VecVT;
3972   if (VecVT.isFixedLengthVector()) {
3973     ContainerVT = getContainerForFixedLengthVector(VecVT);
3974     Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3975   }
3976 
3977   SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
3978   SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
3979 
3980   SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatOne);
3981   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero);
3982 
3983   if (VecVT.isScalableVector()) {
3984     SDValue Trunc = DAG.getNode(ISD::AND, DL, VecVT, Src, SplatOne);
3985     return DAG.getSetCC(DL, MaskVT, Trunc, SplatZero, ISD::SETNE);
3986   }
3987 
3988   SDValue Mask, VL;
3989   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
3990 
3991   MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
3992   SDValue Trunc =
3993       DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, Mask, VL);
3994   Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, Trunc, SplatZero,
3995                       DAG.getCondCode(ISD::SETNE), Mask, VL);
3996   return convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
3997 }
3998 
3999 // Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
4000 // first position of a vector, and that vector is slid up to the insert index.
4001 // By limiting the active vector length to index+1 and merging with the
4002 // original vector (with an undisturbed tail policy for elements >= VL), we
4003 // achieve the desired result of leaving all elements untouched except the one
4004 // at VL-1, which is replaced with the desired value.
4005 SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4006                                                     SelectionDAG &DAG) const {
4007   SDLoc DL(Op);
4008   MVT VecVT = Op.getSimpleValueType();
4009   SDValue Vec = Op.getOperand(0);
4010   SDValue Val = Op.getOperand(1);
4011   SDValue Idx = Op.getOperand(2);
4012 
4013   if (VecVT.getVectorElementType() == MVT::i1) {
4014     // FIXME: For now we just promote to an i8 vector and insert into that,
4015     // but this is probably not optimal.
4016     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4017     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4018     Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
4019     return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
4020   }
4021 
4022   MVT ContainerVT = VecVT;
4023   // If the operand is a fixed-length vector, convert to a scalable one.
4024   if (VecVT.isFixedLengthVector()) {
4025     ContainerVT = getContainerForFixedLengthVector(VecVT);
4026     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4027   }
4028 
4029   MVT XLenVT = Subtarget.getXLenVT();
4030 
4031   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4032   bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
4033   // Even i64-element vectors on RV32 can be lowered without scalar
4034   // legalization if the most-significant 32 bits of the value are not affected
4035   // by the sign-extension of the lower 32 bits.
4036   // TODO: We could also catch sign extensions of a 32-bit value.
4037   if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
4038     const auto *CVal = cast<ConstantSDNode>(Val);
4039     if (isInt<32>(CVal->getSExtValue())) {
4040       IsLegalInsert = true;
4041       Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4042     }
4043   }
4044 
4045   SDValue Mask, VL;
4046   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4047 
4048   SDValue ValInVec;
4049 
4050   if (IsLegalInsert) {
4051     unsigned Opc =
4052         VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
4053     if (isNullConstant(Idx)) {
4054       Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
4055       if (!VecVT.isFixedLengthVector())
4056         return Vec;
4057       return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
4058     }
4059     ValInVec =
4060         DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Val, VL);
4061   } else {
4062     // On RV32, i64-element vectors must be specially handled to place the
4063     // value at element 0, by using two vslide1up instructions in sequence on
4064     // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
4065     // this.
4066     SDValue One = DAG.getConstant(1, DL, XLenVT);
4067     SDValue ValLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, Zero);
4068     SDValue ValHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, One);
4069     MVT I32ContainerVT =
4070         MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2);
4071     SDValue I32Mask =
4072         getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
4073     // Limit the active VL to two.
4074     SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
4075     // Note: We can't pass a UNDEF to the first VSLIDE1UP_VL since an untied
4076     // undef doesn't obey the earlyclobber constraint. Just splat a zero value.
4077     ValInVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, I32ContainerVT, Zero,
4078                            InsertI64VL);
4079     // First slide in the hi value, then the lo in underneath it.
4080     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4081                            ValHi, I32Mask, InsertI64VL);
4082     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4083                            ValLo, I32Mask, InsertI64VL);
4084     // Bitcast back to the right container type.
4085     ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
4086   }
4087 
4088   // Now that the value is in a vector, slide it into position.
4089   SDValue InsertVL =
4090       DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
4091   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
4092                                 ValInVec, Idx, Mask, InsertVL);
4093   if (!VecVT.isFixedLengthVector())
4094     return Slideup;
4095   return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
4096 }
4097 
4098 // Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
4099 // extract the first element: (extractelt (slidedown vec, idx), 0). For integer
4100 // types this is done using VMV_X_S to allow us to glean information about the
4101 // sign bits of the result.
4102 SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4103                                                      SelectionDAG &DAG) const {
4104   SDLoc DL(Op);
4105   SDValue Idx = Op.getOperand(1);
4106   SDValue Vec = Op.getOperand(0);
4107   EVT EltVT = Op.getValueType();
4108   MVT VecVT = Vec.getSimpleValueType();
4109   MVT XLenVT = Subtarget.getXLenVT();
4110 
4111   if (VecVT.getVectorElementType() == MVT::i1) {
4112     // FIXME: For now we just promote to an i8 vector and extract from that,
4113     // but this is probably not optimal.
4114     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4115     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4116     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
4117   }
4118 
4119   // If this is a fixed vector, we need to convert it to a scalable vector.
4120   MVT ContainerVT = VecVT;
4121   if (VecVT.isFixedLengthVector()) {
4122     ContainerVT = getContainerForFixedLengthVector(VecVT);
4123     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4124   }
4125 
4126   // If the index is 0, the vector is already in the right position.
4127   if (!isNullConstant(Idx)) {
4128     // Use a VL of 1 to avoid processing more elements than we need.
4129     SDValue VL = DAG.getConstant(1, DL, XLenVT);
4130     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4131     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4132     Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
4133                       DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
4134   }
4135 
4136   if (!EltVT.isInteger()) {
4137     // Floating-point extracts are handled in TableGen.
4138     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
4139                        DAG.getConstant(0, DL, XLenVT));
4140   }
4141 
4142   SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
4143   return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
4144 }
4145 
4146 // Some RVV intrinsics may claim that they want an integer operand to be
4147 // promoted or expanded.
4148 static SDValue lowerVectorIntrinsicSplats(SDValue Op, SelectionDAG &DAG,
4149                                           const RISCVSubtarget &Subtarget) {
4150   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4151           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
4152          "Unexpected opcode");
4153 
4154   if (!Subtarget.hasVInstructions())
4155     return SDValue();
4156 
4157   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
4158   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
4159   SDLoc DL(Op);
4160 
4161   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
4162       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
4163   if (!II || !II->hasSplatOperand())
4164     return SDValue();
4165 
4166   unsigned SplatOp = II->SplatOperand + 1 + HasChain;
4167   assert(SplatOp < Op.getNumOperands());
4168 
4169   SmallVector<SDValue, 8> Operands(Op->op_begin(), Op->op_end());
4170   SDValue &ScalarOp = Operands[SplatOp];
4171   MVT OpVT = ScalarOp.getSimpleValueType();
4172   MVT XLenVT = Subtarget.getXLenVT();
4173 
4174   // If this isn't a scalar, or its type is XLenVT we're done.
4175   if (!OpVT.isScalarInteger() || OpVT == XLenVT)
4176     return SDValue();
4177 
4178   // Simplest case is that the operand needs to be promoted to XLenVT.
4179   if (OpVT.bitsLT(XLenVT)) {
4180     // If the operand is a constant, sign extend to increase our chances
4181     // of being able to use a .vi instruction. ANY_EXTEND would become a
4182     // a zero extend and the simm5 check in isel would fail.
4183     // FIXME: Should we ignore the upper bits in isel instead?
4184     unsigned ExtOpc =
4185         isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
4186     ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
4187     return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4188   }
4189 
4190   // Use the previous operand to get the vXi64 VT. The result might be a mask
4191   // VT for compares. Using the previous operand assumes that the previous
4192   // operand will never have a smaller element size than a scalar operand and
4193   // that a widening operation never uses SEW=64.
4194   // NOTE: If this fails the below assert, we can probably just find the
4195   // element count from any operand or result and use it to construct the VT.
4196   assert(II->SplatOperand > 0 && "Unexpected splat operand!");
4197   MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
4198 
4199   // The more complex case is when the scalar is larger than XLenVT.
4200   assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
4201          VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
4202 
4203   // If this is a sign-extended 32-bit constant, we can truncate it and rely
4204   // on the instruction to sign-extend since SEW>XLEN.
4205   if (auto *CVal = dyn_cast<ConstantSDNode>(ScalarOp)) {
4206     if (isInt<32>(CVal->getSExtValue())) {
4207       ScalarOp = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4208       return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4209     }
4210   }
4211 
4212   // We need to convert the scalar to a splat vector.
4213   // FIXME: Can we implicitly truncate the scalar if it is known to
4214   // be sign extended?
4215   SDValue VL = Op.getOperand(II->VLOperand + 1 + HasChain);
4216   assert(VL.getValueType() == XLenVT);
4217   ScalarOp = splatSplitI64WithVL(DL, VT, ScalarOp, VL, DAG);
4218   return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4219 }
4220 
4221 SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
4222                                                      SelectionDAG &DAG) const {
4223   unsigned IntNo = Op.getConstantOperandVal(0);
4224   SDLoc DL(Op);
4225   MVT XLenVT = Subtarget.getXLenVT();
4226 
4227   switch (IntNo) {
4228   default:
4229     break; // Don't custom lower most intrinsics.
4230   case Intrinsic::thread_pointer: {
4231     EVT PtrVT = getPointerTy(DAG.getDataLayout());
4232     return DAG.getRegister(RISCV::X4, PtrVT);
4233   }
4234   case Intrinsic::riscv_orc_b:
4235     // Lower to the GORCI encoding for orc.b.
4236     return DAG.getNode(RISCVISD::GORC, DL, XLenVT, Op.getOperand(1),
4237                        DAG.getConstant(7, DL, XLenVT));
4238   case Intrinsic::riscv_grev:
4239   case Intrinsic::riscv_gorc: {
4240     unsigned Opc =
4241         IntNo == Intrinsic::riscv_grev ? RISCVISD::GREV : RISCVISD::GORC;
4242     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4243   }
4244   case Intrinsic::riscv_shfl:
4245   case Intrinsic::riscv_unshfl: {
4246     unsigned Opc =
4247         IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
4248     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4249   }
4250   case Intrinsic::riscv_bcompress:
4251   case Intrinsic::riscv_bdecompress: {
4252     unsigned Opc = IntNo == Intrinsic::riscv_bcompress ? RISCVISD::BCOMPRESS
4253                                                        : RISCVISD::BDECOMPRESS;
4254     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4255   }
4256   case Intrinsic::riscv_bfp:
4257     return DAG.getNode(RISCVISD::BFP, DL, XLenVT, Op.getOperand(1),
4258                        Op.getOperand(2));
4259   case Intrinsic::riscv_fsl:
4260     return DAG.getNode(RISCVISD::FSL, DL, XLenVT, Op.getOperand(1),
4261                        Op.getOperand(2), Op.getOperand(3));
4262   case Intrinsic::riscv_fsr:
4263     return DAG.getNode(RISCVISD::FSR, DL, XLenVT, Op.getOperand(1),
4264                        Op.getOperand(2), Op.getOperand(3));
4265   case Intrinsic::riscv_vmv_x_s:
4266     assert(Op.getValueType() == XLenVT && "Unexpected VT!");
4267     return DAG.getNode(RISCVISD::VMV_X_S, DL, Op.getValueType(),
4268                        Op.getOperand(1));
4269   case Intrinsic::riscv_vmv_v_x:
4270     return lowerScalarSplat(Op.getOperand(1), Op.getOperand(2),
4271                             Op.getSimpleValueType(), DL, DAG, Subtarget);
4272   case Intrinsic::riscv_vfmv_v_f:
4273     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
4274                        Op.getOperand(1), Op.getOperand(2));
4275   case Intrinsic::riscv_vmv_s_x: {
4276     SDValue Scalar = Op.getOperand(2);
4277 
4278     if (Scalar.getValueType().bitsLE(XLenVT)) {
4279       Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
4280       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
4281                          Op.getOperand(1), Scalar, Op.getOperand(3));
4282     }
4283 
4284     assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
4285 
4286     // This is an i64 value that lives in two scalar registers. We have to
4287     // insert this in a convoluted way. First we build vXi64 splat containing
4288     // the/ two values that we assemble using some bit math. Next we'll use
4289     // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
4290     // to merge element 0 from our splat into the source vector.
4291     // FIXME: This is probably not the best way to do this, but it is
4292     // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
4293     // point.
4294     //   sw lo, (a0)
4295     //   sw hi, 4(a0)
4296     //   vlse vX, (a0)
4297     //
4298     //   vid.v      vVid
4299     //   vmseq.vx   mMask, vVid, 0
4300     //   vmerge.vvm vDest, vSrc, vVal, mMask
4301     MVT VT = Op.getSimpleValueType();
4302     SDValue Vec = Op.getOperand(1);
4303     SDValue VL = Op.getOperand(3);
4304 
4305     SDValue SplattedVal = splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
4306     SDValue SplattedIdx = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
4307                                       DAG.getConstant(0, DL, MVT::i32), VL);
4308 
4309     MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
4310     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4311     SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
4312     SDValue SelectCond =
4313         DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, VID, SplattedIdx,
4314                     DAG.getCondCode(ISD::SETEQ), Mask, VL);
4315     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, SelectCond, SplattedVal,
4316                        Vec, VL);
4317   }
4318   case Intrinsic::riscv_vslide1up:
4319   case Intrinsic::riscv_vslide1down:
4320   case Intrinsic::riscv_vslide1up_mask:
4321   case Intrinsic::riscv_vslide1down_mask: {
4322     // We need to special case these when the scalar is larger than XLen.
4323     unsigned NumOps = Op.getNumOperands();
4324     bool IsMasked = NumOps == 7;
4325     unsigned OpOffset = IsMasked ? 1 : 0;
4326     SDValue Scalar = Op.getOperand(2 + OpOffset);
4327     if (Scalar.getValueType().bitsLE(XLenVT))
4328       break;
4329 
4330     // Splatting a sign extended constant is fine.
4331     if (auto *CVal = dyn_cast<ConstantSDNode>(Scalar))
4332       if (isInt<32>(CVal->getSExtValue()))
4333         break;
4334 
4335     MVT VT = Op.getSimpleValueType();
4336     assert(VT.getVectorElementType() == MVT::i64 &&
4337            Scalar.getValueType() == MVT::i64 && "Unexpected VTs");
4338 
4339     // Convert the vector source to the equivalent nxvXi32 vector.
4340     MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
4341     SDValue Vec = DAG.getBitcast(I32VT, Op.getOperand(1 + OpOffset));
4342 
4343     SDValue ScalarLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4344                                    DAG.getConstant(0, DL, XLenVT));
4345     SDValue ScalarHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4346                                    DAG.getConstant(1, DL, XLenVT));
4347 
4348     // Double the VL since we halved SEW.
4349     SDValue VL = Op.getOperand(NumOps - (1 + OpOffset));
4350     SDValue I32VL =
4351         DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
4352 
4353     MVT I32MaskVT = MVT::getVectorVT(MVT::i1, I32VT.getVectorElementCount());
4354     SDValue I32Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, I32MaskVT, VL);
4355 
4356     // Shift the two scalar parts in using SEW=32 slide1up/slide1down
4357     // instructions.
4358     if (IntNo == Intrinsic::riscv_vslide1up ||
4359         IntNo == Intrinsic::riscv_vslide1up_mask) {
4360       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarHi,
4361                         I32Mask, I32VL);
4362       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarLo,
4363                         I32Mask, I32VL);
4364     } else {
4365       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarLo,
4366                         I32Mask, I32VL);
4367       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarHi,
4368                         I32Mask, I32VL);
4369     }
4370 
4371     // Convert back to nxvXi64.
4372     Vec = DAG.getBitcast(VT, Vec);
4373 
4374     if (!IsMasked)
4375       return Vec;
4376 
4377     // Apply mask after the operation.
4378     SDValue Mask = Op.getOperand(NumOps - 3);
4379     SDValue MaskedOff = Op.getOperand(1);
4380     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, Mask, Vec, MaskedOff, VL);
4381   }
4382   }
4383 
4384   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4385 }
4386 
4387 SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
4388                                                     SelectionDAG &DAG) const {
4389   unsigned IntNo = Op.getConstantOperandVal(1);
4390   switch (IntNo) {
4391   default:
4392     break;
4393   case Intrinsic::riscv_masked_strided_load: {
4394     SDLoc DL(Op);
4395     MVT XLenVT = Subtarget.getXLenVT();
4396 
4397     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4398     // the selection of the masked intrinsics doesn't do this for us.
4399     SDValue Mask = Op.getOperand(5);
4400     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4401 
4402     MVT VT = Op->getSimpleValueType(0);
4403     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4404 
4405     SDValue PassThru = Op.getOperand(2);
4406     if (!IsUnmasked) {
4407       MVT MaskVT =
4408           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4409       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4410       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
4411     }
4412 
4413     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4414 
4415     SDValue IntID = DAG.getTargetConstant(
4416         IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask, DL,
4417         XLenVT);
4418 
4419     auto *Load = cast<MemIntrinsicSDNode>(Op);
4420     SmallVector<SDValue, 8> Ops{Load->getChain(), IntID};
4421     if (!IsUnmasked)
4422       Ops.push_back(PassThru);
4423     Ops.push_back(Op.getOperand(3)); // Ptr
4424     Ops.push_back(Op.getOperand(4)); // Stride
4425     if (!IsUnmasked)
4426       Ops.push_back(Mask);
4427     Ops.push_back(VL);
4428     if (!IsUnmasked) {
4429       SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
4430       Ops.push_back(Policy);
4431     }
4432 
4433     SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
4434     SDValue Result =
4435         DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
4436                                 Load->getMemoryVT(), Load->getMemOperand());
4437     SDValue Chain = Result.getValue(1);
4438     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
4439     return DAG.getMergeValues({Result, Chain}, DL);
4440   }
4441   }
4442 
4443   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4444 }
4445 
4446 SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
4447                                                  SelectionDAG &DAG) const {
4448   unsigned IntNo = Op.getConstantOperandVal(1);
4449   switch (IntNo) {
4450   default:
4451     break;
4452   case Intrinsic::riscv_masked_strided_store: {
4453     SDLoc DL(Op);
4454     MVT XLenVT = Subtarget.getXLenVT();
4455 
4456     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4457     // the selection of the masked intrinsics doesn't do this for us.
4458     SDValue Mask = Op.getOperand(5);
4459     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4460 
4461     SDValue Val = Op.getOperand(2);
4462     MVT VT = Val.getSimpleValueType();
4463     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4464 
4465     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
4466     if (!IsUnmasked) {
4467       MVT MaskVT =
4468           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4469       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4470     }
4471 
4472     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4473 
4474     SDValue IntID = DAG.getTargetConstant(
4475         IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask, DL,
4476         XLenVT);
4477 
4478     auto *Store = cast<MemIntrinsicSDNode>(Op);
4479     SmallVector<SDValue, 8> Ops{Store->getChain(), IntID};
4480     Ops.push_back(Val);
4481     Ops.push_back(Op.getOperand(3)); // Ptr
4482     Ops.push_back(Op.getOperand(4)); // Stride
4483     if (!IsUnmasked)
4484       Ops.push_back(Mask);
4485     Ops.push_back(VL);
4486 
4487     return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(),
4488                                    Ops, Store->getMemoryVT(),
4489                                    Store->getMemOperand());
4490   }
4491   }
4492 
4493   return SDValue();
4494 }
4495 
4496 static MVT getLMUL1VT(MVT VT) {
4497   assert(VT.getVectorElementType().getSizeInBits() <= 64 &&
4498          "Unexpected vector MVT");
4499   return MVT::getScalableVectorVT(
4500       VT.getVectorElementType(),
4501       RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits());
4502 }
4503 
4504 static unsigned getRVVReductionOp(unsigned ISDOpcode) {
4505   switch (ISDOpcode) {
4506   default:
4507     llvm_unreachable("Unhandled reduction");
4508   case ISD::VECREDUCE_ADD:
4509     return RISCVISD::VECREDUCE_ADD_VL;
4510   case ISD::VECREDUCE_UMAX:
4511     return RISCVISD::VECREDUCE_UMAX_VL;
4512   case ISD::VECREDUCE_SMAX:
4513     return RISCVISD::VECREDUCE_SMAX_VL;
4514   case ISD::VECREDUCE_UMIN:
4515     return RISCVISD::VECREDUCE_UMIN_VL;
4516   case ISD::VECREDUCE_SMIN:
4517     return RISCVISD::VECREDUCE_SMIN_VL;
4518   case ISD::VECREDUCE_AND:
4519     return RISCVISD::VECREDUCE_AND_VL;
4520   case ISD::VECREDUCE_OR:
4521     return RISCVISD::VECREDUCE_OR_VL;
4522   case ISD::VECREDUCE_XOR:
4523     return RISCVISD::VECREDUCE_XOR_VL;
4524   }
4525 }
4526 
4527 SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
4528                                                          SelectionDAG &DAG,
4529                                                          bool IsVP) const {
4530   SDLoc DL(Op);
4531   SDValue Vec = Op.getOperand(IsVP ? 1 : 0);
4532   MVT VecVT = Vec.getSimpleValueType();
4533   assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
4534           Op.getOpcode() == ISD::VECREDUCE_OR ||
4535           Op.getOpcode() == ISD::VECREDUCE_XOR ||
4536           Op.getOpcode() == ISD::VP_REDUCE_AND ||
4537           Op.getOpcode() == ISD::VP_REDUCE_OR ||
4538           Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
4539          "Unexpected reduction lowering");
4540 
4541   MVT XLenVT = Subtarget.getXLenVT();
4542   assert(Op.getValueType() == XLenVT &&
4543          "Expected reduction output to be legalized to XLenVT");
4544 
4545   MVT ContainerVT = VecVT;
4546   if (VecVT.isFixedLengthVector()) {
4547     ContainerVT = getContainerForFixedLengthVector(VecVT);
4548     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4549   }
4550 
4551   SDValue Mask, VL;
4552   if (IsVP) {
4553     Mask = Op.getOperand(2);
4554     VL = Op.getOperand(3);
4555   } else {
4556     std::tie(Mask, VL) =
4557         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4558   }
4559 
4560   unsigned BaseOpc;
4561   ISD::CondCode CC;
4562   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4563 
4564   switch (Op.getOpcode()) {
4565   default:
4566     llvm_unreachable("Unhandled reduction");
4567   case ISD::VECREDUCE_AND:
4568   case ISD::VP_REDUCE_AND: {
4569     // vcpop ~x == 0
4570     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
4571     Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
4572     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4573     CC = ISD::SETEQ;
4574     BaseOpc = ISD::AND;
4575     break;
4576   }
4577   case ISD::VECREDUCE_OR:
4578   case ISD::VP_REDUCE_OR:
4579     // vcpop x != 0
4580     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4581     CC = ISD::SETNE;
4582     BaseOpc = ISD::OR;
4583     break;
4584   case ISD::VECREDUCE_XOR:
4585   case ISD::VP_REDUCE_XOR: {
4586     // ((vcpop x) & 1) != 0
4587     SDValue One = DAG.getConstant(1, DL, XLenVT);
4588     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4589     Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
4590     CC = ISD::SETNE;
4591     BaseOpc = ISD::XOR;
4592     break;
4593   }
4594   }
4595 
4596   SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC);
4597 
4598   if (!IsVP)
4599     return SetCC;
4600 
4601   // Now include the start value in the operation.
4602   // Note that we must return the start value when no elements are operated
4603   // upon. The vcpop instructions we've emitted in each case above will return
4604   // 0 for an inactive vector, and so we've already received the neutral value:
4605   // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
4606   // can simply include the start value.
4607   return DAG.getNode(BaseOpc, DL, XLenVT, SetCC, Op.getOperand(0));
4608 }
4609 
4610 SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
4611                                             SelectionDAG &DAG) const {
4612   SDLoc DL(Op);
4613   SDValue Vec = Op.getOperand(0);
4614   EVT VecEVT = Vec.getValueType();
4615 
4616   unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
4617 
4618   // Due to ordering in legalize types we may have a vector type that needs to
4619   // be split. Do that manually so we can get down to a legal type.
4620   while (getTypeAction(*DAG.getContext(), VecEVT) ==
4621          TargetLowering::TypeSplitVector) {
4622     SDValue Lo, Hi;
4623     std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
4624     VecEVT = Lo.getValueType();
4625     Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
4626   }
4627 
4628   // TODO: The type may need to be widened rather than split. Or widened before
4629   // it can be split.
4630   if (!isTypeLegal(VecEVT))
4631     return SDValue();
4632 
4633   MVT VecVT = VecEVT.getSimpleVT();
4634   MVT VecEltVT = VecVT.getVectorElementType();
4635   unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());
4636 
4637   MVT ContainerVT = VecVT;
4638   if (VecVT.isFixedLengthVector()) {
4639     ContainerVT = getContainerForFixedLengthVector(VecVT);
4640     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4641   }
4642 
4643   MVT M1VT = getLMUL1VT(ContainerVT);
4644   MVT XLenVT = Subtarget.getXLenVT();
4645 
4646   SDValue Mask, VL;
4647   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4648 
4649   SDValue NeutralElem =
4650       DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
4651   SDValue IdentitySplat = lowerScalarSplat(
4652       NeutralElem, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
4653   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT), Vec,
4654                                   IdentitySplat, Mask, VL);
4655   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
4656                              DAG.getConstant(0, DL, XLenVT));
4657   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
4658 }
4659 
4660 // Given a reduction op, this function returns the matching reduction opcode,
4661 // the vector SDValue and the scalar SDValue required to lower this to a
4662 // RISCVISD node.
4663 static std::tuple<unsigned, SDValue, SDValue>
4664 getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) {
4665   SDLoc DL(Op);
4666   auto Flags = Op->getFlags();
4667   unsigned Opcode = Op.getOpcode();
4668   unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Opcode);
4669   switch (Opcode) {
4670   default:
4671     llvm_unreachable("Unhandled reduction");
4672   case ISD::VECREDUCE_FADD: {
4673     // Use positive zero if we can. It is cheaper to materialize.
4674     SDValue Zero =
4675         DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
4676     return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero);
4677   }
4678   case ISD::VECREDUCE_SEQ_FADD:
4679     return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
4680                            Op.getOperand(0));
4681   case ISD::VECREDUCE_FMIN:
4682     return std::make_tuple(RISCVISD::VECREDUCE_FMIN_VL, Op.getOperand(0),
4683                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
4684   case ISD::VECREDUCE_FMAX:
4685     return std::make_tuple(RISCVISD::VECREDUCE_FMAX_VL, Op.getOperand(0),
4686                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
4687   }
4688 }
4689 
4690 SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
4691                                               SelectionDAG &DAG) const {
4692   SDLoc DL(Op);
4693   MVT VecEltVT = Op.getSimpleValueType();
4694 
4695   unsigned RVVOpcode;
4696   SDValue VectorVal, ScalarVal;
4697   std::tie(RVVOpcode, VectorVal, ScalarVal) =
4698       getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT);
4699   MVT VecVT = VectorVal.getSimpleValueType();
4700 
4701   MVT ContainerVT = VecVT;
4702   if (VecVT.isFixedLengthVector()) {
4703     ContainerVT = getContainerForFixedLengthVector(VecVT);
4704     VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
4705   }
4706 
4707   MVT M1VT = getLMUL1VT(VectorVal.getSimpleValueType());
4708   MVT XLenVT = Subtarget.getXLenVT();
4709 
4710   SDValue Mask, VL;
4711   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4712 
4713   SDValue ScalarSplat = lowerScalarSplat(
4714       ScalarVal, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
4715   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT),
4716                                   VectorVal, ScalarSplat, Mask, VL);
4717   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
4718                      DAG.getConstant(0, DL, XLenVT));
4719 }
4720 
4721 static unsigned getRVVVPReductionOp(unsigned ISDOpcode) {
4722   switch (ISDOpcode) {
4723   default:
4724     llvm_unreachable("Unhandled reduction");
4725   case ISD::VP_REDUCE_ADD:
4726     return RISCVISD::VECREDUCE_ADD_VL;
4727   case ISD::VP_REDUCE_UMAX:
4728     return RISCVISD::VECREDUCE_UMAX_VL;
4729   case ISD::VP_REDUCE_SMAX:
4730     return RISCVISD::VECREDUCE_SMAX_VL;
4731   case ISD::VP_REDUCE_UMIN:
4732     return RISCVISD::VECREDUCE_UMIN_VL;
4733   case ISD::VP_REDUCE_SMIN:
4734     return RISCVISD::VECREDUCE_SMIN_VL;
4735   case ISD::VP_REDUCE_AND:
4736     return RISCVISD::VECREDUCE_AND_VL;
4737   case ISD::VP_REDUCE_OR:
4738     return RISCVISD::VECREDUCE_OR_VL;
4739   case ISD::VP_REDUCE_XOR:
4740     return RISCVISD::VECREDUCE_XOR_VL;
4741   case ISD::VP_REDUCE_FADD:
4742     return RISCVISD::VECREDUCE_FADD_VL;
4743   case ISD::VP_REDUCE_SEQ_FADD:
4744     return RISCVISD::VECREDUCE_SEQ_FADD_VL;
4745   case ISD::VP_REDUCE_FMAX:
4746     return RISCVISD::VECREDUCE_FMAX_VL;
4747   case ISD::VP_REDUCE_FMIN:
4748     return RISCVISD::VECREDUCE_FMIN_VL;
4749   }
4750 }
4751 
4752 SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
4753                                            SelectionDAG &DAG) const {
4754   SDLoc DL(Op);
4755   SDValue Vec = Op.getOperand(1);
4756   EVT VecEVT = Vec.getValueType();
4757 
4758   // TODO: The type may need to be widened rather than split. Or widened before
4759   // it can be split.
4760   if (!isTypeLegal(VecEVT))
4761     return SDValue();
4762 
4763   MVT VecVT = VecEVT.getSimpleVT();
4764   MVT VecEltVT = VecVT.getVectorElementType();
4765   unsigned RVVOpcode = getRVVVPReductionOp(Op.getOpcode());
4766 
4767   MVT ContainerVT = VecVT;
4768   if (VecVT.isFixedLengthVector()) {
4769     ContainerVT = getContainerForFixedLengthVector(VecVT);
4770     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4771   }
4772 
4773   SDValue VL = Op.getOperand(3);
4774   SDValue Mask = Op.getOperand(2);
4775 
4776   MVT M1VT = getLMUL1VT(ContainerVT);
4777   MVT XLenVT = Subtarget.getXLenVT();
4778   MVT ResVT = !VecVT.isInteger() || VecEltVT.bitsGE(XLenVT) ? VecEltVT : XLenVT;
4779 
4780   SDValue StartSplat =
4781       lowerScalarSplat(Op.getOperand(0), DAG.getConstant(1, DL, XLenVT), M1VT,
4782                        DL, DAG, Subtarget);
4783   SDValue Reduction =
4784       DAG.getNode(RVVOpcode, DL, M1VT, StartSplat, Vec, StartSplat, Mask, VL);
4785   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction,
4786                              DAG.getConstant(0, DL, XLenVT));
4787   if (!VecVT.isInteger())
4788     return Elt0;
4789   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
4790 }
4791 
4792 SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
4793                                                    SelectionDAG &DAG) const {
4794   SDValue Vec = Op.getOperand(0);
4795   SDValue SubVec = Op.getOperand(1);
4796   MVT VecVT = Vec.getSimpleValueType();
4797   MVT SubVecVT = SubVec.getSimpleValueType();
4798 
4799   SDLoc DL(Op);
4800   MVT XLenVT = Subtarget.getXLenVT();
4801   unsigned OrigIdx = Op.getConstantOperandVal(2);
4802   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
4803 
4804   // We don't have the ability to slide mask vectors up indexed by their i1
4805   // elements; the smallest we can do is i8. Often we are able to bitcast to
4806   // equivalent i8 vectors. Note that when inserting a fixed-length vector
4807   // into a scalable one, we might not necessarily have enough scalable
4808   // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
4809   if (SubVecVT.getVectorElementType() == MVT::i1 &&
4810       (OrigIdx != 0 || !Vec.isUndef())) {
4811     if (VecVT.getVectorMinNumElements() >= 8 &&
4812         SubVecVT.getVectorMinNumElements() >= 8) {
4813       assert(OrigIdx % 8 == 0 && "Invalid index");
4814       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
4815              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
4816              "Unexpected mask vector lowering");
4817       OrigIdx /= 8;
4818       SubVecVT =
4819           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
4820                            SubVecVT.isScalableVector());
4821       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
4822                                VecVT.isScalableVector());
4823       Vec = DAG.getBitcast(VecVT, Vec);
4824       SubVec = DAG.getBitcast(SubVecVT, SubVec);
4825     } else {
4826       // We can't slide this mask vector up indexed by its i1 elements.
4827       // This poses a problem when we wish to insert a scalable vector which
4828       // can't be re-expressed as a larger type. Just choose the slow path and
4829       // extend to a larger type, then truncate back down.
4830       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
4831       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
4832       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
4833       SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
4834       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
4835                         Op.getOperand(2));
4836       SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
4837       return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
4838     }
4839   }
4840 
4841   // If the subvector vector is a fixed-length type, we cannot use subregister
4842   // manipulation to simplify the codegen; we don't know which register of a
4843   // LMUL group contains the specific subvector as we only know the minimum
4844   // register size. Therefore we must slide the vector group up the full
4845   // amount.
4846   if (SubVecVT.isFixedLengthVector()) {
4847     if (OrigIdx == 0 && Vec.isUndef() && !VecVT.isFixedLengthVector())
4848       return Op;
4849     MVT ContainerVT = VecVT;
4850     if (VecVT.isFixedLengthVector()) {
4851       ContainerVT = getContainerForFixedLengthVector(VecVT);
4852       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4853     }
4854     SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
4855                          DAG.getUNDEF(ContainerVT), SubVec,
4856                          DAG.getConstant(0, DL, XLenVT));
4857     if (OrigIdx == 0 && Vec.isUndef() && VecVT.isFixedLengthVector()) {
4858       SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
4859       return DAG.getBitcast(Op.getValueType(), SubVec);
4860     }
4861     SDValue Mask =
4862         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
4863     // Set the vector length to only the number of elements we care about. Note
4864     // that for slideup this includes the offset.
4865     SDValue VL =
4866         DAG.getConstant(OrigIdx + SubVecVT.getVectorNumElements(), DL, XLenVT);
4867     SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
4868     SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
4869                                   SubVec, SlideupAmt, Mask, VL);
4870     if (VecVT.isFixedLengthVector())
4871       Slideup = convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
4872     return DAG.getBitcast(Op.getValueType(), Slideup);
4873   }
4874 
4875   unsigned SubRegIdx, RemIdx;
4876   std::tie(SubRegIdx, RemIdx) =
4877       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
4878           VecVT, SubVecVT, OrigIdx, TRI);
4879 
4880   RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecVT);
4881   bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
4882                          SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
4883                          SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
4884 
4885   // 1. If the Idx has been completely eliminated and this subvector's size is
4886   // a vector register or a multiple thereof, or the surrounding elements are
4887   // undef, then this is a subvector insert which naturally aligns to a vector
4888   // register. These can easily be handled using subregister manipulation.
4889   // 2. If the subvector is smaller than a vector register, then the insertion
4890   // must preserve the undisturbed elements of the register. We do this by
4891   // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
4892   // (which resolves to a subregister copy), performing a VSLIDEUP to place the
4893   // subvector within the vector register, and an INSERT_SUBVECTOR of that
4894   // LMUL=1 type back into the larger vector (resolving to another subregister
4895   // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
4896   // to avoid allocating a large register group to hold our subvector.
4897   if (RemIdx == 0 && (!IsSubVecPartReg || Vec.isUndef()))
4898     return Op;
4899 
4900   // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
4901   // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
4902   // (in our case undisturbed). This means we can set up a subvector insertion
4903   // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
4904   // size of the subvector.
4905   MVT InterSubVT = VecVT;
4906   SDValue AlignedExtract = Vec;
4907   unsigned AlignedIdx = OrigIdx - RemIdx;
4908   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
4909     InterSubVT = getLMUL1VT(VecVT);
4910     // Extract a subvector equal to the nearest full vector register type. This
4911     // should resolve to a EXTRACT_SUBREG instruction.
4912     AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
4913                                  DAG.getConstant(AlignedIdx, DL, XLenVT));
4914   }
4915 
4916   SDValue SlideupAmt = DAG.getConstant(RemIdx, DL, XLenVT);
4917   // For scalable vectors this must be further multiplied by vscale.
4918   SlideupAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlideupAmt);
4919 
4920   SDValue Mask, VL;
4921   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
4922 
4923   // Construct the vector length corresponding to RemIdx + length(SubVecVT).
4924   VL = DAG.getConstant(SubVecVT.getVectorMinNumElements(), DL, XLenVT);
4925   VL = DAG.getNode(ISD::VSCALE, DL, XLenVT, VL);
4926   VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
4927 
4928   SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
4929                        DAG.getUNDEF(InterSubVT), SubVec,
4930                        DAG.getConstant(0, DL, XLenVT));
4931 
4932   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, InterSubVT,
4933                                 AlignedExtract, SubVec, SlideupAmt, Mask, VL);
4934 
4935   // If required, insert this subvector back into the correct vector register.
4936   // This should resolve to an INSERT_SUBREG instruction.
4937   if (VecVT.bitsGT(InterSubVT))
4938     Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, Slideup,
4939                           DAG.getConstant(AlignedIdx, DL, XLenVT));
4940 
4941   // We might have bitcast from a mask type: cast back to the original type if
4942   // required.
4943   return DAG.getBitcast(Op.getSimpleValueType(), Slideup);
4944 }
4945 
4946 SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
4947                                                     SelectionDAG &DAG) const {
4948   SDValue Vec = Op.getOperand(0);
4949   MVT SubVecVT = Op.getSimpleValueType();
4950   MVT VecVT = Vec.getSimpleValueType();
4951 
4952   SDLoc DL(Op);
4953   MVT XLenVT = Subtarget.getXLenVT();
4954   unsigned OrigIdx = Op.getConstantOperandVal(1);
4955   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
4956 
4957   // We don't have the ability to slide mask vectors down indexed by their i1
4958   // elements; the smallest we can do is i8. Often we are able to bitcast to
4959   // equivalent i8 vectors. Note that when extracting a fixed-length vector
4960   // from a scalable one, we might not necessarily have enough scalable
4961   // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
4962   if (SubVecVT.getVectorElementType() == MVT::i1 && OrigIdx != 0) {
4963     if (VecVT.getVectorMinNumElements() >= 8 &&
4964         SubVecVT.getVectorMinNumElements() >= 8) {
4965       assert(OrigIdx % 8 == 0 && "Invalid index");
4966       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
4967              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
4968              "Unexpected mask vector lowering");
4969       OrigIdx /= 8;
4970       SubVecVT =
4971           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
4972                            SubVecVT.isScalableVector());
4973       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
4974                                VecVT.isScalableVector());
4975       Vec = DAG.getBitcast(VecVT, Vec);
4976     } else {
4977       // We can't slide this mask vector down, indexed by its i1 elements.
4978       // This poses a problem when we wish to extract a scalable vector which
4979       // can't be re-expressed as a larger type. Just choose the slow path and
4980       // extend to a larger type, then truncate back down.
4981       // TODO: We could probably improve this when extracting certain fixed
4982       // from fixed, where we can extract as i8 and shift the correct element
4983       // right to reach the desired subvector?
4984       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
4985       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
4986       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
4987       Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
4988                         Op.getOperand(1));
4989       SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
4990       return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
4991     }
4992   }
4993 
4994   // If the subvector vector is a fixed-length type, we cannot use subregister
4995   // manipulation to simplify the codegen; we don't know which register of a
4996   // LMUL group contains the specific subvector as we only know the minimum
4997   // register size. Therefore we must slide the vector group down the full
4998   // amount.
4999   if (SubVecVT.isFixedLengthVector()) {
5000     // With an index of 0 this is a cast-like subvector, which can be performed
5001     // with subregister operations.
5002     if (OrigIdx == 0)
5003       return Op;
5004     MVT ContainerVT = VecVT;
5005     if (VecVT.isFixedLengthVector()) {
5006       ContainerVT = getContainerForFixedLengthVector(VecVT);
5007       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5008     }
5009     SDValue Mask =
5010         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5011     // Set the vector length to only the number of elements we care about. This
5012     // avoids sliding down elements we're going to discard straight away.
5013     SDValue VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
5014     SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5015     SDValue Slidedown =
5016         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
5017                     DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
5018     // Now we can use a cast-like subvector extract to get the result.
5019     Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5020                             DAG.getConstant(0, DL, XLenVT));
5021     return DAG.getBitcast(Op.getValueType(), Slidedown);
5022   }
5023 
5024   unsigned SubRegIdx, RemIdx;
5025   std::tie(SubRegIdx, RemIdx) =
5026       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5027           VecVT, SubVecVT, OrigIdx, TRI);
5028 
5029   // If the Idx has been completely eliminated then this is a subvector extract
5030   // which naturally aligns to a vector register. These can easily be handled
5031   // using subregister manipulation.
5032   if (RemIdx == 0)
5033     return Op;
5034 
5035   // Else we must shift our vector register directly to extract the subvector.
5036   // Do this using VSLIDEDOWN.
5037 
5038   // If the vector type is an LMUL-group type, extract a subvector equal to the
5039   // nearest full vector register type. This should resolve to a EXTRACT_SUBREG
5040   // instruction.
5041   MVT InterSubVT = VecVT;
5042   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5043     InterSubVT = getLMUL1VT(VecVT);
5044     Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5045                       DAG.getConstant(OrigIdx - RemIdx, DL, XLenVT));
5046   }
5047 
5048   // Slide this vector register down by the desired number of elements in order
5049   // to place the desired subvector starting at element 0.
5050   SDValue SlidedownAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5051   // For scalable vectors this must be further multiplied by vscale.
5052   SlidedownAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlidedownAmt);
5053 
5054   SDValue Mask, VL;
5055   std::tie(Mask, VL) = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
5056   SDValue Slidedown =
5057       DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, InterSubVT,
5058                   DAG.getUNDEF(InterSubVT), Vec, SlidedownAmt, Mask, VL);
5059 
5060   // Now the vector is in the right position, extract our final subvector. This
5061   // should resolve to a COPY.
5062   Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5063                           DAG.getConstant(0, DL, XLenVT));
5064 
5065   // We might have bitcast from a mask type: cast back to the original type if
5066   // required.
5067   return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
5068 }
5069 
5070 // Lower step_vector to the vid instruction. Any non-identity step value must
5071 // be accounted for my manual expansion.
5072 SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
5073                                               SelectionDAG &DAG) const {
5074   SDLoc DL(Op);
5075   MVT VT = Op.getSimpleValueType();
5076   MVT XLenVT = Subtarget.getXLenVT();
5077   SDValue Mask, VL;
5078   std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
5079   SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
5080   uint64_t StepValImm = Op.getConstantOperandVal(0);
5081   if (StepValImm != 1) {
5082     if (isPowerOf2_64(StepValImm)) {
5083       SDValue StepVal =
5084           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
5085                       DAG.getConstant(Log2_64(StepValImm), DL, XLenVT));
5086       StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
5087     } else {
5088       SDValue StepVal = lowerScalarSplat(
5089           DAG.getConstant(StepValImm, DL, VT.getVectorElementType()), VL, VT,
5090           DL, DAG, Subtarget);
5091       StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
5092     }
5093   }
5094   return StepVec;
5095 }
5096 
5097 // Implement vector_reverse using vrgather.vv with indices determined by
5098 // subtracting the id of each element from (VLMAX-1). This will convert
5099 // the indices like so:
5100 // (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
5101 // TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
5102 SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
5103                                                  SelectionDAG &DAG) const {
5104   SDLoc DL(Op);
5105   MVT VecVT = Op.getSimpleValueType();
5106   unsigned EltSize = VecVT.getScalarSizeInBits();
5107   unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
5108 
5109   unsigned MaxVLMAX = 0;
5110   unsigned VectorBitsMax = Subtarget.getMaxRVVVectorSizeInBits();
5111   if (VectorBitsMax != 0)
5112     MaxVLMAX = ((VectorBitsMax / EltSize) * MinSize) / RISCV::RVVBitsPerBlock;
5113 
5114   unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
5115   MVT IntVT = VecVT.changeVectorElementTypeToInteger();
5116 
5117   // If this is SEW=8 and VLMAX is unknown or more than 256, we need
5118   // to use vrgatherei16.vv.
5119   // TODO: It's also possible to use vrgatherei16.vv for other types to
5120   // decrease register width for the index calculation.
5121   if ((MaxVLMAX == 0 || MaxVLMAX > 256) && EltSize == 8) {
5122     // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
5123     // Reverse each half, then reassemble them in reverse order.
5124     // NOTE: It's also possible that after splitting that VLMAX no longer
5125     // requires vrgatherei16.vv.
5126     if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
5127       SDValue Lo, Hi;
5128       std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
5129       EVT LoVT, HiVT;
5130       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
5131       Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
5132       Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
5133       // Reassemble the low and high pieces reversed.
5134       // FIXME: This is a CONCAT_VECTORS.
5135       SDValue Res =
5136           DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
5137                       DAG.getIntPtrConstant(0, DL));
5138       return DAG.getNode(
5139           ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo,
5140           DAG.getIntPtrConstant(LoVT.getVectorMinNumElements(), DL));
5141     }
5142 
5143     // Just promote the int type to i16 which will double the LMUL.
5144     IntVT = MVT::getVectorVT(MVT::i16, VecVT.getVectorElementCount());
5145     GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
5146   }
5147 
5148   MVT XLenVT = Subtarget.getXLenVT();
5149   SDValue Mask, VL;
5150   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5151 
5152   // Calculate VLMAX-1 for the desired SEW.
5153   unsigned MinElts = VecVT.getVectorMinNumElements();
5154   SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT,
5155                               DAG.getConstant(MinElts, DL, XLenVT));
5156   SDValue VLMinus1 =
5157       DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DAG.getConstant(1, DL, XLenVT));
5158 
5159   // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
5160   bool IsRV32E64 =
5161       !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
5162   SDValue SplatVL;
5163   if (!IsRV32E64)
5164     SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
5165   else
5166     SplatVL = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, IntVT, VLMinus1);
5167 
5168   SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
5169   SDValue Indices =
5170       DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, Mask, VL);
5171 
5172   return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices, Mask, VL);
5173 }
5174 
5175 SDValue
5176 RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
5177                                                      SelectionDAG &DAG) const {
5178   SDLoc DL(Op);
5179   auto *Load = cast<LoadSDNode>(Op);
5180 
5181   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5182                                         Load->getMemoryVT(),
5183                                         *Load->getMemOperand()) &&
5184          "Expecting a correctly-aligned load");
5185 
5186   MVT VT = Op.getSimpleValueType();
5187   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5188 
5189   SDValue VL =
5190       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5191 
5192   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5193   SDValue NewLoad = DAG.getMemIntrinsicNode(
5194       RISCVISD::VLE_VL, DL, VTs, {Load->getChain(), Load->getBasePtr(), VL},
5195       Load->getMemoryVT(), Load->getMemOperand());
5196 
5197   SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
5198   return DAG.getMergeValues({Result, Load->getChain()}, DL);
5199 }
5200 
5201 SDValue
5202 RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
5203                                                       SelectionDAG &DAG) const {
5204   SDLoc DL(Op);
5205   auto *Store = cast<StoreSDNode>(Op);
5206 
5207   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5208                                         Store->getMemoryVT(),
5209                                         *Store->getMemOperand()) &&
5210          "Expecting a correctly-aligned store");
5211 
5212   SDValue StoreVal = Store->getValue();
5213   MVT VT = StoreVal.getSimpleValueType();
5214 
5215   // If the size less than a byte, we need to pad with zeros to make a byte.
5216   if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
5217     VT = MVT::v8i1;
5218     StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
5219                            DAG.getConstant(0, DL, VT), StoreVal,
5220                            DAG.getIntPtrConstant(0, DL));
5221   }
5222 
5223   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5224 
5225   SDValue VL =
5226       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5227 
5228   SDValue NewValue =
5229       convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
5230   return DAG.getMemIntrinsicNode(
5231       RISCVISD::VSE_VL, DL, DAG.getVTList(MVT::Other),
5232       {Store->getChain(), NewValue, Store->getBasePtr(), VL},
5233       Store->getMemoryVT(), Store->getMemOperand());
5234 }
5235 
5236 SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
5237                                              SelectionDAG &DAG) const {
5238   SDLoc DL(Op);
5239   MVT VT = Op.getSimpleValueType();
5240 
5241   const auto *MemSD = cast<MemSDNode>(Op);
5242   EVT MemVT = MemSD->getMemoryVT();
5243   MachineMemOperand *MMO = MemSD->getMemOperand();
5244   SDValue Chain = MemSD->getChain();
5245   SDValue BasePtr = MemSD->getBasePtr();
5246 
5247   SDValue Mask, PassThru, VL;
5248   if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Op)) {
5249     Mask = VPLoad->getMask();
5250     PassThru = DAG.getUNDEF(VT);
5251     VL = VPLoad->getVectorLength();
5252   } else {
5253     const auto *MLoad = cast<MaskedLoadSDNode>(Op);
5254     Mask = MLoad->getMask();
5255     PassThru = MLoad->getPassThru();
5256   }
5257 
5258   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5259 
5260   MVT XLenVT = Subtarget.getXLenVT();
5261 
5262   MVT ContainerVT = VT;
5263   if (VT.isFixedLengthVector()) {
5264     ContainerVT = getContainerForFixedLengthVector(VT);
5265     PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5266     if (!IsUnmasked) {
5267       MVT MaskVT =
5268           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5269       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5270     }
5271   }
5272 
5273   if (!VL)
5274     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5275 
5276   unsigned IntID =
5277       IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
5278   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5279   if (!IsUnmasked)
5280     Ops.push_back(PassThru);
5281   Ops.push_back(BasePtr);
5282   if (!IsUnmasked)
5283     Ops.push_back(Mask);
5284   Ops.push_back(VL);
5285   if (!IsUnmasked)
5286     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5287 
5288   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5289 
5290   SDValue Result =
5291       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5292   Chain = Result.getValue(1);
5293 
5294   if (VT.isFixedLengthVector())
5295     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5296 
5297   return DAG.getMergeValues({Result, Chain}, DL);
5298 }
5299 
5300 SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
5301                                               SelectionDAG &DAG) const {
5302   SDLoc DL(Op);
5303 
5304   const auto *MemSD = cast<MemSDNode>(Op);
5305   EVT MemVT = MemSD->getMemoryVT();
5306   MachineMemOperand *MMO = MemSD->getMemOperand();
5307   SDValue Chain = MemSD->getChain();
5308   SDValue BasePtr = MemSD->getBasePtr();
5309   SDValue Val, Mask, VL;
5310 
5311   if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Op)) {
5312     Val = VPStore->getValue();
5313     Mask = VPStore->getMask();
5314     VL = VPStore->getVectorLength();
5315   } else {
5316     const auto *MStore = cast<MaskedStoreSDNode>(Op);
5317     Val = MStore->getValue();
5318     Mask = MStore->getMask();
5319   }
5320 
5321   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5322 
5323   MVT VT = Val.getSimpleValueType();
5324   MVT XLenVT = Subtarget.getXLenVT();
5325 
5326   MVT ContainerVT = VT;
5327   if (VT.isFixedLengthVector()) {
5328     ContainerVT = getContainerForFixedLengthVector(VT);
5329 
5330     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
5331     if (!IsUnmasked) {
5332       MVT MaskVT =
5333           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5334       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5335     }
5336   }
5337 
5338   if (!VL)
5339     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5340 
5341   unsigned IntID =
5342       IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
5343   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5344   Ops.push_back(Val);
5345   Ops.push_back(BasePtr);
5346   if (!IsUnmasked)
5347     Ops.push_back(Mask);
5348   Ops.push_back(VL);
5349 
5350   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
5351                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
5352 }
5353 
5354 SDValue
5355 RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(SDValue Op,
5356                                                       SelectionDAG &DAG) const {
5357   MVT InVT = Op.getOperand(0).getSimpleValueType();
5358   MVT ContainerVT = getContainerForFixedLengthVector(InVT);
5359 
5360   MVT VT = Op.getSimpleValueType();
5361 
5362   SDValue Op1 =
5363       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
5364   SDValue Op2 =
5365       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5366 
5367   SDLoc DL(Op);
5368   SDValue VL =
5369       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5370 
5371   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5372   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
5373 
5374   SDValue Cmp = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op1, Op2,
5375                             Op.getOperand(2), Mask, VL);
5376 
5377   return convertFromScalableVector(VT, Cmp, DAG, Subtarget);
5378 }
5379 
5380 SDValue RISCVTargetLowering::lowerFixedLengthVectorLogicOpToRVV(
5381     SDValue Op, SelectionDAG &DAG, unsigned MaskOpc, unsigned VecOpc) const {
5382   MVT VT = Op.getSimpleValueType();
5383 
5384   if (VT.getVectorElementType() == MVT::i1)
5385     return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMask*/ false);
5386 
5387   return lowerToScalableOp(Op, DAG, VecOpc, /*HasMask*/ true);
5388 }
5389 
5390 SDValue
5391 RISCVTargetLowering::lowerFixedLengthVectorShiftToRVV(SDValue Op,
5392                                                       SelectionDAG &DAG) const {
5393   unsigned Opc;
5394   switch (Op.getOpcode()) {
5395   default: llvm_unreachable("Unexpected opcode!");
5396   case ISD::SHL: Opc = RISCVISD::SHL_VL; break;
5397   case ISD::SRA: Opc = RISCVISD::SRA_VL; break;
5398   case ISD::SRL: Opc = RISCVISD::SRL_VL; break;
5399   }
5400 
5401   return lowerToScalableOp(Op, DAG, Opc);
5402 }
5403 
5404 // Lower vector ABS to smax(X, sub(0, X)).
5405 SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
5406   SDLoc DL(Op);
5407   MVT VT = Op.getSimpleValueType();
5408   SDValue X = Op.getOperand(0);
5409 
5410   assert(VT.isFixedLengthVector() && "Unexpected type");
5411 
5412   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5413   X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
5414 
5415   SDValue Mask, VL;
5416   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5417 
5418   SDValue SplatZero =
5419       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
5420                   DAG.getConstant(0, DL, Subtarget.getXLenVT()));
5421   SDValue NegX =
5422       DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, Mask, VL);
5423   SDValue Max =
5424       DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, Mask, VL);
5425 
5426   return convertFromScalableVector(VT, Max, DAG, Subtarget);
5427 }
5428 
5429 SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
5430     SDValue Op, SelectionDAG &DAG) const {
5431   SDLoc DL(Op);
5432   MVT VT = Op.getSimpleValueType();
5433   SDValue Mag = Op.getOperand(0);
5434   SDValue Sign = Op.getOperand(1);
5435   assert(Mag.getValueType() == Sign.getValueType() &&
5436          "Can only handle COPYSIGN with matching types.");
5437 
5438   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5439   Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget);
5440   Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget);
5441 
5442   SDValue Mask, VL;
5443   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5444 
5445   SDValue CopySign =
5446       DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag, Sign, Mask, VL);
5447 
5448   return convertFromScalableVector(VT, CopySign, DAG, Subtarget);
5449 }
5450 
5451 SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
5452     SDValue Op, SelectionDAG &DAG) const {
5453   MVT VT = Op.getSimpleValueType();
5454   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5455 
5456   MVT I1ContainerVT =
5457       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5458 
5459   SDValue CC =
5460       convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget);
5461   SDValue Op1 =
5462       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5463   SDValue Op2 =
5464       convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget);
5465 
5466   SDLoc DL(Op);
5467   SDValue Mask, VL;
5468   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5469 
5470   SDValue Select =
5471       DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC, Op1, Op2, VL);
5472 
5473   return convertFromScalableVector(VT, Select, DAG, Subtarget);
5474 }
5475 
5476 SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
5477                                                unsigned NewOpc,
5478                                                bool HasMask) const {
5479   MVT VT = Op.getSimpleValueType();
5480   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5481 
5482   // Create list of operands by converting existing ones to scalable types.
5483   SmallVector<SDValue, 6> Ops;
5484   for (const SDValue &V : Op->op_values()) {
5485     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5486 
5487     // Pass through non-vector operands.
5488     if (!V.getValueType().isVector()) {
5489       Ops.push_back(V);
5490       continue;
5491     }
5492 
5493     // "cast" fixed length vector to a scalable vector.
5494     assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
5495            "Only fixed length vectors are supported!");
5496     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5497   }
5498 
5499   SDLoc DL(Op);
5500   SDValue Mask, VL;
5501   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5502   if (HasMask)
5503     Ops.push_back(Mask);
5504   Ops.push_back(VL);
5505 
5506   SDValue ScalableRes = DAG.getNode(NewOpc, DL, ContainerVT, Ops);
5507   return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
5508 }
5509 
5510 // Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
5511 // * Operands of each node are assumed to be in the same order.
5512 // * The EVL operand is promoted from i32 to i64 on RV64.
5513 // * Fixed-length vectors are converted to their scalable-vector container
5514 //   types.
5515 SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
5516                                        unsigned RISCVISDOpc) const {
5517   SDLoc DL(Op);
5518   MVT VT = Op.getSimpleValueType();
5519   SmallVector<SDValue, 4> Ops;
5520 
5521   for (const auto &OpIdx : enumerate(Op->ops())) {
5522     SDValue V = OpIdx.value();
5523     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5524     // Pass through operands which aren't fixed-length vectors.
5525     if (!V.getValueType().isFixedLengthVector()) {
5526       Ops.push_back(V);
5527       continue;
5528     }
5529     // "cast" fixed length vector to a scalable vector.
5530     MVT OpVT = V.getSimpleValueType();
5531     MVT ContainerVT = getContainerForFixedLengthVector(OpVT);
5532     assert(useRVVForFixedLengthVectorVT(OpVT) &&
5533            "Only fixed length vectors are supported!");
5534     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5535   }
5536 
5537   if (!VT.isFixedLengthVector())
5538     return DAG.getNode(RISCVISDOpc, DL, VT, Ops);
5539 
5540   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5541 
5542   SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops);
5543 
5544   return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
5545 }
5546 
5547 SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op, SelectionDAG &DAG,
5548                                             unsigned MaskOpc,
5549                                             unsigned VecOpc) const {
5550   MVT VT = Op.getSimpleValueType();
5551   if (VT.getVectorElementType() != MVT::i1)
5552     return lowerVPOp(Op, DAG, VecOpc);
5553 
5554   // It is safe to drop mask parameter as masked-off elements are undef.
5555   SDValue Op1 = Op->getOperand(0);
5556   SDValue Op2 = Op->getOperand(1);
5557   SDValue VL = Op->getOperand(3);
5558 
5559   MVT ContainerVT = VT;
5560   const bool IsFixed = VT.isFixedLengthVector();
5561   if (IsFixed) {
5562     ContainerVT = getContainerForFixedLengthVector(VT);
5563     Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5564     Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
5565   }
5566 
5567   SDLoc DL(Op);
5568   SDValue Val = DAG.getNode(MaskOpc, DL, ContainerVT, Op1, Op2, VL);
5569   if (!IsFixed)
5570     return Val;
5571   return convertFromScalableVector(VT, Val, DAG, Subtarget);
5572 }
5573 
5574 // Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
5575 // matched to a RVV indexed load. The RVV indexed load instructions only
5576 // support the "unsigned unscaled" addressing mode; indices are implicitly
5577 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
5578 // signed or scaled indexing is extended to the XLEN value type and scaled
5579 // accordingly.
5580 SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
5581                                                SelectionDAG &DAG) const {
5582   SDLoc DL(Op);
5583   MVT VT = Op.getSimpleValueType();
5584 
5585   const auto *MemSD = cast<MemSDNode>(Op.getNode());
5586   EVT MemVT = MemSD->getMemoryVT();
5587   MachineMemOperand *MMO = MemSD->getMemOperand();
5588   SDValue Chain = MemSD->getChain();
5589   SDValue BasePtr = MemSD->getBasePtr();
5590 
5591   ISD::LoadExtType LoadExtType;
5592   SDValue Index, Mask, PassThru, VL;
5593 
5594   if (auto *VPGN = dyn_cast<VPGatherSDNode>(Op.getNode())) {
5595     Index = VPGN->getIndex();
5596     Mask = VPGN->getMask();
5597     PassThru = DAG.getUNDEF(VT);
5598     VL = VPGN->getVectorLength();
5599     // VP doesn't support extending loads.
5600     LoadExtType = ISD::NON_EXTLOAD;
5601   } else {
5602     // Else it must be a MGATHER.
5603     auto *MGN = cast<MaskedGatherSDNode>(Op.getNode());
5604     Index = MGN->getIndex();
5605     Mask = MGN->getMask();
5606     PassThru = MGN->getPassThru();
5607     LoadExtType = MGN->getExtensionType();
5608   }
5609 
5610   MVT IndexVT = Index.getSimpleValueType();
5611   MVT XLenVT = Subtarget.getXLenVT();
5612 
5613   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
5614          "Unexpected VTs!");
5615   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
5616   // Targets have to explicitly opt-in for extending vector loads.
5617   assert(LoadExtType == ISD::NON_EXTLOAD &&
5618          "Unexpected extending MGATHER/VP_GATHER");
5619   (void)LoadExtType;
5620 
5621   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
5622   // the selection of the masked intrinsics doesn't do this for us.
5623   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5624 
5625   MVT ContainerVT = VT;
5626   if (VT.isFixedLengthVector()) {
5627     // We need to use the larger of the result and index type to determine the
5628     // scalable type to use so we don't increase LMUL for any operand/result.
5629     if (VT.bitsGE(IndexVT)) {
5630       ContainerVT = getContainerForFixedLengthVector(VT);
5631       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
5632                                  ContainerVT.getVectorElementCount());
5633     } else {
5634       IndexVT = getContainerForFixedLengthVector(IndexVT);
5635       ContainerVT = MVT::getVectorVT(ContainerVT.getVectorElementType(),
5636                                      IndexVT.getVectorElementCount());
5637     }
5638 
5639     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
5640 
5641     if (!IsUnmasked) {
5642       MVT MaskVT =
5643           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5644       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5645       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5646     }
5647   }
5648 
5649   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
5650       IndexVT = IndexVT.changeVectorElementType(XLenVT);
5651       Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
5652   }
5653 
5654   if (!VL)
5655     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5656 
5657   unsigned IntID =
5658       IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
5659   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5660   if (!IsUnmasked)
5661     Ops.push_back(PassThru);
5662   Ops.push_back(BasePtr);
5663   Ops.push_back(Index);
5664   if (!IsUnmasked)
5665     Ops.push_back(Mask);
5666   Ops.push_back(VL);
5667   if (!IsUnmasked)
5668     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5669 
5670   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5671   SDValue Result =
5672       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5673   Chain = Result.getValue(1);
5674 
5675   if (VT.isFixedLengthVector())
5676     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5677 
5678   return DAG.getMergeValues({Result, Chain}, DL);
5679 }
5680 
5681 // Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
5682 // matched to a RVV indexed store. The RVV indexed store instructions only
5683 // support the "unsigned unscaled" addressing mode; indices are implicitly
5684 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
5685 // signed or scaled indexing is extended to the XLEN value type and scaled
5686 // accordingly.
5687 SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
5688                                                 SelectionDAG &DAG) const {
5689   SDLoc DL(Op);
5690   const auto *MemSD = cast<MemSDNode>(Op.getNode());
5691   EVT MemVT = MemSD->getMemoryVT();
5692   MachineMemOperand *MMO = MemSD->getMemOperand();
5693   SDValue Chain = MemSD->getChain();
5694   SDValue BasePtr = MemSD->getBasePtr();
5695 
5696   bool IsTruncatingStore = false;
5697   SDValue Index, Mask, Val, VL;
5698 
5699   if (auto *VPSN = dyn_cast<VPScatterSDNode>(Op.getNode())) {
5700     Index = VPSN->getIndex();
5701     Mask = VPSN->getMask();
5702     Val = VPSN->getValue();
5703     VL = VPSN->getVectorLength();
5704     // VP doesn't support truncating stores.
5705     IsTruncatingStore = false;
5706   } else {
5707     // Else it must be a MSCATTER.
5708     auto *MSN = cast<MaskedScatterSDNode>(Op.getNode());
5709     Index = MSN->getIndex();
5710     Mask = MSN->getMask();
5711     Val = MSN->getValue();
5712     IsTruncatingStore = MSN->isTruncatingStore();
5713   }
5714 
5715   MVT VT = Val.getSimpleValueType();
5716   MVT IndexVT = Index.getSimpleValueType();
5717   MVT XLenVT = Subtarget.getXLenVT();
5718 
5719   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
5720          "Unexpected VTs!");
5721   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
5722   // Targets have to explicitly opt-in for extending vector loads and
5723   // truncating vector stores.
5724   assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
5725   (void)IsTruncatingStore;
5726 
5727   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
5728   // the selection of the masked intrinsics doesn't do this for us.
5729   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5730 
5731   MVT ContainerVT = VT;
5732   if (VT.isFixedLengthVector()) {
5733     // We need to use the larger of the value and index type to determine the
5734     // scalable type to use so we don't increase LMUL for any operand/result.
5735     if (VT.bitsGE(IndexVT)) {
5736       ContainerVT = getContainerForFixedLengthVector(VT);
5737       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
5738                                  ContainerVT.getVectorElementCount());
5739     } else {
5740       IndexVT = getContainerForFixedLengthVector(IndexVT);
5741       ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
5742                                      IndexVT.getVectorElementCount());
5743     }
5744 
5745     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
5746     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
5747 
5748     if (!IsUnmasked) {
5749       MVT MaskVT =
5750           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5751       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5752     }
5753   }
5754 
5755   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
5756       IndexVT = IndexVT.changeVectorElementType(XLenVT);
5757       Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
5758   }
5759 
5760   if (!VL)
5761     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5762 
5763   unsigned IntID =
5764       IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
5765   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5766   Ops.push_back(Val);
5767   Ops.push_back(BasePtr);
5768   Ops.push_back(Index);
5769   if (!IsUnmasked)
5770     Ops.push_back(Mask);
5771   Ops.push_back(VL);
5772 
5773   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
5774                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
5775 }
5776 
5777 SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
5778                                                SelectionDAG &DAG) const {
5779   const MVT XLenVT = Subtarget.getXLenVT();
5780   SDLoc DL(Op);
5781   SDValue Chain = Op->getOperand(0);
5782   SDValue SysRegNo = DAG.getTargetConstant(
5783       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
5784   SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
5785   SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
5786 
5787   // Encoding used for rounding mode in RISCV differs from that used in
5788   // FLT_ROUNDS. To convert it the RISCV rounding mode is used as an index in a
5789   // table, which consists of a sequence of 4-bit fields, each representing
5790   // corresponding FLT_ROUNDS mode.
5791   static const int Table =
5792       (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
5793       (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
5794       (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
5795       (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
5796       (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
5797 
5798   SDValue Shift =
5799       DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
5800   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
5801                                 DAG.getConstant(Table, DL, XLenVT), Shift);
5802   SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
5803                                DAG.getConstant(7, DL, XLenVT));
5804 
5805   return DAG.getMergeValues({Masked, Chain}, DL);
5806 }
5807 
5808 SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
5809                                                SelectionDAG &DAG) const {
5810   const MVT XLenVT = Subtarget.getXLenVT();
5811   SDLoc DL(Op);
5812   SDValue Chain = Op->getOperand(0);
5813   SDValue RMValue = Op->getOperand(1);
5814   SDValue SysRegNo = DAG.getTargetConstant(
5815       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
5816 
5817   // Encoding used for rounding mode in RISCV differs from that used in
5818   // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
5819   // a table, which consists of a sequence of 4-bit fields, each representing
5820   // corresponding RISCV mode.
5821   static const unsigned Table =
5822       (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
5823       (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
5824       (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
5825       (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
5826       (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
5827 
5828   SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
5829                               DAG.getConstant(2, DL, XLenVT));
5830   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
5831                                 DAG.getConstant(Table, DL, XLenVT), Shift);
5832   RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
5833                         DAG.getConstant(0x7, DL, XLenVT));
5834   return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
5835                      RMValue);
5836 }
5837 
5838 static RISCVISD::NodeType getRISCVWOpcodeByIntr(unsigned IntNo) {
5839   switch (IntNo) {
5840   default:
5841     llvm_unreachable("Unexpected Intrinsic");
5842   case Intrinsic::riscv_grev:
5843     return RISCVISD::GREVW;
5844   case Intrinsic::riscv_gorc:
5845     return RISCVISD::GORCW;
5846   case Intrinsic::riscv_bcompress:
5847     return RISCVISD::BCOMPRESSW;
5848   case Intrinsic::riscv_bdecompress:
5849     return RISCVISD::BDECOMPRESSW;
5850   case Intrinsic::riscv_bfp:
5851     return RISCVISD::BFPW;
5852   case Intrinsic::riscv_fsl:
5853     return RISCVISD::FSLW;
5854   case Intrinsic::riscv_fsr:
5855     return RISCVISD::FSRW;
5856   }
5857 }
5858 
5859 // Converts the given intrinsic to a i64 operation with any extension.
5860 static SDValue customLegalizeToWOpByIntr(SDNode *N, SelectionDAG &DAG,
5861                                          unsigned IntNo) {
5862   SDLoc DL(N);
5863   RISCVISD::NodeType WOpcode = getRISCVWOpcodeByIntr(IntNo);
5864   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5865   SDValue NewOp2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
5866   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp1, NewOp2);
5867   // ReplaceNodeResults requires we maintain the same type for the return value.
5868   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
5869 }
5870 
5871 // Returns the opcode of the target-specific SDNode that implements the 32-bit
5872 // form of the given Opcode.
5873 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
5874   switch (Opcode) {
5875   default:
5876     llvm_unreachable("Unexpected opcode");
5877   case ISD::SHL:
5878     return RISCVISD::SLLW;
5879   case ISD::SRA:
5880     return RISCVISD::SRAW;
5881   case ISD::SRL:
5882     return RISCVISD::SRLW;
5883   case ISD::SDIV:
5884     return RISCVISD::DIVW;
5885   case ISD::UDIV:
5886     return RISCVISD::DIVUW;
5887   case ISD::UREM:
5888     return RISCVISD::REMUW;
5889   case ISD::ROTL:
5890     return RISCVISD::ROLW;
5891   case ISD::ROTR:
5892     return RISCVISD::RORW;
5893   case RISCVISD::GREV:
5894     return RISCVISD::GREVW;
5895   case RISCVISD::GORC:
5896     return RISCVISD::GORCW;
5897   }
5898 }
5899 
5900 // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
5901 // node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
5902 // otherwise be promoted to i64, making it difficult to select the
5903 // SLLW/DIVUW/.../*W later one because the fact the operation was originally of
5904 // type i8/i16/i32 is lost.
5905 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
5906                                    unsigned ExtOpc = ISD::ANY_EXTEND) {
5907   SDLoc DL(N);
5908   RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
5909   SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
5910   SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
5911   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
5912   // ReplaceNodeResults requires we maintain the same type for the return value.
5913   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
5914 }
5915 
5916 // Converts the given 32-bit operation to a i64 operation with signed extension
5917 // semantic to reduce the signed extension instructions.
5918 static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
5919   SDLoc DL(N);
5920   SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
5921   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5922   SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
5923   SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
5924                                DAG.getValueType(MVT::i32));
5925   return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
5926 }
5927 
5928 void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
5929                                              SmallVectorImpl<SDValue> &Results,
5930                                              SelectionDAG &DAG) const {
5931   SDLoc DL(N);
5932   switch (N->getOpcode()) {
5933   default:
5934     llvm_unreachable("Don't know how to custom type legalize this operation!");
5935   case ISD::STRICT_FP_TO_SINT:
5936   case ISD::STRICT_FP_TO_UINT:
5937   case ISD::FP_TO_SINT:
5938   case ISD::FP_TO_UINT: {
5939     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5940            "Unexpected custom legalisation");
5941     bool IsStrict = N->isStrictFPOpcode();
5942     bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
5943                     N->getOpcode() == ISD::STRICT_FP_TO_SINT;
5944     SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
5945     if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
5946         TargetLowering::TypeSoftenFloat) {
5947       if (!isTypeLegal(Op0.getValueType()))
5948         return;
5949       if (IsStrict) {
5950         unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
5951                                 : RISCVISD::STRICT_FCVT_WU_RV64;
5952         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
5953         SDValue Res = DAG.getNode(
5954             Opc, DL, VTs, N->getOperand(0), Op0,
5955             DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
5956         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
5957         Results.push_back(Res.getValue(1));
5958         return;
5959       }
5960       unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
5961       SDValue Res =
5962           DAG.getNode(Opc, DL, MVT::i64, Op0,
5963                       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
5964       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
5965       return;
5966     }
5967     // If the FP type needs to be softened, emit a library call using the 'si'
5968     // version. If we left it to default legalization we'd end up with 'di'. If
5969     // the FP type doesn't need to be softened just let generic type
5970     // legalization promote the result type.
5971     RTLIB::Libcall LC;
5972     if (IsSigned)
5973       LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
5974     else
5975       LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
5976     MakeLibCallOptions CallOptions;
5977     EVT OpVT = Op0.getValueType();
5978     CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
5979     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
5980     SDValue Result;
5981     std::tie(Result, Chain) =
5982         makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
5983     Results.push_back(Result);
5984     if (IsStrict)
5985       Results.push_back(Chain);
5986     break;
5987   }
5988   case ISD::READCYCLECOUNTER: {
5989     assert(!Subtarget.is64Bit() &&
5990            "READCYCLECOUNTER only has custom type legalization on riscv32");
5991 
5992     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
5993     SDValue RCW =
5994         DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
5995 
5996     Results.push_back(
5997         DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
5998     Results.push_back(RCW.getValue(2));
5999     break;
6000   }
6001   case ISD::MUL: {
6002     unsigned Size = N->getSimpleValueType(0).getSizeInBits();
6003     unsigned XLen = Subtarget.getXLen();
6004     // This multiply needs to be expanded, try to use MULHSU+MUL if possible.
6005     if (Size > XLen) {
6006       assert(Size == (XLen * 2) && "Unexpected custom legalisation");
6007       SDValue LHS = N->getOperand(0);
6008       SDValue RHS = N->getOperand(1);
6009       APInt HighMask = APInt::getHighBitsSet(Size, XLen);
6010 
6011       bool LHSIsU = DAG.MaskedValueIsZero(LHS, HighMask);
6012       bool RHSIsU = DAG.MaskedValueIsZero(RHS, HighMask);
6013       // We need exactly one side to be unsigned.
6014       if (LHSIsU == RHSIsU)
6015         return;
6016 
6017       auto MakeMULPair = [&](SDValue S, SDValue U) {
6018         MVT XLenVT = Subtarget.getXLenVT();
6019         S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S);
6020         U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U);
6021         SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U);
6022         SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U);
6023         return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
6024       };
6025 
6026       bool LHSIsS = DAG.ComputeNumSignBits(LHS) > XLen;
6027       bool RHSIsS = DAG.ComputeNumSignBits(RHS) > XLen;
6028 
6029       // The other operand should be signed, but still prefer MULH when
6030       // possible.
6031       if (RHSIsU && LHSIsS && !RHSIsS)
6032         Results.push_back(MakeMULPair(LHS, RHS));
6033       else if (LHSIsU && RHSIsS && !LHSIsS)
6034         Results.push_back(MakeMULPair(RHS, LHS));
6035 
6036       return;
6037     }
6038     LLVM_FALLTHROUGH;
6039   }
6040   case ISD::ADD:
6041   case ISD::SUB:
6042     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6043            "Unexpected custom legalisation");
6044     Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
6045     break;
6046   case ISD::SHL:
6047   case ISD::SRA:
6048   case ISD::SRL:
6049     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6050            "Unexpected custom legalisation");
6051     if (N->getOperand(1).getOpcode() != ISD::Constant) {
6052       Results.push_back(customLegalizeToWOp(N, DAG));
6053       break;
6054     }
6055 
6056     // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
6057     // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
6058     // shift amount.
6059     if (N->getOpcode() == ISD::SHL) {
6060       SDLoc DL(N);
6061       SDValue NewOp0 =
6062           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6063       SDValue NewOp1 =
6064           DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
6065       SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
6066       SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6067                                    DAG.getValueType(MVT::i32));
6068       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6069     }
6070 
6071     break;
6072   case ISD::ROTL:
6073   case ISD::ROTR:
6074     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6075            "Unexpected custom legalisation");
6076     Results.push_back(customLegalizeToWOp(N, DAG));
6077     break;
6078   case ISD::CTTZ:
6079   case ISD::CTTZ_ZERO_UNDEF:
6080   case ISD::CTLZ:
6081   case ISD::CTLZ_ZERO_UNDEF: {
6082     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6083            "Unexpected custom legalisation");
6084 
6085     SDValue NewOp0 =
6086         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6087     bool IsCTZ =
6088         N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF;
6089     unsigned Opc = IsCTZ ? RISCVISD::CTZW : RISCVISD::CLZW;
6090     SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
6091     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6092     return;
6093   }
6094   case ISD::SDIV:
6095   case ISD::UDIV:
6096   case ISD::UREM: {
6097     MVT VT = N->getSimpleValueType(0);
6098     assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
6099            Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
6100            "Unexpected custom legalisation");
6101     // Don't promote division/remainder by constant since we should expand those
6102     // to multiply by magic constant.
6103     // FIXME: What if the expansion is disabled for minsize.
6104     if (N->getOperand(1).getOpcode() == ISD::Constant)
6105       return;
6106 
6107     // If the input is i32, use ANY_EXTEND since the W instructions don't read
6108     // the upper 32 bits. For other types we need to sign or zero extend
6109     // based on the opcode.
6110     unsigned ExtOpc = ISD::ANY_EXTEND;
6111     if (VT != MVT::i32)
6112       ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
6113                                            : ISD::ZERO_EXTEND;
6114 
6115     Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
6116     break;
6117   }
6118   case ISD::UADDO:
6119   case ISD::USUBO: {
6120     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6121            "Unexpected custom legalisation");
6122     bool IsAdd = N->getOpcode() == ISD::UADDO;
6123     // Create an ADDW or SUBW.
6124     SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6125     SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6126     SDValue Res =
6127         DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
6128     Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
6129                       DAG.getValueType(MVT::i32));
6130 
6131     // Sign extend the LHS and perform an unsigned compare with the ADDW result.
6132     // Since the inputs are sign extended from i32, this is equivalent to
6133     // comparing the lower 32 bits.
6134     LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6135     SDValue Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
6136                                     IsAdd ? ISD::SETULT : ISD::SETUGT);
6137 
6138     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6139     Results.push_back(Overflow);
6140     return;
6141   }
6142   case ISD::UADDSAT:
6143   case ISD::USUBSAT: {
6144     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6145            "Unexpected custom legalisation");
6146     if (Subtarget.hasStdExtZbb()) {
6147       // With Zbb we can sign extend and let LegalizeDAG use minu/maxu. Using
6148       // sign extend allows overflow of the lower 32 bits to be detected on
6149       // the promoted size.
6150       SDValue LHS =
6151           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6152       SDValue RHS =
6153           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
6154       SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
6155       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6156       return;
6157     }
6158 
6159     // Without Zbb, expand to UADDO/USUBO+select which will trigger our custom
6160     // promotion for UADDO/USUBO.
6161     Results.push_back(expandAddSubSat(N, DAG));
6162     return;
6163   }
6164   case ISD::BITCAST: {
6165     EVT VT = N->getValueType(0);
6166     assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
6167     SDValue Op0 = N->getOperand(0);
6168     EVT Op0VT = Op0.getValueType();
6169     MVT XLenVT = Subtarget.getXLenVT();
6170     if (VT == MVT::i16 && Op0VT == MVT::f16 && Subtarget.hasStdExtZfh()) {
6171       SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
6172       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
6173     } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
6174                Subtarget.hasStdExtF()) {
6175       SDValue FPConv =
6176           DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
6177       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
6178     } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
6179                isTypeLegal(Op0VT)) {
6180       // Custom-legalize bitcasts from fixed-length vector types to illegal
6181       // scalar types in order to improve codegen. Bitcast the vector to a
6182       // one-element vector type whose element type is the same as the result
6183       // type, and extract the first element.
6184       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
6185       if (isTypeLegal(BVT)) {
6186         SDValue BVec = DAG.getBitcast(BVT, Op0);
6187         Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
6188                                       DAG.getConstant(0, DL, XLenVT)));
6189       }
6190     }
6191     break;
6192   }
6193   case RISCVISD::GREV:
6194   case RISCVISD::GORC: {
6195     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6196            "Unexpected custom legalisation");
6197     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6198     // This is similar to customLegalizeToWOp, except that we pass the second
6199     // operand (a TargetConstant) straight through: it is already of type
6200     // XLenVT.
6201     RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6202     SDValue NewOp0 =
6203         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6204     SDValue NewOp1 =
6205         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6206     SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6207     // ReplaceNodeResults requires we maintain the same type for the return
6208     // value.
6209     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6210     break;
6211   }
6212   case RISCVISD::SHFL: {
6213     // There is no SHFLIW instruction, but we can just promote the operation.
6214     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6215            "Unexpected custom legalisation");
6216     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6217     SDValue NewOp0 =
6218         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6219     SDValue NewOp1 =
6220         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6221     SDValue NewRes = DAG.getNode(RISCVISD::SHFL, DL, MVT::i64, NewOp0, NewOp1);
6222     // ReplaceNodeResults requires we maintain the same type for the return
6223     // value.
6224     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6225     break;
6226   }
6227   case ISD::BSWAP:
6228   case ISD::BITREVERSE: {
6229     MVT VT = N->getSimpleValueType(0);
6230     MVT XLenVT = Subtarget.getXLenVT();
6231     assert((VT == MVT::i8 || VT == MVT::i16 ||
6232             (VT == MVT::i32 && Subtarget.is64Bit())) &&
6233            Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
6234     SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
6235     unsigned Imm = VT.getSizeInBits() - 1;
6236     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
6237     if (N->getOpcode() == ISD::BSWAP)
6238       Imm &= ~0x7U;
6239     unsigned Opc = Subtarget.is64Bit() ? RISCVISD::GREVW : RISCVISD::GREV;
6240     SDValue GREVI =
6241         DAG.getNode(Opc, DL, XLenVT, NewOp0, DAG.getConstant(Imm, DL, XLenVT));
6242     // ReplaceNodeResults requires we maintain the same type for the return
6243     // value.
6244     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, GREVI));
6245     break;
6246   }
6247   case ISD::FSHL:
6248   case ISD::FSHR: {
6249     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6250            Subtarget.hasStdExtZbt() && "Unexpected custom legalisation");
6251     SDValue NewOp0 =
6252         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6253     SDValue NewOp1 =
6254         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6255     SDValue NewShAmt =
6256         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6257     // FSLW/FSRW take a 6 bit shift amount but i32 FSHL/FSHR only use 5 bits.
6258     // Mask the shift amount to 5 bits to prevent accidentally setting bit 5.
6259     NewShAmt = DAG.getNode(ISD::AND, DL, MVT::i64, NewShAmt,
6260                            DAG.getConstant(0x1f, DL, MVT::i64));
6261     // fshl and fshr concatenate their operands in the same order. fsrw and fslw
6262     // instruction use different orders. fshl will return its first operand for
6263     // shift of zero, fshr will return its second operand. fsl and fsr both
6264     // return rs1 so the ISD nodes need to have different operand orders.
6265     // Shift amount is in rs2.
6266     unsigned Opc = RISCVISD::FSLW;
6267     if (N->getOpcode() == ISD::FSHR) {
6268       std::swap(NewOp0, NewOp1);
6269       Opc = RISCVISD::FSRW;
6270     }
6271     SDValue NewOp = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, NewShAmt);
6272     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewOp));
6273     break;
6274   }
6275   case ISD::EXTRACT_VECTOR_ELT: {
6276     // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
6277     // type is illegal (currently only vXi64 RV32).
6278     // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
6279     // transferred to the destination register. We issue two of these from the
6280     // upper- and lower- halves of the SEW-bit vector element, slid down to the
6281     // first element.
6282     SDValue Vec = N->getOperand(0);
6283     SDValue Idx = N->getOperand(1);
6284 
6285     // The vector type hasn't been legalized yet so we can't issue target
6286     // specific nodes if it needs legalization.
6287     // FIXME: We would manually legalize if it's important.
6288     if (!isTypeLegal(Vec.getValueType()))
6289       return;
6290 
6291     MVT VecVT = Vec.getSimpleValueType();
6292 
6293     assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
6294            VecVT.getVectorElementType() == MVT::i64 &&
6295            "Unexpected EXTRACT_VECTOR_ELT legalization");
6296 
6297     // If this is a fixed vector, we need to convert it to a scalable vector.
6298     MVT ContainerVT = VecVT;
6299     if (VecVT.isFixedLengthVector()) {
6300       ContainerVT = getContainerForFixedLengthVector(VecVT);
6301       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
6302     }
6303 
6304     MVT XLenVT = Subtarget.getXLenVT();
6305 
6306     // Use a VL of 1 to avoid processing more elements than we need.
6307     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
6308     SDValue VL = DAG.getConstant(1, DL, XLenVT);
6309     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6310 
6311     // Unless the index is known to be 0, we must slide the vector down to get
6312     // the desired element into index 0.
6313     if (!isNullConstant(Idx)) {
6314       Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
6315                         DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
6316     }
6317 
6318     // Extract the lower XLEN bits of the correct vector element.
6319     SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6320 
6321     // To extract the upper XLEN bits of the vector element, shift the first
6322     // element right by 32 bits and re-extract the lower XLEN bits.
6323     SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
6324                                      DAG.getConstant(32, DL, XLenVT), VL);
6325     SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec,
6326                                  ThirtyTwoV, Mask, VL);
6327 
6328     SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6329 
6330     Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6331     break;
6332   }
6333   case ISD::INTRINSIC_WO_CHAIN: {
6334     unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
6335     switch (IntNo) {
6336     default:
6337       llvm_unreachable(
6338           "Don't know how to custom type legalize this intrinsic!");
6339     case Intrinsic::riscv_grev:
6340     case Intrinsic::riscv_gorc:
6341     case Intrinsic::riscv_bcompress:
6342     case Intrinsic::riscv_bdecompress:
6343     case Intrinsic::riscv_bfp: {
6344       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6345              "Unexpected custom legalisation");
6346       Results.push_back(customLegalizeToWOpByIntr(N, DAG, IntNo));
6347       break;
6348     }
6349     case Intrinsic::riscv_fsl:
6350     case Intrinsic::riscv_fsr: {
6351       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6352              "Unexpected custom legalisation");
6353       SDValue NewOp1 =
6354           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6355       SDValue NewOp2 =
6356           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6357       SDValue NewOp3 =
6358           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3));
6359       unsigned Opc = getRISCVWOpcodeByIntr(IntNo);
6360       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2, NewOp3);
6361       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6362       break;
6363     }
6364     case Intrinsic::riscv_orc_b: {
6365       // Lower to the GORCI encoding for orc.b with the operand extended.
6366       SDValue NewOp =
6367           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6368       // If Zbp is enabled, use GORCIW which will sign extend the result.
6369       unsigned Opc =
6370           Subtarget.hasStdExtZbp() ? RISCVISD::GORCW : RISCVISD::GORC;
6371       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp,
6372                                 DAG.getConstant(7, DL, MVT::i64));
6373       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6374       return;
6375     }
6376     case Intrinsic::riscv_shfl:
6377     case Intrinsic::riscv_unshfl: {
6378       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6379              "Unexpected custom legalisation");
6380       SDValue NewOp1 =
6381           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6382       SDValue NewOp2 =
6383           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6384       unsigned Opc =
6385           IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFLW : RISCVISD::UNSHFLW;
6386       if (isa<ConstantSDNode>(N->getOperand(2))) {
6387         NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2,
6388                              DAG.getConstant(0xf, DL, MVT::i64));
6389         Opc =
6390             IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
6391       }
6392       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2);
6393       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6394       break;
6395     }
6396     case Intrinsic::riscv_vmv_x_s: {
6397       EVT VT = N->getValueType(0);
6398       MVT XLenVT = Subtarget.getXLenVT();
6399       if (VT.bitsLT(XLenVT)) {
6400         // Simple case just extract using vmv.x.s and truncate.
6401         SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
6402                                       Subtarget.getXLenVT(), N->getOperand(1));
6403         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
6404         return;
6405       }
6406 
6407       assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
6408              "Unexpected custom legalization");
6409 
6410       // We need to do the move in two steps.
6411       SDValue Vec = N->getOperand(1);
6412       MVT VecVT = Vec.getSimpleValueType();
6413 
6414       // First extract the lower XLEN bits of the element.
6415       SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6416 
6417       // To extract the upper XLEN bits of the vector element, shift the first
6418       // element right by 32 bits and re-extract the lower XLEN bits.
6419       SDValue VL = DAG.getConstant(1, DL, XLenVT);
6420       MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
6421       SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6422       SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT,
6423                                        DAG.getConstant(32, DL, XLenVT), VL);
6424       SDValue LShr32 =
6425           DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV, Mask, VL);
6426       SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6427 
6428       Results.push_back(
6429           DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6430       break;
6431     }
6432     }
6433     break;
6434   }
6435   case ISD::VECREDUCE_ADD:
6436   case ISD::VECREDUCE_AND:
6437   case ISD::VECREDUCE_OR:
6438   case ISD::VECREDUCE_XOR:
6439   case ISD::VECREDUCE_SMAX:
6440   case ISD::VECREDUCE_UMAX:
6441   case ISD::VECREDUCE_SMIN:
6442   case ISD::VECREDUCE_UMIN:
6443     if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
6444       Results.push_back(V);
6445     break;
6446   case ISD::VP_REDUCE_ADD:
6447   case ISD::VP_REDUCE_AND:
6448   case ISD::VP_REDUCE_OR:
6449   case ISD::VP_REDUCE_XOR:
6450   case ISD::VP_REDUCE_SMAX:
6451   case ISD::VP_REDUCE_UMAX:
6452   case ISD::VP_REDUCE_SMIN:
6453   case ISD::VP_REDUCE_UMIN:
6454     if (SDValue V = lowerVPREDUCE(SDValue(N, 0), DAG))
6455       Results.push_back(V);
6456     break;
6457   case ISD::FLT_ROUNDS_: {
6458     SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other);
6459     SDValue Res = DAG.getNode(ISD::FLT_ROUNDS_, DL, VTs, N->getOperand(0));
6460     Results.push_back(Res.getValue(0));
6461     Results.push_back(Res.getValue(1));
6462     break;
6463   }
6464   }
6465 }
6466 
6467 // A structure to hold one of the bit-manipulation patterns below. Together, a
6468 // SHL and non-SHL pattern may form a bit-manipulation pair on a single source:
6469 //   (or (and (shl x, 1), 0xAAAAAAAA),
6470 //       (and (srl x, 1), 0x55555555))
6471 struct RISCVBitmanipPat {
6472   SDValue Op;
6473   unsigned ShAmt;
6474   bool IsSHL;
6475 
6476   bool formsPairWith(const RISCVBitmanipPat &Other) const {
6477     return Op == Other.Op && ShAmt == Other.ShAmt && IsSHL != Other.IsSHL;
6478   }
6479 };
6480 
6481 // Matches patterns of the form
6482 //   (and (shl x, C2), (C1 << C2))
6483 //   (and (srl x, C2), C1)
6484 //   (shl (and x, C1), C2)
6485 //   (srl (and x, (C1 << C2)), C2)
6486 // Where C2 is a power of 2 and C1 has at least that many leading zeroes.
6487 // The expected masks for each shift amount are specified in BitmanipMasks where
6488 // BitmanipMasks[log2(C2)] specifies the expected C1 value.
6489 // The max allowed shift amount is either XLen/2 or XLen/4 determined by whether
6490 // BitmanipMasks contains 6 or 5 entries assuming that the maximum possible
6491 // XLen is 64.
6492 static Optional<RISCVBitmanipPat>
6493 matchRISCVBitmanipPat(SDValue Op, ArrayRef<uint64_t> BitmanipMasks) {
6494   assert((BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) &&
6495          "Unexpected number of masks");
6496   Optional<uint64_t> Mask;
6497   // Optionally consume a mask around the shift operation.
6498   if (Op.getOpcode() == ISD::AND && isa<ConstantSDNode>(Op.getOperand(1))) {
6499     Mask = Op.getConstantOperandVal(1);
6500     Op = Op.getOperand(0);
6501   }
6502   if (Op.getOpcode() != ISD::SHL && Op.getOpcode() != ISD::SRL)
6503     return None;
6504   bool IsSHL = Op.getOpcode() == ISD::SHL;
6505 
6506   if (!isa<ConstantSDNode>(Op.getOperand(1)))
6507     return None;
6508   uint64_t ShAmt = Op.getConstantOperandVal(1);
6509 
6510   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
6511   if (ShAmt >= Width || !isPowerOf2_64(ShAmt))
6512     return None;
6513   // If we don't have enough masks for 64 bit, then we must be trying to
6514   // match SHFL so we're only allowed to shift 1/4 of the width.
6515   if (BitmanipMasks.size() == 5 && ShAmt >= (Width / 2))
6516     return None;
6517 
6518   SDValue Src = Op.getOperand(0);
6519 
6520   // The expected mask is shifted left when the AND is found around SHL
6521   // patterns.
6522   //   ((x >> 1) & 0x55555555)
6523   //   ((x << 1) & 0xAAAAAAAA)
6524   bool SHLExpMask = IsSHL;
6525 
6526   if (!Mask) {
6527     // Sometimes LLVM keeps the mask as an operand of the shift, typically when
6528     // the mask is all ones: consume that now.
6529     if (Src.getOpcode() == ISD::AND && isa<ConstantSDNode>(Src.getOperand(1))) {
6530       Mask = Src.getConstantOperandVal(1);
6531       Src = Src.getOperand(0);
6532       // The expected mask is now in fact shifted left for SRL, so reverse the
6533       // decision.
6534       //   ((x & 0xAAAAAAAA) >> 1)
6535       //   ((x & 0x55555555) << 1)
6536       SHLExpMask = !SHLExpMask;
6537     } else {
6538       // Use a default shifted mask of all-ones if there's no AND, truncated
6539       // down to the expected width. This simplifies the logic later on.
6540       Mask = maskTrailingOnes<uint64_t>(Width);
6541       *Mask &= (IsSHL ? *Mask << ShAmt : *Mask >> ShAmt);
6542     }
6543   }
6544 
6545   unsigned MaskIdx = Log2_32(ShAmt);
6546   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
6547 
6548   if (SHLExpMask)
6549     ExpMask <<= ShAmt;
6550 
6551   if (Mask != ExpMask)
6552     return None;
6553 
6554   return RISCVBitmanipPat{Src, (unsigned)ShAmt, IsSHL};
6555 }
6556 
6557 // Matches any of the following bit-manipulation patterns:
6558 //   (and (shl x, 1), (0x55555555 << 1))
6559 //   (and (srl x, 1), 0x55555555)
6560 //   (shl (and x, 0x55555555), 1)
6561 //   (srl (and x, (0x55555555 << 1)), 1)
6562 // where the shift amount and mask may vary thus:
6563 //   [1]  = 0x55555555 / 0xAAAAAAAA
6564 //   [2]  = 0x33333333 / 0xCCCCCCCC
6565 //   [4]  = 0x0F0F0F0F / 0xF0F0F0F0
6566 //   [8]  = 0x00FF00FF / 0xFF00FF00
6567 //   [16] = 0x0000FFFF / 0xFFFFFFFF
6568 //   [32] = 0x00000000FFFFFFFF / 0xFFFFFFFF00000000 (for RV64)
6569 static Optional<RISCVBitmanipPat> matchGREVIPat(SDValue Op) {
6570   // These are the unshifted masks which we use to match bit-manipulation
6571   // patterns. They may be shifted left in certain circumstances.
6572   static const uint64_t BitmanipMasks[] = {
6573       0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
6574       0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
6575 
6576   return matchRISCVBitmanipPat(Op, BitmanipMasks);
6577 }
6578 
6579 // Match the following pattern as a GREVI(W) operation
6580 //   (or (BITMANIP_SHL x), (BITMANIP_SRL x))
6581 static SDValue combineORToGREV(SDValue Op, SelectionDAG &DAG,
6582                                const RISCVSubtarget &Subtarget) {
6583   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6584   EVT VT = Op.getValueType();
6585 
6586   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
6587     auto LHS = matchGREVIPat(Op.getOperand(0));
6588     auto RHS = matchGREVIPat(Op.getOperand(1));
6589     if (LHS && RHS && LHS->formsPairWith(*RHS)) {
6590       SDLoc DL(Op);
6591       return DAG.getNode(RISCVISD::GREV, DL, VT, LHS->Op,
6592                          DAG.getConstant(LHS->ShAmt, DL, VT));
6593     }
6594   }
6595   return SDValue();
6596 }
6597 
6598 // Matches any the following pattern as a GORCI(W) operation
6599 // 1.  (or (GREVI x, shamt), x) if shamt is a power of 2
6600 // 2.  (or x, (GREVI x, shamt)) if shamt is a power of 2
6601 // 3.  (or (or (BITMANIP_SHL x), x), (BITMANIP_SRL x))
6602 // Note that with the variant of 3.,
6603 //     (or (or (BITMANIP_SHL x), (BITMANIP_SRL x)), x)
6604 // the inner pattern will first be matched as GREVI and then the outer
6605 // pattern will be matched to GORC via the first rule above.
6606 // 4.  (or (rotl/rotr x, bitwidth/2), x)
6607 static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
6608                                const RISCVSubtarget &Subtarget) {
6609   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6610   EVT VT = Op.getValueType();
6611 
6612   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
6613     SDLoc DL(Op);
6614     SDValue Op0 = Op.getOperand(0);
6615     SDValue Op1 = Op.getOperand(1);
6616 
6617     auto MatchOROfReverse = [&](SDValue Reverse, SDValue X) {
6618       if (Reverse.getOpcode() == RISCVISD::GREV && Reverse.getOperand(0) == X &&
6619           isa<ConstantSDNode>(Reverse.getOperand(1)) &&
6620           isPowerOf2_32(Reverse.getConstantOperandVal(1)))
6621         return DAG.getNode(RISCVISD::GORC, DL, VT, X, Reverse.getOperand(1));
6622       // We can also form GORCI from ROTL/ROTR by half the bitwidth.
6623       if ((Reverse.getOpcode() == ISD::ROTL ||
6624            Reverse.getOpcode() == ISD::ROTR) &&
6625           Reverse.getOperand(0) == X &&
6626           isa<ConstantSDNode>(Reverse.getOperand(1))) {
6627         uint64_t RotAmt = Reverse.getConstantOperandVal(1);
6628         if (RotAmt == (VT.getSizeInBits() / 2))
6629           return DAG.getNode(RISCVISD::GORC, DL, VT, X,
6630                              DAG.getConstant(RotAmt, DL, VT));
6631       }
6632       return SDValue();
6633     };
6634 
6635     // Check for either commutable permutation of (or (GREVI x, shamt), x)
6636     if (SDValue V = MatchOROfReverse(Op0, Op1))
6637       return V;
6638     if (SDValue V = MatchOROfReverse(Op1, Op0))
6639       return V;
6640 
6641     // OR is commutable so canonicalize its OR operand to the left
6642     if (Op0.getOpcode() != ISD::OR && Op1.getOpcode() == ISD::OR)
6643       std::swap(Op0, Op1);
6644     if (Op0.getOpcode() != ISD::OR)
6645       return SDValue();
6646     SDValue OrOp0 = Op0.getOperand(0);
6647     SDValue OrOp1 = Op0.getOperand(1);
6648     auto LHS = matchGREVIPat(OrOp0);
6649     // OR is commutable so swap the operands and try again: x might have been
6650     // on the left
6651     if (!LHS) {
6652       std::swap(OrOp0, OrOp1);
6653       LHS = matchGREVIPat(OrOp0);
6654     }
6655     auto RHS = matchGREVIPat(Op1);
6656     if (LHS && RHS && LHS->formsPairWith(*RHS) && LHS->Op == OrOp1) {
6657       return DAG.getNode(RISCVISD::GORC, DL, VT, LHS->Op,
6658                          DAG.getConstant(LHS->ShAmt, DL, VT));
6659     }
6660   }
6661   return SDValue();
6662 }
6663 
6664 // Matches any of the following bit-manipulation patterns:
6665 //   (and (shl x, 1), (0x22222222 << 1))
6666 //   (and (srl x, 1), 0x22222222)
6667 //   (shl (and x, 0x22222222), 1)
6668 //   (srl (and x, (0x22222222 << 1)), 1)
6669 // where the shift amount and mask may vary thus:
6670 //   [1]  = 0x22222222 / 0x44444444
6671 //   [2]  = 0x0C0C0C0C / 0x3C3C3C3C
6672 //   [4]  = 0x00F000F0 / 0x0F000F00
6673 //   [8]  = 0x0000FF00 / 0x00FF0000
6674 //   [16] = 0x00000000FFFF0000 / 0x0000FFFF00000000 (for RV64)
6675 static Optional<RISCVBitmanipPat> matchSHFLPat(SDValue Op) {
6676   // These are the unshifted masks which we use to match bit-manipulation
6677   // patterns. They may be shifted left in certain circumstances.
6678   static const uint64_t BitmanipMasks[] = {
6679       0x2222222222222222ULL, 0x0C0C0C0C0C0C0C0CULL, 0x00F000F000F000F0ULL,
6680       0x0000FF000000FF00ULL, 0x00000000FFFF0000ULL};
6681 
6682   return matchRISCVBitmanipPat(Op, BitmanipMasks);
6683 }
6684 
6685 // Match (or (or (SHFL_SHL x), (SHFL_SHR x)), (SHFL_AND x)
6686 static SDValue combineORToSHFL(SDValue Op, SelectionDAG &DAG,
6687                                const RISCVSubtarget &Subtarget) {
6688   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6689   EVT VT = Op.getValueType();
6690 
6691   if (VT != MVT::i32 && VT != Subtarget.getXLenVT())
6692     return SDValue();
6693 
6694   SDValue Op0 = Op.getOperand(0);
6695   SDValue Op1 = Op.getOperand(1);
6696 
6697   // Or is commutable so canonicalize the second OR to the LHS.
6698   if (Op0.getOpcode() != ISD::OR)
6699     std::swap(Op0, Op1);
6700   if (Op0.getOpcode() != ISD::OR)
6701     return SDValue();
6702 
6703   // We found an inner OR, so our operands are the operands of the inner OR
6704   // and the other operand of the outer OR.
6705   SDValue A = Op0.getOperand(0);
6706   SDValue B = Op0.getOperand(1);
6707   SDValue C = Op1;
6708 
6709   auto Match1 = matchSHFLPat(A);
6710   auto Match2 = matchSHFLPat(B);
6711 
6712   // If neither matched, we failed.
6713   if (!Match1 && !Match2)
6714     return SDValue();
6715 
6716   // We had at least one match. if one failed, try the remaining C operand.
6717   if (!Match1) {
6718     std::swap(A, C);
6719     Match1 = matchSHFLPat(A);
6720     if (!Match1)
6721       return SDValue();
6722   } else if (!Match2) {
6723     std::swap(B, C);
6724     Match2 = matchSHFLPat(B);
6725     if (!Match2)
6726       return SDValue();
6727   }
6728   assert(Match1 && Match2);
6729 
6730   // Make sure our matches pair up.
6731   if (!Match1->formsPairWith(*Match2))
6732     return SDValue();
6733 
6734   // All the remains is to make sure C is an AND with the same input, that masks
6735   // out the bits that are being shuffled.
6736   if (C.getOpcode() != ISD::AND || !isa<ConstantSDNode>(C.getOperand(1)) ||
6737       C.getOperand(0) != Match1->Op)
6738     return SDValue();
6739 
6740   uint64_t Mask = C.getConstantOperandVal(1);
6741 
6742   static const uint64_t BitmanipMasks[] = {
6743       0x9999999999999999ULL, 0xC3C3C3C3C3C3C3C3ULL, 0xF00FF00FF00FF00FULL,
6744       0xFF0000FFFF0000FFULL, 0xFFFF00000000FFFFULL,
6745   };
6746 
6747   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
6748   unsigned MaskIdx = Log2_32(Match1->ShAmt);
6749   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
6750 
6751   if (Mask != ExpMask)
6752     return SDValue();
6753 
6754   SDLoc DL(Op);
6755   return DAG.getNode(RISCVISD::SHFL, DL, VT, Match1->Op,
6756                      DAG.getConstant(Match1->ShAmt, DL, VT));
6757 }
6758 
6759 // Optimize (add (shl x, c0), (shl y, c1)) ->
6760 //          (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
6761 static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
6762                                   const RISCVSubtarget &Subtarget) {
6763   // Perform this optimization only in the zba extension.
6764   if (!Subtarget.hasStdExtZba())
6765     return SDValue();
6766 
6767   // Skip for vector types and larger types.
6768   EVT VT = N->getValueType(0);
6769   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
6770     return SDValue();
6771 
6772   // The two operand nodes must be SHL and have no other use.
6773   SDValue N0 = N->getOperand(0);
6774   SDValue N1 = N->getOperand(1);
6775   if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
6776       !N0->hasOneUse() || !N1->hasOneUse())
6777     return SDValue();
6778 
6779   // Check c0 and c1.
6780   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
6781   auto *N1C = dyn_cast<ConstantSDNode>(N1->getOperand(1));
6782   if (!N0C || !N1C)
6783     return SDValue();
6784   int64_t C0 = N0C->getSExtValue();
6785   int64_t C1 = N1C->getSExtValue();
6786   if (C0 <= 0 || C1 <= 0)
6787     return SDValue();
6788 
6789   // Skip if SH1ADD/SH2ADD/SH3ADD are not applicable.
6790   int64_t Bits = std::min(C0, C1);
6791   int64_t Diff = std::abs(C0 - C1);
6792   if (Diff != 1 && Diff != 2 && Diff != 3)
6793     return SDValue();
6794 
6795   // Build nodes.
6796   SDLoc DL(N);
6797   SDValue NS = (C0 < C1) ? N0->getOperand(0) : N1->getOperand(0);
6798   SDValue NL = (C0 > C1) ? N0->getOperand(0) : N1->getOperand(0);
6799   SDValue NA0 =
6800       DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT));
6801   SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS);
6802   return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT));
6803 }
6804 
6805 // Combine (GREVI (GREVI x, C2), C1) -> (GREVI x, C1^C2) when C1^C2 is
6806 // non-zero, and to x when it is. Any repeated GREVI stage undoes itself.
6807 // Combine (GORCI (GORCI x, C2), C1) -> (GORCI x, C1|C2). Repeated stage does
6808 // not undo itself, but they are redundant.
6809 static SDValue combineGREVI_GORCI(SDNode *N, SelectionDAG &DAG) {
6810   SDValue Src = N->getOperand(0);
6811 
6812   if (Src.getOpcode() != N->getOpcode())
6813     return SDValue();
6814 
6815   if (!isa<ConstantSDNode>(N->getOperand(1)) ||
6816       !isa<ConstantSDNode>(Src.getOperand(1)))
6817     return SDValue();
6818 
6819   unsigned ShAmt1 = N->getConstantOperandVal(1);
6820   unsigned ShAmt2 = Src.getConstantOperandVal(1);
6821   Src = Src.getOperand(0);
6822 
6823   unsigned CombinedShAmt;
6824   if (N->getOpcode() == RISCVISD::GORC || N->getOpcode() == RISCVISD::GORCW)
6825     CombinedShAmt = ShAmt1 | ShAmt2;
6826   else
6827     CombinedShAmt = ShAmt1 ^ ShAmt2;
6828 
6829   if (CombinedShAmt == 0)
6830     return Src;
6831 
6832   SDLoc DL(N);
6833   return DAG.getNode(
6834       N->getOpcode(), DL, N->getValueType(0), Src,
6835       DAG.getConstant(CombinedShAmt, DL, N->getOperand(1).getValueType()));
6836 }
6837 
6838 // Combine a constant select operand into its use:
6839 //
6840 // (and (select cond, -1, c), x)
6841 //   -> (select cond, x, (and x, c))  [AllOnes=1]
6842 // (or  (select cond, 0, c), x)
6843 //   -> (select cond, x, (or x, c))  [AllOnes=0]
6844 // (xor (select cond, 0, c), x)
6845 //   -> (select cond, x, (xor x, c))  [AllOnes=0]
6846 // (add (select cond, 0, c), x)
6847 //   -> (select cond, x, (add x, c))  [AllOnes=0]
6848 // (sub x, (select cond, 0, c))
6849 //   -> (select cond, x, (sub x, c))  [AllOnes=0]
6850 static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
6851                                    SelectionDAG &DAG, bool AllOnes) {
6852   EVT VT = N->getValueType(0);
6853 
6854   // Skip vectors.
6855   if (VT.isVector())
6856     return SDValue();
6857 
6858   if ((Slct.getOpcode() != ISD::SELECT &&
6859        Slct.getOpcode() != RISCVISD::SELECT_CC) ||
6860       !Slct.hasOneUse())
6861     return SDValue();
6862 
6863   auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
6864     return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
6865   };
6866 
6867   bool SwapSelectOps;
6868   unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
6869   SDValue TrueVal = Slct.getOperand(1 + OpOffset);
6870   SDValue FalseVal = Slct.getOperand(2 + OpOffset);
6871   SDValue NonConstantVal;
6872   if (isZeroOrAllOnes(TrueVal, AllOnes)) {
6873     SwapSelectOps = false;
6874     NonConstantVal = FalseVal;
6875   } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
6876     SwapSelectOps = true;
6877     NonConstantVal = TrueVal;
6878   } else
6879     return SDValue();
6880 
6881   // Slct is now know to be the desired identity constant when CC is true.
6882   TrueVal = OtherOp;
6883   FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
6884   // Unless SwapSelectOps says the condition should be false.
6885   if (SwapSelectOps)
6886     std::swap(TrueVal, FalseVal);
6887 
6888   if (Slct.getOpcode() == RISCVISD::SELECT_CC)
6889     return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
6890                        {Slct.getOperand(0), Slct.getOperand(1),
6891                         Slct.getOperand(2), TrueVal, FalseVal});
6892 
6893   return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
6894                      {Slct.getOperand(0), TrueVal, FalseVal});
6895 }
6896 
6897 // Attempt combineSelectAndUse on each operand of a commutative operator N.
6898 static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
6899                                               bool AllOnes) {
6900   SDValue N0 = N->getOperand(0);
6901   SDValue N1 = N->getOperand(1);
6902   if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes))
6903     return Result;
6904   if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes))
6905     return Result;
6906   return SDValue();
6907 }
6908 
6909 // Transform (add (mul x, c0), c1) ->
6910 //           (add (mul (add x, c1/c0), c0), c1%c0).
6911 // if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
6912 // that should be excluded is when c0*(c1/c0) is simm12, which will lead
6913 // to an infinite loop in DAGCombine if transformed.
6914 // Or transform (add (mul x, c0), c1) ->
6915 //              (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
6916 // if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
6917 // case that should be excluded is when c0*(c1/c0+1) is simm12, which will
6918 // lead to an infinite loop in DAGCombine if transformed.
6919 // Or transform (add (mul x, c0), c1) ->
6920 //              (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
6921 // if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
6922 // case that should be excluded is when c0*(c1/c0-1) is simm12, which will
6923 // lead to an infinite loop in DAGCombine if transformed.
6924 // Or transform (add (mul x, c0), c1) ->
6925 //              (mul (add x, c1/c0), c0).
6926 // if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
6927 static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
6928                                      const RISCVSubtarget &Subtarget) {
6929   // Skip for vector types and larger types.
6930   EVT VT = N->getValueType(0);
6931   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
6932     return SDValue();
6933   // The first operand node must be a MUL and has no other use.
6934   SDValue N0 = N->getOperand(0);
6935   if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
6936     return SDValue();
6937   // Check if c0 and c1 match above conditions.
6938   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
6939   auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
6940   if (!N0C || !N1C)
6941     return SDValue();
6942   int64_t C0 = N0C->getSExtValue();
6943   int64_t C1 = N1C->getSExtValue();
6944   int64_t CA, CB;
6945   if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
6946     return SDValue();
6947   // Search for proper CA (non-zero) and CB that both are simm12.
6948   if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
6949       !isInt<12>(C0 * (C1 / C0))) {
6950     CA = C1 / C0;
6951     CB = C1 % C0;
6952   } else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
6953              isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
6954     CA = C1 / C0 + 1;
6955     CB = C1 % C0 - C0;
6956   } else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
6957              isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
6958     CA = C1 / C0 - 1;
6959     CB = C1 % C0 + C0;
6960   } else
6961     return SDValue();
6962   // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
6963   SDLoc DL(N);
6964   SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
6965                              DAG.getConstant(CA, DL, VT));
6966   SDValue New1 =
6967       DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
6968   return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
6969 }
6970 
6971 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
6972                                  const RISCVSubtarget &Subtarget) {
6973   if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
6974     return V;
6975   if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
6976     return V;
6977   // fold (add (select lhs, rhs, cc, 0, y), x) ->
6978   //      (select lhs, rhs, cc, x, (add x, y))
6979   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
6980 }
6981 
6982 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG) {
6983   // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
6984   //      (select lhs, rhs, cc, x, (sub x, y))
6985   SDValue N0 = N->getOperand(0);
6986   SDValue N1 = N->getOperand(1);
6987   return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false);
6988 }
6989 
6990 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG) {
6991   // fold (and (select lhs, rhs, cc, -1, y), x) ->
6992   //      (select lhs, rhs, cc, x, (and x, y))
6993   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true);
6994 }
6995 
6996 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
6997                                 const RISCVSubtarget &Subtarget) {
6998   if (Subtarget.hasStdExtZbp()) {
6999     if (auto GREV = combineORToGREV(SDValue(N, 0), DAG, Subtarget))
7000       return GREV;
7001     if (auto GORC = combineORToGORC(SDValue(N, 0), DAG, Subtarget))
7002       return GORC;
7003     if (auto SHFL = combineORToSHFL(SDValue(N, 0), DAG, Subtarget))
7004       return SHFL;
7005   }
7006 
7007   // fold (or (select cond, 0, y), x) ->
7008   //      (select cond, x, (or x, y))
7009   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7010 }
7011 
7012 static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG) {
7013   // fold (xor (select cond, 0, y), x) ->
7014   //      (select cond, x, (xor x, y))
7015   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7016 }
7017 
7018 // Attempt to turn ANY_EXTEND into SIGN_EXTEND if the input to the ANY_EXTEND
7019 // has users that require SIGN_EXTEND and the SIGN_EXTEND can be done for free
7020 // by an instruction like ADDW/SUBW/MULW. Without this the ANY_EXTEND would be
7021 // removed during type legalization leaving an ADD/SUB/MUL use that won't use
7022 // ADDW/SUBW/MULW.
7023 static SDValue performANY_EXTENDCombine(SDNode *N,
7024                                         TargetLowering::DAGCombinerInfo &DCI,
7025                                         const RISCVSubtarget &Subtarget) {
7026   if (!Subtarget.is64Bit())
7027     return SDValue();
7028 
7029   SelectionDAG &DAG = DCI.DAG;
7030 
7031   SDValue Src = N->getOperand(0);
7032   EVT VT = N->getValueType(0);
7033   if (VT != MVT::i64 || Src.getValueType() != MVT::i32)
7034     return SDValue();
7035 
7036   // The opcode must be one that can implicitly sign_extend.
7037   // FIXME: Additional opcodes.
7038   switch (Src.getOpcode()) {
7039   default:
7040     return SDValue();
7041   case ISD::MUL:
7042     if (!Subtarget.hasStdExtM())
7043       return SDValue();
7044     LLVM_FALLTHROUGH;
7045   case ISD::ADD:
7046   case ISD::SUB:
7047     break;
7048   }
7049 
7050   // Only handle cases where the result is used by a CopyToReg. That likely
7051   // means the value is a liveout of the basic block. This helps prevent
7052   // infinite combine loops like PR51206.
7053   if (none_of(N->uses(),
7054               [](SDNode *User) { return User->getOpcode() == ISD::CopyToReg; }))
7055     return SDValue();
7056 
7057   SmallVector<SDNode *, 4> SetCCs;
7058   for (SDNode::use_iterator UI = Src.getNode()->use_begin(),
7059                             UE = Src.getNode()->use_end();
7060        UI != UE; ++UI) {
7061     SDNode *User = *UI;
7062     if (User == N)
7063       continue;
7064     if (UI.getUse().getResNo() != Src.getResNo())
7065       continue;
7066     // All i32 setccs are legalized by sign extending operands.
7067     if (User->getOpcode() == ISD::SETCC) {
7068       SetCCs.push_back(User);
7069       continue;
7070     }
7071     // We don't know if we can extend this user.
7072     break;
7073   }
7074 
7075   // If we don't have any SetCCs, this isn't worthwhile.
7076   if (SetCCs.empty())
7077     return SDValue();
7078 
7079   SDLoc DL(N);
7080   SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Src);
7081   DCI.CombineTo(N, SExt);
7082 
7083   // Promote all the setccs.
7084   for (SDNode *SetCC : SetCCs) {
7085     SmallVector<SDValue, 4> Ops;
7086 
7087     for (unsigned j = 0; j != 2; ++j) {
7088       SDValue SOp = SetCC->getOperand(j);
7089       if (SOp == Src)
7090         Ops.push_back(SExt);
7091       else
7092         Ops.push_back(DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, SOp));
7093     }
7094 
7095     Ops.push_back(SetCC->getOperand(2));
7096     DCI.CombineTo(SetCC,
7097                   DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
7098   }
7099   return SDValue(N, 0);
7100 }
7101 
7102 // Try to form VWMUL or VWMULU.
7103 // FIXME: Support VWMULSU.
7104 static SDValue combineMUL_VLToVWMUL(SDNode *N, SDValue Op0, SDValue Op1,
7105                                     SelectionDAG &DAG) {
7106   assert(N->getOpcode() == RISCVISD::MUL_VL && "Unexpected opcode");
7107   bool IsSignExt = Op0.getOpcode() == RISCVISD::VSEXT_VL;
7108   bool IsZeroExt = Op0.getOpcode() == RISCVISD::VZEXT_VL;
7109   if ((!IsSignExt && !IsZeroExt) || !Op0.hasOneUse())
7110     return SDValue();
7111 
7112   SDValue Mask = N->getOperand(2);
7113   SDValue VL = N->getOperand(3);
7114 
7115   // Make sure the mask and VL match.
7116   if (Op0.getOperand(1) != Mask || Op0.getOperand(2) != VL)
7117     return SDValue();
7118 
7119   MVT VT = N->getSimpleValueType(0);
7120 
7121   // Determine the narrow size for a widening multiply.
7122   unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
7123   MVT NarrowVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowSize),
7124                                   VT.getVectorElementCount());
7125 
7126   SDLoc DL(N);
7127 
7128   // See if the other operand is the same opcode.
7129   if (Op0.getOpcode() == Op1.getOpcode()) {
7130     if (!Op1.hasOneUse())
7131       return SDValue();
7132 
7133     // Make sure the mask and VL match.
7134     if (Op1.getOperand(1) != Mask || Op1.getOperand(2) != VL)
7135       return SDValue();
7136 
7137     Op1 = Op1.getOperand(0);
7138   } else if (Op1.getOpcode() == RISCVISD::VMV_V_X_VL) {
7139     // The operand is a splat of a scalar.
7140 
7141     // The VL must be the same.
7142     if (Op1.getOperand(1) != VL)
7143       return SDValue();
7144 
7145     // Get the scalar value.
7146     Op1 = Op1.getOperand(0);
7147 
7148     // See if have enough sign bits or zero bits in the scalar to use a
7149     // widening multiply by splatting to smaller element size.
7150     unsigned EltBits = VT.getScalarSizeInBits();
7151     unsigned ScalarBits = Op1.getValueSizeInBits();
7152     // Make sure we're getting all element bits from the scalar register.
7153     // FIXME: Support implicit sign extension of vmv.v.x?
7154     if (ScalarBits < EltBits)
7155       return SDValue();
7156 
7157     if (IsSignExt) {
7158       if (DAG.ComputeNumSignBits(Op1) <= (ScalarBits - NarrowSize))
7159         return SDValue();
7160     } else {
7161       APInt Mask = APInt::getBitsSetFrom(ScalarBits, NarrowSize);
7162       if (!DAG.MaskedValueIsZero(Op1, Mask))
7163         return SDValue();
7164     }
7165 
7166     Op1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT, Op1, VL);
7167   } else
7168     return SDValue();
7169 
7170   Op0 = Op0.getOperand(0);
7171 
7172   // Re-introduce narrower extends if needed.
7173   unsigned ExtOpc = IsSignExt ? RISCVISD::VSEXT_VL : RISCVISD::VZEXT_VL;
7174   if (Op0.getValueType() != NarrowVT)
7175     Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
7176   if (Op1.getValueType() != NarrowVT)
7177     Op1 = DAG.getNode(ExtOpc, DL, NarrowVT, Op1, Mask, VL);
7178 
7179   unsigned WMulOpc = IsSignExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
7180   return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Mask, VL);
7181 }
7182 
7183 // Fold
7184 //   (fp_to_int (froundeven X)) -> fcvt X, rne
7185 //   (fp_to_int (ftrunc X))     -> fcvt X, rtz
7186 //   (fp_to_int (ffloor X))     -> fcvt X, rdn
7187 //   (fp_to_int (fceil X))      -> fcvt X, rup
7188 //   (fp_to_int (fround X))     -> fcvt X, rmm
7189 // FIXME: We should also do this for fp_to_int_sat.
7190 static SDValue performFP_TO_INTCombine(SDNode *N,
7191                                        TargetLowering::DAGCombinerInfo &DCI,
7192                                        const RISCVSubtarget &Subtarget) {
7193   SelectionDAG &DAG = DCI.DAG;
7194   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7195   MVT XLenVT = Subtarget.getXLenVT();
7196 
7197   // Only handle XLen or i32 types. Other types narrower than XLen will
7198   // eventually be legalized to XLenVT.
7199   EVT VT = N->getValueType(0);
7200   if (VT != MVT::i32 && VT != XLenVT)
7201     return SDValue();
7202 
7203   SDValue Src = N->getOperand(0);
7204 
7205   // Ensure the FP type is also legal.
7206   if (!TLI.isTypeLegal(Src.getValueType()))
7207     return SDValue();
7208 
7209   // Don't do this for f16 with Zfhmin and not Zfh.
7210   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7211     return SDValue();
7212 
7213   RISCVFPRndMode::RoundingMode FRM;
7214   switch (Src->getOpcode()) {
7215   default:
7216     return SDValue();
7217   case ISD::FROUNDEVEN: FRM = RISCVFPRndMode::RNE; break;
7218   case ISD::FTRUNC:     FRM = RISCVFPRndMode::RTZ; break;
7219   case ISD::FFLOOR:     FRM = RISCVFPRndMode::RDN; break;
7220   case ISD::FCEIL:      FRM = RISCVFPRndMode::RUP; break;
7221   case ISD::FROUND:     FRM = RISCVFPRndMode::RMM; break;
7222   }
7223 
7224   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
7225 
7226   unsigned Opc;
7227   if (VT == XLenVT)
7228     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7229   else
7230     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7231 
7232   SDLoc DL(N);
7233   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src.getOperand(0),
7234                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7235   return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
7236 }
7237 
7238 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
7239                                                DAGCombinerInfo &DCI) const {
7240   SelectionDAG &DAG = DCI.DAG;
7241 
7242   // Helper to call SimplifyDemandedBits on an operand of N where only some low
7243   // bits are demanded. N will be added to the Worklist if it was not deleted.
7244   // Caller should return SDValue(N, 0) if this returns true.
7245   auto SimplifyDemandedLowBitsHelper = [&](unsigned OpNo, unsigned LowBits) {
7246     SDValue Op = N->getOperand(OpNo);
7247     APInt Mask = APInt::getLowBitsSet(Op.getValueSizeInBits(), LowBits);
7248     if (!SimplifyDemandedBits(Op, Mask, DCI))
7249       return false;
7250 
7251     if (N->getOpcode() != ISD::DELETED_NODE)
7252       DCI.AddToWorklist(N);
7253     return true;
7254   };
7255 
7256   switch (N->getOpcode()) {
7257   default:
7258     break;
7259   case RISCVISD::SplitF64: {
7260     SDValue Op0 = N->getOperand(0);
7261     // If the input to SplitF64 is just BuildPairF64 then the operation is
7262     // redundant. Instead, use BuildPairF64's operands directly.
7263     if (Op0->getOpcode() == RISCVISD::BuildPairF64)
7264       return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
7265 
7266     SDLoc DL(N);
7267 
7268     // It's cheaper to materialise two 32-bit integers than to load a double
7269     // from the constant pool and transfer it to integer registers through the
7270     // stack.
7271     if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
7272       APInt V = C->getValueAPF().bitcastToAPInt();
7273       SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
7274       SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
7275       return DCI.CombineTo(N, Lo, Hi);
7276     }
7277 
7278     // This is a target-specific version of a DAGCombine performed in
7279     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7280     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7281     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7282     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7283         !Op0.getNode()->hasOneUse())
7284       break;
7285     SDValue NewSplitF64 =
7286         DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
7287                     Op0.getOperand(0));
7288     SDValue Lo = NewSplitF64.getValue(0);
7289     SDValue Hi = NewSplitF64.getValue(1);
7290     APInt SignBit = APInt::getSignMask(32);
7291     if (Op0.getOpcode() == ISD::FNEG) {
7292       SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
7293                                   DAG.getConstant(SignBit, DL, MVT::i32));
7294       return DCI.CombineTo(N, Lo, NewHi);
7295     }
7296     assert(Op0.getOpcode() == ISD::FABS);
7297     SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
7298                                 DAG.getConstant(~SignBit, DL, MVT::i32));
7299     return DCI.CombineTo(N, Lo, NewHi);
7300   }
7301   case RISCVISD::SLLW:
7302   case RISCVISD::SRAW:
7303   case RISCVISD::SRLW:
7304   case RISCVISD::ROLW:
7305   case RISCVISD::RORW: {
7306     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7307     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7308         SimplifyDemandedLowBitsHelper(1, 5))
7309       return SDValue(N, 0);
7310     break;
7311   }
7312   case RISCVISD::CLZW:
7313   case RISCVISD::CTZW: {
7314     // Only the lower 32 bits of the first operand are read
7315     if (SimplifyDemandedLowBitsHelper(0, 32))
7316       return SDValue(N, 0);
7317     break;
7318   }
7319   case RISCVISD::GREV:
7320   case RISCVISD::GORC: {
7321     // Only the lower log2(Bitwidth) bits of the the shift amount are read.
7322     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7323     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7324     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth)))
7325       return SDValue(N, 0);
7326 
7327     return combineGREVI_GORCI(N, DAG);
7328   }
7329   case RISCVISD::GREVW:
7330   case RISCVISD::GORCW: {
7331     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7332     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7333         SimplifyDemandedLowBitsHelper(1, 5))
7334       return SDValue(N, 0);
7335 
7336     return combineGREVI_GORCI(N, DAG);
7337   }
7338   case RISCVISD::SHFL:
7339   case RISCVISD::UNSHFL: {
7340     // Only the lower log2(Bitwidth)-1 bits of the the shift amount are read.
7341     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7342     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7343     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth) - 1))
7344       return SDValue(N, 0);
7345 
7346     break;
7347   }
7348   case RISCVISD::SHFLW:
7349   case RISCVISD::UNSHFLW: {
7350     // Only the lower 32 bits of LHS and lower 4 bits of RHS are read.
7351     SDValue LHS = N->getOperand(0);
7352     SDValue RHS = N->getOperand(1);
7353     APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
7354     APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 4);
7355     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7356         SimplifyDemandedLowBitsHelper(1, 4))
7357       return SDValue(N, 0);
7358 
7359     break;
7360   }
7361   case RISCVISD::BCOMPRESSW:
7362   case RISCVISD::BDECOMPRESSW: {
7363     // Only the lower 32 bits of LHS and RHS are read.
7364     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7365         SimplifyDemandedLowBitsHelper(1, 32))
7366       return SDValue(N, 0);
7367 
7368     break;
7369   }
7370   case RISCVISD::FMV_X_ANYEXTH:
7371   case RISCVISD::FMV_X_ANYEXTW_RV64: {
7372     SDLoc DL(N);
7373     SDValue Op0 = N->getOperand(0);
7374     MVT VT = N->getSimpleValueType(0);
7375     // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
7376     // conversion is unnecessary and can be replaced with the FMV_W_X_RV64
7377     // operand. Similar for FMV_X_ANYEXTH and FMV_H_X.
7378     if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
7379          Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
7380         (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
7381          Op0->getOpcode() == RISCVISD::FMV_H_X)) {
7382       assert(Op0.getOperand(0).getValueType() == VT &&
7383              "Unexpected value type!");
7384       return Op0.getOperand(0);
7385     }
7386 
7387     // This is a target-specific version of a DAGCombine performed in
7388     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7389     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7390     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7391     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7392         !Op0.getNode()->hasOneUse())
7393       break;
7394     SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
7395     unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
7396     APInt SignBit = APInt::getSignMask(FPBits).sextOrSelf(VT.getSizeInBits());
7397     if (Op0.getOpcode() == ISD::FNEG)
7398       return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
7399                          DAG.getConstant(SignBit, DL, VT));
7400 
7401     assert(Op0.getOpcode() == ISD::FABS);
7402     return DAG.getNode(ISD::AND, DL, VT, NewFMV,
7403                        DAG.getConstant(~SignBit, DL, VT));
7404   }
7405   case ISD::ADD:
7406     return performADDCombine(N, DAG, Subtarget);
7407   case ISD::SUB:
7408     return performSUBCombine(N, DAG);
7409   case ISD::AND:
7410     return performANDCombine(N, DAG);
7411   case ISD::OR:
7412     return performORCombine(N, DAG, Subtarget);
7413   case ISD::XOR:
7414     return performXORCombine(N, DAG);
7415   case ISD::ANY_EXTEND:
7416     return performANY_EXTENDCombine(N, DCI, Subtarget);
7417   case ISD::ZERO_EXTEND:
7418     // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
7419     // type legalization. This is safe because fp_to_uint produces poison if
7420     // it overflows.
7421     if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) {
7422       SDValue Src = N->getOperand(0);
7423       if (Src.getOpcode() == ISD::FP_TO_UINT &&
7424           isTypeLegal(Src.getOperand(0).getValueType()))
7425         return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
7426                            Src.getOperand(0));
7427       if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
7428           isTypeLegal(Src.getOperand(1).getValueType())) {
7429         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
7430         SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs,
7431                                   Src.getOperand(0), Src.getOperand(1));
7432         DCI.CombineTo(N, Res);
7433         DAG.ReplaceAllUsesOfValueWith(Src.getValue(1), Res.getValue(1));
7434         DCI.recursivelyDeleteUnusedNodes(Src.getNode());
7435         return SDValue(N, 0); // Return N so it doesn't get rechecked.
7436       }
7437     }
7438     return SDValue();
7439   case RISCVISD::SELECT_CC: {
7440     // Transform
7441     SDValue LHS = N->getOperand(0);
7442     SDValue RHS = N->getOperand(1);
7443     SDValue TrueV = N->getOperand(3);
7444     SDValue FalseV = N->getOperand(4);
7445 
7446     // If the True and False values are the same, we don't need a select_cc.
7447     if (TrueV == FalseV)
7448       return TrueV;
7449 
7450     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(2))->get();
7451     if (!ISD::isIntEqualitySetCC(CCVal))
7452       break;
7453 
7454     // Fold (select_cc (setlt X, Y), 0, ne, trueV, falseV) ->
7455     //      (select_cc X, Y, lt, trueV, falseV)
7456     // Sometimes the setcc is introduced after select_cc has been formed.
7457     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
7458         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
7459       // If we're looking for eq 0 instead of ne 0, we need to invert the
7460       // condition.
7461       bool Invert = CCVal == ISD::SETEQ;
7462       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
7463       if (Invert)
7464         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7465 
7466       SDLoc DL(N);
7467       RHS = LHS.getOperand(1);
7468       LHS = LHS.getOperand(0);
7469       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7470 
7471       SDValue TargetCC = DAG.getCondCode(CCVal);
7472       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
7473                          {LHS, RHS, TargetCC, TrueV, FalseV});
7474     }
7475 
7476     // Fold (select_cc (xor X, Y), 0, eq/ne, trueV, falseV) ->
7477     //      (select_cc X, Y, eq/ne, trueV, falseV)
7478     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
7479       return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), N->getValueType(0),
7480                          {LHS.getOperand(0), LHS.getOperand(1),
7481                           N->getOperand(2), TrueV, FalseV});
7482     // (select_cc X, 1, setne, trueV, falseV) ->
7483     // (select_cc X, 0, seteq, trueV, falseV) if we can prove X is 0/1.
7484     // This can occur when legalizing some floating point comparisons.
7485     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
7486     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
7487       SDLoc DL(N);
7488       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7489       SDValue TargetCC = DAG.getCondCode(CCVal);
7490       RHS = DAG.getConstant(0, DL, LHS.getValueType());
7491       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
7492                          {LHS, RHS, TargetCC, TrueV, FalseV});
7493     }
7494 
7495     break;
7496   }
7497   case RISCVISD::BR_CC: {
7498     SDValue LHS = N->getOperand(1);
7499     SDValue RHS = N->getOperand(2);
7500     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(3))->get();
7501     if (!ISD::isIntEqualitySetCC(CCVal))
7502       break;
7503 
7504     // Fold (br_cc (setlt X, Y), 0, ne, dest) ->
7505     //      (br_cc X, Y, lt, dest)
7506     // Sometimes the setcc is introduced after br_cc has been formed.
7507     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
7508         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
7509       // If we're looking for eq 0 instead of ne 0, we need to invert the
7510       // condition.
7511       bool Invert = CCVal == ISD::SETEQ;
7512       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
7513       if (Invert)
7514         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7515 
7516       SDLoc DL(N);
7517       RHS = LHS.getOperand(1);
7518       LHS = LHS.getOperand(0);
7519       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7520 
7521       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
7522                          N->getOperand(0), LHS, RHS, DAG.getCondCode(CCVal),
7523                          N->getOperand(4));
7524     }
7525 
7526     // Fold (br_cc (xor X, Y), 0, eq/ne, dest) ->
7527     //      (br_cc X, Y, eq/ne, trueV, falseV)
7528     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
7529       return DAG.getNode(RISCVISD::BR_CC, SDLoc(N), N->getValueType(0),
7530                          N->getOperand(0), LHS.getOperand(0), LHS.getOperand(1),
7531                          N->getOperand(3), N->getOperand(4));
7532 
7533     // (br_cc X, 1, setne, br_cc) ->
7534     // (br_cc X, 0, seteq, br_cc) if we can prove X is 0/1.
7535     // This can occur when legalizing some floating point comparisons.
7536     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
7537     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
7538       SDLoc DL(N);
7539       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7540       SDValue TargetCC = DAG.getCondCode(CCVal);
7541       RHS = DAG.getConstant(0, DL, LHS.getValueType());
7542       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
7543                          N->getOperand(0), LHS, RHS, TargetCC,
7544                          N->getOperand(4));
7545     }
7546     break;
7547   }
7548   case ISD::FP_TO_SINT:
7549   case ISD::FP_TO_UINT:
7550     return performFP_TO_INTCombine(N, DCI, Subtarget);
7551   case ISD::FCOPYSIGN: {
7552     EVT VT = N->getValueType(0);
7553     if (!VT.isVector())
7554       break;
7555     // There is a form of VFSGNJ which injects the negated sign of its second
7556     // operand. Try and bubble any FNEG up after the extend/round to produce
7557     // this optimized pattern. Avoid modifying cases where FP_ROUND and
7558     // TRUNC=1.
7559     SDValue In2 = N->getOperand(1);
7560     // Avoid cases where the extend/round has multiple uses, as duplicating
7561     // those is typically more expensive than removing a fneg.
7562     if (!In2.hasOneUse())
7563       break;
7564     if (In2.getOpcode() != ISD::FP_EXTEND &&
7565         (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0))
7566       break;
7567     In2 = In2.getOperand(0);
7568     if (In2.getOpcode() != ISD::FNEG)
7569       break;
7570     SDLoc DL(N);
7571     SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
7572     return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
7573                        DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
7574   }
7575   case ISD::MGATHER:
7576   case ISD::MSCATTER:
7577   case ISD::VP_GATHER:
7578   case ISD::VP_SCATTER: {
7579     if (!DCI.isBeforeLegalize())
7580       break;
7581     SDValue Index, ScaleOp;
7582     bool IsIndexScaled = false;
7583     bool IsIndexSigned = false;
7584     if (const auto *VPGSN = dyn_cast<VPGatherScatterSDNode>(N)) {
7585       Index = VPGSN->getIndex();
7586       ScaleOp = VPGSN->getScale();
7587       IsIndexScaled = VPGSN->isIndexScaled();
7588       IsIndexSigned = VPGSN->isIndexSigned();
7589     } else {
7590       const auto *MGSN = cast<MaskedGatherScatterSDNode>(N);
7591       Index = MGSN->getIndex();
7592       ScaleOp = MGSN->getScale();
7593       IsIndexScaled = MGSN->isIndexScaled();
7594       IsIndexSigned = MGSN->isIndexSigned();
7595     }
7596     EVT IndexVT = Index.getValueType();
7597     MVT XLenVT = Subtarget.getXLenVT();
7598     // RISCV indexed loads only support the "unsigned unscaled" addressing
7599     // mode, so anything else must be manually legalized.
7600     bool NeedsIdxLegalization =
7601         IsIndexScaled ||
7602         (IsIndexSigned && IndexVT.getVectorElementType().bitsLT(XLenVT));
7603     if (!NeedsIdxLegalization)
7604       break;
7605 
7606     SDLoc DL(N);
7607 
7608     // Any index legalization should first promote to XLenVT, so we don't lose
7609     // bits when scaling. This may create an illegal index type so we let
7610     // LLVM's legalization take care of the splitting.
7611     // FIXME: LLVM can't split VP_GATHER or VP_SCATTER yet.
7612     if (IndexVT.getVectorElementType().bitsLT(XLenVT)) {
7613       IndexVT = IndexVT.changeVectorElementType(XLenVT);
7614       Index = DAG.getNode(IsIndexSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
7615                           DL, IndexVT, Index);
7616     }
7617 
7618     unsigned Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue();
7619     if (IsIndexScaled && Scale != 1) {
7620       // Manually scale the indices by the element size.
7621       // TODO: Sanitize the scale operand here?
7622       // TODO: For VP nodes, should we use VP_SHL here?
7623       assert(isPowerOf2_32(Scale) && "Expecting power-of-two types");
7624       SDValue SplatScale = DAG.getConstant(Log2_32(Scale), DL, IndexVT);
7625       Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, SplatScale);
7626     }
7627 
7628     ISD::MemIndexType NewIndexTy = ISD::UNSIGNED_UNSCALED;
7629     if (const auto *VPGN = dyn_cast<VPGatherSDNode>(N))
7630       return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
7631                              {VPGN->getChain(), VPGN->getBasePtr(), Index,
7632                               VPGN->getScale(), VPGN->getMask(),
7633                               VPGN->getVectorLength()},
7634                              VPGN->getMemOperand(), NewIndexTy);
7635     if (const auto *VPSN = dyn_cast<VPScatterSDNode>(N))
7636       return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
7637                               {VPSN->getChain(), VPSN->getValue(),
7638                                VPSN->getBasePtr(), Index, VPSN->getScale(),
7639                                VPSN->getMask(), VPSN->getVectorLength()},
7640                               VPSN->getMemOperand(), NewIndexTy);
7641     if (const auto *MGN = dyn_cast<MaskedGatherSDNode>(N))
7642       return DAG.getMaskedGather(
7643           N->getVTList(), MGN->getMemoryVT(), DL,
7644           {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
7645            MGN->getBasePtr(), Index, MGN->getScale()},
7646           MGN->getMemOperand(), NewIndexTy, MGN->getExtensionType());
7647     const auto *MSN = cast<MaskedScatterSDNode>(N);
7648     return DAG.getMaskedScatter(
7649         N->getVTList(), MSN->getMemoryVT(), DL,
7650         {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
7651          Index, MSN->getScale()},
7652         MSN->getMemOperand(), NewIndexTy, MSN->isTruncatingStore());
7653   }
7654   case RISCVISD::SRA_VL:
7655   case RISCVISD::SRL_VL:
7656   case RISCVISD::SHL_VL: {
7657     SDValue ShAmt = N->getOperand(1);
7658     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
7659       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
7660       SDLoc DL(N);
7661       SDValue VL = N->getOperand(3);
7662       EVT VT = N->getValueType(0);
7663       ShAmt =
7664           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, ShAmt.getOperand(0), VL);
7665       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
7666                          N->getOperand(2), N->getOperand(3));
7667     }
7668     break;
7669   }
7670   case ISD::SRA:
7671   case ISD::SRL:
7672   case ISD::SHL: {
7673     SDValue ShAmt = N->getOperand(1);
7674     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
7675       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
7676       SDLoc DL(N);
7677       EVT VT = N->getValueType(0);
7678       ShAmt =
7679           DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VT, ShAmt.getOperand(0));
7680       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
7681     }
7682     break;
7683   }
7684   case RISCVISD::MUL_VL: {
7685     SDValue Op0 = N->getOperand(0);
7686     SDValue Op1 = N->getOperand(1);
7687     if (SDValue V = combineMUL_VLToVWMUL(N, Op0, Op1, DAG))
7688       return V;
7689     if (SDValue V = combineMUL_VLToVWMUL(N, Op1, Op0, DAG))
7690       return V;
7691     return SDValue();
7692   }
7693   case ISD::STORE: {
7694     auto *Store = cast<StoreSDNode>(N);
7695     SDValue Val = Store->getValue();
7696     // Combine store of vmv.x.s to vse with VL of 1.
7697     // FIXME: Support FP.
7698     if (Val.getOpcode() == RISCVISD::VMV_X_S) {
7699       SDValue Src = Val.getOperand(0);
7700       EVT VecVT = Src.getValueType();
7701       EVT MemVT = Store->getMemoryVT();
7702       // The memory VT and the element type must match.
7703       if (VecVT.getVectorElementType() == MemVT) {
7704         SDLoc DL(N);
7705         MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
7706         return DAG.getStoreVP(
7707             Store->getChain(), DL, Src, Store->getBasePtr(), Store->getOffset(),
7708             DAG.getConstant(1, DL, MaskVT),
7709             DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT,
7710             Store->getMemOperand(), Store->getAddressingMode(),
7711             Store->isTruncatingStore(), /*IsCompress*/ false);
7712       }
7713     }
7714 
7715     break;
7716   }
7717   }
7718 
7719   return SDValue();
7720 }
7721 
7722 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
7723     const SDNode *N, CombineLevel Level) const {
7724   // The following folds are only desirable if `(OP _, c1 << c2)` can be
7725   // materialised in fewer instructions than `(OP _, c1)`:
7726   //
7727   //   (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
7728   //   (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
7729   SDValue N0 = N->getOperand(0);
7730   EVT Ty = N0.getValueType();
7731   if (Ty.isScalarInteger() &&
7732       (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
7733     auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7734     auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
7735     if (C1 && C2) {
7736       const APInt &C1Int = C1->getAPIntValue();
7737       APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
7738 
7739       // We can materialise `c1 << c2` into an add immediate, so it's "free",
7740       // and the combine should happen, to potentially allow further combines
7741       // later.
7742       if (ShiftedC1Int.getMinSignedBits() <= 64 &&
7743           isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
7744         return true;
7745 
7746       // We can materialise `c1` in an add immediate, so it's "free", and the
7747       // combine should be prevented.
7748       if (C1Int.getMinSignedBits() <= 64 &&
7749           isLegalAddImmediate(C1Int.getSExtValue()))
7750         return false;
7751 
7752       // Neither constant will fit into an immediate, so find materialisation
7753       // costs.
7754       int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
7755                                               Subtarget.getFeatureBits(),
7756                                               /*CompressionCost*/true);
7757       int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
7758           ShiftedC1Int, Ty.getSizeInBits(), Subtarget.getFeatureBits(),
7759           /*CompressionCost*/true);
7760 
7761       // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
7762       // combine should be prevented.
7763       if (C1Cost < ShiftedC1Cost)
7764         return false;
7765     }
7766   }
7767   return true;
7768 }
7769 
7770 bool RISCVTargetLowering::targetShrinkDemandedConstant(
7771     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
7772     TargetLoweringOpt &TLO) const {
7773   // Delay this optimization as late as possible.
7774   if (!TLO.LegalOps)
7775     return false;
7776 
7777   EVT VT = Op.getValueType();
7778   if (VT.isVector())
7779     return false;
7780 
7781   // Only handle AND for now.
7782   if (Op.getOpcode() != ISD::AND)
7783     return false;
7784 
7785   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
7786   if (!C)
7787     return false;
7788 
7789   const APInt &Mask = C->getAPIntValue();
7790 
7791   // Clear all non-demanded bits initially.
7792   APInt ShrunkMask = Mask & DemandedBits;
7793 
7794   // Try to make a smaller immediate by setting undemanded bits.
7795 
7796   APInt ExpandedMask = Mask | ~DemandedBits;
7797 
7798   auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
7799     return ShrunkMask.isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
7800   };
7801   auto UseMask = [Mask, Op, VT, &TLO](const APInt &NewMask) -> bool {
7802     if (NewMask == Mask)
7803       return true;
7804     SDLoc DL(Op);
7805     SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
7806     SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
7807     return TLO.CombineTo(Op, NewOp);
7808   };
7809 
7810   // If the shrunk mask fits in sign extended 12 bits, let the target
7811   // independent code apply it.
7812   if (ShrunkMask.isSignedIntN(12))
7813     return false;
7814 
7815   // Preserve (and X, 0xffff) when zext.h is supported.
7816   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
7817     APInt NewMask = APInt(Mask.getBitWidth(), 0xffff);
7818     if (IsLegalMask(NewMask))
7819       return UseMask(NewMask);
7820   }
7821 
7822   // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
7823   if (VT == MVT::i64) {
7824     APInt NewMask = APInt(64, 0xffffffff);
7825     if (IsLegalMask(NewMask))
7826       return UseMask(NewMask);
7827   }
7828 
7829   // For the remaining optimizations, we need to be able to make a negative
7830   // number through a combination of mask and undemanded bits.
7831   if (!ExpandedMask.isNegative())
7832     return false;
7833 
7834   // What is the fewest number of bits we need to represent the negative number.
7835   unsigned MinSignedBits = ExpandedMask.getMinSignedBits();
7836 
7837   // Try to make a 12 bit negative immediate. If that fails try to make a 32
7838   // bit negative immediate unless the shrunk immediate already fits in 32 bits.
7839   APInt NewMask = ShrunkMask;
7840   if (MinSignedBits <= 12)
7841     NewMask.setBitsFrom(11);
7842   else if (MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(32))
7843     NewMask.setBitsFrom(31);
7844   else
7845     return false;
7846 
7847   // Check that our new mask is a subset of the demanded mask.
7848   assert(IsLegalMask(NewMask));
7849   return UseMask(NewMask);
7850 }
7851 
7852 static void computeGREV(APInt &Src, unsigned ShAmt) {
7853   ShAmt &= Src.getBitWidth() - 1;
7854   uint64_t x = Src.getZExtValue();
7855   if (ShAmt & 1)
7856     x = ((x & 0x5555555555555555LL) << 1) | ((x & 0xAAAAAAAAAAAAAAAALL) >> 1);
7857   if (ShAmt & 2)
7858     x = ((x & 0x3333333333333333LL) << 2) | ((x & 0xCCCCCCCCCCCCCCCCLL) >> 2);
7859   if (ShAmt & 4)
7860     x = ((x & 0x0F0F0F0F0F0F0F0FLL) << 4) | ((x & 0xF0F0F0F0F0F0F0F0LL) >> 4);
7861   if (ShAmt & 8)
7862     x = ((x & 0x00FF00FF00FF00FFLL) << 8) | ((x & 0xFF00FF00FF00FF00LL) >> 8);
7863   if (ShAmt & 16)
7864     x = ((x & 0x0000FFFF0000FFFFLL) << 16) | ((x & 0xFFFF0000FFFF0000LL) >> 16);
7865   if (ShAmt & 32)
7866     x = ((x & 0x00000000FFFFFFFFLL) << 32) | ((x & 0xFFFFFFFF00000000LL) >> 32);
7867   Src = x;
7868 }
7869 
7870 void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
7871                                                         KnownBits &Known,
7872                                                         const APInt &DemandedElts,
7873                                                         const SelectionDAG &DAG,
7874                                                         unsigned Depth) const {
7875   unsigned BitWidth = Known.getBitWidth();
7876   unsigned Opc = Op.getOpcode();
7877   assert((Opc >= ISD::BUILTIN_OP_END ||
7878           Opc == ISD::INTRINSIC_WO_CHAIN ||
7879           Opc == ISD::INTRINSIC_W_CHAIN ||
7880           Opc == ISD::INTRINSIC_VOID) &&
7881          "Should use MaskedValueIsZero if you don't know whether Op"
7882          " is a target node!");
7883 
7884   Known.resetAll();
7885   switch (Opc) {
7886   default: break;
7887   case RISCVISD::SELECT_CC: {
7888     Known = DAG.computeKnownBits(Op.getOperand(4), Depth + 1);
7889     // If we don't know any bits, early out.
7890     if (Known.isUnknown())
7891       break;
7892     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(3), Depth + 1);
7893 
7894     // Only known if known in both the LHS and RHS.
7895     Known = KnownBits::commonBits(Known, Known2);
7896     break;
7897   }
7898   case RISCVISD::REMUW: {
7899     KnownBits Known2;
7900     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
7901     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
7902     // We only care about the lower 32 bits.
7903     Known = KnownBits::urem(Known.trunc(32), Known2.trunc(32));
7904     // Restore the original width by sign extending.
7905     Known = Known.sext(BitWidth);
7906     break;
7907   }
7908   case RISCVISD::DIVUW: {
7909     KnownBits Known2;
7910     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
7911     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
7912     // We only care about the lower 32 bits.
7913     Known = KnownBits::udiv(Known.trunc(32), Known2.trunc(32));
7914     // Restore the original width by sign extending.
7915     Known = Known.sext(BitWidth);
7916     break;
7917   }
7918   case RISCVISD::CTZW: {
7919     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
7920     unsigned PossibleTZ = Known2.trunc(32).countMaxTrailingZeros();
7921     unsigned LowBits = Log2_32(PossibleTZ) + 1;
7922     Known.Zero.setBitsFrom(LowBits);
7923     break;
7924   }
7925   case RISCVISD::CLZW: {
7926     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
7927     unsigned PossibleLZ = Known2.trunc(32).countMaxLeadingZeros();
7928     unsigned LowBits = Log2_32(PossibleLZ) + 1;
7929     Known.Zero.setBitsFrom(LowBits);
7930     break;
7931   }
7932   case RISCVISD::GREV:
7933   case RISCVISD::GREVW: {
7934     if (auto *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
7935       Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
7936       if (Opc == RISCVISD::GREVW)
7937         Known = Known.trunc(32);
7938       unsigned ShAmt = C->getZExtValue();
7939       computeGREV(Known.Zero, ShAmt);
7940       computeGREV(Known.One, ShAmt);
7941       if (Opc == RISCVISD::GREVW)
7942         Known = Known.sext(BitWidth);
7943     }
7944     break;
7945   }
7946   case RISCVISD::READ_VLENB:
7947     // We assume VLENB is at least 16 bytes.
7948     Known.Zero.setLowBits(4);
7949     // We assume VLENB is no more than 65536 / 8 bytes.
7950     Known.Zero.setBitsFrom(14);
7951     break;
7952   case ISD::INTRINSIC_W_CHAIN: {
7953     unsigned IntNo = Op.getConstantOperandVal(1);
7954     switch (IntNo) {
7955     default:
7956       // We can't do anything for most intrinsics.
7957       break;
7958     case Intrinsic::riscv_vsetvli:
7959     case Intrinsic::riscv_vsetvlimax:
7960       // Assume that VL output is positive and would fit in an int32_t.
7961       // TODO: VLEN might be capped at 16 bits in a future V spec update.
7962       if (BitWidth >= 32)
7963         Known.Zero.setBitsFrom(31);
7964       break;
7965     }
7966     break;
7967   }
7968   }
7969 }
7970 
7971 unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
7972     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
7973     unsigned Depth) const {
7974   switch (Op.getOpcode()) {
7975   default:
7976     break;
7977   case RISCVISD::SELECT_CC: {
7978     unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth + 1);
7979     if (Tmp == 1) return 1;  // Early out.
7980     unsigned Tmp2 = DAG.ComputeNumSignBits(Op.getOperand(4), DemandedElts, Depth + 1);
7981     return std::min(Tmp, Tmp2);
7982   }
7983   case RISCVISD::SLLW:
7984   case RISCVISD::SRAW:
7985   case RISCVISD::SRLW:
7986   case RISCVISD::DIVW:
7987   case RISCVISD::DIVUW:
7988   case RISCVISD::REMUW:
7989   case RISCVISD::ROLW:
7990   case RISCVISD::RORW:
7991   case RISCVISD::GREVW:
7992   case RISCVISD::GORCW:
7993   case RISCVISD::FSLW:
7994   case RISCVISD::FSRW:
7995   case RISCVISD::SHFLW:
7996   case RISCVISD::UNSHFLW:
7997   case RISCVISD::BCOMPRESSW:
7998   case RISCVISD::BDECOMPRESSW:
7999   case RISCVISD::BFPW:
8000   case RISCVISD::FCVT_W_RV64:
8001   case RISCVISD::FCVT_WU_RV64:
8002   case RISCVISD::STRICT_FCVT_W_RV64:
8003   case RISCVISD::STRICT_FCVT_WU_RV64:
8004     // TODO: As the result is sign-extended, this is conservatively correct. A
8005     // more precise answer could be calculated for SRAW depending on known
8006     // bits in the shift amount.
8007     return 33;
8008   case RISCVISD::SHFL:
8009   case RISCVISD::UNSHFL: {
8010     // There is no SHFLIW, but a i64 SHFLI with bit 4 of the control word
8011     // cleared doesn't affect bit 31. The upper 32 bits will be shuffled, but
8012     // will stay within the upper 32 bits. If there were more than 32 sign bits
8013     // before there will be at least 33 sign bits after.
8014     if (Op.getValueType() == MVT::i64 &&
8015         isa<ConstantSDNode>(Op.getOperand(1)) &&
8016         (Op.getConstantOperandVal(1) & 0x10) == 0) {
8017       unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
8018       if (Tmp > 32)
8019         return 33;
8020     }
8021     break;
8022   }
8023   case RISCVISD::VMV_X_S:
8024     // The number of sign bits of the scalar result is computed by obtaining the
8025     // element type of the input vector operand, subtracting its width from the
8026     // XLEN, and then adding one (sign bit within the element type). If the
8027     // element type is wider than XLen, the least-significant XLEN bits are
8028     // taken.
8029     if (Op.getOperand(0).getScalarValueSizeInBits() > Subtarget.getXLen())
8030       return 1;
8031     return Subtarget.getXLen() - Op.getOperand(0).getScalarValueSizeInBits() + 1;
8032   }
8033 
8034   return 1;
8035 }
8036 
8037 static MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
8038                                                   MachineBasicBlock *BB) {
8039   assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction");
8040 
8041   // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
8042   // Should the count have wrapped while it was being read, we need to try
8043   // again.
8044   // ...
8045   // read:
8046   // rdcycleh x3 # load high word of cycle
8047   // rdcycle  x2 # load low word of cycle
8048   // rdcycleh x4 # load high word of cycle
8049   // bne x3, x4, read # check if high word reads match, otherwise try again
8050   // ...
8051 
8052   MachineFunction &MF = *BB->getParent();
8053   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8054   MachineFunction::iterator It = ++BB->getIterator();
8055 
8056   MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8057   MF.insert(It, LoopMBB);
8058 
8059   MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8060   MF.insert(It, DoneMBB);
8061 
8062   // Transfer the remainder of BB and its successor edges to DoneMBB.
8063   DoneMBB->splice(DoneMBB->begin(), BB,
8064                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
8065   DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
8066 
8067   BB->addSuccessor(LoopMBB);
8068 
8069   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8070   Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
8071   Register LoReg = MI.getOperand(0).getReg();
8072   Register HiReg = MI.getOperand(1).getReg();
8073   DebugLoc DL = MI.getDebugLoc();
8074 
8075   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8076   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
8077       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8078       .addReg(RISCV::X0);
8079   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
8080       .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
8081       .addReg(RISCV::X0);
8082   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
8083       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8084       .addReg(RISCV::X0);
8085 
8086   BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
8087       .addReg(HiReg)
8088       .addReg(ReadAgainReg)
8089       .addMBB(LoopMBB);
8090 
8091   LoopMBB->addSuccessor(LoopMBB);
8092   LoopMBB->addSuccessor(DoneMBB);
8093 
8094   MI.eraseFromParent();
8095 
8096   return DoneMBB;
8097 }
8098 
8099 static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
8100                                              MachineBasicBlock *BB) {
8101   assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
8102 
8103   MachineFunction &MF = *BB->getParent();
8104   DebugLoc DL = MI.getDebugLoc();
8105   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8106   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8107   Register LoReg = MI.getOperand(0).getReg();
8108   Register HiReg = MI.getOperand(1).getReg();
8109   Register SrcReg = MI.getOperand(2).getReg();
8110   const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
8111   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8112 
8113   TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
8114                           RI);
8115   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8116   MachineMemOperand *MMOLo =
8117       MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 4, Align(8));
8118   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8119       MPI.getWithOffset(4), MachineMemOperand::MOLoad, 4, Align(8));
8120   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
8121       .addFrameIndex(FI)
8122       .addImm(0)
8123       .addMemOperand(MMOLo);
8124   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
8125       .addFrameIndex(FI)
8126       .addImm(4)
8127       .addMemOperand(MMOHi);
8128   MI.eraseFromParent(); // The pseudo instruction is gone now.
8129   return BB;
8130 }
8131 
8132 static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
8133                                                  MachineBasicBlock *BB) {
8134   assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
8135          "Unexpected instruction");
8136 
8137   MachineFunction &MF = *BB->getParent();
8138   DebugLoc DL = MI.getDebugLoc();
8139   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8140   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8141   Register DstReg = MI.getOperand(0).getReg();
8142   Register LoReg = MI.getOperand(1).getReg();
8143   Register HiReg = MI.getOperand(2).getReg();
8144   const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
8145   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8146 
8147   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8148   MachineMemOperand *MMOLo =
8149       MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Align(8));
8150   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8151       MPI.getWithOffset(4), MachineMemOperand::MOStore, 4, Align(8));
8152   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8153       .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
8154       .addFrameIndex(FI)
8155       .addImm(0)
8156       .addMemOperand(MMOLo);
8157   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8158       .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
8159       .addFrameIndex(FI)
8160       .addImm(4)
8161       .addMemOperand(MMOHi);
8162   TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
8163   MI.eraseFromParent(); // The pseudo instruction is gone now.
8164   return BB;
8165 }
8166 
8167 static bool isSelectPseudo(MachineInstr &MI) {
8168   switch (MI.getOpcode()) {
8169   default:
8170     return false;
8171   case RISCV::Select_GPR_Using_CC_GPR:
8172   case RISCV::Select_FPR16_Using_CC_GPR:
8173   case RISCV::Select_FPR32_Using_CC_GPR:
8174   case RISCV::Select_FPR64_Using_CC_GPR:
8175     return true;
8176   }
8177 }
8178 
8179 static MachineBasicBlock *emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB,
8180                                         unsigned RelOpcode, unsigned EqOpcode,
8181                                         const RISCVSubtarget &Subtarget) {
8182   DebugLoc DL = MI.getDebugLoc();
8183   Register DstReg = MI.getOperand(0).getReg();
8184   Register Src1Reg = MI.getOperand(1).getReg();
8185   Register Src2Reg = MI.getOperand(2).getReg();
8186   MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
8187   Register SavedFFlags = MRI.createVirtualRegister(&RISCV::GPRRegClass);
8188   const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
8189 
8190   // Save the current FFLAGS.
8191   BuildMI(*BB, MI, DL, TII.get(RISCV::ReadFFLAGS), SavedFFlags);
8192 
8193   auto MIB = BuildMI(*BB, MI, DL, TII.get(RelOpcode), DstReg)
8194                  .addReg(Src1Reg)
8195                  .addReg(Src2Reg);
8196   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8197     MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
8198 
8199   // Restore the FFLAGS.
8200   BuildMI(*BB, MI, DL, TII.get(RISCV::WriteFFLAGS))
8201       .addReg(SavedFFlags, RegState::Kill);
8202 
8203   // Issue a dummy FEQ opcode to raise exception for signaling NaNs.
8204   auto MIB2 = BuildMI(*BB, MI, DL, TII.get(EqOpcode), RISCV::X0)
8205                   .addReg(Src1Reg, getKillRegState(MI.getOperand(1).isKill()))
8206                   .addReg(Src2Reg, getKillRegState(MI.getOperand(2).isKill()));
8207   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8208     MIB2->setFlag(MachineInstr::MIFlag::NoFPExcept);
8209 
8210   // Erase the pseudoinstruction.
8211   MI.eraseFromParent();
8212   return BB;
8213 }
8214 
8215 static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
8216                                            MachineBasicBlock *BB,
8217                                            const RISCVSubtarget &Subtarget) {
8218   // To "insert" Select_* instructions, we actually have to insert the triangle
8219   // control-flow pattern.  The incoming instructions know the destination vreg
8220   // to set, the condition code register to branch on, the true/false values to
8221   // select between, and the condcode to use to select the appropriate branch.
8222   //
8223   // We produce the following control flow:
8224   //     HeadMBB
8225   //     |  \
8226   //     |  IfFalseMBB
8227   //     | /
8228   //    TailMBB
8229   //
8230   // When we find a sequence of selects we attempt to optimize their emission
8231   // by sharing the control flow. Currently we only handle cases where we have
8232   // multiple selects with the exact same condition (same LHS, RHS and CC).
8233   // The selects may be interleaved with other instructions if the other
8234   // instructions meet some requirements we deem safe:
8235   // - They are debug instructions. Otherwise,
8236   // - They do not have side-effects, do not access memory and their inputs do
8237   //   not depend on the results of the select pseudo-instructions.
8238   // The TrueV/FalseV operands of the selects cannot depend on the result of
8239   // previous selects in the sequence.
8240   // These conditions could be further relaxed. See the X86 target for a
8241   // related approach and more information.
8242   Register LHS = MI.getOperand(1).getReg();
8243   Register RHS = MI.getOperand(2).getReg();
8244   auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
8245 
8246   SmallVector<MachineInstr *, 4> SelectDebugValues;
8247   SmallSet<Register, 4> SelectDests;
8248   SelectDests.insert(MI.getOperand(0).getReg());
8249 
8250   MachineInstr *LastSelectPseudo = &MI;
8251 
8252   for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
8253        SequenceMBBI != E; ++SequenceMBBI) {
8254     if (SequenceMBBI->isDebugInstr())
8255       continue;
8256     else if (isSelectPseudo(*SequenceMBBI)) {
8257       if (SequenceMBBI->getOperand(1).getReg() != LHS ||
8258           SequenceMBBI->getOperand(2).getReg() != RHS ||
8259           SequenceMBBI->getOperand(3).getImm() != CC ||
8260           SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
8261           SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
8262         break;
8263       LastSelectPseudo = &*SequenceMBBI;
8264       SequenceMBBI->collectDebugValues(SelectDebugValues);
8265       SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
8266     } else {
8267       if (SequenceMBBI->hasUnmodeledSideEffects() ||
8268           SequenceMBBI->mayLoadOrStore())
8269         break;
8270       if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
8271             return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
8272           }))
8273         break;
8274     }
8275   }
8276 
8277   const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
8278   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8279   DebugLoc DL = MI.getDebugLoc();
8280   MachineFunction::iterator I = ++BB->getIterator();
8281 
8282   MachineBasicBlock *HeadMBB = BB;
8283   MachineFunction *F = BB->getParent();
8284   MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
8285   MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
8286 
8287   F->insert(I, IfFalseMBB);
8288   F->insert(I, TailMBB);
8289 
8290   // Transfer debug instructions associated with the selects to TailMBB.
8291   for (MachineInstr *DebugInstr : SelectDebugValues) {
8292     TailMBB->push_back(DebugInstr->removeFromParent());
8293   }
8294 
8295   // Move all instructions after the sequence to TailMBB.
8296   TailMBB->splice(TailMBB->end(), HeadMBB,
8297                   std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
8298   // Update machine-CFG edges by transferring all successors of the current
8299   // block to the new block which will contain the Phi nodes for the selects.
8300   TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
8301   // Set the successors for HeadMBB.
8302   HeadMBB->addSuccessor(IfFalseMBB);
8303   HeadMBB->addSuccessor(TailMBB);
8304 
8305   // Insert appropriate branch.
8306   BuildMI(HeadMBB, DL, TII.getBrCond(CC))
8307     .addReg(LHS)
8308     .addReg(RHS)
8309     .addMBB(TailMBB);
8310 
8311   // IfFalseMBB just falls through to TailMBB.
8312   IfFalseMBB->addSuccessor(TailMBB);
8313 
8314   // Create PHIs for all of the select pseudo-instructions.
8315   auto SelectMBBI = MI.getIterator();
8316   auto SelectEnd = std::next(LastSelectPseudo->getIterator());
8317   auto InsertionPoint = TailMBB->begin();
8318   while (SelectMBBI != SelectEnd) {
8319     auto Next = std::next(SelectMBBI);
8320     if (isSelectPseudo(*SelectMBBI)) {
8321       // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
8322       BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
8323               TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
8324           .addReg(SelectMBBI->getOperand(4).getReg())
8325           .addMBB(HeadMBB)
8326           .addReg(SelectMBBI->getOperand(5).getReg())
8327           .addMBB(IfFalseMBB);
8328       SelectMBBI->eraseFromParent();
8329     }
8330     SelectMBBI = Next;
8331   }
8332 
8333   F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
8334   return TailMBB;
8335 }
8336 
8337 MachineBasicBlock *
8338 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
8339                                                  MachineBasicBlock *BB) const {
8340   switch (MI.getOpcode()) {
8341   default:
8342     llvm_unreachable("Unexpected instr type to insert");
8343   case RISCV::ReadCycleWide:
8344     assert(!Subtarget.is64Bit() &&
8345            "ReadCycleWrite is only to be used on riscv32");
8346     return emitReadCycleWidePseudo(MI, BB);
8347   case RISCV::Select_GPR_Using_CC_GPR:
8348   case RISCV::Select_FPR16_Using_CC_GPR:
8349   case RISCV::Select_FPR32_Using_CC_GPR:
8350   case RISCV::Select_FPR64_Using_CC_GPR:
8351     return emitSelectPseudo(MI, BB, Subtarget);
8352   case RISCV::BuildPairF64Pseudo:
8353     return emitBuildPairF64Pseudo(MI, BB);
8354   case RISCV::SplitF64Pseudo:
8355     return emitSplitF64Pseudo(MI, BB);
8356   case RISCV::PseudoQuietFLE_H:
8357     return emitQuietFCMP(MI, BB, RISCV::FLE_H, RISCV::FEQ_H, Subtarget);
8358   case RISCV::PseudoQuietFLT_H:
8359     return emitQuietFCMP(MI, BB, RISCV::FLT_H, RISCV::FEQ_H, Subtarget);
8360   case RISCV::PseudoQuietFLE_S:
8361     return emitQuietFCMP(MI, BB, RISCV::FLE_S, RISCV::FEQ_S, Subtarget);
8362   case RISCV::PseudoQuietFLT_S:
8363     return emitQuietFCMP(MI, BB, RISCV::FLT_S, RISCV::FEQ_S, Subtarget);
8364   case RISCV::PseudoQuietFLE_D:
8365     return emitQuietFCMP(MI, BB, RISCV::FLE_D, RISCV::FEQ_D, Subtarget);
8366   case RISCV::PseudoQuietFLT_D:
8367     return emitQuietFCMP(MI, BB, RISCV::FLT_D, RISCV::FEQ_D, Subtarget);
8368   }
8369 }
8370 
8371 void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
8372                                                         SDNode *Node) const {
8373   // Add FRM dependency to any instructions with dynamic rounding mode.
8374   unsigned Opc = MI.getOpcode();
8375   auto Idx = RISCV::getNamedOperandIdx(Opc, RISCV::OpName::frm);
8376   if (Idx < 0)
8377     return;
8378   if (MI.getOperand(Idx).getImm() != RISCVFPRndMode::DYN)
8379     return;
8380   // If the instruction already reads FRM, don't add another read.
8381   if (MI.readsRegister(RISCV::FRM))
8382     return;
8383   MI.addOperand(
8384       MachineOperand::CreateReg(RISCV::FRM, /*isDef*/ false, /*isImp*/ true));
8385 }
8386 
8387 // Calling Convention Implementation.
8388 // The expectations for frontend ABI lowering vary from target to target.
8389 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
8390 // details, but this is a longer term goal. For now, we simply try to keep the
8391 // role of the frontend as simple and well-defined as possible. The rules can
8392 // be summarised as:
8393 // * Never split up large scalar arguments. We handle them here.
8394 // * If a hardfloat calling convention is being used, and the struct may be
8395 // passed in a pair of registers (fp+fp, int+fp), and both registers are
8396 // available, then pass as two separate arguments. If either the GPRs or FPRs
8397 // are exhausted, then pass according to the rule below.
8398 // * If a struct could never be passed in registers or directly in a stack
8399 // slot (as it is larger than 2*XLEN and the floating point rules don't
8400 // apply), then pass it using a pointer with the byval attribute.
8401 // * If a struct is less than 2*XLEN, then coerce to either a two-element
8402 // word-sized array or a 2*XLEN scalar (depending on alignment).
8403 // * The frontend can determine whether a struct is returned by reference or
8404 // not based on its size and fields. If it will be returned by reference, the
8405 // frontend must modify the prototype so a pointer with the sret annotation is
8406 // passed as the first argument. This is not necessary for large scalar
8407 // returns.
8408 // * Struct return values and varargs should be coerced to structs containing
8409 // register-size fields in the same situations they would be for fixed
8410 // arguments.
8411 
8412 static const MCPhysReg ArgGPRs[] = {
8413   RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
8414   RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
8415 };
8416 static const MCPhysReg ArgFPR16s[] = {
8417   RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
8418   RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
8419 };
8420 static const MCPhysReg ArgFPR32s[] = {
8421   RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
8422   RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
8423 };
8424 static const MCPhysReg ArgFPR64s[] = {
8425   RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
8426   RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
8427 };
8428 // This is an interim calling convention and it may be changed in the future.
8429 static const MCPhysReg ArgVRs[] = {
8430     RISCV::V8,  RISCV::V9,  RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
8431     RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
8432     RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
8433 static const MCPhysReg ArgVRM2s[] = {RISCV::V8M2,  RISCV::V10M2, RISCV::V12M2,
8434                                      RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
8435                                      RISCV::V20M2, RISCV::V22M2};
8436 static const MCPhysReg ArgVRM4s[] = {RISCV::V8M4, RISCV::V12M4, RISCV::V16M4,
8437                                      RISCV::V20M4};
8438 static const MCPhysReg ArgVRM8s[] = {RISCV::V8M8, RISCV::V16M8};
8439 
8440 // Pass a 2*XLEN argument that has been split into two XLEN values through
8441 // registers or the stack as necessary.
8442 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
8443                                 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
8444                                 MVT ValVT2, MVT LocVT2,
8445                                 ISD::ArgFlagsTy ArgFlags2) {
8446   unsigned XLenInBytes = XLen / 8;
8447   if (Register Reg = State.AllocateReg(ArgGPRs)) {
8448     // At least one half can be passed via register.
8449     State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
8450                                      VA1.getLocVT(), CCValAssign::Full));
8451   } else {
8452     // Both halves must be passed on the stack, with proper alignment.
8453     Align StackAlign =
8454         std::max(Align(XLenInBytes), ArgFlags1.getNonZeroOrigAlign());
8455     State.addLoc(
8456         CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
8457                             State.AllocateStack(XLenInBytes, StackAlign),
8458                             VA1.getLocVT(), CCValAssign::Full));
8459     State.addLoc(CCValAssign::getMem(
8460         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
8461         LocVT2, CCValAssign::Full));
8462     return false;
8463   }
8464 
8465   if (Register Reg = State.AllocateReg(ArgGPRs)) {
8466     // The second half can also be passed via register.
8467     State.addLoc(
8468         CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
8469   } else {
8470     // The second half is passed via the stack, without additional alignment.
8471     State.addLoc(CCValAssign::getMem(
8472         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
8473         LocVT2, CCValAssign::Full));
8474   }
8475 
8476   return false;
8477 }
8478 
8479 static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo,
8480                                Optional<unsigned> FirstMaskArgument,
8481                                CCState &State, const RISCVTargetLowering &TLI) {
8482   const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT);
8483   if (RC == &RISCV::VRRegClass) {
8484     // Assign the first mask argument to V0.
8485     // This is an interim calling convention and it may be changed in the
8486     // future.
8487     if (FirstMaskArgument.hasValue() && ValNo == FirstMaskArgument.getValue())
8488       return State.AllocateReg(RISCV::V0);
8489     return State.AllocateReg(ArgVRs);
8490   }
8491   if (RC == &RISCV::VRM2RegClass)
8492     return State.AllocateReg(ArgVRM2s);
8493   if (RC == &RISCV::VRM4RegClass)
8494     return State.AllocateReg(ArgVRM4s);
8495   if (RC == &RISCV::VRM8RegClass)
8496     return State.AllocateReg(ArgVRM8s);
8497   llvm_unreachable("Unhandled register class for ValueType");
8498 }
8499 
8500 // Implements the RISC-V calling convention. Returns true upon failure.
8501 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
8502                      MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
8503                      ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
8504                      bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
8505                      Optional<unsigned> FirstMaskArgument) {
8506   unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
8507   assert(XLen == 32 || XLen == 64);
8508   MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
8509 
8510   // Any return value split in to more than two values can't be returned
8511   // directly. Vectors are returned via the available vector registers.
8512   if (!LocVT.isVector() && IsRet && ValNo > 1)
8513     return true;
8514 
8515   // UseGPRForF16_F32 if targeting one of the soft-float ABIs, if passing a
8516   // variadic argument, or if no F16/F32 argument registers are available.
8517   bool UseGPRForF16_F32 = true;
8518   // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
8519   // variadic argument, or if no F64 argument registers are available.
8520   bool UseGPRForF64 = true;
8521 
8522   switch (ABI) {
8523   default:
8524     llvm_unreachable("Unexpected ABI");
8525   case RISCVABI::ABI_ILP32:
8526   case RISCVABI::ABI_LP64:
8527     break;
8528   case RISCVABI::ABI_ILP32F:
8529   case RISCVABI::ABI_LP64F:
8530     UseGPRForF16_F32 = !IsFixed;
8531     break;
8532   case RISCVABI::ABI_ILP32D:
8533   case RISCVABI::ABI_LP64D:
8534     UseGPRForF16_F32 = !IsFixed;
8535     UseGPRForF64 = !IsFixed;
8536     break;
8537   }
8538 
8539   // FPR16, FPR32, and FPR64 alias each other.
8540   if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s)) {
8541     UseGPRForF16_F32 = true;
8542     UseGPRForF64 = true;
8543   }
8544 
8545   // From this point on, rely on UseGPRForF16_F32, UseGPRForF64 and
8546   // similar local variables rather than directly checking against the target
8547   // ABI.
8548 
8549   if (UseGPRForF16_F32 && (ValVT == MVT::f16 || ValVT == MVT::f32)) {
8550     LocVT = XLenVT;
8551     LocInfo = CCValAssign::BCvt;
8552   } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
8553     LocVT = MVT::i64;
8554     LocInfo = CCValAssign::BCvt;
8555   }
8556 
8557   // If this is a variadic argument, the RISC-V calling convention requires
8558   // that it is assigned an 'even' or 'aligned' register if it has 8-byte
8559   // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
8560   // be used regardless of whether the original argument was split during
8561   // legalisation or not. The argument will not be passed by registers if the
8562   // original type is larger than 2*XLEN, so the register alignment rule does
8563   // not apply.
8564   unsigned TwoXLenInBytes = (2 * XLen) / 8;
8565   if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
8566       DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
8567     unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
8568     // Skip 'odd' register if necessary.
8569     if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
8570       State.AllocateReg(ArgGPRs);
8571   }
8572 
8573   SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
8574   SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
8575       State.getPendingArgFlags();
8576 
8577   assert(PendingLocs.size() == PendingArgFlags.size() &&
8578          "PendingLocs and PendingArgFlags out of sync");
8579 
8580   // Handle passing f64 on RV32D with a soft float ABI or when floating point
8581   // registers are exhausted.
8582   if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
8583     assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
8584            "Can't lower f64 if it is split");
8585     // Depending on available argument GPRS, f64 may be passed in a pair of
8586     // GPRs, split between a GPR and the stack, or passed completely on the
8587     // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
8588     // cases.
8589     Register Reg = State.AllocateReg(ArgGPRs);
8590     LocVT = MVT::i32;
8591     if (!Reg) {
8592       unsigned StackOffset = State.AllocateStack(8, Align(8));
8593       State.addLoc(
8594           CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
8595       return false;
8596     }
8597     if (!State.AllocateReg(ArgGPRs))
8598       State.AllocateStack(4, Align(4));
8599     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8600     return false;
8601   }
8602 
8603   // Fixed-length vectors are located in the corresponding scalable-vector
8604   // container types.
8605   if (ValVT.isFixedLengthVector())
8606     LocVT = TLI.getContainerForFixedLengthVector(LocVT);
8607 
8608   // Split arguments might be passed indirectly, so keep track of the pending
8609   // values. Split vectors are passed via a mix of registers and indirectly, so
8610   // treat them as we would any other argument.
8611   if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
8612     LocVT = XLenVT;
8613     LocInfo = CCValAssign::Indirect;
8614     PendingLocs.push_back(
8615         CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
8616     PendingArgFlags.push_back(ArgFlags);
8617     if (!ArgFlags.isSplitEnd()) {
8618       return false;
8619     }
8620   }
8621 
8622   // If the split argument only had two elements, it should be passed directly
8623   // in registers or on the stack.
8624   if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
8625       PendingLocs.size() <= 2) {
8626     assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
8627     // Apply the normal calling convention rules to the first half of the
8628     // split argument.
8629     CCValAssign VA = PendingLocs[0];
8630     ISD::ArgFlagsTy AF = PendingArgFlags[0];
8631     PendingLocs.clear();
8632     PendingArgFlags.clear();
8633     return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
8634                                ArgFlags);
8635   }
8636 
8637   // Allocate to a register if possible, or else a stack slot.
8638   Register Reg;
8639   unsigned StoreSizeBytes = XLen / 8;
8640   Align StackAlign = Align(XLen / 8);
8641 
8642   if (ValVT == MVT::f16 && !UseGPRForF16_F32)
8643     Reg = State.AllocateReg(ArgFPR16s);
8644   else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
8645     Reg = State.AllocateReg(ArgFPR32s);
8646   else if (ValVT == MVT::f64 && !UseGPRForF64)
8647     Reg = State.AllocateReg(ArgFPR64s);
8648   else if (ValVT.isVector()) {
8649     Reg = allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI);
8650     if (!Reg) {
8651       // For return values, the vector must be passed fully via registers or
8652       // via the stack.
8653       // FIXME: The proposed vector ABI only mandates v8-v15 for return values,
8654       // but we're using all of them.
8655       if (IsRet)
8656         return true;
8657       // Try using a GPR to pass the address
8658       if ((Reg = State.AllocateReg(ArgGPRs))) {
8659         LocVT = XLenVT;
8660         LocInfo = CCValAssign::Indirect;
8661       } else if (ValVT.isScalableVector()) {
8662         LocVT = XLenVT;
8663         LocInfo = CCValAssign::Indirect;
8664       } else {
8665         // Pass fixed-length vectors on the stack.
8666         LocVT = ValVT;
8667         StoreSizeBytes = ValVT.getStoreSize();
8668         // Align vectors to their element sizes, being careful for vXi1
8669         // vectors.
8670         StackAlign = MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
8671       }
8672     }
8673   } else {
8674     Reg = State.AllocateReg(ArgGPRs);
8675   }
8676 
8677   unsigned StackOffset =
8678       Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
8679 
8680   // If we reach this point and PendingLocs is non-empty, we must be at the
8681   // end of a split argument that must be passed indirectly.
8682   if (!PendingLocs.empty()) {
8683     assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
8684     assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
8685 
8686     for (auto &It : PendingLocs) {
8687       if (Reg)
8688         It.convertToReg(Reg);
8689       else
8690         It.convertToMem(StackOffset);
8691       State.addLoc(It);
8692     }
8693     PendingLocs.clear();
8694     PendingArgFlags.clear();
8695     return false;
8696   }
8697 
8698   assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
8699           (TLI.getSubtarget().hasVInstructions() && ValVT.isVector())) &&
8700          "Expected an XLenVT or vector types at this stage");
8701 
8702   if (Reg) {
8703     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8704     return false;
8705   }
8706 
8707   // When a floating-point value is passed on the stack, no bit-conversion is
8708   // needed.
8709   if (ValVT.isFloatingPoint()) {
8710     LocVT = ValVT;
8711     LocInfo = CCValAssign::Full;
8712   }
8713   State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
8714   return false;
8715 }
8716 
8717 template <typename ArgTy>
8718 static Optional<unsigned> preAssignMask(const ArgTy &Args) {
8719   for (const auto &ArgIdx : enumerate(Args)) {
8720     MVT ArgVT = ArgIdx.value().VT;
8721     if (ArgVT.isVector() && ArgVT.getVectorElementType() == MVT::i1)
8722       return ArgIdx.index();
8723   }
8724   return None;
8725 }
8726 
8727 void RISCVTargetLowering::analyzeInputArgs(
8728     MachineFunction &MF, CCState &CCInfo,
8729     const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
8730     RISCVCCAssignFn Fn) const {
8731   unsigned NumArgs = Ins.size();
8732   FunctionType *FType = MF.getFunction().getFunctionType();
8733 
8734   Optional<unsigned> FirstMaskArgument;
8735   if (Subtarget.hasVInstructions())
8736     FirstMaskArgument = preAssignMask(Ins);
8737 
8738   for (unsigned i = 0; i != NumArgs; ++i) {
8739     MVT ArgVT = Ins[i].VT;
8740     ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
8741 
8742     Type *ArgTy = nullptr;
8743     if (IsRet)
8744       ArgTy = FType->getReturnType();
8745     else if (Ins[i].isOrigArg())
8746       ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
8747 
8748     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
8749     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
8750            ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this,
8751            FirstMaskArgument)) {
8752       LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
8753                         << EVT(ArgVT).getEVTString() << '\n');
8754       llvm_unreachable(nullptr);
8755     }
8756   }
8757 }
8758 
8759 void RISCVTargetLowering::analyzeOutputArgs(
8760     MachineFunction &MF, CCState &CCInfo,
8761     const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
8762     CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
8763   unsigned NumArgs = Outs.size();
8764 
8765   Optional<unsigned> FirstMaskArgument;
8766   if (Subtarget.hasVInstructions())
8767     FirstMaskArgument = preAssignMask(Outs);
8768 
8769   for (unsigned i = 0; i != NumArgs; i++) {
8770     MVT ArgVT = Outs[i].VT;
8771     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
8772     Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
8773 
8774     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
8775     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
8776            ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this,
8777            FirstMaskArgument)) {
8778       LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
8779                         << EVT(ArgVT).getEVTString() << "\n");
8780       llvm_unreachable(nullptr);
8781     }
8782   }
8783 }
8784 
8785 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
8786 // values.
8787 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
8788                                    const CCValAssign &VA, const SDLoc &DL,
8789                                    const RISCVSubtarget &Subtarget) {
8790   switch (VA.getLocInfo()) {
8791   default:
8792     llvm_unreachable("Unexpected CCValAssign::LocInfo");
8793   case CCValAssign::Full:
8794     if (VA.getValVT().isFixedLengthVector() && VA.getLocVT().isScalableVector())
8795       Val = convertFromScalableVector(VA.getValVT(), Val, DAG, Subtarget);
8796     break;
8797   case CCValAssign::BCvt:
8798     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
8799       Val = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, Val);
8800     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
8801       Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
8802     else
8803       Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
8804     break;
8805   }
8806   return Val;
8807 }
8808 
8809 // The caller is responsible for loading the full value if the argument is
8810 // passed with CCValAssign::Indirect.
8811 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
8812                                 const CCValAssign &VA, const SDLoc &DL,
8813                                 const RISCVTargetLowering &TLI) {
8814   MachineFunction &MF = DAG.getMachineFunction();
8815   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8816   EVT LocVT = VA.getLocVT();
8817   SDValue Val;
8818   const TargetRegisterClass *RC = TLI.getRegClassFor(LocVT.getSimpleVT());
8819   Register VReg = RegInfo.createVirtualRegister(RC);
8820   RegInfo.addLiveIn(VA.getLocReg(), VReg);
8821   Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
8822 
8823   if (VA.getLocInfo() == CCValAssign::Indirect)
8824     return Val;
8825 
8826   return convertLocVTToValVT(DAG, Val, VA, DL, TLI.getSubtarget());
8827 }
8828 
8829 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
8830                                    const CCValAssign &VA, const SDLoc &DL,
8831                                    const RISCVSubtarget &Subtarget) {
8832   EVT LocVT = VA.getLocVT();
8833 
8834   switch (VA.getLocInfo()) {
8835   default:
8836     llvm_unreachable("Unexpected CCValAssign::LocInfo");
8837   case CCValAssign::Full:
8838     if (VA.getValVT().isFixedLengthVector() && LocVT.isScalableVector())
8839       Val = convertToScalableVector(LocVT, Val, DAG, Subtarget);
8840     break;
8841   case CCValAssign::BCvt:
8842     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
8843       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, VA.getLocVT(), Val);
8844     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
8845       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
8846     else
8847       Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
8848     break;
8849   }
8850   return Val;
8851 }
8852 
8853 // The caller is responsible for loading the full value if the argument is
8854 // passed with CCValAssign::Indirect.
8855 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
8856                                 const CCValAssign &VA, const SDLoc &DL) {
8857   MachineFunction &MF = DAG.getMachineFunction();
8858   MachineFrameInfo &MFI = MF.getFrameInfo();
8859   EVT LocVT = VA.getLocVT();
8860   EVT ValVT = VA.getValVT();
8861   EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
8862   if (ValVT.isScalableVector()) {
8863     // When the value is a scalable vector, we save the pointer which points to
8864     // the scalable vector value in the stack. The ValVT will be the pointer
8865     // type, instead of the scalable vector type.
8866     ValVT = LocVT;
8867   }
8868   int FI = MFI.CreateFixedObject(ValVT.getStoreSize(), VA.getLocMemOffset(),
8869                                  /*IsImmutable=*/true);
8870   SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
8871   SDValue Val;
8872 
8873   ISD::LoadExtType ExtType;
8874   switch (VA.getLocInfo()) {
8875   default:
8876     llvm_unreachable("Unexpected CCValAssign::LocInfo");
8877   case CCValAssign::Full:
8878   case CCValAssign::Indirect:
8879   case CCValAssign::BCvt:
8880     ExtType = ISD::NON_EXTLOAD;
8881     break;
8882   }
8883   Val = DAG.getExtLoad(
8884       ExtType, DL, LocVT, Chain, FIN,
8885       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
8886   return Val;
8887 }
8888 
8889 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
8890                                        const CCValAssign &VA, const SDLoc &DL) {
8891   assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
8892          "Unexpected VA");
8893   MachineFunction &MF = DAG.getMachineFunction();
8894   MachineFrameInfo &MFI = MF.getFrameInfo();
8895   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8896 
8897   if (VA.isMemLoc()) {
8898     // f64 is passed on the stack.
8899     int FI =
8900         MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*IsImmutable=*/true);
8901     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
8902     return DAG.getLoad(MVT::f64, DL, Chain, FIN,
8903                        MachinePointerInfo::getFixedStack(MF, FI));
8904   }
8905 
8906   assert(VA.isRegLoc() && "Expected register VA assignment");
8907 
8908   Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
8909   RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
8910   SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
8911   SDValue Hi;
8912   if (VA.getLocReg() == RISCV::X17) {
8913     // Second half of f64 is passed on the stack.
8914     int FI = MFI.CreateFixedObject(4, 0, /*IsImmutable=*/true);
8915     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
8916     Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
8917                      MachinePointerInfo::getFixedStack(MF, FI));
8918   } else {
8919     // Second half of f64 is passed in another GPR.
8920     Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
8921     RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
8922     Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
8923   }
8924   return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
8925 }
8926 
8927 // FastCC has less than 1% performance improvement for some particular
8928 // benchmark. But theoretically, it may has benenfit for some cases.
8929 static bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
8930                             unsigned ValNo, MVT ValVT, MVT LocVT,
8931                             CCValAssign::LocInfo LocInfo,
8932                             ISD::ArgFlagsTy ArgFlags, CCState &State,
8933                             bool IsFixed, bool IsRet, Type *OrigTy,
8934                             const RISCVTargetLowering &TLI,
8935                             Optional<unsigned> FirstMaskArgument) {
8936 
8937   // X5 and X6 might be used for save-restore libcall.
8938   static const MCPhysReg GPRList[] = {
8939       RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
8940       RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7,  RISCV::X28,
8941       RISCV::X29, RISCV::X30, RISCV::X31};
8942 
8943   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
8944     if (unsigned Reg = State.AllocateReg(GPRList)) {
8945       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8946       return false;
8947     }
8948   }
8949 
8950   if (LocVT == MVT::f16) {
8951     static const MCPhysReg FPR16List[] = {
8952         RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
8953         RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H,  RISCV::F1_H,
8954         RISCV::F2_H,  RISCV::F3_H,  RISCV::F4_H,  RISCV::F5_H,  RISCV::F6_H,
8955         RISCV::F7_H,  RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
8956     if (unsigned Reg = State.AllocateReg(FPR16List)) {
8957       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8958       return false;
8959     }
8960   }
8961 
8962   if (LocVT == MVT::f32) {
8963     static const MCPhysReg FPR32List[] = {
8964         RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
8965         RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F,  RISCV::F1_F,
8966         RISCV::F2_F,  RISCV::F3_F,  RISCV::F4_F,  RISCV::F5_F,  RISCV::F6_F,
8967         RISCV::F7_F,  RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
8968     if (unsigned Reg = State.AllocateReg(FPR32List)) {
8969       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8970       return false;
8971     }
8972   }
8973 
8974   if (LocVT == MVT::f64) {
8975     static const MCPhysReg FPR64List[] = {
8976         RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
8977         RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D,  RISCV::F1_D,
8978         RISCV::F2_D,  RISCV::F3_D,  RISCV::F4_D,  RISCV::F5_D,  RISCV::F6_D,
8979         RISCV::F7_D,  RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
8980     if (unsigned Reg = State.AllocateReg(FPR64List)) {
8981       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8982       return false;
8983     }
8984   }
8985 
8986   if (LocVT == MVT::i32 || LocVT == MVT::f32) {
8987     unsigned Offset4 = State.AllocateStack(4, Align(4));
8988     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset4, LocVT, LocInfo));
8989     return false;
8990   }
8991 
8992   if (LocVT == MVT::i64 || LocVT == MVT::f64) {
8993     unsigned Offset5 = State.AllocateStack(8, Align(8));
8994     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset5, LocVT, LocInfo));
8995     return false;
8996   }
8997 
8998   if (LocVT.isVector()) {
8999     if (unsigned Reg =
9000             allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI)) {
9001       // Fixed-length vectors are located in the corresponding scalable-vector
9002       // container types.
9003       if (ValVT.isFixedLengthVector())
9004         LocVT = TLI.getContainerForFixedLengthVector(LocVT);
9005       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9006     } else {
9007       // Try and pass the address via a "fast" GPR.
9008       if (unsigned GPRReg = State.AllocateReg(GPRList)) {
9009         LocInfo = CCValAssign::Indirect;
9010         LocVT = TLI.getSubtarget().getXLenVT();
9011         State.addLoc(CCValAssign::getReg(ValNo, ValVT, GPRReg, LocVT, LocInfo));
9012       } else if (ValVT.isFixedLengthVector()) {
9013         auto StackAlign =
9014             MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
9015         unsigned StackOffset =
9016             State.AllocateStack(ValVT.getStoreSize(), StackAlign);
9017         State.addLoc(
9018             CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
9019       } else {
9020         // Can't pass scalable vectors on the stack.
9021         return true;
9022       }
9023     }
9024 
9025     return false;
9026   }
9027 
9028   return true; // CC didn't match.
9029 }
9030 
9031 static bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
9032                          CCValAssign::LocInfo LocInfo,
9033                          ISD::ArgFlagsTy ArgFlags, CCState &State) {
9034 
9035   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9036     // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, R6, R7, SpLim
9037     //                        s1    s2  s3  s4  s5  s6  s7  s8  s9  s10 s11
9038     static const MCPhysReg GPRList[] = {
9039         RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
9040         RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
9041     if (unsigned Reg = State.AllocateReg(GPRList)) {
9042       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9043       return false;
9044     }
9045   }
9046 
9047   if (LocVT == MVT::f32) {
9048     // Pass in STG registers: F1, ..., F6
9049     //                        fs0 ... fs5
9050     static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
9051                                           RISCV::F18_F, RISCV::F19_F,
9052                                           RISCV::F20_F, RISCV::F21_F};
9053     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9054       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9055       return false;
9056     }
9057   }
9058 
9059   if (LocVT == MVT::f64) {
9060     // Pass in STG registers: D1, ..., D6
9061     //                        fs6 ... fs11
9062     static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
9063                                           RISCV::F24_D, RISCV::F25_D,
9064                                           RISCV::F26_D, RISCV::F27_D};
9065     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9066       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9067       return false;
9068     }
9069   }
9070 
9071   report_fatal_error("No registers left in GHC calling convention");
9072   return true;
9073 }
9074 
9075 // Transform physical registers into virtual registers.
9076 SDValue RISCVTargetLowering::LowerFormalArguments(
9077     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
9078     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
9079     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
9080 
9081   MachineFunction &MF = DAG.getMachineFunction();
9082 
9083   switch (CallConv) {
9084   default:
9085     report_fatal_error("Unsupported calling convention");
9086   case CallingConv::C:
9087   case CallingConv::Fast:
9088     break;
9089   case CallingConv::GHC:
9090     if (!MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtF] ||
9091         !MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtD])
9092       report_fatal_error(
9093         "GHC calling convention requires the F and D instruction set extensions");
9094   }
9095 
9096   const Function &Func = MF.getFunction();
9097   if (Func.hasFnAttribute("interrupt")) {
9098     if (!Func.arg_empty())
9099       report_fatal_error(
9100         "Functions with the interrupt attribute cannot have arguments!");
9101 
9102     StringRef Kind =
9103       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
9104 
9105     if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
9106       report_fatal_error(
9107         "Function interrupt attribute argument not supported!");
9108   }
9109 
9110   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9111   MVT XLenVT = Subtarget.getXLenVT();
9112   unsigned XLenInBytes = Subtarget.getXLen() / 8;
9113   // Used with vargs to acumulate store chains.
9114   std::vector<SDValue> OutChains;
9115 
9116   // Assign locations to all of the incoming arguments.
9117   SmallVector<CCValAssign, 16> ArgLocs;
9118   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9119 
9120   if (CallConv == CallingConv::GHC)
9121     CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_GHC);
9122   else
9123     analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false,
9124                      CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9125                                                    : CC_RISCV);
9126 
9127   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
9128     CCValAssign &VA = ArgLocs[i];
9129     SDValue ArgValue;
9130     // Passing f64 on RV32D with a soft float ABI must be handled as a special
9131     // case.
9132     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
9133       ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
9134     else if (VA.isRegLoc())
9135       ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, *this);
9136     else
9137       ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
9138 
9139     if (VA.getLocInfo() == CCValAssign::Indirect) {
9140       // If the original argument was split and passed by reference (e.g. i128
9141       // on RV32), we need to load all parts of it here (using the same
9142       // address). Vectors may be partly split to registers and partly to the
9143       // stack, in which case the base address is partly offset and subsequent
9144       // stores are relative to that.
9145       InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
9146                                    MachinePointerInfo()));
9147       unsigned ArgIndex = Ins[i].OrigArgIndex;
9148       unsigned ArgPartOffset = Ins[i].PartOffset;
9149       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9150       while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
9151         CCValAssign &PartVA = ArgLocs[i + 1];
9152         unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset;
9153         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9154         if (PartVA.getValVT().isScalableVector())
9155           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9156         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
9157         InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
9158                                      MachinePointerInfo()));
9159         ++i;
9160       }
9161       continue;
9162     }
9163     InVals.push_back(ArgValue);
9164   }
9165 
9166   if (IsVarArg) {
9167     ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
9168     unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
9169     const TargetRegisterClass *RC = &RISCV::GPRRegClass;
9170     MachineFrameInfo &MFI = MF.getFrameInfo();
9171     MachineRegisterInfo &RegInfo = MF.getRegInfo();
9172     RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
9173 
9174     // Offset of the first variable argument from stack pointer, and size of
9175     // the vararg save area. For now, the varargs save area is either zero or
9176     // large enough to hold a0-a7.
9177     int VaArgOffset, VarArgsSaveSize;
9178 
9179     // If all registers are allocated, then all varargs must be passed on the
9180     // stack and we don't need to save any argregs.
9181     if (ArgRegs.size() == Idx) {
9182       VaArgOffset = CCInfo.getNextStackOffset();
9183       VarArgsSaveSize = 0;
9184     } else {
9185       VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
9186       VaArgOffset = -VarArgsSaveSize;
9187     }
9188 
9189     // Record the frame index of the first variable argument
9190     // which is a value necessary to VASTART.
9191     int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9192     RVFI->setVarArgsFrameIndex(FI);
9193 
9194     // If saving an odd number of registers then create an extra stack slot to
9195     // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
9196     // offsets to even-numbered registered remain 2*XLEN-aligned.
9197     if (Idx % 2) {
9198       MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true);
9199       VarArgsSaveSize += XLenInBytes;
9200     }
9201 
9202     // Copy the integer registers that may have been used for passing varargs
9203     // to the vararg save area.
9204     for (unsigned I = Idx; I < ArgRegs.size();
9205          ++I, VaArgOffset += XLenInBytes) {
9206       const Register Reg = RegInfo.createVirtualRegister(RC);
9207       RegInfo.addLiveIn(ArgRegs[I], Reg);
9208       SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
9209       FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9210       SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9211       SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
9212                                    MachinePointerInfo::getFixedStack(MF, FI));
9213       cast<StoreSDNode>(Store.getNode())
9214           ->getMemOperand()
9215           ->setValue((Value *)nullptr);
9216       OutChains.push_back(Store);
9217     }
9218     RVFI->setVarArgsSaveSize(VarArgsSaveSize);
9219   }
9220 
9221   // All stores are grouped in one node to allow the matching between
9222   // the size of Ins and InVals. This only happens for vararg functions.
9223   if (!OutChains.empty()) {
9224     OutChains.push_back(Chain);
9225     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
9226   }
9227 
9228   return Chain;
9229 }
9230 
9231 /// isEligibleForTailCallOptimization - Check whether the call is eligible
9232 /// for tail call optimization.
9233 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
9234 bool RISCVTargetLowering::isEligibleForTailCallOptimization(
9235     CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
9236     const SmallVector<CCValAssign, 16> &ArgLocs) const {
9237 
9238   auto &Callee = CLI.Callee;
9239   auto CalleeCC = CLI.CallConv;
9240   auto &Outs = CLI.Outs;
9241   auto &Caller = MF.getFunction();
9242   auto CallerCC = Caller.getCallingConv();
9243 
9244   // Exception-handling functions need a special set of instructions to
9245   // indicate a return to the hardware. Tail-calling another function would
9246   // probably break this.
9247   // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
9248   // should be expanded as new function attributes are introduced.
9249   if (Caller.hasFnAttribute("interrupt"))
9250     return false;
9251 
9252   // Do not tail call opt if the stack is used to pass parameters.
9253   if (CCInfo.getNextStackOffset() != 0)
9254     return false;
9255 
9256   // Do not tail call opt if any parameters need to be passed indirectly.
9257   // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
9258   // passed indirectly. So the address of the value will be passed in a
9259   // register, or if not available, then the address is put on the stack. In
9260   // order to pass indirectly, space on the stack often needs to be allocated
9261   // in order to store the value. In this case the CCInfo.getNextStackOffset()
9262   // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
9263   // are passed CCValAssign::Indirect.
9264   for (auto &VA : ArgLocs)
9265     if (VA.getLocInfo() == CCValAssign::Indirect)
9266       return false;
9267 
9268   // Do not tail call opt if either caller or callee uses struct return
9269   // semantics.
9270   auto IsCallerStructRet = Caller.hasStructRetAttr();
9271   auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
9272   if (IsCallerStructRet || IsCalleeStructRet)
9273     return false;
9274 
9275   // Externally-defined functions with weak linkage should not be
9276   // tail-called. The behaviour of branch instructions in this situation (as
9277   // used for tail calls) is implementation-defined, so we cannot rely on the
9278   // linker replacing the tail call with a return.
9279   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
9280     const GlobalValue *GV = G->getGlobal();
9281     if (GV->hasExternalWeakLinkage())
9282       return false;
9283   }
9284 
9285   // The callee has to preserve all registers the caller needs to preserve.
9286   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
9287   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
9288   if (CalleeCC != CallerCC) {
9289     const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
9290     if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
9291       return false;
9292   }
9293 
9294   // Byval parameters hand the function a pointer directly into the stack area
9295   // we want to reuse during a tail call. Working around this *is* possible
9296   // but less efficient and uglier in LowerCall.
9297   for (auto &Arg : Outs)
9298     if (Arg.Flags.isByVal())
9299       return false;
9300 
9301   return true;
9302 }
9303 
9304 static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
9305   return DAG.getDataLayout().getPrefTypeAlign(
9306       VT.getTypeForEVT(*DAG.getContext()));
9307 }
9308 
9309 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
9310 // and output parameter nodes.
9311 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
9312                                        SmallVectorImpl<SDValue> &InVals) const {
9313   SelectionDAG &DAG = CLI.DAG;
9314   SDLoc &DL = CLI.DL;
9315   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
9316   SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
9317   SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
9318   SDValue Chain = CLI.Chain;
9319   SDValue Callee = CLI.Callee;
9320   bool &IsTailCall = CLI.IsTailCall;
9321   CallingConv::ID CallConv = CLI.CallConv;
9322   bool IsVarArg = CLI.IsVarArg;
9323   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9324   MVT XLenVT = Subtarget.getXLenVT();
9325 
9326   MachineFunction &MF = DAG.getMachineFunction();
9327 
9328   // Analyze the operands of the call, assigning locations to each operand.
9329   SmallVector<CCValAssign, 16> ArgLocs;
9330   CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9331 
9332   if (CallConv == CallingConv::GHC)
9333     ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_GHC);
9334   else
9335     analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI,
9336                       CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9337                                                     : CC_RISCV);
9338 
9339   // Check if it's really possible to do a tail call.
9340   if (IsTailCall)
9341     IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
9342 
9343   if (IsTailCall)
9344     ++NumTailCalls;
9345   else if (CLI.CB && CLI.CB->isMustTailCall())
9346     report_fatal_error("failed to perform tail call elimination on a call "
9347                        "site marked musttail");
9348 
9349   // Get a count of how many bytes are to be pushed on the stack.
9350   unsigned NumBytes = ArgCCInfo.getNextStackOffset();
9351 
9352   // Create local copies for byval args
9353   SmallVector<SDValue, 8> ByValArgs;
9354   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9355     ISD::ArgFlagsTy Flags = Outs[i].Flags;
9356     if (!Flags.isByVal())
9357       continue;
9358 
9359     SDValue Arg = OutVals[i];
9360     unsigned Size = Flags.getByValSize();
9361     Align Alignment = Flags.getNonZeroByValAlign();
9362 
9363     int FI =
9364         MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false);
9365     SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9366     SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
9367 
9368     Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
9369                           /*IsVolatile=*/false,
9370                           /*AlwaysInline=*/false, IsTailCall,
9371                           MachinePointerInfo(), MachinePointerInfo());
9372     ByValArgs.push_back(FIPtr);
9373   }
9374 
9375   if (!IsTailCall)
9376     Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
9377 
9378   // Copy argument values to their designated locations.
9379   SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
9380   SmallVector<SDValue, 8> MemOpChains;
9381   SDValue StackPtr;
9382   for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
9383     CCValAssign &VA = ArgLocs[i];
9384     SDValue ArgValue = OutVals[i];
9385     ISD::ArgFlagsTy Flags = Outs[i].Flags;
9386 
9387     // Handle passing f64 on RV32D with a soft float ABI as a special case.
9388     bool IsF64OnRV32DSoftABI =
9389         VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
9390     if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
9391       SDValue SplitF64 = DAG.getNode(
9392           RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
9393       SDValue Lo = SplitF64.getValue(0);
9394       SDValue Hi = SplitF64.getValue(1);
9395 
9396       Register RegLo = VA.getLocReg();
9397       RegsToPass.push_back(std::make_pair(RegLo, Lo));
9398 
9399       if (RegLo == RISCV::X17) {
9400         // Second half of f64 is passed on the stack.
9401         // Work out the address of the stack slot.
9402         if (!StackPtr.getNode())
9403           StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
9404         // Emit the store.
9405         MemOpChains.push_back(
9406             DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
9407       } else {
9408         // Second half of f64 is passed in another GPR.
9409         assert(RegLo < RISCV::X31 && "Invalid register pair");
9410         Register RegHigh = RegLo + 1;
9411         RegsToPass.push_back(std::make_pair(RegHigh, Hi));
9412       }
9413       continue;
9414     }
9415 
9416     // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
9417     // as any other MemLoc.
9418 
9419     // Promote the value if needed.
9420     // For now, only handle fully promoted and indirect arguments.
9421     if (VA.getLocInfo() == CCValAssign::Indirect) {
9422       // Store the argument in a stack slot and pass its address.
9423       Align StackAlign =
9424           std::max(getPrefTypeAlign(Outs[i].ArgVT, DAG),
9425                    getPrefTypeAlign(ArgValue.getValueType(), DAG));
9426       TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
9427       // If the original argument was split (e.g. i128), we need
9428       // to store the required parts of it here (and pass just one address).
9429       // Vectors may be partly split to registers and partly to the stack, in
9430       // which case the base address is partly offset and subsequent stores are
9431       // relative to that.
9432       unsigned ArgIndex = Outs[i].OrigArgIndex;
9433       unsigned ArgPartOffset = Outs[i].PartOffset;
9434       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9435       // Calculate the total size to store. We don't have access to what we're
9436       // actually storing other than performing the loop and collecting the
9437       // info.
9438       SmallVector<std::pair<SDValue, SDValue>> Parts;
9439       while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
9440         SDValue PartValue = OutVals[i + 1];
9441         unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset;
9442         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9443         EVT PartVT = PartValue.getValueType();
9444         if (PartVT.isScalableVector())
9445           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9446         StoredSize += PartVT.getStoreSize();
9447         StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
9448         Parts.push_back(std::make_pair(PartValue, Offset));
9449         ++i;
9450       }
9451       SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
9452       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
9453       MemOpChains.push_back(
9454           DAG.getStore(Chain, DL, ArgValue, SpillSlot,
9455                        MachinePointerInfo::getFixedStack(MF, FI)));
9456       for (const auto &Part : Parts) {
9457         SDValue PartValue = Part.first;
9458         SDValue PartOffset = Part.second;
9459         SDValue Address =
9460             DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
9461         MemOpChains.push_back(
9462             DAG.getStore(Chain, DL, PartValue, Address,
9463                          MachinePointerInfo::getFixedStack(MF, FI)));
9464       }
9465       ArgValue = SpillSlot;
9466     } else {
9467       ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget);
9468     }
9469 
9470     // Use local copy if it is a byval arg.
9471     if (Flags.isByVal())
9472       ArgValue = ByValArgs[j++];
9473 
9474     if (VA.isRegLoc()) {
9475       // Queue up the argument copies and emit them at the end.
9476       RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
9477     } else {
9478       assert(VA.isMemLoc() && "Argument not register or memory");
9479       assert(!IsTailCall && "Tail call not allowed if stack is used "
9480                             "for passing parameters");
9481 
9482       // Work out the address of the stack slot.
9483       if (!StackPtr.getNode())
9484         StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
9485       SDValue Address =
9486           DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
9487                       DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
9488 
9489       // Emit the store.
9490       MemOpChains.push_back(
9491           DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
9492     }
9493   }
9494 
9495   // Join the stores, which are independent of one another.
9496   if (!MemOpChains.empty())
9497     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
9498 
9499   SDValue Glue;
9500 
9501   // Build a sequence of copy-to-reg nodes, chained and glued together.
9502   for (auto &Reg : RegsToPass) {
9503     Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
9504     Glue = Chain.getValue(1);
9505   }
9506 
9507   // Validate that none of the argument registers have been marked as
9508   // reserved, if so report an error. Do the same for the return address if this
9509   // is not a tailcall.
9510   validateCCReservedRegs(RegsToPass, MF);
9511   if (!IsTailCall &&
9512       MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1))
9513     MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9514         MF.getFunction(),
9515         "Return address register required, but has been reserved."});
9516 
9517   // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
9518   // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
9519   // split it and then direct call can be matched by PseudoCALL.
9520   if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
9521     const GlobalValue *GV = S->getGlobal();
9522 
9523     unsigned OpFlags = RISCVII::MO_CALL;
9524     if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
9525       OpFlags = RISCVII::MO_PLT;
9526 
9527     Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
9528   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
9529     unsigned OpFlags = RISCVII::MO_CALL;
9530 
9531     if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
9532                                                  nullptr))
9533       OpFlags = RISCVII::MO_PLT;
9534 
9535     Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
9536   }
9537 
9538   // The first call operand is the chain and the second is the target address.
9539   SmallVector<SDValue, 8> Ops;
9540   Ops.push_back(Chain);
9541   Ops.push_back(Callee);
9542 
9543   // Add argument registers to the end of the list so that they are
9544   // known live into the call.
9545   for (auto &Reg : RegsToPass)
9546     Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
9547 
9548   if (!IsTailCall) {
9549     // Add a register mask operand representing the call-preserved registers.
9550     const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
9551     const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
9552     assert(Mask && "Missing call preserved mask for calling convention");
9553     Ops.push_back(DAG.getRegisterMask(Mask));
9554   }
9555 
9556   // Glue the call to the argument copies, if any.
9557   if (Glue.getNode())
9558     Ops.push_back(Glue);
9559 
9560   // Emit the call.
9561   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9562 
9563   if (IsTailCall) {
9564     MF.getFrameInfo().setHasTailCall();
9565     return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
9566   }
9567 
9568   Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
9569   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
9570   Glue = Chain.getValue(1);
9571 
9572   // Mark the end of the call, which is glued to the call itself.
9573   Chain = DAG.getCALLSEQ_END(Chain,
9574                              DAG.getConstant(NumBytes, DL, PtrVT, true),
9575                              DAG.getConstant(0, DL, PtrVT, true),
9576                              Glue, DL);
9577   Glue = Chain.getValue(1);
9578 
9579   // Assign locations to each value returned by this call.
9580   SmallVector<CCValAssign, 16> RVLocs;
9581   CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
9582   analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, CC_RISCV);
9583 
9584   // Copy all of the result registers out of their specified physreg.
9585   for (auto &VA : RVLocs) {
9586     // Copy the value out
9587     SDValue RetValue =
9588         DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
9589     // Glue the RetValue to the end of the call sequence
9590     Chain = RetValue.getValue(1);
9591     Glue = RetValue.getValue(2);
9592 
9593     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
9594       assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
9595       SDValue RetValue2 =
9596           DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
9597       Chain = RetValue2.getValue(1);
9598       Glue = RetValue2.getValue(2);
9599       RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
9600                              RetValue2);
9601     }
9602 
9603     RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget);
9604 
9605     InVals.push_back(RetValue);
9606   }
9607 
9608   return Chain;
9609 }
9610 
9611 bool RISCVTargetLowering::CanLowerReturn(
9612     CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
9613     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
9614   SmallVector<CCValAssign, 16> RVLocs;
9615   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
9616 
9617   Optional<unsigned> FirstMaskArgument;
9618   if (Subtarget.hasVInstructions())
9619     FirstMaskArgument = preAssignMask(Outs);
9620 
9621   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9622     MVT VT = Outs[i].VT;
9623     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
9624     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9625     if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
9626                  ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr,
9627                  *this, FirstMaskArgument))
9628       return false;
9629   }
9630   return true;
9631 }
9632 
9633 SDValue
9634 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
9635                                  bool IsVarArg,
9636                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
9637                                  const SmallVectorImpl<SDValue> &OutVals,
9638                                  const SDLoc &DL, SelectionDAG &DAG) const {
9639   const MachineFunction &MF = DAG.getMachineFunction();
9640   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
9641 
9642   // Stores the assignment of the return value to a location.
9643   SmallVector<CCValAssign, 16> RVLocs;
9644 
9645   // Info about the registers and stack slot.
9646   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
9647                  *DAG.getContext());
9648 
9649   analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
9650                     nullptr, CC_RISCV);
9651 
9652   if (CallConv == CallingConv::GHC && !RVLocs.empty())
9653     report_fatal_error("GHC functions return void only");
9654 
9655   SDValue Glue;
9656   SmallVector<SDValue, 4> RetOps(1, Chain);
9657 
9658   // Copy the result values into the output registers.
9659   for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
9660     SDValue Val = OutVals[i];
9661     CCValAssign &VA = RVLocs[i];
9662     assert(VA.isRegLoc() && "Can only return in registers!");
9663 
9664     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
9665       // Handle returning f64 on RV32D with a soft float ABI.
9666       assert(VA.isRegLoc() && "Expected return via registers");
9667       SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
9668                                      DAG.getVTList(MVT::i32, MVT::i32), Val);
9669       SDValue Lo = SplitF64.getValue(0);
9670       SDValue Hi = SplitF64.getValue(1);
9671       Register RegLo = VA.getLocReg();
9672       assert(RegLo < RISCV::X31 && "Invalid register pair");
9673       Register RegHi = RegLo + 1;
9674 
9675       if (STI.isRegisterReservedByUser(RegLo) ||
9676           STI.isRegisterReservedByUser(RegHi))
9677         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9678             MF.getFunction(),
9679             "Return value register required, but has been reserved."});
9680 
9681       Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
9682       Glue = Chain.getValue(1);
9683       RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
9684       Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
9685       Glue = Chain.getValue(1);
9686       RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
9687     } else {
9688       // Handle a 'normal' return.
9689       Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
9690       Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
9691 
9692       if (STI.isRegisterReservedByUser(VA.getLocReg()))
9693         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9694             MF.getFunction(),
9695             "Return value register required, but has been reserved."});
9696 
9697       // Guarantee that all emitted copies are stuck together.
9698       Glue = Chain.getValue(1);
9699       RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
9700     }
9701   }
9702 
9703   RetOps[0] = Chain; // Update chain.
9704 
9705   // Add the glue node if we have it.
9706   if (Glue.getNode()) {
9707     RetOps.push_back(Glue);
9708   }
9709 
9710   unsigned RetOpc = RISCVISD::RET_FLAG;
9711   // Interrupt service routines use different return instructions.
9712   const Function &Func = DAG.getMachineFunction().getFunction();
9713   if (Func.hasFnAttribute("interrupt")) {
9714     if (!Func.getReturnType()->isVoidTy())
9715       report_fatal_error(
9716           "Functions with the interrupt attribute must have void return type!");
9717 
9718     MachineFunction &MF = DAG.getMachineFunction();
9719     StringRef Kind =
9720       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
9721 
9722     if (Kind == "user")
9723       RetOpc = RISCVISD::URET_FLAG;
9724     else if (Kind == "supervisor")
9725       RetOpc = RISCVISD::SRET_FLAG;
9726     else
9727       RetOpc = RISCVISD::MRET_FLAG;
9728   }
9729 
9730   return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
9731 }
9732 
9733 void RISCVTargetLowering::validateCCReservedRegs(
9734     const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
9735     MachineFunction &MF) const {
9736   const Function &F = MF.getFunction();
9737   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
9738 
9739   if (llvm::any_of(Regs, [&STI](auto Reg) {
9740         return STI.isRegisterReservedByUser(Reg.first);
9741       }))
9742     F.getContext().diagnose(DiagnosticInfoUnsupported{
9743         F, "Argument register required, but has been reserved."});
9744 }
9745 
9746 bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
9747   return CI->isTailCall();
9748 }
9749 
9750 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
9751 #define NODE_NAME_CASE(NODE)                                                   \
9752   case RISCVISD::NODE:                                                         \
9753     return "RISCVISD::" #NODE;
9754   // clang-format off
9755   switch ((RISCVISD::NodeType)Opcode) {
9756   case RISCVISD::FIRST_NUMBER:
9757     break;
9758   NODE_NAME_CASE(RET_FLAG)
9759   NODE_NAME_CASE(URET_FLAG)
9760   NODE_NAME_CASE(SRET_FLAG)
9761   NODE_NAME_CASE(MRET_FLAG)
9762   NODE_NAME_CASE(CALL)
9763   NODE_NAME_CASE(SELECT_CC)
9764   NODE_NAME_CASE(BR_CC)
9765   NODE_NAME_CASE(BuildPairF64)
9766   NODE_NAME_CASE(SplitF64)
9767   NODE_NAME_CASE(TAIL)
9768   NODE_NAME_CASE(MULHSU)
9769   NODE_NAME_CASE(SLLW)
9770   NODE_NAME_CASE(SRAW)
9771   NODE_NAME_CASE(SRLW)
9772   NODE_NAME_CASE(DIVW)
9773   NODE_NAME_CASE(DIVUW)
9774   NODE_NAME_CASE(REMUW)
9775   NODE_NAME_CASE(ROLW)
9776   NODE_NAME_CASE(RORW)
9777   NODE_NAME_CASE(CLZW)
9778   NODE_NAME_CASE(CTZW)
9779   NODE_NAME_CASE(FSLW)
9780   NODE_NAME_CASE(FSRW)
9781   NODE_NAME_CASE(FSL)
9782   NODE_NAME_CASE(FSR)
9783   NODE_NAME_CASE(FMV_H_X)
9784   NODE_NAME_CASE(FMV_X_ANYEXTH)
9785   NODE_NAME_CASE(FMV_W_X_RV64)
9786   NODE_NAME_CASE(FMV_X_ANYEXTW_RV64)
9787   NODE_NAME_CASE(FCVT_X)
9788   NODE_NAME_CASE(FCVT_XU)
9789   NODE_NAME_CASE(FCVT_W_RV64)
9790   NODE_NAME_CASE(FCVT_WU_RV64)
9791   NODE_NAME_CASE(STRICT_FCVT_W_RV64)
9792   NODE_NAME_CASE(STRICT_FCVT_WU_RV64)
9793   NODE_NAME_CASE(READ_CYCLE_WIDE)
9794   NODE_NAME_CASE(GREV)
9795   NODE_NAME_CASE(GREVW)
9796   NODE_NAME_CASE(GORC)
9797   NODE_NAME_CASE(GORCW)
9798   NODE_NAME_CASE(SHFL)
9799   NODE_NAME_CASE(SHFLW)
9800   NODE_NAME_CASE(UNSHFL)
9801   NODE_NAME_CASE(UNSHFLW)
9802   NODE_NAME_CASE(BFP)
9803   NODE_NAME_CASE(BFPW)
9804   NODE_NAME_CASE(BCOMPRESS)
9805   NODE_NAME_CASE(BCOMPRESSW)
9806   NODE_NAME_CASE(BDECOMPRESS)
9807   NODE_NAME_CASE(BDECOMPRESSW)
9808   NODE_NAME_CASE(VMV_V_X_VL)
9809   NODE_NAME_CASE(VFMV_V_F_VL)
9810   NODE_NAME_CASE(VMV_X_S)
9811   NODE_NAME_CASE(VMV_S_X_VL)
9812   NODE_NAME_CASE(VFMV_S_F_VL)
9813   NODE_NAME_CASE(SPLAT_VECTOR_I64)
9814   NODE_NAME_CASE(SPLAT_VECTOR_SPLIT_I64_VL)
9815   NODE_NAME_CASE(READ_VLENB)
9816   NODE_NAME_CASE(TRUNCATE_VECTOR_VL)
9817   NODE_NAME_CASE(VSLIDEUP_VL)
9818   NODE_NAME_CASE(VSLIDE1UP_VL)
9819   NODE_NAME_CASE(VSLIDEDOWN_VL)
9820   NODE_NAME_CASE(VSLIDE1DOWN_VL)
9821   NODE_NAME_CASE(VID_VL)
9822   NODE_NAME_CASE(VFNCVT_ROD_VL)
9823   NODE_NAME_CASE(VECREDUCE_ADD_VL)
9824   NODE_NAME_CASE(VECREDUCE_UMAX_VL)
9825   NODE_NAME_CASE(VECREDUCE_SMAX_VL)
9826   NODE_NAME_CASE(VECREDUCE_UMIN_VL)
9827   NODE_NAME_CASE(VECREDUCE_SMIN_VL)
9828   NODE_NAME_CASE(VECREDUCE_AND_VL)
9829   NODE_NAME_CASE(VECREDUCE_OR_VL)
9830   NODE_NAME_CASE(VECREDUCE_XOR_VL)
9831   NODE_NAME_CASE(VECREDUCE_FADD_VL)
9832   NODE_NAME_CASE(VECREDUCE_SEQ_FADD_VL)
9833   NODE_NAME_CASE(VECREDUCE_FMIN_VL)
9834   NODE_NAME_CASE(VECREDUCE_FMAX_VL)
9835   NODE_NAME_CASE(ADD_VL)
9836   NODE_NAME_CASE(AND_VL)
9837   NODE_NAME_CASE(MUL_VL)
9838   NODE_NAME_CASE(OR_VL)
9839   NODE_NAME_CASE(SDIV_VL)
9840   NODE_NAME_CASE(SHL_VL)
9841   NODE_NAME_CASE(SREM_VL)
9842   NODE_NAME_CASE(SRA_VL)
9843   NODE_NAME_CASE(SRL_VL)
9844   NODE_NAME_CASE(SUB_VL)
9845   NODE_NAME_CASE(UDIV_VL)
9846   NODE_NAME_CASE(UREM_VL)
9847   NODE_NAME_CASE(XOR_VL)
9848   NODE_NAME_CASE(SADDSAT_VL)
9849   NODE_NAME_CASE(UADDSAT_VL)
9850   NODE_NAME_CASE(SSUBSAT_VL)
9851   NODE_NAME_CASE(USUBSAT_VL)
9852   NODE_NAME_CASE(FADD_VL)
9853   NODE_NAME_CASE(FSUB_VL)
9854   NODE_NAME_CASE(FMUL_VL)
9855   NODE_NAME_CASE(FDIV_VL)
9856   NODE_NAME_CASE(FNEG_VL)
9857   NODE_NAME_CASE(FABS_VL)
9858   NODE_NAME_CASE(FSQRT_VL)
9859   NODE_NAME_CASE(FMA_VL)
9860   NODE_NAME_CASE(FCOPYSIGN_VL)
9861   NODE_NAME_CASE(SMIN_VL)
9862   NODE_NAME_CASE(SMAX_VL)
9863   NODE_NAME_CASE(UMIN_VL)
9864   NODE_NAME_CASE(UMAX_VL)
9865   NODE_NAME_CASE(FMINNUM_VL)
9866   NODE_NAME_CASE(FMAXNUM_VL)
9867   NODE_NAME_CASE(MULHS_VL)
9868   NODE_NAME_CASE(MULHU_VL)
9869   NODE_NAME_CASE(FP_TO_SINT_VL)
9870   NODE_NAME_CASE(FP_TO_UINT_VL)
9871   NODE_NAME_CASE(SINT_TO_FP_VL)
9872   NODE_NAME_CASE(UINT_TO_FP_VL)
9873   NODE_NAME_CASE(FP_EXTEND_VL)
9874   NODE_NAME_CASE(FP_ROUND_VL)
9875   NODE_NAME_CASE(VWMUL_VL)
9876   NODE_NAME_CASE(VWMULU_VL)
9877   NODE_NAME_CASE(SETCC_VL)
9878   NODE_NAME_CASE(VSELECT_VL)
9879   NODE_NAME_CASE(VMAND_VL)
9880   NODE_NAME_CASE(VMOR_VL)
9881   NODE_NAME_CASE(VMXOR_VL)
9882   NODE_NAME_CASE(VMCLR_VL)
9883   NODE_NAME_CASE(VMSET_VL)
9884   NODE_NAME_CASE(VRGATHER_VX_VL)
9885   NODE_NAME_CASE(VRGATHER_VV_VL)
9886   NODE_NAME_CASE(VRGATHEREI16_VV_VL)
9887   NODE_NAME_CASE(VSEXT_VL)
9888   NODE_NAME_CASE(VZEXT_VL)
9889   NODE_NAME_CASE(VCPOP_VL)
9890   NODE_NAME_CASE(VLE_VL)
9891   NODE_NAME_CASE(VSE_VL)
9892   NODE_NAME_CASE(READ_CSR)
9893   NODE_NAME_CASE(WRITE_CSR)
9894   NODE_NAME_CASE(SWAP_CSR)
9895   }
9896   // clang-format on
9897   return nullptr;
9898 #undef NODE_NAME_CASE
9899 }
9900 
9901 /// getConstraintType - Given a constraint letter, return the type of
9902 /// constraint it is for this target.
9903 RISCVTargetLowering::ConstraintType
9904 RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
9905   if (Constraint.size() == 1) {
9906     switch (Constraint[0]) {
9907     default:
9908       break;
9909     case 'f':
9910       return C_RegisterClass;
9911     case 'I':
9912     case 'J':
9913     case 'K':
9914       return C_Immediate;
9915     case 'A':
9916       return C_Memory;
9917     case 'S': // A symbolic address
9918       return C_Other;
9919     }
9920   } else {
9921     if (Constraint == "vr" || Constraint == "vm")
9922       return C_RegisterClass;
9923   }
9924   return TargetLowering::getConstraintType(Constraint);
9925 }
9926 
9927 std::pair<unsigned, const TargetRegisterClass *>
9928 RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
9929                                                   StringRef Constraint,
9930                                                   MVT VT) const {
9931   // First, see if this is a constraint that directly corresponds to a
9932   // RISCV register class.
9933   if (Constraint.size() == 1) {
9934     switch (Constraint[0]) {
9935     case 'r':
9936       // TODO: Support fixed vectors up to XLen for P extension?
9937       if (VT.isVector())
9938         break;
9939       return std::make_pair(0U, &RISCV::GPRRegClass);
9940     case 'f':
9941       if (Subtarget.hasStdExtZfh() && VT == MVT::f16)
9942         return std::make_pair(0U, &RISCV::FPR16RegClass);
9943       if (Subtarget.hasStdExtF() && VT == MVT::f32)
9944         return std::make_pair(0U, &RISCV::FPR32RegClass);
9945       if (Subtarget.hasStdExtD() && VT == MVT::f64)
9946         return std::make_pair(0U, &RISCV::FPR64RegClass);
9947       break;
9948     default:
9949       break;
9950     }
9951   } else if (Constraint == "vr") {
9952     for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
9953                            &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
9954       if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
9955         return std::make_pair(0U, RC);
9956     }
9957   } else if (Constraint == "vm") {
9958     if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
9959       return std::make_pair(0U, &RISCV::VMV0RegClass);
9960   }
9961 
9962   // Clang will correctly decode the usage of register name aliases into their
9963   // official names. However, other frontends like `rustc` do not. This allows
9964   // users of these frontends to use the ABI names for registers in LLVM-style
9965   // register constraints.
9966   unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
9967                                .Case("{zero}", RISCV::X0)
9968                                .Case("{ra}", RISCV::X1)
9969                                .Case("{sp}", RISCV::X2)
9970                                .Case("{gp}", RISCV::X3)
9971                                .Case("{tp}", RISCV::X4)
9972                                .Case("{t0}", RISCV::X5)
9973                                .Case("{t1}", RISCV::X6)
9974                                .Case("{t2}", RISCV::X7)
9975                                .Cases("{s0}", "{fp}", RISCV::X8)
9976                                .Case("{s1}", RISCV::X9)
9977                                .Case("{a0}", RISCV::X10)
9978                                .Case("{a1}", RISCV::X11)
9979                                .Case("{a2}", RISCV::X12)
9980                                .Case("{a3}", RISCV::X13)
9981                                .Case("{a4}", RISCV::X14)
9982                                .Case("{a5}", RISCV::X15)
9983                                .Case("{a6}", RISCV::X16)
9984                                .Case("{a7}", RISCV::X17)
9985                                .Case("{s2}", RISCV::X18)
9986                                .Case("{s3}", RISCV::X19)
9987                                .Case("{s4}", RISCV::X20)
9988                                .Case("{s5}", RISCV::X21)
9989                                .Case("{s6}", RISCV::X22)
9990                                .Case("{s7}", RISCV::X23)
9991                                .Case("{s8}", RISCV::X24)
9992                                .Case("{s9}", RISCV::X25)
9993                                .Case("{s10}", RISCV::X26)
9994                                .Case("{s11}", RISCV::X27)
9995                                .Case("{t3}", RISCV::X28)
9996                                .Case("{t4}", RISCV::X29)
9997                                .Case("{t5}", RISCV::X30)
9998                                .Case("{t6}", RISCV::X31)
9999                                .Default(RISCV::NoRegister);
10000   if (XRegFromAlias != RISCV::NoRegister)
10001     return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
10002 
10003   // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
10004   // TableGen record rather than the AsmName to choose registers for InlineAsm
10005   // constraints, plus we want to match those names to the widest floating point
10006   // register type available, manually select floating point registers here.
10007   //
10008   // The second case is the ABI name of the register, so that frontends can also
10009   // use the ABI names in register constraint lists.
10010   if (Subtarget.hasStdExtF()) {
10011     unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
10012                         .Cases("{f0}", "{ft0}", RISCV::F0_F)
10013                         .Cases("{f1}", "{ft1}", RISCV::F1_F)
10014                         .Cases("{f2}", "{ft2}", RISCV::F2_F)
10015                         .Cases("{f3}", "{ft3}", RISCV::F3_F)
10016                         .Cases("{f4}", "{ft4}", RISCV::F4_F)
10017                         .Cases("{f5}", "{ft5}", RISCV::F5_F)
10018                         .Cases("{f6}", "{ft6}", RISCV::F6_F)
10019                         .Cases("{f7}", "{ft7}", RISCV::F7_F)
10020                         .Cases("{f8}", "{fs0}", RISCV::F8_F)
10021                         .Cases("{f9}", "{fs1}", RISCV::F9_F)
10022                         .Cases("{f10}", "{fa0}", RISCV::F10_F)
10023                         .Cases("{f11}", "{fa1}", RISCV::F11_F)
10024                         .Cases("{f12}", "{fa2}", RISCV::F12_F)
10025                         .Cases("{f13}", "{fa3}", RISCV::F13_F)
10026                         .Cases("{f14}", "{fa4}", RISCV::F14_F)
10027                         .Cases("{f15}", "{fa5}", RISCV::F15_F)
10028                         .Cases("{f16}", "{fa6}", RISCV::F16_F)
10029                         .Cases("{f17}", "{fa7}", RISCV::F17_F)
10030                         .Cases("{f18}", "{fs2}", RISCV::F18_F)
10031                         .Cases("{f19}", "{fs3}", RISCV::F19_F)
10032                         .Cases("{f20}", "{fs4}", RISCV::F20_F)
10033                         .Cases("{f21}", "{fs5}", RISCV::F21_F)
10034                         .Cases("{f22}", "{fs6}", RISCV::F22_F)
10035                         .Cases("{f23}", "{fs7}", RISCV::F23_F)
10036                         .Cases("{f24}", "{fs8}", RISCV::F24_F)
10037                         .Cases("{f25}", "{fs9}", RISCV::F25_F)
10038                         .Cases("{f26}", "{fs10}", RISCV::F26_F)
10039                         .Cases("{f27}", "{fs11}", RISCV::F27_F)
10040                         .Cases("{f28}", "{ft8}", RISCV::F28_F)
10041                         .Cases("{f29}", "{ft9}", RISCV::F29_F)
10042                         .Cases("{f30}", "{ft10}", RISCV::F30_F)
10043                         .Cases("{f31}", "{ft11}", RISCV::F31_F)
10044                         .Default(RISCV::NoRegister);
10045     if (FReg != RISCV::NoRegister) {
10046       assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
10047       if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
10048         unsigned RegNo = FReg - RISCV::F0_F;
10049         unsigned DReg = RISCV::F0_D + RegNo;
10050         return std::make_pair(DReg, &RISCV::FPR64RegClass);
10051       }
10052       if (VT == MVT::f32 || VT == MVT::Other)
10053         return std::make_pair(FReg, &RISCV::FPR32RegClass);
10054       if (Subtarget.hasStdExtZfh() && VT == MVT::f16) {
10055         unsigned RegNo = FReg - RISCV::F0_F;
10056         unsigned HReg = RISCV::F0_H + RegNo;
10057         return std::make_pair(HReg, &RISCV::FPR16RegClass);
10058       }
10059     }
10060   }
10061 
10062   if (Subtarget.hasVInstructions()) {
10063     Register VReg = StringSwitch<Register>(Constraint.lower())
10064                         .Case("{v0}", RISCV::V0)
10065                         .Case("{v1}", RISCV::V1)
10066                         .Case("{v2}", RISCV::V2)
10067                         .Case("{v3}", RISCV::V3)
10068                         .Case("{v4}", RISCV::V4)
10069                         .Case("{v5}", RISCV::V5)
10070                         .Case("{v6}", RISCV::V6)
10071                         .Case("{v7}", RISCV::V7)
10072                         .Case("{v8}", RISCV::V8)
10073                         .Case("{v9}", RISCV::V9)
10074                         .Case("{v10}", RISCV::V10)
10075                         .Case("{v11}", RISCV::V11)
10076                         .Case("{v12}", RISCV::V12)
10077                         .Case("{v13}", RISCV::V13)
10078                         .Case("{v14}", RISCV::V14)
10079                         .Case("{v15}", RISCV::V15)
10080                         .Case("{v16}", RISCV::V16)
10081                         .Case("{v17}", RISCV::V17)
10082                         .Case("{v18}", RISCV::V18)
10083                         .Case("{v19}", RISCV::V19)
10084                         .Case("{v20}", RISCV::V20)
10085                         .Case("{v21}", RISCV::V21)
10086                         .Case("{v22}", RISCV::V22)
10087                         .Case("{v23}", RISCV::V23)
10088                         .Case("{v24}", RISCV::V24)
10089                         .Case("{v25}", RISCV::V25)
10090                         .Case("{v26}", RISCV::V26)
10091                         .Case("{v27}", RISCV::V27)
10092                         .Case("{v28}", RISCV::V28)
10093                         .Case("{v29}", RISCV::V29)
10094                         .Case("{v30}", RISCV::V30)
10095                         .Case("{v31}", RISCV::V31)
10096                         .Default(RISCV::NoRegister);
10097     if (VReg != RISCV::NoRegister) {
10098       if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
10099         return std::make_pair(VReg, &RISCV::VMRegClass);
10100       if (TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.SimpleTy))
10101         return std::make_pair(VReg, &RISCV::VRRegClass);
10102       for (const auto *RC :
10103            {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10104         if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) {
10105           VReg = TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
10106           return std::make_pair(VReg, RC);
10107         }
10108       }
10109     }
10110   }
10111 
10112   return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
10113 }
10114 
10115 unsigned
10116 RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
10117   // Currently only support length 1 constraints.
10118   if (ConstraintCode.size() == 1) {
10119     switch (ConstraintCode[0]) {
10120     case 'A':
10121       return InlineAsm::Constraint_A;
10122     default:
10123       break;
10124     }
10125   }
10126 
10127   return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
10128 }
10129 
10130 void RISCVTargetLowering::LowerAsmOperandForConstraint(
10131     SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
10132     SelectionDAG &DAG) const {
10133   // Currently only support length 1 constraints.
10134   if (Constraint.length() == 1) {
10135     switch (Constraint[0]) {
10136     case 'I':
10137       // Validate & create a 12-bit signed immediate operand.
10138       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10139         uint64_t CVal = C->getSExtValue();
10140         if (isInt<12>(CVal))
10141           Ops.push_back(
10142               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10143       }
10144       return;
10145     case 'J':
10146       // Validate & create an integer zero operand.
10147       if (auto *C = dyn_cast<ConstantSDNode>(Op))
10148         if (C->getZExtValue() == 0)
10149           Ops.push_back(
10150               DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
10151       return;
10152     case 'K':
10153       // Validate & create a 5-bit unsigned immediate operand.
10154       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10155         uint64_t CVal = C->getZExtValue();
10156         if (isUInt<5>(CVal))
10157           Ops.push_back(
10158               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10159       }
10160       return;
10161     case 'S':
10162       if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
10163         Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
10164                                                  GA->getValueType(0)));
10165       } else if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
10166         Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(),
10167                                                 BA->getValueType(0)));
10168       }
10169       return;
10170     default:
10171       break;
10172     }
10173   }
10174   TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
10175 }
10176 
10177 Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
10178                                                    Instruction *Inst,
10179                                                    AtomicOrdering Ord) const {
10180   if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
10181     return Builder.CreateFence(Ord);
10182   if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
10183     return Builder.CreateFence(AtomicOrdering::Release);
10184   return nullptr;
10185 }
10186 
10187 Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
10188                                                     Instruction *Inst,
10189                                                     AtomicOrdering Ord) const {
10190   if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
10191     return Builder.CreateFence(AtomicOrdering::Acquire);
10192   return nullptr;
10193 }
10194 
10195 TargetLowering::AtomicExpansionKind
10196 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
10197   // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
10198   // point operations can't be used in an lr/sc sequence without breaking the
10199   // forward-progress guarantee.
10200   if (AI->isFloatingPointOperation())
10201     return AtomicExpansionKind::CmpXChg;
10202 
10203   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
10204   if (Size == 8 || Size == 16)
10205     return AtomicExpansionKind::MaskedIntrinsic;
10206   return AtomicExpansionKind::None;
10207 }
10208 
10209 static Intrinsic::ID
10210 getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
10211   if (XLen == 32) {
10212     switch (BinOp) {
10213     default:
10214       llvm_unreachable("Unexpected AtomicRMW BinOp");
10215     case AtomicRMWInst::Xchg:
10216       return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
10217     case AtomicRMWInst::Add:
10218       return Intrinsic::riscv_masked_atomicrmw_add_i32;
10219     case AtomicRMWInst::Sub:
10220       return Intrinsic::riscv_masked_atomicrmw_sub_i32;
10221     case AtomicRMWInst::Nand:
10222       return Intrinsic::riscv_masked_atomicrmw_nand_i32;
10223     case AtomicRMWInst::Max:
10224       return Intrinsic::riscv_masked_atomicrmw_max_i32;
10225     case AtomicRMWInst::Min:
10226       return Intrinsic::riscv_masked_atomicrmw_min_i32;
10227     case AtomicRMWInst::UMax:
10228       return Intrinsic::riscv_masked_atomicrmw_umax_i32;
10229     case AtomicRMWInst::UMin:
10230       return Intrinsic::riscv_masked_atomicrmw_umin_i32;
10231     }
10232   }
10233 
10234   if (XLen == 64) {
10235     switch (BinOp) {
10236     default:
10237       llvm_unreachable("Unexpected AtomicRMW BinOp");
10238     case AtomicRMWInst::Xchg:
10239       return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
10240     case AtomicRMWInst::Add:
10241       return Intrinsic::riscv_masked_atomicrmw_add_i64;
10242     case AtomicRMWInst::Sub:
10243       return Intrinsic::riscv_masked_atomicrmw_sub_i64;
10244     case AtomicRMWInst::Nand:
10245       return Intrinsic::riscv_masked_atomicrmw_nand_i64;
10246     case AtomicRMWInst::Max:
10247       return Intrinsic::riscv_masked_atomicrmw_max_i64;
10248     case AtomicRMWInst::Min:
10249       return Intrinsic::riscv_masked_atomicrmw_min_i64;
10250     case AtomicRMWInst::UMax:
10251       return Intrinsic::riscv_masked_atomicrmw_umax_i64;
10252     case AtomicRMWInst::UMin:
10253       return Intrinsic::riscv_masked_atomicrmw_umin_i64;
10254     }
10255   }
10256 
10257   llvm_unreachable("Unexpected XLen\n");
10258 }
10259 
10260 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
10261     IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
10262     Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
10263   unsigned XLen = Subtarget.getXLen();
10264   Value *Ordering =
10265       Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
10266   Type *Tys[] = {AlignedAddr->getType()};
10267   Function *LrwOpScwLoop = Intrinsic::getDeclaration(
10268       AI->getModule(),
10269       getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
10270 
10271   if (XLen == 64) {
10272     Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
10273     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10274     ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
10275   }
10276 
10277   Value *Result;
10278 
10279   // Must pass the shift amount needed to sign extend the loaded value prior
10280   // to performing a signed comparison for min/max. ShiftAmt is the number of
10281   // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
10282   // is the number of bits to left+right shift the value in order to
10283   // sign-extend.
10284   if (AI->getOperation() == AtomicRMWInst::Min ||
10285       AI->getOperation() == AtomicRMWInst::Max) {
10286     const DataLayout &DL = AI->getModule()->getDataLayout();
10287     unsigned ValWidth =
10288         DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
10289     Value *SextShamt =
10290         Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
10291     Result = Builder.CreateCall(LrwOpScwLoop,
10292                                 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
10293   } else {
10294     Result =
10295         Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
10296   }
10297 
10298   if (XLen == 64)
10299     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10300   return Result;
10301 }
10302 
10303 TargetLowering::AtomicExpansionKind
10304 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
10305     AtomicCmpXchgInst *CI) const {
10306   unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
10307   if (Size == 8 || Size == 16)
10308     return AtomicExpansionKind::MaskedIntrinsic;
10309   return AtomicExpansionKind::None;
10310 }
10311 
10312 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
10313     IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
10314     Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
10315   unsigned XLen = Subtarget.getXLen();
10316   Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
10317   Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
10318   if (XLen == 64) {
10319     CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
10320     NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
10321     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10322     CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
10323   }
10324   Type *Tys[] = {AlignedAddr->getType()};
10325   Function *MaskedCmpXchg =
10326       Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
10327   Value *Result = Builder.CreateCall(
10328       MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
10329   if (XLen == 64)
10330     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10331   return Result;
10332 }
10333 
10334 bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
10335   return false;
10336 }
10337 
10338 bool RISCVTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
10339                                                EVT VT) const {
10340   if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
10341     return false;
10342 
10343   switch (FPVT.getSimpleVT().SimpleTy) {
10344   case MVT::f16:
10345     return Subtarget.hasStdExtZfh();
10346   case MVT::f32:
10347     return Subtarget.hasStdExtF();
10348   case MVT::f64:
10349     return Subtarget.hasStdExtD();
10350   default:
10351     return false;
10352   }
10353 }
10354 
10355 unsigned RISCVTargetLowering::getJumpTableEncoding() const {
10356   // If we are using the small code model, we can reduce size of jump table
10357   // entry to 4 bytes.
10358   if (Subtarget.is64Bit() && !isPositionIndependent() &&
10359       getTargetMachine().getCodeModel() == CodeModel::Small) {
10360     return MachineJumpTableInfo::EK_Custom32;
10361   }
10362   return TargetLowering::getJumpTableEncoding();
10363 }
10364 
10365 const MCExpr *RISCVTargetLowering::LowerCustomJumpTableEntry(
10366     const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
10367     unsigned uid, MCContext &Ctx) const {
10368   assert(Subtarget.is64Bit() && !isPositionIndependent() &&
10369          getTargetMachine().getCodeModel() == CodeModel::Small);
10370   return MCSymbolRefExpr::create(MBB->getSymbol(), Ctx);
10371 }
10372 
10373 bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
10374                                                      EVT VT) const {
10375   VT = VT.getScalarType();
10376 
10377   if (!VT.isSimple())
10378     return false;
10379 
10380   switch (VT.getSimpleVT().SimpleTy) {
10381   case MVT::f16:
10382     return Subtarget.hasStdExtZfh();
10383   case MVT::f32:
10384     return Subtarget.hasStdExtF();
10385   case MVT::f64:
10386     return Subtarget.hasStdExtD();
10387   default:
10388     break;
10389   }
10390 
10391   return false;
10392 }
10393 
10394 Register RISCVTargetLowering::getExceptionPointerRegister(
10395     const Constant *PersonalityFn) const {
10396   return RISCV::X10;
10397 }
10398 
10399 Register RISCVTargetLowering::getExceptionSelectorRegister(
10400     const Constant *PersonalityFn) const {
10401   return RISCV::X11;
10402 }
10403 
10404 bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
10405   // Return false to suppress the unnecessary extensions if the LibCall
10406   // arguments or return value is f32 type for LP64 ABI.
10407   RISCVABI::ABI ABI = Subtarget.getTargetABI();
10408   if (ABI == RISCVABI::ABI_LP64 && (Type == MVT::f32))
10409     return false;
10410 
10411   return true;
10412 }
10413 
10414 bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
10415   if (Subtarget.is64Bit() && Type == MVT::i32)
10416     return true;
10417 
10418   return IsSigned;
10419 }
10420 
10421 bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
10422                                                  SDValue C) const {
10423   // Check integral scalar types.
10424   if (VT.isScalarInteger()) {
10425     // Omit the optimization if the sub target has the M extension and the data
10426     // size exceeds XLen.
10427     if (Subtarget.hasStdExtM() && VT.getSizeInBits() > Subtarget.getXLen())
10428       return false;
10429     if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
10430       // Break the MUL to a SLLI and an ADD/SUB.
10431       const APInt &Imm = ConstNode->getAPIntValue();
10432       if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
10433           (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
10434         return true;
10435       // Optimize the MUL to (SH*ADD x, (SLLI x, bits)) if Imm is not simm12.
10436       if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
10437           ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
10438            (Imm - 8).isPowerOf2()))
10439         return true;
10440       // Omit the following optimization if the sub target has the M extension
10441       // and the data size >= XLen.
10442       if (Subtarget.hasStdExtM() && VT.getSizeInBits() >= Subtarget.getXLen())
10443         return false;
10444       // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
10445       // a pair of LUI/ADDI.
10446       if (!Imm.isSignedIntN(12) && Imm.countTrailingZeros() < 12) {
10447         APInt ImmS = Imm.ashr(Imm.countTrailingZeros());
10448         if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
10449             (1 - ImmS).isPowerOf2())
10450         return true;
10451       }
10452     }
10453   }
10454 
10455   return false;
10456 }
10457 
10458 bool RISCVTargetLowering::isMulAddWithConstProfitable(
10459     const SDValue &AddNode, const SDValue &ConstNode) const {
10460   // Let the DAGCombiner decide for vectors.
10461   EVT VT = AddNode.getValueType();
10462   if (VT.isVector())
10463     return true;
10464 
10465   // Let the DAGCombiner decide for larger types.
10466   if (VT.getScalarSizeInBits() > Subtarget.getXLen())
10467     return true;
10468 
10469   // It is worse if c1 is simm12 while c1*c2 is not.
10470   ConstantSDNode *C1Node = cast<ConstantSDNode>(AddNode.getOperand(1));
10471   ConstantSDNode *C2Node = cast<ConstantSDNode>(ConstNode);
10472   const APInt &C1 = C1Node->getAPIntValue();
10473   const APInt &C2 = C2Node->getAPIntValue();
10474   if (C1.isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
10475     return false;
10476 
10477   // Default to true and let the DAGCombiner decide.
10478   return true;
10479 }
10480 
10481 bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
10482     EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
10483     bool *Fast) const {
10484   if (!VT.isVector())
10485     return false;
10486 
10487   EVT ElemVT = VT.getVectorElementType();
10488   if (Alignment >= ElemVT.getStoreSize()) {
10489     if (Fast)
10490       *Fast = true;
10491     return true;
10492   }
10493 
10494   return false;
10495 }
10496 
10497 bool RISCVTargetLowering::splitValueIntoRegisterParts(
10498     SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
10499     unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {
10500   bool IsABIRegCopy = CC.hasValue();
10501   EVT ValueVT = Val.getValueType();
10502   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
10503     // Cast the f16 to i16, extend to i32, pad with ones to make a float nan,
10504     // and cast to f32.
10505     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
10506     Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
10507     Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
10508                       DAG.getConstant(0xFFFF0000, DL, MVT::i32));
10509     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
10510     Parts[0] = Val;
10511     return true;
10512   }
10513 
10514   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
10515     LLVMContext &Context = *DAG.getContext();
10516     EVT ValueEltVT = ValueVT.getVectorElementType();
10517     EVT PartEltVT = PartVT.getVectorElementType();
10518     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
10519     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
10520     if (PartVTBitSize % ValueVTBitSize == 0) {
10521       assert(PartVTBitSize >= ValueVTBitSize);
10522       // If the element types are different, bitcast to the same element type of
10523       // PartVT first.
10524       // Give an example here, we want copy a <vscale x 1 x i8> value to
10525       // <vscale x 4 x i16>.
10526       // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert
10527       // subvector, then we can bitcast to <vscale x 4 x i16>.
10528       if (ValueEltVT != PartEltVT) {
10529         if (PartVTBitSize > ValueVTBitSize) {
10530           unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
10531           assert(Count != 0 && "The number of element should not be zero.");
10532           EVT SameEltTypeVT =
10533               EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
10534           Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT,
10535                             DAG.getUNDEF(SameEltTypeVT), Val,
10536                             DAG.getVectorIdxConstant(0, DL));
10537         }
10538         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
10539       } else {
10540         Val =
10541             DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
10542                         Val, DAG.getVectorIdxConstant(0, DL));
10543       }
10544       Parts[0] = Val;
10545       return true;
10546     }
10547   }
10548   return false;
10549 }
10550 
10551 SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
10552     SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
10553     MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {
10554   bool IsABIRegCopy = CC.hasValue();
10555   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
10556     SDValue Val = Parts[0];
10557 
10558     // Cast the f32 to i32, truncate to i16, and cast back to f16.
10559     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
10560     Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
10561     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f16, Val);
10562     return Val;
10563   }
10564 
10565   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
10566     LLVMContext &Context = *DAG.getContext();
10567     SDValue Val = Parts[0];
10568     EVT ValueEltVT = ValueVT.getVectorElementType();
10569     EVT PartEltVT = PartVT.getVectorElementType();
10570     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
10571     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
10572     if (PartVTBitSize % ValueVTBitSize == 0) {
10573       assert(PartVTBitSize >= ValueVTBitSize);
10574       EVT SameEltTypeVT = ValueVT;
10575       // If the element types are different, convert it to the same element type
10576       // of PartVT.
10577       // Give an example here, we want copy a <vscale x 1 x i8> value from
10578       // <vscale x 4 x i16>.
10579       // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first,
10580       // then we can extract <vscale x 1 x i8>.
10581       if (ValueEltVT != PartEltVT) {
10582         unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
10583         assert(Count != 0 && "The number of element should not be zero.");
10584         SameEltTypeVT =
10585             EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
10586         Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
10587       }
10588       Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
10589                         DAG.getVectorIdxConstant(0, DL));
10590       return Val;
10591     }
10592   }
10593   return SDValue();
10594 }
10595 
10596 SDValue
10597 RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
10598                                    SelectionDAG &DAG,
10599                                    SmallVectorImpl<SDNode *> &Created) const {
10600   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
10601   if (isIntDivCheap(N->getValueType(0), Attr))
10602     return SDValue(N, 0); // Lower SDIV as SDIV
10603 
10604   assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
10605          "Unexpected divisor!");
10606 
10607   // Conditional move is needed, so do the transformation iff Zbt is enabled.
10608   if (!Subtarget.hasStdExtZbt())
10609     return SDValue();
10610 
10611   // When |Divisor| >= 2 ^ 12, it isn't profitable to do such transformation.
10612   // Besides, more critical path instructions will be generated when dividing
10613   // by 2. So we keep using the original DAGs for these cases.
10614   unsigned Lg2 = Divisor.countTrailingZeros();
10615   if (Lg2 == 1 || Lg2 >= 12)
10616     return SDValue();
10617 
10618   // fold (sdiv X, pow2)
10619   EVT VT = N->getValueType(0);
10620   if (VT != MVT::i32 && !(Subtarget.is64Bit() && VT == MVT::i64))
10621     return SDValue();
10622 
10623   SDLoc DL(N);
10624   SDValue N0 = N->getOperand(0);
10625   SDValue Zero = DAG.getConstant(0, DL, VT);
10626   SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
10627 
10628   // Add (N0 < 0) ? Pow2 - 1 : 0;
10629   SDValue Cmp = DAG.getSetCC(DL, VT, N0, Zero, ISD::SETLT);
10630   SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
10631   SDValue Sel = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
10632 
10633   Created.push_back(Cmp.getNode());
10634   Created.push_back(Add.getNode());
10635   Created.push_back(Sel.getNode());
10636 
10637   // Divide by pow2.
10638   SDValue SRA =
10639       DAG.getNode(ISD::SRA, DL, VT, Sel, DAG.getConstant(Lg2, DL, VT));
10640 
10641   // If we're dividing by a positive value, we're done.  Otherwise, we must
10642   // negate the result.
10643   if (Divisor.isNonNegative())
10644     return SRA;
10645 
10646   Created.push_back(SRA.getNode());
10647   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
10648 }
10649 
10650 #define GET_REGISTER_MATCHER
10651 #include "RISCVGenAsmMatcher.inc"
10652 
10653 Register
10654 RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
10655                                        const MachineFunction &MF) const {
10656   Register Reg = MatchRegisterAltName(RegName);
10657   if (Reg == RISCV::NoRegister)
10658     Reg = MatchRegisterName(RegName);
10659   if (Reg == RISCV::NoRegister)
10660     report_fatal_error(
10661         Twine("Invalid register name \"" + StringRef(RegName) + "\"."));
10662   BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
10663   if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg))
10664     report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
10665                              StringRef(RegName) + "\"."));
10666   return Reg;
10667 }
10668 
10669 namespace llvm {
10670 namespace RISCVVIntrinsicsTable {
10671 
10672 #define GET_RISCVVIntrinsicsTable_IMPL
10673 #include "RISCVGenSearchableTables.inc"
10674 
10675 } // namespace RISCVVIntrinsicsTable
10676 
10677 } // namespace llvm
10678