1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that RISCV uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "RISCVISelLowering.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCV.h"
17 #include "RISCVMachineFunctionInfo.h"
18 #include "RISCVRegisterInfo.h"
19 #include "RISCVSubtarget.h"
20 #include "RISCVTargetMachine.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/MemoryLocation.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineJumpTableInfo.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
30 #include "llvm/CodeGen/ValueTypes.h"
31 #include "llvm/IR/DiagnosticInfo.h"
32 #include "llvm/IR/DiagnosticPrinter.h"
33 #include "llvm/IR/IRBuilder.h"
34 #include "llvm/IR/IntrinsicsRISCV.h"
35 #include "llvm/IR/PatternMatch.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/KnownBits.h"
39 #include "llvm/Support/MathExtras.h"
40 #include "llvm/Support/raw_ostream.h"
41 
42 using namespace llvm;
43 
44 #define DEBUG_TYPE "riscv-lower"
45 
46 STATISTIC(NumTailCalls, "Number of tail calls");
47 
48 RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
49                                          const RISCVSubtarget &STI)
50     : TargetLowering(TM), Subtarget(STI) {
51 
52   if (Subtarget.isRV32E())
53     report_fatal_error("Codegen not yet implemented for RV32E");
54 
55   RISCVABI::ABI ABI = Subtarget.getTargetABI();
56   assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
57 
58   if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
59       !Subtarget.hasStdExtF()) {
60     errs() << "Hard-float 'f' ABI can't be used for a target that "
61                 "doesn't support the F instruction set extension (ignoring "
62                           "target-abi)\n";
63     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
64   } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
65              !Subtarget.hasStdExtD()) {
66     errs() << "Hard-float 'd' ABI can't be used for a target that "
67               "doesn't support the D instruction set extension (ignoring "
68               "target-abi)\n";
69     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
70   }
71 
72   switch (ABI) {
73   default:
74     report_fatal_error("Don't know how to lower this ABI");
75   case RISCVABI::ABI_ILP32:
76   case RISCVABI::ABI_ILP32F:
77   case RISCVABI::ABI_ILP32D:
78   case RISCVABI::ABI_LP64:
79   case RISCVABI::ABI_LP64F:
80   case RISCVABI::ABI_LP64D:
81     break;
82   }
83 
84   MVT XLenVT = Subtarget.getXLenVT();
85 
86   // Set up the register classes.
87   addRegisterClass(XLenVT, &RISCV::GPRRegClass);
88 
89   if (Subtarget.hasStdExtZfh())
90     addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
91   if (Subtarget.hasStdExtF())
92     addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
93   if (Subtarget.hasStdExtD())
94     addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
95 
96   static const MVT::SimpleValueType BoolVecVTs[] = {
97       MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,
98       MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
99   static const MVT::SimpleValueType IntVecVTs[] = {
100       MVT::nxv1i8,  MVT::nxv2i8,   MVT::nxv4i8,   MVT::nxv8i8,  MVT::nxv16i8,
101       MVT::nxv32i8, MVT::nxv64i8,  MVT::nxv1i16,  MVT::nxv2i16, MVT::nxv4i16,
102       MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
103       MVT::nxv4i32, MVT::nxv8i32,  MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
104       MVT::nxv4i64, MVT::nxv8i64};
105   static const MVT::SimpleValueType F16VecVTs[] = {
106       MVT::nxv1f16, MVT::nxv2f16,  MVT::nxv4f16,
107       MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
108   static const MVT::SimpleValueType F32VecVTs[] = {
109       MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
110   static const MVT::SimpleValueType F64VecVTs[] = {
111       MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
112 
113   if (Subtarget.hasVInstructions()) {
114     auto addRegClassForRVV = [this](MVT VT) {
115       unsigned Size = VT.getSizeInBits().getKnownMinValue();
116       assert(Size <= 512 && isPowerOf2_32(Size));
117       const TargetRegisterClass *RC;
118       if (Size <= 64)
119         RC = &RISCV::VRRegClass;
120       else if (Size == 128)
121         RC = &RISCV::VRM2RegClass;
122       else if (Size == 256)
123         RC = &RISCV::VRM4RegClass;
124       else
125         RC = &RISCV::VRM8RegClass;
126 
127       addRegisterClass(VT, RC);
128     };
129 
130     for (MVT VT : BoolVecVTs)
131       addRegClassForRVV(VT);
132     for (MVT VT : IntVecVTs) {
133       if (VT.getVectorElementType() == MVT::i64 &&
134           !Subtarget.hasVInstructionsI64())
135         continue;
136       addRegClassForRVV(VT);
137     }
138 
139     if (Subtarget.hasVInstructionsF16())
140       for (MVT VT : F16VecVTs)
141         addRegClassForRVV(VT);
142 
143     if (Subtarget.hasVInstructionsF32())
144       for (MVT VT : F32VecVTs)
145         addRegClassForRVV(VT);
146 
147     if (Subtarget.hasVInstructionsF64())
148       for (MVT VT : F64VecVTs)
149         addRegClassForRVV(VT);
150 
151     if (Subtarget.useRVVForFixedLengthVectors()) {
152       auto addRegClassForFixedVectors = [this](MVT VT) {
153         MVT ContainerVT = getContainerForFixedLengthVector(VT);
154         unsigned RCID = getRegClassIDForVecVT(ContainerVT);
155         const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
156         addRegisterClass(VT, TRI.getRegClass(RCID));
157       };
158       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
159         if (useRVVForFixedLengthVectorVT(VT))
160           addRegClassForFixedVectors(VT);
161 
162       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
163         if (useRVVForFixedLengthVectorVT(VT))
164           addRegClassForFixedVectors(VT);
165     }
166   }
167 
168   // Compute derived properties from the register classes.
169   computeRegisterProperties(STI.getRegisterInfo());
170 
171   setStackPointerRegisterToSaveRestore(RISCV::X2);
172 
173   for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
174     setLoadExtAction(N, XLenVT, MVT::i1, Promote);
175 
176   // TODO: add all necessary setOperationAction calls.
177   setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
178 
179   setOperationAction(ISD::BR_JT, MVT::Other, Expand);
180   setOperationAction(ISD::BR_CC, XLenVT, Expand);
181   setOperationAction(ISD::BRCOND, MVT::Other, Custom);
182   setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
183 
184   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
185   setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
186 
187   setOperationAction(ISD::VASTART, MVT::Other, Custom);
188   setOperationAction(ISD::VAARG, MVT::Other, Expand);
189   setOperationAction(ISD::VACOPY, MVT::Other, Expand);
190   setOperationAction(ISD::VAEND, MVT::Other, Expand);
191 
192   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
193   if (!Subtarget.hasStdExtZbb()) {
194     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
195     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
196   }
197 
198   if (Subtarget.is64Bit()) {
199     setOperationAction(ISD::ADD, MVT::i32, Custom);
200     setOperationAction(ISD::SUB, MVT::i32, Custom);
201     setOperationAction(ISD::SHL, MVT::i32, Custom);
202     setOperationAction(ISD::SRA, MVT::i32, Custom);
203     setOperationAction(ISD::SRL, MVT::i32, Custom);
204 
205     setOperationAction(ISD::UADDO, MVT::i32, Custom);
206     setOperationAction(ISD::USUBO, MVT::i32, Custom);
207     setOperationAction(ISD::UADDSAT, MVT::i32, Custom);
208     setOperationAction(ISD::USUBSAT, MVT::i32, Custom);
209   } else {
210     setLibcallName(RTLIB::SHL_I128, nullptr);
211     setLibcallName(RTLIB::SRL_I128, nullptr);
212     setLibcallName(RTLIB::SRA_I128, nullptr);
213     setLibcallName(RTLIB::MUL_I128, nullptr);
214     setLibcallName(RTLIB::MULO_I64, nullptr);
215   }
216 
217   if (!Subtarget.hasStdExtM()) {
218     setOperationAction(ISD::MUL, XLenVT, Expand);
219     setOperationAction(ISD::MULHS, XLenVT, Expand);
220     setOperationAction(ISD::MULHU, XLenVT, Expand);
221     setOperationAction(ISD::SDIV, XLenVT, Expand);
222     setOperationAction(ISD::UDIV, XLenVT, Expand);
223     setOperationAction(ISD::SREM, XLenVT, Expand);
224     setOperationAction(ISD::UREM, XLenVT, Expand);
225   } else {
226     if (Subtarget.is64Bit()) {
227       setOperationAction(ISD::MUL, MVT::i32, Custom);
228       setOperationAction(ISD::MUL, MVT::i128, Custom);
229 
230       setOperationAction(ISD::SDIV, MVT::i8, Custom);
231       setOperationAction(ISD::UDIV, MVT::i8, Custom);
232       setOperationAction(ISD::UREM, MVT::i8, Custom);
233       setOperationAction(ISD::SDIV, MVT::i16, Custom);
234       setOperationAction(ISD::UDIV, MVT::i16, Custom);
235       setOperationAction(ISD::UREM, MVT::i16, Custom);
236       setOperationAction(ISD::SDIV, MVT::i32, Custom);
237       setOperationAction(ISD::UDIV, MVT::i32, Custom);
238       setOperationAction(ISD::UREM, MVT::i32, Custom);
239     } else {
240       setOperationAction(ISD::MUL, MVT::i64, Custom);
241     }
242   }
243 
244   setOperationAction(ISD::SDIVREM, XLenVT, Expand);
245   setOperationAction(ISD::UDIVREM, XLenVT, Expand);
246   setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
247   setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
248 
249   setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
250   setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
251   setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
252 
253   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
254     if (Subtarget.is64Bit()) {
255       setOperationAction(ISD::ROTL, MVT::i32, Custom);
256       setOperationAction(ISD::ROTR, MVT::i32, Custom);
257     }
258   } else {
259     setOperationAction(ISD::ROTL, XLenVT, Expand);
260     setOperationAction(ISD::ROTR, XLenVT, Expand);
261   }
262 
263   if (Subtarget.hasStdExtZbp()) {
264     // Custom lower bswap/bitreverse so we can convert them to GREVI to enable
265     // more combining.
266     setOperationAction(ISD::BITREVERSE, XLenVT,   Custom);
267     setOperationAction(ISD::BSWAP,      XLenVT,   Custom);
268     setOperationAction(ISD::BITREVERSE, MVT::i8,  Custom);
269     // BSWAP i8 doesn't exist.
270     setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
271     setOperationAction(ISD::BSWAP,      MVT::i16, Custom);
272 
273     if (Subtarget.is64Bit()) {
274       setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
275       setOperationAction(ISD::BSWAP,      MVT::i32, Custom);
276     }
277   } else {
278     // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
279     // pattern match it directly in isel.
280     setOperationAction(ISD::BSWAP, XLenVT,
281                        Subtarget.hasStdExtZbb() ? Legal : Expand);
282   }
283 
284   if (Subtarget.hasStdExtZbb()) {
285     setOperationAction(ISD::SMIN, XLenVT, Legal);
286     setOperationAction(ISD::SMAX, XLenVT, Legal);
287     setOperationAction(ISD::UMIN, XLenVT, Legal);
288     setOperationAction(ISD::UMAX, XLenVT, Legal);
289 
290     if (Subtarget.is64Bit()) {
291       setOperationAction(ISD::CTTZ, MVT::i32, Custom);
292       setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom);
293       setOperationAction(ISD::CTLZ, MVT::i32, Custom);
294       setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom);
295     }
296   } else {
297     setOperationAction(ISD::CTTZ, XLenVT, Expand);
298     setOperationAction(ISD::CTLZ, XLenVT, Expand);
299     setOperationAction(ISD::CTPOP, XLenVT, Expand);
300   }
301 
302   if (Subtarget.hasStdExtZbt()) {
303     setOperationAction(ISD::FSHL, XLenVT, Custom);
304     setOperationAction(ISD::FSHR, XLenVT, Custom);
305     setOperationAction(ISD::SELECT, XLenVT, Legal);
306 
307     if (Subtarget.is64Bit()) {
308       setOperationAction(ISD::FSHL, MVT::i32, Custom);
309       setOperationAction(ISD::FSHR, MVT::i32, Custom);
310     }
311   } else {
312     setOperationAction(ISD::SELECT, XLenVT, Custom);
313   }
314 
315   static const ISD::CondCode FPCCToExpand[] = {
316       ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
317       ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
318       ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};
319 
320   static const ISD::NodeType FPOpToExpand[] = {
321       ISD::FSIN, ISD::FCOS,       ISD::FSINCOS,   ISD::FPOW,
322       ISD::FREM, ISD::FP16_TO_FP, ISD::FP_TO_FP16};
323 
324   if (Subtarget.hasStdExtZfh())
325     setOperationAction(ISD::BITCAST, MVT::i16, Custom);
326 
327   if (Subtarget.hasStdExtZfh()) {
328     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
329     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
330     setOperationAction(ISD::LRINT, MVT::f16, Legal);
331     setOperationAction(ISD::LLRINT, MVT::f16, Legal);
332     setOperationAction(ISD::LROUND, MVT::f16, Legal);
333     setOperationAction(ISD::LLROUND, MVT::f16, Legal);
334     setOperationAction(ISD::STRICT_LRINT, MVT::f16, Legal);
335     setOperationAction(ISD::STRICT_LLRINT, MVT::f16, Legal);
336     setOperationAction(ISD::STRICT_LROUND, MVT::f16, Legal);
337     setOperationAction(ISD::STRICT_LLROUND, MVT::f16, Legal);
338     setOperationAction(ISD::STRICT_FADD, MVT::f16, Legal);
339     setOperationAction(ISD::STRICT_FMA, MVT::f16, Legal);
340     setOperationAction(ISD::STRICT_FSUB, MVT::f16, Legal);
341     setOperationAction(ISD::STRICT_FMUL, MVT::f16, Legal);
342     setOperationAction(ISD::STRICT_FDIV, MVT::f16, Legal);
343     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
344     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
345     setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Legal);
346     setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Legal);
347     setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Legal);
348     for (auto CC : FPCCToExpand)
349       setCondCodeAction(CC, MVT::f16, Expand);
350     setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
351     setOperationAction(ISD::SELECT, MVT::f16, Custom);
352     setOperationAction(ISD::BR_CC, MVT::f16, Expand);
353 
354     setOperationAction(ISD::FREM,       MVT::f16, Promote);
355     setOperationAction(ISD::FCEIL,      MVT::f16, Promote);
356     setOperationAction(ISD::FFLOOR,     MVT::f16, Promote);
357     setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote);
358     setOperationAction(ISD::FRINT,      MVT::f16, Promote);
359     setOperationAction(ISD::FROUND,     MVT::f16, Promote);
360     setOperationAction(ISD::FROUNDEVEN, MVT::f16, Promote);
361     setOperationAction(ISD::FTRUNC,     MVT::f16, Promote);
362     setOperationAction(ISD::FPOW,       MVT::f16, Promote);
363     setOperationAction(ISD::FPOWI,      MVT::f16, Promote);
364     setOperationAction(ISD::FCOS,       MVT::f16, Promote);
365     setOperationAction(ISD::FSIN,       MVT::f16, Promote);
366     setOperationAction(ISD::FSINCOS,    MVT::f16, Promote);
367     setOperationAction(ISD::FEXP,       MVT::f16, Promote);
368     setOperationAction(ISD::FEXP2,      MVT::f16, Promote);
369     setOperationAction(ISD::FLOG,       MVT::f16, Promote);
370     setOperationAction(ISD::FLOG2,      MVT::f16, Promote);
371     setOperationAction(ISD::FLOG10,     MVT::f16, Promote);
372 
373     // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
374     // complete support for all operations in LegalizeDAG.
375 
376     // We need to custom promote this.
377     if (Subtarget.is64Bit())
378       setOperationAction(ISD::FPOWI, MVT::i32, Custom);
379   }
380 
381   if (Subtarget.hasStdExtF()) {
382     setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
383     setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
384     setOperationAction(ISD::LRINT, MVT::f32, Legal);
385     setOperationAction(ISD::LLRINT, MVT::f32, Legal);
386     setOperationAction(ISD::LROUND, MVT::f32, Legal);
387     setOperationAction(ISD::LLROUND, MVT::f32, Legal);
388     setOperationAction(ISD::STRICT_LRINT, MVT::f32, Legal);
389     setOperationAction(ISD::STRICT_LLRINT, MVT::f32, Legal);
390     setOperationAction(ISD::STRICT_LROUND, MVT::f32, Legal);
391     setOperationAction(ISD::STRICT_LLROUND, MVT::f32, Legal);
392     setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
393     setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal);
394     setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
395     setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
396     setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
397     setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
398     setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);
399     setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
400     for (auto CC : FPCCToExpand)
401       setCondCodeAction(CC, MVT::f32, Expand);
402     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
403     setOperationAction(ISD::SELECT, MVT::f32, Custom);
404     setOperationAction(ISD::BR_CC, MVT::f32, Expand);
405     for (auto Op : FPOpToExpand)
406       setOperationAction(Op, MVT::f32, Expand);
407     setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
408     setTruncStoreAction(MVT::f32, MVT::f16, Expand);
409   }
410 
411   if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
412     setOperationAction(ISD::BITCAST, MVT::i32, Custom);
413 
414   if (Subtarget.hasStdExtD()) {
415     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
416     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
417     setOperationAction(ISD::LRINT, MVT::f64, Legal);
418     setOperationAction(ISD::LLRINT, MVT::f64, Legal);
419     setOperationAction(ISD::LROUND, MVT::f64, Legal);
420     setOperationAction(ISD::LLROUND, MVT::f64, Legal);
421     setOperationAction(ISD::STRICT_LRINT, MVT::f64, Legal);
422     setOperationAction(ISD::STRICT_LLRINT, MVT::f64, Legal);
423     setOperationAction(ISD::STRICT_LROUND, MVT::f64, Legal);
424     setOperationAction(ISD::STRICT_LLROUND, MVT::f64, Legal);
425     setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal);
426     setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
427     setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
428     setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
429     setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
430     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
431     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
432     setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
433     setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal);
434     setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal);
435     for (auto CC : FPCCToExpand)
436       setCondCodeAction(CC, MVT::f64, Expand);
437     setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
438     setOperationAction(ISD::SELECT, MVT::f64, Custom);
439     setOperationAction(ISD::BR_CC, MVT::f64, Expand);
440     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
441     setTruncStoreAction(MVT::f64, MVT::f32, Expand);
442     for (auto Op : FPOpToExpand)
443       setOperationAction(Op, MVT::f64, Expand);
444     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
445     setTruncStoreAction(MVT::f64, MVT::f16, Expand);
446   }
447 
448   if (Subtarget.is64Bit()) {
449     setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
450     setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
451     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
452     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
453   }
454 
455   if (Subtarget.hasStdExtF()) {
456     setOperationAction(ISD::FP_TO_UINT_SAT, XLenVT, Custom);
457     setOperationAction(ISD::FP_TO_SINT_SAT, XLenVT, Custom);
458 
459     setOperationAction(ISD::STRICT_FP_TO_UINT, XLenVT, Legal);
460     setOperationAction(ISD::STRICT_FP_TO_SINT, XLenVT, Legal);
461     setOperationAction(ISD::STRICT_UINT_TO_FP, XLenVT, Legal);
462     setOperationAction(ISD::STRICT_SINT_TO_FP, XLenVT, Legal);
463 
464     setOperationAction(ISD::FLT_ROUNDS_, XLenVT, Custom);
465     setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
466   }
467 
468   setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
469   setOperationAction(ISD::BlockAddress, XLenVT, Custom);
470   setOperationAction(ISD::ConstantPool, XLenVT, Custom);
471   setOperationAction(ISD::JumpTable, XLenVT, Custom);
472 
473   setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
474 
475   // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
476   // Unfortunately this can't be determined just from the ISA naming string.
477   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
478                      Subtarget.is64Bit() ? Legal : Custom);
479 
480   setOperationAction(ISD::TRAP, MVT::Other, Legal);
481   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
482   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
483   if (Subtarget.is64Bit())
484     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
485 
486   if (Subtarget.hasStdExtA()) {
487     setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
488     setMinCmpXchgSizeInBits(32);
489   } else {
490     setMaxAtomicSizeInBitsSupported(0);
491   }
492 
493   setBooleanContents(ZeroOrOneBooleanContent);
494 
495   if (Subtarget.hasVInstructions()) {
496     setBooleanVectorContents(ZeroOrOneBooleanContent);
497 
498     setOperationAction(ISD::VSCALE, XLenVT, Custom);
499 
500     // RVV intrinsics may have illegal operands.
501     // We also need to custom legalize vmv.x.s.
502     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
503     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
504     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
505     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom);
506     if (Subtarget.is64Bit()) {
507       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);
508     } else {
509       setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
510       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
511     }
512 
513     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
514     setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
515 
516     static const unsigned IntegerVPOps[] = {
517         ISD::VP_ADD,         ISD::VP_SUB,         ISD::VP_MUL,
518         ISD::VP_SDIV,        ISD::VP_UDIV,        ISD::VP_SREM,
519         ISD::VP_UREM,        ISD::VP_AND,         ISD::VP_OR,
520         ISD::VP_XOR,         ISD::VP_ASHR,        ISD::VP_LSHR,
521         ISD::VP_SHL,         ISD::VP_REDUCE_ADD,  ISD::VP_REDUCE_AND,
522         ISD::VP_REDUCE_OR,   ISD::VP_REDUCE_XOR,  ISD::VP_REDUCE_SMAX,
523         ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
524         ISD::VP_MERGE,       ISD::VP_SELECT};
525 
526     static const unsigned FloatingPointVPOps[] = {
527         ISD::VP_FADD,        ISD::VP_FSUB,        ISD::VP_FMUL,
528         ISD::VP_FDIV,        ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
529         ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
530         ISD::VP_SELECT};
531 
532     if (!Subtarget.is64Bit()) {
533       // We must custom-lower certain vXi64 operations on RV32 due to the vector
534       // element type being illegal.
535       setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::i64, Custom);
536       setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::i64, Custom);
537 
538       setOperationAction(ISD::VECREDUCE_ADD, MVT::i64, Custom);
539       setOperationAction(ISD::VECREDUCE_AND, MVT::i64, Custom);
540       setOperationAction(ISD::VECREDUCE_OR, MVT::i64, Custom);
541       setOperationAction(ISD::VECREDUCE_XOR, MVT::i64, Custom);
542       setOperationAction(ISD::VECREDUCE_SMAX, MVT::i64, Custom);
543       setOperationAction(ISD::VECREDUCE_SMIN, MVT::i64, Custom);
544       setOperationAction(ISD::VECREDUCE_UMAX, MVT::i64, Custom);
545       setOperationAction(ISD::VECREDUCE_UMIN, MVT::i64, Custom);
546 
547       setOperationAction(ISD::VP_REDUCE_ADD, MVT::i64, Custom);
548       setOperationAction(ISD::VP_REDUCE_AND, MVT::i64, Custom);
549       setOperationAction(ISD::VP_REDUCE_OR, MVT::i64, Custom);
550       setOperationAction(ISD::VP_REDUCE_XOR, MVT::i64, Custom);
551       setOperationAction(ISD::VP_REDUCE_SMAX, MVT::i64, Custom);
552       setOperationAction(ISD::VP_REDUCE_SMIN, MVT::i64, Custom);
553       setOperationAction(ISD::VP_REDUCE_UMAX, MVT::i64, Custom);
554       setOperationAction(ISD::VP_REDUCE_UMIN, MVT::i64, Custom);
555     }
556 
557     for (MVT VT : BoolVecVTs) {
558       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
559 
560       // Mask VTs are custom-expanded into a series of standard nodes
561       setOperationAction(ISD::TRUNCATE, VT, Custom);
562       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
563       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
564       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
565 
566       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
567       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
568 
569       setOperationAction(ISD::SELECT, VT, Custom);
570       setOperationAction(ISD::SELECT_CC, VT, Expand);
571       setOperationAction(ISD::VSELECT, VT, Expand);
572       setOperationAction(ISD::VP_SELECT, VT, Expand);
573 
574       setOperationAction(ISD::VP_AND, VT, Custom);
575       setOperationAction(ISD::VP_OR, VT, Custom);
576       setOperationAction(ISD::VP_XOR, VT, Custom);
577 
578       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
579       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
580       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
581 
582       setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
583       setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
584       setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
585 
586       // RVV has native int->float & float->int conversions where the
587       // element type sizes are within one power-of-two of each other. Any
588       // wider distances between type sizes have to be lowered as sequences
589       // which progressively narrow the gap in stages.
590       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
591       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
592       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
593       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
594 
595       // Expand all extending loads to types larger than this, and truncating
596       // stores from types larger than this.
597       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
598         setTruncStoreAction(OtherVT, VT, Expand);
599         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
600         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
601         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
602       }
603     }
604 
605     for (MVT VT : IntVecVTs) {
606       if (VT.getVectorElementType() == MVT::i64 &&
607           !Subtarget.hasVInstructionsI64())
608         continue;
609 
610       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
611       setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
612 
613       // Vectors implement MULHS/MULHU.
614       setOperationAction(ISD::SMUL_LOHI, VT, Expand);
615       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
616 
617       // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
618       if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV()) {
619         setOperationAction(ISD::MULHU, VT, Expand);
620         setOperationAction(ISD::MULHS, VT, Expand);
621       }
622 
623       setOperationAction(ISD::SMIN, VT, Legal);
624       setOperationAction(ISD::SMAX, VT, Legal);
625       setOperationAction(ISD::UMIN, VT, Legal);
626       setOperationAction(ISD::UMAX, VT, Legal);
627 
628       setOperationAction(ISD::ROTL, VT, Expand);
629       setOperationAction(ISD::ROTR, VT, Expand);
630 
631       setOperationAction(ISD::CTTZ, VT, Expand);
632       setOperationAction(ISD::CTLZ, VT, Expand);
633       setOperationAction(ISD::CTPOP, VT, Expand);
634 
635       setOperationAction(ISD::BSWAP, VT, Expand);
636 
637       // Custom-lower extensions and truncations from/to mask types.
638       setOperationAction(ISD::ANY_EXTEND, VT, Custom);
639       setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
640       setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
641 
642       // RVV has native int->float & float->int conversions where the
643       // element type sizes are within one power-of-two of each other. Any
644       // wider distances between type sizes have to be lowered as sequences
645       // which progressively narrow the gap in stages.
646       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
647       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
648       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
649       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
650 
651       setOperationAction(ISD::SADDSAT, VT, Legal);
652       setOperationAction(ISD::UADDSAT, VT, Legal);
653       setOperationAction(ISD::SSUBSAT, VT, Legal);
654       setOperationAction(ISD::USUBSAT, VT, Legal);
655 
656       // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
657       // nodes which truncate by one power of two at a time.
658       setOperationAction(ISD::TRUNCATE, VT, Custom);
659 
660       // Custom-lower insert/extract operations to simplify patterns.
661       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
662       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
663 
664       // Custom-lower reduction operations to set up the corresponding custom
665       // nodes' operands.
666       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
667       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
668       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
669       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
670       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
671       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
672       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
673       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
674 
675       for (unsigned VPOpc : IntegerVPOps)
676         setOperationAction(VPOpc, VT, Custom);
677 
678       setOperationAction(ISD::LOAD, VT, Custom);
679       setOperationAction(ISD::STORE, VT, Custom);
680 
681       setOperationAction(ISD::MLOAD, VT, Custom);
682       setOperationAction(ISD::MSTORE, VT, Custom);
683       setOperationAction(ISD::MGATHER, VT, Custom);
684       setOperationAction(ISD::MSCATTER, VT, Custom);
685 
686       setOperationAction(ISD::VP_LOAD, VT, Custom);
687       setOperationAction(ISD::VP_STORE, VT, Custom);
688       setOperationAction(ISD::VP_GATHER, VT, Custom);
689       setOperationAction(ISD::VP_SCATTER, VT, Custom);
690 
691       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
692       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
693       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
694 
695       setOperationAction(ISD::SELECT, VT, Custom);
696       setOperationAction(ISD::SELECT_CC, VT, Expand);
697 
698       setOperationAction(ISD::STEP_VECTOR, VT, Custom);
699       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
700 
701       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
702         setTruncStoreAction(VT, OtherVT, Expand);
703         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
704         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
705         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
706       }
707 
708       // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
709       // type that can represent the value exactly.
710       if (VT.getVectorElementType() != MVT::i64) {
711         MVT FloatEltVT =
712             VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
713         EVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
714         if (isTypeLegal(FloatVT)) {
715           setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
716           setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
717         }
718       }
719     }
720 
721     // Expand various CCs to best match the RVV ISA, which natively supports UNE
722     // but no other unordered comparisons, and supports all ordered comparisons
723     // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
724     // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
725     // and we pattern-match those back to the "original", swapping operands once
726     // more. This way we catch both operations and both "vf" and "fv" forms with
727     // fewer patterns.
728     static const ISD::CondCode VFPCCToExpand[] = {
729         ISD::SETO,   ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
730         ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
731         ISD::SETGT,  ISD::SETOGT, ISD::SETGE,  ISD::SETOGE,
732     };
733 
734     // Sets common operation actions on RVV floating-point vector types.
735     const auto SetCommonVFPActions = [&](MVT VT) {
736       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
737       // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
738       // sizes are within one power-of-two of each other. Therefore conversions
739       // between vXf16 and vXf64 must be lowered as sequences which convert via
740       // vXf32.
741       setOperationAction(ISD::FP_ROUND, VT, Custom);
742       setOperationAction(ISD::FP_EXTEND, VT, Custom);
743       // Custom-lower insert/extract operations to simplify patterns.
744       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
745       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
746       // Expand various condition codes (explained above).
747       for (auto CC : VFPCCToExpand)
748         setCondCodeAction(CC, VT, Expand);
749 
750       setOperationAction(ISD::FMINNUM, VT, Legal);
751       setOperationAction(ISD::FMAXNUM, VT, Legal);
752 
753       setOperationAction(ISD::FTRUNC, VT, Custom);
754       setOperationAction(ISD::FCEIL, VT, Custom);
755       setOperationAction(ISD::FFLOOR, VT, Custom);
756 
757       setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
758       setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
759       setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
760       setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
761 
762       setOperationAction(ISD::FCOPYSIGN, VT, Legal);
763 
764       setOperationAction(ISD::LOAD, VT, Custom);
765       setOperationAction(ISD::STORE, VT, Custom);
766 
767       setOperationAction(ISD::MLOAD, VT, Custom);
768       setOperationAction(ISD::MSTORE, VT, Custom);
769       setOperationAction(ISD::MGATHER, VT, Custom);
770       setOperationAction(ISD::MSCATTER, VT, Custom);
771 
772       setOperationAction(ISD::VP_LOAD, VT, Custom);
773       setOperationAction(ISD::VP_STORE, VT, Custom);
774       setOperationAction(ISD::VP_GATHER, VT, Custom);
775       setOperationAction(ISD::VP_SCATTER, VT, Custom);
776 
777       setOperationAction(ISD::SELECT, VT, Custom);
778       setOperationAction(ISD::SELECT_CC, VT, Expand);
779 
780       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
781       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
782       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
783 
784       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
785 
786       for (unsigned VPOpc : FloatingPointVPOps)
787         setOperationAction(VPOpc, VT, Custom);
788     };
789 
790     // Sets common extload/truncstore actions on RVV floating-point vector
791     // types.
792     const auto SetCommonVFPExtLoadTruncStoreActions =
793         [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
794           for (auto SmallVT : SmallerVTs) {
795             setTruncStoreAction(VT, SmallVT, Expand);
796             setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
797           }
798         };
799 
800     if (Subtarget.hasVInstructionsF16())
801       for (MVT VT : F16VecVTs)
802         SetCommonVFPActions(VT);
803 
804     for (MVT VT : F32VecVTs) {
805       if (Subtarget.hasVInstructionsF32())
806         SetCommonVFPActions(VT);
807       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
808     }
809 
810     for (MVT VT : F64VecVTs) {
811       if (Subtarget.hasVInstructionsF64())
812         SetCommonVFPActions(VT);
813       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
814       SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
815     }
816 
817     if (Subtarget.useRVVForFixedLengthVectors()) {
818       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
819         if (!useRVVForFixedLengthVectorVT(VT))
820           continue;
821 
822         // By default everything must be expanded.
823         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
824           setOperationAction(Op, VT, Expand);
825         for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
826           setTruncStoreAction(VT, OtherVT, Expand);
827           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
828           setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
829           setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
830         }
831 
832         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
833         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
834         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
835 
836         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
837         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
838 
839         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
840         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
841 
842         setOperationAction(ISD::LOAD, VT, Custom);
843         setOperationAction(ISD::STORE, VT, Custom);
844 
845         setOperationAction(ISD::SETCC, VT, Custom);
846 
847         setOperationAction(ISD::SELECT, VT, Custom);
848 
849         setOperationAction(ISD::TRUNCATE, VT, Custom);
850 
851         setOperationAction(ISD::BITCAST, VT, Custom);
852 
853         setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
854         setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
855         setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
856 
857         setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
858         setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
859         setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
860 
861         setOperationAction(ISD::SINT_TO_FP, VT, Custom);
862         setOperationAction(ISD::UINT_TO_FP, VT, Custom);
863         setOperationAction(ISD::FP_TO_SINT, VT, Custom);
864         setOperationAction(ISD::FP_TO_UINT, VT, Custom);
865 
866         // Operations below are different for between masks and other vectors.
867         if (VT.getVectorElementType() == MVT::i1) {
868           setOperationAction(ISD::VP_AND, VT, Custom);
869           setOperationAction(ISD::VP_OR, VT, Custom);
870           setOperationAction(ISD::VP_XOR, VT, Custom);
871           setOperationAction(ISD::AND, VT, Custom);
872           setOperationAction(ISD::OR, VT, Custom);
873           setOperationAction(ISD::XOR, VT, Custom);
874           continue;
875         }
876 
877         // Use SPLAT_VECTOR to prevent type legalization from destroying the
878         // splats when type legalizing i64 scalar on RV32.
879         // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
880         // improvements first.
881         if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
882           setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
883           setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
884         }
885 
886         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
887         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
888 
889         setOperationAction(ISD::MLOAD, VT, Custom);
890         setOperationAction(ISD::MSTORE, VT, Custom);
891         setOperationAction(ISD::MGATHER, VT, Custom);
892         setOperationAction(ISD::MSCATTER, VT, Custom);
893 
894         setOperationAction(ISD::VP_LOAD, VT, Custom);
895         setOperationAction(ISD::VP_STORE, VT, Custom);
896         setOperationAction(ISD::VP_GATHER, VT, Custom);
897         setOperationAction(ISD::VP_SCATTER, VT, Custom);
898 
899         setOperationAction(ISD::ADD, VT, Custom);
900         setOperationAction(ISD::MUL, VT, Custom);
901         setOperationAction(ISD::SUB, VT, Custom);
902         setOperationAction(ISD::AND, VT, Custom);
903         setOperationAction(ISD::OR, VT, Custom);
904         setOperationAction(ISD::XOR, VT, Custom);
905         setOperationAction(ISD::SDIV, VT, Custom);
906         setOperationAction(ISD::SREM, VT, Custom);
907         setOperationAction(ISD::UDIV, VT, Custom);
908         setOperationAction(ISD::UREM, VT, Custom);
909         setOperationAction(ISD::SHL, VT, Custom);
910         setOperationAction(ISD::SRA, VT, Custom);
911         setOperationAction(ISD::SRL, VT, Custom);
912 
913         setOperationAction(ISD::SMIN, VT, Custom);
914         setOperationAction(ISD::SMAX, VT, Custom);
915         setOperationAction(ISD::UMIN, VT, Custom);
916         setOperationAction(ISD::UMAX, VT, Custom);
917         setOperationAction(ISD::ABS,  VT, Custom);
918 
919         // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
920         if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV()) {
921           setOperationAction(ISD::MULHS, VT, Custom);
922           setOperationAction(ISD::MULHU, VT, Custom);
923         }
924 
925         setOperationAction(ISD::SADDSAT, VT, Custom);
926         setOperationAction(ISD::UADDSAT, VT, Custom);
927         setOperationAction(ISD::SSUBSAT, VT, Custom);
928         setOperationAction(ISD::USUBSAT, VT, Custom);
929 
930         setOperationAction(ISD::VSELECT, VT, Custom);
931         setOperationAction(ISD::SELECT_CC, VT, Expand);
932 
933         setOperationAction(ISD::ANY_EXTEND, VT, Custom);
934         setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
935         setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
936 
937         // Custom-lower reduction operations to set up the corresponding custom
938         // nodes' operands.
939         setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
940         setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
941         setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
942         setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
943         setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
944 
945         for (unsigned VPOpc : IntegerVPOps)
946           setOperationAction(VPOpc, VT, Custom);
947 
948         // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
949         // type that can represent the value exactly.
950         if (VT.getVectorElementType() != MVT::i64) {
951           MVT FloatEltVT =
952               VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
953           EVT FloatVT =
954               MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
955           if (isTypeLegal(FloatVT)) {
956             setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
957             setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
958           }
959         }
960       }
961 
962       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
963         if (!useRVVForFixedLengthVectorVT(VT))
964           continue;
965 
966         // By default everything must be expanded.
967         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
968           setOperationAction(Op, VT, Expand);
969         for (MVT OtherVT : MVT::fp_fixedlen_vector_valuetypes()) {
970           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
971           setTruncStoreAction(VT, OtherVT, Expand);
972         }
973 
974         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
975         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
976         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
977 
978         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
979         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
980         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
981         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
982         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
983 
984         setOperationAction(ISD::LOAD, VT, Custom);
985         setOperationAction(ISD::STORE, VT, Custom);
986         setOperationAction(ISD::MLOAD, VT, Custom);
987         setOperationAction(ISD::MSTORE, VT, Custom);
988         setOperationAction(ISD::MGATHER, VT, Custom);
989         setOperationAction(ISD::MSCATTER, VT, Custom);
990 
991         setOperationAction(ISD::VP_LOAD, VT, Custom);
992         setOperationAction(ISD::VP_STORE, VT, Custom);
993         setOperationAction(ISD::VP_GATHER, VT, Custom);
994         setOperationAction(ISD::VP_SCATTER, VT, Custom);
995 
996         setOperationAction(ISD::FADD, VT, Custom);
997         setOperationAction(ISD::FSUB, VT, Custom);
998         setOperationAction(ISD::FMUL, VT, Custom);
999         setOperationAction(ISD::FDIV, VT, Custom);
1000         setOperationAction(ISD::FNEG, VT, Custom);
1001         setOperationAction(ISD::FABS, VT, Custom);
1002         setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1003         setOperationAction(ISD::FSQRT, VT, Custom);
1004         setOperationAction(ISD::FMA, VT, Custom);
1005         setOperationAction(ISD::FMINNUM, VT, Custom);
1006         setOperationAction(ISD::FMAXNUM, VT, Custom);
1007 
1008         setOperationAction(ISD::FP_ROUND, VT, Custom);
1009         setOperationAction(ISD::FP_EXTEND, VT, Custom);
1010 
1011         setOperationAction(ISD::FTRUNC, VT, Custom);
1012         setOperationAction(ISD::FCEIL, VT, Custom);
1013         setOperationAction(ISD::FFLOOR, VT, Custom);
1014 
1015         for (auto CC : VFPCCToExpand)
1016           setCondCodeAction(CC, VT, Expand);
1017 
1018         setOperationAction(ISD::VSELECT, VT, Custom);
1019         setOperationAction(ISD::SELECT, VT, Custom);
1020         setOperationAction(ISD::SELECT_CC, VT, Expand);
1021 
1022         setOperationAction(ISD::BITCAST, VT, Custom);
1023 
1024         setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1025         setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1026         setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1027         setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1028 
1029         for (unsigned VPOpc : FloatingPointVPOps)
1030           setOperationAction(VPOpc, VT, Custom);
1031       }
1032 
1033       // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1034       setOperationAction(ISD::BITCAST, MVT::i8, Custom);
1035       setOperationAction(ISD::BITCAST, MVT::i16, Custom);
1036       setOperationAction(ISD::BITCAST, MVT::i32, Custom);
1037       setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1038       if (Subtarget.hasStdExtZfh())
1039         setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1040       if (Subtarget.hasStdExtF())
1041         setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1042       if (Subtarget.hasStdExtD())
1043         setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1044     }
1045   }
1046 
1047   // Function alignments.
1048   const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
1049   setMinFunctionAlignment(FunctionAlignment);
1050   setPrefFunctionAlignment(FunctionAlignment);
1051 
1052   setMinimumJumpTableEntries(5);
1053 
1054   // Jumps are expensive, compared to logic
1055   setJumpIsExpensive();
1056 
1057   setTargetDAGCombine(ISD::ADD);
1058   setTargetDAGCombine(ISD::SUB);
1059   setTargetDAGCombine(ISD::AND);
1060   setTargetDAGCombine(ISD::OR);
1061   setTargetDAGCombine(ISD::XOR);
1062   setTargetDAGCombine(ISD::ANY_EXTEND);
1063   if (Subtarget.hasStdExtF()) {
1064     setTargetDAGCombine(ISD::ZERO_EXTEND);
1065     setTargetDAGCombine(ISD::FP_TO_SINT);
1066     setTargetDAGCombine(ISD::FP_TO_UINT);
1067     setTargetDAGCombine(ISD::FP_TO_SINT_SAT);
1068     setTargetDAGCombine(ISD::FP_TO_UINT_SAT);
1069   }
1070   if (Subtarget.hasVInstructions()) {
1071     setTargetDAGCombine(ISD::FCOPYSIGN);
1072     setTargetDAGCombine(ISD::MGATHER);
1073     setTargetDAGCombine(ISD::MSCATTER);
1074     setTargetDAGCombine(ISD::VP_GATHER);
1075     setTargetDAGCombine(ISD::VP_SCATTER);
1076     setTargetDAGCombine(ISD::SRA);
1077     setTargetDAGCombine(ISD::SRL);
1078     setTargetDAGCombine(ISD::SHL);
1079     setTargetDAGCombine(ISD::STORE);
1080   }
1081 }
1082 
1083 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
1084                                             LLVMContext &Context,
1085                                             EVT VT) const {
1086   if (!VT.isVector())
1087     return getPointerTy(DL);
1088   if (Subtarget.hasVInstructions() &&
1089       (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
1090     return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
1091   return VT.changeVectorElementTypeToInteger();
1092 }
1093 
1094 MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
1095   return Subtarget.getXLenVT();
1096 }
1097 
1098 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1099                                              const CallInst &I,
1100                                              MachineFunction &MF,
1101                                              unsigned Intrinsic) const {
1102   auto &DL = I.getModule()->getDataLayout();
1103   switch (Intrinsic) {
1104   default:
1105     return false;
1106   case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1107   case Intrinsic::riscv_masked_atomicrmw_add_i32:
1108   case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1109   case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1110   case Intrinsic::riscv_masked_atomicrmw_max_i32:
1111   case Intrinsic::riscv_masked_atomicrmw_min_i32:
1112   case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1113   case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1114   case Intrinsic::riscv_masked_cmpxchg_i32: {
1115     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
1116     Info.opc = ISD::INTRINSIC_W_CHAIN;
1117     Info.memVT = MVT::getVT(PtrTy->getPointerElementType());
1118     Info.ptrVal = I.getArgOperand(0);
1119     Info.offset = 0;
1120     Info.align = Align(4);
1121     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1122                  MachineMemOperand::MOVolatile;
1123     return true;
1124   }
1125   case Intrinsic::riscv_masked_strided_load:
1126     Info.opc = ISD::INTRINSIC_W_CHAIN;
1127     Info.ptrVal = I.getArgOperand(1);
1128     Info.memVT = getValueType(DL, I.getType()->getScalarType());
1129     Info.align = Align(DL.getTypeSizeInBits(I.getType()->getScalarType()) / 8);
1130     Info.size = MemoryLocation::UnknownSize;
1131     Info.flags |= MachineMemOperand::MOLoad;
1132     return true;
1133   case Intrinsic::riscv_masked_strided_store:
1134     Info.opc = ISD::INTRINSIC_VOID;
1135     Info.ptrVal = I.getArgOperand(1);
1136     Info.memVT =
1137         getValueType(DL, I.getArgOperand(0)->getType()->getScalarType());
1138     Info.align = Align(
1139         DL.getTypeSizeInBits(I.getArgOperand(0)->getType()->getScalarType()) /
1140         8);
1141     Info.size = MemoryLocation::UnknownSize;
1142     Info.flags |= MachineMemOperand::MOStore;
1143     return true;
1144   }
1145 }
1146 
1147 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1148                                                 const AddrMode &AM, Type *Ty,
1149                                                 unsigned AS,
1150                                                 Instruction *I) const {
1151   // No global is ever allowed as a base.
1152   if (AM.BaseGV)
1153     return false;
1154 
1155   // Require a 12-bit signed offset.
1156   if (!isInt<12>(AM.BaseOffs))
1157     return false;
1158 
1159   switch (AM.Scale) {
1160   case 0: // "r+i" or just "i", depending on HasBaseReg.
1161     break;
1162   case 1:
1163     if (!AM.HasBaseReg) // allow "r+i".
1164       break;
1165     return false; // disallow "r+r" or "r+r+i".
1166   default:
1167     return false;
1168   }
1169 
1170   return true;
1171 }
1172 
1173 bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1174   return isInt<12>(Imm);
1175 }
1176 
1177 bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1178   return isInt<12>(Imm);
1179 }
1180 
1181 // On RV32, 64-bit integers are split into their high and low parts and held
1182 // in two different registers, so the trunc is free since the low register can
1183 // just be used.
1184 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
1185   if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
1186     return false;
1187   unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1188   unsigned DestBits = DstTy->getPrimitiveSizeInBits();
1189   return (SrcBits == 64 && DestBits == 32);
1190 }
1191 
1192 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
1193   if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
1194       !SrcVT.isInteger() || !DstVT.isInteger())
1195     return false;
1196   unsigned SrcBits = SrcVT.getSizeInBits();
1197   unsigned DestBits = DstVT.getSizeInBits();
1198   return (SrcBits == 64 && DestBits == 32);
1199 }
1200 
1201 bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
1202   // Zexts are free if they can be combined with a load.
1203   // Don't advertise i32->i64 zextload as being free for RV64. It interacts
1204   // poorly with type legalization of compares preferring sext.
1205   if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
1206     EVT MemVT = LD->getMemoryVT();
1207     if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1208         (LD->getExtensionType() == ISD::NON_EXTLOAD ||
1209          LD->getExtensionType() == ISD::ZEXTLOAD))
1210       return true;
1211   }
1212 
1213   return TargetLowering::isZExtFree(Val, VT2);
1214 }
1215 
1216 bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
1217   return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1218 }
1219 
1220 bool RISCVTargetLowering::isCheapToSpeculateCttz() const {
1221   return Subtarget.hasStdExtZbb();
1222 }
1223 
1224 bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
1225   return Subtarget.hasStdExtZbb();
1226 }
1227 
1228 bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
1229   EVT VT = Y.getValueType();
1230 
1231   // FIXME: Support vectors once we have tests.
1232   if (VT.isVector())
1233     return false;
1234 
1235   return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) &&
1236          !isa<ConstantSDNode>(Y);
1237 }
1238 
1239 /// Check if sinking \p I's operands to I's basic block is profitable, because
1240 /// the operands can be folded into a target instruction, e.g.
1241 /// splats of scalars can fold into vector instructions.
1242 bool RISCVTargetLowering::shouldSinkOperands(
1243     Instruction *I, SmallVectorImpl<Use *> &Ops) const {
1244   using namespace llvm::PatternMatch;
1245 
1246   if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
1247     return false;
1248 
1249   auto IsSinker = [&](Instruction *I, int Operand) {
1250     switch (I->getOpcode()) {
1251     case Instruction::Add:
1252     case Instruction::Sub:
1253     case Instruction::Mul:
1254     case Instruction::And:
1255     case Instruction::Or:
1256     case Instruction::Xor:
1257     case Instruction::FAdd:
1258     case Instruction::FSub:
1259     case Instruction::FMul:
1260     case Instruction::FDiv:
1261     case Instruction::ICmp:
1262     case Instruction::FCmp:
1263       return true;
1264     case Instruction::Shl:
1265     case Instruction::LShr:
1266     case Instruction::AShr:
1267     case Instruction::UDiv:
1268     case Instruction::SDiv:
1269     case Instruction::URem:
1270     case Instruction::SRem:
1271       return Operand == 1;
1272     case Instruction::Call:
1273       if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1274         switch (II->getIntrinsicID()) {
1275         case Intrinsic::fma:
1276           return Operand == 0 || Operand == 1;
1277         // FIXME: Our patterns can only match vx/vf instructions when the splat
1278         // it on the RHS, because TableGen doesn't recognize our VP operations
1279         // as commutative.
1280         case Intrinsic::vp_add:
1281         case Intrinsic::vp_mul:
1282         case Intrinsic::vp_and:
1283         case Intrinsic::vp_or:
1284         case Intrinsic::vp_xor:
1285         case Intrinsic::vp_fadd:
1286         case Intrinsic::vp_fmul:
1287         case Intrinsic::vp_shl:
1288         case Intrinsic::vp_lshr:
1289         case Intrinsic::vp_ashr:
1290         case Intrinsic::vp_udiv:
1291         case Intrinsic::vp_sdiv:
1292         case Intrinsic::vp_urem:
1293         case Intrinsic::vp_srem:
1294           return Operand == 1;
1295         // ... with the exception of vp.sub/vp.fsub/vp.fdiv, which have
1296         // explicit patterns for both LHS and RHS (as 'vr' versions).
1297         case Intrinsic::vp_sub:
1298         case Intrinsic::vp_fsub:
1299         case Intrinsic::vp_fdiv:
1300           return Operand == 0 || Operand == 1;
1301         default:
1302           return false;
1303         }
1304       }
1305       return false;
1306     default:
1307       return false;
1308     }
1309   };
1310 
1311   for (auto OpIdx : enumerate(I->operands())) {
1312     if (!IsSinker(I, OpIdx.index()))
1313       continue;
1314 
1315     Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
1316     // Make sure we are not already sinking this operand
1317     if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
1318       continue;
1319 
1320     // We are looking for a splat that can be sunk.
1321     if (!match(Op, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
1322                              m_Undef(), m_ZeroMask())))
1323       continue;
1324 
1325     // All uses of the shuffle should be sunk to avoid duplicating it across gpr
1326     // and vector registers
1327     for (Use &U : Op->uses()) {
1328       Instruction *Insn = cast<Instruction>(U.getUser());
1329       if (!IsSinker(Insn, U.getOperandNo()))
1330         return false;
1331     }
1332 
1333     Ops.push_back(&Op->getOperandUse(0));
1334     Ops.push_back(&OpIdx.value());
1335   }
1336   return true;
1337 }
1338 
1339 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1340                                        bool ForCodeSize) const {
1341   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1342   if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
1343     return false;
1344   if (VT == MVT::f32 && !Subtarget.hasStdExtF())
1345     return false;
1346   if (VT == MVT::f64 && !Subtarget.hasStdExtD())
1347     return false;
1348   return Imm.isZero();
1349 }
1350 
1351 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
1352   return (VT == MVT::f16 && Subtarget.hasStdExtZfh()) ||
1353          (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
1354          (VT == MVT::f64 && Subtarget.hasStdExtD());
1355 }
1356 
1357 MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1358                                                       CallingConv::ID CC,
1359                                                       EVT VT) const {
1360   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1361   // We might still end up using a GPR but that will be decided based on ABI.
1362   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1363   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1364     return MVT::f32;
1365 
1366   return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1367 }
1368 
1369 unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1370                                                            CallingConv::ID CC,
1371                                                            EVT VT) const {
1372   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1373   // We might still end up using a GPR but that will be decided based on ABI.
1374   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1375   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1376     return 1;
1377 
1378   return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1379 }
1380 
1381 // Changes the condition code and swaps operands if necessary, so the SetCC
1382 // operation matches one of the comparisons supported directly by branches
1383 // in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
1384 // with 1/-1.
1385 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1386                                     ISD::CondCode &CC, SelectionDAG &DAG) {
1387   // Convert X > -1 to X >= 0.
1388   if (CC == ISD::SETGT && isAllOnesConstant(RHS)) {
1389     RHS = DAG.getConstant(0, DL, RHS.getValueType());
1390     CC = ISD::SETGE;
1391     return;
1392   }
1393   // Convert X < 1 to 0 >= X.
1394   if (CC == ISD::SETLT && isOneConstant(RHS)) {
1395     RHS = LHS;
1396     LHS = DAG.getConstant(0, DL, RHS.getValueType());
1397     CC = ISD::SETGE;
1398     return;
1399   }
1400 
1401   switch (CC) {
1402   default:
1403     break;
1404   case ISD::SETGT:
1405   case ISD::SETLE:
1406   case ISD::SETUGT:
1407   case ISD::SETULE:
1408     CC = ISD::getSetCCSwappedOperands(CC);
1409     std::swap(LHS, RHS);
1410     break;
1411   }
1412 }
1413 
1414 RISCVII::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
1415   assert(VT.isScalableVector() && "Expecting a scalable vector type");
1416   unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
1417   if (VT.getVectorElementType() == MVT::i1)
1418     KnownSize *= 8;
1419 
1420   switch (KnownSize) {
1421   default:
1422     llvm_unreachable("Invalid LMUL.");
1423   case 8:
1424     return RISCVII::VLMUL::LMUL_F8;
1425   case 16:
1426     return RISCVII::VLMUL::LMUL_F4;
1427   case 32:
1428     return RISCVII::VLMUL::LMUL_F2;
1429   case 64:
1430     return RISCVII::VLMUL::LMUL_1;
1431   case 128:
1432     return RISCVII::VLMUL::LMUL_2;
1433   case 256:
1434     return RISCVII::VLMUL::LMUL_4;
1435   case 512:
1436     return RISCVII::VLMUL::LMUL_8;
1437   }
1438 }
1439 
1440 unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVII::VLMUL LMul) {
1441   switch (LMul) {
1442   default:
1443     llvm_unreachable("Invalid LMUL.");
1444   case RISCVII::VLMUL::LMUL_F8:
1445   case RISCVII::VLMUL::LMUL_F4:
1446   case RISCVII::VLMUL::LMUL_F2:
1447   case RISCVII::VLMUL::LMUL_1:
1448     return RISCV::VRRegClassID;
1449   case RISCVII::VLMUL::LMUL_2:
1450     return RISCV::VRM2RegClassID;
1451   case RISCVII::VLMUL::LMUL_4:
1452     return RISCV::VRM4RegClassID;
1453   case RISCVII::VLMUL::LMUL_8:
1454     return RISCV::VRM8RegClassID;
1455   }
1456 }
1457 
1458 unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
1459   RISCVII::VLMUL LMUL = getLMUL(VT);
1460   if (LMUL == RISCVII::VLMUL::LMUL_F8 ||
1461       LMUL == RISCVII::VLMUL::LMUL_F4 ||
1462       LMUL == RISCVII::VLMUL::LMUL_F2 ||
1463       LMUL == RISCVII::VLMUL::LMUL_1) {
1464     static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
1465                   "Unexpected subreg numbering");
1466     return RISCV::sub_vrm1_0 + Index;
1467   }
1468   if (LMUL == RISCVII::VLMUL::LMUL_2) {
1469     static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
1470                   "Unexpected subreg numbering");
1471     return RISCV::sub_vrm2_0 + Index;
1472   }
1473   if (LMUL == RISCVII::VLMUL::LMUL_4) {
1474     static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
1475                   "Unexpected subreg numbering");
1476     return RISCV::sub_vrm4_0 + Index;
1477   }
1478   llvm_unreachable("Invalid vector type.");
1479 }
1480 
1481 unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
1482   if (VT.getVectorElementType() == MVT::i1)
1483     return RISCV::VRRegClassID;
1484   return getRegClassIDForLMUL(getLMUL(VT));
1485 }
1486 
1487 // Attempt to decompose a subvector insert/extract between VecVT and
1488 // SubVecVT via subregister indices. Returns the subregister index that
1489 // can perform the subvector insert/extract with the given element index, as
1490 // well as the index corresponding to any leftover subvectors that must be
1491 // further inserted/extracted within the register class for SubVecVT.
1492 std::pair<unsigned, unsigned>
1493 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1494     MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
1495     const RISCVRegisterInfo *TRI) {
1496   static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
1497                  RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
1498                  RISCV::VRM2RegClassID > RISCV::VRRegClassID),
1499                 "Register classes not ordered");
1500   unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
1501   unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
1502   // Try to compose a subregister index that takes us from the incoming
1503   // LMUL>1 register class down to the outgoing one. At each step we half
1504   // the LMUL:
1505   //   nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
1506   // Note that this is not guaranteed to find a subregister index, such as
1507   // when we are extracting from one VR type to another.
1508   unsigned SubRegIdx = RISCV::NoSubRegister;
1509   for (const unsigned RCID :
1510        {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
1511     if (VecRegClassID > RCID && SubRegClassID <= RCID) {
1512       VecVT = VecVT.getHalfNumVectorElementsVT();
1513       bool IsHi =
1514           InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
1515       SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
1516                                             getSubregIndexByMVT(VecVT, IsHi));
1517       if (IsHi)
1518         InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
1519     }
1520   return {SubRegIdx, InsertExtractIdx};
1521 }
1522 
1523 // Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
1524 // stores for those types.
1525 bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
1526   return !Subtarget.useRVVForFixedLengthVectors() ||
1527          (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
1528 }
1529 
1530 bool RISCVTargetLowering::isLegalElementTypeForRVV(Type *ScalarTy) const {
1531   if (ScalarTy->isPointerTy())
1532     return true;
1533 
1534   if (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
1535       ScalarTy->isIntegerTy(32))
1536     return true;
1537 
1538   if (ScalarTy->isIntegerTy(64))
1539     return Subtarget.hasVInstructionsI64();
1540 
1541   if (ScalarTy->isHalfTy())
1542     return Subtarget.hasVInstructionsF16();
1543   if (ScalarTy->isFloatTy())
1544     return Subtarget.hasVInstructionsF32();
1545   if (ScalarTy->isDoubleTy())
1546     return Subtarget.hasVInstructionsF64();
1547 
1548   return false;
1549 }
1550 
1551 static SDValue getVLOperand(SDValue Op) {
1552   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
1553           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
1554          "Unexpected opcode");
1555   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
1556   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
1557   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
1558       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
1559   if (!II)
1560     return SDValue();
1561   return Op.getOperand(II->VLOperand + 1 + HasChain);
1562 }
1563 
1564 static bool useRVVForFixedLengthVectorVT(MVT VT,
1565                                          const RISCVSubtarget &Subtarget) {
1566   assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
1567   if (!Subtarget.useRVVForFixedLengthVectors())
1568     return false;
1569 
1570   // We only support a set of vector types with a consistent maximum fixed size
1571   // across all supported vector element types to avoid legalization issues.
1572   // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
1573   // fixed-length vector type we support is 1024 bytes.
1574   if (VT.getFixedSizeInBits() > 1024 * 8)
1575     return false;
1576 
1577   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1578 
1579   MVT EltVT = VT.getVectorElementType();
1580 
1581   // Don't use RVV for vectors we cannot scalarize if required.
1582   switch (EltVT.SimpleTy) {
1583   // i1 is supported but has different rules.
1584   default:
1585     return false;
1586   case MVT::i1:
1587     // Masks can only use a single register.
1588     if (VT.getVectorNumElements() > MinVLen)
1589       return false;
1590     MinVLen /= 8;
1591     break;
1592   case MVT::i8:
1593   case MVT::i16:
1594   case MVT::i32:
1595     break;
1596   case MVT::i64:
1597     if (!Subtarget.hasVInstructionsI64())
1598       return false;
1599     break;
1600   case MVT::f16:
1601     if (!Subtarget.hasVInstructionsF16())
1602       return false;
1603     break;
1604   case MVT::f32:
1605     if (!Subtarget.hasVInstructionsF32())
1606       return false;
1607     break;
1608   case MVT::f64:
1609     if (!Subtarget.hasVInstructionsF64())
1610       return false;
1611     break;
1612   }
1613 
1614   // Reject elements larger than ELEN.
1615   if (EltVT.getSizeInBits() > Subtarget.getMaxELENForFixedLengthVectors())
1616     return false;
1617 
1618   unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
1619   // Don't use RVV for types that don't fit.
1620   if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
1621     return false;
1622 
1623   // TODO: Perhaps an artificial restriction, but worth having whilst getting
1624   // the base fixed length RVV support in place.
1625   if (!VT.isPow2VectorType())
1626     return false;
1627 
1628   return true;
1629 }
1630 
1631 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
1632   return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
1633 }
1634 
1635 // Return the largest legal scalable vector type that matches VT's element type.
1636 static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
1637                                             const RISCVSubtarget &Subtarget) {
1638   // This may be called before legal types are setup.
1639   assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
1640           useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
1641          "Expected legal fixed length vector!");
1642 
1643   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1644   unsigned MaxELen = Subtarget.getMaxELENForFixedLengthVectors();
1645 
1646   MVT EltVT = VT.getVectorElementType();
1647   switch (EltVT.SimpleTy) {
1648   default:
1649     llvm_unreachable("unexpected element type for RVV container");
1650   case MVT::i1:
1651   case MVT::i8:
1652   case MVT::i16:
1653   case MVT::i32:
1654   case MVT::i64:
1655   case MVT::f16:
1656   case MVT::f32:
1657   case MVT::f64: {
1658     // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
1659     // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
1660     // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
1661     unsigned NumElts =
1662         (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
1663     NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
1664     assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
1665     return MVT::getScalableVectorVT(EltVT, NumElts);
1666   }
1667   }
1668 }
1669 
1670 static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
1671                                             const RISCVSubtarget &Subtarget) {
1672   return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
1673                                           Subtarget);
1674 }
1675 
1676 MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
1677   return ::getContainerForFixedLengthVector(*this, VT, getSubtarget());
1678 }
1679 
1680 // Grow V to consume an entire RVV register.
1681 static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1682                                        const RISCVSubtarget &Subtarget) {
1683   assert(VT.isScalableVector() &&
1684          "Expected to convert into a scalable vector!");
1685   assert(V.getValueType().isFixedLengthVector() &&
1686          "Expected a fixed length vector operand!");
1687   SDLoc DL(V);
1688   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1689   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
1690 }
1691 
1692 // Shrink V so it's just big enough to maintain a VT's worth of data.
1693 static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1694                                          const RISCVSubtarget &Subtarget) {
1695   assert(VT.isFixedLengthVector() &&
1696          "Expected to convert into a fixed length vector!");
1697   assert(V.getValueType().isScalableVector() &&
1698          "Expected a scalable vector operand!");
1699   SDLoc DL(V);
1700   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1701   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
1702 }
1703 
1704 // Gets the two common "VL" operands: an all-ones mask and the vector length.
1705 // VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
1706 // the vector type that it is contained in.
1707 static std::pair<SDValue, SDValue>
1708 getDefaultVLOps(MVT VecVT, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
1709                 const RISCVSubtarget &Subtarget) {
1710   assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
1711   MVT XLenVT = Subtarget.getXLenVT();
1712   SDValue VL = VecVT.isFixedLengthVector()
1713                    ? DAG.getConstant(VecVT.getVectorNumElements(), DL, XLenVT)
1714                    : DAG.getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT);
1715   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
1716   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
1717   return {Mask, VL};
1718 }
1719 
1720 // As above but assuming the given type is a scalable vector type.
1721 static std::pair<SDValue, SDValue>
1722 getDefaultScalableVLOps(MVT VecVT, SDLoc DL, SelectionDAG &DAG,
1723                         const RISCVSubtarget &Subtarget) {
1724   assert(VecVT.isScalableVector() && "Expecting a scalable vector");
1725   return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
1726 }
1727 
1728 // The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
1729 // of either is (currently) supported. This can get us into an infinite loop
1730 // where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
1731 // as a ..., etc.
1732 // Until either (or both) of these can reliably lower any node, reporting that
1733 // we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
1734 // the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
1735 // which is not desirable.
1736 bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
1737     EVT VT, unsigned DefinedValues) const {
1738   return false;
1739 }
1740 
1741 bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
1742   // Only splats are currently supported.
1743   if (ShuffleVectorSDNode::isSplatMask(M.data(), VT))
1744     return true;
1745 
1746   return false;
1747 }
1748 
1749 static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
1750                                   const RISCVSubtarget &Subtarget) {
1751   // RISCV FP-to-int conversions saturate to the destination register size, but
1752   // don't produce 0 for nan. We can use a conversion instruction and fix the
1753   // nan case with a compare and a select.
1754   SDValue Src = Op.getOperand(0);
1755 
1756   EVT DstVT = Op.getValueType();
1757   EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1758 
1759   bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
1760   unsigned Opc;
1761   if (SatVT == DstVT)
1762     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
1763   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
1764     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
1765   else
1766     return SDValue();
1767   // FIXME: Support other SatVTs by clamping before or after the conversion.
1768 
1769   SDLoc DL(Op);
1770   SDValue FpToInt = DAG.getNode(
1771       Opc, DL, DstVT, Src,
1772       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
1773 
1774   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
1775   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
1776 }
1777 
1778 // Expand vector FTRUNC, FCEIL, and FFLOOR by converting to the integer domain
1779 // and back. Taking care to avoid converting values that are nan or already
1780 // correct.
1781 // TODO: Floor and ceil could be shorter by changing rounding mode, but we don't
1782 // have FRM dependencies modeled yet.
1783 static SDValue lowerFTRUNC_FCEIL_FFLOOR(SDValue Op, SelectionDAG &DAG) {
1784   MVT VT = Op.getSimpleValueType();
1785   assert(VT.isVector() && "Unexpected type");
1786 
1787   SDLoc DL(Op);
1788 
1789   // Freeze the source since we are increasing the number of uses.
1790   SDValue Src = DAG.getNode(ISD::FREEZE, DL, VT, Op.getOperand(0));
1791 
1792   // Truncate to integer and convert back to FP.
1793   MVT IntVT = VT.changeVectorElementTypeToInteger();
1794   SDValue Truncated = DAG.getNode(ISD::FP_TO_SINT, DL, IntVT, Src);
1795   Truncated = DAG.getNode(ISD::SINT_TO_FP, DL, VT, Truncated);
1796 
1797   MVT SetccVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
1798 
1799   if (Op.getOpcode() == ISD::FCEIL) {
1800     // If the truncated value is the greater than or equal to the original
1801     // value, we've computed the ceil. Otherwise, we went the wrong way and
1802     // need to increase by 1.
1803     // FIXME: This should use a masked operation. Handle here or in isel?
1804     SDValue Adjust = DAG.getNode(ISD::FADD, DL, VT, Truncated,
1805                                  DAG.getConstantFP(1.0, DL, VT));
1806     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOLT);
1807     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1808   } else if (Op.getOpcode() == ISD::FFLOOR) {
1809     // If the truncated value is the less than or equal to the original value,
1810     // we've computed the floor. Otherwise, we went the wrong way and need to
1811     // decrease by 1.
1812     // FIXME: This should use a masked operation. Handle here or in isel?
1813     SDValue Adjust = DAG.getNode(ISD::FSUB, DL, VT, Truncated,
1814                                  DAG.getConstantFP(1.0, DL, VT));
1815     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOGT);
1816     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1817   }
1818 
1819   // Restore the original sign so that -0.0 is preserved.
1820   Truncated = DAG.getNode(ISD::FCOPYSIGN, DL, VT, Truncated, Src);
1821 
1822   // Determine the largest integer that can be represented exactly. This and
1823   // values larger than it don't have any fractional bits so don't need to
1824   // be converted.
1825   const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
1826   unsigned Precision = APFloat::semanticsPrecision(FltSem);
1827   APFloat MaxVal = APFloat(FltSem);
1828   MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
1829                           /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
1830   SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
1831 
1832   // If abs(Src) was larger than MaxVal or nan, keep it.
1833   SDValue Abs = DAG.getNode(ISD::FABS, DL, VT, Src);
1834   SDValue Setcc = DAG.getSetCC(DL, SetccVT, Abs, MaxValNode, ISD::SETOLT);
1835   return DAG.getSelect(DL, VT, Setcc, Truncated, Src);
1836 }
1837 
1838 static SDValue lowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG,
1839                                  const RISCVSubtarget &Subtarget) {
1840   MVT VT = Op.getSimpleValueType();
1841   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1842 
1843   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1844 
1845   SDLoc DL(Op);
1846   SDValue Mask, VL;
1847   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1848 
1849   unsigned Opc =
1850       VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
1851   SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, Op.getOperand(0), VL);
1852   return convertFromScalableVector(VT, Splat, DAG, Subtarget);
1853 }
1854 
1855 struct VIDSequence {
1856   int64_t StepNumerator;
1857   unsigned StepDenominator;
1858   int64_t Addend;
1859 };
1860 
1861 // Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
1862 // to the (non-zero) step S and start value X. This can be then lowered as the
1863 // RVV sequence (VID * S) + X, for example.
1864 // The step S is represented as an integer numerator divided by a positive
1865 // denominator. Note that the implementation currently only identifies
1866 // sequences in which either the numerator is +/- 1 or the denominator is 1. It
1867 // cannot detect 2/3, for example.
1868 // Note that this method will also match potentially unappealing index
1869 // sequences, like <i32 0, i32 50939494>, however it is left to the caller to
1870 // determine whether this is worth generating code for.
1871 static Optional<VIDSequence> isSimpleVIDSequence(SDValue Op) {
1872   unsigned NumElts = Op.getNumOperands();
1873   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
1874   if (!Op.getValueType().isInteger())
1875     return None;
1876 
1877   Optional<unsigned> SeqStepDenom;
1878   Optional<int64_t> SeqStepNum, SeqAddend;
1879   Optional<std::pair<uint64_t, unsigned>> PrevElt;
1880   unsigned EltSizeInBits = Op.getValueType().getScalarSizeInBits();
1881   for (unsigned Idx = 0; Idx < NumElts; Idx++) {
1882     // Assume undef elements match the sequence; we just have to be careful
1883     // when interpolating across them.
1884     if (Op.getOperand(Idx).isUndef())
1885       continue;
1886     // The BUILD_VECTOR must be all constants.
1887     if (!isa<ConstantSDNode>(Op.getOperand(Idx)))
1888       return None;
1889 
1890     uint64_t Val = Op.getConstantOperandVal(Idx) &
1891                    maskTrailingOnes<uint64_t>(EltSizeInBits);
1892 
1893     if (PrevElt) {
1894       // Calculate the step since the last non-undef element, and ensure
1895       // it's consistent across the entire sequence.
1896       unsigned IdxDiff = Idx - PrevElt->second;
1897       int64_t ValDiff = SignExtend64(Val - PrevElt->first, EltSizeInBits);
1898 
1899       // A zero-value value difference means that we're somewhere in the middle
1900       // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
1901       // step change before evaluating the sequence.
1902       if (ValDiff != 0) {
1903         int64_t Remainder = ValDiff % IdxDiff;
1904         // Normalize the step if it's greater than 1.
1905         if (Remainder != ValDiff) {
1906           // The difference must cleanly divide the element span.
1907           if (Remainder != 0)
1908             return None;
1909           ValDiff /= IdxDiff;
1910           IdxDiff = 1;
1911         }
1912 
1913         if (!SeqStepNum)
1914           SeqStepNum = ValDiff;
1915         else if (ValDiff != SeqStepNum)
1916           return None;
1917 
1918         if (!SeqStepDenom)
1919           SeqStepDenom = IdxDiff;
1920         else if (IdxDiff != *SeqStepDenom)
1921           return None;
1922       }
1923     }
1924 
1925     // Record and/or check any addend.
1926     if (SeqStepNum && SeqStepDenom) {
1927       uint64_t ExpectedVal =
1928           (int64_t)(Idx * (uint64_t)*SeqStepNum) / *SeqStepDenom;
1929       int64_t Addend = SignExtend64(Val - ExpectedVal, EltSizeInBits);
1930       if (!SeqAddend)
1931         SeqAddend = Addend;
1932       else if (SeqAddend != Addend)
1933         return None;
1934     }
1935 
1936     // Record this non-undef element for later.
1937     if (!PrevElt || PrevElt->first != Val)
1938       PrevElt = std::make_pair(Val, Idx);
1939   }
1940   // We need to have logged both a step and an addend for this to count as
1941   // a legal index sequence.
1942   if (!SeqStepNum || !SeqStepDenom || !SeqAddend)
1943     return None;
1944 
1945   return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
1946 }
1947 
1948 static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
1949                                  const RISCVSubtarget &Subtarget) {
1950   MVT VT = Op.getSimpleValueType();
1951   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1952 
1953   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1954 
1955   SDLoc DL(Op);
1956   SDValue Mask, VL;
1957   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1958 
1959   MVT XLenVT = Subtarget.getXLenVT();
1960   unsigned NumElts = Op.getNumOperands();
1961 
1962   if (VT.getVectorElementType() == MVT::i1) {
1963     if (ISD::isBuildVectorAllZeros(Op.getNode())) {
1964       SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
1965       return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
1966     }
1967 
1968     if (ISD::isBuildVectorAllOnes(Op.getNode())) {
1969       SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
1970       return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
1971     }
1972 
1973     // Lower constant mask BUILD_VECTORs via an integer vector type, in
1974     // scalar integer chunks whose bit-width depends on the number of mask
1975     // bits and XLEN.
1976     // First, determine the most appropriate scalar integer type to use. This
1977     // is at most XLenVT, but may be shrunk to a smaller vector element type
1978     // according to the size of the final vector - use i8 chunks rather than
1979     // XLenVT if we're producing a v8i1. This results in more consistent
1980     // codegen across RV32 and RV64.
1981     unsigned NumViaIntegerBits =
1982         std::min(std::max(NumElts, 8u), Subtarget.getXLen());
1983     NumViaIntegerBits = std::min(NumViaIntegerBits,
1984                                  Subtarget.getMaxELENForFixedLengthVectors());
1985     if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
1986       // If we have to use more than one INSERT_VECTOR_ELT then this
1987       // optimization is likely to increase code size; avoid peforming it in
1988       // such a case. We can use a load from a constant pool in this case.
1989       if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
1990         return SDValue();
1991       // Now we can create our integer vector type. Note that it may be larger
1992       // than the resulting mask type: v4i1 would use v1i8 as its integer type.
1993       MVT IntegerViaVecVT =
1994           MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
1995                            divideCeil(NumElts, NumViaIntegerBits));
1996 
1997       uint64_t Bits = 0;
1998       unsigned BitPos = 0, IntegerEltIdx = 0;
1999       SDValue Vec = DAG.getUNDEF(IntegerViaVecVT);
2000 
2001       for (unsigned I = 0; I < NumElts; I++, BitPos++) {
2002         // Once we accumulate enough bits to fill our scalar type, insert into
2003         // our vector and clear our accumulated data.
2004         if (I != 0 && I % NumViaIntegerBits == 0) {
2005           if (NumViaIntegerBits <= 32)
2006             Bits = SignExtend64(Bits, 32);
2007           SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2008           Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec,
2009                             Elt, DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2010           Bits = 0;
2011           BitPos = 0;
2012           IntegerEltIdx++;
2013         }
2014         SDValue V = Op.getOperand(I);
2015         bool BitValue = !V.isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
2016         Bits |= ((uint64_t)BitValue << BitPos);
2017       }
2018 
2019       // Insert the (remaining) scalar value into position in our integer
2020       // vector type.
2021       if (NumViaIntegerBits <= 32)
2022         Bits = SignExtend64(Bits, 32);
2023       SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2024       Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, Elt,
2025                         DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2026 
2027       if (NumElts < NumViaIntegerBits) {
2028         // If we're producing a smaller vector than our minimum legal integer
2029         // type, bitcast to the equivalent (known-legal) mask type, and extract
2030         // our final mask.
2031         assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
2032         Vec = DAG.getBitcast(MVT::v8i1, Vec);
2033         Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
2034                           DAG.getConstant(0, DL, XLenVT));
2035       } else {
2036         // Else we must have produced an integer type with the same size as the
2037         // mask type; bitcast for the final result.
2038         assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
2039         Vec = DAG.getBitcast(VT, Vec);
2040       }
2041 
2042       return Vec;
2043     }
2044 
2045     // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
2046     // vector type, we have a legal equivalently-sized i8 type, so we can use
2047     // that.
2048     MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
2049     SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
2050 
2051     SDValue WideVec;
2052     if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2053       // For a splat, perform a scalar truncate before creating the wider
2054       // vector.
2055       assert(Splat.getValueType() == XLenVT &&
2056              "Unexpected type for i1 splat value");
2057       Splat = DAG.getNode(ISD::AND, DL, XLenVT, Splat,
2058                           DAG.getConstant(1, DL, XLenVT));
2059       WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
2060     } else {
2061       SmallVector<SDValue, 8> Ops(Op->op_values());
2062       WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
2063       SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
2064       WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
2065     }
2066 
2067     return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
2068   }
2069 
2070   if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2071     unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
2072                                         : RISCVISD::VMV_V_X_VL;
2073     Splat = DAG.getNode(Opc, DL, ContainerVT, Splat, VL);
2074     return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2075   }
2076 
2077   // Try and match index sequences, which we can lower to the vid instruction
2078   // with optional modifications. An all-undef vector is matched by
2079   // getSplatValue, above.
2080   if (auto SimpleVID = isSimpleVIDSequence(Op)) {
2081     int64_t StepNumerator = SimpleVID->StepNumerator;
2082     unsigned StepDenominator = SimpleVID->StepDenominator;
2083     int64_t Addend = SimpleVID->Addend;
2084 
2085     assert(StepNumerator != 0 && "Invalid step");
2086     bool Negate = false;
2087     int64_t SplatStepVal = StepNumerator;
2088     unsigned StepOpcode = ISD::MUL;
2089     if (StepNumerator != 1) {
2090       if (isPowerOf2_64(std::abs(StepNumerator))) {
2091         Negate = StepNumerator < 0;
2092         StepOpcode = ISD::SHL;
2093         SplatStepVal = Log2_64(std::abs(StepNumerator));
2094       }
2095     }
2096 
2097     // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
2098     // threshold since it's the immediate value many RVV instructions accept.
2099     // There is no vmul.vi instruction so ensure multiply constant can fit in
2100     // a single addi instruction.
2101     if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
2102          (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
2103         isPowerOf2_32(StepDenominator) && isInt<5>(Addend)) {
2104       SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
2105       // Convert right out of the scalable type so we can use standard ISD
2106       // nodes for the rest of the computation. If we used scalable types with
2107       // these, we'd lose the fixed-length vector info and generate worse
2108       // vsetvli code.
2109       VID = convertFromScalableVector(VT, VID, DAG, Subtarget);
2110       if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
2111           (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
2112         SDValue SplatStep = DAG.getSplatVector(
2113             VT, DL, DAG.getConstant(SplatStepVal, DL, XLenVT));
2114         VID = DAG.getNode(StepOpcode, DL, VT, VID, SplatStep);
2115       }
2116       if (StepDenominator != 1) {
2117         SDValue SplatStep = DAG.getSplatVector(
2118             VT, DL, DAG.getConstant(Log2_64(StepDenominator), DL, XLenVT));
2119         VID = DAG.getNode(ISD::SRL, DL, VT, VID, SplatStep);
2120       }
2121       if (Addend != 0 || Negate) {
2122         SDValue SplatAddend =
2123             DAG.getSplatVector(VT, DL, DAG.getConstant(Addend, DL, XLenVT));
2124         VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VT, SplatAddend, VID);
2125       }
2126       return VID;
2127     }
2128   }
2129 
2130   // Attempt to detect "hidden" splats, which only reveal themselves as splats
2131   // when re-interpreted as a vector with a larger element type. For example,
2132   //   v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
2133   // could be instead splat as
2134   //   v2i32 = build_vector i32 0x00010000, i32 0x00010000
2135   // TODO: This optimization could also work on non-constant splats, but it
2136   // would require bit-manipulation instructions to construct the splat value.
2137   SmallVector<SDValue> Sequence;
2138   unsigned EltBitSize = VT.getScalarSizeInBits();
2139   const auto *BV = cast<BuildVectorSDNode>(Op);
2140   if (VT.isInteger() && EltBitSize < 64 &&
2141       ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
2142       BV->getRepeatedSequence(Sequence) &&
2143       (Sequence.size() * EltBitSize) <= 64) {
2144     unsigned SeqLen = Sequence.size();
2145     MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
2146     MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, NumElts / SeqLen);
2147     assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
2148             ViaIntVT == MVT::i64) &&
2149            "Unexpected sequence type");
2150 
2151     unsigned EltIdx = 0;
2152     uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
2153     uint64_t SplatValue = 0;
2154     // Construct the amalgamated value which can be splatted as this larger
2155     // vector type.
2156     for (const auto &SeqV : Sequence) {
2157       if (!SeqV.isUndef())
2158         SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
2159                        << (EltIdx * EltBitSize));
2160       EltIdx++;
2161     }
2162 
2163     // On RV64, sign-extend from 32 to 64 bits where possible in order to
2164     // achieve better constant materializion.
2165     if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
2166       SplatValue = SignExtend64(SplatValue, 32);
2167 
2168     // Since we can't introduce illegal i64 types at this stage, we can only
2169     // perform an i64 splat on RV32 if it is its own sign-extended value. That
2170     // way we can use RVV instructions to splat.
2171     assert((ViaIntVT.bitsLE(XLenVT) ||
2172             (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
2173            "Unexpected bitcast sequence");
2174     if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
2175       SDValue ViaVL =
2176           DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
2177       MVT ViaContainerVT =
2178           getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget);
2179       SDValue Splat =
2180           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
2181                       DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
2182       Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
2183       return DAG.getBitcast(VT, Splat);
2184     }
2185   }
2186 
2187   // Try and optimize BUILD_VECTORs with "dominant values" - these are values
2188   // which constitute a large proportion of the elements. In such cases we can
2189   // splat a vector with the dominant element and make up the shortfall with
2190   // INSERT_VECTOR_ELTs.
2191   // Note that this includes vectors of 2 elements by association. The
2192   // upper-most element is the "dominant" one, allowing us to use a splat to
2193   // "insert" the upper element, and an insert of the lower element at position
2194   // 0, which improves codegen.
2195   SDValue DominantValue;
2196   unsigned MostCommonCount = 0;
2197   DenseMap<SDValue, unsigned> ValueCounts;
2198   unsigned NumUndefElts =
2199       count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
2200 
2201   // Track the number of scalar loads we know we'd be inserting, estimated as
2202   // any non-zero floating-point constant. Other kinds of element are either
2203   // already in registers or are materialized on demand. The threshold at which
2204   // a vector load is more desirable than several scalar materializion and
2205   // vector-insertion instructions is not known.
2206   unsigned NumScalarLoads = 0;
2207 
2208   for (SDValue V : Op->op_values()) {
2209     if (V.isUndef())
2210       continue;
2211 
2212     ValueCounts.insert(std::make_pair(V, 0));
2213     unsigned &Count = ValueCounts[V];
2214 
2215     if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
2216       NumScalarLoads += !CFP->isExactlyValue(+0.0);
2217 
2218     // Is this value dominant? In case of a tie, prefer the highest element as
2219     // it's cheaper to insert near the beginning of a vector than it is at the
2220     // end.
2221     if (++Count >= MostCommonCount) {
2222       DominantValue = V;
2223       MostCommonCount = Count;
2224     }
2225   }
2226 
2227   assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
2228   unsigned NumDefElts = NumElts - NumUndefElts;
2229   unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
2230 
2231   // Don't perform this optimization when optimizing for size, since
2232   // materializing elements and inserting them tends to cause code bloat.
2233   if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
2234       ((MostCommonCount > DominantValueCountThreshold) ||
2235        (ValueCounts.size() <= Log2_32(NumDefElts)))) {
2236     // Start by splatting the most common element.
2237     SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
2238 
2239     DenseSet<SDValue> Processed{DominantValue};
2240     MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
2241     for (const auto &OpIdx : enumerate(Op->ops())) {
2242       const SDValue &V = OpIdx.value();
2243       if (V.isUndef() || !Processed.insert(V).second)
2244         continue;
2245       if (ValueCounts[V] == 1) {
2246         Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
2247                           DAG.getConstant(OpIdx.index(), DL, XLenVT));
2248       } else {
2249         // Blend in all instances of this value using a VSELECT, using a
2250         // mask where each bit signals whether that element is the one
2251         // we're after.
2252         SmallVector<SDValue> Ops;
2253         transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
2254           return DAG.getConstant(V == V1, DL, XLenVT);
2255         });
2256         Vec = DAG.getNode(ISD::VSELECT, DL, VT,
2257                           DAG.getBuildVector(SelMaskTy, DL, Ops),
2258                           DAG.getSplatBuildVector(VT, DL, V), Vec);
2259       }
2260     }
2261 
2262     return Vec;
2263   }
2264 
2265   return SDValue();
2266 }
2267 
2268 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Lo,
2269                                    SDValue Hi, SDValue VL, SelectionDAG &DAG) {
2270   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
2271     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
2272     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
2273     // If Hi constant is all the same sign bit as Lo, lower this as a custom
2274     // node in order to try and match RVV vector/scalar instructions.
2275     if ((LoC >> 31) == HiC)
2276       return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Lo, VL);
2277 
2278     // If vl is equal to VLMax and Hi constant is equal to Lo, we could use
2279     // vmv.v.x whose EEW = 32 to lower it.
2280     auto *Const = dyn_cast<ConstantSDNode>(VL);
2281     if (LoC == HiC && Const && Const->getSExtValue() == RISCV::VLMaxSentinel) {
2282       MVT InterVT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
2283       // TODO: if vl <= min(VLMAX), we can also do this. But we could not
2284       // access the subtarget here now.
2285       auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT, Lo, VL);
2286       return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
2287     }
2288   }
2289 
2290   // Fall back to a stack store and stride x0 vector load.
2291   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Lo, Hi, VL);
2292 }
2293 
2294 // Called by type legalization to handle splat of i64 on RV32.
2295 // FIXME: We can optimize this when the type has sign or zero bits in one
2296 // of the halves.
2297 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Scalar,
2298                                    SDValue VL, SelectionDAG &DAG) {
2299   assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
2300   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2301                            DAG.getConstant(0, DL, MVT::i32));
2302   SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2303                            DAG.getConstant(1, DL, MVT::i32));
2304   return splatPartsI64WithVL(DL, VT, Lo, Hi, VL, DAG);
2305 }
2306 
2307 // This function lowers a splat of a scalar operand Splat with the vector
2308 // length VL. It ensures the final sequence is type legal, which is useful when
2309 // lowering a splat after type legalization.
2310 static SDValue lowerScalarSplat(SDValue Scalar, SDValue VL, MVT VT, SDLoc DL,
2311                                 SelectionDAG &DAG,
2312                                 const RISCVSubtarget &Subtarget) {
2313   if (VT.isFloatingPoint()) {
2314     // If VL is 1, we could use vfmv.s.f.
2315     if (isOneConstant(VL))
2316       return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT),
2317                          Scalar, VL);
2318     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Scalar, VL);
2319   }
2320 
2321   MVT XLenVT = Subtarget.getXLenVT();
2322 
2323   // Simplest case is that the operand needs to be promoted to XLenVT.
2324   if (Scalar.getValueType().bitsLE(XLenVT)) {
2325     // If the operand is a constant, sign extend to increase our chances
2326     // of being able to use a .vi instruction. ANY_EXTEND would become a
2327     // a zero extend and the simm5 check in isel would fail.
2328     // FIXME: Should we ignore the upper bits in isel instead?
2329     unsigned ExtOpc =
2330         isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
2331     Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
2332     ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Scalar);
2333     // If VL is 1 and the scalar value won't benefit from immediate, we could
2334     // use vmv.s.x.
2335     if (isOneConstant(VL) &&
2336         (!Const || isNullConstant(Scalar) || !isInt<5>(Const->getSExtValue())))
2337       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
2338                          VL);
2339     return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Scalar, VL);
2340   }
2341 
2342   assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
2343          "Unexpected scalar for splat lowering!");
2344 
2345   if (isOneConstant(VL) && isNullConstant(Scalar))
2346     return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT),
2347                        DAG.getConstant(0, DL, XLenVT), VL);
2348 
2349   // Otherwise use the more complicated splatting algorithm.
2350   return splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
2351 }
2352 
2353 // Is the mask a slidedown that shifts in undefs.
2354 static int matchShuffleAsSlideDown(ArrayRef<int> Mask) {
2355   int Size = Mask.size();
2356 
2357   // Elements shifted in should be undef.
2358   auto CheckUndefs = [&](int Shift) {
2359     for (int i = Size - Shift; i != Size; ++i)
2360       if (Mask[i] >= 0)
2361         return false;
2362     return true;
2363   };
2364 
2365   // Elements should be shifted or undef.
2366   auto MatchShift = [&](int Shift) {
2367     for (int i = 0; i != Size - Shift; ++i)
2368        if (Mask[i] >= 0 && Mask[i] != Shift + i)
2369          return false;
2370     return true;
2371   };
2372 
2373   // Try all possible shifts.
2374   for (int Shift = 1; Shift != Size; ++Shift)
2375     if (CheckUndefs(Shift) && MatchShift(Shift))
2376       return Shift;
2377 
2378   // No match.
2379   return -1;
2380 }
2381 
2382 static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, bool &SwapSources,
2383                                 const RISCVSubtarget &Subtarget) {
2384   // We need to be able to widen elements to the next larger integer type.
2385   if (VT.getScalarSizeInBits() >= Subtarget.getMaxELENForFixedLengthVectors())
2386     return false;
2387 
2388   int Size = Mask.size();
2389   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
2390 
2391   int Srcs[] = {-1, -1};
2392   for (int i = 0; i != Size; ++i) {
2393     // Ignore undef elements.
2394     if (Mask[i] < 0)
2395       continue;
2396 
2397     // Is this an even or odd element.
2398     int Pol = i % 2;
2399 
2400     // Ensure we consistently use the same source for this element polarity.
2401     int Src = Mask[i] / Size;
2402     if (Srcs[Pol] < 0)
2403       Srcs[Pol] = Src;
2404     if (Srcs[Pol] != Src)
2405       return false;
2406 
2407     // Make sure the element within the source is appropriate for this element
2408     // in the destination.
2409     int Elt = Mask[i] % Size;
2410     if (Elt != i / 2)
2411       return false;
2412   }
2413 
2414   // We need to find a source for each polarity and they can't be the same.
2415   if (Srcs[0] < 0 || Srcs[1] < 0 || Srcs[0] == Srcs[1])
2416     return false;
2417 
2418   // Swap the sources if the second source was in the even polarity.
2419   SwapSources = Srcs[0] > Srcs[1];
2420 
2421   return true;
2422 }
2423 
2424 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
2425                                    const RISCVSubtarget &Subtarget) {
2426   SDValue V1 = Op.getOperand(0);
2427   SDValue V2 = Op.getOperand(1);
2428   SDLoc DL(Op);
2429   MVT XLenVT = Subtarget.getXLenVT();
2430   MVT VT = Op.getSimpleValueType();
2431   unsigned NumElts = VT.getVectorNumElements();
2432   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
2433 
2434   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2435 
2436   SDValue TrueMask, VL;
2437   std::tie(TrueMask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2438 
2439   if (SVN->isSplat()) {
2440     const int Lane = SVN->getSplatIndex();
2441     if (Lane >= 0) {
2442       MVT SVT = VT.getVectorElementType();
2443 
2444       // Turn splatted vector load into a strided load with an X0 stride.
2445       SDValue V = V1;
2446       // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
2447       // with undef.
2448       // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
2449       int Offset = Lane;
2450       if (V.getOpcode() == ISD::CONCAT_VECTORS) {
2451         int OpElements =
2452             V.getOperand(0).getSimpleValueType().getVectorNumElements();
2453         V = V.getOperand(Offset / OpElements);
2454         Offset %= OpElements;
2455       }
2456 
2457       // We need to ensure the load isn't atomic or volatile.
2458       if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
2459         auto *Ld = cast<LoadSDNode>(V);
2460         Offset *= SVT.getStoreSize();
2461         SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(),
2462                                                    TypeSize::Fixed(Offset), DL);
2463 
2464         // If this is SEW=64 on RV32, use a strided load with a stride of x0.
2465         if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
2466           SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
2467           SDValue IntID =
2468               DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
2469           SDValue Ops[] = {Ld->getChain(),
2470                            IntID,
2471                            DAG.getUNDEF(ContainerVT),
2472                            NewAddr,
2473                            DAG.getRegister(RISCV::X0, XLenVT),
2474                            VL};
2475           SDValue NewLoad = DAG.getMemIntrinsicNode(
2476               ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
2477               DAG.getMachineFunction().getMachineMemOperand(
2478                   Ld->getMemOperand(), Offset, SVT.getStoreSize()));
2479           DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
2480           return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
2481         }
2482 
2483         // Otherwise use a scalar load and splat. This will give the best
2484         // opportunity to fold a splat into the operation. ISel can turn it into
2485         // the x0 strided load if we aren't able to fold away the select.
2486         if (SVT.isFloatingPoint())
2487           V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
2488                           Ld->getPointerInfo().getWithOffset(Offset),
2489                           Ld->getOriginalAlign(),
2490                           Ld->getMemOperand()->getFlags());
2491         else
2492           V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
2493                              Ld->getPointerInfo().getWithOffset(Offset), SVT,
2494                              Ld->getOriginalAlign(),
2495                              Ld->getMemOperand()->getFlags());
2496         DAG.makeEquivalentMemoryOrdering(Ld, V);
2497 
2498         unsigned Opc =
2499             VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
2500         SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, V, VL);
2501         return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2502       }
2503 
2504       V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2505       assert(Lane < (int)NumElts && "Unexpected lane!");
2506       SDValue Gather =
2507           DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
2508                       DAG.getConstant(Lane, DL, XLenVT), TrueMask, VL);
2509       return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2510     }
2511   }
2512 
2513   ArrayRef<int> Mask = SVN->getMask();
2514 
2515   // Try to match as a slidedown.
2516   int SlideAmt = matchShuffleAsSlideDown(Mask);
2517   if (SlideAmt >= 0) {
2518     // TODO: Should we reduce the VL to account for the upper undef elements?
2519     // Requires additional vsetvlis, but might be faster to execute.
2520     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2521     SDValue SlideDown =
2522         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
2523                     DAG.getUNDEF(ContainerVT), V1,
2524                     DAG.getConstant(SlideAmt, DL, XLenVT),
2525                     TrueMask, VL);
2526     return convertFromScalableVector(VT, SlideDown, DAG, Subtarget);
2527   }
2528 
2529   // Detect an interleave shuffle and lower to
2530   // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
2531   bool SwapSources;
2532   if (isInterleaveShuffle(Mask, VT, SwapSources, Subtarget)) {
2533     // Swap sources if needed.
2534     if (SwapSources)
2535       std::swap(V1, V2);
2536 
2537     // Extract the lower half of the vectors.
2538     MVT HalfVT = VT.getHalfNumVectorElementsVT();
2539     V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
2540                      DAG.getConstant(0, DL, XLenVT));
2541     V2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V2,
2542                      DAG.getConstant(0, DL, XLenVT));
2543 
2544     // Double the element width and halve the number of elements in an int type.
2545     unsigned EltBits = VT.getScalarSizeInBits();
2546     MVT WideIntEltVT = MVT::getIntegerVT(EltBits * 2);
2547     MVT WideIntVT =
2548         MVT::getVectorVT(WideIntEltVT, VT.getVectorNumElements() / 2);
2549     // Convert this to a scalable vector. We need to base this on the
2550     // destination size to ensure there's always a type with a smaller LMUL.
2551     MVT WideIntContainerVT =
2552         getContainerForFixedLengthVector(DAG, WideIntVT, Subtarget);
2553 
2554     // Convert sources to scalable vectors with the same element count as the
2555     // larger type.
2556     MVT HalfContainerVT = MVT::getVectorVT(
2557         VT.getVectorElementType(), WideIntContainerVT.getVectorElementCount());
2558     V1 = convertToScalableVector(HalfContainerVT, V1, DAG, Subtarget);
2559     V2 = convertToScalableVector(HalfContainerVT, V2, DAG, Subtarget);
2560 
2561     // Cast sources to integer.
2562     MVT IntEltVT = MVT::getIntegerVT(EltBits);
2563     MVT IntHalfVT =
2564         MVT::getVectorVT(IntEltVT, HalfContainerVT.getVectorElementCount());
2565     V1 = DAG.getBitcast(IntHalfVT, V1);
2566     V2 = DAG.getBitcast(IntHalfVT, V2);
2567 
2568     // Freeze V2 since we use it twice and we need to be sure that the add and
2569     // multiply see the same value.
2570     V2 = DAG.getNode(ISD::FREEZE, DL, IntHalfVT, V2);
2571 
2572     // Recreate TrueMask using the widened type's element count.
2573     MVT MaskVT =
2574         MVT::getVectorVT(MVT::i1, HalfContainerVT.getVectorElementCount());
2575     TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
2576 
2577     // Widen V1 and V2 with 0s and add one copy of V2 to V1.
2578     SDValue Add = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideIntContainerVT, V1,
2579                               V2, TrueMask, VL);
2580     // Create 2^eltbits - 1 copies of V2 by multiplying by the largest integer.
2581     SDValue Multiplier = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntHalfVT,
2582                                      DAG.getAllOnesConstant(DL, XLenVT));
2583     SDValue WidenMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideIntContainerVT,
2584                                    V2, Multiplier, TrueMask, VL);
2585     // Add the new copies to our previous addition giving us 2^eltbits copies of
2586     // V2. This is equivalent to shifting V2 left by eltbits. This should
2587     // combine with the vwmulu.vv above to form vwmaccu.vv.
2588     Add = DAG.getNode(RISCVISD::ADD_VL, DL, WideIntContainerVT, Add, WidenMul,
2589                       TrueMask, VL);
2590     // Cast back to ContainerVT. We need to re-create a new ContainerVT in case
2591     // WideIntContainerVT is a larger fractional LMUL than implied by the fixed
2592     // vector VT.
2593     ContainerVT =
2594         MVT::getVectorVT(VT.getVectorElementType(),
2595                          WideIntContainerVT.getVectorElementCount() * 2);
2596     Add = DAG.getBitcast(ContainerVT, Add);
2597     return convertFromScalableVector(VT, Add, DAG, Subtarget);
2598   }
2599 
2600   // Detect shuffles which can be re-expressed as vector selects; these are
2601   // shuffles in which each element in the destination is taken from an element
2602   // at the corresponding index in either source vectors.
2603   bool IsSelect = all_of(enumerate(Mask), [&](const auto &MaskIdx) {
2604     int MaskIndex = MaskIdx.value();
2605     return MaskIndex < 0 || MaskIdx.index() == (unsigned)MaskIndex % NumElts;
2606   });
2607 
2608   assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
2609 
2610   SmallVector<SDValue> MaskVals;
2611   // As a backup, shuffles can be lowered via a vrgather instruction, possibly
2612   // merged with a second vrgather.
2613   SmallVector<SDValue> GatherIndicesLHS, GatherIndicesRHS;
2614 
2615   // By default we preserve the original operand order, and use a mask to
2616   // select LHS as true and RHS as false. However, since RVV vector selects may
2617   // feature splats but only on the LHS, we may choose to invert our mask and
2618   // instead select between RHS and LHS.
2619   bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
2620   bool InvertMask = IsSelect == SwapOps;
2621 
2622   // Keep a track of which non-undef indices are used by each LHS/RHS shuffle
2623   // half.
2624   DenseMap<int, unsigned> LHSIndexCounts, RHSIndexCounts;
2625 
2626   // Now construct the mask that will be used by the vselect or blended
2627   // vrgather operation. For vrgathers, construct the appropriate indices into
2628   // each vector.
2629   for (int MaskIndex : Mask) {
2630     bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ InvertMask;
2631     MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
2632     if (!IsSelect) {
2633       bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
2634       GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
2635                                      ? DAG.getConstant(MaskIndex, DL, XLenVT)
2636                                      : DAG.getUNDEF(XLenVT));
2637       GatherIndicesRHS.push_back(
2638           IsLHSOrUndefIndex ? DAG.getUNDEF(XLenVT)
2639                             : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT));
2640       if (IsLHSOrUndefIndex && MaskIndex >= 0)
2641         ++LHSIndexCounts[MaskIndex];
2642       if (!IsLHSOrUndefIndex)
2643         ++RHSIndexCounts[MaskIndex - NumElts];
2644     }
2645   }
2646 
2647   if (SwapOps) {
2648     std::swap(V1, V2);
2649     std::swap(GatherIndicesLHS, GatherIndicesRHS);
2650   }
2651 
2652   assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
2653   MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
2654   SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
2655 
2656   if (IsSelect)
2657     return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
2658 
2659   if (VT.getScalarSizeInBits() == 8 && VT.getVectorNumElements() > 256) {
2660     // On such a large vector we're unable to use i8 as the index type.
2661     // FIXME: We could promote the index to i16 and use vrgatherei16, but that
2662     // may involve vector splitting if we're already at LMUL=8, or our
2663     // user-supplied maximum fixed-length LMUL.
2664     return SDValue();
2665   }
2666 
2667   unsigned GatherVXOpc = RISCVISD::VRGATHER_VX_VL;
2668   unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
2669   MVT IndexVT = VT.changeTypeToInteger();
2670   // Since we can't introduce illegal index types at this stage, use i16 and
2671   // vrgatherei16 if the corresponding index type for plain vrgather is greater
2672   // than XLenVT.
2673   if (IndexVT.getScalarType().bitsGT(XLenVT)) {
2674     GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
2675     IndexVT = IndexVT.changeVectorElementType(MVT::i16);
2676   }
2677 
2678   MVT IndexContainerVT =
2679       ContainerVT.changeVectorElementType(IndexVT.getScalarType());
2680 
2681   SDValue Gather;
2682   // TODO: This doesn't trigger for i64 vectors on RV32, since there we
2683   // encounter a bitcasted BUILD_VECTOR with low/high i32 values.
2684   if (SDValue SplatValue = DAG.getSplatValue(V1, /*LegalTypes*/ true)) {
2685     Gather = lowerScalarSplat(SplatValue, VL, ContainerVT, DL, DAG, Subtarget);
2686   } else {
2687     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2688     // If only one index is used, we can use a "splat" vrgather.
2689     // TODO: We can splat the most-common index and fix-up any stragglers, if
2690     // that's beneficial.
2691     if (LHSIndexCounts.size() == 1) {
2692       int SplatIndex = LHSIndexCounts.begin()->getFirst();
2693       Gather =
2694           DAG.getNode(GatherVXOpc, DL, ContainerVT, V1,
2695                       DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2696     } else {
2697       SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
2698       LHSIndices =
2699           convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
2700 
2701       Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
2702                            TrueMask, VL);
2703     }
2704   }
2705 
2706   // If a second vector operand is used by this shuffle, blend it in with an
2707   // additional vrgather.
2708   if (!V2.isUndef()) {
2709     V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
2710     // If only one index is used, we can use a "splat" vrgather.
2711     // TODO: We can splat the most-common index and fix-up any stragglers, if
2712     // that's beneficial.
2713     if (RHSIndexCounts.size() == 1) {
2714       int SplatIndex = RHSIndexCounts.begin()->getFirst();
2715       V2 = DAG.getNode(GatherVXOpc, DL, ContainerVT, V2,
2716                        DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2717     } else {
2718       SDValue RHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesRHS);
2719       RHSIndices =
2720           convertToScalableVector(IndexContainerVT, RHSIndices, DAG, Subtarget);
2721       V2 = DAG.getNode(GatherVVOpc, DL, ContainerVT, V2, RHSIndices, TrueMask,
2722                        VL);
2723     }
2724 
2725     MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
2726     SelectMask =
2727         convertToScalableVector(MaskContainerVT, SelectMask, DAG, Subtarget);
2728 
2729     Gather = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, SelectMask, V2,
2730                          Gather, VL);
2731   }
2732 
2733   return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2734 }
2735 
2736 static SDValue getRVVFPExtendOrRound(SDValue Op, MVT VT, MVT ContainerVT,
2737                                      SDLoc DL, SelectionDAG &DAG,
2738                                      const RISCVSubtarget &Subtarget) {
2739   if (VT.isScalableVector())
2740     return DAG.getFPExtendOrRound(Op, DL, VT);
2741   assert(VT.isFixedLengthVector() &&
2742          "Unexpected value type for RVV FP extend/round lowering");
2743   SDValue Mask, VL;
2744   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2745   unsigned RVVOpc = ContainerVT.bitsGT(Op.getSimpleValueType())
2746                         ? RISCVISD::FP_EXTEND_VL
2747                         : RISCVISD::FP_ROUND_VL;
2748   return DAG.getNode(RVVOpc, DL, ContainerVT, Op, Mask, VL);
2749 }
2750 
2751 // Lower CTLZ_ZERO_UNDEF or CTTZ_ZERO_UNDEF by converting to FP and extracting
2752 // the exponent.
2753 static SDValue lowerCTLZ_CTTZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
2754   MVT VT = Op.getSimpleValueType();
2755   unsigned EltSize = VT.getScalarSizeInBits();
2756   SDValue Src = Op.getOperand(0);
2757   SDLoc DL(Op);
2758 
2759   // We need a FP type that can represent the value.
2760   // TODO: Use f16 for i8 when possible?
2761   MVT FloatEltVT = EltSize == 32 ? MVT::f64 : MVT::f32;
2762   MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
2763 
2764   // Legal types should have been checked in the RISCVTargetLowering
2765   // constructor.
2766   // TODO: Splitting may make sense in some cases.
2767   assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
2768          "Expected legal float type!");
2769 
2770   // For CTTZ_ZERO_UNDEF, we need to extract the lowest set bit using X & -X.
2771   // The trailing zero count is equal to log2 of this single bit value.
2772   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
2773     SDValue Neg =
2774         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
2775     Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
2776   }
2777 
2778   // We have a legal FP type, convert to it.
2779   SDValue FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
2780   // Bitcast to integer and shift the exponent to the LSB.
2781   EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
2782   SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
2783   unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
2784   SDValue Shift = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
2785                               DAG.getConstant(ShiftAmt, DL, IntVT));
2786   // Truncate back to original type to allow vnsrl.
2787   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, Shift);
2788   // The exponent contains log2 of the value in biased form.
2789   unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
2790 
2791   // For trailing zeros, we just need to subtract the bias.
2792   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF)
2793     return DAG.getNode(ISD::SUB, DL, VT, Trunc,
2794                        DAG.getConstant(ExponentBias, DL, VT));
2795 
2796   // For leading zeros, we need to remove the bias and convert from log2 to
2797   // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
2798   unsigned Adjust = ExponentBias + (EltSize - 1);
2799   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Trunc);
2800 }
2801 
2802 // While RVV has alignment restrictions, we should always be able to load as a
2803 // legal equivalently-sized byte-typed vector instead. This method is
2804 // responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
2805 // the load is already correctly-aligned, it returns SDValue().
2806 SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
2807                                                     SelectionDAG &DAG) const {
2808   auto *Load = cast<LoadSDNode>(Op);
2809   assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
2810 
2811   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2812                                      Load->getMemoryVT(),
2813                                      *Load->getMemOperand()))
2814     return SDValue();
2815 
2816   SDLoc DL(Op);
2817   MVT VT = Op.getSimpleValueType();
2818   unsigned EltSizeBits = VT.getScalarSizeInBits();
2819   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2820          "Unexpected unaligned RVV load type");
2821   MVT NewVT =
2822       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2823   assert(NewVT.isValid() &&
2824          "Expecting equally-sized RVV vector types to be legal");
2825   SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
2826                           Load->getPointerInfo(), Load->getOriginalAlign(),
2827                           Load->getMemOperand()->getFlags());
2828   return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
2829 }
2830 
2831 // While RVV has alignment restrictions, we should always be able to store as a
2832 // legal equivalently-sized byte-typed vector instead. This method is
2833 // responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
2834 // returns SDValue() if the store is already correctly aligned.
2835 SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
2836                                                      SelectionDAG &DAG) const {
2837   auto *Store = cast<StoreSDNode>(Op);
2838   assert(Store && Store->getValue().getValueType().isVector() &&
2839          "Expected vector store");
2840 
2841   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2842                                      Store->getMemoryVT(),
2843                                      *Store->getMemOperand()))
2844     return SDValue();
2845 
2846   SDLoc DL(Op);
2847   SDValue StoredVal = Store->getValue();
2848   MVT VT = StoredVal.getSimpleValueType();
2849   unsigned EltSizeBits = VT.getScalarSizeInBits();
2850   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2851          "Unexpected unaligned RVV store type");
2852   MVT NewVT =
2853       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2854   assert(NewVT.isValid() &&
2855          "Expecting equally-sized RVV vector types to be legal");
2856   StoredVal = DAG.getBitcast(NewVT, StoredVal);
2857   return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
2858                       Store->getPointerInfo(), Store->getOriginalAlign(),
2859                       Store->getMemOperand()->getFlags());
2860 }
2861 
2862 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
2863                                             SelectionDAG &DAG) const {
2864   switch (Op.getOpcode()) {
2865   default:
2866     report_fatal_error("unimplemented operand");
2867   case ISD::GlobalAddress:
2868     return lowerGlobalAddress(Op, DAG);
2869   case ISD::BlockAddress:
2870     return lowerBlockAddress(Op, DAG);
2871   case ISD::ConstantPool:
2872     return lowerConstantPool(Op, DAG);
2873   case ISD::JumpTable:
2874     return lowerJumpTable(Op, DAG);
2875   case ISD::GlobalTLSAddress:
2876     return lowerGlobalTLSAddress(Op, DAG);
2877   case ISD::SELECT:
2878     return lowerSELECT(Op, DAG);
2879   case ISD::BRCOND:
2880     return lowerBRCOND(Op, DAG);
2881   case ISD::VASTART:
2882     return lowerVASTART(Op, DAG);
2883   case ISD::FRAMEADDR:
2884     return lowerFRAMEADDR(Op, DAG);
2885   case ISD::RETURNADDR:
2886     return lowerRETURNADDR(Op, DAG);
2887   case ISD::SHL_PARTS:
2888     return lowerShiftLeftParts(Op, DAG);
2889   case ISD::SRA_PARTS:
2890     return lowerShiftRightParts(Op, DAG, true);
2891   case ISD::SRL_PARTS:
2892     return lowerShiftRightParts(Op, DAG, false);
2893   case ISD::BITCAST: {
2894     SDLoc DL(Op);
2895     EVT VT = Op.getValueType();
2896     SDValue Op0 = Op.getOperand(0);
2897     EVT Op0VT = Op0.getValueType();
2898     MVT XLenVT = Subtarget.getXLenVT();
2899     if (VT.isFixedLengthVector()) {
2900       // We can handle fixed length vector bitcasts with a simple replacement
2901       // in isel.
2902       if (Op0VT.isFixedLengthVector())
2903         return Op;
2904       // When bitcasting from scalar to fixed-length vector, insert the scalar
2905       // into a one-element vector of the result type, and perform a vector
2906       // bitcast.
2907       if (!Op0VT.isVector()) {
2908         EVT BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
2909         if (!isTypeLegal(BVT))
2910           return SDValue();
2911         return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
2912                                               DAG.getUNDEF(BVT), Op0,
2913                                               DAG.getConstant(0, DL, XLenVT)));
2914       }
2915       return SDValue();
2916     }
2917     // Custom-legalize bitcasts from fixed-length vector types to scalar types
2918     // thus: bitcast the vector to a one-element vector type whose element type
2919     // is the same as the result type, and extract the first element.
2920     if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
2921       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
2922       if (!isTypeLegal(BVT))
2923         return SDValue();
2924       SDValue BVec = DAG.getBitcast(BVT, Op0);
2925       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
2926                          DAG.getConstant(0, DL, XLenVT));
2927     }
2928     if (VT == MVT::f16 && Op0VT == MVT::i16 && Subtarget.hasStdExtZfh()) {
2929       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
2930       SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
2931       return FPConv;
2932     }
2933     if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
2934         Subtarget.hasStdExtF()) {
2935       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
2936       SDValue FPConv =
2937           DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
2938       return FPConv;
2939     }
2940     return SDValue();
2941   }
2942   case ISD::INTRINSIC_WO_CHAIN:
2943     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
2944   case ISD::INTRINSIC_W_CHAIN:
2945     return LowerINTRINSIC_W_CHAIN(Op, DAG);
2946   case ISD::INTRINSIC_VOID:
2947     return LowerINTRINSIC_VOID(Op, DAG);
2948   case ISD::BSWAP:
2949   case ISD::BITREVERSE: {
2950     // Convert BSWAP/BITREVERSE to GREVI to enable GREVI combinining.
2951     assert(Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
2952     MVT VT = Op.getSimpleValueType();
2953     SDLoc DL(Op);
2954     // Start with the maximum immediate value which is the bitwidth - 1.
2955     unsigned Imm = VT.getSizeInBits() - 1;
2956     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
2957     if (Op.getOpcode() == ISD::BSWAP)
2958       Imm &= ~0x7U;
2959     return DAG.getNode(RISCVISD::GREV, DL, VT, Op.getOperand(0),
2960                        DAG.getConstant(Imm, DL, VT));
2961   }
2962   case ISD::FSHL:
2963   case ISD::FSHR: {
2964     MVT VT = Op.getSimpleValueType();
2965     assert(VT == Subtarget.getXLenVT() && "Unexpected custom legalization");
2966     SDLoc DL(Op);
2967     // FSL/FSR take a log2(XLen)+1 bit shift amount but XLenVT FSHL/FSHR only
2968     // use log(XLen) bits. Mask the shift amount accordingly to prevent
2969     // accidentally setting the extra bit.
2970     unsigned ShAmtWidth = Subtarget.getXLen() - 1;
2971     SDValue ShAmt = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(2),
2972                                 DAG.getConstant(ShAmtWidth, DL, VT));
2973     // fshl and fshr concatenate their operands in the same order. fsr and fsl
2974     // instruction use different orders. fshl will return its first operand for
2975     // shift of zero, fshr will return its second operand. fsl and fsr both
2976     // return rs1 so the ISD nodes need to have different operand orders.
2977     // Shift amount is in rs2.
2978     SDValue Op0 = Op.getOperand(0);
2979     SDValue Op1 = Op.getOperand(1);
2980     unsigned Opc = RISCVISD::FSL;
2981     if (Op.getOpcode() == ISD::FSHR) {
2982       std::swap(Op0, Op1);
2983       Opc = RISCVISD::FSR;
2984     }
2985     return DAG.getNode(Opc, DL, VT, Op0, Op1, ShAmt);
2986   }
2987   case ISD::TRUNCATE: {
2988     SDLoc DL(Op);
2989     MVT VT = Op.getSimpleValueType();
2990     // Only custom-lower vector truncates
2991     if (!VT.isVector())
2992       return Op;
2993 
2994     // Truncates to mask types are handled differently
2995     if (VT.getVectorElementType() == MVT::i1)
2996       return lowerVectorMaskTrunc(Op, DAG);
2997 
2998     // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
2999     // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
3000     // truncate by one power of two at a time.
3001     MVT DstEltVT = VT.getVectorElementType();
3002 
3003     SDValue Src = Op.getOperand(0);
3004     MVT SrcVT = Src.getSimpleValueType();
3005     MVT SrcEltVT = SrcVT.getVectorElementType();
3006 
3007     assert(DstEltVT.bitsLT(SrcEltVT) &&
3008            isPowerOf2_64(DstEltVT.getSizeInBits()) &&
3009            isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
3010            "Unexpected vector truncate lowering");
3011 
3012     MVT ContainerVT = SrcVT;
3013     if (SrcVT.isFixedLengthVector()) {
3014       ContainerVT = getContainerForFixedLengthVector(SrcVT);
3015       Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3016     }
3017 
3018     SDValue Result = Src;
3019     SDValue Mask, VL;
3020     std::tie(Mask, VL) =
3021         getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
3022     LLVMContext &Context = *DAG.getContext();
3023     const ElementCount Count = ContainerVT.getVectorElementCount();
3024     do {
3025       SrcEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
3026       EVT ResultVT = EVT::getVectorVT(Context, SrcEltVT, Count);
3027       Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
3028                            Mask, VL);
3029     } while (SrcEltVT != DstEltVT);
3030 
3031     if (SrcVT.isFixedLengthVector())
3032       Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
3033 
3034     return Result;
3035   }
3036   case ISD::ANY_EXTEND:
3037   case ISD::ZERO_EXTEND:
3038     if (Op.getOperand(0).getValueType().isVector() &&
3039         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3040       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
3041     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
3042   case ISD::SIGN_EXTEND:
3043     if (Op.getOperand(0).getValueType().isVector() &&
3044         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3045       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
3046     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
3047   case ISD::SPLAT_VECTOR_PARTS:
3048     return lowerSPLAT_VECTOR_PARTS(Op, DAG);
3049   case ISD::INSERT_VECTOR_ELT:
3050     return lowerINSERT_VECTOR_ELT(Op, DAG);
3051   case ISD::EXTRACT_VECTOR_ELT:
3052     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
3053   case ISD::VSCALE: {
3054     MVT VT = Op.getSimpleValueType();
3055     SDLoc DL(Op);
3056     SDValue VLENB = DAG.getNode(RISCVISD::READ_VLENB, DL, VT);
3057     // We define our scalable vector types for lmul=1 to use a 64 bit known
3058     // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
3059     // vscale as VLENB / 8.
3060     static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
3061     if (isa<ConstantSDNode>(Op.getOperand(0))) {
3062       // We assume VLENB is a multiple of 8. We manually choose the best shift
3063       // here because SimplifyDemandedBits isn't always able to simplify it.
3064       uint64_t Val = Op.getConstantOperandVal(0);
3065       if (isPowerOf2_64(Val)) {
3066         uint64_t Log2 = Log2_64(Val);
3067         if (Log2 < 3)
3068           return DAG.getNode(ISD::SRL, DL, VT, VLENB,
3069                              DAG.getConstant(3 - Log2, DL, VT));
3070         if (Log2 > 3)
3071           return DAG.getNode(ISD::SHL, DL, VT, VLENB,
3072                              DAG.getConstant(Log2 - 3, DL, VT));
3073         return VLENB;
3074       }
3075       // If the multiplier is a multiple of 8, scale it down to avoid needing
3076       // to shift the VLENB value.
3077       if ((Val % 8) == 0)
3078         return DAG.getNode(ISD::MUL, DL, VT, VLENB,
3079                            DAG.getConstant(Val / 8, DL, VT));
3080     }
3081 
3082     SDValue VScale = DAG.getNode(ISD::SRL, DL, VT, VLENB,
3083                                  DAG.getConstant(3, DL, VT));
3084     return DAG.getNode(ISD::MUL, DL, VT, VScale, Op.getOperand(0));
3085   }
3086   case ISD::FPOWI: {
3087     // Custom promote f16 powi with illegal i32 integer type on RV64. Once
3088     // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
3089     if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
3090         Op.getOperand(1).getValueType() == MVT::i32) {
3091       SDLoc DL(Op);
3092       SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
3093       SDValue Powi =
3094           DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
3095       return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
3096                          DAG.getIntPtrConstant(0, DL));
3097     }
3098     return SDValue();
3099   }
3100   case ISD::FP_EXTEND: {
3101     // RVV can only do fp_extend to types double the size as the source. We
3102     // custom-lower f16->f64 extensions to two hops of ISD::FP_EXTEND, going
3103     // via f32.
3104     SDLoc DL(Op);
3105     MVT VT = Op.getSimpleValueType();
3106     SDValue Src = Op.getOperand(0);
3107     MVT SrcVT = Src.getSimpleValueType();
3108 
3109     // Prepare any fixed-length vector operands.
3110     MVT ContainerVT = VT;
3111     if (SrcVT.isFixedLengthVector()) {
3112       ContainerVT = getContainerForFixedLengthVector(VT);
3113       MVT SrcContainerVT =
3114           ContainerVT.changeVectorElementType(SrcVT.getVectorElementType());
3115       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3116     }
3117 
3118     if (!VT.isVector() || VT.getVectorElementType() != MVT::f64 ||
3119         SrcVT.getVectorElementType() != MVT::f16) {
3120       // For scalable vectors, we only need to close the gap between
3121       // vXf16->vXf64.
3122       if (!VT.isFixedLengthVector())
3123         return Op;
3124       // For fixed-length vectors, lower the FP_EXTEND to a custom "VL" version.
3125       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
3126       return convertFromScalableVector(VT, Src, DAG, Subtarget);
3127     }
3128 
3129     MVT InterVT = VT.changeVectorElementType(MVT::f32);
3130     MVT InterContainerVT = ContainerVT.changeVectorElementType(MVT::f32);
3131     SDValue IntermediateExtend = getRVVFPExtendOrRound(
3132         Src, InterVT, InterContainerVT, DL, DAG, Subtarget);
3133 
3134     SDValue Extend = getRVVFPExtendOrRound(IntermediateExtend, VT, ContainerVT,
3135                                            DL, DAG, Subtarget);
3136     if (VT.isFixedLengthVector())
3137       return convertFromScalableVector(VT, Extend, DAG, Subtarget);
3138     return Extend;
3139   }
3140   case ISD::FP_ROUND: {
3141     // RVV can only do fp_round to types half the size as the source. We
3142     // custom-lower f64->f16 rounds via RVV's round-to-odd float
3143     // conversion instruction.
3144     SDLoc DL(Op);
3145     MVT VT = Op.getSimpleValueType();
3146     SDValue Src = Op.getOperand(0);
3147     MVT SrcVT = Src.getSimpleValueType();
3148 
3149     // Prepare any fixed-length vector operands.
3150     MVT ContainerVT = VT;
3151     if (VT.isFixedLengthVector()) {
3152       MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3153       ContainerVT =
3154           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3155       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3156     }
3157 
3158     if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
3159         SrcVT.getVectorElementType() != MVT::f64) {
3160       // For scalable vectors, we only need to close the gap between
3161       // vXf64<->vXf16.
3162       if (!VT.isFixedLengthVector())
3163         return Op;
3164       // For fixed-length vectors, lower the FP_ROUND to a custom "VL" version.
3165       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
3166       return convertFromScalableVector(VT, Src, DAG, Subtarget);
3167     }
3168 
3169     SDValue Mask, VL;
3170     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3171 
3172     MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
3173     SDValue IntermediateRound =
3174         DAG.getNode(RISCVISD::VFNCVT_ROD_VL, DL, InterVT, Src, Mask, VL);
3175     SDValue Round = getRVVFPExtendOrRound(IntermediateRound, VT, ContainerVT,
3176                                           DL, DAG, Subtarget);
3177 
3178     if (VT.isFixedLengthVector())
3179       return convertFromScalableVector(VT, Round, DAG, Subtarget);
3180     return Round;
3181   }
3182   case ISD::FP_TO_SINT:
3183   case ISD::FP_TO_UINT:
3184   case ISD::SINT_TO_FP:
3185   case ISD::UINT_TO_FP: {
3186     // RVV can only do fp<->int conversions to types half/double the size as
3187     // the source. We custom-lower any conversions that do two hops into
3188     // sequences.
3189     MVT VT = Op.getSimpleValueType();
3190     if (!VT.isVector())
3191       return Op;
3192     SDLoc DL(Op);
3193     SDValue Src = Op.getOperand(0);
3194     MVT EltVT = VT.getVectorElementType();
3195     MVT SrcVT = Src.getSimpleValueType();
3196     MVT SrcEltVT = SrcVT.getVectorElementType();
3197     unsigned EltSize = EltVT.getSizeInBits();
3198     unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3199     assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
3200            "Unexpected vector element types");
3201 
3202     bool IsInt2FP = SrcEltVT.isInteger();
3203     // Widening conversions
3204     if (EltSize > SrcEltSize && (EltSize / SrcEltSize >= 4)) {
3205       if (IsInt2FP) {
3206         // Do a regular integer sign/zero extension then convert to float.
3207         MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltVT.getSizeInBits()),
3208                                       VT.getVectorElementCount());
3209         unsigned ExtOpcode = Op.getOpcode() == ISD::UINT_TO_FP
3210                                  ? ISD::ZERO_EXTEND
3211                                  : ISD::SIGN_EXTEND;
3212         SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
3213         return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
3214       }
3215       // FP2Int
3216       assert(SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering");
3217       // Do one doubling fp_extend then complete the operation by converting
3218       // to int.
3219       MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3220       SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
3221       return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
3222     }
3223 
3224     // Narrowing conversions
3225     if (SrcEltSize > EltSize && (SrcEltSize / EltSize >= 4)) {
3226       if (IsInt2FP) {
3227         // One narrowing int_to_fp, then an fp_round.
3228         assert(EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering");
3229         MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3230         SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
3231         return DAG.getFPExtendOrRound(Int2FP, DL, VT);
3232       }
3233       // FP2Int
3234       // One narrowing fp_to_int, then truncate the integer. If the float isn't
3235       // representable by the integer, the result is poison.
3236       MVT IVecVT =
3237           MVT::getVectorVT(MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2),
3238                            VT.getVectorElementCount());
3239       SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
3240       return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
3241     }
3242 
3243     // Scalable vectors can exit here. Patterns will handle equally-sized
3244     // conversions halving/doubling ones.
3245     if (!VT.isFixedLengthVector())
3246       return Op;
3247 
3248     // For fixed-length vectors we lower to a custom "VL" node.
3249     unsigned RVVOpc = 0;
3250     switch (Op.getOpcode()) {
3251     default:
3252       llvm_unreachable("Impossible opcode");
3253     case ISD::FP_TO_SINT:
3254       RVVOpc = RISCVISD::FP_TO_SINT_VL;
3255       break;
3256     case ISD::FP_TO_UINT:
3257       RVVOpc = RISCVISD::FP_TO_UINT_VL;
3258       break;
3259     case ISD::SINT_TO_FP:
3260       RVVOpc = RISCVISD::SINT_TO_FP_VL;
3261       break;
3262     case ISD::UINT_TO_FP:
3263       RVVOpc = RISCVISD::UINT_TO_FP_VL;
3264       break;
3265     }
3266 
3267     MVT ContainerVT, SrcContainerVT;
3268     // Derive the reference container type from the larger vector type.
3269     if (SrcEltSize > EltSize) {
3270       SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3271       ContainerVT =
3272           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3273     } else {
3274       ContainerVT = getContainerForFixedLengthVector(VT);
3275       SrcContainerVT = ContainerVT.changeVectorElementType(SrcEltVT);
3276     }
3277 
3278     SDValue Mask, VL;
3279     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3280 
3281     Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3282     Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
3283     return convertFromScalableVector(VT, Src, DAG, Subtarget);
3284   }
3285   case ISD::FP_TO_SINT_SAT:
3286   case ISD::FP_TO_UINT_SAT:
3287     return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
3288   case ISD::FTRUNC:
3289   case ISD::FCEIL:
3290   case ISD::FFLOOR:
3291     return lowerFTRUNC_FCEIL_FFLOOR(Op, DAG);
3292   case ISD::VECREDUCE_ADD:
3293   case ISD::VECREDUCE_UMAX:
3294   case ISD::VECREDUCE_SMAX:
3295   case ISD::VECREDUCE_UMIN:
3296   case ISD::VECREDUCE_SMIN:
3297     return lowerVECREDUCE(Op, DAG);
3298   case ISD::VECREDUCE_AND:
3299   case ISD::VECREDUCE_OR:
3300   case ISD::VECREDUCE_XOR:
3301     if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3302       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
3303     return lowerVECREDUCE(Op, DAG);
3304   case ISD::VECREDUCE_FADD:
3305   case ISD::VECREDUCE_SEQ_FADD:
3306   case ISD::VECREDUCE_FMIN:
3307   case ISD::VECREDUCE_FMAX:
3308     return lowerFPVECREDUCE(Op, DAG);
3309   case ISD::VP_REDUCE_ADD:
3310   case ISD::VP_REDUCE_UMAX:
3311   case ISD::VP_REDUCE_SMAX:
3312   case ISD::VP_REDUCE_UMIN:
3313   case ISD::VP_REDUCE_SMIN:
3314   case ISD::VP_REDUCE_FADD:
3315   case ISD::VP_REDUCE_SEQ_FADD:
3316   case ISD::VP_REDUCE_FMIN:
3317   case ISD::VP_REDUCE_FMAX:
3318     return lowerVPREDUCE(Op, DAG);
3319   case ISD::VP_REDUCE_AND:
3320   case ISD::VP_REDUCE_OR:
3321   case ISD::VP_REDUCE_XOR:
3322     if (Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
3323       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
3324     return lowerVPREDUCE(Op, DAG);
3325   case ISD::INSERT_SUBVECTOR:
3326     return lowerINSERT_SUBVECTOR(Op, DAG);
3327   case ISD::EXTRACT_SUBVECTOR:
3328     return lowerEXTRACT_SUBVECTOR(Op, DAG);
3329   case ISD::STEP_VECTOR:
3330     return lowerSTEP_VECTOR(Op, DAG);
3331   case ISD::VECTOR_REVERSE:
3332     return lowerVECTOR_REVERSE(Op, DAG);
3333   case ISD::BUILD_VECTOR:
3334     return lowerBUILD_VECTOR(Op, DAG, Subtarget);
3335   case ISD::SPLAT_VECTOR:
3336     if (Op.getValueType().getVectorElementType() == MVT::i1)
3337       return lowerVectorMaskSplat(Op, DAG);
3338     return lowerSPLAT_VECTOR(Op, DAG, Subtarget);
3339   case ISD::VECTOR_SHUFFLE:
3340     return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
3341   case ISD::CONCAT_VECTORS: {
3342     // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
3343     // better than going through the stack, as the default expansion does.
3344     SDLoc DL(Op);
3345     MVT VT = Op.getSimpleValueType();
3346     unsigned NumOpElts =
3347         Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
3348     SDValue Vec = DAG.getUNDEF(VT);
3349     for (const auto &OpIdx : enumerate(Op->ops())) {
3350       SDValue SubVec = OpIdx.value();
3351       // Don't insert undef subvectors.
3352       if (SubVec.isUndef())
3353         continue;
3354       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec,
3355                         DAG.getIntPtrConstant(OpIdx.index() * NumOpElts, DL));
3356     }
3357     return Vec;
3358   }
3359   case ISD::LOAD:
3360     if (auto V = expandUnalignedRVVLoad(Op, DAG))
3361       return V;
3362     if (Op.getValueType().isFixedLengthVector())
3363       return lowerFixedLengthVectorLoadToRVV(Op, DAG);
3364     return Op;
3365   case ISD::STORE:
3366     if (auto V = expandUnalignedRVVStore(Op, DAG))
3367       return V;
3368     if (Op.getOperand(1).getValueType().isFixedLengthVector())
3369       return lowerFixedLengthVectorStoreToRVV(Op, DAG);
3370     return Op;
3371   case ISD::MLOAD:
3372   case ISD::VP_LOAD:
3373     return lowerMaskedLoad(Op, DAG);
3374   case ISD::MSTORE:
3375   case ISD::VP_STORE:
3376     return lowerMaskedStore(Op, DAG);
3377   case ISD::SETCC:
3378     return lowerFixedLengthVectorSetccToRVV(Op, DAG);
3379   case ISD::ADD:
3380     return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);
3381   case ISD::SUB:
3382     return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL);
3383   case ISD::MUL:
3384     return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL);
3385   case ISD::MULHS:
3386     return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL);
3387   case ISD::MULHU:
3388     return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL);
3389   case ISD::AND:
3390     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMAND_VL,
3391                                               RISCVISD::AND_VL);
3392   case ISD::OR:
3393     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMOR_VL,
3394                                               RISCVISD::OR_VL);
3395   case ISD::XOR:
3396     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMXOR_VL,
3397                                               RISCVISD::XOR_VL);
3398   case ISD::SDIV:
3399     return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL);
3400   case ISD::SREM:
3401     return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL);
3402   case ISD::UDIV:
3403     return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL);
3404   case ISD::UREM:
3405     return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL);
3406   case ISD::SHL:
3407   case ISD::SRA:
3408   case ISD::SRL:
3409     if (Op.getSimpleValueType().isFixedLengthVector())
3410       return lowerFixedLengthVectorShiftToRVV(Op, DAG);
3411     // This can be called for an i32 shift amount that needs to be promoted.
3412     assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
3413            "Unexpected custom legalisation");
3414     return SDValue();
3415   case ISD::SADDSAT:
3416     return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL);
3417   case ISD::UADDSAT:
3418     return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL);
3419   case ISD::SSUBSAT:
3420     return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL);
3421   case ISD::USUBSAT:
3422     return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL);
3423   case ISD::FADD:
3424     return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL);
3425   case ISD::FSUB:
3426     return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL);
3427   case ISD::FMUL:
3428     return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL);
3429   case ISD::FDIV:
3430     return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL);
3431   case ISD::FNEG:
3432     return lowerToScalableOp(Op, DAG, RISCVISD::FNEG_VL);
3433   case ISD::FABS:
3434     return lowerToScalableOp(Op, DAG, RISCVISD::FABS_VL);
3435   case ISD::FSQRT:
3436     return lowerToScalableOp(Op, DAG, RISCVISD::FSQRT_VL);
3437   case ISD::FMA:
3438     return lowerToScalableOp(Op, DAG, RISCVISD::FMA_VL);
3439   case ISD::SMIN:
3440     return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL);
3441   case ISD::SMAX:
3442     return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL);
3443   case ISD::UMIN:
3444     return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL);
3445   case ISD::UMAX:
3446     return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL);
3447   case ISD::FMINNUM:
3448     return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL);
3449   case ISD::FMAXNUM:
3450     return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL);
3451   case ISD::ABS:
3452     return lowerABS(Op, DAG);
3453   case ISD::CTLZ_ZERO_UNDEF:
3454   case ISD::CTTZ_ZERO_UNDEF:
3455     return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
3456   case ISD::VSELECT:
3457     return lowerFixedLengthVectorSelectToRVV(Op, DAG);
3458   case ISD::FCOPYSIGN:
3459     return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG);
3460   case ISD::MGATHER:
3461   case ISD::VP_GATHER:
3462     return lowerMaskedGather(Op, DAG);
3463   case ISD::MSCATTER:
3464   case ISD::VP_SCATTER:
3465     return lowerMaskedScatter(Op, DAG);
3466   case ISD::FLT_ROUNDS_:
3467     return lowerGET_ROUNDING(Op, DAG);
3468   case ISD::SET_ROUNDING:
3469     return lowerSET_ROUNDING(Op, DAG);
3470   case ISD::VP_SELECT:
3471     return lowerVPOp(Op, DAG, RISCVISD::VSELECT_VL);
3472   case ISD::VP_MERGE:
3473     return lowerVPOp(Op, DAG, RISCVISD::VP_MERGE_VL);
3474   case ISD::VP_ADD:
3475     return lowerVPOp(Op, DAG, RISCVISD::ADD_VL);
3476   case ISD::VP_SUB:
3477     return lowerVPOp(Op, DAG, RISCVISD::SUB_VL);
3478   case ISD::VP_MUL:
3479     return lowerVPOp(Op, DAG, RISCVISD::MUL_VL);
3480   case ISD::VP_SDIV:
3481     return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL);
3482   case ISD::VP_UDIV:
3483     return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL);
3484   case ISD::VP_SREM:
3485     return lowerVPOp(Op, DAG, RISCVISD::SREM_VL);
3486   case ISD::VP_UREM:
3487     return lowerVPOp(Op, DAG, RISCVISD::UREM_VL);
3488   case ISD::VP_AND:
3489     return lowerLogicVPOp(Op, DAG, RISCVISD::VMAND_VL, RISCVISD::AND_VL);
3490   case ISD::VP_OR:
3491     return lowerLogicVPOp(Op, DAG, RISCVISD::VMOR_VL, RISCVISD::OR_VL);
3492   case ISD::VP_XOR:
3493     return lowerLogicVPOp(Op, DAG, RISCVISD::VMXOR_VL, RISCVISD::XOR_VL);
3494   case ISD::VP_ASHR:
3495     return lowerVPOp(Op, DAG, RISCVISD::SRA_VL);
3496   case ISD::VP_LSHR:
3497     return lowerVPOp(Op, DAG, RISCVISD::SRL_VL);
3498   case ISD::VP_SHL:
3499     return lowerVPOp(Op, DAG, RISCVISD::SHL_VL);
3500   case ISD::VP_FADD:
3501     return lowerVPOp(Op, DAG, RISCVISD::FADD_VL);
3502   case ISD::VP_FSUB:
3503     return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL);
3504   case ISD::VP_FMUL:
3505     return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL);
3506   case ISD::VP_FDIV:
3507     return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL);
3508   }
3509 }
3510 
3511 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
3512                              SelectionDAG &DAG, unsigned Flags) {
3513   return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
3514 }
3515 
3516 static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
3517                              SelectionDAG &DAG, unsigned Flags) {
3518   return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
3519                                    Flags);
3520 }
3521 
3522 static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
3523                              SelectionDAG &DAG, unsigned Flags) {
3524   return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
3525                                    N->getOffset(), Flags);
3526 }
3527 
3528 static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
3529                              SelectionDAG &DAG, unsigned Flags) {
3530   return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
3531 }
3532 
3533 template <class NodeTy>
3534 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
3535                                      bool IsLocal) const {
3536   SDLoc DL(N);
3537   EVT Ty = getPointerTy(DAG.getDataLayout());
3538 
3539   if (isPositionIndependent()) {
3540     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3541     if (IsLocal)
3542       // Use PC-relative addressing to access the symbol. This generates the
3543       // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
3544       // %pcrel_lo(auipc)).
3545       return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3546 
3547     // Use PC-relative addressing to access the GOT for this symbol, then load
3548     // the address from the GOT. This generates the pattern (PseudoLA sym),
3549     // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
3550     return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
3551   }
3552 
3553   switch (getTargetMachine().getCodeModel()) {
3554   default:
3555     report_fatal_error("Unsupported code model for lowering");
3556   case CodeModel::Small: {
3557     // Generate a sequence for accessing addresses within the first 2 GiB of
3558     // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
3559     SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
3560     SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
3561     SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3562     return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
3563   }
3564   case CodeModel::Medium: {
3565     // Generate a sequence for accessing addresses within any 2GiB range within
3566     // the address space. This generates the pattern (PseudoLLA sym), which
3567     // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
3568     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3569     return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3570   }
3571   }
3572 }
3573 
3574 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
3575                                                 SelectionDAG &DAG) const {
3576   SDLoc DL(Op);
3577   EVT Ty = Op.getValueType();
3578   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3579   int64_t Offset = N->getOffset();
3580   MVT XLenVT = Subtarget.getXLenVT();
3581 
3582   const GlobalValue *GV = N->getGlobal();
3583   bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
3584   SDValue Addr = getAddr(N, DAG, IsLocal);
3585 
3586   // In order to maximise the opportunity for common subexpression elimination,
3587   // emit a separate ADD node for the global address offset instead of folding
3588   // it in the global address node. Later peephole optimisations may choose to
3589   // fold it back in when profitable.
3590   if (Offset != 0)
3591     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3592                        DAG.getConstant(Offset, DL, XLenVT));
3593   return Addr;
3594 }
3595 
3596 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
3597                                                SelectionDAG &DAG) const {
3598   BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
3599 
3600   return getAddr(N, DAG);
3601 }
3602 
3603 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
3604                                                SelectionDAG &DAG) const {
3605   ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
3606 
3607   return getAddr(N, DAG);
3608 }
3609 
3610 SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
3611                                             SelectionDAG &DAG) const {
3612   JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
3613 
3614   return getAddr(N, DAG);
3615 }
3616 
3617 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
3618                                               SelectionDAG &DAG,
3619                                               bool UseGOT) const {
3620   SDLoc DL(N);
3621   EVT Ty = getPointerTy(DAG.getDataLayout());
3622   const GlobalValue *GV = N->getGlobal();
3623   MVT XLenVT = Subtarget.getXLenVT();
3624 
3625   if (UseGOT) {
3626     // Use PC-relative addressing to access the GOT for this TLS symbol, then
3627     // load the address from the GOT and add the thread pointer. This generates
3628     // the pattern (PseudoLA_TLS_IE sym), which expands to
3629     // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
3630     SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3631     SDValue Load =
3632         SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
3633 
3634     // Add the thread pointer.
3635     SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3636     return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
3637   }
3638 
3639   // Generate a sequence for accessing the address relative to the thread
3640   // pointer, with the appropriate adjustment for the thread pointer offset.
3641   // This generates the pattern
3642   // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
3643   SDValue AddrHi =
3644       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
3645   SDValue AddrAdd =
3646       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
3647   SDValue AddrLo =
3648       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
3649 
3650   SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3651   SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3652   SDValue MNAdd = SDValue(
3653       DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
3654       0);
3655   return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
3656 }
3657 
3658 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
3659                                                SelectionDAG &DAG) const {
3660   SDLoc DL(N);
3661   EVT Ty = getPointerTy(DAG.getDataLayout());
3662   IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
3663   const GlobalValue *GV = N->getGlobal();
3664 
3665   // Use a PC-relative addressing mode to access the global dynamic GOT address.
3666   // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
3667   // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
3668   SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3669   SDValue Load =
3670       SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
3671 
3672   // Prepare argument list to generate call.
3673   ArgListTy Args;
3674   ArgListEntry Entry;
3675   Entry.Node = Load;
3676   Entry.Ty = CallTy;
3677   Args.push_back(Entry);
3678 
3679   // Setup call to __tls_get_addr.
3680   TargetLowering::CallLoweringInfo CLI(DAG);
3681   CLI.setDebugLoc(DL)
3682       .setChain(DAG.getEntryNode())
3683       .setLibCallee(CallingConv::C, CallTy,
3684                     DAG.getExternalSymbol("__tls_get_addr", Ty),
3685                     std::move(Args));
3686 
3687   return LowerCallTo(CLI).first;
3688 }
3689 
3690 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
3691                                                    SelectionDAG &DAG) const {
3692   SDLoc DL(Op);
3693   EVT Ty = Op.getValueType();
3694   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3695   int64_t Offset = N->getOffset();
3696   MVT XLenVT = Subtarget.getXLenVT();
3697 
3698   TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());
3699 
3700   if (DAG.getMachineFunction().getFunction().getCallingConv() ==
3701       CallingConv::GHC)
3702     report_fatal_error("In GHC calling convention TLS is not supported");
3703 
3704   SDValue Addr;
3705   switch (Model) {
3706   case TLSModel::LocalExec:
3707     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
3708     break;
3709   case TLSModel::InitialExec:
3710     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
3711     break;
3712   case TLSModel::LocalDynamic:
3713   case TLSModel::GeneralDynamic:
3714     Addr = getDynamicTLSAddr(N, DAG);
3715     break;
3716   }
3717 
3718   // In order to maximise the opportunity for common subexpression elimination,
3719   // emit a separate ADD node for the global address offset instead of folding
3720   // it in the global address node. Later peephole optimisations may choose to
3721   // fold it back in when profitable.
3722   if (Offset != 0)
3723     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3724                        DAG.getConstant(Offset, DL, XLenVT));
3725   return Addr;
3726 }
3727 
3728 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
3729   SDValue CondV = Op.getOperand(0);
3730   SDValue TrueV = Op.getOperand(1);
3731   SDValue FalseV = Op.getOperand(2);
3732   SDLoc DL(Op);
3733   MVT VT = Op.getSimpleValueType();
3734   MVT XLenVT = Subtarget.getXLenVT();
3735 
3736   // Lower vector SELECTs to VSELECTs by splatting the condition.
3737   if (VT.isVector()) {
3738     MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
3739     SDValue CondSplat = VT.isScalableVector()
3740                             ? DAG.getSplatVector(SplatCondVT, DL, CondV)
3741                             : DAG.getSplatBuildVector(SplatCondVT, DL, CondV);
3742     return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
3743   }
3744 
3745   // If the result type is XLenVT and CondV is the output of a SETCC node
3746   // which also operated on XLenVT inputs, then merge the SETCC node into the
3747   // lowered RISCVISD::SELECT_CC to take advantage of the integer
3748   // compare+branch instructions. i.e.:
3749   // (select (setcc lhs, rhs, cc), truev, falsev)
3750   // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
3751   if (VT == XLenVT && CondV.getOpcode() == ISD::SETCC &&
3752       CondV.getOperand(0).getSimpleValueType() == XLenVT) {
3753     SDValue LHS = CondV.getOperand(0);
3754     SDValue RHS = CondV.getOperand(1);
3755     const auto *CC = cast<CondCodeSDNode>(CondV.getOperand(2));
3756     ISD::CondCode CCVal = CC->get();
3757 
3758     // Special case for a select of 2 constants that have a diffence of 1.
3759     // Normally this is done by DAGCombine, but if the select is introduced by
3760     // type legalization or op legalization, we miss it. Restricting to SETLT
3761     // case for now because that is what signed saturating add/sub need.
3762     // FIXME: We don't need the condition to be SETLT or even a SETCC,
3763     // but we would probably want to swap the true/false values if the condition
3764     // is SETGE/SETLE to avoid an XORI.
3765     if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
3766         CCVal == ISD::SETLT) {
3767       const APInt &TrueVal = cast<ConstantSDNode>(TrueV)->getAPIntValue();
3768       const APInt &FalseVal = cast<ConstantSDNode>(FalseV)->getAPIntValue();
3769       if (TrueVal - 1 == FalseVal)
3770         return DAG.getNode(ISD::ADD, DL, Op.getValueType(), CondV, FalseV);
3771       if (TrueVal + 1 == FalseVal)
3772         return DAG.getNode(ISD::SUB, DL, Op.getValueType(), FalseV, CondV);
3773     }
3774 
3775     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3776 
3777     SDValue TargetCC = DAG.getCondCode(CCVal);
3778     SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
3779     return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3780   }
3781 
3782   // Otherwise:
3783   // (select condv, truev, falsev)
3784   // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
3785   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
3786   SDValue SetNE = DAG.getCondCode(ISD::SETNE);
3787 
3788   SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
3789 
3790   return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3791 }
3792 
3793 SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
3794   SDValue CondV = Op.getOperand(1);
3795   SDLoc DL(Op);
3796   MVT XLenVT = Subtarget.getXLenVT();
3797 
3798   if (CondV.getOpcode() == ISD::SETCC &&
3799       CondV.getOperand(0).getValueType() == XLenVT) {
3800     SDValue LHS = CondV.getOperand(0);
3801     SDValue RHS = CondV.getOperand(1);
3802     ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
3803 
3804     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3805 
3806     SDValue TargetCC = DAG.getCondCode(CCVal);
3807     return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3808                        LHS, RHS, TargetCC, Op.getOperand(2));
3809   }
3810 
3811   return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3812                      CondV, DAG.getConstant(0, DL, XLenVT),
3813                      DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
3814 }
3815 
3816 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3817   MachineFunction &MF = DAG.getMachineFunction();
3818   RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
3819 
3820   SDLoc DL(Op);
3821   SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
3822                                  getPointerTy(MF.getDataLayout()));
3823 
3824   // vastart just stores the address of the VarArgsFrameIndex slot into the
3825   // memory location argument.
3826   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3827   return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
3828                       MachinePointerInfo(SV));
3829 }
3830 
3831 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
3832                                             SelectionDAG &DAG) const {
3833   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3834   MachineFunction &MF = DAG.getMachineFunction();
3835   MachineFrameInfo &MFI = MF.getFrameInfo();
3836   MFI.setFrameAddressIsTaken(true);
3837   Register FrameReg = RI.getFrameRegister(MF);
3838   int XLenInBytes = Subtarget.getXLen() / 8;
3839 
3840   EVT VT = Op.getValueType();
3841   SDLoc DL(Op);
3842   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
3843   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3844   while (Depth--) {
3845     int Offset = -(XLenInBytes * 2);
3846     SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
3847                               DAG.getIntPtrConstant(Offset, DL));
3848     FrameAddr =
3849         DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
3850   }
3851   return FrameAddr;
3852 }
3853 
3854 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
3855                                              SelectionDAG &DAG) const {
3856   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3857   MachineFunction &MF = DAG.getMachineFunction();
3858   MachineFrameInfo &MFI = MF.getFrameInfo();
3859   MFI.setReturnAddressIsTaken(true);
3860   MVT XLenVT = Subtarget.getXLenVT();
3861   int XLenInBytes = Subtarget.getXLen() / 8;
3862 
3863   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
3864     return SDValue();
3865 
3866   EVT VT = Op.getValueType();
3867   SDLoc DL(Op);
3868   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3869   if (Depth) {
3870     int Off = -XLenInBytes;
3871     SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
3872     SDValue Offset = DAG.getConstant(Off, DL, VT);
3873     return DAG.getLoad(VT, DL, DAG.getEntryNode(),
3874                        DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
3875                        MachinePointerInfo());
3876   }
3877 
3878   // Return the value of the return address register, marking it an implicit
3879   // live-in.
3880   Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
3881   return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
3882 }
3883 
3884 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
3885                                                  SelectionDAG &DAG) const {
3886   SDLoc DL(Op);
3887   SDValue Lo = Op.getOperand(0);
3888   SDValue Hi = Op.getOperand(1);
3889   SDValue Shamt = Op.getOperand(2);
3890   EVT VT = Lo.getValueType();
3891 
3892   // if Shamt-XLEN < 0: // Shamt < XLEN
3893   //   Lo = Lo << Shamt
3894   //   Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
3895   // else:
3896   //   Lo = 0
3897   //   Hi = Lo << (Shamt-XLEN)
3898 
3899   SDValue Zero = DAG.getConstant(0, DL, VT);
3900   SDValue One = DAG.getConstant(1, DL, VT);
3901   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3902   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3903   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3904   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3905 
3906   SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
3907   SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
3908   SDValue ShiftRightLo =
3909       DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
3910   SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
3911   SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
3912   SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
3913 
3914   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3915 
3916   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
3917   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3918 
3919   SDValue Parts[2] = {Lo, Hi};
3920   return DAG.getMergeValues(Parts, DL);
3921 }
3922 
3923 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
3924                                                   bool IsSRA) const {
3925   SDLoc DL(Op);
3926   SDValue Lo = Op.getOperand(0);
3927   SDValue Hi = Op.getOperand(1);
3928   SDValue Shamt = Op.getOperand(2);
3929   EVT VT = Lo.getValueType();
3930 
3931   // SRA expansion:
3932   //   if Shamt-XLEN < 0: // Shamt < XLEN
3933   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3934   //     Hi = Hi >>s Shamt
3935   //   else:
3936   //     Lo = Hi >>s (Shamt-XLEN);
3937   //     Hi = Hi >>s (XLEN-1)
3938   //
3939   // SRL expansion:
3940   //   if Shamt-XLEN < 0: // Shamt < XLEN
3941   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3942   //     Hi = Hi >>u Shamt
3943   //   else:
3944   //     Lo = Hi >>u (Shamt-XLEN);
3945   //     Hi = 0;
3946 
3947   unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
3948 
3949   SDValue Zero = DAG.getConstant(0, DL, VT);
3950   SDValue One = DAG.getConstant(1, DL, VT);
3951   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3952   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3953   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3954   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3955 
3956   SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
3957   SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
3958   SDValue ShiftLeftHi =
3959       DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
3960   SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
3961   SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
3962   SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
3963   SDValue HiFalse =
3964       IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
3965 
3966   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3967 
3968   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
3969   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3970 
3971   SDValue Parts[2] = {Lo, Hi};
3972   return DAG.getMergeValues(Parts, DL);
3973 }
3974 
3975 // Lower splats of i1 types to SETCC. For each mask vector type, we have a
3976 // legal equivalently-sized i8 type, so we can use that as a go-between.
3977 SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
3978                                                   SelectionDAG &DAG) const {
3979   SDLoc DL(Op);
3980   MVT VT = Op.getSimpleValueType();
3981   SDValue SplatVal = Op.getOperand(0);
3982   // All-zeros or all-ones splats are handled specially.
3983   if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) {
3984     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
3985     return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
3986   }
3987   if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) {
3988     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
3989     return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
3990   }
3991   MVT XLenVT = Subtarget.getXLenVT();
3992   assert(SplatVal.getValueType() == XLenVT &&
3993          "Unexpected type for i1 splat value");
3994   MVT InterVT = VT.changeVectorElementType(MVT::i8);
3995   SplatVal = DAG.getNode(ISD::AND, DL, XLenVT, SplatVal,
3996                          DAG.getConstant(1, DL, XLenVT));
3997   SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
3998   SDValue Zero = DAG.getConstant(0, DL, InterVT);
3999   return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
4000 }
4001 
4002 // Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
4003 // illegal (currently only vXi64 RV32).
4004 // FIXME: We could also catch non-constant sign-extended i32 values and lower
4005 // them to SPLAT_VECTOR_I64
4006 SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
4007                                                      SelectionDAG &DAG) const {
4008   SDLoc DL(Op);
4009   MVT VecVT = Op.getSimpleValueType();
4010   assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
4011          "Unexpected SPLAT_VECTOR_PARTS lowering");
4012 
4013   assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
4014   SDValue Lo = Op.getOperand(0);
4015   SDValue Hi = Op.getOperand(1);
4016 
4017   if (VecVT.isFixedLengthVector()) {
4018     MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
4019     SDLoc DL(Op);
4020     SDValue Mask, VL;
4021     std::tie(Mask, VL) =
4022         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4023 
4024     SDValue Res = splatPartsI64WithVL(DL, ContainerVT, Lo, Hi, VL, DAG);
4025     return convertFromScalableVector(VecVT, Res, DAG, Subtarget);
4026   }
4027 
4028   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
4029     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
4030     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
4031     // If Hi constant is all the same sign bit as Lo, lower this as a custom
4032     // node in order to try and match RVV vector/scalar instructions.
4033     if ((LoC >> 31) == HiC)
4034       return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
4035   }
4036 
4037   // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
4038   if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
4039       isa<ConstantSDNode>(Hi.getOperand(1)) &&
4040       Hi.getConstantOperandVal(1) == 31)
4041     return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
4042 
4043   // Fall back to use a stack store and stride x0 vector load. Use X0 as VL.
4044   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VecVT, Lo, Hi,
4045                      DAG.getTargetConstant(RISCV::VLMaxSentinel, DL, MVT::i64));
4046 }
4047 
4048 // Custom-lower extensions from mask vectors by using a vselect either with 1
4049 // for zero/any-extension or -1 for sign-extension:
4050 //   (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
4051 // Note that any-extension is lowered identically to zero-extension.
4052 SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
4053                                                 int64_t ExtTrueVal) const {
4054   SDLoc DL(Op);
4055   MVT VecVT = Op.getSimpleValueType();
4056   SDValue Src = Op.getOperand(0);
4057   // Only custom-lower extensions from mask types
4058   assert(Src.getValueType().isVector() &&
4059          Src.getValueType().getVectorElementType() == MVT::i1);
4060 
4061   MVT XLenVT = Subtarget.getXLenVT();
4062   SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
4063   SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
4064 
4065   if (VecVT.isScalableVector()) {
4066     // Be careful not to introduce illegal scalar types at this stage, and be
4067     // careful also about splatting constants as on RV32, vXi64 SPLAT_VECTOR is
4068     // illegal and must be expanded. Since we know that the constants are
4069     // sign-extended 32-bit values, we use SPLAT_VECTOR_I64 directly.
4070     bool IsRV32E64 =
4071         !Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64;
4072 
4073     if (!IsRV32E64) {
4074       SplatZero = DAG.getSplatVector(VecVT, DL, SplatZero);
4075       SplatTrueVal = DAG.getSplatVector(VecVT, DL, SplatTrueVal);
4076     } else {
4077       SplatZero = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatZero);
4078       SplatTrueVal =
4079           DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatTrueVal);
4080     }
4081 
4082     return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
4083   }
4084 
4085   MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
4086   MVT I1ContainerVT =
4087       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4088 
4089   SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);
4090 
4091   SDValue Mask, VL;
4092   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4093 
4094   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero, VL);
4095   SplatTrueVal =
4096       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatTrueVal, VL);
4097   SDValue Select = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC,
4098                                SplatTrueVal, SplatZero, VL);
4099 
4100   return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
4101 }
4102 
4103 SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
4104     SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const {
4105   MVT ExtVT = Op.getSimpleValueType();
4106   // Only custom-lower extensions from fixed-length vector types.
4107   if (!ExtVT.isFixedLengthVector())
4108     return Op;
4109   MVT VT = Op.getOperand(0).getSimpleValueType();
4110   // Grab the canonical container type for the extended type. Infer the smaller
4111   // type from that to ensure the same number of vector elements, as we know
4112   // the LMUL will be sufficient to hold the smaller type.
4113   MVT ContainerExtVT = getContainerForFixedLengthVector(ExtVT);
4114   // Get the extended container type manually to ensure the same number of
4115   // vector elements between source and dest.
4116   MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
4117                                      ContainerExtVT.getVectorElementCount());
4118 
4119   SDValue Op1 =
4120       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
4121 
4122   SDLoc DL(Op);
4123   SDValue Mask, VL;
4124   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4125 
4126   SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);
4127 
4128   return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget);
4129 }
4130 
4131 // Custom-lower truncations from vectors to mask vectors by using a mask and a
4132 // setcc operation:
4133 //   (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
4134 SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
4135                                                   SelectionDAG &DAG) const {
4136   SDLoc DL(Op);
4137   EVT MaskVT = Op.getValueType();
4138   // Only expect to custom-lower truncations to mask types
4139   assert(MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 &&
4140          "Unexpected type for vector mask lowering");
4141   SDValue Src = Op.getOperand(0);
4142   MVT VecVT = Src.getSimpleValueType();
4143 
4144   // If this is a fixed vector, we need to convert it to a scalable vector.
4145   MVT ContainerVT = VecVT;
4146   if (VecVT.isFixedLengthVector()) {
4147     ContainerVT = getContainerForFixedLengthVector(VecVT);
4148     Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
4149   }
4150 
4151   SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
4152   SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
4153 
4154   SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatOne);
4155   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero);
4156 
4157   if (VecVT.isScalableVector()) {
4158     SDValue Trunc = DAG.getNode(ISD::AND, DL, VecVT, Src, SplatOne);
4159     return DAG.getSetCC(DL, MaskVT, Trunc, SplatZero, ISD::SETNE);
4160   }
4161 
4162   SDValue Mask, VL;
4163   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4164 
4165   MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
4166   SDValue Trunc =
4167       DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, Mask, VL);
4168   Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, Trunc, SplatZero,
4169                       DAG.getCondCode(ISD::SETNE), Mask, VL);
4170   return convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
4171 }
4172 
4173 // Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
4174 // first position of a vector, and that vector is slid up to the insert index.
4175 // By limiting the active vector length to index+1 and merging with the
4176 // original vector (with an undisturbed tail policy for elements >= VL), we
4177 // achieve the desired result of leaving all elements untouched except the one
4178 // at VL-1, which is replaced with the desired value.
4179 SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4180                                                     SelectionDAG &DAG) const {
4181   SDLoc DL(Op);
4182   MVT VecVT = Op.getSimpleValueType();
4183   SDValue Vec = Op.getOperand(0);
4184   SDValue Val = Op.getOperand(1);
4185   SDValue Idx = Op.getOperand(2);
4186 
4187   if (VecVT.getVectorElementType() == MVT::i1) {
4188     // FIXME: For now we just promote to an i8 vector and insert into that,
4189     // but this is probably not optimal.
4190     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4191     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4192     Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
4193     return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
4194   }
4195 
4196   MVT ContainerVT = VecVT;
4197   // If the operand is a fixed-length vector, convert to a scalable one.
4198   if (VecVT.isFixedLengthVector()) {
4199     ContainerVT = getContainerForFixedLengthVector(VecVT);
4200     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4201   }
4202 
4203   MVT XLenVT = Subtarget.getXLenVT();
4204 
4205   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4206   bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
4207   // Even i64-element vectors on RV32 can be lowered without scalar
4208   // legalization if the most-significant 32 bits of the value are not affected
4209   // by the sign-extension of the lower 32 bits.
4210   // TODO: We could also catch sign extensions of a 32-bit value.
4211   if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
4212     const auto *CVal = cast<ConstantSDNode>(Val);
4213     if (isInt<32>(CVal->getSExtValue())) {
4214       IsLegalInsert = true;
4215       Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4216     }
4217   }
4218 
4219   SDValue Mask, VL;
4220   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4221 
4222   SDValue ValInVec;
4223 
4224   if (IsLegalInsert) {
4225     unsigned Opc =
4226         VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
4227     if (isNullConstant(Idx)) {
4228       Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
4229       if (!VecVT.isFixedLengthVector())
4230         return Vec;
4231       return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
4232     }
4233     ValInVec =
4234         DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Val, VL);
4235   } else {
4236     // On RV32, i64-element vectors must be specially handled to place the
4237     // value at element 0, by using two vslide1up instructions in sequence on
4238     // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
4239     // this.
4240     SDValue One = DAG.getConstant(1, DL, XLenVT);
4241     SDValue ValLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, Zero);
4242     SDValue ValHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, One);
4243     MVT I32ContainerVT =
4244         MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2);
4245     SDValue I32Mask =
4246         getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
4247     // Limit the active VL to two.
4248     SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
4249     // Note: We can't pass a UNDEF to the first VSLIDE1UP_VL since an untied
4250     // undef doesn't obey the earlyclobber constraint. Just splat a zero value.
4251     ValInVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, I32ContainerVT, Zero,
4252                            InsertI64VL);
4253     // First slide in the hi value, then the lo in underneath it.
4254     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4255                            ValHi, I32Mask, InsertI64VL);
4256     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4257                            ValLo, I32Mask, InsertI64VL);
4258     // Bitcast back to the right container type.
4259     ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
4260   }
4261 
4262   // Now that the value is in a vector, slide it into position.
4263   SDValue InsertVL =
4264       DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
4265   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
4266                                 ValInVec, Idx, Mask, InsertVL);
4267   if (!VecVT.isFixedLengthVector())
4268     return Slideup;
4269   return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
4270 }
4271 
4272 // Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
4273 // extract the first element: (extractelt (slidedown vec, idx), 0). For integer
4274 // types this is done using VMV_X_S to allow us to glean information about the
4275 // sign bits of the result.
4276 SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4277                                                      SelectionDAG &DAG) const {
4278   SDLoc DL(Op);
4279   SDValue Idx = Op.getOperand(1);
4280   SDValue Vec = Op.getOperand(0);
4281   EVT EltVT = Op.getValueType();
4282   MVT VecVT = Vec.getSimpleValueType();
4283   MVT XLenVT = Subtarget.getXLenVT();
4284 
4285   if (VecVT.getVectorElementType() == MVT::i1) {
4286     // FIXME: For now we just promote to an i8 vector and extract from that,
4287     // but this is probably not optimal.
4288     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4289     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4290     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
4291   }
4292 
4293   // If this is a fixed vector, we need to convert it to a scalable vector.
4294   MVT ContainerVT = VecVT;
4295   if (VecVT.isFixedLengthVector()) {
4296     ContainerVT = getContainerForFixedLengthVector(VecVT);
4297     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4298   }
4299 
4300   // If the index is 0, the vector is already in the right position.
4301   if (!isNullConstant(Idx)) {
4302     // Use a VL of 1 to avoid processing more elements than we need.
4303     SDValue VL = DAG.getConstant(1, DL, XLenVT);
4304     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4305     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4306     Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
4307                       DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
4308   }
4309 
4310   if (!EltVT.isInteger()) {
4311     // Floating-point extracts are handled in TableGen.
4312     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
4313                        DAG.getConstant(0, DL, XLenVT));
4314   }
4315 
4316   SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
4317   return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
4318 }
4319 
4320 // Some RVV intrinsics may claim that they want an integer operand to be
4321 // promoted or expanded.
4322 static SDValue lowerVectorIntrinsicSplats(SDValue Op, SelectionDAG &DAG,
4323                                           const RISCVSubtarget &Subtarget) {
4324   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4325           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
4326          "Unexpected opcode");
4327 
4328   if (!Subtarget.hasVInstructions())
4329     return SDValue();
4330 
4331   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
4332   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
4333   SDLoc DL(Op);
4334 
4335   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
4336       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
4337   if (!II || !II->hasSplatOperand())
4338     return SDValue();
4339 
4340   unsigned SplatOp = II->SplatOperand + 1 + HasChain;
4341   assert(SplatOp < Op.getNumOperands());
4342 
4343   SmallVector<SDValue, 8> Operands(Op->op_begin(), Op->op_end());
4344   SDValue &ScalarOp = Operands[SplatOp];
4345   MVT OpVT = ScalarOp.getSimpleValueType();
4346   MVT XLenVT = Subtarget.getXLenVT();
4347 
4348   // If this isn't a scalar, or its type is XLenVT we're done.
4349   if (!OpVT.isScalarInteger() || OpVT == XLenVT)
4350     return SDValue();
4351 
4352   // Simplest case is that the operand needs to be promoted to XLenVT.
4353   if (OpVT.bitsLT(XLenVT)) {
4354     // If the operand is a constant, sign extend to increase our chances
4355     // of being able to use a .vi instruction. ANY_EXTEND would become a
4356     // a zero extend and the simm5 check in isel would fail.
4357     // FIXME: Should we ignore the upper bits in isel instead?
4358     unsigned ExtOpc =
4359         isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
4360     ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
4361     return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4362   }
4363 
4364   // Use the previous operand to get the vXi64 VT. The result might be a mask
4365   // VT for compares. Using the previous operand assumes that the previous
4366   // operand will never have a smaller element size than a scalar operand and
4367   // that a widening operation never uses SEW=64.
4368   // NOTE: If this fails the below assert, we can probably just find the
4369   // element count from any operand or result and use it to construct the VT.
4370   assert(II->SplatOperand > 0 && "Unexpected splat operand!");
4371   MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
4372 
4373   // The more complex case is when the scalar is larger than XLenVT.
4374   assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
4375          VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
4376 
4377   // If this is a sign-extended 32-bit constant, we can truncate it and rely
4378   // on the instruction to sign-extend since SEW>XLEN.
4379   if (auto *CVal = dyn_cast<ConstantSDNode>(ScalarOp)) {
4380     if (isInt<32>(CVal->getSExtValue())) {
4381       ScalarOp = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4382       return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4383     }
4384   }
4385 
4386   // We need to convert the scalar to a splat vector.
4387   // FIXME: Can we implicitly truncate the scalar if it is known to
4388   // be sign extended?
4389   SDValue VL = getVLOperand(Op);
4390   assert(VL.getValueType() == XLenVT);
4391   ScalarOp = splatSplitI64WithVL(DL, VT, ScalarOp, VL, DAG);
4392   return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4393 }
4394 
4395 SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
4396                                                      SelectionDAG &DAG) const {
4397   unsigned IntNo = Op.getConstantOperandVal(0);
4398   SDLoc DL(Op);
4399   MVT XLenVT = Subtarget.getXLenVT();
4400 
4401   switch (IntNo) {
4402   default:
4403     break; // Don't custom lower most intrinsics.
4404   case Intrinsic::thread_pointer: {
4405     EVT PtrVT = getPointerTy(DAG.getDataLayout());
4406     return DAG.getRegister(RISCV::X4, PtrVT);
4407   }
4408   case Intrinsic::riscv_orc_b:
4409     // Lower to the GORCI encoding for orc.b.
4410     return DAG.getNode(RISCVISD::GORC, DL, XLenVT, Op.getOperand(1),
4411                        DAG.getConstant(7, DL, XLenVT));
4412   case Intrinsic::riscv_grev:
4413   case Intrinsic::riscv_gorc: {
4414     unsigned Opc =
4415         IntNo == Intrinsic::riscv_grev ? RISCVISD::GREV : RISCVISD::GORC;
4416     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4417   }
4418   case Intrinsic::riscv_shfl:
4419   case Intrinsic::riscv_unshfl: {
4420     unsigned Opc =
4421         IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
4422     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4423   }
4424   case Intrinsic::riscv_bcompress:
4425   case Intrinsic::riscv_bdecompress: {
4426     unsigned Opc = IntNo == Intrinsic::riscv_bcompress ? RISCVISD::BCOMPRESS
4427                                                        : RISCVISD::BDECOMPRESS;
4428     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4429   }
4430   case Intrinsic::riscv_bfp:
4431     return DAG.getNode(RISCVISD::BFP, DL, XLenVT, Op.getOperand(1),
4432                        Op.getOperand(2));
4433   case Intrinsic::riscv_fsl:
4434     return DAG.getNode(RISCVISD::FSL, DL, XLenVT, Op.getOperand(1),
4435                        Op.getOperand(2), Op.getOperand(3));
4436   case Intrinsic::riscv_fsr:
4437     return DAG.getNode(RISCVISD::FSR, DL, XLenVT, Op.getOperand(1),
4438                        Op.getOperand(2), Op.getOperand(3));
4439   case Intrinsic::riscv_vmv_x_s:
4440     assert(Op.getValueType() == XLenVT && "Unexpected VT!");
4441     return DAG.getNode(RISCVISD::VMV_X_S, DL, Op.getValueType(),
4442                        Op.getOperand(1));
4443   case Intrinsic::riscv_vmv_v_x:
4444     return lowerScalarSplat(Op.getOperand(1), Op.getOperand(2),
4445                             Op.getSimpleValueType(), DL, DAG, Subtarget);
4446   case Intrinsic::riscv_vfmv_v_f:
4447     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
4448                        Op.getOperand(1), Op.getOperand(2));
4449   case Intrinsic::riscv_vmv_s_x: {
4450     SDValue Scalar = Op.getOperand(2);
4451 
4452     if (Scalar.getValueType().bitsLE(XLenVT)) {
4453       Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
4454       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
4455                          Op.getOperand(1), Scalar, Op.getOperand(3));
4456     }
4457 
4458     assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
4459 
4460     // This is an i64 value that lives in two scalar registers. We have to
4461     // insert this in a convoluted way. First we build vXi64 splat containing
4462     // the/ two values that we assemble using some bit math. Next we'll use
4463     // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
4464     // to merge element 0 from our splat into the source vector.
4465     // FIXME: This is probably not the best way to do this, but it is
4466     // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
4467     // point.
4468     //   sw lo, (a0)
4469     //   sw hi, 4(a0)
4470     //   vlse vX, (a0)
4471     //
4472     //   vid.v      vVid
4473     //   vmseq.vx   mMask, vVid, 0
4474     //   vmerge.vvm vDest, vSrc, vVal, mMask
4475     MVT VT = Op.getSimpleValueType();
4476     SDValue Vec = Op.getOperand(1);
4477     SDValue VL = getVLOperand(Op);
4478 
4479     SDValue SplattedVal = splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
4480     SDValue SplattedIdx = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
4481                                       DAG.getConstant(0, DL, MVT::i32), VL);
4482 
4483     MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
4484     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4485     SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
4486     SDValue SelectCond =
4487         DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, VID, SplattedIdx,
4488                     DAG.getCondCode(ISD::SETEQ), Mask, VL);
4489     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, SelectCond, SplattedVal,
4490                        Vec, VL);
4491   }
4492   case Intrinsic::riscv_vslide1up:
4493   case Intrinsic::riscv_vslide1down:
4494   case Intrinsic::riscv_vslide1up_mask:
4495   case Intrinsic::riscv_vslide1down_mask: {
4496     // We need to special case these when the scalar is larger than XLen.
4497     unsigned NumOps = Op.getNumOperands();
4498     bool IsMasked = NumOps == 7;
4499     unsigned OpOffset = IsMasked ? 1 : 0;
4500     SDValue Scalar = Op.getOperand(2 + OpOffset);
4501     if (Scalar.getValueType().bitsLE(XLenVT))
4502       break;
4503 
4504     // Splatting a sign extended constant is fine.
4505     if (auto *CVal = dyn_cast<ConstantSDNode>(Scalar))
4506       if (isInt<32>(CVal->getSExtValue()))
4507         break;
4508 
4509     MVT VT = Op.getSimpleValueType();
4510     assert(VT.getVectorElementType() == MVT::i64 &&
4511            Scalar.getValueType() == MVT::i64 && "Unexpected VTs");
4512 
4513     // Convert the vector source to the equivalent nxvXi32 vector.
4514     MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
4515     SDValue Vec = DAG.getBitcast(I32VT, Op.getOperand(1 + OpOffset));
4516 
4517     SDValue ScalarLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4518                                    DAG.getConstant(0, DL, XLenVT));
4519     SDValue ScalarHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4520                                    DAG.getConstant(1, DL, XLenVT));
4521 
4522     // Double the VL since we halved SEW.
4523     SDValue VL = getVLOperand(Op);
4524     SDValue I32VL =
4525         DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
4526 
4527     MVT I32MaskVT = MVT::getVectorVT(MVT::i1, I32VT.getVectorElementCount());
4528     SDValue I32Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, I32MaskVT, VL);
4529 
4530     // Shift the two scalar parts in using SEW=32 slide1up/slide1down
4531     // instructions.
4532     if (IntNo == Intrinsic::riscv_vslide1up ||
4533         IntNo == Intrinsic::riscv_vslide1up_mask) {
4534       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarHi,
4535                         I32Mask, I32VL);
4536       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarLo,
4537                         I32Mask, I32VL);
4538     } else {
4539       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarLo,
4540                         I32Mask, I32VL);
4541       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarHi,
4542                         I32Mask, I32VL);
4543     }
4544 
4545     // Convert back to nxvXi64.
4546     Vec = DAG.getBitcast(VT, Vec);
4547 
4548     if (!IsMasked)
4549       return Vec;
4550 
4551     // Apply mask after the operation.
4552     SDValue Mask = Op.getOperand(NumOps - 3);
4553     SDValue MaskedOff = Op.getOperand(1);
4554     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, Mask, Vec, MaskedOff, VL);
4555   }
4556   }
4557 
4558   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4559 }
4560 
4561 SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
4562                                                     SelectionDAG &DAG) const {
4563   unsigned IntNo = Op.getConstantOperandVal(1);
4564   switch (IntNo) {
4565   default:
4566     break;
4567   case Intrinsic::riscv_masked_strided_load: {
4568     SDLoc DL(Op);
4569     MVT XLenVT = Subtarget.getXLenVT();
4570 
4571     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4572     // the selection of the masked intrinsics doesn't do this for us.
4573     SDValue Mask = Op.getOperand(5);
4574     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4575 
4576     MVT VT = Op->getSimpleValueType(0);
4577     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4578 
4579     SDValue PassThru = Op.getOperand(2);
4580     if (!IsUnmasked) {
4581       MVT MaskVT =
4582           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4583       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4584       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
4585     }
4586 
4587     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4588 
4589     SDValue IntID = DAG.getTargetConstant(
4590         IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask, DL,
4591         XLenVT);
4592 
4593     auto *Load = cast<MemIntrinsicSDNode>(Op);
4594     SmallVector<SDValue, 8> Ops{Load->getChain(), IntID};
4595     if (IsUnmasked)
4596       Ops.push_back(DAG.getUNDEF(ContainerVT));
4597     else
4598       Ops.push_back(PassThru);
4599     Ops.push_back(Op.getOperand(3)); // Ptr
4600     Ops.push_back(Op.getOperand(4)); // Stride
4601     if (!IsUnmasked)
4602       Ops.push_back(Mask);
4603     Ops.push_back(VL);
4604     if (!IsUnmasked) {
4605       SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
4606       Ops.push_back(Policy);
4607     }
4608 
4609     SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
4610     SDValue Result =
4611         DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
4612                                 Load->getMemoryVT(), Load->getMemOperand());
4613     SDValue Chain = Result.getValue(1);
4614     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
4615     return DAG.getMergeValues({Result, Chain}, DL);
4616   }
4617   }
4618 
4619   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4620 }
4621 
4622 SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
4623                                                  SelectionDAG &DAG) const {
4624   unsigned IntNo = Op.getConstantOperandVal(1);
4625   switch (IntNo) {
4626   default:
4627     break;
4628   case Intrinsic::riscv_masked_strided_store: {
4629     SDLoc DL(Op);
4630     MVT XLenVT = Subtarget.getXLenVT();
4631 
4632     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4633     // the selection of the masked intrinsics doesn't do this for us.
4634     SDValue Mask = Op.getOperand(5);
4635     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4636 
4637     SDValue Val = Op.getOperand(2);
4638     MVT VT = Val.getSimpleValueType();
4639     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4640 
4641     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
4642     if (!IsUnmasked) {
4643       MVT MaskVT =
4644           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4645       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4646     }
4647 
4648     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4649 
4650     SDValue IntID = DAG.getTargetConstant(
4651         IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask, DL,
4652         XLenVT);
4653 
4654     auto *Store = cast<MemIntrinsicSDNode>(Op);
4655     SmallVector<SDValue, 8> Ops{Store->getChain(), IntID};
4656     Ops.push_back(Val);
4657     Ops.push_back(Op.getOperand(3)); // Ptr
4658     Ops.push_back(Op.getOperand(4)); // Stride
4659     if (!IsUnmasked)
4660       Ops.push_back(Mask);
4661     Ops.push_back(VL);
4662 
4663     return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(),
4664                                    Ops, Store->getMemoryVT(),
4665                                    Store->getMemOperand());
4666   }
4667   }
4668 
4669   return SDValue();
4670 }
4671 
4672 static MVT getLMUL1VT(MVT VT) {
4673   assert(VT.getVectorElementType().getSizeInBits() <= 64 &&
4674          "Unexpected vector MVT");
4675   return MVT::getScalableVectorVT(
4676       VT.getVectorElementType(),
4677       RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits());
4678 }
4679 
4680 static unsigned getRVVReductionOp(unsigned ISDOpcode) {
4681   switch (ISDOpcode) {
4682   default:
4683     llvm_unreachable("Unhandled reduction");
4684   case ISD::VECREDUCE_ADD:
4685     return RISCVISD::VECREDUCE_ADD_VL;
4686   case ISD::VECREDUCE_UMAX:
4687     return RISCVISD::VECREDUCE_UMAX_VL;
4688   case ISD::VECREDUCE_SMAX:
4689     return RISCVISD::VECREDUCE_SMAX_VL;
4690   case ISD::VECREDUCE_UMIN:
4691     return RISCVISD::VECREDUCE_UMIN_VL;
4692   case ISD::VECREDUCE_SMIN:
4693     return RISCVISD::VECREDUCE_SMIN_VL;
4694   case ISD::VECREDUCE_AND:
4695     return RISCVISD::VECREDUCE_AND_VL;
4696   case ISD::VECREDUCE_OR:
4697     return RISCVISD::VECREDUCE_OR_VL;
4698   case ISD::VECREDUCE_XOR:
4699     return RISCVISD::VECREDUCE_XOR_VL;
4700   }
4701 }
4702 
4703 SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
4704                                                          SelectionDAG &DAG,
4705                                                          bool IsVP) const {
4706   SDLoc DL(Op);
4707   SDValue Vec = Op.getOperand(IsVP ? 1 : 0);
4708   MVT VecVT = Vec.getSimpleValueType();
4709   assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
4710           Op.getOpcode() == ISD::VECREDUCE_OR ||
4711           Op.getOpcode() == ISD::VECREDUCE_XOR ||
4712           Op.getOpcode() == ISD::VP_REDUCE_AND ||
4713           Op.getOpcode() == ISD::VP_REDUCE_OR ||
4714           Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
4715          "Unexpected reduction lowering");
4716 
4717   MVT XLenVT = Subtarget.getXLenVT();
4718   assert(Op.getValueType() == XLenVT &&
4719          "Expected reduction output to be legalized to XLenVT");
4720 
4721   MVT ContainerVT = VecVT;
4722   if (VecVT.isFixedLengthVector()) {
4723     ContainerVT = getContainerForFixedLengthVector(VecVT);
4724     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4725   }
4726 
4727   SDValue Mask, VL;
4728   if (IsVP) {
4729     Mask = Op.getOperand(2);
4730     VL = Op.getOperand(3);
4731   } else {
4732     std::tie(Mask, VL) =
4733         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4734   }
4735 
4736   unsigned BaseOpc;
4737   ISD::CondCode CC;
4738   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4739 
4740   switch (Op.getOpcode()) {
4741   default:
4742     llvm_unreachable("Unhandled reduction");
4743   case ISD::VECREDUCE_AND:
4744   case ISD::VP_REDUCE_AND: {
4745     // vcpop ~x == 0
4746     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
4747     Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
4748     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4749     CC = ISD::SETEQ;
4750     BaseOpc = ISD::AND;
4751     break;
4752   }
4753   case ISD::VECREDUCE_OR:
4754   case ISD::VP_REDUCE_OR:
4755     // vcpop x != 0
4756     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4757     CC = ISD::SETNE;
4758     BaseOpc = ISD::OR;
4759     break;
4760   case ISD::VECREDUCE_XOR:
4761   case ISD::VP_REDUCE_XOR: {
4762     // ((vcpop x) & 1) != 0
4763     SDValue One = DAG.getConstant(1, DL, XLenVT);
4764     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4765     Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
4766     CC = ISD::SETNE;
4767     BaseOpc = ISD::XOR;
4768     break;
4769   }
4770   }
4771 
4772   SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC);
4773 
4774   if (!IsVP)
4775     return SetCC;
4776 
4777   // Now include the start value in the operation.
4778   // Note that we must return the start value when no elements are operated
4779   // upon. The vcpop instructions we've emitted in each case above will return
4780   // 0 for an inactive vector, and so we've already received the neutral value:
4781   // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
4782   // can simply include the start value.
4783   return DAG.getNode(BaseOpc, DL, XLenVT, SetCC, Op.getOperand(0));
4784 }
4785 
4786 SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
4787                                             SelectionDAG &DAG) const {
4788   SDLoc DL(Op);
4789   SDValue Vec = Op.getOperand(0);
4790   EVT VecEVT = Vec.getValueType();
4791 
4792   unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
4793 
4794   // Due to ordering in legalize types we may have a vector type that needs to
4795   // be split. Do that manually so we can get down to a legal type.
4796   while (getTypeAction(*DAG.getContext(), VecEVT) ==
4797          TargetLowering::TypeSplitVector) {
4798     SDValue Lo, Hi;
4799     std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
4800     VecEVT = Lo.getValueType();
4801     Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
4802   }
4803 
4804   // TODO: The type may need to be widened rather than split. Or widened before
4805   // it can be split.
4806   if (!isTypeLegal(VecEVT))
4807     return SDValue();
4808 
4809   MVT VecVT = VecEVT.getSimpleVT();
4810   MVT VecEltVT = VecVT.getVectorElementType();
4811   unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());
4812 
4813   MVT ContainerVT = VecVT;
4814   if (VecVT.isFixedLengthVector()) {
4815     ContainerVT = getContainerForFixedLengthVector(VecVT);
4816     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4817   }
4818 
4819   MVT M1VT = getLMUL1VT(ContainerVT);
4820   MVT XLenVT = Subtarget.getXLenVT();
4821 
4822   SDValue Mask, VL;
4823   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4824 
4825   SDValue NeutralElem =
4826       DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
4827   SDValue IdentitySplat = lowerScalarSplat(
4828       NeutralElem, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
4829   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT), Vec,
4830                                   IdentitySplat, Mask, VL);
4831   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
4832                              DAG.getConstant(0, DL, XLenVT));
4833   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
4834 }
4835 
4836 // Given a reduction op, this function returns the matching reduction opcode,
4837 // the vector SDValue and the scalar SDValue required to lower this to a
4838 // RISCVISD node.
4839 static std::tuple<unsigned, SDValue, SDValue>
4840 getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) {
4841   SDLoc DL(Op);
4842   auto Flags = Op->getFlags();
4843   unsigned Opcode = Op.getOpcode();
4844   unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Opcode);
4845   switch (Opcode) {
4846   default:
4847     llvm_unreachable("Unhandled reduction");
4848   case ISD::VECREDUCE_FADD: {
4849     // Use positive zero if we can. It is cheaper to materialize.
4850     SDValue Zero =
4851         DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
4852     return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero);
4853   }
4854   case ISD::VECREDUCE_SEQ_FADD:
4855     return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
4856                            Op.getOperand(0));
4857   case ISD::VECREDUCE_FMIN:
4858     return std::make_tuple(RISCVISD::VECREDUCE_FMIN_VL, Op.getOperand(0),
4859                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
4860   case ISD::VECREDUCE_FMAX:
4861     return std::make_tuple(RISCVISD::VECREDUCE_FMAX_VL, Op.getOperand(0),
4862                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
4863   }
4864 }
4865 
4866 SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
4867                                               SelectionDAG &DAG) const {
4868   SDLoc DL(Op);
4869   MVT VecEltVT = Op.getSimpleValueType();
4870 
4871   unsigned RVVOpcode;
4872   SDValue VectorVal, ScalarVal;
4873   std::tie(RVVOpcode, VectorVal, ScalarVal) =
4874       getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT);
4875   MVT VecVT = VectorVal.getSimpleValueType();
4876 
4877   MVT ContainerVT = VecVT;
4878   if (VecVT.isFixedLengthVector()) {
4879     ContainerVT = getContainerForFixedLengthVector(VecVT);
4880     VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
4881   }
4882 
4883   MVT M1VT = getLMUL1VT(VectorVal.getSimpleValueType());
4884   MVT XLenVT = Subtarget.getXLenVT();
4885 
4886   SDValue Mask, VL;
4887   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4888 
4889   SDValue ScalarSplat = lowerScalarSplat(
4890       ScalarVal, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
4891   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT),
4892                                   VectorVal, ScalarSplat, Mask, VL);
4893   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
4894                      DAG.getConstant(0, DL, XLenVT));
4895 }
4896 
4897 static unsigned getRVVVPReductionOp(unsigned ISDOpcode) {
4898   switch (ISDOpcode) {
4899   default:
4900     llvm_unreachable("Unhandled reduction");
4901   case ISD::VP_REDUCE_ADD:
4902     return RISCVISD::VECREDUCE_ADD_VL;
4903   case ISD::VP_REDUCE_UMAX:
4904     return RISCVISD::VECREDUCE_UMAX_VL;
4905   case ISD::VP_REDUCE_SMAX:
4906     return RISCVISD::VECREDUCE_SMAX_VL;
4907   case ISD::VP_REDUCE_UMIN:
4908     return RISCVISD::VECREDUCE_UMIN_VL;
4909   case ISD::VP_REDUCE_SMIN:
4910     return RISCVISD::VECREDUCE_SMIN_VL;
4911   case ISD::VP_REDUCE_AND:
4912     return RISCVISD::VECREDUCE_AND_VL;
4913   case ISD::VP_REDUCE_OR:
4914     return RISCVISD::VECREDUCE_OR_VL;
4915   case ISD::VP_REDUCE_XOR:
4916     return RISCVISD::VECREDUCE_XOR_VL;
4917   case ISD::VP_REDUCE_FADD:
4918     return RISCVISD::VECREDUCE_FADD_VL;
4919   case ISD::VP_REDUCE_SEQ_FADD:
4920     return RISCVISD::VECREDUCE_SEQ_FADD_VL;
4921   case ISD::VP_REDUCE_FMAX:
4922     return RISCVISD::VECREDUCE_FMAX_VL;
4923   case ISD::VP_REDUCE_FMIN:
4924     return RISCVISD::VECREDUCE_FMIN_VL;
4925   }
4926 }
4927 
4928 SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
4929                                            SelectionDAG &DAG) const {
4930   SDLoc DL(Op);
4931   SDValue Vec = Op.getOperand(1);
4932   EVT VecEVT = Vec.getValueType();
4933 
4934   // TODO: The type may need to be widened rather than split. Or widened before
4935   // it can be split.
4936   if (!isTypeLegal(VecEVT))
4937     return SDValue();
4938 
4939   MVT VecVT = VecEVT.getSimpleVT();
4940   MVT VecEltVT = VecVT.getVectorElementType();
4941   unsigned RVVOpcode = getRVVVPReductionOp(Op.getOpcode());
4942 
4943   MVT ContainerVT = VecVT;
4944   if (VecVT.isFixedLengthVector()) {
4945     ContainerVT = getContainerForFixedLengthVector(VecVT);
4946     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4947   }
4948 
4949   SDValue VL = Op.getOperand(3);
4950   SDValue Mask = Op.getOperand(2);
4951 
4952   MVT M1VT = getLMUL1VT(ContainerVT);
4953   MVT XLenVT = Subtarget.getXLenVT();
4954   MVT ResVT = !VecVT.isInteger() || VecEltVT.bitsGE(XLenVT) ? VecEltVT : XLenVT;
4955 
4956   SDValue StartSplat =
4957       lowerScalarSplat(Op.getOperand(0), DAG.getConstant(1, DL, XLenVT), M1VT,
4958                        DL, DAG, Subtarget);
4959   SDValue Reduction =
4960       DAG.getNode(RVVOpcode, DL, M1VT, StartSplat, Vec, StartSplat, Mask, VL);
4961   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction,
4962                              DAG.getConstant(0, DL, XLenVT));
4963   if (!VecVT.isInteger())
4964     return Elt0;
4965   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
4966 }
4967 
4968 SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
4969                                                    SelectionDAG &DAG) const {
4970   SDValue Vec = Op.getOperand(0);
4971   SDValue SubVec = Op.getOperand(1);
4972   MVT VecVT = Vec.getSimpleValueType();
4973   MVT SubVecVT = SubVec.getSimpleValueType();
4974 
4975   SDLoc DL(Op);
4976   MVT XLenVT = Subtarget.getXLenVT();
4977   unsigned OrigIdx = Op.getConstantOperandVal(2);
4978   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
4979 
4980   // We don't have the ability to slide mask vectors up indexed by their i1
4981   // elements; the smallest we can do is i8. Often we are able to bitcast to
4982   // equivalent i8 vectors. Note that when inserting a fixed-length vector
4983   // into a scalable one, we might not necessarily have enough scalable
4984   // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
4985   if (SubVecVT.getVectorElementType() == MVT::i1 &&
4986       (OrigIdx != 0 || !Vec.isUndef())) {
4987     if (VecVT.getVectorMinNumElements() >= 8 &&
4988         SubVecVT.getVectorMinNumElements() >= 8) {
4989       assert(OrigIdx % 8 == 0 && "Invalid index");
4990       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
4991              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
4992              "Unexpected mask vector lowering");
4993       OrigIdx /= 8;
4994       SubVecVT =
4995           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
4996                            SubVecVT.isScalableVector());
4997       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
4998                                VecVT.isScalableVector());
4999       Vec = DAG.getBitcast(VecVT, Vec);
5000       SubVec = DAG.getBitcast(SubVecVT, SubVec);
5001     } else {
5002       // We can't slide this mask vector up indexed by its i1 elements.
5003       // This poses a problem when we wish to insert a scalable vector which
5004       // can't be re-expressed as a larger type. Just choose the slow path and
5005       // extend to a larger type, then truncate back down.
5006       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
5007       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
5008       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
5009       SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
5010       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
5011                         Op.getOperand(2));
5012       SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
5013       return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
5014     }
5015   }
5016 
5017   // If the subvector vector is a fixed-length type, we cannot use subregister
5018   // manipulation to simplify the codegen; we don't know which register of a
5019   // LMUL group contains the specific subvector as we only know the minimum
5020   // register size. Therefore we must slide the vector group up the full
5021   // amount.
5022   if (SubVecVT.isFixedLengthVector()) {
5023     if (OrigIdx == 0 && Vec.isUndef() && !VecVT.isFixedLengthVector())
5024       return Op;
5025     MVT ContainerVT = VecVT;
5026     if (VecVT.isFixedLengthVector()) {
5027       ContainerVT = getContainerForFixedLengthVector(VecVT);
5028       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5029     }
5030     SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
5031                          DAG.getUNDEF(ContainerVT), SubVec,
5032                          DAG.getConstant(0, DL, XLenVT));
5033     if (OrigIdx == 0 && Vec.isUndef() && VecVT.isFixedLengthVector()) {
5034       SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
5035       return DAG.getBitcast(Op.getValueType(), SubVec);
5036     }
5037     SDValue Mask =
5038         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5039     // Set the vector length to only the number of elements we care about. Note
5040     // that for slideup this includes the offset.
5041     SDValue VL =
5042         DAG.getConstant(OrigIdx + SubVecVT.getVectorNumElements(), DL, XLenVT);
5043     SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5044     SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
5045                                   SubVec, SlideupAmt, Mask, VL);
5046     if (VecVT.isFixedLengthVector())
5047       Slideup = convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
5048     return DAG.getBitcast(Op.getValueType(), Slideup);
5049   }
5050 
5051   unsigned SubRegIdx, RemIdx;
5052   std::tie(SubRegIdx, RemIdx) =
5053       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5054           VecVT, SubVecVT, OrigIdx, TRI);
5055 
5056   RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecVT);
5057   bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
5058                          SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
5059                          SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
5060 
5061   // 1. If the Idx has been completely eliminated and this subvector's size is
5062   // a vector register or a multiple thereof, or the surrounding elements are
5063   // undef, then this is a subvector insert which naturally aligns to a vector
5064   // register. These can easily be handled using subregister manipulation.
5065   // 2. If the subvector is smaller than a vector register, then the insertion
5066   // must preserve the undisturbed elements of the register. We do this by
5067   // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
5068   // (which resolves to a subregister copy), performing a VSLIDEUP to place the
5069   // subvector within the vector register, and an INSERT_SUBVECTOR of that
5070   // LMUL=1 type back into the larger vector (resolving to another subregister
5071   // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
5072   // to avoid allocating a large register group to hold our subvector.
5073   if (RemIdx == 0 && (!IsSubVecPartReg || Vec.isUndef()))
5074     return Op;
5075 
5076   // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
5077   // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
5078   // (in our case undisturbed). This means we can set up a subvector insertion
5079   // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
5080   // size of the subvector.
5081   MVT InterSubVT = VecVT;
5082   SDValue AlignedExtract = Vec;
5083   unsigned AlignedIdx = OrigIdx - RemIdx;
5084   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5085     InterSubVT = getLMUL1VT(VecVT);
5086     // Extract a subvector equal to the nearest full vector register type. This
5087     // should resolve to a EXTRACT_SUBREG instruction.
5088     AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5089                                  DAG.getConstant(AlignedIdx, DL, XLenVT));
5090   }
5091 
5092   SDValue SlideupAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5093   // For scalable vectors this must be further multiplied by vscale.
5094   SlideupAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlideupAmt);
5095 
5096   SDValue Mask, VL;
5097   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5098 
5099   // Construct the vector length corresponding to RemIdx + length(SubVecVT).
5100   VL = DAG.getConstant(SubVecVT.getVectorMinNumElements(), DL, XLenVT);
5101   VL = DAG.getNode(ISD::VSCALE, DL, XLenVT, VL);
5102   VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
5103 
5104   SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
5105                        DAG.getUNDEF(InterSubVT), SubVec,
5106                        DAG.getConstant(0, DL, XLenVT));
5107 
5108   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, InterSubVT,
5109                                 AlignedExtract, SubVec, SlideupAmt, Mask, VL);
5110 
5111   // If required, insert this subvector back into the correct vector register.
5112   // This should resolve to an INSERT_SUBREG instruction.
5113   if (VecVT.bitsGT(InterSubVT))
5114     Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, Slideup,
5115                           DAG.getConstant(AlignedIdx, DL, XLenVT));
5116 
5117   // We might have bitcast from a mask type: cast back to the original type if
5118   // required.
5119   return DAG.getBitcast(Op.getSimpleValueType(), Slideup);
5120 }
5121 
5122 SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
5123                                                     SelectionDAG &DAG) const {
5124   SDValue Vec = Op.getOperand(0);
5125   MVT SubVecVT = Op.getSimpleValueType();
5126   MVT VecVT = Vec.getSimpleValueType();
5127 
5128   SDLoc DL(Op);
5129   MVT XLenVT = Subtarget.getXLenVT();
5130   unsigned OrigIdx = Op.getConstantOperandVal(1);
5131   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
5132 
5133   // We don't have the ability to slide mask vectors down indexed by their i1
5134   // elements; the smallest we can do is i8. Often we are able to bitcast to
5135   // equivalent i8 vectors. Note that when extracting a fixed-length vector
5136   // from a scalable one, we might not necessarily have enough scalable
5137   // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
5138   if (SubVecVT.getVectorElementType() == MVT::i1 && OrigIdx != 0) {
5139     if (VecVT.getVectorMinNumElements() >= 8 &&
5140         SubVecVT.getVectorMinNumElements() >= 8) {
5141       assert(OrigIdx % 8 == 0 && "Invalid index");
5142       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
5143              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
5144              "Unexpected mask vector lowering");
5145       OrigIdx /= 8;
5146       SubVecVT =
5147           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
5148                            SubVecVT.isScalableVector());
5149       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
5150                                VecVT.isScalableVector());
5151       Vec = DAG.getBitcast(VecVT, Vec);
5152     } else {
5153       // We can't slide this mask vector down, indexed by its i1 elements.
5154       // This poses a problem when we wish to extract a scalable vector which
5155       // can't be re-expressed as a larger type. Just choose the slow path and
5156       // extend to a larger type, then truncate back down.
5157       // TODO: We could probably improve this when extracting certain fixed
5158       // from fixed, where we can extract as i8 and shift the correct element
5159       // right to reach the desired subvector?
5160       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
5161       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
5162       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
5163       Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
5164                         Op.getOperand(1));
5165       SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
5166       return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
5167     }
5168   }
5169 
5170   // If the subvector vector is a fixed-length type, we cannot use subregister
5171   // manipulation to simplify the codegen; we don't know which register of a
5172   // LMUL group contains the specific subvector as we only know the minimum
5173   // register size. Therefore we must slide the vector group down the full
5174   // amount.
5175   if (SubVecVT.isFixedLengthVector()) {
5176     // With an index of 0 this is a cast-like subvector, which can be performed
5177     // with subregister operations.
5178     if (OrigIdx == 0)
5179       return Op;
5180     MVT ContainerVT = VecVT;
5181     if (VecVT.isFixedLengthVector()) {
5182       ContainerVT = getContainerForFixedLengthVector(VecVT);
5183       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5184     }
5185     SDValue Mask =
5186         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5187     // Set the vector length to only the number of elements we care about. This
5188     // avoids sliding down elements we're going to discard straight away.
5189     SDValue VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
5190     SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5191     SDValue Slidedown =
5192         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
5193                     DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
5194     // Now we can use a cast-like subvector extract to get the result.
5195     Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5196                             DAG.getConstant(0, DL, XLenVT));
5197     return DAG.getBitcast(Op.getValueType(), Slidedown);
5198   }
5199 
5200   unsigned SubRegIdx, RemIdx;
5201   std::tie(SubRegIdx, RemIdx) =
5202       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5203           VecVT, SubVecVT, OrigIdx, TRI);
5204 
5205   // If the Idx has been completely eliminated then this is a subvector extract
5206   // which naturally aligns to a vector register. These can easily be handled
5207   // using subregister manipulation.
5208   if (RemIdx == 0)
5209     return Op;
5210 
5211   // Else we must shift our vector register directly to extract the subvector.
5212   // Do this using VSLIDEDOWN.
5213 
5214   // If the vector type is an LMUL-group type, extract a subvector equal to the
5215   // nearest full vector register type. This should resolve to a EXTRACT_SUBREG
5216   // instruction.
5217   MVT InterSubVT = VecVT;
5218   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5219     InterSubVT = getLMUL1VT(VecVT);
5220     Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5221                       DAG.getConstant(OrigIdx - RemIdx, DL, XLenVT));
5222   }
5223 
5224   // Slide this vector register down by the desired number of elements in order
5225   // to place the desired subvector starting at element 0.
5226   SDValue SlidedownAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5227   // For scalable vectors this must be further multiplied by vscale.
5228   SlidedownAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlidedownAmt);
5229 
5230   SDValue Mask, VL;
5231   std::tie(Mask, VL) = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
5232   SDValue Slidedown =
5233       DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, InterSubVT,
5234                   DAG.getUNDEF(InterSubVT), Vec, SlidedownAmt, Mask, VL);
5235 
5236   // Now the vector is in the right position, extract our final subvector. This
5237   // should resolve to a COPY.
5238   Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5239                           DAG.getConstant(0, DL, XLenVT));
5240 
5241   // We might have bitcast from a mask type: cast back to the original type if
5242   // required.
5243   return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
5244 }
5245 
5246 // Lower step_vector to the vid instruction. Any non-identity step value must
5247 // be accounted for my manual expansion.
5248 SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
5249                                               SelectionDAG &DAG) const {
5250   SDLoc DL(Op);
5251   MVT VT = Op.getSimpleValueType();
5252   MVT XLenVT = Subtarget.getXLenVT();
5253   SDValue Mask, VL;
5254   std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
5255   SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
5256   uint64_t StepValImm = Op.getConstantOperandVal(0);
5257   if (StepValImm != 1) {
5258     if (isPowerOf2_64(StepValImm)) {
5259       SDValue StepVal =
5260           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
5261                       DAG.getConstant(Log2_64(StepValImm), DL, XLenVT));
5262       StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
5263     } else {
5264       SDValue StepVal = lowerScalarSplat(
5265           DAG.getConstant(StepValImm, DL, VT.getVectorElementType()), VL, VT,
5266           DL, DAG, Subtarget);
5267       StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
5268     }
5269   }
5270   return StepVec;
5271 }
5272 
5273 // Implement vector_reverse using vrgather.vv with indices determined by
5274 // subtracting the id of each element from (VLMAX-1). This will convert
5275 // the indices like so:
5276 // (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
5277 // TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
5278 SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
5279                                                  SelectionDAG &DAG) const {
5280   SDLoc DL(Op);
5281   MVT VecVT = Op.getSimpleValueType();
5282   unsigned EltSize = VecVT.getScalarSizeInBits();
5283   unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
5284 
5285   unsigned MaxVLMAX = 0;
5286   unsigned VectorBitsMax = Subtarget.getMaxRVVVectorSizeInBits();
5287   if (VectorBitsMax != 0)
5288     MaxVLMAX = ((VectorBitsMax / EltSize) * MinSize) / RISCV::RVVBitsPerBlock;
5289 
5290   unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
5291   MVT IntVT = VecVT.changeVectorElementTypeToInteger();
5292 
5293   // If this is SEW=8 and VLMAX is unknown or more than 256, we need
5294   // to use vrgatherei16.vv.
5295   // TODO: It's also possible to use vrgatherei16.vv for other types to
5296   // decrease register width for the index calculation.
5297   if ((MaxVLMAX == 0 || MaxVLMAX > 256) && EltSize == 8) {
5298     // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
5299     // Reverse each half, then reassemble them in reverse order.
5300     // NOTE: It's also possible that after splitting that VLMAX no longer
5301     // requires vrgatherei16.vv.
5302     if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
5303       SDValue Lo, Hi;
5304       std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
5305       EVT LoVT, HiVT;
5306       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
5307       Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
5308       Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
5309       // Reassemble the low and high pieces reversed.
5310       // FIXME: This is a CONCAT_VECTORS.
5311       SDValue Res =
5312           DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
5313                       DAG.getIntPtrConstant(0, DL));
5314       return DAG.getNode(
5315           ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo,
5316           DAG.getIntPtrConstant(LoVT.getVectorMinNumElements(), DL));
5317     }
5318 
5319     // Just promote the int type to i16 which will double the LMUL.
5320     IntVT = MVT::getVectorVT(MVT::i16, VecVT.getVectorElementCount());
5321     GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
5322   }
5323 
5324   MVT XLenVT = Subtarget.getXLenVT();
5325   SDValue Mask, VL;
5326   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5327 
5328   // Calculate VLMAX-1 for the desired SEW.
5329   unsigned MinElts = VecVT.getVectorMinNumElements();
5330   SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT,
5331                               DAG.getConstant(MinElts, DL, XLenVT));
5332   SDValue VLMinus1 =
5333       DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DAG.getConstant(1, DL, XLenVT));
5334 
5335   // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
5336   bool IsRV32E64 =
5337       !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
5338   SDValue SplatVL;
5339   if (!IsRV32E64)
5340     SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
5341   else
5342     SplatVL = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, IntVT, VLMinus1);
5343 
5344   SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
5345   SDValue Indices =
5346       DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, Mask, VL);
5347 
5348   return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices, Mask, VL);
5349 }
5350 
5351 SDValue
5352 RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
5353                                                      SelectionDAG &DAG) const {
5354   SDLoc DL(Op);
5355   auto *Load = cast<LoadSDNode>(Op);
5356 
5357   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5358                                         Load->getMemoryVT(),
5359                                         *Load->getMemOperand()) &&
5360          "Expecting a correctly-aligned load");
5361 
5362   MVT VT = Op.getSimpleValueType();
5363   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5364 
5365   SDValue VL =
5366       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5367 
5368   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5369   SDValue NewLoad = DAG.getMemIntrinsicNode(
5370       RISCVISD::VLE_VL, DL, VTs, {Load->getChain(), Load->getBasePtr(), VL},
5371       Load->getMemoryVT(), Load->getMemOperand());
5372 
5373   SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
5374   return DAG.getMergeValues({Result, Load->getChain()}, DL);
5375 }
5376 
5377 SDValue
5378 RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
5379                                                       SelectionDAG &DAG) const {
5380   SDLoc DL(Op);
5381   auto *Store = cast<StoreSDNode>(Op);
5382 
5383   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5384                                         Store->getMemoryVT(),
5385                                         *Store->getMemOperand()) &&
5386          "Expecting a correctly-aligned store");
5387 
5388   SDValue StoreVal = Store->getValue();
5389   MVT VT = StoreVal.getSimpleValueType();
5390 
5391   // If the size less than a byte, we need to pad with zeros to make a byte.
5392   if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
5393     VT = MVT::v8i1;
5394     StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
5395                            DAG.getConstant(0, DL, VT), StoreVal,
5396                            DAG.getIntPtrConstant(0, DL));
5397   }
5398 
5399   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5400 
5401   SDValue VL =
5402       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5403 
5404   SDValue NewValue =
5405       convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
5406   return DAG.getMemIntrinsicNode(
5407       RISCVISD::VSE_VL, DL, DAG.getVTList(MVT::Other),
5408       {Store->getChain(), NewValue, Store->getBasePtr(), VL},
5409       Store->getMemoryVT(), Store->getMemOperand());
5410 }
5411 
5412 SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
5413                                              SelectionDAG &DAG) const {
5414   SDLoc DL(Op);
5415   MVT VT = Op.getSimpleValueType();
5416 
5417   const auto *MemSD = cast<MemSDNode>(Op);
5418   EVT MemVT = MemSD->getMemoryVT();
5419   MachineMemOperand *MMO = MemSD->getMemOperand();
5420   SDValue Chain = MemSD->getChain();
5421   SDValue BasePtr = MemSD->getBasePtr();
5422 
5423   SDValue Mask, PassThru, VL;
5424   if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Op)) {
5425     Mask = VPLoad->getMask();
5426     PassThru = DAG.getUNDEF(VT);
5427     VL = VPLoad->getVectorLength();
5428   } else {
5429     const auto *MLoad = cast<MaskedLoadSDNode>(Op);
5430     Mask = MLoad->getMask();
5431     PassThru = MLoad->getPassThru();
5432   }
5433 
5434   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5435 
5436   MVT XLenVT = Subtarget.getXLenVT();
5437 
5438   MVT ContainerVT = VT;
5439   if (VT.isFixedLengthVector()) {
5440     ContainerVT = getContainerForFixedLengthVector(VT);
5441     PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5442     if (!IsUnmasked) {
5443       MVT MaskVT =
5444           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5445       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5446     }
5447   }
5448 
5449   if (!VL)
5450     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5451 
5452   unsigned IntID =
5453       IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
5454   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5455   if (IsUnmasked)
5456     Ops.push_back(DAG.getUNDEF(ContainerVT));
5457   else
5458     Ops.push_back(PassThru);
5459   Ops.push_back(BasePtr);
5460   if (!IsUnmasked)
5461     Ops.push_back(Mask);
5462   Ops.push_back(VL);
5463   if (!IsUnmasked)
5464     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5465 
5466   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5467 
5468   SDValue Result =
5469       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5470   Chain = Result.getValue(1);
5471 
5472   if (VT.isFixedLengthVector())
5473     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5474 
5475   return DAG.getMergeValues({Result, Chain}, DL);
5476 }
5477 
5478 SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
5479                                               SelectionDAG &DAG) const {
5480   SDLoc DL(Op);
5481 
5482   const auto *MemSD = cast<MemSDNode>(Op);
5483   EVT MemVT = MemSD->getMemoryVT();
5484   MachineMemOperand *MMO = MemSD->getMemOperand();
5485   SDValue Chain = MemSD->getChain();
5486   SDValue BasePtr = MemSD->getBasePtr();
5487   SDValue Val, Mask, VL;
5488 
5489   if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Op)) {
5490     Val = VPStore->getValue();
5491     Mask = VPStore->getMask();
5492     VL = VPStore->getVectorLength();
5493   } else {
5494     const auto *MStore = cast<MaskedStoreSDNode>(Op);
5495     Val = MStore->getValue();
5496     Mask = MStore->getMask();
5497   }
5498 
5499   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5500 
5501   MVT VT = Val.getSimpleValueType();
5502   MVT XLenVT = Subtarget.getXLenVT();
5503 
5504   MVT ContainerVT = VT;
5505   if (VT.isFixedLengthVector()) {
5506     ContainerVT = getContainerForFixedLengthVector(VT);
5507 
5508     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
5509     if (!IsUnmasked) {
5510       MVT MaskVT =
5511           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5512       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5513     }
5514   }
5515 
5516   if (!VL)
5517     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5518 
5519   unsigned IntID =
5520       IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
5521   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5522   Ops.push_back(Val);
5523   Ops.push_back(BasePtr);
5524   if (!IsUnmasked)
5525     Ops.push_back(Mask);
5526   Ops.push_back(VL);
5527 
5528   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
5529                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
5530 }
5531 
5532 SDValue
5533 RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(SDValue Op,
5534                                                       SelectionDAG &DAG) const {
5535   MVT InVT = Op.getOperand(0).getSimpleValueType();
5536   MVT ContainerVT = getContainerForFixedLengthVector(InVT);
5537 
5538   MVT VT = Op.getSimpleValueType();
5539 
5540   SDValue Op1 =
5541       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
5542   SDValue Op2 =
5543       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5544 
5545   SDLoc DL(Op);
5546   SDValue VL =
5547       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5548 
5549   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5550   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
5551 
5552   SDValue Cmp = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op1, Op2,
5553                             Op.getOperand(2), Mask, VL);
5554 
5555   return convertFromScalableVector(VT, Cmp, DAG, Subtarget);
5556 }
5557 
5558 SDValue RISCVTargetLowering::lowerFixedLengthVectorLogicOpToRVV(
5559     SDValue Op, SelectionDAG &DAG, unsigned MaskOpc, unsigned VecOpc) const {
5560   MVT VT = Op.getSimpleValueType();
5561 
5562   if (VT.getVectorElementType() == MVT::i1)
5563     return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMask*/ false);
5564 
5565   return lowerToScalableOp(Op, DAG, VecOpc, /*HasMask*/ true);
5566 }
5567 
5568 SDValue
5569 RISCVTargetLowering::lowerFixedLengthVectorShiftToRVV(SDValue Op,
5570                                                       SelectionDAG &DAG) const {
5571   unsigned Opc;
5572   switch (Op.getOpcode()) {
5573   default: llvm_unreachable("Unexpected opcode!");
5574   case ISD::SHL: Opc = RISCVISD::SHL_VL; break;
5575   case ISD::SRA: Opc = RISCVISD::SRA_VL; break;
5576   case ISD::SRL: Opc = RISCVISD::SRL_VL; break;
5577   }
5578 
5579   return lowerToScalableOp(Op, DAG, Opc);
5580 }
5581 
5582 // Lower vector ABS to smax(X, sub(0, X)).
5583 SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
5584   SDLoc DL(Op);
5585   MVT VT = Op.getSimpleValueType();
5586   SDValue X = Op.getOperand(0);
5587 
5588   assert(VT.isFixedLengthVector() && "Unexpected type");
5589 
5590   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5591   X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
5592 
5593   SDValue Mask, VL;
5594   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5595 
5596   SDValue SplatZero =
5597       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
5598                   DAG.getConstant(0, DL, Subtarget.getXLenVT()));
5599   SDValue NegX =
5600       DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, Mask, VL);
5601   SDValue Max =
5602       DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, Mask, VL);
5603 
5604   return convertFromScalableVector(VT, Max, DAG, Subtarget);
5605 }
5606 
5607 SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
5608     SDValue Op, SelectionDAG &DAG) const {
5609   SDLoc DL(Op);
5610   MVT VT = Op.getSimpleValueType();
5611   SDValue Mag = Op.getOperand(0);
5612   SDValue Sign = Op.getOperand(1);
5613   assert(Mag.getValueType() == Sign.getValueType() &&
5614          "Can only handle COPYSIGN with matching types.");
5615 
5616   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5617   Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget);
5618   Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget);
5619 
5620   SDValue Mask, VL;
5621   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5622 
5623   SDValue CopySign =
5624       DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag, Sign, Mask, VL);
5625 
5626   return convertFromScalableVector(VT, CopySign, DAG, Subtarget);
5627 }
5628 
5629 SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
5630     SDValue Op, SelectionDAG &DAG) const {
5631   MVT VT = Op.getSimpleValueType();
5632   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5633 
5634   MVT I1ContainerVT =
5635       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5636 
5637   SDValue CC =
5638       convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget);
5639   SDValue Op1 =
5640       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5641   SDValue Op2 =
5642       convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget);
5643 
5644   SDLoc DL(Op);
5645   SDValue Mask, VL;
5646   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5647 
5648   SDValue Select =
5649       DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC, Op1, Op2, VL);
5650 
5651   return convertFromScalableVector(VT, Select, DAG, Subtarget);
5652 }
5653 
5654 SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
5655                                                unsigned NewOpc,
5656                                                bool HasMask) const {
5657   MVT VT = Op.getSimpleValueType();
5658   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5659 
5660   // Create list of operands by converting existing ones to scalable types.
5661   SmallVector<SDValue, 6> Ops;
5662   for (const SDValue &V : Op->op_values()) {
5663     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5664 
5665     // Pass through non-vector operands.
5666     if (!V.getValueType().isVector()) {
5667       Ops.push_back(V);
5668       continue;
5669     }
5670 
5671     // "cast" fixed length vector to a scalable vector.
5672     assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
5673            "Only fixed length vectors are supported!");
5674     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5675   }
5676 
5677   SDLoc DL(Op);
5678   SDValue Mask, VL;
5679   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5680   if (HasMask)
5681     Ops.push_back(Mask);
5682   Ops.push_back(VL);
5683 
5684   SDValue ScalableRes = DAG.getNode(NewOpc, DL, ContainerVT, Ops);
5685   return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
5686 }
5687 
5688 // Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
5689 // * Operands of each node are assumed to be in the same order.
5690 // * The EVL operand is promoted from i32 to i64 on RV64.
5691 // * Fixed-length vectors are converted to their scalable-vector container
5692 //   types.
5693 SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
5694                                        unsigned RISCVISDOpc) const {
5695   SDLoc DL(Op);
5696   MVT VT = Op.getSimpleValueType();
5697   SmallVector<SDValue, 4> Ops;
5698 
5699   for (const auto &OpIdx : enumerate(Op->ops())) {
5700     SDValue V = OpIdx.value();
5701     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5702     // Pass through operands which aren't fixed-length vectors.
5703     if (!V.getValueType().isFixedLengthVector()) {
5704       Ops.push_back(V);
5705       continue;
5706     }
5707     // "cast" fixed length vector to a scalable vector.
5708     MVT OpVT = V.getSimpleValueType();
5709     MVT ContainerVT = getContainerForFixedLengthVector(OpVT);
5710     assert(useRVVForFixedLengthVectorVT(OpVT) &&
5711            "Only fixed length vectors are supported!");
5712     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5713   }
5714 
5715   if (!VT.isFixedLengthVector())
5716     return DAG.getNode(RISCVISDOpc, DL, VT, Ops);
5717 
5718   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5719 
5720   SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops);
5721 
5722   return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
5723 }
5724 
5725 SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op, SelectionDAG &DAG,
5726                                             unsigned MaskOpc,
5727                                             unsigned VecOpc) const {
5728   MVT VT = Op.getSimpleValueType();
5729   if (VT.getVectorElementType() != MVT::i1)
5730     return lowerVPOp(Op, DAG, VecOpc);
5731 
5732   // It is safe to drop mask parameter as masked-off elements are undef.
5733   SDValue Op1 = Op->getOperand(0);
5734   SDValue Op2 = Op->getOperand(1);
5735   SDValue VL = Op->getOperand(3);
5736 
5737   MVT ContainerVT = VT;
5738   const bool IsFixed = VT.isFixedLengthVector();
5739   if (IsFixed) {
5740     ContainerVT = getContainerForFixedLengthVector(VT);
5741     Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5742     Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
5743   }
5744 
5745   SDLoc DL(Op);
5746   SDValue Val = DAG.getNode(MaskOpc, DL, ContainerVT, Op1, Op2, VL);
5747   if (!IsFixed)
5748     return Val;
5749   return convertFromScalableVector(VT, Val, DAG, Subtarget);
5750 }
5751 
5752 // Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
5753 // matched to a RVV indexed load. The RVV indexed load instructions only
5754 // support the "unsigned unscaled" addressing mode; indices are implicitly
5755 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
5756 // signed or scaled indexing is extended to the XLEN value type and scaled
5757 // accordingly.
5758 SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
5759                                                SelectionDAG &DAG) const {
5760   SDLoc DL(Op);
5761   MVT VT = Op.getSimpleValueType();
5762 
5763   const auto *MemSD = cast<MemSDNode>(Op.getNode());
5764   EVT MemVT = MemSD->getMemoryVT();
5765   MachineMemOperand *MMO = MemSD->getMemOperand();
5766   SDValue Chain = MemSD->getChain();
5767   SDValue BasePtr = MemSD->getBasePtr();
5768 
5769   ISD::LoadExtType LoadExtType;
5770   SDValue Index, Mask, PassThru, VL;
5771 
5772   if (auto *VPGN = dyn_cast<VPGatherSDNode>(Op.getNode())) {
5773     Index = VPGN->getIndex();
5774     Mask = VPGN->getMask();
5775     PassThru = DAG.getUNDEF(VT);
5776     VL = VPGN->getVectorLength();
5777     // VP doesn't support extending loads.
5778     LoadExtType = ISD::NON_EXTLOAD;
5779   } else {
5780     // Else it must be a MGATHER.
5781     auto *MGN = cast<MaskedGatherSDNode>(Op.getNode());
5782     Index = MGN->getIndex();
5783     Mask = MGN->getMask();
5784     PassThru = MGN->getPassThru();
5785     LoadExtType = MGN->getExtensionType();
5786   }
5787 
5788   MVT IndexVT = Index.getSimpleValueType();
5789   MVT XLenVT = Subtarget.getXLenVT();
5790 
5791   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
5792          "Unexpected VTs!");
5793   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
5794   // Targets have to explicitly opt-in for extending vector loads.
5795   assert(LoadExtType == ISD::NON_EXTLOAD &&
5796          "Unexpected extending MGATHER/VP_GATHER");
5797   (void)LoadExtType;
5798 
5799   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
5800   // the selection of the masked intrinsics doesn't do this for us.
5801   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5802 
5803   MVT ContainerVT = VT;
5804   if (VT.isFixedLengthVector()) {
5805     // We need to use the larger of the result and index type to determine the
5806     // scalable type to use so we don't increase LMUL for any operand/result.
5807     if (VT.bitsGE(IndexVT)) {
5808       ContainerVT = getContainerForFixedLengthVector(VT);
5809       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
5810                                  ContainerVT.getVectorElementCount());
5811     } else {
5812       IndexVT = getContainerForFixedLengthVector(IndexVT);
5813       ContainerVT = MVT::getVectorVT(ContainerVT.getVectorElementType(),
5814                                      IndexVT.getVectorElementCount());
5815     }
5816 
5817     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
5818 
5819     if (!IsUnmasked) {
5820       MVT MaskVT =
5821           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5822       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5823       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5824     }
5825   }
5826 
5827   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
5828       IndexVT = IndexVT.changeVectorElementType(XLenVT);
5829       Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
5830   }
5831 
5832   if (!VL)
5833     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5834 
5835   unsigned IntID =
5836       IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
5837   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5838   if (IsUnmasked)
5839     Ops.push_back(DAG.getUNDEF(ContainerVT));
5840   else
5841     Ops.push_back(PassThru);
5842   Ops.push_back(BasePtr);
5843   Ops.push_back(Index);
5844   if (!IsUnmasked)
5845     Ops.push_back(Mask);
5846   Ops.push_back(VL);
5847   if (!IsUnmasked)
5848     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5849 
5850   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5851   SDValue Result =
5852       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5853   Chain = Result.getValue(1);
5854 
5855   if (VT.isFixedLengthVector())
5856     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5857 
5858   return DAG.getMergeValues({Result, Chain}, DL);
5859 }
5860 
5861 // Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
5862 // matched to a RVV indexed store. The RVV indexed store instructions only
5863 // support the "unsigned unscaled" addressing mode; indices are implicitly
5864 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
5865 // signed or scaled indexing is extended to the XLEN value type and scaled
5866 // accordingly.
5867 SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
5868                                                 SelectionDAG &DAG) const {
5869   SDLoc DL(Op);
5870   const auto *MemSD = cast<MemSDNode>(Op.getNode());
5871   EVT MemVT = MemSD->getMemoryVT();
5872   MachineMemOperand *MMO = MemSD->getMemOperand();
5873   SDValue Chain = MemSD->getChain();
5874   SDValue BasePtr = MemSD->getBasePtr();
5875 
5876   bool IsTruncatingStore = false;
5877   SDValue Index, Mask, Val, VL;
5878 
5879   if (auto *VPSN = dyn_cast<VPScatterSDNode>(Op.getNode())) {
5880     Index = VPSN->getIndex();
5881     Mask = VPSN->getMask();
5882     Val = VPSN->getValue();
5883     VL = VPSN->getVectorLength();
5884     // VP doesn't support truncating stores.
5885     IsTruncatingStore = false;
5886   } else {
5887     // Else it must be a MSCATTER.
5888     auto *MSN = cast<MaskedScatterSDNode>(Op.getNode());
5889     Index = MSN->getIndex();
5890     Mask = MSN->getMask();
5891     Val = MSN->getValue();
5892     IsTruncatingStore = MSN->isTruncatingStore();
5893   }
5894 
5895   MVT VT = Val.getSimpleValueType();
5896   MVT IndexVT = Index.getSimpleValueType();
5897   MVT XLenVT = Subtarget.getXLenVT();
5898 
5899   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
5900          "Unexpected VTs!");
5901   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
5902   // Targets have to explicitly opt-in for extending vector loads and
5903   // truncating vector stores.
5904   assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
5905   (void)IsTruncatingStore;
5906 
5907   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
5908   // the selection of the masked intrinsics doesn't do this for us.
5909   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5910 
5911   MVT ContainerVT = VT;
5912   if (VT.isFixedLengthVector()) {
5913     // We need to use the larger of the value and index type to determine the
5914     // scalable type to use so we don't increase LMUL for any operand/result.
5915     if (VT.bitsGE(IndexVT)) {
5916       ContainerVT = getContainerForFixedLengthVector(VT);
5917       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
5918                                  ContainerVT.getVectorElementCount());
5919     } else {
5920       IndexVT = getContainerForFixedLengthVector(IndexVT);
5921       ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
5922                                      IndexVT.getVectorElementCount());
5923     }
5924 
5925     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
5926     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
5927 
5928     if (!IsUnmasked) {
5929       MVT MaskVT =
5930           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5931       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5932     }
5933   }
5934 
5935   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
5936       IndexVT = IndexVT.changeVectorElementType(XLenVT);
5937       Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
5938   }
5939 
5940   if (!VL)
5941     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5942 
5943   unsigned IntID =
5944       IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
5945   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5946   Ops.push_back(Val);
5947   Ops.push_back(BasePtr);
5948   Ops.push_back(Index);
5949   if (!IsUnmasked)
5950     Ops.push_back(Mask);
5951   Ops.push_back(VL);
5952 
5953   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
5954                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
5955 }
5956 
5957 SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
5958                                                SelectionDAG &DAG) const {
5959   const MVT XLenVT = Subtarget.getXLenVT();
5960   SDLoc DL(Op);
5961   SDValue Chain = Op->getOperand(0);
5962   SDValue SysRegNo = DAG.getTargetConstant(
5963       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
5964   SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
5965   SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
5966 
5967   // Encoding used for rounding mode in RISCV differs from that used in
5968   // FLT_ROUNDS. To convert it the RISCV rounding mode is used as an index in a
5969   // table, which consists of a sequence of 4-bit fields, each representing
5970   // corresponding FLT_ROUNDS mode.
5971   static const int Table =
5972       (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
5973       (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
5974       (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
5975       (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
5976       (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
5977 
5978   SDValue Shift =
5979       DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
5980   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
5981                                 DAG.getConstant(Table, DL, XLenVT), Shift);
5982   SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
5983                                DAG.getConstant(7, DL, XLenVT));
5984 
5985   return DAG.getMergeValues({Masked, Chain}, DL);
5986 }
5987 
5988 SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
5989                                                SelectionDAG &DAG) const {
5990   const MVT XLenVT = Subtarget.getXLenVT();
5991   SDLoc DL(Op);
5992   SDValue Chain = Op->getOperand(0);
5993   SDValue RMValue = Op->getOperand(1);
5994   SDValue SysRegNo = DAG.getTargetConstant(
5995       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
5996 
5997   // Encoding used for rounding mode in RISCV differs from that used in
5998   // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
5999   // a table, which consists of a sequence of 4-bit fields, each representing
6000   // corresponding RISCV mode.
6001   static const unsigned Table =
6002       (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
6003       (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
6004       (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
6005       (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
6006       (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
6007 
6008   SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
6009                               DAG.getConstant(2, DL, XLenVT));
6010   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
6011                                 DAG.getConstant(Table, DL, XLenVT), Shift);
6012   RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
6013                         DAG.getConstant(0x7, DL, XLenVT));
6014   return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
6015                      RMValue);
6016 }
6017 
6018 static RISCVISD::NodeType getRISCVWOpcodeByIntr(unsigned IntNo) {
6019   switch (IntNo) {
6020   default:
6021     llvm_unreachable("Unexpected Intrinsic");
6022   case Intrinsic::riscv_grev:
6023     return RISCVISD::GREVW;
6024   case Intrinsic::riscv_gorc:
6025     return RISCVISD::GORCW;
6026   case Intrinsic::riscv_bcompress:
6027     return RISCVISD::BCOMPRESSW;
6028   case Intrinsic::riscv_bdecompress:
6029     return RISCVISD::BDECOMPRESSW;
6030   case Intrinsic::riscv_bfp:
6031     return RISCVISD::BFPW;
6032   case Intrinsic::riscv_fsl:
6033     return RISCVISD::FSLW;
6034   case Intrinsic::riscv_fsr:
6035     return RISCVISD::FSRW;
6036   }
6037 }
6038 
6039 // Converts the given intrinsic to a i64 operation with any extension.
6040 static SDValue customLegalizeToWOpByIntr(SDNode *N, SelectionDAG &DAG,
6041                                          unsigned IntNo) {
6042   SDLoc DL(N);
6043   RISCVISD::NodeType WOpcode = getRISCVWOpcodeByIntr(IntNo);
6044   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6045   SDValue NewOp2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6046   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp1, NewOp2);
6047   // ReplaceNodeResults requires we maintain the same type for the return value.
6048   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
6049 }
6050 
6051 // Returns the opcode of the target-specific SDNode that implements the 32-bit
6052 // form of the given Opcode.
6053 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
6054   switch (Opcode) {
6055   default:
6056     llvm_unreachable("Unexpected opcode");
6057   case ISD::SHL:
6058     return RISCVISD::SLLW;
6059   case ISD::SRA:
6060     return RISCVISD::SRAW;
6061   case ISD::SRL:
6062     return RISCVISD::SRLW;
6063   case ISD::SDIV:
6064     return RISCVISD::DIVW;
6065   case ISD::UDIV:
6066     return RISCVISD::DIVUW;
6067   case ISD::UREM:
6068     return RISCVISD::REMUW;
6069   case ISD::ROTL:
6070     return RISCVISD::ROLW;
6071   case ISD::ROTR:
6072     return RISCVISD::RORW;
6073   case RISCVISD::GREV:
6074     return RISCVISD::GREVW;
6075   case RISCVISD::GORC:
6076     return RISCVISD::GORCW;
6077   }
6078 }
6079 
6080 // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
6081 // node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
6082 // otherwise be promoted to i64, making it difficult to select the
6083 // SLLW/DIVUW/.../*W later one because the fact the operation was originally of
6084 // type i8/i16/i32 is lost.
6085 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
6086                                    unsigned ExtOpc = ISD::ANY_EXTEND) {
6087   SDLoc DL(N);
6088   RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6089   SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
6090   SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
6091   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6092   // ReplaceNodeResults requires we maintain the same type for the return value.
6093   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
6094 }
6095 
6096 // Converts the given 32-bit operation to a i64 operation with signed extension
6097 // semantic to reduce the signed extension instructions.
6098 static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
6099   SDLoc DL(N);
6100   SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6101   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6102   SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
6103   SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6104                                DAG.getValueType(MVT::i32));
6105   return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
6106 }
6107 
6108 void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
6109                                              SmallVectorImpl<SDValue> &Results,
6110                                              SelectionDAG &DAG) const {
6111   SDLoc DL(N);
6112   switch (N->getOpcode()) {
6113   default:
6114     llvm_unreachable("Don't know how to custom type legalize this operation!");
6115   case ISD::STRICT_FP_TO_SINT:
6116   case ISD::STRICT_FP_TO_UINT:
6117   case ISD::FP_TO_SINT:
6118   case ISD::FP_TO_UINT: {
6119     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6120            "Unexpected custom legalisation");
6121     bool IsStrict = N->isStrictFPOpcode();
6122     bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
6123                     N->getOpcode() == ISD::STRICT_FP_TO_SINT;
6124     SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
6125     if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
6126         TargetLowering::TypeSoftenFloat) {
6127       if (!isTypeLegal(Op0.getValueType()))
6128         return;
6129       if (IsStrict) {
6130         unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
6131                                 : RISCVISD::STRICT_FCVT_WU_RV64;
6132         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
6133         SDValue Res = DAG.getNode(
6134             Opc, DL, VTs, N->getOperand(0), Op0,
6135             DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
6136         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6137         Results.push_back(Res.getValue(1));
6138         return;
6139       }
6140       unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
6141       SDValue Res =
6142           DAG.getNode(Opc, DL, MVT::i64, Op0,
6143                       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
6144       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6145       return;
6146     }
6147     // If the FP type needs to be softened, emit a library call using the 'si'
6148     // version. If we left it to default legalization we'd end up with 'di'. If
6149     // the FP type doesn't need to be softened just let generic type
6150     // legalization promote the result type.
6151     RTLIB::Libcall LC;
6152     if (IsSigned)
6153       LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
6154     else
6155       LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
6156     MakeLibCallOptions CallOptions;
6157     EVT OpVT = Op0.getValueType();
6158     CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
6159     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
6160     SDValue Result;
6161     std::tie(Result, Chain) =
6162         makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
6163     Results.push_back(Result);
6164     if (IsStrict)
6165       Results.push_back(Chain);
6166     break;
6167   }
6168   case ISD::READCYCLECOUNTER: {
6169     assert(!Subtarget.is64Bit() &&
6170            "READCYCLECOUNTER only has custom type legalization on riscv32");
6171 
6172     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
6173     SDValue RCW =
6174         DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
6175 
6176     Results.push_back(
6177         DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
6178     Results.push_back(RCW.getValue(2));
6179     break;
6180   }
6181   case ISD::MUL: {
6182     unsigned Size = N->getSimpleValueType(0).getSizeInBits();
6183     unsigned XLen = Subtarget.getXLen();
6184     // This multiply needs to be expanded, try to use MULHSU+MUL if possible.
6185     if (Size > XLen) {
6186       assert(Size == (XLen * 2) && "Unexpected custom legalisation");
6187       SDValue LHS = N->getOperand(0);
6188       SDValue RHS = N->getOperand(1);
6189       APInt HighMask = APInt::getHighBitsSet(Size, XLen);
6190 
6191       bool LHSIsU = DAG.MaskedValueIsZero(LHS, HighMask);
6192       bool RHSIsU = DAG.MaskedValueIsZero(RHS, HighMask);
6193       // We need exactly one side to be unsigned.
6194       if (LHSIsU == RHSIsU)
6195         return;
6196 
6197       auto MakeMULPair = [&](SDValue S, SDValue U) {
6198         MVT XLenVT = Subtarget.getXLenVT();
6199         S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S);
6200         U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U);
6201         SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U);
6202         SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U);
6203         return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
6204       };
6205 
6206       bool LHSIsS = DAG.ComputeNumSignBits(LHS) > XLen;
6207       bool RHSIsS = DAG.ComputeNumSignBits(RHS) > XLen;
6208 
6209       // The other operand should be signed, but still prefer MULH when
6210       // possible.
6211       if (RHSIsU && LHSIsS && !RHSIsS)
6212         Results.push_back(MakeMULPair(LHS, RHS));
6213       else if (LHSIsU && RHSIsS && !LHSIsS)
6214         Results.push_back(MakeMULPair(RHS, LHS));
6215 
6216       return;
6217     }
6218     LLVM_FALLTHROUGH;
6219   }
6220   case ISD::ADD:
6221   case ISD::SUB:
6222     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6223            "Unexpected custom legalisation");
6224     Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
6225     break;
6226   case ISD::SHL:
6227   case ISD::SRA:
6228   case ISD::SRL:
6229     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6230            "Unexpected custom legalisation");
6231     if (N->getOperand(1).getOpcode() != ISD::Constant) {
6232       Results.push_back(customLegalizeToWOp(N, DAG));
6233       break;
6234     }
6235 
6236     // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
6237     // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
6238     // shift amount.
6239     if (N->getOpcode() == ISD::SHL) {
6240       SDLoc DL(N);
6241       SDValue NewOp0 =
6242           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6243       SDValue NewOp1 =
6244           DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
6245       SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
6246       SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6247                                    DAG.getValueType(MVT::i32));
6248       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6249     }
6250 
6251     break;
6252   case ISD::ROTL:
6253   case ISD::ROTR:
6254     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6255            "Unexpected custom legalisation");
6256     Results.push_back(customLegalizeToWOp(N, DAG));
6257     break;
6258   case ISD::CTTZ:
6259   case ISD::CTTZ_ZERO_UNDEF:
6260   case ISD::CTLZ:
6261   case ISD::CTLZ_ZERO_UNDEF: {
6262     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6263            "Unexpected custom legalisation");
6264 
6265     SDValue NewOp0 =
6266         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6267     bool IsCTZ =
6268         N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF;
6269     unsigned Opc = IsCTZ ? RISCVISD::CTZW : RISCVISD::CLZW;
6270     SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
6271     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6272     return;
6273   }
6274   case ISD::SDIV:
6275   case ISD::UDIV:
6276   case ISD::UREM: {
6277     MVT VT = N->getSimpleValueType(0);
6278     assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
6279            Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
6280            "Unexpected custom legalisation");
6281     // Don't promote division/remainder by constant since we should expand those
6282     // to multiply by magic constant.
6283     // FIXME: What if the expansion is disabled for minsize.
6284     if (N->getOperand(1).getOpcode() == ISD::Constant)
6285       return;
6286 
6287     // If the input is i32, use ANY_EXTEND since the W instructions don't read
6288     // the upper 32 bits. For other types we need to sign or zero extend
6289     // based on the opcode.
6290     unsigned ExtOpc = ISD::ANY_EXTEND;
6291     if (VT != MVT::i32)
6292       ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
6293                                            : ISD::ZERO_EXTEND;
6294 
6295     Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
6296     break;
6297   }
6298   case ISD::UADDO:
6299   case ISD::USUBO: {
6300     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6301            "Unexpected custom legalisation");
6302     bool IsAdd = N->getOpcode() == ISD::UADDO;
6303     // Create an ADDW or SUBW.
6304     SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6305     SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6306     SDValue Res =
6307         DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
6308     Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
6309                       DAG.getValueType(MVT::i32));
6310 
6311     // Sign extend the LHS and perform an unsigned compare with the ADDW result.
6312     // Since the inputs are sign extended from i32, this is equivalent to
6313     // comparing the lower 32 bits.
6314     LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6315     SDValue Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
6316                                     IsAdd ? ISD::SETULT : ISD::SETUGT);
6317 
6318     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6319     Results.push_back(Overflow);
6320     return;
6321   }
6322   case ISD::UADDSAT:
6323   case ISD::USUBSAT: {
6324     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6325            "Unexpected custom legalisation");
6326     if (Subtarget.hasStdExtZbb()) {
6327       // With Zbb we can sign extend and let LegalizeDAG use minu/maxu. Using
6328       // sign extend allows overflow of the lower 32 bits to be detected on
6329       // the promoted size.
6330       SDValue LHS =
6331           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6332       SDValue RHS =
6333           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
6334       SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
6335       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6336       return;
6337     }
6338 
6339     // Without Zbb, expand to UADDO/USUBO+select which will trigger our custom
6340     // promotion for UADDO/USUBO.
6341     Results.push_back(expandAddSubSat(N, DAG));
6342     return;
6343   }
6344   case ISD::BITCAST: {
6345     EVT VT = N->getValueType(0);
6346     assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
6347     SDValue Op0 = N->getOperand(0);
6348     EVT Op0VT = Op0.getValueType();
6349     MVT XLenVT = Subtarget.getXLenVT();
6350     if (VT == MVT::i16 && Op0VT == MVT::f16 && Subtarget.hasStdExtZfh()) {
6351       SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
6352       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
6353     } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
6354                Subtarget.hasStdExtF()) {
6355       SDValue FPConv =
6356           DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
6357       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
6358     } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
6359                isTypeLegal(Op0VT)) {
6360       // Custom-legalize bitcasts from fixed-length vector types to illegal
6361       // scalar types in order to improve codegen. Bitcast the vector to a
6362       // one-element vector type whose element type is the same as the result
6363       // type, and extract the first element.
6364       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
6365       if (isTypeLegal(BVT)) {
6366         SDValue BVec = DAG.getBitcast(BVT, Op0);
6367         Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
6368                                       DAG.getConstant(0, DL, XLenVT)));
6369       }
6370     }
6371     break;
6372   }
6373   case RISCVISD::GREV:
6374   case RISCVISD::GORC: {
6375     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6376            "Unexpected custom legalisation");
6377     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6378     // This is similar to customLegalizeToWOp, except that we pass the second
6379     // operand (a TargetConstant) straight through: it is already of type
6380     // XLenVT.
6381     RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6382     SDValue NewOp0 =
6383         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6384     SDValue NewOp1 =
6385         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6386     SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6387     // ReplaceNodeResults requires we maintain the same type for the return
6388     // value.
6389     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6390     break;
6391   }
6392   case RISCVISD::SHFL: {
6393     // There is no SHFLIW instruction, but we can just promote the operation.
6394     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6395            "Unexpected custom legalisation");
6396     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6397     SDValue NewOp0 =
6398         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6399     SDValue NewOp1 =
6400         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6401     SDValue NewRes = DAG.getNode(RISCVISD::SHFL, DL, MVT::i64, NewOp0, NewOp1);
6402     // ReplaceNodeResults requires we maintain the same type for the return
6403     // value.
6404     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6405     break;
6406   }
6407   case ISD::BSWAP:
6408   case ISD::BITREVERSE: {
6409     MVT VT = N->getSimpleValueType(0);
6410     MVT XLenVT = Subtarget.getXLenVT();
6411     assert((VT == MVT::i8 || VT == MVT::i16 ||
6412             (VT == MVT::i32 && Subtarget.is64Bit())) &&
6413            Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
6414     SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
6415     unsigned Imm = VT.getSizeInBits() - 1;
6416     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
6417     if (N->getOpcode() == ISD::BSWAP)
6418       Imm &= ~0x7U;
6419     unsigned Opc = Subtarget.is64Bit() ? RISCVISD::GREVW : RISCVISD::GREV;
6420     SDValue GREVI =
6421         DAG.getNode(Opc, DL, XLenVT, NewOp0, DAG.getConstant(Imm, DL, XLenVT));
6422     // ReplaceNodeResults requires we maintain the same type for the return
6423     // value.
6424     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, GREVI));
6425     break;
6426   }
6427   case ISD::FSHL:
6428   case ISD::FSHR: {
6429     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6430            Subtarget.hasStdExtZbt() && "Unexpected custom legalisation");
6431     SDValue NewOp0 =
6432         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6433     SDValue NewOp1 =
6434         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6435     SDValue NewShAmt =
6436         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6437     // FSLW/FSRW take a 6 bit shift amount but i32 FSHL/FSHR only use 5 bits.
6438     // Mask the shift amount to 5 bits to prevent accidentally setting bit 5.
6439     NewShAmt = DAG.getNode(ISD::AND, DL, MVT::i64, NewShAmt,
6440                            DAG.getConstant(0x1f, DL, MVT::i64));
6441     // fshl and fshr concatenate their operands in the same order. fsrw and fslw
6442     // instruction use different orders. fshl will return its first operand for
6443     // shift of zero, fshr will return its second operand. fsl and fsr both
6444     // return rs1 so the ISD nodes need to have different operand orders.
6445     // Shift amount is in rs2.
6446     unsigned Opc = RISCVISD::FSLW;
6447     if (N->getOpcode() == ISD::FSHR) {
6448       std::swap(NewOp0, NewOp1);
6449       Opc = RISCVISD::FSRW;
6450     }
6451     SDValue NewOp = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, NewShAmt);
6452     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewOp));
6453     break;
6454   }
6455   case ISD::EXTRACT_VECTOR_ELT: {
6456     // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
6457     // type is illegal (currently only vXi64 RV32).
6458     // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
6459     // transferred to the destination register. We issue two of these from the
6460     // upper- and lower- halves of the SEW-bit vector element, slid down to the
6461     // first element.
6462     SDValue Vec = N->getOperand(0);
6463     SDValue Idx = N->getOperand(1);
6464 
6465     // The vector type hasn't been legalized yet so we can't issue target
6466     // specific nodes if it needs legalization.
6467     // FIXME: We would manually legalize if it's important.
6468     if (!isTypeLegal(Vec.getValueType()))
6469       return;
6470 
6471     MVT VecVT = Vec.getSimpleValueType();
6472 
6473     assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
6474            VecVT.getVectorElementType() == MVT::i64 &&
6475            "Unexpected EXTRACT_VECTOR_ELT legalization");
6476 
6477     // If this is a fixed vector, we need to convert it to a scalable vector.
6478     MVT ContainerVT = VecVT;
6479     if (VecVT.isFixedLengthVector()) {
6480       ContainerVT = getContainerForFixedLengthVector(VecVT);
6481       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
6482     }
6483 
6484     MVT XLenVT = Subtarget.getXLenVT();
6485 
6486     // Use a VL of 1 to avoid processing more elements than we need.
6487     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
6488     SDValue VL = DAG.getConstant(1, DL, XLenVT);
6489     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6490 
6491     // Unless the index is known to be 0, we must slide the vector down to get
6492     // the desired element into index 0.
6493     if (!isNullConstant(Idx)) {
6494       Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
6495                         DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
6496     }
6497 
6498     // Extract the lower XLEN bits of the correct vector element.
6499     SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6500 
6501     // To extract the upper XLEN bits of the vector element, shift the first
6502     // element right by 32 bits and re-extract the lower XLEN bits.
6503     SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
6504                                      DAG.getConstant(32, DL, XLenVT), VL);
6505     SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec,
6506                                  ThirtyTwoV, Mask, VL);
6507 
6508     SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6509 
6510     Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6511     break;
6512   }
6513   case ISD::INTRINSIC_WO_CHAIN: {
6514     unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
6515     switch (IntNo) {
6516     default:
6517       llvm_unreachable(
6518           "Don't know how to custom type legalize this intrinsic!");
6519     case Intrinsic::riscv_grev:
6520     case Intrinsic::riscv_gorc:
6521     case Intrinsic::riscv_bcompress:
6522     case Intrinsic::riscv_bdecompress:
6523     case Intrinsic::riscv_bfp: {
6524       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6525              "Unexpected custom legalisation");
6526       Results.push_back(customLegalizeToWOpByIntr(N, DAG, IntNo));
6527       break;
6528     }
6529     case Intrinsic::riscv_fsl:
6530     case Intrinsic::riscv_fsr: {
6531       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6532              "Unexpected custom legalisation");
6533       SDValue NewOp1 =
6534           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6535       SDValue NewOp2 =
6536           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6537       SDValue NewOp3 =
6538           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3));
6539       unsigned Opc = getRISCVWOpcodeByIntr(IntNo);
6540       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2, NewOp3);
6541       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6542       break;
6543     }
6544     case Intrinsic::riscv_orc_b: {
6545       // Lower to the GORCI encoding for orc.b with the operand extended.
6546       SDValue NewOp =
6547           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6548       // If Zbp is enabled, use GORCIW which will sign extend the result.
6549       unsigned Opc =
6550           Subtarget.hasStdExtZbp() ? RISCVISD::GORCW : RISCVISD::GORC;
6551       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp,
6552                                 DAG.getConstant(7, DL, MVT::i64));
6553       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6554       return;
6555     }
6556     case Intrinsic::riscv_shfl:
6557     case Intrinsic::riscv_unshfl: {
6558       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6559              "Unexpected custom legalisation");
6560       SDValue NewOp1 =
6561           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6562       SDValue NewOp2 =
6563           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6564       unsigned Opc =
6565           IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFLW : RISCVISD::UNSHFLW;
6566       if (isa<ConstantSDNode>(N->getOperand(2))) {
6567         NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2,
6568                              DAG.getConstant(0xf, DL, MVT::i64));
6569         Opc =
6570             IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
6571       }
6572       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2);
6573       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6574       break;
6575     }
6576     case Intrinsic::riscv_vmv_x_s: {
6577       EVT VT = N->getValueType(0);
6578       MVT XLenVT = Subtarget.getXLenVT();
6579       if (VT.bitsLT(XLenVT)) {
6580         // Simple case just extract using vmv.x.s and truncate.
6581         SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
6582                                       Subtarget.getXLenVT(), N->getOperand(1));
6583         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
6584         return;
6585       }
6586 
6587       assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
6588              "Unexpected custom legalization");
6589 
6590       // We need to do the move in two steps.
6591       SDValue Vec = N->getOperand(1);
6592       MVT VecVT = Vec.getSimpleValueType();
6593 
6594       // First extract the lower XLEN bits of the element.
6595       SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6596 
6597       // To extract the upper XLEN bits of the vector element, shift the first
6598       // element right by 32 bits and re-extract the lower XLEN bits.
6599       SDValue VL = DAG.getConstant(1, DL, XLenVT);
6600       MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
6601       SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6602       SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT,
6603                                        DAG.getConstant(32, DL, XLenVT), VL);
6604       SDValue LShr32 =
6605           DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV, Mask, VL);
6606       SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6607 
6608       Results.push_back(
6609           DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6610       break;
6611     }
6612     }
6613     break;
6614   }
6615   case ISD::VECREDUCE_ADD:
6616   case ISD::VECREDUCE_AND:
6617   case ISD::VECREDUCE_OR:
6618   case ISD::VECREDUCE_XOR:
6619   case ISD::VECREDUCE_SMAX:
6620   case ISD::VECREDUCE_UMAX:
6621   case ISD::VECREDUCE_SMIN:
6622   case ISD::VECREDUCE_UMIN:
6623     if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
6624       Results.push_back(V);
6625     break;
6626   case ISD::VP_REDUCE_ADD:
6627   case ISD::VP_REDUCE_AND:
6628   case ISD::VP_REDUCE_OR:
6629   case ISD::VP_REDUCE_XOR:
6630   case ISD::VP_REDUCE_SMAX:
6631   case ISD::VP_REDUCE_UMAX:
6632   case ISD::VP_REDUCE_SMIN:
6633   case ISD::VP_REDUCE_UMIN:
6634     if (SDValue V = lowerVPREDUCE(SDValue(N, 0), DAG))
6635       Results.push_back(V);
6636     break;
6637   case ISD::FLT_ROUNDS_: {
6638     SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other);
6639     SDValue Res = DAG.getNode(ISD::FLT_ROUNDS_, DL, VTs, N->getOperand(0));
6640     Results.push_back(Res.getValue(0));
6641     Results.push_back(Res.getValue(1));
6642     break;
6643   }
6644   }
6645 }
6646 
6647 // A structure to hold one of the bit-manipulation patterns below. Together, a
6648 // SHL and non-SHL pattern may form a bit-manipulation pair on a single source:
6649 //   (or (and (shl x, 1), 0xAAAAAAAA),
6650 //       (and (srl x, 1), 0x55555555))
6651 struct RISCVBitmanipPat {
6652   SDValue Op;
6653   unsigned ShAmt;
6654   bool IsSHL;
6655 
6656   bool formsPairWith(const RISCVBitmanipPat &Other) const {
6657     return Op == Other.Op && ShAmt == Other.ShAmt && IsSHL != Other.IsSHL;
6658   }
6659 };
6660 
6661 // Matches patterns of the form
6662 //   (and (shl x, C2), (C1 << C2))
6663 //   (and (srl x, C2), C1)
6664 //   (shl (and x, C1), C2)
6665 //   (srl (and x, (C1 << C2)), C2)
6666 // Where C2 is a power of 2 and C1 has at least that many leading zeroes.
6667 // The expected masks for each shift amount are specified in BitmanipMasks where
6668 // BitmanipMasks[log2(C2)] specifies the expected C1 value.
6669 // The max allowed shift amount is either XLen/2 or XLen/4 determined by whether
6670 // BitmanipMasks contains 6 or 5 entries assuming that the maximum possible
6671 // XLen is 64.
6672 static Optional<RISCVBitmanipPat>
6673 matchRISCVBitmanipPat(SDValue Op, ArrayRef<uint64_t> BitmanipMasks) {
6674   assert((BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) &&
6675          "Unexpected number of masks");
6676   Optional<uint64_t> Mask;
6677   // Optionally consume a mask around the shift operation.
6678   if (Op.getOpcode() == ISD::AND && isa<ConstantSDNode>(Op.getOperand(1))) {
6679     Mask = Op.getConstantOperandVal(1);
6680     Op = Op.getOperand(0);
6681   }
6682   if (Op.getOpcode() != ISD::SHL && Op.getOpcode() != ISD::SRL)
6683     return None;
6684   bool IsSHL = Op.getOpcode() == ISD::SHL;
6685 
6686   if (!isa<ConstantSDNode>(Op.getOperand(1)))
6687     return None;
6688   uint64_t ShAmt = Op.getConstantOperandVal(1);
6689 
6690   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
6691   if (ShAmt >= Width || !isPowerOf2_64(ShAmt))
6692     return None;
6693   // If we don't have enough masks for 64 bit, then we must be trying to
6694   // match SHFL so we're only allowed to shift 1/4 of the width.
6695   if (BitmanipMasks.size() == 5 && ShAmt >= (Width / 2))
6696     return None;
6697 
6698   SDValue Src = Op.getOperand(0);
6699 
6700   // The expected mask is shifted left when the AND is found around SHL
6701   // patterns.
6702   //   ((x >> 1) & 0x55555555)
6703   //   ((x << 1) & 0xAAAAAAAA)
6704   bool SHLExpMask = IsSHL;
6705 
6706   if (!Mask) {
6707     // Sometimes LLVM keeps the mask as an operand of the shift, typically when
6708     // the mask is all ones: consume that now.
6709     if (Src.getOpcode() == ISD::AND && isa<ConstantSDNode>(Src.getOperand(1))) {
6710       Mask = Src.getConstantOperandVal(1);
6711       Src = Src.getOperand(0);
6712       // The expected mask is now in fact shifted left for SRL, so reverse the
6713       // decision.
6714       //   ((x & 0xAAAAAAAA) >> 1)
6715       //   ((x & 0x55555555) << 1)
6716       SHLExpMask = !SHLExpMask;
6717     } else {
6718       // Use a default shifted mask of all-ones if there's no AND, truncated
6719       // down to the expected width. This simplifies the logic later on.
6720       Mask = maskTrailingOnes<uint64_t>(Width);
6721       *Mask &= (IsSHL ? *Mask << ShAmt : *Mask >> ShAmt);
6722     }
6723   }
6724 
6725   unsigned MaskIdx = Log2_32(ShAmt);
6726   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
6727 
6728   if (SHLExpMask)
6729     ExpMask <<= ShAmt;
6730 
6731   if (Mask != ExpMask)
6732     return None;
6733 
6734   return RISCVBitmanipPat{Src, (unsigned)ShAmt, IsSHL};
6735 }
6736 
6737 // Matches any of the following bit-manipulation patterns:
6738 //   (and (shl x, 1), (0x55555555 << 1))
6739 //   (and (srl x, 1), 0x55555555)
6740 //   (shl (and x, 0x55555555), 1)
6741 //   (srl (and x, (0x55555555 << 1)), 1)
6742 // where the shift amount and mask may vary thus:
6743 //   [1]  = 0x55555555 / 0xAAAAAAAA
6744 //   [2]  = 0x33333333 / 0xCCCCCCCC
6745 //   [4]  = 0x0F0F0F0F / 0xF0F0F0F0
6746 //   [8]  = 0x00FF00FF / 0xFF00FF00
6747 //   [16] = 0x0000FFFF / 0xFFFFFFFF
6748 //   [32] = 0x00000000FFFFFFFF / 0xFFFFFFFF00000000 (for RV64)
6749 static Optional<RISCVBitmanipPat> matchGREVIPat(SDValue Op) {
6750   // These are the unshifted masks which we use to match bit-manipulation
6751   // patterns. They may be shifted left in certain circumstances.
6752   static const uint64_t BitmanipMasks[] = {
6753       0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
6754       0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
6755 
6756   return matchRISCVBitmanipPat(Op, BitmanipMasks);
6757 }
6758 
6759 // Match the following pattern as a GREVI(W) operation
6760 //   (or (BITMANIP_SHL x), (BITMANIP_SRL x))
6761 static SDValue combineORToGREV(SDValue Op, SelectionDAG &DAG,
6762                                const RISCVSubtarget &Subtarget) {
6763   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6764   EVT VT = Op.getValueType();
6765 
6766   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
6767     auto LHS = matchGREVIPat(Op.getOperand(0));
6768     auto RHS = matchGREVIPat(Op.getOperand(1));
6769     if (LHS && RHS && LHS->formsPairWith(*RHS)) {
6770       SDLoc DL(Op);
6771       return DAG.getNode(RISCVISD::GREV, DL, VT, LHS->Op,
6772                          DAG.getConstant(LHS->ShAmt, DL, VT));
6773     }
6774   }
6775   return SDValue();
6776 }
6777 
6778 // Matches any the following pattern as a GORCI(W) operation
6779 // 1.  (or (GREVI x, shamt), x) if shamt is a power of 2
6780 // 2.  (or x, (GREVI x, shamt)) if shamt is a power of 2
6781 // 3.  (or (or (BITMANIP_SHL x), x), (BITMANIP_SRL x))
6782 // Note that with the variant of 3.,
6783 //     (or (or (BITMANIP_SHL x), (BITMANIP_SRL x)), x)
6784 // the inner pattern will first be matched as GREVI and then the outer
6785 // pattern will be matched to GORC via the first rule above.
6786 // 4.  (or (rotl/rotr x, bitwidth/2), x)
6787 static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
6788                                const RISCVSubtarget &Subtarget) {
6789   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6790   EVT VT = Op.getValueType();
6791 
6792   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
6793     SDLoc DL(Op);
6794     SDValue Op0 = Op.getOperand(0);
6795     SDValue Op1 = Op.getOperand(1);
6796 
6797     auto MatchOROfReverse = [&](SDValue Reverse, SDValue X) {
6798       if (Reverse.getOpcode() == RISCVISD::GREV && Reverse.getOperand(0) == X &&
6799           isa<ConstantSDNode>(Reverse.getOperand(1)) &&
6800           isPowerOf2_32(Reverse.getConstantOperandVal(1)))
6801         return DAG.getNode(RISCVISD::GORC, DL, VT, X, Reverse.getOperand(1));
6802       // We can also form GORCI from ROTL/ROTR by half the bitwidth.
6803       if ((Reverse.getOpcode() == ISD::ROTL ||
6804            Reverse.getOpcode() == ISD::ROTR) &&
6805           Reverse.getOperand(0) == X &&
6806           isa<ConstantSDNode>(Reverse.getOperand(1))) {
6807         uint64_t RotAmt = Reverse.getConstantOperandVal(1);
6808         if (RotAmt == (VT.getSizeInBits() / 2))
6809           return DAG.getNode(RISCVISD::GORC, DL, VT, X,
6810                              DAG.getConstant(RotAmt, DL, VT));
6811       }
6812       return SDValue();
6813     };
6814 
6815     // Check for either commutable permutation of (or (GREVI x, shamt), x)
6816     if (SDValue V = MatchOROfReverse(Op0, Op1))
6817       return V;
6818     if (SDValue V = MatchOROfReverse(Op1, Op0))
6819       return V;
6820 
6821     // OR is commutable so canonicalize its OR operand to the left
6822     if (Op0.getOpcode() != ISD::OR && Op1.getOpcode() == ISD::OR)
6823       std::swap(Op0, Op1);
6824     if (Op0.getOpcode() != ISD::OR)
6825       return SDValue();
6826     SDValue OrOp0 = Op0.getOperand(0);
6827     SDValue OrOp1 = Op0.getOperand(1);
6828     auto LHS = matchGREVIPat(OrOp0);
6829     // OR is commutable so swap the operands and try again: x might have been
6830     // on the left
6831     if (!LHS) {
6832       std::swap(OrOp0, OrOp1);
6833       LHS = matchGREVIPat(OrOp0);
6834     }
6835     auto RHS = matchGREVIPat(Op1);
6836     if (LHS && RHS && LHS->formsPairWith(*RHS) && LHS->Op == OrOp1) {
6837       return DAG.getNode(RISCVISD::GORC, DL, VT, LHS->Op,
6838                          DAG.getConstant(LHS->ShAmt, DL, VT));
6839     }
6840   }
6841   return SDValue();
6842 }
6843 
6844 // Matches any of the following bit-manipulation patterns:
6845 //   (and (shl x, 1), (0x22222222 << 1))
6846 //   (and (srl x, 1), 0x22222222)
6847 //   (shl (and x, 0x22222222), 1)
6848 //   (srl (and x, (0x22222222 << 1)), 1)
6849 // where the shift amount and mask may vary thus:
6850 //   [1]  = 0x22222222 / 0x44444444
6851 //   [2]  = 0x0C0C0C0C / 0x3C3C3C3C
6852 //   [4]  = 0x00F000F0 / 0x0F000F00
6853 //   [8]  = 0x0000FF00 / 0x00FF0000
6854 //   [16] = 0x00000000FFFF0000 / 0x0000FFFF00000000 (for RV64)
6855 static Optional<RISCVBitmanipPat> matchSHFLPat(SDValue Op) {
6856   // These are the unshifted masks which we use to match bit-manipulation
6857   // patterns. They may be shifted left in certain circumstances.
6858   static const uint64_t BitmanipMasks[] = {
6859       0x2222222222222222ULL, 0x0C0C0C0C0C0C0C0CULL, 0x00F000F000F000F0ULL,
6860       0x0000FF000000FF00ULL, 0x00000000FFFF0000ULL};
6861 
6862   return matchRISCVBitmanipPat(Op, BitmanipMasks);
6863 }
6864 
6865 // Match (or (or (SHFL_SHL x), (SHFL_SHR x)), (SHFL_AND x)
6866 static SDValue combineORToSHFL(SDValue Op, SelectionDAG &DAG,
6867                                const RISCVSubtarget &Subtarget) {
6868   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6869   EVT VT = Op.getValueType();
6870 
6871   if (VT != MVT::i32 && VT != Subtarget.getXLenVT())
6872     return SDValue();
6873 
6874   SDValue Op0 = Op.getOperand(0);
6875   SDValue Op1 = Op.getOperand(1);
6876 
6877   // Or is commutable so canonicalize the second OR to the LHS.
6878   if (Op0.getOpcode() != ISD::OR)
6879     std::swap(Op0, Op1);
6880   if (Op0.getOpcode() != ISD::OR)
6881     return SDValue();
6882 
6883   // We found an inner OR, so our operands are the operands of the inner OR
6884   // and the other operand of the outer OR.
6885   SDValue A = Op0.getOperand(0);
6886   SDValue B = Op0.getOperand(1);
6887   SDValue C = Op1;
6888 
6889   auto Match1 = matchSHFLPat(A);
6890   auto Match2 = matchSHFLPat(B);
6891 
6892   // If neither matched, we failed.
6893   if (!Match1 && !Match2)
6894     return SDValue();
6895 
6896   // We had at least one match. if one failed, try the remaining C operand.
6897   if (!Match1) {
6898     std::swap(A, C);
6899     Match1 = matchSHFLPat(A);
6900     if (!Match1)
6901       return SDValue();
6902   } else if (!Match2) {
6903     std::swap(B, C);
6904     Match2 = matchSHFLPat(B);
6905     if (!Match2)
6906       return SDValue();
6907   }
6908   assert(Match1 && Match2);
6909 
6910   // Make sure our matches pair up.
6911   if (!Match1->formsPairWith(*Match2))
6912     return SDValue();
6913 
6914   // All the remains is to make sure C is an AND with the same input, that masks
6915   // out the bits that are being shuffled.
6916   if (C.getOpcode() != ISD::AND || !isa<ConstantSDNode>(C.getOperand(1)) ||
6917       C.getOperand(0) != Match1->Op)
6918     return SDValue();
6919 
6920   uint64_t Mask = C.getConstantOperandVal(1);
6921 
6922   static const uint64_t BitmanipMasks[] = {
6923       0x9999999999999999ULL, 0xC3C3C3C3C3C3C3C3ULL, 0xF00FF00FF00FF00FULL,
6924       0xFF0000FFFF0000FFULL, 0xFFFF00000000FFFFULL,
6925   };
6926 
6927   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
6928   unsigned MaskIdx = Log2_32(Match1->ShAmt);
6929   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
6930 
6931   if (Mask != ExpMask)
6932     return SDValue();
6933 
6934   SDLoc DL(Op);
6935   return DAG.getNode(RISCVISD::SHFL, DL, VT, Match1->Op,
6936                      DAG.getConstant(Match1->ShAmt, DL, VT));
6937 }
6938 
6939 // Optimize (add (shl x, c0), (shl y, c1)) ->
6940 //          (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
6941 static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
6942                                   const RISCVSubtarget &Subtarget) {
6943   // Perform this optimization only in the zba extension.
6944   if (!Subtarget.hasStdExtZba())
6945     return SDValue();
6946 
6947   // Skip for vector types and larger types.
6948   EVT VT = N->getValueType(0);
6949   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
6950     return SDValue();
6951 
6952   // The two operand nodes must be SHL and have no other use.
6953   SDValue N0 = N->getOperand(0);
6954   SDValue N1 = N->getOperand(1);
6955   if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
6956       !N0->hasOneUse() || !N1->hasOneUse())
6957     return SDValue();
6958 
6959   // Check c0 and c1.
6960   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
6961   auto *N1C = dyn_cast<ConstantSDNode>(N1->getOperand(1));
6962   if (!N0C || !N1C)
6963     return SDValue();
6964   int64_t C0 = N0C->getSExtValue();
6965   int64_t C1 = N1C->getSExtValue();
6966   if (C0 <= 0 || C1 <= 0)
6967     return SDValue();
6968 
6969   // Skip if SH1ADD/SH2ADD/SH3ADD are not applicable.
6970   int64_t Bits = std::min(C0, C1);
6971   int64_t Diff = std::abs(C0 - C1);
6972   if (Diff != 1 && Diff != 2 && Diff != 3)
6973     return SDValue();
6974 
6975   // Build nodes.
6976   SDLoc DL(N);
6977   SDValue NS = (C0 < C1) ? N0->getOperand(0) : N1->getOperand(0);
6978   SDValue NL = (C0 > C1) ? N0->getOperand(0) : N1->getOperand(0);
6979   SDValue NA0 =
6980       DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT));
6981   SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS);
6982   return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT));
6983 }
6984 
6985 // Combine (GREVI (GREVI x, C2), C1) -> (GREVI x, C1^C2) when C1^C2 is
6986 // non-zero, and to x when it is. Any repeated GREVI stage undoes itself.
6987 // Combine (GORCI (GORCI x, C2), C1) -> (GORCI x, C1|C2). Repeated stage does
6988 // not undo itself, but they are redundant.
6989 static SDValue combineGREVI_GORCI(SDNode *N, SelectionDAG &DAG) {
6990   SDValue Src = N->getOperand(0);
6991 
6992   if (Src.getOpcode() != N->getOpcode())
6993     return SDValue();
6994 
6995   if (!isa<ConstantSDNode>(N->getOperand(1)) ||
6996       !isa<ConstantSDNode>(Src.getOperand(1)))
6997     return SDValue();
6998 
6999   unsigned ShAmt1 = N->getConstantOperandVal(1);
7000   unsigned ShAmt2 = Src.getConstantOperandVal(1);
7001   Src = Src.getOperand(0);
7002 
7003   unsigned CombinedShAmt;
7004   if (N->getOpcode() == RISCVISD::GORC || N->getOpcode() == RISCVISD::GORCW)
7005     CombinedShAmt = ShAmt1 | ShAmt2;
7006   else
7007     CombinedShAmt = ShAmt1 ^ ShAmt2;
7008 
7009   if (CombinedShAmt == 0)
7010     return Src;
7011 
7012   SDLoc DL(N);
7013   return DAG.getNode(
7014       N->getOpcode(), DL, N->getValueType(0), Src,
7015       DAG.getConstant(CombinedShAmt, DL, N->getOperand(1).getValueType()));
7016 }
7017 
7018 // Combine a constant select operand into its use:
7019 //
7020 // (and (select cond, -1, c), x)
7021 //   -> (select cond, x, (and x, c))  [AllOnes=1]
7022 // (or  (select cond, 0, c), x)
7023 //   -> (select cond, x, (or x, c))  [AllOnes=0]
7024 // (xor (select cond, 0, c), x)
7025 //   -> (select cond, x, (xor x, c))  [AllOnes=0]
7026 // (add (select cond, 0, c), x)
7027 //   -> (select cond, x, (add x, c))  [AllOnes=0]
7028 // (sub x, (select cond, 0, c))
7029 //   -> (select cond, x, (sub x, c))  [AllOnes=0]
7030 static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
7031                                    SelectionDAG &DAG, bool AllOnes) {
7032   EVT VT = N->getValueType(0);
7033 
7034   // Skip vectors.
7035   if (VT.isVector())
7036     return SDValue();
7037 
7038   if ((Slct.getOpcode() != ISD::SELECT &&
7039        Slct.getOpcode() != RISCVISD::SELECT_CC) ||
7040       !Slct.hasOneUse())
7041     return SDValue();
7042 
7043   auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
7044     return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
7045   };
7046 
7047   bool SwapSelectOps;
7048   unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
7049   SDValue TrueVal = Slct.getOperand(1 + OpOffset);
7050   SDValue FalseVal = Slct.getOperand(2 + OpOffset);
7051   SDValue NonConstantVal;
7052   if (isZeroOrAllOnes(TrueVal, AllOnes)) {
7053     SwapSelectOps = false;
7054     NonConstantVal = FalseVal;
7055   } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
7056     SwapSelectOps = true;
7057     NonConstantVal = TrueVal;
7058   } else
7059     return SDValue();
7060 
7061   // Slct is now know to be the desired identity constant when CC is true.
7062   TrueVal = OtherOp;
7063   FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
7064   // Unless SwapSelectOps says the condition should be false.
7065   if (SwapSelectOps)
7066     std::swap(TrueVal, FalseVal);
7067 
7068   if (Slct.getOpcode() == RISCVISD::SELECT_CC)
7069     return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
7070                        {Slct.getOperand(0), Slct.getOperand(1),
7071                         Slct.getOperand(2), TrueVal, FalseVal});
7072 
7073   return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
7074                      {Slct.getOperand(0), TrueVal, FalseVal});
7075 }
7076 
7077 // Attempt combineSelectAndUse on each operand of a commutative operator N.
7078 static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
7079                                               bool AllOnes) {
7080   SDValue N0 = N->getOperand(0);
7081   SDValue N1 = N->getOperand(1);
7082   if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes))
7083     return Result;
7084   if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes))
7085     return Result;
7086   return SDValue();
7087 }
7088 
7089 // Transform (add (mul x, c0), c1) ->
7090 //           (add (mul (add x, c1/c0), c0), c1%c0).
7091 // if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
7092 // that should be excluded is when c0*(c1/c0) is simm12, which will lead
7093 // to an infinite loop in DAGCombine if transformed.
7094 // Or transform (add (mul x, c0), c1) ->
7095 //              (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
7096 // if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
7097 // case that should be excluded is when c0*(c1/c0+1) is simm12, which will
7098 // lead to an infinite loop in DAGCombine if transformed.
7099 // Or transform (add (mul x, c0), c1) ->
7100 //              (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
7101 // if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
7102 // case that should be excluded is when c0*(c1/c0-1) is simm12, which will
7103 // lead to an infinite loop in DAGCombine if transformed.
7104 // Or transform (add (mul x, c0), c1) ->
7105 //              (mul (add x, c1/c0), c0).
7106 // if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
7107 static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
7108                                      const RISCVSubtarget &Subtarget) {
7109   // Skip for vector types and larger types.
7110   EVT VT = N->getValueType(0);
7111   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
7112     return SDValue();
7113   // The first operand node must be a MUL and has no other use.
7114   SDValue N0 = N->getOperand(0);
7115   if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
7116     return SDValue();
7117   // Check if c0 and c1 match above conditions.
7118   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7119   auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
7120   if (!N0C || !N1C)
7121     return SDValue();
7122   int64_t C0 = N0C->getSExtValue();
7123   int64_t C1 = N1C->getSExtValue();
7124   int64_t CA, CB;
7125   if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
7126     return SDValue();
7127   // Search for proper CA (non-zero) and CB that both are simm12.
7128   if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
7129       !isInt<12>(C0 * (C1 / C0))) {
7130     CA = C1 / C0;
7131     CB = C1 % C0;
7132   } else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
7133              isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
7134     CA = C1 / C0 + 1;
7135     CB = C1 % C0 - C0;
7136   } else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
7137              isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
7138     CA = C1 / C0 - 1;
7139     CB = C1 % C0 + C0;
7140   } else
7141     return SDValue();
7142   // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
7143   SDLoc DL(N);
7144   SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
7145                              DAG.getConstant(CA, DL, VT));
7146   SDValue New1 =
7147       DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
7148   return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
7149 }
7150 
7151 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
7152                                  const RISCVSubtarget &Subtarget) {
7153   if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
7154     return V;
7155   if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
7156     return V;
7157   // fold (add (select lhs, rhs, cc, 0, y), x) ->
7158   //      (select lhs, rhs, cc, x, (add x, y))
7159   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7160 }
7161 
7162 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG) {
7163   // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
7164   //      (select lhs, rhs, cc, x, (sub x, y))
7165   SDValue N0 = N->getOperand(0);
7166   SDValue N1 = N->getOperand(1);
7167   return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false);
7168 }
7169 
7170 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG) {
7171   // fold (and (select lhs, rhs, cc, -1, y), x) ->
7172   //      (select lhs, rhs, cc, x, (and x, y))
7173   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true);
7174 }
7175 
7176 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
7177                                 const RISCVSubtarget &Subtarget) {
7178   if (Subtarget.hasStdExtZbp()) {
7179     if (auto GREV = combineORToGREV(SDValue(N, 0), DAG, Subtarget))
7180       return GREV;
7181     if (auto GORC = combineORToGORC(SDValue(N, 0), DAG, Subtarget))
7182       return GORC;
7183     if (auto SHFL = combineORToSHFL(SDValue(N, 0), DAG, Subtarget))
7184       return SHFL;
7185   }
7186 
7187   // fold (or (select cond, 0, y), x) ->
7188   //      (select cond, x, (or x, y))
7189   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7190 }
7191 
7192 static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG) {
7193   // fold (xor (select cond, 0, y), x) ->
7194   //      (select cond, x, (xor x, y))
7195   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7196 }
7197 
7198 // Attempt to turn ANY_EXTEND into SIGN_EXTEND if the input to the ANY_EXTEND
7199 // has users that require SIGN_EXTEND and the SIGN_EXTEND can be done for free
7200 // by an instruction like ADDW/SUBW/MULW. Without this the ANY_EXTEND would be
7201 // removed during type legalization leaving an ADD/SUB/MUL use that won't use
7202 // ADDW/SUBW/MULW.
7203 static SDValue performANY_EXTENDCombine(SDNode *N,
7204                                         TargetLowering::DAGCombinerInfo &DCI,
7205                                         const RISCVSubtarget &Subtarget) {
7206   if (!Subtarget.is64Bit())
7207     return SDValue();
7208 
7209   SelectionDAG &DAG = DCI.DAG;
7210 
7211   SDValue Src = N->getOperand(0);
7212   EVT VT = N->getValueType(0);
7213   if (VT != MVT::i64 || Src.getValueType() != MVT::i32)
7214     return SDValue();
7215 
7216   // The opcode must be one that can implicitly sign_extend.
7217   // FIXME: Additional opcodes.
7218   switch (Src.getOpcode()) {
7219   default:
7220     return SDValue();
7221   case ISD::MUL:
7222     if (!Subtarget.hasStdExtM())
7223       return SDValue();
7224     LLVM_FALLTHROUGH;
7225   case ISD::ADD:
7226   case ISD::SUB:
7227     break;
7228   }
7229 
7230   // Only handle cases where the result is used by a CopyToReg. That likely
7231   // means the value is a liveout of the basic block. This helps prevent
7232   // infinite combine loops like PR51206.
7233   if (none_of(N->uses(),
7234               [](SDNode *User) { return User->getOpcode() == ISD::CopyToReg; }))
7235     return SDValue();
7236 
7237   SmallVector<SDNode *, 4> SetCCs;
7238   for (SDNode::use_iterator UI = Src.getNode()->use_begin(),
7239                             UE = Src.getNode()->use_end();
7240        UI != UE; ++UI) {
7241     SDNode *User = *UI;
7242     if (User == N)
7243       continue;
7244     if (UI.getUse().getResNo() != Src.getResNo())
7245       continue;
7246     // All i32 setccs are legalized by sign extending operands.
7247     if (User->getOpcode() == ISD::SETCC) {
7248       SetCCs.push_back(User);
7249       continue;
7250     }
7251     // We don't know if we can extend this user.
7252     break;
7253   }
7254 
7255   // If we don't have any SetCCs, this isn't worthwhile.
7256   if (SetCCs.empty())
7257     return SDValue();
7258 
7259   SDLoc DL(N);
7260   SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Src);
7261   DCI.CombineTo(N, SExt);
7262 
7263   // Promote all the setccs.
7264   for (SDNode *SetCC : SetCCs) {
7265     SmallVector<SDValue, 4> Ops;
7266 
7267     for (unsigned j = 0; j != 2; ++j) {
7268       SDValue SOp = SetCC->getOperand(j);
7269       if (SOp == Src)
7270         Ops.push_back(SExt);
7271       else
7272         Ops.push_back(DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, SOp));
7273     }
7274 
7275     Ops.push_back(SetCC->getOperand(2));
7276     DCI.CombineTo(SetCC,
7277                   DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
7278   }
7279   return SDValue(N, 0);
7280 }
7281 
7282 // Try to form VWMUL or VWMULU.
7283 // FIXME: Support VWMULSU.
7284 static SDValue combineMUL_VLToVWMUL_VL(SDNode *N, SelectionDAG &DAG,
7285                                        bool Commute) {
7286   assert(N->getOpcode() == RISCVISD::MUL_VL && "Unexpected opcode");
7287   SDValue Op0 = N->getOperand(0);
7288   SDValue Op1 = N->getOperand(1);
7289   if (Commute)
7290     std::swap(Op0, Op1);
7291 
7292   bool IsSignExt = Op0.getOpcode() == RISCVISD::VSEXT_VL;
7293   bool IsZeroExt = Op0.getOpcode() == RISCVISD::VZEXT_VL;
7294   if ((!IsSignExt && !IsZeroExt) || !Op0.hasOneUse())
7295     return SDValue();
7296 
7297   SDValue Mask = N->getOperand(2);
7298   SDValue VL = N->getOperand(3);
7299 
7300   // Make sure the mask and VL match.
7301   if (Op0.getOperand(1) != Mask || Op0.getOperand(2) != VL)
7302     return SDValue();
7303 
7304   MVT VT = N->getSimpleValueType(0);
7305 
7306   // Determine the narrow size for a widening multiply.
7307   unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
7308   MVT NarrowVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowSize),
7309                                   VT.getVectorElementCount());
7310 
7311   SDLoc DL(N);
7312 
7313   // See if the other operand is the same opcode.
7314   if (Op0.getOpcode() == Op1.getOpcode()) {
7315     if (!Op1.hasOneUse())
7316       return SDValue();
7317 
7318     // Make sure the mask and VL match.
7319     if (Op1.getOperand(1) != Mask || Op1.getOperand(2) != VL)
7320       return SDValue();
7321 
7322     Op1 = Op1.getOperand(0);
7323   } else if (Op1.getOpcode() == RISCVISD::VMV_V_X_VL) {
7324     // The operand is a splat of a scalar.
7325 
7326     // The VL must be the same.
7327     if (Op1.getOperand(1) != VL)
7328       return SDValue();
7329 
7330     // Get the scalar value.
7331     Op1 = Op1.getOperand(0);
7332 
7333     // See if have enough sign bits or zero bits in the scalar to use a
7334     // widening multiply by splatting to smaller element size.
7335     unsigned EltBits = VT.getScalarSizeInBits();
7336     unsigned ScalarBits = Op1.getValueSizeInBits();
7337     // Make sure we're getting all element bits from the scalar register.
7338     // FIXME: Support implicit sign extension of vmv.v.x?
7339     if (ScalarBits < EltBits)
7340       return SDValue();
7341 
7342     if (IsSignExt) {
7343       if (DAG.ComputeNumSignBits(Op1) <= (ScalarBits - NarrowSize))
7344         return SDValue();
7345     } else {
7346       APInt Mask = APInt::getBitsSetFrom(ScalarBits, NarrowSize);
7347       if (!DAG.MaskedValueIsZero(Op1, Mask))
7348         return SDValue();
7349     }
7350 
7351     Op1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT, Op1, VL);
7352   } else
7353     return SDValue();
7354 
7355   Op0 = Op0.getOperand(0);
7356 
7357   // Re-introduce narrower extends if needed.
7358   unsigned ExtOpc = IsSignExt ? RISCVISD::VSEXT_VL : RISCVISD::VZEXT_VL;
7359   if (Op0.getValueType() != NarrowVT)
7360     Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
7361   if (Op1.getValueType() != NarrowVT)
7362     Op1 = DAG.getNode(ExtOpc, DL, NarrowVT, Op1, Mask, VL);
7363 
7364   unsigned WMulOpc = IsSignExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
7365   return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Mask, VL);
7366 }
7367 
7368 static RISCVFPRndMode::RoundingMode matchRoundingOp(SDValue Op) {
7369   switch (Op.getOpcode()) {
7370   case ISD::FROUNDEVEN: return RISCVFPRndMode::RNE;
7371   case ISD::FTRUNC:     return RISCVFPRndMode::RTZ;
7372   case ISD::FFLOOR:     return RISCVFPRndMode::RDN;
7373   case ISD::FCEIL:      return RISCVFPRndMode::RUP;
7374   case ISD::FROUND:     return RISCVFPRndMode::RMM;
7375   }
7376 
7377   return RISCVFPRndMode::Invalid;
7378 }
7379 
7380 // Fold
7381 //   (fp_to_int (froundeven X)) -> fcvt X, rne
7382 //   (fp_to_int (ftrunc X))     -> fcvt X, rtz
7383 //   (fp_to_int (ffloor X))     -> fcvt X, rdn
7384 //   (fp_to_int (fceil X))      -> fcvt X, rup
7385 //   (fp_to_int (fround X))     -> fcvt X, rmm
7386 static SDValue performFP_TO_INTCombine(SDNode *N,
7387                                        TargetLowering::DAGCombinerInfo &DCI,
7388                                        const RISCVSubtarget &Subtarget) {
7389   SelectionDAG &DAG = DCI.DAG;
7390   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7391   MVT XLenVT = Subtarget.getXLenVT();
7392 
7393   // Only handle XLen or i32 types. Other types narrower than XLen will
7394   // eventually be legalized to XLenVT.
7395   EVT VT = N->getValueType(0);
7396   if (VT != MVT::i32 && VT != XLenVT)
7397     return SDValue();
7398 
7399   SDValue Src = N->getOperand(0);
7400 
7401   // Ensure the FP type is also legal.
7402   if (!TLI.isTypeLegal(Src.getValueType()))
7403     return SDValue();
7404 
7405   // Don't do this for f16 with Zfhmin and not Zfh.
7406   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7407     return SDValue();
7408 
7409   RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src);
7410   if (FRM == RISCVFPRndMode::Invalid)
7411     return SDValue();
7412 
7413   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
7414 
7415   unsigned Opc;
7416   if (VT == XLenVT)
7417     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7418   else
7419     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7420 
7421   SDLoc DL(N);
7422   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src.getOperand(0),
7423                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7424   return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
7425 }
7426 
7427 // Fold
7428 //   (fp_to_int_sat (froundeven X)) -> (select X == nan, 0, (fcvt X, rne))
7429 //   (fp_to_int_sat (ftrunc X))     -> (select X == nan, 0, (fcvt X, rtz))
7430 //   (fp_to_int_sat (ffloor X))     -> (select X == nan, 0, (fcvt X, rdn))
7431 //   (fp_to_int_sat (fceil X))      -> (select X == nan, 0, (fcvt X, rup))
7432 //   (fp_to_int_sat (fround X))     -> (select X == nan, 0, (fcvt X, rmm))
7433 static SDValue performFP_TO_INT_SATCombine(SDNode *N,
7434                                        TargetLowering::DAGCombinerInfo &DCI,
7435                                        const RISCVSubtarget &Subtarget) {
7436   SelectionDAG &DAG = DCI.DAG;
7437   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7438   MVT XLenVT = Subtarget.getXLenVT();
7439 
7440   // Only handle XLen types. Other types narrower than XLen will eventually be
7441   // legalized to XLenVT.
7442   EVT DstVT = N->getValueType(0);
7443   if (DstVT != XLenVT)
7444     return SDValue();
7445 
7446   SDValue Src = N->getOperand(0);
7447 
7448   // Ensure the FP type is also legal.
7449   if (!TLI.isTypeLegal(Src.getValueType()))
7450     return SDValue();
7451 
7452   // Don't do this for f16 with Zfhmin and not Zfh.
7453   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7454     return SDValue();
7455 
7456   EVT SatVT = cast<VTSDNode>(N->getOperand(1))->getVT();
7457 
7458   RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src);
7459   if (FRM == RISCVFPRndMode::Invalid)
7460     return SDValue();
7461 
7462   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
7463 
7464   unsigned Opc;
7465   if (SatVT == DstVT)
7466     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7467   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
7468     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7469   else
7470     return SDValue();
7471   // FIXME: Support other SatVTs by clamping before or after the conversion.
7472 
7473   Src = Src.getOperand(0);
7474 
7475   SDLoc DL(N);
7476   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src,
7477                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7478 
7479   // RISCV FP-to-int conversions saturate to the destination register size, but
7480   // don't produce 0 for nan.
7481   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
7482   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
7483 }
7484 
7485 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
7486                                                DAGCombinerInfo &DCI) const {
7487   SelectionDAG &DAG = DCI.DAG;
7488 
7489   // Helper to call SimplifyDemandedBits on an operand of N where only some low
7490   // bits are demanded. N will be added to the Worklist if it was not deleted.
7491   // Caller should return SDValue(N, 0) if this returns true.
7492   auto SimplifyDemandedLowBitsHelper = [&](unsigned OpNo, unsigned LowBits) {
7493     SDValue Op = N->getOperand(OpNo);
7494     APInt Mask = APInt::getLowBitsSet(Op.getValueSizeInBits(), LowBits);
7495     if (!SimplifyDemandedBits(Op, Mask, DCI))
7496       return false;
7497 
7498     if (N->getOpcode() != ISD::DELETED_NODE)
7499       DCI.AddToWorklist(N);
7500     return true;
7501   };
7502 
7503   switch (N->getOpcode()) {
7504   default:
7505     break;
7506   case RISCVISD::SplitF64: {
7507     SDValue Op0 = N->getOperand(0);
7508     // If the input to SplitF64 is just BuildPairF64 then the operation is
7509     // redundant. Instead, use BuildPairF64's operands directly.
7510     if (Op0->getOpcode() == RISCVISD::BuildPairF64)
7511       return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
7512 
7513     SDLoc DL(N);
7514 
7515     // It's cheaper to materialise two 32-bit integers than to load a double
7516     // from the constant pool and transfer it to integer registers through the
7517     // stack.
7518     if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
7519       APInt V = C->getValueAPF().bitcastToAPInt();
7520       SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
7521       SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
7522       return DCI.CombineTo(N, Lo, Hi);
7523     }
7524 
7525     // This is a target-specific version of a DAGCombine performed in
7526     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7527     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7528     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7529     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7530         !Op0.getNode()->hasOneUse())
7531       break;
7532     SDValue NewSplitF64 =
7533         DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
7534                     Op0.getOperand(0));
7535     SDValue Lo = NewSplitF64.getValue(0);
7536     SDValue Hi = NewSplitF64.getValue(1);
7537     APInt SignBit = APInt::getSignMask(32);
7538     if (Op0.getOpcode() == ISD::FNEG) {
7539       SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
7540                                   DAG.getConstant(SignBit, DL, MVT::i32));
7541       return DCI.CombineTo(N, Lo, NewHi);
7542     }
7543     assert(Op0.getOpcode() == ISD::FABS);
7544     SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
7545                                 DAG.getConstant(~SignBit, DL, MVT::i32));
7546     return DCI.CombineTo(N, Lo, NewHi);
7547   }
7548   case RISCVISD::SLLW:
7549   case RISCVISD::SRAW:
7550   case RISCVISD::SRLW:
7551   case RISCVISD::ROLW:
7552   case RISCVISD::RORW: {
7553     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7554     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7555         SimplifyDemandedLowBitsHelper(1, 5))
7556       return SDValue(N, 0);
7557     break;
7558   }
7559   case RISCVISD::CLZW:
7560   case RISCVISD::CTZW: {
7561     // Only the lower 32 bits of the first operand are read
7562     if (SimplifyDemandedLowBitsHelper(0, 32))
7563       return SDValue(N, 0);
7564     break;
7565   }
7566   case RISCVISD::GREV:
7567   case RISCVISD::GORC: {
7568     // Only the lower log2(Bitwidth) bits of the the shift amount are read.
7569     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7570     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7571     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth)))
7572       return SDValue(N, 0);
7573 
7574     return combineGREVI_GORCI(N, DAG);
7575   }
7576   case RISCVISD::GREVW:
7577   case RISCVISD::GORCW: {
7578     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7579     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7580         SimplifyDemandedLowBitsHelper(1, 5))
7581       return SDValue(N, 0);
7582 
7583     return combineGREVI_GORCI(N, DAG);
7584   }
7585   case RISCVISD::SHFL:
7586   case RISCVISD::UNSHFL: {
7587     // Only the lower log2(Bitwidth)-1 bits of the the shift amount are read.
7588     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7589     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7590     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth) - 1))
7591       return SDValue(N, 0);
7592 
7593     break;
7594   }
7595   case RISCVISD::SHFLW:
7596   case RISCVISD::UNSHFLW: {
7597     // Only the lower 32 bits of LHS and lower 4 bits of RHS are read.
7598     SDValue LHS = N->getOperand(0);
7599     SDValue RHS = N->getOperand(1);
7600     APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
7601     APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 4);
7602     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7603         SimplifyDemandedLowBitsHelper(1, 4))
7604       return SDValue(N, 0);
7605 
7606     break;
7607   }
7608   case RISCVISD::BCOMPRESSW:
7609   case RISCVISD::BDECOMPRESSW: {
7610     // Only the lower 32 bits of LHS and RHS are read.
7611     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7612         SimplifyDemandedLowBitsHelper(1, 32))
7613       return SDValue(N, 0);
7614 
7615     break;
7616   }
7617   case RISCVISD::FMV_X_ANYEXTH:
7618   case RISCVISD::FMV_X_ANYEXTW_RV64: {
7619     SDLoc DL(N);
7620     SDValue Op0 = N->getOperand(0);
7621     MVT VT = N->getSimpleValueType(0);
7622     // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
7623     // conversion is unnecessary and can be replaced with the FMV_W_X_RV64
7624     // operand. Similar for FMV_X_ANYEXTH and FMV_H_X.
7625     if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
7626          Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
7627         (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
7628          Op0->getOpcode() == RISCVISD::FMV_H_X)) {
7629       assert(Op0.getOperand(0).getValueType() == VT &&
7630              "Unexpected value type!");
7631       return Op0.getOperand(0);
7632     }
7633 
7634     // This is a target-specific version of a DAGCombine performed in
7635     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7636     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7637     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7638     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7639         !Op0.getNode()->hasOneUse())
7640       break;
7641     SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
7642     unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
7643     APInt SignBit = APInt::getSignMask(FPBits).sextOrSelf(VT.getSizeInBits());
7644     if (Op0.getOpcode() == ISD::FNEG)
7645       return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
7646                          DAG.getConstant(SignBit, DL, VT));
7647 
7648     assert(Op0.getOpcode() == ISD::FABS);
7649     return DAG.getNode(ISD::AND, DL, VT, NewFMV,
7650                        DAG.getConstant(~SignBit, DL, VT));
7651   }
7652   case ISD::ADD:
7653     return performADDCombine(N, DAG, Subtarget);
7654   case ISD::SUB:
7655     return performSUBCombine(N, DAG);
7656   case ISD::AND:
7657     return performANDCombine(N, DAG);
7658   case ISD::OR:
7659     return performORCombine(N, DAG, Subtarget);
7660   case ISD::XOR:
7661     return performXORCombine(N, DAG);
7662   case ISD::ANY_EXTEND:
7663     return performANY_EXTENDCombine(N, DCI, Subtarget);
7664   case ISD::ZERO_EXTEND:
7665     // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
7666     // type legalization. This is safe because fp_to_uint produces poison if
7667     // it overflows.
7668     if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) {
7669       SDValue Src = N->getOperand(0);
7670       if (Src.getOpcode() == ISD::FP_TO_UINT &&
7671           isTypeLegal(Src.getOperand(0).getValueType()))
7672         return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
7673                            Src.getOperand(0));
7674       if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
7675           isTypeLegal(Src.getOperand(1).getValueType())) {
7676         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
7677         SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs,
7678                                   Src.getOperand(0), Src.getOperand(1));
7679         DCI.CombineTo(N, Res);
7680         DAG.ReplaceAllUsesOfValueWith(Src.getValue(1), Res.getValue(1));
7681         DCI.recursivelyDeleteUnusedNodes(Src.getNode());
7682         return SDValue(N, 0); // Return N so it doesn't get rechecked.
7683       }
7684     }
7685     return SDValue();
7686   case RISCVISD::SELECT_CC: {
7687     // Transform
7688     SDValue LHS = N->getOperand(0);
7689     SDValue RHS = N->getOperand(1);
7690     SDValue TrueV = N->getOperand(3);
7691     SDValue FalseV = N->getOperand(4);
7692 
7693     // If the True and False values are the same, we don't need a select_cc.
7694     if (TrueV == FalseV)
7695       return TrueV;
7696 
7697     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(2))->get();
7698     if (!ISD::isIntEqualitySetCC(CCVal))
7699       break;
7700 
7701     // Fold (select_cc (setlt X, Y), 0, ne, trueV, falseV) ->
7702     //      (select_cc X, Y, lt, trueV, falseV)
7703     // Sometimes the setcc is introduced after select_cc has been formed.
7704     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
7705         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
7706       // If we're looking for eq 0 instead of ne 0, we need to invert the
7707       // condition.
7708       bool Invert = CCVal == ISD::SETEQ;
7709       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
7710       if (Invert)
7711         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7712 
7713       SDLoc DL(N);
7714       RHS = LHS.getOperand(1);
7715       LHS = LHS.getOperand(0);
7716       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7717 
7718       SDValue TargetCC = DAG.getCondCode(CCVal);
7719       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
7720                          {LHS, RHS, TargetCC, TrueV, FalseV});
7721     }
7722 
7723     // Fold (select_cc (xor X, Y), 0, eq/ne, trueV, falseV) ->
7724     //      (select_cc X, Y, eq/ne, trueV, falseV)
7725     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
7726       return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), N->getValueType(0),
7727                          {LHS.getOperand(0), LHS.getOperand(1),
7728                           N->getOperand(2), TrueV, FalseV});
7729     // (select_cc X, 1, setne, trueV, falseV) ->
7730     // (select_cc X, 0, seteq, trueV, falseV) if we can prove X is 0/1.
7731     // This can occur when legalizing some floating point comparisons.
7732     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
7733     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
7734       SDLoc DL(N);
7735       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7736       SDValue TargetCC = DAG.getCondCode(CCVal);
7737       RHS = DAG.getConstant(0, DL, LHS.getValueType());
7738       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
7739                          {LHS, RHS, TargetCC, TrueV, FalseV});
7740     }
7741 
7742     break;
7743   }
7744   case RISCVISD::BR_CC: {
7745     SDValue LHS = N->getOperand(1);
7746     SDValue RHS = N->getOperand(2);
7747     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(3))->get();
7748     if (!ISD::isIntEqualitySetCC(CCVal))
7749       break;
7750 
7751     // Fold (br_cc (setlt X, Y), 0, ne, dest) ->
7752     //      (br_cc X, Y, lt, dest)
7753     // Sometimes the setcc is introduced after br_cc has been formed.
7754     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
7755         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
7756       // If we're looking for eq 0 instead of ne 0, we need to invert the
7757       // condition.
7758       bool Invert = CCVal == ISD::SETEQ;
7759       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
7760       if (Invert)
7761         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7762 
7763       SDLoc DL(N);
7764       RHS = LHS.getOperand(1);
7765       LHS = LHS.getOperand(0);
7766       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7767 
7768       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
7769                          N->getOperand(0), LHS, RHS, DAG.getCondCode(CCVal),
7770                          N->getOperand(4));
7771     }
7772 
7773     // Fold (br_cc (xor X, Y), 0, eq/ne, dest) ->
7774     //      (br_cc X, Y, eq/ne, trueV, falseV)
7775     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
7776       return DAG.getNode(RISCVISD::BR_CC, SDLoc(N), N->getValueType(0),
7777                          N->getOperand(0), LHS.getOperand(0), LHS.getOperand(1),
7778                          N->getOperand(3), N->getOperand(4));
7779 
7780     // (br_cc X, 1, setne, br_cc) ->
7781     // (br_cc X, 0, seteq, br_cc) if we can prove X is 0/1.
7782     // This can occur when legalizing some floating point comparisons.
7783     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
7784     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
7785       SDLoc DL(N);
7786       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7787       SDValue TargetCC = DAG.getCondCode(CCVal);
7788       RHS = DAG.getConstant(0, DL, LHS.getValueType());
7789       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
7790                          N->getOperand(0), LHS, RHS, TargetCC,
7791                          N->getOperand(4));
7792     }
7793     break;
7794   }
7795   case ISD::FP_TO_SINT:
7796   case ISD::FP_TO_UINT:
7797     return performFP_TO_INTCombine(N, DCI, Subtarget);
7798   case ISD::FP_TO_SINT_SAT:
7799   case ISD::FP_TO_UINT_SAT:
7800     return performFP_TO_INT_SATCombine(N, DCI, Subtarget);
7801   case ISD::FCOPYSIGN: {
7802     EVT VT = N->getValueType(0);
7803     if (!VT.isVector())
7804       break;
7805     // There is a form of VFSGNJ which injects the negated sign of its second
7806     // operand. Try and bubble any FNEG up after the extend/round to produce
7807     // this optimized pattern. Avoid modifying cases where FP_ROUND and
7808     // TRUNC=1.
7809     SDValue In2 = N->getOperand(1);
7810     // Avoid cases where the extend/round has multiple uses, as duplicating
7811     // those is typically more expensive than removing a fneg.
7812     if (!In2.hasOneUse())
7813       break;
7814     if (In2.getOpcode() != ISD::FP_EXTEND &&
7815         (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0))
7816       break;
7817     In2 = In2.getOperand(0);
7818     if (In2.getOpcode() != ISD::FNEG)
7819       break;
7820     SDLoc DL(N);
7821     SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
7822     return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
7823                        DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
7824   }
7825   case ISD::MGATHER:
7826   case ISD::MSCATTER:
7827   case ISD::VP_GATHER:
7828   case ISD::VP_SCATTER: {
7829     if (!DCI.isBeforeLegalize())
7830       break;
7831     SDValue Index, ScaleOp;
7832     bool IsIndexScaled = false;
7833     bool IsIndexSigned = false;
7834     if (const auto *VPGSN = dyn_cast<VPGatherScatterSDNode>(N)) {
7835       Index = VPGSN->getIndex();
7836       ScaleOp = VPGSN->getScale();
7837       IsIndexScaled = VPGSN->isIndexScaled();
7838       IsIndexSigned = VPGSN->isIndexSigned();
7839     } else {
7840       const auto *MGSN = cast<MaskedGatherScatterSDNode>(N);
7841       Index = MGSN->getIndex();
7842       ScaleOp = MGSN->getScale();
7843       IsIndexScaled = MGSN->isIndexScaled();
7844       IsIndexSigned = MGSN->isIndexSigned();
7845     }
7846     EVT IndexVT = Index.getValueType();
7847     MVT XLenVT = Subtarget.getXLenVT();
7848     // RISCV indexed loads only support the "unsigned unscaled" addressing
7849     // mode, so anything else must be manually legalized.
7850     bool NeedsIdxLegalization =
7851         IsIndexScaled ||
7852         (IsIndexSigned && IndexVT.getVectorElementType().bitsLT(XLenVT));
7853     if (!NeedsIdxLegalization)
7854       break;
7855 
7856     SDLoc DL(N);
7857 
7858     // Any index legalization should first promote to XLenVT, so we don't lose
7859     // bits when scaling. This may create an illegal index type so we let
7860     // LLVM's legalization take care of the splitting.
7861     // FIXME: LLVM can't split VP_GATHER or VP_SCATTER yet.
7862     if (IndexVT.getVectorElementType().bitsLT(XLenVT)) {
7863       IndexVT = IndexVT.changeVectorElementType(XLenVT);
7864       Index = DAG.getNode(IsIndexSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
7865                           DL, IndexVT, Index);
7866     }
7867 
7868     unsigned Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue();
7869     if (IsIndexScaled && Scale != 1) {
7870       // Manually scale the indices by the element size.
7871       // TODO: Sanitize the scale operand here?
7872       // TODO: For VP nodes, should we use VP_SHL here?
7873       assert(isPowerOf2_32(Scale) && "Expecting power-of-two types");
7874       SDValue SplatScale = DAG.getConstant(Log2_32(Scale), DL, IndexVT);
7875       Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, SplatScale);
7876     }
7877 
7878     ISD::MemIndexType NewIndexTy = ISD::UNSIGNED_UNSCALED;
7879     if (const auto *VPGN = dyn_cast<VPGatherSDNode>(N))
7880       return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
7881                              {VPGN->getChain(), VPGN->getBasePtr(), Index,
7882                               VPGN->getScale(), VPGN->getMask(),
7883                               VPGN->getVectorLength()},
7884                              VPGN->getMemOperand(), NewIndexTy);
7885     if (const auto *VPSN = dyn_cast<VPScatterSDNode>(N))
7886       return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
7887                               {VPSN->getChain(), VPSN->getValue(),
7888                                VPSN->getBasePtr(), Index, VPSN->getScale(),
7889                                VPSN->getMask(), VPSN->getVectorLength()},
7890                               VPSN->getMemOperand(), NewIndexTy);
7891     if (const auto *MGN = dyn_cast<MaskedGatherSDNode>(N))
7892       return DAG.getMaskedGather(
7893           N->getVTList(), MGN->getMemoryVT(), DL,
7894           {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
7895            MGN->getBasePtr(), Index, MGN->getScale()},
7896           MGN->getMemOperand(), NewIndexTy, MGN->getExtensionType());
7897     const auto *MSN = cast<MaskedScatterSDNode>(N);
7898     return DAG.getMaskedScatter(
7899         N->getVTList(), MSN->getMemoryVT(), DL,
7900         {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
7901          Index, MSN->getScale()},
7902         MSN->getMemOperand(), NewIndexTy, MSN->isTruncatingStore());
7903   }
7904   case RISCVISD::SRA_VL:
7905   case RISCVISD::SRL_VL:
7906   case RISCVISD::SHL_VL: {
7907     SDValue ShAmt = N->getOperand(1);
7908     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
7909       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
7910       SDLoc DL(N);
7911       SDValue VL = N->getOperand(3);
7912       EVT VT = N->getValueType(0);
7913       ShAmt =
7914           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, ShAmt.getOperand(0), VL);
7915       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
7916                          N->getOperand(2), N->getOperand(3));
7917     }
7918     break;
7919   }
7920   case ISD::SRA:
7921   case ISD::SRL:
7922   case ISD::SHL: {
7923     SDValue ShAmt = N->getOperand(1);
7924     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
7925       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
7926       SDLoc DL(N);
7927       EVT VT = N->getValueType(0);
7928       ShAmt =
7929           DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VT, ShAmt.getOperand(0));
7930       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
7931     }
7932     break;
7933   }
7934   case RISCVISD::MUL_VL:
7935     if (SDValue V = combineMUL_VLToVWMUL_VL(N, DAG, /*Commute*/ false))
7936       return V;
7937     // Mul is commutative.
7938     return combineMUL_VLToVWMUL_VL(N, DAG, /*Commute*/ true);
7939   case ISD::STORE: {
7940     auto *Store = cast<StoreSDNode>(N);
7941     SDValue Val = Store->getValue();
7942     // Combine store of vmv.x.s to vse with VL of 1.
7943     // FIXME: Support FP.
7944     if (Val.getOpcode() == RISCVISD::VMV_X_S) {
7945       SDValue Src = Val.getOperand(0);
7946       EVT VecVT = Src.getValueType();
7947       EVT MemVT = Store->getMemoryVT();
7948       // The memory VT and the element type must match.
7949       if (VecVT.getVectorElementType() == MemVT) {
7950         SDLoc DL(N);
7951         MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
7952         return DAG.getStoreVP(
7953             Store->getChain(), DL, Src, Store->getBasePtr(), Store->getOffset(),
7954             DAG.getConstant(1, DL, MaskVT),
7955             DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT,
7956             Store->getMemOperand(), Store->getAddressingMode(),
7957             Store->isTruncatingStore(), /*IsCompress*/ false);
7958       }
7959     }
7960 
7961     break;
7962   }
7963   }
7964 
7965   return SDValue();
7966 }
7967 
7968 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
7969     const SDNode *N, CombineLevel Level) const {
7970   // The following folds are only desirable if `(OP _, c1 << c2)` can be
7971   // materialised in fewer instructions than `(OP _, c1)`:
7972   //
7973   //   (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
7974   //   (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
7975   SDValue N0 = N->getOperand(0);
7976   EVT Ty = N0.getValueType();
7977   if (Ty.isScalarInteger() &&
7978       (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
7979     auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7980     auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
7981     if (C1 && C2) {
7982       const APInt &C1Int = C1->getAPIntValue();
7983       APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
7984 
7985       // We can materialise `c1 << c2` into an add immediate, so it's "free",
7986       // and the combine should happen, to potentially allow further combines
7987       // later.
7988       if (ShiftedC1Int.getMinSignedBits() <= 64 &&
7989           isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
7990         return true;
7991 
7992       // We can materialise `c1` in an add immediate, so it's "free", and the
7993       // combine should be prevented.
7994       if (C1Int.getMinSignedBits() <= 64 &&
7995           isLegalAddImmediate(C1Int.getSExtValue()))
7996         return false;
7997 
7998       // Neither constant will fit into an immediate, so find materialisation
7999       // costs.
8000       int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
8001                                               Subtarget.getFeatureBits(),
8002                                               /*CompressionCost*/true);
8003       int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
8004           ShiftedC1Int, Ty.getSizeInBits(), Subtarget.getFeatureBits(),
8005           /*CompressionCost*/true);
8006 
8007       // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
8008       // combine should be prevented.
8009       if (C1Cost < ShiftedC1Cost)
8010         return false;
8011     }
8012   }
8013   return true;
8014 }
8015 
8016 bool RISCVTargetLowering::targetShrinkDemandedConstant(
8017     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
8018     TargetLoweringOpt &TLO) const {
8019   // Delay this optimization as late as possible.
8020   if (!TLO.LegalOps)
8021     return false;
8022 
8023   EVT VT = Op.getValueType();
8024   if (VT.isVector())
8025     return false;
8026 
8027   // Only handle AND for now.
8028   if (Op.getOpcode() != ISD::AND)
8029     return false;
8030 
8031   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
8032   if (!C)
8033     return false;
8034 
8035   const APInt &Mask = C->getAPIntValue();
8036 
8037   // Clear all non-demanded bits initially.
8038   APInt ShrunkMask = Mask & DemandedBits;
8039 
8040   // Try to make a smaller immediate by setting undemanded bits.
8041 
8042   APInt ExpandedMask = Mask | ~DemandedBits;
8043 
8044   auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
8045     return ShrunkMask.isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
8046   };
8047   auto UseMask = [Mask, Op, VT, &TLO](const APInt &NewMask) -> bool {
8048     if (NewMask == Mask)
8049       return true;
8050     SDLoc DL(Op);
8051     SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
8052     SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
8053     return TLO.CombineTo(Op, NewOp);
8054   };
8055 
8056   // If the shrunk mask fits in sign extended 12 bits, let the target
8057   // independent code apply it.
8058   if (ShrunkMask.isSignedIntN(12))
8059     return false;
8060 
8061   // Preserve (and X, 0xffff) when zext.h is supported.
8062   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
8063     APInt NewMask = APInt(Mask.getBitWidth(), 0xffff);
8064     if (IsLegalMask(NewMask))
8065       return UseMask(NewMask);
8066   }
8067 
8068   // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
8069   if (VT == MVT::i64) {
8070     APInt NewMask = APInt(64, 0xffffffff);
8071     if (IsLegalMask(NewMask))
8072       return UseMask(NewMask);
8073   }
8074 
8075   // For the remaining optimizations, we need to be able to make a negative
8076   // number through a combination of mask and undemanded bits.
8077   if (!ExpandedMask.isNegative())
8078     return false;
8079 
8080   // What is the fewest number of bits we need to represent the negative number.
8081   unsigned MinSignedBits = ExpandedMask.getMinSignedBits();
8082 
8083   // Try to make a 12 bit negative immediate. If that fails try to make a 32
8084   // bit negative immediate unless the shrunk immediate already fits in 32 bits.
8085   APInt NewMask = ShrunkMask;
8086   if (MinSignedBits <= 12)
8087     NewMask.setBitsFrom(11);
8088   else if (MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(32))
8089     NewMask.setBitsFrom(31);
8090   else
8091     return false;
8092 
8093   // Check that our new mask is a subset of the demanded mask.
8094   assert(IsLegalMask(NewMask));
8095   return UseMask(NewMask);
8096 }
8097 
8098 static void computeGREV(APInt &Src, unsigned ShAmt) {
8099   ShAmt &= Src.getBitWidth() - 1;
8100   uint64_t x = Src.getZExtValue();
8101   if (ShAmt & 1)
8102     x = ((x & 0x5555555555555555LL) << 1) | ((x & 0xAAAAAAAAAAAAAAAALL) >> 1);
8103   if (ShAmt & 2)
8104     x = ((x & 0x3333333333333333LL) << 2) | ((x & 0xCCCCCCCCCCCCCCCCLL) >> 2);
8105   if (ShAmt & 4)
8106     x = ((x & 0x0F0F0F0F0F0F0F0FLL) << 4) | ((x & 0xF0F0F0F0F0F0F0F0LL) >> 4);
8107   if (ShAmt & 8)
8108     x = ((x & 0x00FF00FF00FF00FFLL) << 8) | ((x & 0xFF00FF00FF00FF00LL) >> 8);
8109   if (ShAmt & 16)
8110     x = ((x & 0x0000FFFF0000FFFFLL) << 16) | ((x & 0xFFFF0000FFFF0000LL) >> 16);
8111   if (ShAmt & 32)
8112     x = ((x & 0x00000000FFFFFFFFLL) << 32) | ((x & 0xFFFFFFFF00000000LL) >> 32);
8113   Src = x;
8114 }
8115 
8116 void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
8117                                                         KnownBits &Known,
8118                                                         const APInt &DemandedElts,
8119                                                         const SelectionDAG &DAG,
8120                                                         unsigned Depth) const {
8121   unsigned BitWidth = Known.getBitWidth();
8122   unsigned Opc = Op.getOpcode();
8123   assert((Opc >= ISD::BUILTIN_OP_END ||
8124           Opc == ISD::INTRINSIC_WO_CHAIN ||
8125           Opc == ISD::INTRINSIC_W_CHAIN ||
8126           Opc == ISD::INTRINSIC_VOID) &&
8127          "Should use MaskedValueIsZero if you don't know whether Op"
8128          " is a target node!");
8129 
8130   Known.resetAll();
8131   switch (Opc) {
8132   default: break;
8133   case RISCVISD::SELECT_CC: {
8134     Known = DAG.computeKnownBits(Op.getOperand(4), Depth + 1);
8135     // If we don't know any bits, early out.
8136     if (Known.isUnknown())
8137       break;
8138     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(3), Depth + 1);
8139 
8140     // Only known if known in both the LHS and RHS.
8141     Known = KnownBits::commonBits(Known, Known2);
8142     break;
8143   }
8144   case RISCVISD::REMUW: {
8145     KnownBits Known2;
8146     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
8147     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
8148     // We only care about the lower 32 bits.
8149     Known = KnownBits::urem(Known.trunc(32), Known2.trunc(32));
8150     // Restore the original width by sign extending.
8151     Known = Known.sext(BitWidth);
8152     break;
8153   }
8154   case RISCVISD::DIVUW: {
8155     KnownBits Known2;
8156     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
8157     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
8158     // We only care about the lower 32 bits.
8159     Known = KnownBits::udiv(Known.trunc(32), Known2.trunc(32));
8160     // Restore the original width by sign extending.
8161     Known = Known.sext(BitWidth);
8162     break;
8163   }
8164   case RISCVISD::CTZW: {
8165     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8166     unsigned PossibleTZ = Known2.trunc(32).countMaxTrailingZeros();
8167     unsigned LowBits = Log2_32(PossibleTZ) + 1;
8168     Known.Zero.setBitsFrom(LowBits);
8169     break;
8170   }
8171   case RISCVISD::CLZW: {
8172     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8173     unsigned PossibleLZ = Known2.trunc(32).countMaxLeadingZeros();
8174     unsigned LowBits = Log2_32(PossibleLZ) + 1;
8175     Known.Zero.setBitsFrom(LowBits);
8176     break;
8177   }
8178   case RISCVISD::GREV:
8179   case RISCVISD::GREVW: {
8180     if (auto *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
8181       Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8182       if (Opc == RISCVISD::GREVW)
8183         Known = Known.trunc(32);
8184       unsigned ShAmt = C->getZExtValue();
8185       computeGREV(Known.Zero, ShAmt);
8186       computeGREV(Known.One, ShAmt);
8187       if (Opc == RISCVISD::GREVW)
8188         Known = Known.sext(BitWidth);
8189     }
8190     break;
8191   }
8192   case RISCVISD::READ_VLENB:
8193     // We assume VLENB is at least 16 bytes.
8194     Known.Zero.setLowBits(4);
8195     // We assume VLENB is no more than 65536 / 8 bytes.
8196     Known.Zero.setBitsFrom(14);
8197     break;
8198   case ISD::INTRINSIC_W_CHAIN:
8199   case ISD::INTRINSIC_WO_CHAIN: {
8200     unsigned IntNo =
8201         Op.getConstantOperandVal(Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1);
8202     switch (IntNo) {
8203     default:
8204       // We can't do anything for most intrinsics.
8205       break;
8206     case Intrinsic::riscv_vsetvli:
8207     case Intrinsic::riscv_vsetvlimax:
8208     case Intrinsic::riscv_vsetvli_opt:
8209     case Intrinsic::riscv_vsetvlimax_opt:
8210       // Assume that VL output is positive and would fit in an int32_t.
8211       // TODO: VLEN might be capped at 16 bits in a future V spec update.
8212       if (BitWidth >= 32)
8213         Known.Zero.setBitsFrom(31);
8214       break;
8215     }
8216     break;
8217   }
8218   }
8219 }
8220 
8221 unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
8222     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
8223     unsigned Depth) const {
8224   switch (Op.getOpcode()) {
8225   default:
8226     break;
8227   case RISCVISD::SELECT_CC: {
8228     unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth + 1);
8229     if (Tmp == 1) return 1;  // Early out.
8230     unsigned Tmp2 = DAG.ComputeNumSignBits(Op.getOperand(4), DemandedElts, Depth + 1);
8231     return std::min(Tmp, Tmp2);
8232   }
8233   case RISCVISD::SLLW:
8234   case RISCVISD::SRAW:
8235   case RISCVISD::SRLW:
8236   case RISCVISD::DIVW:
8237   case RISCVISD::DIVUW:
8238   case RISCVISD::REMUW:
8239   case RISCVISD::ROLW:
8240   case RISCVISD::RORW:
8241   case RISCVISD::GREVW:
8242   case RISCVISD::GORCW:
8243   case RISCVISD::FSLW:
8244   case RISCVISD::FSRW:
8245   case RISCVISD::SHFLW:
8246   case RISCVISD::UNSHFLW:
8247   case RISCVISD::BCOMPRESSW:
8248   case RISCVISD::BDECOMPRESSW:
8249   case RISCVISD::BFPW:
8250   case RISCVISD::FCVT_W_RV64:
8251   case RISCVISD::FCVT_WU_RV64:
8252   case RISCVISD::STRICT_FCVT_W_RV64:
8253   case RISCVISD::STRICT_FCVT_WU_RV64:
8254     // TODO: As the result is sign-extended, this is conservatively correct. A
8255     // more precise answer could be calculated for SRAW depending on known
8256     // bits in the shift amount.
8257     return 33;
8258   case RISCVISD::SHFL:
8259   case RISCVISD::UNSHFL: {
8260     // There is no SHFLIW, but a i64 SHFLI with bit 4 of the control word
8261     // cleared doesn't affect bit 31. The upper 32 bits will be shuffled, but
8262     // will stay within the upper 32 bits. If there were more than 32 sign bits
8263     // before there will be at least 33 sign bits after.
8264     if (Op.getValueType() == MVT::i64 &&
8265         isa<ConstantSDNode>(Op.getOperand(1)) &&
8266         (Op.getConstantOperandVal(1) & 0x10) == 0) {
8267       unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
8268       if (Tmp > 32)
8269         return 33;
8270     }
8271     break;
8272   }
8273   case RISCVISD::VMV_X_S:
8274     // The number of sign bits of the scalar result is computed by obtaining the
8275     // element type of the input vector operand, subtracting its width from the
8276     // XLEN, and then adding one (sign bit within the element type). If the
8277     // element type is wider than XLen, the least-significant XLEN bits are
8278     // taken.
8279     if (Op.getOperand(0).getScalarValueSizeInBits() > Subtarget.getXLen())
8280       return 1;
8281     return Subtarget.getXLen() - Op.getOperand(0).getScalarValueSizeInBits() + 1;
8282   }
8283 
8284   return 1;
8285 }
8286 
8287 static MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
8288                                                   MachineBasicBlock *BB) {
8289   assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction");
8290 
8291   // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
8292   // Should the count have wrapped while it was being read, we need to try
8293   // again.
8294   // ...
8295   // read:
8296   // rdcycleh x3 # load high word of cycle
8297   // rdcycle  x2 # load low word of cycle
8298   // rdcycleh x4 # load high word of cycle
8299   // bne x3, x4, read # check if high word reads match, otherwise try again
8300   // ...
8301 
8302   MachineFunction &MF = *BB->getParent();
8303   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8304   MachineFunction::iterator It = ++BB->getIterator();
8305 
8306   MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8307   MF.insert(It, LoopMBB);
8308 
8309   MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8310   MF.insert(It, DoneMBB);
8311 
8312   // Transfer the remainder of BB and its successor edges to DoneMBB.
8313   DoneMBB->splice(DoneMBB->begin(), BB,
8314                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
8315   DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
8316 
8317   BB->addSuccessor(LoopMBB);
8318 
8319   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8320   Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
8321   Register LoReg = MI.getOperand(0).getReg();
8322   Register HiReg = MI.getOperand(1).getReg();
8323   DebugLoc DL = MI.getDebugLoc();
8324 
8325   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8326   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
8327       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8328       .addReg(RISCV::X0);
8329   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
8330       .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
8331       .addReg(RISCV::X0);
8332   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
8333       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8334       .addReg(RISCV::X0);
8335 
8336   BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
8337       .addReg(HiReg)
8338       .addReg(ReadAgainReg)
8339       .addMBB(LoopMBB);
8340 
8341   LoopMBB->addSuccessor(LoopMBB);
8342   LoopMBB->addSuccessor(DoneMBB);
8343 
8344   MI.eraseFromParent();
8345 
8346   return DoneMBB;
8347 }
8348 
8349 static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
8350                                              MachineBasicBlock *BB) {
8351   assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
8352 
8353   MachineFunction &MF = *BB->getParent();
8354   DebugLoc DL = MI.getDebugLoc();
8355   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8356   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8357   Register LoReg = MI.getOperand(0).getReg();
8358   Register HiReg = MI.getOperand(1).getReg();
8359   Register SrcReg = MI.getOperand(2).getReg();
8360   const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
8361   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8362 
8363   TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
8364                           RI);
8365   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8366   MachineMemOperand *MMOLo =
8367       MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 4, Align(8));
8368   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8369       MPI.getWithOffset(4), MachineMemOperand::MOLoad, 4, Align(8));
8370   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
8371       .addFrameIndex(FI)
8372       .addImm(0)
8373       .addMemOperand(MMOLo);
8374   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
8375       .addFrameIndex(FI)
8376       .addImm(4)
8377       .addMemOperand(MMOHi);
8378   MI.eraseFromParent(); // The pseudo instruction is gone now.
8379   return BB;
8380 }
8381 
8382 static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
8383                                                  MachineBasicBlock *BB) {
8384   assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
8385          "Unexpected instruction");
8386 
8387   MachineFunction &MF = *BB->getParent();
8388   DebugLoc DL = MI.getDebugLoc();
8389   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8390   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8391   Register DstReg = MI.getOperand(0).getReg();
8392   Register LoReg = MI.getOperand(1).getReg();
8393   Register HiReg = MI.getOperand(2).getReg();
8394   const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
8395   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8396 
8397   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8398   MachineMemOperand *MMOLo =
8399       MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Align(8));
8400   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8401       MPI.getWithOffset(4), MachineMemOperand::MOStore, 4, Align(8));
8402   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8403       .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
8404       .addFrameIndex(FI)
8405       .addImm(0)
8406       .addMemOperand(MMOLo);
8407   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8408       .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
8409       .addFrameIndex(FI)
8410       .addImm(4)
8411       .addMemOperand(MMOHi);
8412   TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
8413   MI.eraseFromParent(); // The pseudo instruction is gone now.
8414   return BB;
8415 }
8416 
8417 static bool isSelectPseudo(MachineInstr &MI) {
8418   switch (MI.getOpcode()) {
8419   default:
8420     return false;
8421   case RISCV::Select_GPR_Using_CC_GPR:
8422   case RISCV::Select_FPR16_Using_CC_GPR:
8423   case RISCV::Select_FPR32_Using_CC_GPR:
8424   case RISCV::Select_FPR64_Using_CC_GPR:
8425     return true;
8426   }
8427 }
8428 
8429 static MachineBasicBlock *emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB,
8430                                         unsigned RelOpcode, unsigned EqOpcode,
8431                                         const RISCVSubtarget &Subtarget) {
8432   DebugLoc DL = MI.getDebugLoc();
8433   Register DstReg = MI.getOperand(0).getReg();
8434   Register Src1Reg = MI.getOperand(1).getReg();
8435   Register Src2Reg = MI.getOperand(2).getReg();
8436   MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
8437   Register SavedFFlags = MRI.createVirtualRegister(&RISCV::GPRRegClass);
8438   const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
8439 
8440   // Save the current FFLAGS.
8441   BuildMI(*BB, MI, DL, TII.get(RISCV::ReadFFLAGS), SavedFFlags);
8442 
8443   auto MIB = BuildMI(*BB, MI, DL, TII.get(RelOpcode), DstReg)
8444                  .addReg(Src1Reg)
8445                  .addReg(Src2Reg);
8446   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8447     MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
8448 
8449   // Restore the FFLAGS.
8450   BuildMI(*BB, MI, DL, TII.get(RISCV::WriteFFLAGS))
8451       .addReg(SavedFFlags, RegState::Kill);
8452 
8453   // Issue a dummy FEQ opcode to raise exception for signaling NaNs.
8454   auto MIB2 = BuildMI(*BB, MI, DL, TII.get(EqOpcode), RISCV::X0)
8455                   .addReg(Src1Reg, getKillRegState(MI.getOperand(1).isKill()))
8456                   .addReg(Src2Reg, getKillRegState(MI.getOperand(2).isKill()));
8457   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8458     MIB2->setFlag(MachineInstr::MIFlag::NoFPExcept);
8459 
8460   // Erase the pseudoinstruction.
8461   MI.eraseFromParent();
8462   return BB;
8463 }
8464 
8465 static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
8466                                            MachineBasicBlock *BB,
8467                                            const RISCVSubtarget &Subtarget) {
8468   // To "insert" Select_* instructions, we actually have to insert the triangle
8469   // control-flow pattern.  The incoming instructions know the destination vreg
8470   // to set, the condition code register to branch on, the true/false values to
8471   // select between, and the condcode to use to select the appropriate branch.
8472   //
8473   // We produce the following control flow:
8474   //     HeadMBB
8475   //     |  \
8476   //     |  IfFalseMBB
8477   //     | /
8478   //    TailMBB
8479   //
8480   // When we find a sequence of selects we attempt to optimize their emission
8481   // by sharing the control flow. Currently we only handle cases where we have
8482   // multiple selects with the exact same condition (same LHS, RHS and CC).
8483   // The selects may be interleaved with other instructions if the other
8484   // instructions meet some requirements we deem safe:
8485   // - They are debug instructions. Otherwise,
8486   // - They do not have side-effects, do not access memory and their inputs do
8487   //   not depend on the results of the select pseudo-instructions.
8488   // The TrueV/FalseV operands of the selects cannot depend on the result of
8489   // previous selects in the sequence.
8490   // These conditions could be further relaxed. See the X86 target for a
8491   // related approach and more information.
8492   Register LHS = MI.getOperand(1).getReg();
8493   Register RHS = MI.getOperand(2).getReg();
8494   auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
8495 
8496   SmallVector<MachineInstr *, 4> SelectDebugValues;
8497   SmallSet<Register, 4> SelectDests;
8498   SelectDests.insert(MI.getOperand(0).getReg());
8499 
8500   MachineInstr *LastSelectPseudo = &MI;
8501 
8502   for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
8503        SequenceMBBI != E; ++SequenceMBBI) {
8504     if (SequenceMBBI->isDebugInstr())
8505       continue;
8506     else if (isSelectPseudo(*SequenceMBBI)) {
8507       if (SequenceMBBI->getOperand(1).getReg() != LHS ||
8508           SequenceMBBI->getOperand(2).getReg() != RHS ||
8509           SequenceMBBI->getOperand(3).getImm() != CC ||
8510           SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
8511           SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
8512         break;
8513       LastSelectPseudo = &*SequenceMBBI;
8514       SequenceMBBI->collectDebugValues(SelectDebugValues);
8515       SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
8516     } else {
8517       if (SequenceMBBI->hasUnmodeledSideEffects() ||
8518           SequenceMBBI->mayLoadOrStore())
8519         break;
8520       if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
8521             return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
8522           }))
8523         break;
8524     }
8525   }
8526 
8527   const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
8528   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8529   DebugLoc DL = MI.getDebugLoc();
8530   MachineFunction::iterator I = ++BB->getIterator();
8531 
8532   MachineBasicBlock *HeadMBB = BB;
8533   MachineFunction *F = BB->getParent();
8534   MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
8535   MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
8536 
8537   F->insert(I, IfFalseMBB);
8538   F->insert(I, TailMBB);
8539 
8540   // Transfer debug instructions associated with the selects to TailMBB.
8541   for (MachineInstr *DebugInstr : SelectDebugValues) {
8542     TailMBB->push_back(DebugInstr->removeFromParent());
8543   }
8544 
8545   // Move all instructions after the sequence to TailMBB.
8546   TailMBB->splice(TailMBB->end(), HeadMBB,
8547                   std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
8548   // Update machine-CFG edges by transferring all successors of the current
8549   // block to the new block which will contain the Phi nodes for the selects.
8550   TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
8551   // Set the successors for HeadMBB.
8552   HeadMBB->addSuccessor(IfFalseMBB);
8553   HeadMBB->addSuccessor(TailMBB);
8554 
8555   // Insert appropriate branch.
8556   BuildMI(HeadMBB, DL, TII.getBrCond(CC))
8557     .addReg(LHS)
8558     .addReg(RHS)
8559     .addMBB(TailMBB);
8560 
8561   // IfFalseMBB just falls through to TailMBB.
8562   IfFalseMBB->addSuccessor(TailMBB);
8563 
8564   // Create PHIs for all of the select pseudo-instructions.
8565   auto SelectMBBI = MI.getIterator();
8566   auto SelectEnd = std::next(LastSelectPseudo->getIterator());
8567   auto InsertionPoint = TailMBB->begin();
8568   while (SelectMBBI != SelectEnd) {
8569     auto Next = std::next(SelectMBBI);
8570     if (isSelectPseudo(*SelectMBBI)) {
8571       // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
8572       BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
8573               TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
8574           .addReg(SelectMBBI->getOperand(4).getReg())
8575           .addMBB(HeadMBB)
8576           .addReg(SelectMBBI->getOperand(5).getReg())
8577           .addMBB(IfFalseMBB);
8578       SelectMBBI->eraseFromParent();
8579     }
8580     SelectMBBI = Next;
8581   }
8582 
8583   F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
8584   return TailMBB;
8585 }
8586 
8587 MachineBasicBlock *
8588 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
8589                                                  MachineBasicBlock *BB) const {
8590   switch (MI.getOpcode()) {
8591   default:
8592     llvm_unreachable("Unexpected instr type to insert");
8593   case RISCV::ReadCycleWide:
8594     assert(!Subtarget.is64Bit() &&
8595            "ReadCycleWrite is only to be used on riscv32");
8596     return emitReadCycleWidePseudo(MI, BB);
8597   case RISCV::Select_GPR_Using_CC_GPR:
8598   case RISCV::Select_FPR16_Using_CC_GPR:
8599   case RISCV::Select_FPR32_Using_CC_GPR:
8600   case RISCV::Select_FPR64_Using_CC_GPR:
8601     return emitSelectPseudo(MI, BB, Subtarget);
8602   case RISCV::BuildPairF64Pseudo:
8603     return emitBuildPairF64Pseudo(MI, BB);
8604   case RISCV::SplitF64Pseudo:
8605     return emitSplitF64Pseudo(MI, BB);
8606   case RISCV::PseudoQuietFLE_H:
8607     return emitQuietFCMP(MI, BB, RISCV::FLE_H, RISCV::FEQ_H, Subtarget);
8608   case RISCV::PseudoQuietFLT_H:
8609     return emitQuietFCMP(MI, BB, RISCV::FLT_H, RISCV::FEQ_H, Subtarget);
8610   case RISCV::PseudoQuietFLE_S:
8611     return emitQuietFCMP(MI, BB, RISCV::FLE_S, RISCV::FEQ_S, Subtarget);
8612   case RISCV::PseudoQuietFLT_S:
8613     return emitQuietFCMP(MI, BB, RISCV::FLT_S, RISCV::FEQ_S, Subtarget);
8614   case RISCV::PseudoQuietFLE_D:
8615     return emitQuietFCMP(MI, BB, RISCV::FLE_D, RISCV::FEQ_D, Subtarget);
8616   case RISCV::PseudoQuietFLT_D:
8617     return emitQuietFCMP(MI, BB, RISCV::FLT_D, RISCV::FEQ_D, Subtarget);
8618   }
8619 }
8620 
8621 void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
8622                                                         SDNode *Node) const {
8623   // Add FRM dependency to any instructions with dynamic rounding mode.
8624   unsigned Opc = MI.getOpcode();
8625   auto Idx = RISCV::getNamedOperandIdx(Opc, RISCV::OpName::frm);
8626   if (Idx < 0)
8627     return;
8628   if (MI.getOperand(Idx).getImm() != RISCVFPRndMode::DYN)
8629     return;
8630   // If the instruction already reads FRM, don't add another read.
8631   if (MI.readsRegister(RISCV::FRM))
8632     return;
8633   MI.addOperand(
8634       MachineOperand::CreateReg(RISCV::FRM, /*isDef*/ false, /*isImp*/ true));
8635 }
8636 
8637 // Calling Convention Implementation.
8638 // The expectations for frontend ABI lowering vary from target to target.
8639 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
8640 // details, but this is a longer term goal. For now, we simply try to keep the
8641 // role of the frontend as simple and well-defined as possible. The rules can
8642 // be summarised as:
8643 // * Never split up large scalar arguments. We handle them here.
8644 // * If a hardfloat calling convention is being used, and the struct may be
8645 // passed in a pair of registers (fp+fp, int+fp), and both registers are
8646 // available, then pass as two separate arguments. If either the GPRs or FPRs
8647 // are exhausted, then pass according to the rule below.
8648 // * If a struct could never be passed in registers or directly in a stack
8649 // slot (as it is larger than 2*XLEN and the floating point rules don't
8650 // apply), then pass it using a pointer with the byval attribute.
8651 // * If a struct is less than 2*XLEN, then coerce to either a two-element
8652 // word-sized array or a 2*XLEN scalar (depending on alignment).
8653 // * The frontend can determine whether a struct is returned by reference or
8654 // not based on its size and fields. If it will be returned by reference, the
8655 // frontend must modify the prototype so a pointer with the sret annotation is
8656 // passed as the first argument. This is not necessary for large scalar
8657 // returns.
8658 // * Struct return values and varargs should be coerced to structs containing
8659 // register-size fields in the same situations they would be for fixed
8660 // arguments.
8661 
8662 static const MCPhysReg ArgGPRs[] = {
8663   RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
8664   RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
8665 };
8666 static const MCPhysReg ArgFPR16s[] = {
8667   RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
8668   RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
8669 };
8670 static const MCPhysReg ArgFPR32s[] = {
8671   RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
8672   RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
8673 };
8674 static const MCPhysReg ArgFPR64s[] = {
8675   RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
8676   RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
8677 };
8678 // This is an interim calling convention and it may be changed in the future.
8679 static const MCPhysReg ArgVRs[] = {
8680     RISCV::V8,  RISCV::V9,  RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
8681     RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
8682     RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
8683 static const MCPhysReg ArgVRM2s[] = {RISCV::V8M2,  RISCV::V10M2, RISCV::V12M2,
8684                                      RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
8685                                      RISCV::V20M2, RISCV::V22M2};
8686 static const MCPhysReg ArgVRM4s[] = {RISCV::V8M4, RISCV::V12M4, RISCV::V16M4,
8687                                      RISCV::V20M4};
8688 static const MCPhysReg ArgVRM8s[] = {RISCV::V8M8, RISCV::V16M8};
8689 
8690 // Pass a 2*XLEN argument that has been split into two XLEN values through
8691 // registers or the stack as necessary.
8692 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
8693                                 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
8694                                 MVT ValVT2, MVT LocVT2,
8695                                 ISD::ArgFlagsTy ArgFlags2) {
8696   unsigned XLenInBytes = XLen / 8;
8697   if (Register Reg = State.AllocateReg(ArgGPRs)) {
8698     // At least one half can be passed via register.
8699     State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
8700                                      VA1.getLocVT(), CCValAssign::Full));
8701   } else {
8702     // Both halves must be passed on the stack, with proper alignment.
8703     Align StackAlign =
8704         std::max(Align(XLenInBytes), ArgFlags1.getNonZeroOrigAlign());
8705     State.addLoc(
8706         CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
8707                             State.AllocateStack(XLenInBytes, StackAlign),
8708                             VA1.getLocVT(), CCValAssign::Full));
8709     State.addLoc(CCValAssign::getMem(
8710         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
8711         LocVT2, CCValAssign::Full));
8712     return false;
8713   }
8714 
8715   if (Register Reg = State.AllocateReg(ArgGPRs)) {
8716     // The second half can also be passed via register.
8717     State.addLoc(
8718         CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
8719   } else {
8720     // The second half is passed via the stack, without additional alignment.
8721     State.addLoc(CCValAssign::getMem(
8722         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
8723         LocVT2, CCValAssign::Full));
8724   }
8725 
8726   return false;
8727 }
8728 
8729 static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo,
8730                                Optional<unsigned> FirstMaskArgument,
8731                                CCState &State, const RISCVTargetLowering &TLI) {
8732   const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT);
8733   if (RC == &RISCV::VRRegClass) {
8734     // Assign the first mask argument to V0.
8735     // This is an interim calling convention and it may be changed in the
8736     // future.
8737     if (FirstMaskArgument.hasValue() && ValNo == FirstMaskArgument.getValue())
8738       return State.AllocateReg(RISCV::V0);
8739     return State.AllocateReg(ArgVRs);
8740   }
8741   if (RC == &RISCV::VRM2RegClass)
8742     return State.AllocateReg(ArgVRM2s);
8743   if (RC == &RISCV::VRM4RegClass)
8744     return State.AllocateReg(ArgVRM4s);
8745   if (RC == &RISCV::VRM8RegClass)
8746     return State.AllocateReg(ArgVRM8s);
8747   llvm_unreachable("Unhandled register class for ValueType");
8748 }
8749 
8750 // Implements the RISC-V calling convention. Returns true upon failure.
8751 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
8752                      MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
8753                      ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
8754                      bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
8755                      Optional<unsigned> FirstMaskArgument) {
8756   unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
8757   assert(XLen == 32 || XLen == 64);
8758   MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
8759 
8760   // Any return value split in to more than two values can't be returned
8761   // directly. Vectors are returned via the available vector registers.
8762   if (!LocVT.isVector() && IsRet && ValNo > 1)
8763     return true;
8764 
8765   // UseGPRForF16_F32 if targeting one of the soft-float ABIs, if passing a
8766   // variadic argument, or if no F16/F32 argument registers are available.
8767   bool UseGPRForF16_F32 = true;
8768   // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
8769   // variadic argument, or if no F64 argument registers are available.
8770   bool UseGPRForF64 = true;
8771 
8772   switch (ABI) {
8773   default:
8774     llvm_unreachable("Unexpected ABI");
8775   case RISCVABI::ABI_ILP32:
8776   case RISCVABI::ABI_LP64:
8777     break;
8778   case RISCVABI::ABI_ILP32F:
8779   case RISCVABI::ABI_LP64F:
8780     UseGPRForF16_F32 = !IsFixed;
8781     break;
8782   case RISCVABI::ABI_ILP32D:
8783   case RISCVABI::ABI_LP64D:
8784     UseGPRForF16_F32 = !IsFixed;
8785     UseGPRForF64 = !IsFixed;
8786     break;
8787   }
8788 
8789   // FPR16, FPR32, and FPR64 alias each other.
8790   if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s)) {
8791     UseGPRForF16_F32 = true;
8792     UseGPRForF64 = true;
8793   }
8794 
8795   // From this point on, rely on UseGPRForF16_F32, UseGPRForF64 and
8796   // similar local variables rather than directly checking against the target
8797   // ABI.
8798 
8799   if (UseGPRForF16_F32 && (ValVT == MVT::f16 || ValVT == MVT::f32)) {
8800     LocVT = XLenVT;
8801     LocInfo = CCValAssign::BCvt;
8802   } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
8803     LocVT = MVT::i64;
8804     LocInfo = CCValAssign::BCvt;
8805   }
8806 
8807   // If this is a variadic argument, the RISC-V calling convention requires
8808   // that it is assigned an 'even' or 'aligned' register if it has 8-byte
8809   // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
8810   // be used regardless of whether the original argument was split during
8811   // legalisation or not. The argument will not be passed by registers if the
8812   // original type is larger than 2*XLEN, so the register alignment rule does
8813   // not apply.
8814   unsigned TwoXLenInBytes = (2 * XLen) / 8;
8815   if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
8816       DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
8817     unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
8818     // Skip 'odd' register if necessary.
8819     if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
8820       State.AllocateReg(ArgGPRs);
8821   }
8822 
8823   SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
8824   SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
8825       State.getPendingArgFlags();
8826 
8827   assert(PendingLocs.size() == PendingArgFlags.size() &&
8828          "PendingLocs and PendingArgFlags out of sync");
8829 
8830   // Handle passing f64 on RV32D with a soft float ABI or when floating point
8831   // registers are exhausted.
8832   if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
8833     assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
8834            "Can't lower f64 if it is split");
8835     // Depending on available argument GPRS, f64 may be passed in a pair of
8836     // GPRs, split between a GPR and the stack, or passed completely on the
8837     // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
8838     // cases.
8839     Register Reg = State.AllocateReg(ArgGPRs);
8840     LocVT = MVT::i32;
8841     if (!Reg) {
8842       unsigned StackOffset = State.AllocateStack(8, Align(8));
8843       State.addLoc(
8844           CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
8845       return false;
8846     }
8847     if (!State.AllocateReg(ArgGPRs))
8848       State.AllocateStack(4, Align(4));
8849     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8850     return false;
8851   }
8852 
8853   // Fixed-length vectors are located in the corresponding scalable-vector
8854   // container types.
8855   if (ValVT.isFixedLengthVector())
8856     LocVT = TLI.getContainerForFixedLengthVector(LocVT);
8857 
8858   // Split arguments might be passed indirectly, so keep track of the pending
8859   // values. Split vectors are passed via a mix of registers and indirectly, so
8860   // treat them as we would any other argument.
8861   if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
8862     LocVT = XLenVT;
8863     LocInfo = CCValAssign::Indirect;
8864     PendingLocs.push_back(
8865         CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
8866     PendingArgFlags.push_back(ArgFlags);
8867     if (!ArgFlags.isSplitEnd()) {
8868       return false;
8869     }
8870   }
8871 
8872   // If the split argument only had two elements, it should be passed directly
8873   // in registers or on the stack.
8874   if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
8875       PendingLocs.size() <= 2) {
8876     assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
8877     // Apply the normal calling convention rules to the first half of the
8878     // split argument.
8879     CCValAssign VA = PendingLocs[0];
8880     ISD::ArgFlagsTy AF = PendingArgFlags[0];
8881     PendingLocs.clear();
8882     PendingArgFlags.clear();
8883     return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
8884                                ArgFlags);
8885   }
8886 
8887   // Allocate to a register if possible, or else a stack slot.
8888   Register Reg;
8889   unsigned StoreSizeBytes = XLen / 8;
8890   Align StackAlign = Align(XLen / 8);
8891 
8892   if (ValVT == MVT::f16 && !UseGPRForF16_F32)
8893     Reg = State.AllocateReg(ArgFPR16s);
8894   else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
8895     Reg = State.AllocateReg(ArgFPR32s);
8896   else if (ValVT == MVT::f64 && !UseGPRForF64)
8897     Reg = State.AllocateReg(ArgFPR64s);
8898   else if (ValVT.isVector()) {
8899     Reg = allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI);
8900     if (!Reg) {
8901       // For return values, the vector must be passed fully via registers or
8902       // via the stack.
8903       // FIXME: The proposed vector ABI only mandates v8-v15 for return values,
8904       // but we're using all of them.
8905       if (IsRet)
8906         return true;
8907       // Try using a GPR to pass the address
8908       if ((Reg = State.AllocateReg(ArgGPRs))) {
8909         LocVT = XLenVT;
8910         LocInfo = CCValAssign::Indirect;
8911       } else if (ValVT.isScalableVector()) {
8912         LocVT = XLenVT;
8913         LocInfo = CCValAssign::Indirect;
8914       } else {
8915         // Pass fixed-length vectors on the stack.
8916         LocVT = ValVT;
8917         StoreSizeBytes = ValVT.getStoreSize();
8918         // Align vectors to their element sizes, being careful for vXi1
8919         // vectors.
8920         StackAlign = MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
8921       }
8922     }
8923   } else {
8924     Reg = State.AllocateReg(ArgGPRs);
8925   }
8926 
8927   unsigned StackOffset =
8928       Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
8929 
8930   // If we reach this point and PendingLocs is non-empty, we must be at the
8931   // end of a split argument that must be passed indirectly.
8932   if (!PendingLocs.empty()) {
8933     assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
8934     assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
8935 
8936     for (auto &It : PendingLocs) {
8937       if (Reg)
8938         It.convertToReg(Reg);
8939       else
8940         It.convertToMem(StackOffset);
8941       State.addLoc(It);
8942     }
8943     PendingLocs.clear();
8944     PendingArgFlags.clear();
8945     return false;
8946   }
8947 
8948   assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
8949           (TLI.getSubtarget().hasVInstructions() && ValVT.isVector())) &&
8950          "Expected an XLenVT or vector types at this stage");
8951 
8952   if (Reg) {
8953     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8954     return false;
8955   }
8956 
8957   // When a floating-point value is passed on the stack, no bit-conversion is
8958   // needed.
8959   if (ValVT.isFloatingPoint()) {
8960     LocVT = ValVT;
8961     LocInfo = CCValAssign::Full;
8962   }
8963   State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
8964   return false;
8965 }
8966 
8967 template <typename ArgTy>
8968 static Optional<unsigned> preAssignMask(const ArgTy &Args) {
8969   for (const auto &ArgIdx : enumerate(Args)) {
8970     MVT ArgVT = ArgIdx.value().VT;
8971     if (ArgVT.isVector() && ArgVT.getVectorElementType() == MVT::i1)
8972       return ArgIdx.index();
8973   }
8974   return None;
8975 }
8976 
8977 void RISCVTargetLowering::analyzeInputArgs(
8978     MachineFunction &MF, CCState &CCInfo,
8979     const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
8980     RISCVCCAssignFn Fn) const {
8981   unsigned NumArgs = Ins.size();
8982   FunctionType *FType = MF.getFunction().getFunctionType();
8983 
8984   Optional<unsigned> FirstMaskArgument;
8985   if (Subtarget.hasVInstructions())
8986     FirstMaskArgument = preAssignMask(Ins);
8987 
8988   for (unsigned i = 0; i != NumArgs; ++i) {
8989     MVT ArgVT = Ins[i].VT;
8990     ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
8991 
8992     Type *ArgTy = nullptr;
8993     if (IsRet)
8994       ArgTy = FType->getReturnType();
8995     else if (Ins[i].isOrigArg())
8996       ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
8997 
8998     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
8999     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
9000            ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this,
9001            FirstMaskArgument)) {
9002       LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
9003                         << EVT(ArgVT).getEVTString() << '\n');
9004       llvm_unreachable(nullptr);
9005     }
9006   }
9007 }
9008 
9009 void RISCVTargetLowering::analyzeOutputArgs(
9010     MachineFunction &MF, CCState &CCInfo,
9011     const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
9012     CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
9013   unsigned NumArgs = Outs.size();
9014 
9015   Optional<unsigned> FirstMaskArgument;
9016   if (Subtarget.hasVInstructions())
9017     FirstMaskArgument = preAssignMask(Outs);
9018 
9019   for (unsigned i = 0; i != NumArgs; i++) {
9020     MVT ArgVT = Outs[i].VT;
9021     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
9022     Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
9023 
9024     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9025     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
9026            ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this,
9027            FirstMaskArgument)) {
9028       LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
9029                         << EVT(ArgVT).getEVTString() << "\n");
9030       llvm_unreachable(nullptr);
9031     }
9032   }
9033 }
9034 
9035 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
9036 // values.
9037 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
9038                                    const CCValAssign &VA, const SDLoc &DL,
9039                                    const RISCVSubtarget &Subtarget) {
9040   switch (VA.getLocInfo()) {
9041   default:
9042     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9043   case CCValAssign::Full:
9044     if (VA.getValVT().isFixedLengthVector() && VA.getLocVT().isScalableVector())
9045       Val = convertFromScalableVector(VA.getValVT(), Val, DAG, Subtarget);
9046     break;
9047   case CCValAssign::BCvt:
9048     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
9049       Val = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, Val);
9050     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9051       Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
9052     else
9053       Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
9054     break;
9055   }
9056   return Val;
9057 }
9058 
9059 // The caller is responsible for loading the full value if the argument is
9060 // passed with CCValAssign::Indirect.
9061 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
9062                                 const CCValAssign &VA, const SDLoc &DL,
9063                                 const RISCVTargetLowering &TLI) {
9064   MachineFunction &MF = DAG.getMachineFunction();
9065   MachineRegisterInfo &RegInfo = MF.getRegInfo();
9066   EVT LocVT = VA.getLocVT();
9067   SDValue Val;
9068   const TargetRegisterClass *RC = TLI.getRegClassFor(LocVT.getSimpleVT());
9069   Register VReg = RegInfo.createVirtualRegister(RC);
9070   RegInfo.addLiveIn(VA.getLocReg(), VReg);
9071   Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
9072 
9073   if (VA.getLocInfo() == CCValAssign::Indirect)
9074     return Val;
9075 
9076   return convertLocVTToValVT(DAG, Val, VA, DL, TLI.getSubtarget());
9077 }
9078 
9079 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
9080                                    const CCValAssign &VA, const SDLoc &DL,
9081                                    const RISCVSubtarget &Subtarget) {
9082   EVT LocVT = VA.getLocVT();
9083 
9084   switch (VA.getLocInfo()) {
9085   default:
9086     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9087   case CCValAssign::Full:
9088     if (VA.getValVT().isFixedLengthVector() && LocVT.isScalableVector())
9089       Val = convertToScalableVector(LocVT, Val, DAG, Subtarget);
9090     break;
9091   case CCValAssign::BCvt:
9092     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
9093       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, VA.getLocVT(), Val);
9094     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9095       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
9096     else
9097       Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
9098     break;
9099   }
9100   return Val;
9101 }
9102 
9103 // The caller is responsible for loading the full value if the argument is
9104 // passed with CCValAssign::Indirect.
9105 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
9106                                 const CCValAssign &VA, const SDLoc &DL) {
9107   MachineFunction &MF = DAG.getMachineFunction();
9108   MachineFrameInfo &MFI = MF.getFrameInfo();
9109   EVT LocVT = VA.getLocVT();
9110   EVT ValVT = VA.getValVT();
9111   EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
9112   if (ValVT.isScalableVector()) {
9113     // When the value is a scalable vector, we save the pointer which points to
9114     // the scalable vector value in the stack. The ValVT will be the pointer
9115     // type, instead of the scalable vector type.
9116     ValVT = LocVT;
9117   }
9118   int FI = MFI.CreateFixedObject(ValVT.getStoreSize(), VA.getLocMemOffset(),
9119                                  /*IsImmutable=*/true);
9120   SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
9121   SDValue Val;
9122 
9123   ISD::LoadExtType ExtType;
9124   switch (VA.getLocInfo()) {
9125   default:
9126     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9127   case CCValAssign::Full:
9128   case CCValAssign::Indirect:
9129   case CCValAssign::BCvt:
9130     ExtType = ISD::NON_EXTLOAD;
9131     break;
9132   }
9133   Val = DAG.getExtLoad(
9134       ExtType, DL, LocVT, Chain, FIN,
9135       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
9136   return Val;
9137 }
9138 
9139 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
9140                                        const CCValAssign &VA, const SDLoc &DL) {
9141   assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
9142          "Unexpected VA");
9143   MachineFunction &MF = DAG.getMachineFunction();
9144   MachineFrameInfo &MFI = MF.getFrameInfo();
9145   MachineRegisterInfo &RegInfo = MF.getRegInfo();
9146 
9147   if (VA.isMemLoc()) {
9148     // f64 is passed on the stack.
9149     int FI =
9150         MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*IsImmutable=*/true);
9151     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
9152     return DAG.getLoad(MVT::f64, DL, Chain, FIN,
9153                        MachinePointerInfo::getFixedStack(MF, FI));
9154   }
9155 
9156   assert(VA.isRegLoc() && "Expected register VA assignment");
9157 
9158   Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
9159   RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
9160   SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
9161   SDValue Hi;
9162   if (VA.getLocReg() == RISCV::X17) {
9163     // Second half of f64 is passed on the stack.
9164     int FI = MFI.CreateFixedObject(4, 0, /*IsImmutable=*/true);
9165     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
9166     Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
9167                      MachinePointerInfo::getFixedStack(MF, FI));
9168   } else {
9169     // Second half of f64 is passed in another GPR.
9170     Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
9171     RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
9172     Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
9173   }
9174   return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
9175 }
9176 
9177 // FastCC has less than 1% performance improvement for some particular
9178 // benchmark. But theoretically, it may has benenfit for some cases.
9179 static bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
9180                             unsigned ValNo, MVT ValVT, MVT LocVT,
9181                             CCValAssign::LocInfo LocInfo,
9182                             ISD::ArgFlagsTy ArgFlags, CCState &State,
9183                             bool IsFixed, bool IsRet, Type *OrigTy,
9184                             const RISCVTargetLowering &TLI,
9185                             Optional<unsigned> FirstMaskArgument) {
9186 
9187   // X5 and X6 might be used for save-restore libcall.
9188   static const MCPhysReg GPRList[] = {
9189       RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
9190       RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7,  RISCV::X28,
9191       RISCV::X29, RISCV::X30, RISCV::X31};
9192 
9193   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9194     if (unsigned Reg = State.AllocateReg(GPRList)) {
9195       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9196       return false;
9197     }
9198   }
9199 
9200   if (LocVT == MVT::f16) {
9201     static const MCPhysReg FPR16List[] = {
9202         RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
9203         RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H,  RISCV::F1_H,
9204         RISCV::F2_H,  RISCV::F3_H,  RISCV::F4_H,  RISCV::F5_H,  RISCV::F6_H,
9205         RISCV::F7_H,  RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
9206     if (unsigned Reg = State.AllocateReg(FPR16List)) {
9207       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9208       return false;
9209     }
9210   }
9211 
9212   if (LocVT == MVT::f32) {
9213     static const MCPhysReg FPR32List[] = {
9214         RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
9215         RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F,  RISCV::F1_F,
9216         RISCV::F2_F,  RISCV::F3_F,  RISCV::F4_F,  RISCV::F5_F,  RISCV::F6_F,
9217         RISCV::F7_F,  RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
9218     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9219       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9220       return false;
9221     }
9222   }
9223 
9224   if (LocVT == MVT::f64) {
9225     static const MCPhysReg FPR64List[] = {
9226         RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
9227         RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D,  RISCV::F1_D,
9228         RISCV::F2_D,  RISCV::F3_D,  RISCV::F4_D,  RISCV::F5_D,  RISCV::F6_D,
9229         RISCV::F7_D,  RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
9230     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9231       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9232       return false;
9233     }
9234   }
9235 
9236   if (LocVT == MVT::i32 || LocVT == MVT::f32) {
9237     unsigned Offset4 = State.AllocateStack(4, Align(4));
9238     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset4, LocVT, LocInfo));
9239     return false;
9240   }
9241 
9242   if (LocVT == MVT::i64 || LocVT == MVT::f64) {
9243     unsigned Offset5 = State.AllocateStack(8, Align(8));
9244     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset5, LocVT, LocInfo));
9245     return false;
9246   }
9247 
9248   if (LocVT.isVector()) {
9249     if (unsigned Reg =
9250             allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI)) {
9251       // Fixed-length vectors are located in the corresponding scalable-vector
9252       // container types.
9253       if (ValVT.isFixedLengthVector())
9254         LocVT = TLI.getContainerForFixedLengthVector(LocVT);
9255       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9256     } else {
9257       // Try and pass the address via a "fast" GPR.
9258       if (unsigned GPRReg = State.AllocateReg(GPRList)) {
9259         LocInfo = CCValAssign::Indirect;
9260         LocVT = TLI.getSubtarget().getXLenVT();
9261         State.addLoc(CCValAssign::getReg(ValNo, ValVT, GPRReg, LocVT, LocInfo));
9262       } else if (ValVT.isFixedLengthVector()) {
9263         auto StackAlign =
9264             MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
9265         unsigned StackOffset =
9266             State.AllocateStack(ValVT.getStoreSize(), StackAlign);
9267         State.addLoc(
9268             CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
9269       } else {
9270         // Can't pass scalable vectors on the stack.
9271         return true;
9272       }
9273     }
9274 
9275     return false;
9276   }
9277 
9278   return true; // CC didn't match.
9279 }
9280 
9281 static bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
9282                          CCValAssign::LocInfo LocInfo,
9283                          ISD::ArgFlagsTy ArgFlags, CCState &State) {
9284 
9285   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9286     // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, R6, R7, SpLim
9287     //                        s1    s2  s3  s4  s5  s6  s7  s8  s9  s10 s11
9288     static const MCPhysReg GPRList[] = {
9289         RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
9290         RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
9291     if (unsigned Reg = State.AllocateReg(GPRList)) {
9292       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9293       return false;
9294     }
9295   }
9296 
9297   if (LocVT == MVT::f32) {
9298     // Pass in STG registers: F1, ..., F6
9299     //                        fs0 ... fs5
9300     static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
9301                                           RISCV::F18_F, RISCV::F19_F,
9302                                           RISCV::F20_F, RISCV::F21_F};
9303     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9304       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9305       return false;
9306     }
9307   }
9308 
9309   if (LocVT == MVT::f64) {
9310     // Pass in STG registers: D1, ..., D6
9311     //                        fs6 ... fs11
9312     static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
9313                                           RISCV::F24_D, RISCV::F25_D,
9314                                           RISCV::F26_D, RISCV::F27_D};
9315     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9316       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9317       return false;
9318     }
9319   }
9320 
9321   report_fatal_error("No registers left in GHC calling convention");
9322   return true;
9323 }
9324 
9325 // Transform physical registers into virtual registers.
9326 SDValue RISCVTargetLowering::LowerFormalArguments(
9327     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
9328     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
9329     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
9330 
9331   MachineFunction &MF = DAG.getMachineFunction();
9332 
9333   switch (CallConv) {
9334   default:
9335     report_fatal_error("Unsupported calling convention");
9336   case CallingConv::C:
9337   case CallingConv::Fast:
9338     break;
9339   case CallingConv::GHC:
9340     if (!MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtF] ||
9341         !MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtD])
9342       report_fatal_error(
9343         "GHC calling convention requires the F and D instruction set extensions");
9344   }
9345 
9346   const Function &Func = MF.getFunction();
9347   if (Func.hasFnAttribute("interrupt")) {
9348     if (!Func.arg_empty())
9349       report_fatal_error(
9350         "Functions with the interrupt attribute cannot have arguments!");
9351 
9352     StringRef Kind =
9353       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
9354 
9355     if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
9356       report_fatal_error(
9357         "Function interrupt attribute argument not supported!");
9358   }
9359 
9360   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9361   MVT XLenVT = Subtarget.getXLenVT();
9362   unsigned XLenInBytes = Subtarget.getXLen() / 8;
9363   // Used with vargs to acumulate store chains.
9364   std::vector<SDValue> OutChains;
9365 
9366   // Assign locations to all of the incoming arguments.
9367   SmallVector<CCValAssign, 16> ArgLocs;
9368   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9369 
9370   if (CallConv == CallingConv::GHC)
9371     CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_GHC);
9372   else
9373     analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false,
9374                      CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9375                                                    : CC_RISCV);
9376 
9377   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
9378     CCValAssign &VA = ArgLocs[i];
9379     SDValue ArgValue;
9380     // Passing f64 on RV32D with a soft float ABI must be handled as a special
9381     // case.
9382     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
9383       ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
9384     else if (VA.isRegLoc())
9385       ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, *this);
9386     else
9387       ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
9388 
9389     if (VA.getLocInfo() == CCValAssign::Indirect) {
9390       // If the original argument was split and passed by reference (e.g. i128
9391       // on RV32), we need to load all parts of it here (using the same
9392       // address). Vectors may be partly split to registers and partly to the
9393       // stack, in which case the base address is partly offset and subsequent
9394       // stores are relative to that.
9395       InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
9396                                    MachinePointerInfo()));
9397       unsigned ArgIndex = Ins[i].OrigArgIndex;
9398       unsigned ArgPartOffset = Ins[i].PartOffset;
9399       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9400       while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
9401         CCValAssign &PartVA = ArgLocs[i + 1];
9402         unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset;
9403         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9404         if (PartVA.getValVT().isScalableVector())
9405           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9406         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
9407         InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
9408                                      MachinePointerInfo()));
9409         ++i;
9410       }
9411       continue;
9412     }
9413     InVals.push_back(ArgValue);
9414   }
9415 
9416   if (IsVarArg) {
9417     ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
9418     unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
9419     const TargetRegisterClass *RC = &RISCV::GPRRegClass;
9420     MachineFrameInfo &MFI = MF.getFrameInfo();
9421     MachineRegisterInfo &RegInfo = MF.getRegInfo();
9422     RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
9423 
9424     // Offset of the first variable argument from stack pointer, and size of
9425     // the vararg save area. For now, the varargs save area is either zero or
9426     // large enough to hold a0-a7.
9427     int VaArgOffset, VarArgsSaveSize;
9428 
9429     // If all registers are allocated, then all varargs must be passed on the
9430     // stack and we don't need to save any argregs.
9431     if (ArgRegs.size() == Idx) {
9432       VaArgOffset = CCInfo.getNextStackOffset();
9433       VarArgsSaveSize = 0;
9434     } else {
9435       VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
9436       VaArgOffset = -VarArgsSaveSize;
9437     }
9438 
9439     // Record the frame index of the first variable argument
9440     // which is a value necessary to VASTART.
9441     int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9442     RVFI->setVarArgsFrameIndex(FI);
9443 
9444     // If saving an odd number of registers then create an extra stack slot to
9445     // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
9446     // offsets to even-numbered registered remain 2*XLEN-aligned.
9447     if (Idx % 2) {
9448       MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true);
9449       VarArgsSaveSize += XLenInBytes;
9450     }
9451 
9452     // Copy the integer registers that may have been used for passing varargs
9453     // to the vararg save area.
9454     for (unsigned I = Idx; I < ArgRegs.size();
9455          ++I, VaArgOffset += XLenInBytes) {
9456       const Register Reg = RegInfo.createVirtualRegister(RC);
9457       RegInfo.addLiveIn(ArgRegs[I], Reg);
9458       SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
9459       FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9460       SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9461       SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
9462                                    MachinePointerInfo::getFixedStack(MF, FI));
9463       cast<StoreSDNode>(Store.getNode())
9464           ->getMemOperand()
9465           ->setValue((Value *)nullptr);
9466       OutChains.push_back(Store);
9467     }
9468     RVFI->setVarArgsSaveSize(VarArgsSaveSize);
9469   }
9470 
9471   // All stores are grouped in one node to allow the matching between
9472   // the size of Ins and InVals. This only happens for vararg functions.
9473   if (!OutChains.empty()) {
9474     OutChains.push_back(Chain);
9475     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
9476   }
9477 
9478   return Chain;
9479 }
9480 
9481 /// isEligibleForTailCallOptimization - Check whether the call is eligible
9482 /// for tail call optimization.
9483 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
9484 bool RISCVTargetLowering::isEligibleForTailCallOptimization(
9485     CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
9486     const SmallVector<CCValAssign, 16> &ArgLocs) const {
9487 
9488   auto &Callee = CLI.Callee;
9489   auto CalleeCC = CLI.CallConv;
9490   auto &Outs = CLI.Outs;
9491   auto &Caller = MF.getFunction();
9492   auto CallerCC = Caller.getCallingConv();
9493 
9494   // Exception-handling functions need a special set of instructions to
9495   // indicate a return to the hardware. Tail-calling another function would
9496   // probably break this.
9497   // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
9498   // should be expanded as new function attributes are introduced.
9499   if (Caller.hasFnAttribute("interrupt"))
9500     return false;
9501 
9502   // Do not tail call opt if the stack is used to pass parameters.
9503   if (CCInfo.getNextStackOffset() != 0)
9504     return false;
9505 
9506   // Do not tail call opt if any parameters need to be passed indirectly.
9507   // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
9508   // passed indirectly. So the address of the value will be passed in a
9509   // register, or if not available, then the address is put on the stack. In
9510   // order to pass indirectly, space on the stack often needs to be allocated
9511   // in order to store the value. In this case the CCInfo.getNextStackOffset()
9512   // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
9513   // are passed CCValAssign::Indirect.
9514   for (auto &VA : ArgLocs)
9515     if (VA.getLocInfo() == CCValAssign::Indirect)
9516       return false;
9517 
9518   // Do not tail call opt if either caller or callee uses struct return
9519   // semantics.
9520   auto IsCallerStructRet = Caller.hasStructRetAttr();
9521   auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
9522   if (IsCallerStructRet || IsCalleeStructRet)
9523     return false;
9524 
9525   // Externally-defined functions with weak linkage should not be
9526   // tail-called. The behaviour of branch instructions in this situation (as
9527   // used for tail calls) is implementation-defined, so we cannot rely on the
9528   // linker replacing the tail call with a return.
9529   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
9530     const GlobalValue *GV = G->getGlobal();
9531     if (GV->hasExternalWeakLinkage())
9532       return false;
9533   }
9534 
9535   // The callee has to preserve all registers the caller needs to preserve.
9536   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
9537   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
9538   if (CalleeCC != CallerCC) {
9539     const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
9540     if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
9541       return false;
9542   }
9543 
9544   // Byval parameters hand the function a pointer directly into the stack area
9545   // we want to reuse during a tail call. Working around this *is* possible
9546   // but less efficient and uglier in LowerCall.
9547   for (auto &Arg : Outs)
9548     if (Arg.Flags.isByVal())
9549       return false;
9550 
9551   return true;
9552 }
9553 
9554 static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
9555   return DAG.getDataLayout().getPrefTypeAlign(
9556       VT.getTypeForEVT(*DAG.getContext()));
9557 }
9558 
9559 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
9560 // and output parameter nodes.
9561 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
9562                                        SmallVectorImpl<SDValue> &InVals) const {
9563   SelectionDAG &DAG = CLI.DAG;
9564   SDLoc &DL = CLI.DL;
9565   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
9566   SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
9567   SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
9568   SDValue Chain = CLI.Chain;
9569   SDValue Callee = CLI.Callee;
9570   bool &IsTailCall = CLI.IsTailCall;
9571   CallingConv::ID CallConv = CLI.CallConv;
9572   bool IsVarArg = CLI.IsVarArg;
9573   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9574   MVT XLenVT = Subtarget.getXLenVT();
9575 
9576   MachineFunction &MF = DAG.getMachineFunction();
9577 
9578   // Analyze the operands of the call, assigning locations to each operand.
9579   SmallVector<CCValAssign, 16> ArgLocs;
9580   CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9581 
9582   if (CallConv == CallingConv::GHC)
9583     ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_GHC);
9584   else
9585     analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI,
9586                       CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9587                                                     : CC_RISCV);
9588 
9589   // Check if it's really possible to do a tail call.
9590   if (IsTailCall)
9591     IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
9592 
9593   if (IsTailCall)
9594     ++NumTailCalls;
9595   else if (CLI.CB && CLI.CB->isMustTailCall())
9596     report_fatal_error("failed to perform tail call elimination on a call "
9597                        "site marked musttail");
9598 
9599   // Get a count of how many bytes are to be pushed on the stack.
9600   unsigned NumBytes = ArgCCInfo.getNextStackOffset();
9601 
9602   // Create local copies for byval args
9603   SmallVector<SDValue, 8> ByValArgs;
9604   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9605     ISD::ArgFlagsTy Flags = Outs[i].Flags;
9606     if (!Flags.isByVal())
9607       continue;
9608 
9609     SDValue Arg = OutVals[i];
9610     unsigned Size = Flags.getByValSize();
9611     Align Alignment = Flags.getNonZeroByValAlign();
9612 
9613     int FI =
9614         MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false);
9615     SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9616     SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
9617 
9618     Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
9619                           /*IsVolatile=*/false,
9620                           /*AlwaysInline=*/false, IsTailCall,
9621                           MachinePointerInfo(), MachinePointerInfo());
9622     ByValArgs.push_back(FIPtr);
9623   }
9624 
9625   if (!IsTailCall)
9626     Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
9627 
9628   // Copy argument values to their designated locations.
9629   SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
9630   SmallVector<SDValue, 8> MemOpChains;
9631   SDValue StackPtr;
9632   for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
9633     CCValAssign &VA = ArgLocs[i];
9634     SDValue ArgValue = OutVals[i];
9635     ISD::ArgFlagsTy Flags = Outs[i].Flags;
9636 
9637     // Handle passing f64 on RV32D with a soft float ABI as a special case.
9638     bool IsF64OnRV32DSoftABI =
9639         VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
9640     if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
9641       SDValue SplitF64 = DAG.getNode(
9642           RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
9643       SDValue Lo = SplitF64.getValue(0);
9644       SDValue Hi = SplitF64.getValue(1);
9645 
9646       Register RegLo = VA.getLocReg();
9647       RegsToPass.push_back(std::make_pair(RegLo, Lo));
9648 
9649       if (RegLo == RISCV::X17) {
9650         // Second half of f64 is passed on the stack.
9651         // Work out the address of the stack slot.
9652         if (!StackPtr.getNode())
9653           StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
9654         // Emit the store.
9655         MemOpChains.push_back(
9656             DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
9657       } else {
9658         // Second half of f64 is passed in another GPR.
9659         assert(RegLo < RISCV::X31 && "Invalid register pair");
9660         Register RegHigh = RegLo + 1;
9661         RegsToPass.push_back(std::make_pair(RegHigh, Hi));
9662       }
9663       continue;
9664     }
9665 
9666     // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
9667     // as any other MemLoc.
9668 
9669     // Promote the value if needed.
9670     // For now, only handle fully promoted and indirect arguments.
9671     if (VA.getLocInfo() == CCValAssign::Indirect) {
9672       // Store the argument in a stack slot and pass its address.
9673       Align StackAlign =
9674           std::max(getPrefTypeAlign(Outs[i].ArgVT, DAG),
9675                    getPrefTypeAlign(ArgValue.getValueType(), DAG));
9676       TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
9677       // If the original argument was split (e.g. i128), we need
9678       // to store the required parts of it here (and pass just one address).
9679       // Vectors may be partly split to registers and partly to the stack, in
9680       // which case the base address is partly offset and subsequent stores are
9681       // relative to that.
9682       unsigned ArgIndex = Outs[i].OrigArgIndex;
9683       unsigned ArgPartOffset = Outs[i].PartOffset;
9684       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9685       // Calculate the total size to store. We don't have access to what we're
9686       // actually storing other than performing the loop and collecting the
9687       // info.
9688       SmallVector<std::pair<SDValue, SDValue>> Parts;
9689       while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
9690         SDValue PartValue = OutVals[i + 1];
9691         unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset;
9692         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9693         EVT PartVT = PartValue.getValueType();
9694         if (PartVT.isScalableVector())
9695           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9696         StoredSize += PartVT.getStoreSize();
9697         StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
9698         Parts.push_back(std::make_pair(PartValue, Offset));
9699         ++i;
9700       }
9701       SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
9702       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
9703       MemOpChains.push_back(
9704           DAG.getStore(Chain, DL, ArgValue, SpillSlot,
9705                        MachinePointerInfo::getFixedStack(MF, FI)));
9706       for (const auto &Part : Parts) {
9707         SDValue PartValue = Part.first;
9708         SDValue PartOffset = Part.second;
9709         SDValue Address =
9710             DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
9711         MemOpChains.push_back(
9712             DAG.getStore(Chain, DL, PartValue, Address,
9713                          MachinePointerInfo::getFixedStack(MF, FI)));
9714       }
9715       ArgValue = SpillSlot;
9716     } else {
9717       ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget);
9718     }
9719 
9720     // Use local copy if it is a byval arg.
9721     if (Flags.isByVal())
9722       ArgValue = ByValArgs[j++];
9723 
9724     if (VA.isRegLoc()) {
9725       // Queue up the argument copies and emit them at the end.
9726       RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
9727     } else {
9728       assert(VA.isMemLoc() && "Argument not register or memory");
9729       assert(!IsTailCall && "Tail call not allowed if stack is used "
9730                             "for passing parameters");
9731 
9732       // Work out the address of the stack slot.
9733       if (!StackPtr.getNode())
9734         StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
9735       SDValue Address =
9736           DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
9737                       DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
9738 
9739       // Emit the store.
9740       MemOpChains.push_back(
9741           DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
9742     }
9743   }
9744 
9745   // Join the stores, which are independent of one another.
9746   if (!MemOpChains.empty())
9747     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
9748 
9749   SDValue Glue;
9750 
9751   // Build a sequence of copy-to-reg nodes, chained and glued together.
9752   for (auto &Reg : RegsToPass) {
9753     Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
9754     Glue = Chain.getValue(1);
9755   }
9756 
9757   // Validate that none of the argument registers have been marked as
9758   // reserved, if so report an error. Do the same for the return address if this
9759   // is not a tailcall.
9760   validateCCReservedRegs(RegsToPass, MF);
9761   if (!IsTailCall &&
9762       MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1))
9763     MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9764         MF.getFunction(),
9765         "Return address register required, but has been reserved."});
9766 
9767   // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
9768   // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
9769   // split it and then direct call can be matched by PseudoCALL.
9770   if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
9771     const GlobalValue *GV = S->getGlobal();
9772 
9773     unsigned OpFlags = RISCVII::MO_CALL;
9774     if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
9775       OpFlags = RISCVII::MO_PLT;
9776 
9777     Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
9778   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
9779     unsigned OpFlags = RISCVII::MO_CALL;
9780 
9781     if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
9782                                                  nullptr))
9783       OpFlags = RISCVII::MO_PLT;
9784 
9785     Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
9786   }
9787 
9788   // The first call operand is the chain and the second is the target address.
9789   SmallVector<SDValue, 8> Ops;
9790   Ops.push_back(Chain);
9791   Ops.push_back(Callee);
9792 
9793   // Add argument registers to the end of the list so that they are
9794   // known live into the call.
9795   for (auto &Reg : RegsToPass)
9796     Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
9797 
9798   if (!IsTailCall) {
9799     // Add a register mask operand representing the call-preserved registers.
9800     const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
9801     const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
9802     assert(Mask && "Missing call preserved mask for calling convention");
9803     Ops.push_back(DAG.getRegisterMask(Mask));
9804   }
9805 
9806   // Glue the call to the argument copies, if any.
9807   if (Glue.getNode())
9808     Ops.push_back(Glue);
9809 
9810   // Emit the call.
9811   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9812 
9813   if (IsTailCall) {
9814     MF.getFrameInfo().setHasTailCall();
9815     return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
9816   }
9817 
9818   Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
9819   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
9820   Glue = Chain.getValue(1);
9821 
9822   // Mark the end of the call, which is glued to the call itself.
9823   Chain = DAG.getCALLSEQ_END(Chain,
9824                              DAG.getConstant(NumBytes, DL, PtrVT, true),
9825                              DAG.getConstant(0, DL, PtrVT, true),
9826                              Glue, DL);
9827   Glue = Chain.getValue(1);
9828 
9829   // Assign locations to each value returned by this call.
9830   SmallVector<CCValAssign, 16> RVLocs;
9831   CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
9832   analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, CC_RISCV);
9833 
9834   // Copy all of the result registers out of their specified physreg.
9835   for (auto &VA : RVLocs) {
9836     // Copy the value out
9837     SDValue RetValue =
9838         DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
9839     // Glue the RetValue to the end of the call sequence
9840     Chain = RetValue.getValue(1);
9841     Glue = RetValue.getValue(2);
9842 
9843     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
9844       assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
9845       SDValue RetValue2 =
9846           DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
9847       Chain = RetValue2.getValue(1);
9848       Glue = RetValue2.getValue(2);
9849       RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
9850                              RetValue2);
9851     }
9852 
9853     RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget);
9854 
9855     InVals.push_back(RetValue);
9856   }
9857 
9858   return Chain;
9859 }
9860 
9861 bool RISCVTargetLowering::CanLowerReturn(
9862     CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
9863     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
9864   SmallVector<CCValAssign, 16> RVLocs;
9865   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
9866 
9867   Optional<unsigned> FirstMaskArgument;
9868   if (Subtarget.hasVInstructions())
9869     FirstMaskArgument = preAssignMask(Outs);
9870 
9871   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9872     MVT VT = Outs[i].VT;
9873     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
9874     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9875     if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
9876                  ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr,
9877                  *this, FirstMaskArgument))
9878       return false;
9879   }
9880   return true;
9881 }
9882 
9883 SDValue
9884 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
9885                                  bool IsVarArg,
9886                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
9887                                  const SmallVectorImpl<SDValue> &OutVals,
9888                                  const SDLoc &DL, SelectionDAG &DAG) const {
9889   const MachineFunction &MF = DAG.getMachineFunction();
9890   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
9891 
9892   // Stores the assignment of the return value to a location.
9893   SmallVector<CCValAssign, 16> RVLocs;
9894 
9895   // Info about the registers and stack slot.
9896   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
9897                  *DAG.getContext());
9898 
9899   analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
9900                     nullptr, CC_RISCV);
9901 
9902   if (CallConv == CallingConv::GHC && !RVLocs.empty())
9903     report_fatal_error("GHC functions return void only");
9904 
9905   SDValue Glue;
9906   SmallVector<SDValue, 4> RetOps(1, Chain);
9907 
9908   // Copy the result values into the output registers.
9909   for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
9910     SDValue Val = OutVals[i];
9911     CCValAssign &VA = RVLocs[i];
9912     assert(VA.isRegLoc() && "Can only return in registers!");
9913 
9914     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
9915       // Handle returning f64 on RV32D with a soft float ABI.
9916       assert(VA.isRegLoc() && "Expected return via registers");
9917       SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
9918                                      DAG.getVTList(MVT::i32, MVT::i32), Val);
9919       SDValue Lo = SplitF64.getValue(0);
9920       SDValue Hi = SplitF64.getValue(1);
9921       Register RegLo = VA.getLocReg();
9922       assert(RegLo < RISCV::X31 && "Invalid register pair");
9923       Register RegHi = RegLo + 1;
9924 
9925       if (STI.isRegisterReservedByUser(RegLo) ||
9926           STI.isRegisterReservedByUser(RegHi))
9927         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9928             MF.getFunction(),
9929             "Return value register required, but has been reserved."});
9930 
9931       Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
9932       Glue = Chain.getValue(1);
9933       RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
9934       Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
9935       Glue = Chain.getValue(1);
9936       RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
9937     } else {
9938       // Handle a 'normal' return.
9939       Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
9940       Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
9941 
9942       if (STI.isRegisterReservedByUser(VA.getLocReg()))
9943         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9944             MF.getFunction(),
9945             "Return value register required, but has been reserved."});
9946 
9947       // Guarantee that all emitted copies are stuck together.
9948       Glue = Chain.getValue(1);
9949       RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
9950     }
9951   }
9952 
9953   RetOps[0] = Chain; // Update chain.
9954 
9955   // Add the glue node if we have it.
9956   if (Glue.getNode()) {
9957     RetOps.push_back(Glue);
9958   }
9959 
9960   unsigned RetOpc = RISCVISD::RET_FLAG;
9961   // Interrupt service routines use different return instructions.
9962   const Function &Func = DAG.getMachineFunction().getFunction();
9963   if (Func.hasFnAttribute("interrupt")) {
9964     if (!Func.getReturnType()->isVoidTy())
9965       report_fatal_error(
9966           "Functions with the interrupt attribute must have void return type!");
9967 
9968     MachineFunction &MF = DAG.getMachineFunction();
9969     StringRef Kind =
9970       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
9971 
9972     if (Kind == "user")
9973       RetOpc = RISCVISD::URET_FLAG;
9974     else if (Kind == "supervisor")
9975       RetOpc = RISCVISD::SRET_FLAG;
9976     else
9977       RetOpc = RISCVISD::MRET_FLAG;
9978   }
9979 
9980   return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
9981 }
9982 
9983 void RISCVTargetLowering::validateCCReservedRegs(
9984     const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
9985     MachineFunction &MF) const {
9986   const Function &F = MF.getFunction();
9987   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
9988 
9989   if (llvm::any_of(Regs, [&STI](auto Reg) {
9990         return STI.isRegisterReservedByUser(Reg.first);
9991       }))
9992     F.getContext().diagnose(DiagnosticInfoUnsupported{
9993         F, "Argument register required, but has been reserved."});
9994 }
9995 
9996 bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
9997   return CI->isTailCall();
9998 }
9999 
10000 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
10001 #define NODE_NAME_CASE(NODE)                                                   \
10002   case RISCVISD::NODE:                                                         \
10003     return "RISCVISD::" #NODE;
10004   // clang-format off
10005   switch ((RISCVISD::NodeType)Opcode) {
10006   case RISCVISD::FIRST_NUMBER:
10007     break;
10008   NODE_NAME_CASE(RET_FLAG)
10009   NODE_NAME_CASE(URET_FLAG)
10010   NODE_NAME_CASE(SRET_FLAG)
10011   NODE_NAME_CASE(MRET_FLAG)
10012   NODE_NAME_CASE(CALL)
10013   NODE_NAME_CASE(SELECT_CC)
10014   NODE_NAME_CASE(BR_CC)
10015   NODE_NAME_CASE(BuildPairF64)
10016   NODE_NAME_CASE(SplitF64)
10017   NODE_NAME_CASE(TAIL)
10018   NODE_NAME_CASE(MULHSU)
10019   NODE_NAME_CASE(SLLW)
10020   NODE_NAME_CASE(SRAW)
10021   NODE_NAME_CASE(SRLW)
10022   NODE_NAME_CASE(DIVW)
10023   NODE_NAME_CASE(DIVUW)
10024   NODE_NAME_CASE(REMUW)
10025   NODE_NAME_CASE(ROLW)
10026   NODE_NAME_CASE(RORW)
10027   NODE_NAME_CASE(CLZW)
10028   NODE_NAME_CASE(CTZW)
10029   NODE_NAME_CASE(FSLW)
10030   NODE_NAME_CASE(FSRW)
10031   NODE_NAME_CASE(FSL)
10032   NODE_NAME_CASE(FSR)
10033   NODE_NAME_CASE(FMV_H_X)
10034   NODE_NAME_CASE(FMV_X_ANYEXTH)
10035   NODE_NAME_CASE(FMV_W_X_RV64)
10036   NODE_NAME_CASE(FMV_X_ANYEXTW_RV64)
10037   NODE_NAME_CASE(FCVT_X)
10038   NODE_NAME_CASE(FCVT_XU)
10039   NODE_NAME_CASE(FCVT_W_RV64)
10040   NODE_NAME_CASE(FCVT_WU_RV64)
10041   NODE_NAME_CASE(STRICT_FCVT_W_RV64)
10042   NODE_NAME_CASE(STRICT_FCVT_WU_RV64)
10043   NODE_NAME_CASE(READ_CYCLE_WIDE)
10044   NODE_NAME_CASE(GREV)
10045   NODE_NAME_CASE(GREVW)
10046   NODE_NAME_CASE(GORC)
10047   NODE_NAME_CASE(GORCW)
10048   NODE_NAME_CASE(SHFL)
10049   NODE_NAME_CASE(SHFLW)
10050   NODE_NAME_CASE(UNSHFL)
10051   NODE_NAME_CASE(UNSHFLW)
10052   NODE_NAME_CASE(BFP)
10053   NODE_NAME_CASE(BFPW)
10054   NODE_NAME_CASE(BCOMPRESS)
10055   NODE_NAME_CASE(BCOMPRESSW)
10056   NODE_NAME_CASE(BDECOMPRESS)
10057   NODE_NAME_CASE(BDECOMPRESSW)
10058   NODE_NAME_CASE(VMV_V_X_VL)
10059   NODE_NAME_CASE(VFMV_V_F_VL)
10060   NODE_NAME_CASE(VMV_X_S)
10061   NODE_NAME_CASE(VMV_S_X_VL)
10062   NODE_NAME_CASE(VFMV_S_F_VL)
10063   NODE_NAME_CASE(SPLAT_VECTOR_I64)
10064   NODE_NAME_CASE(SPLAT_VECTOR_SPLIT_I64_VL)
10065   NODE_NAME_CASE(READ_VLENB)
10066   NODE_NAME_CASE(TRUNCATE_VECTOR_VL)
10067   NODE_NAME_CASE(VSLIDEUP_VL)
10068   NODE_NAME_CASE(VSLIDE1UP_VL)
10069   NODE_NAME_CASE(VSLIDEDOWN_VL)
10070   NODE_NAME_CASE(VSLIDE1DOWN_VL)
10071   NODE_NAME_CASE(VID_VL)
10072   NODE_NAME_CASE(VFNCVT_ROD_VL)
10073   NODE_NAME_CASE(VECREDUCE_ADD_VL)
10074   NODE_NAME_CASE(VECREDUCE_UMAX_VL)
10075   NODE_NAME_CASE(VECREDUCE_SMAX_VL)
10076   NODE_NAME_CASE(VECREDUCE_UMIN_VL)
10077   NODE_NAME_CASE(VECREDUCE_SMIN_VL)
10078   NODE_NAME_CASE(VECREDUCE_AND_VL)
10079   NODE_NAME_CASE(VECREDUCE_OR_VL)
10080   NODE_NAME_CASE(VECREDUCE_XOR_VL)
10081   NODE_NAME_CASE(VECREDUCE_FADD_VL)
10082   NODE_NAME_CASE(VECREDUCE_SEQ_FADD_VL)
10083   NODE_NAME_CASE(VECREDUCE_FMIN_VL)
10084   NODE_NAME_CASE(VECREDUCE_FMAX_VL)
10085   NODE_NAME_CASE(ADD_VL)
10086   NODE_NAME_CASE(AND_VL)
10087   NODE_NAME_CASE(MUL_VL)
10088   NODE_NAME_CASE(OR_VL)
10089   NODE_NAME_CASE(SDIV_VL)
10090   NODE_NAME_CASE(SHL_VL)
10091   NODE_NAME_CASE(SREM_VL)
10092   NODE_NAME_CASE(SRA_VL)
10093   NODE_NAME_CASE(SRL_VL)
10094   NODE_NAME_CASE(SUB_VL)
10095   NODE_NAME_CASE(UDIV_VL)
10096   NODE_NAME_CASE(UREM_VL)
10097   NODE_NAME_CASE(XOR_VL)
10098   NODE_NAME_CASE(SADDSAT_VL)
10099   NODE_NAME_CASE(UADDSAT_VL)
10100   NODE_NAME_CASE(SSUBSAT_VL)
10101   NODE_NAME_CASE(USUBSAT_VL)
10102   NODE_NAME_CASE(FADD_VL)
10103   NODE_NAME_CASE(FSUB_VL)
10104   NODE_NAME_CASE(FMUL_VL)
10105   NODE_NAME_CASE(FDIV_VL)
10106   NODE_NAME_CASE(FNEG_VL)
10107   NODE_NAME_CASE(FABS_VL)
10108   NODE_NAME_CASE(FSQRT_VL)
10109   NODE_NAME_CASE(FMA_VL)
10110   NODE_NAME_CASE(FCOPYSIGN_VL)
10111   NODE_NAME_CASE(SMIN_VL)
10112   NODE_NAME_CASE(SMAX_VL)
10113   NODE_NAME_CASE(UMIN_VL)
10114   NODE_NAME_CASE(UMAX_VL)
10115   NODE_NAME_CASE(FMINNUM_VL)
10116   NODE_NAME_CASE(FMAXNUM_VL)
10117   NODE_NAME_CASE(MULHS_VL)
10118   NODE_NAME_CASE(MULHU_VL)
10119   NODE_NAME_CASE(FP_TO_SINT_VL)
10120   NODE_NAME_CASE(FP_TO_UINT_VL)
10121   NODE_NAME_CASE(SINT_TO_FP_VL)
10122   NODE_NAME_CASE(UINT_TO_FP_VL)
10123   NODE_NAME_CASE(FP_EXTEND_VL)
10124   NODE_NAME_CASE(FP_ROUND_VL)
10125   NODE_NAME_CASE(VWMUL_VL)
10126   NODE_NAME_CASE(VWMULU_VL)
10127   NODE_NAME_CASE(VWADDU_VL)
10128   NODE_NAME_CASE(SETCC_VL)
10129   NODE_NAME_CASE(VSELECT_VL)
10130   NODE_NAME_CASE(VP_MERGE_VL)
10131   NODE_NAME_CASE(VMAND_VL)
10132   NODE_NAME_CASE(VMOR_VL)
10133   NODE_NAME_CASE(VMXOR_VL)
10134   NODE_NAME_CASE(VMCLR_VL)
10135   NODE_NAME_CASE(VMSET_VL)
10136   NODE_NAME_CASE(VRGATHER_VX_VL)
10137   NODE_NAME_CASE(VRGATHER_VV_VL)
10138   NODE_NAME_CASE(VRGATHEREI16_VV_VL)
10139   NODE_NAME_CASE(VSEXT_VL)
10140   NODE_NAME_CASE(VZEXT_VL)
10141   NODE_NAME_CASE(VCPOP_VL)
10142   NODE_NAME_CASE(VLE_VL)
10143   NODE_NAME_CASE(VSE_VL)
10144   NODE_NAME_CASE(READ_CSR)
10145   NODE_NAME_CASE(WRITE_CSR)
10146   NODE_NAME_CASE(SWAP_CSR)
10147   }
10148   // clang-format on
10149   return nullptr;
10150 #undef NODE_NAME_CASE
10151 }
10152 
10153 /// getConstraintType - Given a constraint letter, return the type of
10154 /// constraint it is for this target.
10155 RISCVTargetLowering::ConstraintType
10156 RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
10157   if (Constraint.size() == 1) {
10158     switch (Constraint[0]) {
10159     default:
10160       break;
10161     case 'f':
10162       return C_RegisterClass;
10163     case 'I':
10164     case 'J':
10165     case 'K':
10166       return C_Immediate;
10167     case 'A':
10168       return C_Memory;
10169     case 'S': // A symbolic address
10170       return C_Other;
10171     }
10172   } else {
10173     if (Constraint == "vr" || Constraint == "vm")
10174       return C_RegisterClass;
10175   }
10176   return TargetLowering::getConstraintType(Constraint);
10177 }
10178 
10179 std::pair<unsigned, const TargetRegisterClass *>
10180 RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
10181                                                   StringRef Constraint,
10182                                                   MVT VT) const {
10183   // First, see if this is a constraint that directly corresponds to a
10184   // RISCV register class.
10185   if (Constraint.size() == 1) {
10186     switch (Constraint[0]) {
10187     case 'r':
10188       // TODO: Support fixed vectors up to XLen for P extension?
10189       if (VT.isVector())
10190         break;
10191       return std::make_pair(0U, &RISCV::GPRRegClass);
10192     case 'f':
10193       if (Subtarget.hasStdExtZfh() && VT == MVT::f16)
10194         return std::make_pair(0U, &RISCV::FPR16RegClass);
10195       if (Subtarget.hasStdExtF() && VT == MVT::f32)
10196         return std::make_pair(0U, &RISCV::FPR32RegClass);
10197       if (Subtarget.hasStdExtD() && VT == MVT::f64)
10198         return std::make_pair(0U, &RISCV::FPR64RegClass);
10199       break;
10200     default:
10201       break;
10202     }
10203   } else if (Constraint == "vr") {
10204     for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
10205                            &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10206       if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
10207         return std::make_pair(0U, RC);
10208     }
10209   } else if (Constraint == "vm") {
10210     if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
10211       return std::make_pair(0U, &RISCV::VMV0RegClass);
10212   }
10213 
10214   // Clang will correctly decode the usage of register name aliases into their
10215   // official names. However, other frontends like `rustc` do not. This allows
10216   // users of these frontends to use the ABI names for registers in LLVM-style
10217   // register constraints.
10218   unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
10219                                .Case("{zero}", RISCV::X0)
10220                                .Case("{ra}", RISCV::X1)
10221                                .Case("{sp}", RISCV::X2)
10222                                .Case("{gp}", RISCV::X3)
10223                                .Case("{tp}", RISCV::X4)
10224                                .Case("{t0}", RISCV::X5)
10225                                .Case("{t1}", RISCV::X6)
10226                                .Case("{t2}", RISCV::X7)
10227                                .Cases("{s0}", "{fp}", RISCV::X8)
10228                                .Case("{s1}", RISCV::X9)
10229                                .Case("{a0}", RISCV::X10)
10230                                .Case("{a1}", RISCV::X11)
10231                                .Case("{a2}", RISCV::X12)
10232                                .Case("{a3}", RISCV::X13)
10233                                .Case("{a4}", RISCV::X14)
10234                                .Case("{a5}", RISCV::X15)
10235                                .Case("{a6}", RISCV::X16)
10236                                .Case("{a7}", RISCV::X17)
10237                                .Case("{s2}", RISCV::X18)
10238                                .Case("{s3}", RISCV::X19)
10239                                .Case("{s4}", RISCV::X20)
10240                                .Case("{s5}", RISCV::X21)
10241                                .Case("{s6}", RISCV::X22)
10242                                .Case("{s7}", RISCV::X23)
10243                                .Case("{s8}", RISCV::X24)
10244                                .Case("{s9}", RISCV::X25)
10245                                .Case("{s10}", RISCV::X26)
10246                                .Case("{s11}", RISCV::X27)
10247                                .Case("{t3}", RISCV::X28)
10248                                .Case("{t4}", RISCV::X29)
10249                                .Case("{t5}", RISCV::X30)
10250                                .Case("{t6}", RISCV::X31)
10251                                .Default(RISCV::NoRegister);
10252   if (XRegFromAlias != RISCV::NoRegister)
10253     return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
10254 
10255   // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
10256   // TableGen record rather than the AsmName to choose registers for InlineAsm
10257   // constraints, plus we want to match those names to the widest floating point
10258   // register type available, manually select floating point registers here.
10259   //
10260   // The second case is the ABI name of the register, so that frontends can also
10261   // use the ABI names in register constraint lists.
10262   if (Subtarget.hasStdExtF()) {
10263     unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
10264                         .Cases("{f0}", "{ft0}", RISCV::F0_F)
10265                         .Cases("{f1}", "{ft1}", RISCV::F1_F)
10266                         .Cases("{f2}", "{ft2}", RISCV::F2_F)
10267                         .Cases("{f3}", "{ft3}", RISCV::F3_F)
10268                         .Cases("{f4}", "{ft4}", RISCV::F4_F)
10269                         .Cases("{f5}", "{ft5}", RISCV::F5_F)
10270                         .Cases("{f6}", "{ft6}", RISCV::F6_F)
10271                         .Cases("{f7}", "{ft7}", RISCV::F7_F)
10272                         .Cases("{f8}", "{fs0}", RISCV::F8_F)
10273                         .Cases("{f9}", "{fs1}", RISCV::F9_F)
10274                         .Cases("{f10}", "{fa0}", RISCV::F10_F)
10275                         .Cases("{f11}", "{fa1}", RISCV::F11_F)
10276                         .Cases("{f12}", "{fa2}", RISCV::F12_F)
10277                         .Cases("{f13}", "{fa3}", RISCV::F13_F)
10278                         .Cases("{f14}", "{fa4}", RISCV::F14_F)
10279                         .Cases("{f15}", "{fa5}", RISCV::F15_F)
10280                         .Cases("{f16}", "{fa6}", RISCV::F16_F)
10281                         .Cases("{f17}", "{fa7}", RISCV::F17_F)
10282                         .Cases("{f18}", "{fs2}", RISCV::F18_F)
10283                         .Cases("{f19}", "{fs3}", RISCV::F19_F)
10284                         .Cases("{f20}", "{fs4}", RISCV::F20_F)
10285                         .Cases("{f21}", "{fs5}", RISCV::F21_F)
10286                         .Cases("{f22}", "{fs6}", RISCV::F22_F)
10287                         .Cases("{f23}", "{fs7}", RISCV::F23_F)
10288                         .Cases("{f24}", "{fs8}", RISCV::F24_F)
10289                         .Cases("{f25}", "{fs9}", RISCV::F25_F)
10290                         .Cases("{f26}", "{fs10}", RISCV::F26_F)
10291                         .Cases("{f27}", "{fs11}", RISCV::F27_F)
10292                         .Cases("{f28}", "{ft8}", RISCV::F28_F)
10293                         .Cases("{f29}", "{ft9}", RISCV::F29_F)
10294                         .Cases("{f30}", "{ft10}", RISCV::F30_F)
10295                         .Cases("{f31}", "{ft11}", RISCV::F31_F)
10296                         .Default(RISCV::NoRegister);
10297     if (FReg != RISCV::NoRegister) {
10298       assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
10299       if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
10300         unsigned RegNo = FReg - RISCV::F0_F;
10301         unsigned DReg = RISCV::F0_D + RegNo;
10302         return std::make_pair(DReg, &RISCV::FPR64RegClass);
10303       }
10304       if (VT == MVT::f32 || VT == MVT::Other)
10305         return std::make_pair(FReg, &RISCV::FPR32RegClass);
10306       if (Subtarget.hasStdExtZfh() && VT == MVT::f16) {
10307         unsigned RegNo = FReg - RISCV::F0_F;
10308         unsigned HReg = RISCV::F0_H + RegNo;
10309         return std::make_pair(HReg, &RISCV::FPR16RegClass);
10310       }
10311     }
10312   }
10313 
10314   if (Subtarget.hasVInstructions()) {
10315     Register VReg = StringSwitch<Register>(Constraint.lower())
10316                         .Case("{v0}", RISCV::V0)
10317                         .Case("{v1}", RISCV::V1)
10318                         .Case("{v2}", RISCV::V2)
10319                         .Case("{v3}", RISCV::V3)
10320                         .Case("{v4}", RISCV::V4)
10321                         .Case("{v5}", RISCV::V5)
10322                         .Case("{v6}", RISCV::V6)
10323                         .Case("{v7}", RISCV::V7)
10324                         .Case("{v8}", RISCV::V8)
10325                         .Case("{v9}", RISCV::V9)
10326                         .Case("{v10}", RISCV::V10)
10327                         .Case("{v11}", RISCV::V11)
10328                         .Case("{v12}", RISCV::V12)
10329                         .Case("{v13}", RISCV::V13)
10330                         .Case("{v14}", RISCV::V14)
10331                         .Case("{v15}", RISCV::V15)
10332                         .Case("{v16}", RISCV::V16)
10333                         .Case("{v17}", RISCV::V17)
10334                         .Case("{v18}", RISCV::V18)
10335                         .Case("{v19}", RISCV::V19)
10336                         .Case("{v20}", RISCV::V20)
10337                         .Case("{v21}", RISCV::V21)
10338                         .Case("{v22}", RISCV::V22)
10339                         .Case("{v23}", RISCV::V23)
10340                         .Case("{v24}", RISCV::V24)
10341                         .Case("{v25}", RISCV::V25)
10342                         .Case("{v26}", RISCV::V26)
10343                         .Case("{v27}", RISCV::V27)
10344                         .Case("{v28}", RISCV::V28)
10345                         .Case("{v29}", RISCV::V29)
10346                         .Case("{v30}", RISCV::V30)
10347                         .Case("{v31}", RISCV::V31)
10348                         .Default(RISCV::NoRegister);
10349     if (VReg != RISCV::NoRegister) {
10350       if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
10351         return std::make_pair(VReg, &RISCV::VMRegClass);
10352       if (TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.SimpleTy))
10353         return std::make_pair(VReg, &RISCV::VRRegClass);
10354       for (const auto *RC :
10355            {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10356         if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) {
10357           VReg = TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
10358           return std::make_pair(VReg, RC);
10359         }
10360       }
10361     }
10362   }
10363 
10364   return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
10365 }
10366 
10367 unsigned
10368 RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
10369   // Currently only support length 1 constraints.
10370   if (ConstraintCode.size() == 1) {
10371     switch (ConstraintCode[0]) {
10372     case 'A':
10373       return InlineAsm::Constraint_A;
10374     default:
10375       break;
10376     }
10377   }
10378 
10379   return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
10380 }
10381 
10382 void RISCVTargetLowering::LowerAsmOperandForConstraint(
10383     SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
10384     SelectionDAG &DAG) const {
10385   // Currently only support length 1 constraints.
10386   if (Constraint.length() == 1) {
10387     switch (Constraint[0]) {
10388     case 'I':
10389       // Validate & create a 12-bit signed immediate operand.
10390       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10391         uint64_t CVal = C->getSExtValue();
10392         if (isInt<12>(CVal))
10393           Ops.push_back(
10394               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10395       }
10396       return;
10397     case 'J':
10398       // Validate & create an integer zero operand.
10399       if (auto *C = dyn_cast<ConstantSDNode>(Op))
10400         if (C->getZExtValue() == 0)
10401           Ops.push_back(
10402               DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
10403       return;
10404     case 'K':
10405       // Validate & create a 5-bit unsigned immediate operand.
10406       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10407         uint64_t CVal = C->getZExtValue();
10408         if (isUInt<5>(CVal))
10409           Ops.push_back(
10410               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10411       }
10412       return;
10413     case 'S':
10414       if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
10415         Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
10416                                                  GA->getValueType(0)));
10417       } else if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
10418         Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(),
10419                                                 BA->getValueType(0)));
10420       }
10421       return;
10422     default:
10423       break;
10424     }
10425   }
10426   TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
10427 }
10428 
10429 Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
10430                                                    Instruction *Inst,
10431                                                    AtomicOrdering Ord) const {
10432   if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
10433     return Builder.CreateFence(Ord);
10434   if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
10435     return Builder.CreateFence(AtomicOrdering::Release);
10436   return nullptr;
10437 }
10438 
10439 Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
10440                                                     Instruction *Inst,
10441                                                     AtomicOrdering Ord) const {
10442   if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
10443     return Builder.CreateFence(AtomicOrdering::Acquire);
10444   return nullptr;
10445 }
10446 
10447 TargetLowering::AtomicExpansionKind
10448 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
10449   // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
10450   // point operations can't be used in an lr/sc sequence without breaking the
10451   // forward-progress guarantee.
10452   if (AI->isFloatingPointOperation())
10453     return AtomicExpansionKind::CmpXChg;
10454 
10455   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
10456   if (Size == 8 || Size == 16)
10457     return AtomicExpansionKind::MaskedIntrinsic;
10458   return AtomicExpansionKind::None;
10459 }
10460 
10461 static Intrinsic::ID
10462 getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
10463   if (XLen == 32) {
10464     switch (BinOp) {
10465     default:
10466       llvm_unreachable("Unexpected AtomicRMW BinOp");
10467     case AtomicRMWInst::Xchg:
10468       return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
10469     case AtomicRMWInst::Add:
10470       return Intrinsic::riscv_masked_atomicrmw_add_i32;
10471     case AtomicRMWInst::Sub:
10472       return Intrinsic::riscv_masked_atomicrmw_sub_i32;
10473     case AtomicRMWInst::Nand:
10474       return Intrinsic::riscv_masked_atomicrmw_nand_i32;
10475     case AtomicRMWInst::Max:
10476       return Intrinsic::riscv_masked_atomicrmw_max_i32;
10477     case AtomicRMWInst::Min:
10478       return Intrinsic::riscv_masked_atomicrmw_min_i32;
10479     case AtomicRMWInst::UMax:
10480       return Intrinsic::riscv_masked_atomicrmw_umax_i32;
10481     case AtomicRMWInst::UMin:
10482       return Intrinsic::riscv_masked_atomicrmw_umin_i32;
10483     }
10484   }
10485 
10486   if (XLen == 64) {
10487     switch (BinOp) {
10488     default:
10489       llvm_unreachable("Unexpected AtomicRMW BinOp");
10490     case AtomicRMWInst::Xchg:
10491       return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
10492     case AtomicRMWInst::Add:
10493       return Intrinsic::riscv_masked_atomicrmw_add_i64;
10494     case AtomicRMWInst::Sub:
10495       return Intrinsic::riscv_masked_atomicrmw_sub_i64;
10496     case AtomicRMWInst::Nand:
10497       return Intrinsic::riscv_masked_atomicrmw_nand_i64;
10498     case AtomicRMWInst::Max:
10499       return Intrinsic::riscv_masked_atomicrmw_max_i64;
10500     case AtomicRMWInst::Min:
10501       return Intrinsic::riscv_masked_atomicrmw_min_i64;
10502     case AtomicRMWInst::UMax:
10503       return Intrinsic::riscv_masked_atomicrmw_umax_i64;
10504     case AtomicRMWInst::UMin:
10505       return Intrinsic::riscv_masked_atomicrmw_umin_i64;
10506     }
10507   }
10508 
10509   llvm_unreachable("Unexpected XLen\n");
10510 }
10511 
10512 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
10513     IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
10514     Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
10515   unsigned XLen = Subtarget.getXLen();
10516   Value *Ordering =
10517       Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
10518   Type *Tys[] = {AlignedAddr->getType()};
10519   Function *LrwOpScwLoop = Intrinsic::getDeclaration(
10520       AI->getModule(),
10521       getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
10522 
10523   if (XLen == 64) {
10524     Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
10525     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10526     ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
10527   }
10528 
10529   Value *Result;
10530 
10531   // Must pass the shift amount needed to sign extend the loaded value prior
10532   // to performing a signed comparison for min/max. ShiftAmt is the number of
10533   // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
10534   // is the number of bits to left+right shift the value in order to
10535   // sign-extend.
10536   if (AI->getOperation() == AtomicRMWInst::Min ||
10537       AI->getOperation() == AtomicRMWInst::Max) {
10538     const DataLayout &DL = AI->getModule()->getDataLayout();
10539     unsigned ValWidth =
10540         DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
10541     Value *SextShamt =
10542         Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
10543     Result = Builder.CreateCall(LrwOpScwLoop,
10544                                 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
10545   } else {
10546     Result =
10547         Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
10548   }
10549 
10550   if (XLen == 64)
10551     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10552   return Result;
10553 }
10554 
10555 TargetLowering::AtomicExpansionKind
10556 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
10557     AtomicCmpXchgInst *CI) const {
10558   unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
10559   if (Size == 8 || Size == 16)
10560     return AtomicExpansionKind::MaskedIntrinsic;
10561   return AtomicExpansionKind::None;
10562 }
10563 
10564 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
10565     IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
10566     Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
10567   unsigned XLen = Subtarget.getXLen();
10568   Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
10569   Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
10570   if (XLen == 64) {
10571     CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
10572     NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
10573     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10574     CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
10575   }
10576   Type *Tys[] = {AlignedAddr->getType()};
10577   Function *MaskedCmpXchg =
10578       Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
10579   Value *Result = Builder.CreateCall(
10580       MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
10581   if (XLen == 64)
10582     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10583   return Result;
10584 }
10585 
10586 bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
10587   return false;
10588 }
10589 
10590 bool RISCVTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
10591                                                EVT VT) const {
10592   if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
10593     return false;
10594 
10595   switch (FPVT.getSimpleVT().SimpleTy) {
10596   case MVT::f16:
10597     return Subtarget.hasStdExtZfh();
10598   case MVT::f32:
10599     return Subtarget.hasStdExtF();
10600   case MVT::f64:
10601     return Subtarget.hasStdExtD();
10602   default:
10603     return false;
10604   }
10605 }
10606 
10607 unsigned RISCVTargetLowering::getJumpTableEncoding() const {
10608   // If we are using the small code model, we can reduce size of jump table
10609   // entry to 4 bytes.
10610   if (Subtarget.is64Bit() && !isPositionIndependent() &&
10611       getTargetMachine().getCodeModel() == CodeModel::Small) {
10612     return MachineJumpTableInfo::EK_Custom32;
10613   }
10614   return TargetLowering::getJumpTableEncoding();
10615 }
10616 
10617 const MCExpr *RISCVTargetLowering::LowerCustomJumpTableEntry(
10618     const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
10619     unsigned uid, MCContext &Ctx) const {
10620   assert(Subtarget.is64Bit() && !isPositionIndependent() &&
10621          getTargetMachine().getCodeModel() == CodeModel::Small);
10622   return MCSymbolRefExpr::create(MBB->getSymbol(), Ctx);
10623 }
10624 
10625 bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
10626                                                      EVT VT) const {
10627   VT = VT.getScalarType();
10628 
10629   if (!VT.isSimple())
10630     return false;
10631 
10632   switch (VT.getSimpleVT().SimpleTy) {
10633   case MVT::f16:
10634     return Subtarget.hasStdExtZfh();
10635   case MVT::f32:
10636     return Subtarget.hasStdExtF();
10637   case MVT::f64:
10638     return Subtarget.hasStdExtD();
10639   default:
10640     break;
10641   }
10642 
10643   return false;
10644 }
10645 
10646 Register RISCVTargetLowering::getExceptionPointerRegister(
10647     const Constant *PersonalityFn) const {
10648   return RISCV::X10;
10649 }
10650 
10651 Register RISCVTargetLowering::getExceptionSelectorRegister(
10652     const Constant *PersonalityFn) const {
10653   return RISCV::X11;
10654 }
10655 
10656 bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
10657   // Return false to suppress the unnecessary extensions if the LibCall
10658   // arguments or return value is f32 type for LP64 ABI.
10659   RISCVABI::ABI ABI = Subtarget.getTargetABI();
10660   if (ABI == RISCVABI::ABI_LP64 && (Type == MVT::f32))
10661     return false;
10662 
10663   return true;
10664 }
10665 
10666 bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
10667   if (Subtarget.is64Bit() && Type == MVT::i32)
10668     return true;
10669 
10670   return IsSigned;
10671 }
10672 
10673 bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
10674                                                  SDValue C) const {
10675   // Check integral scalar types.
10676   if (VT.isScalarInteger()) {
10677     // Omit the optimization if the sub target has the M extension and the data
10678     // size exceeds XLen.
10679     if (Subtarget.hasStdExtM() && VT.getSizeInBits() > Subtarget.getXLen())
10680       return false;
10681     if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
10682       // Break the MUL to a SLLI and an ADD/SUB.
10683       const APInt &Imm = ConstNode->getAPIntValue();
10684       if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
10685           (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
10686         return true;
10687       // Optimize the MUL to (SH*ADD x, (SLLI x, bits)) if Imm is not simm12.
10688       if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
10689           ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
10690            (Imm - 8).isPowerOf2()))
10691         return true;
10692       // Omit the following optimization if the sub target has the M extension
10693       // and the data size >= XLen.
10694       if (Subtarget.hasStdExtM() && VT.getSizeInBits() >= Subtarget.getXLen())
10695         return false;
10696       // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
10697       // a pair of LUI/ADDI.
10698       if (!Imm.isSignedIntN(12) && Imm.countTrailingZeros() < 12) {
10699         APInt ImmS = Imm.ashr(Imm.countTrailingZeros());
10700         if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
10701             (1 - ImmS).isPowerOf2())
10702         return true;
10703       }
10704     }
10705   }
10706 
10707   return false;
10708 }
10709 
10710 bool RISCVTargetLowering::isMulAddWithConstProfitable(
10711     const SDValue &AddNode, const SDValue &ConstNode) const {
10712   // Let the DAGCombiner decide for vectors.
10713   EVT VT = AddNode.getValueType();
10714   if (VT.isVector())
10715     return true;
10716 
10717   // Let the DAGCombiner decide for larger types.
10718   if (VT.getScalarSizeInBits() > Subtarget.getXLen())
10719     return true;
10720 
10721   // It is worse if c1 is simm12 while c1*c2 is not.
10722   ConstantSDNode *C1Node = cast<ConstantSDNode>(AddNode.getOperand(1));
10723   ConstantSDNode *C2Node = cast<ConstantSDNode>(ConstNode);
10724   const APInt &C1 = C1Node->getAPIntValue();
10725   const APInt &C2 = C2Node->getAPIntValue();
10726   if (C1.isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
10727     return false;
10728 
10729   // Default to true and let the DAGCombiner decide.
10730   return true;
10731 }
10732 
10733 bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
10734     EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
10735     bool *Fast) const {
10736   if (!VT.isVector())
10737     return false;
10738 
10739   EVT ElemVT = VT.getVectorElementType();
10740   if (Alignment >= ElemVT.getStoreSize()) {
10741     if (Fast)
10742       *Fast = true;
10743     return true;
10744   }
10745 
10746   return false;
10747 }
10748 
10749 bool RISCVTargetLowering::splitValueIntoRegisterParts(
10750     SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
10751     unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {
10752   bool IsABIRegCopy = CC.hasValue();
10753   EVT ValueVT = Val.getValueType();
10754   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
10755     // Cast the f16 to i16, extend to i32, pad with ones to make a float nan,
10756     // and cast to f32.
10757     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
10758     Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
10759     Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
10760                       DAG.getConstant(0xFFFF0000, DL, MVT::i32));
10761     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
10762     Parts[0] = Val;
10763     return true;
10764   }
10765 
10766   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
10767     LLVMContext &Context = *DAG.getContext();
10768     EVT ValueEltVT = ValueVT.getVectorElementType();
10769     EVT PartEltVT = PartVT.getVectorElementType();
10770     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
10771     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
10772     if (PartVTBitSize % ValueVTBitSize == 0) {
10773       assert(PartVTBitSize >= ValueVTBitSize);
10774       // If the element types are different, bitcast to the same element type of
10775       // PartVT first.
10776       // Give an example here, we want copy a <vscale x 1 x i8> value to
10777       // <vscale x 4 x i16>.
10778       // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert
10779       // subvector, then we can bitcast to <vscale x 4 x i16>.
10780       if (ValueEltVT != PartEltVT) {
10781         if (PartVTBitSize > ValueVTBitSize) {
10782           unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
10783           assert(Count != 0 && "The number of element should not be zero.");
10784           EVT SameEltTypeVT =
10785               EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
10786           Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT,
10787                             DAG.getUNDEF(SameEltTypeVT), Val,
10788                             DAG.getVectorIdxConstant(0, DL));
10789         }
10790         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
10791       } else {
10792         Val =
10793             DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
10794                         Val, DAG.getVectorIdxConstant(0, DL));
10795       }
10796       Parts[0] = Val;
10797       return true;
10798     }
10799   }
10800   return false;
10801 }
10802 
10803 SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
10804     SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
10805     MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {
10806   bool IsABIRegCopy = CC.hasValue();
10807   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
10808     SDValue Val = Parts[0];
10809 
10810     // Cast the f32 to i32, truncate to i16, and cast back to f16.
10811     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
10812     Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
10813     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f16, Val);
10814     return Val;
10815   }
10816 
10817   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
10818     LLVMContext &Context = *DAG.getContext();
10819     SDValue Val = Parts[0];
10820     EVT ValueEltVT = ValueVT.getVectorElementType();
10821     EVT PartEltVT = PartVT.getVectorElementType();
10822     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
10823     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
10824     if (PartVTBitSize % ValueVTBitSize == 0) {
10825       assert(PartVTBitSize >= ValueVTBitSize);
10826       EVT SameEltTypeVT = ValueVT;
10827       // If the element types are different, convert it to the same element type
10828       // of PartVT.
10829       // Give an example here, we want copy a <vscale x 1 x i8> value from
10830       // <vscale x 4 x i16>.
10831       // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first,
10832       // then we can extract <vscale x 1 x i8>.
10833       if (ValueEltVT != PartEltVT) {
10834         unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
10835         assert(Count != 0 && "The number of element should not be zero.");
10836         SameEltTypeVT =
10837             EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
10838         Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
10839       }
10840       Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
10841                         DAG.getVectorIdxConstant(0, DL));
10842       return Val;
10843     }
10844   }
10845   return SDValue();
10846 }
10847 
10848 SDValue
10849 RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
10850                                    SelectionDAG &DAG,
10851                                    SmallVectorImpl<SDNode *> &Created) const {
10852   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
10853   if (isIntDivCheap(N->getValueType(0), Attr))
10854     return SDValue(N, 0); // Lower SDIV as SDIV
10855 
10856   assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
10857          "Unexpected divisor!");
10858 
10859   // Conditional move is needed, so do the transformation iff Zbt is enabled.
10860   if (!Subtarget.hasStdExtZbt())
10861     return SDValue();
10862 
10863   // When |Divisor| >= 2 ^ 12, it isn't profitable to do such transformation.
10864   // Besides, more critical path instructions will be generated when dividing
10865   // by 2. So we keep using the original DAGs for these cases.
10866   unsigned Lg2 = Divisor.countTrailingZeros();
10867   if (Lg2 == 1 || Lg2 >= 12)
10868     return SDValue();
10869 
10870   // fold (sdiv X, pow2)
10871   EVT VT = N->getValueType(0);
10872   if (VT != MVT::i32 && !(Subtarget.is64Bit() && VT == MVT::i64))
10873     return SDValue();
10874 
10875   SDLoc DL(N);
10876   SDValue N0 = N->getOperand(0);
10877   SDValue Zero = DAG.getConstant(0, DL, VT);
10878   SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
10879 
10880   // Add (N0 < 0) ? Pow2 - 1 : 0;
10881   SDValue Cmp = DAG.getSetCC(DL, VT, N0, Zero, ISD::SETLT);
10882   SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
10883   SDValue Sel = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
10884 
10885   Created.push_back(Cmp.getNode());
10886   Created.push_back(Add.getNode());
10887   Created.push_back(Sel.getNode());
10888 
10889   // Divide by pow2.
10890   SDValue SRA =
10891       DAG.getNode(ISD::SRA, DL, VT, Sel, DAG.getConstant(Lg2, DL, VT));
10892 
10893   // If we're dividing by a positive value, we're done.  Otherwise, we must
10894   // negate the result.
10895   if (Divisor.isNonNegative())
10896     return SRA;
10897 
10898   Created.push_back(SRA.getNode());
10899   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
10900 }
10901 
10902 #define GET_REGISTER_MATCHER
10903 #include "RISCVGenAsmMatcher.inc"
10904 
10905 Register
10906 RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
10907                                        const MachineFunction &MF) const {
10908   Register Reg = MatchRegisterAltName(RegName);
10909   if (Reg == RISCV::NoRegister)
10910     Reg = MatchRegisterName(RegName);
10911   if (Reg == RISCV::NoRegister)
10912     report_fatal_error(
10913         Twine("Invalid register name \"" + StringRef(RegName) + "\"."));
10914   BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
10915   if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg))
10916     report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
10917                              StringRef(RegName) + "\"."));
10918   return Reg;
10919 }
10920 
10921 namespace llvm {
10922 namespace RISCVVIntrinsicsTable {
10923 
10924 #define GET_RISCVVIntrinsicsTable_IMPL
10925 #include "RISCVGenSearchableTables.inc"
10926 
10927 } // namespace RISCVVIntrinsicsTable
10928 
10929 } // namespace llvm
10930