1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that RISCV uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "RISCVISelLowering.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCV.h"
17 #include "RISCVMachineFunctionInfo.h"
18 #include "RISCVRegisterInfo.h"
19 #include "RISCVSubtarget.h"
20 #include "RISCVTargetMachine.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/MemoryLocation.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineJumpTableInfo.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
30 #include "llvm/CodeGen/ValueTypes.h"
31 #include "llvm/IR/DiagnosticInfo.h"
32 #include "llvm/IR/DiagnosticPrinter.h"
33 #include "llvm/IR/IRBuilder.h"
34 #include "llvm/IR/IntrinsicsRISCV.h"
35 #include "llvm/IR/PatternMatch.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/KnownBits.h"
39 #include "llvm/Support/MathExtras.h"
40 #include "llvm/Support/raw_ostream.h"
41 
42 using namespace llvm;
43 
44 #define DEBUG_TYPE "riscv-lower"
45 
46 STATISTIC(NumTailCalls, "Number of tail calls");
47 
48 RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
49                                          const RISCVSubtarget &STI)
50     : TargetLowering(TM), Subtarget(STI) {
51 
52   if (Subtarget.isRV32E())
53     report_fatal_error("Codegen not yet implemented for RV32E");
54 
55   RISCVABI::ABI ABI = Subtarget.getTargetABI();
56   assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
57 
58   if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
59       !Subtarget.hasStdExtF()) {
60     errs() << "Hard-float 'f' ABI can't be used for a target that "
61                 "doesn't support the F instruction set extension (ignoring "
62                           "target-abi)\n";
63     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
64   } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
65              !Subtarget.hasStdExtD()) {
66     errs() << "Hard-float 'd' ABI can't be used for a target that "
67               "doesn't support the D instruction set extension (ignoring "
68               "target-abi)\n";
69     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
70   }
71 
72   switch (ABI) {
73   default:
74     report_fatal_error("Don't know how to lower this ABI");
75   case RISCVABI::ABI_ILP32:
76   case RISCVABI::ABI_ILP32F:
77   case RISCVABI::ABI_ILP32D:
78   case RISCVABI::ABI_LP64:
79   case RISCVABI::ABI_LP64F:
80   case RISCVABI::ABI_LP64D:
81     break;
82   }
83 
84   MVT XLenVT = Subtarget.getXLenVT();
85 
86   // Set up the register classes.
87   addRegisterClass(XLenVT, &RISCV::GPRRegClass);
88 
89   if (Subtarget.hasStdExtZfh())
90     addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
91   if (Subtarget.hasStdExtF())
92     addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
93   if (Subtarget.hasStdExtD())
94     addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
95 
96   static const MVT::SimpleValueType BoolVecVTs[] = {
97       MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,
98       MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
99   static const MVT::SimpleValueType IntVecVTs[] = {
100       MVT::nxv1i8,  MVT::nxv2i8,   MVT::nxv4i8,   MVT::nxv8i8,  MVT::nxv16i8,
101       MVT::nxv32i8, MVT::nxv64i8,  MVT::nxv1i16,  MVT::nxv2i16, MVT::nxv4i16,
102       MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
103       MVT::nxv4i32, MVT::nxv8i32,  MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
104       MVT::nxv4i64, MVT::nxv8i64};
105   static const MVT::SimpleValueType F16VecVTs[] = {
106       MVT::nxv1f16, MVT::nxv2f16,  MVT::nxv4f16,
107       MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
108   static const MVT::SimpleValueType F32VecVTs[] = {
109       MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
110   static const MVT::SimpleValueType F64VecVTs[] = {
111       MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
112 
113   if (Subtarget.hasVInstructions()) {
114     auto addRegClassForRVV = [this](MVT VT) {
115       unsigned Size = VT.getSizeInBits().getKnownMinValue();
116       assert(Size <= 512 && isPowerOf2_32(Size));
117       const TargetRegisterClass *RC;
118       if (Size <= 64)
119         RC = &RISCV::VRRegClass;
120       else if (Size == 128)
121         RC = &RISCV::VRM2RegClass;
122       else if (Size == 256)
123         RC = &RISCV::VRM4RegClass;
124       else
125         RC = &RISCV::VRM8RegClass;
126 
127       addRegisterClass(VT, RC);
128     };
129 
130     for (MVT VT : BoolVecVTs)
131       addRegClassForRVV(VT);
132     for (MVT VT : IntVecVTs) {
133       if (VT.getVectorElementType() == MVT::i64 &&
134           !Subtarget.hasVInstructionsI64())
135         continue;
136       addRegClassForRVV(VT);
137     }
138 
139     if (Subtarget.hasVInstructionsF16())
140       for (MVT VT : F16VecVTs)
141         addRegClassForRVV(VT);
142 
143     if (Subtarget.hasVInstructionsF32())
144       for (MVT VT : F32VecVTs)
145         addRegClassForRVV(VT);
146 
147     if (Subtarget.hasVInstructionsF64())
148       for (MVT VT : F64VecVTs)
149         addRegClassForRVV(VT);
150 
151     if (Subtarget.useRVVForFixedLengthVectors()) {
152       auto addRegClassForFixedVectors = [this](MVT VT) {
153         MVT ContainerVT = getContainerForFixedLengthVector(VT);
154         unsigned RCID = getRegClassIDForVecVT(ContainerVT);
155         const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
156         addRegisterClass(VT, TRI.getRegClass(RCID));
157       };
158       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
159         if (useRVVForFixedLengthVectorVT(VT))
160           addRegClassForFixedVectors(VT);
161 
162       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
163         if (useRVVForFixedLengthVectorVT(VT))
164           addRegClassForFixedVectors(VT);
165     }
166   }
167 
168   // Compute derived properties from the register classes.
169   computeRegisterProperties(STI.getRegisterInfo());
170 
171   setStackPointerRegisterToSaveRestore(RISCV::X2);
172 
173   for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
174     setLoadExtAction(N, XLenVT, MVT::i1, Promote);
175 
176   // TODO: add all necessary setOperationAction calls.
177   setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
178 
179   setOperationAction(ISD::BR_JT, MVT::Other, Expand);
180   setOperationAction(ISD::BR_CC, XLenVT, Expand);
181   setOperationAction(ISD::BRCOND, MVT::Other, Custom);
182   setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
183 
184   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
185   setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
186 
187   setOperationAction(ISD::VASTART, MVT::Other, Custom);
188   setOperationAction(ISD::VAARG, MVT::Other, Expand);
189   setOperationAction(ISD::VACOPY, MVT::Other, Expand);
190   setOperationAction(ISD::VAEND, MVT::Other, Expand);
191 
192   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
193   if (!Subtarget.hasStdExtZbb()) {
194     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
195     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
196   }
197 
198   if (Subtarget.is64Bit()) {
199     setOperationAction(ISD::ADD, MVT::i32, Custom);
200     setOperationAction(ISD::SUB, MVT::i32, Custom);
201     setOperationAction(ISD::SHL, MVT::i32, Custom);
202     setOperationAction(ISD::SRA, MVT::i32, Custom);
203     setOperationAction(ISD::SRL, MVT::i32, Custom);
204 
205     setOperationAction(ISD::UADDO, MVT::i32, Custom);
206     setOperationAction(ISD::USUBO, MVT::i32, Custom);
207     setOperationAction(ISD::UADDSAT, MVT::i32, Custom);
208     setOperationAction(ISD::USUBSAT, MVT::i32, Custom);
209   } else {
210     setLibcallName(RTLIB::SHL_I128, nullptr);
211     setLibcallName(RTLIB::SRL_I128, nullptr);
212     setLibcallName(RTLIB::SRA_I128, nullptr);
213     setLibcallName(RTLIB::MUL_I128, nullptr);
214     setLibcallName(RTLIB::MULO_I64, nullptr);
215   }
216 
217   if (!Subtarget.hasStdExtM()) {
218     setOperationAction(ISD::MUL, XLenVT, Expand);
219     setOperationAction(ISD::MULHS, XLenVT, Expand);
220     setOperationAction(ISD::MULHU, XLenVT, Expand);
221     setOperationAction(ISD::SDIV, XLenVT, Expand);
222     setOperationAction(ISD::UDIV, XLenVT, Expand);
223     setOperationAction(ISD::SREM, XLenVT, Expand);
224     setOperationAction(ISD::UREM, XLenVT, Expand);
225   } else {
226     if (Subtarget.is64Bit()) {
227       setOperationAction(ISD::MUL, MVT::i32, Custom);
228       setOperationAction(ISD::MUL, MVT::i128, Custom);
229 
230       setOperationAction(ISD::SDIV, MVT::i8, Custom);
231       setOperationAction(ISD::UDIV, MVT::i8, Custom);
232       setOperationAction(ISD::UREM, MVT::i8, Custom);
233       setOperationAction(ISD::SDIV, MVT::i16, Custom);
234       setOperationAction(ISD::UDIV, MVT::i16, Custom);
235       setOperationAction(ISD::UREM, MVT::i16, Custom);
236       setOperationAction(ISD::SDIV, MVT::i32, Custom);
237       setOperationAction(ISD::UDIV, MVT::i32, Custom);
238       setOperationAction(ISD::UREM, MVT::i32, Custom);
239     } else {
240       setOperationAction(ISD::MUL, MVT::i64, Custom);
241     }
242   }
243 
244   setOperationAction(ISD::SDIVREM, XLenVT, Expand);
245   setOperationAction(ISD::UDIVREM, XLenVT, Expand);
246   setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
247   setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
248 
249   setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
250   setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
251   setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
252 
253   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp() ||
254       Subtarget.hasStdExtZbkb()) {
255     if (Subtarget.is64Bit()) {
256       setOperationAction(ISD::ROTL, MVT::i32, Custom);
257       setOperationAction(ISD::ROTR, MVT::i32, Custom);
258     }
259   } else {
260     setOperationAction(ISD::ROTL, XLenVT, Expand);
261     setOperationAction(ISD::ROTR, XLenVT, Expand);
262   }
263 
264   if (Subtarget.hasStdExtZbp()) {
265     // Custom lower bswap/bitreverse so we can convert them to GREVI to enable
266     // more combining.
267     setOperationAction(ISD::BITREVERSE, XLenVT,   Custom);
268     setOperationAction(ISD::BSWAP,      XLenVT,   Custom);
269     setOperationAction(ISD::BITREVERSE, MVT::i8,  Custom);
270     // BSWAP i8 doesn't exist.
271     setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
272     setOperationAction(ISD::BSWAP,      MVT::i16, Custom);
273 
274     if (Subtarget.is64Bit()) {
275       setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
276       setOperationAction(ISD::BSWAP,      MVT::i32, Custom);
277     }
278   } else {
279     // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
280     // pattern match it directly in isel.
281     setOperationAction(ISD::BSWAP, XLenVT,
282                        (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb())
283                            ? Legal
284                            : Expand);
285     // Zbkb can use rev8+brev8 to implement bitreverse.
286     setOperationAction(ISD::BITREVERSE, XLenVT,
287                        Subtarget.hasStdExtZbkb() ? Custom : Expand);
288   }
289 
290   if (Subtarget.hasStdExtZbb()) {
291     setOperationAction(ISD::SMIN, XLenVT, Legal);
292     setOperationAction(ISD::SMAX, XLenVT, Legal);
293     setOperationAction(ISD::UMIN, XLenVT, Legal);
294     setOperationAction(ISD::UMAX, XLenVT, Legal);
295 
296     if (Subtarget.is64Bit()) {
297       setOperationAction(ISD::CTTZ, MVT::i32, Custom);
298       setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom);
299       setOperationAction(ISD::CTLZ, MVT::i32, Custom);
300       setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom);
301     }
302   } else {
303     setOperationAction(ISD::CTTZ, XLenVT, Expand);
304     setOperationAction(ISD::CTLZ, XLenVT, Expand);
305     setOperationAction(ISD::CTPOP, XLenVT, Expand);
306   }
307 
308   if (Subtarget.hasStdExtZbt()) {
309     setOperationAction(ISD::FSHL, XLenVT, Custom);
310     setOperationAction(ISD::FSHR, XLenVT, Custom);
311     setOperationAction(ISD::SELECT, XLenVT, Legal);
312 
313     if (Subtarget.is64Bit()) {
314       setOperationAction(ISD::FSHL, MVT::i32, Custom);
315       setOperationAction(ISD::FSHR, MVT::i32, Custom);
316     }
317   } else {
318     setOperationAction(ISD::SELECT, XLenVT, Custom);
319   }
320 
321   static const ISD::CondCode FPCCToExpand[] = {
322       ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
323       ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
324       ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};
325 
326   static const ISD::NodeType FPOpToExpand[] = {
327       ISD::FSIN, ISD::FCOS,       ISD::FSINCOS,   ISD::FPOW,
328       ISD::FREM, ISD::FP16_TO_FP, ISD::FP_TO_FP16};
329 
330   if (Subtarget.hasStdExtZfh())
331     setOperationAction(ISD::BITCAST, MVT::i16, Custom);
332 
333   if (Subtarget.hasStdExtZfh()) {
334     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
335     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
336     setOperationAction(ISD::LRINT, MVT::f16, Legal);
337     setOperationAction(ISD::LLRINT, MVT::f16, Legal);
338     setOperationAction(ISD::LROUND, MVT::f16, Legal);
339     setOperationAction(ISD::LLROUND, MVT::f16, Legal);
340     setOperationAction(ISD::STRICT_LRINT, MVT::f16, Legal);
341     setOperationAction(ISD::STRICT_LLRINT, MVT::f16, Legal);
342     setOperationAction(ISD::STRICT_LROUND, MVT::f16, Legal);
343     setOperationAction(ISD::STRICT_LLROUND, MVT::f16, Legal);
344     setOperationAction(ISD::STRICT_FADD, MVT::f16, Legal);
345     setOperationAction(ISD::STRICT_FMA, MVT::f16, Legal);
346     setOperationAction(ISD::STRICT_FSUB, MVT::f16, Legal);
347     setOperationAction(ISD::STRICT_FMUL, MVT::f16, Legal);
348     setOperationAction(ISD::STRICT_FDIV, MVT::f16, Legal);
349     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
350     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
351     setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Legal);
352     setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Legal);
353     setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Legal);
354     for (auto CC : FPCCToExpand)
355       setCondCodeAction(CC, MVT::f16, Expand);
356     setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
357     setOperationAction(ISD::SELECT, MVT::f16, Custom);
358     setOperationAction(ISD::BR_CC, MVT::f16, Expand);
359 
360     setOperationAction(ISD::FREM,       MVT::f16, Promote);
361     setOperationAction(ISD::FCEIL,      MVT::f16, Promote);
362     setOperationAction(ISD::FFLOOR,     MVT::f16, Promote);
363     setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote);
364     setOperationAction(ISD::FRINT,      MVT::f16, Promote);
365     setOperationAction(ISD::FROUND,     MVT::f16, Promote);
366     setOperationAction(ISD::FROUNDEVEN, MVT::f16, Promote);
367     setOperationAction(ISD::FTRUNC,     MVT::f16, Promote);
368     setOperationAction(ISD::FPOW,       MVT::f16, Promote);
369     setOperationAction(ISD::FPOWI,      MVT::f16, Promote);
370     setOperationAction(ISD::FCOS,       MVT::f16, Promote);
371     setOperationAction(ISD::FSIN,       MVT::f16, Promote);
372     setOperationAction(ISD::FSINCOS,    MVT::f16, Promote);
373     setOperationAction(ISD::FEXP,       MVT::f16, Promote);
374     setOperationAction(ISD::FEXP2,      MVT::f16, Promote);
375     setOperationAction(ISD::FLOG,       MVT::f16, Promote);
376     setOperationAction(ISD::FLOG2,      MVT::f16, Promote);
377     setOperationAction(ISD::FLOG10,     MVT::f16, Promote);
378 
379     // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
380     // complete support for all operations in LegalizeDAG.
381 
382     // We need to custom promote this.
383     if (Subtarget.is64Bit())
384       setOperationAction(ISD::FPOWI, MVT::i32, Custom);
385   }
386 
387   if (Subtarget.hasStdExtF()) {
388     setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
389     setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
390     setOperationAction(ISD::LRINT, MVT::f32, Legal);
391     setOperationAction(ISD::LLRINT, MVT::f32, Legal);
392     setOperationAction(ISD::LROUND, MVT::f32, Legal);
393     setOperationAction(ISD::LLROUND, MVT::f32, Legal);
394     setOperationAction(ISD::STRICT_LRINT, MVT::f32, Legal);
395     setOperationAction(ISD::STRICT_LLRINT, MVT::f32, Legal);
396     setOperationAction(ISD::STRICT_LROUND, MVT::f32, Legal);
397     setOperationAction(ISD::STRICT_LLROUND, MVT::f32, Legal);
398     setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
399     setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal);
400     setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
401     setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
402     setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
403     setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
404     setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);
405     setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
406     for (auto CC : FPCCToExpand)
407       setCondCodeAction(CC, MVT::f32, Expand);
408     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
409     setOperationAction(ISD::SELECT, MVT::f32, Custom);
410     setOperationAction(ISD::BR_CC, MVT::f32, Expand);
411     for (auto Op : FPOpToExpand)
412       setOperationAction(Op, MVT::f32, Expand);
413     setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
414     setTruncStoreAction(MVT::f32, MVT::f16, Expand);
415   }
416 
417   if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
418     setOperationAction(ISD::BITCAST, MVT::i32, Custom);
419 
420   if (Subtarget.hasStdExtD()) {
421     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
422     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
423     setOperationAction(ISD::LRINT, MVT::f64, Legal);
424     setOperationAction(ISD::LLRINT, MVT::f64, Legal);
425     setOperationAction(ISD::LROUND, MVT::f64, Legal);
426     setOperationAction(ISD::LLROUND, MVT::f64, Legal);
427     setOperationAction(ISD::STRICT_LRINT, MVT::f64, Legal);
428     setOperationAction(ISD::STRICT_LLRINT, MVT::f64, Legal);
429     setOperationAction(ISD::STRICT_LROUND, MVT::f64, Legal);
430     setOperationAction(ISD::STRICT_LLROUND, MVT::f64, Legal);
431     setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal);
432     setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
433     setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
434     setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
435     setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
436     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
437     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
438     setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
439     setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal);
440     setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal);
441     for (auto CC : FPCCToExpand)
442       setCondCodeAction(CC, MVT::f64, Expand);
443     setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
444     setOperationAction(ISD::SELECT, MVT::f64, Custom);
445     setOperationAction(ISD::BR_CC, MVT::f64, Expand);
446     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
447     setTruncStoreAction(MVT::f64, MVT::f32, Expand);
448     for (auto Op : FPOpToExpand)
449       setOperationAction(Op, MVT::f64, Expand);
450     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
451     setTruncStoreAction(MVT::f64, MVT::f16, Expand);
452   }
453 
454   if (Subtarget.is64Bit()) {
455     setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
456     setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
457     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
458     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
459   }
460 
461   if (Subtarget.hasStdExtF()) {
462     setOperationAction(ISD::FP_TO_UINT_SAT, XLenVT, Custom);
463     setOperationAction(ISD::FP_TO_SINT_SAT, XLenVT, Custom);
464 
465     setOperationAction(ISD::STRICT_FP_TO_UINT, XLenVT, Legal);
466     setOperationAction(ISD::STRICT_FP_TO_SINT, XLenVT, Legal);
467     setOperationAction(ISD::STRICT_UINT_TO_FP, XLenVT, Legal);
468     setOperationAction(ISD::STRICT_SINT_TO_FP, XLenVT, Legal);
469 
470     setOperationAction(ISD::FLT_ROUNDS_, XLenVT, Custom);
471     setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
472   }
473 
474   setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
475   setOperationAction(ISD::BlockAddress, XLenVT, Custom);
476   setOperationAction(ISD::ConstantPool, XLenVT, Custom);
477   setOperationAction(ISD::JumpTable, XLenVT, Custom);
478 
479   setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
480 
481   // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
482   // Unfortunately this can't be determined just from the ISA naming string.
483   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
484                      Subtarget.is64Bit() ? Legal : Custom);
485 
486   setOperationAction(ISD::TRAP, MVT::Other, Legal);
487   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
488   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
489   if (Subtarget.is64Bit())
490     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
491 
492   if (Subtarget.hasStdExtA()) {
493     setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
494     setMinCmpXchgSizeInBits(32);
495   } else {
496     setMaxAtomicSizeInBitsSupported(0);
497   }
498 
499   setBooleanContents(ZeroOrOneBooleanContent);
500 
501   if (Subtarget.hasVInstructions()) {
502     setBooleanVectorContents(ZeroOrOneBooleanContent);
503 
504     setOperationAction(ISD::VSCALE, XLenVT, Custom);
505 
506     // RVV intrinsics may have illegal operands.
507     // We also need to custom legalize vmv.x.s.
508     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
509     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
510     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
511     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom);
512     if (Subtarget.is64Bit()) {
513       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);
514     } else {
515       setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
516       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
517     }
518 
519     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
520     setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
521 
522     static const unsigned IntegerVPOps[] = {
523         ISD::VP_ADD,         ISD::VP_SUB,         ISD::VP_MUL,
524         ISD::VP_SDIV,        ISD::VP_UDIV,        ISD::VP_SREM,
525         ISD::VP_UREM,        ISD::VP_AND,         ISD::VP_OR,
526         ISD::VP_XOR,         ISD::VP_ASHR,        ISD::VP_LSHR,
527         ISD::VP_SHL,         ISD::VP_REDUCE_ADD,  ISD::VP_REDUCE_AND,
528         ISD::VP_REDUCE_OR,   ISD::VP_REDUCE_XOR,  ISD::VP_REDUCE_SMAX,
529         ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
530         ISD::VP_MERGE,       ISD::VP_SELECT};
531 
532     static const unsigned FloatingPointVPOps[] = {
533         ISD::VP_FADD,        ISD::VP_FSUB,        ISD::VP_FMUL,
534         ISD::VP_FDIV,        ISD::VP_FNEG,        ISD::VP_FMA,
535         ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD, ISD::VP_REDUCE_FMIN,
536         ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,       ISD::VP_SELECT};
537 
538     if (!Subtarget.is64Bit()) {
539       // We must custom-lower certain vXi64 operations on RV32 due to the vector
540       // element type being illegal.
541       setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::i64, Custom);
542       setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::i64, Custom);
543 
544       setOperationAction(ISD::VECREDUCE_ADD, MVT::i64, Custom);
545       setOperationAction(ISD::VECREDUCE_AND, MVT::i64, Custom);
546       setOperationAction(ISD::VECREDUCE_OR, MVT::i64, Custom);
547       setOperationAction(ISD::VECREDUCE_XOR, MVT::i64, Custom);
548       setOperationAction(ISD::VECREDUCE_SMAX, MVT::i64, Custom);
549       setOperationAction(ISD::VECREDUCE_SMIN, MVT::i64, Custom);
550       setOperationAction(ISD::VECREDUCE_UMAX, MVT::i64, Custom);
551       setOperationAction(ISD::VECREDUCE_UMIN, MVT::i64, Custom);
552 
553       setOperationAction(ISD::VP_REDUCE_ADD, MVT::i64, Custom);
554       setOperationAction(ISD::VP_REDUCE_AND, MVT::i64, Custom);
555       setOperationAction(ISD::VP_REDUCE_OR, MVT::i64, Custom);
556       setOperationAction(ISD::VP_REDUCE_XOR, MVT::i64, Custom);
557       setOperationAction(ISD::VP_REDUCE_SMAX, MVT::i64, Custom);
558       setOperationAction(ISD::VP_REDUCE_SMIN, MVT::i64, Custom);
559       setOperationAction(ISD::VP_REDUCE_UMAX, MVT::i64, Custom);
560       setOperationAction(ISD::VP_REDUCE_UMIN, MVT::i64, Custom);
561     }
562 
563     for (MVT VT : BoolVecVTs) {
564       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
565 
566       // Mask VTs are custom-expanded into a series of standard nodes
567       setOperationAction(ISD::TRUNCATE, VT, Custom);
568       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
569       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
570       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
571 
572       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
573       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
574 
575       setOperationAction(ISD::SELECT, VT, Custom);
576       setOperationAction(ISD::SELECT_CC, VT, Expand);
577       setOperationAction(ISD::VSELECT, VT, Expand);
578       setOperationAction(ISD::VP_MERGE, VT, Expand);
579       setOperationAction(ISD::VP_SELECT, VT, Expand);
580 
581       setOperationAction(ISD::VP_AND, VT, Custom);
582       setOperationAction(ISD::VP_OR, VT, Custom);
583       setOperationAction(ISD::VP_XOR, VT, Custom);
584 
585       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
586       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
587       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
588 
589       setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
590       setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
591       setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
592 
593       // RVV has native int->float & float->int conversions where the
594       // element type sizes are within one power-of-two of each other. Any
595       // wider distances between type sizes have to be lowered as sequences
596       // which progressively narrow the gap in stages.
597       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
598       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
599       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
600       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
601 
602       // Expand all extending loads to types larger than this, and truncating
603       // stores from types larger than this.
604       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
605         setTruncStoreAction(OtherVT, VT, Expand);
606         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
607         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
608         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
609       }
610     }
611 
612     for (MVT VT : IntVecVTs) {
613       if (VT.getVectorElementType() == MVT::i64 &&
614           !Subtarget.hasVInstructionsI64())
615         continue;
616 
617       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
618       setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
619 
620       // Vectors implement MULHS/MULHU.
621       setOperationAction(ISD::SMUL_LOHI, VT, Expand);
622       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
623 
624       // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
625       if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV()) {
626         setOperationAction(ISD::MULHU, VT, Expand);
627         setOperationAction(ISD::MULHS, VT, Expand);
628       }
629 
630       setOperationAction(ISD::SMIN, VT, Legal);
631       setOperationAction(ISD::SMAX, VT, Legal);
632       setOperationAction(ISD::UMIN, VT, Legal);
633       setOperationAction(ISD::UMAX, VT, Legal);
634 
635       setOperationAction(ISD::ROTL, VT, Expand);
636       setOperationAction(ISD::ROTR, VT, Expand);
637 
638       setOperationAction(ISD::CTTZ, VT, Expand);
639       setOperationAction(ISD::CTLZ, VT, Expand);
640       setOperationAction(ISD::CTPOP, VT, Expand);
641 
642       setOperationAction(ISD::BSWAP, VT, Expand);
643 
644       // Custom-lower extensions and truncations from/to mask types.
645       setOperationAction(ISD::ANY_EXTEND, VT, Custom);
646       setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
647       setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
648 
649       // RVV has native int->float & float->int conversions where the
650       // element type sizes are within one power-of-two of each other. Any
651       // wider distances between type sizes have to be lowered as sequences
652       // which progressively narrow the gap in stages.
653       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
654       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
655       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
656       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
657 
658       setOperationAction(ISD::SADDSAT, VT, Legal);
659       setOperationAction(ISD::UADDSAT, VT, Legal);
660       setOperationAction(ISD::SSUBSAT, VT, Legal);
661       setOperationAction(ISD::USUBSAT, VT, Legal);
662 
663       // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
664       // nodes which truncate by one power of two at a time.
665       setOperationAction(ISD::TRUNCATE, VT, Custom);
666 
667       // Custom-lower insert/extract operations to simplify patterns.
668       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
669       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
670 
671       // Custom-lower reduction operations to set up the corresponding custom
672       // nodes' operands.
673       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
674       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
675       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
676       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
677       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
678       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
679       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
680       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
681 
682       for (unsigned VPOpc : IntegerVPOps)
683         setOperationAction(VPOpc, VT, Custom);
684 
685       setOperationAction(ISD::LOAD, VT, Custom);
686       setOperationAction(ISD::STORE, VT, Custom);
687 
688       setOperationAction(ISD::MLOAD, VT, Custom);
689       setOperationAction(ISD::MSTORE, VT, Custom);
690       setOperationAction(ISD::MGATHER, VT, Custom);
691       setOperationAction(ISD::MSCATTER, VT, Custom);
692 
693       setOperationAction(ISD::VP_LOAD, VT, Custom);
694       setOperationAction(ISD::VP_STORE, VT, Custom);
695       setOperationAction(ISD::VP_GATHER, VT, Custom);
696       setOperationAction(ISD::VP_SCATTER, VT, Custom);
697 
698       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
699       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
700       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
701 
702       setOperationAction(ISD::SELECT, VT, Custom);
703       setOperationAction(ISD::SELECT_CC, VT, Expand);
704 
705       setOperationAction(ISD::STEP_VECTOR, VT, Custom);
706       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
707 
708       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
709         setTruncStoreAction(VT, OtherVT, Expand);
710         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
711         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
712         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
713       }
714 
715       // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
716       // type that can represent the value exactly.
717       if (VT.getVectorElementType() != MVT::i64) {
718         MVT FloatEltVT =
719             VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
720         EVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
721         if (isTypeLegal(FloatVT)) {
722           setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
723           setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
724         }
725       }
726     }
727 
728     // Expand various CCs to best match the RVV ISA, which natively supports UNE
729     // but no other unordered comparisons, and supports all ordered comparisons
730     // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
731     // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
732     // and we pattern-match those back to the "original", swapping operands once
733     // more. This way we catch both operations and both "vf" and "fv" forms with
734     // fewer patterns.
735     static const ISD::CondCode VFPCCToExpand[] = {
736         ISD::SETO,   ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
737         ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
738         ISD::SETGT,  ISD::SETOGT, ISD::SETGE,  ISD::SETOGE,
739     };
740 
741     // Sets common operation actions on RVV floating-point vector types.
742     const auto SetCommonVFPActions = [&](MVT VT) {
743       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
744       // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
745       // sizes are within one power-of-two of each other. Therefore conversions
746       // between vXf16 and vXf64 must be lowered as sequences which convert via
747       // vXf32.
748       setOperationAction(ISD::FP_ROUND, VT, Custom);
749       setOperationAction(ISD::FP_EXTEND, VT, Custom);
750       // Custom-lower insert/extract operations to simplify patterns.
751       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
752       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
753       // Expand various condition codes (explained above).
754       for (auto CC : VFPCCToExpand)
755         setCondCodeAction(CC, VT, Expand);
756 
757       setOperationAction(ISD::FMINNUM, VT, Legal);
758       setOperationAction(ISD::FMAXNUM, VT, Legal);
759 
760       setOperationAction(ISD::FTRUNC, VT, Custom);
761       setOperationAction(ISD::FCEIL, VT, Custom);
762       setOperationAction(ISD::FFLOOR, VT, Custom);
763       setOperationAction(ISD::FROUND, VT, Custom);
764 
765       setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
766       setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
767       setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
768       setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
769 
770       setOperationAction(ISD::FCOPYSIGN, VT, Legal);
771 
772       setOperationAction(ISD::LOAD, VT, Custom);
773       setOperationAction(ISD::STORE, VT, Custom);
774 
775       setOperationAction(ISD::MLOAD, VT, Custom);
776       setOperationAction(ISD::MSTORE, VT, Custom);
777       setOperationAction(ISD::MGATHER, VT, Custom);
778       setOperationAction(ISD::MSCATTER, VT, Custom);
779 
780       setOperationAction(ISD::VP_LOAD, VT, Custom);
781       setOperationAction(ISD::VP_STORE, VT, Custom);
782       setOperationAction(ISD::VP_GATHER, VT, Custom);
783       setOperationAction(ISD::VP_SCATTER, VT, Custom);
784 
785       setOperationAction(ISD::SELECT, VT, Custom);
786       setOperationAction(ISD::SELECT_CC, VT, Expand);
787 
788       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
789       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
790       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
791 
792       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
793 
794       for (unsigned VPOpc : FloatingPointVPOps)
795         setOperationAction(VPOpc, VT, Custom);
796     };
797 
798     // Sets common extload/truncstore actions on RVV floating-point vector
799     // types.
800     const auto SetCommonVFPExtLoadTruncStoreActions =
801         [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
802           for (auto SmallVT : SmallerVTs) {
803             setTruncStoreAction(VT, SmallVT, Expand);
804             setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
805           }
806         };
807 
808     if (Subtarget.hasVInstructionsF16())
809       for (MVT VT : F16VecVTs)
810         SetCommonVFPActions(VT);
811 
812     for (MVT VT : F32VecVTs) {
813       if (Subtarget.hasVInstructionsF32())
814         SetCommonVFPActions(VT);
815       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
816     }
817 
818     for (MVT VT : F64VecVTs) {
819       if (Subtarget.hasVInstructionsF64())
820         SetCommonVFPActions(VT);
821       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
822       SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
823     }
824 
825     if (Subtarget.useRVVForFixedLengthVectors()) {
826       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
827         if (!useRVVForFixedLengthVectorVT(VT))
828           continue;
829 
830         // By default everything must be expanded.
831         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
832           setOperationAction(Op, VT, Expand);
833         for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
834           setTruncStoreAction(VT, OtherVT, Expand);
835           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
836           setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
837           setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
838         }
839 
840         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
841         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
842         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
843 
844         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
845         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
846 
847         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
848         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
849 
850         setOperationAction(ISD::LOAD, VT, Custom);
851         setOperationAction(ISD::STORE, VT, Custom);
852 
853         setOperationAction(ISD::SETCC, VT, Custom);
854 
855         setOperationAction(ISD::SELECT, VT, Custom);
856 
857         setOperationAction(ISD::TRUNCATE, VT, Custom);
858 
859         setOperationAction(ISD::BITCAST, VT, Custom);
860 
861         setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
862         setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
863         setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
864 
865         setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
866         setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
867         setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
868 
869         setOperationAction(ISD::SINT_TO_FP, VT, Custom);
870         setOperationAction(ISD::UINT_TO_FP, VT, Custom);
871         setOperationAction(ISD::FP_TO_SINT, VT, Custom);
872         setOperationAction(ISD::FP_TO_UINT, VT, Custom);
873 
874         // Operations below are different for between masks and other vectors.
875         if (VT.getVectorElementType() == MVT::i1) {
876           setOperationAction(ISD::VP_AND, VT, Custom);
877           setOperationAction(ISD::VP_OR, VT, Custom);
878           setOperationAction(ISD::VP_XOR, VT, Custom);
879           setOperationAction(ISD::AND, VT, Custom);
880           setOperationAction(ISD::OR, VT, Custom);
881           setOperationAction(ISD::XOR, VT, Custom);
882           continue;
883         }
884 
885         // Use SPLAT_VECTOR to prevent type legalization from destroying the
886         // splats when type legalizing i64 scalar on RV32.
887         // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
888         // improvements first.
889         if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
890           setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
891           setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
892         }
893 
894         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
895         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
896 
897         setOperationAction(ISD::MLOAD, VT, Custom);
898         setOperationAction(ISD::MSTORE, VT, Custom);
899         setOperationAction(ISD::MGATHER, VT, Custom);
900         setOperationAction(ISD::MSCATTER, VT, Custom);
901 
902         setOperationAction(ISD::VP_LOAD, VT, Custom);
903         setOperationAction(ISD::VP_STORE, VT, Custom);
904         setOperationAction(ISD::VP_GATHER, VT, Custom);
905         setOperationAction(ISD::VP_SCATTER, VT, Custom);
906 
907         setOperationAction(ISD::ADD, VT, Custom);
908         setOperationAction(ISD::MUL, VT, Custom);
909         setOperationAction(ISD::SUB, VT, Custom);
910         setOperationAction(ISD::AND, VT, Custom);
911         setOperationAction(ISD::OR, VT, Custom);
912         setOperationAction(ISD::XOR, VT, Custom);
913         setOperationAction(ISD::SDIV, VT, Custom);
914         setOperationAction(ISD::SREM, VT, Custom);
915         setOperationAction(ISD::UDIV, VT, Custom);
916         setOperationAction(ISD::UREM, VT, Custom);
917         setOperationAction(ISD::SHL, VT, Custom);
918         setOperationAction(ISD::SRA, VT, Custom);
919         setOperationAction(ISD::SRL, VT, Custom);
920 
921         setOperationAction(ISD::SMIN, VT, Custom);
922         setOperationAction(ISD::SMAX, VT, Custom);
923         setOperationAction(ISD::UMIN, VT, Custom);
924         setOperationAction(ISD::UMAX, VT, Custom);
925         setOperationAction(ISD::ABS,  VT, Custom);
926 
927         // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
928         if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV()) {
929           setOperationAction(ISD::MULHS, VT, Custom);
930           setOperationAction(ISD::MULHU, VT, Custom);
931         }
932 
933         setOperationAction(ISD::SADDSAT, VT, Custom);
934         setOperationAction(ISD::UADDSAT, VT, Custom);
935         setOperationAction(ISD::SSUBSAT, VT, Custom);
936         setOperationAction(ISD::USUBSAT, VT, Custom);
937 
938         setOperationAction(ISD::VSELECT, VT, Custom);
939         setOperationAction(ISD::SELECT_CC, VT, Expand);
940 
941         setOperationAction(ISD::ANY_EXTEND, VT, Custom);
942         setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
943         setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
944 
945         // Custom-lower reduction operations to set up the corresponding custom
946         // nodes' operands.
947         setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
948         setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
949         setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
950         setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
951         setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
952 
953         for (unsigned VPOpc : IntegerVPOps)
954           setOperationAction(VPOpc, VT, Custom);
955 
956         // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
957         // type that can represent the value exactly.
958         if (VT.getVectorElementType() != MVT::i64) {
959           MVT FloatEltVT =
960               VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
961           EVT FloatVT =
962               MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
963           if (isTypeLegal(FloatVT)) {
964             setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
965             setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
966           }
967         }
968       }
969 
970       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
971         if (!useRVVForFixedLengthVectorVT(VT))
972           continue;
973 
974         // By default everything must be expanded.
975         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
976           setOperationAction(Op, VT, Expand);
977         for (MVT OtherVT : MVT::fp_fixedlen_vector_valuetypes()) {
978           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
979           setTruncStoreAction(VT, OtherVT, Expand);
980         }
981 
982         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
983         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
984         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
985 
986         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
987         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
988         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
989         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
990         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
991 
992         setOperationAction(ISD::LOAD, VT, Custom);
993         setOperationAction(ISD::STORE, VT, Custom);
994         setOperationAction(ISD::MLOAD, VT, Custom);
995         setOperationAction(ISD::MSTORE, VT, Custom);
996         setOperationAction(ISD::MGATHER, VT, Custom);
997         setOperationAction(ISD::MSCATTER, VT, Custom);
998 
999         setOperationAction(ISD::VP_LOAD, VT, Custom);
1000         setOperationAction(ISD::VP_STORE, VT, Custom);
1001         setOperationAction(ISD::VP_GATHER, VT, Custom);
1002         setOperationAction(ISD::VP_SCATTER, VT, Custom);
1003 
1004         setOperationAction(ISD::FADD, VT, Custom);
1005         setOperationAction(ISD::FSUB, VT, Custom);
1006         setOperationAction(ISD::FMUL, VT, Custom);
1007         setOperationAction(ISD::FDIV, VT, Custom);
1008         setOperationAction(ISD::FNEG, VT, Custom);
1009         setOperationAction(ISD::FABS, VT, Custom);
1010         setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1011         setOperationAction(ISD::FSQRT, VT, Custom);
1012         setOperationAction(ISD::FMA, VT, Custom);
1013         setOperationAction(ISD::FMINNUM, VT, Custom);
1014         setOperationAction(ISD::FMAXNUM, VT, Custom);
1015 
1016         setOperationAction(ISD::FP_ROUND, VT, Custom);
1017         setOperationAction(ISD::FP_EXTEND, VT, Custom);
1018 
1019         setOperationAction(ISD::FTRUNC, VT, Custom);
1020         setOperationAction(ISD::FCEIL, VT, Custom);
1021         setOperationAction(ISD::FFLOOR, VT, Custom);
1022         setOperationAction(ISD::FROUND, VT, Custom);
1023 
1024         for (auto CC : VFPCCToExpand)
1025           setCondCodeAction(CC, VT, Expand);
1026 
1027         setOperationAction(ISD::VSELECT, VT, Custom);
1028         setOperationAction(ISD::SELECT, VT, Custom);
1029         setOperationAction(ISD::SELECT_CC, VT, Expand);
1030 
1031         setOperationAction(ISD::BITCAST, VT, Custom);
1032 
1033         setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1034         setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1035         setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1036         setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1037 
1038         for (unsigned VPOpc : FloatingPointVPOps)
1039           setOperationAction(VPOpc, VT, Custom);
1040       }
1041 
1042       // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1043       setOperationAction(ISD::BITCAST, MVT::i8, Custom);
1044       setOperationAction(ISD::BITCAST, MVT::i16, Custom);
1045       setOperationAction(ISD::BITCAST, MVT::i32, Custom);
1046       setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1047       if (Subtarget.hasStdExtZfh())
1048         setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1049       if (Subtarget.hasStdExtF())
1050         setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1051       if (Subtarget.hasStdExtD())
1052         setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1053     }
1054   }
1055 
1056   // Function alignments.
1057   const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
1058   setMinFunctionAlignment(FunctionAlignment);
1059   setPrefFunctionAlignment(FunctionAlignment);
1060 
1061   setMinimumJumpTableEntries(5);
1062 
1063   // Jumps are expensive, compared to logic
1064   setJumpIsExpensive();
1065 
1066   setTargetDAGCombine(ISD::ADD);
1067   setTargetDAGCombine(ISD::SUB);
1068   setTargetDAGCombine(ISD::AND);
1069   setTargetDAGCombine(ISD::OR);
1070   setTargetDAGCombine(ISD::XOR);
1071   setTargetDAGCombine(ISD::ANY_EXTEND);
1072   if (Subtarget.hasStdExtF()) {
1073     setTargetDAGCombine(ISD::ZERO_EXTEND);
1074     setTargetDAGCombine(ISD::FP_TO_SINT);
1075     setTargetDAGCombine(ISD::FP_TO_UINT);
1076     setTargetDAGCombine(ISD::FP_TO_SINT_SAT);
1077     setTargetDAGCombine(ISD::FP_TO_UINT_SAT);
1078   }
1079   if (Subtarget.hasVInstructions()) {
1080     setTargetDAGCombine(ISD::FCOPYSIGN);
1081     setTargetDAGCombine(ISD::MGATHER);
1082     setTargetDAGCombine(ISD::MSCATTER);
1083     setTargetDAGCombine(ISD::VP_GATHER);
1084     setTargetDAGCombine(ISD::VP_SCATTER);
1085     setTargetDAGCombine(ISD::SRA);
1086     setTargetDAGCombine(ISD::SRL);
1087     setTargetDAGCombine(ISD::SHL);
1088     setTargetDAGCombine(ISD::STORE);
1089     setTargetDAGCombine(ISD::SPLAT_VECTOR);
1090   }
1091 
1092   setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
1093   setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
1094 }
1095 
1096 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
1097                                             LLVMContext &Context,
1098                                             EVT VT) const {
1099   if (!VT.isVector())
1100     return getPointerTy(DL);
1101   if (Subtarget.hasVInstructions() &&
1102       (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
1103     return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
1104   return VT.changeVectorElementTypeToInteger();
1105 }
1106 
1107 MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
1108   return Subtarget.getXLenVT();
1109 }
1110 
1111 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1112                                              const CallInst &I,
1113                                              MachineFunction &MF,
1114                                              unsigned Intrinsic) const {
1115   auto &DL = I.getModule()->getDataLayout();
1116   switch (Intrinsic) {
1117   default:
1118     return false;
1119   case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1120   case Intrinsic::riscv_masked_atomicrmw_add_i32:
1121   case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1122   case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1123   case Intrinsic::riscv_masked_atomicrmw_max_i32:
1124   case Intrinsic::riscv_masked_atomicrmw_min_i32:
1125   case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1126   case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1127   case Intrinsic::riscv_masked_cmpxchg_i32:
1128     Info.opc = ISD::INTRINSIC_W_CHAIN;
1129     Info.memVT = MVT::i32;
1130     Info.ptrVal = I.getArgOperand(0);
1131     Info.offset = 0;
1132     Info.align = Align(4);
1133     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1134                  MachineMemOperand::MOVolatile;
1135     return true;
1136   case Intrinsic::riscv_masked_strided_load:
1137     Info.opc = ISD::INTRINSIC_W_CHAIN;
1138     Info.ptrVal = I.getArgOperand(1);
1139     Info.memVT = getValueType(DL, I.getType()->getScalarType());
1140     Info.align = Align(DL.getTypeSizeInBits(I.getType()->getScalarType()) / 8);
1141     Info.size = MemoryLocation::UnknownSize;
1142     Info.flags |= MachineMemOperand::MOLoad;
1143     return true;
1144   case Intrinsic::riscv_masked_strided_store:
1145     Info.opc = ISD::INTRINSIC_VOID;
1146     Info.ptrVal = I.getArgOperand(1);
1147     Info.memVT =
1148         getValueType(DL, I.getArgOperand(0)->getType()->getScalarType());
1149     Info.align = Align(
1150         DL.getTypeSizeInBits(I.getArgOperand(0)->getType()->getScalarType()) /
1151         8);
1152     Info.size = MemoryLocation::UnknownSize;
1153     Info.flags |= MachineMemOperand::MOStore;
1154     return true;
1155   }
1156 }
1157 
1158 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1159                                                 const AddrMode &AM, Type *Ty,
1160                                                 unsigned AS,
1161                                                 Instruction *I) const {
1162   // No global is ever allowed as a base.
1163   if (AM.BaseGV)
1164     return false;
1165 
1166   // Require a 12-bit signed offset.
1167   if (!isInt<12>(AM.BaseOffs))
1168     return false;
1169 
1170   switch (AM.Scale) {
1171   case 0: // "r+i" or just "i", depending on HasBaseReg.
1172     break;
1173   case 1:
1174     if (!AM.HasBaseReg) // allow "r+i".
1175       break;
1176     return false; // disallow "r+r" or "r+r+i".
1177   default:
1178     return false;
1179   }
1180 
1181   return true;
1182 }
1183 
1184 bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1185   return isInt<12>(Imm);
1186 }
1187 
1188 bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1189   return isInt<12>(Imm);
1190 }
1191 
1192 // On RV32, 64-bit integers are split into their high and low parts and held
1193 // in two different registers, so the trunc is free since the low register can
1194 // just be used.
1195 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
1196   if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
1197     return false;
1198   unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1199   unsigned DestBits = DstTy->getPrimitiveSizeInBits();
1200   return (SrcBits == 64 && DestBits == 32);
1201 }
1202 
1203 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
1204   if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
1205       !SrcVT.isInteger() || !DstVT.isInteger())
1206     return false;
1207   unsigned SrcBits = SrcVT.getSizeInBits();
1208   unsigned DestBits = DstVT.getSizeInBits();
1209   return (SrcBits == 64 && DestBits == 32);
1210 }
1211 
1212 bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
1213   // Zexts are free if they can be combined with a load.
1214   // Don't advertise i32->i64 zextload as being free for RV64. It interacts
1215   // poorly with type legalization of compares preferring sext.
1216   if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
1217     EVT MemVT = LD->getMemoryVT();
1218     if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1219         (LD->getExtensionType() == ISD::NON_EXTLOAD ||
1220          LD->getExtensionType() == ISD::ZEXTLOAD))
1221       return true;
1222   }
1223 
1224   return TargetLowering::isZExtFree(Val, VT2);
1225 }
1226 
1227 bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
1228   return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1229 }
1230 
1231 bool RISCVTargetLowering::isCheapToSpeculateCttz() const {
1232   return Subtarget.hasStdExtZbb();
1233 }
1234 
1235 bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
1236   return Subtarget.hasStdExtZbb();
1237 }
1238 
1239 bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
1240   EVT VT = Y.getValueType();
1241 
1242   // FIXME: Support vectors once we have tests.
1243   if (VT.isVector())
1244     return false;
1245 
1246   return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp() ||
1247           Subtarget.hasStdExtZbkb()) &&
1248          !isa<ConstantSDNode>(Y);
1249 }
1250 
1251 /// Check if sinking \p I's operands to I's basic block is profitable, because
1252 /// the operands can be folded into a target instruction, e.g.
1253 /// splats of scalars can fold into vector instructions.
1254 bool RISCVTargetLowering::shouldSinkOperands(
1255     Instruction *I, SmallVectorImpl<Use *> &Ops) const {
1256   using namespace llvm::PatternMatch;
1257 
1258   if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
1259     return false;
1260 
1261   auto IsSinker = [&](Instruction *I, int Operand) {
1262     switch (I->getOpcode()) {
1263     case Instruction::Add:
1264     case Instruction::Sub:
1265     case Instruction::Mul:
1266     case Instruction::And:
1267     case Instruction::Or:
1268     case Instruction::Xor:
1269     case Instruction::FAdd:
1270     case Instruction::FSub:
1271     case Instruction::FMul:
1272     case Instruction::FDiv:
1273     case Instruction::ICmp:
1274     case Instruction::FCmp:
1275       return true;
1276     case Instruction::Shl:
1277     case Instruction::LShr:
1278     case Instruction::AShr:
1279     case Instruction::UDiv:
1280     case Instruction::SDiv:
1281     case Instruction::URem:
1282     case Instruction::SRem:
1283       return Operand == 1;
1284     case Instruction::Call:
1285       if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1286         switch (II->getIntrinsicID()) {
1287         case Intrinsic::fma:
1288           return Operand == 0 || Operand == 1;
1289         // FIXME: Our patterns can only match vx/vf instructions when the splat
1290         // it on the RHS, because TableGen doesn't recognize our VP operations
1291         // as commutative.
1292         case Intrinsic::vp_add:
1293         case Intrinsic::vp_mul:
1294         case Intrinsic::vp_and:
1295         case Intrinsic::vp_or:
1296         case Intrinsic::vp_xor:
1297         case Intrinsic::vp_fadd:
1298         case Intrinsic::vp_fmul:
1299         case Intrinsic::vp_shl:
1300         case Intrinsic::vp_lshr:
1301         case Intrinsic::vp_ashr:
1302         case Intrinsic::vp_udiv:
1303         case Intrinsic::vp_sdiv:
1304         case Intrinsic::vp_urem:
1305         case Intrinsic::vp_srem:
1306           return Operand == 1;
1307         // ... with the exception of vp.sub/vp.fsub/vp.fdiv, which have
1308         // explicit patterns for both LHS and RHS (as 'vr' versions).
1309         case Intrinsic::vp_sub:
1310         case Intrinsic::vp_fsub:
1311         case Intrinsic::vp_fdiv:
1312           return Operand == 0 || Operand == 1;
1313         default:
1314           return false;
1315         }
1316       }
1317       return false;
1318     default:
1319       return false;
1320     }
1321   };
1322 
1323   for (auto OpIdx : enumerate(I->operands())) {
1324     if (!IsSinker(I, OpIdx.index()))
1325       continue;
1326 
1327     Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
1328     // Make sure we are not already sinking this operand
1329     if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
1330       continue;
1331 
1332     // We are looking for a splat that can be sunk.
1333     if (!match(Op, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
1334                              m_Undef(), m_ZeroMask())))
1335       continue;
1336 
1337     // All uses of the shuffle should be sunk to avoid duplicating it across gpr
1338     // and vector registers
1339     for (Use &U : Op->uses()) {
1340       Instruction *Insn = cast<Instruction>(U.getUser());
1341       if (!IsSinker(Insn, U.getOperandNo()))
1342         return false;
1343     }
1344 
1345     Ops.push_back(&Op->getOperandUse(0));
1346     Ops.push_back(&OpIdx.value());
1347   }
1348   return true;
1349 }
1350 
1351 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1352                                        bool ForCodeSize) const {
1353   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1354   if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
1355     return false;
1356   if (VT == MVT::f32 && !Subtarget.hasStdExtF())
1357     return false;
1358   if (VT == MVT::f64 && !Subtarget.hasStdExtD())
1359     return false;
1360   return Imm.isZero();
1361 }
1362 
1363 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
1364   return (VT == MVT::f16 && Subtarget.hasStdExtZfh()) ||
1365          (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
1366          (VT == MVT::f64 && Subtarget.hasStdExtD());
1367 }
1368 
1369 MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1370                                                       CallingConv::ID CC,
1371                                                       EVT VT) const {
1372   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1373   // We might still end up using a GPR but that will be decided based on ABI.
1374   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1375   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1376     return MVT::f32;
1377 
1378   return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1379 }
1380 
1381 unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1382                                                            CallingConv::ID CC,
1383                                                            EVT VT) const {
1384   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1385   // We might still end up using a GPR but that will be decided based on ABI.
1386   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1387   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1388     return 1;
1389 
1390   return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1391 }
1392 
1393 // Changes the condition code and swaps operands if necessary, so the SetCC
1394 // operation matches one of the comparisons supported directly by branches
1395 // in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
1396 // with 1/-1.
1397 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1398                                     ISD::CondCode &CC, SelectionDAG &DAG) {
1399   // Convert X > -1 to X >= 0.
1400   if (CC == ISD::SETGT && isAllOnesConstant(RHS)) {
1401     RHS = DAG.getConstant(0, DL, RHS.getValueType());
1402     CC = ISD::SETGE;
1403     return;
1404   }
1405   // Convert X < 1 to 0 >= X.
1406   if (CC == ISD::SETLT && isOneConstant(RHS)) {
1407     RHS = LHS;
1408     LHS = DAG.getConstant(0, DL, RHS.getValueType());
1409     CC = ISD::SETGE;
1410     return;
1411   }
1412 
1413   switch (CC) {
1414   default:
1415     break;
1416   case ISD::SETGT:
1417   case ISD::SETLE:
1418   case ISD::SETUGT:
1419   case ISD::SETULE:
1420     CC = ISD::getSetCCSwappedOperands(CC);
1421     std::swap(LHS, RHS);
1422     break;
1423   }
1424 }
1425 
1426 RISCVII::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
1427   assert(VT.isScalableVector() && "Expecting a scalable vector type");
1428   unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
1429   if (VT.getVectorElementType() == MVT::i1)
1430     KnownSize *= 8;
1431 
1432   switch (KnownSize) {
1433   default:
1434     llvm_unreachable("Invalid LMUL.");
1435   case 8:
1436     return RISCVII::VLMUL::LMUL_F8;
1437   case 16:
1438     return RISCVII::VLMUL::LMUL_F4;
1439   case 32:
1440     return RISCVII::VLMUL::LMUL_F2;
1441   case 64:
1442     return RISCVII::VLMUL::LMUL_1;
1443   case 128:
1444     return RISCVII::VLMUL::LMUL_2;
1445   case 256:
1446     return RISCVII::VLMUL::LMUL_4;
1447   case 512:
1448     return RISCVII::VLMUL::LMUL_8;
1449   }
1450 }
1451 
1452 unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVII::VLMUL LMul) {
1453   switch (LMul) {
1454   default:
1455     llvm_unreachable("Invalid LMUL.");
1456   case RISCVII::VLMUL::LMUL_F8:
1457   case RISCVII::VLMUL::LMUL_F4:
1458   case RISCVII::VLMUL::LMUL_F2:
1459   case RISCVII::VLMUL::LMUL_1:
1460     return RISCV::VRRegClassID;
1461   case RISCVII::VLMUL::LMUL_2:
1462     return RISCV::VRM2RegClassID;
1463   case RISCVII::VLMUL::LMUL_4:
1464     return RISCV::VRM4RegClassID;
1465   case RISCVII::VLMUL::LMUL_8:
1466     return RISCV::VRM8RegClassID;
1467   }
1468 }
1469 
1470 unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
1471   RISCVII::VLMUL LMUL = getLMUL(VT);
1472   if (LMUL == RISCVII::VLMUL::LMUL_F8 ||
1473       LMUL == RISCVII::VLMUL::LMUL_F4 ||
1474       LMUL == RISCVII::VLMUL::LMUL_F2 ||
1475       LMUL == RISCVII::VLMUL::LMUL_1) {
1476     static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
1477                   "Unexpected subreg numbering");
1478     return RISCV::sub_vrm1_0 + Index;
1479   }
1480   if (LMUL == RISCVII::VLMUL::LMUL_2) {
1481     static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
1482                   "Unexpected subreg numbering");
1483     return RISCV::sub_vrm2_0 + Index;
1484   }
1485   if (LMUL == RISCVII::VLMUL::LMUL_4) {
1486     static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
1487                   "Unexpected subreg numbering");
1488     return RISCV::sub_vrm4_0 + Index;
1489   }
1490   llvm_unreachable("Invalid vector type.");
1491 }
1492 
1493 unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
1494   if (VT.getVectorElementType() == MVT::i1)
1495     return RISCV::VRRegClassID;
1496   return getRegClassIDForLMUL(getLMUL(VT));
1497 }
1498 
1499 // Attempt to decompose a subvector insert/extract between VecVT and
1500 // SubVecVT via subregister indices. Returns the subregister index that
1501 // can perform the subvector insert/extract with the given element index, as
1502 // well as the index corresponding to any leftover subvectors that must be
1503 // further inserted/extracted within the register class for SubVecVT.
1504 std::pair<unsigned, unsigned>
1505 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1506     MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
1507     const RISCVRegisterInfo *TRI) {
1508   static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
1509                  RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
1510                  RISCV::VRM2RegClassID > RISCV::VRRegClassID),
1511                 "Register classes not ordered");
1512   unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
1513   unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
1514   // Try to compose a subregister index that takes us from the incoming
1515   // LMUL>1 register class down to the outgoing one. At each step we half
1516   // the LMUL:
1517   //   nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
1518   // Note that this is not guaranteed to find a subregister index, such as
1519   // when we are extracting from one VR type to another.
1520   unsigned SubRegIdx = RISCV::NoSubRegister;
1521   for (const unsigned RCID :
1522        {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
1523     if (VecRegClassID > RCID && SubRegClassID <= RCID) {
1524       VecVT = VecVT.getHalfNumVectorElementsVT();
1525       bool IsHi =
1526           InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
1527       SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
1528                                             getSubregIndexByMVT(VecVT, IsHi));
1529       if (IsHi)
1530         InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
1531     }
1532   return {SubRegIdx, InsertExtractIdx};
1533 }
1534 
1535 // Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
1536 // stores for those types.
1537 bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
1538   return !Subtarget.useRVVForFixedLengthVectors() ||
1539          (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
1540 }
1541 
1542 bool RISCVTargetLowering::isLegalElementTypeForRVV(Type *ScalarTy) const {
1543   if (ScalarTy->isPointerTy())
1544     return true;
1545 
1546   if (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
1547       ScalarTy->isIntegerTy(32))
1548     return true;
1549 
1550   if (ScalarTy->isIntegerTy(64))
1551     return Subtarget.hasVInstructionsI64();
1552 
1553   if (ScalarTy->isHalfTy())
1554     return Subtarget.hasVInstructionsF16();
1555   if (ScalarTy->isFloatTy())
1556     return Subtarget.hasVInstructionsF32();
1557   if (ScalarTy->isDoubleTy())
1558     return Subtarget.hasVInstructionsF64();
1559 
1560   return false;
1561 }
1562 
1563 static SDValue getVLOperand(SDValue Op) {
1564   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
1565           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
1566          "Unexpected opcode");
1567   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
1568   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
1569   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
1570       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
1571   if (!II)
1572     return SDValue();
1573   return Op.getOperand(II->VLOperand + 1 + HasChain);
1574 }
1575 
1576 static bool useRVVForFixedLengthVectorVT(MVT VT,
1577                                          const RISCVSubtarget &Subtarget) {
1578   assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
1579   if (!Subtarget.useRVVForFixedLengthVectors())
1580     return false;
1581 
1582   // We only support a set of vector types with a consistent maximum fixed size
1583   // across all supported vector element types to avoid legalization issues.
1584   // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
1585   // fixed-length vector type we support is 1024 bytes.
1586   if (VT.getFixedSizeInBits() > 1024 * 8)
1587     return false;
1588 
1589   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1590 
1591   MVT EltVT = VT.getVectorElementType();
1592 
1593   // Don't use RVV for vectors we cannot scalarize if required.
1594   switch (EltVT.SimpleTy) {
1595   // i1 is supported but has different rules.
1596   default:
1597     return false;
1598   case MVT::i1:
1599     // Masks can only use a single register.
1600     if (VT.getVectorNumElements() > MinVLen)
1601       return false;
1602     MinVLen /= 8;
1603     break;
1604   case MVT::i8:
1605   case MVT::i16:
1606   case MVT::i32:
1607     break;
1608   case MVT::i64:
1609     if (!Subtarget.hasVInstructionsI64())
1610       return false;
1611     break;
1612   case MVT::f16:
1613     if (!Subtarget.hasVInstructionsF16())
1614       return false;
1615     break;
1616   case MVT::f32:
1617     if (!Subtarget.hasVInstructionsF32())
1618       return false;
1619     break;
1620   case MVT::f64:
1621     if (!Subtarget.hasVInstructionsF64())
1622       return false;
1623     break;
1624   }
1625 
1626   // Reject elements larger than ELEN.
1627   if (EltVT.getSizeInBits() > Subtarget.getMaxELENForFixedLengthVectors())
1628     return false;
1629 
1630   unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
1631   // Don't use RVV for types that don't fit.
1632   if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
1633     return false;
1634 
1635   // TODO: Perhaps an artificial restriction, but worth having whilst getting
1636   // the base fixed length RVV support in place.
1637   if (!VT.isPow2VectorType())
1638     return false;
1639 
1640   return true;
1641 }
1642 
1643 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
1644   return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
1645 }
1646 
1647 // Return the largest legal scalable vector type that matches VT's element type.
1648 static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
1649                                             const RISCVSubtarget &Subtarget) {
1650   // This may be called before legal types are setup.
1651   assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
1652           useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
1653          "Expected legal fixed length vector!");
1654 
1655   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1656   unsigned MaxELen = Subtarget.getMaxELENForFixedLengthVectors();
1657 
1658   MVT EltVT = VT.getVectorElementType();
1659   switch (EltVT.SimpleTy) {
1660   default:
1661     llvm_unreachable("unexpected element type for RVV container");
1662   case MVT::i1:
1663   case MVT::i8:
1664   case MVT::i16:
1665   case MVT::i32:
1666   case MVT::i64:
1667   case MVT::f16:
1668   case MVT::f32:
1669   case MVT::f64: {
1670     // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
1671     // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
1672     // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
1673     unsigned NumElts =
1674         (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
1675     NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
1676     assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
1677     return MVT::getScalableVectorVT(EltVT, NumElts);
1678   }
1679   }
1680 }
1681 
1682 static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
1683                                             const RISCVSubtarget &Subtarget) {
1684   return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
1685                                           Subtarget);
1686 }
1687 
1688 MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
1689   return ::getContainerForFixedLengthVector(*this, VT, getSubtarget());
1690 }
1691 
1692 // Grow V to consume an entire RVV register.
1693 static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1694                                        const RISCVSubtarget &Subtarget) {
1695   assert(VT.isScalableVector() &&
1696          "Expected to convert into a scalable vector!");
1697   assert(V.getValueType().isFixedLengthVector() &&
1698          "Expected a fixed length vector operand!");
1699   SDLoc DL(V);
1700   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1701   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
1702 }
1703 
1704 // Shrink V so it's just big enough to maintain a VT's worth of data.
1705 static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1706                                          const RISCVSubtarget &Subtarget) {
1707   assert(VT.isFixedLengthVector() &&
1708          "Expected to convert into a fixed length vector!");
1709   assert(V.getValueType().isScalableVector() &&
1710          "Expected a scalable vector operand!");
1711   SDLoc DL(V);
1712   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1713   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
1714 }
1715 
1716 // Gets the two common "VL" operands: an all-ones mask and the vector length.
1717 // VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
1718 // the vector type that it is contained in.
1719 static std::pair<SDValue, SDValue>
1720 getDefaultVLOps(MVT VecVT, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
1721                 const RISCVSubtarget &Subtarget) {
1722   assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
1723   MVT XLenVT = Subtarget.getXLenVT();
1724   SDValue VL = VecVT.isFixedLengthVector()
1725                    ? DAG.getConstant(VecVT.getVectorNumElements(), DL, XLenVT)
1726                    : DAG.getRegister(RISCV::X0, XLenVT);
1727   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
1728   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
1729   return {Mask, VL};
1730 }
1731 
1732 // As above but assuming the given type is a scalable vector type.
1733 static std::pair<SDValue, SDValue>
1734 getDefaultScalableVLOps(MVT VecVT, SDLoc DL, SelectionDAG &DAG,
1735                         const RISCVSubtarget &Subtarget) {
1736   assert(VecVT.isScalableVector() && "Expecting a scalable vector");
1737   return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
1738 }
1739 
1740 // The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
1741 // of either is (currently) supported. This can get us into an infinite loop
1742 // where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
1743 // as a ..., etc.
1744 // Until either (or both) of these can reliably lower any node, reporting that
1745 // we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
1746 // the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
1747 // which is not desirable.
1748 bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
1749     EVT VT, unsigned DefinedValues) const {
1750   return false;
1751 }
1752 
1753 static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
1754                                   const RISCVSubtarget &Subtarget) {
1755   // RISCV FP-to-int conversions saturate to the destination register size, but
1756   // don't produce 0 for nan. We can use a conversion instruction and fix the
1757   // nan case with a compare and a select.
1758   SDValue Src = Op.getOperand(0);
1759 
1760   EVT DstVT = Op.getValueType();
1761   EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1762 
1763   bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
1764   unsigned Opc;
1765   if (SatVT == DstVT)
1766     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
1767   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
1768     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
1769   else
1770     return SDValue();
1771   // FIXME: Support other SatVTs by clamping before or after the conversion.
1772 
1773   SDLoc DL(Op);
1774   SDValue FpToInt = DAG.getNode(
1775       Opc, DL, DstVT, Src,
1776       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
1777 
1778   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
1779   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
1780 }
1781 
1782 // Expand vector FTRUNC, FCEIL, and FFLOOR by converting to the integer domain
1783 // and back. Taking care to avoid converting values that are nan or already
1784 // correct.
1785 // TODO: Floor and ceil could be shorter by changing rounding mode, but we don't
1786 // have FRM dependencies modeled yet.
1787 static SDValue lowerFTRUNC_FCEIL_FFLOOR(SDValue Op, SelectionDAG &DAG) {
1788   MVT VT = Op.getSimpleValueType();
1789   assert(VT.isVector() && "Unexpected type");
1790 
1791   SDLoc DL(Op);
1792 
1793   // Freeze the source since we are increasing the number of uses.
1794   SDValue Src = DAG.getNode(ISD::FREEZE, DL, VT, Op.getOperand(0));
1795 
1796   // Truncate to integer and convert back to FP.
1797   MVT IntVT = VT.changeVectorElementTypeToInteger();
1798   SDValue Truncated = DAG.getNode(ISD::FP_TO_SINT, DL, IntVT, Src);
1799   Truncated = DAG.getNode(ISD::SINT_TO_FP, DL, VT, Truncated);
1800 
1801   MVT SetccVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
1802 
1803   if (Op.getOpcode() == ISD::FCEIL) {
1804     // If the truncated value is the greater than or equal to the original
1805     // value, we've computed the ceil. Otherwise, we went the wrong way and
1806     // need to increase by 1.
1807     // FIXME: This should use a masked operation. Handle here or in isel?
1808     SDValue Adjust = DAG.getNode(ISD::FADD, DL, VT, Truncated,
1809                                  DAG.getConstantFP(1.0, DL, VT));
1810     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOLT);
1811     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1812   } else if (Op.getOpcode() == ISD::FFLOOR) {
1813     // If the truncated value is the less than or equal to the original value,
1814     // we've computed the floor. Otherwise, we went the wrong way and need to
1815     // decrease by 1.
1816     // FIXME: This should use a masked operation. Handle here or in isel?
1817     SDValue Adjust = DAG.getNode(ISD::FSUB, DL, VT, Truncated,
1818                                  DAG.getConstantFP(1.0, DL, VT));
1819     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOGT);
1820     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1821   }
1822 
1823   // Restore the original sign so that -0.0 is preserved.
1824   Truncated = DAG.getNode(ISD::FCOPYSIGN, DL, VT, Truncated, Src);
1825 
1826   // Determine the largest integer that can be represented exactly. This and
1827   // values larger than it don't have any fractional bits so don't need to
1828   // be converted.
1829   const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
1830   unsigned Precision = APFloat::semanticsPrecision(FltSem);
1831   APFloat MaxVal = APFloat(FltSem);
1832   MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
1833                           /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
1834   SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
1835 
1836   // If abs(Src) was larger than MaxVal or nan, keep it.
1837   SDValue Abs = DAG.getNode(ISD::FABS, DL, VT, Src);
1838   SDValue Setcc = DAG.getSetCC(DL, SetccVT, Abs, MaxValNode, ISD::SETOLT);
1839   return DAG.getSelect(DL, VT, Setcc, Truncated, Src);
1840 }
1841 
1842 // ISD::FROUND is defined to round to nearest with ties rounding away from 0.
1843 // This mode isn't supported in vector hardware on RISCV. But as long as we
1844 // aren't compiling with trapping math, we can emulate this with
1845 // floor(X + copysign(nextafter(0.5, 0.0), X)).
1846 // FIXME: Could be shorter by changing rounding mode, but we don't have FRM
1847 // dependencies modeled yet.
1848 // FIXME: Use masked operations to avoid final merge.
1849 static SDValue lowerFROUND(SDValue Op, SelectionDAG &DAG) {
1850   MVT VT = Op.getSimpleValueType();
1851   assert(VT.isVector() && "Unexpected type");
1852 
1853   SDLoc DL(Op);
1854 
1855   // Freeze the source since we are increasing the number of uses.
1856   SDValue Src = DAG.getNode(ISD::FREEZE, DL, VT, Op.getOperand(0));
1857 
1858   // We do the conversion on the absolute value and fix the sign at the end.
1859   SDValue Abs = DAG.getNode(ISD::FABS, DL, VT, Src);
1860 
1861   const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
1862   bool Ignored;
1863   APFloat Point5Pred = APFloat(0.5f);
1864   Point5Pred.convert(FltSem, APFloat::rmNearestTiesToEven, &Ignored);
1865   Point5Pred.next(/*nextDown*/ true);
1866 
1867   // Add the adjustment.
1868   SDValue Adjust = DAG.getNode(ISD::FADD, DL, VT, Abs,
1869                                DAG.getConstantFP(Point5Pred, DL, VT));
1870 
1871   // Truncate to integer and convert back to fp.
1872   MVT IntVT = VT.changeVectorElementTypeToInteger();
1873   SDValue Truncated = DAG.getNode(ISD::FP_TO_SINT, DL, IntVT, Adjust);
1874   Truncated = DAG.getNode(ISD::SINT_TO_FP, DL, VT, Truncated);
1875 
1876   // Restore the original sign.
1877   Truncated = DAG.getNode(ISD::FCOPYSIGN, DL, VT, Truncated, Src);
1878 
1879   // Determine the largest integer that can be represented exactly. This and
1880   // values larger than it don't have any fractional bits so don't need to
1881   // be converted.
1882   unsigned Precision = APFloat::semanticsPrecision(FltSem);
1883   APFloat MaxVal = APFloat(FltSem);
1884   MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
1885                           /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
1886   SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
1887 
1888   // If abs(Src) was larger than MaxVal or nan, keep it.
1889   MVT SetccVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
1890   SDValue Setcc = DAG.getSetCC(DL, SetccVT, Abs, MaxValNode, ISD::SETOLT);
1891   return DAG.getSelect(DL, VT, Setcc, Truncated, Src);
1892 }
1893 
1894 static SDValue lowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG,
1895                                  const RISCVSubtarget &Subtarget) {
1896   MVT VT = Op.getSimpleValueType();
1897   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1898 
1899   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1900 
1901   SDLoc DL(Op);
1902   SDValue Mask, VL;
1903   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1904 
1905   unsigned Opc =
1906       VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
1907   SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, Op.getOperand(0), VL);
1908   return convertFromScalableVector(VT, Splat, DAG, Subtarget);
1909 }
1910 
1911 struct VIDSequence {
1912   int64_t StepNumerator;
1913   unsigned StepDenominator;
1914   int64_t Addend;
1915 };
1916 
1917 // Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
1918 // to the (non-zero) step S and start value X. This can be then lowered as the
1919 // RVV sequence (VID * S) + X, for example.
1920 // The step S is represented as an integer numerator divided by a positive
1921 // denominator. Note that the implementation currently only identifies
1922 // sequences in which either the numerator is +/- 1 or the denominator is 1. It
1923 // cannot detect 2/3, for example.
1924 // Note that this method will also match potentially unappealing index
1925 // sequences, like <i32 0, i32 50939494>, however it is left to the caller to
1926 // determine whether this is worth generating code for.
1927 static Optional<VIDSequence> isSimpleVIDSequence(SDValue Op) {
1928   unsigned NumElts = Op.getNumOperands();
1929   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
1930   if (!Op.getValueType().isInteger())
1931     return None;
1932 
1933   Optional<unsigned> SeqStepDenom;
1934   Optional<int64_t> SeqStepNum, SeqAddend;
1935   Optional<std::pair<uint64_t, unsigned>> PrevElt;
1936   unsigned EltSizeInBits = Op.getValueType().getScalarSizeInBits();
1937   for (unsigned Idx = 0; Idx < NumElts; Idx++) {
1938     // Assume undef elements match the sequence; we just have to be careful
1939     // when interpolating across them.
1940     if (Op.getOperand(Idx).isUndef())
1941       continue;
1942     // The BUILD_VECTOR must be all constants.
1943     if (!isa<ConstantSDNode>(Op.getOperand(Idx)))
1944       return None;
1945 
1946     uint64_t Val = Op.getConstantOperandVal(Idx) &
1947                    maskTrailingOnes<uint64_t>(EltSizeInBits);
1948 
1949     if (PrevElt) {
1950       // Calculate the step since the last non-undef element, and ensure
1951       // it's consistent across the entire sequence.
1952       unsigned IdxDiff = Idx - PrevElt->second;
1953       int64_t ValDiff = SignExtend64(Val - PrevElt->first, EltSizeInBits);
1954 
1955       // A zero-value value difference means that we're somewhere in the middle
1956       // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
1957       // step change before evaluating the sequence.
1958       if (ValDiff != 0) {
1959         int64_t Remainder = ValDiff % IdxDiff;
1960         // Normalize the step if it's greater than 1.
1961         if (Remainder != ValDiff) {
1962           // The difference must cleanly divide the element span.
1963           if (Remainder != 0)
1964             return None;
1965           ValDiff /= IdxDiff;
1966           IdxDiff = 1;
1967         }
1968 
1969         if (!SeqStepNum)
1970           SeqStepNum = ValDiff;
1971         else if (ValDiff != SeqStepNum)
1972           return None;
1973 
1974         if (!SeqStepDenom)
1975           SeqStepDenom = IdxDiff;
1976         else if (IdxDiff != *SeqStepDenom)
1977           return None;
1978       }
1979     }
1980 
1981     // Record and/or check any addend.
1982     if (SeqStepNum && SeqStepDenom) {
1983       uint64_t ExpectedVal =
1984           (int64_t)(Idx * (uint64_t)*SeqStepNum) / *SeqStepDenom;
1985       int64_t Addend = SignExtend64(Val - ExpectedVal, EltSizeInBits);
1986       if (!SeqAddend)
1987         SeqAddend = Addend;
1988       else if (SeqAddend != Addend)
1989         return None;
1990     }
1991 
1992     // Record this non-undef element for later.
1993     if (!PrevElt || PrevElt->first != Val)
1994       PrevElt = std::make_pair(Val, Idx);
1995   }
1996   // We need to have logged both a step and an addend for this to count as
1997   // a legal index sequence.
1998   if (!SeqStepNum || !SeqStepDenom || !SeqAddend)
1999     return None;
2000 
2001   return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
2002 }
2003 
2004 // Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
2005 // and lower it as a VRGATHER_VX_VL from the source vector.
2006 static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
2007                                   SelectionDAG &DAG,
2008                                   const RISCVSubtarget &Subtarget) {
2009   if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2010     return SDValue();
2011   SDValue Vec = SplatVal.getOperand(0);
2012   // Only perform this optimization on vectors of the same size for simplicity.
2013   if (Vec.getValueType() != VT)
2014     return SDValue();
2015   SDValue Idx = SplatVal.getOperand(1);
2016   // The index must be a legal type.
2017   if (Idx.getValueType() != Subtarget.getXLenVT())
2018     return SDValue();
2019 
2020   MVT ContainerVT = VT;
2021   if (VT.isFixedLengthVector()) {
2022     ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2023     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
2024   }
2025 
2026   SDValue Mask, VL;
2027   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2028 
2029   SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Vec,
2030                                Idx, Mask, VL);
2031 
2032   if (!VT.isFixedLengthVector())
2033     return Gather;
2034 
2035   return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2036 }
2037 
2038 static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
2039                                  const RISCVSubtarget &Subtarget) {
2040   MVT VT = Op.getSimpleValueType();
2041   assert(VT.isFixedLengthVector() && "Unexpected vector!");
2042 
2043   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2044 
2045   SDLoc DL(Op);
2046   SDValue Mask, VL;
2047   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2048 
2049   MVT XLenVT = Subtarget.getXLenVT();
2050   unsigned NumElts = Op.getNumOperands();
2051 
2052   if (VT.getVectorElementType() == MVT::i1) {
2053     if (ISD::isBuildVectorAllZeros(Op.getNode())) {
2054       SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
2055       return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
2056     }
2057 
2058     if (ISD::isBuildVectorAllOnes(Op.getNode())) {
2059       SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
2060       return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
2061     }
2062 
2063     // Lower constant mask BUILD_VECTORs via an integer vector type, in
2064     // scalar integer chunks whose bit-width depends on the number of mask
2065     // bits and XLEN.
2066     // First, determine the most appropriate scalar integer type to use. This
2067     // is at most XLenVT, but may be shrunk to a smaller vector element type
2068     // according to the size of the final vector - use i8 chunks rather than
2069     // XLenVT if we're producing a v8i1. This results in more consistent
2070     // codegen across RV32 and RV64.
2071     unsigned NumViaIntegerBits =
2072         std::min(std::max(NumElts, 8u), Subtarget.getXLen());
2073     NumViaIntegerBits = std::min(NumViaIntegerBits,
2074                                  Subtarget.getMaxELENForFixedLengthVectors());
2075     if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
2076       // If we have to use more than one INSERT_VECTOR_ELT then this
2077       // optimization is likely to increase code size; avoid peforming it in
2078       // such a case. We can use a load from a constant pool in this case.
2079       if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
2080         return SDValue();
2081       // Now we can create our integer vector type. Note that it may be larger
2082       // than the resulting mask type: v4i1 would use v1i8 as its integer type.
2083       MVT IntegerViaVecVT =
2084           MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
2085                            divideCeil(NumElts, NumViaIntegerBits));
2086 
2087       uint64_t Bits = 0;
2088       unsigned BitPos = 0, IntegerEltIdx = 0;
2089       SDValue Vec = DAG.getUNDEF(IntegerViaVecVT);
2090 
2091       for (unsigned I = 0; I < NumElts; I++, BitPos++) {
2092         // Once we accumulate enough bits to fill our scalar type, insert into
2093         // our vector and clear our accumulated data.
2094         if (I != 0 && I % NumViaIntegerBits == 0) {
2095           if (NumViaIntegerBits <= 32)
2096             Bits = SignExtend64(Bits, 32);
2097           SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2098           Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec,
2099                             Elt, DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2100           Bits = 0;
2101           BitPos = 0;
2102           IntegerEltIdx++;
2103         }
2104         SDValue V = Op.getOperand(I);
2105         bool BitValue = !V.isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
2106         Bits |= ((uint64_t)BitValue << BitPos);
2107       }
2108 
2109       // Insert the (remaining) scalar value into position in our integer
2110       // vector type.
2111       if (NumViaIntegerBits <= 32)
2112         Bits = SignExtend64(Bits, 32);
2113       SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2114       Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, Elt,
2115                         DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2116 
2117       if (NumElts < NumViaIntegerBits) {
2118         // If we're producing a smaller vector than our minimum legal integer
2119         // type, bitcast to the equivalent (known-legal) mask type, and extract
2120         // our final mask.
2121         assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
2122         Vec = DAG.getBitcast(MVT::v8i1, Vec);
2123         Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
2124                           DAG.getConstant(0, DL, XLenVT));
2125       } else {
2126         // Else we must have produced an integer type with the same size as the
2127         // mask type; bitcast for the final result.
2128         assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
2129         Vec = DAG.getBitcast(VT, Vec);
2130       }
2131 
2132       return Vec;
2133     }
2134 
2135     // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
2136     // vector type, we have a legal equivalently-sized i8 type, so we can use
2137     // that.
2138     MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
2139     SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
2140 
2141     SDValue WideVec;
2142     if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2143       // For a splat, perform a scalar truncate before creating the wider
2144       // vector.
2145       assert(Splat.getValueType() == XLenVT &&
2146              "Unexpected type for i1 splat value");
2147       Splat = DAG.getNode(ISD::AND, DL, XLenVT, Splat,
2148                           DAG.getConstant(1, DL, XLenVT));
2149       WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
2150     } else {
2151       SmallVector<SDValue, 8> Ops(Op->op_values());
2152       WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
2153       SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
2154       WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
2155     }
2156 
2157     return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
2158   }
2159 
2160   if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2161     if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
2162       return Gather;
2163     unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
2164                                         : RISCVISD::VMV_V_X_VL;
2165     Splat = DAG.getNode(Opc, DL, ContainerVT, Splat, VL);
2166     return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2167   }
2168 
2169   // Try and match index sequences, which we can lower to the vid instruction
2170   // with optional modifications. An all-undef vector is matched by
2171   // getSplatValue, above.
2172   if (auto SimpleVID = isSimpleVIDSequence(Op)) {
2173     int64_t StepNumerator = SimpleVID->StepNumerator;
2174     unsigned StepDenominator = SimpleVID->StepDenominator;
2175     int64_t Addend = SimpleVID->Addend;
2176 
2177     assert(StepNumerator != 0 && "Invalid step");
2178     bool Negate = false;
2179     int64_t SplatStepVal = StepNumerator;
2180     unsigned StepOpcode = ISD::MUL;
2181     if (StepNumerator != 1) {
2182       if (isPowerOf2_64(std::abs(StepNumerator))) {
2183         Negate = StepNumerator < 0;
2184         StepOpcode = ISD::SHL;
2185         SplatStepVal = Log2_64(std::abs(StepNumerator));
2186       }
2187     }
2188 
2189     // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
2190     // threshold since it's the immediate value many RVV instructions accept.
2191     // There is no vmul.vi instruction so ensure multiply constant can fit in
2192     // a single addi instruction.
2193     if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
2194          (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
2195         isPowerOf2_32(StepDenominator) && isInt<5>(Addend)) {
2196       SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
2197       // Convert right out of the scalable type so we can use standard ISD
2198       // nodes for the rest of the computation. If we used scalable types with
2199       // these, we'd lose the fixed-length vector info and generate worse
2200       // vsetvli code.
2201       VID = convertFromScalableVector(VT, VID, DAG, Subtarget);
2202       if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
2203           (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
2204         SDValue SplatStep = DAG.getSplatVector(
2205             VT, DL, DAG.getConstant(SplatStepVal, DL, XLenVT));
2206         VID = DAG.getNode(StepOpcode, DL, VT, VID, SplatStep);
2207       }
2208       if (StepDenominator != 1) {
2209         SDValue SplatStep = DAG.getSplatVector(
2210             VT, DL, DAG.getConstant(Log2_64(StepDenominator), DL, XLenVT));
2211         VID = DAG.getNode(ISD::SRL, DL, VT, VID, SplatStep);
2212       }
2213       if (Addend != 0 || Negate) {
2214         SDValue SplatAddend =
2215             DAG.getSplatVector(VT, DL, DAG.getConstant(Addend, DL, XLenVT));
2216         VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VT, SplatAddend, VID);
2217       }
2218       return VID;
2219     }
2220   }
2221 
2222   // Attempt to detect "hidden" splats, which only reveal themselves as splats
2223   // when re-interpreted as a vector with a larger element type. For example,
2224   //   v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
2225   // could be instead splat as
2226   //   v2i32 = build_vector i32 0x00010000, i32 0x00010000
2227   // TODO: This optimization could also work on non-constant splats, but it
2228   // would require bit-manipulation instructions to construct the splat value.
2229   SmallVector<SDValue> Sequence;
2230   unsigned EltBitSize = VT.getScalarSizeInBits();
2231   const auto *BV = cast<BuildVectorSDNode>(Op);
2232   if (VT.isInteger() && EltBitSize < 64 &&
2233       ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
2234       BV->getRepeatedSequence(Sequence) &&
2235       (Sequence.size() * EltBitSize) <= 64) {
2236     unsigned SeqLen = Sequence.size();
2237     MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
2238     MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, NumElts / SeqLen);
2239     assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
2240             ViaIntVT == MVT::i64) &&
2241            "Unexpected sequence type");
2242 
2243     unsigned EltIdx = 0;
2244     uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
2245     uint64_t SplatValue = 0;
2246     // Construct the amalgamated value which can be splatted as this larger
2247     // vector type.
2248     for (const auto &SeqV : Sequence) {
2249       if (!SeqV.isUndef())
2250         SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
2251                        << (EltIdx * EltBitSize));
2252       EltIdx++;
2253     }
2254 
2255     // On RV64, sign-extend from 32 to 64 bits where possible in order to
2256     // achieve better constant materializion.
2257     if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
2258       SplatValue = SignExtend64(SplatValue, 32);
2259 
2260     // Since we can't introduce illegal i64 types at this stage, we can only
2261     // perform an i64 splat on RV32 if it is its own sign-extended value. That
2262     // way we can use RVV instructions to splat.
2263     assert((ViaIntVT.bitsLE(XLenVT) ||
2264             (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
2265            "Unexpected bitcast sequence");
2266     if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
2267       SDValue ViaVL =
2268           DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
2269       MVT ViaContainerVT =
2270           getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget);
2271       SDValue Splat =
2272           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
2273                       DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
2274       Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
2275       return DAG.getBitcast(VT, Splat);
2276     }
2277   }
2278 
2279   // Try and optimize BUILD_VECTORs with "dominant values" - these are values
2280   // which constitute a large proportion of the elements. In such cases we can
2281   // splat a vector with the dominant element and make up the shortfall with
2282   // INSERT_VECTOR_ELTs.
2283   // Note that this includes vectors of 2 elements by association. The
2284   // upper-most element is the "dominant" one, allowing us to use a splat to
2285   // "insert" the upper element, and an insert of the lower element at position
2286   // 0, which improves codegen.
2287   SDValue DominantValue;
2288   unsigned MostCommonCount = 0;
2289   DenseMap<SDValue, unsigned> ValueCounts;
2290   unsigned NumUndefElts =
2291       count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
2292 
2293   // Track the number of scalar loads we know we'd be inserting, estimated as
2294   // any non-zero floating-point constant. Other kinds of element are either
2295   // already in registers or are materialized on demand. The threshold at which
2296   // a vector load is more desirable than several scalar materializion and
2297   // vector-insertion instructions is not known.
2298   unsigned NumScalarLoads = 0;
2299 
2300   for (SDValue V : Op->op_values()) {
2301     if (V.isUndef())
2302       continue;
2303 
2304     ValueCounts.insert(std::make_pair(V, 0));
2305     unsigned &Count = ValueCounts[V];
2306 
2307     if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
2308       NumScalarLoads += !CFP->isExactlyValue(+0.0);
2309 
2310     // Is this value dominant? In case of a tie, prefer the highest element as
2311     // it's cheaper to insert near the beginning of a vector than it is at the
2312     // end.
2313     if (++Count >= MostCommonCount) {
2314       DominantValue = V;
2315       MostCommonCount = Count;
2316     }
2317   }
2318 
2319   assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
2320   unsigned NumDefElts = NumElts - NumUndefElts;
2321   unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
2322 
2323   // Don't perform this optimization when optimizing for size, since
2324   // materializing elements and inserting them tends to cause code bloat.
2325   if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
2326       ((MostCommonCount > DominantValueCountThreshold) ||
2327        (ValueCounts.size() <= Log2_32(NumDefElts)))) {
2328     // Start by splatting the most common element.
2329     SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
2330 
2331     DenseSet<SDValue> Processed{DominantValue};
2332     MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
2333     for (const auto &OpIdx : enumerate(Op->ops())) {
2334       const SDValue &V = OpIdx.value();
2335       if (V.isUndef() || !Processed.insert(V).second)
2336         continue;
2337       if (ValueCounts[V] == 1) {
2338         Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
2339                           DAG.getConstant(OpIdx.index(), DL, XLenVT));
2340       } else {
2341         // Blend in all instances of this value using a VSELECT, using a
2342         // mask where each bit signals whether that element is the one
2343         // we're after.
2344         SmallVector<SDValue> Ops;
2345         transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
2346           return DAG.getConstant(V == V1, DL, XLenVT);
2347         });
2348         Vec = DAG.getNode(ISD::VSELECT, DL, VT,
2349                           DAG.getBuildVector(SelMaskTy, DL, Ops),
2350                           DAG.getSplatBuildVector(VT, DL, V), Vec);
2351       }
2352     }
2353 
2354     return Vec;
2355   }
2356 
2357   return SDValue();
2358 }
2359 
2360 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Lo,
2361                                    SDValue Hi, SDValue VL, SelectionDAG &DAG) {
2362   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
2363     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
2364     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
2365     // If Hi constant is all the same sign bit as Lo, lower this as a custom
2366     // node in order to try and match RVV vector/scalar instructions.
2367     if ((LoC >> 31) == HiC)
2368       return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Lo, VL);
2369 
2370     // If vl is equal to XLEN_MAX and Hi constant is equal to Lo, we could use
2371     // vmv.v.x whose EEW = 32 to lower it.
2372     auto *Const = dyn_cast<ConstantSDNode>(VL);
2373     if (LoC == HiC && Const && Const->isAllOnesValue()) {
2374       MVT InterVT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
2375       // TODO: if vl <= min(VLMAX), we can also do this. But we could not
2376       // access the subtarget here now.
2377       auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT, Lo,
2378                                   DAG.getRegister(RISCV::X0, MVT::i32));
2379       return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
2380     }
2381   }
2382 
2383   // Fall back to a stack store and stride x0 vector load.
2384   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Lo, Hi, VL);
2385 }
2386 
2387 // Called by type legalization to handle splat of i64 on RV32.
2388 // FIXME: We can optimize this when the type has sign or zero bits in one
2389 // of the halves.
2390 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Scalar,
2391                                    SDValue VL, SelectionDAG &DAG) {
2392   assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
2393   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2394                            DAG.getConstant(0, DL, MVT::i32));
2395   SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2396                            DAG.getConstant(1, DL, MVT::i32));
2397   return splatPartsI64WithVL(DL, VT, Lo, Hi, VL, DAG);
2398 }
2399 
2400 // This function lowers a splat of a scalar operand Splat with the vector
2401 // length VL. It ensures the final sequence is type legal, which is useful when
2402 // lowering a splat after type legalization.
2403 static SDValue lowerScalarSplat(SDValue Scalar, SDValue VL, MVT VT, SDLoc DL,
2404                                 SelectionDAG &DAG,
2405                                 const RISCVSubtarget &Subtarget) {
2406   if (VT.isFloatingPoint()) {
2407     // If VL is 1, we could use vfmv.s.f.
2408     if (isOneConstant(VL))
2409       return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT),
2410                          Scalar, VL);
2411     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Scalar, VL);
2412   }
2413 
2414   MVT XLenVT = Subtarget.getXLenVT();
2415 
2416   // Simplest case is that the operand needs to be promoted to XLenVT.
2417   if (Scalar.getValueType().bitsLE(XLenVT)) {
2418     // If the operand is a constant, sign extend to increase our chances
2419     // of being able to use a .vi instruction. ANY_EXTEND would become a
2420     // a zero extend and the simm5 check in isel would fail.
2421     // FIXME: Should we ignore the upper bits in isel instead?
2422     unsigned ExtOpc =
2423         isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
2424     Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
2425     ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Scalar);
2426     // If VL is 1 and the scalar value won't benefit from immediate, we could
2427     // use vmv.s.x.
2428     if (isOneConstant(VL) &&
2429         (!Const || isNullConstant(Scalar) || !isInt<5>(Const->getSExtValue())))
2430       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
2431                          VL);
2432     return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Scalar, VL);
2433   }
2434 
2435   assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
2436          "Unexpected scalar for splat lowering!");
2437 
2438   if (isOneConstant(VL) && isNullConstant(Scalar))
2439     return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT),
2440                        DAG.getConstant(0, DL, XLenVT), VL);
2441 
2442   // Otherwise use the more complicated splatting algorithm.
2443   return splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
2444 }
2445 
2446 // Is the mask a slidedown that shifts in undefs.
2447 static int matchShuffleAsSlideDown(ArrayRef<int> Mask) {
2448   int Size = Mask.size();
2449 
2450   // Elements shifted in should be undef.
2451   auto CheckUndefs = [&](int Shift) {
2452     for (int i = Size - Shift; i != Size; ++i)
2453       if (Mask[i] >= 0)
2454         return false;
2455     return true;
2456   };
2457 
2458   // Elements should be shifted or undef.
2459   auto MatchShift = [&](int Shift) {
2460     for (int i = 0; i != Size - Shift; ++i)
2461        if (Mask[i] >= 0 && Mask[i] != Shift + i)
2462          return false;
2463     return true;
2464   };
2465 
2466   // Try all possible shifts.
2467   for (int Shift = 1; Shift != Size; ++Shift)
2468     if (CheckUndefs(Shift) && MatchShift(Shift))
2469       return Shift;
2470 
2471   // No match.
2472   return -1;
2473 }
2474 
2475 static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, bool &SwapSources,
2476                                 const RISCVSubtarget &Subtarget) {
2477   // We need to be able to widen elements to the next larger integer type.
2478   if (VT.getScalarSizeInBits() >= Subtarget.getMaxELENForFixedLengthVectors())
2479     return false;
2480 
2481   int Size = Mask.size();
2482   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
2483 
2484   int Srcs[] = {-1, -1};
2485   for (int i = 0; i != Size; ++i) {
2486     // Ignore undef elements.
2487     if (Mask[i] < 0)
2488       continue;
2489 
2490     // Is this an even or odd element.
2491     int Pol = i % 2;
2492 
2493     // Ensure we consistently use the same source for this element polarity.
2494     int Src = Mask[i] / Size;
2495     if (Srcs[Pol] < 0)
2496       Srcs[Pol] = Src;
2497     if (Srcs[Pol] != Src)
2498       return false;
2499 
2500     // Make sure the element within the source is appropriate for this element
2501     // in the destination.
2502     int Elt = Mask[i] % Size;
2503     if (Elt != i / 2)
2504       return false;
2505   }
2506 
2507   // We need to find a source for each polarity and they can't be the same.
2508   if (Srcs[0] < 0 || Srcs[1] < 0 || Srcs[0] == Srcs[1])
2509     return false;
2510 
2511   // Swap the sources if the second source was in the even polarity.
2512   SwapSources = Srcs[0] > Srcs[1];
2513 
2514   return true;
2515 }
2516 
2517 static int isElementRotate(SDValue &V1, SDValue &V2, ArrayRef<int> Mask) {
2518   int Size = Mask.size();
2519 
2520   // We need to detect various ways of spelling a rotation:
2521   //   [11, 12, 13, 14, 15,  0,  1,  2]
2522   //   [-1, 12, 13, 14, -1, -1,  1, -1]
2523   //   [-1, -1, -1, -1, -1, -1,  1,  2]
2524   //   [ 3,  4,  5,  6,  7,  8,  9, 10]
2525   //   [-1,  4,  5,  6, -1, -1,  9, -1]
2526   //   [-1,  4,  5,  6, -1, -1, -1, -1]
2527   int Rotation = 0;
2528   SDValue Lo, Hi;
2529   for (int i = 0; i != Size; ++i) {
2530     int M = Mask[i];
2531     if (M < 0)
2532       continue;
2533 
2534     // Determine where a rotate vector would have started.
2535     int StartIdx = i - (M % Size);
2536     // The identity rotation isn't interesting, stop.
2537     if (StartIdx == 0)
2538       return -1;
2539 
2540     // If we found the tail of a vector the rotation must be the missing
2541     // front. If we found the head of a vector, it must be how much of the
2542     // head.
2543     int CandidateRotation = StartIdx < 0 ? -StartIdx : Size - StartIdx;
2544 
2545     if (Rotation == 0)
2546       Rotation = CandidateRotation;
2547     else if (Rotation != CandidateRotation)
2548       // The rotations don't match, so we can't match this mask.
2549       return -1;
2550 
2551     // Compute which value this mask is pointing at.
2552     SDValue MaskV = M < Size ? V1 : V2;
2553 
2554     // Compute which of the two target values this index should be assigned to.
2555     // This reflects whether the high elements are remaining or the low elemnts
2556     // are remaining.
2557     SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
2558 
2559     // Either set up this value if we've not encountered it before, or check
2560     // that it remains consistent.
2561     if (!TargetV)
2562       TargetV = MaskV;
2563     else if (TargetV != MaskV)
2564       // This may be a rotation, but it pulls from the inputs in some
2565       // unsupported interleaving.
2566       return -1;
2567   }
2568 
2569   // Check that we successfully analyzed the mask, and normalize the results.
2570   assert(Rotation != 0 && "Failed to locate a viable rotation!");
2571   assert((Lo || Hi) && "Failed to find a rotated input vector!");
2572 
2573   // Make sure we've found a value for both halves.
2574   if (!Lo || !Hi)
2575     return -1;
2576 
2577   V1 = Lo;
2578   V2 = Hi;
2579 
2580   return Rotation;
2581 }
2582 
2583 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
2584                                    const RISCVSubtarget &Subtarget) {
2585   SDValue V1 = Op.getOperand(0);
2586   SDValue V2 = Op.getOperand(1);
2587   SDLoc DL(Op);
2588   MVT XLenVT = Subtarget.getXLenVT();
2589   MVT VT = Op.getSimpleValueType();
2590   unsigned NumElts = VT.getVectorNumElements();
2591   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
2592 
2593   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2594 
2595   SDValue TrueMask, VL;
2596   std::tie(TrueMask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2597 
2598   if (SVN->isSplat()) {
2599     const int Lane = SVN->getSplatIndex();
2600     if (Lane >= 0) {
2601       MVT SVT = VT.getVectorElementType();
2602 
2603       // Turn splatted vector load into a strided load with an X0 stride.
2604       SDValue V = V1;
2605       // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
2606       // with undef.
2607       // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
2608       int Offset = Lane;
2609       if (V.getOpcode() == ISD::CONCAT_VECTORS) {
2610         int OpElements =
2611             V.getOperand(0).getSimpleValueType().getVectorNumElements();
2612         V = V.getOperand(Offset / OpElements);
2613         Offset %= OpElements;
2614       }
2615 
2616       // We need to ensure the load isn't atomic or volatile.
2617       if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
2618         auto *Ld = cast<LoadSDNode>(V);
2619         Offset *= SVT.getStoreSize();
2620         SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(),
2621                                                    TypeSize::Fixed(Offset), DL);
2622 
2623         // If this is SEW=64 on RV32, use a strided load with a stride of x0.
2624         if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
2625           SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
2626           SDValue IntID =
2627               DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
2628           SDValue Ops[] = {Ld->getChain(),
2629                            IntID,
2630                            DAG.getUNDEF(ContainerVT),
2631                            NewAddr,
2632                            DAG.getRegister(RISCV::X0, XLenVT),
2633                            VL};
2634           SDValue NewLoad = DAG.getMemIntrinsicNode(
2635               ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
2636               DAG.getMachineFunction().getMachineMemOperand(
2637                   Ld->getMemOperand(), Offset, SVT.getStoreSize()));
2638           DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
2639           return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
2640         }
2641 
2642         // Otherwise use a scalar load and splat. This will give the best
2643         // opportunity to fold a splat into the operation. ISel can turn it into
2644         // the x0 strided load if we aren't able to fold away the select.
2645         if (SVT.isFloatingPoint())
2646           V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
2647                           Ld->getPointerInfo().getWithOffset(Offset),
2648                           Ld->getOriginalAlign(),
2649                           Ld->getMemOperand()->getFlags());
2650         else
2651           V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
2652                              Ld->getPointerInfo().getWithOffset(Offset), SVT,
2653                              Ld->getOriginalAlign(),
2654                              Ld->getMemOperand()->getFlags());
2655         DAG.makeEquivalentMemoryOrdering(Ld, V);
2656 
2657         unsigned Opc =
2658             VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
2659         SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, V, VL);
2660         return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2661       }
2662 
2663       V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2664       assert(Lane < (int)NumElts && "Unexpected lane!");
2665       SDValue Gather =
2666           DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
2667                       DAG.getConstant(Lane, DL, XLenVT), TrueMask, VL);
2668       return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2669     }
2670   }
2671 
2672   ArrayRef<int> Mask = SVN->getMask();
2673 
2674   // Try to match as a slidedown.
2675   int SlideAmt = matchShuffleAsSlideDown(Mask);
2676   if (SlideAmt >= 0) {
2677     // TODO: Should we reduce the VL to account for the upper undef elements?
2678     // Requires additional vsetvlis, but might be faster to execute.
2679     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2680     SDValue SlideDown =
2681         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
2682                     DAG.getUNDEF(ContainerVT), V1,
2683                     DAG.getConstant(SlideAmt, DL, XLenVT),
2684                     TrueMask, VL);
2685     return convertFromScalableVector(VT, SlideDown, DAG, Subtarget);
2686   }
2687 
2688   // Match shuffles that concatenate two vectors, rotate the concatenation,
2689   // and then extract the original number of elements from the rotated result.
2690   // This is equivalent to vector.splice or X86's PALIGNR instruction. Lower
2691   // it to a SLIDEDOWN and a SLIDEUP.
2692   // FIXME: We don't really need it to be a concatenation. We just need two
2693   // regions with contiguous elements that need to be shifted down and up.
2694   int Rotation = isElementRotate(V1, V2, Mask);
2695   if (Rotation > 0) {
2696     // We found a rotation. We need to slide V1 down by Rotation. Using
2697     // (NumElts - Rotation) for VL. Then we need to slide V2 up by
2698     // (NumElts - Rotation) using NumElts for VL.
2699     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2700     V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
2701 
2702     unsigned InvRotate = NumElts - Rotation;
2703     SDValue SlideDown =
2704         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
2705                     DAG.getUNDEF(ContainerVT), V2,
2706                     DAG.getConstant(Rotation, DL, XLenVT),
2707                     TrueMask, DAG.getConstant(InvRotate, DL, XLenVT));
2708     SDValue SlideUp =
2709         DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, SlideDown, V1,
2710                     DAG.getConstant(InvRotate, DL, XLenVT),
2711                     TrueMask, VL);
2712     return convertFromScalableVector(VT, SlideUp, DAG, Subtarget);
2713   }
2714 
2715   // Detect an interleave shuffle and lower to
2716   // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
2717   bool SwapSources;
2718   if (isInterleaveShuffle(Mask, VT, SwapSources, Subtarget)) {
2719     // Swap sources if needed.
2720     if (SwapSources)
2721       std::swap(V1, V2);
2722 
2723     // Extract the lower half of the vectors.
2724     MVT HalfVT = VT.getHalfNumVectorElementsVT();
2725     V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
2726                      DAG.getConstant(0, DL, XLenVT));
2727     V2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V2,
2728                      DAG.getConstant(0, DL, XLenVT));
2729 
2730     // Double the element width and halve the number of elements in an int type.
2731     unsigned EltBits = VT.getScalarSizeInBits();
2732     MVT WideIntEltVT = MVT::getIntegerVT(EltBits * 2);
2733     MVT WideIntVT =
2734         MVT::getVectorVT(WideIntEltVT, VT.getVectorNumElements() / 2);
2735     // Convert this to a scalable vector. We need to base this on the
2736     // destination size to ensure there's always a type with a smaller LMUL.
2737     MVT WideIntContainerVT =
2738         getContainerForFixedLengthVector(DAG, WideIntVT, Subtarget);
2739 
2740     // Convert sources to scalable vectors with the same element count as the
2741     // larger type.
2742     MVT HalfContainerVT = MVT::getVectorVT(
2743         VT.getVectorElementType(), WideIntContainerVT.getVectorElementCount());
2744     V1 = convertToScalableVector(HalfContainerVT, V1, DAG, Subtarget);
2745     V2 = convertToScalableVector(HalfContainerVT, V2, DAG, Subtarget);
2746 
2747     // Cast sources to integer.
2748     MVT IntEltVT = MVT::getIntegerVT(EltBits);
2749     MVT IntHalfVT =
2750         MVT::getVectorVT(IntEltVT, HalfContainerVT.getVectorElementCount());
2751     V1 = DAG.getBitcast(IntHalfVT, V1);
2752     V2 = DAG.getBitcast(IntHalfVT, V2);
2753 
2754     // Freeze V2 since we use it twice and we need to be sure that the add and
2755     // multiply see the same value.
2756     V2 = DAG.getNode(ISD::FREEZE, DL, IntHalfVT, V2);
2757 
2758     // Recreate TrueMask using the widened type's element count.
2759     MVT MaskVT =
2760         MVT::getVectorVT(MVT::i1, HalfContainerVT.getVectorElementCount());
2761     TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
2762 
2763     // Widen V1 and V2 with 0s and add one copy of V2 to V1.
2764     SDValue Add = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideIntContainerVT, V1,
2765                               V2, TrueMask, VL);
2766     // Create 2^eltbits - 1 copies of V2 by multiplying by the largest integer.
2767     SDValue Multiplier = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntHalfVT,
2768                                      DAG.getAllOnesConstant(DL, XLenVT));
2769     SDValue WidenMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideIntContainerVT,
2770                                    V2, Multiplier, TrueMask, VL);
2771     // Add the new copies to our previous addition giving us 2^eltbits copies of
2772     // V2. This is equivalent to shifting V2 left by eltbits. This should
2773     // combine with the vwmulu.vv above to form vwmaccu.vv.
2774     Add = DAG.getNode(RISCVISD::ADD_VL, DL, WideIntContainerVT, Add, WidenMul,
2775                       TrueMask, VL);
2776     // Cast back to ContainerVT. We need to re-create a new ContainerVT in case
2777     // WideIntContainerVT is a larger fractional LMUL than implied by the fixed
2778     // vector VT.
2779     ContainerVT =
2780         MVT::getVectorVT(VT.getVectorElementType(),
2781                          WideIntContainerVT.getVectorElementCount() * 2);
2782     Add = DAG.getBitcast(ContainerVT, Add);
2783     return convertFromScalableVector(VT, Add, DAG, Subtarget);
2784   }
2785 
2786   // Detect shuffles which can be re-expressed as vector selects; these are
2787   // shuffles in which each element in the destination is taken from an element
2788   // at the corresponding index in either source vectors.
2789   bool IsSelect = all_of(enumerate(Mask), [&](const auto &MaskIdx) {
2790     int MaskIndex = MaskIdx.value();
2791     return MaskIndex < 0 || MaskIdx.index() == (unsigned)MaskIndex % NumElts;
2792   });
2793 
2794   assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
2795 
2796   SmallVector<SDValue> MaskVals;
2797   // As a backup, shuffles can be lowered via a vrgather instruction, possibly
2798   // merged with a second vrgather.
2799   SmallVector<SDValue> GatherIndicesLHS, GatherIndicesRHS;
2800 
2801   // By default we preserve the original operand order, and use a mask to
2802   // select LHS as true and RHS as false. However, since RVV vector selects may
2803   // feature splats but only on the LHS, we may choose to invert our mask and
2804   // instead select between RHS and LHS.
2805   bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
2806   bool InvertMask = IsSelect == SwapOps;
2807 
2808   // Keep a track of which non-undef indices are used by each LHS/RHS shuffle
2809   // half.
2810   DenseMap<int, unsigned> LHSIndexCounts, RHSIndexCounts;
2811 
2812   // Now construct the mask that will be used by the vselect or blended
2813   // vrgather operation. For vrgathers, construct the appropriate indices into
2814   // each vector.
2815   for (int MaskIndex : Mask) {
2816     bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ InvertMask;
2817     MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
2818     if (!IsSelect) {
2819       bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
2820       GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
2821                                      ? DAG.getConstant(MaskIndex, DL, XLenVT)
2822                                      : DAG.getUNDEF(XLenVT));
2823       GatherIndicesRHS.push_back(
2824           IsLHSOrUndefIndex ? DAG.getUNDEF(XLenVT)
2825                             : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT));
2826       if (IsLHSOrUndefIndex && MaskIndex >= 0)
2827         ++LHSIndexCounts[MaskIndex];
2828       if (!IsLHSOrUndefIndex)
2829         ++RHSIndexCounts[MaskIndex - NumElts];
2830     }
2831   }
2832 
2833   if (SwapOps) {
2834     std::swap(V1, V2);
2835     std::swap(GatherIndicesLHS, GatherIndicesRHS);
2836   }
2837 
2838   assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
2839   MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
2840   SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
2841 
2842   if (IsSelect)
2843     return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
2844 
2845   if (VT.getScalarSizeInBits() == 8 && VT.getVectorNumElements() > 256) {
2846     // On such a large vector we're unable to use i8 as the index type.
2847     // FIXME: We could promote the index to i16 and use vrgatherei16, but that
2848     // may involve vector splitting if we're already at LMUL=8, or our
2849     // user-supplied maximum fixed-length LMUL.
2850     return SDValue();
2851   }
2852 
2853   unsigned GatherVXOpc = RISCVISD::VRGATHER_VX_VL;
2854   unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
2855   MVT IndexVT = VT.changeTypeToInteger();
2856   // Since we can't introduce illegal index types at this stage, use i16 and
2857   // vrgatherei16 if the corresponding index type for plain vrgather is greater
2858   // than XLenVT.
2859   if (IndexVT.getScalarType().bitsGT(XLenVT)) {
2860     GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
2861     IndexVT = IndexVT.changeVectorElementType(MVT::i16);
2862   }
2863 
2864   MVT IndexContainerVT =
2865       ContainerVT.changeVectorElementType(IndexVT.getScalarType());
2866 
2867   SDValue Gather;
2868   // TODO: This doesn't trigger for i64 vectors on RV32, since there we
2869   // encounter a bitcasted BUILD_VECTOR with low/high i32 values.
2870   if (SDValue SplatValue = DAG.getSplatValue(V1, /*LegalTypes*/ true)) {
2871     Gather = lowerScalarSplat(SplatValue, VL, ContainerVT, DL, DAG, Subtarget);
2872   } else {
2873     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2874     // If only one index is used, we can use a "splat" vrgather.
2875     // TODO: We can splat the most-common index and fix-up any stragglers, if
2876     // that's beneficial.
2877     if (LHSIndexCounts.size() == 1) {
2878       int SplatIndex = LHSIndexCounts.begin()->getFirst();
2879       Gather =
2880           DAG.getNode(GatherVXOpc, DL, ContainerVT, V1,
2881                       DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2882     } else {
2883       SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
2884       LHSIndices =
2885           convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
2886 
2887       Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
2888                            TrueMask, VL);
2889     }
2890   }
2891 
2892   // If a second vector operand is used by this shuffle, blend it in with an
2893   // additional vrgather.
2894   if (!V2.isUndef()) {
2895     V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
2896     // If only one index is used, we can use a "splat" vrgather.
2897     // TODO: We can splat the most-common index and fix-up any stragglers, if
2898     // that's beneficial.
2899     if (RHSIndexCounts.size() == 1) {
2900       int SplatIndex = RHSIndexCounts.begin()->getFirst();
2901       V2 = DAG.getNode(GatherVXOpc, DL, ContainerVT, V2,
2902                        DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2903     } else {
2904       SDValue RHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesRHS);
2905       RHSIndices =
2906           convertToScalableVector(IndexContainerVT, RHSIndices, DAG, Subtarget);
2907       V2 = DAG.getNode(GatherVVOpc, DL, ContainerVT, V2, RHSIndices, TrueMask,
2908                        VL);
2909     }
2910 
2911     MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
2912     SelectMask =
2913         convertToScalableVector(MaskContainerVT, SelectMask, DAG, Subtarget);
2914 
2915     Gather = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, SelectMask, V2,
2916                          Gather, VL);
2917   }
2918 
2919   return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2920 }
2921 
2922 bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
2923   // Support splats for any type. These should type legalize well.
2924   if (ShuffleVectorSDNode::isSplatMask(M.data(), VT))
2925     return true;
2926 
2927   // Only support legal VTs for other shuffles for now.
2928   if (!isTypeLegal(VT))
2929     return false;
2930 
2931   MVT SVT = VT.getSimpleVT();
2932 
2933   bool SwapSources;
2934   return (matchShuffleAsSlideDown(M) >= 0) ||
2935          isInterleaveShuffle(M, SVT, SwapSources, Subtarget);
2936 }
2937 
2938 static SDValue getRVVFPExtendOrRound(SDValue Op, MVT VT, MVT ContainerVT,
2939                                      SDLoc DL, SelectionDAG &DAG,
2940                                      const RISCVSubtarget &Subtarget) {
2941   if (VT.isScalableVector())
2942     return DAG.getFPExtendOrRound(Op, DL, VT);
2943   assert(VT.isFixedLengthVector() &&
2944          "Unexpected value type for RVV FP extend/round lowering");
2945   SDValue Mask, VL;
2946   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2947   unsigned RVVOpc = ContainerVT.bitsGT(Op.getSimpleValueType())
2948                         ? RISCVISD::FP_EXTEND_VL
2949                         : RISCVISD::FP_ROUND_VL;
2950   return DAG.getNode(RVVOpc, DL, ContainerVT, Op, Mask, VL);
2951 }
2952 
2953 // Lower CTLZ_ZERO_UNDEF or CTTZ_ZERO_UNDEF by converting to FP and extracting
2954 // the exponent.
2955 static SDValue lowerCTLZ_CTTZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
2956   MVT VT = Op.getSimpleValueType();
2957   unsigned EltSize = VT.getScalarSizeInBits();
2958   SDValue Src = Op.getOperand(0);
2959   SDLoc DL(Op);
2960 
2961   // We need a FP type that can represent the value.
2962   // TODO: Use f16 for i8 when possible?
2963   MVT FloatEltVT = EltSize == 32 ? MVT::f64 : MVT::f32;
2964   MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
2965 
2966   // Legal types should have been checked in the RISCVTargetLowering
2967   // constructor.
2968   // TODO: Splitting may make sense in some cases.
2969   assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
2970          "Expected legal float type!");
2971 
2972   // For CTTZ_ZERO_UNDEF, we need to extract the lowest set bit using X & -X.
2973   // The trailing zero count is equal to log2 of this single bit value.
2974   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
2975     SDValue Neg =
2976         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
2977     Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
2978   }
2979 
2980   // We have a legal FP type, convert to it.
2981   SDValue FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
2982   // Bitcast to integer and shift the exponent to the LSB.
2983   EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
2984   SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
2985   unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
2986   SDValue Shift = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
2987                               DAG.getConstant(ShiftAmt, DL, IntVT));
2988   // Truncate back to original type to allow vnsrl.
2989   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, Shift);
2990   // The exponent contains log2 of the value in biased form.
2991   unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
2992 
2993   // For trailing zeros, we just need to subtract the bias.
2994   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF)
2995     return DAG.getNode(ISD::SUB, DL, VT, Trunc,
2996                        DAG.getConstant(ExponentBias, DL, VT));
2997 
2998   // For leading zeros, we need to remove the bias and convert from log2 to
2999   // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
3000   unsigned Adjust = ExponentBias + (EltSize - 1);
3001   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Trunc);
3002 }
3003 
3004 // While RVV has alignment restrictions, we should always be able to load as a
3005 // legal equivalently-sized byte-typed vector instead. This method is
3006 // responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
3007 // the load is already correctly-aligned, it returns SDValue().
3008 SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
3009                                                     SelectionDAG &DAG) const {
3010   auto *Load = cast<LoadSDNode>(Op);
3011   assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
3012 
3013   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
3014                                      Load->getMemoryVT(),
3015                                      *Load->getMemOperand()))
3016     return SDValue();
3017 
3018   SDLoc DL(Op);
3019   MVT VT = Op.getSimpleValueType();
3020   unsigned EltSizeBits = VT.getScalarSizeInBits();
3021   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
3022          "Unexpected unaligned RVV load type");
3023   MVT NewVT =
3024       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
3025   assert(NewVT.isValid() &&
3026          "Expecting equally-sized RVV vector types to be legal");
3027   SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
3028                           Load->getPointerInfo(), Load->getOriginalAlign(),
3029                           Load->getMemOperand()->getFlags());
3030   return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
3031 }
3032 
3033 // While RVV has alignment restrictions, we should always be able to store as a
3034 // legal equivalently-sized byte-typed vector instead. This method is
3035 // responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
3036 // returns SDValue() if the store is already correctly aligned.
3037 SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
3038                                                      SelectionDAG &DAG) const {
3039   auto *Store = cast<StoreSDNode>(Op);
3040   assert(Store && Store->getValue().getValueType().isVector() &&
3041          "Expected vector store");
3042 
3043   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
3044                                      Store->getMemoryVT(),
3045                                      *Store->getMemOperand()))
3046     return SDValue();
3047 
3048   SDLoc DL(Op);
3049   SDValue StoredVal = Store->getValue();
3050   MVT VT = StoredVal.getSimpleValueType();
3051   unsigned EltSizeBits = VT.getScalarSizeInBits();
3052   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
3053          "Unexpected unaligned RVV store type");
3054   MVT NewVT =
3055       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
3056   assert(NewVT.isValid() &&
3057          "Expecting equally-sized RVV vector types to be legal");
3058   StoredVal = DAG.getBitcast(NewVT, StoredVal);
3059   return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
3060                       Store->getPointerInfo(), Store->getOriginalAlign(),
3061                       Store->getMemOperand()->getFlags());
3062 }
3063 
3064 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
3065                                             SelectionDAG &DAG) const {
3066   switch (Op.getOpcode()) {
3067   default:
3068     report_fatal_error("unimplemented operand");
3069   case ISD::GlobalAddress:
3070     return lowerGlobalAddress(Op, DAG);
3071   case ISD::BlockAddress:
3072     return lowerBlockAddress(Op, DAG);
3073   case ISD::ConstantPool:
3074     return lowerConstantPool(Op, DAG);
3075   case ISD::JumpTable:
3076     return lowerJumpTable(Op, DAG);
3077   case ISD::GlobalTLSAddress:
3078     return lowerGlobalTLSAddress(Op, DAG);
3079   case ISD::SELECT:
3080     return lowerSELECT(Op, DAG);
3081   case ISD::BRCOND:
3082     return lowerBRCOND(Op, DAG);
3083   case ISD::VASTART:
3084     return lowerVASTART(Op, DAG);
3085   case ISD::FRAMEADDR:
3086     return lowerFRAMEADDR(Op, DAG);
3087   case ISD::RETURNADDR:
3088     return lowerRETURNADDR(Op, DAG);
3089   case ISD::SHL_PARTS:
3090     return lowerShiftLeftParts(Op, DAG);
3091   case ISD::SRA_PARTS:
3092     return lowerShiftRightParts(Op, DAG, true);
3093   case ISD::SRL_PARTS:
3094     return lowerShiftRightParts(Op, DAG, false);
3095   case ISD::BITCAST: {
3096     SDLoc DL(Op);
3097     EVT VT = Op.getValueType();
3098     SDValue Op0 = Op.getOperand(0);
3099     EVT Op0VT = Op0.getValueType();
3100     MVT XLenVT = Subtarget.getXLenVT();
3101     if (VT.isFixedLengthVector()) {
3102       // We can handle fixed length vector bitcasts with a simple replacement
3103       // in isel.
3104       if (Op0VT.isFixedLengthVector())
3105         return Op;
3106       // When bitcasting from scalar to fixed-length vector, insert the scalar
3107       // into a one-element vector of the result type, and perform a vector
3108       // bitcast.
3109       if (!Op0VT.isVector()) {
3110         EVT BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
3111         if (!isTypeLegal(BVT))
3112           return SDValue();
3113         return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
3114                                               DAG.getUNDEF(BVT), Op0,
3115                                               DAG.getConstant(0, DL, XLenVT)));
3116       }
3117       return SDValue();
3118     }
3119     // Custom-legalize bitcasts from fixed-length vector types to scalar types
3120     // thus: bitcast the vector to a one-element vector type whose element type
3121     // is the same as the result type, and extract the first element.
3122     if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
3123       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
3124       if (!isTypeLegal(BVT))
3125         return SDValue();
3126       SDValue BVec = DAG.getBitcast(BVT, Op0);
3127       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
3128                          DAG.getConstant(0, DL, XLenVT));
3129     }
3130     if (VT == MVT::f16 && Op0VT == MVT::i16 && Subtarget.hasStdExtZfh()) {
3131       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
3132       SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
3133       return FPConv;
3134     }
3135     if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
3136         Subtarget.hasStdExtF()) {
3137       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
3138       SDValue FPConv =
3139           DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
3140       return FPConv;
3141     }
3142     return SDValue();
3143   }
3144   case ISD::INTRINSIC_WO_CHAIN:
3145     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3146   case ISD::INTRINSIC_W_CHAIN:
3147     return LowerINTRINSIC_W_CHAIN(Op, DAG);
3148   case ISD::INTRINSIC_VOID:
3149     return LowerINTRINSIC_VOID(Op, DAG);
3150   case ISD::BSWAP:
3151   case ISD::BITREVERSE: {
3152     MVT VT = Op.getSimpleValueType();
3153     SDLoc DL(Op);
3154     if (Subtarget.hasStdExtZbp()) {
3155       // Convert BSWAP/BITREVERSE to GREVI to enable GREVI combinining.
3156       // Start with the maximum immediate value which is the bitwidth - 1.
3157       unsigned Imm = VT.getSizeInBits() - 1;
3158       // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
3159       if (Op.getOpcode() == ISD::BSWAP)
3160         Imm &= ~0x7U;
3161       return DAG.getNode(RISCVISD::GREV, DL, VT, Op.getOperand(0),
3162                          DAG.getConstant(Imm, DL, VT));
3163     }
3164     assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization");
3165     assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode");
3166     // Expand bitreverse to a bswap(rev8) followed by brev8.
3167     SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0));
3168     // We use the Zbp grevi encoding for rev.b/brev8 which will be recognized
3169     // as brev8 by an isel pattern.
3170     return DAG.getNode(RISCVISD::GREV, DL, VT, BSwap,
3171                        DAG.getConstant(7, DL, VT));
3172   }
3173   case ISD::FSHL:
3174   case ISD::FSHR: {
3175     MVT VT = Op.getSimpleValueType();
3176     assert(VT == Subtarget.getXLenVT() && "Unexpected custom legalization");
3177     SDLoc DL(Op);
3178     // FSL/FSR take a log2(XLen)+1 bit shift amount but XLenVT FSHL/FSHR only
3179     // use log(XLen) bits. Mask the shift amount accordingly to prevent
3180     // accidentally setting the extra bit.
3181     unsigned ShAmtWidth = Subtarget.getXLen() - 1;
3182     SDValue ShAmt = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(2),
3183                                 DAG.getConstant(ShAmtWidth, DL, VT));
3184     // fshl and fshr concatenate their operands in the same order. fsr and fsl
3185     // instruction use different orders. fshl will return its first operand for
3186     // shift of zero, fshr will return its second operand. fsl and fsr both
3187     // return rs1 so the ISD nodes need to have different operand orders.
3188     // Shift amount is in rs2.
3189     SDValue Op0 = Op.getOperand(0);
3190     SDValue Op1 = Op.getOperand(1);
3191     unsigned Opc = RISCVISD::FSL;
3192     if (Op.getOpcode() == ISD::FSHR) {
3193       std::swap(Op0, Op1);
3194       Opc = RISCVISD::FSR;
3195     }
3196     return DAG.getNode(Opc, DL, VT, Op0, Op1, ShAmt);
3197   }
3198   case ISD::TRUNCATE: {
3199     SDLoc DL(Op);
3200     MVT VT = Op.getSimpleValueType();
3201     // Only custom-lower vector truncates
3202     if (!VT.isVector())
3203       return Op;
3204 
3205     // Truncates to mask types are handled differently
3206     if (VT.getVectorElementType() == MVT::i1)
3207       return lowerVectorMaskTrunc(Op, DAG);
3208 
3209     // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
3210     // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
3211     // truncate by one power of two at a time.
3212     MVT DstEltVT = VT.getVectorElementType();
3213 
3214     SDValue Src = Op.getOperand(0);
3215     MVT SrcVT = Src.getSimpleValueType();
3216     MVT SrcEltVT = SrcVT.getVectorElementType();
3217 
3218     assert(DstEltVT.bitsLT(SrcEltVT) &&
3219            isPowerOf2_64(DstEltVT.getSizeInBits()) &&
3220            isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
3221            "Unexpected vector truncate lowering");
3222 
3223     MVT ContainerVT = SrcVT;
3224     if (SrcVT.isFixedLengthVector()) {
3225       ContainerVT = getContainerForFixedLengthVector(SrcVT);
3226       Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3227     }
3228 
3229     SDValue Result = Src;
3230     SDValue Mask, VL;
3231     std::tie(Mask, VL) =
3232         getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
3233     LLVMContext &Context = *DAG.getContext();
3234     const ElementCount Count = ContainerVT.getVectorElementCount();
3235     do {
3236       SrcEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
3237       EVT ResultVT = EVT::getVectorVT(Context, SrcEltVT, Count);
3238       Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
3239                            Mask, VL);
3240     } while (SrcEltVT != DstEltVT);
3241 
3242     if (SrcVT.isFixedLengthVector())
3243       Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
3244 
3245     return Result;
3246   }
3247   case ISD::ANY_EXTEND:
3248   case ISD::ZERO_EXTEND:
3249     if (Op.getOperand(0).getValueType().isVector() &&
3250         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3251       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
3252     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
3253   case ISD::SIGN_EXTEND:
3254     if (Op.getOperand(0).getValueType().isVector() &&
3255         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3256       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
3257     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
3258   case ISD::SPLAT_VECTOR_PARTS:
3259     return lowerSPLAT_VECTOR_PARTS(Op, DAG);
3260   case ISD::INSERT_VECTOR_ELT:
3261     return lowerINSERT_VECTOR_ELT(Op, DAG);
3262   case ISD::EXTRACT_VECTOR_ELT:
3263     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
3264   case ISD::VSCALE: {
3265     MVT VT = Op.getSimpleValueType();
3266     SDLoc DL(Op);
3267     SDValue VLENB = DAG.getNode(RISCVISD::READ_VLENB, DL, VT);
3268     // We define our scalable vector types for lmul=1 to use a 64 bit known
3269     // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
3270     // vscale as VLENB / 8.
3271     static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
3272     if (Subtarget.getMinVLen() < RISCV::RVVBitsPerBlock)
3273       report_fatal_error("Support for VLEN==32 is incomplete.");
3274     if (isa<ConstantSDNode>(Op.getOperand(0))) {
3275       // We assume VLENB is a multiple of 8. We manually choose the best shift
3276       // here because SimplifyDemandedBits isn't always able to simplify it.
3277       uint64_t Val = Op.getConstantOperandVal(0);
3278       if (isPowerOf2_64(Val)) {
3279         uint64_t Log2 = Log2_64(Val);
3280         if (Log2 < 3)
3281           return DAG.getNode(ISD::SRL, DL, VT, VLENB,
3282                              DAG.getConstant(3 - Log2, DL, VT));
3283         if (Log2 > 3)
3284           return DAG.getNode(ISD::SHL, DL, VT, VLENB,
3285                              DAG.getConstant(Log2 - 3, DL, VT));
3286         return VLENB;
3287       }
3288       // If the multiplier is a multiple of 8, scale it down to avoid needing
3289       // to shift the VLENB value.
3290       if ((Val % 8) == 0)
3291         return DAG.getNode(ISD::MUL, DL, VT, VLENB,
3292                            DAG.getConstant(Val / 8, DL, VT));
3293     }
3294 
3295     SDValue VScale = DAG.getNode(ISD::SRL, DL, VT, VLENB,
3296                                  DAG.getConstant(3, DL, VT));
3297     return DAG.getNode(ISD::MUL, DL, VT, VScale, Op.getOperand(0));
3298   }
3299   case ISD::FPOWI: {
3300     // Custom promote f16 powi with illegal i32 integer type on RV64. Once
3301     // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
3302     if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
3303         Op.getOperand(1).getValueType() == MVT::i32) {
3304       SDLoc DL(Op);
3305       SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
3306       SDValue Powi =
3307           DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
3308       return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
3309                          DAG.getIntPtrConstant(0, DL));
3310     }
3311     return SDValue();
3312   }
3313   case ISD::FP_EXTEND: {
3314     // RVV can only do fp_extend to types double the size as the source. We
3315     // custom-lower f16->f64 extensions to two hops of ISD::FP_EXTEND, going
3316     // via f32.
3317     SDLoc DL(Op);
3318     MVT VT = Op.getSimpleValueType();
3319     SDValue Src = Op.getOperand(0);
3320     MVT SrcVT = Src.getSimpleValueType();
3321 
3322     // Prepare any fixed-length vector operands.
3323     MVT ContainerVT = VT;
3324     if (SrcVT.isFixedLengthVector()) {
3325       ContainerVT = getContainerForFixedLengthVector(VT);
3326       MVT SrcContainerVT =
3327           ContainerVT.changeVectorElementType(SrcVT.getVectorElementType());
3328       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3329     }
3330 
3331     if (!VT.isVector() || VT.getVectorElementType() != MVT::f64 ||
3332         SrcVT.getVectorElementType() != MVT::f16) {
3333       // For scalable vectors, we only need to close the gap between
3334       // vXf16->vXf64.
3335       if (!VT.isFixedLengthVector())
3336         return Op;
3337       // For fixed-length vectors, lower the FP_EXTEND to a custom "VL" version.
3338       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
3339       return convertFromScalableVector(VT, Src, DAG, Subtarget);
3340     }
3341 
3342     MVT InterVT = VT.changeVectorElementType(MVT::f32);
3343     MVT InterContainerVT = ContainerVT.changeVectorElementType(MVT::f32);
3344     SDValue IntermediateExtend = getRVVFPExtendOrRound(
3345         Src, InterVT, InterContainerVT, DL, DAG, Subtarget);
3346 
3347     SDValue Extend = getRVVFPExtendOrRound(IntermediateExtend, VT, ContainerVT,
3348                                            DL, DAG, Subtarget);
3349     if (VT.isFixedLengthVector())
3350       return convertFromScalableVector(VT, Extend, DAG, Subtarget);
3351     return Extend;
3352   }
3353   case ISD::FP_ROUND: {
3354     // RVV can only do fp_round to types half the size as the source. We
3355     // custom-lower f64->f16 rounds via RVV's round-to-odd float
3356     // conversion instruction.
3357     SDLoc DL(Op);
3358     MVT VT = Op.getSimpleValueType();
3359     SDValue Src = Op.getOperand(0);
3360     MVT SrcVT = Src.getSimpleValueType();
3361 
3362     // Prepare any fixed-length vector operands.
3363     MVT ContainerVT = VT;
3364     if (VT.isFixedLengthVector()) {
3365       MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3366       ContainerVT =
3367           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3368       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3369     }
3370 
3371     if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
3372         SrcVT.getVectorElementType() != MVT::f64) {
3373       // For scalable vectors, we only need to close the gap between
3374       // vXf64<->vXf16.
3375       if (!VT.isFixedLengthVector())
3376         return Op;
3377       // For fixed-length vectors, lower the FP_ROUND to a custom "VL" version.
3378       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
3379       return convertFromScalableVector(VT, Src, DAG, Subtarget);
3380     }
3381 
3382     SDValue Mask, VL;
3383     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3384 
3385     MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
3386     SDValue IntermediateRound =
3387         DAG.getNode(RISCVISD::VFNCVT_ROD_VL, DL, InterVT, Src, Mask, VL);
3388     SDValue Round = getRVVFPExtendOrRound(IntermediateRound, VT, ContainerVT,
3389                                           DL, DAG, Subtarget);
3390 
3391     if (VT.isFixedLengthVector())
3392       return convertFromScalableVector(VT, Round, DAG, Subtarget);
3393     return Round;
3394   }
3395   case ISD::FP_TO_SINT:
3396   case ISD::FP_TO_UINT:
3397   case ISD::SINT_TO_FP:
3398   case ISD::UINT_TO_FP: {
3399     // RVV can only do fp<->int conversions to types half/double the size as
3400     // the source. We custom-lower any conversions that do two hops into
3401     // sequences.
3402     MVT VT = Op.getSimpleValueType();
3403     if (!VT.isVector())
3404       return Op;
3405     SDLoc DL(Op);
3406     SDValue Src = Op.getOperand(0);
3407     MVT EltVT = VT.getVectorElementType();
3408     MVT SrcVT = Src.getSimpleValueType();
3409     MVT SrcEltVT = SrcVT.getVectorElementType();
3410     unsigned EltSize = EltVT.getSizeInBits();
3411     unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3412     assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
3413            "Unexpected vector element types");
3414 
3415     bool IsInt2FP = SrcEltVT.isInteger();
3416     // Widening conversions
3417     if (EltSize > SrcEltSize && (EltSize / SrcEltSize >= 4)) {
3418       if (IsInt2FP) {
3419         // Do a regular integer sign/zero extension then convert to float.
3420         MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltVT.getSizeInBits()),
3421                                       VT.getVectorElementCount());
3422         unsigned ExtOpcode = Op.getOpcode() == ISD::UINT_TO_FP
3423                                  ? ISD::ZERO_EXTEND
3424                                  : ISD::SIGN_EXTEND;
3425         SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
3426         return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
3427       }
3428       // FP2Int
3429       assert(SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering");
3430       // Do one doubling fp_extend then complete the operation by converting
3431       // to int.
3432       MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3433       SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
3434       return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
3435     }
3436 
3437     // Narrowing conversions
3438     if (SrcEltSize > EltSize && (SrcEltSize / EltSize >= 4)) {
3439       if (IsInt2FP) {
3440         // One narrowing int_to_fp, then an fp_round.
3441         assert(EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering");
3442         MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3443         SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
3444         return DAG.getFPExtendOrRound(Int2FP, DL, VT);
3445       }
3446       // FP2Int
3447       // One narrowing fp_to_int, then truncate the integer. If the float isn't
3448       // representable by the integer, the result is poison.
3449       MVT IVecVT =
3450           MVT::getVectorVT(MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2),
3451                            VT.getVectorElementCount());
3452       SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
3453       return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
3454     }
3455 
3456     // Scalable vectors can exit here. Patterns will handle equally-sized
3457     // conversions halving/doubling ones.
3458     if (!VT.isFixedLengthVector())
3459       return Op;
3460 
3461     // For fixed-length vectors we lower to a custom "VL" node.
3462     unsigned RVVOpc = 0;
3463     switch (Op.getOpcode()) {
3464     default:
3465       llvm_unreachable("Impossible opcode");
3466     case ISD::FP_TO_SINT:
3467       RVVOpc = RISCVISD::FP_TO_SINT_VL;
3468       break;
3469     case ISD::FP_TO_UINT:
3470       RVVOpc = RISCVISD::FP_TO_UINT_VL;
3471       break;
3472     case ISD::SINT_TO_FP:
3473       RVVOpc = RISCVISD::SINT_TO_FP_VL;
3474       break;
3475     case ISD::UINT_TO_FP:
3476       RVVOpc = RISCVISD::UINT_TO_FP_VL;
3477       break;
3478     }
3479 
3480     MVT ContainerVT, SrcContainerVT;
3481     // Derive the reference container type from the larger vector type.
3482     if (SrcEltSize > EltSize) {
3483       SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3484       ContainerVT =
3485           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3486     } else {
3487       ContainerVT = getContainerForFixedLengthVector(VT);
3488       SrcContainerVT = ContainerVT.changeVectorElementType(SrcEltVT);
3489     }
3490 
3491     SDValue Mask, VL;
3492     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3493 
3494     Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3495     Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
3496     return convertFromScalableVector(VT, Src, DAG, Subtarget);
3497   }
3498   case ISD::FP_TO_SINT_SAT:
3499   case ISD::FP_TO_UINT_SAT:
3500     return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
3501   case ISD::FTRUNC:
3502   case ISD::FCEIL:
3503   case ISD::FFLOOR:
3504     return lowerFTRUNC_FCEIL_FFLOOR(Op, DAG);
3505   case ISD::FROUND:
3506     return lowerFROUND(Op, DAG);
3507   case ISD::VECREDUCE_ADD:
3508   case ISD::VECREDUCE_UMAX:
3509   case ISD::VECREDUCE_SMAX:
3510   case ISD::VECREDUCE_UMIN:
3511   case ISD::VECREDUCE_SMIN:
3512     return lowerVECREDUCE(Op, DAG);
3513   case ISD::VECREDUCE_AND:
3514   case ISD::VECREDUCE_OR:
3515   case ISD::VECREDUCE_XOR:
3516     if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3517       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
3518     return lowerVECREDUCE(Op, DAG);
3519   case ISD::VECREDUCE_FADD:
3520   case ISD::VECREDUCE_SEQ_FADD:
3521   case ISD::VECREDUCE_FMIN:
3522   case ISD::VECREDUCE_FMAX:
3523     return lowerFPVECREDUCE(Op, DAG);
3524   case ISD::VP_REDUCE_ADD:
3525   case ISD::VP_REDUCE_UMAX:
3526   case ISD::VP_REDUCE_SMAX:
3527   case ISD::VP_REDUCE_UMIN:
3528   case ISD::VP_REDUCE_SMIN:
3529   case ISD::VP_REDUCE_FADD:
3530   case ISD::VP_REDUCE_SEQ_FADD:
3531   case ISD::VP_REDUCE_FMIN:
3532   case ISD::VP_REDUCE_FMAX:
3533     return lowerVPREDUCE(Op, DAG);
3534   case ISD::VP_REDUCE_AND:
3535   case ISD::VP_REDUCE_OR:
3536   case ISD::VP_REDUCE_XOR:
3537     if (Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
3538       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
3539     return lowerVPREDUCE(Op, DAG);
3540   case ISD::INSERT_SUBVECTOR:
3541     return lowerINSERT_SUBVECTOR(Op, DAG);
3542   case ISD::EXTRACT_SUBVECTOR:
3543     return lowerEXTRACT_SUBVECTOR(Op, DAG);
3544   case ISD::STEP_VECTOR:
3545     return lowerSTEP_VECTOR(Op, DAG);
3546   case ISD::VECTOR_REVERSE:
3547     return lowerVECTOR_REVERSE(Op, DAG);
3548   case ISD::BUILD_VECTOR:
3549     return lowerBUILD_VECTOR(Op, DAG, Subtarget);
3550   case ISD::SPLAT_VECTOR:
3551     if (Op.getValueType().getVectorElementType() == MVT::i1)
3552       return lowerVectorMaskSplat(Op, DAG);
3553     return lowerSPLAT_VECTOR(Op, DAG, Subtarget);
3554   case ISD::VECTOR_SHUFFLE:
3555     return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
3556   case ISD::CONCAT_VECTORS: {
3557     // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
3558     // better than going through the stack, as the default expansion does.
3559     SDLoc DL(Op);
3560     MVT VT = Op.getSimpleValueType();
3561     unsigned NumOpElts =
3562         Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
3563     SDValue Vec = DAG.getUNDEF(VT);
3564     for (const auto &OpIdx : enumerate(Op->ops())) {
3565       SDValue SubVec = OpIdx.value();
3566       // Don't insert undef subvectors.
3567       if (SubVec.isUndef())
3568         continue;
3569       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec,
3570                         DAG.getIntPtrConstant(OpIdx.index() * NumOpElts, DL));
3571     }
3572     return Vec;
3573   }
3574   case ISD::LOAD:
3575     if (auto V = expandUnalignedRVVLoad(Op, DAG))
3576       return V;
3577     if (Op.getValueType().isFixedLengthVector())
3578       return lowerFixedLengthVectorLoadToRVV(Op, DAG);
3579     return Op;
3580   case ISD::STORE:
3581     if (auto V = expandUnalignedRVVStore(Op, DAG))
3582       return V;
3583     if (Op.getOperand(1).getValueType().isFixedLengthVector())
3584       return lowerFixedLengthVectorStoreToRVV(Op, DAG);
3585     return Op;
3586   case ISD::MLOAD:
3587   case ISD::VP_LOAD:
3588     return lowerMaskedLoad(Op, DAG);
3589   case ISD::MSTORE:
3590   case ISD::VP_STORE:
3591     return lowerMaskedStore(Op, DAG);
3592   case ISD::SETCC:
3593     return lowerFixedLengthVectorSetccToRVV(Op, DAG);
3594   case ISD::ADD:
3595     return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);
3596   case ISD::SUB:
3597     return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL);
3598   case ISD::MUL:
3599     return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL);
3600   case ISD::MULHS:
3601     return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL);
3602   case ISD::MULHU:
3603     return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL);
3604   case ISD::AND:
3605     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMAND_VL,
3606                                               RISCVISD::AND_VL);
3607   case ISD::OR:
3608     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMOR_VL,
3609                                               RISCVISD::OR_VL);
3610   case ISD::XOR:
3611     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMXOR_VL,
3612                                               RISCVISD::XOR_VL);
3613   case ISD::SDIV:
3614     return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL);
3615   case ISD::SREM:
3616     return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL);
3617   case ISD::UDIV:
3618     return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL);
3619   case ISD::UREM:
3620     return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL);
3621   case ISD::SHL:
3622   case ISD::SRA:
3623   case ISD::SRL:
3624     if (Op.getSimpleValueType().isFixedLengthVector())
3625       return lowerFixedLengthVectorShiftToRVV(Op, DAG);
3626     // This can be called for an i32 shift amount that needs to be promoted.
3627     assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
3628            "Unexpected custom legalisation");
3629     return SDValue();
3630   case ISD::SADDSAT:
3631     return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL);
3632   case ISD::UADDSAT:
3633     return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL);
3634   case ISD::SSUBSAT:
3635     return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL);
3636   case ISD::USUBSAT:
3637     return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL);
3638   case ISD::FADD:
3639     return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL);
3640   case ISD::FSUB:
3641     return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL);
3642   case ISD::FMUL:
3643     return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL);
3644   case ISD::FDIV:
3645     return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL);
3646   case ISD::FNEG:
3647     return lowerToScalableOp(Op, DAG, RISCVISD::FNEG_VL);
3648   case ISD::FABS:
3649     return lowerToScalableOp(Op, DAG, RISCVISD::FABS_VL);
3650   case ISD::FSQRT:
3651     return lowerToScalableOp(Op, DAG, RISCVISD::FSQRT_VL);
3652   case ISD::FMA:
3653     return lowerToScalableOp(Op, DAG, RISCVISD::FMA_VL);
3654   case ISD::SMIN:
3655     return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL);
3656   case ISD::SMAX:
3657     return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL);
3658   case ISD::UMIN:
3659     return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL);
3660   case ISD::UMAX:
3661     return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL);
3662   case ISD::FMINNUM:
3663     return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL);
3664   case ISD::FMAXNUM:
3665     return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL);
3666   case ISD::ABS:
3667     return lowerABS(Op, DAG);
3668   case ISD::CTLZ_ZERO_UNDEF:
3669   case ISD::CTTZ_ZERO_UNDEF:
3670     return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
3671   case ISD::VSELECT:
3672     return lowerFixedLengthVectorSelectToRVV(Op, DAG);
3673   case ISD::FCOPYSIGN:
3674     return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG);
3675   case ISD::MGATHER:
3676   case ISD::VP_GATHER:
3677     return lowerMaskedGather(Op, DAG);
3678   case ISD::MSCATTER:
3679   case ISD::VP_SCATTER:
3680     return lowerMaskedScatter(Op, DAG);
3681   case ISD::FLT_ROUNDS_:
3682     return lowerGET_ROUNDING(Op, DAG);
3683   case ISD::SET_ROUNDING:
3684     return lowerSET_ROUNDING(Op, DAG);
3685   case ISD::VP_SELECT:
3686     return lowerVPOp(Op, DAG, RISCVISD::VSELECT_VL);
3687   case ISD::VP_MERGE:
3688     return lowerVPOp(Op, DAG, RISCVISD::VP_MERGE_VL);
3689   case ISD::VP_ADD:
3690     return lowerVPOp(Op, DAG, RISCVISD::ADD_VL);
3691   case ISD::VP_SUB:
3692     return lowerVPOp(Op, DAG, RISCVISD::SUB_VL);
3693   case ISD::VP_MUL:
3694     return lowerVPOp(Op, DAG, RISCVISD::MUL_VL);
3695   case ISD::VP_SDIV:
3696     return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL);
3697   case ISD::VP_UDIV:
3698     return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL);
3699   case ISD::VP_SREM:
3700     return lowerVPOp(Op, DAG, RISCVISD::SREM_VL);
3701   case ISD::VP_UREM:
3702     return lowerVPOp(Op, DAG, RISCVISD::UREM_VL);
3703   case ISD::VP_AND:
3704     return lowerLogicVPOp(Op, DAG, RISCVISD::VMAND_VL, RISCVISD::AND_VL);
3705   case ISD::VP_OR:
3706     return lowerLogicVPOp(Op, DAG, RISCVISD::VMOR_VL, RISCVISD::OR_VL);
3707   case ISD::VP_XOR:
3708     return lowerLogicVPOp(Op, DAG, RISCVISD::VMXOR_VL, RISCVISD::XOR_VL);
3709   case ISD::VP_ASHR:
3710     return lowerVPOp(Op, DAG, RISCVISD::SRA_VL);
3711   case ISD::VP_LSHR:
3712     return lowerVPOp(Op, DAG, RISCVISD::SRL_VL);
3713   case ISD::VP_SHL:
3714     return lowerVPOp(Op, DAG, RISCVISD::SHL_VL);
3715   case ISD::VP_FADD:
3716     return lowerVPOp(Op, DAG, RISCVISD::FADD_VL);
3717   case ISD::VP_FSUB:
3718     return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL);
3719   case ISD::VP_FMUL:
3720     return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL);
3721   case ISD::VP_FDIV:
3722     return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL);
3723   case ISD::VP_FNEG:
3724     return lowerVPOp(Op, DAG, RISCVISD::FNEG_VL);
3725   case ISD::VP_FMA:
3726     return lowerVPOp(Op, DAG, RISCVISD::FMA_VL);
3727   }
3728 }
3729 
3730 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
3731                              SelectionDAG &DAG, unsigned Flags) {
3732   return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
3733 }
3734 
3735 static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
3736                              SelectionDAG &DAG, unsigned Flags) {
3737   return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
3738                                    Flags);
3739 }
3740 
3741 static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
3742                              SelectionDAG &DAG, unsigned Flags) {
3743   return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
3744                                    N->getOffset(), Flags);
3745 }
3746 
3747 static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
3748                              SelectionDAG &DAG, unsigned Flags) {
3749   return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
3750 }
3751 
3752 template <class NodeTy>
3753 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
3754                                      bool IsLocal) const {
3755   SDLoc DL(N);
3756   EVT Ty = getPointerTy(DAG.getDataLayout());
3757 
3758   if (isPositionIndependent()) {
3759     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3760     if (IsLocal)
3761       // Use PC-relative addressing to access the symbol. This generates the
3762       // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
3763       // %pcrel_lo(auipc)).
3764       return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3765 
3766     // Use PC-relative addressing to access the GOT for this symbol, then load
3767     // the address from the GOT. This generates the pattern (PseudoLA sym),
3768     // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
3769     return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
3770   }
3771 
3772   switch (getTargetMachine().getCodeModel()) {
3773   default:
3774     report_fatal_error("Unsupported code model for lowering");
3775   case CodeModel::Small: {
3776     // Generate a sequence for accessing addresses within the first 2 GiB of
3777     // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
3778     SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
3779     SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
3780     SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3781     return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
3782   }
3783   case CodeModel::Medium: {
3784     // Generate a sequence for accessing addresses within any 2GiB range within
3785     // the address space. This generates the pattern (PseudoLLA sym), which
3786     // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
3787     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3788     return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3789   }
3790   }
3791 }
3792 
3793 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
3794                                                 SelectionDAG &DAG) const {
3795   SDLoc DL(Op);
3796   EVT Ty = Op.getValueType();
3797   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3798   int64_t Offset = N->getOffset();
3799   MVT XLenVT = Subtarget.getXLenVT();
3800 
3801   const GlobalValue *GV = N->getGlobal();
3802   bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
3803   SDValue Addr = getAddr(N, DAG, IsLocal);
3804 
3805   // In order to maximise the opportunity for common subexpression elimination,
3806   // emit a separate ADD node for the global address offset instead of folding
3807   // it in the global address node. Later peephole optimisations may choose to
3808   // fold it back in when profitable.
3809   if (Offset != 0)
3810     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3811                        DAG.getConstant(Offset, DL, XLenVT));
3812   return Addr;
3813 }
3814 
3815 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
3816                                                SelectionDAG &DAG) const {
3817   BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
3818 
3819   return getAddr(N, DAG);
3820 }
3821 
3822 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
3823                                                SelectionDAG &DAG) const {
3824   ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
3825 
3826   return getAddr(N, DAG);
3827 }
3828 
3829 SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
3830                                             SelectionDAG &DAG) const {
3831   JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
3832 
3833   return getAddr(N, DAG);
3834 }
3835 
3836 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
3837                                               SelectionDAG &DAG,
3838                                               bool UseGOT) const {
3839   SDLoc DL(N);
3840   EVT Ty = getPointerTy(DAG.getDataLayout());
3841   const GlobalValue *GV = N->getGlobal();
3842   MVT XLenVT = Subtarget.getXLenVT();
3843 
3844   if (UseGOT) {
3845     // Use PC-relative addressing to access the GOT for this TLS symbol, then
3846     // load the address from the GOT and add the thread pointer. This generates
3847     // the pattern (PseudoLA_TLS_IE sym), which expands to
3848     // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
3849     SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3850     SDValue Load =
3851         SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
3852 
3853     // Add the thread pointer.
3854     SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3855     return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
3856   }
3857 
3858   // Generate a sequence for accessing the address relative to the thread
3859   // pointer, with the appropriate adjustment for the thread pointer offset.
3860   // This generates the pattern
3861   // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
3862   SDValue AddrHi =
3863       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
3864   SDValue AddrAdd =
3865       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
3866   SDValue AddrLo =
3867       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
3868 
3869   SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3870   SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3871   SDValue MNAdd = SDValue(
3872       DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
3873       0);
3874   return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
3875 }
3876 
3877 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
3878                                                SelectionDAG &DAG) const {
3879   SDLoc DL(N);
3880   EVT Ty = getPointerTy(DAG.getDataLayout());
3881   IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
3882   const GlobalValue *GV = N->getGlobal();
3883 
3884   // Use a PC-relative addressing mode to access the global dynamic GOT address.
3885   // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
3886   // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
3887   SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3888   SDValue Load =
3889       SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
3890 
3891   // Prepare argument list to generate call.
3892   ArgListTy Args;
3893   ArgListEntry Entry;
3894   Entry.Node = Load;
3895   Entry.Ty = CallTy;
3896   Args.push_back(Entry);
3897 
3898   // Setup call to __tls_get_addr.
3899   TargetLowering::CallLoweringInfo CLI(DAG);
3900   CLI.setDebugLoc(DL)
3901       .setChain(DAG.getEntryNode())
3902       .setLibCallee(CallingConv::C, CallTy,
3903                     DAG.getExternalSymbol("__tls_get_addr", Ty),
3904                     std::move(Args));
3905 
3906   return LowerCallTo(CLI).first;
3907 }
3908 
3909 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
3910                                                    SelectionDAG &DAG) const {
3911   SDLoc DL(Op);
3912   EVT Ty = Op.getValueType();
3913   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3914   int64_t Offset = N->getOffset();
3915   MVT XLenVT = Subtarget.getXLenVT();
3916 
3917   TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());
3918 
3919   if (DAG.getMachineFunction().getFunction().getCallingConv() ==
3920       CallingConv::GHC)
3921     report_fatal_error("In GHC calling convention TLS is not supported");
3922 
3923   SDValue Addr;
3924   switch (Model) {
3925   case TLSModel::LocalExec:
3926     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
3927     break;
3928   case TLSModel::InitialExec:
3929     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
3930     break;
3931   case TLSModel::LocalDynamic:
3932   case TLSModel::GeneralDynamic:
3933     Addr = getDynamicTLSAddr(N, DAG);
3934     break;
3935   }
3936 
3937   // In order to maximise the opportunity for common subexpression elimination,
3938   // emit a separate ADD node for the global address offset instead of folding
3939   // it in the global address node. Later peephole optimisations may choose to
3940   // fold it back in when profitable.
3941   if (Offset != 0)
3942     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3943                        DAG.getConstant(Offset, DL, XLenVT));
3944   return Addr;
3945 }
3946 
3947 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
3948   SDValue CondV = Op.getOperand(0);
3949   SDValue TrueV = Op.getOperand(1);
3950   SDValue FalseV = Op.getOperand(2);
3951   SDLoc DL(Op);
3952   MVT VT = Op.getSimpleValueType();
3953   MVT XLenVT = Subtarget.getXLenVT();
3954 
3955   // Lower vector SELECTs to VSELECTs by splatting the condition.
3956   if (VT.isVector()) {
3957     MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
3958     SDValue CondSplat = VT.isScalableVector()
3959                             ? DAG.getSplatVector(SplatCondVT, DL, CondV)
3960                             : DAG.getSplatBuildVector(SplatCondVT, DL, CondV);
3961     return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
3962   }
3963 
3964   // If the result type is XLenVT and CondV is the output of a SETCC node
3965   // which also operated on XLenVT inputs, then merge the SETCC node into the
3966   // lowered RISCVISD::SELECT_CC to take advantage of the integer
3967   // compare+branch instructions. i.e.:
3968   // (select (setcc lhs, rhs, cc), truev, falsev)
3969   // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
3970   if (VT == XLenVT && CondV.getOpcode() == ISD::SETCC &&
3971       CondV.getOperand(0).getSimpleValueType() == XLenVT) {
3972     SDValue LHS = CondV.getOperand(0);
3973     SDValue RHS = CondV.getOperand(1);
3974     const auto *CC = cast<CondCodeSDNode>(CondV.getOperand(2));
3975     ISD::CondCode CCVal = CC->get();
3976 
3977     // Special case for a select of 2 constants that have a diffence of 1.
3978     // Normally this is done by DAGCombine, but if the select is introduced by
3979     // type legalization or op legalization, we miss it. Restricting to SETLT
3980     // case for now because that is what signed saturating add/sub need.
3981     // FIXME: We don't need the condition to be SETLT or even a SETCC,
3982     // but we would probably want to swap the true/false values if the condition
3983     // is SETGE/SETLE to avoid an XORI.
3984     if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
3985         CCVal == ISD::SETLT) {
3986       const APInt &TrueVal = cast<ConstantSDNode>(TrueV)->getAPIntValue();
3987       const APInt &FalseVal = cast<ConstantSDNode>(FalseV)->getAPIntValue();
3988       if (TrueVal - 1 == FalseVal)
3989         return DAG.getNode(ISD::ADD, DL, Op.getValueType(), CondV, FalseV);
3990       if (TrueVal + 1 == FalseVal)
3991         return DAG.getNode(ISD::SUB, DL, Op.getValueType(), FalseV, CondV);
3992     }
3993 
3994     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3995 
3996     SDValue TargetCC = DAG.getCondCode(CCVal);
3997     SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
3998     return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3999   }
4000 
4001   // Otherwise:
4002   // (select condv, truev, falsev)
4003   // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
4004   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4005   SDValue SetNE = DAG.getCondCode(ISD::SETNE);
4006 
4007   SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
4008 
4009   return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
4010 }
4011 
4012 SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
4013   SDValue CondV = Op.getOperand(1);
4014   SDLoc DL(Op);
4015   MVT XLenVT = Subtarget.getXLenVT();
4016 
4017   if (CondV.getOpcode() == ISD::SETCC &&
4018       CondV.getOperand(0).getValueType() == XLenVT) {
4019     SDValue LHS = CondV.getOperand(0);
4020     SDValue RHS = CondV.getOperand(1);
4021     ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
4022 
4023     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
4024 
4025     SDValue TargetCC = DAG.getCondCode(CCVal);
4026     return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
4027                        LHS, RHS, TargetCC, Op.getOperand(2));
4028   }
4029 
4030   return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
4031                      CondV, DAG.getConstant(0, DL, XLenVT),
4032                      DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
4033 }
4034 
4035 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
4036   MachineFunction &MF = DAG.getMachineFunction();
4037   RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
4038 
4039   SDLoc DL(Op);
4040   SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
4041                                  getPointerTy(MF.getDataLayout()));
4042 
4043   // vastart just stores the address of the VarArgsFrameIndex slot into the
4044   // memory location argument.
4045   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4046   return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
4047                       MachinePointerInfo(SV));
4048 }
4049 
4050 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
4051                                             SelectionDAG &DAG) const {
4052   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
4053   MachineFunction &MF = DAG.getMachineFunction();
4054   MachineFrameInfo &MFI = MF.getFrameInfo();
4055   MFI.setFrameAddressIsTaken(true);
4056   Register FrameReg = RI.getFrameRegister(MF);
4057   int XLenInBytes = Subtarget.getXLen() / 8;
4058 
4059   EVT VT = Op.getValueType();
4060   SDLoc DL(Op);
4061   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
4062   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
4063   while (Depth--) {
4064     int Offset = -(XLenInBytes * 2);
4065     SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
4066                               DAG.getIntPtrConstant(Offset, DL));
4067     FrameAddr =
4068         DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
4069   }
4070   return FrameAddr;
4071 }
4072 
4073 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
4074                                              SelectionDAG &DAG) const {
4075   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
4076   MachineFunction &MF = DAG.getMachineFunction();
4077   MachineFrameInfo &MFI = MF.getFrameInfo();
4078   MFI.setReturnAddressIsTaken(true);
4079   MVT XLenVT = Subtarget.getXLenVT();
4080   int XLenInBytes = Subtarget.getXLen() / 8;
4081 
4082   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
4083     return SDValue();
4084 
4085   EVT VT = Op.getValueType();
4086   SDLoc DL(Op);
4087   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
4088   if (Depth) {
4089     int Off = -XLenInBytes;
4090     SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
4091     SDValue Offset = DAG.getConstant(Off, DL, VT);
4092     return DAG.getLoad(VT, DL, DAG.getEntryNode(),
4093                        DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
4094                        MachinePointerInfo());
4095   }
4096 
4097   // Return the value of the return address register, marking it an implicit
4098   // live-in.
4099   Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
4100   return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
4101 }
4102 
4103 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
4104                                                  SelectionDAG &DAG) const {
4105   SDLoc DL(Op);
4106   SDValue Lo = Op.getOperand(0);
4107   SDValue Hi = Op.getOperand(1);
4108   SDValue Shamt = Op.getOperand(2);
4109   EVT VT = Lo.getValueType();
4110 
4111   // if Shamt-XLEN < 0: // Shamt < XLEN
4112   //   Lo = Lo << Shamt
4113   //   Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
4114   // else:
4115   //   Lo = 0
4116   //   Hi = Lo << (Shamt-XLEN)
4117 
4118   SDValue Zero = DAG.getConstant(0, DL, VT);
4119   SDValue One = DAG.getConstant(1, DL, VT);
4120   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
4121   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
4122   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
4123   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
4124 
4125   SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
4126   SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
4127   SDValue ShiftRightLo =
4128       DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
4129   SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
4130   SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
4131   SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
4132 
4133   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
4134 
4135   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
4136   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
4137 
4138   SDValue Parts[2] = {Lo, Hi};
4139   return DAG.getMergeValues(Parts, DL);
4140 }
4141 
4142 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
4143                                                   bool IsSRA) const {
4144   SDLoc DL(Op);
4145   SDValue Lo = Op.getOperand(0);
4146   SDValue Hi = Op.getOperand(1);
4147   SDValue Shamt = Op.getOperand(2);
4148   EVT VT = Lo.getValueType();
4149 
4150   // SRA expansion:
4151   //   if Shamt-XLEN < 0: // Shamt < XLEN
4152   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
4153   //     Hi = Hi >>s Shamt
4154   //   else:
4155   //     Lo = Hi >>s (Shamt-XLEN);
4156   //     Hi = Hi >>s (XLEN-1)
4157   //
4158   // SRL expansion:
4159   //   if Shamt-XLEN < 0: // Shamt < XLEN
4160   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
4161   //     Hi = Hi >>u Shamt
4162   //   else:
4163   //     Lo = Hi >>u (Shamt-XLEN);
4164   //     Hi = 0;
4165 
4166   unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
4167 
4168   SDValue Zero = DAG.getConstant(0, DL, VT);
4169   SDValue One = DAG.getConstant(1, DL, VT);
4170   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
4171   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
4172   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
4173   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
4174 
4175   SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
4176   SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
4177   SDValue ShiftLeftHi =
4178       DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
4179   SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
4180   SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
4181   SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
4182   SDValue HiFalse =
4183       IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
4184 
4185   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
4186 
4187   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
4188   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
4189 
4190   SDValue Parts[2] = {Lo, Hi};
4191   return DAG.getMergeValues(Parts, DL);
4192 }
4193 
4194 // Lower splats of i1 types to SETCC. For each mask vector type, we have a
4195 // legal equivalently-sized i8 type, so we can use that as a go-between.
4196 SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
4197                                                   SelectionDAG &DAG) const {
4198   SDLoc DL(Op);
4199   MVT VT = Op.getSimpleValueType();
4200   SDValue SplatVal = Op.getOperand(0);
4201   // All-zeros or all-ones splats are handled specially.
4202   if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) {
4203     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
4204     return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
4205   }
4206   if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) {
4207     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
4208     return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
4209   }
4210   MVT XLenVT = Subtarget.getXLenVT();
4211   assert(SplatVal.getValueType() == XLenVT &&
4212          "Unexpected type for i1 splat value");
4213   MVT InterVT = VT.changeVectorElementType(MVT::i8);
4214   SplatVal = DAG.getNode(ISD::AND, DL, XLenVT, SplatVal,
4215                          DAG.getConstant(1, DL, XLenVT));
4216   SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
4217   SDValue Zero = DAG.getConstant(0, DL, InterVT);
4218   return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
4219 }
4220 
4221 // Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
4222 // illegal (currently only vXi64 RV32).
4223 // FIXME: We could also catch non-constant sign-extended i32 values and lower
4224 // them to VMV_V_X_VL.
4225 SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
4226                                                      SelectionDAG &DAG) const {
4227   SDLoc DL(Op);
4228   MVT VecVT = Op.getSimpleValueType();
4229   assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
4230          "Unexpected SPLAT_VECTOR_PARTS lowering");
4231 
4232   assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
4233   SDValue Lo = Op.getOperand(0);
4234   SDValue Hi = Op.getOperand(1);
4235 
4236   if (VecVT.isFixedLengthVector()) {
4237     MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
4238     SDLoc DL(Op);
4239     SDValue Mask, VL;
4240     std::tie(Mask, VL) =
4241         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4242 
4243     SDValue Res = splatPartsI64WithVL(DL, ContainerVT, Lo, Hi, VL, DAG);
4244     return convertFromScalableVector(VecVT, Res, DAG, Subtarget);
4245   }
4246 
4247   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
4248     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
4249     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
4250     // If Hi constant is all the same sign bit as Lo, lower this as a custom
4251     // node in order to try and match RVV vector/scalar instructions.
4252     if ((LoC >> 31) == HiC)
4253       return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, Lo,
4254                          DAG.getRegister(RISCV::X0, MVT::i32));
4255   }
4256 
4257   // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
4258   if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
4259       isa<ConstantSDNode>(Hi.getOperand(1)) &&
4260       Hi.getConstantOperandVal(1) == 31)
4261     return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, Lo,
4262                        DAG.getRegister(RISCV::X0, MVT::i32));
4263 
4264   // Fall back to use a stack store and stride x0 vector load. Use X0 as VL.
4265   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VecVT, Lo, Hi,
4266                      DAG.getRegister(RISCV::X0, MVT::i32));
4267 }
4268 
4269 // Custom-lower extensions from mask vectors by using a vselect either with 1
4270 // for zero/any-extension or -1 for sign-extension:
4271 //   (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
4272 // Note that any-extension is lowered identically to zero-extension.
4273 SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
4274                                                 int64_t ExtTrueVal) const {
4275   SDLoc DL(Op);
4276   MVT VecVT = Op.getSimpleValueType();
4277   SDValue Src = Op.getOperand(0);
4278   // Only custom-lower extensions from mask types
4279   assert(Src.getValueType().isVector() &&
4280          Src.getValueType().getVectorElementType() == MVT::i1);
4281 
4282   MVT XLenVT = Subtarget.getXLenVT();
4283   SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
4284   SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
4285 
4286   if (VecVT.isScalableVector()) {
4287     // Be careful not to introduce illegal scalar types at this stage, and be
4288     // careful also about splatting constants as on RV32, vXi64 SPLAT_VECTOR is
4289     // illegal and must be expanded. Since we know that the constants are
4290     // sign-extended 32-bit values, we use VMV_V_X_VL directly.
4291     bool IsRV32E64 =
4292         !Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64;
4293 
4294     if (!IsRV32E64) {
4295       SplatZero = DAG.getSplatVector(VecVT, DL, SplatZero);
4296       SplatTrueVal = DAG.getSplatVector(VecVT, DL, SplatTrueVal);
4297     } else {
4298       SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, SplatZero,
4299                               DAG.getRegister(RISCV::X0, XLenVT));
4300       SplatTrueVal = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, SplatTrueVal,
4301                                  DAG.getRegister(RISCV::X0, XLenVT));
4302     }
4303 
4304     return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
4305   }
4306 
4307   MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
4308   MVT I1ContainerVT =
4309       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4310 
4311   SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);
4312 
4313   SDValue Mask, VL;
4314   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4315 
4316   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero, VL);
4317   SplatTrueVal =
4318       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatTrueVal, VL);
4319   SDValue Select = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC,
4320                                SplatTrueVal, SplatZero, VL);
4321 
4322   return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
4323 }
4324 
4325 SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
4326     SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const {
4327   MVT ExtVT = Op.getSimpleValueType();
4328   // Only custom-lower extensions from fixed-length vector types.
4329   if (!ExtVT.isFixedLengthVector())
4330     return Op;
4331   MVT VT = Op.getOperand(0).getSimpleValueType();
4332   // Grab the canonical container type for the extended type. Infer the smaller
4333   // type from that to ensure the same number of vector elements, as we know
4334   // the LMUL will be sufficient to hold the smaller type.
4335   MVT ContainerExtVT = getContainerForFixedLengthVector(ExtVT);
4336   // Get the extended container type manually to ensure the same number of
4337   // vector elements between source and dest.
4338   MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
4339                                      ContainerExtVT.getVectorElementCount());
4340 
4341   SDValue Op1 =
4342       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
4343 
4344   SDLoc DL(Op);
4345   SDValue Mask, VL;
4346   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4347 
4348   SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);
4349 
4350   return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget);
4351 }
4352 
4353 // Custom-lower truncations from vectors to mask vectors by using a mask and a
4354 // setcc operation:
4355 //   (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
4356 SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
4357                                                   SelectionDAG &DAG) const {
4358   SDLoc DL(Op);
4359   EVT MaskVT = Op.getValueType();
4360   // Only expect to custom-lower truncations to mask types
4361   assert(MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 &&
4362          "Unexpected type for vector mask lowering");
4363   SDValue Src = Op.getOperand(0);
4364   MVT VecVT = Src.getSimpleValueType();
4365 
4366   // If this is a fixed vector, we need to convert it to a scalable vector.
4367   MVT ContainerVT = VecVT;
4368   if (VecVT.isFixedLengthVector()) {
4369     ContainerVT = getContainerForFixedLengthVector(VecVT);
4370     Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
4371   }
4372 
4373   SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
4374   SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
4375 
4376   SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatOne);
4377   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero);
4378 
4379   if (VecVT.isScalableVector()) {
4380     SDValue Trunc = DAG.getNode(ISD::AND, DL, VecVT, Src, SplatOne);
4381     return DAG.getSetCC(DL, MaskVT, Trunc, SplatZero, ISD::SETNE);
4382   }
4383 
4384   SDValue Mask, VL;
4385   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4386 
4387   MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
4388   SDValue Trunc =
4389       DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, Mask, VL);
4390   Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, Trunc, SplatZero,
4391                       DAG.getCondCode(ISD::SETNE), Mask, VL);
4392   return convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
4393 }
4394 
4395 // Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
4396 // first position of a vector, and that vector is slid up to the insert index.
4397 // By limiting the active vector length to index+1 and merging with the
4398 // original vector (with an undisturbed tail policy for elements >= VL), we
4399 // achieve the desired result of leaving all elements untouched except the one
4400 // at VL-1, which is replaced with the desired value.
4401 SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4402                                                     SelectionDAG &DAG) const {
4403   SDLoc DL(Op);
4404   MVT VecVT = Op.getSimpleValueType();
4405   SDValue Vec = Op.getOperand(0);
4406   SDValue Val = Op.getOperand(1);
4407   SDValue Idx = Op.getOperand(2);
4408 
4409   if (VecVT.getVectorElementType() == MVT::i1) {
4410     // FIXME: For now we just promote to an i8 vector and insert into that,
4411     // but this is probably not optimal.
4412     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4413     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4414     Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
4415     return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
4416   }
4417 
4418   MVT ContainerVT = VecVT;
4419   // If the operand is a fixed-length vector, convert to a scalable one.
4420   if (VecVT.isFixedLengthVector()) {
4421     ContainerVT = getContainerForFixedLengthVector(VecVT);
4422     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4423   }
4424 
4425   MVT XLenVT = Subtarget.getXLenVT();
4426 
4427   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4428   bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
4429   // Even i64-element vectors on RV32 can be lowered without scalar
4430   // legalization if the most-significant 32 bits of the value are not affected
4431   // by the sign-extension of the lower 32 bits.
4432   // TODO: We could also catch sign extensions of a 32-bit value.
4433   if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
4434     const auto *CVal = cast<ConstantSDNode>(Val);
4435     if (isInt<32>(CVal->getSExtValue())) {
4436       IsLegalInsert = true;
4437       Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4438     }
4439   }
4440 
4441   SDValue Mask, VL;
4442   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4443 
4444   SDValue ValInVec;
4445 
4446   if (IsLegalInsert) {
4447     unsigned Opc =
4448         VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
4449     if (isNullConstant(Idx)) {
4450       Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
4451       if (!VecVT.isFixedLengthVector())
4452         return Vec;
4453       return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
4454     }
4455     ValInVec =
4456         DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Val, VL);
4457   } else {
4458     // On RV32, i64-element vectors must be specially handled to place the
4459     // value at element 0, by using two vslide1up instructions in sequence on
4460     // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
4461     // this.
4462     SDValue One = DAG.getConstant(1, DL, XLenVT);
4463     SDValue ValLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, Zero);
4464     SDValue ValHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, One);
4465     MVT I32ContainerVT =
4466         MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2);
4467     SDValue I32Mask =
4468         getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
4469     // Limit the active VL to two.
4470     SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
4471     // Note: We can't pass a UNDEF to the first VSLIDE1UP_VL since an untied
4472     // undef doesn't obey the earlyclobber constraint. Just splat a zero value.
4473     ValInVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, I32ContainerVT, Zero,
4474                            InsertI64VL);
4475     // First slide in the hi value, then the lo in underneath it.
4476     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4477                            ValHi, I32Mask, InsertI64VL);
4478     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4479                            ValLo, I32Mask, InsertI64VL);
4480     // Bitcast back to the right container type.
4481     ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
4482   }
4483 
4484   // Now that the value is in a vector, slide it into position.
4485   SDValue InsertVL =
4486       DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
4487   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
4488                                 ValInVec, Idx, Mask, InsertVL);
4489   if (!VecVT.isFixedLengthVector())
4490     return Slideup;
4491   return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
4492 }
4493 
4494 // Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
4495 // extract the first element: (extractelt (slidedown vec, idx), 0). For integer
4496 // types this is done using VMV_X_S to allow us to glean information about the
4497 // sign bits of the result.
4498 SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4499                                                      SelectionDAG &DAG) const {
4500   SDLoc DL(Op);
4501   SDValue Idx = Op.getOperand(1);
4502   SDValue Vec = Op.getOperand(0);
4503   EVT EltVT = Op.getValueType();
4504   MVT VecVT = Vec.getSimpleValueType();
4505   MVT XLenVT = Subtarget.getXLenVT();
4506 
4507   if (VecVT.getVectorElementType() == MVT::i1) {
4508     if (VecVT.isFixedLengthVector()) {
4509       unsigned NumElts = VecVT.getVectorNumElements();
4510       if (NumElts >= 8) {
4511         MVT WideEltVT;
4512         unsigned WidenVecLen;
4513         SDValue ExtractElementIdx;
4514         SDValue ExtractBitIdx;
4515         unsigned MaxEEW = Subtarget.getMaxELENForFixedLengthVectors();
4516         MVT LargestEltVT = MVT::getIntegerVT(
4517             std::min(MaxEEW, unsigned(XLenVT.getSizeInBits())));
4518         if (NumElts <= LargestEltVT.getSizeInBits()) {
4519           assert(isPowerOf2_32(NumElts) &&
4520                  "the number of elements should be power of 2");
4521           WideEltVT = MVT::getIntegerVT(NumElts);
4522           WidenVecLen = 1;
4523           ExtractElementIdx = DAG.getConstant(0, DL, XLenVT);
4524           ExtractBitIdx = Idx;
4525         } else {
4526           WideEltVT = LargestEltVT;
4527           WidenVecLen = NumElts / WideEltVT.getSizeInBits();
4528           // extract element index = index / element width
4529           ExtractElementIdx = DAG.getNode(
4530               ISD::SRL, DL, XLenVT, Idx,
4531               DAG.getConstant(Log2_64(WideEltVT.getSizeInBits()), DL, XLenVT));
4532           // mask bit index = index % element width
4533           ExtractBitIdx = DAG.getNode(
4534               ISD::AND, DL, XLenVT, Idx,
4535               DAG.getConstant(WideEltVT.getSizeInBits() - 1, DL, XLenVT));
4536         }
4537         MVT WideVT = MVT::getVectorVT(WideEltVT, WidenVecLen);
4538         Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec);
4539         SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT,
4540                                          Vec, ExtractElementIdx);
4541         // Extract the bit from GPR.
4542         SDValue ShiftRight =
4543             DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx);
4544         return DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight,
4545                            DAG.getConstant(1, DL, XLenVT));
4546       }
4547     }
4548     // Otherwise, promote to an i8 vector and extract from that.
4549     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4550     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4551     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
4552   }
4553 
4554   // If this is a fixed vector, we need to convert it to a scalable vector.
4555   MVT ContainerVT = VecVT;
4556   if (VecVT.isFixedLengthVector()) {
4557     ContainerVT = getContainerForFixedLengthVector(VecVT);
4558     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4559   }
4560 
4561   // If the index is 0, the vector is already in the right position.
4562   if (!isNullConstant(Idx)) {
4563     // Use a VL of 1 to avoid processing more elements than we need.
4564     SDValue VL = DAG.getConstant(1, DL, XLenVT);
4565     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4566     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4567     Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
4568                       DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
4569   }
4570 
4571   if (!EltVT.isInteger()) {
4572     // Floating-point extracts are handled in TableGen.
4573     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
4574                        DAG.getConstant(0, DL, XLenVT));
4575   }
4576 
4577   SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
4578   return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
4579 }
4580 
4581 // Some RVV intrinsics may claim that they want an integer operand to be
4582 // promoted or expanded.
4583 static SDValue lowerVectorIntrinsicSplats(SDValue Op, SelectionDAG &DAG,
4584                                           const RISCVSubtarget &Subtarget) {
4585   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4586           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
4587          "Unexpected opcode");
4588 
4589   if (!Subtarget.hasVInstructions())
4590     return SDValue();
4591 
4592   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
4593   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
4594   SDLoc DL(Op);
4595 
4596   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
4597       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
4598   if (!II || !II->hasSplatOperand())
4599     return SDValue();
4600 
4601   unsigned SplatOp = II->SplatOperand + 1 + HasChain;
4602   assert(SplatOp < Op.getNumOperands());
4603 
4604   SmallVector<SDValue, 8> Operands(Op->op_begin(), Op->op_end());
4605   SDValue &ScalarOp = Operands[SplatOp];
4606   MVT OpVT = ScalarOp.getSimpleValueType();
4607   MVT XLenVT = Subtarget.getXLenVT();
4608 
4609   // If this isn't a scalar, or its type is XLenVT we're done.
4610   if (!OpVT.isScalarInteger() || OpVT == XLenVT)
4611     return SDValue();
4612 
4613   // Simplest case is that the operand needs to be promoted to XLenVT.
4614   if (OpVT.bitsLT(XLenVT)) {
4615     // If the operand is a constant, sign extend to increase our chances
4616     // of being able to use a .vi instruction. ANY_EXTEND would become a
4617     // a zero extend and the simm5 check in isel would fail.
4618     // FIXME: Should we ignore the upper bits in isel instead?
4619     unsigned ExtOpc =
4620         isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
4621     ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
4622     return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4623   }
4624 
4625   // Use the previous operand to get the vXi64 VT. The result might be a mask
4626   // VT for compares. Using the previous operand assumes that the previous
4627   // operand will never have a smaller element size than a scalar operand and
4628   // that a widening operation never uses SEW=64.
4629   // NOTE: If this fails the below assert, we can probably just find the
4630   // element count from any operand or result and use it to construct the VT.
4631   assert(II->SplatOperand > 0 && "Unexpected splat operand!");
4632   MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
4633 
4634   // The more complex case is when the scalar is larger than XLenVT.
4635   assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
4636          VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
4637 
4638   // If this is a sign-extended 32-bit constant, we can truncate it and rely
4639   // on the instruction to sign-extend since SEW>XLEN.
4640   if (auto *CVal = dyn_cast<ConstantSDNode>(ScalarOp)) {
4641     if (isInt<32>(CVal->getSExtValue())) {
4642       ScalarOp = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4643       return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4644     }
4645   }
4646 
4647   // We need to convert the scalar to a splat vector.
4648   // FIXME: Can we implicitly truncate the scalar if it is known to
4649   // be sign extended?
4650   SDValue VL = getVLOperand(Op);
4651   assert(VL.getValueType() == XLenVT);
4652   ScalarOp = splatSplitI64WithVL(DL, VT, ScalarOp, VL, DAG);
4653   return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4654 }
4655 
4656 SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
4657                                                      SelectionDAG &DAG) const {
4658   unsigned IntNo = Op.getConstantOperandVal(0);
4659   SDLoc DL(Op);
4660   MVT XLenVT = Subtarget.getXLenVT();
4661 
4662   switch (IntNo) {
4663   default:
4664     break; // Don't custom lower most intrinsics.
4665   case Intrinsic::thread_pointer: {
4666     EVT PtrVT = getPointerTy(DAG.getDataLayout());
4667     return DAG.getRegister(RISCV::X4, PtrVT);
4668   }
4669   case Intrinsic::riscv_orc_b:
4670   case Intrinsic::riscv_brev8: {
4671     // Lower to the GORCI encoding for orc.b or the GREVI encoding for brev8.
4672     unsigned Opc =
4673         IntNo == Intrinsic::riscv_brev8 ? RISCVISD::GREV : RISCVISD::GORC;
4674     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1),
4675                        DAG.getConstant(7, DL, XLenVT));
4676   }
4677   case Intrinsic::riscv_grev:
4678   case Intrinsic::riscv_gorc: {
4679     unsigned Opc =
4680         IntNo == Intrinsic::riscv_grev ? RISCVISD::GREV : RISCVISD::GORC;
4681     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4682   }
4683   case Intrinsic::riscv_zip:
4684   case Intrinsic::riscv_unzip: {
4685     // Lower to the SHFLI encoding for zip or the UNSHFLI encoding for unzip.
4686     // For i32 the immdiate is 15. For i64 the immediate is 31.
4687     unsigned Opc =
4688         IntNo == Intrinsic::riscv_zip ? RISCVISD::SHFL : RISCVISD::UNSHFL;
4689     unsigned BitWidth = Op.getValueSizeInBits();
4690     assert(isPowerOf2_32(BitWidth) && BitWidth >= 2 && "Unexpected bit width");
4691     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1),
4692                        DAG.getConstant((BitWidth / 2) - 1, DL, XLenVT));
4693   }
4694   case Intrinsic::riscv_shfl:
4695   case Intrinsic::riscv_unshfl: {
4696     unsigned Opc =
4697         IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
4698     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4699   }
4700   case Intrinsic::riscv_bcompress:
4701   case Intrinsic::riscv_bdecompress: {
4702     unsigned Opc = IntNo == Intrinsic::riscv_bcompress ? RISCVISD::BCOMPRESS
4703                                                        : RISCVISD::BDECOMPRESS;
4704     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4705   }
4706   case Intrinsic::riscv_bfp:
4707     return DAG.getNode(RISCVISD::BFP, DL, XLenVT, Op.getOperand(1),
4708                        Op.getOperand(2));
4709   case Intrinsic::riscv_fsl:
4710     return DAG.getNode(RISCVISD::FSL, DL, XLenVT, Op.getOperand(1),
4711                        Op.getOperand(2), Op.getOperand(3));
4712   case Intrinsic::riscv_fsr:
4713     return DAG.getNode(RISCVISD::FSR, DL, XLenVT, Op.getOperand(1),
4714                        Op.getOperand(2), Op.getOperand(3));
4715   case Intrinsic::riscv_vmv_x_s:
4716     assert(Op.getValueType() == XLenVT && "Unexpected VT!");
4717     return DAG.getNode(RISCVISD::VMV_X_S, DL, Op.getValueType(),
4718                        Op.getOperand(1));
4719   case Intrinsic::riscv_vmv_v_x:
4720     return lowerScalarSplat(Op.getOperand(1), Op.getOperand(2),
4721                             Op.getSimpleValueType(), DL, DAG, Subtarget);
4722   case Intrinsic::riscv_vfmv_v_f:
4723     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
4724                        Op.getOperand(1), Op.getOperand(2));
4725   case Intrinsic::riscv_vmv_s_x: {
4726     SDValue Scalar = Op.getOperand(2);
4727 
4728     if (Scalar.getValueType().bitsLE(XLenVT)) {
4729       Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
4730       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
4731                          Op.getOperand(1), Scalar, Op.getOperand(3));
4732     }
4733 
4734     assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
4735 
4736     // This is an i64 value that lives in two scalar registers. We have to
4737     // insert this in a convoluted way. First we build vXi64 splat containing
4738     // the/ two values that we assemble using some bit math. Next we'll use
4739     // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
4740     // to merge element 0 from our splat into the source vector.
4741     // FIXME: This is probably not the best way to do this, but it is
4742     // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
4743     // point.
4744     //   sw lo, (a0)
4745     //   sw hi, 4(a0)
4746     //   vlse vX, (a0)
4747     //
4748     //   vid.v      vVid
4749     //   vmseq.vx   mMask, vVid, 0
4750     //   vmerge.vvm vDest, vSrc, vVal, mMask
4751     MVT VT = Op.getSimpleValueType();
4752     SDValue Vec = Op.getOperand(1);
4753     SDValue VL = getVLOperand(Op);
4754 
4755     SDValue SplattedVal = splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
4756     SDValue SplattedIdx = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
4757                                       DAG.getConstant(0, DL, MVT::i32), VL);
4758 
4759     MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
4760     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4761     SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
4762     SDValue SelectCond =
4763         DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, VID, SplattedIdx,
4764                     DAG.getCondCode(ISD::SETEQ), Mask, VL);
4765     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, SelectCond, SplattedVal,
4766                        Vec, VL);
4767   }
4768   case Intrinsic::riscv_vslide1up:
4769   case Intrinsic::riscv_vslide1down:
4770   case Intrinsic::riscv_vslide1up_mask:
4771   case Intrinsic::riscv_vslide1down_mask: {
4772     // We need to special case these when the scalar is larger than XLen.
4773     unsigned NumOps = Op.getNumOperands();
4774     bool IsMasked = NumOps == 7;
4775     unsigned OpOffset = IsMasked ? 1 : 0;
4776     SDValue Scalar = Op.getOperand(2 + OpOffset);
4777     if (Scalar.getValueType().bitsLE(XLenVT))
4778       break;
4779 
4780     // Splatting a sign extended constant is fine.
4781     if (auto *CVal = dyn_cast<ConstantSDNode>(Scalar))
4782       if (isInt<32>(CVal->getSExtValue()))
4783         break;
4784 
4785     MVT VT = Op.getSimpleValueType();
4786     assert(VT.getVectorElementType() == MVT::i64 &&
4787            Scalar.getValueType() == MVT::i64 && "Unexpected VTs");
4788 
4789     // Convert the vector source to the equivalent nxvXi32 vector.
4790     MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
4791     SDValue Vec = DAG.getBitcast(I32VT, Op.getOperand(1 + OpOffset));
4792 
4793     SDValue ScalarLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4794                                    DAG.getConstant(0, DL, XLenVT));
4795     SDValue ScalarHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4796                                    DAG.getConstant(1, DL, XLenVT));
4797 
4798     // Double the VL since we halved SEW.
4799     SDValue VL = getVLOperand(Op);
4800     SDValue I32VL =
4801         DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
4802 
4803     MVT I32MaskVT = MVT::getVectorVT(MVT::i1, I32VT.getVectorElementCount());
4804     SDValue I32Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, I32MaskVT, VL);
4805 
4806     // Shift the two scalar parts in using SEW=32 slide1up/slide1down
4807     // instructions.
4808     if (IntNo == Intrinsic::riscv_vslide1up ||
4809         IntNo == Intrinsic::riscv_vslide1up_mask) {
4810       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarHi,
4811                         I32Mask, I32VL);
4812       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarLo,
4813                         I32Mask, I32VL);
4814     } else {
4815       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarLo,
4816                         I32Mask, I32VL);
4817       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarHi,
4818                         I32Mask, I32VL);
4819     }
4820 
4821     // Convert back to nxvXi64.
4822     Vec = DAG.getBitcast(VT, Vec);
4823 
4824     if (!IsMasked)
4825       return Vec;
4826 
4827     // Apply mask after the operation.
4828     SDValue Mask = Op.getOperand(NumOps - 3);
4829     SDValue MaskedOff = Op.getOperand(1);
4830     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, Mask, Vec, MaskedOff, VL);
4831   }
4832   }
4833 
4834   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4835 }
4836 
4837 SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
4838                                                     SelectionDAG &DAG) const {
4839   unsigned IntNo = Op.getConstantOperandVal(1);
4840   switch (IntNo) {
4841   default:
4842     break;
4843   case Intrinsic::riscv_masked_strided_load: {
4844     SDLoc DL(Op);
4845     MVT XLenVT = Subtarget.getXLenVT();
4846 
4847     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4848     // the selection of the masked intrinsics doesn't do this for us.
4849     SDValue Mask = Op.getOperand(5);
4850     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4851 
4852     MVT VT = Op->getSimpleValueType(0);
4853     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4854 
4855     SDValue PassThru = Op.getOperand(2);
4856     if (!IsUnmasked) {
4857       MVT MaskVT =
4858           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4859       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4860       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
4861     }
4862 
4863     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4864 
4865     SDValue IntID = DAG.getTargetConstant(
4866         IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask, DL,
4867         XLenVT);
4868 
4869     auto *Load = cast<MemIntrinsicSDNode>(Op);
4870     SmallVector<SDValue, 8> Ops{Load->getChain(), IntID};
4871     if (IsUnmasked)
4872       Ops.push_back(DAG.getUNDEF(ContainerVT));
4873     else
4874       Ops.push_back(PassThru);
4875     Ops.push_back(Op.getOperand(3)); // Ptr
4876     Ops.push_back(Op.getOperand(4)); // Stride
4877     if (!IsUnmasked)
4878       Ops.push_back(Mask);
4879     Ops.push_back(VL);
4880     if (!IsUnmasked) {
4881       SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
4882       Ops.push_back(Policy);
4883     }
4884 
4885     SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
4886     SDValue Result =
4887         DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
4888                                 Load->getMemoryVT(), Load->getMemOperand());
4889     SDValue Chain = Result.getValue(1);
4890     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
4891     return DAG.getMergeValues({Result, Chain}, DL);
4892   }
4893   }
4894 
4895   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4896 }
4897 
4898 SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
4899                                                  SelectionDAG &DAG) const {
4900   unsigned IntNo = Op.getConstantOperandVal(1);
4901   switch (IntNo) {
4902   default:
4903     break;
4904   case Intrinsic::riscv_masked_strided_store: {
4905     SDLoc DL(Op);
4906     MVT XLenVT = Subtarget.getXLenVT();
4907 
4908     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4909     // the selection of the masked intrinsics doesn't do this for us.
4910     SDValue Mask = Op.getOperand(5);
4911     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4912 
4913     SDValue Val = Op.getOperand(2);
4914     MVT VT = Val.getSimpleValueType();
4915     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4916 
4917     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
4918     if (!IsUnmasked) {
4919       MVT MaskVT =
4920           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4921       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4922     }
4923 
4924     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4925 
4926     SDValue IntID = DAG.getTargetConstant(
4927         IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask, DL,
4928         XLenVT);
4929 
4930     auto *Store = cast<MemIntrinsicSDNode>(Op);
4931     SmallVector<SDValue, 8> Ops{Store->getChain(), IntID};
4932     Ops.push_back(Val);
4933     Ops.push_back(Op.getOperand(3)); // Ptr
4934     Ops.push_back(Op.getOperand(4)); // Stride
4935     if (!IsUnmasked)
4936       Ops.push_back(Mask);
4937     Ops.push_back(VL);
4938 
4939     return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(),
4940                                    Ops, Store->getMemoryVT(),
4941                                    Store->getMemOperand());
4942   }
4943   }
4944 
4945   return SDValue();
4946 }
4947 
4948 static MVT getLMUL1VT(MVT VT) {
4949   assert(VT.getVectorElementType().getSizeInBits() <= 64 &&
4950          "Unexpected vector MVT");
4951   return MVT::getScalableVectorVT(
4952       VT.getVectorElementType(),
4953       RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits());
4954 }
4955 
4956 static unsigned getRVVReductionOp(unsigned ISDOpcode) {
4957   switch (ISDOpcode) {
4958   default:
4959     llvm_unreachable("Unhandled reduction");
4960   case ISD::VECREDUCE_ADD:
4961     return RISCVISD::VECREDUCE_ADD_VL;
4962   case ISD::VECREDUCE_UMAX:
4963     return RISCVISD::VECREDUCE_UMAX_VL;
4964   case ISD::VECREDUCE_SMAX:
4965     return RISCVISD::VECREDUCE_SMAX_VL;
4966   case ISD::VECREDUCE_UMIN:
4967     return RISCVISD::VECREDUCE_UMIN_VL;
4968   case ISD::VECREDUCE_SMIN:
4969     return RISCVISD::VECREDUCE_SMIN_VL;
4970   case ISD::VECREDUCE_AND:
4971     return RISCVISD::VECREDUCE_AND_VL;
4972   case ISD::VECREDUCE_OR:
4973     return RISCVISD::VECREDUCE_OR_VL;
4974   case ISD::VECREDUCE_XOR:
4975     return RISCVISD::VECREDUCE_XOR_VL;
4976   }
4977 }
4978 
4979 SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
4980                                                          SelectionDAG &DAG,
4981                                                          bool IsVP) const {
4982   SDLoc DL(Op);
4983   SDValue Vec = Op.getOperand(IsVP ? 1 : 0);
4984   MVT VecVT = Vec.getSimpleValueType();
4985   assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
4986           Op.getOpcode() == ISD::VECREDUCE_OR ||
4987           Op.getOpcode() == ISD::VECREDUCE_XOR ||
4988           Op.getOpcode() == ISD::VP_REDUCE_AND ||
4989           Op.getOpcode() == ISD::VP_REDUCE_OR ||
4990           Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
4991          "Unexpected reduction lowering");
4992 
4993   MVT XLenVT = Subtarget.getXLenVT();
4994   assert(Op.getValueType() == XLenVT &&
4995          "Expected reduction output to be legalized to XLenVT");
4996 
4997   MVT ContainerVT = VecVT;
4998   if (VecVT.isFixedLengthVector()) {
4999     ContainerVT = getContainerForFixedLengthVector(VecVT);
5000     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5001   }
5002 
5003   SDValue Mask, VL;
5004   if (IsVP) {
5005     Mask = Op.getOperand(2);
5006     VL = Op.getOperand(3);
5007   } else {
5008     std::tie(Mask, VL) =
5009         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
5010   }
5011 
5012   unsigned BaseOpc;
5013   ISD::CondCode CC;
5014   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
5015 
5016   switch (Op.getOpcode()) {
5017   default:
5018     llvm_unreachable("Unhandled reduction");
5019   case ISD::VECREDUCE_AND:
5020   case ISD::VP_REDUCE_AND: {
5021     // vcpop ~x == 0
5022     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
5023     Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
5024     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
5025     CC = ISD::SETEQ;
5026     BaseOpc = ISD::AND;
5027     break;
5028   }
5029   case ISD::VECREDUCE_OR:
5030   case ISD::VP_REDUCE_OR:
5031     // vcpop x != 0
5032     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
5033     CC = ISD::SETNE;
5034     BaseOpc = ISD::OR;
5035     break;
5036   case ISD::VECREDUCE_XOR:
5037   case ISD::VP_REDUCE_XOR: {
5038     // ((vcpop x) & 1) != 0
5039     SDValue One = DAG.getConstant(1, DL, XLenVT);
5040     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
5041     Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
5042     CC = ISD::SETNE;
5043     BaseOpc = ISD::XOR;
5044     break;
5045   }
5046   }
5047 
5048   SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC);
5049 
5050   if (!IsVP)
5051     return SetCC;
5052 
5053   // Now include the start value in the operation.
5054   // Note that we must return the start value when no elements are operated
5055   // upon. The vcpop instructions we've emitted in each case above will return
5056   // 0 for an inactive vector, and so we've already received the neutral value:
5057   // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
5058   // can simply include the start value.
5059   return DAG.getNode(BaseOpc, DL, XLenVT, SetCC, Op.getOperand(0));
5060 }
5061 
5062 SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
5063                                             SelectionDAG &DAG) const {
5064   SDLoc DL(Op);
5065   SDValue Vec = Op.getOperand(0);
5066   EVT VecEVT = Vec.getValueType();
5067 
5068   unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
5069 
5070   // Due to ordering in legalize types we may have a vector type that needs to
5071   // be split. Do that manually so we can get down to a legal type.
5072   while (getTypeAction(*DAG.getContext(), VecEVT) ==
5073          TargetLowering::TypeSplitVector) {
5074     SDValue Lo, Hi;
5075     std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
5076     VecEVT = Lo.getValueType();
5077     Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
5078   }
5079 
5080   // TODO: The type may need to be widened rather than split. Or widened before
5081   // it can be split.
5082   if (!isTypeLegal(VecEVT))
5083     return SDValue();
5084 
5085   MVT VecVT = VecEVT.getSimpleVT();
5086   MVT VecEltVT = VecVT.getVectorElementType();
5087   unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());
5088 
5089   MVT ContainerVT = VecVT;
5090   if (VecVT.isFixedLengthVector()) {
5091     ContainerVT = getContainerForFixedLengthVector(VecVT);
5092     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5093   }
5094 
5095   MVT M1VT = getLMUL1VT(ContainerVT);
5096   MVT XLenVT = Subtarget.getXLenVT();
5097 
5098   SDValue Mask, VL;
5099   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
5100 
5101   SDValue NeutralElem =
5102       DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
5103   SDValue IdentitySplat = lowerScalarSplat(
5104       NeutralElem, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
5105   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT), Vec,
5106                                   IdentitySplat, Mask, VL);
5107   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
5108                              DAG.getConstant(0, DL, XLenVT));
5109   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
5110 }
5111 
5112 // Given a reduction op, this function returns the matching reduction opcode,
5113 // the vector SDValue and the scalar SDValue required to lower this to a
5114 // RISCVISD node.
5115 static std::tuple<unsigned, SDValue, SDValue>
5116 getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) {
5117   SDLoc DL(Op);
5118   auto Flags = Op->getFlags();
5119   unsigned Opcode = Op.getOpcode();
5120   unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Opcode);
5121   switch (Opcode) {
5122   default:
5123     llvm_unreachable("Unhandled reduction");
5124   case ISD::VECREDUCE_FADD: {
5125     // Use positive zero if we can. It is cheaper to materialize.
5126     SDValue Zero =
5127         DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
5128     return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero);
5129   }
5130   case ISD::VECREDUCE_SEQ_FADD:
5131     return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
5132                            Op.getOperand(0));
5133   case ISD::VECREDUCE_FMIN:
5134     return std::make_tuple(RISCVISD::VECREDUCE_FMIN_VL, Op.getOperand(0),
5135                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
5136   case ISD::VECREDUCE_FMAX:
5137     return std::make_tuple(RISCVISD::VECREDUCE_FMAX_VL, Op.getOperand(0),
5138                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
5139   }
5140 }
5141 
5142 SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
5143                                               SelectionDAG &DAG) const {
5144   SDLoc DL(Op);
5145   MVT VecEltVT = Op.getSimpleValueType();
5146 
5147   unsigned RVVOpcode;
5148   SDValue VectorVal, ScalarVal;
5149   std::tie(RVVOpcode, VectorVal, ScalarVal) =
5150       getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT);
5151   MVT VecVT = VectorVal.getSimpleValueType();
5152 
5153   MVT ContainerVT = VecVT;
5154   if (VecVT.isFixedLengthVector()) {
5155     ContainerVT = getContainerForFixedLengthVector(VecVT);
5156     VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
5157   }
5158 
5159   MVT M1VT = getLMUL1VT(VectorVal.getSimpleValueType());
5160   MVT XLenVT = Subtarget.getXLenVT();
5161 
5162   SDValue Mask, VL;
5163   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
5164 
5165   SDValue ScalarSplat = lowerScalarSplat(
5166       ScalarVal, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
5167   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT),
5168                                   VectorVal, ScalarSplat, Mask, VL);
5169   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
5170                      DAG.getConstant(0, DL, XLenVT));
5171 }
5172 
5173 static unsigned getRVVVPReductionOp(unsigned ISDOpcode) {
5174   switch (ISDOpcode) {
5175   default:
5176     llvm_unreachable("Unhandled reduction");
5177   case ISD::VP_REDUCE_ADD:
5178     return RISCVISD::VECREDUCE_ADD_VL;
5179   case ISD::VP_REDUCE_UMAX:
5180     return RISCVISD::VECREDUCE_UMAX_VL;
5181   case ISD::VP_REDUCE_SMAX:
5182     return RISCVISD::VECREDUCE_SMAX_VL;
5183   case ISD::VP_REDUCE_UMIN:
5184     return RISCVISD::VECREDUCE_UMIN_VL;
5185   case ISD::VP_REDUCE_SMIN:
5186     return RISCVISD::VECREDUCE_SMIN_VL;
5187   case ISD::VP_REDUCE_AND:
5188     return RISCVISD::VECREDUCE_AND_VL;
5189   case ISD::VP_REDUCE_OR:
5190     return RISCVISD::VECREDUCE_OR_VL;
5191   case ISD::VP_REDUCE_XOR:
5192     return RISCVISD::VECREDUCE_XOR_VL;
5193   case ISD::VP_REDUCE_FADD:
5194     return RISCVISD::VECREDUCE_FADD_VL;
5195   case ISD::VP_REDUCE_SEQ_FADD:
5196     return RISCVISD::VECREDUCE_SEQ_FADD_VL;
5197   case ISD::VP_REDUCE_FMAX:
5198     return RISCVISD::VECREDUCE_FMAX_VL;
5199   case ISD::VP_REDUCE_FMIN:
5200     return RISCVISD::VECREDUCE_FMIN_VL;
5201   }
5202 }
5203 
5204 SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
5205                                            SelectionDAG &DAG) const {
5206   SDLoc DL(Op);
5207   SDValue Vec = Op.getOperand(1);
5208   EVT VecEVT = Vec.getValueType();
5209 
5210   // TODO: The type may need to be widened rather than split. Or widened before
5211   // it can be split.
5212   if (!isTypeLegal(VecEVT))
5213     return SDValue();
5214 
5215   MVT VecVT = VecEVT.getSimpleVT();
5216   MVT VecEltVT = VecVT.getVectorElementType();
5217   unsigned RVVOpcode = getRVVVPReductionOp(Op.getOpcode());
5218 
5219   MVT ContainerVT = VecVT;
5220   if (VecVT.isFixedLengthVector()) {
5221     ContainerVT = getContainerForFixedLengthVector(VecVT);
5222     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5223   }
5224 
5225   SDValue VL = Op.getOperand(3);
5226   SDValue Mask = Op.getOperand(2);
5227 
5228   MVT M1VT = getLMUL1VT(ContainerVT);
5229   MVT XLenVT = Subtarget.getXLenVT();
5230   MVT ResVT = !VecVT.isInteger() || VecEltVT.bitsGE(XLenVT) ? VecEltVT : XLenVT;
5231 
5232   SDValue StartSplat =
5233       lowerScalarSplat(Op.getOperand(0), DAG.getConstant(1, DL, XLenVT), M1VT,
5234                        DL, DAG, Subtarget);
5235   SDValue Reduction =
5236       DAG.getNode(RVVOpcode, DL, M1VT, StartSplat, Vec, StartSplat, Mask, VL);
5237   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction,
5238                              DAG.getConstant(0, DL, XLenVT));
5239   if (!VecVT.isInteger())
5240     return Elt0;
5241   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
5242 }
5243 
5244 SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
5245                                                    SelectionDAG &DAG) const {
5246   SDValue Vec = Op.getOperand(0);
5247   SDValue SubVec = Op.getOperand(1);
5248   MVT VecVT = Vec.getSimpleValueType();
5249   MVT SubVecVT = SubVec.getSimpleValueType();
5250 
5251   SDLoc DL(Op);
5252   MVT XLenVT = Subtarget.getXLenVT();
5253   unsigned OrigIdx = Op.getConstantOperandVal(2);
5254   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
5255 
5256   // We don't have the ability to slide mask vectors up indexed by their i1
5257   // elements; the smallest we can do is i8. Often we are able to bitcast to
5258   // equivalent i8 vectors. Note that when inserting a fixed-length vector
5259   // into a scalable one, we might not necessarily have enough scalable
5260   // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
5261   if (SubVecVT.getVectorElementType() == MVT::i1 &&
5262       (OrigIdx != 0 || !Vec.isUndef())) {
5263     if (VecVT.getVectorMinNumElements() >= 8 &&
5264         SubVecVT.getVectorMinNumElements() >= 8) {
5265       assert(OrigIdx % 8 == 0 && "Invalid index");
5266       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
5267              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
5268              "Unexpected mask vector lowering");
5269       OrigIdx /= 8;
5270       SubVecVT =
5271           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
5272                            SubVecVT.isScalableVector());
5273       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
5274                                VecVT.isScalableVector());
5275       Vec = DAG.getBitcast(VecVT, Vec);
5276       SubVec = DAG.getBitcast(SubVecVT, SubVec);
5277     } else {
5278       // We can't slide this mask vector up indexed by its i1 elements.
5279       // This poses a problem when we wish to insert a scalable vector which
5280       // can't be re-expressed as a larger type. Just choose the slow path and
5281       // extend to a larger type, then truncate back down.
5282       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
5283       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
5284       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
5285       SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
5286       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
5287                         Op.getOperand(2));
5288       SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
5289       return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
5290     }
5291   }
5292 
5293   // If the subvector vector is a fixed-length type, we cannot use subregister
5294   // manipulation to simplify the codegen; we don't know which register of a
5295   // LMUL group contains the specific subvector as we only know the minimum
5296   // register size. Therefore we must slide the vector group up the full
5297   // amount.
5298   if (SubVecVT.isFixedLengthVector()) {
5299     if (OrigIdx == 0 && Vec.isUndef() && !VecVT.isFixedLengthVector())
5300       return Op;
5301     MVT ContainerVT = VecVT;
5302     if (VecVT.isFixedLengthVector()) {
5303       ContainerVT = getContainerForFixedLengthVector(VecVT);
5304       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5305     }
5306     SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
5307                          DAG.getUNDEF(ContainerVT), SubVec,
5308                          DAG.getConstant(0, DL, XLenVT));
5309     if (OrigIdx == 0 && Vec.isUndef() && VecVT.isFixedLengthVector()) {
5310       SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
5311       return DAG.getBitcast(Op.getValueType(), SubVec);
5312     }
5313     SDValue Mask =
5314         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5315     // Set the vector length to only the number of elements we care about. Note
5316     // that for slideup this includes the offset.
5317     SDValue VL =
5318         DAG.getConstant(OrigIdx + SubVecVT.getVectorNumElements(), DL, XLenVT);
5319     SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5320     SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
5321                                   SubVec, SlideupAmt, Mask, VL);
5322     if (VecVT.isFixedLengthVector())
5323       Slideup = convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
5324     return DAG.getBitcast(Op.getValueType(), Slideup);
5325   }
5326 
5327   unsigned SubRegIdx, RemIdx;
5328   std::tie(SubRegIdx, RemIdx) =
5329       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5330           VecVT, SubVecVT, OrigIdx, TRI);
5331 
5332   RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecVT);
5333   bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
5334                          SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
5335                          SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
5336 
5337   // 1. If the Idx has been completely eliminated and this subvector's size is
5338   // a vector register or a multiple thereof, or the surrounding elements are
5339   // undef, then this is a subvector insert which naturally aligns to a vector
5340   // register. These can easily be handled using subregister manipulation.
5341   // 2. If the subvector is smaller than a vector register, then the insertion
5342   // must preserve the undisturbed elements of the register. We do this by
5343   // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
5344   // (which resolves to a subregister copy), performing a VSLIDEUP to place the
5345   // subvector within the vector register, and an INSERT_SUBVECTOR of that
5346   // LMUL=1 type back into the larger vector (resolving to another subregister
5347   // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
5348   // to avoid allocating a large register group to hold our subvector.
5349   if (RemIdx == 0 && (!IsSubVecPartReg || Vec.isUndef()))
5350     return Op;
5351 
5352   // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
5353   // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
5354   // (in our case undisturbed). This means we can set up a subvector insertion
5355   // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
5356   // size of the subvector.
5357   MVT InterSubVT = VecVT;
5358   SDValue AlignedExtract = Vec;
5359   unsigned AlignedIdx = OrigIdx - RemIdx;
5360   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5361     InterSubVT = getLMUL1VT(VecVT);
5362     // Extract a subvector equal to the nearest full vector register type. This
5363     // should resolve to a EXTRACT_SUBREG instruction.
5364     AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5365                                  DAG.getConstant(AlignedIdx, DL, XLenVT));
5366   }
5367 
5368   SDValue SlideupAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5369   // For scalable vectors this must be further multiplied by vscale.
5370   SlideupAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlideupAmt);
5371 
5372   SDValue Mask, VL;
5373   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5374 
5375   // Construct the vector length corresponding to RemIdx + length(SubVecVT).
5376   VL = DAG.getConstant(SubVecVT.getVectorMinNumElements(), DL, XLenVT);
5377   VL = DAG.getNode(ISD::VSCALE, DL, XLenVT, VL);
5378   VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
5379 
5380   SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
5381                        DAG.getUNDEF(InterSubVT), SubVec,
5382                        DAG.getConstant(0, DL, XLenVT));
5383 
5384   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, InterSubVT,
5385                                 AlignedExtract, SubVec, SlideupAmt, Mask, VL);
5386 
5387   // If required, insert this subvector back into the correct vector register.
5388   // This should resolve to an INSERT_SUBREG instruction.
5389   if (VecVT.bitsGT(InterSubVT))
5390     Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, Slideup,
5391                           DAG.getConstant(AlignedIdx, DL, XLenVT));
5392 
5393   // We might have bitcast from a mask type: cast back to the original type if
5394   // required.
5395   return DAG.getBitcast(Op.getSimpleValueType(), Slideup);
5396 }
5397 
5398 SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
5399                                                     SelectionDAG &DAG) const {
5400   SDValue Vec = Op.getOperand(0);
5401   MVT SubVecVT = Op.getSimpleValueType();
5402   MVT VecVT = Vec.getSimpleValueType();
5403 
5404   SDLoc DL(Op);
5405   MVT XLenVT = Subtarget.getXLenVT();
5406   unsigned OrigIdx = Op.getConstantOperandVal(1);
5407   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
5408 
5409   // We don't have the ability to slide mask vectors down indexed by their i1
5410   // elements; the smallest we can do is i8. Often we are able to bitcast to
5411   // equivalent i8 vectors. Note that when extracting a fixed-length vector
5412   // from a scalable one, we might not necessarily have enough scalable
5413   // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
5414   if (SubVecVT.getVectorElementType() == MVT::i1 && OrigIdx != 0) {
5415     if (VecVT.getVectorMinNumElements() >= 8 &&
5416         SubVecVT.getVectorMinNumElements() >= 8) {
5417       assert(OrigIdx % 8 == 0 && "Invalid index");
5418       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
5419              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
5420              "Unexpected mask vector lowering");
5421       OrigIdx /= 8;
5422       SubVecVT =
5423           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
5424                            SubVecVT.isScalableVector());
5425       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
5426                                VecVT.isScalableVector());
5427       Vec = DAG.getBitcast(VecVT, Vec);
5428     } else {
5429       // We can't slide this mask vector down, indexed by its i1 elements.
5430       // This poses a problem when we wish to extract a scalable vector which
5431       // can't be re-expressed as a larger type. Just choose the slow path and
5432       // extend to a larger type, then truncate back down.
5433       // TODO: We could probably improve this when extracting certain fixed
5434       // from fixed, where we can extract as i8 and shift the correct element
5435       // right to reach the desired subvector?
5436       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
5437       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
5438       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
5439       Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
5440                         Op.getOperand(1));
5441       SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
5442       return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
5443     }
5444   }
5445 
5446   // If the subvector vector is a fixed-length type, we cannot use subregister
5447   // manipulation to simplify the codegen; we don't know which register of a
5448   // LMUL group contains the specific subvector as we only know the minimum
5449   // register size. Therefore we must slide the vector group down the full
5450   // amount.
5451   if (SubVecVT.isFixedLengthVector()) {
5452     // With an index of 0 this is a cast-like subvector, which can be performed
5453     // with subregister operations.
5454     if (OrigIdx == 0)
5455       return Op;
5456     MVT ContainerVT = VecVT;
5457     if (VecVT.isFixedLengthVector()) {
5458       ContainerVT = getContainerForFixedLengthVector(VecVT);
5459       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5460     }
5461     SDValue Mask =
5462         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5463     // Set the vector length to only the number of elements we care about. This
5464     // avoids sliding down elements we're going to discard straight away.
5465     SDValue VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
5466     SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5467     SDValue Slidedown =
5468         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
5469                     DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
5470     // Now we can use a cast-like subvector extract to get the result.
5471     Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5472                             DAG.getConstant(0, DL, XLenVT));
5473     return DAG.getBitcast(Op.getValueType(), Slidedown);
5474   }
5475 
5476   unsigned SubRegIdx, RemIdx;
5477   std::tie(SubRegIdx, RemIdx) =
5478       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5479           VecVT, SubVecVT, OrigIdx, TRI);
5480 
5481   // If the Idx has been completely eliminated then this is a subvector extract
5482   // which naturally aligns to a vector register. These can easily be handled
5483   // using subregister manipulation.
5484   if (RemIdx == 0)
5485     return Op;
5486 
5487   // Else we must shift our vector register directly to extract the subvector.
5488   // Do this using VSLIDEDOWN.
5489 
5490   // If the vector type is an LMUL-group type, extract a subvector equal to the
5491   // nearest full vector register type. This should resolve to a EXTRACT_SUBREG
5492   // instruction.
5493   MVT InterSubVT = VecVT;
5494   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5495     InterSubVT = getLMUL1VT(VecVT);
5496     Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5497                       DAG.getConstant(OrigIdx - RemIdx, DL, XLenVT));
5498   }
5499 
5500   // Slide this vector register down by the desired number of elements in order
5501   // to place the desired subvector starting at element 0.
5502   SDValue SlidedownAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5503   // For scalable vectors this must be further multiplied by vscale.
5504   SlidedownAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlidedownAmt);
5505 
5506   SDValue Mask, VL;
5507   std::tie(Mask, VL) = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
5508   SDValue Slidedown =
5509       DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, InterSubVT,
5510                   DAG.getUNDEF(InterSubVT), Vec, SlidedownAmt, Mask, VL);
5511 
5512   // Now the vector is in the right position, extract our final subvector. This
5513   // should resolve to a COPY.
5514   Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5515                           DAG.getConstant(0, DL, XLenVT));
5516 
5517   // We might have bitcast from a mask type: cast back to the original type if
5518   // required.
5519   return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
5520 }
5521 
5522 // Lower step_vector to the vid instruction. Any non-identity step value must
5523 // be accounted for my manual expansion.
5524 SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
5525                                               SelectionDAG &DAG) const {
5526   SDLoc DL(Op);
5527   MVT VT = Op.getSimpleValueType();
5528   MVT XLenVT = Subtarget.getXLenVT();
5529   SDValue Mask, VL;
5530   std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
5531   SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
5532   uint64_t StepValImm = Op.getConstantOperandVal(0);
5533   if (StepValImm != 1) {
5534     if (isPowerOf2_64(StepValImm)) {
5535       SDValue StepVal =
5536           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
5537                       DAG.getConstant(Log2_64(StepValImm), DL, XLenVT));
5538       StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
5539     } else {
5540       SDValue StepVal = lowerScalarSplat(
5541           DAG.getConstant(StepValImm, DL, VT.getVectorElementType()), VL, VT,
5542           DL, DAG, Subtarget);
5543       StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
5544     }
5545   }
5546   return StepVec;
5547 }
5548 
5549 // Implement vector_reverse using vrgather.vv with indices determined by
5550 // subtracting the id of each element from (VLMAX-1). This will convert
5551 // the indices like so:
5552 // (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
5553 // TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
5554 SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
5555                                                  SelectionDAG &DAG) const {
5556   SDLoc DL(Op);
5557   MVT VecVT = Op.getSimpleValueType();
5558   unsigned EltSize = VecVT.getScalarSizeInBits();
5559   unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
5560 
5561   unsigned MaxVLMAX = 0;
5562   unsigned VectorBitsMax = Subtarget.getMaxRVVVectorSizeInBits();
5563   if (VectorBitsMax != 0)
5564     MaxVLMAX = ((VectorBitsMax / EltSize) * MinSize) / RISCV::RVVBitsPerBlock;
5565 
5566   unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
5567   MVT IntVT = VecVT.changeVectorElementTypeToInteger();
5568 
5569   // If this is SEW=8 and VLMAX is unknown or more than 256, we need
5570   // to use vrgatherei16.vv.
5571   // TODO: It's also possible to use vrgatherei16.vv for other types to
5572   // decrease register width for the index calculation.
5573   if ((MaxVLMAX == 0 || MaxVLMAX > 256) && EltSize == 8) {
5574     // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
5575     // Reverse each half, then reassemble them in reverse order.
5576     // NOTE: It's also possible that after splitting that VLMAX no longer
5577     // requires vrgatherei16.vv.
5578     if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
5579       SDValue Lo, Hi;
5580       std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
5581       EVT LoVT, HiVT;
5582       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
5583       Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
5584       Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
5585       // Reassemble the low and high pieces reversed.
5586       // FIXME: This is a CONCAT_VECTORS.
5587       SDValue Res =
5588           DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
5589                       DAG.getIntPtrConstant(0, DL));
5590       return DAG.getNode(
5591           ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo,
5592           DAG.getIntPtrConstant(LoVT.getVectorMinNumElements(), DL));
5593     }
5594 
5595     // Just promote the int type to i16 which will double the LMUL.
5596     IntVT = MVT::getVectorVT(MVT::i16, VecVT.getVectorElementCount());
5597     GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
5598   }
5599 
5600   MVT XLenVT = Subtarget.getXLenVT();
5601   SDValue Mask, VL;
5602   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5603 
5604   // Calculate VLMAX-1 for the desired SEW.
5605   unsigned MinElts = VecVT.getVectorMinNumElements();
5606   SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT,
5607                               DAG.getConstant(MinElts, DL, XLenVT));
5608   SDValue VLMinus1 =
5609       DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DAG.getConstant(1, DL, XLenVT));
5610 
5611   // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
5612   bool IsRV32E64 =
5613       !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
5614   SDValue SplatVL;
5615   if (!IsRV32E64)
5616     SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
5617   else
5618     SplatVL = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT, VLMinus1,
5619                           DAG.getRegister(RISCV::X0, XLenVT));
5620 
5621   SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
5622   SDValue Indices =
5623       DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, Mask, VL);
5624 
5625   return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices, Mask, VL);
5626 }
5627 
5628 SDValue
5629 RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
5630                                                      SelectionDAG &DAG) const {
5631   SDLoc DL(Op);
5632   auto *Load = cast<LoadSDNode>(Op);
5633 
5634   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5635                                         Load->getMemoryVT(),
5636                                         *Load->getMemOperand()) &&
5637          "Expecting a correctly-aligned load");
5638 
5639   MVT VT = Op.getSimpleValueType();
5640   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5641 
5642   SDValue VL =
5643       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5644 
5645   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5646   SDValue NewLoad = DAG.getMemIntrinsicNode(
5647       RISCVISD::VLE_VL, DL, VTs, {Load->getChain(), Load->getBasePtr(), VL},
5648       Load->getMemoryVT(), Load->getMemOperand());
5649 
5650   SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
5651   return DAG.getMergeValues({Result, Load->getChain()}, DL);
5652 }
5653 
5654 SDValue
5655 RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
5656                                                       SelectionDAG &DAG) const {
5657   SDLoc DL(Op);
5658   auto *Store = cast<StoreSDNode>(Op);
5659 
5660   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5661                                         Store->getMemoryVT(),
5662                                         *Store->getMemOperand()) &&
5663          "Expecting a correctly-aligned store");
5664 
5665   SDValue StoreVal = Store->getValue();
5666   MVT VT = StoreVal.getSimpleValueType();
5667 
5668   // If the size less than a byte, we need to pad with zeros to make a byte.
5669   if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
5670     VT = MVT::v8i1;
5671     StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
5672                            DAG.getConstant(0, DL, VT), StoreVal,
5673                            DAG.getIntPtrConstant(0, DL));
5674   }
5675 
5676   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5677 
5678   SDValue VL =
5679       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5680 
5681   SDValue NewValue =
5682       convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
5683   return DAG.getMemIntrinsicNode(
5684       RISCVISD::VSE_VL, DL, DAG.getVTList(MVT::Other),
5685       {Store->getChain(), NewValue, Store->getBasePtr(), VL},
5686       Store->getMemoryVT(), Store->getMemOperand());
5687 }
5688 
5689 SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
5690                                              SelectionDAG &DAG) const {
5691   SDLoc DL(Op);
5692   MVT VT = Op.getSimpleValueType();
5693 
5694   const auto *MemSD = cast<MemSDNode>(Op);
5695   EVT MemVT = MemSD->getMemoryVT();
5696   MachineMemOperand *MMO = MemSD->getMemOperand();
5697   SDValue Chain = MemSD->getChain();
5698   SDValue BasePtr = MemSD->getBasePtr();
5699 
5700   SDValue Mask, PassThru, VL;
5701   if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Op)) {
5702     Mask = VPLoad->getMask();
5703     PassThru = DAG.getUNDEF(VT);
5704     VL = VPLoad->getVectorLength();
5705   } else {
5706     const auto *MLoad = cast<MaskedLoadSDNode>(Op);
5707     Mask = MLoad->getMask();
5708     PassThru = MLoad->getPassThru();
5709   }
5710 
5711   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5712 
5713   MVT XLenVT = Subtarget.getXLenVT();
5714 
5715   MVT ContainerVT = VT;
5716   if (VT.isFixedLengthVector()) {
5717     ContainerVT = getContainerForFixedLengthVector(VT);
5718     PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5719     if (!IsUnmasked) {
5720       MVT MaskVT =
5721           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5722       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5723     }
5724   }
5725 
5726   if (!VL)
5727     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5728 
5729   unsigned IntID =
5730       IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
5731   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5732   if (IsUnmasked)
5733     Ops.push_back(DAG.getUNDEF(ContainerVT));
5734   else
5735     Ops.push_back(PassThru);
5736   Ops.push_back(BasePtr);
5737   if (!IsUnmasked)
5738     Ops.push_back(Mask);
5739   Ops.push_back(VL);
5740   if (!IsUnmasked)
5741     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5742 
5743   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5744 
5745   SDValue Result =
5746       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5747   Chain = Result.getValue(1);
5748 
5749   if (VT.isFixedLengthVector())
5750     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5751 
5752   return DAG.getMergeValues({Result, Chain}, DL);
5753 }
5754 
5755 SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
5756                                               SelectionDAG &DAG) const {
5757   SDLoc DL(Op);
5758 
5759   const auto *MemSD = cast<MemSDNode>(Op);
5760   EVT MemVT = MemSD->getMemoryVT();
5761   MachineMemOperand *MMO = MemSD->getMemOperand();
5762   SDValue Chain = MemSD->getChain();
5763   SDValue BasePtr = MemSD->getBasePtr();
5764   SDValue Val, Mask, VL;
5765 
5766   if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Op)) {
5767     Val = VPStore->getValue();
5768     Mask = VPStore->getMask();
5769     VL = VPStore->getVectorLength();
5770   } else {
5771     const auto *MStore = cast<MaskedStoreSDNode>(Op);
5772     Val = MStore->getValue();
5773     Mask = MStore->getMask();
5774   }
5775 
5776   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5777 
5778   MVT VT = Val.getSimpleValueType();
5779   MVT XLenVT = Subtarget.getXLenVT();
5780 
5781   MVT ContainerVT = VT;
5782   if (VT.isFixedLengthVector()) {
5783     ContainerVT = getContainerForFixedLengthVector(VT);
5784 
5785     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
5786     if (!IsUnmasked) {
5787       MVT MaskVT =
5788           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5789       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5790     }
5791   }
5792 
5793   if (!VL)
5794     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5795 
5796   unsigned IntID =
5797       IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
5798   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5799   Ops.push_back(Val);
5800   Ops.push_back(BasePtr);
5801   if (!IsUnmasked)
5802     Ops.push_back(Mask);
5803   Ops.push_back(VL);
5804 
5805   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
5806                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
5807 }
5808 
5809 SDValue
5810 RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(SDValue Op,
5811                                                       SelectionDAG &DAG) const {
5812   MVT InVT = Op.getOperand(0).getSimpleValueType();
5813   MVT ContainerVT = getContainerForFixedLengthVector(InVT);
5814 
5815   MVT VT = Op.getSimpleValueType();
5816 
5817   SDValue Op1 =
5818       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
5819   SDValue Op2 =
5820       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5821 
5822   SDLoc DL(Op);
5823   SDValue VL =
5824       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5825 
5826   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5827   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
5828 
5829   SDValue Cmp = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op1, Op2,
5830                             Op.getOperand(2), Mask, VL);
5831 
5832   return convertFromScalableVector(VT, Cmp, DAG, Subtarget);
5833 }
5834 
5835 SDValue RISCVTargetLowering::lowerFixedLengthVectorLogicOpToRVV(
5836     SDValue Op, SelectionDAG &DAG, unsigned MaskOpc, unsigned VecOpc) const {
5837   MVT VT = Op.getSimpleValueType();
5838 
5839   if (VT.getVectorElementType() == MVT::i1)
5840     return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMask*/ false);
5841 
5842   return lowerToScalableOp(Op, DAG, VecOpc, /*HasMask*/ true);
5843 }
5844 
5845 SDValue
5846 RISCVTargetLowering::lowerFixedLengthVectorShiftToRVV(SDValue Op,
5847                                                       SelectionDAG &DAG) const {
5848   unsigned Opc;
5849   switch (Op.getOpcode()) {
5850   default: llvm_unreachable("Unexpected opcode!");
5851   case ISD::SHL: Opc = RISCVISD::SHL_VL; break;
5852   case ISD::SRA: Opc = RISCVISD::SRA_VL; break;
5853   case ISD::SRL: Opc = RISCVISD::SRL_VL; break;
5854   }
5855 
5856   return lowerToScalableOp(Op, DAG, Opc);
5857 }
5858 
5859 // Lower vector ABS to smax(X, sub(0, X)).
5860 SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
5861   SDLoc DL(Op);
5862   MVT VT = Op.getSimpleValueType();
5863   SDValue X = Op.getOperand(0);
5864 
5865   assert(VT.isFixedLengthVector() && "Unexpected type");
5866 
5867   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5868   X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
5869 
5870   SDValue Mask, VL;
5871   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5872 
5873   SDValue SplatZero =
5874       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
5875                   DAG.getConstant(0, DL, Subtarget.getXLenVT()));
5876   SDValue NegX =
5877       DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, Mask, VL);
5878   SDValue Max =
5879       DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, Mask, VL);
5880 
5881   return convertFromScalableVector(VT, Max, DAG, Subtarget);
5882 }
5883 
5884 SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
5885     SDValue Op, SelectionDAG &DAG) const {
5886   SDLoc DL(Op);
5887   MVT VT = Op.getSimpleValueType();
5888   SDValue Mag = Op.getOperand(0);
5889   SDValue Sign = Op.getOperand(1);
5890   assert(Mag.getValueType() == Sign.getValueType() &&
5891          "Can only handle COPYSIGN with matching types.");
5892 
5893   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5894   Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget);
5895   Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget);
5896 
5897   SDValue Mask, VL;
5898   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5899 
5900   SDValue CopySign =
5901       DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag, Sign, Mask, VL);
5902 
5903   return convertFromScalableVector(VT, CopySign, DAG, Subtarget);
5904 }
5905 
5906 SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
5907     SDValue Op, SelectionDAG &DAG) const {
5908   MVT VT = Op.getSimpleValueType();
5909   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5910 
5911   MVT I1ContainerVT =
5912       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5913 
5914   SDValue CC =
5915       convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget);
5916   SDValue Op1 =
5917       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5918   SDValue Op2 =
5919       convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget);
5920 
5921   SDLoc DL(Op);
5922   SDValue Mask, VL;
5923   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5924 
5925   SDValue Select =
5926       DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC, Op1, Op2, VL);
5927 
5928   return convertFromScalableVector(VT, Select, DAG, Subtarget);
5929 }
5930 
5931 SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
5932                                                unsigned NewOpc,
5933                                                bool HasMask) const {
5934   MVT VT = Op.getSimpleValueType();
5935   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5936 
5937   // Create list of operands by converting existing ones to scalable types.
5938   SmallVector<SDValue, 6> Ops;
5939   for (const SDValue &V : Op->op_values()) {
5940     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5941 
5942     // Pass through non-vector operands.
5943     if (!V.getValueType().isVector()) {
5944       Ops.push_back(V);
5945       continue;
5946     }
5947 
5948     // "cast" fixed length vector to a scalable vector.
5949     assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
5950            "Only fixed length vectors are supported!");
5951     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5952   }
5953 
5954   SDLoc DL(Op);
5955   SDValue Mask, VL;
5956   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5957   if (HasMask)
5958     Ops.push_back(Mask);
5959   Ops.push_back(VL);
5960 
5961   SDValue ScalableRes = DAG.getNode(NewOpc, DL, ContainerVT, Ops);
5962   return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
5963 }
5964 
5965 // Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
5966 // * Operands of each node are assumed to be in the same order.
5967 // * The EVL operand is promoted from i32 to i64 on RV64.
5968 // * Fixed-length vectors are converted to their scalable-vector container
5969 //   types.
5970 SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
5971                                        unsigned RISCVISDOpc) const {
5972   SDLoc DL(Op);
5973   MVT VT = Op.getSimpleValueType();
5974   SmallVector<SDValue, 4> Ops;
5975 
5976   for (const auto &OpIdx : enumerate(Op->ops())) {
5977     SDValue V = OpIdx.value();
5978     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5979     // Pass through operands which aren't fixed-length vectors.
5980     if (!V.getValueType().isFixedLengthVector()) {
5981       Ops.push_back(V);
5982       continue;
5983     }
5984     // "cast" fixed length vector to a scalable vector.
5985     MVT OpVT = V.getSimpleValueType();
5986     MVT ContainerVT = getContainerForFixedLengthVector(OpVT);
5987     assert(useRVVForFixedLengthVectorVT(OpVT) &&
5988            "Only fixed length vectors are supported!");
5989     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5990   }
5991 
5992   if (!VT.isFixedLengthVector())
5993     return DAG.getNode(RISCVISDOpc, DL, VT, Ops);
5994 
5995   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5996 
5997   SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops);
5998 
5999   return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
6000 }
6001 
6002 SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op, SelectionDAG &DAG,
6003                                             unsigned MaskOpc,
6004                                             unsigned VecOpc) const {
6005   MVT VT = Op.getSimpleValueType();
6006   if (VT.getVectorElementType() != MVT::i1)
6007     return lowerVPOp(Op, DAG, VecOpc);
6008 
6009   // It is safe to drop mask parameter as masked-off elements are undef.
6010   SDValue Op1 = Op->getOperand(0);
6011   SDValue Op2 = Op->getOperand(1);
6012   SDValue VL = Op->getOperand(3);
6013 
6014   MVT ContainerVT = VT;
6015   const bool IsFixed = VT.isFixedLengthVector();
6016   if (IsFixed) {
6017     ContainerVT = getContainerForFixedLengthVector(VT);
6018     Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
6019     Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
6020   }
6021 
6022   SDLoc DL(Op);
6023   SDValue Val = DAG.getNode(MaskOpc, DL, ContainerVT, Op1, Op2, VL);
6024   if (!IsFixed)
6025     return Val;
6026   return convertFromScalableVector(VT, Val, DAG, Subtarget);
6027 }
6028 
6029 // Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
6030 // matched to a RVV indexed load. The RVV indexed load instructions only
6031 // support the "unsigned unscaled" addressing mode; indices are implicitly
6032 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
6033 // signed or scaled indexing is extended to the XLEN value type and scaled
6034 // accordingly.
6035 SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
6036                                                SelectionDAG &DAG) const {
6037   SDLoc DL(Op);
6038   MVT VT = Op.getSimpleValueType();
6039 
6040   const auto *MemSD = cast<MemSDNode>(Op.getNode());
6041   EVT MemVT = MemSD->getMemoryVT();
6042   MachineMemOperand *MMO = MemSD->getMemOperand();
6043   SDValue Chain = MemSD->getChain();
6044   SDValue BasePtr = MemSD->getBasePtr();
6045 
6046   ISD::LoadExtType LoadExtType;
6047   SDValue Index, Mask, PassThru, VL;
6048 
6049   if (auto *VPGN = dyn_cast<VPGatherSDNode>(Op.getNode())) {
6050     Index = VPGN->getIndex();
6051     Mask = VPGN->getMask();
6052     PassThru = DAG.getUNDEF(VT);
6053     VL = VPGN->getVectorLength();
6054     // VP doesn't support extending loads.
6055     LoadExtType = ISD::NON_EXTLOAD;
6056   } else {
6057     // Else it must be a MGATHER.
6058     auto *MGN = cast<MaskedGatherSDNode>(Op.getNode());
6059     Index = MGN->getIndex();
6060     Mask = MGN->getMask();
6061     PassThru = MGN->getPassThru();
6062     LoadExtType = MGN->getExtensionType();
6063   }
6064 
6065   MVT IndexVT = Index.getSimpleValueType();
6066   MVT XLenVT = Subtarget.getXLenVT();
6067 
6068   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
6069          "Unexpected VTs!");
6070   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
6071   // Targets have to explicitly opt-in for extending vector loads.
6072   assert(LoadExtType == ISD::NON_EXTLOAD &&
6073          "Unexpected extending MGATHER/VP_GATHER");
6074   (void)LoadExtType;
6075 
6076   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
6077   // the selection of the masked intrinsics doesn't do this for us.
6078   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
6079 
6080   MVT ContainerVT = VT;
6081   if (VT.isFixedLengthVector()) {
6082     // We need to use the larger of the result and index type to determine the
6083     // scalable type to use so we don't increase LMUL for any operand/result.
6084     if (VT.bitsGE(IndexVT)) {
6085       ContainerVT = getContainerForFixedLengthVector(VT);
6086       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
6087                                  ContainerVT.getVectorElementCount());
6088     } else {
6089       IndexVT = getContainerForFixedLengthVector(IndexVT);
6090       ContainerVT = MVT::getVectorVT(ContainerVT.getVectorElementType(),
6091                                      IndexVT.getVectorElementCount());
6092     }
6093 
6094     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
6095 
6096     if (!IsUnmasked) {
6097       MVT MaskVT =
6098           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
6099       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
6100       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
6101     }
6102   }
6103 
6104   if (!VL)
6105     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
6106 
6107   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
6108     IndexVT = IndexVT.changeVectorElementType(XLenVT);
6109     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, Mask.getValueType(),
6110                                    VL);
6111     Index = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, IndexVT, Index,
6112                         TrueMask, VL);
6113   }
6114 
6115   unsigned IntID =
6116       IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
6117   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
6118   if (IsUnmasked)
6119     Ops.push_back(DAG.getUNDEF(ContainerVT));
6120   else
6121     Ops.push_back(PassThru);
6122   Ops.push_back(BasePtr);
6123   Ops.push_back(Index);
6124   if (!IsUnmasked)
6125     Ops.push_back(Mask);
6126   Ops.push_back(VL);
6127   if (!IsUnmasked)
6128     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
6129 
6130   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
6131   SDValue Result =
6132       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
6133   Chain = Result.getValue(1);
6134 
6135   if (VT.isFixedLengthVector())
6136     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
6137 
6138   return DAG.getMergeValues({Result, Chain}, DL);
6139 }
6140 
6141 // Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
6142 // matched to a RVV indexed store. The RVV indexed store instructions only
6143 // support the "unsigned unscaled" addressing mode; indices are implicitly
6144 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
6145 // signed or scaled indexing is extended to the XLEN value type and scaled
6146 // accordingly.
6147 SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
6148                                                 SelectionDAG &DAG) const {
6149   SDLoc DL(Op);
6150   const auto *MemSD = cast<MemSDNode>(Op.getNode());
6151   EVT MemVT = MemSD->getMemoryVT();
6152   MachineMemOperand *MMO = MemSD->getMemOperand();
6153   SDValue Chain = MemSD->getChain();
6154   SDValue BasePtr = MemSD->getBasePtr();
6155 
6156   bool IsTruncatingStore = false;
6157   SDValue Index, Mask, Val, VL;
6158 
6159   if (auto *VPSN = dyn_cast<VPScatterSDNode>(Op.getNode())) {
6160     Index = VPSN->getIndex();
6161     Mask = VPSN->getMask();
6162     Val = VPSN->getValue();
6163     VL = VPSN->getVectorLength();
6164     // VP doesn't support truncating stores.
6165     IsTruncatingStore = false;
6166   } else {
6167     // Else it must be a MSCATTER.
6168     auto *MSN = cast<MaskedScatterSDNode>(Op.getNode());
6169     Index = MSN->getIndex();
6170     Mask = MSN->getMask();
6171     Val = MSN->getValue();
6172     IsTruncatingStore = MSN->isTruncatingStore();
6173   }
6174 
6175   MVT VT = Val.getSimpleValueType();
6176   MVT IndexVT = Index.getSimpleValueType();
6177   MVT XLenVT = Subtarget.getXLenVT();
6178 
6179   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
6180          "Unexpected VTs!");
6181   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
6182   // Targets have to explicitly opt-in for extending vector loads and
6183   // truncating vector stores.
6184   assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
6185   (void)IsTruncatingStore;
6186 
6187   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
6188   // the selection of the masked intrinsics doesn't do this for us.
6189   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
6190 
6191   MVT ContainerVT = VT;
6192   if (VT.isFixedLengthVector()) {
6193     // We need to use the larger of the value and index type to determine the
6194     // scalable type to use so we don't increase LMUL for any operand/result.
6195     if (VT.bitsGE(IndexVT)) {
6196       ContainerVT = getContainerForFixedLengthVector(VT);
6197       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
6198                                  ContainerVT.getVectorElementCount());
6199     } else {
6200       IndexVT = getContainerForFixedLengthVector(IndexVT);
6201       ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
6202                                      IndexVT.getVectorElementCount());
6203     }
6204 
6205     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
6206     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
6207 
6208     if (!IsUnmasked) {
6209       MVT MaskVT =
6210           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
6211       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
6212     }
6213   }
6214 
6215   if (!VL)
6216     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
6217 
6218   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
6219     IndexVT = IndexVT.changeVectorElementType(XLenVT);
6220     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, Mask.getValueType(),
6221                                    VL);
6222     Index = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, IndexVT, Index,
6223                         TrueMask, VL);
6224   }
6225 
6226   unsigned IntID =
6227       IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
6228   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
6229   Ops.push_back(Val);
6230   Ops.push_back(BasePtr);
6231   Ops.push_back(Index);
6232   if (!IsUnmasked)
6233     Ops.push_back(Mask);
6234   Ops.push_back(VL);
6235 
6236   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
6237                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
6238 }
6239 
6240 SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
6241                                                SelectionDAG &DAG) const {
6242   const MVT XLenVT = Subtarget.getXLenVT();
6243   SDLoc DL(Op);
6244   SDValue Chain = Op->getOperand(0);
6245   SDValue SysRegNo = DAG.getTargetConstant(
6246       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
6247   SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
6248   SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
6249 
6250   // Encoding used for rounding mode in RISCV differs from that used in
6251   // FLT_ROUNDS. To convert it the RISCV rounding mode is used as an index in a
6252   // table, which consists of a sequence of 4-bit fields, each representing
6253   // corresponding FLT_ROUNDS mode.
6254   static const int Table =
6255       (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
6256       (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
6257       (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
6258       (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
6259       (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
6260 
6261   SDValue Shift =
6262       DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
6263   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
6264                                 DAG.getConstant(Table, DL, XLenVT), Shift);
6265   SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
6266                                DAG.getConstant(7, DL, XLenVT));
6267 
6268   return DAG.getMergeValues({Masked, Chain}, DL);
6269 }
6270 
6271 SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
6272                                                SelectionDAG &DAG) const {
6273   const MVT XLenVT = Subtarget.getXLenVT();
6274   SDLoc DL(Op);
6275   SDValue Chain = Op->getOperand(0);
6276   SDValue RMValue = Op->getOperand(1);
6277   SDValue SysRegNo = DAG.getTargetConstant(
6278       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
6279 
6280   // Encoding used for rounding mode in RISCV differs from that used in
6281   // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
6282   // a table, which consists of a sequence of 4-bit fields, each representing
6283   // corresponding RISCV mode.
6284   static const unsigned Table =
6285       (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
6286       (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
6287       (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
6288       (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
6289       (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
6290 
6291   SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
6292                               DAG.getConstant(2, DL, XLenVT));
6293   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
6294                                 DAG.getConstant(Table, DL, XLenVT), Shift);
6295   RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
6296                         DAG.getConstant(0x7, DL, XLenVT));
6297   return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
6298                      RMValue);
6299 }
6300 
6301 static RISCVISD::NodeType getRISCVWOpcodeByIntr(unsigned IntNo) {
6302   switch (IntNo) {
6303   default:
6304     llvm_unreachable("Unexpected Intrinsic");
6305   case Intrinsic::riscv_grev:
6306     return RISCVISD::GREVW;
6307   case Intrinsic::riscv_gorc:
6308     return RISCVISD::GORCW;
6309   case Intrinsic::riscv_bcompress:
6310     return RISCVISD::BCOMPRESSW;
6311   case Intrinsic::riscv_bdecompress:
6312     return RISCVISD::BDECOMPRESSW;
6313   case Intrinsic::riscv_bfp:
6314     return RISCVISD::BFPW;
6315   case Intrinsic::riscv_fsl:
6316     return RISCVISD::FSLW;
6317   case Intrinsic::riscv_fsr:
6318     return RISCVISD::FSRW;
6319   }
6320 }
6321 
6322 // Converts the given intrinsic to a i64 operation with any extension.
6323 static SDValue customLegalizeToWOpByIntr(SDNode *N, SelectionDAG &DAG,
6324                                          unsigned IntNo) {
6325   SDLoc DL(N);
6326   RISCVISD::NodeType WOpcode = getRISCVWOpcodeByIntr(IntNo);
6327   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6328   SDValue NewOp2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6329   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp1, NewOp2);
6330   // ReplaceNodeResults requires we maintain the same type for the return value.
6331   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
6332 }
6333 
6334 // Returns the opcode of the target-specific SDNode that implements the 32-bit
6335 // form of the given Opcode.
6336 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
6337   switch (Opcode) {
6338   default:
6339     llvm_unreachable("Unexpected opcode");
6340   case ISD::SHL:
6341     return RISCVISD::SLLW;
6342   case ISD::SRA:
6343     return RISCVISD::SRAW;
6344   case ISD::SRL:
6345     return RISCVISD::SRLW;
6346   case ISD::SDIV:
6347     return RISCVISD::DIVW;
6348   case ISD::UDIV:
6349     return RISCVISD::DIVUW;
6350   case ISD::UREM:
6351     return RISCVISD::REMUW;
6352   case ISD::ROTL:
6353     return RISCVISD::ROLW;
6354   case ISD::ROTR:
6355     return RISCVISD::RORW;
6356   case RISCVISD::GREV:
6357     return RISCVISD::GREVW;
6358   case RISCVISD::GORC:
6359     return RISCVISD::GORCW;
6360   }
6361 }
6362 
6363 // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
6364 // node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
6365 // otherwise be promoted to i64, making it difficult to select the
6366 // SLLW/DIVUW/.../*W later one because the fact the operation was originally of
6367 // type i8/i16/i32 is lost.
6368 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
6369                                    unsigned ExtOpc = ISD::ANY_EXTEND) {
6370   SDLoc DL(N);
6371   RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6372   SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
6373   SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
6374   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6375   // ReplaceNodeResults requires we maintain the same type for the return value.
6376   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
6377 }
6378 
6379 // Converts the given 32-bit operation to a i64 operation with signed extension
6380 // semantic to reduce the signed extension instructions.
6381 static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
6382   SDLoc DL(N);
6383   SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6384   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6385   SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
6386   SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6387                                DAG.getValueType(MVT::i32));
6388   return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
6389 }
6390 
6391 void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
6392                                              SmallVectorImpl<SDValue> &Results,
6393                                              SelectionDAG &DAG) const {
6394   SDLoc DL(N);
6395   switch (N->getOpcode()) {
6396   default:
6397     llvm_unreachable("Don't know how to custom type legalize this operation!");
6398   case ISD::STRICT_FP_TO_SINT:
6399   case ISD::STRICT_FP_TO_UINT:
6400   case ISD::FP_TO_SINT:
6401   case ISD::FP_TO_UINT: {
6402     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6403            "Unexpected custom legalisation");
6404     bool IsStrict = N->isStrictFPOpcode();
6405     bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
6406                     N->getOpcode() == ISD::STRICT_FP_TO_SINT;
6407     SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
6408     if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
6409         TargetLowering::TypeSoftenFloat) {
6410       if (!isTypeLegal(Op0.getValueType()))
6411         return;
6412       if (IsStrict) {
6413         unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
6414                                 : RISCVISD::STRICT_FCVT_WU_RV64;
6415         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
6416         SDValue Res = DAG.getNode(
6417             Opc, DL, VTs, N->getOperand(0), Op0,
6418             DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
6419         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6420         Results.push_back(Res.getValue(1));
6421         return;
6422       }
6423       unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
6424       SDValue Res =
6425           DAG.getNode(Opc, DL, MVT::i64, Op0,
6426                       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
6427       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6428       return;
6429     }
6430     // If the FP type needs to be softened, emit a library call using the 'si'
6431     // version. If we left it to default legalization we'd end up with 'di'. If
6432     // the FP type doesn't need to be softened just let generic type
6433     // legalization promote the result type.
6434     RTLIB::Libcall LC;
6435     if (IsSigned)
6436       LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
6437     else
6438       LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
6439     MakeLibCallOptions CallOptions;
6440     EVT OpVT = Op0.getValueType();
6441     CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
6442     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
6443     SDValue Result;
6444     std::tie(Result, Chain) =
6445         makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
6446     Results.push_back(Result);
6447     if (IsStrict)
6448       Results.push_back(Chain);
6449     break;
6450   }
6451   case ISD::READCYCLECOUNTER: {
6452     assert(!Subtarget.is64Bit() &&
6453            "READCYCLECOUNTER only has custom type legalization on riscv32");
6454 
6455     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
6456     SDValue RCW =
6457         DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
6458 
6459     Results.push_back(
6460         DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
6461     Results.push_back(RCW.getValue(2));
6462     break;
6463   }
6464   case ISD::MUL: {
6465     unsigned Size = N->getSimpleValueType(0).getSizeInBits();
6466     unsigned XLen = Subtarget.getXLen();
6467     // This multiply needs to be expanded, try to use MULHSU+MUL if possible.
6468     if (Size > XLen) {
6469       assert(Size == (XLen * 2) && "Unexpected custom legalisation");
6470       SDValue LHS = N->getOperand(0);
6471       SDValue RHS = N->getOperand(1);
6472       APInt HighMask = APInt::getHighBitsSet(Size, XLen);
6473 
6474       bool LHSIsU = DAG.MaskedValueIsZero(LHS, HighMask);
6475       bool RHSIsU = DAG.MaskedValueIsZero(RHS, HighMask);
6476       // We need exactly one side to be unsigned.
6477       if (LHSIsU == RHSIsU)
6478         return;
6479 
6480       auto MakeMULPair = [&](SDValue S, SDValue U) {
6481         MVT XLenVT = Subtarget.getXLenVT();
6482         S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S);
6483         U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U);
6484         SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U);
6485         SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U);
6486         return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
6487       };
6488 
6489       bool LHSIsS = DAG.ComputeNumSignBits(LHS) > XLen;
6490       bool RHSIsS = DAG.ComputeNumSignBits(RHS) > XLen;
6491 
6492       // The other operand should be signed, but still prefer MULH when
6493       // possible.
6494       if (RHSIsU && LHSIsS && !RHSIsS)
6495         Results.push_back(MakeMULPair(LHS, RHS));
6496       else if (LHSIsU && RHSIsS && !LHSIsS)
6497         Results.push_back(MakeMULPair(RHS, LHS));
6498 
6499       return;
6500     }
6501     LLVM_FALLTHROUGH;
6502   }
6503   case ISD::ADD:
6504   case ISD::SUB:
6505     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6506            "Unexpected custom legalisation");
6507     Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
6508     break;
6509   case ISD::SHL:
6510   case ISD::SRA:
6511   case ISD::SRL:
6512     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6513            "Unexpected custom legalisation");
6514     if (N->getOperand(1).getOpcode() != ISD::Constant) {
6515       Results.push_back(customLegalizeToWOp(N, DAG));
6516       break;
6517     }
6518 
6519     // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
6520     // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
6521     // shift amount.
6522     if (N->getOpcode() == ISD::SHL) {
6523       SDLoc DL(N);
6524       SDValue NewOp0 =
6525           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6526       SDValue NewOp1 =
6527           DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
6528       SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
6529       SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6530                                    DAG.getValueType(MVT::i32));
6531       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6532     }
6533 
6534     break;
6535   case ISD::ROTL:
6536   case ISD::ROTR:
6537     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6538            "Unexpected custom legalisation");
6539     Results.push_back(customLegalizeToWOp(N, DAG));
6540     break;
6541   case ISD::CTTZ:
6542   case ISD::CTTZ_ZERO_UNDEF:
6543   case ISD::CTLZ:
6544   case ISD::CTLZ_ZERO_UNDEF: {
6545     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6546            "Unexpected custom legalisation");
6547 
6548     SDValue NewOp0 =
6549         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6550     bool IsCTZ =
6551         N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF;
6552     unsigned Opc = IsCTZ ? RISCVISD::CTZW : RISCVISD::CLZW;
6553     SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
6554     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6555     return;
6556   }
6557   case ISD::SDIV:
6558   case ISD::UDIV:
6559   case ISD::UREM: {
6560     MVT VT = N->getSimpleValueType(0);
6561     assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
6562            Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
6563            "Unexpected custom legalisation");
6564     // Don't promote division/remainder by constant since we should expand those
6565     // to multiply by magic constant.
6566     // FIXME: What if the expansion is disabled for minsize.
6567     if (N->getOperand(1).getOpcode() == ISD::Constant)
6568       return;
6569 
6570     // If the input is i32, use ANY_EXTEND since the W instructions don't read
6571     // the upper 32 bits. For other types we need to sign or zero extend
6572     // based on the opcode.
6573     unsigned ExtOpc = ISD::ANY_EXTEND;
6574     if (VT != MVT::i32)
6575       ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
6576                                            : ISD::ZERO_EXTEND;
6577 
6578     Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
6579     break;
6580   }
6581   case ISD::UADDO:
6582   case ISD::USUBO: {
6583     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6584            "Unexpected custom legalisation");
6585     bool IsAdd = N->getOpcode() == ISD::UADDO;
6586     // Create an ADDW or SUBW.
6587     SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6588     SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6589     SDValue Res =
6590         DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
6591     Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
6592                       DAG.getValueType(MVT::i32));
6593 
6594     // Sign extend the LHS and perform an unsigned compare with the ADDW result.
6595     // Since the inputs are sign extended from i32, this is equivalent to
6596     // comparing the lower 32 bits.
6597     LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6598     SDValue Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
6599                                     IsAdd ? ISD::SETULT : ISD::SETUGT);
6600 
6601     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6602     Results.push_back(Overflow);
6603     return;
6604   }
6605   case ISD::UADDSAT:
6606   case ISD::USUBSAT: {
6607     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6608            "Unexpected custom legalisation");
6609     if (Subtarget.hasStdExtZbb()) {
6610       // With Zbb we can sign extend and let LegalizeDAG use minu/maxu. Using
6611       // sign extend allows overflow of the lower 32 bits to be detected on
6612       // the promoted size.
6613       SDValue LHS =
6614           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6615       SDValue RHS =
6616           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
6617       SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
6618       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6619       return;
6620     }
6621 
6622     // Without Zbb, expand to UADDO/USUBO+select which will trigger our custom
6623     // promotion for UADDO/USUBO.
6624     Results.push_back(expandAddSubSat(N, DAG));
6625     return;
6626   }
6627   case ISD::BITCAST: {
6628     EVT VT = N->getValueType(0);
6629     assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
6630     SDValue Op0 = N->getOperand(0);
6631     EVT Op0VT = Op0.getValueType();
6632     MVT XLenVT = Subtarget.getXLenVT();
6633     if (VT == MVT::i16 && Op0VT == MVT::f16 && Subtarget.hasStdExtZfh()) {
6634       SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
6635       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
6636     } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
6637                Subtarget.hasStdExtF()) {
6638       SDValue FPConv =
6639           DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
6640       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
6641     } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
6642                isTypeLegal(Op0VT)) {
6643       // Custom-legalize bitcasts from fixed-length vector types to illegal
6644       // scalar types in order to improve codegen. Bitcast the vector to a
6645       // one-element vector type whose element type is the same as the result
6646       // type, and extract the first element.
6647       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
6648       if (isTypeLegal(BVT)) {
6649         SDValue BVec = DAG.getBitcast(BVT, Op0);
6650         Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
6651                                       DAG.getConstant(0, DL, XLenVT)));
6652       }
6653     }
6654     break;
6655   }
6656   case RISCVISD::GREV:
6657   case RISCVISD::GORC: {
6658     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6659            "Unexpected custom legalisation");
6660     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6661     // This is similar to customLegalizeToWOp, except that we pass the second
6662     // operand (a TargetConstant) straight through: it is already of type
6663     // XLenVT.
6664     RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6665     SDValue NewOp0 =
6666         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6667     SDValue NewOp1 =
6668         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6669     SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6670     // ReplaceNodeResults requires we maintain the same type for the return
6671     // value.
6672     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6673     break;
6674   }
6675   case RISCVISD::SHFL: {
6676     // There is no SHFLIW instruction, but we can just promote the operation.
6677     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6678            "Unexpected custom legalisation");
6679     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6680     SDValue NewOp0 =
6681         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6682     SDValue NewOp1 =
6683         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6684     SDValue NewRes = DAG.getNode(RISCVISD::SHFL, DL, MVT::i64, NewOp0, NewOp1);
6685     // ReplaceNodeResults requires we maintain the same type for the return
6686     // value.
6687     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6688     break;
6689   }
6690   case ISD::BSWAP:
6691   case ISD::BITREVERSE: {
6692     MVT VT = N->getSimpleValueType(0);
6693     MVT XLenVT = Subtarget.getXLenVT();
6694     assert((VT == MVT::i8 || VT == MVT::i16 ||
6695             (VT == MVT::i32 && Subtarget.is64Bit())) &&
6696            Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
6697     SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
6698     unsigned Imm = VT.getSizeInBits() - 1;
6699     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
6700     if (N->getOpcode() == ISD::BSWAP)
6701       Imm &= ~0x7U;
6702     unsigned Opc = Subtarget.is64Bit() ? RISCVISD::GREVW : RISCVISD::GREV;
6703     SDValue GREVI =
6704         DAG.getNode(Opc, DL, XLenVT, NewOp0, DAG.getConstant(Imm, DL, XLenVT));
6705     // ReplaceNodeResults requires we maintain the same type for the return
6706     // value.
6707     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, GREVI));
6708     break;
6709   }
6710   case ISD::FSHL:
6711   case ISD::FSHR: {
6712     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6713            Subtarget.hasStdExtZbt() && "Unexpected custom legalisation");
6714     SDValue NewOp0 =
6715         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6716     SDValue NewOp1 =
6717         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6718     SDValue NewShAmt =
6719         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6720     // FSLW/FSRW take a 6 bit shift amount but i32 FSHL/FSHR only use 5 bits.
6721     // Mask the shift amount to 5 bits to prevent accidentally setting bit 5.
6722     NewShAmt = DAG.getNode(ISD::AND, DL, MVT::i64, NewShAmt,
6723                            DAG.getConstant(0x1f, DL, MVT::i64));
6724     // fshl and fshr concatenate their operands in the same order. fsrw and fslw
6725     // instruction use different orders. fshl will return its first operand for
6726     // shift of zero, fshr will return its second operand. fsl and fsr both
6727     // return rs1 so the ISD nodes need to have different operand orders.
6728     // Shift amount is in rs2.
6729     unsigned Opc = RISCVISD::FSLW;
6730     if (N->getOpcode() == ISD::FSHR) {
6731       std::swap(NewOp0, NewOp1);
6732       Opc = RISCVISD::FSRW;
6733     }
6734     SDValue NewOp = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, NewShAmt);
6735     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewOp));
6736     break;
6737   }
6738   case ISD::EXTRACT_VECTOR_ELT: {
6739     // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
6740     // type is illegal (currently only vXi64 RV32).
6741     // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
6742     // transferred to the destination register. We issue two of these from the
6743     // upper- and lower- halves of the SEW-bit vector element, slid down to the
6744     // first element.
6745     SDValue Vec = N->getOperand(0);
6746     SDValue Idx = N->getOperand(1);
6747 
6748     // The vector type hasn't been legalized yet so we can't issue target
6749     // specific nodes if it needs legalization.
6750     // FIXME: We would manually legalize if it's important.
6751     if (!isTypeLegal(Vec.getValueType()))
6752       return;
6753 
6754     MVT VecVT = Vec.getSimpleValueType();
6755 
6756     assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
6757            VecVT.getVectorElementType() == MVT::i64 &&
6758            "Unexpected EXTRACT_VECTOR_ELT legalization");
6759 
6760     // If this is a fixed vector, we need to convert it to a scalable vector.
6761     MVT ContainerVT = VecVT;
6762     if (VecVT.isFixedLengthVector()) {
6763       ContainerVT = getContainerForFixedLengthVector(VecVT);
6764       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
6765     }
6766 
6767     MVT XLenVT = Subtarget.getXLenVT();
6768 
6769     // Use a VL of 1 to avoid processing more elements than we need.
6770     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
6771     SDValue VL = DAG.getConstant(1, DL, XLenVT);
6772     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6773 
6774     // Unless the index is known to be 0, we must slide the vector down to get
6775     // the desired element into index 0.
6776     if (!isNullConstant(Idx)) {
6777       Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
6778                         DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
6779     }
6780 
6781     // Extract the lower XLEN bits of the correct vector element.
6782     SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6783 
6784     // To extract the upper XLEN bits of the vector element, shift the first
6785     // element right by 32 bits and re-extract the lower XLEN bits.
6786     SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
6787                                      DAG.getConstant(32, DL, XLenVT), VL);
6788     SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec,
6789                                  ThirtyTwoV, Mask, VL);
6790 
6791     SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6792 
6793     Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6794     break;
6795   }
6796   case ISD::INTRINSIC_WO_CHAIN: {
6797     unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
6798     switch (IntNo) {
6799     default:
6800       llvm_unreachable(
6801           "Don't know how to custom type legalize this intrinsic!");
6802     case Intrinsic::riscv_grev:
6803     case Intrinsic::riscv_gorc:
6804     case Intrinsic::riscv_bcompress:
6805     case Intrinsic::riscv_bdecompress:
6806     case Intrinsic::riscv_bfp: {
6807       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6808              "Unexpected custom legalisation");
6809       Results.push_back(customLegalizeToWOpByIntr(N, DAG, IntNo));
6810       break;
6811     }
6812     case Intrinsic::riscv_fsl:
6813     case Intrinsic::riscv_fsr: {
6814       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6815              "Unexpected custom legalisation");
6816       SDValue NewOp1 =
6817           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6818       SDValue NewOp2 =
6819           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6820       SDValue NewOp3 =
6821           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3));
6822       unsigned Opc = getRISCVWOpcodeByIntr(IntNo);
6823       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2, NewOp3);
6824       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6825       break;
6826     }
6827     case Intrinsic::riscv_orc_b: {
6828       // Lower to the GORCI encoding for orc.b with the operand extended.
6829       SDValue NewOp =
6830           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6831       // If Zbp is enabled, use GORCIW which will sign extend the result.
6832       unsigned Opc =
6833           Subtarget.hasStdExtZbp() ? RISCVISD::GORCW : RISCVISD::GORC;
6834       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp,
6835                                 DAG.getConstant(7, DL, MVT::i64));
6836       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6837       return;
6838     }
6839     case Intrinsic::riscv_shfl:
6840     case Intrinsic::riscv_unshfl: {
6841       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6842              "Unexpected custom legalisation");
6843       SDValue NewOp1 =
6844           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6845       SDValue NewOp2 =
6846           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6847       unsigned Opc =
6848           IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFLW : RISCVISD::UNSHFLW;
6849       // There is no (UN)SHFLIW. If the control word is a constant, we can use
6850       // (UN)SHFLI with bit 4 of the control word cleared. The upper 32 bit half
6851       // will be shuffled the same way as the lower 32 bit half, but the two
6852       // halves won't cross.
6853       if (isa<ConstantSDNode>(NewOp2)) {
6854         NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2,
6855                              DAG.getConstant(0xf, DL, MVT::i64));
6856         Opc =
6857             IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
6858       }
6859       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2);
6860       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6861       break;
6862     }
6863     case Intrinsic::riscv_vmv_x_s: {
6864       EVT VT = N->getValueType(0);
6865       MVT XLenVT = Subtarget.getXLenVT();
6866       if (VT.bitsLT(XLenVT)) {
6867         // Simple case just extract using vmv.x.s and truncate.
6868         SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
6869                                       Subtarget.getXLenVT(), N->getOperand(1));
6870         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
6871         return;
6872       }
6873 
6874       assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
6875              "Unexpected custom legalization");
6876 
6877       // We need to do the move in two steps.
6878       SDValue Vec = N->getOperand(1);
6879       MVT VecVT = Vec.getSimpleValueType();
6880 
6881       // First extract the lower XLEN bits of the element.
6882       SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6883 
6884       // To extract the upper XLEN bits of the vector element, shift the first
6885       // element right by 32 bits and re-extract the lower XLEN bits.
6886       SDValue VL = DAG.getConstant(1, DL, XLenVT);
6887       MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
6888       SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6889       SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT,
6890                                        DAG.getConstant(32, DL, XLenVT), VL);
6891       SDValue LShr32 =
6892           DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV, Mask, VL);
6893       SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6894 
6895       Results.push_back(
6896           DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6897       break;
6898     }
6899     }
6900     break;
6901   }
6902   case ISD::VECREDUCE_ADD:
6903   case ISD::VECREDUCE_AND:
6904   case ISD::VECREDUCE_OR:
6905   case ISD::VECREDUCE_XOR:
6906   case ISD::VECREDUCE_SMAX:
6907   case ISD::VECREDUCE_UMAX:
6908   case ISD::VECREDUCE_SMIN:
6909   case ISD::VECREDUCE_UMIN:
6910     if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
6911       Results.push_back(V);
6912     break;
6913   case ISD::VP_REDUCE_ADD:
6914   case ISD::VP_REDUCE_AND:
6915   case ISD::VP_REDUCE_OR:
6916   case ISD::VP_REDUCE_XOR:
6917   case ISD::VP_REDUCE_SMAX:
6918   case ISD::VP_REDUCE_UMAX:
6919   case ISD::VP_REDUCE_SMIN:
6920   case ISD::VP_REDUCE_UMIN:
6921     if (SDValue V = lowerVPREDUCE(SDValue(N, 0), DAG))
6922       Results.push_back(V);
6923     break;
6924   case ISD::FLT_ROUNDS_: {
6925     SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other);
6926     SDValue Res = DAG.getNode(ISD::FLT_ROUNDS_, DL, VTs, N->getOperand(0));
6927     Results.push_back(Res.getValue(0));
6928     Results.push_back(Res.getValue(1));
6929     break;
6930   }
6931   }
6932 }
6933 
6934 // A structure to hold one of the bit-manipulation patterns below. Together, a
6935 // SHL and non-SHL pattern may form a bit-manipulation pair on a single source:
6936 //   (or (and (shl x, 1), 0xAAAAAAAA),
6937 //       (and (srl x, 1), 0x55555555))
6938 struct RISCVBitmanipPat {
6939   SDValue Op;
6940   unsigned ShAmt;
6941   bool IsSHL;
6942 
6943   bool formsPairWith(const RISCVBitmanipPat &Other) const {
6944     return Op == Other.Op && ShAmt == Other.ShAmt && IsSHL != Other.IsSHL;
6945   }
6946 };
6947 
6948 // Matches patterns of the form
6949 //   (and (shl x, C2), (C1 << C2))
6950 //   (and (srl x, C2), C1)
6951 //   (shl (and x, C1), C2)
6952 //   (srl (and x, (C1 << C2)), C2)
6953 // Where C2 is a power of 2 and C1 has at least that many leading zeroes.
6954 // The expected masks for each shift amount are specified in BitmanipMasks where
6955 // BitmanipMasks[log2(C2)] specifies the expected C1 value.
6956 // The max allowed shift amount is either XLen/2 or XLen/4 determined by whether
6957 // BitmanipMasks contains 6 or 5 entries assuming that the maximum possible
6958 // XLen is 64.
6959 static Optional<RISCVBitmanipPat>
6960 matchRISCVBitmanipPat(SDValue Op, ArrayRef<uint64_t> BitmanipMasks) {
6961   assert((BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) &&
6962          "Unexpected number of masks");
6963   Optional<uint64_t> Mask;
6964   // Optionally consume a mask around the shift operation.
6965   if (Op.getOpcode() == ISD::AND && isa<ConstantSDNode>(Op.getOperand(1))) {
6966     Mask = Op.getConstantOperandVal(1);
6967     Op = Op.getOperand(0);
6968   }
6969   if (Op.getOpcode() != ISD::SHL && Op.getOpcode() != ISD::SRL)
6970     return None;
6971   bool IsSHL = Op.getOpcode() == ISD::SHL;
6972 
6973   if (!isa<ConstantSDNode>(Op.getOperand(1)))
6974     return None;
6975   uint64_t ShAmt = Op.getConstantOperandVal(1);
6976 
6977   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
6978   if (ShAmt >= Width || !isPowerOf2_64(ShAmt))
6979     return None;
6980   // If we don't have enough masks for 64 bit, then we must be trying to
6981   // match SHFL so we're only allowed to shift 1/4 of the width.
6982   if (BitmanipMasks.size() == 5 && ShAmt >= (Width / 2))
6983     return None;
6984 
6985   SDValue Src = Op.getOperand(0);
6986 
6987   // The expected mask is shifted left when the AND is found around SHL
6988   // patterns.
6989   //   ((x >> 1) & 0x55555555)
6990   //   ((x << 1) & 0xAAAAAAAA)
6991   bool SHLExpMask = IsSHL;
6992 
6993   if (!Mask) {
6994     // Sometimes LLVM keeps the mask as an operand of the shift, typically when
6995     // the mask is all ones: consume that now.
6996     if (Src.getOpcode() == ISD::AND && isa<ConstantSDNode>(Src.getOperand(1))) {
6997       Mask = Src.getConstantOperandVal(1);
6998       Src = Src.getOperand(0);
6999       // The expected mask is now in fact shifted left for SRL, so reverse the
7000       // decision.
7001       //   ((x & 0xAAAAAAAA) >> 1)
7002       //   ((x & 0x55555555) << 1)
7003       SHLExpMask = !SHLExpMask;
7004     } else {
7005       // Use a default shifted mask of all-ones if there's no AND, truncated
7006       // down to the expected width. This simplifies the logic later on.
7007       Mask = maskTrailingOnes<uint64_t>(Width);
7008       *Mask &= (IsSHL ? *Mask << ShAmt : *Mask >> ShAmt);
7009     }
7010   }
7011 
7012   unsigned MaskIdx = Log2_32(ShAmt);
7013   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
7014 
7015   if (SHLExpMask)
7016     ExpMask <<= ShAmt;
7017 
7018   if (Mask != ExpMask)
7019     return None;
7020 
7021   return RISCVBitmanipPat{Src, (unsigned)ShAmt, IsSHL};
7022 }
7023 
7024 // Matches any of the following bit-manipulation patterns:
7025 //   (and (shl x, 1), (0x55555555 << 1))
7026 //   (and (srl x, 1), 0x55555555)
7027 //   (shl (and x, 0x55555555), 1)
7028 //   (srl (and x, (0x55555555 << 1)), 1)
7029 // where the shift amount and mask may vary thus:
7030 //   [1]  = 0x55555555 / 0xAAAAAAAA
7031 //   [2]  = 0x33333333 / 0xCCCCCCCC
7032 //   [4]  = 0x0F0F0F0F / 0xF0F0F0F0
7033 //   [8]  = 0x00FF00FF / 0xFF00FF00
7034 //   [16] = 0x0000FFFF / 0xFFFFFFFF
7035 //   [32] = 0x00000000FFFFFFFF / 0xFFFFFFFF00000000 (for RV64)
7036 static Optional<RISCVBitmanipPat> matchGREVIPat(SDValue Op) {
7037   // These are the unshifted masks which we use to match bit-manipulation
7038   // patterns. They may be shifted left in certain circumstances.
7039   static const uint64_t BitmanipMasks[] = {
7040       0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
7041       0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
7042 
7043   return matchRISCVBitmanipPat(Op, BitmanipMasks);
7044 }
7045 
7046 // Match the following pattern as a GREVI(W) operation
7047 //   (or (BITMANIP_SHL x), (BITMANIP_SRL x))
7048 static SDValue combineORToGREV(SDValue Op, SelectionDAG &DAG,
7049                                const RISCVSubtarget &Subtarget) {
7050   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
7051   EVT VT = Op.getValueType();
7052 
7053   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
7054     auto LHS = matchGREVIPat(Op.getOperand(0));
7055     auto RHS = matchGREVIPat(Op.getOperand(1));
7056     if (LHS && RHS && LHS->formsPairWith(*RHS)) {
7057       SDLoc DL(Op);
7058       return DAG.getNode(RISCVISD::GREV, DL, VT, LHS->Op,
7059                          DAG.getConstant(LHS->ShAmt, DL, VT));
7060     }
7061   }
7062   return SDValue();
7063 }
7064 
7065 // Matches any the following pattern as a GORCI(W) operation
7066 // 1.  (or (GREVI x, shamt), x) if shamt is a power of 2
7067 // 2.  (or x, (GREVI x, shamt)) if shamt is a power of 2
7068 // 3.  (or (or (BITMANIP_SHL x), x), (BITMANIP_SRL x))
7069 // Note that with the variant of 3.,
7070 //     (or (or (BITMANIP_SHL x), (BITMANIP_SRL x)), x)
7071 // the inner pattern will first be matched as GREVI and then the outer
7072 // pattern will be matched to GORC via the first rule above.
7073 // 4.  (or (rotl/rotr x, bitwidth/2), x)
7074 static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
7075                                const RISCVSubtarget &Subtarget) {
7076   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
7077   EVT VT = Op.getValueType();
7078 
7079   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
7080     SDLoc DL(Op);
7081     SDValue Op0 = Op.getOperand(0);
7082     SDValue Op1 = Op.getOperand(1);
7083 
7084     auto MatchOROfReverse = [&](SDValue Reverse, SDValue X) {
7085       if (Reverse.getOpcode() == RISCVISD::GREV && Reverse.getOperand(0) == X &&
7086           isa<ConstantSDNode>(Reverse.getOperand(1)) &&
7087           isPowerOf2_32(Reverse.getConstantOperandVal(1)))
7088         return DAG.getNode(RISCVISD::GORC, DL, VT, X, Reverse.getOperand(1));
7089       // We can also form GORCI from ROTL/ROTR by half the bitwidth.
7090       if ((Reverse.getOpcode() == ISD::ROTL ||
7091            Reverse.getOpcode() == ISD::ROTR) &&
7092           Reverse.getOperand(0) == X &&
7093           isa<ConstantSDNode>(Reverse.getOperand(1))) {
7094         uint64_t RotAmt = Reverse.getConstantOperandVal(1);
7095         if (RotAmt == (VT.getSizeInBits() / 2))
7096           return DAG.getNode(RISCVISD::GORC, DL, VT, X,
7097                              DAG.getConstant(RotAmt, DL, VT));
7098       }
7099       return SDValue();
7100     };
7101 
7102     // Check for either commutable permutation of (or (GREVI x, shamt), x)
7103     if (SDValue V = MatchOROfReverse(Op0, Op1))
7104       return V;
7105     if (SDValue V = MatchOROfReverse(Op1, Op0))
7106       return V;
7107 
7108     // OR is commutable so canonicalize its OR operand to the left
7109     if (Op0.getOpcode() != ISD::OR && Op1.getOpcode() == ISD::OR)
7110       std::swap(Op0, Op1);
7111     if (Op0.getOpcode() != ISD::OR)
7112       return SDValue();
7113     SDValue OrOp0 = Op0.getOperand(0);
7114     SDValue OrOp1 = Op0.getOperand(1);
7115     auto LHS = matchGREVIPat(OrOp0);
7116     // OR is commutable so swap the operands and try again: x might have been
7117     // on the left
7118     if (!LHS) {
7119       std::swap(OrOp0, OrOp1);
7120       LHS = matchGREVIPat(OrOp0);
7121     }
7122     auto RHS = matchGREVIPat(Op1);
7123     if (LHS && RHS && LHS->formsPairWith(*RHS) && LHS->Op == OrOp1) {
7124       return DAG.getNode(RISCVISD::GORC, DL, VT, LHS->Op,
7125                          DAG.getConstant(LHS->ShAmt, DL, VT));
7126     }
7127   }
7128   return SDValue();
7129 }
7130 
7131 // Matches any of the following bit-manipulation patterns:
7132 //   (and (shl x, 1), (0x22222222 << 1))
7133 //   (and (srl x, 1), 0x22222222)
7134 //   (shl (and x, 0x22222222), 1)
7135 //   (srl (and x, (0x22222222 << 1)), 1)
7136 // where the shift amount and mask may vary thus:
7137 //   [1]  = 0x22222222 / 0x44444444
7138 //   [2]  = 0x0C0C0C0C / 0x3C3C3C3C
7139 //   [4]  = 0x00F000F0 / 0x0F000F00
7140 //   [8]  = 0x0000FF00 / 0x00FF0000
7141 //   [16] = 0x00000000FFFF0000 / 0x0000FFFF00000000 (for RV64)
7142 static Optional<RISCVBitmanipPat> matchSHFLPat(SDValue Op) {
7143   // These are the unshifted masks which we use to match bit-manipulation
7144   // patterns. They may be shifted left in certain circumstances.
7145   static const uint64_t BitmanipMasks[] = {
7146       0x2222222222222222ULL, 0x0C0C0C0C0C0C0C0CULL, 0x00F000F000F000F0ULL,
7147       0x0000FF000000FF00ULL, 0x00000000FFFF0000ULL};
7148 
7149   return matchRISCVBitmanipPat(Op, BitmanipMasks);
7150 }
7151 
7152 // Match (or (or (SHFL_SHL x), (SHFL_SHR x)), (SHFL_AND x)
7153 static SDValue combineORToSHFL(SDValue Op, SelectionDAG &DAG,
7154                                const RISCVSubtarget &Subtarget) {
7155   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
7156   EVT VT = Op.getValueType();
7157 
7158   if (VT != MVT::i32 && VT != Subtarget.getXLenVT())
7159     return SDValue();
7160 
7161   SDValue Op0 = Op.getOperand(0);
7162   SDValue Op1 = Op.getOperand(1);
7163 
7164   // Or is commutable so canonicalize the second OR to the LHS.
7165   if (Op0.getOpcode() != ISD::OR)
7166     std::swap(Op0, Op1);
7167   if (Op0.getOpcode() != ISD::OR)
7168     return SDValue();
7169 
7170   // We found an inner OR, so our operands are the operands of the inner OR
7171   // and the other operand of the outer OR.
7172   SDValue A = Op0.getOperand(0);
7173   SDValue B = Op0.getOperand(1);
7174   SDValue C = Op1;
7175 
7176   auto Match1 = matchSHFLPat(A);
7177   auto Match2 = matchSHFLPat(B);
7178 
7179   // If neither matched, we failed.
7180   if (!Match1 && !Match2)
7181     return SDValue();
7182 
7183   // We had at least one match. if one failed, try the remaining C operand.
7184   if (!Match1) {
7185     std::swap(A, C);
7186     Match1 = matchSHFLPat(A);
7187     if (!Match1)
7188       return SDValue();
7189   } else if (!Match2) {
7190     std::swap(B, C);
7191     Match2 = matchSHFLPat(B);
7192     if (!Match2)
7193       return SDValue();
7194   }
7195   assert(Match1 && Match2);
7196 
7197   // Make sure our matches pair up.
7198   if (!Match1->formsPairWith(*Match2))
7199     return SDValue();
7200 
7201   // All the remains is to make sure C is an AND with the same input, that masks
7202   // out the bits that are being shuffled.
7203   if (C.getOpcode() != ISD::AND || !isa<ConstantSDNode>(C.getOperand(1)) ||
7204       C.getOperand(0) != Match1->Op)
7205     return SDValue();
7206 
7207   uint64_t Mask = C.getConstantOperandVal(1);
7208 
7209   static const uint64_t BitmanipMasks[] = {
7210       0x9999999999999999ULL, 0xC3C3C3C3C3C3C3C3ULL, 0xF00FF00FF00FF00FULL,
7211       0xFF0000FFFF0000FFULL, 0xFFFF00000000FFFFULL,
7212   };
7213 
7214   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
7215   unsigned MaskIdx = Log2_32(Match1->ShAmt);
7216   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
7217 
7218   if (Mask != ExpMask)
7219     return SDValue();
7220 
7221   SDLoc DL(Op);
7222   return DAG.getNode(RISCVISD::SHFL, DL, VT, Match1->Op,
7223                      DAG.getConstant(Match1->ShAmt, DL, VT));
7224 }
7225 
7226 // Optimize (add (shl x, c0), (shl y, c1)) ->
7227 //          (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
7228 static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
7229                                   const RISCVSubtarget &Subtarget) {
7230   // Perform this optimization only in the zba extension.
7231   if (!Subtarget.hasStdExtZba())
7232     return SDValue();
7233 
7234   // Skip for vector types and larger types.
7235   EVT VT = N->getValueType(0);
7236   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
7237     return SDValue();
7238 
7239   // The two operand nodes must be SHL and have no other use.
7240   SDValue N0 = N->getOperand(0);
7241   SDValue N1 = N->getOperand(1);
7242   if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
7243       !N0->hasOneUse() || !N1->hasOneUse())
7244     return SDValue();
7245 
7246   // Check c0 and c1.
7247   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7248   auto *N1C = dyn_cast<ConstantSDNode>(N1->getOperand(1));
7249   if (!N0C || !N1C)
7250     return SDValue();
7251   int64_t C0 = N0C->getSExtValue();
7252   int64_t C1 = N1C->getSExtValue();
7253   if (C0 <= 0 || C1 <= 0)
7254     return SDValue();
7255 
7256   // Skip if SH1ADD/SH2ADD/SH3ADD are not applicable.
7257   int64_t Bits = std::min(C0, C1);
7258   int64_t Diff = std::abs(C0 - C1);
7259   if (Diff != 1 && Diff != 2 && Diff != 3)
7260     return SDValue();
7261 
7262   // Build nodes.
7263   SDLoc DL(N);
7264   SDValue NS = (C0 < C1) ? N0->getOperand(0) : N1->getOperand(0);
7265   SDValue NL = (C0 > C1) ? N0->getOperand(0) : N1->getOperand(0);
7266   SDValue NA0 =
7267       DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT));
7268   SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS);
7269   return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT));
7270 }
7271 
7272 // Combine (GREVI (GREVI x, C2), C1) -> (GREVI x, C1^C2) when C1^C2 is
7273 // non-zero, and to x when it is. Any repeated GREVI stage undoes itself.
7274 // Combine (GORCI (GORCI x, C2), C1) -> (GORCI x, C1|C2). Repeated stage does
7275 // not undo itself, but they are redundant.
7276 static SDValue combineGREVI_GORCI(SDNode *N, SelectionDAG &DAG) {
7277   SDValue Src = N->getOperand(0);
7278 
7279   if (Src.getOpcode() != N->getOpcode())
7280     return SDValue();
7281 
7282   if (!isa<ConstantSDNode>(N->getOperand(1)) ||
7283       !isa<ConstantSDNode>(Src.getOperand(1)))
7284     return SDValue();
7285 
7286   unsigned ShAmt1 = N->getConstantOperandVal(1);
7287   unsigned ShAmt2 = Src.getConstantOperandVal(1);
7288   Src = Src.getOperand(0);
7289 
7290   unsigned CombinedShAmt;
7291   if (N->getOpcode() == RISCVISD::GORC || N->getOpcode() == RISCVISD::GORCW)
7292     CombinedShAmt = ShAmt1 | ShAmt2;
7293   else
7294     CombinedShAmt = ShAmt1 ^ ShAmt2;
7295 
7296   if (CombinedShAmt == 0)
7297     return Src;
7298 
7299   SDLoc DL(N);
7300   return DAG.getNode(
7301       N->getOpcode(), DL, N->getValueType(0), Src,
7302       DAG.getConstant(CombinedShAmt, DL, N->getOperand(1).getValueType()));
7303 }
7304 
7305 // Combine a constant select operand into its use:
7306 //
7307 // (and (select cond, -1, c), x)
7308 //   -> (select cond, x, (and x, c))  [AllOnes=1]
7309 // (or  (select cond, 0, c), x)
7310 //   -> (select cond, x, (or x, c))  [AllOnes=0]
7311 // (xor (select cond, 0, c), x)
7312 //   -> (select cond, x, (xor x, c))  [AllOnes=0]
7313 // (add (select cond, 0, c), x)
7314 //   -> (select cond, x, (add x, c))  [AllOnes=0]
7315 // (sub x, (select cond, 0, c))
7316 //   -> (select cond, x, (sub x, c))  [AllOnes=0]
7317 static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
7318                                    SelectionDAG &DAG, bool AllOnes) {
7319   EVT VT = N->getValueType(0);
7320 
7321   // Skip vectors.
7322   if (VT.isVector())
7323     return SDValue();
7324 
7325   if ((Slct.getOpcode() != ISD::SELECT &&
7326        Slct.getOpcode() != RISCVISD::SELECT_CC) ||
7327       !Slct.hasOneUse())
7328     return SDValue();
7329 
7330   auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
7331     return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
7332   };
7333 
7334   bool SwapSelectOps;
7335   unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
7336   SDValue TrueVal = Slct.getOperand(1 + OpOffset);
7337   SDValue FalseVal = Slct.getOperand(2 + OpOffset);
7338   SDValue NonConstantVal;
7339   if (isZeroOrAllOnes(TrueVal, AllOnes)) {
7340     SwapSelectOps = false;
7341     NonConstantVal = FalseVal;
7342   } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
7343     SwapSelectOps = true;
7344     NonConstantVal = TrueVal;
7345   } else
7346     return SDValue();
7347 
7348   // Slct is now know to be the desired identity constant when CC is true.
7349   TrueVal = OtherOp;
7350   FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
7351   // Unless SwapSelectOps says the condition should be false.
7352   if (SwapSelectOps)
7353     std::swap(TrueVal, FalseVal);
7354 
7355   if (Slct.getOpcode() == RISCVISD::SELECT_CC)
7356     return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
7357                        {Slct.getOperand(0), Slct.getOperand(1),
7358                         Slct.getOperand(2), TrueVal, FalseVal});
7359 
7360   return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
7361                      {Slct.getOperand(0), TrueVal, FalseVal});
7362 }
7363 
7364 // Attempt combineSelectAndUse on each operand of a commutative operator N.
7365 static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
7366                                               bool AllOnes) {
7367   SDValue N0 = N->getOperand(0);
7368   SDValue N1 = N->getOperand(1);
7369   if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes))
7370     return Result;
7371   if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes))
7372     return Result;
7373   return SDValue();
7374 }
7375 
7376 // Transform (add (mul x, c0), c1) ->
7377 //           (add (mul (add x, c1/c0), c0), c1%c0).
7378 // if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
7379 // that should be excluded is when c0*(c1/c0) is simm12, which will lead
7380 // to an infinite loop in DAGCombine if transformed.
7381 // Or transform (add (mul x, c0), c1) ->
7382 //              (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
7383 // if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
7384 // case that should be excluded is when c0*(c1/c0+1) is simm12, which will
7385 // lead to an infinite loop in DAGCombine if transformed.
7386 // Or transform (add (mul x, c0), c1) ->
7387 //              (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
7388 // if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
7389 // case that should be excluded is when c0*(c1/c0-1) is simm12, which will
7390 // lead to an infinite loop in DAGCombine if transformed.
7391 // Or transform (add (mul x, c0), c1) ->
7392 //              (mul (add x, c1/c0), c0).
7393 // if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
7394 static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
7395                                      const RISCVSubtarget &Subtarget) {
7396   // Skip for vector types and larger types.
7397   EVT VT = N->getValueType(0);
7398   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
7399     return SDValue();
7400   // The first operand node must be a MUL and has no other use.
7401   SDValue N0 = N->getOperand(0);
7402   if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
7403     return SDValue();
7404   // Check if c0 and c1 match above conditions.
7405   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7406   auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
7407   if (!N0C || !N1C)
7408     return SDValue();
7409   int64_t C0 = N0C->getSExtValue();
7410   int64_t C1 = N1C->getSExtValue();
7411   int64_t CA, CB;
7412   if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
7413     return SDValue();
7414   // Search for proper CA (non-zero) and CB that both are simm12.
7415   if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
7416       !isInt<12>(C0 * (C1 / C0))) {
7417     CA = C1 / C0;
7418     CB = C1 % C0;
7419   } else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
7420              isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
7421     CA = C1 / C0 + 1;
7422     CB = C1 % C0 - C0;
7423   } else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
7424              isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
7425     CA = C1 / C0 - 1;
7426     CB = C1 % C0 + C0;
7427   } else
7428     return SDValue();
7429   // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
7430   SDLoc DL(N);
7431   SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
7432                              DAG.getConstant(CA, DL, VT));
7433   SDValue New1 =
7434       DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
7435   return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
7436 }
7437 
7438 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
7439                                  const RISCVSubtarget &Subtarget) {
7440   if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
7441     return V;
7442   if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
7443     return V;
7444   // fold (add (select lhs, rhs, cc, 0, y), x) ->
7445   //      (select lhs, rhs, cc, x, (add x, y))
7446   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7447 }
7448 
7449 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG) {
7450   // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
7451   //      (select lhs, rhs, cc, x, (sub x, y))
7452   SDValue N0 = N->getOperand(0);
7453   SDValue N1 = N->getOperand(1);
7454   return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false);
7455 }
7456 
7457 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG) {
7458   // fold (and (select lhs, rhs, cc, -1, y), x) ->
7459   //      (select lhs, rhs, cc, x, (and x, y))
7460   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true);
7461 }
7462 
7463 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
7464                                 const RISCVSubtarget &Subtarget) {
7465   if (Subtarget.hasStdExtZbp()) {
7466     if (auto GREV = combineORToGREV(SDValue(N, 0), DAG, Subtarget))
7467       return GREV;
7468     if (auto GORC = combineORToGORC(SDValue(N, 0), DAG, Subtarget))
7469       return GORC;
7470     if (auto SHFL = combineORToSHFL(SDValue(N, 0), DAG, Subtarget))
7471       return SHFL;
7472   }
7473 
7474   // fold (or (select cond, 0, y), x) ->
7475   //      (select cond, x, (or x, y))
7476   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7477 }
7478 
7479 static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG) {
7480   // fold (xor (select cond, 0, y), x) ->
7481   //      (select cond, x, (xor x, y))
7482   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7483 }
7484 
7485 // Attempt to turn ANY_EXTEND into SIGN_EXTEND if the input to the ANY_EXTEND
7486 // has users that require SIGN_EXTEND and the SIGN_EXTEND can be done for free
7487 // by an instruction like ADDW/SUBW/MULW. Without this the ANY_EXTEND would be
7488 // removed during type legalization leaving an ADD/SUB/MUL use that won't use
7489 // ADDW/SUBW/MULW.
7490 static SDValue performANY_EXTENDCombine(SDNode *N,
7491                                         TargetLowering::DAGCombinerInfo &DCI,
7492                                         const RISCVSubtarget &Subtarget) {
7493   if (!Subtarget.is64Bit())
7494     return SDValue();
7495 
7496   SelectionDAG &DAG = DCI.DAG;
7497 
7498   SDValue Src = N->getOperand(0);
7499   EVT VT = N->getValueType(0);
7500   if (VT != MVT::i64 || Src.getValueType() != MVT::i32)
7501     return SDValue();
7502 
7503   // The opcode must be one that can implicitly sign_extend.
7504   // FIXME: Additional opcodes.
7505   switch (Src.getOpcode()) {
7506   default:
7507     return SDValue();
7508   case ISD::MUL:
7509     if (!Subtarget.hasStdExtM())
7510       return SDValue();
7511     LLVM_FALLTHROUGH;
7512   case ISD::ADD:
7513   case ISD::SUB:
7514     break;
7515   }
7516 
7517   // Only handle cases where the result is used by a CopyToReg. That likely
7518   // means the value is a liveout of the basic block. This helps prevent
7519   // infinite combine loops like PR51206.
7520   if (none_of(N->uses(),
7521               [](SDNode *User) { return User->getOpcode() == ISD::CopyToReg; }))
7522     return SDValue();
7523 
7524   SmallVector<SDNode *, 4> SetCCs;
7525   for (SDNode::use_iterator UI = Src.getNode()->use_begin(),
7526                             UE = Src.getNode()->use_end();
7527        UI != UE; ++UI) {
7528     SDNode *User = *UI;
7529     if (User == N)
7530       continue;
7531     if (UI.getUse().getResNo() != Src.getResNo())
7532       continue;
7533     // All i32 setccs are legalized by sign extending operands.
7534     if (User->getOpcode() == ISD::SETCC) {
7535       SetCCs.push_back(User);
7536       continue;
7537     }
7538     // We don't know if we can extend this user.
7539     break;
7540   }
7541 
7542   // If we don't have any SetCCs, this isn't worthwhile.
7543   if (SetCCs.empty())
7544     return SDValue();
7545 
7546   SDLoc DL(N);
7547   SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Src);
7548   DCI.CombineTo(N, SExt);
7549 
7550   // Promote all the setccs.
7551   for (SDNode *SetCC : SetCCs) {
7552     SmallVector<SDValue, 4> Ops;
7553 
7554     for (unsigned j = 0; j != 2; ++j) {
7555       SDValue SOp = SetCC->getOperand(j);
7556       if (SOp == Src)
7557         Ops.push_back(SExt);
7558       else
7559         Ops.push_back(DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, SOp));
7560     }
7561 
7562     Ops.push_back(SetCC->getOperand(2));
7563     DCI.CombineTo(SetCC,
7564                   DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
7565   }
7566   return SDValue(N, 0);
7567 }
7568 
7569 // Try to form vwadd(u).wv/wx or vwsub(u).wv/wx. It might later be optimized to
7570 // vwadd(u).vv/vx or vwsub(u).vv/vx.
7571 static SDValue combineADDSUB_VLToVWADDSUB_VL(SDNode *N, SelectionDAG &DAG,
7572                                              bool Commute = false) {
7573   assert((N->getOpcode() == RISCVISD::ADD_VL ||
7574           N->getOpcode() == RISCVISD::SUB_VL) &&
7575          "Unexpected opcode");
7576   bool IsAdd = N->getOpcode() == RISCVISD::ADD_VL;
7577   SDValue Op0 = N->getOperand(0);
7578   SDValue Op1 = N->getOperand(1);
7579   if (Commute)
7580     std::swap(Op0, Op1);
7581 
7582   MVT VT = N->getSimpleValueType(0);
7583 
7584   // Determine the narrow size for a widening add/sub.
7585   unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
7586   MVT NarrowVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowSize),
7587                                   VT.getVectorElementCount());
7588 
7589   SDValue Mask = N->getOperand(2);
7590   SDValue VL = N->getOperand(3);
7591 
7592   SDLoc DL(N);
7593 
7594   // If the RHS is a sext or zext, we can form a widening op.
7595   if ((Op1.getOpcode() == RISCVISD::VZEXT_VL ||
7596        Op1.getOpcode() == RISCVISD::VSEXT_VL) &&
7597       Op1.hasOneUse() && Op1.getOperand(1) == Mask && Op1.getOperand(2) == VL) {
7598     unsigned ExtOpc = Op1.getOpcode();
7599     Op1 = Op1.getOperand(0);
7600     // Re-introduce narrower extends if needed.
7601     if (Op1.getValueType() != NarrowVT)
7602       Op1 = DAG.getNode(ExtOpc, DL, NarrowVT, Op1, Mask, VL);
7603 
7604     unsigned WOpc;
7605     if (ExtOpc == RISCVISD::VSEXT_VL)
7606       WOpc = IsAdd ? RISCVISD::VWADD_W_VL : RISCVISD::VWSUB_W_VL;
7607     else
7608       WOpc = IsAdd ? RISCVISD::VWADDU_W_VL : RISCVISD::VWSUBU_W_VL;
7609 
7610     return DAG.getNode(WOpc, DL, VT, Op0, Op1, Mask, VL);
7611   }
7612 
7613   // FIXME: Is it useful to form a vwadd.wx or vwsub.wx if it removes a scalar
7614   // sext/zext?
7615 
7616   return SDValue();
7617 }
7618 
7619 // Try to convert vwadd(u).wv/wx or vwsub(u).wv/wx to vwadd(u).vv/vx or
7620 // vwsub(u).vv/vx.
7621 static SDValue combineVWADD_W_VL_VWSUB_W_VL(SDNode *N, SelectionDAG &DAG) {
7622   SDValue Op0 = N->getOperand(0);
7623   SDValue Op1 = N->getOperand(1);
7624   SDValue Mask = N->getOperand(2);
7625   SDValue VL = N->getOperand(3);
7626 
7627   MVT VT = N->getSimpleValueType(0);
7628   MVT NarrowVT = Op1.getSimpleValueType();
7629   unsigned NarrowSize = NarrowVT.getScalarSizeInBits();
7630 
7631   unsigned VOpc;
7632   switch (N->getOpcode()) {
7633   default: llvm_unreachable("Unexpected opcode");
7634   case RISCVISD::VWADD_W_VL:  VOpc = RISCVISD::VWADD_VL;  break;
7635   case RISCVISD::VWSUB_W_VL:  VOpc = RISCVISD::VWSUB_VL;  break;
7636   case RISCVISD::VWADDU_W_VL: VOpc = RISCVISD::VWADDU_VL; break;
7637   case RISCVISD::VWSUBU_W_VL: VOpc = RISCVISD::VWSUBU_VL; break;
7638   }
7639 
7640   bool IsSigned = N->getOpcode() == RISCVISD::VWADD_W_VL ||
7641                   N->getOpcode() == RISCVISD::VWSUB_W_VL;
7642 
7643   SDLoc DL(N);
7644 
7645   // If the LHS is a sext or zext, we can narrow this op to the same size as
7646   // the RHS.
7647   if (((Op0.getOpcode() == RISCVISD::VZEXT_VL && !IsSigned) ||
7648        (Op0.getOpcode() == RISCVISD::VSEXT_VL && IsSigned)) &&
7649       Op0.hasOneUse() && Op0.getOperand(1) == Mask && Op0.getOperand(2) == VL) {
7650     unsigned ExtOpc = Op0.getOpcode();
7651     Op0 = Op0.getOperand(0);
7652     // Re-introduce narrower extends if needed.
7653     if (Op0.getValueType() != NarrowVT)
7654       Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
7655     return DAG.getNode(VOpc, DL, VT, Op0, Op1, Mask, VL);
7656   }
7657 
7658   bool IsAdd = N->getOpcode() == RISCVISD::VWADD_W_VL ||
7659                N->getOpcode() == RISCVISD::VWADDU_W_VL;
7660 
7661   // Look for splats on the left hand side of a vwadd(u).wv. We might be able
7662   // to commute and use a vwadd(u).vx instead.
7663   if (IsAdd && Op0.getOpcode() == RISCVISD::VMV_V_X_VL &&
7664       Op0.getOperand(1) == VL) {
7665     Op0 = Op0.getOperand(0);
7666 
7667     // See if have enough sign bits or zero bits in the scalar to use a
7668     // widening add/sub by splatting to smaller element size.
7669     unsigned EltBits = VT.getScalarSizeInBits();
7670     unsigned ScalarBits = Op0.getValueSizeInBits();
7671     // Make sure we're getting all element bits from the scalar register.
7672     // FIXME: Support implicit sign extension of vmv.v.x?
7673     if (ScalarBits < EltBits)
7674       return SDValue();
7675 
7676     if (IsSigned) {
7677       if (DAG.ComputeNumSignBits(Op0) <= (ScalarBits - NarrowSize))
7678         return SDValue();
7679     } else {
7680       APInt Mask = APInt::getBitsSetFrom(ScalarBits, NarrowSize);
7681       if (!DAG.MaskedValueIsZero(Op0, Mask))
7682         return SDValue();
7683     }
7684 
7685     Op0 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT, Op0, VL);
7686     return DAG.getNode(VOpc, DL, VT, Op1, Op0, Mask, VL);
7687   }
7688 
7689   return SDValue();
7690 }
7691 
7692 // Try to form VWMUL, VWMULU or VWMULSU.
7693 // TODO: Support VWMULSU.vx with a sign extend Op and a splat of scalar Op.
7694 static SDValue combineMUL_VLToVWMUL_VL(SDNode *N, SelectionDAG &DAG,
7695                                        bool Commute) {
7696   assert(N->getOpcode() == RISCVISD::MUL_VL && "Unexpected opcode");
7697   SDValue Op0 = N->getOperand(0);
7698   SDValue Op1 = N->getOperand(1);
7699   if (Commute)
7700     std::swap(Op0, Op1);
7701 
7702   bool IsSignExt = Op0.getOpcode() == RISCVISD::VSEXT_VL;
7703   bool IsZeroExt = Op0.getOpcode() == RISCVISD::VZEXT_VL;
7704   bool IsVWMULSU = IsSignExt && Op1.getOpcode() == RISCVISD::VZEXT_VL;
7705   if ((!IsSignExt && !IsZeroExt) || !Op0.hasOneUse())
7706     return SDValue();
7707 
7708   SDValue Mask = N->getOperand(2);
7709   SDValue VL = N->getOperand(3);
7710 
7711   // Make sure the mask and VL match.
7712   if (Op0.getOperand(1) != Mask || Op0.getOperand(2) != VL)
7713     return SDValue();
7714 
7715   MVT VT = N->getSimpleValueType(0);
7716 
7717   // Determine the narrow size for a widening multiply.
7718   unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
7719   MVT NarrowVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowSize),
7720                                   VT.getVectorElementCount());
7721 
7722   SDLoc DL(N);
7723 
7724   // See if the other operand is the same opcode.
7725   if (IsVWMULSU || Op0.getOpcode() == Op1.getOpcode()) {
7726     if (!Op1.hasOneUse())
7727       return SDValue();
7728 
7729     // Make sure the mask and VL match.
7730     if (Op1.getOperand(1) != Mask || Op1.getOperand(2) != VL)
7731       return SDValue();
7732 
7733     Op1 = Op1.getOperand(0);
7734   } else if (Op1.getOpcode() == RISCVISD::VMV_V_X_VL) {
7735     // The operand is a splat of a scalar.
7736 
7737     // The VL must be the same.
7738     if (Op1.getOperand(1) != VL)
7739       return SDValue();
7740 
7741     // Get the scalar value.
7742     Op1 = Op1.getOperand(0);
7743 
7744     // See if have enough sign bits or zero bits in the scalar to use a
7745     // widening multiply by splatting to smaller element size.
7746     unsigned EltBits = VT.getScalarSizeInBits();
7747     unsigned ScalarBits = Op1.getValueSizeInBits();
7748     // Make sure we're getting all element bits from the scalar register.
7749     // FIXME: Support implicit sign extension of vmv.v.x?
7750     if (ScalarBits < EltBits)
7751       return SDValue();
7752 
7753     if (IsSignExt) {
7754       if (DAG.ComputeNumSignBits(Op1) <= (ScalarBits - NarrowSize))
7755         return SDValue();
7756     } else {
7757       APInt Mask = APInt::getBitsSetFrom(ScalarBits, NarrowSize);
7758       if (!DAG.MaskedValueIsZero(Op1, Mask))
7759         return SDValue();
7760     }
7761 
7762     Op1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT, Op1, VL);
7763   } else
7764     return SDValue();
7765 
7766   Op0 = Op0.getOperand(0);
7767 
7768   // Re-introduce narrower extends if needed.
7769   unsigned ExtOpc = IsSignExt ? RISCVISD::VSEXT_VL : RISCVISD::VZEXT_VL;
7770   if (Op0.getValueType() != NarrowVT)
7771     Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
7772   if (Op1.getValueType() != NarrowVT)
7773     Op1 = DAG.getNode(ExtOpc, DL, NarrowVT, Op1, Mask, VL);
7774 
7775   unsigned WMulOpc = RISCVISD::VWMULSU_VL;
7776   if (!IsVWMULSU)
7777     WMulOpc = IsSignExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
7778   return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Mask, VL);
7779 }
7780 
7781 static RISCVFPRndMode::RoundingMode matchRoundingOp(SDValue Op) {
7782   switch (Op.getOpcode()) {
7783   case ISD::FROUNDEVEN: return RISCVFPRndMode::RNE;
7784   case ISD::FTRUNC:     return RISCVFPRndMode::RTZ;
7785   case ISD::FFLOOR:     return RISCVFPRndMode::RDN;
7786   case ISD::FCEIL:      return RISCVFPRndMode::RUP;
7787   case ISD::FROUND:     return RISCVFPRndMode::RMM;
7788   }
7789 
7790   return RISCVFPRndMode::Invalid;
7791 }
7792 
7793 // Fold
7794 //   (fp_to_int (froundeven X)) -> fcvt X, rne
7795 //   (fp_to_int (ftrunc X))     -> fcvt X, rtz
7796 //   (fp_to_int (ffloor X))     -> fcvt X, rdn
7797 //   (fp_to_int (fceil X))      -> fcvt X, rup
7798 //   (fp_to_int (fround X))     -> fcvt X, rmm
7799 static SDValue performFP_TO_INTCombine(SDNode *N,
7800                                        TargetLowering::DAGCombinerInfo &DCI,
7801                                        const RISCVSubtarget &Subtarget) {
7802   SelectionDAG &DAG = DCI.DAG;
7803   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7804   MVT XLenVT = Subtarget.getXLenVT();
7805 
7806   // Only handle XLen or i32 types. Other types narrower than XLen will
7807   // eventually be legalized to XLenVT.
7808   EVT VT = N->getValueType(0);
7809   if (VT != MVT::i32 && VT != XLenVT)
7810     return SDValue();
7811 
7812   SDValue Src = N->getOperand(0);
7813 
7814   // Ensure the FP type is also legal.
7815   if (!TLI.isTypeLegal(Src.getValueType()))
7816     return SDValue();
7817 
7818   // Don't do this for f16 with Zfhmin and not Zfh.
7819   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7820     return SDValue();
7821 
7822   RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src);
7823   if (FRM == RISCVFPRndMode::Invalid)
7824     return SDValue();
7825 
7826   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
7827 
7828   unsigned Opc;
7829   if (VT == XLenVT)
7830     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7831   else
7832     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7833 
7834   SDLoc DL(N);
7835   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src.getOperand(0),
7836                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7837   return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
7838 }
7839 
7840 // Fold
7841 //   (fp_to_int_sat (froundeven X)) -> (select X == nan, 0, (fcvt X, rne))
7842 //   (fp_to_int_sat (ftrunc X))     -> (select X == nan, 0, (fcvt X, rtz))
7843 //   (fp_to_int_sat (ffloor X))     -> (select X == nan, 0, (fcvt X, rdn))
7844 //   (fp_to_int_sat (fceil X))      -> (select X == nan, 0, (fcvt X, rup))
7845 //   (fp_to_int_sat (fround X))     -> (select X == nan, 0, (fcvt X, rmm))
7846 static SDValue performFP_TO_INT_SATCombine(SDNode *N,
7847                                        TargetLowering::DAGCombinerInfo &DCI,
7848                                        const RISCVSubtarget &Subtarget) {
7849   SelectionDAG &DAG = DCI.DAG;
7850   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7851   MVT XLenVT = Subtarget.getXLenVT();
7852 
7853   // Only handle XLen types. Other types narrower than XLen will eventually be
7854   // legalized to XLenVT.
7855   EVT DstVT = N->getValueType(0);
7856   if (DstVT != XLenVT)
7857     return SDValue();
7858 
7859   SDValue Src = N->getOperand(0);
7860 
7861   // Ensure the FP type is also legal.
7862   if (!TLI.isTypeLegal(Src.getValueType()))
7863     return SDValue();
7864 
7865   // Don't do this for f16 with Zfhmin and not Zfh.
7866   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7867     return SDValue();
7868 
7869   EVT SatVT = cast<VTSDNode>(N->getOperand(1))->getVT();
7870 
7871   RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src);
7872   if (FRM == RISCVFPRndMode::Invalid)
7873     return SDValue();
7874 
7875   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
7876 
7877   unsigned Opc;
7878   if (SatVT == DstVT)
7879     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7880   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
7881     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7882   else
7883     return SDValue();
7884   // FIXME: Support other SatVTs by clamping before or after the conversion.
7885 
7886   Src = Src.getOperand(0);
7887 
7888   SDLoc DL(N);
7889   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src,
7890                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7891 
7892   // RISCV FP-to-int conversions saturate to the destination register size, but
7893   // don't produce 0 for nan.
7894   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
7895   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
7896 }
7897 
7898 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
7899                                                DAGCombinerInfo &DCI) const {
7900   SelectionDAG &DAG = DCI.DAG;
7901 
7902   // Helper to call SimplifyDemandedBits on an operand of N where only some low
7903   // bits are demanded. N will be added to the Worklist if it was not deleted.
7904   // Caller should return SDValue(N, 0) if this returns true.
7905   auto SimplifyDemandedLowBitsHelper = [&](unsigned OpNo, unsigned LowBits) {
7906     SDValue Op = N->getOperand(OpNo);
7907     APInt Mask = APInt::getLowBitsSet(Op.getValueSizeInBits(), LowBits);
7908     if (!SimplifyDemandedBits(Op, Mask, DCI))
7909       return false;
7910 
7911     if (N->getOpcode() != ISD::DELETED_NODE)
7912       DCI.AddToWorklist(N);
7913     return true;
7914   };
7915 
7916   switch (N->getOpcode()) {
7917   default:
7918     break;
7919   case RISCVISD::SplitF64: {
7920     SDValue Op0 = N->getOperand(0);
7921     // If the input to SplitF64 is just BuildPairF64 then the operation is
7922     // redundant. Instead, use BuildPairF64's operands directly.
7923     if (Op0->getOpcode() == RISCVISD::BuildPairF64)
7924       return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
7925 
7926     if (Op0->isUndef()) {
7927       SDValue Lo = DAG.getUNDEF(MVT::i32);
7928       SDValue Hi = DAG.getUNDEF(MVT::i32);
7929       return DCI.CombineTo(N, Lo, Hi);
7930     }
7931 
7932     SDLoc DL(N);
7933 
7934     // It's cheaper to materialise two 32-bit integers than to load a double
7935     // from the constant pool and transfer it to integer registers through the
7936     // stack.
7937     if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
7938       APInt V = C->getValueAPF().bitcastToAPInt();
7939       SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
7940       SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
7941       return DCI.CombineTo(N, Lo, Hi);
7942     }
7943 
7944     // This is a target-specific version of a DAGCombine performed in
7945     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7946     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7947     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7948     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7949         !Op0.getNode()->hasOneUse())
7950       break;
7951     SDValue NewSplitF64 =
7952         DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
7953                     Op0.getOperand(0));
7954     SDValue Lo = NewSplitF64.getValue(0);
7955     SDValue Hi = NewSplitF64.getValue(1);
7956     APInt SignBit = APInt::getSignMask(32);
7957     if (Op0.getOpcode() == ISD::FNEG) {
7958       SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
7959                                   DAG.getConstant(SignBit, DL, MVT::i32));
7960       return DCI.CombineTo(N, Lo, NewHi);
7961     }
7962     assert(Op0.getOpcode() == ISD::FABS);
7963     SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
7964                                 DAG.getConstant(~SignBit, DL, MVT::i32));
7965     return DCI.CombineTo(N, Lo, NewHi);
7966   }
7967   case RISCVISD::SLLW:
7968   case RISCVISD::SRAW:
7969   case RISCVISD::SRLW:
7970   case RISCVISD::ROLW:
7971   case RISCVISD::RORW: {
7972     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7973     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7974         SimplifyDemandedLowBitsHelper(1, 5))
7975       return SDValue(N, 0);
7976     break;
7977   }
7978   case RISCVISD::CLZW:
7979   case RISCVISD::CTZW: {
7980     // Only the lower 32 bits of the first operand are read
7981     if (SimplifyDemandedLowBitsHelper(0, 32))
7982       return SDValue(N, 0);
7983     break;
7984   }
7985   case RISCVISD::GREV:
7986   case RISCVISD::GORC: {
7987     // Only the lower log2(Bitwidth) bits of the the shift amount are read.
7988     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7989     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7990     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth)))
7991       return SDValue(N, 0);
7992 
7993     return combineGREVI_GORCI(N, DAG);
7994   }
7995   case RISCVISD::GREVW:
7996   case RISCVISD::GORCW: {
7997     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7998     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7999         SimplifyDemandedLowBitsHelper(1, 5))
8000       return SDValue(N, 0);
8001 
8002     return combineGREVI_GORCI(N, DAG);
8003   }
8004   case RISCVISD::SHFL:
8005   case RISCVISD::UNSHFL: {
8006     // Only the lower log2(Bitwidth)-1 bits of the the shift amount are read.
8007     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
8008     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
8009     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth) - 1))
8010       return SDValue(N, 0);
8011 
8012     break;
8013   }
8014   case RISCVISD::SHFLW:
8015   case RISCVISD::UNSHFLW: {
8016     // Only the lower 32 bits of LHS and lower 4 bits of RHS are read.
8017     if (SimplifyDemandedLowBitsHelper(0, 32) ||
8018         SimplifyDemandedLowBitsHelper(1, 4))
8019       return SDValue(N, 0);
8020 
8021     break;
8022   }
8023   case RISCVISD::BCOMPRESSW:
8024   case RISCVISD::BDECOMPRESSW: {
8025     // Only the lower 32 bits of LHS and RHS are read.
8026     if (SimplifyDemandedLowBitsHelper(0, 32) ||
8027         SimplifyDemandedLowBitsHelper(1, 32))
8028       return SDValue(N, 0);
8029 
8030     break;
8031   }
8032   case RISCVISD::FMV_X_ANYEXTH:
8033   case RISCVISD::FMV_X_ANYEXTW_RV64: {
8034     SDLoc DL(N);
8035     SDValue Op0 = N->getOperand(0);
8036     MVT VT = N->getSimpleValueType(0);
8037     // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
8038     // conversion is unnecessary and can be replaced with the FMV_W_X_RV64
8039     // operand. Similar for FMV_X_ANYEXTH and FMV_H_X.
8040     if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
8041          Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
8042         (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
8043          Op0->getOpcode() == RISCVISD::FMV_H_X)) {
8044       assert(Op0.getOperand(0).getValueType() == VT &&
8045              "Unexpected value type!");
8046       return Op0.getOperand(0);
8047     }
8048 
8049     // This is a target-specific version of a DAGCombine performed in
8050     // DAGCombiner::visitBITCAST. It performs the equivalent of:
8051     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
8052     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
8053     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
8054         !Op0.getNode()->hasOneUse())
8055       break;
8056     SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
8057     unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
8058     APInt SignBit = APInt::getSignMask(FPBits).sextOrSelf(VT.getSizeInBits());
8059     if (Op0.getOpcode() == ISD::FNEG)
8060       return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
8061                          DAG.getConstant(SignBit, DL, VT));
8062 
8063     assert(Op0.getOpcode() == ISD::FABS);
8064     return DAG.getNode(ISD::AND, DL, VT, NewFMV,
8065                        DAG.getConstant(~SignBit, DL, VT));
8066   }
8067   case ISD::ADD:
8068     return performADDCombine(N, DAG, Subtarget);
8069   case ISD::SUB:
8070     return performSUBCombine(N, DAG);
8071   case ISD::AND:
8072     return performANDCombine(N, DAG);
8073   case ISD::OR:
8074     return performORCombine(N, DAG, Subtarget);
8075   case ISD::XOR:
8076     return performXORCombine(N, DAG);
8077   case ISD::ANY_EXTEND:
8078     return performANY_EXTENDCombine(N, DCI, Subtarget);
8079   case ISD::ZERO_EXTEND:
8080     // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
8081     // type legalization. This is safe because fp_to_uint produces poison if
8082     // it overflows.
8083     if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) {
8084       SDValue Src = N->getOperand(0);
8085       if (Src.getOpcode() == ISD::FP_TO_UINT &&
8086           isTypeLegal(Src.getOperand(0).getValueType()))
8087         return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
8088                            Src.getOperand(0));
8089       if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
8090           isTypeLegal(Src.getOperand(1).getValueType())) {
8091         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
8092         SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs,
8093                                   Src.getOperand(0), Src.getOperand(1));
8094         DCI.CombineTo(N, Res);
8095         DAG.ReplaceAllUsesOfValueWith(Src.getValue(1), Res.getValue(1));
8096         DCI.recursivelyDeleteUnusedNodes(Src.getNode());
8097         return SDValue(N, 0); // Return N so it doesn't get rechecked.
8098       }
8099     }
8100     return SDValue();
8101   case RISCVISD::SELECT_CC: {
8102     // Transform
8103     SDValue LHS = N->getOperand(0);
8104     SDValue RHS = N->getOperand(1);
8105     SDValue TrueV = N->getOperand(3);
8106     SDValue FalseV = N->getOperand(4);
8107 
8108     // If the True and False values are the same, we don't need a select_cc.
8109     if (TrueV == FalseV)
8110       return TrueV;
8111 
8112     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(2))->get();
8113     if (!ISD::isIntEqualitySetCC(CCVal))
8114       break;
8115 
8116     // Fold (select_cc (setlt X, Y), 0, ne, trueV, falseV) ->
8117     //      (select_cc X, Y, lt, trueV, falseV)
8118     // Sometimes the setcc is introduced after select_cc has been formed.
8119     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
8120         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
8121       // If we're looking for eq 0 instead of ne 0, we need to invert the
8122       // condition.
8123       bool Invert = CCVal == ISD::SETEQ;
8124       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
8125       if (Invert)
8126         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
8127 
8128       SDLoc DL(N);
8129       RHS = LHS.getOperand(1);
8130       LHS = LHS.getOperand(0);
8131       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
8132 
8133       SDValue TargetCC = DAG.getCondCode(CCVal);
8134       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
8135                          {LHS, RHS, TargetCC, TrueV, FalseV});
8136     }
8137 
8138     // Fold (select_cc (xor X, Y), 0, eq/ne, trueV, falseV) ->
8139     //      (select_cc X, Y, eq/ne, trueV, falseV)
8140     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
8141       return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), N->getValueType(0),
8142                          {LHS.getOperand(0), LHS.getOperand(1),
8143                           N->getOperand(2), TrueV, FalseV});
8144     // (select_cc X, 1, setne, trueV, falseV) ->
8145     // (select_cc X, 0, seteq, trueV, falseV) if we can prove X is 0/1.
8146     // This can occur when legalizing some floating point comparisons.
8147     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
8148     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
8149       SDLoc DL(N);
8150       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
8151       SDValue TargetCC = DAG.getCondCode(CCVal);
8152       RHS = DAG.getConstant(0, DL, LHS.getValueType());
8153       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
8154                          {LHS, RHS, TargetCC, TrueV, FalseV});
8155     }
8156 
8157     break;
8158   }
8159   case RISCVISD::BR_CC: {
8160     SDValue LHS = N->getOperand(1);
8161     SDValue RHS = N->getOperand(2);
8162     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(3))->get();
8163     if (!ISD::isIntEqualitySetCC(CCVal))
8164       break;
8165 
8166     // Fold (br_cc (setlt X, Y), 0, ne, dest) ->
8167     //      (br_cc X, Y, lt, dest)
8168     // Sometimes the setcc is introduced after br_cc has been formed.
8169     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
8170         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
8171       // If we're looking for eq 0 instead of ne 0, we need to invert the
8172       // condition.
8173       bool Invert = CCVal == ISD::SETEQ;
8174       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
8175       if (Invert)
8176         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
8177 
8178       SDLoc DL(N);
8179       RHS = LHS.getOperand(1);
8180       LHS = LHS.getOperand(0);
8181       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
8182 
8183       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
8184                          N->getOperand(0), LHS, RHS, DAG.getCondCode(CCVal),
8185                          N->getOperand(4));
8186     }
8187 
8188     // Fold (br_cc (xor X, Y), 0, eq/ne, dest) ->
8189     //      (br_cc X, Y, eq/ne, trueV, falseV)
8190     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
8191       return DAG.getNode(RISCVISD::BR_CC, SDLoc(N), N->getValueType(0),
8192                          N->getOperand(0), LHS.getOperand(0), LHS.getOperand(1),
8193                          N->getOperand(3), N->getOperand(4));
8194 
8195     // (br_cc X, 1, setne, br_cc) ->
8196     // (br_cc X, 0, seteq, br_cc) if we can prove X is 0/1.
8197     // This can occur when legalizing some floating point comparisons.
8198     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
8199     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
8200       SDLoc DL(N);
8201       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
8202       SDValue TargetCC = DAG.getCondCode(CCVal);
8203       RHS = DAG.getConstant(0, DL, LHS.getValueType());
8204       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
8205                          N->getOperand(0), LHS, RHS, TargetCC,
8206                          N->getOperand(4));
8207     }
8208     break;
8209   }
8210   case ISD::FP_TO_SINT:
8211   case ISD::FP_TO_UINT:
8212     return performFP_TO_INTCombine(N, DCI, Subtarget);
8213   case ISD::FP_TO_SINT_SAT:
8214   case ISD::FP_TO_UINT_SAT:
8215     return performFP_TO_INT_SATCombine(N, DCI, Subtarget);
8216   case ISD::FCOPYSIGN: {
8217     EVT VT = N->getValueType(0);
8218     if (!VT.isVector())
8219       break;
8220     // There is a form of VFSGNJ which injects the negated sign of its second
8221     // operand. Try and bubble any FNEG up after the extend/round to produce
8222     // this optimized pattern. Avoid modifying cases where FP_ROUND and
8223     // TRUNC=1.
8224     SDValue In2 = N->getOperand(1);
8225     // Avoid cases where the extend/round has multiple uses, as duplicating
8226     // those is typically more expensive than removing a fneg.
8227     if (!In2.hasOneUse())
8228       break;
8229     if (In2.getOpcode() != ISD::FP_EXTEND &&
8230         (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0))
8231       break;
8232     In2 = In2.getOperand(0);
8233     if (In2.getOpcode() != ISD::FNEG)
8234       break;
8235     SDLoc DL(N);
8236     SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
8237     return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
8238                        DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
8239   }
8240   case ISD::MGATHER:
8241   case ISD::MSCATTER:
8242   case ISD::VP_GATHER:
8243   case ISD::VP_SCATTER: {
8244     if (!DCI.isBeforeLegalize())
8245       break;
8246     SDValue Index, ScaleOp;
8247     bool IsIndexScaled = false;
8248     bool IsIndexSigned = false;
8249     if (const auto *VPGSN = dyn_cast<VPGatherScatterSDNode>(N)) {
8250       Index = VPGSN->getIndex();
8251       ScaleOp = VPGSN->getScale();
8252       IsIndexScaled = VPGSN->isIndexScaled();
8253       IsIndexSigned = VPGSN->isIndexSigned();
8254     } else {
8255       const auto *MGSN = cast<MaskedGatherScatterSDNode>(N);
8256       Index = MGSN->getIndex();
8257       ScaleOp = MGSN->getScale();
8258       IsIndexScaled = MGSN->isIndexScaled();
8259       IsIndexSigned = MGSN->isIndexSigned();
8260     }
8261     EVT IndexVT = Index.getValueType();
8262     MVT XLenVT = Subtarget.getXLenVT();
8263     // RISCV indexed loads only support the "unsigned unscaled" addressing
8264     // mode, so anything else must be manually legalized.
8265     bool NeedsIdxLegalization =
8266         IsIndexScaled ||
8267         (IsIndexSigned && IndexVT.getVectorElementType().bitsLT(XLenVT));
8268     if (!NeedsIdxLegalization)
8269       break;
8270 
8271     SDLoc DL(N);
8272 
8273     // Any index legalization should first promote to XLenVT, so we don't lose
8274     // bits when scaling. This may create an illegal index type so we let
8275     // LLVM's legalization take care of the splitting.
8276     // FIXME: LLVM can't split VP_GATHER or VP_SCATTER yet.
8277     if (IndexVT.getVectorElementType().bitsLT(XLenVT)) {
8278       IndexVT = IndexVT.changeVectorElementType(XLenVT);
8279       Index = DAG.getNode(IsIndexSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
8280                           DL, IndexVT, Index);
8281     }
8282 
8283     unsigned Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue();
8284     if (IsIndexScaled && Scale != 1) {
8285       // Manually scale the indices by the element size.
8286       // TODO: Sanitize the scale operand here?
8287       // TODO: For VP nodes, should we use VP_SHL here?
8288       assert(isPowerOf2_32(Scale) && "Expecting power-of-two types");
8289       SDValue SplatScale = DAG.getConstant(Log2_32(Scale), DL, IndexVT);
8290       Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, SplatScale);
8291     }
8292 
8293     ISD::MemIndexType NewIndexTy = ISD::UNSIGNED_UNSCALED;
8294     if (const auto *VPGN = dyn_cast<VPGatherSDNode>(N))
8295       return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
8296                              {VPGN->getChain(), VPGN->getBasePtr(), Index,
8297                               VPGN->getScale(), VPGN->getMask(),
8298                               VPGN->getVectorLength()},
8299                              VPGN->getMemOperand(), NewIndexTy);
8300     if (const auto *VPSN = dyn_cast<VPScatterSDNode>(N))
8301       return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
8302                               {VPSN->getChain(), VPSN->getValue(),
8303                                VPSN->getBasePtr(), Index, VPSN->getScale(),
8304                                VPSN->getMask(), VPSN->getVectorLength()},
8305                               VPSN->getMemOperand(), NewIndexTy);
8306     if (const auto *MGN = dyn_cast<MaskedGatherSDNode>(N))
8307       return DAG.getMaskedGather(
8308           N->getVTList(), MGN->getMemoryVT(), DL,
8309           {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
8310            MGN->getBasePtr(), Index, MGN->getScale()},
8311           MGN->getMemOperand(), NewIndexTy, MGN->getExtensionType());
8312     const auto *MSN = cast<MaskedScatterSDNode>(N);
8313     return DAG.getMaskedScatter(
8314         N->getVTList(), MSN->getMemoryVT(), DL,
8315         {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
8316          Index, MSN->getScale()},
8317         MSN->getMemOperand(), NewIndexTy, MSN->isTruncatingStore());
8318   }
8319   case RISCVISD::SRA_VL:
8320   case RISCVISD::SRL_VL:
8321   case RISCVISD::SHL_VL: {
8322     SDValue ShAmt = N->getOperand(1);
8323     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
8324       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
8325       SDLoc DL(N);
8326       SDValue VL = N->getOperand(3);
8327       EVT VT = N->getValueType(0);
8328       ShAmt =
8329           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, ShAmt.getOperand(0), VL);
8330       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
8331                          N->getOperand(2), N->getOperand(3));
8332     }
8333     break;
8334   }
8335   case ISD::SRA:
8336   case ISD::SRL:
8337   case ISD::SHL: {
8338     SDValue ShAmt = N->getOperand(1);
8339     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
8340       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
8341       SDLoc DL(N);
8342       EVT VT = N->getValueType(0);
8343       ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, ShAmt.getOperand(0),
8344                           DAG.getRegister(RISCV::X0, Subtarget.getXLenVT()));
8345       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
8346     }
8347     break;
8348   }
8349   case RISCVISD::ADD_VL:
8350     if (SDValue V = combineADDSUB_VLToVWADDSUB_VL(N, DAG, /*Commute*/ false))
8351       return V;
8352     return combineADDSUB_VLToVWADDSUB_VL(N, DAG, /*Commute*/ true);
8353   case RISCVISD::SUB_VL:
8354     return combineADDSUB_VLToVWADDSUB_VL(N, DAG);
8355   case RISCVISD::VWADD_W_VL:
8356   case RISCVISD::VWADDU_W_VL:
8357   case RISCVISD::VWSUB_W_VL:
8358   case RISCVISD::VWSUBU_W_VL:
8359     return combineVWADD_W_VL_VWSUB_W_VL(N, DAG);
8360   case RISCVISD::MUL_VL:
8361     if (SDValue V = combineMUL_VLToVWMUL_VL(N, DAG, /*Commute*/ false))
8362       return V;
8363     // Mul is commutative.
8364     return combineMUL_VLToVWMUL_VL(N, DAG, /*Commute*/ true);
8365   case ISD::STORE: {
8366     auto *Store = cast<StoreSDNode>(N);
8367     SDValue Val = Store->getValue();
8368     // Combine store of vmv.x.s to vse with VL of 1.
8369     // FIXME: Support FP.
8370     if (Val.getOpcode() == RISCVISD::VMV_X_S) {
8371       SDValue Src = Val.getOperand(0);
8372       EVT VecVT = Src.getValueType();
8373       EVT MemVT = Store->getMemoryVT();
8374       // The memory VT and the element type must match.
8375       if (VecVT.getVectorElementType() == MemVT) {
8376         SDLoc DL(N);
8377         MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
8378         return DAG.getStoreVP(
8379             Store->getChain(), DL, Src, Store->getBasePtr(), Store->getOffset(),
8380             DAG.getConstant(1, DL, MaskVT),
8381             DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT,
8382             Store->getMemOperand(), Store->getAddressingMode(),
8383             Store->isTruncatingStore(), /*IsCompress*/ false);
8384       }
8385     }
8386 
8387     break;
8388   }
8389   case ISD::SPLAT_VECTOR: {
8390     EVT VT = N->getValueType(0);
8391     // Only perform this combine on legal MVT types.
8392     if (!isTypeLegal(VT))
8393       break;
8394     if (auto Gather = matchSplatAsGather(N->getOperand(0), VT.getSimpleVT(), N,
8395                                          DAG, Subtarget))
8396       return Gather;
8397     break;
8398   }
8399   }
8400 
8401   return SDValue();
8402 }
8403 
8404 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
8405     const SDNode *N, CombineLevel Level) const {
8406   // The following folds are only desirable if `(OP _, c1 << c2)` can be
8407   // materialised in fewer instructions than `(OP _, c1)`:
8408   //
8409   //   (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
8410   //   (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
8411   SDValue N0 = N->getOperand(0);
8412   EVT Ty = N0.getValueType();
8413   if (Ty.isScalarInteger() &&
8414       (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
8415     auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
8416     auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
8417     if (C1 && C2) {
8418       const APInt &C1Int = C1->getAPIntValue();
8419       APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
8420 
8421       // We can materialise `c1 << c2` into an add immediate, so it's "free",
8422       // and the combine should happen, to potentially allow further combines
8423       // later.
8424       if (ShiftedC1Int.getMinSignedBits() <= 64 &&
8425           isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
8426         return true;
8427 
8428       // We can materialise `c1` in an add immediate, so it's "free", and the
8429       // combine should be prevented.
8430       if (C1Int.getMinSignedBits() <= 64 &&
8431           isLegalAddImmediate(C1Int.getSExtValue()))
8432         return false;
8433 
8434       // Neither constant will fit into an immediate, so find materialisation
8435       // costs.
8436       int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
8437                                               Subtarget.getFeatureBits(),
8438                                               /*CompressionCost*/true);
8439       int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
8440           ShiftedC1Int, Ty.getSizeInBits(), Subtarget.getFeatureBits(),
8441           /*CompressionCost*/true);
8442 
8443       // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
8444       // combine should be prevented.
8445       if (C1Cost < ShiftedC1Cost)
8446         return false;
8447     }
8448   }
8449   return true;
8450 }
8451 
8452 bool RISCVTargetLowering::targetShrinkDemandedConstant(
8453     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
8454     TargetLoweringOpt &TLO) const {
8455   // Delay this optimization as late as possible.
8456   if (!TLO.LegalOps)
8457     return false;
8458 
8459   EVT VT = Op.getValueType();
8460   if (VT.isVector())
8461     return false;
8462 
8463   // Only handle AND for now.
8464   if (Op.getOpcode() != ISD::AND)
8465     return false;
8466 
8467   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
8468   if (!C)
8469     return false;
8470 
8471   const APInt &Mask = C->getAPIntValue();
8472 
8473   // Clear all non-demanded bits initially.
8474   APInt ShrunkMask = Mask & DemandedBits;
8475 
8476   // Try to make a smaller immediate by setting undemanded bits.
8477 
8478   APInt ExpandedMask = Mask | ~DemandedBits;
8479 
8480   auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
8481     return ShrunkMask.isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
8482   };
8483   auto UseMask = [Mask, Op, VT, &TLO](const APInt &NewMask) -> bool {
8484     if (NewMask == Mask)
8485       return true;
8486     SDLoc DL(Op);
8487     SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
8488     SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
8489     return TLO.CombineTo(Op, NewOp);
8490   };
8491 
8492   // If the shrunk mask fits in sign extended 12 bits, let the target
8493   // independent code apply it.
8494   if (ShrunkMask.isSignedIntN(12))
8495     return false;
8496 
8497   // Preserve (and X, 0xffff) when zext.h is supported.
8498   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
8499     APInt NewMask = APInt(Mask.getBitWidth(), 0xffff);
8500     if (IsLegalMask(NewMask))
8501       return UseMask(NewMask);
8502   }
8503 
8504   // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
8505   if (VT == MVT::i64) {
8506     APInt NewMask = APInt(64, 0xffffffff);
8507     if (IsLegalMask(NewMask))
8508       return UseMask(NewMask);
8509   }
8510 
8511   // For the remaining optimizations, we need to be able to make a negative
8512   // number through a combination of mask and undemanded bits.
8513   if (!ExpandedMask.isNegative())
8514     return false;
8515 
8516   // What is the fewest number of bits we need to represent the negative number.
8517   unsigned MinSignedBits = ExpandedMask.getMinSignedBits();
8518 
8519   // Try to make a 12 bit negative immediate. If that fails try to make a 32
8520   // bit negative immediate unless the shrunk immediate already fits in 32 bits.
8521   APInt NewMask = ShrunkMask;
8522   if (MinSignedBits <= 12)
8523     NewMask.setBitsFrom(11);
8524   else if (MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(32))
8525     NewMask.setBitsFrom(31);
8526   else
8527     return false;
8528 
8529   // Check that our new mask is a subset of the demanded mask.
8530   assert(IsLegalMask(NewMask));
8531   return UseMask(NewMask);
8532 }
8533 
8534 static void computeGREV(APInt &Src, unsigned ShAmt) {
8535   ShAmt &= Src.getBitWidth() - 1;
8536   uint64_t x = Src.getZExtValue();
8537   if (ShAmt & 1)
8538     x = ((x & 0x5555555555555555LL) << 1) | ((x & 0xAAAAAAAAAAAAAAAALL) >> 1);
8539   if (ShAmt & 2)
8540     x = ((x & 0x3333333333333333LL) << 2) | ((x & 0xCCCCCCCCCCCCCCCCLL) >> 2);
8541   if (ShAmt & 4)
8542     x = ((x & 0x0F0F0F0F0F0F0F0FLL) << 4) | ((x & 0xF0F0F0F0F0F0F0F0LL) >> 4);
8543   if (ShAmt & 8)
8544     x = ((x & 0x00FF00FF00FF00FFLL) << 8) | ((x & 0xFF00FF00FF00FF00LL) >> 8);
8545   if (ShAmt & 16)
8546     x = ((x & 0x0000FFFF0000FFFFLL) << 16) | ((x & 0xFFFF0000FFFF0000LL) >> 16);
8547   if (ShAmt & 32)
8548     x = ((x & 0x00000000FFFFFFFFLL) << 32) | ((x & 0xFFFFFFFF00000000LL) >> 32);
8549   Src = x;
8550 }
8551 
8552 void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
8553                                                         KnownBits &Known,
8554                                                         const APInt &DemandedElts,
8555                                                         const SelectionDAG &DAG,
8556                                                         unsigned Depth) const {
8557   unsigned BitWidth = Known.getBitWidth();
8558   unsigned Opc = Op.getOpcode();
8559   assert((Opc >= ISD::BUILTIN_OP_END ||
8560           Opc == ISD::INTRINSIC_WO_CHAIN ||
8561           Opc == ISD::INTRINSIC_W_CHAIN ||
8562           Opc == ISD::INTRINSIC_VOID) &&
8563          "Should use MaskedValueIsZero if you don't know whether Op"
8564          " is a target node!");
8565 
8566   Known.resetAll();
8567   switch (Opc) {
8568   default: break;
8569   case RISCVISD::SELECT_CC: {
8570     Known = DAG.computeKnownBits(Op.getOperand(4), Depth + 1);
8571     // If we don't know any bits, early out.
8572     if (Known.isUnknown())
8573       break;
8574     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(3), Depth + 1);
8575 
8576     // Only known if known in both the LHS and RHS.
8577     Known = KnownBits::commonBits(Known, Known2);
8578     break;
8579   }
8580   case RISCVISD::REMUW: {
8581     KnownBits Known2;
8582     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
8583     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
8584     // We only care about the lower 32 bits.
8585     Known = KnownBits::urem(Known.trunc(32), Known2.trunc(32));
8586     // Restore the original width by sign extending.
8587     Known = Known.sext(BitWidth);
8588     break;
8589   }
8590   case RISCVISD::DIVUW: {
8591     KnownBits Known2;
8592     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
8593     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
8594     // We only care about the lower 32 bits.
8595     Known = KnownBits::udiv(Known.trunc(32), Known2.trunc(32));
8596     // Restore the original width by sign extending.
8597     Known = Known.sext(BitWidth);
8598     break;
8599   }
8600   case RISCVISD::CTZW: {
8601     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8602     unsigned PossibleTZ = Known2.trunc(32).countMaxTrailingZeros();
8603     unsigned LowBits = Log2_32(PossibleTZ) + 1;
8604     Known.Zero.setBitsFrom(LowBits);
8605     break;
8606   }
8607   case RISCVISD::CLZW: {
8608     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8609     unsigned PossibleLZ = Known2.trunc(32).countMaxLeadingZeros();
8610     unsigned LowBits = Log2_32(PossibleLZ) + 1;
8611     Known.Zero.setBitsFrom(LowBits);
8612     break;
8613   }
8614   case RISCVISD::GREV:
8615   case RISCVISD::GREVW: {
8616     if (auto *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
8617       Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8618       if (Opc == RISCVISD::GREVW)
8619         Known = Known.trunc(32);
8620       unsigned ShAmt = C->getZExtValue();
8621       computeGREV(Known.Zero, ShAmt);
8622       computeGREV(Known.One, ShAmt);
8623       if (Opc == RISCVISD::GREVW)
8624         Known = Known.sext(BitWidth);
8625     }
8626     break;
8627   }
8628   case RISCVISD::READ_VLENB: {
8629     // If we know the minimum VLen from Zvl extensions, we can use that to
8630     // determine the trailing zeros of VLENB.
8631     // FIXME: Limit to 128 bit vectors until we have more testing.
8632     unsigned MinVLenB = std::min(128U, Subtarget.getMinVLen()) / 8;
8633     if (MinVLenB > 0)
8634       Known.Zero.setLowBits(Log2_32(MinVLenB));
8635     // We assume VLENB is no more than 65536 / 8 bytes.
8636     Known.Zero.setBitsFrom(14);
8637     break;
8638   }
8639   case ISD::INTRINSIC_W_CHAIN:
8640   case ISD::INTRINSIC_WO_CHAIN: {
8641     unsigned IntNo =
8642         Op.getConstantOperandVal(Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1);
8643     switch (IntNo) {
8644     default:
8645       // We can't do anything for most intrinsics.
8646       break;
8647     case Intrinsic::riscv_vsetvli:
8648     case Intrinsic::riscv_vsetvlimax:
8649     case Intrinsic::riscv_vsetvli_opt:
8650     case Intrinsic::riscv_vsetvlimax_opt:
8651       // Assume that VL output is positive and would fit in an int32_t.
8652       // TODO: VLEN might be capped at 16 bits in a future V spec update.
8653       if (BitWidth >= 32)
8654         Known.Zero.setBitsFrom(31);
8655       break;
8656     }
8657     break;
8658   }
8659   }
8660 }
8661 
8662 unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
8663     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
8664     unsigned Depth) const {
8665   switch (Op.getOpcode()) {
8666   default:
8667     break;
8668   case RISCVISD::SELECT_CC: {
8669     unsigned Tmp =
8670         DAG.ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth + 1);
8671     if (Tmp == 1) return 1;  // Early out.
8672     unsigned Tmp2 =
8673         DAG.ComputeNumSignBits(Op.getOperand(4), DemandedElts, Depth + 1);
8674     return std::min(Tmp, Tmp2);
8675   }
8676   case RISCVISD::SLLW:
8677   case RISCVISD::SRAW:
8678   case RISCVISD::SRLW:
8679   case RISCVISD::DIVW:
8680   case RISCVISD::DIVUW:
8681   case RISCVISD::REMUW:
8682   case RISCVISD::ROLW:
8683   case RISCVISD::RORW:
8684   case RISCVISD::GREVW:
8685   case RISCVISD::GORCW:
8686   case RISCVISD::FSLW:
8687   case RISCVISD::FSRW:
8688   case RISCVISD::SHFLW:
8689   case RISCVISD::UNSHFLW:
8690   case RISCVISD::BCOMPRESSW:
8691   case RISCVISD::BDECOMPRESSW:
8692   case RISCVISD::BFPW:
8693   case RISCVISD::FCVT_W_RV64:
8694   case RISCVISD::FCVT_WU_RV64:
8695   case RISCVISD::STRICT_FCVT_W_RV64:
8696   case RISCVISD::STRICT_FCVT_WU_RV64:
8697     // TODO: As the result is sign-extended, this is conservatively correct. A
8698     // more precise answer could be calculated for SRAW depending on known
8699     // bits in the shift amount.
8700     return 33;
8701   case RISCVISD::SHFL:
8702   case RISCVISD::UNSHFL: {
8703     // There is no SHFLIW, but a i64 SHFLI with bit 4 of the control word
8704     // cleared doesn't affect bit 31. The upper 32 bits will be shuffled, but
8705     // will stay within the upper 32 bits. If there were more than 32 sign bits
8706     // before there will be at least 33 sign bits after.
8707     if (Op.getValueType() == MVT::i64 &&
8708         isa<ConstantSDNode>(Op.getOperand(1)) &&
8709         (Op.getConstantOperandVal(1) & 0x10) == 0) {
8710       unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
8711       if (Tmp > 32)
8712         return 33;
8713     }
8714     break;
8715   }
8716   case RISCVISD::VMV_X_S: {
8717     // The number of sign bits of the scalar result is computed by obtaining the
8718     // element type of the input vector operand, subtracting its width from the
8719     // XLEN, and then adding one (sign bit within the element type). If the
8720     // element type is wider than XLen, the least-significant XLEN bits are
8721     // taken.
8722     unsigned XLen = Subtarget.getXLen();
8723     unsigned EltBits = Op.getOperand(0).getScalarValueSizeInBits();
8724     if (EltBits <= XLen)
8725       return XLen - EltBits + 1;
8726     break;
8727   }
8728   }
8729 
8730   return 1;
8731 }
8732 
8733 static MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
8734                                                   MachineBasicBlock *BB) {
8735   assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction");
8736 
8737   // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
8738   // Should the count have wrapped while it was being read, we need to try
8739   // again.
8740   // ...
8741   // read:
8742   // rdcycleh x3 # load high word of cycle
8743   // rdcycle  x2 # load low word of cycle
8744   // rdcycleh x4 # load high word of cycle
8745   // bne x3, x4, read # check if high word reads match, otherwise try again
8746   // ...
8747 
8748   MachineFunction &MF = *BB->getParent();
8749   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8750   MachineFunction::iterator It = ++BB->getIterator();
8751 
8752   MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8753   MF.insert(It, LoopMBB);
8754 
8755   MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8756   MF.insert(It, DoneMBB);
8757 
8758   // Transfer the remainder of BB and its successor edges to DoneMBB.
8759   DoneMBB->splice(DoneMBB->begin(), BB,
8760                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
8761   DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
8762 
8763   BB->addSuccessor(LoopMBB);
8764 
8765   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8766   Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
8767   Register LoReg = MI.getOperand(0).getReg();
8768   Register HiReg = MI.getOperand(1).getReg();
8769   DebugLoc DL = MI.getDebugLoc();
8770 
8771   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8772   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
8773       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8774       .addReg(RISCV::X0);
8775   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
8776       .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
8777       .addReg(RISCV::X0);
8778   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
8779       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8780       .addReg(RISCV::X0);
8781 
8782   BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
8783       .addReg(HiReg)
8784       .addReg(ReadAgainReg)
8785       .addMBB(LoopMBB);
8786 
8787   LoopMBB->addSuccessor(LoopMBB);
8788   LoopMBB->addSuccessor(DoneMBB);
8789 
8790   MI.eraseFromParent();
8791 
8792   return DoneMBB;
8793 }
8794 
8795 static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
8796                                              MachineBasicBlock *BB) {
8797   assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
8798 
8799   MachineFunction &MF = *BB->getParent();
8800   DebugLoc DL = MI.getDebugLoc();
8801   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8802   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8803   Register LoReg = MI.getOperand(0).getReg();
8804   Register HiReg = MI.getOperand(1).getReg();
8805   Register SrcReg = MI.getOperand(2).getReg();
8806   const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
8807   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8808 
8809   TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
8810                           RI);
8811   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8812   MachineMemOperand *MMOLo =
8813       MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 4, Align(8));
8814   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8815       MPI.getWithOffset(4), MachineMemOperand::MOLoad, 4, Align(8));
8816   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
8817       .addFrameIndex(FI)
8818       .addImm(0)
8819       .addMemOperand(MMOLo);
8820   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
8821       .addFrameIndex(FI)
8822       .addImm(4)
8823       .addMemOperand(MMOHi);
8824   MI.eraseFromParent(); // The pseudo instruction is gone now.
8825   return BB;
8826 }
8827 
8828 static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
8829                                                  MachineBasicBlock *BB) {
8830   assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
8831          "Unexpected instruction");
8832 
8833   MachineFunction &MF = *BB->getParent();
8834   DebugLoc DL = MI.getDebugLoc();
8835   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8836   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8837   Register DstReg = MI.getOperand(0).getReg();
8838   Register LoReg = MI.getOperand(1).getReg();
8839   Register HiReg = MI.getOperand(2).getReg();
8840   const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
8841   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8842 
8843   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8844   MachineMemOperand *MMOLo =
8845       MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Align(8));
8846   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8847       MPI.getWithOffset(4), MachineMemOperand::MOStore, 4, Align(8));
8848   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8849       .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
8850       .addFrameIndex(FI)
8851       .addImm(0)
8852       .addMemOperand(MMOLo);
8853   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8854       .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
8855       .addFrameIndex(FI)
8856       .addImm(4)
8857       .addMemOperand(MMOHi);
8858   TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
8859   MI.eraseFromParent(); // The pseudo instruction is gone now.
8860   return BB;
8861 }
8862 
8863 static bool isSelectPseudo(MachineInstr &MI) {
8864   switch (MI.getOpcode()) {
8865   default:
8866     return false;
8867   case RISCV::Select_GPR_Using_CC_GPR:
8868   case RISCV::Select_FPR16_Using_CC_GPR:
8869   case RISCV::Select_FPR32_Using_CC_GPR:
8870   case RISCV::Select_FPR64_Using_CC_GPR:
8871     return true;
8872   }
8873 }
8874 
8875 static MachineBasicBlock *emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB,
8876                                         unsigned RelOpcode, unsigned EqOpcode,
8877                                         const RISCVSubtarget &Subtarget) {
8878   DebugLoc DL = MI.getDebugLoc();
8879   Register DstReg = MI.getOperand(0).getReg();
8880   Register Src1Reg = MI.getOperand(1).getReg();
8881   Register Src2Reg = MI.getOperand(2).getReg();
8882   MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
8883   Register SavedFFlags = MRI.createVirtualRegister(&RISCV::GPRRegClass);
8884   const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
8885 
8886   // Save the current FFLAGS.
8887   BuildMI(*BB, MI, DL, TII.get(RISCV::ReadFFLAGS), SavedFFlags);
8888 
8889   auto MIB = BuildMI(*BB, MI, DL, TII.get(RelOpcode), DstReg)
8890                  .addReg(Src1Reg)
8891                  .addReg(Src2Reg);
8892   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8893     MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
8894 
8895   // Restore the FFLAGS.
8896   BuildMI(*BB, MI, DL, TII.get(RISCV::WriteFFLAGS))
8897       .addReg(SavedFFlags, RegState::Kill);
8898 
8899   // Issue a dummy FEQ opcode to raise exception for signaling NaNs.
8900   auto MIB2 = BuildMI(*BB, MI, DL, TII.get(EqOpcode), RISCV::X0)
8901                   .addReg(Src1Reg, getKillRegState(MI.getOperand(1).isKill()))
8902                   .addReg(Src2Reg, getKillRegState(MI.getOperand(2).isKill()));
8903   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8904     MIB2->setFlag(MachineInstr::MIFlag::NoFPExcept);
8905 
8906   // Erase the pseudoinstruction.
8907   MI.eraseFromParent();
8908   return BB;
8909 }
8910 
8911 static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
8912                                            MachineBasicBlock *BB,
8913                                            const RISCVSubtarget &Subtarget) {
8914   // To "insert" Select_* instructions, we actually have to insert the triangle
8915   // control-flow pattern.  The incoming instructions know the destination vreg
8916   // to set, the condition code register to branch on, the true/false values to
8917   // select between, and the condcode to use to select the appropriate branch.
8918   //
8919   // We produce the following control flow:
8920   //     HeadMBB
8921   //     |  \
8922   //     |  IfFalseMBB
8923   //     | /
8924   //    TailMBB
8925   //
8926   // When we find a sequence of selects we attempt to optimize their emission
8927   // by sharing the control flow. Currently we only handle cases where we have
8928   // multiple selects with the exact same condition (same LHS, RHS and CC).
8929   // The selects may be interleaved with other instructions if the other
8930   // instructions meet some requirements we deem safe:
8931   // - They are debug instructions. Otherwise,
8932   // - They do not have side-effects, do not access memory and their inputs do
8933   //   not depend on the results of the select pseudo-instructions.
8934   // The TrueV/FalseV operands of the selects cannot depend on the result of
8935   // previous selects in the sequence.
8936   // These conditions could be further relaxed. See the X86 target for a
8937   // related approach and more information.
8938   Register LHS = MI.getOperand(1).getReg();
8939   Register RHS = MI.getOperand(2).getReg();
8940   auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
8941 
8942   SmallVector<MachineInstr *, 4> SelectDebugValues;
8943   SmallSet<Register, 4> SelectDests;
8944   SelectDests.insert(MI.getOperand(0).getReg());
8945 
8946   MachineInstr *LastSelectPseudo = &MI;
8947 
8948   for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
8949        SequenceMBBI != E; ++SequenceMBBI) {
8950     if (SequenceMBBI->isDebugInstr())
8951       continue;
8952     else if (isSelectPseudo(*SequenceMBBI)) {
8953       if (SequenceMBBI->getOperand(1).getReg() != LHS ||
8954           SequenceMBBI->getOperand(2).getReg() != RHS ||
8955           SequenceMBBI->getOperand(3).getImm() != CC ||
8956           SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
8957           SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
8958         break;
8959       LastSelectPseudo = &*SequenceMBBI;
8960       SequenceMBBI->collectDebugValues(SelectDebugValues);
8961       SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
8962     } else {
8963       if (SequenceMBBI->hasUnmodeledSideEffects() ||
8964           SequenceMBBI->mayLoadOrStore())
8965         break;
8966       if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
8967             return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
8968           }))
8969         break;
8970     }
8971   }
8972 
8973   const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
8974   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8975   DebugLoc DL = MI.getDebugLoc();
8976   MachineFunction::iterator I = ++BB->getIterator();
8977 
8978   MachineBasicBlock *HeadMBB = BB;
8979   MachineFunction *F = BB->getParent();
8980   MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
8981   MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
8982 
8983   F->insert(I, IfFalseMBB);
8984   F->insert(I, TailMBB);
8985 
8986   // Transfer debug instructions associated with the selects to TailMBB.
8987   for (MachineInstr *DebugInstr : SelectDebugValues) {
8988     TailMBB->push_back(DebugInstr->removeFromParent());
8989   }
8990 
8991   // Move all instructions after the sequence to TailMBB.
8992   TailMBB->splice(TailMBB->end(), HeadMBB,
8993                   std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
8994   // Update machine-CFG edges by transferring all successors of the current
8995   // block to the new block which will contain the Phi nodes for the selects.
8996   TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
8997   // Set the successors for HeadMBB.
8998   HeadMBB->addSuccessor(IfFalseMBB);
8999   HeadMBB->addSuccessor(TailMBB);
9000 
9001   // Insert appropriate branch.
9002   BuildMI(HeadMBB, DL, TII.getBrCond(CC))
9003     .addReg(LHS)
9004     .addReg(RHS)
9005     .addMBB(TailMBB);
9006 
9007   // IfFalseMBB just falls through to TailMBB.
9008   IfFalseMBB->addSuccessor(TailMBB);
9009 
9010   // Create PHIs for all of the select pseudo-instructions.
9011   auto SelectMBBI = MI.getIterator();
9012   auto SelectEnd = std::next(LastSelectPseudo->getIterator());
9013   auto InsertionPoint = TailMBB->begin();
9014   while (SelectMBBI != SelectEnd) {
9015     auto Next = std::next(SelectMBBI);
9016     if (isSelectPseudo(*SelectMBBI)) {
9017       // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
9018       BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
9019               TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
9020           .addReg(SelectMBBI->getOperand(4).getReg())
9021           .addMBB(HeadMBB)
9022           .addReg(SelectMBBI->getOperand(5).getReg())
9023           .addMBB(IfFalseMBB);
9024       SelectMBBI->eraseFromParent();
9025     }
9026     SelectMBBI = Next;
9027   }
9028 
9029   F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
9030   return TailMBB;
9031 }
9032 
9033 MachineBasicBlock *
9034 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
9035                                                  MachineBasicBlock *BB) const {
9036   switch (MI.getOpcode()) {
9037   default:
9038     llvm_unreachable("Unexpected instr type to insert");
9039   case RISCV::ReadCycleWide:
9040     assert(!Subtarget.is64Bit() &&
9041            "ReadCycleWrite is only to be used on riscv32");
9042     return emitReadCycleWidePseudo(MI, BB);
9043   case RISCV::Select_GPR_Using_CC_GPR:
9044   case RISCV::Select_FPR16_Using_CC_GPR:
9045   case RISCV::Select_FPR32_Using_CC_GPR:
9046   case RISCV::Select_FPR64_Using_CC_GPR:
9047     return emitSelectPseudo(MI, BB, Subtarget);
9048   case RISCV::BuildPairF64Pseudo:
9049     return emitBuildPairF64Pseudo(MI, BB);
9050   case RISCV::SplitF64Pseudo:
9051     return emitSplitF64Pseudo(MI, BB);
9052   case RISCV::PseudoQuietFLE_H:
9053     return emitQuietFCMP(MI, BB, RISCV::FLE_H, RISCV::FEQ_H, Subtarget);
9054   case RISCV::PseudoQuietFLT_H:
9055     return emitQuietFCMP(MI, BB, RISCV::FLT_H, RISCV::FEQ_H, Subtarget);
9056   case RISCV::PseudoQuietFLE_S:
9057     return emitQuietFCMP(MI, BB, RISCV::FLE_S, RISCV::FEQ_S, Subtarget);
9058   case RISCV::PseudoQuietFLT_S:
9059     return emitQuietFCMP(MI, BB, RISCV::FLT_S, RISCV::FEQ_S, Subtarget);
9060   case RISCV::PseudoQuietFLE_D:
9061     return emitQuietFCMP(MI, BB, RISCV::FLE_D, RISCV::FEQ_D, Subtarget);
9062   case RISCV::PseudoQuietFLT_D:
9063     return emitQuietFCMP(MI, BB, RISCV::FLT_D, RISCV::FEQ_D, Subtarget);
9064   }
9065 }
9066 
9067 void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
9068                                                         SDNode *Node) const {
9069   // Add FRM dependency to any instructions with dynamic rounding mode.
9070   unsigned Opc = MI.getOpcode();
9071   auto Idx = RISCV::getNamedOperandIdx(Opc, RISCV::OpName::frm);
9072   if (Idx < 0)
9073     return;
9074   if (MI.getOperand(Idx).getImm() != RISCVFPRndMode::DYN)
9075     return;
9076   // If the instruction already reads FRM, don't add another read.
9077   if (MI.readsRegister(RISCV::FRM))
9078     return;
9079   MI.addOperand(
9080       MachineOperand::CreateReg(RISCV::FRM, /*isDef*/ false, /*isImp*/ true));
9081 }
9082 
9083 // Calling Convention Implementation.
9084 // The expectations for frontend ABI lowering vary from target to target.
9085 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
9086 // details, but this is a longer term goal. For now, we simply try to keep the
9087 // role of the frontend as simple and well-defined as possible. The rules can
9088 // be summarised as:
9089 // * Never split up large scalar arguments. We handle them here.
9090 // * If a hardfloat calling convention is being used, and the struct may be
9091 // passed in a pair of registers (fp+fp, int+fp), and both registers are
9092 // available, then pass as two separate arguments. If either the GPRs or FPRs
9093 // are exhausted, then pass according to the rule below.
9094 // * If a struct could never be passed in registers or directly in a stack
9095 // slot (as it is larger than 2*XLEN and the floating point rules don't
9096 // apply), then pass it using a pointer with the byval attribute.
9097 // * If a struct is less than 2*XLEN, then coerce to either a two-element
9098 // word-sized array or a 2*XLEN scalar (depending on alignment).
9099 // * The frontend can determine whether a struct is returned by reference or
9100 // not based on its size and fields. If it will be returned by reference, the
9101 // frontend must modify the prototype so a pointer with the sret annotation is
9102 // passed as the first argument. This is not necessary for large scalar
9103 // returns.
9104 // * Struct return values and varargs should be coerced to structs containing
9105 // register-size fields in the same situations they would be for fixed
9106 // arguments.
9107 
9108 static const MCPhysReg ArgGPRs[] = {
9109   RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
9110   RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
9111 };
9112 static const MCPhysReg ArgFPR16s[] = {
9113   RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
9114   RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
9115 };
9116 static const MCPhysReg ArgFPR32s[] = {
9117   RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
9118   RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
9119 };
9120 static const MCPhysReg ArgFPR64s[] = {
9121   RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
9122   RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
9123 };
9124 // This is an interim calling convention and it may be changed in the future.
9125 static const MCPhysReg ArgVRs[] = {
9126     RISCV::V8,  RISCV::V9,  RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
9127     RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
9128     RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
9129 static const MCPhysReg ArgVRM2s[] = {RISCV::V8M2,  RISCV::V10M2, RISCV::V12M2,
9130                                      RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
9131                                      RISCV::V20M2, RISCV::V22M2};
9132 static const MCPhysReg ArgVRM4s[] = {RISCV::V8M4, RISCV::V12M4, RISCV::V16M4,
9133                                      RISCV::V20M4};
9134 static const MCPhysReg ArgVRM8s[] = {RISCV::V8M8, RISCV::V16M8};
9135 
9136 // Pass a 2*XLEN argument that has been split into two XLEN values through
9137 // registers or the stack as necessary.
9138 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
9139                                 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
9140                                 MVT ValVT2, MVT LocVT2,
9141                                 ISD::ArgFlagsTy ArgFlags2) {
9142   unsigned XLenInBytes = XLen / 8;
9143   if (Register Reg = State.AllocateReg(ArgGPRs)) {
9144     // At least one half can be passed via register.
9145     State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
9146                                      VA1.getLocVT(), CCValAssign::Full));
9147   } else {
9148     // Both halves must be passed on the stack, with proper alignment.
9149     Align StackAlign =
9150         std::max(Align(XLenInBytes), ArgFlags1.getNonZeroOrigAlign());
9151     State.addLoc(
9152         CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
9153                             State.AllocateStack(XLenInBytes, StackAlign),
9154                             VA1.getLocVT(), CCValAssign::Full));
9155     State.addLoc(CCValAssign::getMem(
9156         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
9157         LocVT2, CCValAssign::Full));
9158     return false;
9159   }
9160 
9161   if (Register Reg = State.AllocateReg(ArgGPRs)) {
9162     // The second half can also be passed via register.
9163     State.addLoc(
9164         CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
9165   } else {
9166     // The second half is passed via the stack, without additional alignment.
9167     State.addLoc(CCValAssign::getMem(
9168         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
9169         LocVT2, CCValAssign::Full));
9170   }
9171 
9172   return false;
9173 }
9174 
9175 static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo,
9176                                Optional<unsigned> FirstMaskArgument,
9177                                CCState &State, const RISCVTargetLowering &TLI) {
9178   const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT);
9179   if (RC == &RISCV::VRRegClass) {
9180     // Assign the first mask argument to V0.
9181     // This is an interim calling convention and it may be changed in the
9182     // future.
9183     if (FirstMaskArgument.hasValue() && ValNo == FirstMaskArgument.getValue())
9184       return State.AllocateReg(RISCV::V0);
9185     return State.AllocateReg(ArgVRs);
9186   }
9187   if (RC == &RISCV::VRM2RegClass)
9188     return State.AllocateReg(ArgVRM2s);
9189   if (RC == &RISCV::VRM4RegClass)
9190     return State.AllocateReg(ArgVRM4s);
9191   if (RC == &RISCV::VRM8RegClass)
9192     return State.AllocateReg(ArgVRM8s);
9193   llvm_unreachable("Unhandled register class for ValueType");
9194 }
9195 
9196 // Implements the RISC-V calling convention. Returns true upon failure.
9197 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
9198                      MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
9199                      ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
9200                      bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
9201                      Optional<unsigned> FirstMaskArgument) {
9202   unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
9203   assert(XLen == 32 || XLen == 64);
9204   MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
9205 
9206   // Any return value split in to more than two values can't be returned
9207   // directly. Vectors are returned via the available vector registers.
9208   if (!LocVT.isVector() && IsRet && ValNo > 1)
9209     return true;
9210 
9211   // UseGPRForF16_F32 if targeting one of the soft-float ABIs, if passing a
9212   // variadic argument, or if no F16/F32 argument registers are available.
9213   bool UseGPRForF16_F32 = true;
9214   // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
9215   // variadic argument, or if no F64 argument registers are available.
9216   bool UseGPRForF64 = true;
9217 
9218   switch (ABI) {
9219   default:
9220     llvm_unreachable("Unexpected ABI");
9221   case RISCVABI::ABI_ILP32:
9222   case RISCVABI::ABI_LP64:
9223     break;
9224   case RISCVABI::ABI_ILP32F:
9225   case RISCVABI::ABI_LP64F:
9226     UseGPRForF16_F32 = !IsFixed;
9227     break;
9228   case RISCVABI::ABI_ILP32D:
9229   case RISCVABI::ABI_LP64D:
9230     UseGPRForF16_F32 = !IsFixed;
9231     UseGPRForF64 = !IsFixed;
9232     break;
9233   }
9234 
9235   // FPR16, FPR32, and FPR64 alias each other.
9236   if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s)) {
9237     UseGPRForF16_F32 = true;
9238     UseGPRForF64 = true;
9239   }
9240 
9241   // From this point on, rely on UseGPRForF16_F32, UseGPRForF64 and
9242   // similar local variables rather than directly checking against the target
9243   // ABI.
9244 
9245   if (UseGPRForF16_F32 && (ValVT == MVT::f16 || ValVT == MVT::f32)) {
9246     LocVT = XLenVT;
9247     LocInfo = CCValAssign::BCvt;
9248   } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
9249     LocVT = MVT::i64;
9250     LocInfo = CCValAssign::BCvt;
9251   }
9252 
9253   // If this is a variadic argument, the RISC-V calling convention requires
9254   // that it is assigned an 'even' or 'aligned' register if it has 8-byte
9255   // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
9256   // be used regardless of whether the original argument was split during
9257   // legalisation or not. The argument will not be passed by registers if the
9258   // original type is larger than 2*XLEN, so the register alignment rule does
9259   // not apply.
9260   unsigned TwoXLenInBytes = (2 * XLen) / 8;
9261   if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
9262       DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
9263     unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
9264     // Skip 'odd' register if necessary.
9265     if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
9266       State.AllocateReg(ArgGPRs);
9267   }
9268 
9269   SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
9270   SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
9271       State.getPendingArgFlags();
9272 
9273   assert(PendingLocs.size() == PendingArgFlags.size() &&
9274          "PendingLocs and PendingArgFlags out of sync");
9275 
9276   // Handle passing f64 on RV32D with a soft float ABI or when floating point
9277   // registers are exhausted.
9278   if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
9279     assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
9280            "Can't lower f64 if it is split");
9281     // Depending on available argument GPRS, f64 may be passed in a pair of
9282     // GPRs, split between a GPR and the stack, or passed completely on the
9283     // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
9284     // cases.
9285     Register Reg = State.AllocateReg(ArgGPRs);
9286     LocVT = MVT::i32;
9287     if (!Reg) {
9288       unsigned StackOffset = State.AllocateStack(8, Align(8));
9289       State.addLoc(
9290           CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
9291       return false;
9292     }
9293     if (!State.AllocateReg(ArgGPRs))
9294       State.AllocateStack(4, Align(4));
9295     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9296     return false;
9297   }
9298 
9299   // Fixed-length vectors are located in the corresponding scalable-vector
9300   // container types.
9301   if (ValVT.isFixedLengthVector())
9302     LocVT = TLI.getContainerForFixedLengthVector(LocVT);
9303 
9304   // Split arguments might be passed indirectly, so keep track of the pending
9305   // values. Split vectors are passed via a mix of registers and indirectly, so
9306   // treat them as we would any other argument.
9307   if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
9308     LocVT = XLenVT;
9309     LocInfo = CCValAssign::Indirect;
9310     PendingLocs.push_back(
9311         CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
9312     PendingArgFlags.push_back(ArgFlags);
9313     if (!ArgFlags.isSplitEnd()) {
9314       return false;
9315     }
9316   }
9317 
9318   // If the split argument only had two elements, it should be passed directly
9319   // in registers or on the stack.
9320   if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
9321       PendingLocs.size() <= 2) {
9322     assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
9323     // Apply the normal calling convention rules to the first half of the
9324     // split argument.
9325     CCValAssign VA = PendingLocs[0];
9326     ISD::ArgFlagsTy AF = PendingArgFlags[0];
9327     PendingLocs.clear();
9328     PendingArgFlags.clear();
9329     return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
9330                                ArgFlags);
9331   }
9332 
9333   // Allocate to a register if possible, or else a stack slot.
9334   Register Reg;
9335   unsigned StoreSizeBytes = XLen / 8;
9336   Align StackAlign = Align(XLen / 8);
9337 
9338   if (ValVT == MVT::f16 && !UseGPRForF16_F32)
9339     Reg = State.AllocateReg(ArgFPR16s);
9340   else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
9341     Reg = State.AllocateReg(ArgFPR32s);
9342   else if (ValVT == MVT::f64 && !UseGPRForF64)
9343     Reg = State.AllocateReg(ArgFPR64s);
9344   else if (ValVT.isVector()) {
9345     Reg = allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI);
9346     if (!Reg) {
9347       // For return values, the vector must be passed fully via registers or
9348       // via the stack.
9349       // FIXME: The proposed vector ABI only mandates v8-v15 for return values,
9350       // but we're using all of them.
9351       if (IsRet)
9352         return true;
9353       // Try using a GPR to pass the address
9354       if ((Reg = State.AllocateReg(ArgGPRs))) {
9355         LocVT = XLenVT;
9356         LocInfo = CCValAssign::Indirect;
9357       } else if (ValVT.isScalableVector()) {
9358         LocVT = XLenVT;
9359         LocInfo = CCValAssign::Indirect;
9360       } else {
9361         // Pass fixed-length vectors on the stack.
9362         LocVT = ValVT;
9363         StoreSizeBytes = ValVT.getStoreSize();
9364         // Align vectors to their element sizes, being careful for vXi1
9365         // vectors.
9366         StackAlign = MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
9367       }
9368     }
9369   } else {
9370     Reg = State.AllocateReg(ArgGPRs);
9371   }
9372 
9373   unsigned StackOffset =
9374       Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
9375 
9376   // If we reach this point and PendingLocs is non-empty, we must be at the
9377   // end of a split argument that must be passed indirectly.
9378   if (!PendingLocs.empty()) {
9379     assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
9380     assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
9381 
9382     for (auto &It : PendingLocs) {
9383       if (Reg)
9384         It.convertToReg(Reg);
9385       else
9386         It.convertToMem(StackOffset);
9387       State.addLoc(It);
9388     }
9389     PendingLocs.clear();
9390     PendingArgFlags.clear();
9391     return false;
9392   }
9393 
9394   assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
9395           (TLI.getSubtarget().hasVInstructions() && ValVT.isVector())) &&
9396          "Expected an XLenVT or vector types at this stage");
9397 
9398   if (Reg) {
9399     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9400     return false;
9401   }
9402 
9403   // When a floating-point value is passed on the stack, no bit-conversion is
9404   // needed.
9405   if (ValVT.isFloatingPoint()) {
9406     LocVT = ValVT;
9407     LocInfo = CCValAssign::Full;
9408   }
9409   State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
9410   return false;
9411 }
9412 
9413 template <typename ArgTy>
9414 static Optional<unsigned> preAssignMask(const ArgTy &Args) {
9415   for (const auto &ArgIdx : enumerate(Args)) {
9416     MVT ArgVT = ArgIdx.value().VT;
9417     if (ArgVT.isVector() && ArgVT.getVectorElementType() == MVT::i1)
9418       return ArgIdx.index();
9419   }
9420   return None;
9421 }
9422 
9423 void RISCVTargetLowering::analyzeInputArgs(
9424     MachineFunction &MF, CCState &CCInfo,
9425     const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
9426     RISCVCCAssignFn Fn) const {
9427   unsigned NumArgs = Ins.size();
9428   FunctionType *FType = MF.getFunction().getFunctionType();
9429 
9430   Optional<unsigned> FirstMaskArgument;
9431   if (Subtarget.hasVInstructions())
9432     FirstMaskArgument = preAssignMask(Ins);
9433 
9434   for (unsigned i = 0; i != NumArgs; ++i) {
9435     MVT ArgVT = Ins[i].VT;
9436     ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
9437 
9438     Type *ArgTy = nullptr;
9439     if (IsRet)
9440       ArgTy = FType->getReturnType();
9441     else if (Ins[i].isOrigArg())
9442       ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
9443 
9444     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9445     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
9446            ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this,
9447            FirstMaskArgument)) {
9448       LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
9449                         << EVT(ArgVT).getEVTString() << '\n');
9450       llvm_unreachable(nullptr);
9451     }
9452   }
9453 }
9454 
9455 void RISCVTargetLowering::analyzeOutputArgs(
9456     MachineFunction &MF, CCState &CCInfo,
9457     const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
9458     CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
9459   unsigned NumArgs = Outs.size();
9460 
9461   Optional<unsigned> FirstMaskArgument;
9462   if (Subtarget.hasVInstructions())
9463     FirstMaskArgument = preAssignMask(Outs);
9464 
9465   for (unsigned i = 0; i != NumArgs; i++) {
9466     MVT ArgVT = Outs[i].VT;
9467     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
9468     Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
9469 
9470     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9471     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
9472            ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this,
9473            FirstMaskArgument)) {
9474       LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
9475                         << EVT(ArgVT).getEVTString() << "\n");
9476       llvm_unreachable(nullptr);
9477     }
9478   }
9479 }
9480 
9481 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
9482 // values.
9483 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
9484                                    const CCValAssign &VA, const SDLoc &DL,
9485                                    const RISCVSubtarget &Subtarget) {
9486   switch (VA.getLocInfo()) {
9487   default:
9488     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9489   case CCValAssign::Full:
9490     if (VA.getValVT().isFixedLengthVector() && VA.getLocVT().isScalableVector())
9491       Val = convertFromScalableVector(VA.getValVT(), Val, DAG, Subtarget);
9492     break;
9493   case CCValAssign::BCvt:
9494     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
9495       Val = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, Val);
9496     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9497       Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
9498     else
9499       Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
9500     break;
9501   }
9502   return Val;
9503 }
9504 
9505 // The caller is responsible for loading the full value if the argument is
9506 // passed with CCValAssign::Indirect.
9507 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
9508                                 const CCValAssign &VA, const SDLoc &DL,
9509                                 const RISCVTargetLowering &TLI) {
9510   MachineFunction &MF = DAG.getMachineFunction();
9511   MachineRegisterInfo &RegInfo = MF.getRegInfo();
9512   EVT LocVT = VA.getLocVT();
9513   SDValue Val;
9514   const TargetRegisterClass *RC = TLI.getRegClassFor(LocVT.getSimpleVT());
9515   Register VReg = RegInfo.createVirtualRegister(RC);
9516   RegInfo.addLiveIn(VA.getLocReg(), VReg);
9517   Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
9518 
9519   if (VA.getLocInfo() == CCValAssign::Indirect)
9520     return Val;
9521 
9522   return convertLocVTToValVT(DAG, Val, VA, DL, TLI.getSubtarget());
9523 }
9524 
9525 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
9526                                    const CCValAssign &VA, const SDLoc &DL,
9527                                    const RISCVSubtarget &Subtarget) {
9528   EVT LocVT = VA.getLocVT();
9529 
9530   switch (VA.getLocInfo()) {
9531   default:
9532     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9533   case CCValAssign::Full:
9534     if (VA.getValVT().isFixedLengthVector() && LocVT.isScalableVector())
9535       Val = convertToScalableVector(LocVT, Val, DAG, Subtarget);
9536     break;
9537   case CCValAssign::BCvt:
9538     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
9539       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, VA.getLocVT(), Val);
9540     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9541       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
9542     else
9543       Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
9544     break;
9545   }
9546   return Val;
9547 }
9548 
9549 // The caller is responsible for loading the full value if the argument is
9550 // passed with CCValAssign::Indirect.
9551 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
9552                                 const CCValAssign &VA, const SDLoc &DL) {
9553   MachineFunction &MF = DAG.getMachineFunction();
9554   MachineFrameInfo &MFI = MF.getFrameInfo();
9555   EVT LocVT = VA.getLocVT();
9556   EVT ValVT = VA.getValVT();
9557   EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
9558   if (ValVT.isScalableVector()) {
9559     // When the value is a scalable vector, we save the pointer which points to
9560     // the scalable vector value in the stack. The ValVT will be the pointer
9561     // type, instead of the scalable vector type.
9562     ValVT = LocVT;
9563   }
9564   int FI = MFI.CreateFixedObject(ValVT.getStoreSize(), VA.getLocMemOffset(),
9565                                  /*IsImmutable=*/true);
9566   SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
9567   SDValue Val;
9568 
9569   ISD::LoadExtType ExtType;
9570   switch (VA.getLocInfo()) {
9571   default:
9572     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9573   case CCValAssign::Full:
9574   case CCValAssign::Indirect:
9575   case CCValAssign::BCvt:
9576     ExtType = ISD::NON_EXTLOAD;
9577     break;
9578   }
9579   Val = DAG.getExtLoad(
9580       ExtType, DL, LocVT, Chain, FIN,
9581       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
9582   return Val;
9583 }
9584 
9585 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
9586                                        const CCValAssign &VA, const SDLoc &DL) {
9587   assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
9588          "Unexpected VA");
9589   MachineFunction &MF = DAG.getMachineFunction();
9590   MachineFrameInfo &MFI = MF.getFrameInfo();
9591   MachineRegisterInfo &RegInfo = MF.getRegInfo();
9592 
9593   if (VA.isMemLoc()) {
9594     // f64 is passed on the stack.
9595     int FI =
9596         MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*IsImmutable=*/true);
9597     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
9598     return DAG.getLoad(MVT::f64, DL, Chain, FIN,
9599                        MachinePointerInfo::getFixedStack(MF, FI));
9600   }
9601 
9602   assert(VA.isRegLoc() && "Expected register VA assignment");
9603 
9604   Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
9605   RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
9606   SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
9607   SDValue Hi;
9608   if (VA.getLocReg() == RISCV::X17) {
9609     // Second half of f64 is passed on the stack.
9610     int FI = MFI.CreateFixedObject(4, 0, /*IsImmutable=*/true);
9611     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
9612     Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
9613                      MachinePointerInfo::getFixedStack(MF, FI));
9614   } else {
9615     // Second half of f64 is passed in another GPR.
9616     Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
9617     RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
9618     Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
9619   }
9620   return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
9621 }
9622 
9623 // FastCC has less than 1% performance improvement for some particular
9624 // benchmark. But theoretically, it may has benenfit for some cases.
9625 static bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
9626                             unsigned ValNo, MVT ValVT, MVT LocVT,
9627                             CCValAssign::LocInfo LocInfo,
9628                             ISD::ArgFlagsTy ArgFlags, CCState &State,
9629                             bool IsFixed, bool IsRet, Type *OrigTy,
9630                             const RISCVTargetLowering &TLI,
9631                             Optional<unsigned> FirstMaskArgument) {
9632 
9633   // X5 and X6 might be used for save-restore libcall.
9634   static const MCPhysReg GPRList[] = {
9635       RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
9636       RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7,  RISCV::X28,
9637       RISCV::X29, RISCV::X30, RISCV::X31};
9638 
9639   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9640     if (unsigned Reg = State.AllocateReg(GPRList)) {
9641       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9642       return false;
9643     }
9644   }
9645 
9646   if (LocVT == MVT::f16) {
9647     static const MCPhysReg FPR16List[] = {
9648         RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
9649         RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H,  RISCV::F1_H,
9650         RISCV::F2_H,  RISCV::F3_H,  RISCV::F4_H,  RISCV::F5_H,  RISCV::F6_H,
9651         RISCV::F7_H,  RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
9652     if (unsigned Reg = State.AllocateReg(FPR16List)) {
9653       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9654       return false;
9655     }
9656   }
9657 
9658   if (LocVT == MVT::f32) {
9659     static const MCPhysReg FPR32List[] = {
9660         RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
9661         RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F,  RISCV::F1_F,
9662         RISCV::F2_F,  RISCV::F3_F,  RISCV::F4_F,  RISCV::F5_F,  RISCV::F6_F,
9663         RISCV::F7_F,  RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
9664     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9665       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9666       return false;
9667     }
9668   }
9669 
9670   if (LocVT == MVT::f64) {
9671     static const MCPhysReg FPR64List[] = {
9672         RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
9673         RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D,  RISCV::F1_D,
9674         RISCV::F2_D,  RISCV::F3_D,  RISCV::F4_D,  RISCV::F5_D,  RISCV::F6_D,
9675         RISCV::F7_D,  RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
9676     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9677       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9678       return false;
9679     }
9680   }
9681 
9682   if (LocVT == MVT::i32 || LocVT == MVT::f32) {
9683     unsigned Offset4 = State.AllocateStack(4, Align(4));
9684     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset4, LocVT, LocInfo));
9685     return false;
9686   }
9687 
9688   if (LocVT == MVT::i64 || LocVT == MVT::f64) {
9689     unsigned Offset5 = State.AllocateStack(8, Align(8));
9690     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset5, LocVT, LocInfo));
9691     return false;
9692   }
9693 
9694   if (LocVT.isVector()) {
9695     if (unsigned Reg =
9696             allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI)) {
9697       // Fixed-length vectors are located in the corresponding scalable-vector
9698       // container types.
9699       if (ValVT.isFixedLengthVector())
9700         LocVT = TLI.getContainerForFixedLengthVector(LocVT);
9701       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9702     } else {
9703       // Try and pass the address via a "fast" GPR.
9704       if (unsigned GPRReg = State.AllocateReg(GPRList)) {
9705         LocInfo = CCValAssign::Indirect;
9706         LocVT = TLI.getSubtarget().getXLenVT();
9707         State.addLoc(CCValAssign::getReg(ValNo, ValVT, GPRReg, LocVT, LocInfo));
9708       } else if (ValVT.isFixedLengthVector()) {
9709         auto StackAlign =
9710             MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
9711         unsigned StackOffset =
9712             State.AllocateStack(ValVT.getStoreSize(), StackAlign);
9713         State.addLoc(
9714             CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
9715       } else {
9716         // Can't pass scalable vectors on the stack.
9717         return true;
9718       }
9719     }
9720 
9721     return false;
9722   }
9723 
9724   return true; // CC didn't match.
9725 }
9726 
9727 static bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
9728                          CCValAssign::LocInfo LocInfo,
9729                          ISD::ArgFlagsTy ArgFlags, CCState &State) {
9730 
9731   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9732     // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, R6, R7, SpLim
9733     //                        s1    s2  s3  s4  s5  s6  s7  s8  s9  s10 s11
9734     static const MCPhysReg GPRList[] = {
9735         RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
9736         RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
9737     if (unsigned Reg = State.AllocateReg(GPRList)) {
9738       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9739       return false;
9740     }
9741   }
9742 
9743   if (LocVT == MVT::f32) {
9744     // Pass in STG registers: F1, ..., F6
9745     //                        fs0 ... fs5
9746     static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
9747                                           RISCV::F18_F, RISCV::F19_F,
9748                                           RISCV::F20_F, RISCV::F21_F};
9749     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9750       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9751       return false;
9752     }
9753   }
9754 
9755   if (LocVT == MVT::f64) {
9756     // Pass in STG registers: D1, ..., D6
9757     //                        fs6 ... fs11
9758     static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
9759                                           RISCV::F24_D, RISCV::F25_D,
9760                                           RISCV::F26_D, RISCV::F27_D};
9761     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9762       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9763       return false;
9764     }
9765   }
9766 
9767   report_fatal_error("No registers left in GHC calling convention");
9768   return true;
9769 }
9770 
9771 // Transform physical registers into virtual registers.
9772 SDValue RISCVTargetLowering::LowerFormalArguments(
9773     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
9774     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
9775     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
9776 
9777   MachineFunction &MF = DAG.getMachineFunction();
9778 
9779   switch (CallConv) {
9780   default:
9781     report_fatal_error("Unsupported calling convention");
9782   case CallingConv::C:
9783   case CallingConv::Fast:
9784     break;
9785   case CallingConv::GHC:
9786     if (!MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtF] ||
9787         !MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtD])
9788       report_fatal_error(
9789         "GHC calling convention requires the F and D instruction set extensions");
9790   }
9791 
9792   const Function &Func = MF.getFunction();
9793   if (Func.hasFnAttribute("interrupt")) {
9794     if (!Func.arg_empty())
9795       report_fatal_error(
9796         "Functions with the interrupt attribute cannot have arguments!");
9797 
9798     StringRef Kind =
9799       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
9800 
9801     if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
9802       report_fatal_error(
9803         "Function interrupt attribute argument not supported!");
9804   }
9805 
9806   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9807   MVT XLenVT = Subtarget.getXLenVT();
9808   unsigned XLenInBytes = Subtarget.getXLen() / 8;
9809   // Used with vargs to acumulate store chains.
9810   std::vector<SDValue> OutChains;
9811 
9812   // Assign locations to all of the incoming arguments.
9813   SmallVector<CCValAssign, 16> ArgLocs;
9814   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9815 
9816   if (CallConv == CallingConv::GHC)
9817     CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_GHC);
9818   else
9819     analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false,
9820                      CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9821                                                    : CC_RISCV);
9822 
9823   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
9824     CCValAssign &VA = ArgLocs[i];
9825     SDValue ArgValue;
9826     // Passing f64 on RV32D with a soft float ABI must be handled as a special
9827     // case.
9828     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
9829       ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
9830     else if (VA.isRegLoc())
9831       ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, *this);
9832     else
9833       ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
9834 
9835     if (VA.getLocInfo() == CCValAssign::Indirect) {
9836       // If the original argument was split and passed by reference (e.g. i128
9837       // on RV32), we need to load all parts of it here (using the same
9838       // address). Vectors may be partly split to registers and partly to the
9839       // stack, in which case the base address is partly offset and subsequent
9840       // stores are relative to that.
9841       InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
9842                                    MachinePointerInfo()));
9843       unsigned ArgIndex = Ins[i].OrigArgIndex;
9844       unsigned ArgPartOffset = Ins[i].PartOffset;
9845       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9846       while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
9847         CCValAssign &PartVA = ArgLocs[i + 1];
9848         unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset;
9849         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9850         if (PartVA.getValVT().isScalableVector())
9851           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9852         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
9853         InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
9854                                      MachinePointerInfo()));
9855         ++i;
9856       }
9857       continue;
9858     }
9859     InVals.push_back(ArgValue);
9860   }
9861 
9862   if (IsVarArg) {
9863     ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
9864     unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
9865     const TargetRegisterClass *RC = &RISCV::GPRRegClass;
9866     MachineFrameInfo &MFI = MF.getFrameInfo();
9867     MachineRegisterInfo &RegInfo = MF.getRegInfo();
9868     RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
9869 
9870     // Offset of the first variable argument from stack pointer, and size of
9871     // the vararg save area. For now, the varargs save area is either zero or
9872     // large enough to hold a0-a7.
9873     int VaArgOffset, VarArgsSaveSize;
9874 
9875     // If all registers are allocated, then all varargs must be passed on the
9876     // stack and we don't need to save any argregs.
9877     if (ArgRegs.size() == Idx) {
9878       VaArgOffset = CCInfo.getNextStackOffset();
9879       VarArgsSaveSize = 0;
9880     } else {
9881       VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
9882       VaArgOffset = -VarArgsSaveSize;
9883     }
9884 
9885     // Record the frame index of the first variable argument
9886     // which is a value necessary to VASTART.
9887     int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9888     RVFI->setVarArgsFrameIndex(FI);
9889 
9890     // If saving an odd number of registers then create an extra stack slot to
9891     // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
9892     // offsets to even-numbered registered remain 2*XLEN-aligned.
9893     if (Idx % 2) {
9894       MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true);
9895       VarArgsSaveSize += XLenInBytes;
9896     }
9897 
9898     // Copy the integer registers that may have been used for passing varargs
9899     // to the vararg save area.
9900     for (unsigned I = Idx; I < ArgRegs.size();
9901          ++I, VaArgOffset += XLenInBytes) {
9902       const Register Reg = RegInfo.createVirtualRegister(RC);
9903       RegInfo.addLiveIn(ArgRegs[I], Reg);
9904       SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
9905       FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9906       SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9907       SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
9908                                    MachinePointerInfo::getFixedStack(MF, FI));
9909       cast<StoreSDNode>(Store.getNode())
9910           ->getMemOperand()
9911           ->setValue((Value *)nullptr);
9912       OutChains.push_back(Store);
9913     }
9914     RVFI->setVarArgsSaveSize(VarArgsSaveSize);
9915   }
9916 
9917   // All stores are grouped in one node to allow the matching between
9918   // the size of Ins and InVals. This only happens for vararg functions.
9919   if (!OutChains.empty()) {
9920     OutChains.push_back(Chain);
9921     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
9922   }
9923 
9924   return Chain;
9925 }
9926 
9927 /// isEligibleForTailCallOptimization - Check whether the call is eligible
9928 /// for tail call optimization.
9929 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
9930 bool RISCVTargetLowering::isEligibleForTailCallOptimization(
9931     CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
9932     const SmallVector<CCValAssign, 16> &ArgLocs) const {
9933 
9934   auto &Callee = CLI.Callee;
9935   auto CalleeCC = CLI.CallConv;
9936   auto &Outs = CLI.Outs;
9937   auto &Caller = MF.getFunction();
9938   auto CallerCC = Caller.getCallingConv();
9939 
9940   // Exception-handling functions need a special set of instructions to
9941   // indicate a return to the hardware. Tail-calling another function would
9942   // probably break this.
9943   // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
9944   // should be expanded as new function attributes are introduced.
9945   if (Caller.hasFnAttribute("interrupt"))
9946     return false;
9947 
9948   // Do not tail call opt if the stack is used to pass parameters.
9949   if (CCInfo.getNextStackOffset() != 0)
9950     return false;
9951 
9952   // Do not tail call opt if any parameters need to be passed indirectly.
9953   // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
9954   // passed indirectly. So the address of the value will be passed in a
9955   // register, or if not available, then the address is put on the stack. In
9956   // order to pass indirectly, space on the stack often needs to be allocated
9957   // in order to store the value. In this case the CCInfo.getNextStackOffset()
9958   // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
9959   // are passed CCValAssign::Indirect.
9960   for (auto &VA : ArgLocs)
9961     if (VA.getLocInfo() == CCValAssign::Indirect)
9962       return false;
9963 
9964   // Do not tail call opt if either caller or callee uses struct return
9965   // semantics.
9966   auto IsCallerStructRet = Caller.hasStructRetAttr();
9967   auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
9968   if (IsCallerStructRet || IsCalleeStructRet)
9969     return false;
9970 
9971   // Externally-defined functions with weak linkage should not be
9972   // tail-called. The behaviour of branch instructions in this situation (as
9973   // used for tail calls) is implementation-defined, so we cannot rely on the
9974   // linker replacing the tail call with a return.
9975   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
9976     const GlobalValue *GV = G->getGlobal();
9977     if (GV->hasExternalWeakLinkage())
9978       return false;
9979   }
9980 
9981   // The callee has to preserve all registers the caller needs to preserve.
9982   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
9983   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
9984   if (CalleeCC != CallerCC) {
9985     const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
9986     if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
9987       return false;
9988   }
9989 
9990   // Byval parameters hand the function a pointer directly into the stack area
9991   // we want to reuse during a tail call. Working around this *is* possible
9992   // but less efficient and uglier in LowerCall.
9993   for (auto &Arg : Outs)
9994     if (Arg.Flags.isByVal())
9995       return false;
9996 
9997   return true;
9998 }
9999 
10000 static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
10001   return DAG.getDataLayout().getPrefTypeAlign(
10002       VT.getTypeForEVT(*DAG.getContext()));
10003 }
10004 
10005 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
10006 // and output parameter nodes.
10007 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
10008                                        SmallVectorImpl<SDValue> &InVals) const {
10009   SelectionDAG &DAG = CLI.DAG;
10010   SDLoc &DL = CLI.DL;
10011   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
10012   SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
10013   SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
10014   SDValue Chain = CLI.Chain;
10015   SDValue Callee = CLI.Callee;
10016   bool &IsTailCall = CLI.IsTailCall;
10017   CallingConv::ID CallConv = CLI.CallConv;
10018   bool IsVarArg = CLI.IsVarArg;
10019   EVT PtrVT = getPointerTy(DAG.getDataLayout());
10020   MVT XLenVT = Subtarget.getXLenVT();
10021 
10022   MachineFunction &MF = DAG.getMachineFunction();
10023 
10024   // Analyze the operands of the call, assigning locations to each operand.
10025   SmallVector<CCValAssign, 16> ArgLocs;
10026   CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
10027 
10028   if (CallConv == CallingConv::GHC)
10029     ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_GHC);
10030   else
10031     analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI,
10032                       CallConv == CallingConv::Fast ? CC_RISCV_FastCC
10033                                                     : CC_RISCV);
10034 
10035   // Check if it's really possible to do a tail call.
10036   if (IsTailCall)
10037     IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
10038 
10039   if (IsTailCall)
10040     ++NumTailCalls;
10041   else if (CLI.CB && CLI.CB->isMustTailCall())
10042     report_fatal_error("failed to perform tail call elimination on a call "
10043                        "site marked musttail");
10044 
10045   // Get a count of how many bytes are to be pushed on the stack.
10046   unsigned NumBytes = ArgCCInfo.getNextStackOffset();
10047 
10048   // Create local copies for byval args
10049   SmallVector<SDValue, 8> ByValArgs;
10050   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
10051     ISD::ArgFlagsTy Flags = Outs[i].Flags;
10052     if (!Flags.isByVal())
10053       continue;
10054 
10055     SDValue Arg = OutVals[i];
10056     unsigned Size = Flags.getByValSize();
10057     Align Alignment = Flags.getNonZeroByValAlign();
10058 
10059     int FI =
10060         MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false);
10061     SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
10062     SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
10063 
10064     Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
10065                           /*IsVolatile=*/false,
10066                           /*AlwaysInline=*/false, IsTailCall,
10067                           MachinePointerInfo(), MachinePointerInfo());
10068     ByValArgs.push_back(FIPtr);
10069   }
10070 
10071   if (!IsTailCall)
10072     Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
10073 
10074   // Copy argument values to their designated locations.
10075   SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
10076   SmallVector<SDValue, 8> MemOpChains;
10077   SDValue StackPtr;
10078   for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
10079     CCValAssign &VA = ArgLocs[i];
10080     SDValue ArgValue = OutVals[i];
10081     ISD::ArgFlagsTy Flags = Outs[i].Flags;
10082 
10083     // Handle passing f64 on RV32D with a soft float ABI as a special case.
10084     bool IsF64OnRV32DSoftABI =
10085         VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
10086     if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
10087       SDValue SplitF64 = DAG.getNode(
10088           RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
10089       SDValue Lo = SplitF64.getValue(0);
10090       SDValue Hi = SplitF64.getValue(1);
10091 
10092       Register RegLo = VA.getLocReg();
10093       RegsToPass.push_back(std::make_pair(RegLo, Lo));
10094 
10095       if (RegLo == RISCV::X17) {
10096         // Second half of f64 is passed on the stack.
10097         // Work out the address of the stack slot.
10098         if (!StackPtr.getNode())
10099           StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
10100         // Emit the store.
10101         MemOpChains.push_back(
10102             DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
10103       } else {
10104         // Second half of f64 is passed in another GPR.
10105         assert(RegLo < RISCV::X31 && "Invalid register pair");
10106         Register RegHigh = RegLo + 1;
10107         RegsToPass.push_back(std::make_pair(RegHigh, Hi));
10108       }
10109       continue;
10110     }
10111 
10112     // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
10113     // as any other MemLoc.
10114 
10115     // Promote the value if needed.
10116     // For now, only handle fully promoted and indirect arguments.
10117     if (VA.getLocInfo() == CCValAssign::Indirect) {
10118       // Store the argument in a stack slot and pass its address.
10119       Align StackAlign =
10120           std::max(getPrefTypeAlign(Outs[i].ArgVT, DAG),
10121                    getPrefTypeAlign(ArgValue.getValueType(), DAG));
10122       TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
10123       // If the original argument was split (e.g. i128), we need
10124       // to store the required parts of it here (and pass just one address).
10125       // Vectors may be partly split to registers and partly to the stack, in
10126       // which case the base address is partly offset and subsequent stores are
10127       // relative to that.
10128       unsigned ArgIndex = Outs[i].OrigArgIndex;
10129       unsigned ArgPartOffset = Outs[i].PartOffset;
10130       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
10131       // Calculate the total size to store. We don't have access to what we're
10132       // actually storing other than performing the loop and collecting the
10133       // info.
10134       SmallVector<std::pair<SDValue, SDValue>> Parts;
10135       while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
10136         SDValue PartValue = OutVals[i + 1];
10137         unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset;
10138         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
10139         EVT PartVT = PartValue.getValueType();
10140         if (PartVT.isScalableVector())
10141           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
10142         StoredSize += PartVT.getStoreSize();
10143         StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
10144         Parts.push_back(std::make_pair(PartValue, Offset));
10145         ++i;
10146       }
10147       SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
10148       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
10149       MemOpChains.push_back(
10150           DAG.getStore(Chain, DL, ArgValue, SpillSlot,
10151                        MachinePointerInfo::getFixedStack(MF, FI)));
10152       for (const auto &Part : Parts) {
10153         SDValue PartValue = Part.first;
10154         SDValue PartOffset = Part.second;
10155         SDValue Address =
10156             DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
10157         MemOpChains.push_back(
10158             DAG.getStore(Chain, DL, PartValue, Address,
10159                          MachinePointerInfo::getFixedStack(MF, FI)));
10160       }
10161       ArgValue = SpillSlot;
10162     } else {
10163       ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget);
10164     }
10165 
10166     // Use local copy if it is a byval arg.
10167     if (Flags.isByVal())
10168       ArgValue = ByValArgs[j++];
10169 
10170     if (VA.isRegLoc()) {
10171       // Queue up the argument copies and emit them at the end.
10172       RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
10173     } else {
10174       assert(VA.isMemLoc() && "Argument not register or memory");
10175       assert(!IsTailCall && "Tail call not allowed if stack is used "
10176                             "for passing parameters");
10177 
10178       // Work out the address of the stack slot.
10179       if (!StackPtr.getNode())
10180         StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
10181       SDValue Address =
10182           DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
10183                       DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
10184 
10185       // Emit the store.
10186       MemOpChains.push_back(
10187           DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
10188     }
10189   }
10190 
10191   // Join the stores, which are independent of one another.
10192   if (!MemOpChains.empty())
10193     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
10194 
10195   SDValue Glue;
10196 
10197   // Build a sequence of copy-to-reg nodes, chained and glued together.
10198   for (auto &Reg : RegsToPass) {
10199     Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
10200     Glue = Chain.getValue(1);
10201   }
10202 
10203   // Validate that none of the argument registers have been marked as
10204   // reserved, if so report an error. Do the same for the return address if this
10205   // is not a tailcall.
10206   validateCCReservedRegs(RegsToPass, MF);
10207   if (!IsTailCall &&
10208       MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1))
10209     MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
10210         MF.getFunction(),
10211         "Return address register required, but has been reserved."});
10212 
10213   // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
10214   // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
10215   // split it and then direct call can be matched by PseudoCALL.
10216   if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
10217     const GlobalValue *GV = S->getGlobal();
10218 
10219     unsigned OpFlags = RISCVII::MO_CALL;
10220     if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
10221       OpFlags = RISCVII::MO_PLT;
10222 
10223     Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
10224   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
10225     unsigned OpFlags = RISCVII::MO_CALL;
10226 
10227     if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
10228                                                  nullptr))
10229       OpFlags = RISCVII::MO_PLT;
10230 
10231     Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
10232   }
10233 
10234   // The first call operand is the chain and the second is the target address.
10235   SmallVector<SDValue, 8> Ops;
10236   Ops.push_back(Chain);
10237   Ops.push_back(Callee);
10238 
10239   // Add argument registers to the end of the list so that they are
10240   // known live into the call.
10241   for (auto &Reg : RegsToPass)
10242     Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
10243 
10244   if (!IsTailCall) {
10245     // Add a register mask operand representing the call-preserved registers.
10246     const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
10247     const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
10248     assert(Mask && "Missing call preserved mask for calling convention");
10249     Ops.push_back(DAG.getRegisterMask(Mask));
10250   }
10251 
10252   // Glue the call to the argument copies, if any.
10253   if (Glue.getNode())
10254     Ops.push_back(Glue);
10255 
10256   // Emit the call.
10257   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10258 
10259   if (IsTailCall) {
10260     MF.getFrameInfo().setHasTailCall();
10261     return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
10262   }
10263 
10264   Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
10265   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
10266   Glue = Chain.getValue(1);
10267 
10268   // Mark the end of the call, which is glued to the call itself.
10269   Chain = DAG.getCALLSEQ_END(Chain,
10270                              DAG.getConstant(NumBytes, DL, PtrVT, true),
10271                              DAG.getConstant(0, DL, PtrVT, true),
10272                              Glue, DL);
10273   Glue = Chain.getValue(1);
10274 
10275   // Assign locations to each value returned by this call.
10276   SmallVector<CCValAssign, 16> RVLocs;
10277   CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
10278   analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, CC_RISCV);
10279 
10280   // Copy all of the result registers out of their specified physreg.
10281   for (auto &VA : RVLocs) {
10282     // Copy the value out
10283     SDValue RetValue =
10284         DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
10285     // Glue the RetValue to the end of the call sequence
10286     Chain = RetValue.getValue(1);
10287     Glue = RetValue.getValue(2);
10288 
10289     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10290       assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
10291       SDValue RetValue2 =
10292           DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
10293       Chain = RetValue2.getValue(1);
10294       Glue = RetValue2.getValue(2);
10295       RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
10296                              RetValue2);
10297     }
10298 
10299     RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget);
10300 
10301     InVals.push_back(RetValue);
10302   }
10303 
10304   return Chain;
10305 }
10306 
10307 bool RISCVTargetLowering::CanLowerReturn(
10308     CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
10309     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
10310   SmallVector<CCValAssign, 16> RVLocs;
10311   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
10312 
10313   Optional<unsigned> FirstMaskArgument;
10314   if (Subtarget.hasVInstructions())
10315     FirstMaskArgument = preAssignMask(Outs);
10316 
10317   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
10318     MVT VT = Outs[i].VT;
10319     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
10320     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
10321     if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
10322                  ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr,
10323                  *this, FirstMaskArgument))
10324       return false;
10325   }
10326   return true;
10327 }
10328 
10329 SDValue
10330 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
10331                                  bool IsVarArg,
10332                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
10333                                  const SmallVectorImpl<SDValue> &OutVals,
10334                                  const SDLoc &DL, SelectionDAG &DAG) const {
10335   const MachineFunction &MF = DAG.getMachineFunction();
10336   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
10337 
10338   // Stores the assignment of the return value to a location.
10339   SmallVector<CCValAssign, 16> RVLocs;
10340 
10341   // Info about the registers and stack slot.
10342   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
10343                  *DAG.getContext());
10344 
10345   analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
10346                     nullptr, CC_RISCV);
10347 
10348   if (CallConv == CallingConv::GHC && !RVLocs.empty())
10349     report_fatal_error("GHC functions return void only");
10350 
10351   SDValue Glue;
10352   SmallVector<SDValue, 4> RetOps(1, Chain);
10353 
10354   // Copy the result values into the output registers.
10355   for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
10356     SDValue Val = OutVals[i];
10357     CCValAssign &VA = RVLocs[i];
10358     assert(VA.isRegLoc() && "Can only return in registers!");
10359 
10360     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10361       // Handle returning f64 on RV32D with a soft float ABI.
10362       assert(VA.isRegLoc() && "Expected return via registers");
10363       SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
10364                                      DAG.getVTList(MVT::i32, MVT::i32), Val);
10365       SDValue Lo = SplitF64.getValue(0);
10366       SDValue Hi = SplitF64.getValue(1);
10367       Register RegLo = VA.getLocReg();
10368       assert(RegLo < RISCV::X31 && "Invalid register pair");
10369       Register RegHi = RegLo + 1;
10370 
10371       if (STI.isRegisterReservedByUser(RegLo) ||
10372           STI.isRegisterReservedByUser(RegHi))
10373         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
10374             MF.getFunction(),
10375             "Return value register required, but has been reserved."});
10376 
10377       Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
10378       Glue = Chain.getValue(1);
10379       RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
10380       Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
10381       Glue = Chain.getValue(1);
10382       RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
10383     } else {
10384       // Handle a 'normal' return.
10385       Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
10386       Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
10387 
10388       if (STI.isRegisterReservedByUser(VA.getLocReg()))
10389         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
10390             MF.getFunction(),
10391             "Return value register required, but has been reserved."});
10392 
10393       // Guarantee that all emitted copies are stuck together.
10394       Glue = Chain.getValue(1);
10395       RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
10396     }
10397   }
10398 
10399   RetOps[0] = Chain; // Update chain.
10400 
10401   // Add the glue node if we have it.
10402   if (Glue.getNode()) {
10403     RetOps.push_back(Glue);
10404   }
10405 
10406   unsigned RetOpc = RISCVISD::RET_FLAG;
10407   // Interrupt service routines use different return instructions.
10408   const Function &Func = DAG.getMachineFunction().getFunction();
10409   if (Func.hasFnAttribute("interrupt")) {
10410     if (!Func.getReturnType()->isVoidTy())
10411       report_fatal_error(
10412           "Functions with the interrupt attribute must have void return type!");
10413 
10414     MachineFunction &MF = DAG.getMachineFunction();
10415     StringRef Kind =
10416       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
10417 
10418     if (Kind == "user")
10419       RetOpc = RISCVISD::URET_FLAG;
10420     else if (Kind == "supervisor")
10421       RetOpc = RISCVISD::SRET_FLAG;
10422     else
10423       RetOpc = RISCVISD::MRET_FLAG;
10424   }
10425 
10426   return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
10427 }
10428 
10429 void RISCVTargetLowering::validateCCReservedRegs(
10430     const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
10431     MachineFunction &MF) const {
10432   const Function &F = MF.getFunction();
10433   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
10434 
10435   if (llvm::any_of(Regs, [&STI](auto Reg) {
10436         return STI.isRegisterReservedByUser(Reg.first);
10437       }))
10438     F.getContext().diagnose(DiagnosticInfoUnsupported{
10439         F, "Argument register required, but has been reserved."});
10440 }
10441 
10442 bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
10443   return CI->isTailCall();
10444 }
10445 
10446 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
10447 #define NODE_NAME_CASE(NODE)                                                   \
10448   case RISCVISD::NODE:                                                         \
10449     return "RISCVISD::" #NODE;
10450   // clang-format off
10451   switch ((RISCVISD::NodeType)Opcode) {
10452   case RISCVISD::FIRST_NUMBER:
10453     break;
10454   NODE_NAME_CASE(RET_FLAG)
10455   NODE_NAME_CASE(URET_FLAG)
10456   NODE_NAME_CASE(SRET_FLAG)
10457   NODE_NAME_CASE(MRET_FLAG)
10458   NODE_NAME_CASE(CALL)
10459   NODE_NAME_CASE(SELECT_CC)
10460   NODE_NAME_CASE(BR_CC)
10461   NODE_NAME_CASE(BuildPairF64)
10462   NODE_NAME_CASE(SplitF64)
10463   NODE_NAME_CASE(TAIL)
10464   NODE_NAME_CASE(MULHSU)
10465   NODE_NAME_CASE(SLLW)
10466   NODE_NAME_CASE(SRAW)
10467   NODE_NAME_CASE(SRLW)
10468   NODE_NAME_CASE(DIVW)
10469   NODE_NAME_CASE(DIVUW)
10470   NODE_NAME_CASE(REMUW)
10471   NODE_NAME_CASE(ROLW)
10472   NODE_NAME_CASE(RORW)
10473   NODE_NAME_CASE(CLZW)
10474   NODE_NAME_CASE(CTZW)
10475   NODE_NAME_CASE(FSLW)
10476   NODE_NAME_CASE(FSRW)
10477   NODE_NAME_CASE(FSL)
10478   NODE_NAME_CASE(FSR)
10479   NODE_NAME_CASE(FMV_H_X)
10480   NODE_NAME_CASE(FMV_X_ANYEXTH)
10481   NODE_NAME_CASE(FMV_W_X_RV64)
10482   NODE_NAME_CASE(FMV_X_ANYEXTW_RV64)
10483   NODE_NAME_CASE(FCVT_X)
10484   NODE_NAME_CASE(FCVT_XU)
10485   NODE_NAME_CASE(FCVT_W_RV64)
10486   NODE_NAME_CASE(FCVT_WU_RV64)
10487   NODE_NAME_CASE(STRICT_FCVT_W_RV64)
10488   NODE_NAME_CASE(STRICT_FCVT_WU_RV64)
10489   NODE_NAME_CASE(READ_CYCLE_WIDE)
10490   NODE_NAME_CASE(GREV)
10491   NODE_NAME_CASE(GREVW)
10492   NODE_NAME_CASE(GORC)
10493   NODE_NAME_CASE(GORCW)
10494   NODE_NAME_CASE(SHFL)
10495   NODE_NAME_CASE(SHFLW)
10496   NODE_NAME_CASE(UNSHFL)
10497   NODE_NAME_CASE(UNSHFLW)
10498   NODE_NAME_CASE(BFP)
10499   NODE_NAME_CASE(BFPW)
10500   NODE_NAME_CASE(BCOMPRESS)
10501   NODE_NAME_CASE(BCOMPRESSW)
10502   NODE_NAME_CASE(BDECOMPRESS)
10503   NODE_NAME_CASE(BDECOMPRESSW)
10504   NODE_NAME_CASE(VMV_V_X_VL)
10505   NODE_NAME_CASE(VFMV_V_F_VL)
10506   NODE_NAME_CASE(VMV_X_S)
10507   NODE_NAME_CASE(VMV_S_X_VL)
10508   NODE_NAME_CASE(VFMV_S_F_VL)
10509   NODE_NAME_CASE(SPLAT_VECTOR_SPLIT_I64_VL)
10510   NODE_NAME_CASE(READ_VLENB)
10511   NODE_NAME_CASE(TRUNCATE_VECTOR_VL)
10512   NODE_NAME_CASE(VSLIDEUP_VL)
10513   NODE_NAME_CASE(VSLIDE1UP_VL)
10514   NODE_NAME_CASE(VSLIDEDOWN_VL)
10515   NODE_NAME_CASE(VSLIDE1DOWN_VL)
10516   NODE_NAME_CASE(VID_VL)
10517   NODE_NAME_CASE(VFNCVT_ROD_VL)
10518   NODE_NAME_CASE(VECREDUCE_ADD_VL)
10519   NODE_NAME_CASE(VECREDUCE_UMAX_VL)
10520   NODE_NAME_CASE(VECREDUCE_SMAX_VL)
10521   NODE_NAME_CASE(VECREDUCE_UMIN_VL)
10522   NODE_NAME_CASE(VECREDUCE_SMIN_VL)
10523   NODE_NAME_CASE(VECREDUCE_AND_VL)
10524   NODE_NAME_CASE(VECREDUCE_OR_VL)
10525   NODE_NAME_CASE(VECREDUCE_XOR_VL)
10526   NODE_NAME_CASE(VECREDUCE_FADD_VL)
10527   NODE_NAME_CASE(VECREDUCE_SEQ_FADD_VL)
10528   NODE_NAME_CASE(VECREDUCE_FMIN_VL)
10529   NODE_NAME_CASE(VECREDUCE_FMAX_VL)
10530   NODE_NAME_CASE(ADD_VL)
10531   NODE_NAME_CASE(AND_VL)
10532   NODE_NAME_CASE(MUL_VL)
10533   NODE_NAME_CASE(OR_VL)
10534   NODE_NAME_CASE(SDIV_VL)
10535   NODE_NAME_CASE(SHL_VL)
10536   NODE_NAME_CASE(SREM_VL)
10537   NODE_NAME_CASE(SRA_VL)
10538   NODE_NAME_CASE(SRL_VL)
10539   NODE_NAME_CASE(SUB_VL)
10540   NODE_NAME_CASE(UDIV_VL)
10541   NODE_NAME_CASE(UREM_VL)
10542   NODE_NAME_CASE(XOR_VL)
10543   NODE_NAME_CASE(SADDSAT_VL)
10544   NODE_NAME_CASE(UADDSAT_VL)
10545   NODE_NAME_CASE(SSUBSAT_VL)
10546   NODE_NAME_CASE(USUBSAT_VL)
10547   NODE_NAME_CASE(FADD_VL)
10548   NODE_NAME_CASE(FSUB_VL)
10549   NODE_NAME_CASE(FMUL_VL)
10550   NODE_NAME_CASE(FDIV_VL)
10551   NODE_NAME_CASE(FNEG_VL)
10552   NODE_NAME_CASE(FABS_VL)
10553   NODE_NAME_CASE(FSQRT_VL)
10554   NODE_NAME_CASE(FMA_VL)
10555   NODE_NAME_CASE(FCOPYSIGN_VL)
10556   NODE_NAME_CASE(SMIN_VL)
10557   NODE_NAME_CASE(SMAX_VL)
10558   NODE_NAME_CASE(UMIN_VL)
10559   NODE_NAME_CASE(UMAX_VL)
10560   NODE_NAME_CASE(FMINNUM_VL)
10561   NODE_NAME_CASE(FMAXNUM_VL)
10562   NODE_NAME_CASE(MULHS_VL)
10563   NODE_NAME_CASE(MULHU_VL)
10564   NODE_NAME_CASE(FP_TO_SINT_VL)
10565   NODE_NAME_CASE(FP_TO_UINT_VL)
10566   NODE_NAME_CASE(SINT_TO_FP_VL)
10567   NODE_NAME_CASE(UINT_TO_FP_VL)
10568   NODE_NAME_CASE(FP_EXTEND_VL)
10569   NODE_NAME_CASE(FP_ROUND_VL)
10570   NODE_NAME_CASE(VWMUL_VL)
10571   NODE_NAME_CASE(VWMULU_VL)
10572   NODE_NAME_CASE(VWMULSU_VL)
10573   NODE_NAME_CASE(VWADD_VL)
10574   NODE_NAME_CASE(VWADDU_VL)
10575   NODE_NAME_CASE(VWSUB_VL)
10576   NODE_NAME_CASE(VWSUBU_VL)
10577   NODE_NAME_CASE(VWADD_W_VL)
10578   NODE_NAME_CASE(VWADDU_W_VL)
10579   NODE_NAME_CASE(VWSUB_W_VL)
10580   NODE_NAME_CASE(VWSUBU_W_VL)
10581   NODE_NAME_CASE(SETCC_VL)
10582   NODE_NAME_CASE(VSELECT_VL)
10583   NODE_NAME_CASE(VP_MERGE_VL)
10584   NODE_NAME_CASE(VMAND_VL)
10585   NODE_NAME_CASE(VMOR_VL)
10586   NODE_NAME_CASE(VMXOR_VL)
10587   NODE_NAME_CASE(VMCLR_VL)
10588   NODE_NAME_CASE(VMSET_VL)
10589   NODE_NAME_CASE(VRGATHER_VX_VL)
10590   NODE_NAME_CASE(VRGATHER_VV_VL)
10591   NODE_NAME_CASE(VRGATHEREI16_VV_VL)
10592   NODE_NAME_CASE(VSEXT_VL)
10593   NODE_NAME_CASE(VZEXT_VL)
10594   NODE_NAME_CASE(VCPOP_VL)
10595   NODE_NAME_CASE(VLE_VL)
10596   NODE_NAME_CASE(VSE_VL)
10597   NODE_NAME_CASE(READ_CSR)
10598   NODE_NAME_CASE(WRITE_CSR)
10599   NODE_NAME_CASE(SWAP_CSR)
10600   }
10601   // clang-format on
10602   return nullptr;
10603 #undef NODE_NAME_CASE
10604 }
10605 
10606 /// getConstraintType - Given a constraint letter, return the type of
10607 /// constraint it is for this target.
10608 RISCVTargetLowering::ConstraintType
10609 RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
10610   if (Constraint.size() == 1) {
10611     switch (Constraint[0]) {
10612     default:
10613       break;
10614     case 'f':
10615       return C_RegisterClass;
10616     case 'I':
10617     case 'J':
10618     case 'K':
10619       return C_Immediate;
10620     case 'A':
10621       return C_Memory;
10622     case 'S': // A symbolic address
10623       return C_Other;
10624     }
10625   } else {
10626     if (Constraint == "vr" || Constraint == "vm")
10627       return C_RegisterClass;
10628   }
10629   return TargetLowering::getConstraintType(Constraint);
10630 }
10631 
10632 std::pair<unsigned, const TargetRegisterClass *>
10633 RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
10634                                                   StringRef Constraint,
10635                                                   MVT VT) const {
10636   // First, see if this is a constraint that directly corresponds to a
10637   // RISCV register class.
10638   if (Constraint.size() == 1) {
10639     switch (Constraint[0]) {
10640     case 'r':
10641       // TODO: Support fixed vectors up to XLen for P extension?
10642       if (VT.isVector())
10643         break;
10644       return std::make_pair(0U, &RISCV::GPRRegClass);
10645     case 'f':
10646       if (Subtarget.hasStdExtZfh() && VT == MVT::f16)
10647         return std::make_pair(0U, &RISCV::FPR16RegClass);
10648       if (Subtarget.hasStdExtF() && VT == MVT::f32)
10649         return std::make_pair(0U, &RISCV::FPR32RegClass);
10650       if (Subtarget.hasStdExtD() && VT == MVT::f64)
10651         return std::make_pair(0U, &RISCV::FPR64RegClass);
10652       break;
10653     default:
10654       break;
10655     }
10656   } else if (Constraint == "vr") {
10657     for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
10658                            &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10659       if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
10660         return std::make_pair(0U, RC);
10661     }
10662   } else if (Constraint == "vm") {
10663     if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
10664       return std::make_pair(0U, &RISCV::VMV0RegClass);
10665   }
10666 
10667   // Clang will correctly decode the usage of register name aliases into their
10668   // official names. However, other frontends like `rustc` do not. This allows
10669   // users of these frontends to use the ABI names for registers in LLVM-style
10670   // register constraints.
10671   unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
10672                                .Case("{zero}", RISCV::X0)
10673                                .Case("{ra}", RISCV::X1)
10674                                .Case("{sp}", RISCV::X2)
10675                                .Case("{gp}", RISCV::X3)
10676                                .Case("{tp}", RISCV::X4)
10677                                .Case("{t0}", RISCV::X5)
10678                                .Case("{t1}", RISCV::X6)
10679                                .Case("{t2}", RISCV::X7)
10680                                .Cases("{s0}", "{fp}", RISCV::X8)
10681                                .Case("{s1}", RISCV::X9)
10682                                .Case("{a0}", RISCV::X10)
10683                                .Case("{a1}", RISCV::X11)
10684                                .Case("{a2}", RISCV::X12)
10685                                .Case("{a3}", RISCV::X13)
10686                                .Case("{a4}", RISCV::X14)
10687                                .Case("{a5}", RISCV::X15)
10688                                .Case("{a6}", RISCV::X16)
10689                                .Case("{a7}", RISCV::X17)
10690                                .Case("{s2}", RISCV::X18)
10691                                .Case("{s3}", RISCV::X19)
10692                                .Case("{s4}", RISCV::X20)
10693                                .Case("{s5}", RISCV::X21)
10694                                .Case("{s6}", RISCV::X22)
10695                                .Case("{s7}", RISCV::X23)
10696                                .Case("{s8}", RISCV::X24)
10697                                .Case("{s9}", RISCV::X25)
10698                                .Case("{s10}", RISCV::X26)
10699                                .Case("{s11}", RISCV::X27)
10700                                .Case("{t3}", RISCV::X28)
10701                                .Case("{t4}", RISCV::X29)
10702                                .Case("{t5}", RISCV::X30)
10703                                .Case("{t6}", RISCV::X31)
10704                                .Default(RISCV::NoRegister);
10705   if (XRegFromAlias != RISCV::NoRegister)
10706     return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
10707 
10708   // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
10709   // TableGen record rather than the AsmName to choose registers for InlineAsm
10710   // constraints, plus we want to match those names to the widest floating point
10711   // register type available, manually select floating point registers here.
10712   //
10713   // The second case is the ABI name of the register, so that frontends can also
10714   // use the ABI names in register constraint lists.
10715   if (Subtarget.hasStdExtF()) {
10716     unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
10717                         .Cases("{f0}", "{ft0}", RISCV::F0_F)
10718                         .Cases("{f1}", "{ft1}", RISCV::F1_F)
10719                         .Cases("{f2}", "{ft2}", RISCV::F2_F)
10720                         .Cases("{f3}", "{ft3}", RISCV::F3_F)
10721                         .Cases("{f4}", "{ft4}", RISCV::F4_F)
10722                         .Cases("{f5}", "{ft5}", RISCV::F5_F)
10723                         .Cases("{f6}", "{ft6}", RISCV::F6_F)
10724                         .Cases("{f7}", "{ft7}", RISCV::F7_F)
10725                         .Cases("{f8}", "{fs0}", RISCV::F8_F)
10726                         .Cases("{f9}", "{fs1}", RISCV::F9_F)
10727                         .Cases("{f10}", "{fa0}", RISCV::F10_F)
10728                         .Cases("{f11}", "{fa1}", RISCV::F11_F)
10729                         .Cases("{f12}", "{fa2}", RISCV::F12_F)
10730                         .Cases("{f13}", "{fa3}", RISCV::F13_F)
10731                         .Cases("{f14}", "{fa4}", RISCV::F14_F)
10732                         .Cases("{f15}", "{fa5}", RISCV::F15_F)
10733                         .Cases("{f16}", "{fa6}", RISCV::F16_F)
10734                         .Cases("{f17}", "{fa7}", RISCV::F17_F)
10735                         .Cases("{f18}", "{fs2}", RISCV::F18_F)
10736                         .Cases("{f19}", "{fs3}", RISCV::F19_F)
10737                         .Cases("{f20}", "{fs4}", RISCV::F20_F)
10738                         .Cases("{f21}", "{fs5}", RISCV::F21_F)
10739                         .Cases("{f22}", "{fs6}", RISCV::F22_F)
10740                         .Cases("{f23}", "{fs7}", RISCV::F23_F)
10741                         .Cases("{f24}", "{fs8}", RISCV::F24_F)
10742                         .Cases("{f25}", "{fs9}", RISCV::F25_F)
10743                         .Cases("{f26}", "{fs10}", RISCV::F26_F)
10744                         .Cases("{f27}", "{fs11}", RISCV::F27_F)
10745                         .Cases("{f28}", "{ft8}", RISCV::F28_F)
10746                         .Cases("{f29}", "{ft9}", RISCV::F29_F)
10747                         .Cases("{f30}", "{ft10}", RISCV::F30_F)
10748                         .Cases("{f31}", "{ft11}", RISCV::F31_F)
10749                         .Default(RISCV::NoRegister);
10750     if (FReg != RISCV::NoRegister) {
10751       assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
10752       if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
10753         unsigned RegNo = FReg - RISCV::F0_F;
10754         unsigned DReg = RISCV::F0_D + RegNo;
10755         return std::make_pair(DReg, &RISCV::FPR64RegClass);
10756       }
10757       if (VT == MVT::f32 || VT == MVT::Other)
10758         return std::make_pair(FReg, &RISCV::FPR32RegClass);
10759       if (Subtarget.hasStdExtZfh() && VT == MVT::f16) {
10760         unsigned RegNo = FReg - RISCV::F0_F;
10761         unsigned HReg = RISCV::F0_H + RegNo;
10762         return std::make_pair(HReg, &RISCV::FPR16RegClass);
10763       }
10764     }
10765   }
10766 
10767   if (Subtarget.hasVInstructions()) {
10768     Register VReg = StringSwitch<Register>(Constraint.lower())
10769                         .Case("{v0}", RISCV::V0)
10770                         .Case("{v1}", RISCV::V1)
10771                         .Case("{v2}", RISCV::V2)
10772                         .Case("{v3}", RISCV::V3)
10773                         .Case("{v4}", RISCV::V4)
10774                         .Case("{v5}", RISCV::V5)
10775                         .Case("{v6}", RISCV::V6)
10776                         .Case("{v7}", RISCV::V7)
10777                         .Case("{v8}", RISCV::V8)
10778                         .Case("{v9}", RISCV::V9)
10779                         .Case("{v10}", RISCV::V10)
10780                         .Case("{v11}", RISCV::V11)
10781                         .Case("{v12}", RISCV::V12)
10782                         .Case("{v13}", RISCV::V13)
10783                         .Case("{v14}", RISCV::V14)
10784                         .Case("{v15}", RISCV::V15)
10785                         .Case("{v16}", RISCV::V16)
10786                         .Case("{v17}", RISCV::V17)
10787                         .Case("{v18}", RISCV::V18)
10788                         .Case("{v19}", RISCV::V19)
10789                         .Case("{v20}", RISCV::V20)
10790                         .Case("{v21}", RISCV::V21)
10791                         .Case("{v22}", RISCV::V22)
10792                         .Case("{v23}", RISCV::V23)
10793                         .Case("{v24}", RISCV::V24)
10794                         .Case("{v25}", RISCV::V25)
10795                         .Case("{v26}", RISCV::V26)
10796                         .Case("{v27}", RISCV::V27)
10797                         .Case("{v28}", RISCV::V28)
10798                         .Case("{v29}", RISCV::V29)
10799                         .Case("{v30}", RISCV::V30)
10800                         .Case("{v31}", RISCV::V31)
10801                         .Default(RISCV::NoRegister);
10802     if (VReg != RISCV::NoRegister) {
10803       if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
10804         return std::make_pair(VReg, &RISCV::VMRegClass);
10805       if (TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.SimpleTy))
10806         return std::make_pair(VReg, &RISCV::VRRegClass);
10807       for (const auto *RC :
10808            {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10809         if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) {
10810           VReg = TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
10811           return std::make_pair(VReg, RC);
10812         }
10813       }
10814     }
10815   }
10816 
10817   return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
10818 }
10819 
10820 unsigned
10821 RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
10822   // Currently only support length 1 constraints.
10823   if (ConstraintCode.size() == 1) {
10824     switch (ConstraintCode[0]) {
10825     case 'A':
10826       return InlineAsm::Constraint_A;
10827     default:
10828       break;
10829     }
10830   }
10831 
10832   return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
10833 }
10834 
10835 void RISCVTargetLowering::LowerAsmOperandForConstraint(
10836     SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
10837     SelectionDAG &DAG) const {
10838   // Currently only support length 1 constraints.
10839   if (Constraint.length() == 1) {
10840     switch (Constraint[0]) {
10841     case 'I':
10842       // Validate & create a 12-bit signed immediate operand.
10843       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10844         uint64_t CVal = C->getSExtValue();
10845         if (isInt<12>(CVal))
10846           Ops.push_back(
10847               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10848       }
10849       return;
10850     case 'J':
10851       // Validate & create an integer zero operand.
10852       if (auto *C = dyn_cast<ConstantSDNode>(Op))
10853         if (C->getZExtValue() == 0)
10854           Ops.push_back(
10855               DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
10856       return;
10857     case 'K':
10858       // Validate & create a 5-bit unsigned immediate operand.
10859       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10860         uint64_t CVal = C->getZExtValue();
10861         if (isUInt<5>(CVal))
10862           Ops.push_back(
10863               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10864       }
10865       return;
10866     case 'S':
10867       if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
10868         Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
10869                                                  GA->getValueType(0)));
10870       } else if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
10871         Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(),
10872                                                 BA->getValueType(0)));
10873       }
10874       return;
10875     default:
10876       break;
10877     }
10878   }
10879   TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
10880 }
10881 
10882 Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
10883                                                    Instruction *Inst,
10884                                                    AtomicOrdering Ord) const {
10885   if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
10886     return Builder.CreateFence(Ord);
10887   if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
10888     return Builder.CreateFence(AtomicOrdering::Release);
10889   return nullptr;
10890 }
10891 
10892 Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
10893                                                     Instruction *Inst,
10894                                                     AtomicOrdering Ord) const {
10895   if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
10896     return Builder.CreateFence(AtomicOrdering::Acquire);
10897   return nullptr;
10898 }
10899 
10900 TargetLowering::AtomicExpansionKind
10901 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
10902   // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
10903   // point operations can't be used in an lr/sc sequence without breaking the
10904   // forward-progress guarantee.
10905   if (AI->isFloatingPointOperation())
10906     return AtomicExpansionKind::CmpXChg;
10907 
10908   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
10909   if (Size == 8 || Size == 16)
10910     return AtomicExpansionKind::MaskedIntrinsic;
10911   return AtomicExpansionKind::None;
10912 }
10913 
10914 static Intrinsic::ID
10915 getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
10916   if (XLen == 32) {
10917     switch (BinOp) {
10918     default:
10919       llvm_unreachable("Unexpected AtomicRMW BinOp");
10920     case AtomicRMWInst::Xchg:
10921       return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
10922     case AtomicRMWInst::Add:
10923       return Intrinsic::riscv_masked_atomicrmw_add_i32;
10924     case AtomicRMWInst::Sub:
10925       return Intrinsic::riscv_masked_atomicrmw_sub_i32;
10926     case AtomicRMWInst::Nand:
10927       return Intrinsic::riscv_masked_atomicrmw_nand_i32;
10928     case AtomicRMWInst::Max:
10929       return Intrinsic::riscv_masked_atomicrmw_max_i32;
10930     case AtomicRMWInst::Min:
10931       return Intrinsic::riscv_masked_atomicrmw_min_i32;
10932     case AtomicRMWInst::UMax:
10933       return Intrinsic::riscv_masked_atomicrmw_umax_i32;
10934     case AtomicRMWInst::UMin:
10935       return Intrinsic::riscv_masked_atomicrmw_umin_i32;
10936     }
10937   }
10938 
10939   if (XLen == 64) {
10940     switch (BinOp) {
10941     default:
10942       llvm_unreachable("Unexpected AtomicRMW BinOp");
10943     case AtomicRMWInst::Xchg:
10944       return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
10945     case AtomicRMWInst::Add:
10946       return Intrinsic::riscv_masked_atomicrmw_add_i64;
10947     case AtomicRMWInst::Sub:
10948       return Intrinsic::riscv_masked_atomicrmw_sub_i64;
10949     case AtomicRMWInst::Nand:
10950       return Intrinsic::riscv_masked_atomicrmw_nand_i64;
10951     case AtomicRMWInst::Max:
10952       return Intrinsic::riscv_masked_atomicrmw_max_i64;
10953     case AtomicRMWInst::Min:
10954       return Intrinsic::riscv_masked_atomicrmw_min_i64;
10955     case AtomicRMWInst::UMax:
10956       return Intrinsic::riscv_masked_atomicrmw_umax_i64;
10957     case AtomicRMWInst::UMin:
10958       return Intrinsic::riscv_masked_atomicrmw_umin_i64;
10959     }
10960   }
10961 
10962   llvm_unreachable("Unexpected XLen\n");
10963 }
10964 
10965 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
10966     IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
10967     Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
10968   unsigned XLen = Subtarget.getXLen();
10969   Value *Ordering =
10970       Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
10971   Type *Tys[] = {AlignedAddr->getType()};
10972   Function *LrwOpScwLoop = Intrinsic::getDeclaration(
10973       AI->getModule(),
10974       getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
10975 
10976   if (XLen == 64) {
10977     Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
10978     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10979     ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
10980   }
10981 
10982   Value *Result;
10983 
10984   // Must pass the shift amount needed to sign extend the loaded value prior
10985   // to performing a signed comparison for min/max. ShiftAmt is the number of
10986   // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
10987   // is the number of bits to left+right shift the value in order to
10988   // sign-extend.
10989   if (AI->getOperation() == AtomicRMWInst::Min ||
10990       AI->getOperation() == AtomicRMWInst::Max) {
10991     const DataLayout &DL = AI->getModule()->getDataLayout();
10992     unsigned ValWidth =
10993         DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
10994     Value *SextShamt =
10995         Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
10996     Result = Builder.CreateCall(LrwOpScwLoop,
10997                                 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
10998   } else {
10999     Result =
11000         Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
11001   }
11002 
11003   if (XLen == 64)
11004     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
11005   return Result;
11006 }
11007 
11008 TargetLowering::AtomicExpansionKind
11009 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
11010     AtomicCmpXchgInst *CI) const {
11011   unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
11012   if (Size == 8 || Size == 16)
11013     return AtomicExpansionKind::MaskedIntrinsic;
11014   return AtomicExpansionKind::None;
11015 }
11016 
11017 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
11018     IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
11019     Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
11020   unsigned XLen = Subtarget.getXLen();
11021   Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
11022   Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
11023   if (XLen == 64) {
11024     CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
11025     NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
11026     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
11027     CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
11028   }
11029   Type *Tys[] = {AlignedAddr->getType()};
11030   Function *MaskedCmpXchg =
11031       Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
11032   Value *Result = Builder.CreateCall(
11033       MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
11034   if (XLen == 64)
11035     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
11036   return Result;
11037 }
11038 
11039 bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
11040   return false;
11041 }
11042 
11043 bool RISCVTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
11044                                                EVT VT) const {
11045   if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
11046     return false;
11047 
11048   switch (FPVT.getSimpleVT().SimpleTy) {
11049   case MVT::f16:
11050     return Subtarget.hasStdExtZfh();
11051   case MVT::f32:
11052     return Subtarget.hasStdExtF();
11053   case MVT::f64:
11054     return Subtarget.hasStdExtD();
11055   default:
11056     return false;
11057   }
11058 }
11059 
11060 unsigned RISCVTargetLowering::getJumpTableEncoding() const {
11061   // If we are using the small code model, we can reduce size of jump table
11062   // entry to 4 bytes.
11063   if (Subtarget.is64Bit() && !isPositionIndependent() &&
11064       getTargetMachine().getCodeModel() == CodeModel::Small) {
11065     return MachineJumpTableInfo::EK_Custom32;
11066   }
11067   return TargetLowering::getJumpTableEncoding();
11068 }
11069 
11070 const MCExpr *RISCVTargetLowering::LowerCustomJumpTableEntry(
11071     const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
11072     unsigned uid, MCContext &Ctx) const {
11073   assert(Subtarget.is64Bit() && !isPositionIndependent() &&
11074          getTargetMachine().getCodeModel() == CodeModel::Small);
11075   return MCSymbolRefExpr::create(MBB->getSymbol(), Ctx);
11076 }
11077 
11078 bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
11079                                                      EVT VT) const {
11080   VT = VT.getScalarType();
11081 
11082   if (!VT.isSimple())
11083     return false;
11084 
11085   switch (VT.getSimpleVT().SimpleTy) {
11086   case MVT::f16:
11087     return Subtarget.hasStdExtZfh();
11088   case MVT::f32:
11089     return Subtarget.hasStdExtF();
11090   case MVT::f64:
11091     return Subtarget.hasStdExtD();
11092   default:
11093     break;
11094   }
11095 
11096   return false;
11097 }
11098 
11099 Register RISCVTargetLowering::getExceptionPointerRegister(
11100     const Constant *PersonalityFn) const {
11101   return RISCV::X10;
11102 }
11103 
11104 Register RISCVTargetLowering::getExceptionSelectorRegister(
11105     const Constant *PersonalityFn) const {
11106   return RISCV::X11;
11107 }
11108 
11109 bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
11110   // Return false to suppress the unnecessary extensions if the LibCall
11111   // arguments or return value is f32 type for LP64 ABI.
11112   RISCVABI::ABI ABI = Subtarget.getTargetABI();
11113   if (ABI == RISCVABI::ABI_LP64 && (Type == MVT::f32))
11114     return false;
11115 
11116   return true;
11117 }
11118 
11119 bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
11120   if (Subtarget.is64Bit() && Type == MVT::i32)
11121     return true;
11122 
11123   return IsSigned;
11124 }
11125 
11126 bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
11127                                                  SDValue C) const {
11128   // Check integral scalar types.
11129   if (VT.isScalarInteger()) {
11130     // Omit the optimization if the sub target has the M extension and the data
11131     // size exceeds XLen.
11132     if (Subtarget.hasStdExtM() && VT.getSizeInBits() > Subtarget.getXLen())
11133       return false;
11134     if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
11135       // Break the MUL to a SLLI and an ADD/SUB.
11136       const APInt &Imm = ConstNode->getAPIntValue();
11137       if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
11138           (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
11139         return true;
11140       // Optimize the MUL to (SH*ADD x, (SLLI x, bits)) if Imm is not simm12.
11141       if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
11142           ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
11143            (Imm - 8).isPowerOf2()))
11144         return true;
11145       // Omit the following optimization if the sub target has the M extension
11146       // and the data size >= XLen.
11147       if (Subtarget.hasStdExtM() && VT.getSizeInBits() >= Subtarget.getXLen())
11148         return false;
11149       // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
11150       // a pair of LUI/ADDI.
11151       if (!Imm.isSignedIntN(12) && Imm.countTrailingZeros() < 12) {
11152         APInt ImmS = Imm.ashr(Imm.countTrailingZeros());
11153         if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
11154             (1 - ImmS).isPowerOf2())
11155         return true;
11156       }
11157     }
11158   }
11159 
11160   return false;
11161 }
11162 
11163 bool RISCVTargetLowering::isMulAddWithConstProfitable(
11164     const SDValue &AddNode, const SDValue &ConstNode) const {
11165   // Let the DAGCombiner decide for vectors.
11166   EVT VT = AddNode.getValueType();
11167   if (VT.isVector())
11168     return true;
11169 
11170   // Let the DAGCombiner decide for larger types.
11171   if (VT.getScalarSizeInBits() > Subtarget.getXLen())
11172     return true;
11173 
11174   // It is worse if c1 is simm12 while c1*c2 is not.
11175   ConstantSDNode *C1Node = cast<ConstantSDNode>(AddNode.getOperand(1));
11176   ConstantSDNode *C2Node = cast<ConstantSDNode>(ConstNode);
11177   const APInt &C1 = C1Node->getAPIntValue();
11178   const APInt &C2 = C2Node->getAPIntValue();
11179   if (C1.isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
11180     return false;
11181 
11182   // Default to true and let the DAGCombiner decide.
11183   return true;
11184 }
11185 
11186 bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
11187     EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
11188     bool *Fast) const {
11189   if (!VT.isVector())
11190     return false;
11191 
11192   EVT ElemVT = VT.getVectorElementType();
11193   if (Alignment >= ElemVT.getStoreSize()) {
11194     if (Fast)
11195       *Fast = true;
11196     return true;
11197   }
11198 
11199   return false;
11200 }
11201 
11202 bool RISCVTargetLowering::splitValueIntoRegisterParts(
11203     SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
11204     unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {
11205   bool IsABIRegCopy = CC.hasValue();
11206   EVT ValueVT = Val.getValueType();
11207   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
11208     // Cast the f16 to i16, extend to i32, pad with ones to make a float nan,
11209     // and cast to f32.
11210     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
11211     Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
11212     Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
11213                       DAG.getConstant(0xFFFF0000, DL, MVT::i32));
11214     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
11215     Parts[0] = Val;
11216     return true;
11217   }
11218 
11219   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
11220     LLVMContext &Context = *DAG.getContext();
11221     EVT ValueEltVT = ValueVT.getVectorElementType();
11222     EVT PartEltVT = PartVT.getVectorElementType();
11223     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
11224     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
11225     if (PartVTBitSize % ValueVTBitSize == 0) {
11226       assert(PartVTBitSize >= ValueVTBitSize);
11227       // If the element types are different, bitcast to the same element type of
11228       // PartVT first.
11229       // Give an example here, we want copy a <vscale x 1 x i8> value to
11230       // <vscale x 4 x i16>.
11231       // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert
11232       // subvector, then we can bitcast to <vscale x 4 x i16>.
11233       if (ValueEltVT != PartEltVT) {
11234         if (PartVTBitSize > ValueVTBitSize) {
11235           unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
11236           assert(Count != 0 && "The number of element should not be zero.");
11237           EVT SameEltTypeVT =
11238               EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
11239           Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT,
11240                             DAG.getUNDEF(SameEltTypeVT), Val,
11241                             DAG.getVectorIdxConstant(0, DL));
11242         }
11243         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
11244       } else {
11245         Val =
11246             DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
11247                         Val, DAG.getVectorIdxConstant(0, DL));
11248       }
11249       Parts[0] = Val;
11250       return true;
11251     }
11252   }
11253   return false;
11254 }
11255 
11256 SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
11257     SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
11258     MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {
11259   bool IsABIRegCopy = CC.hasValue();
11260   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
11261     SDValue Val = Parts[0];
11262 
11263     // Cast the f32 to i32, truncate to i16, and cast back to f16.
11264     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
11265     Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
11266     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f16, Val);
11267     return Val;
11268   }
11269 
11270   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
11271     LLVMContext &Context = *DAG.getContext();
11272     SDValue Val = Parts[0];
11273     EVT ValueEltVT = ValueVT.getVectorElementType();
11274     EVT PartEltVT = PartVT.getVectorElementType();
11275     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
11276     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
11277     if (PartVTBitSize % ValueVTBitSize == 0) {
11278       assert(PartVTBitSize >= ValueVTBitSize);
11279       EVT SameEltTypeVT = ValueVT;
11280       // If the element types are different, convert it to the same element type
11281       // of PartVT.
11282       // Give an example here, we want copy a <vscale x 1 x i8> value from
11283       // <vscale x 4 x i16>.
11284       // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first,
11285       // then we can extract <vscale x 1 x i8>.
11286       if (ValueEltVT != PartEltVT) {
11287         unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
11288         assert(Count != 0 && "The number of element should not be zero.");
11289         SameEltTypeVT =
11290             EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
11291         Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
11292       }
11293       Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
11294                         DAG.getVectorIdxConstant(0, DL));
11295       return Val;
11296     }
11297   }
11298   return SDValue();
11299 }
11300 
11301 SDValue
11302 RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
11303                                    SelectionDAG &DAG,
11304                                    SmallVectorImpl<SDNode *> &Created) const {
11305   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
11306   if (isIntDivCheap(N->getValueType(0), Attr))
11307     return SDValue(N, 0); // Lower SDIV as SDIV
11308 
11309   assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
11310          "Unexpected divisor!");
11311 
11312   // Conditional move is needed, so do the transformation iff Zbt is enabled.
11313   if (!Subtarget.hasStdExtZbt())
11314     return SDValue();
11315 
11316   // When |Divisor| >= 2 ^ 12, it isn't profitable to do such transformation.
11317   // Besides, more critical path instructions will be generated when dividing
11318   // by 2. So we keep using the original DAGs for these cases.
11319   unsigned Lg2 = Divisor.countTrailingZeros();
11320   if (Lg2 == 1 || Lg2 >= 12)
11321     return SDValue();
11322 
11323   // fold (sdiv X, pow2)
11324   EVT VT = N->getValueType(0);
11325   if (VT != MVT::i32 && !(Subtarget.is64Bit() && VT == MVT::i64))
11326     return SDValue();
11327 
11328   SDLoc DL(N);
11329   SDValue N0 = N->getOperand(0);
11330   SDValue Zero = DAG.getConstant(0, DL, VT);
11331   SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
11332 
11333   // Add (N0 < 0) ? Pow2 - 1 : 0;
11334   SDValue Cmp = DAG.getSetCC(DL, VT, N0, Zero, ISD::SETLT);
11335   SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
11336   SDValue Sel = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
11337 
11338   Created.push_back(Cmp.getNode());
11339   Created.push_back(Add.getNode());
11340   Created.push_back(Sel.getNode());
11341 
11342   // Divide by pow2.
11343   SDValue SRA =
11344       DAG.getNode(ISD::SRA, DL, VT, Sel, DAG.getConstant(Lg2, DL, VT));
11345 
11346   // If we're dividing by a positive value, we're done.  Otherwise, we must
11347   // negate the result.
11348   if (Divisor.isNonNegative())
11349     return SRA;
11350 
11351   Created.push_back(SRA.getNode());
11352   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
11353 }
11354 
11355 #define GET_REGISTER_MATCHER
11356 #include "RISCVGenAsmMatcher.inc"
11357 
11358 Register
11359 RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
11360                                        const MachineFunction &MF) const {
11361   Register Reg = MatchRegisterAltName(RegName);
11362   if (Reg == RISCV::NoRegister)
11363     Reg = MatchRegisterName(RegName);
11364   if (Reg == RISCV::NoRegister)
11365     report_fatal_error(
11366         Twine("Invalid register name \"" + StringRef(RegName) + "\"."));
11367   BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
11368   if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg))
11369     report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
11370                              StringRef(RegName) + "\"."));
11371   return Reg;
11372 }
11373 
11374 namespace llvm {
11375 namespace RISCVVIntrinsicsTable {
11376 
11377 #define GET_RISCVVIntrinsicsTable_IMPL
11378 #include "RISCVGenSearchableTables.inc"
11379 
11380 } // namespace RISCVVIntrinsicsTable
11381 
11382 } // namespace llvm
11383