1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that RISCV uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "RISCVISelLowering.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCV.h"
17 #include "RISCVMachineFunctionInfo.h"
18 #include "RISCVRegisterInfo.h"
19 #include "RISCVSubtarget.h"
20 #include "RISCVTargetMachine.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/MemoryLocation.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineJumpTableInfo.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
30 #include "llvm/CodeGen/ValueTypes.h"
31 #include "llvm/IR/DiagnosticInfo.h"
32 #include "llvm/IR/DiagnosticPrinter.h"
33 #include "llvm/IR/IRBuilder.h"
34 #include "llvm/IR/IntrinsicsRISCV.h"
35 #include "llvm/IR/PatternMatch.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/KnownBits.h"
39 #include "llvm/Support/MathExtras.h"
40 #include "llvm/Support/raw_ostream.h"
41 
42 using namespace llvm;
43 
44 #define DEBUG_TYPE "riscv-lower"
45 
46 STATISTIC(NumTailCalls, "Number of tail calls");
47 
48 RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
49                                          const RISCVSubtarget &STI)
50     : TargetLowering(TM), Subtarget(STI) {
51 
52   if (Subtarget.isRV32E())
53     report_fatal_error("Codegen not yet implemented for RV32E");
54 
55   RISCVABI::ABI ABI = Subtarget.getTargetABI();
56   assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
57 
58   if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
59       !Subtarget.hasStdExtF()) {
60     errs() << "Hard-float 'f' ABI can't be used for a target that "
61                 "doesn't support the F instruction set extension (ignoring "
62                           "target-abi)\n";
63     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
64   } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
65              !Subtarget.hasStdExtD()) {
66     errs() << "Hard-float 'd' ABI can't be used for a target that "
67               "doesn't support the D instruction set extension (ignoring "
68               "target-abi)\n";
69     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
70   }
71 
72   switch (ABI) {
73   default:
74     report_fatal_error("Don't know how to lower this ABI");
75   case RISCVABI::ABI_ILP32:
76   case RISCVABI::ABI_ILP32F:
77   case RISCVABI::ABI_ILP32D:
78   case RISCVABI::ABI_LP64:
79   case RISCVABI::ABI_LP64F:
80   case RISCVABI::ABI_LP64D:
81     break;
82   }
83 
84   MVT XLenVT = Subtarget.getXLenVT();
85 
86   // Set up the register classes.
87   addRegisterClass(XLenVT, &RISCV::GPRRegClass);
88 
89   if (Subtarget.hasStdExtZfh())
90     addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
91   if (Subtarget.hasStdExtF())
92     addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
93   if (Subtarget.hasStdExtD())
94     addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
95 
96   static const MVT::SimpleValueType BoolVecVTs[] = {
97       MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,
98       MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
99   static const MVT::SimpleValueType IntVecVTs[] = {
100       MVT::nxv1i8,  MVT::nxv2i8,   MVT::nxv4i8,   MVT::nxv8i8,  MVT::nxv16i8,
101       MVT::nxv32i8, MVT::nxv64i8,  MVT::nxv1i16,  MVT::nxv2i16, MVT::nxv4i16,
102       MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
103       MVT::nxv4i32, MVT::nxv8i32,  MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
104       MVT::nxv4i64, MVT::nxv8i64};
105   static const MVT::SimpleValueType F16VecVTs[] = {
106       MVT::nxv1f16, MVT::nxv2f16,  MVT::nxv4f16,
107       MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
108   static const MVT::SimpleValueType F32VecVTs[] = {
109       MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
110   static const MVT::SimpleValueType F64VecVTs[] = {
111       MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
112 
113   if (Subtarget.hasVInstructions()) {
114     auto addRegClassForRVV = [this](MVT VT) {
115       unsigned Size = VT.getSizeInBits().getKnownMinValue();
116       assert(Size <= 512 && isPowerOf2_32(Size));
117       const TargetRegisterClass *RC;
118       if (Size <= 64)
119         RC = &RISCV::VRRegClass;
120       else if (Size == 128)
121         RC = &RISCV::VRM2RegClass;
122       else if (Size == 256)
123         RC = &RISCV::VRM4RegClass;
124       else
125         RC = &RISCV::VRM8RegClass;
126 
127       addRegisterClass(VT, RC);
128     };
129 
130     for (MVT VT : BoolVecVTs)
131       addRegClassForRVV(VT);
132     for (MVT VT : IntVecVTs) {
133       if (VT.getVectorElementType() == MVT::i64 &&
134           !Subtarget.hasVInstructionsI64())
135         continue;
136       addRegClassForRVV(VT);
137     }
138 
139     if (Subtarget.hasVInstructionsF16())
140       for (MVT VT : F16VecVTs)
141         addRegClassForRVV(VT);
142 
143     if (Subtarget.hasVInstructionsF32())
144       for (MVT VT : F32VecVTs)
145         addRegClassForRVV(VT);
146 
147     if (Subtarget.hasVInstructionsF64())
148       for (MVT VT : F64VecVTs)
149         addRegClassForRVV(VT);
150 
151     if (Subtarget.useRVVForFixedLengthVectors()) {
152       auto addRegClassForFixedVectors = [this](MVT VT) {
153         MVT ContainerVT = getContainerForFixedLengthVector(VT);
154         unsigned RCID = getRegClassIDForVecVT(ContainerVT);
155         const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
156         addRegisterClass(VT, TRI.getRegClass(RCID));
157       };
158       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
159         if (useRVVForFixedLengthVectorVT(VT))
160           addRegClassForFixedVectors(VT);
161 
162       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
163         if (useRVVForFixedLengthVectorVT(VT))
164           addRegClassForFixedVectors(VT);
165     }
166   }
167 
168   // Compute derived properties from the register classes.
169   computeRegisterProperties(STI.getRegisterInfo());
170 
171   setStackPointerRegisterToSaveRestore(RISCV::X2);
172 
173   for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
174     setLoadExtAction(N, XLenVT, MVT::i1, Promote);
175 
176   // TODO: add all necessary setOperationAction calls.
177   setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
178 
179   setOperationAction(ISD::BR_JT, MVT::Other, Expand);
180   setOperationAction(ISD::BR_CC, XLenVT, Expand);
181   setOperationAction(ISD::BRCOND, MVT::Other, Custom);
182   setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
183 
184   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
185   setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
186 
187   setOperationAction(ISD::VASTART, MVT::Other, Custom);
188   setOperationAction(ISD::VAARG, MVT::Other, Expand);
189   setOperationAction(ISD::VACOPY, MVT::Other, Expand);
190   setOperationAction(ISD::VAEND, MVT::Other, Expand);
191 
192   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
193   if (!Subtarget.hasStdExtZbb()) {
194     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
195     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
196   }
197 
198   if (Subtarget.is64Bit()) {
199     setOperationAction(ISD::ADD, MVT::i32, Custom);
200     setOperationAction(ISD::SUB, MVT::i32, Custom);
201     setOperationAction(ISD::SHL, MVT::i32, Custom);
202     setOperationAction(ISD::SRA, MVT::i32, Custom);
203     setOperationAction(ISD::SRL, MVT::i32, Custom);
204 
205     setOperationAction(ISD::UADDO, MVT::i32, Custom);
206     setOperationAction(ISD::USUBO, MVT::i32, Custom);
207     setOperationAction(ISD::UADDSAT, MVT::i32, Custom);
208     setOperationAction(ISD::USUBSAT, MVT::i32, Custom);
209   } else {
210     setLibcallName(RTLIB::SHL_I128, nullptr);
211     setLibcallName(RTLIB::SRL_I128, nullptr);
212     setLibcallName(RTLIB::SRA_I128, nullptr);
213     setLibcallName(RTLIB::MUL_I128, nullptr);
214     setLibcallName(RTLIB::MULO_I64, nullptr);
215   }
216 
217   if (!Subtarget.hasStdExtM()) {
218     setOperationAction(ISD::MUL, XLenVT, Expand);
219     setOperationAction(ISD::MULHS, XLenVT, Expand);
220     setOperationAction(ISD::MULHU, XLenVT, Expand);
221     setOperationAction(ISD::SDIV, XLenVT, Expand);
222     setOperationAction(ISD::UDIV, XLenVT, Expand);
223     setOperationAction(ISD::SREM, XLenVT, Expand);
224     setOperationAction(ISD::UREM, XLenVT, Expand);
225   } else {
226     if (Subtarget.is64Bit()) {
227       setOperationAction(ISD::MUL, MVT::i32, Custom);
228       setOperationAction(ISD::MUL, MVT::i128, Custom);
229 
230       setOperationAction(ISD::SDIV, MVT::i8, Custom);
231       setOperationAction(ISD::UDIV, MVT::i8, Custom);
232       setOperationAction(ISD::UREM, MVT::i8, Custom);
233       setOperationAction(ISD::SDIV, MVT::i16, Custom);
234       setOperationAction(ISD::UDIV, MVT::i16, Custom);
235       setOperationAction(ISD::UREM, MVT::i16, Custom);
236       setOperationAction(ISD::SDIV, MVT::i32, Custom);
237       setOperationAction(ISD::UDIV, MVT::i32, Custom);
238       setOperationAction(ISD::UREM, MVT::i32, Custom);
239     } else {
240       setOperationAction(ISD::MUL, MVT::i64, Custom);
241     }
242   }
243 
244   setOperationAction(ISD::SDIVREM, XLenVT, Expand);
245   setOperationAction(ISD::UDIVREM, XLenVT, Expand);
246   setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
247   setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
248 
249   setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
250   setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
251   setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
252 
253   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp() ||
254       Subtarget.hasStdExtZbkb()) {
255     if (Subtarget.is64Bit()) {
256       setOperationAction(ISD::ROTL, MVT::i32, Custom);
257       setOperationAction(ISD::ROTR, MVT::i32, Custom);
258     }
259   } else {
260     setOperationAction(ISD::ROTL, XLenVT, Expand);
261     setOperationAction(ISD::ROTR, XLenVT, Expand);
262   }
263 
264   if (Subtarget.hasStdExtZbp()) {
265     // Custom lower bswap/bitreverse so we can convert them to GREVI to enable
266     // more combining.
267     setOperationAction(ISD::BITREVERSE, XLenVT,   Custom);
268     setOperationAction(ISD::BSWAP,      XLenVT,   Custom);
269     setOperationAction(ISD::BITREVERSE, MVT::i8,  Custom);
270     // BSWAP i8 doesn't exist.
271     setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
272     setOperationAction(ISD::BSWAP,      MVT::i16, Custom);
273 
274     if (Subtarget.is64Bit()) {
275       setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
276       setOperationAction(ISD::BSWAP,      MVT::i32, Custom);
277     }
278   } else {
279     // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
280     // pattern match it directly in isel.
281     setOperationAction(ISD::BSWAP, XLenVT,
282                        (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb())
283                            ? Legal
284                            : Expand);
285   }
286 
287   if (Subtarget.hasStdExtZbb()) {
288     setOperationAction(ISD::SMIN, XLenVT, Legal);
289     setOperationAction(ISD::SMAX, XLenVT, Legal);
290     setOperationAction(ISD::UMIN, XLenVT, Legal);
291     setOperationAction(ISD::UMAX, XLenVT, Legal);
292 
293     if (Subtarget.is64Bit()) {
294       setOperationAction(ISD::CTTZ, MVT::i32, Custom);
295       setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom);
296       setOperationAction(ISD::CTLZ, MVT::i32, Custom);
297       setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom);
298     }
299   } else {
300     setOperationAction(ISD::CTTZ, XLenVT, Expand);
301     setOperationAction(ISD::CTLZ, XLenVT, Expand);
302     setOperationAction(ISD::CTPOP, XLenVT, Expand);
303   }
304 
305   if (Subtarget.hasStdExtZbt()) {
306     setOperationAction(ISD::FSHL, XLenVT, Custom);
307     setOperationAction(ISD::FSHR, XLenVT, Custom);
308     setOperationAction(ISD::SELECT, XLenVT, Legal);
309 
310     if (Subtarget.is64Bit()) {
311       setOperationAction(ISD::FSHL, MVT::i32, Custom);
312       setOperationAction(ISD::FSHR, MVT::i32, Custom);
313     }
314   } else {
315     setOperationAction(ISD::SELECT, XLenVT, Custom);
316   }
317 
318   static const ISD::CondCode FPCCToExpand[] = {
319       ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
320       ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
321       ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};
322 
323   static const ISD::NodeType FPOpToExpand[] = {
324       ISD::FSIN, ISD::FCOS,       ISD::FSINCOS,   ISD::FPOW,
325       ISD::FREM, ISD::FP16_TO_FP, ISD::FP_TO_FP16};
326 
327   if (Subtarget.hasStdExtZfh())
328     setOperationAction(ISD::BITCAST, MVT::i16, Custom);
329 
330   if (Subtarget.hasStdExtZfh()) {
331     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
332     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
333     setOperationAction(ISD::LRINT, MVT::f16, Legal);
334     setOperationAction(ISD::LLRINT, MVT::f16, Legal);
335     setOperationAction(ISD::LROUND, MVT::f16, Legal);
336     setOperationAction(ISD::LLROUND, MVT::f16, Legal);
337     setOperationAction(ISD::STRICT_LRINT, MVT::f16, Legal);
338     setOperationAction(ISD::STRICT_LLRINT, MVT::f16, Legal);
339     setOperationAction(ISD::STRICT_LROUND, MVT::f16, Legal);
340     setOperationAction(ISD::STRICT_LLROUND, MVT::f16, Legal);
341     setOperationAction(ISD::STRICT_FADD, MVT::f16, Legal);
342     setOperationAction(ISD::STRICT_FMA, MVT::f16, Legal);
343     setOperationAction(ISD::STRICT_FSUB, MVT::f16, Legal);
344     setOperationAction(ISD::STRICT_FMUL, MVT::f16, Legal);
345     setOperationAction(ISD::STRICT_FDIV, MVT::f16, Legal);
346     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
347     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
348     setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Legal);
349     setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Legal);
350     setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Legal);
351     for (auto CC : FPCCToExpand)
352       setCondCodeAction(CC, MVT::f16, Expand);
353     setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
354     setOperationAction(ISD::SELECT, MVT::f16, Custom);
355     setOperationAction(ISD::BR_CC, MVT::f16, Expand);
356 
357     setOperationAction(ISD::FREM,       MVT::f16, Promote);
358     setOperationAction(ISD::FCEIL,      MVT::f16, Promote);
359     setOperationAction(ISD::FFLOOR,     MVT::f16, Promote);
360     setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote);
361     setOperationAction(ISD::FRINT,      MVT::f16, Promote);
362     setOperationAction(ISD::FROUND,     MVT::f16, Promote);
363     setOperationAction(ISD::FROUNDEVEN, MVT::f16, Promote);
364     setOperationAction(ISD::FTRUNC,     MVT::f16, Promote);
365     setOperationAction(ISD::FPOW,       MVT::f16, Promote);
366     setOperationAction(ISD::FPOWI,      MVT::f16, Promote);
367     setOperationAction(ISD::FCOS,       MVT::f16, Promote);
368     setOperationAction(ISD::FSIN,       MVT::f16, Promote);
369     setOperationAction(ISD::FSINCOS,    MVT::f16, Promote);
370     setOperationAction(ISD::FEXP,       MVT::f16, Promote);
371     setOperationAction(ISD::FEXP2,      MVT::f16, Promote);
372     setOperationAction(ISD::FLOG,       MVT::f16, Promote);
373     setOperationAction(ISD::FLOG2,      MVT::f16, Promote);
374     setOperationAction(ISD::FLOG10,     MVT::f16, Promote);
375 
376     // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
377     // complete support for all operations in LegalizeDAG.
378 
379     // We need to custom promote this.
380     if (Subtarget.is64Bit())
381       setOperationAction(ISD::FPOWI, MVT::i32, Custom);
382   }
383 
384   if (Subtarget.hasStdExtF()) {
385     setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
386     setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
387     setOperationAction(ISD::LRINT, MVT::f32, Legal);
388     setOperationAction(ISD::LLRINT, MVT::f32, Legal);
389     setOperationAction(ISD::LROUND, MVT::f32, Legal);
390     setOperationAction(ISD::LLROUND, MVT::f32, Legal);
391     setOperationAction(ISD::STRICT_LRINT, MVT::f32, Legal);
392     setOperationAction(ISD::STRICT_LLRINT, MVT::f32, Legal);
393     setOperationAction(ISD::STRICT_LROUND, MVT::f32, Legal);
394     setOperationAction(ISD::STRICT_LLROUND, MVT::f32, Legal);
395     setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
396     setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal);
397     setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
398     setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
399     setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
400     setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
401     setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);
402     setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
403     for (auto CC : FPCCToExpand)
404       setCondCodeAction(CC, MVT::f32, Expand);
405     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
406     setOperationAction(ISD::SELECT, MVT::f32, Custom);
407     setOperationAction(ISD::BR_CC, MVT::f32, Expand);
408     for (auto Op : FPOpToExpand)
409       setOperationAction(Op, MVT::f32, Expand);
410     setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
411     setTruncStoreAction(MVT::f32, MVT::f16, Expand);
412   }
413 
414   if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
415     setOperationAction(ISD::BITCAST, MVT::i32, Custom);
416 
417   if (Subtarget.hasStdExtD()) {
418     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
419     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
420     setOperationAction(ISD::LRINT, MVT::f64, Legal);
421     setOperationAction(ISD::LLRINT, MVT::f64, Legal);
422     setOperationAction(ISD::LROUND, MVT::f64, Legal);
423     setOperationAction(ISD::LLROUND, MVT::f64, Legal);
424     setOperationAction(ISD::STRICT_LRINT, MVT::f64, Legal);
425     setOperationAction(ISD::STRICT_LLRINT, MVT::f64, Legal);
426     setOperationAction(ISD::STRICT_LROUND, MVT::f64, Legal);
427     setOperationAction(ISD::STRICT_LLROUND, MVT::f64, Legal);
428     setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal);
429     setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
430     setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
431     setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
432     setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
433     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
434     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
435     setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
436     setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal);
437     setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal);
438     for (auto CC : FPCCToExpand)
439       setCondCodeAction(CC, MVT::f64, Expand);
440     setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
441     setOperationAction(ISD::SELECT, MVT::f64, Custom);
442     setOperationAction(ISD::BR_CC, MVT::f64, Expand);
443     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
444     setTruncStoreAction(MVT::f64, MVT::f32, Expand);
445     for (auto Op : FPOpToExpand)
446       setOperationAction(Op, MVT::f64, Expand);
447     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
448     setTruncStoreAction(MVT::f64, MVT::f16, Expand);
449   }
450 
451   if (Subtarget.is64Bit()) {
452     setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
453     setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
454     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
455     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
456   }
457 
458   if (Subtarget.hasStdExtF()) {
459     setOperationAction(ISD::FP_TO_UINT_SAT, XLenVT, Custom);
460     setOperationAction(ISD::FP_TO_SINT_SAT, XLenVT, Custom);
461 
462     setOperationAction(ISD::STRICT_FP_TO_UINT, XLenVT, Legal);
463     setOperationAction(ISD::STRICT_FP_TO_SINT, XLenVT, Legal);
464     setOperationAction(ISD::STRICT_UINT_TO_FP, XLenVT, Legal);
465     setOperationAction(ISD::STRICT_SINT_TO_FP, XLenVT, Legal);
466 
467     setOperationAction(ISD::FLT_ROUNDS_, XLenVT, Custom);
468     setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
469   }
470 
471   setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
472   setOperationAction(ISD::BlockAddress, XLenVT, Custom);
473   setOperationAction(ISD::ConstantPool, XLenVT, Custom);
474   setOperationAction(ISD::JumpTable, XLenVT, Custom);
475 
476   setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
477 
478   // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
479   // Unfortunately this can't be determined just from the ISA naming string.
480   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
481                      Subtarget.is64Bit() ? Legal : Custom);
482 
483   setOperationAction(ISD::TRAP, MVT::Other, Legal);
484   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
485   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
486   if (Subtarget.is64Bit())
487     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
488 
489   if (Subtarget.hasStdExtA()) {
490     setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
491     setMinCmpXchgSizeInBits(32);
492   } else {
493     setMaxAtomicSizeInBitsSupported(0);
494   }
495 
496   setBooleanContents(ZeroOrOneBooleanContent);
497 
498   if (Subtarget.hasVInstructions()) {
499     setBooleanVectorContents(ZeroOrOneBooleanContent);
500 
501     setOperationAction(ISD::VSCALE, XLenVT, Custom);
502 
503     // RVV intrinsics may have illegal operands.
504     // We also need to custom legalize vmv.x.s.
505     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
506     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
507     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
508     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom);
509     if (Subtarget.is64Bit()) {
510       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);
511     } else {
512       setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
513       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
514     }
515 
516     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
517     setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
518 
519     static const unsigned IntegerVPOps[] = {
520         ISD::VP_ADD,         ISD::VP_SUB,         ISD::VP_MUL,
521         ISD::VP_SDIV,        ISD::VP_UDIV,        ISD::VP_SREM,
522         ISD::VP_UREM,        ISD::VP_AND,         ISD::VP_OR,
523         ISD::VP_XOR,         ISD::VP_ASHR,        ISD::VP_LSHR,
524         ISD::VP_SHL,         ISD::VP_REDUCE_ADD,  ISD::VP_REDUCE_AND,
525         ISD::VP_REDUCE_OR,   ISD::VP_REDUCE_XOR,  ISD::VP_REDUCE_SMAX,
526         ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
527         ISD::VP_MERGE,       ISD::VP_SELECT};
528 
529     static const unsigned FloatingPointVPOps[] = {
530         ISD::VP_FADD,        ISD::VP_FSUB,        ISD::VP_FMUL,
531         ISD::VP_FDIV,        ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
532         ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
533         ISD::VP_SELECT};
534 
535     if (!Subtarget.is64Bit()) {
536       // We must custom-lower certain vXi64 operations on RV32 due to the vector
537       // element type being illegal.
538       setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::i64, Custom);
539       setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::i64, Custom);
540 
541       setOperationAction(ISD::VECREDUCE_ADD, MVT::i64, Custom);
542       setOperationAction(ISD::VECREDUCE_AND, MVT::i64, Custom);
543       setOperationAction(ISD::VECREDUCE_OR, MVT::i64, Custom);
544       setOperationAction(ISD::VECREDUCE_XOR, MVT::i64, Custom);
545       setOperationAction(ISD::VECREDUCE_SMAX, MVT::i64, Custom);
546       setOperationAction(ISD::VECREDUCE_SMIN, MVT::i64, Custom);
547       setOperationAction(ISD::VECREDUCE_UMAX, MVT::i64, Custom);
548       setOperationAction(ISD::VECREDUCE_UMIN, MVT::i64, Custom);
549 
550       setOperationAction(ISD::VP_REDUCE_ADD, MVT::i64, Custom);
551       setOperationAction(ISD::VP_REDUCE_AND, MVT::i64, Custom);
552       setOperationAction(ISD::VP_REDUCE_OR, MVT::i64, Custom);
553       setOperationAction(ISD::VP_REDUCE_XOR, MVT::i64, Custom);
554       setOperationAction(ISD::VP_REDUCE_SMAX, MVT::i64, Custom);
555       setOperationAction(ISD::VP_REDUCE_SMIN, MVT::i64, Custom);
556       setOperationAction(ISD::VP_REDUCE_UMAX, MVT::i64, Custom);
557       setOperationAction(ISD::VP_REDUCE_UMIN, MVT::i64, Custom);
558     }
559 
560     for (MVT VT : BoolVecVTs) {
561       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
562 
563       // Mask VTs are custom-expanded into a series of standard nodes
564       setOperationAction(ISD::TRUNCATE, VT, Custom);
565       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
566       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
567       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
568 
569       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
570       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
571 
572       setOperationAction(ISD::SELECT, VT, Custom);
573       setOperationAction(ISD::SELECT_CC, VT, Expand);
574       setOperationAction(ISD::VSELECT, VT, Expand);
575       setOperationAction(ISD::VP_MERGE, VT, Expand);
576       setOperationAction(ISD::VP_SELECT, VT, Expand);
577 
578       setOperationAction(ISD::VP_AND, VT, Custom);
579       setOperationAction(ISD::VP_OR, VT, Custom);
580       setOperationAction(ISD::VP_XOR, VT, Custom);
581 
582       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
583       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
584       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
585 
586       setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
587       setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
588       setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
589 
590       // RVV has native int->float & float->int conversions where the
591       // element type sizes are within one power-of-two of each other. Any
592       // wider distances between type sizes have to be lowered as sequences
593       // which progressively narrow the gap in stages.
594       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
595       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
596       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
597       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
598 
599       // Expand all extending loads to types larger than this, and truncating
600       // stores from types larger than this.
601       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
602         setTruncStoreAction(OtherVT, VT, Expand);
603         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
604         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
605         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
606       }
607     }
608 
609     for (MVT VT : IntVecVTs) {
610       if (VT.getVectorElementType() == MVT::i64 &&
611           !Subtarget.hasVInstructionsI64())
612         continue;
613 
614       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
615       setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
616 
617       // Vectors implement MULHS/MULHU.
618       setOperationAction(ISD::SMUL_LOHI, VT, Expand);
619       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
620 
621       // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
622       if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV()) {
623         setOperationAction(ISD::MULHU, VT, Expand);
624         setOperationAction(ISD::MULHS, VT, Expand);
625       }
626 
627       setOperationAction(ISD::SMIN, VT, Legal);
628       setOperationAction(ISD::SMAX, VT, Legal);
629       setOperationAction(ISD::UMIN, VT, Legal);
630       setOperationAction(ISD::UMAX, VT, Legal);
631 
632       setOperationAction(ISD::ROTL, VT, Expand);
633       setOperationAction(ISD::ROTR, VT, Expand);
634 
635       setOperationAction(ISD::CTTZ, VT, Expand);
636       setOperationAction(ISD::CTLZ, VT, Expand);
637       setOperationAction(ISD::CTPOP, VT, Expand);
638 
639       setOperationAction(ISD::BSWAP, VT, Expand);
640 
641       // Custom-lower extensions and truncations from/to mask types.
642       setOperationAction(ISD::ANY_EXTEND, VT, Custom);
643       setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
644       setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
645 
646       // RVV has native int->float & float->int conversions where the
647       // element type sizes are within one power-of-two of each other. Any
648       // wider distances between type sizes have to be lowered as sequences
649       // which progressively narrow the gap in stages.
650       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
651       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
652       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
653       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
654 
655       setOperationAction(ISD::SADDSAT, VT, Legal);
656       setOperationAction(ISD::UADDSAT, VT, Legal);
657       setOperationAction(ISD::SSUBSAT, VT, Legal);
658       setOperationAction(ISD::USUBSAT, VT, Legal);
659 
660       // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
661       // nodes which truncate by one power of two at a time.
662       setOperationAction(ISD::TRUNCATE, VT, Custom);
663 
664       // Custom-lower insert/extract operations to simplify patterns.
665       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
666       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
667 
668       // Custom-lower reduction operations to set up the corresponding custom
669       // nodes' operands.
670       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
671       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
672       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
673       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
674       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
675       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
676       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
677       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
678 
679       for (unsigned VPOpc : IntegerVPOps)
680         setOperationAction(VPOpc, VT, Custom);
681 
682       setOperationAction(ISD::LOAD, VT, Custom);
683       setOperationAction(ISD::STORE, VT, Custom);
684 
685       setOperationAction(ISD::MLOAD, VT, Custom);
686       setOperationAction(ISD::MSTORE, VT, Custom);
687       setOperationAction(ISD::MGATHER, VT, Custom);
688       setOperationAction(ISD::MSCATTER, VT, Custom);
689 
690       setOperationAction(ISD::VP_LOAD, VT, Custom);
691       setOperationAction(ISD::VP_STORE, VT, Custom);
692       setOperationAction(ISD::VP_GATHER, VT, Custom);
693       setOperationAction(ISD::VP_SCATTER, VT, Custom);
694 
695       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
696       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
697       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
698 
699       setOperationAction(ISD::SELECT, VT, Custom);
700       setOperationAction(ISD::SELECT_CC, VT, Expand);
701 
702       setOperationAction(ISD::STEP_VECTOR, VT, Custom);
703       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
704 
705       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
706         setTruncStoreAction(VT, OtherVT, Expand);
707         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
708         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
709         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
710       }
711 
712       // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
713       // type that can represent the value exactly.
714       if (VT.getVectorElementType() != MVT::i64) {
715         MVT FloatEltVT =
716             VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
717         EVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
718         if (isTypeLegal(FloatVT)) {
719           setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
720           setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
721         }
722       }
723     }
724 
725     // Expand various CCs to best match the RVV ISA, which natively supports UNE
726     // but no other unordered comparisons, and supports all ordered comparisons
727     // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
728     // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
729     // and we pattern-match those back to the "original", swapping operands once
730     // more. This way we catch both operations and both "vf" and "fv" forms with
731     // fewer patterns.
732     static const ISD::CondCode VFPCCToExpand[] = {
733         ISD::SETO,   ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
734         ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
735         ISD::SETGT,  ISD::SETOGT, ISD::SETGE,  ISD::SETOGE,
736     };
737 
738     // Sets common operation actions on RVV floating-point vector types.
739     const auto SetCommonVFPActions = [&](MVT VT) {
740       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
741       // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
742       // sizes are within one power-of-two of each other. Therefore conversions
743       // between vXf16 and vXf64 must be lowered as sequences which convert via
744       // vXf32.
745       setOperationAction(ISD::FP_ROUND, VT, Custom);
746       setOperationAction(ISD::FP_EXTEND, VT, Custom);
747       // Custom-lower insert/extract operations to simplify patterns.
748       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
749       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
750       // Expand various condition codes (explained above).
751       for (auto CC : VFPCCToExpand)
752         setCondCodeAction(CC, VT, Expand);
753 
754       setOperationAction(ISD::FMINNUM, VT, Legal);
755       setOperationAction(ISD::FMAXNUM, VT, Legal);
756 
757       setOperationAction(ISD::FTRUNC, VT, Custom);
758       setOperationAction(ISD::FCEIL, VT, Custom);
759       setOperationAction(ISD::FFLOOR, VT, Custom);
760 
761       setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
762       setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
763       setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
764       setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
765 
766       setOperationAction(ISD::FCOPYSIGN, VT, Legal);
767 
768       setOperationAction(ISD::LOAD, VT, Custom);
769       setOperationAction(ISD::STORE, VT, Custom);
770 
771       setOperationAction(ISD::MLOAD, VT, Custom);
772       setOperationAction(ISD::MSTORE, VT, Custom);
773       setOperationAction(ISD::MGATHER, VT, Custom);
774       setOperationAction(ISD::MSCATTER, VT, Custom);
775 
776       setOperationAction(ISD::VP_LOAD, VT, Custom);
777       setOperationAction(ISD::VP_STORE, VT, Custom);
778       setOperationAction(ISD::VP_GATHER, VT, Custom);
779       setOperationAction(ISD::VP_SCATTER, VT, Custom);
780 
781       setOperationAction(ISD::SELECT, VT, Custom);
782       setOperationAction(ISD::SELECT_CC, VT, Expand);
783 
784       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
785       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
786       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
787 
788       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
789 
790       for (unsigned VPOpc : FloatingPointVPOps)
791         setOperationAction(VPOpc, VT, Custom);
792     };
793 
794     // Sets common extload/truncstore actions on RVV floating-point vector
795     // types.
796     const auto SetCommonVFPExtLoadTruncStoreActions =
797         [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
798           for (auto SmallVT : SmallerVTs) {
799             setTruncStoreAction(VT, SmallVT, Expand);
800             setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
801           }
802         };
803 
804     if (Subtarget.hasVInstructionsF16())
805       for (MVT VT : F16VecVTs)
806         SetCommonVFPActions(VT);
807 
808     for (MVT VT : F32VecVTs) {
809       if (Subtarget.hasVInstructionsF32())
810         SetCommonVFPActions(VT);
811       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
812     }
813 
814     for (MVT VT : F64VecVTs) {
815       if (Subtarget.hasVInstructionsF64())
816         SetCommonVFPActions(VT);
817       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
818       SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
819     }
820 
821     if (Subtarget.useRVVForFixedLengthVectors()) {
822       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
823         if (!useRVVForFixedLengthVectorVT(VT))
824           continue;
825 
826         // By default everything must be expanded.
827         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
828           setOperationAction(Op, VT, Expand);
829         for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
830           setTruncStoreAction(VT, OtherVT, Expand);
831           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
832           setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
833           setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
834         }
835 
836         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
837         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
838         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
839 
840         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
841         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
842 
843         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
844         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
845 
846         setOperationAction(ISD::LOAD, VT, Custom);
847         setOperationAction(ISD::STORE, VT, Custom);
848 
849         setOperationAction(ISD::SETCC, VT, Custom);
850 
851         setOperationAction(ISD::SELECT, VT, Custom);
852 
853         setOperationAction(ISD::TRUNCATE, VT, Custom);
854 
855         setOperationAction(ISD::BITCAST, VT, Custom);
856 
857         setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
858         setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
859         setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
860 
861         setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
862         setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
863         setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
864 
865         setOperationAction(ISD::SINT_TO_FP, VT, Custom);
866         setOperationAction(ISD::UINT_TO_FP, VT, Custom);
867         setOperationAction(ISD::FP_TO_SINT, VT, Custom);
868         setOperationAction(ISD::FP_TO_UINT, VT, Custom);
869 
870         // Operations below are different for between masks and other vectors.
871         if (VT.getVectorElementType() == MVT::i1) {
872           setOperationAction(ISD::VP_AND, VT, Custom);
873           setOperationAction(ISD::VP_OR, VT, Custom);
874           setOperationAction(ISD::VP_XOR, VT, Custom);
875           setOperationAction(ISD::AND, VT, Custom);
876           setOperationAction(ISD::OR, VT, Custom);
877           setOperationAction(ISD::XOR, VT, Custom);
878           continue;
879         }
880 
881         // Use SPLAT_VECTOR to prevent type legalization from destroying the
882         // splats when type legalizing i64 scalar on RV32.
883         // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
884         // improvements first.
885         if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
886           setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
887           setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
888         }
889 
890         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
891         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
892 
893         setOperationAction(ISD::MLOAD, VT, Custom);
894         setOperationAction(ISD::MSTORE, VT, Custom);
895         setOperationAction(ISD::MGATHER, VT, Custom);
896         setOperationAction(ISD::MSCATTER, VT, Custom);
897 
898         setOperationAction(ISD::VP_LOAD, VT, Custom);
899         setOperationAction(ISD::VP_STORE, VT, Custom);
900         setOperationAction(ISD::VP_GATHER, VT, Custom);
901         setOperationAction(ISD::VP_SCATTER, VT, Custom);
902 
903         setOperationAction(ISD::ADD, VT, Custom);
904         setOperationAction(ISD::MUL, VT, Custom);
905         setOperationAction(ISD::SUB, VT, Custom);
906         setOperationAction(ISD::AND, VT, Custom);
907         setOperationAction(ISD::OR, VT, Custom);
908         setOperationAction(ISD::XOR, VT, Custom);
909         setOperationAction(ISD::SDIV, VT, Custom);
910         setOperationAction(ISD::SREM, VT, Custom);
911         setOperationAction(ISD::UDIV, VT, Custom);
912         setOperationAction(ISD::UREM, VT, Custom);
913         setOperationAction(ISD::SHL, VT, Custom);
914         setOperationAction(ISD::SRA, VT, Custom);
915         setOperationAction(ISD::SRL, VT, Custom);
916 
917         setOperationAction(ISD::SMIN, VT, Custom);
918         setOperationAction(ISD::SMAX, VT, Custom);
919         setOperationAction(ISD::UMIN, VT, Custom);
920         setOperationAction(ISD::UMAX, VT, Custom);
921         setOperationAction(ISD::ABS,  VT, Custom);
922 
923         // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
924         if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV()) {
925           setOperationAction(ISD::MULHS, VT, Custom);
926           setOperationAction(ISD::MULHU, VT, Custom);
927         }
928 
929         setOperationAction(ISD::SADDSAT, VT, Custom);
930         setOperationAction(ISD::UADDSAT, VT, Custom);
931         setOperationAction(ISD::SSUBSAT, VT, Custom);
932         setOperationAction(ISD::USUBSAT, VT, Custom);
933 
934         setOperationAction(ISD::VSELECT, VT, Custom);
935         setOperationAction(ISD::SELECT_CC, VT, Expand);
936 
937         setOperationAction(ISD::ANY_EXTEND, VT, Custom);
938         setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
939         setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
940 
941         // Custom-lower reduction operations to set up the corresponding custom
942         // nodes' operands.
943         setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
944         setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
945         setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
946         setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
947         setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
948 
949         for (unsigned VPOpc : IntegerVPOps)
950           setOperationAction(VPOpc, VT, Custom);
951 
952         // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
953         // type that can represent the value exactly.
954         if (VT.getVectorElementType() != MVT::i64) {
955           MVT FloatEltVT =
956               VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
957           EVT FloatVT =
958               MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
959           if (isTypeLegal(FloatVT)) {
960             setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
961             setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
962           }
963         }
964       }
965 
966       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
967         if (!useRVVForFixedLengthVectorVT(VT))
968           continue;
969 
970         // By default everything must be expanded.
971         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
972           setOperationAction(Op, VT, Expand);
973         for (MVT OtherVT : MVT::fp_fixedlen_vector_valuetypes()) {
974           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
975           setTruncStoreAction(VT, OtherVT, Expand);
976         }
977 
978         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
979         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
980         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
981 
982         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
983         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
984         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
985         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
986         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
987 
988         setOperationAction(ISD::LOAD, VT, Custom);
989         setOperationAction(ISD::STORE, VT, Custom);
990         setOperationAction(ISD::MLOAD, VT, Custom);
991         setOperationAction(ISD::MSTORE, VT, Custom);
992         setOperationAction(ISD::MGATHER, VT, Custom);
993         setOperationAction(ISD::MSCATTER, VT, Custom);
994 
995         setOperationAction(ISD::VP_LOAD, VT, Custom);
996         setOperationAction(ISD::VP_STORE, VT, Custom);
997         setOperationAction(ISD::VP_GATHER, VT, Custom);
998         setOperationAction(ISD::VP_SCATTER, VT, Custom);
999 
1000         setOperationAction(ISD::FADD, VT, Custom);
1001         setOperationAction(ISD::FSUB, VT, Custom);
1002         setOperationAction(ISD::FMUL, VT, Custom);
1003         setOperationAction(ISD::FDIV, VT, Custom);
1004         setOperationAction(ISD::FNEG, VT, Custom);
1005         setOperationAction(ISD::FABS, VT, Custom);
1006         setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1007         setOperationAction(ISD::FSQRT, VT, Custom);
1008         setOperationAction(ISD::FMA, VT, Custom);
1009         setOperationAction(ISD::FMINNUM, VT, Custom);
1010         setOperationAction(ISD::FMAXNUM, VT, Custom);
1011 
1012         setOperationAction(ISD::FP_ROUND, VT, Custom);
1013         setOperationAction(ISD::FP_EXTEND, VT, Custom);
1014 
1015         setOperationAction(ISD::FTRUNC, VT, Custom);
1016         setOperationAction(ISD::FCEIL, VT, Custom);
1017         setOperationAction(ISD::FFLOOR, VT, Custom);
1018 
1019         for (auto CC : VFPCCToExpand)
1020           setCondCodeAction(CC, VT, Expand);
1021 
1022         setOperationAction(ISD::VSELECT, VT, Custom);
1023         setOperationAction(ISD::SELECT, VT, Custom);
1024         setOperationAction(ISD::SELECT_CC, VT, Expand);
1025 
1026         setOperationAction(ISD::BITCAST, VT, Custom);
1027 
1028         setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1029         setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1030         setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1031         setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1032 
1033         for (unsigned VPOpc : FloatingPointVPOps)
1034           setOperationAction(VPOpc, VT, Custom);
1035       }
1036 
1037       // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1038       setOperationAction(ISD::BITCAST, MVT::i8, Custom);
1039       setOperationAction(ISD::BITCAST, MVT::i16, Custom);
1040       setOperationAction(ISD::BITCAST, MVT::i32, Custom);
1041       setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1042       if (Subtarget.hasStdExtZfh())
1043         setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1044       if (Subtarget.hasStdExtF())
1045         setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1046       if (Subtarget.hasStdExtD())
1047         setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1048     }
1049   }
1050 
1051   // Function alignments.
1052   const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
1053   setMinFunctionAlignment(FunctionAlignment);
1054   setPrefFunctionAlignment(FunctionAlignment);
1055 
1056   setMinimumJumpTableEntries(5);
1057 
1058   // Jumps are expensive, compared to logic
1059   setJumpIsExpensive();
1060 
1061   setTargetDAGCombine(ISD::ADD);
1062   setTargetDAGCombine(ISD::SUB);
1063   setTargetDAGCombine(ISD::AND);
1064   setTargetDAGCombine(ISD::OR);
1065   setTargetDAGCombine(ISD::XOR);
1066   setTargetDAGCombine(ISD::ANY_EXTEND);
1067   if (Subtarget.hasStdExtF()) {
1068     setTargetDAGCombine(ISD::ZERO_EXTEND);
1069     setTargetDAGCombine(ISD::FP_TO_SINT);
1070     setTargetDAGCombine(ISD::FP_TO_UINT);
1071     setTargetDAGCombine(ISD::FP_TO_SINT_SAT);
1072     setTargetDAGCombine(ISD::FP_TO_UINT_SAT);
1073   }
1074   if (Subtarget.hasVInstructions()) {
1075     setTargetDAGCombine(ISD::FCOPYSIGN);
1076     setTargetDAGCombine(ISD::MGATHER);
1077     setTargetDAGCombine(ISD::MSCATTER);
1078     setTargetDAGCombine(ISD::VP_GATHER);
1079     setTargetDAGCombine(ISD::VP_SCATTER);
1080     setTargetDAGCombine(ISD::SRA);
1081     setTargetDAGCombine(ISD::SRL);
1082     setTargetDAGCombine(ISD::SHL);
1083     setTargetDAGCombine(ISD::STORE);
1084   }
1085 
1086   setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
1087   setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
1088 }
1089 
1090 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
1091                                             LLVMContext &Context,
1092                                             EVT VT) const {
1093   if (!VT.isVector())
1094     return getPointerTy(DL);
1095   if (Subtarget.hasVInstructions() &&
1096       (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
1097     return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
1098   return VT.changeVectorElementTypeToInteger();
1099 }
1100 
1101 MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
1102   return Subtarget.getXLenVT();
1103 }
1104 
1105 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1106                                              const CallInst &I,
1107                                              MachineFunction &MF,
1108                                              unsigned Intrinsic) const {
1109   auto &DL = I.getModule()->getDataLayout();
1110   switch (Intrinsic) {
1111   default:
1112     return false;
1113   case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1114   case Intrinsic::riscv_masked_atomicrmw_add_i32:
1115   case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1116   case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1117   case Intrinsic::riscv_masked_atomicrmw_max_i32:
1118   case Intrinsic::riscv_masked_atomicrmw_min_i32:
1119   case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1120   case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1121   case Intrinsic::riscv_masked_cmpxchg_i32: {
1122     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
1123     Info.opc = ISD::INTRINSIC_W_CHAIN;
1124     Info.memVT = MVT::getVT(PtrTy->getPointerElementType());
1125     Info.ptrVal = I.getArgOperand(0);
1126     Info.offset = 0;
1127     Info.align = Align(4);
1128     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1129                  MachineMemOperand::MOVolatile;
1130     return true;
1131   }
1132   case Intrinsic::riscv_masked_strided_load:
1133     Info.opc = ISD::INTRINSIC_W_CHAIN;
1134     Info.ptrVal = I.getArgOperand(1);
1135     Info.memVT = getValueType(DL, I.getType()->getScalarType());
1136     Info.align = Align(DL.getTypeSizeInBits(I.getType()->getScalarType()) / 8);
1137     Info.size = MemoryLocation::UnknownSize;
1138     Info.flags |= MachineMemOperand::MOLoad;
1139     return true;
1140   case Intrinsic::riscv_masked_strided_store:
1141     Info.opc = ISD::INTRINSIC_VOID;
1142     Info.ptrVal = I.getArgOperand(1);
1143     Info.memVT =
1144         getValueType(DL, I.getArgOperand(0)->getType()->getScalarType());
1145     Info.align = Align(
1146         DL.getTypeSizeInBits(I.getArgOperand(0)->getType()->getScalarType()) /
1147         8);
1148     Info.size = MemoryLocation::UnknownSize;
1149     Info.flags |= MachineMemOperand::MOStore;
1150     return true;
1151   }
1152 }
1153 
1154 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1155                                                 const AddrMode &AM, Type *Ty,
1156                                                 unsigned AS,
1157                                                 Instruction *I) const {
1158   // No global is ever allowed as a base.
1159   if (AM.BaseGV)
1160     return false;
1161 
1162   // Require a 12-bit signed offset.
1163   if (!isInt<12>(AM.BaseOffs))
1164     return false;
1165 
1166   switch (AM.Scale) {
1167   case 0: // "r+i" or just "i", depending on HasBaseReg.
1168     break;
1169   case 1:
1170     if (!AM.HasBaseReg) // allow "r+i".
1171       break;
1172     return false; // disallow "r+r" or "r+r+i".
1173   default:
1174     return false;
1175   }
1176 
1177   return true;
1178 }
1179 
1180 bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1181   return isInt<12>(Imm);
1182 }
1183 
1184 bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1185   return isInt<12>(Imm);
1186 }
1187 
1188 // On RV32, 64-bit integers are split into their high and low parts and held
1189 // in two different registers, so the trunc is free since the low register can
1190 // just be used.
1191 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
1192   if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
1193     return false;
1194   unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1195   unsigned DestBits = DstTy->getPrimitiveSizeInBits();
1196   return (SrcBits == 64 && DestBits == 32);
1197 }
1198 
1199 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
1200   if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
1201       !SrcVT.isInteger() || !DstVT.isInteger())
1202     return false;
1203   unsigned SrcBits = SrcVT.getSizeInBits();
1204   unsigned DestBits = DstVT.getSizeInBits();
1205   return (SrcBits == 64 && DestBits == 32);
1206 }
1207 
1208 bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
1209   // Zexts are free if they can be combined with a load.
1210   // Don't advertise i32->i64 zextload as being free for RV64. It interacts
1211   // poorly with type legalization of compares preferring sext.
1212   if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
1213     EVT MemVT = LD->getMemoryVT();
1214     if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1215         (LD->getExtensionType() == ISD::NON_EXTLOAD ||
1216          LD->getExtensionType() == ISD::ZEXTLOAD))
1217       return true;
1218   }
1219 
1220   return TargetLowering::isZExtFree(Val, VT2);
1221 }
1222 
1223 bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
1224   return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1225 }
1226 
1227 bool RISCVTargetLowering::isCheapToSpeculateCttz() const {
1228   return Subtarget.hasStdExtZbb();
1229 }
1230 
1231 bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
1232   return Subtarget.hasStdExtZbb();
1233 }
1234 
1235 bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
1236   EVT VT = Y.getValueType();
1237 
1238   // FIXME: Support vectors once we have tests.
1239   if (VT.isVector())
1240     return false;
1241 
1242   return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp() ||
1243           Subtarget.hasStdExtZbkb()) &&
1244          !isa<ConstantSDNode>(Y);
1245 }
1246 
1247 /// Check if sinking \p I's operands to I's basic block is profitable, because
1248 /// the operands can be folded into a target instruction, e.g.
1249 /// splats of scalars can fold into vector instructions.
1250 bool RISCVTargetLowering::shouldSinkOperands(
1251     Instruction *I, SmallVectorImpl<Use *> &Ops) const {
1252   using namespace llvm::PatternMatch;
1253 
1254   if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
1255     return false;
1256 
1257   auto IsSinker = [&](Instruction *I, int Operand) {
1258     switch (I->getOpcode()) {
1259     case Instruction::Add:
1260     case Instruction::Sub:
1261     case Instruction::Mul:
1262     case Instruction::And:
1263     case Instruction::Or:
1264     case Instruction::Xor:
1265     case Instruction::FAdd:
1266     case Instruction::FSub:
1267     case Instruction::FMul:
1268     case Instruction::FDiv:
1269     case Instruction::ICmp:
1270     case Instruction::FCmp:
1271       return true;
1272     case Instruction::Shl:
1273     case Instruction::LShr:
1274     case Instruction::AShr:
1275     case Instruction::UDiv:
1276     case Instruction::SDiv:
1277     case Instruction::URem:
1278     case Instruction::SRem:
1279       return Operand == 1;
1280     case Instruction::Call:
1281       if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1282         switch (II->getIntrinsicID()) {
1283         case Intrinsic::fma:
1284           return Operand == 0 || Operand == 1;
1285         // FIXME: Our patterns can only match vx/vf instructions when the splat
1286         // it on the RHS, because TableGen doesn't recognize our VP operations
1287         // as commutative.
1288         case Intrinsic::vp_add:
1289         case Intrinsic::vp_mul:
1290         case Intrinsic::vp_and:
1291         case Intrinsic::vp_or:
1292         case Intrinsic::vp_xor:
1293         case Intrinsic::vp_fadd:
1294         case Intrinsic::vp_fmul:
1295         case Intrinsic::vp_shl:
1296         case Intrinsic::vp_lshr:
1297         case Intrinsic::vp_ashr:
1298         case Intrinsic::vp_udiv:
1299         case Intrinsic::vp_sdiv:
1300         case Intrinsic::vp_urem:
1301         case Intrinsic::vp_srem:
1302           return Operand == 1;
1303         // ... with the exception of vp.sub/vp.fsub/vp.fdiv, which have
1304         // explicit patterns for both LHS and RHS (as 'vr' versions).
1305         case Intrinsic::vp_sub:
1306         case Intrinsic::vp_fsub:
1307         case Intrinsic::vp_fdiv:
1308           return Operand == 0 || Operand == 1;
1309         default:
1310           return false;
1311         }
1312       }
1313       return false;
1314     default:
1315       return false;
1316     }
1317   };
1318 
1319   for (auto OpIdx : enumerate(I->operands())) {
1320     if (!IsSinker(I, OpIdx.index()))
1321       continue;
1322 
1323     Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
1324     // Make sure we are not already sinking this operand
1325     if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
1326       continue;
1327 
1328     // We are looking for a splat that can be sunk.
1329     if (!match(Op, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
1330                              m_Undef(), m_ZeroMask())))
1331       continue;
1332 
1333     // All uses of the shuffle should be sunk to avoid duplicating it across gpr
1334     // and vector registers
1335     for (Use &U : Op->uses()) {
1336       Instruction *Insn = cast<Instruction>(U.getUser());
1337       if (!IsSinker(Insn, U.getOperandNo()))
1338         return false;
1339     }
1340 
1341     Ops.push_back(&Op->getOperandUse(0));
1342     Ops.push_back(&OpIdx.value());
1343   }
1344   return true;
1345 }
1346 
1347 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1348                                        bool ForCodeSize) const {
1349   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1350   if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
1351     return false;
1352   if (VT == MVT::f32 && !Subtarget.hasStdExtF())
1353     return false;
1354   if (VT == MVT::f64 && !Subtarget.hasStdExtD())
1355     return false;
1356   return Imm.isZero();
1357 }
1358 
1359 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
1360   return (VT == MVT::f16 && Subtarget.hasStdExtZfh()) ||
1361          (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
1362          (VT == MVT::f64 && Subtarget.hasStdExtD());
1363 }
1364 
1365 MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1366                                                       CallingConv::ID CC,
1367                                                       EVT VT) const {
1368   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1369   // We might still end up using a GPR but that will be decided based on ABI.
1370   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1371   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1372     return MVT::f32;
1373 
1374   return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1375 }
1376 
1377 unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1378                                                            CallingConv::ID CC,
1379                                                            EVT VT) const {
1380   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1381   // We might still end up using a GPR but that will be decided based on ABI.
1382   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1383   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1384     return 1;
1385 
1386   return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1387 }
1388 
1389 // Changes the condition code and swaps operands if necessary, so the SetCC
1390 // operation matches one of the comparisons supported directly by branches
1391 // in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
1392 // with 1/-1.
1393 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1394                                     ISD::CondCode &CC, SelectionDAG &DAG) {
1395   // Convert X > -1 to X >= 0.
1396   if (CC == ISD::SETGT && isAllOnesConstant(RHS)) {
1397     RHS = DAG.getConstant(0, DL, RHS.getValueType());
1398     CC = ISD::SETGE;
1399     return;
1400   }
1401   // Convert X < 1 to 0 >= X.
1402   if (CC == ISD::SETLT && isOneConstant(RHS)) {
1403     RHS = LHS;
1404     LHS = DAG.getConstant(0, DL, RHS.getValueType());
1405     CC = ISD::SETGE;
1406     return;
1407   }
1408 
1409   switch (CC) {
1410   default:
1411     break;
1412   case ISD::SETGT:
1413   case ISD::SETLE:
1414   case ISD::SETUGT:
1415   case ISD::SETULE:
1416     CC = ISD::getSetCCSwappedOperands(CC);
1417     std::swap(LHS, RHS);
1418     break;
1419   }
1420 }
1421 
1422 RISCVII::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
1423   assert(VT.isScalableVector() && "Expecting a scalable vector type");
1424   unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
1425   if (VT.getVectorElementType() == MVT::i1)
1426     KnownSize *= 8;
1427 
1428   switch (KnownSize) {
1429   default:
1430     llvm_unreachable("Invalid LMUL.");
1431   case 8:
1432     return RISCVII::VLMUL::LMUL_F8;
1433   case 16:
1434     return RISCVII::VLMUL::LMUL_F4;
1435   case 32:
1436     return RISCVII::VLMUL::LMUL_F2;
1437   case 64:
1438     return RISCVII::VLMUL::LMUL_1;
1439   case 128:
1440     return RISCVII::VLMUL::LMUL_2;
1441   case 256:
1442     return RISCVII::VLMUL::LMUL_4;
1443   case 512:
1444     return RISCVII::VLMUL::LMUL_8;
1445   }
1446 }
1447 
1448 unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVII::VLMUL LMul) {
1449   switch (LMul) {
1450   default:
1451     llvm_unreachable("Invalid LMUL.");
1452   case RISCVII::VLMUL::LMUL_F8:
1453   case RISCVII::VLMUL::LMUL_F4:
1454   case RISCVII::VLMUL::LMUL_F2:
1455   case RISCVII::VLMUL::LMUL_1:
1456     return RISCV::VRRegClassID;
1457   case RISCVII::VLMUL::LMUL_2:
1458     return RISCV::VRM2RegClassID;
1459   case RISCVII::VLMUL::LMUL_4:
1460     return RISCV::VRM4RegClassID;
1461   case RISCVII::VLMUL::LMUL_8:
1462     return RISCV::VRM8RegClassID;
1463   }
1464 }
1465 
1466 unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
1467   RISCVII::VLMUL LMUL = getLMUL(VT);
1468   if (LMUL == RISCVII::VLMUL::LMUL_F8 ||
1469       LMUL == RISCVII::VLMUL::LMUL_F4 ||
1470       LMUL == RISCVII::VLMUL::LMUL_F2 ||
1471       LMUL == RISCVII::VLMUL::LMUL_1) {
1472     static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
1473                   "Unexpected subreg numbering");
1474     return RISCV::sub_vrm1_0 + Index;
1475   }
1476   if (LMUL == RISCVII::VLMUL::LMUL_2) {
1477     static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
1478                   "Unexpected subreg numbering");
1479     return RISCV::sub_vrm2_0 + Index;
1480   }
1481   if (LMUL == RISCVII::VLMUL::LMUL_4) {
1482     static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
1483                   "Unexpected subreg numbering");
1484     return RISCV::sub_vrm4_0 + Index;
1485   }
1486   llvm_unreachable("Invalid vector type.");
1487 }
1488 
1489 unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
1490   if (VT.getVectorElementType() == MVT::i1)
1491     return RISCV::VRRegClassID;
1492   return getRegClassIDForLMUL(getLMUL(VT));
1493 }
1494 
1495 // Attempt to decompose a subvector insert/extract between VecVT and
1496 // SubVecVT via subregister indices. Returns the subregister index that
1497 // can perform the subvector insert/extract with the given element index, as
1498 // well as the index corresponding to any leftover subvectors that must be
1499 // further inserted/extracted within the register class for SubVecVT.
1500 std::pair<unsigned, unsigned>
1501 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1502     MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
1503     const RISCVRegisterInfo *TRI) {
1504   static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
1505                  RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
1506                  RISCV::VRM2RegClassID > RISCV::VRRegClassID),
1507                 "Register classes not ordered");
1508   unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
1509   unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
1510   // Try to compose a subregister index that takes us from the incoming
1511   // LMUL>1 register class down to the outgoing one. At each step we half
1512   // the LMUL:
1513   //   nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
1514   // Note that this is not guaranteed to find a subregister index, such as
1515   // when we are extracting from one VR type to another.
1516   unsigned SubRegIdx = RISCV::NoSubRegister;
1517   for (const unsigned RCID :
1518        {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
1519     if (VecRegClassID > RCID && SubRegClassID <= RCID) {
1520       VecVT = VecVT.getHalfNumVectorElementsVT();
1521       bool IsHi =
1522           InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
1523       SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
1524                                             getSubregIndexByMVT(VecVT, IsHi));
1525       if (IsHi)
1526         InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
1527     }
1528   return {SubRegIdx, InsertExtractIdx};
1529 }
1530 
1531 // Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
1532 // stores for those types.
1533 bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
1534   return !Subtarget.useRVVForFixedLengthVectors() ||
1535          (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
1536 }
1537 
1538 bool RISCVTargetLowering::isLegalElementTypeForRVV(Type *ScalarTy) const {
1539   if (ScalarTy->isPointerTy())
1540     return true;
1541 
1542   if (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
1543       ScalarTy->isIntegerTy(32))
1544     return true;
1545 
1546   if (ScalarTy->isIntegerTy(64))
1547     return Subtarget.hasVInstructionsI64();
1548 
1549   if (ScalarTy->isHalfTy())
1550     return Subtarget.hasVInstructionsF16();
1551   if (ScalarTy->isFloatTy())
1552     return Subtarget.hasVInstructionsF32();
1553   if (ScalarTy->isDoubleTy())
1554     return Subtarget.hasVInstructionsF64();
1555 
1556   return false;
1557 }
1558 
1559 static SDValue getVLOperand(SDValue Op) {
1560   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
1561           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
1562          "Unexpected opcode");
1563   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
1564   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
1565   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
1566       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
1567   if (!II)
1568     return SDValue();
1569   return Op.getOperand(II->VLOperand + 1 + HasChain);
1570 }
1571 
1572 static bool useRVVForFixedLengthVectorVT(MVT VT,
1573                                          const RISCVSubtarget &Subtarget) {
1574   assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
1575   if (!Subtarget.useRVVForFixedLengthVectors())
1576     return false;
1577 
1578   // We only support a set of vector types with a consistent maximum fixed size
1579   // across all supported vector element types to avoid legalization issues.
1580   // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
1581   // fixed-length vector type we support is 1024 bytes.
1582   if (VT.getFixedSizeInBits() > 1024 * 8)
1583     return false;
1584 
1585   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1586 
1587   MVT EltVT = VT.getVectorElementType();
1588 
1589   // Don't use RVV for vectors we cannot scalarize if required.
1590   switch (EltVT.SimpleTy) {
1591   // i1 is supported but has different rules.
1592   default:
1593     return false;
1594   case MVT::i1:
1595     // Masks can only use a single register.
1596     if (VT.getVectorNumElements() > MinVLen)
1597       return false;
1598     MinVLen /= 8;
1599     break;
1600   case MVT::i8:
1601   case MVT::i16:
1602   case MVT::i32:
1603     break;
1604   case MVT::i64:
1605     if (!Subtarget.hasVInstructionsI64())
1606       return false;
1607     break;
1608   case MVT::f16:
1609     if (!Subtarget.hasVInstructionsF16())
1610       return false;
1611     break;
1612   case MVT::f32:
1613     if (!Subtarget.hasVInstructionsF32())
1614       return false;
1615     break;
1616   case MVT::f64:
1617     if (!Subtarget.hasVInstructionsF64())
1618       return false;
1619     break;
1620   }
1621 
1622   // Reject elements larger than ELEN.
1623   if (EltVT.getSizeInBits() > Subtarget.getMaxELENForFixedLengthVectors())
1624     return false;
1625 
1626   unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
1627   // Don't use RVV for types that don't fit.
1628   if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
1629     return false;
1630 
1631   // TODO: Perhaps an artificial restriction, but worth having whilst getting
1632   // the base fixed length RVV support in place.
1633   if (!VT.isPow2VectorType())
1634     return false;
1635 
1636   return true;
1637 }
1638 
1639 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
1640   return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
1641 }
1642 
1643 // Return the largest legal scalable vector type that matches VT's element type.
1644 static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
1645                                             const RISCVSubtarget &Subtarget) {
1646   // This may be called before legal types are setup.
1647   assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
1648           useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
1649          "Expected legal fixed length vector!");
1650 
1651   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1652   unsigned MaxELen = Subtarget.getMaxELENForFixedLengthVectors();
1653 
1654   MVT EltVT = VT.getVectorElementType();
1655   switch (EltVT.SimpleTy) {
1656   default:
1657     llvm_unreachable("unexpected element type for RVV container");
1658   case MVT::i1:
1659   case MVT::i8:
1660   case MVT::i16:
1661   case MVT::i32:
1662   case MVT::i64:
1663   case MVT::f16:
1664   case MVT::f32:
1665   case MVT::f64: {
1666     // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
1667     // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
1668     // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
1669     unsigned NumElts =
1670         (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
1671     NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
1672     assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
1673     return MVT::getScalableVectorVT(EltVT, NumElts);
1674   }
1675   }
1676 }
1677 
1678 static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
1679                                             const RISCVSubtarget &Subtarget) {
1680   return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
1681                                           Subtarget);
1682 }
1683 
1684 MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
1685   return ::getContainerForFixedLengthVector(*this, VT, getSubtarget());
1686 }
1687 
1688 // Grow V to consume an entire RVV register.
1689 static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1690                                        const RISCVSubtarget &Subtarget) {
1691   assert(VT.isScalableVector() &&
1692          "Expected to convert into a scalable vector!");
1693   assert(V.getValueType().isFixedLengthVector() &&
1694          "Expected a fixed length vector operand!");
1695   SDLoc DL(V);
1696   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1697   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
1698 }
1699 
1700 // Shrink V so it's just big enough to maintain a VT's worth of data.
1701 static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1702                                          const RISCVSubtarget &Subtarget) {
1703   assert(VT.isFixedLengthVector() &&
1704          "Expected to convert into a fixed length vector!");
1705   assert(V.getValueType().isScalableVector() &&
1706          "Expected a scalable vector operand!");
1707   SDLoc DL(V);
1708   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1709   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
1710 }
1711 
1712 // Gets the two common "VL" operands: an all-ones mask and the vector length.
1713 // VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
1714 // the vector type that it is contained in.
1715 static std::pair<SDValue, SDValue>
1716 getDefaultVLOps(MVT VecVT, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
1717                 const RISCVSubtarget &Subtarget) {
1718   assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
1719   MVT XLenVT = Subtarget.getXLenVT();
1720   SDValue VL = VecVT.isFixedLengthVector()
1721                    ? DAG.getConstant(VecVT.getVectorNumElements(), DL, XLenVT)
1722                    : DAG.getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT);
1723   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
1724   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
1725   return {Mask, VL};
1726 }
1727 
1728 // As above but assuming the given type is a scalable vector type.
1729 static std::pair<SDValue, SDValue>
1730 getDefaultScalableVLOps(MVT VecVT, SDLoc DL, SelectionDAG &DAG,
1731                         const RISCVSubtarget &Subtarget) {
1732   assert(VecVT.isScalableVector() && "Expecting a scalable vector");
1733   return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
1734 }
1735 
1736 // The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
1737 // of either is (currently) supported. This can get us into an infinite loop
1738 // where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
1739 // as a ..., etc.
1740 // Until either (or both) of these can reliably lower any node, reporting that
1741 // we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
1742 // the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
1743 // which is not desirable.
1744 bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
1745     EVT VT, unsigned DefinedValues) const {
1746   return false;
1747 }
1748 
1749 bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
1750   // Only splats are currently supported.
1751   if (ShuffleVectorSDNode::isSplatMask(M.data(), VT))
1752     return true;
1753 
1754   return false;
1755 }
1756 
1757 static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
1758                                   const RISCVSubtarget &Subtarget) {
1759   // RISCV FP-to-int conversions saturate to the destination register size, but
1760   // don't produce 0 for nan. We can use a conversion instruction and fix the
1761   // nan case with a compare and a select.
1762   SDValue Src = Op.getOperand(0);
1763 
1764   EVT DstVT = Op.getValueType();
1765   EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1766 
1767   bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
1768   unsigned Opc;
1769   if (SatVT == DstVT)
1770     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
1771   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
1772     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
1773   else
1774     return SDValue();
1775   // FIXME: Support other SatVTs by clamping before or after the conversion.
1776 
1777   SDLoc DL(Op);
1778   SDValue FpToInt = DAG.getNode(
1779       Opc, DL, DstVT, Src,
1780       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
1781 
1782   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
1783   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
1784 }
1785 
1786 // Expand vector FTRUNC, FCEIL, and FFLOOR by converting to the integer domain
1787 // and back. Taking care to avoid converting values that are nan or already
1788 // correct.
1789 // TODO: Floor and ceil could be shorter by changing rounding mode, but we don't
1790 // have FRM dependencies modeled yet.
1791 static SDValue lowerFTRUNC_FCEIL_FFLOOR(SDValue Op, SelectionDAG &DAG) {
1792   MVT VT = Op.getSimpleValueType();
1793   assert(VT.isVector() && "Unexpected type");
1794 
1795   SDLoc DL(Op);
1796 
1797   // Freeze the source since we are increasing the number of uses.
1798   SDValue Src = DAG.getNode(ISD::FREEZE, DL, VT, Op.getOperand(0));
1799 
1800   // Truncate to integer and convert back to FP.
1801   MVT IntVT = VT.changeVectorElementTypeToInteger();
1802   SDValue Truncated = DAG.getNode(ISD::FP_TO_SINT, DL, IntVT, Src);
1803   Truncated = DAG.getNode(ISD::SINT_TO_FP, DL, VT, Truncated);
1804 
1805   MVT SetccVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
1806 
1807   if (Op.getOpcode() == ISD::FCEIL) {
1808     // If the truncated value is the greater than or equal to the original
1809     // value, we've computed the ceil. Otherwise, we went the wrong way and
1810     // need to increase by 1.
1811     // FIXME: This should use a masked operation. Handle here or in isel?
1812     SDValue Adjust = DAG.getNode(ISD::FADD, DL, VT, Truncated,
1813                                  DAG.getConstantFP(1.0, DL, VT));
1814     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOLT);
1815     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1816   } else if (Op.getOpcode() == ISD::FFLOOR) {
1817     // If the truncated value is the less than or equal to the original value,
1818     // we've computed the floor. Otherwise, we went the wrong way and need to
1819     // decrease by 1.
1820     // FIXME: This should use a masked operation. Handle here or in isel?
1821     SDValue Adjust = DAG.getNode(ISD::FSUB, DL, VT, Truncated,
1822                                  DAG.getConstantFP(1.0, DL, VT));
1823     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOGT);
1824     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1825   }
1826 
1827   // Restore the original sign so that -0.0 is preserved.
1828   Truncated = DAG.getNode(ISD::FCOPYSIGN, DL, VT, Truncated, Src);
1829 
1830   // Determine the largest integer that can be represented exactly. This and
1831   // values larger than it don't have any fractional bits so don't need to
1832   // be converted.
1833   const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
1834   unsigned Precision = APFloat::semanticsPrecision(FltSem);
1835   APFloat MaxVal = APFloat(FltSem);
1836   MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
1837                           /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
1838   SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
1839 
1840   // If abs(Src) was larger than MaxVal or nan, keep it.
1841   SDValue Abs = DAG.getNode(ISD::FABS, DL, VT, Src);
1842   SDValue Setcc = DAG.getSetCC(DL, SetccVT, Abs, MaxValNode, ISD::SETOLT);
1843   return DAG.getSelect(DL, VT, Setcc, Truncated, Src);
1844 }
1845 
1846 static SDValue lowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG,
1847                                  const RISCVSubtarget &Subtarget) {
1848   MVT VT = Op.getSimpleValueType();
1849   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1850 
1851   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1852 
1853   SDLoc DL(Op);
1854   SDValue Mask, VL;
1855   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1856 
1857   unsigned Opc =
1858       VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
1859   SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, Op.getOperand(0), VL);
1860   return convertFromScalableVector(VT, Splat, DAG, Subtarget);
1861 }
1862 
1863 struct VIDSequence {
1864   int64_t StepNumerator;
1865   unsigned StepDenominator;
1866   int64_t Addend;
1867 };
1868 
1869 // Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
1870 // to the (non-zero) step S and start value X. This can be then lowered as the
1871 // RVV sequence (VID * S) + X, for example.
1872 // The step S is represented as an integer numerator divided by a positive
1873 // denominator. Note that the implementation currently only identifies
1874 // sequences in which either the numerator is +/- 1 or the denominator is 1. It
1875 // cannot detect 2/3, for example.
1876 // Note that this method will also match potentially unappealing index
1877 // sequences, like <i32 0, i32 50939494>, however it is left to the caller to
1878 // determine whether this is worth generating code for.
1879 static Optional<VIDSequence> isSimpleVIDSequence(SDValue Op) {
1880   unsigned NumElts = Op.getNumOperands();
1881   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
1882   if (!Op.getValueType().isInteger())
1883     return None;
1884 
1885   Optional<unsigned> SeqStepDenom;
1886   Optional<int64_t> SeqStepNum, SeqAddend;
1887   Optional<std::pair<uint64_t, unsigned>> PrevElt;
1888   unsigned EltSizeInBits = Op.getValueType().getScalarSizeInBits();
1889   for (unsigned Idx = 0; Idx < NumElts; Idx++) {
1890     // Assume undef elements match the sequence; we just have to be careful
1891     // when interpolating across them.
1892     if (Op.getOperand(Idx).isUndef())
1893       continue;
1894     // The BUILD_VECTOR must be all constants.
1895     if (!isa<ConstantSDNode>(Op.getOperand(Idx)))
1896       return None;
1897 
1898     uint64_t Val = Op.getConstantOperandVal(Idx) &
1899                    maskTrailingOnes<uint64_t>(EltSizeInBits);
1900 
1901     if (PrevElt) {
1902       // Calculate the step since the last non-undef element, and ensure
1903       // it's consistent across the entire sequence.
1904       unsigned IdxDiff = Idx - PrevElt->second;
1905       int64_t ValDiff = SignExtend64(Val - PrevElt->first, EltSizeInBits);
1906 
1907       // A zero-value value difference means that we're somewhere in the middle
1908       // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
1909       // step change before evaluating the sequence.
1910       if (ValDiff != 0) {
1911         int64_t Remainder = ValDiff % IdxDiff;
1912         // Normalize the step if it's greater than 1.
1913         if (Remainder != ValDiff) {
1914           // The difference must cleanly divide the element span.
1915           if (Remainder != 0)
1916             return None;
1917           ValDiff /= IdxDiff;
1918           IdxDiff = 1;
1919         }
1920 
1921         if (!SeqStepNum)
1922           SeqStepNum = ValDiff;
1923         else if (ValDiff != SeqStepNum)
1924           return None;
1925 
1926         if (!SeqStepDenom)
1927           SeqStepDenom = IdxDiff;
1928         else if (IdxDiff != *SeqStepDenom)
1929           return None;
1930       }
1931     }
1932 
1933     // Record and/or check any addend.
1934     if (SeqStepNum && SeqStepDenom) {
1935       uint64_t ExpectedVal =
1936           (int64_t)(Idx * (uint64_t)*SeqStepNum) / *SeqStepDenom;
1937       int64_t Addend = SignExtend64(Val - ExpectedVal, EltSizeInBits);
1938       if (!SeqAddend)
1939         SeqAddend = Addend;
1940       else if (SeqAddend != Addend)
1941         return None;
1942     }
1943 
1944     // Record this non-undef element for later.
1945     if (!PrevElt || PrevElt->first != Val)
1946       PrevElt = std::make_pair(Val, Idx);
1947   }
1948   // We need to have logged both a step and an addend for this to count as
1949   // a legal index sequence.
1950   if (!SeqStepNum || !SeqStepDenom || !SeqAddend)
1951     return None;
1952 
1953   return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
1954 }
1955 
1956 static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
1957                                  const RISCVSubtarget &Subtarget) {
1958   MVT VT = Op.getSimpleValueType();
1959   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1960 
1961   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1962 
1963   SDLoc DL(Op);
1964   SDValue Mask, VL;
1965   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1966 
1967   MVT XLenVT = Subtarget.getXLenVT();
1968   unsigned NumElts = Op.getNumOperands();
1969 
1970   if (VT.getVectorElementType() == MVT::i1) {
1971     if (ISD::isBuildVectorAllZeros(Op.getNode())) {
1972       SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
1973       return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
1974     }
1975 
1976     if (ISD::isBuildVectorAllOnes(Op.getNode())) {
1977       SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
1978       return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
1979     }
1980 
1981     // Lower constant mask BUILD_VECTORs via an integer vector type, in
1982     // scalar integer chunks whose bit-width depends on the number of mask
1983     // bits and XLEN.
1984     // First, determine the most appropriate scalar integer type to use. This
1985     // is at most XLenVT, but may be shrunk to a smaller vector element type
1986     // according to the size of the final vector - use i8 chunks rather than
1987     // XLenVT if we're producing a v8i1. This results in more consistent
1988     // codegen across RV32 and RV64.
1989     unsigned NumViaIntegerBits =
1990         std::min(std::max(NumElts, 8u), Subtarget.getXLen());
1991     NumViaIntegerBits = std::min(NumViaIntegerBits,
1992                                  Subtarget.getMaxELENForFixedLengthVectors());
1993     if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
1994       // If we have to use more than one INSERT_VECTOR_ELT then this
1995       // optimization is likely to increase code size; avoid peforming it in
1996       // such a case. We can use a load from a constant pool in this case.
1997       if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
1998         return SDValue();
1999       // Now we can create our integer vector type. Note that it may be larger
2000       // than the resulting mask type: v4i1 would use v1i8 as its integer type.
2001       MVT IntegerViaVecVT =
2002           MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
2003                            divideCeil(NumElts, NumViaIntegerBits));
2004 
2005       uint64_t Bits = 0;
2006       unsigned BitPos = 0, IntegerEltIdx = 0;
2007       SDValue Vec = DAG.getUNDEF(IntegerViaVecVT);
2008 
2009       for (unsigned I = 0; I < NumElts; I++, BitPos++) {
2010         // Once we accumulate enough bits to fill our scalar type, insert into
2011         // our vector and clear our accumulated data.
2012         if (I != 0 && I % NumViaIntegerBits == 0) {
2013           if (NumViaIntegerBits <= 32)
2014             Bits = SignExtend64(Bits, 32);
2015           SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2016           Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec,
2017                             Elt, DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2018           Bits = 0;
2019           BitPos = 0;
2020           IntegerEltIdx++;
2021         }
2022         SDValue V = Op.getOperand(I);
2023         bool BitValue = !V.isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
2024         Bits |= ((uint64_t)BitValue << BitPos);
2025       }
2026 
2027       // Insert the (remaining) scalar value into position in our integer
2028       // vector type.
2029       if (NumViaIntegerBits <= 32)
2030         Bits = SignExtend64(Bits, 32);
2031       SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2032       Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, Elt,
2033                         DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2034 
2035       if (NumElts < NumViaIntegerBits) {
2036         // If we're producing a smaller vector than our minimum legal integer
2037         // type, bitcast to the equivalent (known-legal) mask type, and extract
2038         // our final mask.
2039         assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
2040         Vec = DAG.getBitcast(MVT::v8i1, Vec);
2041         Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
2042                           DAG.getConstant(0, DL, XLenVT));
2043       } else {
2044         // Else we must have produced an integer type with the same size as the
2045         // mask type; bitcast for the final result.
2046         assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
2047         Vec = DAG.getBitcast(VT, Vec);
2048       }
2049 
2050       return Vec;
2051     }
2052 
2053     // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
2054     // vector type, we have a legal equivalently-sized i8 type, so we can use
2055     // that.
2056     MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
2057     SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
2058 
2059     SDValue WideVec;
2060     if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2061       // For a splat, perform a scalar truncate before creating the wider
2062       // vector.
2063       assert(Splat.getValueType() == XLenVT &&
2064              "Unexpected type for i1 splat value");
2065       Splat = DAG.getNode(ISD::AND, DL, XLenVT, Splat,
2066                           DAG.getConstant(1, DL, XLenVT));
2067       WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
2068     } else {
2069       SmallVector<SDValue, 8> Ops(Op->op_values());
2070       WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
2071       SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
2072       WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
2073     }
2074 
2075     return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
2076   }
2077 
2078   if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2079     unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
2080                                         : RISCVISD::VMV_V_X_VL;
2081     Splat = DAG.getNode(Opc, DL, ContainerVT, Splat, VL);
2082     return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2083   }
2084 
2085   // Try and match index sequences, which we can lower to the vid instruction
2086   // with optional modifications. An all-undef vector is matched by
2087   // getSplatValue, above.
2088   if (auto SimpleVID = isSimpleVIDSequence(Op)) {
2089     int64_t StepNumerator = SimpleVID->StepNumerator;
2090     unsigned StepDenominator = SimpleVID->StepDenominator;
2091     int64_t Addend = SimpleVID->Addend;
2092 
2093     assert(StepNumerator != 0 && "Invalid step");
2094     bool Negate = false;
2095     int64_t SplatStepVal = StepNumerator;
2096     unsigned StepOpcode = ISD::MUL;
2097     if (StepNumerator != 1) {
2098       if (isPowerOf2_64(std::abs(StepNumerator))) {
2099         Negate = StepNumerator < 0;
2100         StepOpcode = ISD::SHL;
2101         SplatStepVal = Log2_64(std::abs(StepNumerator));
2102       }
2103     }
2104 
2105     // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
2106     // threshold since it's the immediate value many RVV instructions accept.
2107     // There is no vmul.vi instruction so ensure multiply constant can fit in
2108     // a single addi instruction.
2109     if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
2110          (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
2111         isPowerOf2_32(StepDenominator) && isInt<5>(Addend)) {
2112       SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
2113       // Convert right out of the scalable type so we can use standard ISD
2114       // nodes for the rest of the computation. If we used scalable types with
2115       // these, we'd lose the fixed-length vector info and generate worse
2116       // vsetvli code.
2117       VID = convertFromScalableVector(VT, VID, DAG, Subtarget);
2118       if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
2119           (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
2120         SDValue SplatStep = DAG.getSplatVector(
2121             VT, DL, DAG.getConstant(SplatStepVal, DL, XLenVT));
2122         VID = DAG.getNode(StepOpcode, DL, VT, VID, SplatStep);
2123       }
2124       if (StepDenominator != 1) {
2125         SDValue SplatStep = DAG.getSplatVector(
2126             VT, DL, DAG.getConstant(Log2_64(StepDenominator), DL, XLenVT));
2127         VID = DAG.getNode(ISD::SRL, DL, VT, VID, SplatStep);
2128       }
2129       if (Addend != 0 || Negate) {
2130         SDValue SplatAddend =
2131             DAG.getSplatVector(VT, DL, DAG.getConstant(Addend, DL, XLenVT));
2132         VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VT, SplatAddend, VID);
2133       }
2134       return VID;
2135     }
2136   }
2137 
2138   // Attempt to detect "hidden" splats, which only reveal themselves as splats
2139   // when re-interpreted as a vector with a larger element type. For example,
2140   //   v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
2141   // could be instead splat as
2142   //   v2i32 = build_vector i32 0x00010000, i32 0x00010000
2143   // TODO: This optimization could also work on non-constant splats, but it
2144   // would require bit-manipulation instructions to construct the splat value.
2145   SmallVector<SDValue> Sequence;
2146   unsigned EltBitSize = VT.getScalarSizeInBits();
2147   const auto *BV = cast<BuildVectorSDNode>(Op);
2148   if (VT.isInteger() && EltBitSize < 64 &&
2149       ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
2150       BV->getRepeatedSequence(Sequence) &&
2151       (Sequence.size() * EltBitSize) <= 64) {
2152     unsigned SeqLen = Sequence.size();
2153     MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
2154     MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, NumElts / SeqLen);
2155     assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
2156             ViaIntVT == MVT::i64) &&
2157            "Unexpected sequence type");
2158 
2159     unsigned EltIdx = 0;
2160     uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
2161     uint64_t SplatValue = 0;
2162     // Construct the amalgamated value which can be splatted as this larger
2163     // vector type.
2164     for (const auto &SeqV : Sequence) {
2165       if (!SeqV.isUndef())
2166         SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
2167                        << (EltIdx * EltBitSize));
2168       EltIdx++;
2169     }
2170 
2171     // On RV64, sign-extend from 32 to 64 bits where possible in order to
2172     // achieve better constant materializion.
2173     if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
2174       SplatValue = SignExtend64(SplatValue, 32);
2175 
2176     // Since we can't introduce illegal i64 types at this stage, we can only
2177     // perform an i64 splat on RV32 if it is its own sign-extended value. That
2178     // way we can use RVV instructions to splat.
2179     assert((ViaIntVT.bitsLE(XLenVT) ||
2180             (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
2181            "Unexpected bitcast sequence");
2182     if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
2183       SDValue ViaVL =
2184           DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
2185       MVT ViaContainerVT =
2186           getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget);
2187       SDValue Splat =
2188           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
2189                       DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
2190       Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
2191       return DAG.getBitcast(VT, Splat);
2192     }
2193   }
2194 
2195   // Try and optimize BUILD_VECTORs with "dominant values" - these are values
2196   // which constitute a large proportion of the elements. In such cases we can
2197   // splat a vector with the dominant element and make up the shortfall with
2198   // INSERT_VECTOR_ELTs.
2199   // Note that this includes vectors of 2 elements by association. The
2200   // upper-most element is the "dominant" one, allowing us to use a splat to
2201   // "insert" the upper element, and an insert of the lower element at position
2202   // 0, which improves codegen.
2203   SDValue DominantValue;
2204   unsigned MostCommonCount = 0;
2205   DenseMap<SDValue, unsigned> ValueCounts;
2206   unsigned NumUndefElts =
2207       count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
2208 
2209   // Track the number of scalar loads we know we'd be inserting, estimated as
2210   // any non-zero floating-point constant. Other kinds of element are either
2211   // already in registers or are materialized on demand. The threshold at which
2212   // a vector load is more desirable than several scalar materializion and
2213   // vector-insertion instructions is not known.
2214   unsigned NumScalarLoads = 0;
2215 
2216   for (SDValue V : Op->op_values()) {
2217     if (V.isUndef())
2218       continue;
2219 
2220     ValueCounts.insert(std::make_pair(V, 0));
2221     unsigned &Count = ValueCounts[V];
2222 
2223     if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
2224       NumScalarLoads += !CFP->isExactlyValue(+0.0);
2225 
2226     // Is this value dominant? In case of a tie, prefer the highest element as
2227     // it's cheaper to insert near the beginning of a vector than it is at the
2228     // end.
2229     if (++Count >= MostCommonCount) {
2230       DominantValue = V;
2231       MostCommonCount = Count;
2232     }
2233   }
2234 
2235   assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
2236   unsigned NumDefElts = NumElts - NumUndefElts;
2237   unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
2238 
2239   // Don't perform this optimization when optimizing for size, since
2240   // materializing elements and inserting them tends to cause code bloat.
2241   if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
2242       ((MostCommonCount > DominantValueCountThreshold) ||
2243        (ValueCounts.size() <= Log2_32(NumDefElts)))) {
2244     // Start by splatting the most common element.
2245     SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
2246 
2247     DenseSet<SDValue> Processed{DominantValue};
2248     MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
2249     for (const auto &OpIdx : enumerate(Op->ops())) {
2250       const SDValue &V = OpIdx.value();
2251       if (V.isUndef() || !Processed.insert(V).second)
2252         continue;
2253       if (ValueCounts[V] == 1) {
2254         Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
2255                           DAG.getConstant(OpIdx.index(), DL, XLenVT));
2256       } else {
2257         // Blend in all instances of this value using a VSELECT, using a
2258         // mask where each bit signals whether that element is the one
2259         // we're after.
2260         SmallVector<SDValue> Ops;
2261         transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
2262           return DAG.getConstant(V == V1, DL, XLenVT);
2263         });
2264         Vec = DAG.getNode(ISD::VSELECT, DL, VT,
2265                           DAG.getBuildVector(SelMaskTy, DL, Ops),
2266                           DAG.getSplatBuildVector(VT, DL, V), Vec);
2267       }
2268     }
2269 
2270     return Vec;
2271   }
2272 
2273   return SDValue();
2274 }
2275 
2276 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Lo,
2277                                    SDValue Hi, SDValue VL, SelectionDAG &DAG) {
2278   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
2279     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
2280     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
2281     // If Hi constant is all the same sign bit as Lo, lower this as a custom
2282     // node in order to try and match RVV vector/scalar instructions.
2283     if ((LoC >> 31) == HiC)
2284       return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Lo, VL);
2285 
2286     // If vl is equal to VLMax and Hi constant is equal to Lo, we could use
2287     // vmv.v.x whose EEW = 32 to lower it.
2288     auto *Const = dyn_cast<ConstantSDNode>(VL);
2289     if (LoC == HiC && Const && Const->getSExtValue() == RISCV::VLMaxSentinel) {
2290       MVT InterVT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
2291       // TODO: if vl <= min(VLMAX), we can also do this. But we could not
2292       // access the subtarget here now.
2293       auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT, Lo, VL);
2294       return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
2295     }
2296   }
2297 
2298   // Fall back to a stack store and stride x0 vector load.
2299   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Lo, Hi, VL);
2300 }
2301 
2302 // Called by type legalization to handle splat of i64 on RV32.
2303 // FIXME: We can optimize this when the type has sign or zero bits in one
2304 // of the halves.
2305 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Scalar,
2306                                    SDValue VL, SelectionDAG &DAG) {
2307   assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
2308   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2309                            DAG.getConstant(0, DL, MVT::i32));
2310   SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2311                            DAG.getConstant(1, DL, MVT::i32));
2312   return splatPartsI64WithVL(DL, VT, Lo, Hi, VL, DAG);
2313 }
2314 
2315 // This function lowers a splat of a scalar operand Splat with the vector
2316 // length VL. It ensures the final sequence is type legal, which is useful when
2317 // lowering a splat after type legalization.
2318 static SDValue lowerScalarSplat(SDValue Scalar, SDValue VL, MVT VT, SDLoc DL,
2319                                 SelectionDAG &DAG,
2320                                 const RISCVSubtarget &Subtarget) {
2321   if (VT.isFloatingPoint()) {
2322     // If VL is 1, we could use vfmv.s.f.
2323     if (isOneConstant(VL))
2324       return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT),
2325                          Scalar, VL);
2326     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Scalar, VL);
2327   }
2328 
2329   MVT XLenVT = Subtarget.getXLenVT();
2330 
2331   // Simplest case is that the operand needs to be promoted to XLenVT.
2332   if (Scalar.getValueType().bitsLE(XLenVT)) {
2333     // If the operand is a constant, sign extend to increase our chances
2334     // of being able to use a .vi instruction. ANY_EXTEND would become a
2335     // a zero extend and the simm5 check in isel would fail.
2336     // FIXME: Should we ignore the upper bits in isel instead?
2337     unsigned ExtOpc =
2338         isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
2339     Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
2340     ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Scalar);
2341     // If VL is 1 and the scalar value won't benefit from immediate, we could
2342     // use vmv.s.x.
2343     if (isOneConstant(VL) &&
2344         (!Const || isNullConstant(Scalar) || !isInt<5>(Const->getSExtValue())))
2345       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
2346                          VL);
2347     return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Scalar, VL);
2348   }
2349 
2350   assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
2351          "Unexpected scalar for splat lowering!");
2352 
2353   if (isOneConstant(VL) && isNullConstant(Scalar))
2354     return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT),
2355                        DAG.getConstant(0, DL, XLenVT), VL);
2356 
2357   // Otherwise use the more complicated splatting algorithm.
2358   return splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
2359 }
2360 
2361 // Is the mask a slidedown that shifts in undefs.
2362 static int matchShuffleAsSlideDown(ArrayRef<int> Mask) {
2363   int Size = Mask.size();
2364 
2365   // Elements shifted in should be undef.
2366   auto CheckUndefs = [&](int Shift) {
2367     for (int i = Size - Shift; i != Size; ++i)
2368       if (Mask[i] >= 0)
2369         return false;
2370     return true;
2371   };
2372 
2373   // Elements should be shifted or undef.
2374   auto MatchShift = [&](int Shift) {
2375     for (int i = 0; i != Size - Shift; ++i)
2376        if (Mask[i] >= 0 && Mask[i] != Shift + i)
2377          return false;
2378     return true;
2379   };
2380 
2381   // Try all possible shifts.
2382   for (int Shift = 1; Shift != Size; ++Shift)
2383     if (CheckUndefs(Shift) && MatchShift(Shift))
2384       return Shift;
2385 
2386   // No match.
2387   return -1;
2388 }
2389 
2390 static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, bool &SwapSources,
2391                                 const RISCVSubtarget &Subtarget) {
2392   // We need to be able to widen elements to the next larger integer type.
2393   if (VT.getScalarSizeInBits() >= Subtarget.getMaxELENForFixedLengthVectors())
2394     return false;
2395 
2396   int Size = Mask.size();
2397   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
2398 
2399   int Srcs[] = {-1, -1};
2400   for (int i = 0; i != Size; ++i) {
2401     // Ignore undef elements.
2402     if (Mask[i] < 0)
2403       continue;
2404 
2405     // Is this an even or odd element.
2406     int Pol = i % 2;
2407 
2408     // Ensure we consistently use the same source for this element polarity.
2409     int Src = Mask[i] / Size;
2410     if (Srcs[Pol] < 0)
2411       Srcs[Pol] = Src;
2412     if (Srcs[Pol] != Src)
2413       return false;
2414 
2415     // Make sure the element within the source is appropriate for this element
2416     // in the destination.
2417     int Elt = Mask[i] % Size;
2418     if (Elt != i / 2)
2419       return false;
2420   }
2421 
2422   // We need to find a source for each polarity and they can't be the same.
2423   if (Srcs[0] < 0 || Srcs[1] < 0 || Srcs[0] == Srcs[1])
2424     return false;
2425 
2426   // Swap the sources if the second source was in the even polarity.
2427   SwapSources = Srcs[0] > Srcs[1];
2428 
2429   return true;
2430 }
2431 
2432 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
2433                                    const RISCVSubtarget &Subtarget) {
2434   SDValue V1 = Op.getOperand(0);
2435   SDValue V2 = Op.getOperand(1);
2436   SDLoc DL(Op);
2437   MVT XLenVT = Subtarget.getXLenVT();
2438   MVT VT = Op.getSimpleValueType();
2439   unsigned NumElts = VT.getVectorNumElements();
2440   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
2441 
2442   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2443 
2444   SDValue TrueMask, VL;
2445   std::tie(TrueMask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2446 
2447   if (SVN->isSplat()) {
2448     const int Lane = SVN->getSplatIndex();
2449     if (Lane >= 0) {
2450       MVT SVT = VT.getVectorElementType();
2451 
2452       // Turn splatted vector load into a strided load with an X0 stride.
2453       SDValue V = V1;
2454       // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
2455       // with undef.
2456       // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
2457       int Offset = Lane;
2458       if (V.getOpcode() == ISD::CONCAT_VECTORS) {
2459         int OpElements =
2460             V.getOperand(0).getSimpleValueType().getVectorNumElements();
2461         V = V.getOperand(Offset / OpElements);
2462         Offset %= OpElements;
2463       }
2464 
2465       // We need to ensure the load isn't atomic or volatile.
2466       if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
2467         auto *Ld = cast<LoadSDNode>(V);
2468         Offset *= SVT.getStoreSize();
2469         SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(),
2470                                                    TypeSize::Fixed(Offset), DL);
2471 
2472         // If this is SEW=64 on RV32, use a strided load with a stride of x0.
2473         if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
2474           SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
2475           SDValue IntID =
2476               DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
2477           SDValue Ops[] = {Ld->getChain(),
2478                            IntID,
2479                            DAG.getUNDEF(ContainerVT),
2480                            NewAddr,
2481                            DAG.getRegister(RISCV::X0, XLenVT),
2482                            VL};
2483           SDValue NewLoad = DAG.getMemIntrinsicNode(
2484               ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
2485               DAG.getMachineFunction().getMachineMemOperand(
2486                   Ld->getMemOperand(), Offset, SVT.getStoreSize()));
2487           DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
2488           return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
2489         }
2490 
2491         // Otherwise use a scalar load and splat. This will give the best
2492         // opportunity to fold a splat into the operation. ISel can turn it into
2493         // the x0 strided load if we aren't able to fold away the select.
2494         if (SVT.isFloatingPoint())
2495           V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
2496                           Ld->getPointerInfo().getWithOffset(Offset),
2497                           Ld->getOriginalAlign(),
2498                           Ld->getMemOperand()->getFlags());
2499         else
2500           V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
2501                              Ld->getPointerInfo().getWithOffset(Offset), SVT,
2502                              Ld->getOriginalAlign(),
2503                              Ld->getMemOperand()->getFlags());
2504         DAG.makeEquivalentMemoryOrdering(Ld, V);
2505 
2506         unsigned Opc =
2507             VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
2508         SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, V, VL);
2509         return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2510       }
2511 
2512       V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2513       assert(Lane < (int)NumElts && "Unexpected lane!");
2514       SDValue Gather =
2515           DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
2516                       DAG.getConstant(Lane, DL, XLenVT), TrueMask, VL);
2517       return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2518     }
2519   }
2520 
2521   ArrayRef<int> Mask = SVN->getMask();
2522 
2523   // Try to match as a slidedown.
2524   int SlideAmt = matchShuffleAsSlideDown(Mask);
2525   if (SlideAmt >= 0) {
2526     // TODO: Should we reduce the VL to account for the upper undef elements?
2527     // Requires additional vsetvlis, but might be faster to execute.
2528     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2529     SDValue SlideDown =
2530         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
2531                     DAG.getUNDEF(ContainerVT), V1,
2532                     DAG.getConstant(SlideAmt, DL, XLenVT),
2533                     TrueMask, VL);
2534     return convertFromScalableVector(VT, SlideDown, DAG, Subtarget);
2535   }
2536 
2537   // Detect an interleave shuffle and lower to
2538   // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
2539   bool SwapSources;
2540   if (isInterleaveShuffle(Mask, VT, SwapSources, Subtarget)) {
2541     // Swap sources if needed.
2542     if (SwapSources)
2543       std::swap(V1, V2);
2544 
2545     // Extract the lower half of the vectors.
2546     MVT HalfVT = VT.getHalfNumVectorElementsVT();
2547     V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
2548                      DAG.getConstant(0, DL, XLenVT));
2549     V2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V2,
2550                      DAG.getConstant(0, DL, XLenVT));
2551 
2552     // Double the element width and halve the number of elements in an int type.
2553     unsigned EltBits = VT.getScalarSizeInBits();
2554     MVT WideIntEltVT = MVT::getIntegerVT(EltBits * 2);
2555     MVT WideIntVT =
2556         MVT::getVectorVT(WideIntEltVT, VT.getVectorNumElements() / 2);
2557     // Convert this to a scalable vector. We need to base this on the
2558     // destination size to ensure there's always a type with a smaller LMUL.
2559     MVT WideIntContainerVT =
2560         getContainerForFixedLengthVector(DAG, WideIntVT, Subtarget);
2561 
2562     // Convert sources to scalable vectors with the same element count as the
2563     // larger type.
2564     MVT HalfContainerVT = MVT::getVectorVT(
2565         VT.getVectorElementType(), WideIntContainerVT.getVectorElementCount());
2566     V1 = convertToScalableVector(HalfContainerVT, V1, DAG, Subtarget);
2567     V2 = convertToScalableVector(HalfContainerVT, V2, DAG, Subtarget);
2568 
2569     // Cast sources to integer.
2570     MVT IntEltVT = MVT::getIntegerVT(EltBits);
2571     MVT IntHalfVT =
2572         MVT::getVectorVT(IntEltVT, HalfContainerVT.getVectorElementCount());
2573     V1 = DAG.getBitcast(IntHalfVT, V1);
2574     V2 = DAG.getBitcast(IntHalfVT, V2);
2575 
2576     // Freeze V2 since we use it twice and we need to be sure that the add and
2577     // multiply see the same value.
2578     V2 = DAG.getNode(ISD::FREEZE, DL, IntHalfVT, V2);
2579 
2580     // Recreate TrueMask using the widened type's element count.
2581     MVT MaskVT =
2582         MVT::getVectorVT(MVT::i1, HalfContainerVT.getVectorElementCount());
2583     TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
2584 
2585     // Widen V1 and V2 with 0s and add one copy of V2 to V1.
2586     SDValue Add = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideIntContainerVT, V1,
2587                               V2, TrueMask, VL);
2588     // Create 2^eltbits - 1 copies of V2 by multiplying by the largest integer.
2589     SDValue Multiplier = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntHalfVT,
2590                                      DAG.getAllOnesConstant(DL, XLenVT));
2591     SDValue WidenMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideIntContainerVT,
2592                                    V2, Multiplier, TrueMask, VL);
2593     // Add the new copies to our previous addition giving us 2^eltbits copies of
2594     // V2. This is equivalent to shifting V2 left by eltbits. This should
2595     // combine with the vwmulu.vv above to form vwmaccu.vv.
2596     Add = DAG.getNode(RISCVISD::ADD_VL, DL, WideIntContainerVT, Add, WidenMul,
2597                       TrueMask, VL);
2598     // Cast back to ContainerVT. We need to re-create a new ContainerVT in case
2599     // WideIntContainerVT is a larger fractional LMUL than implied by the fixed
2600     // vector VT.
2601     ContainerVT =
2602         MVT::getVectorVT(VT.getVectorElementType(),
2603                          WideIntContainerVT.getVectorElementCount() * 2);
2604     Add = DAG.getBitcast(ContainerVT, Add);
2605     return convertFromScalableVector(VT, Add, DAG, Subtarget);
2606   }
2607 
2608   // Detect shuffles which can be re-expressed as vector selects; these are
2609   // shuffles in which each element in the destination is taken from an element
2610   // at the corresponding index in either source vectors.
2611   bool IsSelect = all_of(enumerate(Mask), [&](const auto &MaskIdx) {
2612     int MaskIndex = MaskIdx.value();
2613     return MaskIndex < 0 || MaskIdx.index() == (unsigned)MaskIndex % NumElts;
2614   });
2615 
2616   assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
2617 
2618   SmallVector<SDValue> MaskVals;
2619   // As a backup, shuffles can be lowered via a vrgather instruction, possibly
2620   // merged with a second vrgather.
2621   SmallVector<SDValue> GatherIndicesLHS, GatherIndicesRHS;
2622 
2623   // By default we preserve the original operand order, and use a mask to
2624   // select LHS as true and RHS as false. However, since RVV vector selects may
2625   // feature splats but only on the LHS, we may choose to invert our mask and
2626   // instead select between RHS and LHS.
2627   bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
2628   bool InvertMask = IsSelect == SwapOps;
2629 
2630   // Keep a track of which non-undef indices are used by each LHS/RHS shuffle
2631   // half.
2632   DenseMap<int, unsigned> LHSIndexCounts, RHSIndexCounts;
2633 
2634   // Now construct the mask that will be used by the vselect or blended
2635   // vrgather operation. For vrgathers, construct the appropriate indices into
2636   // each vector.
2637   for (int MaskIndex : Mask) {
2638     bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ InvertMask;
2639     MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
2640     if (!IsSelect) {
2641       bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
2642       GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
2643                                      ? DAG.getConstant(MaskIndex, DL, XLenVT)
2644                                      : DAG.getUNDEF(XLenVT));
2645       GatherIndicesRHS.push_back(
2646           IsLHSOrUndefIndex ? DAG.getUNDEF(XLenVT)
2647                             : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT));
2648       if (IsLHSOrUndefIndex && MaskIndex >= 0)
2649         ++LHSIndexCounts[MaskIndex];
2650       if (!IsLHSOrUndefIndex)
2651         ++RHSIndexCounts[MaskIndex - NumElts];
2652     }
2653   }
2654 
2655   if (SwapOps) {
2656     std::swap(V1, V2);
2657     std::swap(GatherIndicesLHS, GatherIndicesRHS);
2658   }
2659 
2660   assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
2661   MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
2662   SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
2663 
2664   if (IsSelect)
2665     return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
2666 
2667   if (VT.getScalarSizeInBits() == 8 && VT.getVectorNumElements() > 256) {
2668     // On such a large vector we're unable to use i8 as the index type.
2669     // FIXME: We could promote the index to i16 and use vrgatherei16, but that
2670     // may involve vector splitting if we're already at LMUL=8, or our
2671     // user-supplied maximum fixed-length LMUL.
2672     return SDValue();
2673   }
2674 
2675   unsigned GatherVXOpc = RISCVISD::VRGATHER_VX_VL;
2676   unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
2677   MVT IndexVT = VT.changeTypeToInteger();
2678   // Since we can't introduce illegal index types at this stage, use i16 and
2679   // vrgatherei16 if the corresponding index type for plain vrgather is greater
2680   // than XLenVT.
2681   if (IndexVT.getScalarType().bitsGT(XLenVT)) {
2682     GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
2683     IndexVT = IndexVT.changeVectorElementType(MVT::i16);
2684   }
2685 
2686   MVT IndexContainerVT =
2687       ContainerVT.changeVectorElementType(IndexVT.getScalarType());
2688 
2689   SDValue Gather;
2690   // TODO: This doesn't trigger for i64 vectors on RV32, since there we
2691   // encounter a bitcasted BUILD_VECTOR with low/high i32 values.
2692   if (SDValue SplatValue = DAG.getSplatValue(V1, /*LegalTypes*/ true)) {
2693     Gather = lowerScalarSplat(SplatValue, VL, ContainerVT, DL, DAG, Subtarget);
2694   } else {
2695     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2696     // If only one index is used, we can use a "splat" vrgather.
2697     // TODO: We can splat the most-common index and fix-up any stragglers, if
2698     // that's beneficial.
2699     if (LHSIndexCounts.size() == 1) {
2700       int SplatIndex = LHSIndexCounts.begin()->getFirst();
2701       Gather =
2702           DAG.getNode(GatherVXOpc, DL, ContainerVT, V1,
2703                       DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2704     } else {
2705       SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
2706       LHSIndices =
2707           convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
2708 
2709       Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
2710                            TrueMask, VL);
2711     }
2712   }
2713 
2714   // If a second vector operand is used by this shuffle, blend it in with an
2715   // additional vrgather.
2716   if (!V2.isUndef()) {
2717     V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
2718     // If only one index is used, we can use a "splat" vrgather.
2719     // TODO: We can splat the most-common index and fix-up any stragglers, if
2720     // that's beneficial.
2721     if (RHSIndexCounts.size() == 1) {
2722       int SplatIndex = RHSIndexCounts.begin()->getFirst();
2723       V2 = DAG.getNode(GatherVXOpc, DL, ContainerVT, V2,
2724                        DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2725     } else {
2726       SDValue RHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesRHS);
2727       RHSIndices =
2728           convertToScalableVector(IndexContainerVT, RHSIndices, DAG, Subtarget);
2729       V2 = DAG.getNode(GatherVVOpc, DL, ContainerVT, V2, RHSIndices, TrueMask,
2730                        VL);
2731     }
2732 
2733     MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
2734     SelectMask =
2735         convertToScalableVector(MaskContainerVT, SelectMask, DAG, Subtarget);
2736 
2737     Gather = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, SelectMask, V2,
2738                          Gather, VL);
2739   }
2740 
2741   return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2742 }
2743 
2744 static SDValue getRVVFPExtendOrRound(SDValue Op, MVT VT, MVT ContainerVT,
2745                                      SDLoc DL, SelectionDAG &DAG,
2746                                      const RISCVSubtarget &Subtarget) {
2747   if (VT.isScalableVector())
2748     return DAG.getFPExtendOrRound(Op, DL, VT);
2749   assert(VT.isFixedLengthVector() &&
2750          "Unexpected value type for RVV FP extend/round lowering");
2751   SDValue Mask, VL;
2752   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2753   unsigned RVVOpc = ContainerVT.bitsGT(Op.getSimpleValueType())
2754                         ? RISCVISD::FP_EXTEND_VL
2755                         : RISCVISD::FP_ROUND_VL;
2756   return DAG.getNode(RVVOpc, DL, ContainerVT, Op, Mask, VL);
2757 }
2758 
2759 // Lower CTLZ_ZERO_UNDEF or CTTZ_ZERO_UNDEF by converting to FP and extracting
2760 // the exponent.
2761 static SDValue lowerCTLZ_CTTZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
2762   MVT VT = Op.getSimpleValueType();
2763   unsigned EltSize = VT.getScalarSizeInBits();
2764   SDValue Src = Op.getOperand(0);
2765   SDLoc DL(Op);
2766 
2767   // We need a FP type that can represent the value.
2768   // TODO: Use f16 for i8 when possible?
2769   MVT FloatEltVT = EltSize == 32 ? MVT::f64 : MVT::f32;
2770   MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
2771 
2772   // Legal types should have been checked in the RISCVTargetLowering
2773   // constructor.
2774   // TODO: Splitting may make sense in some cases.
2775   assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
2776          "Expected legal float type!");
2777 
2778   // For CTTZ_ZERO_UNDEF, we need to extract the lowest set bit using X & -X.
2779   // The trailing zero count is equal to log2 of this single bit value.
2780   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
2781     SDValue Neg =
2782         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
2783     Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
2784   }
2785 
2786   // We have a legal FP type, convert to it.
2787   SDValue FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
2788   // Bitcast to integer and shift the exponent to the LSB.
2789   EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
2790   SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
2791   unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
2792   SDValue Shift = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
2793                               DAG.getConstant(ShiftAmt, DL, IntVT));
2794   // Truncate back to original type to allow vnsrl.
2795   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, Shift);
2796   // The exponent contains log2 of the value in biased form.
2797   unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
2798 
2799   // For trailing zeros, we just need to subtract the bias.
2800   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF)
2801     return DAG.getNode(ISD::SUB, DL, VT, Trunc,
2802                        DAG.getConstant(ExponentBias, DL, VT));
2803 
2804   // For leading zeros, we need to remove the bias and convert from log2 to
2805   // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
2806   unsigned Adjust = ExponentBias + (EltSize - 1);
2807   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Trunc);
2808 }
2809 
2810 // While RVV has alignment restrictions, we should always be able to load as a
2811 // legal equivalently-sized byte-typed vector instead. This method is
2812 // responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
2813 // the load is already correctly-aligned, it returns SDValue().
2814 SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
2815                                                     SelectionDAG &DAG) const {
2816   auto *Load = cast<LoadSDNode>(Op);
2817   assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
2818 
2819   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2820                                      Load->getMemoryVT(),
2821                                      *Load->getMemOperand()))
2822     return SDValue();
2823 
2824   SDLoc DL(Op);
2825   MVT VT = Op.getSimpleValueType();
2826   unsigned EltSizeBits = VT.getScalarSizeInBits();
2827   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2828          "Unexpected unaligned RVV load type");
2829   MVT NewVT =
2830       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2831   assert(NewVT.isValid() &&
2832          "Expecting equally-sized RVV vector types to be legal");
2833   SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
2834                           Load->getPointerInfo(), Load->getOriginalAlign(),
2835                           Load->getMemOperand()->getFlags());
2836   return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
2837 }
2838 
2839 // While RVV has alignment restrictions, we should always be able to store as a
2840 // legal equivalently-sized byte-typed vector instead. This method is
2841 // responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
2842 // returns SDValue() if the store is already correctly aligned.
2843 SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
2844                                                      SelectionDAG &DAG) const {
2845   auto *Store = cast<StoreSDNode>(Op);
2846   assert(Store && Store->getValue().getValueType().isVector() &&
2847          "Expected vector store");
2848 
2849   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2850                                      Store->getMemoryVT(),
2851                                      *Store->getMemOperand()))
2852     return SDValue();
2853 
2854   SDLoc DL(Op);
2855   SDValue StoredVal = Store->getValue();
2856   MVT VT = StoredVal.getSimpleValueType();
2857   unsigned EltSizeBits = VT.getScalarSizeInBits();
2858   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2859          "Unexpected unaligned RVV store type");
2860   MVT NewVT =
2861       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2862   assert(NewVT.isValid() &&
2863          "Expecting equally-sized RVV vector types to be legal");
2864   StoredVal = DAG.getBitcast(NewVT, StoredVal);
2865   return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
2866                       Store->getPointerInfo(), Store->getOriginalAlign(),
2867                       Store->getMemOperand()->getFlags());
2868 }
2869 
2870 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
2871                                             SelectionDAG &DAG) const {
2872   switch (Op.getOpcode()) {
2873   default:
2874     report_fatal_error("unimplemented operand");
2875   case ISD::GlobalAddress:
2876     return lowerGlobalAddress(Op, DAG);
2877   case ISD::BlockAddress:
2878     return lowerBlockAddress(Op, DAG);
2879   case ISD::ConstantPool:
2880     return lowerConstantPool(Op, DAG);
2881   case ISD::JumpTable:
2882     return lowerJumpTable(Op, DAG);
2883   case ISD::GlobalTLSAddress:
2884     return lowerGlobalTLSAddress(Op, DAG);
2885   case ISD::SELECT:
2886     return lowerSELECT(Op, DAG);
2887   case ISD::BRCOND:
2888     return lowerBRCOND(Op, DAG);
2889   case ISD::VASTART:
2890     return lowerVASTART(Op, DAG);
2891   case ISD::FRAMEADDR:
2892     return lowerFRAMEADDR(Op, DAG);
2893   case ISD::RETURNADDR:
2894     return lowerRETURNADDR(Op, DAG);
2895   case ISD::SHL_PARTS:
2896     return lowerShiftLeftParts(Op, DAG);
2897   case ISD::SRA_PARTS:
2898     return lowerShiftRightParts(Op, DAG, true);
2899   case ISD::SRL_PARTS:
2900     return lowerShiftRightParts(Op, DAG, false);
2901   case ISD::BITCAST: {
2902     SDLoc DL(Op);
2903     EVT VT = Op.getValueType();
2904     SDValue Op0 = Op.getOperand(0);
2905     EVT Op0VT = Op0.getValueType();
2906     MVT XLenVT = Subtarget.getXLenVT();
2907     if (VT.isFixedLengthVector()) {
2908       // We can handle fixed length vector bitcasts with a simple replacement
2909       // in isel.
2910       if (Op0VT.isFixedLengthVector())
2911         return Op;
2912       // When bitcasting from scalar to fixed-length vector, insert the scalar
2913       // into a one-element vector of the result type, and perform a vector
2914       // bitcast.
2915       if (!Op0VT.isVector()) {
2916         EVT BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
2917         if (!isTypeLegal(BVT))
2918           return SDValue();
2919         return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
2920                                               DAG.getUNDEF(BVT), Op0,
2921                                               DAG.getConstant(0, DL, XLenVT)));
2922       }
2923       return SDValue();
2924     }
2925     // Custom-legalize bitcasts from fixed-length vector types to scalar types
2926     // thus: bitcast the vector to a one-element vector type whose element type
2927     // is the same as the result type, and extract the first element.
2928     if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
2929       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
2930       if (!isTypeLegal(BVT))
2931         return SDValue();
2932       SDValue BVec = DAG.getBitcast(BVT, Op0);
2933       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
2934                          DAG.getConstant(0, DL, XLenVT));
2935     }
2936     if (VT == MVT::f16 && Op0VT == MVT::i16 && Subtarget.hasStdExtZfh()) {
2937       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
2938       SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
2939       return FPConv;
2940     }
2941     if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
2942         Subtarget.hasStdExtF()) {
2943       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
2944       SDValue FPConv =
2945           DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
2946       return FPConv;
2947     }
2948     return SDValue();
2949   }
2950   case ISD::INTRINSIC_WO_CHAIN:
2951     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
2952   case ISD::INTRINSIC_W_CHAIN:
2953     return LowerINTRINSIC_W_CHAIN(Op, DAG);
2954   case ISD::INTRINSIC_VOID:
2955     return LowerINTRINSIC_VOID(Op, DAG);
2956   case ISD::BSWAP:
2957   case ISD::BITREVERSE: {
2958     // Convert BSWAP/BITREVERSE to GREVI to enable GREVI combinining.
2959     assert(Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
2960     MVT VT = Op.getSimpleValueType();
2961     SDLoc DL(Op);
2962     // Start with the maximum immediate value which is the bitwidth - 1.
2963     unsigned Imm = VT.getSizeInBits() - 1;
2964     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
2965     if (Op.getOpcode() == ISD::BSWAP)
2966       Imm &= ~0x7U;
2967     return DAG.getNode(RISCVISD::GREV, DL, VT, Op.getOperand(0),
2968                        DAG.getConstant(Imm, DL, VT));
2969   }
2970   case ISD::FSHL:
2971   case ISD::FSHR: {
2972     MVT VT = Op.getSimpleValueType();
2973     assert(VT == Subtarget.getXLenVT() && "Unexpected custom legalization");
2974     SDLoc DL(Op);
2975     // FSL/FSR take a log2(XLen)+1 bit shift amount but XLenVT FSHL/FSHR only
2976     // use log(XLen) bits. Mask the shift amount accordingly to prevent
2977     // accidentally setting the extra bit.
2978     unsigned ShAmtWidth = Subtarget.getXLen() - 1;
2979     SDValue ShAmt = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(2),
2980                                 DAG.getConstant(ShAmtWidth, DL, VT));
2981     // fshl and fshr concatenate their operands in the same order. fsr and fsl
2982     // instruction use different orders. fshl will return its first operand for
2983     // shift of zero, fshr will return its second operand. fsl and fsr both
2984     // return rs1 so the ISD nodes need to have different operand orders.
2985     // Shift amount is in rs2.
2986     SDValue Op0 = Op.getOperand(0);
2987     SDValue Op1 = Op.getOperand(1);
2988     unsigned Opc = RISCVISD::FSL;
2989     if (Op.getOpcode() == ISD::FSHR) {
2990       std::swap(Op0, Op1);
2991       Opc = RISCVISD::FSR;
2992     }
2993     return DAG.getNode(Opc, DL, VT, Op0, Op1, ShAmt);
2994   }
2995   case ISD::TRUNCATE: {
2996     SDLoc DL(Op);
2997     MVT VT = Op.getSimpleValueType();
2998     // Only custom-lower vector truncates
2999     if (!VT.isVector())
3000       return Op;
3001 
3002     // Truncates to mask types are handled differently
3003     if (VT.getVectorElementType() == MVT::i1)
3004       return lowerVectorMaskTrunc(Op, DAG);
3005 
3006     // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
3007     // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
3008     // truncate by one power of two at a time.
3009     MVT DstEltVT = VT.getVectorElementType();
3010 
3011     SDValue Src = Op.getOperand(0);
3012     MVT SrcVT = Src.getSimpleValueType();
3013     MVT SrcEltVT = SrcVT.getVectorElementType();
3014 
3015     assert(DstEltVT.bitsLT(SrcEltVT) &&
3016            isPowerOf2_64(DstEltVT.getSizeInBits()) &&
3017            isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
3018            "Unexpected vector truncate lowering");
3019 
3020     MVT ContainerVT = SrcVT;
3021     if (SrcVT.isFixedLengthVector()) {
3022       ContainerVT = getContainerForFixedLengthVector(SrcVT);
3023       Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3024     }
3025 
3026     SDValue Result = Src;
3027     SDValue Mask, VL;
3028     std::tie(Mask, VL) =
3029         getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
3030     LLVMContext &Context = *DAG.getContext();
3031     const ElementCount Count = ContainerVT.getVectorElementCount();
3032     do {
3033       SrcEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
3034       EVT ResultVT = EVT::getVectorVT(Context, SrcEltVT, Count);
3035       Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
3036                            Mask, VL);
3037     } while (SrcEltVT != DstEltVT);
3038 
3039     if (SrcVT.isFixedLengthVector())
3040       Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
3041 
3042     return Result;
3043   }
3044   case ISD::ANY_EXTEND:
3045   case ISD::ZERO_EXTEND:
3046     if (Op.getOperand(0).getValueType().isVector() &&
3047         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3048       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
3049     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
3050   case ISD::SIGN_EXTEND:
3051     if (Op.getOperand(0).getValueType().isVector() &&
3052         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3053       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
3054     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
3055   case ISD::SPLAT_VECTOR_PARTS:
3056     return lowerSPLAT_VECTOR_PARTS(Op, DAG);
3057   case ISD::INSERT_VECTOR_ELT:
3058     return lowerINSERT_VECTOR_ELT(Op, DAG);
3059   case ISD::EXTRACT_VECTOR_ELT:
3060     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
3061   case ISD::VSCALE: {
3062     MVT VT = Op.getSimpleValueType();
3063     SDLoc DL(Op);
3064     SDValue VLENB = DAG.getNode(RISCVISD::READ_VLENB, DL, VT);
3065     // We define our scalable vector types for lmul=1 to use a 64 bit known
3066     // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
3067     // vscale as VLENB / 8.
3068     static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
3069     if (isa<ConstantSDNode>(Op.getOperand(0))) {
3070       // We assume VLENB is a multiple of 8. We manually choose the best shift
3071       // here because SimplifyDemandedBits isn't always able to simplify it.
3072       uint64_t Val = Op.getConstantOperandVal(0);
3073       if (isPowerOf2_64(Val)) {
3074         uint64_t Log2 = Log2_64(Val);
3075         if (Log2 < 3)
3076           return DAG.getNode(ISD::SRL, DL, VT, VLENB,
3077                              DAG.getConstant(3 - Log2, DL, VT));
3078         if (Log2 > 3)
3079           return DAG.getNode(ISD::SHL, DL, VT, VLENB,
3080                              DAG.getConstant(Log2 - 3, DL, VT));
3081         return VLENB;
3082       }
3083       // If the multiplier is a multiple of 8, scale it down to avoid needing
3084       // to shift the VLENB value.
3085       if ((Val % 8) == 0)
3086         return DAG.getNode(ISD::MUL, DL, VT, VLENB,
3087                            DAG.getConstant(Val / 8, DL, VT));
3088     }
3089 
3090     SDValue VScale = DAG.getNode(ISD::SRL, DL, VT, VLENB,
3091                                  DAG.getConstant(3, DL, VT));
3092     return DAG.getNode(ISD::MUL, DL, VT, VScale, Op.getOperand(0));
3093   }
3094   case ISD::FPOWI: {
3095     // Custom promote f16 powi with illegal i32 integer type on RV64. Once
3096     // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
3097     if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
3098         Op.getOperand(1).getValueType() == MVT::i32) {
3099       SDLoc DL(Op);
3100       SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
3101       SDValue Powi =
3102           DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
3103       return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
3104                          DAG.getIntPtrConstant(0, DL));
3105     }
3106     return SDValue();
3107   }
3108   case ISD::FP_EXTEND: {
3109     // RVV can only do fp_extend to types double the size as the source. We
3110     // custom-lower f16->f64 extensions to two hops of ISD::FP_EXTEND, going
3111     // via f32.
3112     SDLoc DL(Op);
3113     MVT VT = Op.getSimpleValueType();
3114     SDValue Src = Op.getOperand(0);
3115     MVT SrcVT = Src.getSimpleValueType();
3116 
3117     // Prepare any fixed-length vector operands.
3118     MVT ContainerVT = VT;
3119     if (SrcVT.isFixedLengthVector()) {
3120       ContainerVT = getContainerForFixedLengthVector(VT);
3121       MVT SrcContainerVT =
3122           ContainerVT.changeVectorElementType(SrcVT.getVectorElementType());
3123       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3124     }
3125 
3126     if (!VT.isVector() || VT.getVectorElementType() != MVT::f64 ||
3127         SrcVT.getVectorElementType() != MVT::f16) {
3128       // For scalable vectors, we only need to close the gap between
3129       // vXf16->vXf64.
3130       if (!VT.isFixedLengthVector())
3131         return Op;
3132       // For fixed-length vectors, lower the FP_EXTEND to a custom "VL" version.
3133       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
3134       return convertFromScalableVector(VT, Src, DAG, Subtarget);
3135     }
3136 
3137     MVT InterVT = VT.changeVectorElementType(MVT::f32);
3138     MVT InterContainerVT = ContainerVT.changeVectorElementType(MVT::f32);
3139     SDValue IntermediateExtend = getRVVFPExtendOrRound(
3140         Src, InterVT, InterContainerVT, DL, DAG, Subtarget);
3141 
3142     SDValue Extend = getRVVFPExtendOrRound(IntermediateExtend, VT, ContainerVT,
3143                                            DL, DAG, Subtarget);
3144     if (VT.isFixedLengthVector())
3145       return convertFromScalableVector(VT, Extend, DAG, Subtarget);
3146     return Extend;
3147   }
3148   case ISD::FP_ROUND: {
3149     // RVV can only do fp_round to types half the size as the source. We
3150     // custom-lower f64->f16 rounds via RVV's round-to-odd float
3151     // conversion instruction.
3152     SDLoc DL(Op);
3153     MVT VT = Op.getSimpleValueType();
3154     SDValue Src = Op.getOperand(0);
3155     MVT SrcVT = Src.getSimpleValueType();
3156 
3157     // Prepare any fixed-length vector operands.
3158     MVT ContainerVT = VT;
3159     if (VT.isFixedLengthVector()) {
3160       MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3161       ContainerVT =
3162           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3163       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3164     }
3165 
3166     if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
3167         SrcVT.getVectorElementType() != MVT::f64) {
3168       // For scalable vectors, we only need to close the gap between
3169       // vXf64<->vXf16.
3170       if (!VT.isFixedLengthVector())
3171         return Op;
3172       // For fixed-length vectors, lower the FP_ROUND to a custom "VL" version.
3173       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
3174       return convertFromScalableVector(VT, Src, DAG, Subtarget);
3175     }
3176 
3177     SDValue Mask, VL;
3178     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3179 
3180     MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
3181     SDValue IntermediateRound =
3182         DAG.getNode(RISCVISD::VFNCVT_ROD_VL, DL, InterVT, Src, Mask, VL);
3183     SDValue Round = getRVVFPExtendOrRound(IntermediateRound, VT, ContainerVT,
3184                                           DL, DAG, Subtarget);
3185 
3186     if (VT.isFixedLengthVector())
3187       return convertFromScalableVector(VT, Round, DAG, Subtarget);
3188     return Round;
3189   }
3190   case ISD::FP_TO_SINT:
3191   case ISD::FP_TO_UINT:
3192   case ISD::SINT_TO_FP:
3193   case ISD::UINT_TO_FP: {
3194     // RVV can only do fp<->int conversions to types half/double the size as
3195     // the source. We custom-lower any conversions that do two hops into
3196     // sequences.
3197     MVT VT = Op.getSimpleValueType();
3198     if (!VT.isVector())
3199       return Op;
3200     SDLoc DL(Op);
3201     SDValue Src = Op.getOperand(0);
3202     MVT EltVT = VT.getVectorElementType();
3203     MVT SrcVT = Src.getSimpleValueType();
3204     MVT SrcEltVT = SrcVT.getVectorElementType();
3205     unsigned EltSize = EltVT.getSizeInBits();
3206     unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3207     assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
3208            "Unexpected vector element types");
3209 
3210     bool IsInt2FP = SrcEltVT.isInteger();
3211     // Widening conversions
3212     if (EltSize > SrcEltSize && (EltSize / SrcEltSize >= 4)) {
3213       if (IsInt2FP) {
3214         // Do a regular integer sign/zero extension then convert to float.
3215         MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltVT.getSizeInBits()),
3216                                       VT.getVectorElementCount());
3217         unsigned ExtOpcode = Op.getOpcode() == ISD::UINT_TO_FP
3218                                  ? ISD::ZERO_EXTEND
3219                                  : ISD::SIGN_EXTEND;
3220         SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
3221         return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
3222       }
3223       // FP2Int
3224       assert(SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering");
3225       // Do one doubling fp_extend then complete the operation by converting
3226       // to int.
3227       MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3228       SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
3229       return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
3230     }
3231 
3232     // Narrowing conversions
3233     if (SrcEltSize > EltSize && (SrcEltSize / EltSize >= 4)) {
3234       if (IsInt2FP) {
3235         // One narrowing int_to_fp, then an fp_round.
3236         assert(EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering");
3237         MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3238         SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
3239         return DAG.getFPExtendOrRound(Int2FP, DL, VT);
3240       }
3241       // FP2Int
3242       // One narrowing fp_to_int, then truncate the integer. If the float isn't
3243       // representable by the integer, the result is poison.
3244       MVT IVecVT =
3245           MVT::getVectorVT(MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2),
3246                            VT.getVectorElementCount());
3247       SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
3248       return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
3249     }
3250 
3251     // Scalable vectors can exit here. Patterns will handle equally-sized
3252     // conversions halving/doubling ones.
3253     if (!VT.isFixedLengthVector())
3254       return Op;
3255 
3256     // For fixed-length vectors we lower to a custom "VL" node.
3257     unsigned RVVOpc = 0;
3258     switch (Op.getOpcode()) {
3259     default:
3260       llvm_unreachable("Impossible opcode");
3261     case ISD::FP_TO_SINT:
3262       RVVOpc = RISCVISD::FP_TO_SINT_VL;
3263       break;
3264     case ISD::FP_TO_UINT:
3265       RVVOpc = RISCVISD::FP_TO_UINT_VL;
3266       break;
3267     case ISD::SINT_TO_FP:
3268       RVVOpc = RISCVISD::SINT_TO_FP_VL;
3269       break;
3270     case ISD::UINT_TO_FP:
3271       RVVOpc = RISCVISD::UINT_TO_FP_VL;
3272       break;
3273     }
3274 
3275     MVT ContainerVT, SrcContainerVT;
3276     // Derive the reference container type from the larger vector type.
3277     if (SrcEltSize > EltSize) {
3278       SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3279       ContainerVT =
3280           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3281     } else {
3282       ContainerVT = getContainerForFixedLengthVector(VT);
3283       SrcContainerVT = ContainerVT.changeVectorElementType(SrcEltVT);
3284     }
3285 
3286     SDValue Mask, VL;
3287     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3288 
3289     Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3290     Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
3291     return convertFromScalableVector(VT, Src, DAG, Subtarget);
3292   }
3293   case ISD::FP_TO_SINT_SAT:
3294   case ISD::FP_TO_UINT_SAT:
3295     return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
3296   case ISD::FTRUNC:
3297   case ISD::FCEIL:
3298   case ISD::FFLOOR:
3299     return lowerFTRUNC_FCEIL_FFLOOR(Op, DAG);
3300   case ISD::VECREDUCE_ADD:
3301   case ISD::VECREDUCE_UMAX:
3302   case ISD::VECREDUCE_SMAX:
3303   case ISD::VECREDUCE_UMIN:
3304   case ISD::VECREDUCE_SMIN:
3305     return lowerVECREDUCE(Op, DAG);
3306   case ISD::VECREDUCE_AND:
3307   case ISD::VECREDUCE_OR:
3308   case ISD::VECREDUCE_XOR:
3309     if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3310       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
3311     return lowerVECREDUCE(Op, DAG);
3312   case ISD::VECREDUCE_FADD:
3313   case ISD::VECREDUCE_SEQ_FADD:
3314   case ISD::VECREDUCE_FMIN:
3315   case ISD::VECREDUCE_FMAX:
3316     return lowerFPVECREDUCE(Op, DAG);
3317   case ISD::VP_REDUCE_ADD:
3318   case ISD::VP_REDUCE_UMAX:
3319   case ISD::VP_REDUCE_SMAX:
3320   case ISD::VP_REDUCE_UMIN:
3321   case ISD::VP_REDUCE_SMIN:
3322   case ISD::VP_REDUCE_FADD:
3323   case ISD::VP_REDUCE_SEQ_FADD:
3324   case ISD::VP_REDUCE_FMIN:
3325   case ISD::VP_REDUCE_FMAX:
3326     return lowerVPREDUCE(Op, DAG);
3327   case ISD::VP_REDUCE_AND:
3328   case ISD::VP_REDUCE_OR:
3329   case ISD::VP_REDUCE_XOR:
3330     if (Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
3331       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
3332     return lowerVPREDUCE(Op, DAG);
3333   case ISD::INSERT_SUBVECTOR:
3334     return lowerINSERT_SUBVECTOR(Op, DAG);
3335   case ISD::EXTRACT_SUBVECTOR:
3336     return lowerEXTRACT_SUBVECTOR(Op, DAG);
3337   case ISD::STEP_VECTOR:
3338     return lowerSTEP_VECTOR(Op, DAG);
3339   case ISD::VECTOR_REVERSE:
3340     return lowerVECTOR_REVERSE(Op, DAG);
3341   case ISD::BUILD_VECTOR:
3342     return lowerBUILD_VECTOR(Op, DAG, Subtarget);
3343   case ISD::SPLAT_VECTOR:
3344     if (Op.getValueType().getVectorElementType() == MVT::i1)
3345       return lowerVectorMaskSplat(Op, DAG);
3346     return lowerSPLAT_VECTOR(Op, DAG, Subtarget);
3347   case ISD::VECTOR_SHUFFLE:
3348     return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
3349   case ISD::CONCAT_VECTORS: {
3350     // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
3351     // better than going through the stack, as the default expansion does.
3352     SDLoc DL(Op);
3353     MVT VT = Op.getSimpleValueType();
3354     unsigned NumOpElts =
3355         Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
3356     SDValue Vec = DAG.getUNDEF(VT);
3357     for (const auto &OpIdx : enumerate(Op->ops())) {
3358       SDValue SubVec = OpIdx.value();
3359       // Don't insert undef subvectors.
3360       if (SubVec.isUndef())
3361         continue;
3362       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec,
3363                         DAG.getIntPtrConstant(OpIdx.index() * NumOpElts, DL));
3364     }
3365     return Vec;
3366   }
3367   case ISD::LOAD:
3368     if (auto V = expandUnalignedRVVLoad(Op, DAG))
3369       return V;
3370     if (Op.getValueType().isFixedLengthVector())
3371       return lowerFixedLengthVectorLoadToRVV(Op, DAG);
3372     return Op;
3373   case ISD::STORE:
3374     if (auto V = expandUnalignedRVVStore(Op, DAG))
3375       return V;
3376     if (Op.getOperand(1).getValueType().isFixedLengthVector())
3377       return lowerFixedLengthVectorStoreToRVV(Op, DAG);
3378     return Op;
3379   case ISD::MLOAD:
3380   case ISD::VP_LOAD:
3381     return lowerMaskedLoad(Op, DAG);
3382   case ISD::MSTORE:
3383   case ISD::VP_STORE:
3384     return lowerMaskedStore(Op, DAG);
3385   case ISD::SETCC:
3386     return lowerFixedLengthVectorSetccToRVV(Op, DAG);
3387   case ISD::ADD:
3388     return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);
3389   case ISD::SUB:
3390     return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL);
3391   case ISD::MUL:
3392     return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL);
3393   case ISD::MULHS:
3394     return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL);
3395   case ISD::MULHU:
3396     return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL);
3397   case ISD::AND:
3398     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMAND_VL,
3399                                               RISCVISD::AND_VL);
3400   case ISD::OR:
3401     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMOR_VL,
3402                                               RISCVISD::OR_VL);
3403   case ISD::XOR:
3404     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMXOR_VL,
3405                                               RISCVISD::XOR_VL);
3406   case ISD::SDIV:
3407     return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL);
3408   case ISD::SREM:
3409     return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL);
3410   case ISD::UDIV:
3411     return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL);
3412   case ISD::UREM:
3413     return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL);
3414   case ISD::SHL:
3415   case ISD::SRA:
3416   case ISD::SRL:
3417     if (Op.getSimpleValueType().isFixedLengthVector())
3418       return lowerFixedLengthVectorShiftToRVV(Op, DAG);
3419     // This can be called for an i32 shift amount that needs to be promoted.
3420     assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
3421            "Unexpected custom legalisation");
3422     return SDValue();
3423   case ISD::SADDSAT:
3424     return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL);
3425   case ISD::UADDSAT:
3426     return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL);
3427   case ISD::SSUBSAT:
3428     return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL);
3429   case ISD::USUBSAT:
3430     return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL);
3431   case ISD::FADD:
3432     return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL);
3433   case ISD::FSUB:
3434     return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL);
3435   case ISD::FMUL:
3436     return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL);
3437   case ISD::FDIV:
3438     return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL);
3439   case ISD::FNEG:
3440     return lowerToScalableOp(Op, DAG, RISCVISD::FNEG_VL);
3441   case ISD::FABS:
3442     return lowerToScalableOp(Op, DAG, RISCVISD::FABS_VL);
3443   case ISD::FSQRT:
3444     return lowerToScalableOp(Op, DAG, RISCVISD::FSQRT_VL);
3445   case ISD::FMA:
3446     return lowerToScalableOp(Op, DAG, RISCVISD::FMA_VL);
3447   case ISD::SMIN:
3448     return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL);
3449   case ISD::SMAX:
3450     return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL);
3451   case ISD::UMIN:
3452     return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL);
3453   case ISD::UMAX:
3454     return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL);
3455   case ISD::FMINNUM:
3456     return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL);
3457   case ISD::FMAXNUM:
3458     return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL);
3459   case ISD::ABS:
3460     return lowerABS(Op, DAG);
3461   case ISD::CTLZ_ZERO_UNDEF:
3462   case ISD::CTTZ_ZERO_UNDEF:
3463     return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
3464   case ISD::VSELECT:
3465     return lowerFixedLengthVectorSelectToRVV(Op, DAG);
3466   case ISD::FCOPYSIGN:
3467     return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG);
3468   case ISD::MGATHER:
3469   case ISD::VP_GATHER:
3470     return lowerMaskedGather(Op, DAG);
3471   case ISD::MSCATTER:
3472   case ISD::VP_SCATTER:
3473     return lowerMaskedScatter(Op, DAG);
3474   case ISD::FLT_ROUNDS_:
3475     return lowerGET_ROUNDING(Op, DAG);
3476   case ISD::SET_ROUNDING:
3477     return lowerSET_ROUNDING(Op, DAG);
3478   case ISD::VP_SELECT:
3479     return lowerVPOp(Op, DAG, RISCVISD::VSELECT_VL);
3480   case ISD::VP_MERGE:
3481     return lowerVPOp(Op, DAG, RISCVISD::VP_MERGE_VL);
3482   case ISD::VP_ADD:
3483     return lowerVPOp(Op, DAG, RISCVISD::ADD_VL);
3484   case ISD::VP_SUB:
3485     return lowerVPOp(Op, DAG, RISCVISD::SUB_VL);
3486   case ISD::VP_MUL:
3487     return lowerVPOp(Op, DAG, RISCVISD::MUL_VL);
3488   case ISD::VP_SDIV:
3489     return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL);
3490   case ISD::VP_UDIV:
3491     return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL);
3492   case ISD::VP_SREM:
3493     return lowerVPOp(Op, DAG, RISCVISD::SREM_VL);
3494   case ISD::VP_UREM:
3495     return lowerVPOp(Op, DAG, RISCVISD::UREM_VL);
3496   case ISD::VP_AND:
3497     return lowerLogicVPOp(Op, DAG, RISCVISD::VMAND_VL, RISCVISD::AND_VL);
3498   case ISD::VP_OR:
3499     return lowerLogicVPOp(Op, DAG, RISCVISD::VMOR_VL, RISCVISD::OR_VL);
3500   case ISD::VP_XOR:
3501     return lowerLogicVPOp(Op, DAG, RISCVISD::VMXOR_VL, RISCVISD::XOR_VL);
3502   case ISD::VP_ASHR:
3503     return lowerVPOp(Op, DAG, RISCVISD::SRA_VL);
3504   case ISD::VP_LSHR:
3505     return lowerVPOp(Op, DAG, RISCVISD::SRL_VL);
3506   case ISD::VP_SHL:
3507     return lowerVPOp(Op, DAG, RISCVISD::SHL_VL);
3508   case ISD::VP_FADD:
3509     return lowerVPOp(Op, DAG, RISCVISD::FADD_VL);
3510   case ISD::VP_FSUB:
3511     return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL);
3512   case ISD::VP_FMUL:
3513     return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL);
3514   case ISD::VP_FDIV:
3515     return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL);
3516   }
3517 }
3518 
3519 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
3520                              SelectionDAG &DAG, unsigned Flags) {
3521   return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
3522 }
3523 
3524 static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
3525                              SelectionDAG &DAG, unsigned Flags) {
3526   return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
3527                                    Flags);
3528 }
3529 
3530 static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
3531                              SelectionDAG &DAG, unsigned Flags) {
3532   return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
3533                                    N->getOffset(), Flags);
3534 }
3535 
3536 static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
3537                              SelectionDAG &DAG, unsigned Flags) {
3538   return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
3539 }
3540 
3541 template <class NodeTy>
3542 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
3543                                      bool IsLocal) const {
3544   SDLoc DL(N);
3545   EVT Ty = getPointerTy(DAG.getDataLayout());
3546 
3547   if (isPositionIndependent()) {
3548     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3549     if (IsLocal)
3550       // Use PC-relative addressing to access the symbol. This generates the
3551       // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
3552       // %pcrel_lo(auipc)).
3553       return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3554 
3555     // Use PC-relative addressing to access the GOT for this symbol, then load
3556     // the address from the GOT. This generates the pattern (PseudoLA sym),
3557     // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
3558     return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
3559   }
3560 
3561   switch (getTargetMachine().getCodeModel()) {
3562   default:
3563     report_fatal_error("Unsupported code model for lowering");
3564   case CodeModel::Small: {
3565     // Generate a sequence for accessing addresses within the first 2 GiB of
3566     // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
3567     SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
3568     SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
3569     SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3570     return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
3571   }
3572   case CodeModel::Medium: {
3573     // Generate a sequence for accessing addresses within any 2GiB range within
3574     // the address space. This generates the pattern (PseudoLLA sym), which
3575     // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
3576     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3577     return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3578   }
3579   }
3580 }
3581 
3582 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
3583                                                 SelectionDAG &DAG) const {
3584   SDLoc DL(Op);
3585   EVT Ty = Op.getValueType();
3586   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3587   int64_t Offset = N->getOffset();
3588   MVT XLenVT = Subtarget.getXLenVT();
3589 
3590   const GlobalValue *GV = N->getGlobal();
3591   bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
3592   SDValue Addr = getAddr(N, DAG, IsLocal);
3593 
3594   // In order to maximise the opportunity for common subexpression elimination,
3595   // emit a separate ADD node for the global address offset instead of folding
3596   // it in the global address node. Later peephole optimisations may choose to
3597   // fold it back in when profitable.
3598   if (Offset != 0)
3599     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3600                        DAG.getConstant(Offset, DL, XLenVT));
3601   return Addr;
3602 }
3603 
3604 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
3605                                                SelectionDAG &DAG) const {
3606   BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
3607 
3608   return getAddr(N, DAG);
3609 }
3610 
3611 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
3612                                                SelectionDAG &DAG) const {
3613   ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
3614 
3615   return getAddr(N, DAG);
3616 }
3617 
3618 SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
3619                                             SelectionDAG &DAG) const {
3620   JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
3621 
3622   return getAddr(N, DAG);
3623 }
3624 
3625 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
3626                                               SelectionDAG &DAG,
3627                                               bool UseGOT) const {
3628   SDLoc DL(N);
3629   EVT Ty = getPointerTy(DAG.getDataLayout());
3630   const GlobalValue *GV = N->getGlobal();
3631   MVT XLenVT = Subtarget.getXLenVT();
3632 
3633   if (UseGOT) {
3634     // Use PC-relative addressing to access the GOT for this TLS symbol, then
3635     // load the address from the GOT and add the thread pointer. This generates
3636     // the pattern (PseudoLA_TLS_IE sym), which expands to
3637     // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
3638     SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3639     SDValue Load =
3640         SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
3641 
3642     // Add the thread pointer.
3643     SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3644     return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
3645   }
3646 
3647   // Generate a sequence for accessing the address relative to the thread
3648   // pointer, with the appropriate adjustment for the thread pointer offset.
3649   // This generates the pattern
3650   // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
3651   SDValue AddrHi =
3652       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
3653   SDValue AddrAdd =
3654       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
3655   SDValue AddrLo =
3656       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
3657 
3658   SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3659   SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3660   SDValue MNAdd = SDValue(
3661       DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
3662       0);
3663   return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
3664 }
3665 
3666 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
3667                                                SelectionDAG &DAG) const {
3668   SDLoc DL(N);
3669   EVT Ty = getPointerTy(DAG.getDataLayout());
3670   IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
3671   const GlobalValue *GV = N->getGlobal();
3672 
3673   // Use a PC-relative addressing mode to access the global dynamic GOT address.
3674   // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
3675   // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
3676   SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3677   SDValue Load =
3678       SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
3679 
3680   // Prepare argument list to generate call.
3681   ArgListTy Args;
3682   ArgListEntry Entry;
3683   Entry.Node = Load;
3684   Entry.Ty = CallTy;
3685   Args.push_back(Entry);
3686 
3687   // Setup call to __tls_get_addr.
3688   TargetLowering::CallLoweringInfo CLI(DAG);
3689   CLI.setDebugLoc(DL)
3690       .setChain(DAG.getEntryNode())
3691       .setLibCallee(CallingConv::C, CallTy,
3692                     DAG.getExternalSymbol("__tls_get_addr", Ty),
3693                     std::move(Args));
3694 
3695   return LowerCallTo(CLI).first;
3696 }
3697 
3698 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
3699                                                    SelectionDAG &DAG) const {
3700   SDLoc DL(Op);
3701   EVT Ty = Op.getValueType();
3702   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3703   int64_t Offset = N->getOffset();
3704   MVT XLenVT = Subtarget.getXLenVT();
3705 
3706   TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());
3707 
3708   if (DAG.getMachineFunction().getFunction().getCallingConv() ==
3709       CallingConv::GHC)
3710     report_fatal_error("In GHC calling convention TLS is not supported");
3711 
3712   SDValue Addr;
3713   switch (Model) {
3714   case TLSModel::LocalExec:
3715     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
3716     break;
3717   case TLSModel::InitialExec:
3718     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
3719     break;
3720   case TLSModel::LocalDynamic:
3721   case TLSModel::GeneralDynamic:
3722     Addr = getDynamicTLSAddr(N, DAG);
3723     break;
3724   }
3725 
3726   // In order to maximise the opportunity for common subexpression elimination,
3727   // emit a separate ADD node for the global address offset instead of folding
3728   // it in the global address node. Later peephole optimisations may choose to
3729   // fold it back in when profitable.
3730   if (Offset != 0)
3731     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3732                        DAG.getConstant(Offset, DL, XLenVT));
3733   return Addr;
3734 }
3735 
3736 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
3737   SDValue CondV = Op.getOperand(0);
3738   SDValue TrueV = Op.getOperand(1);
3739   SDValue FalseV = Op.getOperand(2);
3740   SDLoc DL(Op);
3741   MVT VT = Op.getSimpleValueType();
3742   MVT XLenVT = Subtarget.getXLenVT();
3743 
3744   // Lower vector SELECTs to VSELECTs by splatting the condition.
3745   if (VT.isVector()) {
3746     MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
3747     SDValue CondSplat = VT.isScalableVector()
3748                             ? DAG.getSplatVector(SplatCondVT, DL, CondV)
3749                             : DAG.getSplatBuildVector(SplatCondVT, DL, CondV);
3750     return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
3751   }
3752 
3753   // If the result type is XLenVT and CondV is the output of a SETCC node
3754   // which also operated on XLenVT inputs, then merge the SETCC node into the
3755   // lowered RISCVISD::SELECT_CC to take advantage of the integer
3756   // compare+branch instructions. i.e.:
3757   // (select (setcc lhs, rhs, cc), truev, falsev)
3758   // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
3759   if (VT == XLenVT && CondV.getOpcode() == ISD::SETCC &&
3760       CondV.getOperand(0).getSimpleValueType() == XLenVT) {
3761     SDValue LHS = CondV.getOperand(0);
3762     SDValue RHS = CondV.getOperand(1);
3763     const auto *CC = cast<CondCodeSDNode>(CondV.getOperand(2));
3764     ISD::CondCode CCVal = CC->get();
3765 
3766     // Special case for a select of 2 constants that have a diffence of 1.
3767     // Normally this is done by DAGCombine, but if the select is introduced by
3768     // type legalization or op legalization, we miss it. Restricting to SETLT
3769     // case for now because that is what signed saturating add/sub need.
3770     // FIXME: We don't need the condition to be SETLT or even a SETCC,
3771     // but we would probably want to swap the true/false values if the condition
3772     // is SETGE/SETLE to avoid an XORI.
3773     if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
3774         CCVal == ISD::SETLT) {
3775       const APInt &TrueVal = cast<ConstantSDNode>(TrueV)->getAPIntValue();
3776       const APInt &FalseVal = cast<ConstantSDNode>(FalseV)->getAPIntValue();
3777       if (TrueVal - 1 == FalseVal)
3778         return DAG.getNode(ISD::ADD, DL, Op.getValueType(), CondV, FalseV);
3779       if (TrueVal + 1 == FalseVal)
3780         return DAG.getNode(ISD::SUB, DL, Op.getValueType(), FalseV, CondV);
3781     }
3782 
3783     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3784 
3785     SDValue TargetCC = DAG.getCondCode(CCVal);
3786     SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
3787     return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3788   }
3789 
3790   // Otherwise:
3791   // (select condv, truev, falsev)
3792   // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
3793   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
3794   SDValue SetNE = DAG.getCondCode(ISD::SETNE);
3795 
3796   SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
3797 
3798   return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3799 }
3800 
3801 SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
3802   SDValue CondV = Op.getOperand(1);
3803   SDLoc DL(Op);
3804   MVT XLenVT = Subtarget.getXLenVT();
3805 
3806   if (CondV.getOpcode() == ISD::SETCC &&
3807       CondV.getOperand(0).getValueType() == XLenVT) {
3808     SDValue LHS = CondV.getOperand(0);
3809     SDValue RHS = CondV.getOperand(1);
3810     ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
3811 
3812     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3813 
3814     SDValue TargetCC = DAG.getCondCode(CCVal);
3815     return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3816                        LHS, RHS, TargetCC, Op.getOperand(2));
3817   }
3818 
3819   return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3820                      CondV, DAG.getConstant(0, DL, XLenVT),
3821                      DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
3822 }
3823 
3824 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3825   MachineFunction &MF = DAG.getMachineFunction();
3826   RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
3827 
3828   SDLoc DL(Op);
3829   SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
3830                                  getPointerTy(MF.getDataLayout()));
3831 
3832   // vastart just stores the address of the VarArgsFrameIndex slot into the
3833   // memory location argument.
3834   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3835   return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
3836                       MachinePointerInfo(SV));
3837 }
3838 
3839 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
3840                                             SelectionDAG &DAG) const {
3841   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3842   MachineFunction &MF = DAG.getMachineFunction();
3843   MachineFrameInfo &MFI = MF.getFrameInfo();
3844   MFI.setFrameAddressIsTaken(true);
3845   Register FrameReg = RI.getFrameRegister(MF);
3846   int XLenInBytes = Subtarget.getXLen() / 8;
3847 
3848   EVT VT = Op.getValueType();
3849   SDLoc DL(Op);
3850   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
3851   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3852   while (Depth--) {
3853     int Offset = -(XLenInBytes * 2);
3854     SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
3855                               DAG.getIntPtrConstant(Offset, DL));
3856     FrameAddr =
3857         DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
3858   }
3859   return FrameAddr;
3860 }
3861 
3862 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
3863                                              SelectionDAG &DAG) const {
3864   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3865   MachineFunction &MF = DAG.getMachineFunction();
3866   MachineFrameInfo &MFI = MF.getFrameInfo();
3867   MFI.setReturnAddressIsTaken(true);
3868   MVT XLenVT = Subtarget.getXLenVT();
3869   int XLenInBytes = Subtarget.getXLen() / 8;
3870 
3871   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
3872     return SDValue();
3873 
3874   EVT VT = Op.getValueType();
3875   SDLoc DL(Op);
3876   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3877   if (Depth) {
3878     int Off = -XLenInBytes;
3879     SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
3880     SDValue Offset = DAG.getConstant(Off, DL, VT);
3881     return DAG.getLoad(VT, DL, DAG.getEntryNode(),
3882                        DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
3883                        MachinePointerInfo());
3884   }
3885 
3886   // Return the value of the return address register, marking it an implicit
3887   // live-in.
3888   Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
3889   return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
3890 }
3891 
3892 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
3893                                                  SelectionDAG &DAG) const {
3894   SDLoc DL(Op);
3895   SDValue Lo = Op.getOperand(0);
3896   SDValue Hi = Op.getOperand(1);
3897   SDValue Shamt = Op.getOperand(2);
3898   EVT VT = Lo.getValueType();
3899 
3900   // if Shamt-XLEN < 0: // Shamt < XLEN
3901   //   Lo = Lo << Shamt
3902   //   Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
3903   // else:
3904   //   Lo = 0
3905   //   Hi = Lo << (Shamt-XLEN)
3906 
3907   SDValue Zero = DAG.getConstant(0, DL, VT);
3908   SDValue One = DAG.getConstant(1, DL, VT);
3909   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3910   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3911   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3912   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3913 
3914   SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
3915   SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
3916   SDValue ShiftRightLo =
3917       DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
3918   SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
3919   SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
3920   SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
3921 
3922   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3923 
3924   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
3925   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3926 
3927   SDValue Parts[2] = {Lo, Hi};
3928   return DAG.getMergeValues(Parts, DL);
3929 }
3930 
3931 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
3932                                                   bool IsSRA) const {
3933   SDLoc DL(Op);
3934   SDValue Lo = Op.getOperand(0);
3935   SDValue Hi = Op.getOperand(1);
3936   SDValue Shamt = Op.getOperand(2);
3937   EVT VT = Lo.getValueType();
3938 
3939   // SRA expansion:
3940   //   if Shamt-XLEN < 0: // Shamt < XLEN
3941   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3942   //     Hi = Hi >>s Shamt
3943   //   else:
3944   //     Lo = Hi >>s (Shamt-XLEN);
3945   //     Hi = Hi >>s (XLEN-1)
3946   //
3947   // SRL expansion:
3948   //   if Shamt-XLEN < 0: // Shamt < XLEN
3949   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3950   //     Hi = Hi >>u Shamt
3951   //   else:
3952   //     Lo = Hi >>u (Shamt-XLEN);
3953   //     Hi = 0;
3954 
3955   unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
3956 
3957   SDValue Zero = DAG.getConstant(0, DL, VT);
3958   SDValue One = DAG.getConstant(1, DL, VT);
3959   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3960   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3961   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3962   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3963 
3964   SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
3965   SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
3966   SDValue ShiftLeftHi =
3967       DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
3968   SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
3969   SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
3970   SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
3971   SDValue HiFalse =
3972       IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
3973 
3974   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3975 
3976   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
3977   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3978 
3979   SDValue Parts[2] = {Lo, Hi};
3980   return DAG.getMergeValues(Parts, DL);
3981 }
3982 
3983 // Lower splats of i1 types to SETCC. For each mask vector type, we have a
3984 // legal equivalently-sized i8 type, so we can use that as a go-between.
3985 SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
3986                                                   SelectionDAG &DAG) const {
3987   SDLoc DL(Op);
3988   MVT VT = Op.getSimpleValueType();
3989   SDValue SplatVal = Op.getOperand(0);
3990   // All-zeros or all-ones splats are handled specially.
3991   if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) {
3992     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
3993     return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
3994   }
3995   if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) {
3996     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
3997     return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
3998   }
3999   MVT XLenVT = Subtarget.getXLenVT();
4000   assert(SplatVal.getValueType() == XLenVT &&
4001          "Unexpected type for i1 splat value");
4002   MVT InterVT = VT.changeVectorElementType(MVT::i8);
4003   SplatVal = DAG.getNode(ISD::AND, DL, XLenVT, SplatVal,
4004                          DAG.getConstant(1, DL, XLenVT));
4005   SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
4006   SDValue Zero = DAG.getConstant(0, DL, InterVT);
4007   return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
4008 }
4009 
4010 // Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
4011 // illegal (currently only vXi64 RV32).
4012 // FIXME: We could also catch non-constant sign-extended i32 values and lower
4013 // them to SPLAT_VECTOR_I64
4014 SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
4015                                                      SelectionDAG &DAG) const {
4016   SDLoc DL(Op);
4017   MVT VecVT = Op.getSimpleValueType();
4018   assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
4019          "Unexpected SPLAT_VECTOR_PARTS lowering");
4020 
4021   assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
4022   SDValue Lo = Op.getOperand(0);
4023   SDValue Hi = Op.getOperand(1);
4024 
4025   if (VecVT.isFixedLengthVector()) {
4026     MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
4027     SDLoc DL(Op);
4028     SDValue Mask, VL;
4029     std::tie(Mask, VL) =
4030         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4031 
4032     SDValue Res = splatPartsI64WithVL(DL, ContainerVT, Lo, Hi, VL, DAG);
4033     return convertFromScalableVector(VecVT, Res, DAG, Subtarget);
4034   }
4035 
4036   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
4037     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
4038     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
4039     // If Hi constant is all the same sign bit as Lo, lower this as a custom
4040     // node in order to try and match RVV vector/scalar instructions.
4041     if ((LoC >> 31) == HiC)
4042       return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
4043   }
4044 
4045   // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
4046   if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
4047       isa<ConstantSDNode>(Hi.getOperand(1)) &&
4048       Hi.getConstantOperandVal(1) == 31)
4049     return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
4050 
4051   // Fall back to use a stack store and stride x0 vector load. Use X0 as VL.
4052   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VecVT, Lo, Hi,
4053                      DAG.getTargetConstant(RISCV::VLMaxSentinel, DL, MVT::i64));
4054 }
4055 
4056 // Custom-lower extensions from mask vectors by using a vselect either with 1
4057 // for zero/any-extension or -1 for sign-extension:
4058 //   (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
4059 // Note that any-extension is lowered identically to zero-extension.
4060 SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
4061                                                 int64_t ExtTrueVal) const {
4062   SDLoc DL(Op);
4063   MVT VecVT = Op.getSimpleValueType();
4064   SDValue Src = Op.getOperand(0);
4065   // Only custom-lower extensions from mask types
4066   assert(Src.getValueType().isVector() &&
4067          Src.getValueType().getVectorElementType() == MVT::i1);
4068 
4069   MVT XLenVT = Subtarget.getXLenVT();
4070   SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
4071   SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
4072 
4073   if (VecVT.isScalableVector()) {
4074     // Be careful not to introduce illegal scalar types at this stage, and be
4075     // careful also about splatting constants as on RV32, vXi64 SPLAT_VECTOR is
4076     // illegal and must be expanded. Since we know that the constants are
4077     // sign-extended 32-bit values, we use SPLAT_VECTOR_I64 directly.
4078     bool IsRV32E64 =
4079         !Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64;
4080 
4081     if (!IsRV32E64) {
4082       SplatZero = DAG.getSplatVector(VecVT, DL, SplatZero);
4083       SplatTrueVal = DAG.getSplatVector(VecVT, DL, SplatTrueVal);
4084     } else {
4085       SplatZero = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatZero);
4086       SplatTrueVal =
4087           DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatTrueVal);
4088     }
4089 
4090     return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
4091   }
4092 
4093   MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
4094   MVT I1ContainerVT =
4095       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4096 
4097   SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);
4098 
4099   SDValue Mask, VL;
4100   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4101 
4102   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero, VL);
4103   SplatTrueVal =
4104       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatTrueVal, VL);
4105   SDValue Select = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC,
4106                                SplatTrueVal, SplatZero, VL);
4107 
4108   return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
4109 }
4110 
4111 SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
4112     SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const {
4113   MVT ExtVT = Op.getSimpleValueType();
4114   // Only custom-lower extensions from fixed-length vector types.
4115   if (!ExtVT.isFixedLengthVector())
4116     return Op;
4117   MVT VT = Op.getOperand(0).getSimpleValueType();
4118   // Grab the canonical container type for the extended type. Infer the smaller
4119   // type from that to ensure the same number of vector elements, as we know
4120   // the LMUL will be sufficient to hold the smaller type.
4121   MVT ContainerExtVT = getContainerForFixedLengthVector(ExtVT);
4122   // Get the extended container type manually to ensure the same number of
4123   // vector elements between source and dest.
4124   MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
4125                                      ContainerExtVT.getVectorElementCount());
4126 
4127   SDValue Op1 =
4128       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
4129 
4130   SDLoc DL(Op);
4131   SDValue Mask, VL;
4132   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4133 
4134   SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);
4135 
4136   return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget);
4137 }
4138 
4139 // Custom-lower truncations from vectors to mask vectors by using a mask and a
4140 // setcc operation:
4141 //   (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
4142 SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
4143                                                   SelectionDAG &DAG) const {
4144   SDLoc DL(Op);
4145   EVT MaskVT = Op.getValueType();
4146   // Only expect to custom-lower truncations to mask types
4147   assert(MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 &&
4148          "Unexpected type for vector mask lowering");
4149   SDValue Src = Op.getOperand(0);
4150   MVT VecVT = Src.getSimpleValueType();
4151 
4152   // If this is a fixed vector, we need to convert it to a scalable vector.
4153   MVT ContainerVT = VecVT;
4154   if (VecVT.isFixedLengthVector()) {
4155     ContainerVT = getContainerForFixedLengthVector(VecVT);
4156     Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
4157   }
4158 
4159   SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
4160   SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
4161 
4162   SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatOne);
4163   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero);
4164 
4165   if (VecVT.isScalableVector()) {
4166     SDValue Trunc = DAG.getNode(ISD::AND, DL, VecVT, Src, SplatOne);
4167     return DAG.getSetCC(DL, MaskVT, Trunc, SplatZero, ISD::SETNE);
4168   }
4169 
4170   SDValue Mask, VL;
4171   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4172 
4173   MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
4174   SDValue Trunc =
4175       DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, Mask, VL);
4176   Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, Trunc, SplatZero,
4177                       DAG.getCondCode(ISD::SETNE), Mask, VL);
4178   return convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
4179 }
4180 
4181 // Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
4182 // first position of a vector, and that vector is slid up to the insert index.
4183 // By limiting the active vector length to index+1 and merging with the
4184 // original vector (with an undisturbed tail policy for elements >= VL), we
4185 // achieve the desired result of leaving all elements untouched except the one
4186 // at VL-1, which is replaced with the desired value.
4187 SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4188                                                     SelectionDAG &DAG) const {
4189   SDLoc DL(Op);
4190   MVT VecVT = Op.getSimpleValueType();
4191   SDValue Vec = Op.getOperand(0);
4192   SDValue Val = Op.getOperand(1);
4193   SDValue Idx = Op.getOperand(2);
4194 
4195   if (VecVT.getVectorElementType() == MVT::i1) {
4196     // FIXME: For now we just promote to an i8 vector and insert into that,
4197     // but this is probably not optimal.
4198     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4199     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4200     Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
4201     return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
4202   }
4203 
4204   MVT ContainerVT = VecVT;
4205   // If the operand is a fixed-length vector, convert to a scalable one.
4206   if (VecVT.isFixedLengthVector()) {
4207     ContainerVT = getContainerForFixedLengthVector(VecVT);
4208     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4209   }
4210 
4211   MVT XLenVT = Subtarget.getXLenVT();
4212 
4213   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4214   bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
4215   // Even i64-element vectors on RV32 can be lowered without scalar
4216   // legalization if the most-significant 32 bits of the value are not affected
4217   // by the sign-extension of the lower 32 bits.
4218   // TODO: We could also catch sign extensions of a 32-bit value.
4219   if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
4220     const auto *CVal = cast<ConstantSDNode>(Val);
4221     if (isInt<32>(CVal->getSExtValue())) {
4222       IsLegalInsert = true;
4223       Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4224     }
4225   }
4226 
4227   SDValue Mask, VL;
4228   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4229 
4230   SDValue ValInVec;
4231 
4232   if (IsLegalInsert) {
4233     unsigned Opc =
4234         VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
4235     if (isNullConstant(Idx)) {
4236       Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
4237       if (!VecVT.isFixedLengthVector())
4238         return Vec;
4239       return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
4240     }
4241     ValInVec =
4242         DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Val, VL);
4243   } else {
4244     // On RV32, i64-element vectors must be specially handled to place the
4245     // value at element 0, by using two vslide1up instructions in sequence on
4246     // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
4247     // this.
4248     SDValue One = DAG.getConstant(1, DL, XLenVT);
4249     SDValue ValLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, Zero);
4250     SDValue ValHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, One);
4251     MVT I32ContainerVT =
4252         MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2);
4253     SDValue I32Mask =
4254         getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
4255     // Limit the active VL to two.
4256     SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
4257     // Note: We can't pass a UNDEF to the first VSLIDE1UP_VL since an untied
4258     // undef doesn't obey the earlyclobber constraint. Just splat a zero value.
4259     ValInVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, I32ContainerVT, Zero,
4260                            InsertI64VL);
4261     // First slide in the hi value, then the lo in underneath it.
4262     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4263                            ValHi, I32Mask, InsertI64VL);
4264     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4265                            ValLo, I32Mask, InsertI64VL);
4266     // Bitcast back to the right container type.
4267     ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
4268   }
4269 
4270   // Now that the value is in a vector, slide it into position.
4271   SDValue InsertVL =
4272       DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
4273   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
4274                                 ValInVec, Idx, Mask, InsertVL);
4275   if (!VecVT.isFixedLengthVector())
4276     return Slideup;
4277   return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
4278 }
4279 
4280 // Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
4281 // extract the first element: (extractelt (slidedown vec, idx), 0). For integer
4282 // types this is done using VMV_X_S to allow us to glean information about the
4283 // sign bits of the result.
4284 SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4285                                                      SelectionDAG &DAG) const {
4286   SDLoc DL(Op);
4287   SDValue Idx = Op.getOperand(1);
4288   SDValue Vec = Op.getOperand(0);
4289   EVT EltVT = Op.getValueType();
4290   MVT VecVT = Vec.getSimpleValueType();
4291   MVT XLenVT = Subtarget.getXLenVT();
4292 
4293   if (VecVT.getVectorElementType() == MVT::i1) {
4294     if (VecVT.isFixedLengthVector()) {
4295       unsigned NumElts = VecVT.getVectorNumElements();
4296       if (NumElts >= 8) {
4297         MVT WideEltVT;
4298         unsigned WidenVecLen;
4299         SDValue ExtractElementIdx;
4300         SDValue ExtractBitIdx;
4301         unsigned MaxEEW = Subtarget.getMaxELENForFixedLengthVectors();
4302         MVT LargestEltVT = MVT::getIntegerVT(
4303             std::min(MaxEEW, unsigned(XLenVT.getSizeInBits())));
4304         if (NumElts <= LargestEltVT.getSizeInBits()) {
4305           assert(isPowerOf2_32(NumElts) &&
4306                  "the number of elements should be power of 2");
4307           WideEltVT = MVT::getIntegerVT(NumElts);
4308           WidenVecLen = 1;
4309           ExtractElementIdx = DAG.getConstant(0, DL, XLenVT);
4310           ExtractBitIdx = Idx;
4311         } else {
4312           WideEltVT = LargestEltVT;
4313           WidenVecLen = NumElts / WideEltVT.getSizeInBits();
4314           // extract element index = index / element width
4315           ExtractElementIdx = DAG.getNode(
4316               ISD::SRL, DL, XLenVT, Idx,
4317               DAG.getConstant(Log2_64(WideEltVT.getSizeInBits()), DL, XLenVT));
4318           // mask bit index = index % element width
4319           ExtractBitIdx = DAG.getNode(
4320               ISD::AND, DL, XLenVT, Idx,
4321               DAG.getConstant(WideEltVT.getSizeInBits() - 1, DL, XLenVT));
4322         }
4323         MVT WideVT = MVT::getVectorVT(WideEltVT, WidenVecLen);
4324         Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec);
4325         SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT,
4326                                          Vec, ExtractElementIdx);
4327         // Extract the bit from GPR.
4328         SDValue ShiftRight =
4329             DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx);
4330         return DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight,
4331                            DAG.getConstant(1, DL, XLenVT));
4332       }
4333     }
4334     // Otherwise, promote to an i8 vector and extract from that.
4335     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4336     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4337     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
4338   }
4339 
4340   // If this is a fixed vector, we need to convert it to a scalable vector.
4341   MVT ContainerVT = VecVT;
4342   if (VecVT.isFixedLengthVector()) {
4343     ContainerVT = getContainerForFixedLengthVector(VecVT);
4344     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4345   }
4346 
4347   // If the index is 0, the vector is already in the right position.
4348   if (!isNullConstant(Idx)) {
4349     // Use a VL of 1 to avoid processing more elements than we need.
4350     SDValue VL = DAG.getConstant(1, DL, XLenVT);
4351     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4352     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4353     Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
4354                       DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
4355   }
4356 
4357   if (!EltVT.isInteger()) {
4358     // Floating-point extracts are handled in TableGen.
4359     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
4360                        DAG.getConstant(0, DL, XLenVT));
4361   }
4362 
4363   SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
4364   return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
4365 }
4366 
4367 // Some RVV intrinsics may claim that they want an integer operand to be
4368 // promoted or expanded.
4369 static SDValue lowerVectorIntrinsicSplats(SDValue Op, SelectionDAG &DAG,
4370                                           const RISCVSubtarget &Subtarget) {
4371   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4372           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
4373          "Unexpected opcode");
4374 
4375   if (!Subtarget.hasVInstructions())
4376     return SDValue();
4377 
4378   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
4379   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
4380   SDLoc DL(Op);
4381 
4382   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
4383       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
4384   if (!II || !II->hasSplatOperand())
4385     return SDValue();
4386 
4387   unsigned SplatOp = II->SplatOperand + 1 + HasChain;
4388   assert(SplatOp < Op.getNumOperands());
4389 
4390   SmallVector<SDValue, 8> Operands(Op->op_begin(), Op->op_end());
4391   SDValue &ScalarOp = Operands[SplatOp];
4392   MVT OpVT = ScalarOp.getSimpleValueType();
4393   MVT XLenVT = Subtarget.getXLenVT();
4394 
4395   // If this isn't a scalar, or its type is XLenVT we're done.
4396   if (!OpVT.isScalarInteger() || OpVT == XLenVT)
4397     return SDValue();
4398 
4399   // Simplest case is that the operand needs to be promoted to XLenVT.
4400   if (OpVT.bitsLT(XLenVT)) {
4401     // If the operand is a constant, sign extend to increase our chances
4402     // of being able to use a .vi instruction. ANY_EXTEND would become a
4403     // a zero extend and the simm5 check in isel would fail.
4404     // FIXME: Should we ignore the upper bits in isel instead?
4405     unsigned ExtOpc =
4406         isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
4407     ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
4408     return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4409   }
4410 
4411   // Use the previous operand to get the vXi64 VT. The result might be a mask
4412   // VT for compares. Using the previous operand assumes that the previous
4413   // operand will never have a smaller element size than a scalar operand and
4414   // that a widening operation never uses SEW=64.
4415   // NOTE: If this fails the below assert, we can probably just find the
4416   // element count from any operand or result and use it to construct the VT.
4417   assert(II->SplatOperand > 0 && "Unexpected splat operand!");
4418   MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
4419 
4420   // The more complex case is when the scalar is larger than XLenVT.
4421   assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
4422          VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
4423 
4424   // If this is a sign-extended 32-bit constant, we can truncate it and rely
4425   // on the instruction to sign-extend since SEW>XLEN.
4426   if (auto *CVal = dyn_cast<ConstantSDNode>(ScalarOp)) {
4427     if (isInt<32>(CVal->getSExtValue())) {
4428       ScalarOp = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4429       return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4430     }
4431   }
4432 
4433   // We need to convert the scalar to a splat vector.
4434   // FIXME: Can we implicitly truncate the scalar if it is known to
4435   // be sign extended?
4436   SDValue VL = getVLOperand(Op);
4437   assert(VL.getValueType() == XLenVT);
4438   ScalarOp = splatSplitI64WithVL(DL, VT, ScalarOp, VL, DAG);
4439   return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4440 }
4441 
4442 SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
4443                                                      SelectionDAG &DAG) const {
4444   unsigned IntNo = Op.getConstantOperandVal(0);
4445   SDLoc DL(Op);
4446   MVT XLenVT = Subtarget.getXLenVT();
4447 
4448   switch (IntNo) {
4449   default:
4450     break; // Don't custom lower most intrinsics.
4451   case Intrinsic::thread_pointer: {
4452     EVT PtrVT = getPointerTy(DAG.getDataLayout());
4453     return DAG.getRegister(RISCV::X4, PtrVT);
4454   }
4455   case Intrinsic::riscv_orc_b:
4456     // Lower to the GORCI encoding for orc.b.
4457     return DAG.getNode(RISCVISD::GORC, DL, XLenVT, Op.getOperand(1),
4458                        DAG.getConstant(7, DL, XLenVT));
4459   case Intrinsic::riscv_grev:
4460   case Intrinsic::riscv_gorc: {
4461     unsigned Opc =
4462         IntNo == Intrinsic::riscv_grev ? RISCVISD::GREV : RISCVISD::GORC;
4463     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4464   }
4465   case Intrinsic::riscv_shfl:
4466   case Intrinsic::riscv_unshfl: {
4467     unsigned Opc =
4468         IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
4469     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4470   }
4471   case Intrinsic::riscv_bcompress:
4472   case Intrinsic::riscv_bdecompress: {
4473     unsigned Opc = IntNo == Intrinsic::riscv_bcompress ? RISCVISD::BCOMPRESS
4474                                                        : RISCVISD::BDECOMPRESS;
4475     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4476   }
4477   case Intrinsic::riscv_bfp:
4478     return DAG.getNode(RISCVISD::BFP, DL, XLenVT, Op.getOperand(1),
4479                        Op.getOperand(2));
4480   case Intrinsic::riscv_fsl:
4481     return DAG.getNode(RISCVISD::FSL, DL, XLenVT, Op.getOperand(1),
4482                        Op.getOperand(2), Op.getOperand(3));
4483   case Intrinsic::riscv_fsr:
4484     return DAG.getNode(RISCVISD::FSR, DL, XLenVT, Op.getOperand(1),
4485                        Op.getOperand(2), Op.getOperand(3));
4486   case Intrinsic::riscv_vmv_x_s:
4487     assert(Op.getValueType() == XLenVT && "Unexpected VT!");
4488     return DAG.getNode(RISCVISD::VMV_X_S, DL, Op.getValueType(),
4489                        Op.getOperand(1));
4490   case Intrinsic::riscv_vmv_v_x:
4491     return lowerScalarSplat(Op.getOperand(1), Op.getOperand(2),
4492                             Op.getSimpleValueType(), DL, DAG, Subtarget);
4493   case Intrinsic::riscv_vfmv_v_f:
4494     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
4495                        Op.getOperand(1), Op.getOperand(2));
4496   case Intrinsic::riscv_vmv_s_x: {
4497     SDValue Scalar = Op.getOperand(2);
4498 
4499     if (Scalar.getValueType().bitsLE(XLenVT)) {
4500       Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
4501       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
4502                          Op.getOperand(1), Scalar, Op.getOperand(3));
4503     }
4504 
4505     assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
4506 
4507     // This is an i64 value that lives in two scalar registers. We have to
4508     // insert this in a convoluted way. First we build vXi64 splat containing
4509     // the/ two values that we assemble using some bit math. Next we'll use
4510     // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
4511     // to merge element 0 from our splat into the source vector.
4512     // FIXME: This is probably not the best way to do this, but it is
4513     // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
4514     // point.
4515     //   sw lo, (a0)
4516     //   sw hi, 4(a0)
4517     //   vlse vX, (a0)
4518     //
4519     //   vid.v      vVid
4520     //   vmseq.vx   mMask, vVid, 0
4521     //   vmerge.vvm vDest, vSrc, vVal, mMask
4522     MVT VT = Op.getSimpleValueType();
4523     SDValue Vec = Op.getOperand(1);
4524     SDValue VL = getVLOperand(Op);
4525 
4526     SDValue SplattedVal = splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
4527     SDValue SplattedIdx = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
4528                                       DAG.getConstant(0, DL, MVT::i32), VL);
4529 
4530     MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
4531     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4532     SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
4533     SDValue SelectCond =
4534         DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, VID, SplattedIdx,
4535                     DAG.getCondCode(ISD::SETEQ), Mask, VL);
4536     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, SelectCond, SplattedVal,
4537                        Vec, VL);
4538   }
4539   case Intrinsic::riscv_vslide1up:
4540   case Intrinsic::riscv_vslide1down:
4541   case Intrinsic::riscv_vslide1up_mask:
4542   case Intrinsic::riscv_vslide1down_mask: {
4543     // We need to special case these when the scalar is larger than XLen.
4544     unsigned NumOps = Op.getNumOperands();
4545     bool IsMasked = NumOps == 7;
4546     unsigned OpOffset = IsMasked ? 1 : 0;
4547     SDValue Scalar = Op.getOperand(2 + OpOffset);
4548     if (Scalar.getValueType().bitsLE(XLenVT))
4549       break;
4550 
4551     // Splatting a sign extended constant is fine.
4552     if (auto *CVal = dyn_cast<ConstantSDNode>(Scalar))
4553       if (isInt<32>(CVal->getSExtValue()))
4554         break;
4555 
4556     MVT VT = Op.getSimpleValueType();
4557     assert(VT.getVectorElementType() == MVT::i64 &&
4558            Scalar.getValueType() == MVT::i64 && "Unexpected VTs");
4559 
4560     // Convert the vector source to the equivalent nxvXi32 vector.
4561     MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
4562     SDValue Vec = DAG.getBitcast(I32VT, Op.getOperand(1 + OpOffset));
4563 
4564     SDValue ScalarLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4565                                    DAG.getConstant(0, DL, XLenVT));
4566     SDValue ScalarHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4567                                    DAG.getConstant(1, DL, XLenVT));
4568 
4569     // Double the VL since we halved SEW.
4570     SDValue VL = getVLOperand(Op);
4571     SDValue I32VL =
4572         DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
4573 
4574     MVT I32MaskVT = MVT::getVectorVT(MVT::i1, I32VT.getVectorElementCount());
4575     SDValue I32Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, I32MaskVT, VL);
4576 
4577     // Shift the two scalar parts in using SEW=32 slide1up/slide1down
4578     // instructions.
4579     if (IntNo == Intrinsic::riscv_vslide1up ||
4580         IntNo == Intrinsic::riscv_vslide1up_mask) {
4581       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarHi,
4582                         I32Mask, I32VL);
4583       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarLo,
4584                         I32Mask, I32VL);
4585     } else {
4586       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarLo,
4587                         I32Mask, I32VL);
4588       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarHi,
4589                         I32Mask, I32VL);
4590     }
4591 
4592     // Convert back to nxvXi64.
4593     Vec = DAG.getBitcast(VT, Vec);
4594 
4595     if (!IsMasked)
4596       return Vec;
4597 
4598     // Apply mask after the operation.
4599     SDValue Mask = Op.getOperand(NumOps - 3);
4600     SDValue MaskedOff = Op.getOperand(1);
4601     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, Mask, Vec, MaskedOff, VL);
4602   }
4603   }
4604 
4605   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4606 }
4607 
4608 SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
4609                                                     SelectionDAG &DAG) const {
4610   unsigned IntNo = Op.getConstantOperandVal(1);
4611   switch (IntNo) {
4612   default:
4613     break;
4614   case Intrinsic::riscv_masked_strided_load: {
4615     SDLoc DL(Op);
4616     MVT XLenVT = Subtarget.getXLenVT();
4617 
4618     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4619     // the selection of the masked intrinsics doesn't do this for us.
4620     SDValue Mask = Op.getOperand(5);
4621     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4622 
4623     MVT VT = Op->getSimpleValueType(0);
4624     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4625 
4626     SDValue PassThru = Op.getOperand(2);
4627     if (!IsUnmasked) {
4628       MVT MaskVT =
4629           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4630       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4631       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
4632     }
4633 
4634     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4635 
4636     SDValue IntID = DAG.getTargetConstant(
4637         IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask, DL,
4638         XLenVT);
4639 
4640     auto *Load = cast<MemIntrinsicSDNode>(Op);
4641     SmallVector<SDValue, 8> Ops{Load->getChain(), IntID};
4642     if (IsUnmasked)
4643       Ops.push_back(DAG.getUNDEF(ContainerVT));
4644     else
4645       Ops.push_back(PassThru);
4646     Ops.push_back(Op.getOperand(3)); // Ptr
4647     Ops.push_back(Op.getOperand(4)); // Stride
4648     if (!IsUnmasked)
4649       Ops.push_back(Mask);
4650     Ops.push_back(VL);
4651     if (!IsUnmasked) {
4652       SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
4653       Ops.push_back(Policy);
4654     }
4655 
4656     SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
4657     SDValue Result =
4658         DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
4659                                 Load->getMemoryVT(), Load->getMemOperand());
4660     SDValue Chain = Result.getValue(1);
4661     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
4662     return DAG.getMergeValues({Result, Chain}, DL);
4663   }
4664   }
4665 
4666   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4667 }
4668 
4669 SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
4670                                                  SelectionDAG &DAG) const {
4671   unsigned IntNo = Op.getConstantOperandVal(1);
4672   switch (IntNo) {
4673   default:
4674     break;
4675   case Intrinsic::riscv_masked_strided_store: {
4676     SDLoc DL(Op);
4677     MVT XLenVT = Subtarget.getXLenVT();
4678 
4679     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4680     // the selection of the masked intrinsics doesn't do this for us.
4681     SDValue Mask = Op.getOperand(5);
4682     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4683 
4684     SDValue Val = Op.getOperand(2);
4685     MVT VT = Val.getSimpleValueType();
4686     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4687 
4688     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
4689     if (!IsUnmasked) {
4690       MVT MaskVT =
4691           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4692       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4693     }
4694 
4695     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4696 
4697     SDValue IntID = DAG.getTargetConstant(
4698         IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask, DL,
4699         XLenVT);
4700 
4701     auto *Store = cast<MemIntrinsicSDNode>(Op);
4702     SmallVector<SDValue, 8> Ops{Store->getChain(), IntID};
4703     Ops.push_back(Val);
4704     Ops.push_back(Op.getOperand(3)); // Ptr
4705     Ops.push_back(Op.getOperand(4)); // Stride
4706     if (!IsUnmasked)
4707       Ops.push_back(Mask);
4708     Ops.push_back(VL);
4709 
4710     return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(),
4711                                    Ops, Store->getMemoryVT(),
4712                                    Store->getMemOperand());
4713   }
4714   }
4715 
4716   return SDValue();
4717 }
4718 
4719 static MVT getLMUL1VT(MVT VT) {
4720   assert(VT.getVectorElementType().getSizeInBits() <= 64 &&
4721          "Unexpected vector MVT");
4722   return MVT::getScalableVectorVT(
4723       VT.getVectorElementType(),
4724       RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits());
4725 }
4726 
4727 static unsigned getRVVReductionOp(unsigned ISDOpcode) {
4728   switch (ISDOpcode) {
4729   default:
4730     llvm_unreachable("Unhandled reduction");
4731   case ISD::VECREDUCE_ADD:
4732     return RISCVISD::VECREDUCE_ADD_VL;
4733   case ISD::VECREDUCE_UMAX:
4734     return RISCVISD::VECREDUCE_UMAX_VL;
4735   case ISD::VECREDUCE_SMAX:
4736     return RISCVISD::VECREDUCE_SMAX_VL;
4737   case ISD::VECREDUCE_UMIN:
4738     return RISCVISD::VECREDUCE_UMIN_VL;
4739   case ISD::VECREDUCE_SMIN:
4740     return RISCVISD::VECREDUCE_SMIN_VL;
4741   case ISD::VECREDUCE_AND:
4742     return RISCVISD::VECREDUCE_AND_VL;
4743   case ISD::VECREDUCE_OR:
4744     return RISCVISD::VECREDUCE_OR_VL;
4745   case ISD::VECREDUCE_XOR:
4746     return RISCVISD::VECREDUCE_XOR_VL;
4747   }
4748 }
4749 
4750 SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
4751                                                          SelectionDAG &DAG,
4752                                                          bool IsVP) const {
4753   SDLoc DL(Op);
4754   SDValue Vec = Op.getOperand(IsVP ? 1 : 0);
4755   MVT VecVT = Vec.getSimpleValueType();
4756   assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
4757           Op.getOpcode() == ISD::VECREDUCE_OR ||
4758           Op.getOpcode() == ISD::VECREDUCE_XOR ||
4759           Op.getOpcode() == ISD::VP_REDUCE_AND ||
4760           Op.getOpcode() == ISD::VP_REDUCE_OR ||
4761           Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
4762          "Unexpected reduction lowering");
4763 
4764   MVT XLenVT = Subtarget.getXLenVT();
4765   assert(Op.getValueType() == XLenVT &&
4766          "Expected reduction output to be legalized to XLenVT");
4767 
4768   MVT ContainerVT = VecVT;
4769   if (VecVT.isFixedLengthVector()) {
4770     ContainerVT = getContainerForFixedLengthVector(VecVT);
4771     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4772   }
4773 
4774   SDValue Mask, VL;
4775   if (IsVP) {
4776     Mask = Op.getOperand(2);
4777     VL = Op.getOperand(3);
4778   } else {
4779     std::tie(Mask, VL) =
4780         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4781   }
4782 
4783   unsigned BaseOpc;
4784   ISD::CondCode CC;
4785   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4786 
4787   switch (Op.getOpcode()) {
4788   default:
4789     llvm_unreachable("Unhandled reduction");
4790   case ISD::VECREDUCE_AND:
4791   case ISD::VP_REDUCE_AND: {
4792     // vcpop ~x == 0
4793     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
4794     Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
4795     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4796     CC = ISD::SETEQ;
4797     BaseOpc = ISD::AND;
4798     break;
4799   }
4800   case ISD::VECREDUCE_OR:
4801   case ISD::VP_REDUCE_OR:
4802     // vcpop x != 0
4803     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4804     CC = ISD::SETNE;
4805     BaseOpc = ISD::OR;
4806     break;
4807   case ISD::VECREDUCE_XOR:
4808   case ISD::VP_REDUCE_XOR: {
4809     // ((vcpop x) & 1) != 0
4810     SDValue One = DAG.getConstant(1, DL, XLenVT);
4811     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4812     Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
4813     CC = ISD::SETNE;
4814     BaseOpc = ISD::XOR;
4815     break;
4816   }
4817   }
4818 
4819   SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC);
4820 
4821   if (!IsVP)
4822     return SetCC;
4823 
4824   // Now include the start value in the operation.
4825   // Note that we must return the start value when no elements are operated
4826   // upon. The vcpop instructions we've emitted in each case above will return
4827   // 0 for an inactive vector, and so we've already received the neutral value:
4828   // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
4829   // can simply include the start value.
4830   return DAG.getNode(BaseOpc, DL, XLenVT, SetCC, Op.getOperand(0));
4831 }
4832 
4833 SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
4834                                             SelectionDAG &DAG) const {
4835   SDLoc DL(Op);
4836   SDValue Vec = Op.getOperand(0);
4837   EVT VecEVT = Vec.getValueType();
4838 
4839   unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
4840 
4841   // Due to ordering in legalize types we may have a vector type that needs to
4842   // be split. Do that manually so we can get down to a legal type.
4843   while (getTypeAction(*DAG.getContext(), VecEVT) ==
4844          TargetLowering::TypeSplitVector) {
4845     SDValue Lo, Hi;
4846     std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
4847     VecEVT = Lo.getValueType();
4848     Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
4849   }
4850 
4851   // TODO: The type may need to be widened rather than split. Or widened before
4852   // it can be split.
4853   if (!isTypeLegal(VecEVT))
4854     return SDValue();
4855 
4856   MVT VecVT = VecEVT.getSimpleVT();
4857   MVT VecEltVT = VecVT.getVectorElementType();
4858   unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());
4859 
4860   MVT ContainerVT = VecVT;
4861   if (VecVT.isFixedLengthVector()) {
4862     ContainerVT = getContainerForFixedLengthVector(VecVT);
4863     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4864   }
4865 
4866   MVT M1VT = getLMUL1VT(ContainerVT);
4867   MVT XLenVT = Subtarget.getXLenVT();
4868 
4869   SDValue Mask, VL;
4870   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4871 
4872   SDValue NeutralElem =
4873       DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
4874   SDValue IdentitySplat = lowerScalarSplat(
4875       NeutralElem, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
4876   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT), Vec,
4877                                   IdentitySplat, Mask, VL);
4878   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
4879                              DAG.getConstant(0, DL, XLenVT));
4880   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
4881 }
4882 
4883 // Given a reduction op, this function returns the matching reduction opcode,
4884 // the vector SDValue and the scalar SDValue required to lower this to a
4885 // RISCVISD node.
4886 static std::tuple<unsigned, SDValue, SDValue>
4887 getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) {
4888   SDLoc DL(Op);
4889   auto Flags = Op->getFlags();
4890   unsigned Opcode = Op.getOpcode();
4891   unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Opcode);
4892   switch (Opcode) {
4893   default:
4894     llvm_unreachable("Unhandled reduction");
4895   case ISD::VECREDUCE_FADD: {
4896     // Use positive zero if we can. It is cheaper to materialize.
4897     SDValue Zero =
4898         DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
4899     return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero);
4900   }
4901   case ISD::VECREDUCE_SEQ_FADD:
4902     return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
4903                            Op.getOperand(0));
4904   case ISD::VECREDUCE_FMIN:
4905     return std::make_tuple(RISCVISD::VECREDUCE_FMIN_VL, Op.getOperand(0),
4906                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
4907   case ISD::VECREDUCE_FMAX:
4908     return std::make_tuple(RISCVISD::VECREDUCE_FMAX_VL, Op.getOperand(0),
4909                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
4910   }
4911 }
4912 
4913 SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
4914                                               SelectionDAG &DAG) const {
4915   SDLoc DL(Op);
4916   MVT VecEltVT = Op.getSimpleValueType();
4917 
4918   unsigned RVVOpcode;
4919   SDValue VectorVal, ScalarVal;
4920   std::tie(RVVOpcode, VectorVal, ScalarVal) =
4921       getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT);
4922   MVT VecVT = VectorVal.getSimpleValueType();
4923 
4924   MVT ContainerVT = VecVT;
4925   if (VecVT.isFixedLengthVector()) {
4926     ContainerVT = getContainerForFixedLengthVector(VecVT);
4927     VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
4928   }
4929 
4930   MVT M1VT = getLMUL1VT(VectorVal.getSimpleValueType());
4931   MVT XLenVT = Subtarget.getXLenVT();
4932 
4933   SDValue Mask, VL;
4934   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4935 
4936   SDValue ScalarSplat = lowerScalarSplat(
4937       ScalarVal, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
4938   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT),
4939                                   VectorVal, ScalarSplat, Mask, VL);
4940   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
4941                      DAG.getConstant(0, DL, XLenVT));
4942 }
4943 
4944 static unsigned getRVVVPReductionOp(unsigned ISDOpcode) {
4945   switch (ISDOpcode) {
4946   default:
4947     llvm_unreachable("Unhandled reduction");
4948   case ISD::VP_REDUCE_ADD:
4949     return RISCVISD::VECREDUCE_ADD_VL;
4950   case ISD::VP_REDUCE_UMAX:
4951     return RISCVISD::VECREDUCE_UMAX_VL;
4952   case ISD::VP_REDUCE_SMAX:
4953     return RISCVISD::VECREDUCE_SMAX_VL;
4954   case ISD::VP_REDUCE_UMIN:
4955     return RISCVISD::VECREDUCE_UMIN_VL;
4956   case ISD::VP_REDUCE_SMIN:
4957     return RISCVISD::VECREDUCE_SMIN_VL;
4958   case ISD::VP_REDUCE_AND:
4959     return RISCVISD::VECREDUCE_AND_VL;
4960   case ISD::VP_REDUCE_OR:
4961     return RISCVISD::VECREDUCE_OR_VL;
4962   case ISD::VP_REDUCE_XOR:
4963     return RISCVISD::VECREDUCE_XOR_VL;
4964   case ISD::VP_REDUCE_FADD:
4965     return RISCVISD::VECREDUCE_FADD_VL;
4966   case ISD::VP_REDUCE_SEQ_FADD:
4967     return RISCVISD::VECREDUCE_SEQ_FADD_VL;
4968   case ISD::VP_REDUCE_FMAX:
4969     return RISCVISD::VECREDUCE_FMAX_VL;
4970   case ISD::VP_REDUCE_FMIN:
4971     return RISCVISD::VECREDUCE_FMIN_VL;
4972   }
4973 }
4974 
4975 SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
4976                                            SelectionDAG &DAG) const {
4977   SDLoc DL(Op);
4978   SDValue Vec = Op.getOperand(1);
4979   EVT VecEVT = Vec.getValueType();
4980 
4981   // TODO: The type may need to be widened rather than split. Or widened before
4982   // it can be split.
4983   if (!isTypeLegal(VecEVT))
4984     return SDValue();
4985 
4986   MVT VecVT = VecEVT.getSimpleVT();
4987   MVT VecEltVT = VecVT.getVectorElementType();
4988   unsigned RVVOpcode = getRVVVPReductionOp(Op.getOpcode());
4989 
4990   MVT ContainerVT = VecVT;
4991   if (VecVT.isFixedLengthVector()) {
4992     ContainerVT = getContainerForFixedLengthVector(VecVT);
4993     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4994   }
4995 
4996   SDValue VL = Op.getOperand(3);
4997   SDValue Mask = Op.getOperand(2);
4998 
4999   MVT M1VT = getLMUL1VT(ContainerVT);
5000   MVT XLenVT = Subtarget.getXLenVT();
5001   MVT ResVT = !VecVT.isInteger() || VecEltVT.bitsGE(XLenVT) ? VecEltVT : XLenVT;
5002 
5003   SDValue StartSplat =
5004       lowerScalarSplat(Op.getOperand(0), DAG.getConstant(1, DL, XLenVT), M1VT,
5005                        DL, DAG, Subtarget);
5006   SDValue Reduction =
5007       DAG.getNode(RVVOpcode, DL, M1VT, StartSplat, Vec, StartSplat, Mask, VL);
5008   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction,
5009                              DAG.getConstant(0, DL, XLenVT));
5010   if (!VecVT.isInteger())
5011     return Elt0;
5012   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
5013 }
5014 
5015 SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
5016                                                    SelectionDAG &DAG) const {
5017   SDValue Vec = Op.getOperand(0);
5018   SDValue SubVec = Op.getOperand(1);
5019   MVT VecVT = Vec.getSimpleValueType();
5020   MVT SubVecVT = SubVec.getSimpleValueType();
5021 
5022   SDLoc DL(Op);
5023   MVT XLenVT = Subtarget.getXLenVT();
5024   unsigned OrigIdx = Op.getConstantOperandVal(2);
5025   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
5026 
5027   // We don't have the ability to slide mask vectors up indexed by their i1
5028   // elements; the smallest we can do is i8. Often we are able to bitcast to
5029   // equivalent i8 vectors. Note that when inserting a fixed-length vector
5030   // into a scalable one, we might not necessarily have enough scalable
5031   // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
5032   if (SubVecVT.getVectorElementType() == MVT::i1 &&
5033       (OrigIdx != 0 || !Vec.isUndef())) {
5034     if (VecVT.getVectorMinNumElements() >= 8 &&
5035         SubVecVT.getVectorMinNumElements() >= 8) {
5036       assert(OrigIdx % 8 == 0 && "Invalid index");
5037       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
5038              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
5039              "Unexpected mask vector lowering");
5040       OrigIdx /= 8;
5041       SubVecVT =
5042           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
5043                            SubVecVT.isScalableVector());
5044       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
5045                                VecVT.isScalableVector());
5046       Vec = DAG.getBitcast(VecVT, Vec);
5047       SubVec = DAG.getBitcast(SubVecVT, SubVec);
5048     } else {
5049       // We can't slide this mask vector up indexed by its i1 elements.
5050       // This poses a problem when we wish to insert a scalable vector which
5051       // can't be re-expressed as a larger type. Just choose the slow path and
5052       // extend to a larger type, then truncate back down.
5053       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
5054       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
5055       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
5056       SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
5057       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
5058                         Op.getOperand(2));
5059       SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
5060       return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
5061     }
5062   }
5063 
5064   // If the subvector vector is a fixed-length type, we cannot use subregister
5065   // manipulation to simplify the codegen; we don't know which register of a
5066   // LMUL group contains the specific subvector as we only know the minimum
5067   // register size. Therefore we must slide the vector group up the full
5068   // amount.
5069   if (SubVecVT.isFixedLengthVector()) {
5070     if (OrigIdx == 0 && Vec.isUndef() && !VecVT.isFixedLengthVector())
5071       return Op;
5072     MVT ContainerVT = VecVT;
5073     if (VecVT.isFixedLengthVector()) {
5074       ContainerVT = getContainerForFixedLengthVector(VecVT);
5075       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5076     }
5077     SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
5078                          DAG.getUNDEF(ContainerVT), SubVec,
5079                          DAG.getConstant(0, DL, XLenVT));
5080     if (OrigIdx == 0 && Vec.isUndef() && VecVT.isFixedLengthVector()) {
5081       SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
5082       return DAG.getBitcast(Op.getValueType(), SubVec);
5083     }
5084     SDValue Mask =
5085         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5086     // Set the vector length to only the number of elements we care about. Note
5087     // that for slideup this includes the offset.
5088     SDValue VL =
5089         DAG.getConstant(OrigIdx + SubVecVT.getVectorNumElements(), DL, XLenVT);
5090     SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5091     SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
5092                                   SubVec, SlideupAmt, Mask, VL);
5093     if (VecVT.isFixedLengthVector())
5094       Slideup = convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
5095     return DAG.getBitcast(Op.getValueType(), Slideup);
5096   }
5097 
5098   unsigned SubRegIdx, RemIdx;
5099   std::tie(SubRegIdx, RemIdx) =
5100       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5101           VecVT, SubVecVT, OrigIdx, TRI);
5102 
5103   RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecVT);
5104   bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
5105                          SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
5106                          SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
5107 
5108   // 1. If the Idx has been completely eliminated and this subvector's size is
5109   // a vector register or a multiple thereof, or the surrounding elements are
5110   // undef, then this is a subvector insert which naturally aligns to a vector
5111   // register. These can easily be handled using subregister manipulation.
5112   // 2. If the subvector is smaller than a vector register, then the insertion
5113   // must preserve the undisturbed elements of the register. We do this by
5114   // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
5115   // (which resolves to a subregister copy), performing a VSLIDEUP to place the
5116   // subvector within the vector register, and an INSERT_SUBVECTOR of that
5117   // LMUL=1 type back into the larger vector (resolving to another subregister
5118   // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
5119   // to avoid allocating a large register group to hold our subvector.
5120   if (RemIdx == 0 && (!IsSubVecPartReg || Vec.isUndef()))
5121     return Op;
5122 
5123   // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
5124   // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
5125   // (in our case undisturbed). This means we can set up a subvector insertion
5126   // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
5127   // size of the subvector.
5128   MVT InterSubVT = VecVT;
5129   SDValue AlignedExtract = Vec;
5130   unsigned AlignedIdx = OrigIdx - RemIdx;
5131   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5132     InterSubVT = getLMUL1VT(VecVT);
5133     // Extract a subvector equal to the nearest full vector register type. This
5134     // should resolve to a EXTRACT_SUBREG instruction.
5135     AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5136                                  DAG.getConstant(AlignedIdx, DL, XLenVT));
5137   }
5138 
5139   SDValue SlideupAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5140   // For scalable vectors this must be further multiplied by vscale.
5141   SlideupAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlideupAmt);
5142 
5143   SDValue Mask, VL;
5144   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5145 
5146   // Construct the vector length corresponding to RemIdx + length(SubVecVT).
5147   VL = DAG.getConstant(SubVecVT.getVectorMinNumElements(), DL, XLenVT);
5148   VL = DAG.getNode(ISD::VSCALE, DL, XLenVT, VL);
5149   VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
5150 
5151   SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
5152                        DAG.getUNDEF(InterSubVT), SubVec,
5153                        DAG.getConstant(0, DL, XLenVT));
5154 
5155   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, InterSubVT,
5156                                 AlignedExtract, SubVec, SlideupAmt, Mask, VL);
5157 
5158   // If required, insert this subvector back into the correct vector register.
5159   // This should resolve to an INSERT_SUBREG instruction.
5160   if (VecVT.bitsGT(InterSubVT))
5161     Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, Slideup,
5162                           DAG.getConstant(AlignedIdx, DL, XLenVT));
5163 
5164   // We might have bitcast from a mask type: cast back to the original type if
5165   // required.
5166   return DAG.getBitcast(Op.getSimpleValueType(), Slideup);
5167 }
5168 
5169 SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
5170                                                     SelectionDAG &DAG) const {
5171   SDValue Vec = Op.getOperand(0);
5172   MVT SubVecVT = Op.getSimpleValueType();
5173   MVT VecVT = Vec.getSimpleValueType();
5174 
5175   SDLoc DL(Op);
5176   MVT XLenVT = Subtarget.getXLenVT();
5177   unsigned OrigIdx = Op.getConstantOperandVal(1);
5178   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
5179 
5180   // We don't have the ability to slide mask vectors down indexed by their i1
5181   // elements; the smallest we can do is i8. Often we are able to bitcast to
5182   // equivalent i8 vectors. Note that when extracting a fixed-length vector
5183   // from a scalable one, we might not necessarily have enough scalable
5184   // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
5185   if (SubVecVT.getVectorElementType() == MVT::i1 && OrigIdx != 0) {
5186     if (VecVT.getVectorMinNumElements() >= 8 &&
5187         SubVecVT.getVectorMinNumElements() >= 8) {
5188       assert(OrigIdx % 8 == 0 && "Invalid index");
5189       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
5190              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
5191              "Unexpected mask vector lowering");
5192       OrigIdx /= 8;
5193       SubVecVT =
5194           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
5195                            SubVecVT.isScalableVector());
5196       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
5197                                VecVT.isScalableVector());
5198       Vec = DAG.getBitcast(VecVT, Vec);
5199     } else {
5200       // We can't slide this mask vector down, indexed by its i1 elements.
5201       // This poses a problem when we wish to extract a scalable vector which
5202       // can't be re-expressed as a larger type. Just choose the slow path and
5203       // extend to a larger type, then truncate back down.
5204       // TODO: We could probably improve this when extracting certain fixed
5205       // from fixed, where we can extract as i8 and shift the correct element
5206       // right to reach the desired subvector?
5207       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
5208       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
5209       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
5210       Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
5211                         Op.getOperand(1));
5212       SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
5213       return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
5214     }
5215   }
5216 
5217   // If the subvector vector is a fixed-length type, we cannot use subregister
5218   // manipulation to simplify the codegen; we don't know which register of a
5219   // LMUL group contains the specific subvector as we only know the minimum
5220   // register size. Therefore we must slide the vector group down the full
5221   // amount.
5222   if (SubVecVT.isFixedLengthVector()) {
5223     // With an index of 0 this is a cast-like subvector, which can be performed
5224     // with subregister operations.
5225     if (OrigIdx == 0)
5226       return Op;
5227     MVT ContainerVT = VecVT;
5228     if (VecVT.isFixedLengthVector()) {
5229       ContainerVT = getContainerForFixedLengthVector(VecVT);
5230       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5231     }
5232     SDValue Mask =
5233         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5234     // Set the vector length to only the number of elements we care about. This
5235     // avoids sliding down elements we're going to discard straight away.
5236     SDValue VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
5237     SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5238     SDValue Slidedown =
5239         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
5240                     DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
5241     // Now we can use a cast-like subvector extract to get the result.
5242     Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5243                             DAG.getConstant(0, DL, XLenVT));
5244     return DAG.getBitcast(Op.getValueType(), Slidedown);
5245   }
5246 
5247   unsigned SubRegIdx, RemIdx;
5248   std::tie(SubRegIdx, RemIdx) =
5249       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5250           VecVT, SubVecVT, OrigIdx, TRI);
5251 
5252   // If the Idx has been completely eliminated then this is a subvector extract
5253   // which naturally aligns to a vector register. These can easily be handled
5254   // using subregister manipulation.
5255   if (RemIdx == 0)
5256     return Op;
5257 
5258   // Else we must shift our vector register directly to extract the subvector.
5259   // Do this using VSLIDEDOWN.
5260 
5261   // If the vector type is an LMUL-group type, extract a subvector equal to the
5262   // nearest full vector register type. This should resolve to a EXTRACT_SUBREG
5263   // instruction.
5264   MVT InterSubVT = VecVT;
5265   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5266     InterSubVT = getLMUL1VT(VecVT);
5267     Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5268                       DAG.getConstant(OrigIdx - RemIdx, DL, XLenVT));
5269   }
5270 
5271   // Slide this vector register down by the desired number of elements in order
5272   // to place the desired subvector starting at element 0.
5273   SDValue SlidedownAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5274   // For scalable vectors this must be further multiplied by vscale.
5275   SlidedownAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlidedownAmt);
5276 
5277   SDValue Mask, VL;
5278   std::tie(Mask, VL) = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
5279   SDValue Slidedown =
5280       DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, InterSubVT,
5281                   DAG.getUNDEF(InterSubVT), Vec, SlidedownAmt, Mask, VL);
5282 
5283   // Now the vector is in the right position, extract our final subvector. This
5284   // should resolve to a COPY.
5285   Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5286                           DAG.getConstant(0, DL, XLenVT));
5287 
5288   // We might have bitcast from a mask type: cast back to the original type if
5289   // required.
5290   return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
5291 }
5292 
5293 // Lower step_vector to the vid instruction. Any non-identity step value must
5294 // be accounted for my manual expansion.
5295 SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
5296                                               SelectionDAG &DAG) const {
5297   SDLoc DL(Op);
5298   MVT VT = Op.getSimpleValueType();
5299   MVT XLenVT = Subtarget.getXLenVT();
5300   SDValue Mask, VL;
5301   std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
5302   SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
5303   uint64_t StepValImm = Op.getConstantOperandVal(0);
5304   if (StepValImm != 1) {
5305     if (isPowerOf2_64(StepValImm)) {
5306       SDValue StepVal =
5307           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
5308                       DAG.getConstant(Log2_64(StepValImm), DL, XLenVT));
5309       StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
5310     } else {
5311       SDValue StepVal = lowerScalarSplat(
5312           DAG.getConstant(StepValImm, DL, VT.getVectorElementType()), VL, VT,
5313           DL, DAG, Subtarget);
5314       StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
5315     }
5316   }
5317   return StepVec;
5318 }
5319 
5320 // Implement vector_reverse using vrgather.vv with indices determined by
5321 // subtracting the id of each element from (VLMAX-1). This will convert
5322 // the indices like so:
5323 // (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
5324 // TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
5325 SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
5326                                                  SelectionDAG &DAG) const {
5327   SDLoc DL(Op);
5328   MVT VecVT = Op.getSimpleValueType();
5329   unsigned EltSize = VecVT.getScalarSizeInBits();
5330   unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
5331 
5332   unsigned MaxVLMAX = 0;
5333   unsigned VectorBitsMax = Subtarget.getMaxRVVVectorSizeInBits();
5334   if (VectorBitsMax != 0)
5335     MaxVLMAX = ((VectorBitsMax / EltSize) * MinSize) / RISCV::RVVBitsPerBlock;
5336 
5337   unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
5338   MVT IntVT = VecVT.changeVectorElementTypeToInteger();
5339 
5340   // If this is SEW=8 and VLMAX is unknown or more than 256, we need
5341   // to use vrgatherei16.vv.
5342   // TODO: It's also possible to use vrgatherei16.vv for other types to
5343   // decrease register width for the index calculation.
5344   if ((MaxVLMAX == 0 || MaxVLMAX > 256) && EltSize == 8) {
5345     // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
5346     // Reverse each half, then reassemble them in reverse order.
5347     // NOTE: It's also possible that after splitting that VLMAX no longer
5348     // requires vrgatherei16.vv.
5349     if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
5350       SDValue Lo, Hi;
5351       std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
5352       EVT LoVT, HiVT;
5353       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
5354       Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
5355       Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
5356       // Reassemble the low and high pieces reversed.
5357       // FIXME: This is a CONCAT_VECTORS.
5358       SDValue Res =
5359           DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
5360                       DAG.getIntPtrConstant(0, DL));
5361       return DAG.getNode(
5362           ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo,
5363           DAG.getIntPtrConstant(LoVT.getVectorMinNumElements(), DL));
5364     }
5365 
5366     // Just promote the int type to i16 which will double the LMUL.
5367     IntVT = MVT::getVectorVT(MVT::i16, VecVT.getVectorElementCount());
5368     GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
5369   }
5370 
5371   MVT XLenVT = Subtarget.getXLenVT();
5372   SDValue Mask, VL;
5373   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5374 
5375   // Calculate VLMAX-1 for the desired SEW.
5376   unsigned MinElts = VecVT.getVectorMinNumElements();
5377   SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT,
5378                               DAG.getConstant(MinElts, DL, XLenVT));
5379   SDValue VLMinus1 =
5380       DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DAG.getConstant(1, DL, XLenVT));
5381 
5382   // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
5383   bool IsRV32E64 =
5384       !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
5385   SDValue SplatVL;
5386   if (!IsRV32E64)
5387     SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
5388   else
5389     SplatVL = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, IntVT, VLMinus1);
5390 
5391   SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
5392   SDValue Indices =
5393       DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, Mask, VL);
5394 
5395   return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices, Mask, VL);
5396 }
5397 
5398 SDValue
5399 RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
5400                                                      SelectionDAG &DAG) const {
5401   SDLoc DL(Op);
5402   auto *Load = cast<LoadSDNode>(Op);
5403 
5404   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5405                                         Load->getMemoryVT(),
5406                                         *Load->getMemOperand()) &&
5407          "Expecting a correctly-aligned load");
5408 
5409   MVT VT = Op.getSimpleValueType();
5410   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5411 
5412   SDValue VL =
5413       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5414 
5415   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5416   SDValue NewLoad = DAG.getMemIntrinsicNode(
5417       RISCVISD::VLE_VL, DL, VTs, {Load->getChain(), Load->getBasePtr(), VL},
5418       Load->getMemoryVT(), Load->getMemOperand());
5419 
5420   SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
5421   return DAG.getMergeValues({Result, Load->getChain()}, DL);
5422 }
5423 
5424 SDValue
5425 RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
5426                                                       SelectionDAG &DAG) const {
5427   SDLoc DL(Op);
5428   auto *Store = cast<StoreSDNode>(Op);
5429 
5430   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5431                                         Store->getMemoryVT(),
5432                                         *Store->getMemOperand()) &&
5433          "Expecting a correctly-aligned store");
5434 
5435   SDValue StoreVal = Store->getValue();
5436   MVT VT = StoreVal.getSimpleValueType();
5437 
5438   // If the size less than a byte, we need to pad with zeros to make a byte.
5439   if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
5440     VT = MVT::v8i1;
5441     StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
5442                            DAG.getConstant(0, DL, VT), StoreVal,
5443                            DAG.getIntPtrConstant(0, DL));
5444   }
5445 
5446   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5447 
5448   SDValue VL =
5449       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5450 
5451   SDValue NewValue =
5452       convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
5453   return DAG.getMemIntrinsicNode(
5454       RISCVISD::VSE_VL, DL, DAG.getVTList(MVT::Other),
5455       {Store->getChain(), NewValue, Store->getBasePtr(), VL},
5456       Store->getMemoryVT(), Store->getMemOperand());
5457 }
5458 
5459 SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
5460                                              SelectionDAG &DAG) const {
5461   SDLoc DL(Op);
5462   MVT VT = Op.getSimpleValueType();
5463 
5464   const auto *MemSD = cast<MemSDNode>(Op);
5465   EVT MemVT = MemSD->getMemoryVT();
5466   MachineMemOperand *MMO = MemSD->getMemOperand();
5467   SDValue Chain = MemSD->getChain();
5468   SDValue BasePtr = MemSD->getBasePtr();
5469 
5470   SDValue Mask, PassThru, VL;
5471   if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Op)) {
5472     Mask = VPLoad->getMask();
5473     PassThru = DAG.getUNDEF(VT);
5474     VL = VPLoad->getVectorLength();
5475   } else {
5476     const auto *MLoad = cast<MaskedLoadSDNode>(Op);
5477     Mask = MLoad->getMask();
5478     PassThru = MLoad->getPassThru();
5479   }
5480 
5481   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5482 
5483   MVT XLenVT = Subtarget.getXLenVT();
5484 
5485   MVT ContainerVT = VT;
5486   if (VT.isFixedLengthVector()) {
5487     ContainerVT = getContainerForFixedLengthVector(VT);
5488     PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5489     if (!IsUnmasked) {
5490       MVT MaskVT =
5491           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5492       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5493     }
5494   }
5495 
5496   if (!VL)
5497     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5498 
5499   unsigned IntID =
5500       IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
5501   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5502   if (IsUnmasked)
5503     Ops.push_back(DAG.getUNDEF(ContainerVT));
5504   else
5505     Ops.push_back(PassThru);
5506   Ops.push_back(BasePtr);
5507   if (!IsUnmasked)
5508     Ops.push_back(Mask);
5509   Ops.push_back(VL);
5510   if (!IsUnmasked)
5511     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5512 
5513   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5514 
5515   SDValue Result =
5516       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5517   Chain = Result.getValue(1);
5518 
5519   if (VT.isFixedLengthVector())
5520     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5521 
5522   return DAG.getMergeValues({Result, Chain}, DL);
5523 }
5524 
5525 SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
5526                                               SelectionDAG &DAG) const {
5527   SDLoc DL(Op);
5528 
5529   const auto *MemSD = cast<MemSDNode>(Op);
5530   EVT MemVT = MemSD->getMemoryVT();
5531   MachineMemOperand *MMO = MemSD->getMemOperand();
5532   SDValue Chain = MemSD->getChain();
5533   SDValue BasePtr = MemSD->getBasePtr();
5534   SDValue Val, Mask, VL;
5535 
5536   if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Op)) {
5537     Val = VPStore->getValue();
5538     Mask = VPStore->getMask();
5539     VL = VPStore->getVectorLength();
5540   } else {
5541     const auto *MStore = cast<MaskedStoreSDNode>(Op);
5542     Val = MStore->getValue();
5543     Mask = MStore->getMask();
5544   }
5545 
5546   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5547 
5548   MVT VT = Val.getSimpleValueType();
5549   MVT XLenVT = Subtarget.getXLenVT();
5550 
5551   MVT ContainerVT = VT;
5552   if (VT.isFixedLengthVector()) {
5553     ContainerVT = getContainerForFixedLengthVector(VT);
5554 
5555     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
5556     if (!IsUnmasked) {
5557       MVT MaskVT =
5558           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5559       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5560     }
5561   }
5562 
5563   if (!VL)
5564     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5565 
5566   unsigned IntID =
5567       IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
5568   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5569   Ops.push_back(Val);
5570   Ops.push_back(BasePtr);
5571   if (!IsUnmasked)
5572     Ops.push_back(Mask);
5573   Ops.push_back(VL);
5574 
5575   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
5576                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
5577 }
5578 
5579 SDValue
5580 RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(SDValue Op,
5581                                                       SelectionDAG &DAG) const {
5582   MVT InVT = Op.getOperand(0).getSimpleValueType();
5583   MVT ContainerVT = getContainerForFixedLengthVector(InVT);
5584 
5585   MVT VT = Op.getSimpleValueType();
5586 
5587   SDValue Op1 =
5588       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
5589   SDValue Op2 =
5590       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5591 
5592   SDLoc DL(Op);
5593   SDValue VL =
5594       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5595 
5596   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5597   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
5598 
5599   SDValue Cmp = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op1, Op2,
5600                             Op.getOperand(2), Mask, VL);
5601 
5602   return convertFromScalableVector(VT, Cmp, DAG, Subtarget);
5603 }
5604 
5605 SDValue RISCVTargetLowering::lowerFixedLengthVectorLogicOpToRVV(
5606     SDValue Op, SelectionDAG &DAG, unsigned MaskOpc, unsigned VecOpc) const {
5607   MVT VT = Op.getSimpleValueType();
5608 
5609   if (VT.getVectorElementType() == MVT::i1)
5610     return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMask*/ false);
5611 
5612   return lowerToScalableOp(Op, DAG, VecOpc, /*HasMask*/ true);
5613 }
5614 
5615 SDValue
5616 RISCVTargetLowering::lowerFixedLengthVectorShiftToRVV(SDValue Op,
5617                                                       SelectionDAG &DAG) const {
5618   unsigned Opc;
5619   switch (Op.getOpcode()) {
5620   default: llvm_unreachable("Unexpected opcode!");
5621   case ISD::SHL: Opc = RISCVISD::SHL_VL; break;
5622   case ISD::SRA: Opc = RISCVISD::SRA_VL; break;
5623   case ISD::SRL: Opc = RISCVISD::SRL_VL; break;
5624   }
5625 
5626   return lowerToScalableOp(Op, DAG, Opc);
5627 }
5628 
5629 // Lower vector ABS to smax(X, sub(0, X)).
5630 SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
5631   SDLoc DL(Op);
5632   MVT VT = Op.getSimpleValueType();
5633   SDValue X = Op.getOperand(0);
5634 
5635   assert(VT.isFixedLengthVector() && "Unexpected type");
5636 
5637   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5638   X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
5639 
5640   SDValue Mask, VL;
5641   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5642 
5643   SDValue SplatZero =
5644       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
5645                   DAG.getConstant(0, DL, Subtarget.getXLenVT()));
5646   SDValue NegX =
5647       DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, Mask, VL);
5648   SDValue Max =
5649       DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, Mask, VL);
5650 
5651   return convertFromScalableVector(VT, Max, DAG, Subtarget);
5652 }
5653 
5654 SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
5655     SDValue Op, SelectionDAG &DAG) const {
5656   SDLoc DL(Op);
5657   MVT VT = Op.getSimpleValueType();
5658   SDValue Mag = Op.getOperand(0);
5659   SDValue Sign = Op.getOperand(1);
5660   assert(Mag.getValueType() == Sign.getValueType() &&
5661          "Can only handle COPYSIGN with matching types.");
5662 
5663   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5664   Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget);
5665   Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget);
5666 
5667   SDValue Mask, VL;
5668   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5669 
5670   SDValue CopySign =
5671       DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag, Sign, Mask, VL);
5672 
5673   return convertFromScalableVector(VT, CopySign, DAG, Subtarget);
5674 }
5675 
5676 SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
5677     SDValue Op, SelectionDAG &DAG) const {
5678   MVT VT = Op.getSimpleValueType();
5679   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5680 
5681   MVT I1ContainerVT =
5682       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5683 
5684   SDValue CC =
5685       convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget);
5686   SDValue Op1 =
5687       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5688   SDValue Op2 =
5689       convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget);
5690 
5691   SDLoc DL(Op);
5692   SDValue Mask, VL;
5693   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5694 
5695   SDValue Select =
5696       DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC, Op1, Op2, VL);
5697 
5698   return convertFromScalableVector(VT, Select, DAG, Subtarget);
5699 }
5700 
5701 SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
5702                                                unsigned NewOpc,
5703                                                bool HasMask) const {
5704   MVT VT = Op.getSimpleValueType();
5705   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5706 
5707   // Create list of operands by converting existing ones to scalable types.
5708   SmallVector<SDValue, 6> Ops;
5709   for (const SDValue &V : Op->op_values()) {
5710     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5711 
5712     // Pass through non-vector operands.
5713     if (!V.getValueType().isVector()) {
5714       Ops.push_back(V);
5715       continue;
5716     }
5717 
5718     // "cast" fixed length vector to a scalable vector.
5719     assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
5720            "Only fixed length vectors are supported!");
5721     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5722   }
5723 
5724   SDLoc DL(Op);
5725   SDValue Mask, VL;
5726   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5727   if (HasMask)
5728     Ops.push_back(Mask);
5729   Ops.push_back(VL);
5730 
5731   SDValue ScalableRes = DAG.getNode(NewOpc, DL, ContainerVT, Ops);
5732   return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
5733 }
5734 
5735 // Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
5736 // * Operands of each node are assumed to be in the same order.
5737 // * The EVL operand is promoted from i32 to i64 on RV64.
5738 // * Fixed-length vectors are converted to their scalable-vector container
5739 //   types.
5740 SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
5741                                        unsigned RISCVISDOpc) const {
5742   SDLoc DL(Op);
5743   MVT VT = Op.getSimpleValueType();
5744   SmallVector<SDValue, 4> Ops;
5745 
5746   for (const auto &OpIdx : enumerate(Op->ops())) {
5747     SDValue V = OpIdx.value();
5748     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5749     // Pass through operands which aren't fixed-length vectors.
5750     if (!V.getValueType().isFixedLengthVector()) {
5751       Ops.push_back(V);
5752       continue;
5753     }
5754     // "cast" fixed length vector to a scalable vector.
5755     MVT OpVT = V.getSimpleValueType();
5756     MVT ContainerVT = getContainerForFixedLengthVector(OpVT);
5757     assert(useRVVForFixedLengthVectorVT(OpVT) &&
5758            "Only fixed length vectors are supported!");
5759     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5760   }
5761 
5762   if (!VT.isFixedLengthVector())
5763     return DAG.getNode(RISCVISDOpc, DL, VT, Ops);
5764 
5765   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5766 
5767   SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops);
5768 
5769   return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
5770 }
5771 
5772 SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op, SelectionDAG &DAG,
5773                                             unsigned MaskOpc,
5774                                             unsigned VecOpc) const {
5775   MVT VT = Op.getSimpleValueType();
5776   if (VT.getVectorElementType() != MVT::i1)
5777     return lowerVPOp(Op, DAG, VecOpc);
5778 
5779   // It is safe to drop mask parameter as masked-off elements are undef.
5780   SDValue Op1 = Op->getOperand(0);
5781   SDValue Op2 = Op->getOperand(1);
5782   SDValue VL = Op->getOperand(3);
5783 
5784   MVT ContainerVT = VT;
5785   const bool IsFixed = VT.isFixedLengthVector();
5786   if (IsFixed) {
5787     ContainerVT = getContainerForFixedLengthVector(VT);
5788     Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5789     Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
5790   }
5791 
5792   SDLoc DL(Op);
5793   SDValue Val = DAG.getNode(MaskOpc, DL, ContainerVT, Op1, Op2, VL);
5794   if (!IsFixed)
5795     return Val;
5796   return convertFromScalableVector(VT, Val, DAG, Subtarget);
5797 }
5798 
5799 // Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
5800 // matched to a RVV indexed load. The RVV indexed load instructions only
5801 // support the "unsigned unscaled" addressing mode; indices are implicitly
5802 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
5803 // signed or scaled indexing is extended to the XLEN value type and scaled
5804 // accordingly.
5805 SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
5806                                                SelectionDAG &DAG) const {
5807   SDLoc DL(Op);
5808   MVT VT = Op.getSimpleValueType();
5809 
5810   const auto *MemSD = cast<MemSDNode>(Op.getNode());
5811   EVT MemVT = MemSD->getMemoryVT();
5812   MachineMemOperand *MMO = MemSD->getMemOperand();
5813   SDValue Chain = MemSD->getChain();
5814   SDValue BasePtr = MemSD->getBasePtr();
5815 
5816   ISD::LoadExtType LoadExtType;
5817   SDValue Index, Mask, PassThru, VL;
5818 
5819   if (auto *VPGN = dyn_cast<VPGatherSDNode>(Op.getNode())) {
5820     Index = VPGN->getIndex();
5821     Mask = VPGN->getMask();
5822     PassThru = DAG.getUNDEF(VT);
5823     VL = VPGN->getVectorLength();
5824     // VP doesn't support extending loads.
5825     LoadExtType = ISD::NON_EXTLOAD;
5826   } else {
5827     // Else it must be a MGATHER.
5828     auto *MGN = cast<MaskedGatherSDNode>(Op.getNode());
5829     Index = MGN->getIndex();
5830     Mask = MGN->getMask();
5831     PassThru = MGN->getPassThru();
5832     LoadExtType = MGN->getExtensionType();
5833   }
5834 
5835   MVT IndexVT = Index.getSimpleValueType();
5836   MVT XLenVT = Subtarget.getXLenVT();
5837 
5838   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
5839          "Unexpected VTs!");
5840   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
5841   // Targets have to explicitly opt-in for extending vector loads.
5842   assert(LoadExtType == ISD::NON_EXTLOAD &&
5843          "Unexpected extending MGATHER/VP_GATHER");
5844   (void)LoadExtType;
5845 
5846   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
5847   // the selection of the masked intrinsics doesn't do this for us.
5848   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5849 
5850   MVT ContainerVT = VT;
5851   if (VT.isFixedLengthVector()) {
5852     // We need to use the larger of the result and index type to determine the
5853     // scalable type to use so we don't increase LMUL for any operand/result.
5854     if (VT.bitsGE(IndexVT)) {
5855       ContainerVT = getContainerForFixedLengthVector(VT);
5856       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
5857                                  ContainerVT.getVectorElementCount());
5858     } else {
5859       IndexVT = getContainerForFixedLengthVector(IndexVT);
5860       ContainerVT = MVT::getVectorVT(ContainerVT.getVectorElementType(),
5861                                      IndexVT.getVectorElementCount());
5862     }
5863 
5864     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
5865 
5866     if (!IsUnmasked) {
5867       MVT MaskVT =
5868           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5869       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5870       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5871     }
5872   }
5873 
5874   if (!VL)
5875     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5876 
5877   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
5878     IndexVT = IndexVT.changeVectorElementType(XLenVT);
5879     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, Mask.getValueType(),
5880                                    VL);
5881     Index = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, IndexVT, Index,
5882                         TrueMask, VL);
5883   }
5884 
5885   unsigned IntID =
5886       IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
5887   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5888   if (IsUnmasked)
5889     Ops.push_back(DAG.getUNDEF(ContainerVT));
5890   else
5891     Ops.push_back(PassThru);
5892   Ops.push_back(BasePtr);
5893   Ops.push_back(Index);
5894   if (!IsUnmasked)
5895     Ops.push_back(Mask);
5896   Ops.push_back(VL);
5897   if (!IsUnmasked)
5898     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5899 
5900   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5901   SDValue Result =
5902       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5903   Chain = Result.getValue(1);
5904 
5905   if (VT.isFixedLengthVector())
5906     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5907 
5908   return DAG.getMergeValues({Result, Chain}, DL);
5909 }
5910 
5911 // Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
5912 // matched to a RVV indexed store. The RVV indexed store instructions only
5913 // support the "unsigned unscaled" addressing mode; indices are implicitly
5914 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
5915 // signed or scaled indexing is extended to the XLEN value type and scaled
5916 // accordingly.
5917 SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
5918                                                 SelectionDAG &DAG) const {
5919   SDLoc DL(Op);
5920   const auto *MemSD = cast<MemSDNode>(Op.getNode());
5921   EVT MemVT = MemSD->getMemoryVT();
5922   MachineMemOperand *MMO = MemSD->getMemOperand();
5923   SDValue Chain = MemSD->getChain();
5924   SDValue BasePtr = MemSD->getBasePtr();
5925 
5926   bool IsTruncatingStore = false;
5927   SDValue Index, Mask, Val, VL;
5928 
5929   if (auto *VPSN = dyn_cast<VPScatterSDNode>(Op.getNode())) {
5930     Index = VPSN->getIndex();
5931     Mask = VPSN->getMask();
5932     Val = VPSN->getValue();
5933     VL = VPSN->getVectorLength();
5934     // VP doesn't support truncating stores.
5935     IsTruncatingStore = false;
5936   } else {
5937     // Else it must be a MSCATTER.
5938     auto *MSN = cast<MaskedScatterSDNode>(Op.getNode());
5939     Index = MSN->getIndex();
5940     Mask = MSN->getMask();
5941     Val = MSN->getValue();
5942     IsTruncatingStore = MSN->isTruncatingStore();
5943   }
5944 
5945   MVT VT = Val.getSimpleValueType();
5946   MVT IndexVT = Index.getSimpleValueType();
5947   MVT XLenVT = Subtarget.getXLenVT();
5948 
5949   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
5950          "Unexpected VTs!");
5951   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
5952   // Targets have to explicitly opt-in for extending vector loads and
5953   // truncating vector stores.
5954   assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
5955   (void)IsTruncatingStore;
5956 
5957   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
5958   // the selection of the masked intrinsics doesn't do this for us.
5959   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5960 
5961   MVT ContainerVT = VT;
5962   if (VT.isFixedLengthVector()) {
5963     // We need to use the larger of the value and index type to determine the
5964     // scalable type to use so we don't increase LMUL for any operand/result.
5965     if (VT.bitsGE(IndexVT)) {
5966       ContainerVT = getContainerForFixedLengthVector(VT);
5967       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
5968                                  ContainerVT.getVectorElementCount());
5969     } else {
5970       IndexVT = getContainerForFixedLengthVector(IndexVT);
5971       ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
5972                                      IndexVT.getVectorElementCount());
5973     }
5974 
5975     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
5976     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
5977 
5978     if (!IsUnmasked) {
5979       MVT MaskVT =
5980           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5981       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5982     }
5983   }
5984 
5985   if (!VL)
5986     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5987 
5988   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
5989     IndexVT = IndexVT.changeVectorElementType(XLenVT);
5990     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, Mask.getValueType(),
5991                                    VL);
5992     Index = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, IndexVT, Index,
5993                         TrueMask, VL);
5994   }
5995 
5996   unsigned IntID =
5997       IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
5998   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5999   Ops.push_back(Val);
6000   Ops.push_back(BasePtr);
6001   Ops.push_back(Index);
6002   if (!IsUnmasked)
6003     Ops.push_back(Mask);
6004   Ops.push_back(VL);
6005 
6006   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
6007                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
6008 }
6009 
6010 SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
6011                                                SelectionDAG &DAG) const {
6012   const MVT XLenVT = Subtarget.getXLenVT();
6013   SDLoc DL(Op);
6014   SDValue Chain = Op->getOperand(0);
6015   SDValue SysRegNo = DAG.getTargetConstant(
6016       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
6017   SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
6018   SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
6019 
6020   // Encoding used for rounding mode in RISCV differs from that used in
6021   // FLT_ROUNDS. To convert it the RISCV rounding mode is used as an index in a
6022   // table, which consists of a sequence of 4-bit fields, each representing
6023   // corresponding FLT_ROUNDS mode.
6024   static const int Table =
6025       (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
6026       (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
6027       (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
6028       (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
6029       (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
6030 
6031   SDValue Shift =
6032       DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
6033   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
6034                                 DAG.getConstant(Table, DL, XLenVT), Shift);
6035   SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
6036                                DAG.getConstant(7, DL, XLenVT));
6037 
6038   return DAG.getMergeValues({Masked, Chain}, DL);
6039 }
6040 
6041 SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
6042                                                SelectionDAG &DAG) const {
6043   const MVT XLenVT = Subtarget.getXLenVT();
6044   SDLoc DL(Op);
6045   SDValue Chain = Op->getOperand(0);
6046   SDValue RMValue = Op->getOperand(1);
6047   SDValue SysRegNo = DAG.getTargetConstant(
6048       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
6049 
6050   // Encoding used for rounding mode in RISCV differs from that used in
6051   // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
6052   // a table, which consists of a sequence of 4-bit fields, each representing
6053   // corresponding RISCV mode.
6054   static const unsigned Table =
6055       (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
6056       (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
6057       (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
6058       (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
6059       (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
6060 
6061   SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
6062                               DAG.getConstant(2, DL, XLenVT));
6063   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
6064                                 DAG.getConstant(Table, DL, XLenVT), Shift);
6065   RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
6066                         DAG.getConstant(0x7, DL, XLenVT));
6067   return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
6068                      RMValue);
6069 }
6070 
6071 static RISCVISD::NodeType getRISCVWOpcodeByIntr(unsigned IntNo) {
6072   switch (IntNo) {
6073   default:
6074     llvm_unreachable("Unexpected Intrinsic");
6075   case Intrinsic::riscv_grev:
6076     return RISCVISD::GREVW;
6077   case Intrinsic::riscv_gorc:
6078     return RISCVISD::GORCW;
6079   case Intrinsic::riscv_bcompress:
6080     return RISCVISD::BCOMPRESSW;
6081   case Intrinsic::riscv_bdecompress:
6082     return RISCVISD::BDECOMPRESSW;
6083   case Intrinsic::riscv_bfp:
6084     return RISCVISD::BFPW;
6085   case Intrinsic::riscv_fsl:
6086     return RISCVISD::FSLW;
6087   case Intrinsic::riscv_fsr:
6088     return RISCVISD::FSRW;
6089   }
6090 }
6091 
6092 // Converts the given intrinsic to a i64 operation with any extension.
6093 static SDValue customLegalizeToWOpByIntr(SDNode *N, SelectionDAG &DAG,
6094                                          unsigned IntNo) {
6095   SDLoc DL(N);
6096   RISCVISD::NodeType WOpcode = getRISCVWOpcodeByIntr(IntNo);
6097   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6098   SDValue NewOp2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6099   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp1, NewOp2);
6100   // ReplaceNodeResults requires we maintain the same type for the return value.
6101   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
6102 }
6103 
6104 // Returns the opcode of the target-specific SDNode that implements the 32-bit
6105 // form of the given Opcode.
6106 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
6107   switch (Opcode) {
6108   default:
6109     llvm_unreachable("Unexpected opcode");
6110   case ISD::SHL:
6111     return RISCVISD::SLLW;
6112   case ISD::SRA:
6113     return RISCVISD::SRAW;
6114   case ISD::SRL:
6115     return RISCVISD::SRLW;
6116   case ISD::SDIV:
6117     return RISCVISD::DIVW;
6118   case ISD::UDIV:
6119     return RISCVISD::DIVUW;
6120   case ISD::UREM:
6121     return RISCVISD::REMUW;
6122   case ISD::ROTL:
6123     return RISCVISD::ROLW;
6124   case ISD::ROTR:
6125     return RISCVISD::RORW;
6126   case RISCVISD::GREV:
6127     return RISCVISD::GREVW;
6128   case RISCVISD::GORC:
6129     return RISCVISD::GORCW;
6130   }
6131 }
6132 
6133 // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
6134 // node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
6135 // otherwise be promoted to i64, making it difficult to select the
6136 // SLLW/DIVUW/.../*W later one because the fact the operation was originally of
6137 // type i8/i16/i32 is lost.
6138 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
6139                                    unsigned ExtOpc = ISD::ANY_EXTEND) {
6140   SDLoc DL(N);
6141   RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6142   SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
6143   SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
6144   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6145   // ReplaceNodeResults requires we maintain the same type for the return value.
6146   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
6147 }
6148 
6149 // Converts the given 32-bit operation to a i64 operation with signed extension
6150 // semantic to reduce the signed extension instructions.
6151 static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
6152   SDLoc DL(N);
6153   SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6154   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6155   SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
6156   SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6157                                DAG.getValueType(MVT::i32));
6158   return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
6159 }
6160 
6161 void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
6162                                              SmallVectorImpl<SDValue> &Results,
6163                                              SelectionDAG &DAG) const {
6164   SDLoc DL(N);
6165   switch (N->getOpcode()) {
6166   default:
6167     llvm_unreachable("Don't know how to custom type legalize this operation!");
6168   case ISD::STRICT_FP_TO_SINT:
6169   case ISD::STRICT_FP_TO_UINT:
6170   case ISD::FP_TO_SINT:
6171   case ISD::FP_TO_UINT: {
6172     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6173            "Unexpected custom legalisation");
6174     bool IsStrict = N->isStrictFPOpcode();
6175     bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
6176                     N->getOpcode() == ISD::STRICT_FP_TO_SINT;
6177     SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
6178     if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
6179         TargetLowering::TypeSoftenFloat) {
6180       if (!isTypeLegal(Op0.getValueType()))
6181         return;
6182       if (IsStrict) {
6183         unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
6184                                 : RISCVISD::STRICT_FCVT_WU_RV64;
6185         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
6186         SDValue Res = DAG.getNode(
6187             Opc, DL, VTs, N->getOperand(0), Op0,
6188             DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
6189         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6190         Results.push_back(Res.getValue(1));
6191         return;
6192       }
6193       unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
6194       SDValue Res =
6195           DAG.getNode(Opc, DL, MVT::i64, Op0,
6196                       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
6197       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6198       return;
6199     }
6200     // If the FP type needs to be softened, emit a library call using the 'si'
6201     // version. If we left it to default legalization we'd end up with 'di'. If
6202     // the FP type doesn't need to be softened just let generic type
6203     // legalization promote the result type.
6204     RTLIB::Libcall LC;
6205     if (IsSigned)
6206       LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
6207     else
6208       LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
6209     MakeLibCallOptions CallOptions;
6210     EVT OpVT = Op0.getValueType();
6211     CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
6212     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
6213     SDValue Result;
6214     std::tie(Result, Chain) =
6215         makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
6216     Results.push_back(Result);
6217     if (IsStrict)
6218       Results.push_back(Chain);
6219     break;
6220   }
6221   case ISD::READCYCLECOUNTER: {
6222     assert(!Subtarget.is64Bit() &&
6223            "READCYCLECOUNTER only has custom type legalization on riscv32");
6224 
6225     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
6226     SDValue RCW =
6227         DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
6228 
6229     Results.push_back(
6230         DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
6231     Results.push_back(RCW.getValue(2));
6232     break;
6233   }
6234   case ISD::MUL: {
6235     unsigned Size = N->getSimpleValueType(0).getSizeInBits();
6236     unsigned XLen = Subtarget.getXLen();
6237     // This multiply needs to be expanded, try to use MULHSU+MUL if possible.
6238     if (Size > XLen) {
6239       assert(Size == (XLen * 2) && "Unexpected custom legalisation");
6240       SDValue LHS = N->getOperand(0);
6241       SDValue RHS = N->getOperand(1);
6242       APInt HighMask = APInt::getHighBitsSet(Size, XLen);
6243 
6244       bool LHSIsU = DAG.MaskedValueIsZero(LHS, HighMask);
6245       bool RHSIsU = DAG.MaskedValueIsZero(RHS, HighMask);
6246       // We need exactly one side to be unsigned.
6247       if (LHSIsU == RHSIsU)
6248         return;
6249 
6250       auto MakeMULPair = [&](SDValue S, SDValue U) {
6251         MVT XLenVT = Subtarget.getXLenVT();
6252         S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S);
6253         U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U);
6254         SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U);
6255         SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U);
6256         return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
6257       };
6258 
6259       bool LHSIsS = DAG.ComputeNumSignBits(LHS) > XLen;
6260       bool RHSIsS = DAG.ComputeNumSignBits(RHS) > XLen;
6261 
6262       // The other operand should be signed, but still prefer MULH when
6263       // possible.
6264       if (RHSIsU && LHSIsS && !RHSIsS)
6265         Results.push_back(MakeMULPair(LHS, RHS));
6266       else if (LHSIsU && RHSIsS && !LHSIsS)
6267         Results.push_back(MakeMULPair(RHS, LHS));
6268 
6269       return;
6270     }
6271     LLVM_FALLTHROUGH;
6272   }
6273   case ISD::ADD:
6274   case ISD::SUB:
6275     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6276            "Unexpected custom legalisation");
6277     Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
6278     break;
6279   case ISD::SHL:
6280   case ISD::SRA:
6281   case ISD::SRL:
6282     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6283            "Unexpected custom legalisation");
6284     if (N->getOperand(1).getOpcode() != ISD::Constant) {
6285       Results.push_back(customLegalizeToWOp(N, DAG));
6286       break;
6287     }
6288 
6289     // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
6290     // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
6291     // shift amount.
6292     if (N->getOpcode() == ISD::SHL) {
6293       SDLoc DL(N);
6294       SDValue NewOp0 =
6295           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6296       SDValue NewOp1 =
6297           DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
6298       SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
6299       SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6300                                    DAG.getValueType(MVT::i32));
6301       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6302     }
6303 
6304     break;
6305   case ISD::ROTL:
6306   case ISD::ROTR:
6307     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6308            "Unexpected custom legalisation");
6309     Results.push_back(customLegalizeToWOp(N, DAG));
6310     break;
6311   case ISD::CTTZ:
6312   case ISD::CTTZ_ZERO_UNDEF:
6313   case ISD::CTLZ:
6314   case ISD::CTLZ_ZERO_UNDEF: {
6315     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6316            "Unexpected custom legalisation");
6317 
6318     SDValue NewOp0 =
6319         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6320     bool IsCTZ =
6321         N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF;
6322     unsigned Opc = IsCTZ ? RISCVISD::CTZW : RISCVISD::CLZW;
6323     SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
6324     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6325     return;
6326   }
6327   case ISD::SDIV:
6328   case ISD::UDIV:
6329   case ISD::UREM: {
6330     MVT VT = N->getSimpleValueType(0);
6331     assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
6332            Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
6333            "Unexpected custom legalisation");
6334     // Don't promote division/remainder by constant since we should expand those
6335     // to multiply by magic constant.
6336     // FIXME: What if the expansion is disabled for minsize.
6337     if (N->getOperand(1).getOpcode() == ISD::Constant)
6338       return;
6339 
6340     // If the input is i32, use ANY_EXTEND since the W instructions don't read
6341     // the upper 32 bits. For other types we need to sign or zero extend
6342     // based on the opcode.
6343     unsigned ExtOpc = ISD::ANY_EXTEND;
6344     if (VT != MVT::i32)
6345       ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
6346                                            : ISD::ZERO_EXTEND;
6347 
6348     Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
6349     break;
6350   }
6351   case ISD::UADDO:
6352   case ISD::USUBO: {
6353     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6354            "Unexpected custom legalisation");
6355     bool IsAdd = N->getOpcode() == ISD::UADDO;
6356     // Create an ADDW or SUBW.
6357     SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6358     SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6359     SDValue Res =
6360         DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
6361     Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
6362                       DAG.getValueType(MVT::i32));
6363 
6364     // Sign extend the LHS and perform an unsigned compare with the ADDW result.
6365     // Since the inputs are sign extended from i32, this is equivalent to
6366     // comparing the lower 32 bits.
6367     LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6368     SDValue Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
6369                                     IsAdd ? ISD::SETULT : ISD::SETUGT);
6370 
6371     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6372     Results.push_back(Overflow);
6373     return;
6374   }
6375   case ISD::UADDSAT:
6376   case ISD::USUBSAT: {
6377     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6378            "Unexpected custom legalisation");
6379     if (Subtarget.hasStdExtZbb()) {
6380       // With Zbb we can sign extend and let LegalizeDAG use minu/maxu. Using
6381       // sign extend allows overflow of the lower 32 bits to be detected on
6382       // the promoted size.
6383       SDValue LHS =
6384           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6385       SDValue RHS =
6386           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
6387       SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
6388       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6389       return;
6390     }
6391 
6392     // Without Zbb, expand to UADDO/USUBO+select which will trigger our custom
6393     // promotion for UADDO/USUBO.
6394     Results.push_back(expandAddSubSat(N, DAG));
6395     return;
6396   }
6397   case ISD::BITCAST: {
6398     EVT VT = N->getValueType(0);
6399     assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
6400     SDValue Op0 = N->getOperand(0);
6401     EVT Op0VT = Op0.getValueType();
6402     MVT XLenVT = Subtarget.getXLenVT();
6403     if (VT == MVT::i16 && Op0VT == MVT::f16 && Subtarget.hasStdExtZfh()) {
6404       SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
6405       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
6406     } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
6407                Subtarget.hasStdExtF()) {
6408       SDValue FPConv =
6409           DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
6410       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
6411     } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
6412                isTypeLegal(Op0VT)) {
6413       // Custom-legalize bitcasts from fixed-length vector types to illegal
6414       // scalar types in order to improve codegen. Bitcast the vector to a
6415       // one-element vector type whose element type is the same as the result
6416       // type, and extract the first element.
6417       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
6418       if (isTypeLegal(BVT)) {
6419         SDValue BVec = DAG.getBitcast(BVT, Op0);
6420         Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
6421                                       DAG.getConstant(0, DL, XLenVT)));
6422       }
6423     }
6424     break;
6425   }
6426   case RISCVISD::GREV:
6427   case RISCVISD::GORC: {
6428     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6429            "Unexpected custom legalisation");
6430     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6431     // This is similar to customLegalizeToWOp, except that we pass the second
6432     // operand (a TargetConstant) straight through: it is already of type
6433     // XLenVT.
6434     RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6435     SDValue NewOp0 =
6436         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6437     SDValue NewOp1 =
6438         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6439     SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6440     // ReplaceNodeResults requires we maintain the same type for the return
6441     // value.
6442     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6443     break;
6444   }
6445   case RISCVISD::SHFL: {
6446     // There is no SHFLIW instruction, but we can just promote the operation.
6447     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6448            "Unexpected custom legalisation");
6449     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6450     SDValue NewOp0 =
6451         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6452     SDValue NewOp1 =
6453         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6454     SDValue NewRes = DAG.getNode(RISCVISD::SHFL, DL, MVT::i64, NewOp0, NewOp1);
6455     // ReplaceNodeResults requires we maintain the same type for the return
6456     // value.
6457     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6458     break;
6459   }
6460   case ISD::BSWAP:
6461   case ISD::BITREVERSE: {
6462     MVT VT = N->getSimpleValueType(0);
6463     MVT XLenVT = Subtarget.getXLenVT();
6464     assert((VT == MVT::i8 || VT == MVT::i16 ||
6465             (VT == MVT::i32 && Subtarget.is64Bit())) &&
6466            Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
6467     SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
6468     unsigned Imm = VT.getSizeInBits() - 1;
6469     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
6470     if (N->getOpcode() == ISD::BSWAP)
6471       Imm &= ~0x7U;
6472     unsigned Opc = Subtarget.is64Bit() ? RISCVISD::GREVW : RISCVISD::GREV;
6473     SDValue GREVI =
6474         DAG.getNode(Opc, DL, XLenVT, NewOp0, DAG.getConstant(Imm, DL, XLenVT));
6475     // ReplaceNodeResults requires we maintain the same type for the return
6476     // value.
6477     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, GREVI));
6478     break;
6479   }
6480   case ISD::FSHL:
6481   case ISD::FSHR: {
6482     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6483            Subtarget.hasStdExtZbt() && "Unexpected custom legalisation");
6484     SDValue NewOp0 =
6485         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6486     SDValue NewOp1 =
6487         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6488     SDValue NewShAmt =
6489         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6490     // FSLW/FSRW take a 6 bit shift amount but i32 FSHL/FSHR only use 5 bits.
6491     // Mask the shift amount to 5 bits to prevent accidentally setting bit 5.
6492     NewShAmt = DAG.getNode(ISD::AND, DL, MVT::i64, NewShAmt,
6493                            DAG.getConstant(0x1f, DL, MVT::i64));
6494     // fshl and fshr concatenate their operands in the same order. fsrw and fslw
6495     // instruction use different orders. fshl will return its first operand for
6496     // shift of zero, fshr will return its second operand. fsl and fsr both
6497     // return rs1 so the ISD nodes need to have different operand orders.
6498     // Shift amount is in rs2.
6499     unsigned Opc = RISCVISD::FSLW;
6500     if (N->getOpcode() == ISD::FSHR) {
6501       std::swap(NewOp0, NewOp1);
6502       Opc = RISCVISD::FSRW;
6503     }
6504     SDValue NewOp = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, NewShAmt);
6505     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewOp));
6506     break;
6507   }
6508   case ISD::EXTRACT_VECTOR_ELT: {
6509     // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
6510     // type is illegal (currently only vXi64 RV32).
6511     // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
6512     // transferred to the destination register. We issue two of these from the
6513     // upper- and lower- halves of the SEW-bit vector element, slid down to the
6514     // first element.
6515     SDValue Vec = N->getOperand(0);
6516     SDValue Idx = N->getOperand(1);
6517 
6518     // The vector type hasn't been legalized yet so we can't issue target
6519     // specific nodes if it needs legalization.
6520     // FIXME: We would manually legalize if it's important.
6521     if (!isTypeLegal(Vec.getValueType()))
6522       return;
6523 
6524     MVT VecVT = Vec.getSimpleValueType();
6525 
6526     assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
6527            VecVT.getVectorElementType() == MVT::i64 &&
6528            "Unexpected EXTRACT_VECTOR_ELT legalization");
6529 
6530     // If this is a fixed vector, we need to convert it to a scalable vector.
6531     MVT ContainerVT = VecVT;
6532     if (VecVT.isFixedLengthVector()) {
6533       ContainerVT = getContainerForFixedLengthVector(VecVT);
6534       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
6535     }
6536 
6537     MVT XLenVT = Subtarget.getXLenVT();
6538 
6539     // Use a VL of 1 to avoid processing more elements than we need.
6540     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
6541     SDValue VL = DAG.getConstant(1, DL, XLenVT);
6542     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6543 
6544     // Unless the index is known to be 0, we must slide the vector down to get
6545     // the desired element into index 0.
6546     if (!isNullConstant(Idx)) {
6547       Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
6548                         DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
6549     }
6550 
6551     // Extract the lower XLEN bits of the correct vector element.
6552     SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6553 
6554     // To extract the upper XLEN bits of the vector element, shift the first
6555     // element right by 32 bits and re-extract the lower XLEN bits.
6556     SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
6557                                      DAG.getConstant(32, DL, XLenVT), VL);
6558     SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec,
6559                                  ThirtyTwoV, Mask, VL);
6560 
6561     SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6562 
6563     Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6564     break;
6565   }
6566   case ISD::INTRINSIC_WO_CHAIN: {
6567     unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
6568     switch (IntNo) {
6569     default:
6570       llvm_unreachable(
6571           "Don't know how to custom type legalize this intrinsic!");
6572     case Intrinsic::riscv_grev:
6573     case Intrinsic::riscv_gorc:
6574     case Intrinsic::riscv_bcompress:
6575     case Intrinsic::riscv_bdecompress:
6576     case Intrinsic::riscv_bfp: {
6577       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6578              "Unexpected custom legalisation");
6579       Results.push_back(customLegalizeToWOpByIntr(N, DAG, IntNo));
6580       break;
6581     }
6582     case Intrinsic::riscv_fsl:
6583     case Intrinsic::riscv_fsr: {
6584       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6585              "Unexpected custom legalisation");
6586       SDValue NewOp1 =
6587           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6588       SDValue NewOp2 =
6589           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6590       SDValue NewOp3 =
6591           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3));
6592       unsigned Opc = getRISCVWOpcodeByIntr(IntNo);
6593       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2, NewOp3);
6594       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6595       break;
6596     }
6597     case Intrinsic::riscv_orc_b: {
6598       // Lower to the GORCI encoding for orc.b with the operand extended.
6599       SDValue NewOp =
6600           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6601       // If Zbp is enabled, use GORCIW which will sign extend the result.
6602       unsigned Opc =
6603           Subtarget.hasStdExtZbp() ? RISCVISD::GORCW : RISCVISD::GORC;
6604       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp,
6605                                 DAG.getConstant(7, DL, MVT::i64));
6606       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6607       return;
6608     }
6609     case Intrinsic::riscv_shfl:
6610     case Intrinsic::riscv_unshfl: {
6611       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6612              "Unexpected custom legalisation");
6613       SDValue NewOp1 =
6614           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6615       SDValue NewOp2 =
6616           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6617       unsigned Opc =
6618           IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFLW : RISCVISD::UNSHFLW;
6619       if (isa<ConstantSDNode>(N->getOperand(2))) {
6620         NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2,
6621                              DAG.getConstant(0xf, DL, MVT::i64));
6622         Opc =
6623             IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
6624       }
6625       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2);
6626       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6627       break;
6628     }
6629     case Intrinsic::riscv_vmv_x_s: {
6630       EVT VT = N->getValueType(0);
6631       MVT XLenVT = Subtarget.getXLenVT();
6632       if (VT.bitsLT(XLenVT)) {
6633         // Simple case just extract using vmv.x.s and truncate.
6634         SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
6635                                       Subtarget.getXLenVT(), N->getOperand(1));
6636         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
6637         return;
6638       }
6639 
6640       assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
6641              "Unexpected custom legalization");
6642 
6643       // We need to do the move in two steps.
6644       SDValue Vec = N->getOperand(1);
6645       MVT VecVT = Vec.getSimpleValueType();
6646 
6647       // First extract the lower XLEN bits of the element.
6648       SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6649 
6650       // To extract the upper XLEN bits of the vector element, shift the first
6651       // element right by 32 bits and re-extract the lower XLEN bits.
6652       SDValue VL = DAG.getConstant(1, DL, XLenVT);
6653       MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
6654       SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6655       SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT,
6656                                        DAG.getConstant(32, DL, XLenVT), VL);
6657       SDValue LShr32 =
6658           DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV, Mask, VL);
6659       SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6660 
6661       Results.push_back(
6662           DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6663       break;
6664     }
6665     }
6666     break;
6667   }
6668   case ISD::VECREDUCE_ADD:
6669   case ISD::VECREDUCE_AND:
6670   case ISD::VECREDUCE_OR:
6671   case ISD::VECREDUCE_XOR:
6672   case ISD::VECREDUCE_SMAX:
6673   case ISD::VECREDUCE_UMAX:
6674   case ISD::VECREDUCE_SMIN:
6675   case ISD::VECREDUCE_UMIN:
6676     if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
6677       Results.push_back(V);
6678     break;
6679   case ISD::VP_REDUCE_ADD:
6680   case ISD::VP_REDUCE_AND:
6681   case ISD::VP_REDUCE_OR:
6682   case ISD::VP_REDUCE_XOR:
6683   case ISD::VP_REDUCE_SMAX:
6684   case ISD::VP_REDUCE_UMAX:
6685   case ISD::VP_REDUCE_SMIN:
6686   case ISD::VP_REDUCE_UMIN:
6687     if (SDValue V = lowerVPREDUCE(SDValue(N, 0), DAG))
6688       Results.push_back(V);
6689     break;
6690   case ISD::FLT_ROUNDS_: {
6691     SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other);
6692     SDValue Res = DAG.getNode(ISD::FLT_ROUNDS_, DL, VTs, N->getOperand(0));
6693     Results.push_back(Res.getValue(0));
6694     Results.push_back(Res.getValue(1));
6695     break;
6696   }
6697   }
6698 }
6699 
6700 // A structure to hold one of the bit-manipulation patterns below. Together, a
6701 // SHL and non-SHL pattern may form a bit-manipulation pair on a single source:
6702 //   (or (and (shl x, 1), 0xAAAAAAAA),
6703 //       (and (srl x, 1), 0x55555555))
6704 struct RISCVBitmanipPat {
6705   SDValue Op;
6706   unsigned ShAmt;
6707   bool IsSHL;
6708 
6709   bool formsPairWith(const RISCVBitmanipPat &Other) const {
6710     return Op == Other.Op && ShAmt == Other.ShAmt && IsSHL != Other.IsSHL;
6711   }
6712 };
6713 
6714 // Matches patterns of the form
6715 //   (and (shl x, C2), (C1 << C2))
6716 //   (and (srl x, C2), C1)
6717 //   (shl (and x, C1), C2)
6718 //   (srl (and x, (C1 << C2)), C2)
6719 // Where C2 is a power of 2 and C1 has at least that many leading zeroes.
6720 // The expected masks for each shift amount are specified in BitmanipMasks where
6721 // BitmanipMasks[log2(C2)] specifies the expected C1 value.
6722 // The max allowed shift amount is either XLen/2 or XLen/4 determined by whether
6723 // BitmanipMasks contains 6 or 5 entries assuming that the maximum possible
6724 // XLen is 64.
6725 static Optional<RISCVBitmanipPat>
6726 matchRISCVBitmanipPat(SDValue Op, ArrayRef<uint64_t> BitmanipMasks) {
6727   assert((BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) &&
6728          "Unexpected number of masks");
6729   Optional<uint64_t> Mask;
6730   // Optionally consume a mask around the shift operation.
6731   if (Op.getOpcode() == ISD::AND && isa<ConstantSDNode>(Op.getOperand(1))) {
6732     Mask = Op.getConstantOperandVal(1);
6733     Op = Op.getOperand(0);
6734   }
6735   if (Op.getOpcode() != ISD::SHL && Op.getOpcode() != ISD::SRL)
6736     return None;
6737   bool IsSHL = Op.getOpcode() == ISD::SHL;
6738 
6739   if (!isa<ConstantSDNode>(Op.getOperand(1)))
6740     return None;
6741   uint64_t ShAmt = Op.getConstantOperandVal(1);
6742 
6743   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
6744   if (ShAmt >= Width || !isPowerOf2_64(ShAmt))
6745     return None;
6746   // If we don't have enough masks for 64 bit, then we must be trying to
6747   // match SHFL so we're only allowed to shift 1/4 of the width.
6748   if (BitmanipMasks.size() == 5 && ShAmt >= (Width / 2))
6749     return None;
6750 
6751   SDValue Src = Op.getOperand(0);
6752 
6753   // The expected mask is shifted left when the AND is found around SHL
6754   // patterns.
6755   //   ((x >> 1) & 0x55555555)
6756   //   ((x << 1) & 0xAAAAAAAA)
6757   bool SHLExpMask = IsSHL;
6758 
6759   if (!Mask) {
6760     // Sometimes LLVM keeps the mask as an operand of the shift, typically when
6761     // the mask is all ones: consume that now.
6762     if (Src.getOpcode() == ISD::AND && isa<ConstantSDNode>(Src.getOperand(1))) {
6763       Mask = Src.getConstantOperandVal(1);
6764       Src = Src.getOperand(0);
6765       // The expected mask is now in fact shifted left for SRL, so reverse the
6766       // decision.
6767       //   ((x & 0xAAAAAAAA) >> 1)
6768       //   ((x & 0x55555555) << 1)
6769       SHLExpMask = !SHLExpMask;
6770     } else {
6771       // Use a default shifted mask of all-ones if there's no AND, truncated
6772       // down to the expected width. This simplifies the logic later on.
6773       Mask = maskTrailingOnes<uint64_t>(Width);
6774       *Mask &= (IsSHL ? *Mask << ShAmt : *Mask >> ShAmt);
6775     }
6776   }
6777 
6778   unsigned MaskIdx = Log2_32(ShAmt);
6779   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
6780 
6781   if (SHLExpMask)
6782     ExpMask <<= ShAmt;
6783 
6784   if (Mask != ExpMask)
6785     return None;
6786 
6787   return RISCVBitmanipPat{Src, (unsigned)ShAmt, IsSHL};
6788 }
6789 
6790 // Matches any of the following bit-manipulation patterns:
6791 //   (and (shl x, 1), (0x55555555 << 1))
6792 //   (and (srl x, 1), 0x55555555)
6793 //   (shl (and x, 0x55555555), 1)
6794 //   (srl (and x, (0x55555555 << 1)), 1)
6795 // where the shift amount and mask may vary thus:
6796 //   [1]  = 0x55555555 / 0xAAAAAAAA
6797 //   [2]  = 0x33333333 / 0xCCCCCCCC
6798 //   [4]  = 0x0F0F0F0F / 0xF0F0F0F0
6799 //   [8]  = 0x00FF00FF / 0xFF00FF00
6800 //   [16] = 0x0000FFFF / 0xFFFFFFFF
6801 //   [32] = 0x00000000FFFFFFFF / 0xFFFFFFFF00000000 (for RV64)
6802 static Optional<RISCVBitmanipPat> matchGREVIPat(SDValue Op) {
6803   // These are the unshifted masks which we use to match bit-manipulation
6804   // patterns. They may be shifted left in certain circumstances.
6805   static const uint64_t BitmanipMasks[] = {
6806       0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
6807       0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
6808 
6809   return matchRISCVBitmanipPat(Op, BitmanipMasks);
6810 }
6811 
6812 // Match the following pattern as a GREVI(W) operation
6813 //   (or (BITMANIP_SHL x), (BITMANIP_SRL x))
6814 static SDValue combineORToGREV(SDValue Op, SelectionDAG &DAG,
6815                                const RISCVSubtarget &Subtarget) {
6816   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6817   EVT VT = Op.getValueType();
6818 
6819   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
6820     auto LHS = matchGREVIPat(Op.getOperand(0));
6821     auto RHS = matchGREVIPat(Op.getOperand(1));
6822     if (LHS && RHS && LHS->formsPairWith(*RHS)) {
6823       SDLoc DL(Op);
6824       return DAG.getNode(RISCVISD::GREV, DL, VT, LHS->Op,
6825                          DAG.getConstant(LHS->ShAmt, DL, VT));
6826     }
6827   }
6828   return SDValue();
6829 }
6830 
6831 // Matches any the following pattern as a GORCI(W) operation
6832 // 1.  (or (GREVI x, shamt), x) if shamt is a power of 2
6833 // 2.  (or x, (GREVI x, shamt)) if shamt is a power of 2
6834 // 3.  (or (or (BITMANIP_SHL x), x), (BITMANIP_SRL x))
6835 // Note that with the variant of 3.,
6836 //     (or (or (BITMANIP_SHL x), (BITMANIP_SRL x)), x)
6837 // the inner pattern will first be matched as GREVI and then the outer
6838 // pattern will be matched to GORC via the first rule above.
6839 // 4.  (or (rotl/rotr x, bitwidth/2), x)
6840 static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
6841                                const RISCVSubtarget &Subtarget) {
6842   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6843   EVT VT = Op.getValueType();
6844 
6845   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
6846     SDLoc DL(Op);
6847     SDValue Op0 = Op.getOperand(0);
6848     SDValue Op1 = Op.getOperand(1);
6849 
6850     auto MatchOROfReverse = [&](SDValue Reverse, SDValue X) {
6851       if (Reverse.getOpcode() == RISCVISD::GREV && Reverse.getOperand(0) == X &&
6852           isa<ConstantSDNode>(Reverse.getOperand(1)) &&
6853           isPowerOf2_32(Reverse.getConstantOperandVal(1)))
6854         return DAG.getNode(RISCVISD::GORC, DL, VT, X, Reverse.getOperand(1));
6855       // We can also form GORCI from ROTL/ROTR by half the bitwidth.
6856       if ((Reverse.getOpcode() == ISD::ROTL ||
6857            Reverse.getOpcode() == ISD::ROTR) &&
6858           Reverse.getOperand(0) == X &&
6859           isa<ConstantSDNode>(Reverse.getOperand(1))) {
6860         uint64_t RotAmt = Reverse.getConstantOperandVal(1);
6861         if (RotAmt == (VT.getSizeInBits() / 2))
6862           return DAG.getNode(RISCVISD::GORC, DL, VT, X,
6863                              DAG.getConstant(RotAmt, DL, VT));
6864       }
6865       return SDValue();
6866     };
6867 
6868     // Check for either commutable permutation of (or (GREVI x, shamt), x)
6869     if (SDValue V = MatchOROfReverse(Op0, Op1))
6870       return V;
6871     if (SDValue V = MatchOROfReverse(Op1, Op0))
6872       return V;
6873 
6874     // OR is commutable so canonicalize its OR operand to the left
6875     if (Op0.getOpcode() != ISD::OR && Op1.getOpcode() == ISD::OR)
6876       std::swap(Op0, Op1);
6877     if (Op0.getOpcode() != ISD::OR)
6878       return SDValue();
6879     SDValue OrOp0 = Op0.getOperand(0);
6880     SDValue OrOp1 = Op0.getOperand(1);
6881     auto LHS = matchGREVIPat(OrOp0);
6882     // OR is commutable so swap the operands and try again: x might have been
6883     // on the left
6884     if (!LHS) {
6885       std::swap(OrOp0, OrOp1);
6886       LHS = matchGREVIPat(OrOp0);
6887     }
6888     auto RHS = matchGREVIPat(Op1);
6889     if (LHS && RHS && LHS->formsPairWith(*RHS) && LHS->Op == OrOp1) {
6890       return DAG.getNode(RISCVISD::GORC, DL, VT, LHS->Op,
6891                          DAG.getConstant(LHS->ShAmt, DL, VT));
6892     }
6893   }
6894   return SDValue();
6895 }
6896 
6897 // Matches any of the following bit-manipulation patterns:
6898 //   (and (shl x, 1), (0x22222222 << 1))
6899 //   (and (srl x, 1), 0x22222222)
6900 //   (shl (and x, 0x22222222), 1)
6901 //   (srl (and x, (0x22222222 << 1)), 1)
6902 // where the shift amount and mask may vary thus:
6903 //   [1]  = 0x22222222 / 0x44444444
6904 //   [2]  = 0x0C0C0C0C / 0x3C3C3C3C
6905 //   [4]  = 0x00F000F0 / 0x0F000F00
6906 //   [8]  = 0x0000FF00 / 0x00FF0000
6907 //   [16] = 0x00000000FFFF0000 / 0x0000FFFF00000000 (for RV64)
6908 static Optional<RISCVBitmanipPat> matchSHFLPat(SDValue Op) {
6909   // These are the unshifted masks which we use to match bit-manipulation
6910   // patterns. They may be shifted left in certain circumstances.
6911   static const uint64_t BitmanipMasks[] = {
6912       0x2222222222222222ULL, 0x0C0C0C0C0C0C0C0CULL, 0x00F000F000F000F0ULL,
6913       0x0000FF000000FF00ULL, 0x00000000FFFF0000ULL};
6914 
6915   return matchRISCVBitmanipPat(Op, BitmanipMasks);
6916 }
6917 
6918 // Match (or (or (SHFL_SHL x), (SHFL_SHR x)), (SHFL_AND x)
6919 static SDValue combineORToSHFL(SDValue Op, SelectionDAG &DAG,
6920                                const RISCVSubtarget &Subtarget) {
6921   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6922   EVT VT = Op.getValueType();
6923 
6924   if (VT != MVT::i32 && VT != Subtarget.getXLenVT())
6925     return SDValue();
6926 
6927   SDValue Op0 = Op.getOperand(0);
6928   SDValue Op1 = Op.getOperand(1);
6929 
6930   // Or is commutable so canonicalize the second OR to the LHS.
6931   if (Op0.getOpcode() != ISD::OR)
6932     std::swap(Op0, Op1);
6933   if (Op0.getOpcode() != ISD::OR)
6934     return SDValue();
6935 
6936   // We found an inner OR, so our operands are the operands of the inner OR
6937   // and the other operand of the outer OR.
6938   SDValue A = Op0.getOperand(0);
6939   SDValue B = Op0.getOperand(1);
6940   SDValue C = Op1;
6941 
6942   auto Match1 = matchSHFLPat(A);
6943   auto Match2 = matchSHFLPat(B);
6944 
6945   // If neither matched, we failed.
6946   if (!Match1 && !Match2)
6947     return SDValue();
6948 
6949   // We had at least one match. if one failed, try the remaining C operand.
6950   if (!Match1) {
6951     std::swap(A, C);
6952     Match1 = matchSHFLPat(A);
6953     if (!Match1)
6954       return SDValue();
6955   } else if (!Match2) {
6956     std::swap(B, C);
6957     Match2 = matchSHFLPat(B);
6958     if (!Match2)
6959       return SDValue();
6960   }
6961   assert(Match1 && Match2);
6962 
6963   // Make sure our matches pair up.
6964   if (!Match1->formsPairWith(*Match2))
6965     return SDValue();
6966 
6967   // All the remains is to make sure C is an AND with the same input, that masks
6968   // out the bits that are being shuffled.
6969   if (C.getOpcode() != ISD::AND || !isa<ConstantSDNode>(C.getOperand(1)) ||
6970       C.getOperand(0) != Match1->Op)
6971     return SDValue();
6972 
6973   uint64_t Mask = C.getConstantOperandVal(1);
6974 
6975   static const uint64_t BitmanipMasks[] = {
6976       0x9999999999999999ULL, 0xC3C3C3C3C3C3C3C3ULL, 0xF00FF00FF00FF00FULL,
6977       0xFF0000FFFF0000FFULL, 0xFFFF00000000FFFFULL,
6978   };
6979 
6980   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
6981   unsigned MaskIdx = Log2_32(Match1->ShAmt);
6982   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
6983 
6984   if (Mask != ExpMask)
6985     return SDValue();
6986 
6987   SDLoc DL(Op);
6988   return DAG.getNode(RISCVISD::SHFL, DL, VT, Match1->Op,
6989                      DAG.getConstant(Match1->ShAmt, DL, VT));
6990 }
6991 
6992 // Optimize (add (shl x, c0), (shl y, c1)) ->
6993 //          (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
6994 static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
6995                                   const RISCVSubtarget &Subtarget) {
6996   // Perform this optimization only in the zba extension.
6997   if (!Subtarget.hasStdExtZba())
6998     return SDValue();
6999 
7000   // Skip for vector types and larger types.
7001   EVT VT = N->getValueType(0);
7002   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
7003     return SDValue();
7004 
7005   // The two operand nodes must be SHL and have no other use.
7006   SDValue N0 = N->getOperand(0);
7007   SDValue N1 = N->getOperand(1);
7008   if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
7009       !N0->hasOneUse() || !N1->hasOneUse())
7010     return SDValue();
7011 
7012   // Check c0 and c1.
7013   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7014   auto *N1C = dyn_cast<ConstantSDNode>(N1->getOperand(1));
7015   if (!N0C || !N1C)
7016     return SDValue();
7017   int64_t C0 = N0C->getSExtValue();
7018   int64_t C1 = N1C->getSExtValue();
7019   if (C0 <= 0 || C1 <= 0)
7020     return SDValue();
7021 
7022   // Skip if SH1ADD/SH2ADD/SH3ADD are not applicable.
7023   int64_t Bits = std::min(C0, C1);
7024   int64_t Diff = std::abs(C0 - C1);
7025   if (Diff != 1 && Diff != 2 && Diff != 3)
7026     return SDValue();
7027 
7028   // Build nodes.
7029   SDLoc DL(N);
7030   SDValue NS = (C0 < C1) ? N0->getOperand(0) : N1->getOperand(0);
7031   SDValue NL = (C0 > C1) ? N0->getOperand(0) : N1->getOperand(0);
7032   SDValue NA0 =
7033       DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT));
7034   SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS);
7035   return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT));
7036 }
7037 
7038 // Combine (GREVI (GREVI x, C2), C1) -> (GREVI x, C1^C2) when C1^C2 is
7039 // non-zero, and to x when it is. Any repeated GREVI stage undoes itself.
7040 // Combine (GORCI (GORCI x, C2), C1) -> (GORCI x, C1|C2). Repeated stage does
7041 // not undo itself, but they are redundant.
7042 static SDValue combineGREVI_GORCI(SDNode *N, SelectionDAG &DAG) {
7043   SDValue Src = N->getOperand(0);
7044 
7045   if (Src.getOpcode() != N->getOpcode())
7046     return SDValue();
7047 
7048   if (!isa<ConstantSDNode>(N->getOperand(1)) ||
7049       !isa<ConstantSDNode>(Src.getOperand(1)))
7050     return SDValue();
7051 
7052   unsigned ShAmt1 = N->getConstantOperandVal(1);
7053   unsigned ShAmt2 = Src.getConstantOperandVal(1);
7054   Src = Src.getOperand(0);
7055 
7056   unsigned CombinedShAmt;
7057   if (N->getOpcode() == RISCVISD::GORC || N->getOpcode() == RISCVISD::GORCW)
7058     CombinedShAmt = ShAmt1 | ShAmt2;
7059   else
7060     CombinedShAmt = ShAmt1 ^ ShAmt2;
7061 
7062   if (CombinedShAmt == 0)
7063     return Src;
7064 
7065   SDLoc DL(N);
7066   return DAG.getNode(
7067       N->getOpcode(), DL, N->getValueType(0), Src,
7068       DAG.getConstant(CombinedShAmt, DL, N->getOperand(1).getValueType()));
7069 }
7070 
7071 // Combine a constant select operand into its use:
7072 //
7073 // (and (select cond, -1, c), x)
7074 //   -> (select cond, x, (and x, c))  [AllOnes=1]
7075 // (or  (select cond, 0, c), x)
7076 //   -> (select cond, x, (or x, c))  [AllOnes=0]
7077 // (xor (select cond, 0, c), x)
7078 //   -> (select cond, x, (xor x, c))  [AllOnes=0]
7079 // (add (select cond, 0, c), x)
7080 //   -> (select cond, x, (add x, c))  [AllOnes=0]
7081 // (sub x, (select cond, 0, c))
7082 //   -> (select cond, x, (sub x, c))  [AllOnes=0]
7083 static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
7084                                    SelectionDAG &DAG, bool AllOnes) {
7085   EVT VT = N->getValueType(0);
7086 
7087   // Skip vectors.
7088   if (VT.isVector())
7089     return SDValue();
7090 
7091   if ((Slct.getOpcode() != ISD::SELECT &&
7092        Slct.getOpcode() != RISCVISD::SELECT_CC) ||
7093       !Slct.hasOneUse())
7094     return SDValue();
7095 
7096   auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
7097     return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
7098   };
7099 
7100   bool SwapSelectOps;
7101   unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
7102   SDValue TrueVal = Slct.getOperand(1 + OpOffset);
7103   SDValue FalseVal = Slct.getOperand(2 + OpOffset);
7104   SDValue NonConstantVal;
7105   if (isZeroOrAllOnes(TrueVal, AllOnes)) {
7106     SwapSelectOps = false;
7107     NonConstantVal = FalseVal;
7108   } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
7109     SwapSelectOps = true;
7110     NonConstantVal = TrueVal;
7111   } else
7112     return SDValue();
7113 
7114   // Slct is now know to be the desired identity constant when CC is true.
7115   TrueVal = OtherOp;
7116   FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
7117   // Unless SwapSelectOps says the condition should be false.
7118   if (SwapSelectOps)
7119     std::swap(TrueVal, FalseVal);
7120 
7121   if (Slct.getOpcode() == RISCVISD::SELECT_CC)
7122     return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
7123                        {Slct.getOperand(0), Slct.getOperand(1),
7124                         Slct.getOperand(2), TrueVal, FalseVal});
7125 
7126   return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
7127                      {Slct.getOperand(0), TrueVal, FalseVal});
7128 }
7129 
7130 // Attempt combineSelectAndUse on each operand of a commutative operator N.
7131 static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
7132                                               bool AllOnes) {
7133   SDValue N0 = N->getOperand(0);
7134   SDValue N1 = N->getOperand(1);
7135   if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes))
7136     return Result;
7137   if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes))
7138     return Result;
7139   return SDValue();
7140 }
7141 
7142 // Transform (add (mul x, c0), c1) ->
7143 //           (add (mul (add x, c1/c0), c0), c1%c0).
7144 // if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
7145 // that should be excluded is when c0*(c1/c0) is simm12, which will lead
7146 // to an infinite loop in DAGCombine if transformed.
7147 // Or transform (add (mul x, c0), c1) ->
7148 //              (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
7149 // if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
7150 // case that should be excluded is when c0*(c1/c0+1) is simm12, which will
7151 // lead to an infinite loop in DAGCombine if transformed.
7152 // Or transform (add (mul x, c0), c1) ->
7153 //              (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
7154 // if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
7155 // case that should be excluded is when c0*(c1/c0-1) is simm12, which will
7156 // lead to an infinite loop in DAGCombine if transformed.
7157 // Or transform (add (mul x, c0), c1) ->
7158 //              (mul (add x, c1/c0), c0).
7159 // if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
7160 static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
7161                                      const RISCVSubtarget &Subtarget) {
7162   // Skip for vector types and larger types.
7163   EVT VT = N->getValueType(0);
7164   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
7165     return SDValue();
7166   // The first operand node must be a MUL and has no other use.
7167   SDValue N0 = N->getOperand(0);
7168   if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
7169     return SDValue();
7170   // Check if c0 and c1 match above conditions.
7171   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7172   auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
7173   if (!N0C || !N1C)
7174     return SDValue();
7175   int64_t C0 = N0C->getSExtValue();
7176   int64_t C1 = N1C->getSExtValue();
7177   int64_t CA, CB;
7178   if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
7179     return SDValue();
7180   // Search for proper CA (non-zero) and CB that both are simm12.
7181   if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
7182       !isInt<12>(C0 * (C1 / C0))) {
7183     CA = C1 / C0;
7184     CB = C1 % C0;
7185   } else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
7186              isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
7187     CA = C1 / C0 + 1;
7188     CB = C1 % C0 - C0;
7189   } else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
7190              isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
7191     CA = C1 / C0 - 1;
7192     CB = C1 % C0 + C0;
7193   } else
7194     return SDValue();
7195   // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
7196   SDLoc DL(N);
7197   SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
7198                              DAG.getConstant(CA, DL, VT));
7199   SDValue New1 =
7200       DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
7201   return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
7202 }
7203 
7204 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
7205                                  const RISCVSubtarget &Subtarget) {
7206   if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
7207     return V;
7208   if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
7209     return V;
7210   // fold (add (select lhs, rhs, cc, 0, y), x) ->
7211   //      (select lhs, rhs, cc, x, (add x, y))
7212   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7213 }
7214 
7215 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG) {
7216   // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
7217   //      (select lhs, rhs, cc, x, (sub x, y))
7218   SDValue N0 = N->getOperand(0);
7219   SDValue N1 = N->getOperand(1);
7220   return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false);
7221 }
7222 
7223 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG) {
7224   // fold (and (select lhs, rhs, cc, -1, y), x) ->
7225   //      (select lhs, rhs, cc, x, (and x, y))
7226   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true);
7227 }
7228 
7229 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
7230                                 const RISCVSubtarget &Subtarget) {
7231   if (Subtarget.hasStdExtZbp()) {
7232     if (auto GREV = combineORToGREV(SDValue(N, 0), DAG, Subtarget))
7233       return GREV;
7234     if (auto GORC = combineORToGORC(SDValue(N, 0), DAG, Subtarget))
7235       return GORC;
7236     if (auto SHFL = combineORToSHFL(SDValue(N, 0), DAG, Subtarget))
7237       return SHFL;
7238   }
7239 
7240   // fold (or (select cond, 0, y), x) ->
7241   //      (select cond, x, (or x, y))
7242   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7243 }
7244 
7245 static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG) {
7246   // fold (xor (select cond, 0, y), x) ->
7247   //      (select cond, x, (xor x, y))
7248   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7249 }
7250 
7251 // Attempt to turn ANY_EXTEND into SIGN_EXTEND if the input to the ANY_EXTEND
7252 // has users that require SIGN_EXTEND and the SIGN_EXTEND can be done for free
7253 // by an instruction like ADDW/SUBW/MULW. Without this the ANY_EXTEND would be
7254 // removed during type legalization leaving an ADD/SUB/MUL use that won't use
7255 // ADDW/SUBW/MULW.
7256 static SDValue performANY_EXTENDCombine(SDNode *N,
7257                                         TargetLowering::DAGCombinerInfo &DCI,
7258                                         const RISCVSubtarget &Subtarget) {
7259   if (!Subtarget.is64Bit())
7260     return SDValue();
7261 
7262   SelectionDAG &DAG = DCI.DAG;
7263 
7264   SDValue Src = N->getOperand(0);
7265   EVT VT = N->getValueType(0);
7266   if (VT != MVT::i64 || Src.getValueType() != MVT::i32)
7267     return SDValue();
7268 
7269   // The opcode must be one that can implicitly sign_extend.
7270   // FIXME: Additional opcodes.
7271   switch (Src.getOpcode()) {
7272   default:
7273     return SDValue();
7274   case ISD::MUL:
7275     if (!Subtarget.hasStdExtM())
7276       return SDValue();
7277     LLVM_FALLTHROUGH;
7278   case ISD::ADD:
7279   case ISD::SUB:
7280     break;
7281   }
7282 
7283   // Only handle cases where the result is used by a CopyToReg. That likely
7284   // means the value is a liveout of the basic block. This helps prevent
7285   // infinite combine loops like PR51206.
7286   if (none_of(N->uses(),
7287               [](SDNode *User) { return User->getOpcode() == ISD::CopyToReg; }))
7288     return SDValue();
7289 
7290   SmallVector<SDNode *, 4> SetCCs;
7291   for (SDNode::use_iterator UI = Src.getNode()->use_begin(),
7292                             UE = Src.getNode()->use_end();
7293        UI != UE; ++UI) {
7294     SDNode *User = *UI;
7295     if (User == N)
7296       continue;
7297     if (UI.getUse().getResNo() != Src.getResNo())
7298       continue;
7299     // All i32 setccs are legalized by sign extending operands.
7300     if (User->getOpcode() == ISD::SETCC) {
7301       SetCCs.push_back(User);
7302       continue;
7303     }
7304     // We don't know if we can extend this user.
7305     break;
7306   }
7307 
7308   // If we don't have any SetCCs, this isn't worthwhile.
7309   if (SetCCs.empty())
7310     return SDValue();
7311 
7312   SDLoc DL(N);
7313   SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Src);
7314   DCI.CombineTo(N, SExt);
7315 
7316   // Promote all the setccs.
7317   for (SDNode *SetCC : SetCCs) {
7318     SmallVector<SDValue, 4> Ops;
7319 
7320     for (unsigned j = 0; j != 2; ++j) {
7321       SDValue SOp = SetCC->getOperand(j);
7322       if (SOp == Src)
7323         Ops.push_back(SExt);
7324       else
7325         Ops.push_back(DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, SOp));
7326     }
7327 
7328     Ops.push_back(SetCC->getOperand(2));
7329     DCI.CombineTo(SetCC,
7330                   DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
7331   }
7332   return SDValue(N, 0);
7333 }
7334 
7335 // Try to form VWMUL, VWMULU or VWMULSU.
7336 // TODO: Support VWMULSU.vx with a sign extend Op and a splat of scalar Op.
7337 static SDValue combineMUL_VLToVWMUL_VL(SDNode *N, SelectionDAG &DAG,
7338                                        bool Commute) {
7339   assert(N->getOpcode() == RISCVISD::MUL_VL && "Unexpected opcode");
7340   SDValue Op0 = N->getOperand(0);
7341   SDValue Op1 = N->getOperand(1);
7342   if (Commute)
7343     std::swap(Op0, Op1);
7344 
7345   bool IsSignExt = Op0.getOpcode() == RISCVISD::VSEXT_VL;
7346   bool IsZeroExt = Op0.getOpcode() == RISCVISD::VZEXT_VL;
7347   bool IsVWMULSU = IsSignExt && Op1.getOpcode() == RISCVISD::VZEXT_VL;
7348   if ((!IsSignExt && !IsZeroExt) || !Op0.hasOneUse())
7349     return SDValue();
7350 
7351   SDValue Mask = N->getOperand(2);
7352   SDValue VL = N->getOperand(3);
7353 
7354   // Make sure the mask and VL match.
7355   if (Op0.getOperand(1) != Mask || Op0.getOperand(2) != VL)
7356     return SDValue();
7357 
7358   MVT VT = N->getSimpleValueType(0);
7359 
7360   // Determine the narrow size for a widening multiply.
7361   unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
7362   MVT NarrowVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowSize),
7363                                   VT.getVectorElementCount());
7364 
7365   SDLoc DL(N);
7366 
7367   // See if the other operand is the same opcode.
7368   if (IsVWMULSU || Op0.getOpcode() == Op1.getOpcode()) {
7369     if (!Op1.hasOneUse())
7370       return SDValue();
7371 
7372     // Make sure the mask and VL match.
7373     if (Op1.getOperand(1) != Mask || Op1.getOperand(2) != VL)
7374       return SDValue();
7375 
7376     Op1 = Op1.getOperand(0);
7377   } else if (Op1.getOpcode() == RISCVISD::VMV_V_X_VL) {
7378     // The operand is a splat of a scalar.
7379 
7380     // The VL must be the same.
7381     if (Op1.getOperand(1) != VL)
7382       return SDValue();
7383 
7384     // Get the scalar value.
7385     Op1 = Op1.getOperand(0);
7386 
7387     // See if have enough sign bits or zero bits in the scalar to use a
7388     // widening multiply by splatting to smaller element size.
7389     unsigned EltBits = VT.getScalarSizeInBits();
7390     unsigned ScalarBits = Op1.getValueSizeInBits();
7391     // Make sure we're getting all element bits from the scalar register.
7392     // FIXME: Support implicit sign extension of vmv.v.x?
7393     if (ScalarBits < EltBits)
7394       return SDValue();
7395 
7396     if (IsSignExt) {
7397       if (DAG.ComputeNumSignBits(Op1) <= (ScalarBits - NarrowSize))
7398         return SDValue();
7399     } else {
7400       APInt Mask = APInt::getBitsSetFrom(ScalarBits, NarrowSize);
7401       if (!DAG.MaskedValueIsZero(Op1, Mask))
7402         return SDValue();
7403     }
7404 
7405     Op1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT, Op1, VL);
7406   } else
7407     return SDValue();
7408 
7409   Op0 = Op0.getOperand(0);
7410 
7411   // Re-introduce narrower extends if needed.
7412   unsigned ExtOpc = IsSignExt ? RISCVISD::VSEXT_VL : RISCVISD::VZEXT_VL;
7413   if (Op0.getValueType() != NarrowVT)
7414     Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
7415   if (Op1.getValueType() != NarrowVT)
7416     Op1 = DAG.getNode(ExtOpc, DL, NarrowVT, Op1, Mask, VL);
7417 
7418   unsigned WMulOpc = RISCVISD::VWMULSU_VL;
7419   if (!IsVWMULSU)
7420     WMulOpc = IsSignExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
7421   return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Mask, VL);
7422 }
7423 
7424 static RISCVFPRndMode::RoundingMode matchRoundingOp(SDValue Op) {
7425   switch (Op.getOpcode()) {
7426   case ISD::FROUNDEVEN: return RISCVFPRndMode::RNE;
7427   case ISD::FTRUNC:     return RISCVFPRndMode::RTZ;
7428   case ISD::FFLOOR:     return RISCVFPRndMode::RDN;
7429   case ISD::FCEIL:      return RISCVFPRndMode::RUP;
7430   case ISD::FROUND:     return RISCVFPRndMode::RMM;
7431   }
7432 
7433   return RISCVFPRndMode::Invalid;
7434 }
7435 
7436 // Fold
7437 //   (fp_to_int (froundeven X)) -> fcvt X, rne
7438 //   (fp_to_int (ftrunc X))     -> fcvt X, rtz
7439 //   (fp_to_int (ffloor X))     -> fcvt X, rdn
7440 //   (fp_to_int (fceil X))      -> fcvt X, rup
7441 //   (fp_to_int (fround X))     -> fcvt X, rmm
7442 static SDValue performFP_TO_INTCombine(SDNode *N,
7443                                        TargetLowering::DAGCombinerInfo &DCI,
7444                                        const RISCVSubtarget &Subtarget) {
7445   SelectionDAG &DAG = DCI.DAG;
7446   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7447   MVT XLenVT = Subtarget.getXLenVT();
7448 
7449   // Only handle XLen or i32 types. Other types narrower than XLen will
7450   // eventually be legalized to XLenVT.
7451   EVT VT = N->getValueType(0);
7452   if (VT != MVT::i32 && VT != XLenVT)
7453     return SDValue();
7454 
7455   SDValue Src = N->getOperand(0);
7456 
7457   // Ensure the FP type is also legal.
7458   if (!TLI.isTypeLegal(Src.getValueType()))
7459     return SDValue();
7460 
7461   // Don't do this for f16 with Zfhmin and not Zfh.
7462   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7463     return SDValue();
7464 
7465   RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src);
7466   if (FRM == RISCVFPRndMode::Invalid)
7467     return SDValue();
7468 
7469   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
7470 
7471   unsigned Opc;
7472   if (VT == XLenVT)
7473     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7474   else
7475     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7476 
7477   SDLoc DL(N);
7478   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src.getOperand(0),
7479                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7480   return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
7481 }
7482 
7483 // Fold
7484 //   (fp_to_int_sat (froundeven X)) -> (select X == nan, 0, (fcvt X, rne))
7485 //   (fp_to_int_sat (ftrunc X))     -> (select X == nan, 0, (fcvt X, rtz))
7486 //   (fp_to_int_sat (ffloor X))     -> (select X == nan, 0, (fcvt X, rdn))
7487 //   (fp_to_int_sat (fceil X))      -> (select X == nan, 0, (fcvt X, rup))
7488 //   (fp_to_int_sat (fround X))     -> (select X == nan, 0, (fcvt X, rmm))
7489 static SDValue performFP_TO_INT_SATCombine(SDNode *N,
7490                                        TargetLowering::DAGCombinerInfo &DCI,
7491                                        const RISCVSubtarget &Subtarget) {
7492   SelectionDAG &DAG = DCI.DAG;
7493   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7494   MVT XLenVT = Subtarget.getXLenVT();
7495 
7496   // Only handle XLen types. Other types narrower than XLen will eventually be
7497   // legalized to XLenVT.
7498   EVT DstVT = N->getValueType(0);
7499   if (DstVT != XLenVT)
7500     return SDValue();
7501 
7502   SDValue Src = N->getOperand(0);
7503 
7504   // Ensure the FP type is also legal.
7505   if (!TLI.isTypeLegal(Src.getValueType()))
7506     return SDValue();
7507 
7508   // Don't do this for f16 with Zfhmin and not Zfh.
7509   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7510     return SDValue();
7511 
7512   EVT SatVT = cast<VTSDNode>(N->getOperand(1))->getVT();
7513 
7514   RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src);
7515   if (FRM == RISCVFPRndMode::Invalid)
7516     return SDValue();
7517 
7518   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
7519 
7520   unsigned Opc;
7521   if (SatVT == DstVT)
7522     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7523   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
7524     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7525   else
7526     return SDValue();
7527   // FIXME: Support other SatVTs by clamping before or after the conversion.
7528 
7529   Src = Src.getOperand(0);
7530 
7531   SDLoc DL(N);
7532   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src,
7533                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7534 
7535   // RISCV FP-to-int conversions saturate to the destination register size, but
7536   // don't produce 0 for nan.
7537   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
7538   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
7539 }
7540 
7541 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
7542                                                DAGCombinerInfo &DCI) const {
7543   SelectionDAG &DAG = DCI.DAG;
7544 
7545   // Helper to call SimplifyDemandedBits on an operand of N where only some low
7546   // bits are demanded. N will be added to the Worklist if it was not deleted.
7547   // Caller should return SDValue(N, 0) if this returns true.
7548   auto SimplifyDemandedLowBitsHelper = [&](unsigned OpNo, unsigned LowBits) {
7549     SDValue Op = N->getOperand(OpNo);
7550     APInt Mask = APInt::getLowBitsSet(Op.getValueSizeInBits(), LowBits);
7551     if (!SimplifyDemandedBits(Op, Mask, DCI))
7552       return false;
7553 
7554     if (N->getOpcode() != ISD::DELETED_NODE)
7555       DCI.AddToWorklist(N);
7556     return true;
7557   };
7558 
7559   switch (N->getOpcode()) {
7560   default:
7561     break;
7562   case RISCVISD::SplitF64: {
7563     SDValue Op0 = N->getOperand(0);
7564     // If the input to SplitF64 is just BuildPairF64 then the operation is
7565     // redundant. Instead, use BuildPairF64's operands directly.
7566     if (Op0->getOpcode() == RISCVISD::BuildPairF64)
7567       return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
7568 
7569     SDLoc DL(N);
7570 
7571     // It's cheaper to materialise two 32-bit integers than to load a double
7572     // from the constant pool and transfer it to integer registers through the
7573     // stack.
7574     if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
7575       APInt V = C->getValueAPF().bitcastToAPInt();
7576       SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
7577       SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
7578       return DCI.CombineTo(N, Lo, Hi);
7579     }
7580 
7581     // This is a target-specific version of a DAGCombine performed in
7582     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7583     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7584     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7585     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7586         !Op0.getNode()->hasOneUse())
7587       break;
7588     SDValue NewSplitF64 =
7589         DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
7590                     Op0.getOperand(0));
7591     SDValue Lo = NewSplitF64.getValue(0);
7592     SDValue Hi = NewSplitF64.getValue(1);
7593     APInt SignBit = APInt::getSignMask(32);
7594     if (Op0.getOpcode() == ISD::FNEG) {
7595       SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
7596                                   DAG.getConstant(SignBit, DL, MVT::i32));
7597       return DCI.CombineTo(N, Lo, NewHi);
7598     }
7599     assert(Op0.getOpcode() == ISD::FABS);
7600     SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
7601                                 DAG.getConstant(~SignBit, DL, MVT::i32));
7602     return DCI.CombineTo(N, Lo, NewHi);
7603   }
7604   case RISCVISD::SLLW:
7605   case RISCVISD::SRAW:
7606   case RISCVISD::SRLW:
7607   case RISCVISD::ROLW:
7608   case RISCVISD::RORW: {
7609     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7610     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7611         SimplifyDemandedLowBitsHelper(1, 5))
7612       return SDValue(N, 0);
7613     break;
7614   }
7615   case RISCVISD::CLZW:
7616   case RISCVISD::CTZW: {
7617     // Only the lower 32 bits of the first operand are read
7618     if (SimplifyDemandedLowBitsHelper(0, 32))
7619       return SDValue(N, 0);
7620     break;
7621   }
7622   case RISCVISD::GREV:
7623   case RISCVISD::GORC: {
7624     // Only the lower log2(Bitwidth) bits of the the shift amount are read.
7625     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7626     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7627     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth)))
7628       return SDValue(N, 0);
7629 
7630     return combineGREVI_GORCI(N, DAG);
7631   }
7632   case RISCVISD::GREVW:
7633   case RISCVISD::GORCW: {
7634     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7635     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7636         SimplifyDemandedLowBitsHelper(1, 5))
7637       return SDValue(N, 0);
7638 
7639     return combineGREVI_GORCI(N, DAG);
7640   }
7641   case RISCVISD::SHFL:
7642   case RISCVISD::UNSHFL: {
7643     // Only the lower log2(Bitwidth)-1 bits of the the shift amount are read.
7644     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7645     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7646     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth) - 1))
7647       return SDValue(N, 0);
7648 
7649     break;
7650   }
7651   case RISCVISD::SHFLW:
7652   case RISCVISD::UNSHFLW: {
7653     // Only the lower 32 bits of LHS and lower 4 bits of RHS are read.
7654     SDValue LHS = N->getOperand(0);
7655     SDValue RHS = N->getOperand(1);
7656     APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
7657     APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 4);
7658     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7659         SimplifyDemandedLowBitsHelper(1, 4))
7660       return SDValue(N, 0);
7661 
7662     break;
7663   }
7664   case RISCVISD::BCOMPRESSW:
7665   case RISCVISD::BDECOMPRESSW: {
7666     // Only the lower 32 bits of LHS and RHS are read.
7667     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7668         SimplifyDemandedLowBitsHelper(1, 32))
7669       return SDValue(N, 0);
7670 
7671     break;
7672   }
7673   case RISCVISD::FMV_X_ANYEXTH:
7674   case RISCVISD::FMV_X_ANYEXTW_RV64: {
7675     SDLoc DL(N);
7676     SDValue Op0 = N->getOperand(0);
7677     MVT VT = N->getSimpleValueType(0);
7678     // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
7679     // conversion is unnecessary and can be replaced with the FMV_W_X_RV64
7680     // operand. Similar for FMV_X_ANYEXTH and FMV_H_X.
7681     if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
7682          Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
7683         (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
7684          Op0->getOpcode() == RISCVISD::FMV_H_X)) {
7685       assert(Op0.getOperand(0).getValueType() == VT &&
7686              "Unexpected value type!");
7687       return Op0.getOperand(0);
7688     }
7689 
7690     // This is a target-specific version of a DAGCombine performed in
7691     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7692     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7693     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7694     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7695         !Op0.getNode()->hasOneUse())
7696       break;
7697     SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
7698     unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
7699     APInt SignBit = APInt::getSignMask(FPBits).sextOrSelf(VT.getSizeInBits());
7700     if (Op0.getOpcode() == ISD::FNEG)
7701       return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
7702                          DAG.getConstant(SignBit, DL, VT));
7703 
7704     assert(Op0.getOpcode() == ISD::FABS);
7705     return DAG.getNode(ISD::AND, DL, VT, NewFMV,
7706                        DAG.getConstant(~SignBit, DL, VT));
7707   }
7708   case ISD::ADD:
7709     return performADDCombine(N, DAG, Subtarget);
7710   case ISD::SUB:
7711     return performSUBCombine(N, DAG);
7712   case ISD::AND:
7713     return performANDCombine(N, DAG);
7714   case ISD::OR:
7715     return performORCombine(N, DAG, Subtarget);
7716   case ISD::XOR:
7717     return performXORCombine(N, DAG);
7718   case ISD::ANY_EXTEND:
7719     return performANY_EXTENDCombine(N, DCI, Subtarget);
7720   case ISD::ZERO_EXTEND:
7721     // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
7722     // type legalization. This is safe because fp_to_uint produces poison if
7723     // it overflows.
7724     if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) {
7725       SDValue Src = N->getOperand(0);
7726       if (Src.getOpcode() == ISD::FP_TO_UINT &&
7727           isTypeLegal(Src.getOperand(0).getValueType()))
7728         return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
7729                            Src.getOperand(0));
7730       if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
7731           isTypeLegal(Src.getOperand(1).getValueType())) {
7732         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
7733         SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs,
7734                                   Src.getOperand(0), Src.getOperand(1));
7735         DCI.CombineTo(N, Res);
7736         DAG.ReplaceAllUsesOfValueWith(Src.getValue(1), Res.getValue(1));
7737         DCI.recursivelyDeleteUnusedNodes(Src.getNode());
7738         return SDValue(N, 0); // Return N so it doesn't get rechecked.
7739       }
7740     }
7741     return SDValue();
7742   case RISCVISD::SELECT_CC: {
7743     // Transform
7744     SDValue LHS = N->getOperand(0);
7745     SDValue RHS = N->getOperand(1);
7746     SDValue TrueV = N->getOperand(3);
7747     SDValue FalseV = N->getOperand(4);
7748 
7749     // If the True and False values are the same, we don't need a select_cc.
7750     if (TrueV == FalseV)
7751       return TrueV;
7752 
7753     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(2))->get();
7754     if (!ISD::isIntEqualitySetCC(CCVal))
7755       break;
7756 
7757     // Fold (select_cc (setlt X, Y), 0, ne, trueV, falseV) ->
7758     //      (select_cc X, Y, lt, trueV, falseV)
7759     // Sometimes the setcc is introduced after select_cc has been formed.
7760     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
7761         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
7762       // If we're looking for eq 0 instead of ne 0, we need to invert the
7763       // condition.
7764       bool Invert = CCVal == ISD::SETEQ;
7765       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
7766       if (Invert)
7767         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7768 
7769       SDLoc DL(N);
7770       RHS = LHS.getOperand(1);
7771       LHS = LHS.getOperand(0);
7772       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7773 
7774       SDValue TargetCC = DAG.getCondCode(CCVal);
7775       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
7776                          {LHS, RHS, TargetCC, TrueV, FalseV});
7777     }
7778 
7779     // Fold (select_cc (xor X, Y), 0, eq/ne, trueV, falseV) ->
7780     //      (select_cc X, Y, eq/ne, trueV, falseV)
7781     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
7782       return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), N->getValueType(0),
7783                          {LHS.getOperand(0), LHS.getOperand(1),
7784                           N->getOperand(2), TrueV, FalseV});
7785     // (select_cc X, 1, setne, trueV, falseV) ->
7786     // (select_cc X, 0, seteq, trueV, falseV) if we can prove X is 0/1.
7787     // This can occur when legalizing some floating point comparisons.
7788     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
7789     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
7790       SDLoc DL(N);
7791       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7792       SDValue TargetCC = DAG.getCondCode(CCVal);
7793       RHS = DAG.getConstant(0, DL, LHS.getValueType());
7794       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
7795                          {LHS, RHS, TargetCC, TrueV, FalseV});
7796     }
7797 
7798     break;
7799   }
7800   case RISCVISD::BR_CC: {
7801     SDValue LHS = N->getOperand(1);
7802     SDValue RHS = N->getOperand(2);
7803     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(3))->get();
7804     if (!ISD::isIntEqualitySetCC(CCVal))
7805       break;
7806 
7807     // Fold (br_cc (setlt X, Y), 0, ne, dest) ->
7808     //      (br_cc X, Y, lt, dest)
7809     // Sometimes the setcc is introduced after br_cc has been formed.
7810     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
7811         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
7812       // If we're looking for eq 0 instead of ne 0, we need to invert the
7813       // condition.
7814       bool Invert = CCVal == ISD::SETEQ;
7815       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
7816       if (Invert)
7817         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7818 
7819       SDLoc DL(N);
7820       RHS = LHS.getOperand(1);
7821       LHS = LHS.getOperand(0);
7822       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7823 
7824       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
7825                          N->getOperand(0), LHS, RHS, DAG.getCondCode(CCVal),
7826                          N->getOperand(4));
7827     }
7828 
7829     // Fold (br_cc (xor X, Y), 0, eq/ne, dest) ->
7830     //      (br_cc X, Y, eq/ne, trueV, falseV)
7831     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
7832       return DAG.getNode(RISCVISD::BR_CC, SDLoc(N), N->getValueType(0),
7833                          N->getOperand(0), LHS.getOperand(0), LHS.getOperand(1),
7834                          N->getOperand(3), N->getOperand(4));
7835 
7836     // (br_cc X, 1, setne, br_cc) ->
7837     // (br_cc X, 0, seteq, br_cc) if we can prove X is 0/1.
7838     // This can occur when legalizing some floating point comparisons.
7839     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
7840     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
7841       SDLoc DL(N);
7842       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7843       SDValue TargetCC = DAG.getCondCode(CCVal);
7844       RHS = DAG.getConstant(0, DL, LHS.getValueType());
7845       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
7846                          N->getOperand(0), LHS, RHS, TargetCC,
7847                          N->getOperand(4));
7848     }
7849     break;
7850   }
7851   case ISD::FP_TO_SINT:
7852   case ISD::FP_TO_UINT:
7853     return performFP_TO_INTCombine(N, DCI, Subtarget);
7854   case ISD::FP_TO_SINT_SAT:
7855   case ISD::FP_TO_UINT_SAT:
7856     return performFP_TO_INT_SATCombine(N, DCI, Subtarget);
7857   case ISD::FCOPYSIGN: {
7858     EVT VT = N->getValueType(0);
7859     if (!VT.isVector())
7860       break;
7861     // There is a form of VFSGNJ which injects the negated sign of its second
7862     // operand. Try and bubble any FNEG up after the extend/round to produce
7863     // this optimized pattern. Avoid modifying cases where FP_ROUND and
7864     // TRUNC=1.
7865     SDValue In2 = N->getOperand(1);
7866     // Avoid cases where the extend/round has multiple uses, as duplicating
7867     // those is typically more expensive than removing a fneg.
7868     if (!In2.hasOneUse())
7869       break;
7870     if (In2.getOpcode() != ISD::FP_EXTEND &&
7871         (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0))
7872       break;
7873     In2 = In2.getOperand(0);
7874     if (In2.getOpcode() != ISD::FNEG)
7875       break;
7876     SDLoc DL(N);
7877     SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
7878     return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
7879                        DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
7880   }
7881   case ISD::MGATHER:
7882   case ISD::MSCATTER:
7883   case ISD::VP_GATHER:
7884   case ISD::VP_SCATTER: {
7885     if (!DCI.isBeforeLegalize())
7886       break;
7887     SDValue Index, ScaleOp;
7888     bool IsIndexScaled = false;
7889     bool IsIndexSigned = false;
7890     if (const auto *VPGSN = dyn_cast<VPGatherScatterSDNode>(N)) {
7891       Index = VPGSN->getIndex();
7892       ScaleOp = VPGSN->getScale();
7893       IsIndexScaled = VPGSN->isIndexScaled();
7894       IsIndexSigned = VPGSN->isIndexSigned();
7895     } else {
7896       const auto *MGSN = cast<MaskedGatherScatterSDNode>(N);
7897       Index = MGSN->getIndex();
7898       ScaleOp = MGSN->getScale();
7899       IsIndexScaled = MGSN->isIndexScaled();
7900       IsIndexSigned = MGSN->isIndexSigned();
7901     }
7902     EVT IndexVT = Index.getValueType();
7903     MVT XLenVT = Subtarget.getXLenVT();
7904     // RISCV indexed loads only support the "unsigned unscaled" addressing
7905     // mode, so anything else must be manually legalized.
7906     bool NeedsIdxLegalization =
7907         IsIndexScaled ||
7908         (IsIndexSigned && IndexVT.getVectorElementType().bitsLT(XLenVT));
7909     if (!NeedsIdxLegalization)
7910       break;
7911 
7912     SDLoc DL(N);
7913 
7914     // Any index legalization should first promote to XLenVT, so we don't lose
7915     // bits when scaling. This may create an illegal index type so we let
7916     // LLVM's legalization take care of the splitting.
7917     // FIXME: LLVM can't split VP_GATHER or VP_SCATTER yet.
7918     if (IndexVT.getVectorElementType().bitsLT(XLenVT)) {
7919       IndexVT = IndexVT.changeVectorElementType(XLenVT);
7920       Index = DAG.getNode(IsIndexSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
7921                           DL, IndexVT, Index);
7922     }
7923 
7924     unsigned Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue();
7925     if (IsIndexScaled && Scale != 1) {
7926       // Manually scale the indices by the element size.
7927       // TODO: Sanitize the scale operand here?
7928       // TODO: For VP nodes, should we use VP_SHL here?
7929       assert(isPowerOf2_32(Scale) && "Expecting power-of-two types");
7930       SDValue SplatScale = DAG.getConstant(Log2_32(Scale), DL, IndexVT);
7931       Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, SplatScale);
7932     }
7933 
7934     ISD::MemIndexType NewIndexTy = ISD::UNSIGNED_UNSCALED;
7935     if (const auto *VPGN = dyn_cast<VPGatherSDNode>(N))
7936       return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
7937                              {VPGN->getChain(), VPGN->getBasePtr(), Index,
7938                               VPGN->getScale(), VPGN->getMask(),
7939                               VPGN->getVectorLength()},
7940                              VPGN->getMemOperand(), NewIndexTy);
7941     if (const auto *VPSN = dyn_cast<VPScatterSDNode>(N))
7942       return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
7943                               {VPSN->getChain(), VPSN->getValue(),
7944                                VPSN->getBasePtr(), Index, VPSN->getScale(),
7945                                VPSN->getMask(), VPSN->getVectorLength()},
7946                               VPSN->getMemOperand(), NewIndexTy);
7947     if (const auto *MGN = dyn_cast<MaskedGatherSDNode>(N))
7948       return DAG.getMaskedGather(
7949           N->getVTList(), MGN->getMemoryVT(), DL,
7950           {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
7951            MGN->getBasePtr(), Index, MGN->getScale()},
7952           MGN->getMemOperand(), NewIndexTy, MGN->getExtensionType());
7953     const auto *MSN = cast<MaskedScatterSDNode>(N);
7954     return DAG.getMaskedScatter(
7955         N->getVTList(), MSN->getMemoryVT(), DL,
7956         {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
7957          Index, MSN->getScale()},
7958         MSN->getMemOperand(), NewIndexTy, MSN->isTruncatingStore());
7959   }
7960   case RISCVISD::SRA_VL:
7961   case RISCVISD::SRL_VL:
7962   case RISCVISD::SHL_VL: {
7963     SDValue ShAmt = N->getOperand(1);
7964     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
7965       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
7966       SDLoc DL(N);
7967       SDValue VL = N->getOperand(3);
7968       EVT VT = N->getValueType(0);
7969       ShAmt =
7970           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, ShAmt.getOperand(0), VL);
7971       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
7972                          N->getOperand(2), N->getOperand(3));
7973     }
7974     break;
7975   }
7976   case ISD::SRA:
7977   case ISD::SRL:
7978   case ISD::SHL: {
7979     SDValue ShAmt = N->getOperand(1);
7980     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
7981       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
7982       SDLoc DL(N);
7983       EVT VT = N->getValueType(0);
7984       ShAmt =
7985           DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VT, ShAmt.getOperand(0));
7986       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
7987     }
7988     break;
7989   }
7990   case RISCVISD::MUL_VL:
7991     if (SDValue V = combineMUL_VLToVWMUL_VL(N, DAG, /*Commute*/ false))
7992       return V;
7993     // Mul is commutative.
7994     return combineMUL_VLToVWMUL_VL(N, DAG, /*Commute*/ true);
7995   case ISD::STORE: {
7996     auto *Store = cast<StoreSDNode>(N);
7997     SDValue Val = Store->getValue();
7998     // Combine store of vmv.x.s to vse with VL of 1.
7999     // FIXME: Support FP.
8000     if (Val.getOpcode() == RISCVISD::VMV_X_S) {
8001       SDValue Src = Val.getOperand(0);
8002       EVT VecVT = Src.getValueType();
8003       EVT MemVT = Store->getMemoryVT();
8004       // The memory VT and the element type must match.
8005       if (VecVT.getVectorElementType() == MemVT) {
8006         SDLoc DL(N);
8007         MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
8008         return DAG.getStoreVP(
8009             Store->getChain(), DL, Src, Store->getBasePtr(), Store->getOffset(),
8010             DAG.getConstant(1, DL, MaskVT),
8011             DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT,
8012             Store->getMemOperand(), Store->getAddressingMode(),
8013             Store->isTruncatingStore(), /*IsCompress*/ false);
8014       }
8015     }
8016 
8017     break;
8018   }
8019   }
8020 
8021   return SDValue();
8022 }
8023 
8024 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
8025     const SDNode *N, CombineLevel Level) const {
8026   // The following folds are only desirable if `(OP _, c1 << c2)` can be
8027   // materialised in fewer instructions than `(OP _, c1)`:
8028   //
8029   //   (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
8030   //   (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
8031   SDValue N0 = N->getOperand(0);
8032   EVT Ty = N0.getValueType();
8033   if (Ty.isScalarInteger() &&
8034       (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
8035     auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
8036     auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
8037     if (C1 && C2) {
8038       const APInt &C1Int = C1->getAPIntValue();
8039       APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
8040 
8041       // We can materialise `c1 << c2` into an add immediate, so it's "free",
8042       // and the combine should happen, to potentially allow further combines
8043       // later.
8044       if (ShiftedC1Int.getMinSignedBits() <= 64 &&
8045           isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
8046         return true;
8047 
8048       // We can materialise `c1` in an add immediate, so it's "free", and the
8049       // combine should be prevented.
8050       if (C1Int.getMinSignedBits() <= 64 &&
8051           isLegalAddImmediate(C1Int.getSExtValue()))
8052         return false;
8053 
8054       // Neither constant will fit into an immediate, so find materialisation
8055       // costs.
8056       int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
8057                                               Subtarget.getFeatureBits(),
8058                                               /*CompressionCost*/true);
8059       int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
8060           ShiftedC1Int, Ty.getSizeInBits(), Subtarget.getFeatureBits(),
8061           /*CompressionCost*/true);
8062 
8063       // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
8064       // combine should be prevented.
8065       if (C1Cost < ShiftedC1Cost)
8066         return false;
8067     }
8068   }
8069   return true;
8070 }
8071 
8072 bool RISCVTargetLowering::targetShrinkDemandedConstant(
8073     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
8074     TargetLoweringOpt &TLO) const {
8075   // Delay this optimization as late as possible.
8076   if (!TLO.LegalOps)
8077     return false;
8078 
8079   EVT VT = Op.getValueType();
8080   if (VT.isVector())
8081     return false;
8082 
8083   // Only handle AND for now.
8084   if (Op.getOpcode() != ISD::AND)
8085     return false;
8086 
8087   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
8088   if (!C)
8089     return false;
8090 
8091   const APInt &Mask = C->getAPIntValue();
8092 
8093   // Clear all non-demanded bits initially.
8094   APInt ShrunkMask = Mask & DemandedBits;
8095 
8096   // Try to make a smaller immediate by setting undemanded bits.
8097 
8098   APInt ExpandedMask = Mask | ~DemandedBits;
8099 
8100   auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
8101     return ShrunkMask.isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
8102   };
8103   auto UseMask = [Mask, Op, VT, &TLO](const APInt &NewMask) -> bool {
8104     if (NewMask == Mask)
8105       return true;
8106     SDLoc DL(Op);
8107     SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
8108     SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
8109     return TLO.CombineTo(Op, NewOp);
8110   };
8111 
8112   // If the shrunk mask fits in sign extended 12 bits, let the target
8113   // independent code apply it.
8114   if (ShrunkMask.isSignedIntN(12))
8115     return false;
8116 
8117   // Preserve (and X, 0xffff) when zext.h is supported.
8118   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
8119     APInt NewMask = APInt(Mask.getBitWidth(), 0xffff);
8120     if (IsLegalMask(NewMask))
8121       return UseMask(NewMask);
8122   }
8123 
8124   // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
8125   if (VT == MVT::i64) {
8126     APInt NewMask = APInt(64, 0xffffffff);
8127     if (IsLegalMask(NewMask))
8128       return UseMask(NewMask);
8129   }
8130 
8131   // For the remaining optimizations, we need to be able to make a negative
8132   // number through a combination of mask and undemanded bits.
8133   if (!ExpandedMask.isNegative())
8134     return false;
8135 
8136   // What is the fewest number of bits we need to represent the negative number.
8137   unsigned MinSignedBits = ExpandedMask.getMinSignedBits();
8138 
8139   // Try to make a 12 bit negative immediate. If that fails try to make a 32
8140   // bit negative immediate unless the shrunk immediate already fits in 32 bits.
8141   APInt NewMask = ShrunkMask;
8142   if (MinSignedBits <= 12)
8143     NewMask.setBitsFrom(11);
8144   else if (MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(32))
8145     NewMask.setBitsFrom(31);
8146   else
8147     return false;
8148 
8149   // Check that our new mask is a subset of the demanded mask.
8150   assert(IsLegalMask(NewMask));
8151   return UseMask(NewMask);
8152 }
8153 
8154 static void computeGREV(APInt &Src, unsigned ShAmt) {
8155   ShAmt &= Src.getBitWidth() - 1;
8156   uint64_t x = Src.getZExtValue();
8157   if (ShAmt & 1)
8158     x = ((x & 0x5555555555555555LL) << 1) | ((x & 0xAAAAAAAAAAAAAAAALL) >> 1);
8159   if (ShAmt & 2)
8160     x = ((x & 0x3333333333333333LL) << 2) | ((x & 0xCCCCCCCCCCCCCCCCLL) >> 2);
8161   if (ShAmt & 4)
8162     x = ((x & 0x0F0F0F0F0F0F0F0FLL) << 4) | ((x & 0xF0F0F0F0F0F0F0F0LL) >> 4);
8163   if (ShAmt & 8)
8164     x = ((x & 0x00FF00FF00FF00FFLL) << 8) | ((x & 0xFF00FF00FF00FF00LL) >> 8);
8165   if (ShAmt & 16)
8166     x = ((x & 0x0000FFFF0000FFFFLL) << 16) | ((x & 0xFFFF0000FFFF0000LL) >> 16);
8167   if (ShAmt & 32)
8168     x = ((x & 0x00000000FFFFFFFFLL) << 32) | ((x & 0xFFFFFFFF00000000LL) >> 32);
8169   Src = x;
8170 }
8171 
8172 void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
8173                                                         KnownBits &Known,
8174                                                         const APInt &DemandedElts,
8175                                                         const SelectionDAG &DAG,
8176                                                         unsigned Depth) const {
8177   unsigned BitWidth = Known.getBitWidth();
8178   unsigned Opc = Op.getOpcode();
8179   assert((Opc >= ISD::BUILTIN_OP_END ||
8180           Opc == ISD::INTRINSIC_WO_CHAIN ||
8181           Opc == ISD::INTRINSIC_W_CHAIN ||
8182           Opc == ISD::INTRINSIC_VOID) &&
8183          "Should use MaskedValueIsZero if you don't know whether Op"
8184          " is a target node!");
8185 
8186   Known.resetAll();
8187   switch (Opc) {
8188   default: break;
8189   case RISCVISD::SELECT_CC: {
8190     Known = DAG.computeKnownBits(Op.getOperand(4), Depth + 1);
8191     // If we don't know any bits, early out.
8192     if (Known.isUnknown())
8193       break;
8194     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(3), Depth + 1);
8195 
8196     // Only known if known in both the LHS and RHS.
8197     Known = KnownBits::commonBits(Known, Known2);
8198     break;
8199   }
8200   case RISCVISD::REMUW: {
8201     KnownBits Known2;
8202     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
8203     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
8204     // We only care about the lower 32 bits.
8205     Known = KnownBits::urem(Known.trunc(32), Known2.trunc(32));
8206     // Restore the original width by sign extending.
8207     Known = Known.sext(BitWidth);
8208     break;
8209   }
8210   case RISCVISD::DIVUW: {
8211     KnownBits Known2;
8212     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
8213     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
8214     // We only care about the lower 32 bits.
8215     Known = KnownBits::udiv(Known.trunc(32), Known2.trunc(32));
8216     // Restore the original width by sign extending.
8217     Known = Known.sext(BitWidth);
8218     break;
8219   }
8220   case RISCVISD::CTZW: {
8221     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8222     unsigned PossibleTZ = Known2.trunc(32).countMaxTrailingZeros();
8223     unsigned LowBits = Log2_32(PossibleTZ) + 1;
8224     Known.Zero.setBitsFrom(LowBits);
8225     break;
8226   }
8227   case RISCVISD::CLZW: {
8228     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8229     unsigned PossibleLZ = Known2.trunc(32).countMaxLeadingZeros();
8230     unsigned LowBits = Log2_32(PossibleLZ) + 1;
8231     Known.Zero.setBitsFrom(LowBits);
8232     break;
8233   }
8234   case RISCVISD::GREV:
8235   case RISCVISD::GREVW: {
8236     if (auto *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
8237       Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8238       if (Opc == RISCVISD::GREVW)
8239         Known = Known.trunc(32);
8240       unsigned ShAmt = C->getZExtValue();
8241       computeGREV(Known.Zero, ShAmt);
8242       computeGREV(Known.One, ShAmt);
8243       if (Opc == RISCVISD::GREVW)
8244         Known = Known.sext(BitWidth);
8245     }
8246     break;
8247   }
8248   case RISCVISD::READ_VLENB:
8249     // We assume VLENB is at least 16 bytes.
8250     Known.Zero.setLowBits(4);
8251     // We assume VLENB is no more than 65536 / 8 bytes.
8252     Known.Zero.setBitsFrom(14);
8253     break;
8254   case ISD::INTRINSIC_W_CHAIN:
8255   case ISD::INTRINSIC_WO_CHAIN: {
8256     unsigned IntNo =
8257         Op.getConstantOperandVal(Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1);
8258     switch (IntNo) {
8259     default:
8260       // We can't do anything for most intrinsics.
8261       break;
8262     case Intrinsic::riscv_vsetvli:
8263     case Intrinsic::riscv_vsetvlimax:
8264     case Intrinsic::riscv_vsetvli_opt:
8265     case Intrinsic::riscv_vsetvlimax_opt:
8266       // Assume that VL output is positive and would fit in an int32_t.
8267       // TODO: VLEN might be capped at 16 bits in a future V spec update.
8268       if (BitWidth >= 32)
8269         Known.Zero.setBitsFrom(31);
8270       break;
8271     }
8272     break;
8273   }
8274   }
8275 }
8276 
8277 unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
8278     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
8279     unsigned Depth) const {
8280   switch (Op.getOpcode()) {
8281   default:
8282     break;
8283   case RISCVISD::SELECT_CC: {
8284     unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth + 1);
8285     if (Tmp == 1) return 1;  // Early out.
8286     unsigned Tmp2 = DAG.ComputeNumSignBits(Op.getOperand(4), DemandedElts, Depth + 1);
8287     return std::min(Tmp, Tmp2);
8288   }
8289   case RISCVISD::SLLW:
8290   case RISCVISD::SRAW:
8291   case RISCVISD::SRLW:
8292   case RISCVISD::DIVW:
8293   case RISCVISD::DIVUW:
8294   case RISCVISD::REMUW:
8295   case RISCVISD::ROLW:
8296   case RISCVISD::RORW:
8297   case RISCVISD::GREVW:
8298   case RISCVISD::GORCW:
8299   case RISCVISD::FSLW:
8300   case RISCVISD::FSRW:
8301   case RISCVISD::SHFLW:
8302   case RISCVISD::UNSHFLW:
8303   case RISCVISD::BCOMPRESSW:
8304   case RISCVISD::BDECOMPRESSW:
8305   case RISCVISD::BFPW:
8306   case RISCVISD::FCVT_W_RV64:
8307   case RISCVISD::FCVT_WU_RV64:
8308   case RISCVISD::STRICT_FCVT_W_RV64:
8309   case RISCVISD::STRICT_FCVT_WU_RV64:
8310     // TODO: As the result is sign-extended, this is conservatively correct. A
8311     // more precise answer could be calculated for SRAW depending on known
8312     // bits in the shift amount.
8313     return 33;
8314   case RISCVISD::SHFL:
8315   case RISCVISD::UNSHFL: {
8316     // There is no SHFLIW, but a i64 SHFLI with bit 4 of the control word
8317     // cleared doesn't affect bit 31. The upper 32 bits will be shuffled, but
8318     // will stay within the upper 32 bits. If there were more than 32 sign bits
8319     // before there will be at least 33 sign bits after.
8320     if (Op.getValueType() == MVT::i64 &&
8321         isa<ConstantSDNode>(Op.getOperand(1)) &&
8322         (Op.getConstantOperandVal(1) & 0x10) == 0) {
8323       unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
8324       if (Tmp > 32)
8325         return 33;
8326     }
8327     break;
8328   }
8329   case RISCVISD::VMV_X_S:
8330     // The number of sign bits of the scalar result is computed by obtaining the
8331     // element type of the input vector operand, subtracting its width from the
8332     // XLEN, and then adding one (sign bit within the element type). If the
8333     // element type is wider than XLen, the least-significant XLEN bits are
8334     // taken.
8335     if (Op.getOperand(0).getScalarValueSizeInBits() > Subtarget.getXLen())
8336       return 1;
8337     return Subtarget.getXLen() - Op.getOperand(0).getScalarValueSizeInBits() + 1;
8338   }
8339 
8340   return 1;
8341 }
8342 
8343 static MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
8344                                                   MachineBasicBlock *BB) {
8345   assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction");
8346 
8347   // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
8348   // Should the count have wrapped while it was being read, we need to try
8349   // again.
8350   // ...
8351   // read:
8352   // rdcycleh x3 # load high word of cycle
8353   // rdcycle  x2 # load low word of cycle
8354   // rdcycleh x4 # load high word of cycle
8355   // bne x3, x4, read # check if high word reads match, otherwise try again
8356   // ...
8357 
8358   MachineFunction &MF = *BB->getParent();
8359   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8360   MachineFunction::iterator It = ++BB->getIterator();
8361 
8362   MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8363   MF.insert(It, LoopMBB);
8364 
8365   MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8366   MF.insert(It, DoneMBB);
8367 
8368   // Transfer the remainder of BB and its successor edges to DoneMBB.
8369   DoneMBB->splice(DoneMBB->begin(), BB,
8370                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
8371   DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
8372 
8373   BB->addSuccessor(LoopMBB);
8374 
8375   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8376   Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
8377   Register LoReg = MI.getOperand(0).getReg();
8378   Register HiReg = MI.getOperand(1).getReg();
8379   DebugLoc DL = MI.getDebugLoc();
8380 
8381   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8382   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
8383       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8384       .addReg(RISCV::X0);
8385   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
8386       .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
8387       .addReg(RISCV::X0);
8388   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
8389       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8390       .addReg(RISCV::X0);
8391 
8392   BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
8393       .addReg(HiReg)
8394       .addReg(ReadAgainReg)
8395       .addMBB(LoopMBB);
8396 
8397   LoopMBB->addSuccessor(LoopMBB);
8398   LoopMBB->addSuccessor(DoneMBB);
8399 
8400   MI.eraseFromParent();
8401 
8402   return DoneMBB;
8403 }
8404 
8405 static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
8406                                              MachineBasicBlock *BB) {
8407   assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
8408 
8409   MachineFunction &MF = *BB->getParent();
8410   DebugLoc DL = MI.getDebugLoc();
8411   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8412   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8413   Register LoReg = MI.getOperand(0).getReg();
8414   Register HiReg = MI.getOperand(1).getReg();
8415   Register SrcReg = MI.getOperand(2).getReg();
8416   const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
8417   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8418 
8419   TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
8420                           RI);
8421   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8422   MachineMemOperand *MMOLo =
8423       MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 4, Align(8));
8424   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8425       MPI.getWithOffset(4), MachineMemOperand::MOLoad, 4, Align(8));
8426   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
8427       .addFrameIndex(FI)
8428       .addImm(0)
8429       .addMemOperand(MMOLo);
8430   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
8431       .addFrameIndex(FI)
8432       .addImm(4)
8433       .addMemOperand(MMOHi);
8434   MI.eraseFromParent(); // The pseudo instruction is gone now.
8435   return BB;
8436 }
8437 
8438 static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
8439                                                  MachineBasicBlock *BB) {
8440   assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
8441          "Unexpected instruction");
8442 
8443   MachineFunction &MF = *BB->getParent();
8444   DebugLoc DL = MI.getDebugLoc();
8445   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8446   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8447   Register DstReg = MI.getOperand(0).getReg();
8448   Register LoReg = MI.getOperand(1).getReg();
8449   Register HiReg = MI.getOperand(2).getReg();
8450   const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
8451   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8452 
8453   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8454   MachineMemOperand *MMOLo =
8455       MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Align(8));
8456   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8457       MPI.getWithOffset(4), MachineMemOperand::MOStore, 4, Align(8));
8458   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8459       .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
8460       .addFrameIndex(FI)
8461       .addImm(0)
8462       .addMemOperand(MMOLo);
8463   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8464       .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
8465       .addFrameIndex(FI)
8466       .addImm(4)
8467       .addMemOperand(MMOHi);
8468   TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
8469   MI.eraseFromParent(); // The pseudo instruction is gone now.
8470   return BB;
8471 }
8472 
8473 static bool isSelectPseudo(MachineInstr &MI) {
8474   switch (MI.getOpcode()) {
8475   default:
8476     return false;
8477   case RISCV::Select_GPR_Using_CC_GPR:
8478   case RISCV::Select_FPR16_Using_CC_GPR:
8479   case RISCV::Select_FPR32_Using_CC_GPR:
8480   case RISCV::Select_FPR64_Using_CC_GPR:
8481     return true;
8482   }
8483 }
8484 
8485 static MachineBasicBlock *emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB,
8486                                         unsigned RelOpcode, unsigned EqOpcode,
8487                                         const RISCVSubtarget &Subtarget) {
8488   DebugLoc DL = MI.getDebugLoc();
8489   Register DstReg = MI.getOperand(0).getReg();
8490   Register Src1Reg = MI.getOperand(1).getReg();
8491   Register Src2Reg = MI.getOperand(2).getReg();
8492   MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
8493   Register SavedFFlags = MRI.createVirtualRegister(&RISCV::GPRRegClass);
8494   const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
8495 
8496   // Save the current FFLAGS.
8497   BuildMI(*BB, MI, DL, TII.get(RISCV::ReadFFLAGS), SavedFFlags);
8498 
8499   auto MIB = BuildMI(*BB, MI, DL, TII.get(RelOpcode), DstReg)
8500                  .addReg(Src1Reg)
8501                  .addReg(Src2Reg);
8502   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8503     MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
8504 
8505   // Restore the FFLAGS.
8506   BuildMI(*BB, MI, DL, TII.get(RISCV::WriteFFLAGS))
8507       .addReg(SavedFFlags, RegState::Kill);
8508 
8509   // Issue a dummy FEQ opcode to raise exception for signaling NaNs.
8510   auto MIB2 = BuildMI(*BB, MI, DL, TII.get(EqOpcode), RISCV::X0)
8511                   .addReg(Src1Reg, getKillRegState(MI.getOperand(1).isKill()))
8512                   .addReg(Src2Reg, getKillRegState(MI.getOperand(2).isKill()));
8513   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8514     MIB2->setFlag(MachineInstr::MIFlag::NoFPExcept);
8515 
8516   // Erase the pseudoinstruction.
8517   MI.eraseFromParent();
8518   return BB;
8519 }
8520 
8521 static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
8522                                            MachineBasicBlock *BB,
8523                                            const RISCVSubtarget &Subtarget) {
8524   // To "insert" Select_* instructions, we actually have to insert the triangle
8525   // control-flow pattern.  The incoming instructions know the destination vreg
8526   // to set, the condition code register to branch on, the true/false values to
8527   // select between, and the condcode to use to select the appropriate branch.
8528   //
8529   // We produce the following control flow:
8530   //     HeadMBB
8531   //     |  \
8532   //     |  IfFalseMBB
8533   //     | /
8534   //    TailMBB
8535   //
8536   // When we find a sequence of selects we attempt to optimize their emission
8537   // by sharing the control flow. Currently we only handle cases where we have
8538   // multiple selects with the exact same condition (same LHS, RHS and CC).
8539   // The selects may be interleaved with other instructions if the other
8540   // instructions meet some requirements we deem safe:
8541   // - They are debug instructions. Otherwise,
8542   // - They do not have side-effects, do not access memory and their inputs do
8543   //   not depend on the results of the select pseudo-instructions.
8544   // The TrueV/FalseV operands of the selects cannot depend on the result of
8545   // previous selects in the sequence.
8546   // These conditions could be further relaxed. See the X86 target for a
8547   // related approach and more information.
8548   Register LHS = MI.getOperand(1).getReg();
8549   Register RHS = MI.getOperand(2).getReg();
8550   auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
8551 
8552   SmallVector<MachineInstr *, 4> SelectDebugValues;
8553   SmallSet<Register, 4> SelectDests;
8554   SelectDests.insert(MI.getOperand(0).getReg());
8555 
8556   MachineInstr *LastSelectPseudo = &MI;
8557 
8558   for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
8559        SequenceMBBI != E; ++SequenceMBBI) {
8560     if (SequenceMBBI->isDebugInstr())
8561       continue;
8562     else if (isSelectPseudo(*SequenceMBBI)) {
8563       if (SequenceMBBI->getOperand(1).getReg() != LHS ||
8564           SequenceMBBI->getOperand(2).getReg() != RHS ||
8565           SequenceMBBI->getOperand(3).getImm() != CC ||
8566           SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
8567           SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
8568         break;
8569       LastSelectPseudo = &*SequenceMBBI;
8570       SequenceMBBI->collectDebugValues(SelectDebugValues);
8571       SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
8572     } else {
8573       if (SequenceMBBI->hasUnmodeledSideEffects() ||
8574           SequenceMBBI->mayLoadOrStore())
8575         break;
8576       if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
8577             return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
8578           }))
8579         break;
8580     }
8581   }
8582 
8583   const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
8584   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8585   DebugLoc DL = MI.getDebugLoc();
8586   MachineFunction::iterator I = ++BB->getIterator();
8587 
8588   MachineBasicBlock *HeadMBB = BB;
8589   MachineFunction *F = BB->getParent();
8590   MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
8591   MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
8592 
8593   F->insert(I, IfFalseMBB);
8594   F->insert(I, TailMBB);
8595 
8596   // Transfer debug instructions associated with the selects to TailMBB.
8597   for (MachineInstr *DebugInstr : SelectDebugValues) {
8598     TailMBB->push_back(DebugInstr->removeFromParent());
8599   }
8600 
8601   // Move all instructions after the sequence to TailMBB.
8602   TailMBB->splice(TailMBB->end(), HeadMBB,
8603                   std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
8604   // Update machine-CFG edges by transferring all successors of the current
8605   // block to the new block which will contain the Phi nodes for the selects.
8606   TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
8607   // Set the successors for HeadMBB.
8608   HeadMBB->addSuccessor(IfFalseMBB);
8609   HeadMBB->addSuccessor(TailMBB);
8610 
8611   // Insert appropriate branch.
8612   BuildMI(HeadMBB, DL, TII.getBrCond(CC))
8613     .addReg(LHS)
8614     .addReg(RHS)
8615     .addMBB(TailMBB);
8616 
8617   // IfFalseMBB just falls through to TailMBB.
8618   IfFalseMBB->addSuccessor(TailMBB);
8619 
8620   // Create PHIs for all of the select pseudo-instructions.
8621   auto SelectMBBI = MI.getIterator();
8622   auto SelectEnd = std::next(LastSelectPseudo->getIterator());
8623   auto InsertionPoint = TailMBB->begin();
8624   while (SelectMBBI != SelectEnd) {
8625     auto Next = std::next(SelectMBBI);
8626     if (isSelectPseudo(*SelectMBBI)) {
8627       // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
8628       BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
8629               TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
8630           .addReg(SelectMBBI->getOperand(4).getReg())
8631           .addMBB(HeadMBB)
8632           .addReg(SelectMBBI->getOperand(5).getReg())
8633           .addMBB(IfFalseMBB);
8634       SelectMBBI->eraseFromParent();
8635     }
8636     SelectMBBI = Next;
8637   }
8638 
8639   F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
8640   return TailMBB;
8641 }
8642 
8643 MachineBasicBlock *
8644 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
8645                                                  MachineBasicBlock *BB) const {
8646   switch (MI.getOpcode()) {
8647   default:
8648     llvm_unreachable("Unexpected instr type to insert");
8649   case RISCV::ReadCycleWide:
8650     assert(!Subtarget.is64Bit() &&
8651            "ReadCycleWrite is only to be used on riscv32");
8652     return emitReadCycleWidePseudo(MI, BB);
8653   case RISCV::Select_GPR_Using_CC_GPR:
8654   case RISCV::Select_FPR16_Using_CC_GPR:
8655   case RISCV::Select_FPR32_Using_CC_GPR:
8656   case RISCV::Select_FPR64_Using_CC_GPR:
8657     return emitSelectPseudo(MI, BB, Subtarget);
8658   case RISCV::BuildPairF64Pseudo:
8659     return emitBuildPairF64Pseudo(MI, BB);
8660   case RISCV::SplitF64Pseudo:
8661     return emitSplitF64Pseudo(MI, BB);
8662   case RISCV::PseudoQuietFLE_H:
8663     return emitQuietFCMP(MI, BB, RISCV::FLE_H, RISCV::FEQ_H, Subtarget);
8664   case RISCV::PseudoQuietFLT_H:
8665     return emitQuietFCMP(MI, BB, RISCV::FLT_H, RISCV::FEQ_H, Subtarget);
8666   case RISCV::PseudoQuietFLE_S:
8667     return emitQuietFCMP(MI, BB, RISCV::FLE_S, RISCV::FEQ_S, Subtarget);
8668   case RISCV::PseudoQuietFLT_S:
8669     return emitQuietFCMP(MI, BB, RISCV::FLT_S, RISCV::FEQ_S, Subtarget);
8670   case RISCV::PseudoQuietFLE_D:
8671     return emitQuietFCMP(MI, BB, RISCV::FLE_D, RISCV::FEQ_D, Subtarget);
8672   case RISCV::PseudoQuietFLT_D:
8673     return emitQuietFCMP(MI, BB, RISCV::FLT_D, RISCV::FEQ_D, Subtarget);
8674   }
8675 }
8676 
8677 void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
8678                                                         SDNode *Node) const {
8679   // Add FRM dependency to any instructions with dynamic rounding mode.
8680   unsigned Opc = MI.getOpcode();
8681   auto Idx = RISCV::getNamedOperandIdx(Opc, RISCV::OpName::frm);
8682   if (Idx < 0)
8683     return;
8684   if (MI.getOperand(Idx).getImm() != RISCVFPRndMode::DYN)
8685     return;
8686   // If the instruction already reads FRM, don't add another read.
8687   if (MI.readsRegister(RISCV::FRM))
8688     return;
8689   MI.addOperand(
8690       MachineOperand::CreateReg(RISCV::FRM, /*isDef*/ false, /*isImp*/ true));
8691 }
8692 
8693 // Calling Convention Implementation.
8694 // The expectations for frontend ABI lowering vary from target to target.
8695 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
8696 // details, but this is a longer term goal. For now, we simply try to keep the
8697 // role of the frontend as simple and well-defined as possible. The rules can
8698 // be summarised as:
8699 // * Never split up large scalar arguments. We handle them here.
8700 // * If a hardfloat calling convention is being used, and the struct may be
8701 // passed in a pair of registers (fp+fp, int+fp), and both registers are
8702 // available, then pass as two separate arguments. If either the GPRs or FPRs
8703 // are exhausted, then pass according to the rule below.
8704 // * If a struct could never be passed in registers or directly in a stack
8705 // slot (as it is larger than 2*XLEN and the floating point rules don't
8706 // apply), then pass it using a pointer with the byval attribute.
8707 // * If a struct is less than 2*XLEN, then coerce to either a two-element
8708 // word-sized array or a 2*XLEN scalar (depending on alignment).
8709 // * The frontend can determine whether a struct is returned by reference or
8710 // not based on its size and fields. If it will be returned by reference, the
8711 // frontend must modify the prototype so a pointer with the sret annotation is
8712 // passed as the first argument. This is not necessary for large scalar
8713 // returns.
8714 // * Struct return values and varargs should be coerced to structs containing
8715 // register-size fields in the same situations they would be for fixed
8716 // arguments.
8717 
8718 static const MCPhysReg ArgGPRs[] = {
8719   RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
8720   RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
8721 };
8722 static const MCPhysReg ArgFPR16s[] = {
8723   RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
8724   RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
8725 };
8726 static const MCPhysReg ArgFPR32s[] = {
8727   RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
8728   RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
8729 };
8730 static const MCPhysReg ArgFPR64s[] = {
8731   RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
8732   RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
8733 };
8734 // This is an interim calling convention and it may be changed in the future.
8735 static const MCPhysReg ArgVRs[] = {
8736     RISCV::V8,  RISCV::V9,  RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
8737     RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
8738     RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
8739 static const MCPhysReg ArgVRM2s[] = {RISCV::V8M2,  RISCV::V10M2, RISCV::V12M2,
8740                                      RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
8741                                      RISCV::V20M2, RISCV::V22M2};
8742 static const MCPhysReg ArgVRM4s[] = {RISCV::V8M4, RISCV::V12M4, RISCV::V16M4,
8743                                      RISCV::V20M4};
8744 static const MCPhysReg ArgVRM8s[] = {RISCV::V8M8, RISCV::V16M8};
8745 
8746 // Pass a 2*XLEN argument that has been split into two XLEN values through
8747 // registers or the stack as necessary.
8748 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
8749                                 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
8750                                 MVT ValVT2, MVT LocVT2,
8751                                 ISD::ArgFlagsTy ArgFlags2) {
8752   unsigned XLenInBytes = XLen / 8;
8753   if (Register Reg = State.AllocateReg(ArgGPRs)) {
8754     // At least one half can be passed via register.
8755     State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
8756                                      VA1.getLocVT(), CCValAssign::Full));
8757   } else {
8758     // Both halves must be passed on the stack, with proper alignment.
8759     Align StackAlign =
8760         std::max(Align(XLenInBytes), ArgFlags1.getNonZeroOrigAlign());
8761     State.addLoc(
8762         CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
8763                             State.AllocateStack(XLenInBytes, StackAlign),
8764                             VA1.getLocVT(), CCValAssign::Full));
8765     State.addLoc(CCValAssign::getMem(
8766         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
8767         LocVT2, CCValAssign::Full));
8768     return false;
8769   }
8770 
8771   if (Register Reg = State.AllocateReg(ArgGPRs)) {
8772     // The second half can also be passed via register.
8773     State.addLoc(
8774         CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
8775   } else {
8776     // The second half is passed via the stack, without additional alignment.
8777     State.addLoc(CCValAssign::getMem(
8778         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
8779         LocVT2, CCValAssign::Full));
8780   }
8781 
8782   return false;
8783 }
8784 
8785 static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo,
8786                                Optional<unsigned> FirstMaskArgument,
8787                                CCState &State, const RISCVTargetLowering &TLI) {
8788   const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT);
8789   if (RC == &RISCV::VRRegClass) {
8790     // Assign the first mask argument to V0.
8791     // This is an interim calling convention and it may be changed in the
8792     // future.
8793     if (FirstMaskArgument.hasValue() && ValNo == FirstMaskArgument.getValue())
8794       return State.AllocateReg(RISCV::V0);
8795     return State.AllocateReg(ArgVRs);
8796   }
8797   if (RC == &RISCV::VRM2RegClass)
8798     return State.AllocateReg(ArgVRM2s);
8799   if (RC == &RISCV::VRM4RegClass)
8800     return State.AllocateReg(ArgVRM4s);
8801   if (RC == &RISCV::VRM8RegClass)
8802     return State.AllocateReg(ArgVRM8s);
8803   llvm_unreachable("Unhandled register class for ValueType");
8804 }
8805 
8806 // Implements the RISC-V calling convention. Returns true upon failure.
8807 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
8808                      MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
8809                      ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
8810                      bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
8811                      Optional<unsigned> FirstMaskArgument) {
8812   unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
8813   assert(XLen == 32 || XLen == 64);
8814   MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
8815 
8816   // Any return value split in to more than two values can't be returned
8817   // directly. Vectors are returned via the available vector registers.
8818   if (!LocVT.isVector() && IsRet && ValNo > 1)
8819     return true;
8820 
8821   // UseGPRForF16_F32 if targeting one of the soft-float ABIs, if passing a
8822   // variadic argument, or if no F16/F32 argument registers are available.
8823   bool UseGPRForF16_F32 = true;
8824   // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
8825   // variadic argument, or if no F64 argument registers are available.
8826   bool UseGPRForF64 = true;
8827 
8828   switch (ABI) {
8829   default:
8830     llvm_unreachable("Unexpected ABI");
8831   case RISCVABI::ABI_ILP32:
8832   case RISCVABI::ABI_LP64:
8833     break;
8834   case RISCVABI::ABI_ILP32F:
8835   case RISCVABI::ABI_LP64F:
8836     UseGPRForF16_F32 = !IsFixed;
8837     break;
8838   case RISCVABI::ABI_ILP32D:
8839   case RISCVABI::ABI_LP64D:
8840     UseGPRForF16_F32 = !IsFixed;
8841     UseGPRForF64 = !IsFixed;
8842     break;
8843   }
8844 
8845   // FPR16, FPR32, and FPR64 alias each other.
8846   if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s)) {
8847     UseGPRForF16_F32 = true;
8848     UseGPRForF64 = true;
8849   }
8850 
8851   // From this point on, rely on UseGPRForF16_F32, UseGPRForF64 and
8852   // similar local variables rather than directly checking against the target
8853   // ABI.
8854 
8855   if (UseGPRForF16_F32 && (ValVT == MVT::f16 || ValVT == MVT::f32)) {
8856     LocVT = XLenVT;
8857     LocInfo = CCValAssign::BCvt;
8858   } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
8859     LocVT = MVT::i64;
8860     LocInfo = CCValAssign::BCvt;
8861   }
8862 
8863   // If this is a variadic argument, the RISC-V calling convention requires
8864   // that it is assigned an 'even' or 'aligned' register if it has 8-byte
8865   // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
8866   // be used regardless of whether the original argument was split during
8867   // legalisation or not. The argument will not be passed by registers if the
8868   // original type is larger than 2*XLEN, so the register alignment rule does
8869   // not apply.
8870   unsigned TwoXLenInBytes = (2 * XLen) / 8;
8871   if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
8872       DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
8873     unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
8874     // Skip 'odd' register if necessary.
8875     if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
8876       State.AllocateReg(ArgGPRs);
8877   }
8878 
8879   SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
8880   SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
8881       State.getPendingArgFlags();
8882 
8883   assert(PendingLocs.size() == PendingArgFlags.size() &&
8884          "PendingLocs and PendingArgFlags out of sync");
8885 
8886   // Handle passing f64 on RV32D with a soft float ABI or when floating point
8887   // registers are exhausted.
8888   if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
8889     assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
8890            "Can't lower f64 if it is split");
8891     // Depending on available argument GPRS, f64 may be passed in a pair of
8892     // GPRs, split between a GPR and the stack, or passed completely on the
8893     // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
8894     // cases.
8895     Register Reg = State.AllocateReg(ArgGPRs);
8896     LocVT = MVT::i32;
8897     if (!Reg) {
8898       unsigned StackOffset = State.AllocateStack(8, Align(8));
8899       State.addLoc(
8900           CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
8901       return false;
8902     }
8903     if (!State.AllocateReg(ArgGPRs))
8904       State.AllocateStack(4, Align(4));
8905     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8906     return false;
8907   }
8908 
8909   // Fixed-length vectors are located in the corresponding scalable-vector
8910   // container types.
8911   if (ValVT.isFixedLengthVector())
8912     LocVT = TLI.getContainerForFixedLengthVector(LocVT);
8913 
8914   // Split arguments might be passed indirectly, so keep track of the pending
8915   // values. Split vectors are passed via a mix of registers and indirectly, so
8916   // treat them as we would any other argument.
8917   if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
8918     LocVT = XLenVT;
8919     LocInfo = CCValAssign::Indirect;
8920     PendingLocs.push_back(
8921         CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
8922     PendingArgFlags.push_back(ArgFlags);
8923     if (!ArgFlags.isSplitEnd()) {
8924       return false;
8925     }
8926   }
8927 
8928   // If the split argument only had two elements, it should be passed directly
8929   // in registers or on the stack.
8930   if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
8931       PendingLocs.size() <= 2) {
8932     assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
8933     // Apply the normal calling convention rules to the first half of the
8934     // split argument.
8935     CCValAssign VA = PendingLocs[0];
8936     ISD::ArgFlagsTy AF = PendingArgFlags[0];
8937     PendingLocs.clear();
8938     PendingArgFlags.clear();
8939     return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
8940                                ArgFlags);
8941   }
8942 
8943   // Allocate to a register if possible, or else a stack slot.
8944   Register Reg;
8945   unsigned StoreSizeBytes = XLen / 8;
8946   Align StackAlign = Align(XLen / 8);
8947 
8948   if (ValVT == MVT::f16 && !UseGPRForF16_F32)
8949     Reg = State.AllocateReg(ArgFPR16s);
8950   else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
8951     Reg = State.AllocateReg(ArgFPR32s);
8952   else if (ValVT == MVT::f64 && !UseGPRForF64)
8953     Reg = State.AllocateReg(ArgFPR64s);
8954   else if (ValVT.isVector()) {
8955     Reg = allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI);
8956     if (!Reg) {
8957       // For return values, the vector must be passed fully via registers or
8958       // via the stack.
8959       // FIXME: The proposed vector ABI only mandates v8-v15 for return values,
8960       // but we're using all of them.
8961       if (IsRet)
8962         return true;
8963       // Try using a GPR to pass the address
8964       if ((Reg = State.AllocateReg(ArgGPRs))) {
8965         LocVT = XLenVT;
8966         LocInfo = CCValAssign::Indirect;
8967       } else if (ValVT.isScalableVector()) {
8968         LocVT = XLenVT;
8969         LocInfo = CCValAssign::Indirect;
8970       } else {
8971         // Pass fixed-length vectors on the stack.
8972         LocVT = ValVT;
8973         StoreSizeBytes = ValVT.getStoreSize();
8974         // Align vectors to their element sizes, being careful for vXi1
8975         // vectors.
8976         StackAlign = MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
8977       }
8978     }
8979   } else {
8980     Reg = State.AllocateReg(ArgGPRs);
8981   }
8982 
8983   unsigned StackOffset =
8984       Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
8985 
8986   // If we reach this point and PendingLocs is non-empty, we must be at the
8987   // end of a split argument that must be passed indirectly.
8988   if (!PendingLocs.empty()) {
8989     assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
8990     assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
8991 
8992     for (auto &It : PendingLocs) {
8993       if (Reg)
8994         It.convertToReg(Reg);
8995       else
8996         It.convertToMem(StackOffset);
8997       State.addLoc(It);
8998     }
8999     PendingLocs.clear();
9000     PendingArgFlags.clear();
9001     return false;
9002   }
9003 
9004   assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
9005           (TLI.getSubtarget().hasVInstructions() && ValVT.isVector())) &&
9006          "Expected an XLenVT or vector types at this stage");
9007 
9008   if (Reg) {
9009     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9010     return false;
9011   }
9012 
9013   // When a floating-point value is passed on the stack, no bit-conversion is
9014   // needed.
9015   if (ValVT.isFloatingPoint()) {
9016     LocVT = ValVT;
9017     LocInfo = CCValAssign::Full;
9018   }
9019   State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
9020   return false;
9021 }
9022 
9023 template <typename ArgTy>
9024 static Optional<unsigned> preAssignMask(const ArgTy &Args) {
9025   for (const auto &ArgIdx : enumerate(Args)) {
9026     MVT ArgVT = ArgIdx.value().VT;
9027     if (ArgVT.isVector() && ArgVT.getVectorElementType() == MVT::i1)
9028       return ArgIdx.index();
9029   }
9030   return None;
9031 }
9032 
9033 void RISCVTargetLowering::analyzeInputArgs(
9034     MachineFunction &MF, CCState &CCInfo,
9035     const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
9036     RISCVCCAssignFn Fn) const {
9037   unsigned NumArgs = Ins.size();
9038   FunctionType *FType = MF.getFunction().getFunctionType();
9039 
9040   Optional<unsigned> FirstMaskArgument;
9041   if (Subtarget.hasVInstructions())
9042     FirstMaskArgument = preAssignMask(Ins);
9043 
9044   for (unsigned i = 0; i != NumArgs; ++i) {
9045     MVT ArgVT = Ins[i].VT;
9046     ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
9047 
9048     Type *ArgTy = nullptr;
9049     if (IsRet)
9050       ArgTy = FType->getReturnType();
9051     else if (Ins[i].isOrigArg())
9052       ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
9053 
9054     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9055     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
9056            ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this,
9057            FirstMaskArgument)) {
9058       LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
9059                         << EVT(ArgVT).getEVTString() << '\n');
9060       llvm_unreachable(nullptr);
9061     }
9062   }
9063 }
9064 
9065 void RISCVTargetLowering::analyzeOutputArgs(
9066     MachineFunction &MF, CCState &CCInfo,
9067     const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
9068     CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
9069   unsigned NumArgs = Outs.size();
9070 
9071   Optional<unsigned> FirstMaskArgument;
9072   if (Subtarget.hasVInstructions())
9073     FirstMaskArgument = preAssignMask(Outs);
9074 
9075   for (unsigned i = 0; i != NumArgs; i++) {
9076     MVT ArgVT = Outs[i].VT;
9077     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
9078     Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
9079 
9080     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9081     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
9082            ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this,
9083            FirstMaskArgument)) {
9084       LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
9085                         << EVT(ArgVT).getEVTString() << "\n");
9086       llvm_unreachable(nullptr);
9087     }
9088   }
9089 }
9090 
9091 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
9092 // values.
9093 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
9094                                    const CCValAssign &VA, const SDLoc &DL,
9095                                    const RISCVSubtarget &Subtarget) {
9096   switch (VA.getLocInfo()) {
9097   default:
9098     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9099   case CCValAssign::Full:
9100     if (VA.getValVT().isFixedLengthVector() && VA.getLocVT().isScalableVector())
9101       Val = convertFromScalableVector(VA.getValVT(), Val, DAG, Subtarget);
9102     break;
9103   case CCValAssign::BCvt:
9104     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
9105       Val = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, Val);
9106     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9107       Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
9108     else
9109       Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
9110     break;
9111   }
9112   return Val;
9113 }
9114 
9115 // The caller is responsible for loading the full value if the argument is
9116 // passed with CCValAssign::Indirect.
9117 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
9118                                 const CCValAssign &VA, const SDLoc &DL,
9119                                 const RISCVTargetLowering &TLI) {
9120   MachineFunction &MF = DAG.getMachineFunction();
9121   MachineRegisterInfo &RegInfo = MF.getRegInfo();
9122   EVT LocVT = VA.getLocVT();
9123   SDValue Val;
9124   const TargetRegisterClass *RC = TLI.getRegClassFor(LocVT.getSimpleVT());
9125   Register VReg = RegInfo.createVirtualRegister(RC);
9126   RegInfo.addLiveIn(VA.getLocReg(), VReg);
9127   Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
9128 
9129   if (VA.getLocInfo() == CCValAssign::Indirect)
9130     return Val;
9131 
9132   return convertLocVTToValVT(DAG, Val, VA, DL, TLI.getSubtarget());
9133 }
9134 
9135 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
9136                                    const CCValAssign &VA, const SDLoc &DL,
9137                                    const RISCVSubtarget &Subtarget) {
9138   EVT LocVT = VA.getLocVT();
9139 
9140   switch (VA.getLocInfo()) {
9141   default:
9142     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9143   case CCValAssign::Full:
9144     if (VA.getValVT().isFixedLengthVector() && LocVT.isScalableVector())
9145       Val = convertToScalableVector(LocVT, Val, DAG, Subtarget);
9146     break;
9147   case CCValAssign::BCvt:
9148     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
9149       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, VA.getLocVT(), Val);
9150     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9151       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
9152     else
9153       Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
9154     break;
9155   }
9156   return Val;
9157 }
9158 
9159 // The caller is responsible for loading the full value if the argument is
9160 // passed with CCValAssign::Indirect.
9161 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
9162                                 const CCValAssign &VA, const SDLoc &DL) {
9163   MachineFunction &MF = DAG.getMachineFunction();
9164   MachineFrameInfo &MFI = MF.getFrameInfo();
9165   EVT LocVT = VA.getLocVT();
9166   EVT ValVT = VA.getValVT();
9167   EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
9168   if (ValVT.isScalableVector()) {
9169     // When the value is a scalable vector, we save the pointer which points to
9170     // the scalable vector value in the stack. The ValVT will be the pointer
9171     // type, instead of the scalable vector type.
9172     ValVT = LocVT;
9173   }
9174   int FI = MFI.CreateFixedObject(ValVT.getStoreSize(), VA.getLocMemOffset(),
9175                                  /*IsImmutable=*/true);
9176   SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
9177   SDValue Val;
9178 
9179   ISD::LoadExtType ExtType;
9180   switch (VA.getLocInfo()) {
9181   default:
9182     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9183   case CCValAssign::Full:
9184   case CCValAssign::Indirect:
9185   case CCValAssign::BCvt:
9186     ExtType = ISD::NON_EXTLOAD;
9187     break;
9188   }
9189   Val = DAG.getExtLoad(
9190       ExtType, DL, LocVT, Chain, FIN,
9191       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
9192   return Val;
9193 }
9194 
9195 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
9196                                        const CCValAssign &VA, const SDLoc &DL) {
9197   assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
9198          "Unexpected VA");
9199   MachineFunction &MF = DAG.getMachineFunction();
9200   MachineFrameInfo &MFI = MF.getFrameInfo();
9201   MachineRegisterInfo &RegInfo = MF.getRegInfo();
9202 
9203   if (VA.isMemLoc()) {
9204     // f64 is passed on the stack.
9205     int FI =
9206         MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*IsImmutable=*/true);
9207     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
9208     return DAG.getLoad(MVT::f64, DL, Chain, FIN,
9209                        MachinePointerInfo::getFixedStack(MF, FI));
9210   }
9211 
9212   assert(VA.isRegLoc() && "Expected register VA assignment");
9213 
9214   Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
9215   RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
9216   SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
9217   SDValue Hi;
9218   if (VA.getLocReg() == RISCV::X17) {
9219     // Second half of f64 is passed on the stack.
9220     int FI = MFI.CreateFixedObject(4, 0, /*IsImmutable=*/true);
9221     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
9222     Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
9223                      MachinePointerInfo::getFixedStack(MF, FI));
9224   } else {
9225     // Second half of f64 is passed in another GPR.
9226     Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
9227     RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
9228     Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
9229   }
9230   return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
9231 }
9232 
9233 // FastCC has less than 1% performance improvement for some particular
9234 // benchmark. But theoretically, it may has benenfit for some cases.
9235 static bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
9236                             unsigned ValNo, MVT ValVT, MVT LocVT,
9237                             CCValAssign::LocInfo LocInfo,
9238                             ISD::ArgFlagsTy ArgFlags, CCState &State,
9239                             bool IsFixed, bool IsRet, Type *OrigTy,
9240                             const RISCVTargetLowering &TLI,
9241                             Optional<unsigned> FirstMaskArgument) {
9242 
9243   // X5 and X6 might be used for save-restore libcall.
9244   static const MCPhysReg GPRList[] = {
9245       RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
9246       RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7,  RISCV::X28,
9247       RISCV::X29, RISCV::X30, RISCV::X31};
9248 
9249   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9250     if (unsigned Reg = State.AllocateReg(GPRList)) {
9251       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9252       return false;
9253     }
9254   }
9255 
9256   if (LocVT == MVT::f16) {
9257     static const MCPhysReg FPR16List[] = {
9258         RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
9259         RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H,  RISCV::F1_H,
9260         RISCV::F2_H,  RISCV::F3_H,  RISCV::F4_H,  RISCV::F5_H,  RISCV::F6_H,
9261         RISCV::F7_H,  RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
9262     if (unsigned Reg = State.AllocateReg(FPR16List)) {
9263       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9264       return false;
9265     }
9266   }
9267 
9268   if (LocVT == MVT::f32) {
9269     static const MCPhysReg FPR32List[] = {
9270         RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
9271         RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F,  RISCV::F1_F,
9272         RISCV::F2_F,  RISCV::F3_F,  RISCV::F4_F,  RISCV::F5_F,  RISCV::F6_F,
9273         RISCV::F7_F,  RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
9274     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9275       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9276       return false;
9277     }
9278   }
9279 
9280   if (LocVT == MVT::f64) {
9281     static const MCPhysReg FPR64List[] = {
9282         RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
9283         RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D,  RISCV::F1_D,
9284         RISCV::F2_D,  RISCV::F3_D,  RISCV::F4_D,  RISCV::F5_D,  RISCV::F6_D,
9285         RISCV::F7_D,  RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
9286     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9287       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9288       return false;
9289     }
9290   }
9291 
9292   if (LocVT == MVT::i32 || LocVT == MVT::f32) {
9293     unsigned Offset4 = State.AllocateStack(4, Align(4));
9294     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset4, LocVT, LocInfo));
9295     return false;
9296   }
9297 
9298   if (LocVT == MVT::i64 || LocVT == MVT::f64) {
9299     unsigned Offset5 = State.AllocateStack(8, Align(8));
9300     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset5, LocVT, LocInfo));
9301     return false;
9302   }
9303 
9304   if (LocVT.isVector()) {
9305     if (unsigned Reg =
9306             allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI)) {
9307       // Fixed-length vectors are located in the corresponding scalable-vector
9308       // container types.
9309       if (ValVT.isFixedLengthVector())
9310         LocVT = TLI.getContainerForFixedLengthVector(LocVT);
9311       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9312     } else {
9313       // Try and pass the address via a "fast" GPR.
9314       if (unsigned GPRReg = State.AllocateReg(GPRList)) {
9315         LocInfo = CCValAssign::Indirect;
9316         LocVT = TLI.getSubtarget().getXLenVT();
9317         State.addLoc(CCValAssign::getReg(ValNo, ValVT, GPRReg, LocVT, LocInfo));
9318       } else if (ValVT.isFixedLengthVector()) {
9319         auto StackAlign =
9320             MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
9321         unsigned StackOffset =
9322             State.AllocateStack(ValVT.getStoreSize(), StackAlign);
9323         State.addLoc(
9324             CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
9325       } else {
9326         // Can't pass scalable vectors on the stack.
9327         return true;
9328       }
9329     }
9330 
9331     return false;
9332   }
9333 
9334   return true; // CC didn't match.
9335 }
9336 
9337 static bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
9338                          CCValAssign::LocInfo LocInfo,
9339                          ISD::ArgFlagsTy ArgFlags, CCState &State) {
9340 
9341   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9342     // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, R6, R7, SpLim
9343     //                        s1    s2  s3  s4  s5  s6  s7  s8  s9  s10 s11
9344     static const MCPhysReg GPRList[] = {
9345         RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
9346         RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
9347     if (unsigned Reg = State.AllocateReg(GPRList)) {
9348       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9349       return false;
9350     }
9351   }
9352 
9353   if (LocVT == MVT::f32) {
9354     // Pass in STG registers: F1, ..., F6
9355     //                        fs0 ... fs5
9356     static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
9357                                           RISCV::F18_F, RISCV::F19_F,
9358                                           RISCV::F20_F, RISCV::F21_F};
9359     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9360       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9361       return false;
9362     }
9363   }
9364 
9365   if (LocVT == MVT::f64) {
9366     // Pass in STG registers: D1, ..., D6
9367     //                        fs6 ... fs11
9368     static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
9369                                           RISCV::F24_D, RISCV::F25_D,
9370                                           RISCV::F26_D, RISCV::F27_D};
9371     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9372       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9373       return false;
9374     }
9375   }
9376 
9377   report_fatal_error("No registers left in GHC calling convention");
9378   return true;
9379 }
9380 
9381 // Transform physical registers into virtual registers.
9382 SDValue RISCVTargetLowering::LowerFormalArguments(
9383     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
9384     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
9385     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
9386 
9387   MachineFunction &MF = DAG.getMachineFunction();
9388 
9389   switch (CallConv) {
9390   default:
9391     report_fatal_error("Unsupported calling convention");
9392   case CallingConv::C:
9393   case CallingConv::Fast:
9394     break;
9395   case CallingConv::GHC:
9396     if (!MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtF] ||
9397         !MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtD])
9398       report_fatal_error(
9399         "GHC calling convention requires the F and D instruction set extensions");
9400   }
9401 
9402   const Function &Func = MF.getFunction();
9403   if (Func.hasFnAttribute("interrupt")) {
9404     if (!Func.arg_empty())
9405       report_fatal_error(
9406         "Functions with the interrupt attribute cannot have arguments!");
9407 
9408     StringRef Kind =
9409       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
9410 
9411     if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
9412       report_fatal_error(
9413         "Function interrupt attribute argument not supported!");
9414   }
9415 
9416   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9417   MVT XLenVT = Subtarget.getXLenVT();
9418   unsigned XLenInBytes = Subtarget.getXLen() / 8;
9419   // Used with vargs to acumulate store chains.
9420   std::vector<SDValue> OutChains;
9421 
9422   // Assign locations to all of the incoming arguments.
9423   SmallVector<CCValAssign, 16> ArgLocs;
9424   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9425 
9426   if (CallConv == CallingConv::GHC)
9427     CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_GHC);
9428   else
9429     analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false,
9430                      CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9431                                                    : CC_RISCV);
9432 
9433   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
9434     CCValAssign &VA = ArgLocs[i];
9435     SDValue ArgValue;
9436     // Passing f64 on RV32D with a soft float ABI must be handled as a special
9437     // case.
9438     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
9439       ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
9440     else if (VA.isRegLoc())
9441       ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, *this);
9442     else
9443       ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
9444 
9445     if (VA.getLocInfo() == CCValAssign::Indirect) {
9446       // If the original argument was split and passed by reference (e.g. i128
9447       // on RV32), we need to load all parts of it here (using the same
9448       // address). Vectors may be partly split to registers and partly to the
9449       // stack, in which case the base address is partly offset and subsequent
9450       // stores are relative to that.
9451       InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
9452                                    MachinePointerInfo()));
9453       unsigned ArgIndex = Ins[i].OrigArgIndex;
9454       unsigned ArgPartOffset = Ins[i].PartOffset;
9455       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9456       while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
9457         CCValAssign &PartVA = ArgLocs[i + 1];
9458         unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset;
9459         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9460         if (PartVA.getValVT().isScalableVector())
9461           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9462         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
9463         InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
9464                                      MachinePointerInfo()));
9465         ++i;
9466       }
9467       continue;
9468     }
9469     InVals.push_back(ArgValue);
9470   }
9471 
9472   if (IsVarArg) {
9473     ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
9474     unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
9475     const TargetRegisterClass *RC = &RISCV::GPRRegClass;
9476     MachineFrameInfo &MFI = MF.getFrameInfo();
9477     MachineRegisterInfo &RegInfo = MF.getRegInfo();
9478     RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
9479 
9480     // Offset of the first variable argument from stack pointer, and size of
9481     // the vararg save area. For now, the varargs save area is either zero or
9482     // large enough to hold a0-a7.
9483     int VaArgOffset, VarArgsSaveSize;
9484 
9485     // If all registers are allocated, then all varargs must be passed on the
9486     // stack and we don't need to save any argregs.
9487     if (ArgRegs.size() == Idx) {
9488       VaArgOffset = CCInfo.getNextStackOffset();
9489       VarArgsSaveSize = 0;
9490     } else {
9491       VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
9492       VaArgOffset = -VarArgsSaveSize;
9493     }
9494 
9495     // Record the frame index of the first variable argument
9496     // which is a value necessary to VASTART.
9497     int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9498     RVFI->setVarArgsFrameIndex(FI);
9499 
9500     // If saving an odd number of registers then create an extra stack slot to
9501     // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
9502     // offsets to even-numbered registered remain 2*XLEN-aligned.
9503     if (Idx % 2) {
9504       MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true);
9505       VarArgsSaveSize += XLenInBytes;
9506     }
9507 
9508     // Copy the integer registers that may have been used for passing varargs
9509     // to the vararg save area.
9510     for (unsigned I = Idx; I < ArgRegs.size();
9511          ++I, VaArgOffset += XLenInBytes) {
9512       const Register Reg = RegInfo.createVirtualRegister(RC);
9513       RegInfo.addLiveIn(ArgRegs[I], Reg);
9514       SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
9515       FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9516       SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9517       SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
9518                                    MachinePointerInfo::getFixedStack(MF, FI));
9519       cast<StoreSDNode>(Store.getNode())
9520           ->getMemOperand()
9521           ->setValue((Value *)nullptr);
9522       OutChains.push_back(Store);
9523     }
9524     RVFI->setVarArgsSaveSize(VarArgsSaveSize);
9525   }
9526 
9527   // All stores are grouped in one node to allow the matching between
9528   // the size of Ins and InVals. This only happens for vararg functions.
9529   if (!OutChains.empty()) {
9530     OutChains.push_back(Chain);
9531     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
9532   }
9533 
9534   return Chain;
9535 }
9536 
9537 /// isEligibleForTailCallOptimization - Check whether the call is eligible
9538 /// for tail call optimization.
9539 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
9540 bool RISCVTargetLowering::isEligibleForTailCallOptimization(
9541     CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
9542     const SmallVector<CCValAssign, 16> &ArgLocs) const {
9543 
9544   auto &Callee = CLI.Callee;
9545   auto CalleeCC = CLI.CallConv;
9546   auto &Outs = CLI.Outs;
9547   auto &Caller = MF.getFunction();
9548   auto CallerCC = Caller.getCallingConv();
9549 
9550   // Exception-handling functions need a special set of instructions to
9551   // indicate a return to the hardware. Tail-calling another function would
9552   // probably break this.
9553   // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
9554   // should be expanded as new function attributes are introduced.
9555   if (Caller.hasFnAttribute("interrupt"))
9556     return false;
9557 
9558   // Do not tail call opt if the stack is used to pass parameters.
9559   if (CCInfo.getNextStackOffset() != 0)
9560     return false;
9561 
9562   // Do not tail call opt if any parameters need to be passed indirectly.
9563   // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
9564   // passed indirectly. So the address of the value will be passed in a
9565   // register, or if not available, then the address is put on the stack. In
9566   // order to pass indirectly, space on the stack often needs to be allocated
9567   // in order to store the value. In this case the CCInfo.getNextStackOffset()
9568   // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
9569   // are passed CCValAssign::Indirect.
9570   for (auto &VA : ArgLocs)
9571     if (VA.getLocInfo() == CCValAssign::Indirect)
9572       return false;
9573 
9574   // Do not tail call opt if either caller or callee uses struct return
9575   // semantics.
9576   auto IsCallerStructRet = Caller.hasStructRetAttr();
9577   auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
9578   if (IsCallerStructRet || IsCalleeStructRet)
9579     return false;
9580 
9581   // Externally-defined functions with weak linkage should not be
9582   // tail-called. The behaviour of branch instructions in this situation (as
9583   // used for tail calls) is implementation-defined, so we cannot rely on the
9584   // linker replacing the tail call with a return.
9585   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
9586     const GlobalValue *GV = G->getGlobal();
9587     if (GV->hasExternalWeakLinkage())
9588       return false;
9589   }
9590 
9591   // The callee has to preserve all registers the caller needs to preserve.
9592   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
9593   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
9594   if (CalleeCC != CallerCC) {
9595     const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
9596     if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
9597       return false;
9598   }
9599 
9600   // Byval parameters hand the function a pointer directly into the stack area
9601   // we want to reuse during a tail call. Working around this *is* possible
9602   // but less efficient and uglier in LowerCall.
9603   for (auto &Arg : Outs)
9604     if (Arg.Flags.isByVal())
9605       return false;
9606 
9607   return true;
9608 }
9609 
9610 static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
9611   return DAG.getDataLayout().getPrefTypeAlign(
9612       VT.getTypeForEVT(*DAG.getContext()));
9613 }
9614 
9615 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
9616 // and output parameter nodes.
9617 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
9618                                        SmallVectorImpl<SDValue> &InVals) const {
9619   SelectionDAG &DAG = CLI.DAG;
9620   SDLoc &DL = CLI.DL;
9621   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
9622   SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
9623   SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
9624   SDValue Chain = CLI.Chain;
9625   SDValue Callee = CLI.Callee;
9626   bool &IsTailCall = CLI.IsTailCall;
9627   CallingConv::ID CallConv = CLI.CallConv;
9628   bool IsVarArg = CLI.IsVarArg;
9629   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9630   MVT XLenVT = Subtarget.getXLenVT();
9631 
9632   MachineFunction &MF = DAG.getMachineFunction();
9633 
9634   // Analyze the operands of the call, assigning locations to each operand.
9635   SmallVector<CCValAssign, 16> ArgLocs;
9636   CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9637 
9638   if (CallConv == CallingConv::GHC)
9639     ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_GHC);
9640   else
9641     analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI,
9642                       CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9643                                                     : CC_RISCV);
9644 
9645   // Check if it's really possible to do a tail call.
9646   if (IsTailCall)
9647     IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
9648 
9649   if (IsTailCall)
9650     ++NumTailCalls;
9651   else if (CLI.CB && CLI.CB->isMustTailCall())
9652     report_fatal_error("failed to perform tail call elimination on a call "
9653                        "site marked musttail");
9654 
9655   // Get a count of how many bytes are to be pushed on the stack.
9656   unsigned NumBytes = ArgCCInfo.getNextStackOffset();
9657 
9658   // Create local copies for byval args
9659   SmallVector<SDValue, 8> ByValArgs;
9660   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9661     ISD::ArgFlagsTy Flags = Outs[i].Flags;
9662     if (!Flags.isByVal())
9663       continue;
9664 
9665     SDValue Arg = OutVals[i];
9666     unsigned Size = Flags.getByValSize();
9667     Align Alignment = Flags.getNonZeroByValAlign();
9668 
9669     int FI =
9670         MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false);
9671     SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9672     SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
9673 
9674     Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
9675                           /*IsVolatile=*/false,
9676                           /*AlwaysInline=*/false, IsTailCall,
9677                           MachinePointerInfo(), MachinePointerInfo());
9678     ByValArgs.push_back(FIPtr);
9679   }
9680 
9681   if (!IsTailCall)
9682     Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
9683 
9684   // Copy argument values to their designated locations.
9685   SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
9686   SmallVector<SDValue, 8> MemOpChains;
9687   SDValue StackPtr;
9688   for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
9689     CCValAssign &VA = ArgLocs[i];
9690     SDValue ArgValue = OutVals[i];
9691     ISD::ArgFlagsTy Flags = Outs[i].Flags;
9692 
9693     // Handle passing f64 on RV32D with a soft float ABI as a special case.
9694     bool IsF64OnRV32DSoftABI =
9695         VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
9696     if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
9697       SDValue SplitF64 = DAG.getNode(
9698           RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
9699       SDValue Lo = SplitF64.getValue(0);
9700       SDValue Hi = SplitF64.getValue(1);
9701 
9702       Register RegLo = VA.getLocReg();
9703       RegsToPass.push_back(std::make_pair(RegLo, Lo));
9704 
9705       if (RegLo == RISCV::X17) {
9706         // Second half of f64 is passed on the stack.
9707         // Work out the address of the stack slot.
9708         if (!StackPtr.getNode())
9709           StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
9710         // Emit the store.
9711         MemOpChains.push_back(
9712             DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
9713       } else {
9714         // Second half of f64 is passed in another GPR.
9715         assert(RegLo < RISCV::X31 && "Invalid register pair");
9716         Register RegHigh = RegLo + 1;
9717         RegsToPass.push_back(std::make_pair(RegHigh, Hi));
9718       }
9719       continue;
9720     }
9721 
9722     // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
9723     // as any other MemLoc.
9724 
9725     // Promote the value if needed.
9726     // For now, only handle fully promoted and indirect arguments.
9727     if (VA.getLocInfo() == CCValAssign::Indirect) {
9728       // Store the argument in a stack slot and pass its address.
9729       Align StackAlign =
9730           std::max(getPrefTypeAlign(Outs[i].ArgVT, DAG),
9731                    getPrefTypeAlign(ArgValue.getValueType(), DAG));
9732       TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
9733       // If the original argument was split (e.g. i128), we need
9734       // to store the required parts of it here (and pass just one address).
9735       // Vectors may be partly split to registers and partly to the stack, in
9736       // which case the base address is partly offset and subsequent stores are
9737       // relative to that.
9738       unsigned ArgIndex = Outs[i].OrigArgIndex;
9739       unsigned ArgPartOffset = Outs[i].PartOffset;
9740       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9741       // Calculate the total size to store. We don't have access to what we're
9742       // actually storing other than performing the loop and collecting the
9743       // info.
9744       SmallVector<std::pair<SDValue, SDValue>> Parts;
9745       while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
9746         SDValue PartValue = OutVals[i + 1];
9747         unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset;
9748         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9749         EVT PartVT = PartValue.getValueType();
9750         if (PartVT.isScalableVector())
9751           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9752         StoredSize += PartVT.getStoreSize();
9753         StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
9754         Parts.push_back(std::make_pair(PartValue, Offset));
9755         ++i;
9756       }
9757       SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
9758       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
9759       MemOpChains.push_back(
9760           DAG.getStore(Chain, DL, ArgValue, SpillSlot,
9761                        MachinePointerInfo::getFixedStack(MF, FI)));
9762       for (const auto &Part : Parts) {
9763         SDValue PartValue = Part.first;
9764         SDValue PartOffset = Part.second;
9765         SDValue Address =
9766             DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
9767         MemOpChains.push_back(
9768             DAG.getStore(Chain, DL, PartValue, Address,
9769                          MachinePointerInfo::getFixedStack(MF, FI)));
9770       }
9771       ArgValue = SpillSlot;
9772     } else {
9773       ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget);
9774     }
9775 
9776     // Use local copy if it is a byval arg.
9777     if (Flags.isByVal())
9778       ArgValue = ByValArgs[j++];
9779 
9780     if (VA.isRegLoc()) {
9781       // Queue up the argument copies and emit them at the end.
9782       RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
9783     } else {
9784       assert(VA.isMemLoc() && "Argument not register or memory");
9785       assert(!IsTailCall && "Tail call not allowed if stack is used "
9786                             "for passing parameters");
9787 
9788       // Work out the address of the stack slot.
9789       if (!StackPtr.getNode())
9790         StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
9791       SDValue Address =
9792           DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
9793                       DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
9794 
9795       // Emit the store.
9796       MemOpChains.push_back(
9797           DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
9798     }
9799   }
9800 
9801   // Join the stores, which are independent of one another.
9802   if (!MemOpChains.empty())
9803     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
9804 
9805   SDValue Glue;
9806 
9807   // Build a sequence of copy-to-reg nodes, chained and glued together.
9808   for (auto &Reg : RegsToPass) {
9809     Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
9810     Glue = Chain.getValue(1);
9811   }
9812 
9813   // Validate that none of the argument registers have been marked as
9814   // reserved, if so report an error. Do the same for the return address if this
9815   // is not a tailcall.
9816   validateCCReservedRegs(RegsToPass, MF);
9817   if (!IsTailCall &&
9818       MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1))
9819     MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9820         MF.getFunction(),
9821         "Return address register required, but has been reserved."});
9822 
9823   // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
9824   // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
9825   // split it and then direct call can be matched by PseudoCALL.
9826   if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
9827     const GlobalValue *GV = S->getGlobal();
9828 
9829     unsigned OpFlags = RISCVII::MO_CALL;
9830     if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
9831       OpFlags = RISCVII::MO_PLT;
9832 
9833     Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
9834   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
9835     unsigned OpFlags = RISCVII::MO_CALL;
9836 
9837     if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
9838                                                  nullptr))
9839       OpFlags = RISCVII::MO_PLT;
9840 
9841     Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
9842   }
9843 
9844   // The first call operand is the chain and the second is the target address.
9845   SmallVector<SDValue, 8> Ops;
9846   Ops.push_back(Chain);
9847   Ops.push_back(Callee);
9848 
9849   // Add argument registers to the end of the list so that they are
9850   // known live into the call.
9851   for (auto &Reg : RegsToPass)
9852     Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
9853 
9854   if (!IsTailCall) {
9855     // Add a register mask operand representing the call-preserved registers.
9856     const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
9857     const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
9858     assert(Mask && "Missing call preserved mask for calling convention");
9859     Ops.push_back(DAG.getRegisterMask(Mask));
9860   }
9861 
9862   // Glue the call to the argument copies, if any.
9863   if (Glue.getNode())
9864     Ops.push_back(Glue);
9865 
9866   // Emit the call.
9867   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9868 
9869   if (IsTailCall) {
9870     MF.getFrameInfo().setHasTailCall();
9871     return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
9872   }
9873 
9874   Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
9875   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
9876   Glue = Chain.getValue(1);
9877 
9878   // Mark the end of the call, which is glued to the call itself.
9879   Chain = DAG.getCALLSEQ_END(Chain,
9880                              DAG.getConstant(NumBytes, DL, PtrVT, true),
9881                              DAG.getConstant(0, DL, PtrVT, true),
9882                              Glue, DL);
9883   Glue = Chain.getValue(1);
9884 
9885   // Assign locations to each value returned by this call.
9886   SmallVector<CCValAssign, 16> RVLocs;
9887   CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
9888   analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, CC_RISCV);
9889 
9890   // Copy all of the result registers out of their specified physreg.
9891   for (auto &VA : RVLocs) {
9892     // Copy the value out
9893     SDValue RetValue =
9894         DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
9895     // Glue the RetValue to the end of the call sequence
9896     Chain = RetValue.getValue(1);
9897     Glue = RetValue.getValue(2);
9898 
9899     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
9900       assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
9901       SDValue RetValue2 =
9902           DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
9903       Chain = RetValue2.getValue(1);
9904       Glue = RetValue2.getValue(2);
9905       RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
9906                              RetValue2);
9907     }
9908 
9909     RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget);
9910 
9911     InVals.push_back(RetValue);
9912   }
9913 
9914   return Chain;
9915 }
9916 
9917 bool RISCVTargetLowering::CanLowerReturn(
9918     CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
9919     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
9920   SmallVector<CCValAssign, 16> RVLocs;
9921   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
9922 
9923   Optional<unsigned> FirstMaskArgument;
9924   if (Subtarget.hasVInstructions())
9925     FirstMaskArgument = preAssignMask(Outs);
9926 
9927   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9928     MVT VT = Outs[i].VT;
9929     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
9930     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9931     if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
9932                  ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr,
9933                  *this, FirstMaskArgument))
9934       return false;
9935   }
9936   return true;
9937 }
9938 
9939 SDValue
9940 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
9941                                  bool IsVarArg,
9942                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
9943                                  const SmallVectorImpl<SDValue> &OutVals,
9944                                  const SDLoc &DL, SelectionDAG &DAG) const {
9945   const MachineFunction &MF = DAG.getMachineFunction();
9946   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
9947 
9948   // Stores the assignment of the return value to a location.
9949   SmallVector<CCValAssign, 16> RVLocs;
9950 
9951   // Info about the registers and stack slot.
9952   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
9953                  *DAG.getContext());
9954 
9955   analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
9956                     nullptr, CC_RISCV);
9957 
9958   if (CallConv == CallingConv::GHC && !RVLocs.empty())
9959     report_fatal_error("GHC functions return void only");
9960 
9961   SDValue Glue;
9962   SmallVector<SDValue, 4> RetOps(1, Chain);
9963 
9964   // Copy the result values into the output registers.
9965   for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
9966     SDValue Val = OutVals[i];
9967     CCValAssign &VA = RVLocs[i];
9968     assert(VA.isRegLoc() && "Can only return in registers!");
9969 
9970     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
9971       // Handle returning f64 on RV32D with a soft float ABI.
9972       assert(VA.isRegLoc() && "Expected return via registers");
9973       SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
9974                                      DAG.getVTList(MVT::i32, MVT::i32), Val);
9975       SDValue Lo = SplitF64.getValue(0);
9976       SDValue Hi = SplitF64.getValue(1);
9977       Register RegLo = VA.getLocReg();
9978       assert(RegLo < RISCV::X31 && "Invalid register pair");
9979       Register RegHi = RegLo + 1;
9980 
9981       if (STI.isRegisterReservedByUser(RegLo) ||
9982           STI.isRegisterReservedByUser(RegHi))
9983         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9984             MF.getFunction(),
9985             "Return value register required, but has been reserved."});
9986 
9987       Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
9988       Glue = Chain.getValue(1);
9989       RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
9990       Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
9991       Glue = Chain.getValue(1);
9992       RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
9993     } else {
9994       // Handle a 'normal' return.
9995       Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
9996       Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
9997 
9998       if (STI.isRegisterReservedByUser(VA.getLocReg()))
9999         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
10000             MF.getFunction(),
10001             "Return value register required, but has been reserved."});
10002 
10003       // Guarantee that all emitted copies are stuck together.
10004       Glue = Chain.getValue(1);
10005       RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
10006     }
10007   }
10008 
10009   RetOps[0] = Chain; // Update chain.
10010 
10011   // Add the glue node if we have it.
10012   if (Glue.getNode()) {
10013     RetOps.push_back(Glue);
10014   }
10015 
10016   unsigned RetOpc = RISCVISD::RET_FLAG;
10017   // Interrupt service routines use different return instructions.
10018   const Function &Func = DAG.getMachineFunction().getFunction();
10019   if (Func.hasFnAttribute("interrupt")) {
10020     if (!Func.getReturnType()->isVoidTy())
10021       report_fatal_error(
10022           "Functions with the interrupt attribute must have void return type!");
10023 
10024     MachineFunction &MF = DAG.getMachineFunction();
10025     StringRef Kind =
10026       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
10027 
10028     if (Kind == "user")
10029       RetOpc = RISCVISD::URET_FLAG;
10030     else if (Kind == "supervisor")
10031       RetOpc = RISCVISD::SRET_FLAG;
10032     else
10033       RetOpc = RISCVISD::MRET_FLAG;
10034   }
10035 
10036   return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
10037 }
10038 
10039 void RISCVTargetLowering::validateCCReservedRegs(
10040     const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
10041     MachineFunction &MF) const {
10042   const Function &F = MF.getFunction();
10043   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
10044 
10045   if (llvm::any_of(Regs, [&STI](auto Reg) {
10046         return STI.isRegisterReservedByUser(Reg.first);
10047       }))
10048     F.getContext().diagnose(DiagnosticInfoUnsupported{
10049         F, "Argument register required, but has been reserved."});
10050 }
10051 
10052 bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
10053   return CI->isTailCall();
10054 }
10055 
10056 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
10057 #define NODE_NAME_CASE(NODE)                                                   \
10058   case RISCVISD::NODE:                                                         \
10059     return "RISCVISD::" #NODE;
10060   // clang-format off
10061   switch ((RISCVISD::NodeType)Opcode) {
10062   case RISCVISD::FIRST_NUMBER:
10063     break;
10064   NODE_NAME_CASE(RET_FLAG)
10065   NODE_NAME_CASE(URET_FLAG)
10066   NODE_NAME_CASE(SRET_FLAG)
10067   NODE_NAME_CASE(MRET_FLAG)
10068   NODE_NAME_CASE(CALL)
10069   NODE_NAME_CASE(SELECT_CC)
10070   NODE_NAME_CASE(BR_CC)
10071   NODE_NAME_CASE(BuildPairF64)
10072   NODE_NAME_CASE(SplitF64)
10073   NODE_NAME_CASE(TAIL)
10074   NODE_NAME_CASE(MULHSU)
10075   NODE_NAME_CASE(SLLW)
10076   NODE_NAME_CASE(SRAW)
10077   NODE_NAME_CASE(SRLW)
10078   NODE_NAME_CASE(DIVW)
10079   NODE_NAME_CASE(DIVUW)
10080   NODE_NAME_CASE(REMUW)
10081   NODE_NAME_CASE(ROLW)
10082   NODE_NAME_CASE(RORW)
10083   NODE_NAME_CASE(CLZW)
10084   NODE_NAME_CASE(CTZW)
10085   NODE_NAME_CASE(FSLW)
10086   NODE_NAME_CASE(FSRW)
10087   NODE_NAME_CASE(FSL)
10088   NODE_NAME_CASE(FSR)
10089   NODE_NAME_CASE(FMV_H_X)
10090   NODE_NAME_CASE(FMV_X_ANYEXTH)
10091   NODE_NAME_CASE(FMV_W_X_RV64)
10092   NODE_NAME_CASE(FMV_X_ANYEXTW_RV64)
10093   NODE_NAME_CASE(FCVT_X)
10094   NODE_NAME_CASE(FCVT_XU)
10095   NODE_NAME_CASE(FCVT_W_RV64)
10096   NODE_NAME_CASE(FCVT_WU_RV64)
10097   NODE_NAME_CASE(STRICT_FCVT_W_RV64)
10098   NODE_NAME_CASE(STRICT_FCVT_WU_RV64)
10099   NODE_NAME_CASE(READ_CYCLE_WIDE)
10100   NODE_NAME_CASE(GREV)
10101   NODE_NAME_CASE(GREVW)
10102   NODE_NAME_CASE(GORC)
10103   NODE_NAME_CASE(GORCW)
10104   NODE_NAME_CASE(SHFL)
10105   NODE_NAME_CASE(SHFLW)
10106   NODE_NAME_CASE(UNSHFL)
10107   NODE_NAME_CASE(UNSHFLW)
10108   NODE_NAME_CASE(BFP)
10109   NODE_NAME_CASE(BFPW)
10110   NODE_NAME_CASE(BCOMPRESS)
10111   NODE_NAME_CASE(BCOMPRESSW)
10112   NODE_NAME_CASE(BDECOMPRESS)
10113   NODE_NAME_CASE(BDECOMPRESSW)
10114   NODE_NAME_CASE(VMV_V_X_VL)
10115   NODE_NAME_CASE(VFMV_V_F_VL)
10116   NODE_NAME_CASE(VMV_X_S)
10117   NODE_NAME_CASE(VMV_S_X_VL)
10118   NODE_NAME_CASE(VFMV_S_F_VL)
10119   NODE_NAME_CASE(SPLAT_VECTOR_I64)
10120   NODE_NAME_CASE(SPLAT_VECTOR_SPLIT_I64_VL)
10121   NODE_NAME_CASE(READ_VLENB)
10122   NODE_NAME_CASE(TRUNCATE_VECTOR_VL)
10123   NODE_NAME_CASE(VSLIDEUP_VL)
10124   NODE_NAME_CASE(VSLIDE1UP_VL)
10125   NODE_NAME_CASE(VSLIDEDOWN_VL)
10126   NODE_NAME_CASE(VSLIDE1DOWN_VL)
10127   NODE_NAME_CASE(VID_VL)
10128   NODE_NAME_CASE(VFNCVT_ROD_VL)
10129   NODE_NAME_CASE(VECREDUCE_ADD_VL)
10130   NODE_NAME_CASE(VECREDUCE_UMAX_VL)
10131   NODE_NAME_CASE(VECREDUCE_SMAX_VL)
10132   NODE_NAME_CASE(VECREDUCE_UMIN_VL)
10133   NODE_NAME_CASE(VECREDUCE_SMIN_VL)
10134   NODE_NAME_CASE(VECREDUCE_AND_VL)
10135   NODE_NAME_CASE(VECREDUCE_OR_VL)
10136   NODE_NAME_CASE(VECREDUCE_XOR_VL)
10137   NODE_NAME_CASE(VECREDUCE_FADD_VL)
10138   NODE_NAME_CASE(VECREDUCE_SEQ_FADD_VL)
10139   NODE_NAME_CASE(VECREDUCE_FMIN_VL)
10140   NODE_NAME_CASE(VECREDUCE_FMAX_VL)
10141   NODE_NAME_CASE(ADD_VL)
10142   NODE_NAME_CASE(AND_VL)
10143   NODE_NAME_CASE(MUL_VL)
10144   NODE_NAME_CASE(OR_VL)
10145   NODE_NAME_CASE(SDIV_VL)
10146   NODE_NAME_CASE(SHL_VL)
10147   NODE_NAME_CASE(SREM_VL)
10148   NODE_NAME_CASE(SRA_VL)
10149   NODE_NAME_CASE(SRL_VL)
10150   NODE_NAME_CASE(SUB_VL)
10151   NODE_NAME_CASE(UDIV_VL)
10152   NODE_NAME_CASE(UREM_VL)
10153   NODE_NAME_CASE(XOR_VL)
10154   NODE_NAME_CASE(SADDSAT_VL)
10155   NODE_NAME_CASE(UADDSAT_VL)
10156   NODE_NAME_CASE(SSUBSAT_VL)
10157   NODE_NAME_CASE(USUBSAT_VL)
10158   NODE_NAME_CASE(FADD_VL)
10159   NODE_NAME_CASE(FSUB_VL)
10160   NODE_NAME_CASE(FMUL_VL)
10161   NODE_NAME_CASE(FDIV_VL)
10162   NODE_NAME_CASE(FNEG_VL)
10163   NODE_NAME_CASE(FABS_VL)
10164   NODE_NAME_CASE(FSQRT_VL)
10165   NODE_NAME_CASE(FMA_VL)
10166   NODE_NAME_CASE(FCOPYSIGN_VL)
10167   NODE_NAME_CASE(SMIN_VL)
10168   NODE_NAME_CASE(SMAX_VL)
10169   NODE_NAME_CASE(UMIN_VL)
10170   NODE_NAME_CASE(UMAX_VL)
10171   NODE_NAME_CASE(FMINNUM_VL)
10172   NODE_NAME_CASE(FMAXNUM_VL)
10173   NODE_NAME_CASE(MULHS_VL)
10174   NODE_NAME_CASE(MULHU_VL)
10175   NODE_NAME_CASE(FP_TO_SINT_VL)
10176   NODE_NAME_CASE(FP_TO_UINT_VL)
10177   NODE_NAME_CASE(SINT_TO_FP_VL)
10178   NODE_NAME_CASE(UINT_TO_FP_VL)
10179   NODE_NAME_CASE(FP_EXTEND_VL)
10180   NODE_NAME_CASE(FP_ROUND_VL)
10181   NODE_NAME_CASE(VWMUL_VL)
10182   NODE_NAME_CASE(VWMULU_VL)
10183   NODE_NAME_CASE(VWMULSU_VL)
10184   NODE_NAME_CASE(VWADDU_VL)
10185   NODE_NAME_CASE(SETCC_VL)
10186   NODE_NAME_CASE(VSELECT_VL)
10187   NODE_NAME_CASE(VP_MERGE_VL)
10188   NODE_NAME_CASE(VMAND_VL)
10189   NODE_NAME_CASE(VMOR_VL)
10190   NODE_NAME_CASE(VMXOR_VL)
10191   NODE_NAME_CASE(VMCLR_VL)
10192   NODE_NAME_CASE(VMSET_VL)
10193   NODE_NAME_CASE(VRGATHER_VX_VL)
10194   NODE_NAME_CASE(VRGATHER_VV_VL)
10195   NODE_NAME_CASE(VRGATHEREI16_VV_VL)
10196   NODE_NAME_CASE(VSEXT_VL)
10197   NODE_NAME_CASE(VZEXT_VL)
10198   NODE_NAME_CASE(VCPOP_VL)
10199   NODE_NAME_CASE(VLE_VL)
10200   NODE_NAME_CASE(VSE_VL)
10201   NODE_NAME_CASE(READ_CSR)
10202   NODE_NAME_CASE(WRITE_CSR)
10203   NODE_NAME_CASE(SWAP_CSR)
10204   }
10205   // clang-format on
10206   return nullptr;
10207 #undef NODE_NAME_CASE
10208 }
10209 
10210 /// getConstraintType - Given a constraint letter, return the type of
10211 /// constraint it is for this target.
10212 RISCVTargetLowering::ConstraintType
10213 RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
10214   if (Constraint.size() == 1) {
10215     switch (Constraint[0]) {
10216     default:
10217       break;
10218     case 'f':
10219       return C_RegisterClass;
10220     case 'I':
10221     case 'J':
10222     case 'K':
10223       return C_Immediate;
10224     case 'A':
10225       return C_Memory;
10226     case 'S': // A symbolic address
10227       return C_Other;
10228     }
10229   } else {
10230     if (Constraint == "vr" || Constraint == "vm")
10231       return C_RegisterClass;
10232   }
10233   return TargetLowering::getConstraintType(Constraint);
10234 }
10235 
10236 std::pair<unsigned, const TargetRegisterClass *>
10237 RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
10238                                                   StringRef Constraint,
10239                                                   MVT VT) const {
10240   // First, see if this is a constraint that directly corresponds to a
10241   // RISCV register class.
10242   if (Constraint.size() == 1) {
10243     switch (Constraint[0]) {
10244     case 'r':
10245       // TODO: Support fixed vectors up to XLen for P extension?
10246       if (VT.isVector())
10247         break;
10248       return std::make_pair(0U, &RISCV::GPRRegClass);
10249     case 'f':
10250       if (Subtarget.hasStdExtZfh() && VT == MVT::f16)
10251         return std::make_pair(0U, &RISCV::FPR16RegClass);
10252       if (Subtarget.hasStdExtF() && VT == MVT::f32)
10253         return std::make_pair(0U, &RISCV::FPR32RegClass);
10254       if (Subtarget.hasStdExtD() && VT == MVT::f64)
10255         return std::make_pair(0U, &RISCV::FPR64RegClass);
10256       break;
10257     default:
10258       break;
10259     }
10260   } else if (Constraint == "vr") {
10261     for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
10262                            &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10263       if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
10264         return std::make_pair(0U, RC);
10265     }
10266   } else if (Constraint == "vm") {
10267     if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
10268       return std::make_pair(0U, &RISCV::VMV0RegClass);
10269   }
10270 
10271   // Clang will correctly decode the usage of register name aliases into their
10272   // official names. However, other frontends like `rustc` do not. This allows
10273   // users of these frontends to use the ABI names for registers in LLVM-style
10274   // register constraints.
10275   unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
10276                                .Case("{zero}", RISCV::X0)
10277                                .Case("{ra}", RISCV::X1)
10278                                .Case("{sp}", RISCV::X2)
10279                                .Case("{gp}", RISCV::X3)
10280                                .Case("{tp}", RISCV::X4)
10281                                .Case("{t0}", RISCV::X5)
10282                                .Case("{t1}", RISCV::X6)
10283                                .Case("{t2}", RISCV::X7)
10284                                .Cases("{s0}", "{fp}", RISCV::X8)
10285                                .Case("{s1}", RISCV::X9)
10286                                .Case("{a0}", RISCV::X10)
10287                                .Case("{a1}", RISCV::X11)
10288                                .Case("{a2}", RISCV::X12)
10289                                .Case("{a3}", RISCV::X13)
10290                                .Case("{a4}", RISCV::X14)
10291                                .Case("{a5}", RISCV::X15)
10292                                .Case("{a6}", RISCV::X16)
10293                                .Case("{a7}", RISCV::X17)
10294                                .Case("{s2}", RISCV::X18)
10295                                .Case("{s3}", RISCV::X19)
10296                                .Case("{s4}", RISCV::X20)
10297                                .Case("{s5}", RISCV::X21)
10298                                .Case("{s6}", RISCV::X22)
10299                                .Case("{s7}", RISCV::X23)
10300                                .Case("{s8}", RISCV::X24)
10301                                .Case("{s9}", RISCV::X25)
10302                                .Case("{s10}", RISCV::X26)
10303                                .Case("{s11}", RISCV::X27)
10304                                .Case("{t3}", RISCV::X28)
10305                                .Case("{t4}", RISCV::X29)
10306                                .Case("{t5}", RISCV::X30)
10307                                .Case("{t6}", RISCV::X31)
10308                                .Default(RISCV::NoRegister);
10309   if (XRegFromAlias != RISCV::NoRegister)
10310     return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
10311 
10312   // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
10313   // TableGen record rather than the AsmName to choose registers for InlineAsm
10314   // constraints, plus we want to match those names to the widest floating point
10315   // register type available, manually select floating point registers here.
10316   //
10317   // The second case is the ABI name of the register, so that frontends can also
10318   // use the ABI names in register constraint lists.
10319   if (Subtarget.hasStdExtF()) {
10320     unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
10321                         .Cases("{f0}", "{ft0}", RISCV::F0_F)
10322                         .Cases("{f1}", "{ft1}", RISCV::F1_F)
10323                         .Cases("{f2}", "{ft2}", RISCV::F2_F)
10324                         .Cases("{f3}", "{ft3}", RISCV::F3_F)
10325                         .Cases("{f4}", "{ft4}", RISCV::F4_F)
10326                         .Cases("{f5}", "{ft5}", RISCV::F5_F)
10327                         .Cases("{f6}", "{ft6}", RISCV::F6_F)
10328                         .Cases("{f7}", "{ft7}", RISCV::F7_F)
10329                         .Cases("{f8}", "{fs0}", RISCV::F8_F)
10330                         .Cases("{f9}", "{fs1}", RISCV::F9_F)
10331                         .Cases("{f10}", "{fa0}", RISCV::F10_F)
10332                         .Cases("{f11}", "{fa1}", RISCV::F11_F)
10333                         .Cases("{f12}", "{fa2}", RISCV::F12_F)
10334                         .Cases("{f13}", "{fa3}", RISCV::F13_F)
10335                         .Cases("{f14}", "{fa4}", RISCV::F14_F)
10336                         .Cases("{f15}", "{fa5}", RISCV::F15_F)
10337                         .Cases("{f16}", "{fa6}", RISCV::F16_F)
10338                         .Cases("{f17}", "{fa7}", RISCV::F17_F)
10339                         .Cases("{f18}", "{fs2}", RISCV::F18_F)
10340                         .Cases("{f19}", "{fs3}", RISCV::F19_F)
10341                         .Cases("{f20}", "{fs4}", RISCV::F20_F)
10342                         .Cases("{f21}", "{fs5}", RISCV::F21_F)
10343                         .Cases("{f22}", "{fs6}", RISCV::F22_F)
10344                         .Cases("{f23}", "{fs7}", RISCV::F23_F)
10345                         .Cases("{f24}", "{fs8}", RISCV::F24_F)
10346                         .Cases("{f25}", "{fs9}", RISCV::F25_F)
10347                         .Cases("{f26}", "{fs10}", RISCV::F26_F)
10348                         .Cases("{f27}", "{fs11}", RISCV::F27_F)
10349                         .Cases("{f28}", "{ft8}", RISCV::F28_F)
10350                         .Cases("{f29}", "{ft9}", RISCV::F29_F)
10351                         .Cases("{f30}", "{ft10}", RISCV::F30_F)
10352                         .Cases("{f31}", "{ft11}", RISCV::F31_F)
10353                         .Default(RISCV::NoRegister);
10354     if (FReg != RISCV::NoRegister) {
10355       assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
10356       if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
10357         unsigned RegNo = FReg - RISCV::F0_F;
10358         unsigned DReg = RISCV::F0_D + RegNo;
10359         return std::make_pair(DReg, &RISCV::FPR64RegClass);
10360       }
10361       if (VT == MVT::f32 || VT == MVT::Other)
10362         return std::make_pair(FReg, &RISCV::FPR32RegClass);
10363       if (Subtarget.hasStdExtZfh() && VT == MVT::f16) {
10364         unsigned RegNo = FReg - RISCV::F0_F;
10365         unsigned HReg = RISCV::F0_H + RegNo;
10366         return std::make_pair(HReg, &RISCV::FPR16RegClass);
10367       }
10368     }
10369   }
10370 
10371   if (Subtarget.hasVInstructions()) {
10372     Register VReg = StringSwitch<Register>(Constraint.lower())
10373                         .Case("{v0}", RISCV::V0)
10374                         .Case("{v1}", RISCV::V1)
10375                         .Case("{v2}", RISCV::V2)
10376                         .Case("{v3}", RISCV::V3)
10377                         .Case("{v4}", RISCV::V4)
10378                         .Case("{v5}", RISCV::V5)
10379                         .Case("{v6}", RISCV::V6)
10380                         .Case("{v7}", RISCV::V7)
10381                         .Case("{v8}", RISCV::V8)
10382                         .Case("{v9}", RISCV::V9)
10383                         .Case("{v10}", RISCV::V10)
10384                         .Case("{v11}", RISCV::V11)
10385                         .Case("{v12}", RISCV::V12)
10386                         .Case("{v13}", RISCV::V13)
10387                         .Case("{v14}", RISCV::V14)
10388                         .Case("{v15}", RISCV::V15)
10389                         .Case("{v16}", RISCV::V16)
10390                         .Case("{v17}", RISCV::V17)
10391                         .Case("{v18}", RISCV::V18)
10392                         .Case("{v19}", RISCV::V19)
10393                         .Case("{v20}", RISCV::V20)
10394                         .Case("{v21}", RISCV::V21)
10395                         .Case("{v22}", RISCV::V22)
10396                         .Case("{v23}", RISCV::V23)
10397                         .Case("{v24}", RISCV::V24)
10398                         .Case("{v25}", RISCV::V25)
10399                         .Case("{v26}", RISCV::V26)
10400                         .Case("{v27}", RISCV::V27)
10401                         .Case("{v28}", RISCV::V28)
10402                         .Case("{v29}", RISCV::V29)
10403                         .Case("{v30}", RISCV::V30)
10404                         .Case("{v31}", RISCV::V31)
10405                         .Default(RISCV::NoRegister);
10406     if (VReg != RISCV::NoRegister) {
10407       if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
10408         return std::make_pair(VReg, &RISCV::VMRegClass);
10409       if (TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.SimpleTy))
10410         return std::make_pair(VReg, &RISCV::VRRegClass);
10411       for (const auto *RC :
10412            {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10413         if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) {
10414           VReg = TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
10415           return std::make_pair(VReg, RC);
10416         }
10417       }
10418     }
10419   }
10420 
10421   return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
10422 }
10423 
10424 unsigned
10425 RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
10426   // Currently only support length 1 constraints.
10427   if (ConstraintCode.size() == 1) {
10428     switch (ConstraintCode[0]) {
10429     case 'A':
10430       return InlineAsm::Constraint_A;
10431     default:
10432       break;
10433     }
10434   }
10435 
10436   return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
10437 }
10438 
10439 void RISCVTargetLowering::LowerAsmOperandForConstraint(
10440     SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
10441     SelectionDAG &DAG) const {
10442   // Currently only support length 1 constraints.
10443   if (Constraint.length() == 1) {
10444     switch (Constraint[0]) {
10445     case 'I':
10446       // Validate & create a 12-bit signed immediate operand.
10447       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10448         uint64_t CVal = C->getSExtValue();
10449         if (isInt<12>(CVal))
10450           Ops.push_back(
10451               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10452       }
10453       return;
10454     case 'J':
10455       // Validate & create an integer zero operand.
10456       if (auto *C = dyn_cast<ConstantSDNode>(Op))
10457         if (C->getZExtValue() == 0)
10458           Ops.push_back(
10459               DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
10460       return;
10461     case 'K':
10462       // Validate & create a 5-bit unsigned immediate operand.
10463       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10464         uint64_t CVal = C->getZExtValue();
10465         if (isUInt<5>(CVal))
10466           Ops.push_back(
10467               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10468       }
10469       return;
10470     case 'S':
10471       if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
10472         Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
10473                                                  GA->getValueType(0)));
10474       } else if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
10475         Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(),
10476                                                 BA->getValueType(0)));
10477       }
10478       return;
10479     default:
10480       break;
10481     }
10482   }
10483   TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
10484 }
10485 
10486 Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
10487                                                    Instruction *Inst,
10488                                                    AtomicOrdering Ord) const {
10489   if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
10490     return Builder.CreateFence(Ord);
10491   if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
10492     return Builder.CreateFence(AtomicOrdering::Release);
10493   return nullptr;
10494 }
10495 
10496 Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
10497                                                     Instruction *Inst,
10498                                                     AtomicOrdering Ord) const {
10499   if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
10500     return Builder.CreateFence(AtomicOrdering::Acquire);
10501   return nullptr;
10502 }
10503 
10504 TargetLowering::AtomicExpansionKind
10505 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
10506   // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
10507   // point operations can't be used in an lr/sc sequence without breaking the
10508   // forward-progress guarantee.
10509   if (AI->isFloatingPointOperation())
10510     return AtomicExpansionKind::CmpXChg;
10511 
10512   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
10513   if (Size == 8 || Size == 16)
10514     return AtomicExpansionKind::MaskedIntrinsic;
10515   return AtomicExpansionKind::None;
10516 }
10517 
10518 static Intrinsic::ID
10519 getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
10520   if (XLen == 32) {
10521     switch (BinOp) {
10522     default:
10523       llvm_unreachable("Unexpected AtomicRMW BinOp");
10524     case AtomicRMWInst::Xchg:
10525       return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
10526     case AtomicRMWInst::Add:
10527       return Intrinsic::riscv_masked_atomicrmw_add_i32;
10528     case AtomicRMWInst::Sub:
10529       return Intrinsic::riscv_masked_atomicrmw_sub_i32;
10530     case AtomicRMWInst::Nand:
10531       return Intrinsic::riscv_masked_atomicrmw_nand_i32;
10532     case AtomicRMWInst::Max:
10533       return Intrinsic::riscv_masked_atomicrmw_max_i32;
10534     case AtomicRMWInst::Min:
10535       return Intrinsic::riscv_masked_atomicrmw_min_i32;
10536     case AtomicRMWInst::UMax:
10537       return Intrinsic::riscv_masked_atomicrmw_umax_i32;
10538     case AtomicRMWInst::UMin:
10539       return Intrinsic::riscv_masked_atomicrmw_umin_i32;
10540     }
10541   }
10542 
10543   if (XLen == 64) {
10544     switch (BinOp) {
10545     default:
10546       llvm_unreachable("Unexpected AtomicRMW BinOp");
10547     case AtomicRMWInst::Xchg:
10548       return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
10549     case AtomicRMWInst::Add:
10550       return Intrinsic::riscv_masked_atomicrmw_add_i64;
10551     case AtomicRMWInst::Sub:
10552       return Intrinsic::riscv_masked_atomicrmw_sub_i64;
10553     case AtomicRMWInst::Nand:
10554       return Intrinsic::riscv_masked_atomicrmw_nand_i64;
10555     case AtomicRMWInst::Max:
10556       return Intrinsic::riscv_masked_atomicrmw_max_i64;
10557     case AtomicRMWInst::Min:
10558       return Intrinsic::riscv_masked_atomicrmw_min_i64;
10559     case AtomicRMWInst::UMax:
10560       return Intrinsic::riscv_masked_atomicrmw_umax_i64;
10561     case AtomicRMWInst::UMin:
10562       return Intrinsic::riscv_masked_atomicrmw_umin_i64;
10563     }
10564   }
10565 
10566   llvm_unreachable("Unexpected XLen\n");
10567 }
10568 
10569 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
10570     IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
10571     Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
10572   unsigned XLen = Subtarget.getXLen();
10573   Value *Ordering =
10574       Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
10575   Type *Tys[] = {AlignedAddr->getType()};
10576   Function *LrwOpScwLoop = Intrinsic::getDeclaration(
10577       AI->getModule(),
10578       getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
10579 
10580   if (XLen == 64) {
10581     Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
10582     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10583     ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
10584   }
10585 
10586   Value *Result;
10587 
10588   // Must pass the shift amount needed to sign extend the loaded value prior
10589   // to performing a signed comparison for min/max. ShiftAmt is the number of
10590   // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
10591   // is the number of bits to left+right shift the value in order to
10592   // sign-extend.
10593   if (AI->getOperation() == AtomicRMWInst::Min ||
10594       AI->getOperation() == AtomicRMWInst::Max) {
10595     const DataLayout &DL = AI->getModule()->getDataLayout();
10596     unsigned ValWidth =
10597         DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
10598     Value *SextShamt =
10599         Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
10600     Result = Builder.CreateCall(LrwOpScwLoop,
10601                                 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
10602   } else {
10603     Result =
10604         Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
10605   }
10606 
10607   if (XLen == 64)
10608     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10609   return Result;
10610 }
10611 
10612 TargetLowering::AtomicExpansionKind
10613 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
10614     AtomicCmpXchgInst *CI) const {
10615   unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
10616   if (Size == 8 || Size == 16)
10617     return AtomicExpansionKind::MaskedIntrinsic;
10618   return AtomicExpansionKind::None;
10619 }
10620 
10621 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
10622     IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
10623     Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
10624   unsigned XLen = Subtarget.getXLen();
10625   Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
10626   Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
10627   if (XLen == 64) {
10628     CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
10629     NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
10630     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10631     CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
10632   }
10633   Type *Tys[] = {AlignedAddr->getType()};
10634   Function *MaskedCmpXchg =
10635       Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
10636   Value *Result = Builder.CreateCall(
10637       MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
10638   if (XLen == 64)
10639     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10640   return Result;
10641 }
10642 
10643 bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
10644   return false;
10645 }
10646 
10647 bool RISCVTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
10648                                                EVT VT) const {
10649   if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
10650     return false;
10651 
10652   switch (FPVT.getSimpleVT().SimpleTy) {
10653   case MVT::f16:
10654     return Subtarget.hasStdExtZfh();
10655   case MVT::f32:
10656     return Subtarget.hasStdExtF();
10657   case MVT::f64:
10658     return Subtarget.hasStdExtD();
10659   default:
10660     return false;
10661   }
10662 }
10663 
10664 unsigned RISCVTargetLowering::getJumpTableEncoding() const {
10665   // If we are using the small code model, we can reduce size of jump table
10666   // entry to 4 bytes.
10667   if (Subtarget.is64Bit() && !isPositionIndependent() &&
10668       getTargetMachine().getCodeModel() == CodeModel::Small) {
10669     return MachineJumpTableInfo::EK_Custom32;
10670   }
10671   return TargetLowering::getJumpTableEncoding();
10672 }
10673 
10674 const MCExpr *RISCVTargetLowering::LowerCustomJumpTableEntry(
10675     const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
10676     unsigned uid, MCContext &Ctx) const {
10677   assert(Subtarget.is64Bit() && !isPositionIndependent() &&
10678          getTargetMachine().getCodeModel() == CodeModel::Small);
10679   return MCSymbolRefExpr::create(MBB->getSymbol(), Ctx);
10680 }
10681 
10682 bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
10683                                                      EVT VT) const {
10684   VT = VT.getScalarType();
10685 
10686   if (!VT.isSimple())
10687     return false;
10688 
10689   switch (VT.getSimpleVT().SimpleTy) {
10690   case MVT::f16:
10691     return Subtarget.hasStdExtZfh();
10692   case MVT::f32:
10693     return Subtarget.hasStdExtF();
10694   case MVT::f64:
10695     return Subtarget.hasStdExtD();
10696   default:
10697     break;
10698   }
10699 
10700   return false;
10701 }
10702 
10703 Register RISCVTargetLowering::getExceptionPointerRegister(
10704     const Constant *PersonalityFn) const {
10705   return RISCV::X10;
10706 }
10707 
10708 Register RISCVTargetLowering::getExceptionSelectorRegister(
10709     const Constant *PersonalityFn) const {
10710   return RISCV::X11;
10711 }
10712 
10713 bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
10714   // Return false to suppress the unnecessary extensions if the LibCall
10715   // arguments or return value is f32 type for LP64 ABI.
10716   RISCVABI::ABI ABI = Subtarget.getTargetABI();
10717   if (ABI == RISCVABI::ABI_LP64 && (Type == MVT::f32))
10718     return false;
10719 
10720   return true;
10721 }
10722 
10723 bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
10724   if (Subtarget.is64Bit() && Type == MVT::i32)
10725     return true;
10726 
10727   return IsSigned;
10728 }
10729 
10730 bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
10731                                                  SDValue C) const {
10732   // Check integral scalar types.
10733   if (VT.isScalarInteger()) {
10734     // Omit the optimization if the sub target has the M extension and the data
10735     // size exceeds XLen.
10736     if (Subtarget.hasStdExtM() && VT.getSizeInBits() > Subtarget.getXLen())
10737       return false;
10738     if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
10739       // Break the MUL to a SLLI and an ADD/SUB.
10740       const APInt &Imm = ConstNode->getAPIntValue();
10741       if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
10742           (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
10743         return true;
10744       // Optimize the MUL to (SH*ADD x, (SLLI x, bits)) if Imm is not simm12.
10745       if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
10746           ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
10747            (Imm - 8).isPowerOf2()))
10748         return true;
10749       // Omit the following optimization if the sub target has the M extension
10750       // and the data size >= XLen.
10751       if (Subtarget.hasStdExtM() && VT.getSizeInBits() >= Subtarget.getXLen())
10752         return false;
10753       // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
10754       // a pair of LUI/ADDI.
10755       if (!Imm.isSignedIntN(12) && Imm.countTrailingZeros() < 12) {
10756         APInt ImmS = Imm.ashr(Imm.countTrailingZeros());
10757         if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
10758             (1 - ImmS).isPowerOf2())
10759         return true;
10760       }
10761     }
10762   }
10763 
10764   return false;
10765 }
10766 
10767 bool RISCVTargetLowering::isMulAddWithConstProfitable(
10768     const SDValue &AddNode, const SDValue &ConstNode) const {
10769   // Let the DAGCombiner decide for vectors.
10770   EVT VT = AddNode.getValueType();
10771   if (VT.isVector())
10772     return true;
10773 
10774   // Let the DAGCombiner decide for larger types.
10775   if (VT.getScalarSizeInBits() > Subtarget.getXLen())
10776     return true;
10777 
10778   // It is worse if c1 is simm12 while c1*c2 is not.
10779   ConstantSDNode *C1Node = cast<ConstantSDNode>(AddNode.getOperand(1));
10780   ConstantSDNode *C2Node = cast<ConstantSDNode>(ConstNode);
10781   const APInt &C1 = C1Node->getAPIntValue();
10782   const APInt &C2 = C2Node->getAPIntValue();
10783   if (C1.isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
10784     return false;
10785 
10786   // Default to true and let the DAGCombiner decide.
10787   return true;
10788 }
10789 
10790 bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
10791     EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
10792     bool *Fast) const {
10793   if (!VT.isVector())
10794     return false;
10795 
10796   EVT ElemVT = VT.getVectorElementType();
10797   if (Alignment >= ElemVT.getStoreSize()) {
10798     if (Fast)
10799       *Fast = true;
10800     return true;
10801   }
10802 
10803   return false;
10804 }
10805 
10806 bool RISCVTargetLowering::splitValueIntoRegisterParts(
10807     SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
10808     unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {
10809   bool IsABIRegCopy = CC.hasValue();
10810   EVT ValueVT = Val.getValueType();
10811   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
10812     // Cast the f16 to i16, extend to i32, pad with ones to make a float nan,
10813     // and cast to f32.
10814     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
10815     Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
10816     Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
10817                       DAG.getConstant(0xFFFF0000, DL, MVT::i32));
10818     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
10819     Parts[0] = Val;
10820     return true;
10821   }
10822 
10823   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
10824     LLVMContext &Context = *DAG.getContext();
10825     EVT ValueEltVT = ValueVT.getVectorElementType();
10826     EVT PartEltVT = PartVT.getVectorElementType();
10827     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
10828     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
10829     if (PartVTBitSize % ValueVTBitSize == 0) {
10830       assert(PartVTBitSize >= ValueVTBitSize);
10831       // If the element types are different, bitcast to the same element type of
10832       // PartVT first.
10833       // Give an example here, we want copy a <vscale x 1 x i8> value to
10834       // <vscale x 4 x i16>.
10835       // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert
10836       // subvector, then we can bitcast to <vscale x 4 x i16>.
10837       if (ValueEltVT != PartEltVT) {
10838         if (PartVTBitSize > ValueVTBitSize) {
10839           unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
10840           assert(Count != 0 && "The number of element should not be zero.");
10841           EVT SameEltTypeVT =
10842               EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
10843           Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT,
10844                             DAG.getUNDEF(SameEltTypeVT), Val,
10845                             DAG.getVectorIdxConstant(0, DL));
10846         }
10847         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
10848       } else {
10849         Val =
10850             DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
10851                         Val, DAG.getVectorIdxConstant(0, DL));
10852       }
10853       Parts[0] = Val;
10854       return true;
10855     }
10856   }
10857   return false;
10858 }
10859 
10860 SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
10861     SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
10862     MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {
10863   bool IsABIRegCopy = CC.hasValue();
10864   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
10865     SDValue Val = Parts[0];
10866 
10867     // Cast the f32 to i32, truncate to i16, and cast back to f16.
10868     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
10869     Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
10870     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f16, Val);
10871     return Val;
10872   }
10873 
10874   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
10875     LLVMContext &Context = *DAG.getContext();
10876     SDValue Val = Parts[0];
10877     EVT ValueEltVT = ValueVT.getVectorElementType();
10878     EVT PartEltVT = PartVT.getVectorElementType();
10879     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
10880     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
10881     if (PartVTBitSize % ValueVTBitSize == 0) {
10882       assert(PartVTBitSize >= ValueVTBitSize);
10883       EVT SameEltTypeVT = ValueVT;
10884       // If the element types are different, convert it to the same element type
10885       // of PartVT.
10886       // Give an example here, we want copy a <vscale x 1 x i8> value from
10887       // <vscale x 4 x i16>.
10888       // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first,
10889       // then we can extract <vscale x 1 x i8>.
10890       if (ValueEltVT != PartEltVT) {
10891         unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
10892         assert(Count != 0 && "The number of element should not be zero.");
10893         SameEltTypeVT =
10894             EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
10895         Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
10896       }
10897       Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
10898                         DAG.getVectorIdxConstant(0, DL));
10899       return Val;
10900     }
10901   }
10902   return SDValue();
10903 }
10904 
10905 SDValue
10906 RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
10907                                    SelectionDAG &DAG,
10908                                    SmallVectorImpl<SDNode *> &Created) const {
10909   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
10910   if (isIntDivCheap(N->getValueType(0), Attr))
10911     return SDValue(N, 0); // Lower SDIV as SDIV
10912 
10913   assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
10914          "Unexpected divisor!");
10915 
10916   // Conditional move is needed, so do the transformation iff Zbt is enabled.
10917   if (!Subtarget.hasStdExtZbt())
10918     return SDValue();
10919 
10920   // When |Divisor| >= 2 ^ 12, it isn't profitable to do such transformation.
10921   // Besides, more critical path instructions will be generated when dividing
10922   // by 2. So we keep using the original DAGs for these cases.
10923   unsigned Lg2 = Divisor.countTrailingZeros();
10924   if (Lg2 == 1 || Lg2 >= 12)
10925     return SDValue();
10926 
10927   // fold (sdiv X, pow2)
10928   EVT VT = N->getValueType(0);
10929   if (VT != MVT::i32 && !(Subtarget.is64Bit() && VT == MVT::i64))
10930     return SDValue();
10931 
10932   SDLoc DL(N);
10933   SDValue N0 = N->getOperand(0);
10934   SDValue Zero = DAG.getConstant(0, DL, VT);
10935   SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
10936 
10937   // Add (N0 < 0) ? Pow2 - 1 : 0;
10938   SDValue Cmp = DAG.getSetCC(DL, VT, N0, Zero, ISD::SETLT);
10939   SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
10940   SDValue Sel = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
10941 
10942   Created.push_back(Cmp.getNode());
10943   Created.push_back(Add.getNode());
10944   Created.push_back(Sel.getNode());
10945 
10946   // Divide by pow2.
10947   SDValue SRA =
10948       DAG.getNode(ISD::SRA, DL, VT, Sel, DAG.getConstant(Lg2, DL, VT));
10949 
10950   // If we're dividing by a positive value, we're done.  Otherwise, we must
10951   // negate the result.
10952   if (Divisor.isNonNegative())
10953     return SRA;
10954 
10955   Created.push_back(SRA.getNode());
10956   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
10957 }
10958 
10959 #define GET_REGISTER_MATCHER
10960 #include "RISCVGenAsmMatcher.inc"
10961 
10962 Register
10963 RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
10964                                        const MachineFunction &MF) const {
10965   Register Reg = MatchRegisterAltName(RegName);
10966   if (Reg == RISCV::NoRegister)
10967     Reg = MatchRegisterName(RegName);
10968   if (Reg == RISCV::NoRegister)
10969     report_fatal_error(
10970         Twine("Invalid register name \"" + StringRef(RegName) + "\"."));
10971   BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
10972   if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg))
10973     report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
10974                              StringRef(RegName) + "\"."));
10975   return Reg;
10976 }
10977 
10978 namespace llvm {
10979 namespace RISCVVIntrinsicsTable {
10980 
10981 #define GET_RISCVVIntrinsicsTable_IMPL
10982 #include "RISCVGenSearchableTables.inc"
10983 
10984 } // namespace RISCVVIntrinsicsTable
10985 
10986 } // namespace llvm
10987