1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that RISCV uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "RISCVISelLowering.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCV.h"
17 #include "RISCVMachineFunctionInfo.h"
18 #include "RISCVRegisterInfo.h"
19 #include "RISCVSubtarget.h"
20 #include "RISCVTargetMachine.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/MemoryLocation.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineJumpTableInfo.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
30 #include "llvm/CodeGen/ValueTypes.h"
31 #include "llvm/IR/DiagnosticInfo.h"
32 #include "llvm/IR/DiagnosticPrinter.h"
33 #include "llvm/IR/IRBuilder.h"
34 #include "llvm/IR/IntrinsicsRISCV.h"
35 #include "llvm/IR/PatternMatch.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/KnownBits.h"
39 #include "llvm/Support/MathExtras.h"
40 #include "llvm/Support/raw_ostream.h"
41 
42 using namespace llvm;
43 
44 #define DEBUG_TYPE "riscv-lower"
45 
46 STATISTIC(NumTailCalls, "Number of tail calls");
47 
48 RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
49                                          const RISCVSubtarget &STI)
50     : TargetLowering(TM), Subtarget(STI) {
51 
52   if (Subtarget.isRV32E())
53     report_fatal_error("Codegen not yet implemented for RV32E");
54 
55   RISCVABI::ABI ABI = Subtarget.getTargetABI();
56   assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
57 
58   if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
59       !Subtarget.hasStdExtF()) {
60     errs() << "Hard-float 'f' ABI can't be used for a target that "
61                 "doesn't support the F instruction set extension (ignoring "
62                           "target-abi)\n";
63     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
64   } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
65              !Subtarget.hasStdExtD()) {
66     errs() << "Hard-float 'd' ABI can't be used for a target that "
67               "doesn't support the D instruction set extension (ignoring "
68               "target-abi)\n";
69     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
70   }
71 
72   switch (ABI) {
73   default:
74     report_fatal_error("Don't know how to lower this ABI");
75   case RISCVABI::ABI_ILP32:
76   case RISCVABI::ABI_ILP32F:
77   case RISCVABI::ABI_ILP32D:
78   case RISCVABI::ABI_LP64:
79   case RISCVABI::ABI_LP64F:
80   case RISCVABI::ABI_LP64D:
81     break;
82   }
83 
84   MVT XLenVT = Subtarget.getXLenVT();
85 
86   // Set up the register classes.
87   addRegisterClass(XLenVT, &RISCV::GPRRegClass);
88 
89   if (Subtarget.hasStdExtZfh())
90     addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
91   if (Subtarget.hasStdExtF())
92     addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
93   if (Subtarget.hasStdExtD())
94     addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
95 
96   static const MVT::SimpleValueType BoolVecVTs[] = {
97       MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,
98       MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
99   static const MVT::SimpleValueType IntVecVTs[] = {
100       MVT::nxv1i8,  MVT::nxv2i8,   MVT::nxv4i8,   MVT::nxv8i8,  MVT::nxv16i8,
101       MVT::nxv32i8, MVT::nxv64i8,  MVT::nxv1i16,  MVT::nxv2i16, MVT::nxv4i16,
102       MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
103       MVT::nxv4i32, MVT::nxv8i32,  MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
104       MVT::nxv4i64, MVT::nxv8i64};
105   static const MVT::SimpleValueType F16VecVTs[] = {
106       MVT::nxv1f16, MVT::nxv2f16,  MVT::nxv4f16,
107       MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
108   static const MVT::SimpleValueType F32VecVTs[] = {
109       MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
110   static const MVT::SimpleValueType F64VecVTs[] = {
111       MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
112 
113   if (Subtarget.hasVInstructions()) {
114     auto addRegClassForRVV = [this](MVT VT) {
115       unsigned Size = VT.getSizeInBits().getKnownMinValue();
116       assert(Size <= 512 && isPowerOf2_32(Size));
117       const TargetRegisterClass *RC;
118       if (Size <= 64)
119         RC = &RISCV::VRRegClass;
120       else if (Size == 128)
121         RC = &RISCV::VRM2RegClass;
122       else if (Size == 256)
123         RC = &RISCV::VRM4RegClass;
124       else
125         RC = &RISCV::VRM8RegClass;
126 
127       addRegisterClass(VT, RC);
128     };
129 
130     for (MVT VT : BoolVecVTs)
131       addRegClassForRVV(VT);
132     for (MVT VT : IntVecVTs) {
133       if (VT.getVectorElementType() == MVT::i64 &&
134           !Subtarget.hasVInstructionsI64())
135         continue;
136       addRegClassForRVV(VT);
137     }
138 
139     if (Subtarget.hasVInstructionsF16())
140       for (MVT VT : F16VecVTs)
141         addRegClassForRVV(VT);
142 
143     if (Subtarget.hasVInstructionsF32())
144       for (MVT VT : F32VecVTs)
145         addRegClassForRVV(VT);
146 
147     if (Subtarget.hasVInstructionsF64())
148       for (MVT VT : F64VecVTs)
149         addRegClassForRVV(VT);
150 
151     if (Subtarget.useRVVForFixedLengthVectors()) {
152       auto addRegClassForFixedVectors = [this](MVT VT) {
153         MVT ContainerVT = getContainerForFixedLengthVector(VT);
154         unsigned RCID = getRegClassIDForVecVT(ContainerVT);
155         const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
156         addRegisterClass(VT, TRI.getRegClass(RCID));
157       };
158       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
159         if (useRVVForFixedLengthVectorVT(VT))
160           addRegClassForFixedVectors(VT);
161 
162       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
163         if (useRVVForFixedLengthVectorVT(VT))
164           addRegClassForFixedVectors(VT);
165     }
166   }
167 
168   // Compute derived properties from the register classes.
169   computeRegisterProperties(STI.getRegisterInfo());
170 
171   setStackPointerRegisterToSaveRestore(RISCV::X2);
172 
173   for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
174     setLoadExtAction(N, XLenVT, MVT::i1, Promote);
175 
176   // TODO: add all necessary setOperationAction calls.
177   setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
178 
179   setOperationAction(ISD::BR_JT, MVT::Other, Expand);
180   setOperationAction(ISD::BR_CC, XLenVT, Expand);
181   setOperationAction(ISD::BRCOND, MVT::Other, Custom);
182   setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
183 
184   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
185   setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
186 
187   setOperationAction(ISD::VASTART, MVT::Other, Custom);
188   setOperationAction(ISD::VAARG, MVT::Other, Expand);
189   setOperationAction(ISD::VACOPY, MVT::Other, Expand);
190   setOperationAction(ISD::VAEND, MVT::Other, Expand);
191 
192   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
193   if (!Subtarget.hasStdExtZbb()) {
194     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
195     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
196   }
197 
198   if (Subtarget.is64Bit()) {
199     setOperationAction(ISD::ADD, MVT::i32, Custom);
200     setOperationAction(ISD::SUB, MVT::i32, Custom);
201     setOperationAction(ISD::SHL, MVT::i32, Custom);
202     setOperationAction(ISD::SRA, MVT::i32, Custom);
203     setOperationAction(ISD::SRL, MVT::i32, Custom);
204 
205     setOperationAction(ISD::UADDO, MVT::i32, Custom);
206     setOperationAction(ISD::USUBO, MVT::i32, Custom);
207     setOperationAction(ISD::UADDSAT, MVT::i32, Custom);
208     setOperationAction(ISD::USUBSAT, MVT::i32, Custom);
209   } else {
210     setLibcallName(RTLIB::SHL_I128, nullptr);
211     setLibcallName(RTLIB::SRL_I128, nullptr);
212     setLibcallName(RTLIB::SRA_I128, nullptr);
213     setLibcallName(RTLIB::MUL_I128, nullptr);
214     setLibcallName(RTLIB::MULO_I64, nullptr);
215   }
216 
217   if (!Subtarget.hasStdExtM()) {
218     setOperationAction(ISD::MUL, XLenVT, Expand);
219     setOperationAction(ISD::MULHS, XLenVT, Expand);
220     setOperationAction(ISD::MULHU, XLenVT, Expand);
221     setOperationAction(ISD::SDIV, XLenVT, Expand);
222     setOperationAction(ISD::UDIV, XLenVT, Expand);
223     setOperationAction(ISD::SREM, XLenVT, Expand);
224     setOperationAction(ISD::UREM, XLenVT, Expand);
225   } else {
226     if (Subtarget.is64Bit()) {
227       setOperationAction(ISD::MUL, MVT::i32, Custom);
228       setOperationAction(ISD::MUL, MVT::i128, Custom);
229 
230       setOperationAction(ISD::SDIV, MVT::i8, Custom);
231       setOperationAction(ISD::UDIV, MVT::i8, Custom);
232       setOperationAction(ISD::UREM, MVT::i8, Custom);
233       setOperationAction(ISD::SDIV, MVT::i16, Custom);
234       setOperationAction(ISD::UDIV, MVT::i16, Custom);
235       setOperationAction(ISD::UREM, MVT::i16, Custom);
236       setOperationAction(ISD::SDIV, MVT::i32, Custom);
237       setOperationAction(ISD::UDIV, MVT::i32, Custom);
238       setOperationAction(ISD::UREM, MVT::i32, Custom);
239     } else {
240       setOperationAction(ISD::MUL, MVT::i64, Custom);
241     }
242   }
243 
244   setOperationAction(ISD::SDIVREM, XLenVT, Expand);
245   setOperationAction(ISD::UDIVREM, XLenVT, Expand);
246   setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
247   setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
248 
249   setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
250   setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
251   setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
252 
253   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp() ||
254       Subtarget.hasStdExtZbkb()) {
255     if (Subtarget.is64Bit()) {
256       setOperationAction(ISD::ROTL, MVT::i32, Custom);
257       setOperationAction(ISD::ROTR, MVT::i32, Custom);
258     }
259   } else {
260     setOperationAction(ISD::ROTL, XLenVT, Expand);
261     setOperationAction(ISD::ROTR, XLenVT, Expand);
262   }
263 
264   if (Subtarget.hasStdExtZbp()) {
265     // Custom lower bswap/bitreverse so we can convert them to GREVI to enable
266     // more combining.
267     setOperationAction(ISD::BITREVERSE, XLenVT,   Custom);
268     setOperationAction(ISD::BSWAP,      XLenVT,   Custom);
269     setOperationAction(ISD::BITREVERSE, MVT::i8,  Custom);
270     // BSWAP i8 doesn't exist.
271     setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
272     setOperationAction(ISD::BSWAP,      MVT::i16, Custom);
273 
274     if (Subtarget.is64Bit()) {
275       setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
276       setOperationAction(ISD::BSWAP,      MVT::i32, Custom);
277     }
278   } else {
279     // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
280     // pattern match it directly in isel.
281     setOperationAction(ISD::BSWAP, XLenVT,
282                        (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb())
283                            ? Legal
284                            : Expand);
285   }
286 
287   if (Subtarget.hasStdExtZbb()) {
288     setOperationAction(ISD::SMIN, XLenVT, Legal);
289     setOperationAction(ISD::SMAX, XLenVT, Legal);
290     setOperationAction(ISD::UMIN, XLenVT, Legal);
291     setOperationAction(ISD::UMAX, XLenVT, Legal);
292 
293     if (Subtarget.is64Bit()) {
294       setOperationAction(ISD::CTTZ, MVT::i32, Custom);
295       setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom);
296       setOperationAction(ISD::CTLZ, MVT::i32, Custom);
297       setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom);
298     }
299   } else {
300     setOperationAction(ISD::CTTZ, XLenVT, Expand);
301     setOperationAction(ISD::CTLZ, XLenVT, Expand);
302     setOperationAction(ISD::CTPOP, XLenVT, Expand);
303   }
304 
305   if (Subtarget.hasStdExtZbt()) {
306     setOperationAction(ISD::FSHL, XLenVT, Custom);
307     setOperationAction(ISD::FSHR, XLenVT, Custom);
308     setOperationAction(ISD::SELECT, XLenVT, Legal);
309 
310     if (Subtarget.is64Bit()) {
311       setOperationAction(ISD::FSHL, MVT::i32, Custom);
312       setOperationAction(ISD::FSHR, MVT::i32, Custom);
313     }
314   } else {
315     setOperationAction(ISD::SELECT, XLenVT, Custom);
316   }
317 
318   static const ISD::CondCode FPCCToExpand[] = {
319       ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
320       ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
321       ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};
322 
323   static const ISD::NodeType FPOpToExpand[] = {
324       ISD::FSIN, ISD::FCOS,       ISD::FSINCOS,   ISD::FPOW,
325       ISD::FREM, ISD::FP16_TO_FP, ISD::FP_TO_FP16};
326 
327   if (Subtarget.hasStdExtZfh())
328     setOperationAction(ISD::BITCAST, MVT::i16, Custom);
329 
330   if (Subtarget.hasStdExtZfh()) {
331     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
332     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
333     setOperationAction(ISD::LRINT, MVT::f16, Legal);
334     setOperationAction(ISD::LLRINT, MVT::f16, Legal);
335     setOperationAction(ISD::LROUND, MVT::f16, Legal);
336     setOperationAction(ISD::LLROUND, MVT::f16, Legal);
337     setOperationAction(ISD::STRICT_LRINT, MVT::f16, Legal);
338     setOperationAction(ISD::STRICT_LLRINT, MVT::f16, Legal);
339     setOperationAction(ISD::STRICT_LROUND, MVT::f16, Legal);
340     setOperationAction(ISD::STRICT_LLROUND, MVT::f16, Legal);
341     setOperationAction(ISD::STRICT_FADD, MVT::f16, Legal);
342     setOperationAction(ISD::STRICT_FMA, MVT::f16, Legal);
343     setOperationAction(ISD::STRICT_FSUB, MVT::f16, Legal);
344     setOperationAction(ISD::STRICT_FMUL, MVT::f16, Legal);
345     setOperationAction(ISD::STRICT_FDIV, MVT::f16, Legal);
346     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
347     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
348     setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Legal);
349     setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Legal);
350     setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Legal);
351     for (auto CC : FPCCToExpand)
352       setCondCodeAction(CC, MVT::f16, Expand);
353     setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
354     setOperationAction(ISD::SELECT, MVT::f16, Custom);
355     setOperationAction(ISD::BR_CC, MVT::f16, Expand);
356 
357     setOperationAction(ISD::FREM,       MVT::f16, Promote);
358     setOperationAction(ISD::FCEIL,      MVT::f16, Promote);
359     setOperationAction(ISD::FFLOOR,     MVT::f16, Promote);
360     setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote);
361     setOperationAction(ISD::FRINT,      MVT::f16, Promote);
362     setOperationAction(ISD::FROUND,     MVT::f16, Promote);
363     setOperationAction(ISD::FROUNDEVEN, MVT::f16, Promote);
364     setOperationAction(ISD::FTRUNC,     MVT::f16, Promote);
365     setOperationAction(ISD::FPOW,       MVT::f16, Promote);
366     setOperationAction(ISD::FPOWI,      MVT::f16, Promote);
367     setOperationAction(ISD::FCOS,       MVT::f16, Promote);
368     setOperationAction(ISD::FSIN,       MVT::f16, Promote);
369     setOperationAction(ISD::FSINCOS,    MVT::f16, Promote);
370     setOperationAction(ISD::FEXP,       MVT::f16, Promote);
371     setOperationAction(ISD::FEXP2,      MVT::f16, Promote);
372     setOperationAction(ISD::FLOG,       MVT::f16, Promote);
373     setOperationAction(ISD::FLOG2,      MVT::f16, Promote);
374     setOperationAction(ISD::FLOG10,     MVT::f16, Promote);
375 
376     // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
377     // complete support for all operations in LegalizeDAG.
378 
379     // We need to custom promote this.
380     if (Subtarget.is64Bit())
381       setOperationAction(ISD::FPOWI, MVT::i32, Custom);
382   }
383 
384   if (Subtarget.hasStdExtF()) {
385     setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
386     setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
387     setOperationAction(ISD::LRINT, MVT::f32, Legal);
388     setOperationAction(ISD::LLRINT, MVT::f32, Legal);
389     setOperationAction(ISD::LROUND, MVT::f32, Legal);
390     setOperationAction(ISD::LLROUND, MVT::f32, Legal);
391     setOperationAction(ISD::STRICT_LRINT, MVT::f32, Legal);
392     setOperationAction(ISD::STRICT_LLRINT, MVT::f32, Legal);
393     setOperationAction(ISD::STRICT_LROUND, MVT::f32, Legal);
394     setOperationAction(ISD::STRICT_LLROUND, MVT::f32, Legal);
395     setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
396     setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal);
397     setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
398     setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
399     setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
400     setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
401     setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);
402     setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
403     for (auto CC : FPCCToExpand)
404       setCondCodeAction(CC, MVT::f32, Expand);
405     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
406     setOperationAction(ISD::SELECT, MVT::f32, Custom);
407     setOperationAction(ISD::BR_CC, MVT::f32, Expand);
408     for (auto Op : FPOpToExpand)
409       setOperationAction(Op, MVT::f32, Expand);
410     setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
411     setTruncStoreAction(MVT::f32, MVT::f16, Expand);
412   }
413 
414   if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
415     setOperationAction(ISD::BITCAST, MVT::i32, Custom);
416 
417   if (Subtarget.hasStdExtD()) {
418     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
419     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
420     setOperationAction(ISD::LRINT, MVT::f64, Legal);
421     setOperationAction(ISD::LLRINT, MVT::f64, Legal);
422     setOperationAction(ISD::LROUND, MVT::f64, Legal);
423     setOperationAction(ISD::LLROUND, MVT::f64, Legal);
424     setOperationAction(ISD::STRICT_LRINT, MVT::f64, Legal);
425     setOperationAction(ISD::STRICT_LLRINT, MVT::f64, Legal);
426     setOperationAction(ISD::STRICT_LROUND, MVT::f64, Legal);
427     setOperationAction(ISD::STRICT_LLROUND, MVT::f64, Legal);
428     setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal);
429     setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
430     setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
431     setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
432     setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
433     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
434     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
435     setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
436     setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal);
437     setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal);
438     for (auto CC : FPCCToExpand)
439       setCondCodeAction(CC, MVT::f64, Expand);
440     setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
441     setOperationAction(ISD::SELECT, MVT::f64, Custom);
442     setOperationAction(ISD::BR_CC, MVT::f64, Expand);
443     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
444     setTruncStoreAction(MVT::f64, MVT::f32, Expand);
445     for (auto Op : FPOpToExpand)
446       setOperationAction(Op, MVT::f64, Expand);
447     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
448     setTruncStoreAction(MVT::f64, MVT::f16, Expand);
449   }
450 
451   if (Subtarget.is64Bit()) {
452     setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
453     setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
454     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
455     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
456   }
457 
458   if (Subtarget.hasStdExtF()) {
459     setOperationAction(ISD::FP_TO_UINT_SAT, XLenVT, Custom);
460     setOperationAction(ISD::FP_TO_SINT_SAT, XLenVT, Custom);
461 
462     setOperationAction(ISD::STRICT_FP_TO_UINT, XLenVT, Legal);
463     setOperationAction(ISD::STRICT_FP_TO_SINT, XLenVT, Legal);
464     setOperationAction(ISD::STRICT_UINT_TO_FP, XLenVT, Legal);
465     setOperationAction(ISD::STRICT_SINT_TO_FP, XLenVT, Legal);
466 
467     setOperationAction(ISD::FLT_ROUNDS_, XLenVT, Custom);
468     setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
469   }
470 
471   setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
472   setOperationAction(ISD::BlockAddress, XLenVT, Custom);
473   setOperationAction(ISD::ConstantPool, XLenVT, Custom);
474   setOperationAction(ISD::JumpTable, XLenVT, Custom);
475 
476   setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
477 
478   // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
479   // Unfortunately this can't be determined just from the ISA naming string.
480   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
481                      Subtarget.is64Bit() ? Legal : Custom);
482 
483   setOperationAction(ISD::TRAP, MVT::Other, Legal);
484   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
485   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
486   if (Subtarget.is64Bit())
487     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
488 
489   if (Subtarget.hasStdExtA()) {
490     setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
491     setMinCmpXchgSizeInBits(32);
492   } else {
493     setMaxAtomicSizeInBitsSupported(0);
494   }
495 
496   setBooleanContents(ZeroOrOneBooleanContent);
497 
498   if (Subtarget.hasVInstructions()) {
499     setBooleanVectorContents(ZeroOrOneBooleanContent);
500 
501     setOperationAction(ISD::VSCALE, XLenVT, Custom);
502 
503     // RVV intrinsics may have illegal operands.
504     // We also need to custom legalize vmv.x.s.
505     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
506     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
507     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
508     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom);
509     if (Subtarget.is64Bit()) {
510       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);
511     } else {
512       setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
513       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
514     }
515 
516     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
517     setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
518 
519     static const unsigned IntegerVPOps[] = {
520         ISD::VP_ADD,         ISD::VP_SUB,         ISD::VP_MUL,
521         ISD::VP_SDIV,        ISD::VP_UDIV,        ISD::VP_SREM,
522         ISD::VP_UREM,        ISD::VP_AND,         ISD::VP_OR,
523         ISD::VP_XOR,         ISD::VP_ASHR,        ISD::VP_LSHR,
524         ISD::VP_SHL,         ISD::VP_REDUCE_ADD,  ISD::VP_REDUCE_AND,
525         ISD::VP_REDUCE_OR,   ISD::VP_REDUCE_XOR,  ISD::VP_REDUCE_SMAX,
526         ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
527         ISD::VP_MERGE,       ISD::VP_SELECT};
528 
529     static const unsigned FloatingPointVPOps[] = {
530         ISD::VP_FADD,        ISD::VP_FSUB,        ISD::VP_FMUL,
531         ISD::VP_FDIV,        ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
532         ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
533         ISD::VP_SELECT};
534 
535     if (!Subtarget.is64Bit()) {
536       // We must custom-lower certain vXi64 operations on RV32 due to the vector
537       // element type being illegal.
538       setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::i64, Custom);
539       setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::i64, Custom);
540 
541       setOperationAction(ISD::VECREDUCE_ADD, MVT::i64, Custom);
542       setOperationAction(ISD::VECREDUCE_AND, MVT::i64, Custom);
543       setOperationAction(ISD::VECREDUCE_OR, MVT::i64, Custom);
544       setOperationAction(ISD::VECREDUCE_XOR, MVT::i64, Custom);
545       setOperationAction(ISD::VECREDUCE_SMAX, MVT::i64, Custom);
546       setOperationAction(ISD::VECREDUCE_SMIN, MVT::i64, Custom);
547       setOperationAction(ISD::VECREDUCE_UMAX, MVT::i64, Custom);
548       setOperationAction(ISD::VECREDUCE_UMIN, MVT::i64, Custom);
549 
550       setOperationAction(ISD::VP_REDUCE_ADD, MVT::i64, Custom);
551       setOperationAction(ISD::VP_REDUCE_AND, MVT::i64, Custom);
552       setOperationAction(ISD::VP_REDUCE_OR, MVT::i64, Custom);
553       setOperationAction(ISD::VP_REDUCE_XOR, MVT::i64, Custom);
554       setOperationAction(ISD::VP_REDUCE_SMAX, MVT::i64, Custom);
555       setOperationAction(ISD::VP_REDUCE_SMIN, MVT::i64, Custom);
556       setOperationAction(ISD::VP_REDUCE_UMAX, MVT::i64, Custom);
557       setOperationAction(ISD::VP_REDUCE_UMIN, MVT::i64, Custom);
558     }
559 
560     for (MVT VT : BoolVecVTs) {
561       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
562 
563       // Mask VTs are custom-expanded into a series of standard nodes
564       setOperationAction(ISD::TRUNCATE, VT, Custom);
565       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
566       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
567       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
568 
569       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
570       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
571 
572       setOperationAction(ISD::SELECT, VT, Custom);
573       setOperationAction(ISD::SELECT_CC, VT, Expand);
574       setOperationAction(ISD::VSELECT, VT, Expand);
575       setOperationAction(ISD::VP_MERGE, VT, Expand);
576       setOperationAction(ISD::VP_SELECT, VT, Expand);
577 
578       setOperationAction(ISD::VP_AND, VT, Custom);
579       setOperationAction(ISD::VP_OR, VT, Custom);
580       setOperationAction(ISD::VP_XOR, VT, Custom);
581 
582       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
583       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
584       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
585 
586       setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
587       setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
588       setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
589 
590       // RVV has native int->float & float->int conversions where the
591       // element type sizes are within one power-of-two of each other. Any
592       // wider distances between type sizes have to be lowered as sequences
593       // which progressively narrow the gap in stages.
594       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
595       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
596       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
597       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
598 
599       // Expand all extending loads to types larger than this, and truncating
600       // stores from types larger than this.
601       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
602         setTruncStoreAction(OtherVT, VT, Expand);
603         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
604         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
605         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
606       }
607     }
608 
609     for (MVT VT : IntVecVTs) {
610       if (VT.getVectorElementType() == MVT::i64 &&
611           !Subtarget.hasVInstructionsI64())
612         continue;
613 
614       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
615       setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
616 
617       // Vectors implement MULHS/MULHU.
618       setOperationAction(ISD::SMUL_LOHI, VT, Expand);
619       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
620 
621       // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
622       if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV()) {
623         setOperationAction(ISD::MULHU, VT, Expand);
624         setOperationAction(ISD::MULHS, VT, Expand);
625       }
626 
627       setOperationAction(ISD::SMIN, VT, Legal);
628       setOperationAction(ISD::SMAX, VT, Legal);
629       setOperationAction(ISD::UMIN, VT, Legal);
630       setOperationAction(ISD::UMAX, VT, Legal);
631 
632       setOperationAction(ISD::ROTL, VT, Expand);
633       setOperationAction(ISD::ROTR, VT, Expand);
634 
635       setOperationAction(ISD::CTTZ, VT, Expand);
636       setOperationAction(ISD::CTLZ, VT, Expand);
637       setOperationAction(ISD::CTPOP, VT, Expand);
638 
639       setOperationAction(ISD::BSWAP, VT, Expand);
640 
641       // Custom-lower extensions and truncations from/to mask types.
642       setOperationAction(ISD::ANY_EXTEND, VT, Custom);
643       setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
644       setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
645 
646       // RVV has native int->float & float->int conversions where the
647       // element type sizes are within one power-of-two of each other. Any
648       // wider distances between type sizes have to be lowered as sequences
649       // which progressively narrow the gap in stages.
650       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
651       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
652       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
653       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
654 
655       setOperationAction(ISD::SADDSAT, VT, Legal);
656       setOperationAction(ISD::UADDSAT, VT, Legal);
657       setOperationAction(ISD::SSUBSAT, VT, Legal);
658       setOperationAction(ISD::USUBSAT, VT, Legal);
659 
660       // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
661       // nodes which truncate by one power of two at a time.
662       setOperationAction(ISD::TRUNCATE, VT, Custom);
663 
664       // Custom-lower insert/extract operations to simplify patterns.
665       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
666       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
667 
668       // Custom-lower reduction operations to set up the corresponding custom
669       // nodes' operands.
670       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
671       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
672       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
673       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
674       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
675       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
676       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
677       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
678 
679       for (unsigned VPOpc : IntegerVPOps)
680         setOperationAction(VPOpc, VT, Custom);
681 
682       setOperationAction(ISD::LOAD, VT, Custom);
683       setOperationAction(ISD::STORE, VT, Custom);
684 
685       setOperationAction(ISD::MLOAD, VT, Custom);
686       setOperationAction(ISD::MSTORE, VT, Custom);
687       setOperationAction(ISD::MGATHER, VT, Custom);
688       setOperationAction(ISD::MSCATTER, VT, Custom);
689 
690       setOperationAction(ISD::VP_LOAD, VT, Custom);
691       setOperationAction(ISD::VP_STORE, VT, Custom);
692       setOperationAction(ISD::VP_GATHER, VT, Custom);
693       setOperationAction(ISD::VP_SCATTER, VT, Custom);
694 
695       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
696       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
697       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
698 
699       setOperationAction(ISD::SELECT, VT, Custom);
700       setOperationAction(ISD::SELECT_CC, VT, Expand);
701 
702       setOperationAction(ISD::STEP_VECTOR, VT, Custom);
703       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
704 
705       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
706         setTruncStoreAction(VT, OtherVT, Expand);
707         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
708         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
709         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
710       }
711 
712       // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
713       // type that can represent the value exactly.
714       if (VT.getVectorElementType() != MVT::i64) {
715         MVT FloatEltVT =
716             VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
717         EVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
718         if (isTypeLegal(FloatVT)) {
719           setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
720           setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
721         }
722       }
723     }
724 
725     // Expand various CCs to best match the RVV ISA, which natively supports UNE
726     // but no other unordered comparisons, and supports all ordered comparisons
727     // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
728     // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
729     // and we pattern-match those back to the "original", swapping operands once
730     // more. This way we catch both operations and both "vf" and "fv" forms with
731     // fewer patterns.
732     static const ISD::CondCode VFPCCToExpand[] = {
733         ISD::SETO,   ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
734         ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
735         ISD::SETGT,  ISD::SETOGT, ISD::SETGE,  ISD::SETOGE,
736     };
737 
738     // Sets common operation actions on RVV floating-point vector types.
739     const auto SetCommonVFPActions = [&](MVT VT) {
740       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
741       // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
742       // sizes are within one power-of-two of each other. Therefore conversions
743       // between vXf16 and vXf64 must be lowered as sequences which convert via
744       // vXf32.
745       setOperationAction(ISD::FP_ROUND, VT, Custom);
746       setOperationAction(ISD::FP_EXTEND, VT, Custom);
747       // Custom-lower insert/extract operations to simplify patterns.
748       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
749       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
750       // Expand various condition codes (explained above).
751       for (auto CC : VFPCCToExpand)
752         setCondCodeAction(CC, VT, Expand);
753 
754       setOperationAction(ISD::FMINNUM, VT, Legal);
755       setOperationAction(ISD::FMAXNUM, VT, Legal);
756 
757       setOperationAction(ISD::FTRUNC, VT, Custom);
758       setOperationAction(ISD::FCEIL, VT, Custom);
759       setOperationAction(ISD::FFLOOR, VT, Custom);
760 
761       setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
762       setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
763       setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
764       setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
765 
766       setOperationAction(ISD::FCOPYSIGN, VT, Legal);
767 
768       setOperationAction(ISD::LOAD, VT, Custom);
769       setOperationAction(ISD::STORE, VT, Custom);
770 
771       setOperationAction(ISD::MLOAD, VT, Custom);
772       setOperationAction(ISD::MSTORE, VT, Custom);
773       setOperationAction(ISD::MGATHER, VT, Custom);
774       setOperationAction(ISD::MSCATTER, VT, Custom);
775 
776       setOperationAction(ISD::VP_LOAD, VT, Custom);
777       setOperationAction(ISD::VP_STORE, VT, Custom);
778       setOperationAction(ISD::VP_GATHER, VT, Custom);
779       setOperationAction(ISD::VP_SCATTER, VT, Custom);
780 
781       setOperationAction(ISD::SELECT, VT, Custom);
782       setOperationAction(ISD::SELECT_CC, VT, Expand);
783 
784       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
785       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
786       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
787 
788       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
789 
790       for (unsigned VPOpc : FloatingPointVPOps)
791         setOperationAction(VPOpc, VT, Custom);
792     };
793 
794     // Sets common extload/truncstore actions on RVV floating-point vector
795     // types.
796     const auto SetCommonVFPExtLoadTruncStoreActions =
797         [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
798           for (auto SmallVT : SmallerVTs) {
799             setTruncStoreAction(VT, SmallVT, Expand);
800             setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
801           }
802         };
803 
804     if (Subtarget.hasVInstructionsF16())
805       for (MVT VT : F16VecVTs)
806         SetCommonVFPActions(VT);
807 
808     for (MVT VT : F32VecVTs) {
809       if (Subtarget.hasVInstructionsF32())
810         SetCommonVFPActions(VT);
811       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
812     }
813 
814     for (MVT VT : F64VecVTs) {
815       if (Subtarget.hasVInstructionsF64())
816         SetCommonVFPActions(VT);
817       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
818       SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
819     }
820 
821     if (Subtarget.useRVVForFixedLengthVectors()) {
822       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
823         if (!useRVVForFixedLengthVectorVT(VT))
824           continue;
825 
826         // By default everything must be expanded.
827         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
828           setOperationAction(Op, VT, Expand);
829         for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
830           setTruncStoreAction(VT, OtherVT, Expand);
831           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
832           setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
833           setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
834         }
835 
836         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
837         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
838         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
839 
840         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
841         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
842 
843         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
844         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
845 
846         setOperationAction(ISD::LOAD, VT, Custom);
847         setOperationAction(ISD::STORE, VT, Custom);
848 
849         setOperationAction(ISD::SETCC, VT, Custom);
850 
851         setOperationAction(ISD::SELECT, VT, Custom);
852 
853         setOperationAction(ISD::TRUNCATE, VT, Custom);
854 
855         setOperationAction(ISD::BITCAST, VT, Custom);
856 
857         setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
858         setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
859         setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
860 
861         setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
862         setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
863         setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
864 
865         setOperationAction(ISD::SINT_TO_FP, VT, Custom);
866         setOperationAction(ISD::UINT_TO_FP, VT, Custom);
867         setOperationAction(ISD::FP_TO_SINT, VT, Custom);
868         setOperationAction(ISD::FP_TO_UINT, VT, Custom);
869 
870         // Operations below are different for between masks and other vectors.
871         if (VT.getVectorElementType() == MVT::i1) {
872           setOperationAction(ISD::VP_AND, VT, Custom);
873           setOperationAction(ISD::VP_OR, VT, Custom);
874           setOperationAction(ISD::VP_XOR, VT, Custom);
875           setOperationAction(ISD::AND, VT, Custom);
876           setOperationAction(ISD::OR, VT, Custom);
877           setOperationAction(ISD::XOR, VT, Custom);
878           continue;
879         }
880 
881         // Use SPLAT_VECTOR to prevent type legalization from destroying the
882         // splats when type legalizing i64 scalar on RV32.
883         // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
884         // improvements first.
885         if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
886           setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
887           setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
888         }
889 
890         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
891         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
892 
893         setOperationAction(ISD::MLOAD, VT, Custom);
894         setOperationAction(ISD::MSTORE, VT, Custom);
895         setOperationAction(ISD::MGATHER, VT, Custom);
896         setOperationAction(ISD::MSCATTER, VT, Custom);
897 
898         setOperationAction(ISD::VP_LOAD, VT, Custom);
899         setOperationAction(ISD::VP_STORE, VT, Custom);
900         setOperationAction(ISD::VP_GATHER, VT, Custom);
901         setOperationAction(ISD::VP_SCATTER, VT, Custom);
902 
903         setOperationAction(ISD::ADD, VT, Custom);
904         setOperationAction(ISD::MUL, VT, Custom);
905         setOperationAction(ISD::SUB, VT, Custom);
906         setOperationAction(ISD::AND, VT, Custom);
907         setOperationAction(ISD::OR, VT, Custom);
908         setOperationAction(ISD::XOR, VT, Custom);
909         setOperationAction(ISD::SDIV, VT, Custom);
910         setOperationAction(ISD::SREM, VT, Custom);
911         setOperationAction(ISD::UDIV, VT, Custom);
912         setOperationAction(ISD::UREM, VT, Custom);
913         setOperationAction(ISD::SHL, VT, Custom);
914         setOperationAction(ISD::SRA, VT, Custom);
915         setOperationAction(ISD::SRL, VT, Custom);
916 
917         setOperationAction(ISD::SMIN, VT, Custom);
918         setOperationAction(ISD::SMAX, VT, Custom);
919         setOperationAction(ISD::UMIN, VT, Custom);
920         setOperationAction(ISD::UMAX, VT, Custom);
921         setOperationAction(ISD::ABS,  VT, Custom);
922 
923         // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
924         if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV()) {
925           setOperationAction(ISD::MULHS, VT, Custom);
926           setOperationAction(ISD::MULHU, VT, Custom);
927         }
928 
929         setOperationAction(ISD::SADDSAT, VT, Custom);
930         setOperationAction(ISD::UADDSAT, VT, Custom);
931         setOperationAction(ISD::SSUBSAT, VT, Custom);
932         setOperationAction(ISD::USUBSAT, VT, Custom);
933 
934         setOperationAction(ISD::VSELECT, VT, Custom);
935         setOperationAction(ISD::SELECT_CC, VT, Expand);
936 
937         setOperationAction(ISD::ANY_EXTEND, VT, Custom);
938         setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
939         setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
940 
941         // Custom-lower reduction operations to set up the corresponding custom
942         // nodes' operands.
943         setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
944         setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
945         setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
946         setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
947         setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
948 
949         for (unsigned VPOpc : IntegerVPOps)
950           setOperationAction(VPOpc, VT, Custom);
951 
952         // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
953         // type that can represent the value exactly.
954         if (VT.getVectorElementType() != MVT::i64) {
955           MVT FloatEltVT =
956               VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
957           EVT FloatVT =
958               MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
959           if (isTypeLegal(FloatVT)) {
960             setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
961             setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
962           }
963         }
964       }
965 
966       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
967         if (!useRVVForFixedLengthVectorVT(VT))
968           continue;
969 
970         // By default everything must be expanded.
971         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
972           setOperationAction(Op, VT, Expand);
973         for (MVT OtherVT : MVT::fp_fixedlen_vector_valuetypes()) {
974           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
975           setTruncStoreAction(VT, OtherVT, Expand);
976         }
977 
978         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
979         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
980         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
981 
982         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
983         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
984         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
985         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
986         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
987 
988         setOperationAction(ISD::LOAD, VT, Custom);
989         setOperationAction(ISD::STORE, VT, Custom);
990         setOperationAction(ISD::MLOAD, VT, Custom);
991         setOperationAction(ISD::MSTORE, VT, Custom);
992         setOperationAction(ISD::MGATHER, VT, Custom);
993         setOperationAction(ISD::MSCATTER, VT, Custom);
994 
995         setOperationAction(ISD::VP_LOAD, VT, Custom);
996         setOperationAction(ISD::VP_STORE, VT, Custom);
997         setOperationAction(ISD::VP_GATHER, VT, Custom);
998         setOperationAction(ISD::VP_SCATTER, VT, Custom);
999 
1000         setOperationAction(ISD::FADD, VT, Custom);
1001         setOperationAction(ISD::FSUB, VT, Custom);
1002         setOperationAction(ISD::FMUL, VT, Custom);
1003         setOperationAction(ISD::FDIV, VT, Custom);
1004         setOperationAction(ISD::FNEG, VT, Custom);
1005         setOperationAction(ISD::FABS, VT, Custom);
1006         setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1007         setOperationAction(ISD::FSQRT, VT, Custom);
1008         setOperationAction(ISD::FMA, VT, Custom);
1009         setOperationAction(ISD::FMINNUM, VT, Custom);
1010         setOperationAction(ISD::FMAXNUM, VT, Custom);
1011 
1012         setOperationAction(ISD::FP_ROUND, VT, Custom);
1013         setOperationAction(ISD::FP_EXTEND, VT, Custom);
1014 
1015         setOperationAction(ISD::FTRUNC, VT, Custom);
1016         setOperationAction(ISD::FCEIL, VT, Custom);
1017         setOperationAction(ISD::FFLOOR, VT, Custom);
1018 
1019         for (auto CC : VFPCCToExpand)
1020           setCondCodeAction(CC, VT, Expand);
1021 
1022         setOperationAction(ISD::VSELECT, VT, Custom);
1023         setOperationAction(ISD::SELECT, VT, Custom);
1024         setOperationAction(ISD::SELECT_CC, VT, Expand);
1025 
1026         setOperationAction(ISD::BITCAST, VT, Custom);
1027 
1028         setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1029         setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1030         setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1031         setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1032 
1033         for (unsigned VPOpc : FloatingPointVPOps)
1034           setOperationAction(VPOpc, VT, Custom);
1035       }
1036 
1037       // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1038       setOperationAction(ISD::BITCAST, MVT::i8, Custom);
1039       setOperationAction(ISD::BITCAST, MVT::i16, Custom);
1040       setOperationAction(ISD::BITCAST, MVT::i32, Custom);
1041       setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1042       if (Subtarget.hasStdExtZfh())
1043         setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1044       if (Subtarget.hasStdExtF())
1045         setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1046       if (Subtarget.hasStdExtD())
1047         setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1048     }
1049   }
1050 
1051   // Function alignments.
1052   const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
1053   setMinFunctionAlignment(FunctionAlignment);
1054   setPrefFunctionAlignment(FunctionAlignment);
1055 
1056   setMinimumJumpTableEntries(5);
1057 
1058   // Jumps are expensive, compared to logic
1059   setJumpIsExpensive();
1060 
1061   setTargetDAGCombine(ISD::ADD);
1062   setTargetDAGCombine(ISD::SUB);
1063   setTargetDAGCombine(ISD::AND);
1064   setTargetDAGCombine(ISD::OR);
1065   setTargetDAGCombine(ISD::XOR);
1066   setTargetDAGCombine(ISD::ANY_EXTEND);
1067   if (Subtarget.hasStdExtF()) {
1068     setTargetDAGCombine(ISD::ZERO_EXTEND);
1069     setTargetDAGCombine(ISD::FP_TO_SINT);
1070     setTargetDAGCombine(ISD::FP_TO_UINT);
1071     setTargetDAGCombine(ISD::FP_TO_SINT_SAT);
1072     setTargetDAGCombine(ISD::FP_TO_UINT_SAT);
1073   }
1074   if (Subtarget.hasVInstructions()) {
1075     setTargetDAGCombine(ISD::FCOPYSIGN);
1076     setTargetDAGCombine(ISD::MGATHER);
1077     setTargetDAGCombine(ISD::MSCATTER);
1078     setTargetDAGCombine(ISD::VP_GATHER);
1079     setTargetDAGCombine(ISD::VP_SCATTER);
1080     setTargetDAGCombine(ISD::SRA);
1081     setTargetDAGCombine(ISD::SRL);
1082     setTargetDAGCombine(ISD::SHL);
1083     setTargetDAGCombine(ISD::STORE);
1084   }
1085 }
1086 
1087 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
1088                                             LLVMContext &Context,
1089                                             EVT VT) const {
1090   if (!VT.isVector())
1091     return getPointerTy(DL);
1092   if (Subtarget.hasVInstructions() &&
1093       (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
1094     return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
1095   return VT.changeVectorElementTypeToInteger();
1096 }
1097 
1098 MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
1099   return Subtarget.getXLenVT();
1100 }
1101 
1102 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1103                                              const CallInst &I,
1104                                              MachineFunction &MF,
1105                                              unsigned Intrinsic) const {
1106   auto &DL = I.getModule()->getDataLayout();
1107   switch (Intrinsic) {
1108   default:
1109     return false;
1110   case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1111   case Intrinsic::riscv_masked_atomicrmw_add_i32:
1112   case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1113   case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1114   case Intrinsic::riscv_masked_atomicrmw_max_i32:
1115   case Intrinsic::riscv_masked_atomicrmw_min_i32:
1116   case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1117   case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1118   case Intrinsic::riscv_masked_cmpxchg_i32: {
1119     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
1120     Info.opc = ISD::INTRINSIC_W_CHAIN;
1121     Info.memVT = MVT::getVT(PtrTy->getPointerElementType());
1122     Info.ptrVal = I.getArgOperand(0);
1123     Info.offset = 0;
1124     Info.align = Align(4);
1125     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1126                  MachineMemOperand::MOVolatile;
1127     return true;
1128   }
1129   case Intrinsic::riscv_masked_strided_load:
1130     Info.opc = ISD::INTRINSIC_W_CHAIN;
1131     Info.ptrVal = I.getArgOperand(1);
1132     Info.memVT = getValueType(DL, I.getType()->getScalarType());
1133     Info.align = Align(DL.getTypeSizeInBits(I.getType()->getScalarType()) / 8);
1134     Info.size = MemoryLocation::UnknownSize;
1135     Info.flags |= MachineMemOperand::MOLoad;
1136     return true;
1137   case Intrinsic::riscv_masked_strided_store:
1138     Info.opc = ISD::INTRINSIC_VOID;
1139     Info.ptrVal = I.getArgOperand(1);
1140     Info.memVT =
1141         getValueType(DL, I.getArgOperand(0)->getType()->getScalarType());
1142     Info.align = Align(
1143         DL.getTypeSizeInBits(I.getArgOperand(0)->getType()->getScalarType()) /
1144         8);
1145     Info.size = MemoryLocation::UnknownSize;
1146     Info.flags |= MachineMemOperand::MOStore;
1147     return true;
1148   }
1149 }
1150 
1151 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1152                                                 const AddrMode &AM, Type *Ty,
1153                                                 unsigned AS,
1154                                                 Instruction *I) const {
1155   // No global is ever allowed as a base.
1156   if (AM.BaseGV)
1157     return false;
1158 
1159   // Require a 12-bit signed offset.
1160   if (!isInt<12>(AM.BaseOffs))
1161     return false;
1162 
1163   switch (AM.Scale) {
1164   case 0: // "r+i" or just "i", depending on HasBaseReg.
1165     break;
1166   case 1:
1167     if (!AM.HasBaseReg) // allow "r+i".
1168       break;
1169     return false; // disallow "r+r" or "r+r+i".
1170   default:
1171     return false;
1172   }
1173 
1174   return true;
1175 }
1176 
1177 bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1178   return isInt<12>(Imm);
1179 }
1180 
1181 bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1182   return isInt<12>(Imm);
1183 }
1184 
1185 // On RV32, 64-bit integers are split into their high and low parts and held
1186 // in two different registers, so the trunc is free since the low register can
1187 // just be used.
1188 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
1189   if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
1190     return false;
1191   unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1192   unsigned DestBits = DstTy->getPrimitiveSizeInBits();
1193   return (SrcBits == 64 && DestBits == 32);
1194 }
1195 
1196 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
1197   if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
1198       !SrcVT.isInteger() || !DstVT.isInteger())
1199     return false;
1200   unsigned SrcBits = SrcVT.getSizeInBits();
1201   unsigned DestBits = DstVT.getSizeInBits();
1202   return (SrcBits == 64 && DestBits == 32);
1203 }
1204 
1205 bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
1206   // Zexts are free if they can be combined with a load.
1207   // Don't advertise i32->i64 zextload as being free for RV64. It interacts
1208   // poorly with type legalization of compares preferring sext.
1209   if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
1210     EVT MemVT = LD->getMemoryVT();
1211     if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1212         (LD->getExtensionType() == ISD::NON_EXTLOAD ||
1213          LD->getExtensionType() == ISD::ZEXTLOAD))
1214       return true;
1215   }
1216 
1217   return TargetLowering::isZExtFree(Val, VT2);
1218 }
1219 
1220 bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
1221   return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1222 }
1223 
1224 bool RISCVTargetLowering::isCheapToSpeculateCttz() const {
1225   return Subtarget.hasStdExtZbb();
1226 }
1227 
1228 bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
1229   return Subtarget.hasStdExtZbb();
1230 }
1231 
1232 bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
1233   EVT VT = Y.getValueType();
1234 
1235   // FIXME: Support vectors once we have tests.
1236   if (VT.isVector())
1237     return false;
1238 
1239   return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp() ||
1240           Subtarget.hasStdExtZbkb()) &&
1241          !isa<ConstantSDNode>(Y);
1242 }
1243 
1244 /// Check if sinking \p I's operands to I's basic block is profitable, because
1245 /// the operands can be folded into a target instruction, e.g.
1246 /// splats of scalars can fold into vector instructions.
1247 bool RISCVTargetLowering::shouldSinkOperands(
1248     Instruction *I, SmallVectorImpl<Use *> &Ops) const {
1249   using namespace llvm::PatternMatch;
1250 
1251   if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
1252     return false;
1253 
1254   auto IsSinker = [&](Instruction *I, int Operand) {
1255     switch (I->getOpcode()) {
1256     case Instruction::Add:
1257     case Instruction::Sub:
1258     case Instruction::Mul:
1259     case Instruction::And:
1260     case Instruction::Or:
1261     case Instruction::Xor:
1262     case Instruction::FAdd:
1263     case Instruction::FSub:
1264     case Instruction::FMul:
1265     case Instruction::FDiv:
1266     case Instruction::ICmp:
1267     case Instruction::FCmp:
1268       return true;
1269     case Instruction::Shl:
1270     case Instruction::LShr:
1271     case Instruction::AShr:
1272     case Instruction::UDiv:
1273     case Instruction::SDiv:
1274     case Instruction::URem:
1275     case Instruction::SRem:
1276       return Operand == 1;
1277     case Instruction::Call:
1278       if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1279         switch (II->getIntrinsicID()) {
1280         case Intrinsic::fma:
1281           return Operand == 0 || Operand == 1;
1282         // FIXME: Our patterns can only match vx/vf instructions when the splat
1283         // it on the RHS, because TableGen doesn't recognize our VP operations
1284         // as commutative.
1285         case Intrinsic::vp_add:
1286         case Intrinsic::vp_mul:
1287         case Intrinsic::vp_and:
1288         case Intrinsic::vp_or:
1289         case Intrinsic::vp_xor:
1290         case Intrinsic::vp_fadd:
1291         case Intrinsic::vp_fmul:
1292         case Intrinsic::vp_shl:
1293         case Intrinsic::vp_lshr:
1294         case Intrinsic::vp_ashr:
1295         case Intrinsic::vp_udiv:
1296         case Intrinsic::vp_sdiv:
1297         case Intrinsic::vp_urem:
1298         case Intrinsic::vp_srem:
1299           return Operand == 1;
1300         // ... with the exception of vp.sub/vp.fsub/vp.fdiv, which have
1301         // explicit patterns for both LHS and RHS (as 'vr' versions).
1302         case Intrinsic::vp_sub:
1303         case Intrinsic::vp_fsub:
1304         case Intrinsic::vp_fdiv:
1305           return Operand == 0 || Operand == 1;
1306         default:
1307           return false;
1308         }
1309       }
1310       return false;
1311     default:
1312       return false;
1313     }
1314   };
1315 
1316   for (auto OpIdx : enumerate(I->operands())) {
1317     if (!IsSinker(I, OpIdx.index()))
1318       continue;
1319 
1320     Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
1321     // Make sure we are not already sinking this operand
1322     if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
1323       continue;
1324 
1325     // We are looking for a splat that can be sunk.
1326     if (!match(Op, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
1327                              m_Undef(), m_ZeroMask())))
1328       continue;
1329 
1330     // All uses of the shuffle should be sunk to avoid duplicating it across gpr
1331     // and vector registers
1332     for (Use &U : Op->uses()) {
1333       Instruction *Insn = cast<Instruction>(U.getUser());
1334       if (!IsSinker(Insn, U.getOperandNo()))
1335         return false;
1336     }
1337 
1338     Ops.push_back(&Op->getOperandUse(0));
1339     Ops.push_back(&OpIdx.value());
1340   }
1341   return true;
1342 }
1343 
1344 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1345                                        bool ForCodeSize) const {
1346   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1347   if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
1348     return false;
1349   if (VT == MVT::f32 && !Subtarget.hasStdExtF())
1350     return false;
1351   if (VT == MVT::f64 && !Subtarget.hasStdExtD())
1352     return false;
1353   return Imm.isZero();
1354 }
1355 
1356 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
1357   return (VT == MVT::f16 && Subtarget.hasStdExtZfh()) ||
1358          (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
1359          (VT == MVT::f64 && Subtarget.hasStdExtD());
1360 }
1361 
1362 MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1363                                                       CallingConv::ID CC,
1364                                                       EVT VT) const {
1365   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1366   // We might still end up using a GPR but that will be decided based on ABI.
1367   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1368   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1369     return MVT::f32;
1370 
1371   return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1372 }
1373 
1374 unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1375                                                            CallingConv::ID CC,
1376                                                            EVT VT) const {
1377   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1378   // We might still end up using a GPR but that will be decided based on ABI.
1379   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1380   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1381     return 1;
1382 
1383   return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1384 }
1385 
1386 // Changes the condition code and swaps operands if necessary, so the SetCC
1387 // operation matches one of the comparisons supported directly by branches
1388 // in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
1389 // with 1/-1.
1390 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1391                                     ISD::CondCode &CC, SelectionDAG &DAG) {
1392   // Convert X > -1 to X >= 0.
1393   if (CC == ISD::SETGT && isAllOnesConstant(RHS)) {
1394     RHS = DAG.getConstant(0, DL, RHS.getValueType());
1395     CC = ISD::SETGE;
1396     return;
1397   }
1398   // Convert X < 1 to 0 >= X.
1399   if (CC == ISD::SETLT && isOneConstant(RHS)) {
1400     RHS = LHS;
1401     LHS = DAG.getConstant(0, DL, RHS.getValueType());
1402     CC = ISD::SETGE;
1403     return;
1404   }
1405 
1406   switch (CC) {
1407   default:
1408     break;
1409   case ISD::SETGT:
1410   case ISD::SETLE:
1411   case ISD::SETUGT:
1412   case ISD::SETULE:
1413     CC = ISD::getSetCCSwappedOperands(CC);
1414     std::swap(LHS, RHS);
1415     break;
1416   }
1417 }
1418 
1419 RISCVII::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
1420   assert(VT.isScalableVector() && "Expecting a scalable vector type");
1421   unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
1422   if (VT.getVectorElementType() == MVT::i1)
1423     KnownSize *= 8;
1424 
1425   switch (KnownSize) {
1426   default:
1427     llvm_unreachable("Invalid LMUL.");
1428   case 8:
1429     return RISCVII::VLMUL::LMUL_F8;
1430   case 16:
1431     return RISCVII::VLMUL::LMUL_F4;
1432   case 32:
1433     return RISCVII::VLMUL::LMUL_F2;
1434   case 64:
1435     return RISCVII::VLMUL::LMUL_1;
1436   case 128:
1437     return RISCVII::VLMUL::LMUL_2;
1438   case 256:
1439     return RISCVII::VLMUL::LMUL_4;
1440   case 512:
1441     return RISCVII::VLMUL::LMUL_8;
1442   }
1443 }
1444 
1445 unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVII::VLMUL LMul) {
1446   switch (LMul) {
1447   default:
1448     llvm_unreachable("Invalid LMUL.");
1449   case RISCVII::VLMUL::LMUL_F8:
1450   case RISCVII::VLMUL::LMUL_F4:
1451   case RISCVII::VLMUL::LMUL_F2:
1452   case RISCVII::VLMUL::LMUL_1:
1453     return RISCV::VRRegClassID;
1454   case RISCVII::VLMUL::LMUL_2:
1455     return RISCV::VRM2RegClassID;
1456   case RISCVII::VLMUL::LMUL_4:
1457     return RISCV::VRM4RegClassID;
1458   case RISCVII::VLMUL::LMUL_8:
1459     return RISCV::VRM8RegClassID;
1460   }
1461 }
1462 
1463 unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
1464   RISCVII::VLMUL LMUL = getLMUL(VT);
1465   if (LMUL == RISCVII::VLMUL::LMUL_F8 ||
1466       LMUL == RISCVII::VLMUL::LMUL_F4 ||
1467       LMUL == RISCVII::VLMUL::LMUL_F2 ||
1468       LMUL == RISCVII::VLMUL::LMUL_1) {
1469     static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
1470                   "Unexpected subreg numbering");
1471     return RISCV::sub_vrm1_0 + Index;
1472   }
1473   if (LMUL == RISCVII::VLMUL::LMUL_2) {
1474     static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
1475                   "Unexpected subreg numbering");
1476     return RISCV::sub_vrm2_0 + Index;
1477   }
1478   if (LMUL == RISCVII::VLMUL::LMUL_4) {
1479     static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
1480                   "Unexpected subreg numbering");
1481     return RISCV::sub_vrm4_0 + Index;
1482   }
1483   llvm_unreachable("Invalid vector type.");
1484 }
1485 
1486 unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
1487   if (VT.getVectorElementType() == MVT::i1)
1488     return RISCV::VRRegClassID;
1489   return getRegClassIDForLMUL(getLMUL(VT));
1490 }
1491 
1492 // Attempt to decompose a subvector insert/extract between VecVT and
1493 // SubVecVT via subregister indices. Returns the subregister index that
1494 // can perform the subvector insert/extract with the given element index, as
1495 // well as the index corresponding to any leftover subvectors that must be
1496 // further inserted/extracted within the register class for SubVecVT.
1497 std::pair<unsigned, unsigned>
1498 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1499     MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
1500     const RISCVRegisterInfo *TRI) {
1501   static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
1502                  RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
1503                  RISCV::VRM2RegClassID > RISCV::VRRegClassID),
1504                 "Register classes not ordered");
1505   unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
1506   unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
1507   // Try to compose a subregister index that takes us from the incoming
1508   // LMUL>1 register class down to the outgoing one. At each step we half
1509   // the LMUL:
1510   //   nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
1511   // Note that this is not guaranteed to find a subregister index, such as
1512   // when we are extracting from one VR type to another.
1513   unsigned SubRegIdx = RISCV::NoSubRegister;
1514   for (const unsigned RCID :
1515        {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
1516     if (VecRegClassID > RCID && SubRegClassID <= RCID) {
1517       VecVT = VecVT.getHalfNumVectorElementsVT();
1518       bool IsHi =
1519           InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
1520       SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
1521                                             getSubregIndexByMVT(VecVT, IsHi));
1522       if (IsHi)
1523         InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
1524     }
1525   return {SubRegIdx, InsertExtractIdx};
1526 }
1527 
1528 // Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
1529 // stores for those types.
1530 bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
1531   return !Subtarget.useRVVForFixedLengthVectors() ||
1532          (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
1533 }
1534 
1535 bool RISCVTargetLowering::isLegalElementTypeForRVV(Type *ScalarTy) const {
1536   if (ScalarTy->isPointerTy())
1537     return true;
1538 
1539   if (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
1540       ScalarTy->isIntegerTy(32))
1541     return true;
1542 
1543   if (ScalarTy->isIntegerTy(64))
1544     return Subtarget.hasVInstructionsI64();
1545 
1546   if (ScalarTy->isHalfTy())
1547     return Subtarget.hasVInstructionsF16();
1548   if (ScalarTy->isFloatTy())
1549     return Subtarget.hasVInstructionsF32();
1550   if (ScalarTy->isDoubleTy())
1551     return Subtarget.hasVInstructionsF64();
1552 
1553   return false;
1554 }
1555 
1556 static SDValue getVLOperand(SDValue Op) {
1557   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
1558           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
1559          "Unexpected opcode");
1560   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
1561   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
1562   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
1563       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
1564   if (!II)
1565     return SDValue();
1566   return Op.getOperand(II->VLOperand + 1 + HasChain);
1567 }
1568 
1569 static bool useRVVForFixedLengthVectorVT(MVT VT,
1570                                          const RISCVSubtarget &Subtarget) {
1571   assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
1572   if (!Subtarget.useRVVForFixedLengthVectors())
1573     return false;
1574 
1575   // We only support a set of vector types with a consistent maximum fixed size
1576   // across all supported vector element types to avoid legalization issues.
1577   // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
1578   // fixed-length vector type we support is 1024 bytes.
1579   if (VT.getFixedSizeInBits() > 1024 * 8)
1580     return false;
1581 
1582   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1583 
1584   MVT EltVT = VT.getVectorElementType();
1585 
1586   // Don't use RVV for vectors we cannot scalarize if required.
1587   switch (EltVT.SimpleTy) {
1588   // i1 is supported but has different rules.
1589   default:
1590     return false;
1591   case MVT::i1:
1592     // Masks can only use a single register.
1593     if (VT.getVectorNumElements() > MinVLen)
1594       return false;
1595     MinVLen /= 8;
1596     break;
1597   case MVT::i8:
1598   case MVT::i16:
1599   case MVT::i32:
1600     break;
1601   case MVT::i64:
1602     if (!Subtarget.hasVInstructionsI64())
1603       return false;
1604     break;
1605   case MVT::f16:
1606     if (!Subtarget.hasVInstructionsF16())
1607       return false;
1608     break;
1609   case MVT::f32:
1610     if (!Subtarget.hasVInstructionsF32())
1611       return false;
1612     break;
1613   case MVT::f64:
1614     if (!Subtarget.hasVInstructionsF64())
1615       return false;
1616     break;
1617   }
1618 
1619   // Reject elements larger than ELEN.
1620   if (EltVT.getSizeInBits() > Subtarget.getMaxELENForFixedLengthVectors())
1621     return false;
1622 
1623   unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
1624   // Don't use RVV for types that don't fit.
1625   if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
1626     return false;
1627 
1628   // TODO: Perhaps an artificial restriction, but worth having whilst getting
1629   // the base fixed length RVV support in place.
1630   if (!VT.isPow2VectorType())
1631     return false;
1632 
1633   return true;
1634 }
1635 
1636 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
1637   return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
1638 }
1639 
1640 // Return the largest legal scalable vector type that matches VT's element type.
1641 static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
1642                                             const RISCVSubtarget &Subtarget) {
1643   // This may be called before legal types are setup.
1644   assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
1645           useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
1646          "Expected legal fixed length vector!");
1647 
1648   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1649   unsigned MaxELen = Subtarget.getMaxELENForFixedLengthVectors();
1650 
1651   MVT EltVT = VT.getVectorElementType();
1652   switch (EltVT.SimpleTy) {
1653   default:
1654     llvm_unreachable("unexpected element type for RVV container");
1655   case MVT::i1:
1656   case MVT::i8:
1657   case MVT::i16:
1658   case MVT::i32:
1659   case MVT::i64:
1660   case MVT::f16:
1661   case MVT::f32:
1662   case MVT::f64: {
1663     // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
1664     // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
1665     // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
1666     unsigned NumElts =
1667         (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
1668     NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
1669     assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
1670     return MVT::getScalableVectorVT(EltVT, NumElts);
1671   }
1672   }
1673 }
1674 
1675 static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
1676                                             const RISCVSubtarget &Subtarget) {
1677   return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
1678                                           Subtarget);
1679 }
1680 
1681 MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
1682   return ::getContainerForFixedLengthVector(*this, VT, getSubtarget());
1683 }
1684 
1685 // Grow V to consume an entire RVV register.
1686 static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1687                                        const RISCVSubtarget &Subtarget) {
1688   assert(VT.isScalableVector() &&
1689          "Expected to convert into a scalable vector!");
1690   assert(V.getValueType().isFixedLengthVector() &&
1691          "Expected a fixed length vector operand!");
1692   SDLoc DL(V);
1693   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1694   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
1695 }
1696 
1697 // Shrink V so it's just big enough to maintain a VT's worth of data.
1698 static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1699                                          const RISCVSubtarget &Subtarget) {
1700   assert(VT.isFixedLengthVector() &&
1701          "Expected to convert into a fixed length vector!");
1702   assert(V.getValueType().isScalableVector() &&
1703          "Expected a scalable vector operand!");
1704   SDLoc DL(V);
1705   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1706   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
1707 }
1708 
1709 // Gets the two common "VL" operands: an all-ones mask and the vector length.
1710 // VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
1711 // the vector type that it is contained in.
1712 static std::pair<SDValue, SDValue>
1713 getDefaultVLOps(MVT VecVT, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
1714                 const RISCVSubtarget &Subtarget) {
1715   assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
1716   MVT XLenVT = Subtarget.getXLenVT();
1717   SDValue VL = VecVT.isFixedLengthVector()
1718                    ? DAG.getConstant(VecVT.getVectorNumElements(), DL, XLenVT)
1719                    : DAG.getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT);
1720   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
1721   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
1722   return {Mask, VL};
1723 }
1724 
1725 // As above but assuming the given type is a scalable vector type.
1726 static std::pair<SDValue, SDValue>
1727 getDefaultScalableVLOps(MVT VecVT, SDLoc DL, SelectionDAG &DAG,
1728                         const RISCVSubtarget &Subtarget) {
1729   assert(VecVT.isScalableVector() && "Expecting a scalable vector");
1730   return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
1731 }
1732 
1733 // The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
1734 // of either is (currently) supported. This can get us into an infinite loop
1735 // where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
1736 // as a ..., etc.
1737 // Until either (or both) of these can reliably lower any node, reporting that
1738 // we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
1739 // the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
1740 // which is not desirable.
1741 bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
1742     EVT VT, unsigned DefinedValues) const {
1743   return false;
1744 }
1745 
1746 bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
1747   // Only splats are currently supported.
1748   if (ShuffleVectorSDNode::isSplatMask(M.data(), VT))
1749     return true;
1750 
1751   return false;
1752 }
1753 
1754 static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
1755                                   const RISCVSubtarget &Subtarget) {
1756   // RISCV FP-to-int conversions saturate to the destination register size, but
1757   // don't produce 0 for nan. We can use a conversion instruction and fix the
1758   // nan case with a compare and a select.
1759   SDValue Src = Op.getOperand(0);
1760 
1761   EVT DstVT = Op.getValueType();
1762   EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1763 
1764   bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
1765   unsigned Opc;
1766   if (SatVT == DstVT)
1767     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
1768   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
1769     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
1770   else
1771     return SDValue();
1772   // FIXME: Support other SatVTs by clamping before or after the conversion.
1773 
1774   SDLoc DL(Op);
1775   SDValue FpToInt = DAG.getNode(
1776       Opc, DL, DstVT, Src,
1777       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
1778 
1779   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
1780   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
1781 }
1782 
1783 // Expand vector FTRUNC, FCEIL, and FFLOOR by converting to the integer domain
1784 // and back. Taking care to avoid converting values that are nan or already
1785 // correct.
1786 // TODO: Floor and ceil could be shorter by changing rounding mode, but we don't
1787 // have FRM dependencies modeled yet.
1788 static SDValue lowerFTRUNC_FCEIL_FFLOOR(SDValue Op, SelectionDAG &DAG) {
1789   MVT VT = Op.getSimpleValueType();
1790   assert(VT.isVector() && "Unexpected type");
1791 
1792   SDLoc DL(Op);
1793 
1794   // Freeze the source since we are increasing the number of uses.
1795   SDValue Src = DAG.getNode(ISD::FREEZE, DL, VT, Op.getOperand(0));
1796 
1797   // Truncate to integer and convert back to FP.
1798   MVT IntVT = VT.changeVectorElementTypeToInteger();
1799   SDValue Truncated = DAG.getNode(ISD::FP_TO_SINT, DL, IntVT, Src);
1800   Truncated = DAG.getNode(ISD::SINT_TO_FP, DL, VT, Truncated);
1801 
1802   MVT SetccVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
1803 
1804   if (Op.getOpcode() == ISD::FCEIL) {
1805     // If the truncated value is the greater than or equal to the original
1806     // value, we've computed the ceil. Otherwise, we went the wrong way and
1807     // need to increase by 1.
1808     // FIXME: This should use a masked operation. Handle here or in isel?
1809     SDValue Adjust = DAG.getNode(ISD::FADD, DL, VT, Truncated,
1810                                  DAG.getConstantFP(1.0, DL, VT));
1811     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOLT);
1812     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1813   } else if (Op.getOpcode() == ISD::FFLOOR) {
1814     // If the truncated value is the less than or equal to the original value,
1815     // we've computed the floor. Otherwise, we went the wrong way and need to
1816     // decrease by 1.
1817     // FIXME: This should use a masked operation. Handle here or in isel?
1818     SDValue Adjust = DAG.getNode(ISD::FSUB, DL, VT, Truncated,
1819                                  DAG.getConstantFP(1.0, DL, VT));
1820     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOGT);
1821     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1822   }
1823 
1824   // Restore the original sign so that -0.0 is preserved.
1825   Truncated = DAG.getNode(ISD::FCOPYSIGN, DL, VT, Truncated, Src);
1826 
1827   // Determine the largest integer that can be represented exactly. This and
1828   // values larger than it don't have any fractional bits so don't need to
1829   // be converted.
1830   const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
1831   unsigned Precision = APFloat::semanticsPrecision(FltSem);
1832   APFloat MaxVal = APFloat(FltSem);
1833   MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
1834                           /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
1835   SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
1836 
1837   // If abs(Src) was larger than MaxVal or nan, keep it.
1838   SDValue Abs = DAG.getNode(ISD::FABS, DL, VT, Src);
1839   SDValue Setcc = DAG.getSetCC(DL, SetccVT, Abs, MaxValNode, ISD::SETOLT);
1840   return DAG.getSelect(DL, VT, Setcc, Truncated, Src);
1841 }
1842 
1843 static SDValue lowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG,
1844                                  const RISCVSubtarget &Subtarget) {
1845   MVT VT = Op.getSimpleValueType();
1846   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1847 
1848   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1849 
1850   SDLoc DL(Op);
1851   SDValue Mask, VL;
1852   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1853 
1854   unsigned Opc =
1855       VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
1856   SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, Op.getOperand(0), VL);
1857   return convertFromScalableVector(VT, Splat, DAG, Subtarget);
1858 }
1859 
1860 struct VIDSequence {
1861   int64_t StepNumerator;
1862   unsigned StepDenominator;
1863   int64_t Addend;
1864 };
1865 
1866 // Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
1867 // to the (non-zero) step S and start value X. This can be then lowered as the
1868 // RVV sequence (VID * S) + X, for example.
1869 // The step S is represented as an integer numerator divided by a positive
1870 // denominator. Note that the implementation currently only identifies
1871 // sequences in which either the numerator is +/- 1 or the denominator is 1. It
1872 // cannot detect 2/3, for example.
1873 // Note that this method will also match potentially unappealing index
1874 // sequences, like <i32 0, i32 50939494>, however it is left to the caller to
1875 // determine whether this is worth generating code for.
1876 static Optional<VIDSequence> isSimpleVIDSequence(SDValue Op) {
1877   unsigned NumElts = Op.getNumOperands();
1878   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
1879   if (!Op.getValueType().isInteger())
1880     return None;
1881 
1882   Optional<unsigned> SeqStepDenom;
1883   Optional<int64_t> SeqStepNum, SeqAddend;
1884   Optional<std::pair<uint64_t, unsigned>> PrevElt;
1885   unsigned EltSizeInBits = Op.getValueType().getScalarSizeInBits();
1886   for (unsigned Idx = 0; Idx < NumElts; Idx++) {
1887     // Assume undef elements match the sequence; we just have to be careful
1888     // when interpolating across them.
1889     if (Op.getOperand(Idx).isUndef())
1890       continue;
1891     // The BUILD_VECTOR must be all constants.
1892     if (!isa<ConstantSDNode>(Op.getOperand(Idx)))
1893       return None;
1894 
1895     uint64_t Val = Op.getConstantOperandVal(Idx) &
1896                    maskTrailingOnes<uint64_t>(EltSizeInBits);
1897 
1898     if (PrevElt) {
1899       // Calculate the step since the last non-undef element, and ensure
1900       // it's consistent across the entire sequence.
1901       unsigned IdxDiff = Idx - PrevElt->second;
1902       int64_t ValDiff = SignExtend64(Val - PrevElt->first, EltSizeInBits);
1903 
1904       // A zero-value value difference means that we're somewhere in the middle
1905       // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
1906       // step change before evaluating the sequence.
1907       if (ValDiff != 0) {
1908         int64_t Remainder = ValDiff % IdxDiff;
1909         // Normalize the step if it's greater than 1.
1910         if (Remainder != ValDiff) {
1911           // The difference must cleanly divide the element span.
1912           if (Remainder != 0)
1913             return None;
1914           ValDiff /= IdxDiff;
1915           IdxDiff = 1;
1916         }
1917 
1918         if (!SeqStepNum)
1919           SeqStepNum = ValDiff;
1920         else if (ValDiff != SeqStepNum)
1921           return None;
1922 
1923         if (!SeqStepDenom)
1924           SeqStepDenom = IdxDiff;
1925         else if (IdxDiff != *SeqStepDenom)
1926           return None;
1927       }
1928     }
1929 
1930     // Record and/or check any addend.
1931     if (SeqStepNum && SeqStepDenom) {
1932       uint64_t ExpectedVal =
1933           (int64_t)(Idx * (uint64_t)*SeqStepNum) / *SeqStepDenom;
1934       int64_t Addend = SignExtend64(Val - ExpectedVal, EltSizeInBits);
1935       if (!SeqAddend)
1936         SeqAddend = Addend;
1937       else if (SeqAddend != Addend)
1938         return None;
1939     }
1940 
1941     // Record this non-undef element for later.
1942     if (!PrevElt || PrevElt->first != Val)
1943       PrevElt = std::make_pair(Val, Idx);
1944   }
1945   // We need to have logged both a step and an addend for this to count as
1946   // a legal index sequence.
1947   if (!SeqStepNum || !SeqStepDenom || !SeqAddend)
1948     return None;
1949 
1950   return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
1951 }
1952 
1953 static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
1954                                  const RISCVSubtarget &Subtarget) {
1955   MVT VT = Op.getSimpleValueType();
1956   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1957 
1958   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1959 
1960   SDLoc DL(Op);
1961   SDValue Mask, VL;
1962   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1963 
1964   MVT XLenVT = Subtarget.getXLenVT();
1965   unsigned NumElts = Op.getNumOperands();
1966 
1967   if (VT.getVectorElementType() == MVT::i1) {
1968     if (ISD::isBuildVectorAllZeros(Op.getNode())) {
1969       SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
1970       return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
1971     }
1972 
1973     if (ISD::isBuildVectorAllOnes(Op.getNode())) {
1974       SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
1975       return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
1976     }
1977 
1978     // Lower constant mask BUILD_VECTORs via an integer vector type, in
1979     // scalar integer chunks whose bit-width depends on the number of mask
1980     // bits and XLEN.
1981     // First, determine the most appropriate scalar integer type to use. This
1982     // is at most XLenVT, but may be shrunk to a smaller vector element type
1983     // according to the size of the final vector - use i8 chunks rather than
1984     // XLenVT if we're producing a v8i1. This results in more consistent
1985     // codegen across RV32 and RV64.
1986     unsigned NumViaIntegerBits =
1987         std::min(std::max(NumElts, 8u), Subtarget.getXLen());
1988     NumViaIntegerBits = std::min(NumViaIntegerBits,
1989                                  Subtarget.getMaxELENForFixedLengthVectors());
1990     if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
1991       // If we have to use more than one INSERT_VECTOR_ELT then this
1992       // optimization is likely to increase code size; avoid peforming it in
1993       // such a case. We can use a load from a constant pool in this case.
1994       if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
1995         return SDValue();
1996       // Now we can create our integer vector type. Note that it may be larger
1997       // than the resulting mask type: v4i1 would use v1i8 as its integer type.
1998       MVT IntegerViaVecVT =
1999           MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
2000                            divideCeil(NumElts, NumViaIntegerBits));
2001 
2002       uint64_t Bits = 0;
2003       unsigned BitPos = 0, IntegerEltIdx = 0;
2004       SDValue Vec = DAG.getUNDEF(IntegerViaVecVT);
2005 
2006       for (unsigned I = 0; I < NumElts; I++, BitPos++) {
2007         // Once we accumulate enough bits to fill our scalar type, insert into
2008         // our vector and clear our accumulated data.
2009         if (I != 0 && I % NumViaIntegerBits == 0) {
2010           if (NumViaIntegerBits <= 32)
2011             Bits = SignExtend64(Bits, 32);
2012           SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2013           Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec,
2014                             Elt, DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2015           Bits = 0;
2016           BitPos = 0;
2017           IntegerEltIdx++;
2018         }
2019         SDValue V = Op.getOperand(I);
2020         bool BitValue = !V.isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
2021         Bits |= ((uint64_t)BitValue << BitPos);
2022       }
2023 
2024       // Insert the (remaining) scalar value into position in our integer
2025       // vector type.
2026       if (NumViaIntegerBits <= 32)
2027         Bits = SignExtend64(Bits, 32);
2028       SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2029       Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, Elt,
2030                         DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2031 
2032       if (NumElts < NumViaIntegerBits) {
2033         // If we're producing a smaller vector than our minimum legal integer
2034         // type, bitcast to the equivalent (known-legal) mask type, and extract
2035         // our final mask.
2036         assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
2037         Vec = DAG.getBitcast(MVT::v8i1, Vec);
2038         Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
2039                           DAG.getConstant(0, DL, XLenVT));
2040       } else {
2041         // Else we must have produced an integer type with the same size as the
2042         // mask type; bitcast for the final result.
2043         assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
2044         Vec = DAG.getBitcast(VT, Vec);
2045       }
2046 
2047       return Vec;
2048     }
2049 
2050     // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
2051     // vector type, we have a legal equivalently-sized i8 type, so we can use
2052     // that.
2053     MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
2054     SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
2055 
2056     SDValue WideVec;
2057     if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2058       // For a splat, perform a scalar truncate before creating the wider
2059       // vector.
2060       assert(Splat.getValueType() == XLenVT &&
2061              "Unexpected type for i1 splat value");
2062       Splat = DAG.getNode(ISD::AND, DL, XLenVT, Splat,
2063                           DAG.getConstant(1, DL, XLenVT));
2064       WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
2065     } else {
2066       SmallVector<SDValue, 8> Ops(Op->op_values());
2067       WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
2068       SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
2069       WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
2070     }
2071 
2072     return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
2073   }
2074 
2075   if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2076     unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
2077                                         : RISCVISD::VMV_V_X_VL;
2078     Splat = DAG.getNode(Opc, DL, ContainerVT, Splat, VL);
2079     return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2080   }
2081 
2082   // Try and match index sequences, which we can lower to the vid instruction
2083   // with optional modifications. An all-undef vector is matched by
2084   // getSplatValue, above.
2085   if (auto SimpleVID = isSimpleVIDSequence(Op)) {
2086     int64_t StepNumerator = SimpleVID->StepNumerator;
2087     unsigned StepDenominator = SimpleVID->StepDenominator;
2088     int64_t Addend = SimpleVID->Addend;
2089 
2090     assert(StepNumerator != 0 && "Invalid step");
2091     bool Negate = false;
2092     int64_t SplatStepVal = StepNumerator;
2093     unsigned StepOpcode = ISD::MUL;
2094     if (StepNumerator != 1) {
2095       if (isPowerOf2_64(std::abs(StepNumerator))) {
2096         Negate = StepNumerator < 0;
2097         StepOpcode = ISD::SHL;
2098         SplatStepVal = Log2_64(std::abs(StepNumerator));
2099       }
2100     }
2101 
2102     // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
2103     // threshold since it's the immediate value many RVV instructions accept.
2104     // There is no vmul.vi instruction so ensure multiply constant can fit in
2105     // a single addi instruction.
2106     if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
2107          (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
2108         isPowerOf2_32(StepDenominator) && isInt<5>(Addend)) {
2109       SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
2110       // Convert right out of the scalable type so we can use standard ISD
2111       // nodes for the rest of the computation. If we used scalable types with
2112       // these, we'd lose the fixed-length vector info and generate worse
2113       // vsetvli code.
2114       VID = convertFromScalableVector(VT, VID, DAG, Subtarget);
2115       if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
2116           (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
2117         SDValue SplatStep = DAG.getSplatVector(
2118             VT, DL, DAG.getConstant(SplatStepVal, DL, XLenVT));
2119         VID = DAG.getNode(StepOpcode, DL, VT, VID, SplatStep);
2120       }
2121       if (StepDenominator != 1) {
2122         SDValue SplatStep = DAG.getSplatVector(
2123             VT, DL, DAG.getConstant(Log2_64(StepDenominator), DL, XLenVT));
2124         VID = DAG.getNode(ISD::SRL, DL, VT, VID, SplatStep);
2125       }
2126       if (Addend != 0 || Negate) {
2127         SDValue SplatAddend =
2128             DAG.getSplatVector(VT, DL, DAG.getConstant(Addend, DL, XLenVT));
2129         VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VT, SplatAddend, VID);
2130       }
2131       return VID;
2132     }
2133   }
2134 
2135   // Attempt to detect "hidden" splats, which only reveal themselves as splats
2136   // when re-interpreted as a vector with a larger element type. For example,
2137   //   v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
2138   // could be instead splat as
2139   //   v2i32 = build_vector i32 0x00010000, i32 0x00010000
2140   // TODO: This optimization could also work on non-constant splats, but it
2141   // would require bit-manipulation instructions to construct the splat value.
2142   SmallVector<SDValue> Sequence;
2143   unsigned EltBitSize = VT.getScalarSizeInBits();
2144   const auto *BV = cast<BuildVectorSDNode>(Op);
2145   if (VT.isInteger() && EltBitSize < 64 &&
2146       ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
2147       BV->getRepeatedSequence(Sequence) &&
2148       (Sequence.size() * EltBitSize) <= 64) {
2149     unsigned SeqLen = Sequence.size();
2150     MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
2151     MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, NumElts / SeqLen);
2152     assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
2153             ViaIntVT == MVT::i64) &&
2154            "Unexpected sequence type");
2155 
2156     unsigned EltIdx = 0;
2157     uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
2158     uint64_t SplatValue = 0;
2159     // Construct the amalgamated value which can be splatted as this larger
2160     // vector type.
2161     for (const auto &SeqV : Sequence) {
2162       if (!SeqV.isUndef())
2163         SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
2164                        << (EltIdx * EltBitSize));
2165       EltIdx++;
2166     }
2167 
2168     // On RV64, sign-extend from 32 to 64 bits where possible in order to
2169     // achieve better constant materializion.
2170     if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
2171       SplatValue = SignExtend64(SplatValue, 32);
2172 
2173     // Since we can't introduce illegal i64 types at this stage, we can only
2174     // perform an i64 splat on RV32 if it is its own sign-extended value. That
2175     // way we can use RVV instructions to splat.
2176     assert((ViaIntVT.bitsLE(XLenVT) ||
2177             (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
2178            "Unexpected bitcast sequence");
2179     if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
2180       SDValue ViaVL =
2181           DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
2182       MVT ViaContainerVT =
2183           getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget);
2184       SDValue Splat =
2185           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
2186                       DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
2187       Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
2188       return DAG.getBitcast(VT, Splat);
2189     }
2190   }
2191 
2192   // Try and optimize BUILD_VECTORs with "dominant values" - these are values
2193   // which constitute a large proportion of the elements. In such cases we can
2194   // splat a vector with the dominant element and make up the shortfall with
2195   // INSERT_VECTOR_ELTs.
2196   // Note that this includes vectors of 2 elements by association. The
2197   // upper-most element is the "dominant" one, allowing us to use a splat to
2198   // "insert" the upper element, and an insert of the lower element at position
2199   // 0, which improves codegen.
2200   SDValue DominantValue;
2201   unsigned MostCommonCount = 0;
2202   DenseMap<SDValue, unsigned> ValueCounts;
2203   unsigned NumUndefElts =
2204       count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
2205 
2206   // Track the number of scalar loads we know we'd be inserting, estimated as
2207   // any non-zero floating-point constant. Other kinds of element are either
2208   // already in registers or are materialized on demand. The threshold at which
2209   // a vector load is more desirable than several scalar materializion and
2210   // vector-insertion instructions is not known.
2211   unsigned NumScalarLoads = 0;
2212 
2213   for (SDValue V : Op->op_values()) {
2214     if (V.isUndef())
2215       continue;
2216 
2217     ValueCounts.insert(std::make_pair(V, 0));
2218     unsigned &Count = ValueCounts[V];
2219 
2220     if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
2221       NumScalarLoads += !CFP->isExactlyValue(+0.0);
2222 
2223     // Is this value dominant? In case of a tie, prefer the highest element as
2224     // it's cheaper to insert near the beginning of a vector than it is at the
2225     // end.
2226     if (++Count >= MostCommonCount) {
2227       DominantValue = V;
2228       MostCommonCount = Count;
2229     }
2230   }
2231 
2232   assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
2233   unsigned NumDefElts = NumElts - NumUndefElts;
2234   unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
2235 
2236   // Don't perform this optimization when optimizing for size, since
2237   // materializing elements and inserting them tends to cause code bloat.
2238   if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
2239       ((MostCommonCount > DominantValueCountThreshold) ||
2240        (ValueCounts.size() <= Log2_32(NumDefElts)))) {
2241     // Start by splatting the most common element.
2242     SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
2243 
2244     DenseSet<SDValue> Processed{DominantValue};
2245     MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
2246     for (const auto &OpIdx : enumerate(Op->ops())) {
2247       const SDValue &V = OpIdx.value();
2248       if (V.isUndef() || !Processed.insert(V).second)
2249         continue;
2250       if (ValueCounts[V] == 1) {
2251         Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
2252                           DAG.getConstant(OpIdx.index(), DL, XLenVT));
2253       } else {
2254         // Blend in all instances of this value using a VSELECT, using a
2255         // mask where each bit signals whether that element is the one
2256         // we're after.
2257         SmallVector<SDValue> Ops;
2258         transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
2259           return DAG.getConstant(V == V1, DL, XLenVT);
2260         });
2261         Vec = DAG.getNode(ISD::VSELECT, DL, VT,
2262                           DAG.getBuildVector(SelMaskTy, DL, Ops),
2263                           DAG.getSplatBuildVector(VT, DL, V), Vec);
2264       }
2265     }
2266 
2267     return Vec;
2268   }
2269 
2270   return SDValue();
2271 }
2272 
2273 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Lo,
2274                                    SDValue Hi, SDValue VL, SelectionDAG &DAG) {
2275   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
2276     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
2277     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
2278     // If Hi constant is all the same sign bit as Lo, lower this as a custom
2279     // node in order to try and match RVV vector/scalar instructions.
2280     if ((LoC >> 31) == HiC)
2281       return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Lo, VL);
2282 
2283     // If vl is equal to VLMax and Hi constant is equal to Lo, we could use
2284     // vmv.v.x whose EEW = 32 to lower it.
2285     auto *Const = dyn_cast<ConstantSDNode>(VL);
2286     if (LoC == HiC && Const && Const->getSExtValue() == RISCV::VLMaxSentinel) {
2287       MVT InterVT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
2288       // TODO: if vl <= min(VLMAX), we can also do this. But we could not
2289       // access the subtarget here now.
2290       auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT, Lo, VL);
2291       return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
2292     }
2293   }
2294 
2295   // Fall back to a stack store and stride x0 vector load.
2296   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Lo, Hi, VL);
2297 }
2298 
2299 // Called by type legalization to handle splat of i64 on RV32.
2300 // FIXME: We can optimize this when the type has sign or zero bits in one
2301 // of the halves.
2302 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Scalar,
2303                                    SDValue VL, SelectionDAG &DAG) {
2304   assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
2305   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2306                            DAG.getConstant(0, DL, MVT::i32));
2307   SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2308                            DAG.getConstant(1, DL, MVT::i32));
2309   return splatPartsI64WithVL(DL, VT, Lo, Hi, VL, DAG);
2310 }
2311 
2312 // This function lowers a splat of a scalar operand Splat with the vector
2313 // length VL. It ensures the final sequence is type legal, which is useful when
2314 // lowering a splat after type legalization.
2315 static SDValue lowerScalarSplat(SDValue Scalar, SDValue VL, MVT VT, SDLoc DL,
2316                                 SelectionDAG &DAG,
2317                                 const RISCVSubtarget &Subtarget) {
2318   if (VT.isFloatingPoint()) {
2319     // If VL is 1, we could use vfmv.s.f.
2320     if (isOneConstant(VL))
2321       return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT),
2322                          Scalar, VL);
2323     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Scalar, VL);
2324   }
2325 
2326   MVT XLenVT = Subtarget.getXLenVT();
2327 
2328   // Simplest case is that the operand needs to be promoted to XLenVT.
2329   if (Scalar.getValueType().bitsLE(XLenVT)) {
2330     // If the operand is a constant, sign extend to increase our chances
2331     // of being able to use a .vi instruction. ANY_EXTEND would become a
2332     // a zero extend and the simm5 check in isel would fail.
2333     // FIXME: Should we ignore the upper bits in isel instead?
2334     unsigned ExtOpc =
2335         isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
2336     Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
2337     ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Scalar);
2338     // If VL is 1 and the scalar value won't benefit from immediate, we could
2339     // use vmv.s.x.
2340     if (isOneConstant(VL) &&
2341         (!Const || isNullConstant(Scalar) || !isInt<5>(Const->getSExtValue())))
2342       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
2343                          VL);
2344     return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Scalar, VL);
2345   }
2346 
2347   assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
2348          "Unexpected scalar for splat lowering!");
2349 
2350   if (isOneConstant(VL) && isNullConstant(Scalar))
2351     return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT),
2352                        DAG.getConstant(0, DL, XLenVT), VL);
2353 
2354   // Otherwise use the more complicated splatting algorithm.
2355   return splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
2356 }
2357 
2358 // Is the mask a slidedown that shifts in undefs.
2359 static int matchShuffleAsSlideDown(ArrayRef<int> Mask) {
2360   int Size = Mask.size();
2361 
2362   // Elements shifted in should be undef.
2363   auto CheckUndefs = [&](int Shift) {
2364     for (int i = Size - Shift; i != Size; ++i)
2365       if (Mask[i] >= 0)
2366         return false;
2367     return true;
2368   };
2369 
2370   // Elements should be shifted or undef.
2371   auto MatchShift = [&](int Shift) {
2372     for (int i = 0; i != Size - Shift; ++i)
2373        if (Mask[i] >= 0 && Mask[i] != Shift + i)
2374          return false;
2375     return true;
2376   };
2377 
2378   // Try all possible shifts.
2379   for (int Shift = 1; Shift != Size; ++Shift)
2380     if (CheckUndefs(Shift) && MatchShift(Shift))
2381       return Shift;
2382 
2383   // No match.
2384   return -1;
2385 }
2386 
2387 static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, bool &SwapSources,
2388                                 const RISCVSubtarget &Subtarget) {
2389   // We need to be able to widen elements to the next larger integer type.
2390   if (VT.getScalarSizeInBits() >= Subtarget.getMaxELENForFixedLengthVectors())
2391     return false;
2392 
2393   int Size = Mask.size();
2394   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
2395 
2396   int Srcs[] = {-1, -1};
2397   for (int i = 0; i != Size; ++i) {
2398     // Ignore undef elements.
2399     if (Mask[i] < 0)
2400       continue;
2401 
2402     // Is this an even or odd element.
2403     int Pol = i % 2;
2404 
2405     // Ensure we consistently use the same source for this element polarity.
2406     int Src = Mask[i] / Size;
2407     if (Srcs[Pol] < 0)
2408       Srcs[Pol] = Src;
2409     if (Srcs[Pol] != Src)
2410       return false;
2411 
2412     // Make sure the element within the source is appropriate for this element
2413     // in the destination.
2414     int Elt = Mask[i] % Size;
2415     if (Elt != i / 2)
2416       return false;
2417   }
2418 
2419   // We need to find a source for each polarity and they can't be the same.
2420   if (Srcs[0] < 0 || Srcs[1] < 0 || Srcs[0] == Srcs[1])
2421     return false;
2422 
2423   // Swap the sources if the second source was in the even polarity.
2424   SwapSources = Srcs[0] > Srcs[1];
2425 
2426   return true;
2427 }
2428 
2429 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
2430                                    const RISCVSubtarget &Subtarget) {
2431   SDValue V1 = Op.getOperand(0);
2432   SDValue V2 = Op.getOperand(1);
2433   SDLoc DL(Op);
2434   MVT XLenVT = Subtarget.getXLenVT();
2435   MVT VT = Op.getSimpleValueType();
2436   unsigned NumElts = VT.getVectorNumElements();
2437   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
2438 
2439   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2440 
2441   SDValue TrueMask, VL;
2442   std::tie(TrueMask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2443 
2444   if (SVN->isSplat()) {
2445     const int Lane = SVN->getSplatIndex();
2446     if (Lane >= 0) {
2447       MVT SVT = VT.getVectorElementType();
2448 
2449       // Turn splatted vector load into a strided load with an X0 stride.
2450       SDValue V = V1;
2451       // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
2452       // with undef.
2453       // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
2454       int Offset = Lane;
2455       if (V.getOpcode() == ISD::CONCAT_VECTORS) {
2456         int OpElements =
2457             V.getOperand(0).getSimpleValueType().getVectorNumElements();
2458         V = V.getOperand(Offset / OpElements);
2459         Offset %= OpElements;
2460       }
2461 
2462       // We need to ensure the load isn't atomic or volatile.
2463       if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
2464         auto *Ld = cast<LoadSDNode>(V);
2465         Offset *= SVT.getStoreSize();
2466         SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(),
2467                                                    TypeSize::Fixed(Offset), DL);
2468 
2469         // If this is SEW=64 on RV32, use a strided load with a stride of x0.
2470         if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
2471           SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
2472           SDValue IntID =
2473               DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
2474           SDValue Ops[] = {Ld->getChain(),
2475                            IntID,
2476                            DAG.getUNDEF(ContainerVT),
2477                            NewAddr,
2478                            DAG.getRegister(RISCV::X0, XLenVT),
2479                            VL};
2480           SDValue NewLoad = DAG.getMemIntrinsicNode(
2481               ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
2482               DAG.getMachineFunction().getMachineMemOperand(
2483                   Ld->getMemOperand(), Offset, SVT.getStoreSize()));
2484           DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
2485           return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
2486         }
2487 
2488         // Otherwise use a scalar load and splat. This will give the best
2489         // opportunity to fold a splat into the operation. ISel can turn it into
2490         // the x0 strided load if we aren't able to fold away the select.
2491         if (SVT.isFloatingPoint())
2492           V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
2493                           Ld->getPointerInfo().getWithOffset(Offset),
2494                           Ld->getOriginalAlign(),
2495                           Ld->getMemOperand()->getFlags());
2496         else
2497           V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
2498                              Ld->getPointerInfo().getWithOffset(Offset), SVT,
2499                              Ld->getOriginalAlign(),
2500                              Ld->getMemOperand()->getFlags());
2501         DAG.makeEquivalentMemoryOrdering(Ld, V);
2502 
2503         unsigned Opc =
2504             VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
2505         SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, V, VL);
2506         return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2507       }
2508 
2509       V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2510       assert(Lane < (int)NumElts && "Unexpected lane!");
2511       SDValue Gather =
2512           DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
2513                       DAG.getConstant(Lane, DL, XLenVT), TrueMask, VL);
2514       return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2515     }
2516   }
2517 
2518   ArrayRef<int> Mask = SVN->getMask();
2519 
2520   // Try to match as a slidedown.
2521   int SlideAmt = matchShuffleAsSlideDown(Mask);
2522   if (SlideAmt >= 0) {
2523     // TODO: Should we reduce the VL to account for the upper undef elements?
2524     // Requires additional vsetvlis, but might be faster to execute.
2525     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2526     SDValue SlideDown =
2527         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
2528                     DAG.getUNDEF(ContainerVT), V1,
2529                     DAG.getConstant(SlideAmt, DL, XLenVT),
2530                     TrueMask, VL);
2531     return convertFromScalableVector(VT, SlideDown, DAG, Subtarget);
2532   }
2533 
2534   // Detect an interleave shuffle and lower to
2535   // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
2536   bool SwapSources;
2537   if (isInterleaveShuffle(Mask, VT, SwapSources, Subtarget)) {
2538     // Swap sources if needed.
2539     if (SwapSources)
2540       std::swap(V1, V2);
2541 
2542     // Extract the lower half of the vectors.
2543     MVT HalfVT = VT.getHalfNumVectorElementsVT();
2544     V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
2545                      DAG.getConstant(0, DL, XLenVT));
2546     V2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V2,
2547                      DAG.getConstant(0, DL, XLenVT));
2548 
2549     // Double the element width and halve the number of elements in an int type.
2550     unsigned EltBits = VT.getScalarSizeInBits();
2551     MVT WideIntEltVT = MVT::getIntegerVT(EltBits * 2);
2552     MVT WideIntVT =
2553         MVT::getVectorVT(WideIntEltVT, VT.getVectorNumElements() / 2);
2554     // Convert this to a scalable vector. We need to base this on the
2555     // destination size to ensure there's always a type with a smaller LMUL.
2556     MVT WideIntContainerVT =
2557         getContainerForFixedLengthVector(DAG, WideIntVT, Subtarget);
2558 
2559     // Convert sources to scalable vectors with the same element count as the
2560     // larger type.
2561     MVT HalfContainerVT = MVT::getVectorVT(
2562         VT.getVectorElementType(), WideIntContainerVT.getVectorElementCount());
2563     V1 = convertToScalableVector(HalfContainerVT, V1, DAG, Subtarget);
2564     V2 = convertToScalableVector(HalfContainerVT, V2, DAG, Subtarget);
2565 
2566     // Cast sources to integer.
2567     MVT IntEltVT = MVT::getIntegerVT(EltBits);
2568     MVT IntHalfVT =
2569         MVT::getVectorVT(IntEltVT, HalfContainerVT.getVectorElementCount());
2570     V1 = DAG.getBitcast(IntHalfVT, V1);
2571     V2 = DAG.getBitcast(IntHalfVT, V2);
2572 
2573     // Freeze V2 since we use it twice and we need to be sure that the add and
2574     // multiply see the same value.
2575     V2 = DAG.getNode(ISD::FREEZE, DL, IntHalfVT, V2);
2576 
2577     // Recreate TrueMask using the widened type's element count.
2578     MVT MaskVT =
2579         MVT::getVectorVT(MVT::i1, HalfContainerVT.getVectorElementCount());
2580     TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
2581 
2582     // Widen V1 and V2 with 0s and add one copy of V2 to V1.
2583     SDValue Add = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideIntContainerVT, V1,
2584                               V2, TrueMask, VL);
2585     // Create 2^eltbits - 1 copies of V2 by multiplying by the largest integer.
2586     SDValue Multiplier = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntHalfVT,
2587                                      DAG.getAllOnesConstant(DL, XLenVT));
2588     SDValue WidenMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideIntContainerVT,
2589                                    V2, Multiplier, TrueMask, VL);
2590     // Add the new copies to our previous addition giving us 2^eltbits copies of
2591     // V2. This is equivalent to shifting V2 left by eltbits. This should
2592     // combine with the vwmulu.vv above to form vwmaccu.vv.
2593     Add = DAG.getNode(RISCVISD::ADD_VL, DL, WideIntContainerVT, Add, WidenMul,
2594                       TrueMask, VL);
2595     // Cast back to ContainerVT. We need to re-create a new ContainerVT in case
2596     // WideIntContainerVT is a larger fractional LMUL than implied by the fixed
2597     // vector VT.
2598     ContainerVT =
2599         MVT::getVectorVT(VT.getVectorElementType(),
2600                          WideIntContainerVT.getVectorElementCount() * 2);
2601     Add = DAG.getBitcast(ContainerVT, Add);
2602     return convertFromScalableVector(VT, Add, DAG, Subtarget);
2603   }
2604 
2605   // Detect shuffles which can be re-expressed as vector selects; these are
2606   // shuffles in which each element in the destination is taken from an element
2607   // at the corresponding index in either source vectors.
2608   bool IsSelect = all_of(enumerate(Mask), [&](const auto &MaskIdx) {
2609     int MaskIndex = MaskIdx.value();
2610     return MaskIndex < 0 || MaskIdx.index() == (unsigned)MaskIndex % NumElts;
2611   });
2612 
2613   assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
2614 
2615   SmallVector<SDValue> MaskVals;
2616   // As a backup, shuffles can be lowered via a vrgather instruction, possibly
2617   // merged with a second vrgather.
2618   SmallVector<SDValue> GatherIndicesLHS, GatherIndicesRHS;
2619 
2620   // By default we preserve the original operand order, and use a mask to
2621   // select LHS as true and RHS as false. However, since RVV vector selects may
2622   // feature splats but only on the LHS, we may choose to invert our mask and
2623   // instead select between RHS and LHS.
2624   bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
2625   bool InvertMask = IsSelect == SwapOps;
2626 
2627   // Keep a track of which non-undef indices are used by each LHS/RHS shuffle
2628   // half.
2629   DenseMap<int, unsigned> LHSIndexCounts, RHSIndexCounts;
2630 
2631   // Now construct the mask that will be used by the vselect or blended
2632   // vrgather operation. For vrgathers, construct the appropriate indices into
2633   // each vector.
2634   for (int MaskIndex : Mask) {
2635     bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ InvertMask;
2636     MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
2637     if (!IsSelect) {
2638       bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
2639       GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
2640                                      ? DAG.getConstant(MaskIndex, DL, XLenVT)
2641                                      : DAG.getUNDEF(XLenVT));
2642       GatherIndicesRHS.push_back(
2643           IsLHSOrUndefIndex ? DAG.getUNDEF(XLenVT)
2644                             : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT));
2645       if (IsLHSOrUndefIndex && MaskIndex >= 0)
2646         ++LHSIndexCounts[MaskIndex];
2647       if (!IsLHSOrUndefIndex)
2648         ++RHSIndexCounts[MaskIndex - NumElts];
2649     }
2650   }
2651 
2652   if (SwapOps) {
2653     std::swap(V1, V2);
2654     std::swap(GatherIndicesLHS, GatherIndicesRHS);
2655   }
2656 
2657   assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
2658   MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
2659   SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
2660 
2661   if (IsSelect)
2662     return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
2663 
2664   if (VT.getScalarSizeInBits() == 8 && VT.getVectorNumElements() > 256) {
2665     // On such a large vector we're unable to use i8 as the index type.
2666     // FIXME: We could promote the index to i16 and use vrgatherei16, but that
2667     // may involve vector splitting if we're already at LMUL=8, or our
2668     // user-supplied maximum fixed-length LMUL.
2669     return SDValue();
2670   }
2671 
2672   unsigned GatherVXOpc = RISCVISD::VRGATHER_VX_VL;
2673   unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
2674   MVT IndexVT = VT.changeTypeToInteger();
2675   // Since we can't introduce illegal index types at this stage, use i16 and
2676   // vrgatherei16 if the corresponding index type for plain vrgather is greater
2677   // than XLenVT.
2678   if (IndexVT.getScalarType().bitsGT(XLenVT)) {
2679     GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
2680     IndexVT = IndexVT.changeVectorElementType(MVT::i16);
2681   }
2682 
2683   MVT IndexContainerVT =
2684       ContainerVT.changeVectorElementType(IndexVT.getScalarType());
2685 
2686   SDValue Gather;
2687   // TODO: This doesn't trigger for i64 vectors on RV32, since there we
2688   // encounter a bitcasted BUILD_VECTOR with low/high i32 values.
2689   if (SDValue SplatValue = DAG.getSplatValue(V1, /*LegalTypes*/ true)) {
2690     Gather = lowerScalarSplat(SplatValue, VL, ContainerVT, DL, DAG, Subtarget);
2691   } else {
2692     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2693     // If only one index is used, we can use a "splat" vrgather.
2694     // TODO: We can splat the most-common index and fix-up any stragglers, if
2695     // that's beneficial.
2696     if (LHSIndexCounts.size() == 1) {
2697       int SplatIndex = LHSIndexCounts.begin()->getFirst();
2698       Gather =
2699           DAG.getNode(GatherVXOpc, DL, ContainerVT, V1,
2700                       DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2701     } else {
2702       SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
2703       LHSIndices =
2704           convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
2705 
2706       Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
2707                            TrueMask, VL);
2708     }
2709   }
2710 
2711   // If a second vector operand is used by this shuffle, blend it in with an
2712   // additional vrgather.
2713   if (!V2.isUndef()) {
2714     V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
2715     // If only one index is used, we can use a "splat" vrgather.
2716     // TODO: We can splat the most-common index and fix-up any stragglers, if
2717     // that's beneficial.
2718     if (RHSIndexCounts.size() == 1) {
2719       int SplatIndex = RHSIndexCounts.begin()->getFirst();
2720       V2 = DAG.getNode(GatherVXOpc, DL, ContainerVT, V2,
2721                        DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2722     } else {
2723       SDValue RHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesRHS);
2724       RHSIndices =
2725           convertToScalableVector(IndexContainerVT, RHSIndices, DAG, Subtarget);
2726       V2 = DAG.getNode(GatherVVOpc, DL, ContainerVT, V2, RHSIndices, TrueMask,
2727                        VL);
2728     }
2729 
2730     MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
2731     SelectMask =
2732         convertToScalableVector(MaskContainerVT, SelectMask, DAG, Subtarget);
2733 
2734     Gather = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, SelectMask, V2,
2735                          Gather, VL);
2736   }
2737 
2738   return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2739 }
2740 
2741 static SDValue getRVVFPExtendOrRound(SDValue Op, MVT VT, MVT ContainerVT,
2742                                      SDLoc DL, SelectionDAG &DAG,
2743                                      const RISCVSubtarget &Subtarget) {
2744   if (VT.isScalableVector())
2745     return DAG.getFPExtendOrRound(Op, DL, VT);
2746   assert(VT.isFixedLengthVector() &&
2747          "Unexpected value type for RVV FP extend/round lowering");
2748   SDValue Mask, VL;
2749   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2750   unsigned RVVOpc = ContainerVT.bitsGT(Op.getSimpleValueType())
2751                         ? RISCVISD::FP_EXTEND_VL
2752                         : RISCVISD::FP_ROUND_VL;
2753   return DAG.getNode(RVVOpc, DL, ContainerVT, Op, Mask, VL);
2754 }
2755 
2756 // Lower CTLZ_ZERO_UNDEF or CTTZ_ZERO_UNDEF by converting to FP and extracting
2757 // the exponent.
2758 static SDValue lowerCTLZ_CTTZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
2759   MVT VT = Op.getSimpleValueType();
2760   unsigned EltSize = VT.getScalarSizeInBits();
2761   SDValue Src = Op.getOperand(0);
2762   SDLoc DL(Op);
2763 
2764   // We need a FP type that can represent the value.
2765   // TODO: Use f16 for i8 when possible?
2766   MVT FloatEltVT = EltSize == 32 ? MVT::f64 : MVT::f32;
2767   MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
2768 
2769   // Legal types should have been checked in the RISCVTargetLowering
2770   // constructor.
2771   // TODO: Splitting may make sense in some cases.
2772   assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
2773          "Expected legal float type!");
2774 
2775   // For CTTZ_ZERO_UNDEF, we need to extract the lowest set bit using X & -X.
2776   // The trailing zero count is equal to log2 of this single bit value.
2777   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
2778     SDValue Neg =
2779         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
2780     Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
2781   }
2782 
2783   // We have a legal FP type, convert to it.
2784   SDValue FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
2785   // Bitcast to integer and shift the exponent to the LSB.
2786   EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
2787   SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
2788   unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
2789   SDValue Shift = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
2790                               DAG.getConstant(ShiftAmt, DL, IntVT));
2791   // Truncate back to original type to allow vnsrl.
2792   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, Shift);
2793   // The exponent contains log2 of the value in biased form.
2794   unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
2795 
2796   // For trailing zeros, we just need to subtract the bias.
2797   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF)
2798     return DAG.getNode(ISD::SUB, DL, VT, Trunc,
2799                        DAG.getConstant(ExponentBias, DL, VT));
2800 
2801   // For leading zeros, we need to remove the bias and convert from log2 to
2802   // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
2803   unsigned Adjust = ExponentBias + (EltSize - 1);
2804   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Trunc);
2805 }
2806 
2807 // While RVV has alignment restrictions, we should always be able to load as a
2808 // legal equivalently-sized byte-typed vector instead. This method is
2809 // responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
2810 // the load is already correctly-aligned, it returns SDValue().
2811 SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
2812                                                     SelectionDAG &DAG) const {
2813   auto *Load = cast<LoadSDNode>(Op);
2814   assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
2815 
2816   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2817                                      Load->getMemoryVT(),
2818                                      *Load->getMemOperand()))
2819     return SDValue();
2820 
2821   SDLoc DL(Op);
2822   MVT VT = Op.getSimpleValueType();
2823   unsigned EltSizeBits = VT.getScalarSizeInBits();
2824   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2825          "Unexpected unaligned RVV load type");
2826   MVT NewVT =
2827       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2828   assert(NewVT.isValid() &&
2829          "Expecting equally-sized RVV vector types to be legal");
2830   SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
2831                           Load->getPointerInfo(), Load->getOriginalAlign(),
2832                           Load->getMemOperand()->getFlags());
2833   return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
2834 }
2835 
2836 // While RVV has alignment restrictions, we should always be able to store as a
2837 // legal equivalently-sized byte-typed vector instead. This method is
2838 // responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
2839 // returns SDValue() if the store is already correctly aligned.
2840 SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
2841                                                      SelectionDAG &DAG) const {
2842   auto *Store = cast<StoreSDNode>(Op);
2843   assert(Store && Store->getValue().getValueType().isVector() &&
2844          "Expected vector store");
2845 
2846   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2847                                      Store->getMemoryVT(),
2848                                      *Store->getMemOperand()))
2849     return SDValue();
2850 
2851   SDLoc DL(Op);
2852   SDValue StoredVal = Store->getValue();
2853   MVT VT = StoredVal.getSimpleValueType();
2854   unsigned EltSizeBits = VT.getScalarSizeInBits();
2855   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2856          "Unexpected unaligned RVV store type");
2857   MVT NewVT =
2858       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2859   assert(NewVT.isValid() &&
2860          "Expecting equally-sized RVV vector types to be legal");
2861   StoredVal = DAG.getBitcast(NewVT, StoredVal);
2862   return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
2863                       Store->getPointerInfo(), Store->getOriginalAlign(),
2864                       Store->getMemOperand()->getFlags());
2865 }
2866 
2867 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
2868                                             SelectionDAG &DAG) const {
2869   switch (Op.getOpcode()) {
2870   default:
2871     report_fatal_error("unimplemented operand");
2872   case ISD::GlobalAddress:
2873     return lowerGlobalAddress(Op, DAG);
2874   case ISD::BlockAddress:
2875     return lowerBlockAddress(Op, DAG);
2876   case ISD::ConstantPool:
2877     return lowerConstantPool(Op, DAG);
2878   case ISD::JumpTable:
2879     return lowerJumpTable(Op, DAG);
2880   case ISD::GlobalTLSAddress:
2881     return lowerGlobalTLSAddress(Op, DAG);
2882   case ISD::SELECT:
2883     return lowerSELECT(Op, DAG);
2884   case ISD::BRCOND:
2885     return lowerBRCOND(Op, DAG);
2886   case ISD::VASTART:
2887     return lowerVASTART(Op, DAG);
2888   case ISD::FRAMEADDR:
2889     return lowerFRAMEADDR(Op, DAG);
2890   case ISD::RETURNADDR:
2891     return lowerRETURNADDR(Op, DAG);
2892   case ISD::SHL_PARTS:
2893     return lowerShiftLeftParts(Op, DAG);
2894   case ISD::SRA_PARTS:
2895     return lowerShiftRightParts(Op, DAG, true);
2896   case ISD::SRL_PARTS:
2897     return lowerShiftRightParts(Op, DAG, false);
2898   case ISD::BITCAST: {
2899     SDLoc DL(Op);
2900     EVT VT = Op.getValueType();
2901     SDValue Op0 = Op.getOperand(0);
2902     EVT Op0VT = Op0.getValueType();
2903     MVT XLenVT = Subtarget.getXLenVT();
2904     if (VT.isFixedLengthVector()) {
2905       // We can handle fixed length vector bitcasts with a simple replacement
2906       // in isel.
2907       if (Op0VT.isFixedLengthVector())
2908         return Op;
2909       // When bitcasting from scalar to fixed-length vector, insert the scalar
2910       // into a one-element vector of the result type, and perform a vector
2911       // bitcast.
2912       if (!Op0VT.isVector()) {
2913         EVT BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
2914         if (!isTypeLegal(BVT))
2915           return SDValue();
2916         return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
2917                                               DAG.getUNDEF(BVT), Op0,
2918                                               DAG.getConstant(0, DL, XLenVT)));
2919       }
2920       return SDValue();
2921     }
2922     // Custom-legalize bitcasts from fixed-length vector types to scalar types
2923     // thus: bitcast the vector to a one-element vector type whose element type
2924     // is the same as the result type, and extract the first element.
2925     if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
2926       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
2927       if (!isTypeLegal(BVT))
2928         return SDValue();
2929       SDValue BVec = DAG.getBitcast(BVT, Op0);
2930       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
2931                          DAG.getConstant(0, DL, XLenVT));
2932     }
2933     if (VT == MVT::f16 && Op0VT == MVT::i16 && Subtarget.hasStdExtZfh()) {
2934       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
2935       SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
2936       return FPConv;
2937     }
2938     if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
2939         Subtarget.hasStdExtF()) {
2940       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
2941       SDValue FPConv =
2942           DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
2943       return FPConv;
2944     }
2945     return SDValue();
2946   }
2947   case ISD::INTRINSIC_WO_CHAIN:
2948     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
2949   case ISD::INTRINSIC_W_CHAIN:
2950     return LowerINTRINSIC_W_CHAIN(Op, DAG);
2951   case ISD::INTRINSIC_VOID:
2952     return LowerINTRINSIC_VOID(Op, DAG);
2953   case ISD::BSWAP:
2954   case ISD::BITREVERSE: {
2955     // Convert BSWAP/BITREVERSE to GREVI to enable GREVI combinining.
2956     assert(Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
2957     MVT VT = Op.getSimpleValueType();
2958     SDLoc DL(Op);
2959     // Start with the maximum immediate value which is the bitwidth - 1.
2960     unsigned Imm = VT.getSizeInBits() - 1;
2961     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
2962     if (Op.getOpcode() == ISD::BSWAP)
2963       Imm &= ~0x7U;
2964     return DAG.getNode(RISCVISD::GREV, DL, VT, Op.getOperand(0),
2965                        DAG.getConstant(Imm, DL, VT));
2966   }
2967   case ISD::FSHL:
2968   case ISD::FSHR: {
2969     MVT VT = Op.getSimpleValueType();
2970     assert(VT == Subtarget.getXLenVT() && "Unexpected custom legalization");
2971     SDLoc DL(Op);
2972     // FSL/FSR take a log2(XLen)+1 bit shift amount but XLenVT FSHL/FSHR only
2973     // use log(XLen) bits. Mask the shift amount accordingly to prevent
2974     // accidentally setting the extra bit.
2975     unsigned ShAmtWidth = Subtarget.getXLen() - 1;
2976     SDValue ShAmt = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(2),
2977                                 DAG.getConstant(ShAmtWidth, DL, VT));
2978     // fshl and fshr concatenate their operands in the same order. fsr and fsl
2979     // instruction use different orders. fshl will return its first operand for
2980     // shift of zero, fshr will return its second operand. fsl and fsr both
2981     // return rs1 so the ISD nodes need to have different operand orders.
2982     // Shift amount is in rs2.
2983     SDValue Op0 = Op.getOperand(0);
2984     SDValue Op1 = Op.getOperand(1);
2985     unsigned Opc = RISCVISD::FSL;
2986     if (Op.getOpcode() == ISD::FSHR) {
2987       std::swap(Op0, Op1);
2988       Opc = RISCVISD::FSR;
2989     }
2990     return DAG.getNode(Opc, DL, VT, Op0, Op1, ShAmt);
2991   }
2992   case ISD::TRUNCATE: {
2993     SDLoc DL(Op);
2994     MVT VT = Op.getSimpleValueType();
2995     // Only custom-lower vector truncates
2996     if (!VT.isVector())
2997       return Op;
2998 
2999     // Truncates to mask types are handled differently
3000     if (VT.getVectorElementType() == MVT::i1)
3001       return lowerVectorMaskTrunc(Op, DAG);
3002 
3003     // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
3004     // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
3005     // truncate by one power of two at a time.
3006     MVT DstEltVT = VT.getVectorElementType();
3007 
3008     SDValue Src = Op.getOperand(0);
3009     MVT SrcVT = Src.getSimpleValueType();
3010     MVT SrcEltVT = SrcVT.getVectorElementType();
3011 
3012     assert(DstEltVT.bitsLT(SrcEltVT) &&
3013            isPowerOf2_64(DstEltVT.getSizeInBits()) &&
3014            isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
3015            "Unexpected vector truncate lowering");
3016 
3017     MVT ContainerVT = SrcVT;
3018     if (SrcVT.isFixedLengthVector()) {
3019       ContainerVT = getContainerForFixedLengthVector(SrcVT);
3020       Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3021     }
3022 
3023     SDValue Result = Src;
3024     SDValue Mask, VL;
3025     std::tie(Mask, VL) =
3026         getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
3027     LLVMContext &Context = *DAG.getContext();
3028     const ElementCount Count = ContainerVT.getVectorElementCount();
3029     do {
3030       SrcEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
3031       EVT ResultVT = EVT::getVectorVT(Context, SrcEltVT, Count);
3032       Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
3033                            Mask, VL);
3034     } while (SrcEltVT != DstEltVT);
3035 
3036     if (SrcVT.isFixedLengthVector())
3037       Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
3038 
3039     return Result;
3040   }
3041   case ISD::ANY_EXTEND:
3042   case ISD::ZERO_EXTEND:
3043     if (Op.getOperand(0).getValueType().isVector() &&
3044         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3045       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
3046     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
3047   case ISD::SIGN_EXTEND:
3048     if (Op.getOperand(0).getValueType().isVector() &&
3049         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3050       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
3051     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
3052   case ISD::SPLAT_VECTOR_PARTS:
3053     return lowerSPLAT_VECTOR_PARTS(Op, DAG);
3054   case ISD::INSERT_VECTOR_ELT:
3055     return lowerINSERT_VECTOR_ELT(Op, DAG);
3056   case ISD::EXTRACT_VECTOR_ELT:
3057     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
3058   case ISD::VSCALE: {
3059     MVT VT = Op.getSimpleValueType();
3060     SDLoc DL(Op);
3061     SDValue VLENB = DAG.getNode(RISCVISD::READ_VLENB, DL, VT);
3062     // We define our scalable vector types for lmul=1 to use a 64 bit known
3063     // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
3064     // vscale as VLENB / 8.
3065     static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
3066     if (isa<ConstantSDNode>(Op.getOperand(0))) {
3067       // We assume VLENB is a multiple of 8. We manually choose the best shift
3068       // here because SimplifyDemandedBits isn't always able to simplify it.
3069       uint64_t Val = Op.getConstantOperandVal(0);
3070       if (isPowerOf2_64(Val)) {
3071         uint64_t Log2 = Log2_64(Val);
3072         if (Log2 < 3)
3073           return DAG.getNode(ISD::SRL, DL, VT, VLENB,
3074                              DAG.getConstant(3 - Log2, DL, VT));
3075         if (Log2 > 3)
3076           return DAG.getNode(ISD::SHL, DL, VT, VLENB,
3077                              DAG.getConstant(Log2 - 3, DL, VT));
3078         return VLENB;
3079       }
3080       // If the multiplier is a multiple of 8, scale it down to avoid needing
3081       // to shift the VLENB value.
3082       if ((Val % 8) == 0)
3083         return DAG.getNode(ISD::MUL, DL, VT, VLENB,
3084                            DAG.getConstant(Val / 8, DL, VT));
3085     }
3086 
3087     SDValue VScale = DAG.getNode(ISD::SRL, DL, VT, VLENB,
3088                                  DAG.getConstant(3, DL, VT));
3089     return DAG.getNode(ISD::MUL, DL, VT, VScale, Op.getOperand(0));
3090   }
3091   case ISD::FPOWI: {
3092     // Custom promote f16 powi with illegal i32 integer type on RV64. Once
3093     // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
3094     if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
3095         Op.getOperand(1).getValueType() == MVT::i32) {
3096       SDLoc DL(Op);
3097       SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
3098       SDValue Powi =
3099           DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
3100       return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
3101                          DAG.getIntPtrConstant(0, DL));
3102     }
3103     return SDValue();
3104   }
3105   case ISD::FP_EXTEND: {
3106     // RVV can only do fp_extend to types double the size as the source. We
3107     // custom-lower f16->f64 extensions to two hops of ISD::FP_EXTEND, going
3108     // via f32.
3109     SDLoc DL(Op);
3110     MVT VT = Op.getSimpleValueType();
3111     SDValue Src = Op.getOperand(0);
3112     MVT SrcVT = Src.getSimpleValueType();
3113 
3114     // Prepare any fixed-length vector operands.
3115     MVT ContainerVT = VT;
3116     if (SrcVT.isFixedLengthVector()) {
3117       ContainerVT = getContainerForFixedLengthVector(VT);
3118       MVT SrcContainerVT =
3119           ContainerVT.changeVectorElementType(SrcVT.getVectorElementType());
3120       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3121     }
3122 
3123     if (!VT.isVector() || VT.getVectorElementType() != MVT::f64 ||
3124         SrcVT.getVectorElementType() != MVT::f16) {
3125       // For scalable vectors, we only need to close the gap between
3126       // vXf16->vXf64.
3127       if (!VT.isFixedLengthVector())
3128         return Op;
3129       // For fixed-length vectors, lower the FP_EXTEND to a custom "VL" version.
3130       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
3131       return convertFromScalableVector(VT, Src, DAG, Subtarget);
3132     }
3133 
3134     MVT InterVT = VT.changeVectorElementType(MVT::f32);
3135     MVT InterContainerVT = ContainerVT.changeVectorElementType(MVT::f32);
3136     SDValue IntermediateExtend = getRVVFPExtendOrRound(
3137         Src, InterVT, InterContainerVT, DL, DAG, Subtarget);
3138 
3139     SDValue Extend = getRVVFPExtendOrRound(IntermediateExtend, VT, ContainerVT,
3140                                            DL, DAG, Subtarget);
3141     if (VT.isFixedLengthVector())
3142       return convertFromScalableVector(VT, Extend, DAG, Subtarget);
3143     return Extend;
3144   }
3145   case ISD::FP_ROUND: {
3146     // RVV can only do fp_round to types half the size as the source. We
3147     // custom-lower f64->f16 rounds via RVV's round-to-odd float
3148     // conversion instruction.
3149     SDLoc DL(Op);
3150     MVT VT = Op.getSimpleValueType();
3151     SDValue Src = Op.getOperand(0);
3152     MVT SrcVT = Src.getSimpleValueType();
3153 
3154     // Prepare any fixed-length vector operands.
3155     MVT ContainerVT = VT;
3156     if (VT.isFixedLengthVector()) {
3157       MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3158       ContainerVT =
3159           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3160       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3161     }
3162 
3163     if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
3164         SrcVT.getVectorElementType() != MVT::f64) {
3165       // For scalable vectors, we only need to close the gap between
3166       // vXf64<->vXf16.
3167       if (!VT.isFixedLengthVector())
3168         return Op;
3169       // For fixed-length vectors, lower the FP_ROUND to a custom "VL" version.
3170       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
3171       return convertFromScalableVector(VT, Src, DAG, Subtarget);
3172     }
3173 
3174     SDValue Mask, VL;
3175     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3176 
3177     MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
3178     SDValue IntermediateRound =
3179         DAG.getNode(RISCVISD::VFNCVT_ROD_VL, DL, InterVT, Src, Mask, VL);
3180     SDValue Round = getRVVFPExtendOrRound(IntermediateRound, VT, ContainerVT,
3181                                           DL, DAG, Subtarget);
3182 
3183     if (VT.isFixedLengthVector())
3184       return convertFromScalableVector(VT, Round, DAG, Subtarget);
3185     return Round;
3186   }
3187   case ISD::FP_TO_SINT:
3188   case ISD::FP_TO_UINT:
3189   case ISD::SINT_TO_FP:
3190   case ISD::UINT_TO_FP: {
3191     // RVV can only do fp<->int conversions to types half/double the size as
3192     // the source. We custom-lower any conversions that do two hops into
3193     // sequences.
3194     MVT VT = Op.getSimpleValueType();
3195     if (!VT.isVector())
3196       return Op;
3197     SDLoc DL(Op);
3198     SDValue Src = Op.getOperand(0);
3199     MVT EltVT = VT.getVectorElementType();
3200     MVT SrcVT = Src.getSimpleValueType();
3201     MVT SrcEltVT = SrcVT.getVectorElementType();
3202     unsigned EltSize = EltVT.getSizeInBits();
3203     unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3204     assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
3205            "Unexpected vector element types");
3206 
3207     bool IsInt2FP = SrcEltVT.isInteger();
3208     // Widening conversions
3209     if (EltSize > SrcEltSize && (EltSize / SrcEltSize >= 4)) {
3210       if (IsInt2FP) {
3211         // Do a regular integer sign/zero extension then convert to float.
3212         MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltVT.getSizeInBits()),
3213                                       VT.getVectorElementCount());
3214         unsigned ExtOpcode = Op.getOpcode() == ISD::UINT_TO_FP
3215                                  ? ISD::ZERO_EXTEND
3216                                  : ISD::SIGN_EXTEND;
3217         SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
3218         return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
3219       }
3220       // FP2Int
3221       assert(SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering");
3222       // Do one doubling fp_extend then complete the operation by converting
3223       // to int.
3224       MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3225       SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
3226       return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
3227     }
3228 
3229     // Narrowing conversions
3230     if (SrcEltSize > EltSize && (SrcEltSize / EltSize >= 4)) {
3231       if (IsInt2FP) {
3232         // One narrowing int_to_fp, then an fp_round.
3233         assert(EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering");
3234         MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3235         SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
3236         return DAG.getFPExtendOrRound(Int2FP, DL, VT);
3237       }
3238       // FP2Int
3239       // One narrowing fp_to_int, then truncate the integer. If the float isn't
3240       // representable by the integer, the result is poison.
3241       MVT IVecVT =
3242           MVT::getVectorVT(MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2),
3243                            VT.getVectorElementCount());
3244       SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
3245       return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
3246     }
3247 
3248     // Scalable vectors can exit here. Patterns will handle equally-sized
3249     // conversions halving/doubling ones.
3250     if (!VT.isFixedLengthVector())
3251       return Op;
3252 
3253     // For fixed-length vectors we lower to a custom "VL" node.
3254     unsigned RVVOpc = 0;
3255     switch (Op.getOpcode()) {
3256     default:
3257       llvm_unreachable("Impossible opcode");
3258     case ISD::FP_TO_SINT:
3259       RVVOpc = RISCVISD::FP_TO_SINT_VL;
3260       break;
3261     case ISD::FP_TO_UINT:
3262       RVVOpc = RISCVISD::FP_TO_UINT_VL;
3263       break;
3264     case ISD::SINT_TO_FP:
3265       RVVOpc = RISCVISD::SINT_TO_FP_VL;
3266       break;
3267     case ISD::UINT_TO_FP:
3268       RVVOpc = RISCVISD::UINT_TO_FP_VL;
3269       break;
3270     }
3271 
3272     MVT ContainerVT, SrcContainerVT;
3273     // Derive the reference container type from the larger vector type.
3274     if (SrcEltSize > EltSize) {
3275       SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3276       ContainerVT =
3277           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3278     } else {
3279       ContainerVT = getContainerForFixedLengthVector(VT);
3280       SrcContainerVT = ContainerVT.changeVectorElementType(SrcEltVT);
3281     }
3282 
3283     SDValue Mask, VL;
3284     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3285 
3286     Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3287     Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
3288     return convertFromScalableVector(VT, Src, DAG, Subtarget);
3289   }
3290   case ISD::FP_TO_SINT_SAT:
3291   case ISD::FP_TO_UINT_SAT:
3292     return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
3293   case ISD::FTRUNC:
3294   case ISD::FCEIL:
3295   case ISD::FFLOOR:
3296     return lowerFTRUNC_FCEIL_FFLOOR(Op, DAG);
3297   case ISD::VECREDUCE_ADD:
3298   case ISD::VECREDUCE_UMAX:
3299   case ISD::VECREDUCE_SMAX:
3300   case ISD::VECREDUCE_UMIN:
3301   case ISD::VECREDUCE_SMIN:
3302     return lowerVECREDUCE(Op, DAG);
3303   case ISD::VECREDUCE_AND:
3304   case ISD::VECREDUCE_OR:
3305   case ISD::VECREDUCE_XOR:
3306     if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3307       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
3308     return lowerVECREDUCE(Op, DAG);
3309   case ISD::VECREDUCE_FADD:
3310   case ISD::VECREDUCE_SEQ_FADD:
3311   case ISD::VECREDUCE_FMIN:
3312   case ISD::VECREDUCE_FMAX:
3313     return lowerFPVECREDUCE(Op, DAG);
3314   case ISD::VP_REDUCE_ADD:
3315   case ISD::VP_REDUCE_UMAX:
3316   case ISD::VP_REDUCE_SMAX:
3317   case ISD::VP_REDUCE_UMIN:
3318   case ISD::VP_REDUCE_SMIN:
3319   case ISD::VP_REDUCE_FADD:
3320   case ISD::VP_REDUCE_SEQ_FADD:
3321   case ISD::VP_REDUCE_FMIN:
3322   case ISD::VP_REDUCE_FMAX:
3323     return lowerVPREDUCE(Op, DAG);
3324   case ISD::VP_REDUCE_AND:
3325   case ISD::VP_REDUCE_OR:
3326   case ISD::VP_REDUCE_XOR:
3327     if (Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
3328       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
3329     return lowerVPREDUCE(Op, DAG);
3330   case ISD::INSERT_SUBVECTOR:
3331     return lowerINSERT_SUBVECTOR(Op, DAG);
3332   case ISD::EXTRACT_SUBVECTOR:
3333     return lowerEXTRACT_SUBVECTOR(Op, DAG);
3334   case ISD::STEP_VECTOR:
3335     return lowerSTEP_VECTOR(Op, DAG);
3336   case ISD::VECTOR_REVERSE:
3337     return lowerVECTOR_REVERSE(Op, DAG);
3338   case ISD::BUILD_VECTOR:
3339     return lowerBUILD_VECTOR(Op, DAG, Subtarget);
3340   case ISD::SPLAT_VECTOR:
3341     if (Op.getValueType().getVectorElementType() == MVT::i1)
3342       return lowerVectorMaskSplat(Op, DAG);
3343     return lowerSPLAT_VECTOR(Op, DAG, Subtarget);
3344   case ISD::VECTOR_SHUFFLE:
3345     return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
3346   case ISD::CONCAT_VECTORS: {
3347     // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
3348     // better than going through the stack, as the default expansion does.
3349     SDLoc DL(Op);
3350     MVT VT = Op.getSimpleValueType();
3351     unsigned NumOpElts =
3352         Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
3353     SDValue Vec = DAG.getUNDEF(VT);
3354     for (const auto &OpIdx : enumerate(Op->ops())) {
3355       SDValue SubVec = OpIdx.value();
3356       // Don't insert undef subvectors.
3357       if (SubVec.isUndef())
3358         continue;
3359       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec,
3360                         DAG.getIntPtrConstant(OpIdx.index() * NumOpElts, DL));
3361     }
3362     return Vec;
3363   }
3364   case ISD::LOAD:
3365     if (auto V = expandUnalignedRVVLoad(Op, DAG))
3366       return V;
3367     if (Op.getValueType().isFixedLengthVector())
3368       return lowerFixedLengthVectorLoadToRVV(Op, DAG);
3369     return Op;
3370   case ISD::STORE:
3371     if (auto V = expandUnalignedRVVStore(Op, DAG))
3372       return V;
3373     if (Op.getOperand(1).getValueType().isFixedLengthVector())
3374       return lowerFixedLengthVectorStoreToRVV(Op, DAG);
3375     return Op;
3376   case ISD::MLOAD:
3377   case ISD::VP_LOAD:
3378     return lowerMaskedLoad(Op, DAG);
3379   case ISD::MSTORE:
3380   case ISD::VP_STORE:
3381     return lowerMaskedStore(Op, DAG);
3382   case ISD::SETCC:
3383     return lowerFixedLengthVectorSetccToRVV(Op, DAG);
3384   case ISD::ADD:
3385     return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);
3386   case ISD::SUB:
3387     return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL);
3388   case ISD::MUL:
3389     return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL);
3390   case ISD::MULHS:
3391     return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL);
3392   case ISD::MULHU:
3393     return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL);
3394   case ISD::AND:
3395     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMAND_VL,
3396                                               RISCVISD::AND_VL);
3397   case ISD::OR:
3398     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMOR_VL,
3399                                               RISCVISD::OR_VL);
3400   case ISD::XOR:
3401     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMXOR_VL,
3402                                               RISCVISD::XOR_VL);
3403   case ISD::SDIV:
3404     return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL);
3405   case ISD::SREM:
3406     return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL);
3407   case ISD::UDIV:
3408     return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL);
3409   case ISD::UREM:
3410     return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL);
3411   case ISD::SHL:
3412   case ISD::SRA:
3413   case ISD::SRL:
3414     if (Op.getSimpleValueType().isFixedLengthVector())
3415       return lowerFixedLengthVectorShiftToRVV(Op, DAG);
3416     // This can be called for an i32 shift amount that needs to be promoted.
3417     assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
3418            "Unexpected custom legalisation");
3419     return SDValue();
3420   case ISD::SADDSAT:
3421     return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL);
3422   case ISD::UADDSAT:
3423     return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL);
3424   case ISD::SSUBSAT:
3425     return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL);
3426   case ISD::USUBSAT:
3427     return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL);
3428   case ISD::FADD:
3429     return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL);
3430   case ISD::FSUB:
3431     return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL);
3432   case ISD::FMUL:
3433     return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL);
3434   case ISD::FDIV:
3435     return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL);
3436   case ISD::FNEG:
3437     return lowerToScalableOp(Op, DAG, RISCVISD::FNEG_VL);
3438   case ISD::FABS:
3439     return lowerToScalableOp(Op, DAG, RISCVISD::FABS_VL);
3440   case ISD::FSQRT:
3441     return lowerToScalableOp(Op, DAG, RISCVISD::FSQRT_VL);
3442   case ISD::FMA:
3443     return lowerToScalableOp(Op, DAG, RISCVISD::FMA_VL);
3444   case ISD::SMIN:
3445     return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL);
3446   case ISD::SMAX:
3447     return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL);
3448   case ISD::UMIN:
3449     return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL);
3450   case ISD::UMAX:
3451     return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL);
3452   case ISD::FMINNUM:
3453     return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL);
3454   case ISD::FMAXNUM:
3455     return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL);
3456   case ISD::ABS:
3457     return lowerABS(Op, DAG);
3458   case ISD::CTLZ_ZERO_UNDEF:
3459   case ISD::CTTZ_ZERO_UNDEF:
3460     return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
3461   case ISD::VSELECT:
3462     return lowerFixedLengthVectorSelectToRVV(Op, DAG);
3463   case ISD::FCOPYSIGN:
3464     return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG);
3465   case ISD::MGATHER:
3466   case ISD::VP_GATHER:
3467     return lowerMaskedGather(Op, DAG);
3468   case ISD::MSCATTER:
3469   case ISD::VP_SCATTER:
3470     return lowerMaskedScatter(Op, DAG);
3471   case ISD::FLT_ROUNDS_:
3472     return lowerGET_ROUNDING(Op, DAG);
3473   case ISD::SET_ROUNDING:
3474     return lowerSET_ROUNDING(Op, DAG);
3475   case ISD::VP_SELECT:
3476     return lowerVPOp(Op, DAG, RISCVISD::VSELECT_VL);
3477   case ISD::VP_MERGE:
3478     return lowerVPOp(Op, DAG, RISCVISD::VP_MERGE_VL);
3479   case ISD::VP_ADD:
3480     return lowerVPOp(Op, DAG, RISCVISD::ADD_VL);
3481   case ISD::VP_SUB:
3482     return lowerVPOp(Op, DAG, RISCVISD::SUB_VL);
3483   case ISD::VP_MUL:
3484     return lowerVPOp(Op, DAG, RISCVISD::MUL_VL);
3485   case ISD::VP_SDIV:
3486     return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL);
3487   case ISD::VP_UDIV:
3488     return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL);
3489   case ISD::VP_SREM:
3490     return lowerVPOp(Op, DAG, RISCVISD::SREM_VL);
3491   case ISD::VP_UREM:
3492     return lowerVPOp(Op, DAG, RISCVISD::UREM_VL);
3493   case ISD::VP_AND:
3494     return lowerLogicVPOp(Op, DAG, RISCVISD::VMAND_VL, RISCVISD::AND_VL);
3495   case ISD::VP_OR:
3496     return lowerLogicVPOp(Op, DAG, RISCVISD::VMOR_VL, RISCVISD::OR_VL);
3497   case ISD::VP_XOR:
3498     return lowerLogicVPOp(Op, DAG, RISCVISD::VMXOR_VL, RISCVISD::XOR_VL);
3499   case ISD::VP_ASHR:
3500     return lowerVPOp(Op, DAG, RISCVISD::SRA_VL);
3501   case ISD::VP_LSHR:
3502     return lowerVPOp(Op, DAG, RISCVISD::SRL_VL);
3503   case ISD::VP_SHL:
3504     return lowerVPOp(Op, DAG, RISCVISD::SHL_VL);
3505   case ISD::VP_FADD:
3506     return lowerVPOp(Op, DAG, RISCVISD::FADD_VL);
3507   case ISD::VP_FSUB:
3508     return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL);
3509   case ISD::VP_FMUL:
3510     return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL);
3511   case ISD::VP_FDIV:
3512     return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL);
3513   }
3514 }
3515 
3516 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
3517                              SelectionDAG &DAG, unsigned Flags) {
3518   return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
3519 }
3520 
3521 static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
3522                              SelectionDAG &DAG, unsigned Flags) {
3523   return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
3524                                    Flags);
3525 }
3526 
3527 static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
3528                              SelectionDAG &DAG, unsigned Flags) {
3529   return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
3530                                    N->getOffset(), Flags);
3531 }
3532 
3533 static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
3534                              SelectionDAG &DAG, unsigned Flags) {
3535   return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
3536 }
3537 
3538 template <class NodeTy>
3539 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
3540                                      bool IsLocal) const {
3541   SDLoc DL(N);
3542   EVT Ty = getPointerTy(DAG.getDataLayout());
3543 
3544   if (isPositionIndependent()) {
3545     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3546     if (IsLocal)
3547       // Use PC-relative addressing to access the symbol. This generates the
3548       // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
3549       // %pcrel_lo(auipc)).
3550       return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3551 
3552     // Use PC-relative addressing to access the GOT for this symbol, then load
3553     // the address from the GOT. This generates the pattern (PseudoLA sym),
3554     // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
3555     return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
3556   }
3557 
3558   switch (getTargetMachine().getCodeModel()) {
3559   default:
3560     report_fatal_error("Unsupported code model for lowering");
3561   case CodeModel::Small: {
3562     // Generate a sequence for accessing addresses within the first 2 GiB of
3563     // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
3564     SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
3565     SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
3566     SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3567     return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
3568   }
3569   case CodeModel::Medium: {
3570     // Generate a sequence for accessing addresses within any 2GiB range within
3571     // the address space. This generates the pattern (PseudoLLA sym), which
3572     // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
3573     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3574     return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3575   }
3576   }
3577 }
3578 
3579 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
3580                                                 SelectionDAG &DAG) const {
3581   SDLoc DL(Op);
3582   EVT Ty = Op.getValueType();
3583   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3584   int64_t Offset = N->getOffset();
3585   MVT XLenVT = Subtarget.getXLenVT();
3586 
3587   const GlobalValue *GV = N->getGlobal();
3588   bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
3589   SDValue Addr = getAddr(N, DAG, IsLocal);
3590 
3591   // In order to maximise the opportunity for common subexpression elimination,
3592   // emit a separate ADD node for the global address offset instead of folding
3593   // it in the global address node. Later peephole optimisations may choose to
3594   // fold it back in when profitable.
3595   if (Offset != 0)
3596     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3597                        DAG.getConstant(Offset, DL, XLenVT));
3598   return Addr;
3599 }
3600 
3601 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
3602                                                SelectionDAG &DAG) const {
3603   BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
3604 
3605   return getAddr(N, DAG);
3606 }
3607 
3608 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
3609                                                SelectionDAG &DAG) const {
3610   ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
3611 
3612   return getAddr(N, DAG);
3613 }
3614 
3615 SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
3616                                             SelectionDAG &DAG) const {
3617   JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
3618 
3619   return getAddr(N, DAG);
3620 }
3621 
3622 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
3623                                               SelectionDAG &DAG,
3624                                               bool UseGOT) const {
3625   SDLoc DL(N);
3626   EVT Ty = getPointerTy(DAG.getDataLayout());
3627   const GlobalValue *GV = N->getGlobal();
3628   MVT XLenVT = Subtarget.getXLenVT();
3629 
3630   if (UseGOT) {
3631     // Use PC-relative addressing to access the GOT for this TLS symbol, then
3632     // load the address from the GOT and add the thread pointer. This generates
3633     // the pattern (PseudoLA_TLS_IE sym), which expands to
3634     // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
3635     SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3636     SDValue Load =
3637         SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
3638 
3639     // Add the thread pointer.
3640     SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3641     return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
3642   }
3643 
3644   // Generate a sequence for accessing the address relative to the thread
3645   // pointer, with the appropriate adjustment for the thread pointer offset.
3646   // This generates the pattern
3647   // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
3648   SDValue AddrHi =
3649       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
3650   SDValue AddrAdd =
3651       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
3652   SDValue AddrLo =
3653       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
3654 
3655   SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3656   SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3657   SDValue MNAdd = SDValue(
3658       DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
3659       0);
3660   return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
3661 }
3662 
3663 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
3664                                                SelectionDAG &DAG) const {
3665   SDLoc DL(N);
3666   EVT Ty = getPointerTy(DAG.getDataLayout());
3667   IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
3668   const GlobalValue *GV = N->getGlobal();
3669 
3670   // Use a PC-relative addressing mode to access the global dynamic GOT address.
3671   // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
3672   // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
3673   SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3674   SDValue Load =
3675       SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
3676 
3677   // Prepare argument list to generate call.
3678   ArgListTy Args;
3679   ArgListEntry Entry;
3680   Entry.Node = Load;
3681   Entry.Ty = CallTy;
3682   Args.push_back(Entry);
3683 
3684   // Setup call to __tls_get_addr.
3685   TargetLowering::CallLoweringInfo CLI(DAG);
3686   CLI.setDebugLoc(DL)
3687       .setChain(DAG.getEntryNode())
3688       .setLibCallee(CallingConv::C, CallTy,
3689                     DAG.getExternalSymbol("__tls_get_addr", Ty),
3690                     std::move(Args));
3691 
3692   return LowerCallTo(CLI).first;
3693 }
3694 
3695 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
3696                                                    SelectionDAG &DAG) const {
3697   SDLoc DL(Op);
3698   EVT Ty = Op.getValueType();
3699   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3700   int64_t Offset = N->getOffset();
3701   MVT XLenVT = Subtarget.getXLenVT();
3702 
3703   TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());
3704 
3705   if (DAG.getMachineFunction().getFunction().getCallingConv() ==
3706       CallingConv::GHC)
3707     report_fatal_error("In GHC calling convention TLS is not supported");
3708 
3709   SDValue Addr;
3710   switch (Model) {
3711   case TLSModel::LocalExec:
3712     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
3713     break;
3714   case TLSModel::InitialExec:
3715     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
3716     break;
3717   case TLSModel::LocalDynamic:
3718   case TLSModel::GeneralDynamic:
3719     Addr = getDynamicTLSAddr(N, DAG);
3720     break;
3721   }
3722 
3723   // In order to maximise the opportunity for common subexpression elimination,
3724   // emit a separate ADD node for the global address offset instead of folding
3725   // it in the global address node. Later peephole optimisations may choose to
3726   // fold it back in when profitable.
3727   if (Offset != 0)
3728     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3729                        DAG.getConstant(Offset, DL, XLenVT));
3730   return Addr;
3731 }
3732 
3733 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
3734   SDValue CondV = Op.getOperand(0);
3735   SDValue TrueV = Op.getOperand(1);
3736   SDValue FalseV = Op.getOperand(2);
3737   SDLoc DL(Op);
3738   MVT VT = Op.getSimpleValueType();
3739   MVT XLenVT = Subtarget.getXLenVT();
3740 
3741   // Lower vector SELECTs to VSELECTs by splatting the condition.
3742   if (VT.isVector()) {
3743     MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
3744     SDValue CondSplat = VT.isScalableVector()
3745                             ? DAG.getSplatVector(SplatCondVT, DL, CondV)
3746                             : DAG.getSplatBuildVector(SplatCondVT, DL, CondV);
3747     return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
3748   }
3749 
3750   // If the result type is XLenVT and CondV is the output of a SETCC node
3751   // which also operated on XLenVT inputs, then merge the SETCC node into the
3752   // lowered RISCVISD::SELECT_CC to take advantage of the integer
3753   // compare+branch instructions. i.e.:
3754   // (select (setcc lhs, rhs, cc), truev, falsev)
3755   // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
3756   if (VT == XLenVT && CondV.getOpcode() == ISD::SETCC &&
3757       CondV.getOperand(0).getSimpleValueType() == XLenVT) {
3758     SDValue LHS = CondV.getOperand(0);
3759     SDValue RHS = CondV.getOperand(1);
3760     const auto *CC = cast<CondCodeSDNode>(CondV.getOperand(2));
3761     ISD::CondCode CCVal = CC->get();
3762 
3763     // Special case for a select of 2 constants that have a diffence of 1.
3764     // Normally this is done by DAGCombine, but if the select is introduced by
3765     // type legalization or op legalization, we miss it. Restricting to SETLT
3766     // case for now because that is what signed saturating add/sub need.
3767     // FIXME: We don't need the condition to be SETLT or even a SETCC,
3768     // but we would probably want to swap the true/false values if the condition
3769     // is SETGE/SETLE to avoid an XORI.
3770     if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
3771         CCVal == ISD::SETLT) {
3772       const APInt &TrueVal = cast<ConstantSDNode>(TrueV)->getAPIntValue();
3773       const APInt &FalseVal = cast<ConstantSDNode>(FalseV)->getAPIntValue();
3774       if (TrueVal - 1 == FalseVal)
3775         return DAG.getNode(ISD::ADD, DL, Op.getValueType(), CondV, FalseV);
3776       if (TrueVal + 1 == FalseVal)
3777         return DAG.getNode(ISD::SUB, DL, Op.getValueType(), FalseV, CondV);
3778     }
3779 
3780     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3781 
3782     SDValue TargetCC = DAG.getCondCode(CCVal);
3783     SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
3784     return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3785   }
3786 
3787   // Otherwise:
3788   // (select condv, truev, falsev)
3789   // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
3790   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
3791   SDValue SetNE = DAG.getCondCode(ISD::SETNE);
3792 
3793   SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
3794 
3795   return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3796 }
3797 
3798 SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
3799   SDValue CondV = Op.getOperand(1);
3800   SDLoc DL(Op);
3801   MVT XLenVT = Subtarget.getXLenVT();
3802 
3803   if (CondV.getOpcode() == ISD::SETCC &&
3804       CondV.getOperand(0).getValueType() == XLenVT) {
3805     SDValue LHS = CondV.getOperand(0);
3806     SDValue RHS = CondV.getOperand(1);
3807     ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
3808 
3809     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3810 
3811     SDValue TargetCC = DAG.getCondCode(CCVal);
3812     return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3813                        LHS, RHS, TargetCC, Op.getOperand(2));
3814   }
3815 
3816   return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3817                      CondV, DAG.getConstant(0, DL, XLenVT),
3818                      DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
3819 }
3820 
3821 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3822   MachineFunction &MF = DAG.getMachineFunction();
3823   RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
3824 
3825   SDLoc DL(Op);
3826   SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
3827                                  getPointerTy(MF.getDataLayout()));
3828 
3829   // vastart just stores the address of the VarArgsFrameIndex slot into the
3830   // memory location argument.
3831   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3832   return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
3833                       MachinePointerInfo(SV));
3834 }
3835 
3836 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
3837                                             SelectionDAG &DAG) const {
3838   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3839   MachineFunction &MF = DAG.getMachineFunction();
3840   MachineFrameInfo &MFI = MF.getFrameInfo();
3841   MFI.setFrameAddressIsTaken(true);
3842   Register FrameReg = RI.getFrameRegister(MF);
3843   int XLenInBytes = Subtarget.getXLen() / 8;
3844 
3845   EVT VT = Op.getValueType();
3846   SDLoc DL(Op);
3847   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
3848   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3849   while (Depth--) {
3850     int Offset = -(XLenInBytes * 2);
3851     SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
3852                               DAG.getIntPtrConstant(Offset, DL));
3853     FrameAddr =
3854         DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
3855   }
3856   return FrameAddr;
3857 }
3858 
3859 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
3860                                              SelectionDAG &DAG) const {
3861   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3862   MachineFunction &MF = DAG.getMachineFunction();
3863   MachineFrameInfo &MFI = MF.getFrameInfo();
3864   MFI.setReturnAddressIsTaken(true);
3865   MVT XLenVT = Subtarget.getXLenVT();
3866   int XLenInBytes = Subtarget.getXLen() / 8;
3867 
3868   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
3869     return SDValue();
3870 
3871   EVT VT = Op.getValueType();
3872   SDLoc DL(Op);
3873   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3874   if (Depth) {
3875     int Off = -XLenInBytes;
3876     SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
3877     SDValue Offset = DAG.getConstant(Off, DL, VT);
3878     return DAG.getLoad(VT, DL, DAG.getEntryNode(),
3879                        DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
3880                        MachinePointerInfo());
3881   }
3882 
3883   // Return the value of the return address register, marking it an implicit
3884   // live-in.
3885   Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
3886   return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
3887 }
3888 
3889 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
3890                                                  SelectionDAG &DAG) const {
3891   SDLoc DL(Op);
3892   SDValue Lo = Op.getOperand(0);
3893   SDValue Hi = Op.getOperand(1);
3894   SDValue Shamt = Op.getOperand(2);
3895   EVT VT = Lo.getValueType();
3896 
3897   // if Shamt-XLEN < 0: // Shamt < XLEN
3898   //   Lo = Lo << Shamt
3899   //   Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
3900   // else:
3901   //   Lo = 0
3902   //   Hi = Lo << (Shamt-XLEN)
3903 
3904   SDValue Zero = DAG.getConstant(0, DL, VT);
3905   SDValue One = DAG.getConstant(1, DL, VT);
3906   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3907   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3908   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3909   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3910 
3911   SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
3912   SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
3913   SDValue ShiftRightLo =
3914       DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
3915   SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
3916   SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
3917   SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
3918 
3919   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3920 
3921   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
3922   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3923 
3924   SDValue Parts[2] = {Lo, Hi};
3925   return DAG.getMergeValues(Parts, DL);
3926 }
3927 
3928 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
3929                                                   bool IsSRA) const {
3930   SDLoc DL(Op);
3931   SDValue Lo = Op.getOperand(0);
3932   SDValue Hi = Op.getOperand(1);
3933   SDValue Shamt = Op.getOperand(2);
3934   EVT VT = Lo.getValueType();
3935 
3936   // SRA expansion:
3937   //   if Shamt-XLEN < 0: // Shamt < XLEN
3938   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3939   //     Hi = Hi >>s Shamt
3940   //   else:
3941   //     Lo = Hi >>s (Shamt-XLEN);
3942   //     Hi = Hi >>s (XLEN-1)
3943   //
3944   // SRL expansion:
3945   //   if Shamt-XLEN < 0: // Shamt < XLEN
3946   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3947   //     Hi = Hi >>u Shamt
3948   //   else:
3949   //     Lo = Hi >>u (Shamt-XLEN);
3950   //     Hi = 0;
3951 
3952   unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
3953 
3954   SDValue Zero = DAG.getConstant(0, DL, VT);
3955   SDValue One = DAG.getConstant(1, DL, VT);
3956   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3957   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3958   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3959   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3960 
3961   SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
3962   SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
3963   SDValue ShiftLeftHi =
3964       DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
3965   SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
3966   SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
3967   SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
3968   SDValue HiFalse =
3969       IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
3970 
3971   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3972 
3973   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
3974   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3975 
3976   SDValue Parts[2] = {Lo, Hi};
3977   return DAG.getMergeValues(Parts, DL);
3978 }
3979 
3980 // Lower splats of i1 types to SETCC. For each mask vector type, we have a
3981 // legal equivalently-sized i8 type, so we can use that as a go-between.
3982 SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
3983                                                   SelectionDAG &DAG) const {
3984   SDLoc DL(Op);
3985   MVT VT = Op.getSimpleValueType();
3986   SDValue SplatVal = Op.getOperand(0);
3987   // All-zeros or all-ones splats are handled specially.
3988   if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) {
3989     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
3990     return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
3991   }
3992   if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) {
3993     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
3994     return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
3995   }
3996   MVT XLenVT = Subtarget.getXLenVT();
3997   assert(SplatVal.getValueType() == XLenVT &&
3998          "Unexpected type for i1 splat value");
3999   MVT InterVT = VT.changeVectorElementType(MVT::i8);
4000   SplatVal = DAG.getNode(ISD::AND, DL, XLenVT, SplatVal,
4001                          DAG.getConstant(1, DL, XLenVT));
4002   SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
4003   SDValue Zero = DAG.getConstant(0, DL, InterVT);
4004   return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
4005 }
4006 
4007 // Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
4008 // illegal (currently only vXi64 RV32).
4009 // FIXME: We could also catch non-constant sign-extended i32 values and lower
4010 // them to SPLAT_VECTOR_I64
4011 SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
4012                                                      SelectionDAG &DAG) const {
4013   SDLoc DL(Op);
4014   MVT VecVT = Op.getSimpleValueType();
4015   assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
4016          "Unexpected SPLAT_VECTOR_PARTS lowering");
4017 
4018   assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
4019   SDValue Lo = Op.getOperand(0);
4020   SDValue Hi = Op.getOperand(1);
4021 
4022   if (VecVT.isFixedLengthVector()) {
4023     MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
4024     SDLoc DL(Op);
4025     SDValue Mask, VL;
4026     std::tie(Mask, VL) =
4027         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4028 
4029     SDValue Res = splatPartsI64WithVL(DL, ContainerVT, Lo, Hi, VL, DAG);
4030     return convertFromScalableVector(VecVT, Res, DAG, Subtarget);
4031   }
4032 
4033   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
4034     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
4035     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
4036     // If Hi constant is all the same sign bit as Lo, lower this as a custom
4037     // node in order to try and match RVV vector/scalar instructions.
4038     if ((LoC >> 31) == HiC)
4039       return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
4040   }
4041 
4042   // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
4043   if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
4044       isa<ConstantSDNode>(Hi.getOperand(1)) &&
4045       Hi.getConstantOperandVal(1) == 31)
4046     return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
4047 
4048   // Fall back to use a stack store and stride x0 vector load. Use X0 as VL.
4049   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VecVT, Lo, Hi,
4050                      DAG.getTargetConstant(RISCV::VLMaxSentinel, DL, MVT::i64));
4051 }
4052 
4053 // Custom-lower extensions from mask vectors by using a vselect either with 1
4054 // for zero/any-extension or -1 for sign-extension:
4055 //   (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
4056 // Note that any-extension is lowered identically to zero-extension.
4057 SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
4058                                                 int64_t ExtTrueVal) const {
4059   SDLoc DL(Op);
4060   MVT VecVT = Op.getSimpleValueType();
4061   SDValue Src = Op.getOperand(0);
4062   // Only custom-lower extensions from mask types
4063   assert(Src.getValueType().isVector() &&
4064          Src.getValueType().getVectorElementType() == MVT::i1);
4065 
4066   MVT XLenVT = Subtarget.getXLenVT();
4067   SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
4068   SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
4069 
4070   if (VecVT.isScalableVector()) {
4071     // Be careful not to introduce illegal scalar types at this stage, and be
4072     // careful also about splatting constants as on RV32, vXi64 SPLAT_VECTOR is
4073     // illegal and must be expanded. Since we know that the constants are
4074     // sign-extended 32-bit values, we use SPLAT_VECTOR_I64 directly.
4075     bool IsRV32E64 =
4076         !Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64;
4077 
4078     if (!IsRV32E64) {
4079       SplatZero = DAG.getSplatVector(VecVT, DL, SplatZero);
4080       SplatTrueVal = DAG.getSplatVector(VecVT, DL, SplatTrueVal);
4081     } else {
4082       SplatZero = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatZero);
4083       SplatTrueVal =
4084           DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatTrueVal);
4085     }
4086 
4087     return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
4088   }
4089 
4090   MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
4091   MVT I1ContainerVT =
4092       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4093 
4094   SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);
4095 
4096   SDValue Mask, VL;
4097   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4098 
4099   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero, VL);
4100   SplatTrueVal =
4101       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatTrueVal, VL);
4102   SDValue Select = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC,
4103                                SplatTrueVal, SplatZero, VL);
4104 
4105   return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
4106 }
4107 
4108 SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
4109     SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const {
4110   MVT ExtVT = Op.getSimpleValueType();
4111   // Only custom-lower extensions from fixed-length vector types.
4112   if (!ExtVT.isFixedLengthVector())
4113     return Op;
4114   MVT VT = Op.getOperand(0).getSimpleValueType();
4115   // Grab the canonical container type for the extended type. Infer the smaller
4116   // type from that to ensure the same number of vector elements, as we know
4117   // the LMUL will be sufficient to hold the smaller type.
4118   MVT ContainerExtVT = getContainerForFixedLengthVector(ExtVT);
4119   // Get the extended container type manually to ensure the same number of
4120   // vector elements between source and dest.
4121   MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
4122                                      ContainerExtVT.getVectorElementCount());
4123 
4124   SDValue Op1 =
4125       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
4126 
4127   SDLoc DL(Op);
4128   SDValue Mask, VL;
4129   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4130 
4131   SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);
4132 
4133   return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget);
4134 }
4135 
4136 // Custom-lower truncations from vectors to mask vectors by using a mask and a
4137 // setcc operation:
4138 //   (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
4139 SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
4140                                                   SelectionDAG &DAG) const {
4141   SDLoc DL(Op);
4142   EVT MaskVT = Op.getValueType();
4143   // Only expect to custom-lower truncations to mask types
4144   assert(MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 &&
4145          "Unexpected type for vector mask lowering");
4146   SDValue Src = Op.getOperand(0);
4147   MVT VecVT = Src.getSimpleValueType();
4148 
4149   // If this is a fixed vector, we need to convert it to a scalable vector.
4150   MVT ContainerVT = VecVT;
4151   if (VecVT.isFixedLengthVector()) {
4152     ContainerVT = getContainerForFixedLengthVector(VecVT);
4153     Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
4154   }
4155 
4156   SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
4157   SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
4158 
4159   SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatOne);
4160   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero);
4161 
4162   if (VecVT.isScalableVector()) {
4163     SDValue Trunc = DAG.getNode(ISD::AND, DL, VecVT, Src, SplatOne);
4164     return DAG.getSetCC(DL, MaskVT, Trunc, SplatZero, ISD::SETNE);
4165   }
4166 
4167   SDValue Mask, VL;
4168   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4169 
4170   MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
4171   SDValue Trunc =
4172       DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, Mask, VL);
4173   Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, Trunc, SplatZero,
4174                       DAG.getCondCode(ISD::SETNE), Mask, VL);
4175   return convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
4176 }
4177 
4178 // Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
4179 // first position of a vector, and that vector is slid up to the insert index.
4180 // By limiting the active vector length to index+1 and merging with the
4181 // original vector (with an undisturbed tail policy for elements >= VL), we
4182 // achieve the desired result of leaving all elements untouched except the one
4183 // at VL-1, which is replaced with the desired value.
4184 SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4185                                                     SelectionDAG &DAG) const {
4186   SDLoc DL(Op);
4187   MVT VecVT = Op.getSimpleValueType();
4188   SDValue Vec = Op.getOperand(0);
4189   SDValue Val = Op.getOperand(1);
4190   SDValue Idx = Op.getOperand(2);
4191 
4192   if (VecVT.getVectorElementType() == MVT::i1) {
4193     // FIXME: For now we just promote to an i8 vector and insert into that,
4194     // but this is probably not optimal.
4195     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4196     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4197     Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
4198     return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
4199   }
4200 
4201   MVT ContainerVT = VecVT;
4202   // If the operand is a fixed-length vector, convert to a scalable one.
4203   if (VecVT.isFixedLengthVector()) {
4204     ContainerVT = getContainerForFixedLengthVector(VecVT);
4205     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4206   }
4207 
4208   MVT XLenVT = Subtarget.getXLenVT();
4209 
4210   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4211   bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
4212   // Even i64-element vectors on RV32 can be lowered without scalar
4213   // legalization if the most-significant 32 bits of the value are not affected
4214   // by the sign-extension of the lower 32 bits.
4215   // TODO: We could also catch sign extensions of a 32-bit value.
4216   if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
4217     const auto *CVal = cast<ConstantSDNode>(Val);
4218     if (isInt<32>(CVal->getSExtValue())) {
4219       IsLegalInsert = true;
4220       Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4221     }
4222   }
4223 
4224   SDValue Mask, VL;
4225   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4226 
4227   SDValue ValInVec;
4228 
4229   if (IsLegalInsert) {
4230     unsigned Opc =
4231         VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
4232     if (isNullConstant(Idx)) {
4233       Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
4234       if (!VecVT.isFixedLengthVector())
4235         return Vec;
4236       return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
4237     }
4238     ValInVec =
4239         DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Val, VL);
4240   } else {
4241     // On RV32, i64-element vectors must be specially handled to place the
4242     // value at element 0, by using two vslide1up instructions in sequence on
4243     // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
4244     // this.
4245     SDValue One = DAG.getConstant(1, DL, XLenVT);
4246     SDValue ValLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, Zero);
4247     SDValue ValHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, One);
4248     MVT I32ContainerVT =
4249         MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2);
4250     SDValue I32Mask =
4251         getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
4252     // Limit the active VL to two.
4253     SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
4254     // Note: We can't pass a UNDEF to the first VSLIDE1UP_VL since an untied
4255     // undef doesn't obey the earlyclobber constraint. Just splat a zero value.
4256     ValInVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, I32ContainerVT, Zero,
4257                            InsertI64VL);
4258     // First slide in the hi value, then the lo in underneath it.
4259     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4260                            ValHi, I32Mask, InsertI64VL);
4261     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4262                            ValLo, I32Mask, InsertI64VL);
4263     // Bitcast back to the right container type.
4264     ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
4265   }
4266 
4267   // Now that the value is in a vector, slide it into position.
4268   SDValue InsertVL =
4269       DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
4270   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
4271                                 ValInVec, Idx, Mask, InsertVL);
4272   if (!VecVT.isFixedLengthVector())
4273     return Slideup;
4274   return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
4275 }
4276 
4277 // Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
4278 // extract the first element: (extractelt (slidedown vec, idx), 0). For integer
4279 // types this is done using VMV_X_S to allow us to glean information about the
4280 // sign bits of the result.
4281 SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4282                                                      SelectionDAG &DAG) const {
4283   SDLoc DL(Op);
4284   SDValue Idx = Op.getOperand(1);
4285   SDValue Vec = Op.getOperand(0);
4286   EVT EltVT = Op.getValueType();
4287   MVT VecVT = Vec.getSimpleValueType();
4288   MVT XLenVT = Subtarget.getXLenVT();
4289 
4290   if (VecVT.getVectorElementType() == MVT::i1) {
4291     // FIXME: For now we just promote to an i8 vector and extract from that,
4292     // but this is probably not optimal.
4293     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4294     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4295     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
4296   }
4297 
4298   // If this is a fixed vector, we need to convert it to a scalable vector.
4299   MVT ContainerVT = VecVT;
4300   if (VecVT.isFixedLengthVector()) {
4301     ContainerVT = getContainerForFixedLengthVector(VecVT);
4302     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4303   }
4304 
4305   // If the index is 0, the vector is already in the right position.
4306   if (!isNullConstant(Idx)) {
4307     // Use a VL of 1 to avoid processing more elements than we need.
4308     SDValue VL = DAG.getConstant(1, DL, XLenVT);
4309     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4310     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4311     Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
4312                       DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
4313   }
4314 
4315   if (!EltVT.isInteger()) {
4316     // Floating-point extracts are handled in TableGen.
4317     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
4318                        DAG.getConstant(0, DL, XLenVT));
4319   }
4320 
4321   SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
4322   return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
4323 }
4324 
4325 // Some RVV intrinsics may claim that they want an integer operand to be
4326 // promoted or expanded.
4327 static SDValue lowerVectorIntrinsicSplats(SDValue Op, SelectionDAG &DAG,
4328                                           const RISCVSubtarget &Subtarget) {
4329   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4330           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
4331          "Unexpected opcode");
4332 
4333   if (!Subtarget.hasVInstructions())
4334     return SDValue();
4335 
4336   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
4337   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
4338   SDLoc DL(Op);
4339 
4340   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
4341       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
4342   if (!II || !II->hasSplatOperand())
4343     return SDValue();
4344 
4345   unsigned SplatOp = II->SplatOperand + 1 + HasChain;
4346   assert(SplatOp < Op.getNumOperands());
4347 
4348   SmallVector<SDValue, 8> Operands(Op->op_begin(), Op->op_end());
4349   SDValue &ScalarOp = Operands[SplatOp];
4350   MVT OpVT = ScalarOp.getSimpleValueType();
4351   MVT XLenVT = Subtarget.getXLenVT();
4352 
4353   // If this isn't a scalar, or its type is XLenVT we're done.
4354   if (!OpVT.isScalarInteger() || OpVT == XLenVT)
4355     return SDValue();
4356 
4357   // Simplest case is that the operand needs to be promoted to XLenVT.
4358   if (OpVT.bitsLT(XLenVT)) {
4359     // If the operand is a constant, sign extend to increase our chances
4360     // of being able to use a .vi instruction. ANY_EXTEND would become a
4361     // a zero extend and the simm5 check in isel would fail.
4362     // FIXME: Should we ignore the upper bits in isel instead?
4363     unsigned ExtOpc =
4364         isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
4365     ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
4366     return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4367   }
4368 
4369   // Use the previous operand to get the vXi64 VT. The result might be a mask
4370   // VT for compares. Using the previous operand assumes that the previous
4371   // operand will never have a smaller element size than a scalar operand and
4372   // that a widening operation never uses SEW=64.
4373   // NOTE: If this fails the below assert, we can probably just find the
4374   // element count from any operand or result and use it to construct the VT.
4375   assert(II->SplatOperand > 0 && "Unexpected splat operand!");
4376   MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
4377 
4378   // The more complex case is when the scalar is larger than XLenVT.
4379   assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
4380          VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
4381 
4382   // If this is a sign-extended 32-bit constant, we can truncate it and rely
4383   // on the instruction to sign-extend since SEW>XLEN.
4384   if (auto *CVal = dyn_cast<ConstantSDNode>(ScalarOp)) {
4385     if (isInt<32>(CVal->getSExtValue())) {
4386       ScalarOp = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4387       return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4388     }
4389   }
4390 
4391   // We need to convert the scalar to a splat vector.
4392   // FIXME: Can we implicitly truncate the scalar if it is known to
4393   // be sign extended?
4394   SDValue VL = getVLOperand(Op);
4395   assert(VL.getValueType() == XLenVT);
4396   ScalarOp = splatSplitI64WithVL(DL, VT, ScalarOp, VL, DAG);
4397   return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4398 }
4399 
4400 SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
4401                                                      SelectionDAG &DAG) const {
4402   unsigned IntNo = Op.getConstantOperandVal(0);
4403   SDLoc DL(Op);
4404   MVT XLenVT = Subtarget.getXLenVT();
4405 
4406   switch (IntNo) {
4407   default:
4408     break; // Don't custom lower most intrinsics.
4409   case Intrinsic::thread_pointer: {
4410     EVT PtrVT = getPointerTy(DAG.getDataLayout());
4411     return DAG.getRegister(RISCV::X4, PtrVT);
4412   }
4413   case Intrinsic::riscv_orc_b:
4414     // Lower to the GORCI encoding for orc.b.
4415     return DAG.getNode(RISCVISD::GORC, DL, XLenVT, Op.getOperand(1),
4416                        DAG.getConstant(7, DL, XLenVT));
4417   case Intrinsic::riscv_grev:
4418   case Intrinsic::riscv_gorc: {
4419     unsigned Opc =
4420         IntNo == Intrinsic::riscv_grev ? RISCVISD::GREV : RISCVISD::GORC;
4421     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4422   }
4423   case Intrinsic::riscv_shfl:
4424   case Intrinsic::riscv_unshfl: {
4425     unsigned Opc =
4426         IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
4427     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4428   }
4429   case Intrinsic::riscv_bcompress:
4430   case Intrinsic::riscv_bdecompress: {
4431     unsigned Opc = IntNo == Intrinsic::riscv_bcompress ? RISCVISD::BCOMPRESS
4432                                                        : RISCVISD::BDECOMPRESS;
4433     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4434   }
4435   case Intrinsic::riscv_bfp:
4436     return DAG.getNode(RISCVISD::BFP, DL, XLenVT, Op.getOperand(1),
4437                        Op.getOperand(2));
4438   case Intrinsic::riscv_fsl:
4439     return DAG.getNode(RISCVISD::FSL, DL, XLenVT, Op.getOperand(1),
4440                        Op.getOperand(2), Op.getOperand(3));
4441   case Intrinsic::riscv_fsr:
4442     return DAG.getNode(RISCVISD::FSR, DL, XLenVT, Op.getOperand(1),
4443                        Op.getOperand(2), Op.getOperand(3));
4444   case Intrinsic::riscv_vmv_x_s:
4445     assert(Op.getValueType() == XLenVT && "Unexpected VT!");
4446     return DAG.getNode(RISCVISD::VMV_X_S, DL, Op.getValueType(),
4447                        Op.getOperand(1));
4448   case Intrinsic::riscv_vmv_v_x:
4449     return lowerScalarSplat(Op.getOperand(1), Op.getOperand(2),
4450                             Op.getSimpleValueType(), DL, DAG, Subtarget);
4451   case Intrinsic::riscv_vfmv_v_f:
4452     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
4453                        Op.getOperand(1), Op.getOperand(2));
4454   case Intrinsic::riscv_vmv_s_x: {
4455     SDValue Scalar = Op.getOperand(2);
4456 
4457     if (Scalar.getValueType().bitsLE(XLenVT)) {
4458       Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
4459       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
4460                          Op.getOperand(1), Scalar, Op.getOperand(3));
4461     }
4462 
4463     assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
4464 
4465     // This is an i64 value that lives in two scalar registers. We have to
4466     // insert this in a convoluted way. First we build vXi64 splat containing
4467     // the/ two values that we assemble using some bit math. Next we'll use
4468     // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
4469     // to merge element 0 from our splat into the source vector.
4470     // FIXME: This is probably not the best way to do this, but it is
4471     // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
4472     // point.
4473     //   sw lo, (a0)
4474     //   sw hi, 4(a0)
4475     //   vlse vX, (a0)
4476     //
4477     //   vid.v      vVid
4478     //   vmseq.vx   mMask, vVid, 0
4479     //   vmerge.vvm vDest, vSrc, vVal, mMask
4480     MVT VT = Op.getSimpleValueType();
4481     SDValue Vec = Op.getOperand(1);
4482     SDValue VL = getVLOperand(Op);
4483 
4484     SDValue SplattedVal = splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
4485     SDValue SplattedIdx = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
4486                                       DAG.getConstant(0, DL, MVT::i32), VL);
4487 
4488     MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
4489     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4490     SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
4491     SDValue SelectCond =
4492         DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, VID, SplattedIdx,
4493                     DAG.getCondCode(ISD::SETEQ), Mask, VL);
4494     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, SelectCond, SplattedVal,
4495                        Vec, VL);
4496   }
4497   case Intrinsic::riscv_vslide1up:
4498   case Intrinsic::riscv_vslide1down:
4499   case Intrinsic::riscv_vslide1up_mask:
4500   case Intrinsic::riscv_vslide1down_mask: {
4501     // We need to special case these when the scalar is larger than XLen.
4502     unsigned NumOps = Op.getNumOperands();
4503     bool IsMasked = NumOps == 7;
4504     unsigned OpOffset = IsMasked ? 1 : 0;
4505     SDValue Scalar = Op.getOperand(2 + OpOffset);
4506     if (Scalar.getValueType().bitsLE(XLenVT))
4507       break;
4508 
4509     // Splatting a sign extended constant is fine.
4510     if (auto *CVal = dyn_cast<ConstantSDNode>(Scalar))
4511       if (isInt<32>(CVal->getSExtValue()))
4512         break;
4513 
4514     MVT VT = Op.getSimpleValueType();
4515     assert(VT.getVectorElementType() == MVT::i64 &&
4516            Scalar.getValueType() == MVT::i64 && "Unexpected VTs");
4517 
4518     // Convert the vector source to the equivalent nxvXi32 vector.
4519     MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
4520     SDValue Vec = DAG.getBitcast(I32VT, Op.getOperand(1 + OpOffset));
4521 
4522     SDValue ScalarLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4523                                    DAG.getConstant(0, DL, XLenVT));
4524     SDValue ScalarHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4525                                    DAG.getConstant(1, DL, XLenVT));
4526 
4527     // Double the VL since we halved SEW.
4528     SDValue VL = getVLOperand(Op);
4529     SDValue I32VL =
4530         DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
4531 
4532     MVT I32MaskVT = MVT::getVectorVT(MVT::i1, I32VT.getVectorElementCount());
4533     SDValue I32Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, I32MaskVT, VL);
4534 
4535     // Shift the two scalar parts in using SEW=32 slide1up/slide1down
4536     // instructions.
4537     if (IntNo == Intrinsic::riscv_vslide1up ||
4538         IntNo == Intrinsic::riscv_vslide1up_mask) {
4539       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarHi,
4540                         I32Mask, I32VL);
4541       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarLo,
4542                         I32Mask, I32VL);
4543     } else {
4544       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarLo,
4545                         I32Mask, I32VL);
4546       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarHi,
4547                         I32Mask, I32VL);
4548     }
4549 
4550     // Convert back to nxvXi64.
4551     Vec = DAG.getBitcast(VT, Vec);
4552 
4553     if (!IsMasked)
4554       return Vec;
4555 
4556     // Apply mask after the operation.
4557     SDValue Mask = Op.getOperand(NumOps - 3);
4558     SDValue MaskedOff = Op.getOperand(1);
4559     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, Mask, Vec, MaskedOff, VL);
4560   }
4561   }
4562 
4563   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4564 }
4565 
4566 SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
4567                                                     SelectionDAG &DAG) const {
4568   unsigned IntNo = Op.getConstantOperandVal(1);
4569   switch (IntNo) {
4570   default:
4571     break;
4572   case Intrinsic::riscv_masked_strided_load: {
4573     SDLoc DL(Op);
4574     MVT XLenVT = Subtarget.getXLenVT();
4575 
4576     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4577     // the selection of the masked intrinsics doesn't do this for us.
4578     SDValue Mask = Op.getOperand(5);
4579     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4580 
4581     MVT VT = Op->getSimpleValueType(0);
4582     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4583 
4584     SDValue PassThru = Op.getOperand(2);
4585     if (!IsUnmasked) {
4586       MVT MaskVT =
4587           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4588       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4589       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
4590     }
4591 
4592     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4593 
4594     SDValue IntID = DAG.getTargetConstant(
4595         IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask, DL,
4596         XLenVT);
4597 
4598     auto *Load = cast<MemIntrinsicSDNode>(Op);
4599     SmallVector<SDValue, 8> Ops{Load->getChain(), IntID};
4600     if (IsUnmasked)
4601       Ops.push_back(DAG.getUNDEF(ContainerVT));
4602     else
4603       Ops.push_back(PassThru);
4604     Ops.push_back(Op.getOperand(3)); // Ptr
4605     Ops.push_back(Op.getOperand(4)); // Stride
4606     if (!IsUnmasked)
4607       Ops.push_back(Mask);
4608     Ops.push_back(VL);
4609     if (!IsUnmasked) {
4610       SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
4611       Ops.push_back(Policy);
4612     }
4613 
4614     SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
4615     SDValue Result =
4616         DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
4617                                 Load->getMemoryVT(), Load->getMemOperand());
4618     SDValue Chain = Result.getValue(1);
4619     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
4620     return DAG.getMergeValues({Result, Chain}, DL);
4621   }
4622   }
4623 
4624   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4625 }
4626 
4627 SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
4628                                                  SelectionDAG &DAG) const {
4629   unsigned IntNo = Op.getConstantOperandVal(1);
4630   switch (IntNo) {
4631   default:
4632     break;
4633   case Intrinsic::riscv_masked_strided_store: {
4634     SDLoc DL(Op);
4635     MVT XLenVT = Subtarget.getXLenVT();
4636 
4637     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4638     // the selection of the masked intrinsics doesn't do this for us.
4639     SDValue Mask = Op.getOperand(5);
4640     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4641 
4642     SDValue Val = Op.getOperand(2);
4643     MVT VT = Val.getSimpleValueType();
4644     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4645 
4646     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
4647     if (!IsUnmasked) {
4648       MVT MaskVT =
4649           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4650       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4651     }
4652 
4653     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4654 
4655     SDValue IntID = DAG.getTargetConstant(
4656         IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask, DL,
4657         XLenVT);
4658 
4659     auto *Store = cast<MemIntrinsicSDNode>(Op);
4660     SmallVector<SDValue, 8> Ops{Store->getChain(), IntID};
4661     Ops.push_back(Val);
4662     Ops.push_back(Op.getOperand(3)); // Ptr
4663     Ops.push_back(Op.getOperand(4)); // Stride
4664     if (!IsUnmasked)
4665       Ops.push_back(Mask);
4666     Ops.push_back(VL);
4667 
4668     return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(),
4669                                    Ops, Store->getMemoryVT(),
4670                                    Store->getMemOperand());
4671   }
4672   }
4673 
4674   return SDValue();
4675 }
4676 
4677 static MVT getLMUL1VT(MVT VT) {
4678   assert(VT.getVectorElementType().getSizeInBits() <= 64 &&
4679          "Unexpected vector MVT");
4680   return MVT::getScalableVectorVT(
4681       VT.getVectorElementType(),
4682       RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits());
4683 }
4684 
4685 static unsigned getRVVReductionOp(unsigned ISDOpcode) {
4686   switch (ISDOpcode) {
4687   default:
4688     llvm_unreachable("Unhandled reduction");
4689   case ISD::VECREDUCE_ADD:
4690     return RISCVISD::VECREDUCE_ADD_VL;
4691   case ISD::VECREDUCE_UMAX:
4692     return RISCVISD::VECREDUCE_UMAX_VL;
4693   case ISD::VECREDUCE_SMAX:
4694     return RISCVISD::VECREDUCE_SMAX_VL;
4695   case ISD::VECREDUCE_UMIN:
4696     return RISCVISD::VECREDUCE_UMIN_VL;
4697   case ISD::VECREDUCE_SMIN:
4698     return RISCVISD::VECREDUCE_SMIN_VL;
4699   case ISD::VECREDUCE_AND:
4700     return RISCVISD::VECREDUCE_AND_VL;
4701   case ISD::VECREDUCE_OR:
4702     return RISCVISD::VECREDUCE_OR_VL;
4703   case ISD::VECREDUCE_XOR:
4704     return RISCVISD::VECREDUCE_XOR_VL;
4705   }
4706 }
4707 
4708 SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
4709                                                          SelectionDAG &DAG,
4710                                                          bool IsVP) const {
4711   SDLoc DL(Op);
4712   SDValue Vec = Op.getOperand(IsVP ? 1 : 0);
4713   MVT VecVT = Vec.getSimpleValueType();
4714   assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
4715           Op.getOpcode() == ISD::VECREDUCE_OR ||
4716           Op.getOpcode() == ISD::VECREDUCE_XOR ||
4717           Op.getOpcode() == ISD::VP_REDUCE_AND ||
4718           Op.getOpcode() == ISD::VP_REDUCE_OR ||
4719           Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
4720          "Unexpected reduction lowering");
4721 
4722   MVT XLenVT = Subtarget.getXLenVT();
4723   assert(Op.getValueType() == XLenVT &&
4724          "Expected reduction output to be legalized to XLenVT");
4725 
4726   MVT ContainerVT = VecVT;
4727   if (VecVT.isFixedLengthVector()) {
4728     ContainerVT = getContainerForFixedLengthVector(VecVT);
4729     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4730   }
4731 
4732   SDValue Mask, VL;
4733   if (IsVP) {
4734     Mask = Op.getOperand(2);
4735     VL = Op.getOperand(3);
4736   } else {
4737     std::tie(Mask, VL) =
4738         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4739   }
4740 
4741   unsigned BaseOpc;
4742   ISD::CondCode CC;
4743   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4744 
4745   switch (Op.getOpcode()) {
4746   default:
4747     llvm_unreachable("Unhandled reduction");
4748   case ISD::VECREDUCE_AND:
4749   case ISD::VP_REDUCE_AND: {
4750     // vcpop ~x == 0
4751     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
4752     Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
4753     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4754     CC = ISD::SETEQ;
4755     BaseOpc = ISD::AND;
4756     break;
4757   }
4758   case ISD::VECREDUCE_OR:
4759   case ISD::VP_REDUCE_OR:
4760     // vcpop x != 0
4761     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4762     CC = ISD::SETNE;
4763     BaseOpc = ISD::OR;
4764     break;
4765   case ISD::VECREDUCE_XOR:
4766   case ISD::VP_REDUCE_XOR: {
4767     // ((vcpop x) & 1) != 0
4768     SDValue One = DAG.getConstant(1, DL, XLenVT);
4769     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4770     Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
4771     CC = ISD::SETNE;
4772     BaseOpc = ISD::XOR;
4773     break;
4774   }
4775   }
4776 
4777   SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC);
4778 
4779   if (!IsVP)
4780     return SetCC;
4781 
4782   // Now include the start value in the operation.
4783   // Note that we must return the start value when no elements are operated
4784   // upon. The vcpop instructions we've emitted in each case above will return
4785   // 0 for an inactive vector, and so we've already received the neutral value:
4786   // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
4787   // can simply include the start value.
4788   return DAG.getNode(BaseOpc, DL, XLenVT, SetCC, Op.getOperand(0));
4789 }
4790 
4791 SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
4792                                             SelectionDAG &DAG) const {
4793   SDLoc DL(Op);
4794   SDValue Vec = Op.getOperand(0);
4795   EVT VecEVT = Vec.getValueType();
4796 
4797   unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
4798 
4799   // Due to ordering in legalize types we may have a vector type that needs to
4800   // be split. Do that manually so we can get down to a legal type.
4801   while (getTypeAction(*DAG.getContext(), VecEVT) ==
4802          TargetLowering::TypeSplitVector) {
4803     SDValue Lo, Hi;
4804     std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
4805     VecEVT = Lo.getValueType();
4806     Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
4807   }
4808 
4809   // TODO: The type may need to be widened rather than split. Or widened before
4810   // it can be split.
4811   if (!isTypeLegal(VecEVT))
4812     return SDValue();
4813 
4814   MVT VecVT = VecEVT.getSimpleVT();
4815   MVT VecEltVT = VecVT.getVectorElementType();
4816   unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());
4817 
4818   MVT ContainerVT = VecVT;
4819   if (VecVT.isFixedLengthVector()) {
4820     ContainerVT = getContainerForFixedLengthVector(VecVT);
4821     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4822   }
4823 
4824   MVT M1VT = getLMUL1VT(ContainerVT);
4825   MVT XLenVT = Subtarget.getXLenVT();
4826 
4827   SDValue Mask, VL;
4828   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4829 
4830   SDValue NeutralElem =
4831       DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
4832   SDValue IdentitySplat = lowerScalarSplat(
4833       NeutralElem, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
4834   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT), Vec,
4835                                   IdentitySplat, Mask, VL);
4836   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
4837                              DAG.getConstant(0, DL, XLenVT));
4838   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
4839 }
4840 
4841 // Given a reduction op, this function returns the matching reduction opcode,
4842 // the vector SDValue and the scalar SDValue required to lower this to a
4843 // RISCVISD node.
4844 static std::tuple<unsigned, SDValue, SDValue>
4845 getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) {
4846   SDLoc DL(Op);
4847   auto Flags = Op->getFlags();
4848   unsigned Opcode = Op.getOpcode();
4849   unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Opcode);
4850   switch (Opcode) {
4851   default:
4852     llvm_unreachable("Unhandled reduction");
4853   case ISD::VECREDUCE_FADD: {
4854     // Use positive zero if we can. It is cheaper to materialize.
4855     SDValue Zero =
4856         DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
4857     return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero);
4858   }
4859   case ISD::VECREDUCE_SEQ_FADD:
4860     return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
4861                            Op.getOperand(0));
4862   case ISD::VECREDUCE_FMIN:
4863     return std::make_tuple(RISCVISD::VECREDUCE_FMIN_VL, Op.getOperand(0),
4864                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
4865   case ISD::VECREDUCE_FMAX:
4866     return std::make_tuple(RISCVISD::VECREDUCE_FMAX_VL, Op.getOperand(0),
4867                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
4868   }
4869 }
4870 
4871 SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
4872                                               SelectionDAG &DAG) const {
4873   SDLoc DL(Op);
4874   MVT VecEltVT = Op.getSimpleValueType();
4875 
4876   unsigned RVVOpcode;
4877   SDValue VectorVal, ScalarVal;
4878   std::tie(RVVOpcode, VectorVal, ScalarVal) =
4879       getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT);
4880   MVT VecVT = VectorVal.getSimpleValueType();
4881 
4882   MVT ContainerVT = VecVT;
4883   if (VecVT.isFixedLengthVector()) {
4884     ContainerVT = getContainerForFixedLengthVector(VecVT);
4885     VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
4886   }
4887 
4888   MVT M1VT = getLMUL1VT(VectorVal.getSimpleValueType());
4889   MVT XLenVT = Subtarget.getXLenVT();
4890 
4891   SDValue Mask, VL;
4892   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4893 
4894   SDValue ScalarSplat = lowerScalarSplat(
4895       ScalarVal, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
4896   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT),
4897                                   VectorVal, ScalarSplat, Mask, VL);
4898   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
4899                      DAG.getConstant(0, DL, XLenVT));
4900 }
4901 
4902 static unsigned getRVVVPReductionOp(unsigned ISDOpcode) {
4903   switch (ISDOpcode) {
4904   default:
4905     llvm_unreachable("Unhandled reduction");
4906   case ISD::VP_REDUCE_ADD:
4907     return RISCVISD::VECREDUCE_ADD_VL;
4908   case ISD::VP_REDUCE_UMAX:
4909     return RISCVISD::VECREDUCE_UMAX_VL;
4910   case ISD::VP_REDUCE_SMAX:
4911     return RISCVISD::VECREDUCE_SMAX_VL;
4912   case ISD::VP_REDUCE_UMIN:
4913     return RISCVISD::VECREDUCE_UMIN_VL;
4914   case ISD::VP_REDUCE_SMIN:
4915     return RISCVISD::VECREDUCE_SMIN_VL;
4916   case ISD::VP_REDUCE_AND:
4917     return RISCVISD::VECREDUCE_AND_VL;
4918   case ISD::VP_REDUCE_OR:
4919     return RISCVISD::VECREDUCE_OR_VL;
4920   case ISD::VP_REDUCE_XOR:
4921     return RISCVISD::VECREDUCE_XOR_VL;
4922   case ISD::VP_REDUCE_FADD:
4923     return RISCVISD::VECREDUCE_FADD_VL;
4924   case ISD::VP_REDUCE_SEQ_FADD:
4925     return RISCVISD::VECREDUCE_SEQ_FADD_VL;
4926   case ISD::VP_REDUCE_FMAX:
4927     return RISCVISD::VECREDUCE_FMAX_VL;
4928   case ISD::VP_REDUCE_FMIN:
4929     return RISCVISD::VECREDUCE_FMIN_VL;
4930   }
4931 }
4932 
4933 SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
4934                                            SelectionDAG &DAG) const {
4935   SDLoc DL(Op);
4936   SDValue Vec = Op.getOperand(1);
4937   EVT VecEVT = Vec.getValueType();
4938 
4939   // TODO: The type may need to be widened rather than split. Or widened before
4940   // it can be split.
4941   if (!isTypeLegal(VecEVT))
4942     return SDValue();
4943 
4944   MVT VecVT = VecEVT.getSimpleVT();
4945   MVT VecEltVT = VecVT.getVectorElementType();
4946   unsigned RVVOpcode = getRVVVPReductionOp(Op.getOpcode());
4947 
4948   MVT ContainerVT = VecVT;
4949   if (VecVT.isFixedLengthVector()) {
4950     ContainerVT = getContainerForFixedLengthVector(VecVT);
4951     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4952   }
4953 
4954   SDValue VL = Op.getOperand(3);
4955   SDValue Mask = Op.getOperand(2);
4956 
4957   MVT M1VT = getLMUL1VT(ContainerVT);
4958   MVT XLenVT = Subtarget.getXLenVT();
4959   MVT ResVT = !VecVT.isInteger() || VecEltVT.bitsGE(XLenVT) ? VecEltVT : XLenVT;
4960 
4961   SDValue StartSplat =
4962       lowerScalarSplat(Op.getOperand(0), DAG.getConstant(1, DL, XLenVT), M1VT,
4963                        DL, DAG, Subtarget);
4964   SDValue Reduction =
4965       DAG.getNode(RVVOpcode, DL, M1VT, StartSplat, Vec, StartSplat, Mask, VL);
4966   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction,
4967                              DAG.getConstant(0, DL, XLenVT));
4968   if (!VecVT.isInteger())
4969     return Elt0;
4970   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
4971 }
4972 
4973 SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
4974                                                    SelectionDAG &DAG) const {
4975   SDValue Vec = Op.getOperand(0);
4976   SDValue SubVec = Op.getOperand(1);
4977   MVT VecVT = Vec.getSimpleValueType();
4978   MVT SubVecVT = SubVec.getSimpleValueType();
4979 
4980   SDLoc DL(Op);
4981   MVT XLenVT = Subtarget.getXLenVT();
4982   unsigned OrigIdx = Op.getConstantOperandVal(2);
4983   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
4984 
4985   // We don't have the ability to slide mask vectors up indexed by their i1
4986   // elements; the smallest we can do is i8. Often we are able to bitcast to
4987   // equivalent i8 vectors. Note that when inserting a fixed-length vector
4988   // into a scalable one, we might not necessarily have enough scalable
4989   // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
4990   if (SubVecVT.getVectorElementType() == MVT::i1 &&
4991       (OrigIdx != 0 || !Vec.isUndef())) {
4992     if (VecVT.getVectorMinNumElements() >= 8 &&
4993         SubVecVT.getVectorMinNumElements() >= 8) {
4994       assert(OrigIdx % 8 == 0 && "Invalid index");
4995       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
4996              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
4997              "Unexpected mask vector lowering");
4998       OrigIdx /= 8;
4999       SubVecVT =
5000           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
5001                            SubVecVT.isScalableVector());
5002       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
5003                                VecVT.isScalableVector());
5004       Vec = DAG.getBitcast(VecVT, Vec);
5005       SubVec = DAG.getBitcast(SubVecVT, SubVec);
5006     } else {
5007       // We can't slide this mask vector up indexed by its i1 elements.
5008       // This poses a problem when we wish to insert a scalable vector which
5009       // can't be re-expressed as a larger type. Just choose the slow path and
5010       // extend to a larger type, then truncate back down.
5011       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
5012       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
5013       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
5014       SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
5015       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
5016                         Op.getOperand(2));
5017       SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
5018       return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
5019     }
5020   }
5021 
5022   // If the subvector vector is a fixed-length type, we cannot use subregister
5023   // manipulation to simplify the codegen; we don't know which register of a
5024   // LMUL group contains the specific subvector as we only know the minimum
5025   // register size. Therefore we must slide the vector group up the full
5026   // amount.
5027   if (SubVecVT.isFixedLengthVector()) {
5028     if (OrigIdx == 0 && Vec.isUndef() && !VecVT.isFixedLengthVector())
5029       return Op;
5030     MVT ContainerVT = VecVT;
5031     if (VecVT.isFixedLengthVector()) {
5032       ContainerVT = getContainerForFixedLengthVector(VecVT);
5033       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5034     }
5035     SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
5036                          DAG.getUNDEF(ContainerVT), SubVec,
5037                          DAG.getConstant(0, DL, XLenVT));
5038     if (OrigIdx == 0 && Vec.isUndef() && VecVT.isFixedLengthVector()) {
5039       SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
5040       return DAG.getBitcast(Op.getValueType(), SubVec);
5041     }
5042     SDValue Mask =
5043         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5044     // Set the vector length to only the number of elements we care about. Note
5045     // that for slideup this includes the offset.
5046     SDValue VL =
5047         DAG.getConstant(OrigIdx + SubVecVT.getVectorNumElements(), DL, XLenVT);
5048     SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5049     SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
5050                                   SubVec, SlideupAmt, Mask, VL);
5051     if (VecVT.isFixedLengthVector())
5052       Slideup = convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
5053     return DAG.getBitcast(Op.getValueType(), Slideup);
5054   }
5055 
5056   unsigned SubRegIdx, RemIdx;
5057   std::tie(SubRegIdx, RemIdx) =
5058       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5059           VecVT, SubVecVT, OrigIdx, TRI);
5060 
5061   RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecVT);
5062   bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
5063                          SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
5064                          SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
5065 
5066   // 1. If the Idx has been completely eliminated and this subvector's size is
5067   // a vector register or a multiple thereof, or the surrounding elements are
5068   // undef, then this is a subvector insert which naturally aligns to a vector
5069   // register. These can easily be handled using subregister manipulation.
5070   // 2. If the subvector is smaller than a vector register, then the insertion
5071   // must preserve the undisturbed elements of the register. We do this by
5072   // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
5073   // (which resolves to a subregister copy), performing a VSLIDEUP to place the
5074   // subvector within the vector register, and an INSERT_SUBVECTOR of that
5075   // LMUL=1 type back into the larger vector (resolving to another subregister
5076   // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
5077   // to avoid allocating a large register group to hold our subvector.
5078   if (RemIdx == 0 && (!IsSubVecPartReg || Vec.isUndef()))
5079     return Op;
5080 
5081   // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
5082   // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
5083   // (in our case undisturbed). This means we can set up a subvector insertion
5084   // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
5085   // size of the subvector.
5086   MVT InterSubVT = VecVT;
5087   SDValue AlignedExtract = Vec;
5088   unsigned AlignedIdx = OrigIdx - RemIdx;
5089   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5090     InterSubVT = getLMUL1VT(VecVT);
5091     // Extract a subvector equal to the nearest full vector register type. This
5092     // should resolve to a EXTRACT_SUBREG instruction.
5093     AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5094                                  DAG.getConstant(AlignedIdx, DL, XLenVT));
5095   }
5096 
5097   SDValue SlideupAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5098   // For scalable vectors this must be further multiplied by vscale.
5099   SlideupAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlideupAmt);
5100 
5101   SDValue Mask, VL;
5102   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5103 
5104   // Construct the vector length corresponding to RemIdx + length(SubVecVT).
5105   VL = DAG.getConstant(SubVecVT.getVectorMinNumElements(), DL, XLenVT);
5106   VL = DAG.getNode(ISD::VSCALE, DL, XLenVT, VL);
5107   VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
5108 
5109   SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
5110                        DAG.getUNDEF(InterSubVT), SubVec,
5111                        DAG.getConstant(0, DL, XLenVT));
5112 
5113   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, InterSubVT,
5114                                 AlignedExtract, SubVec, SlideupAmt, Mask, VL);
5115 
5116   // If required, insert this subvector back into the correct vector register.
5117   // This should resolve to an INSERT_SUBREG instruction.
5118   if (VecVT.bitsGT(InterSubVT))
5119     Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, Slideup,
5120                           DAG.getConstant(AlignedIdx, DL, XLenVT));
5121 
5122   // We might have bitcast from a mask type: cast back to the original type if
5123   // required.
5124   return DAG.getBitcast(Op.getSimpleValueType(), Slideup);
5125 }
5126 
5127 SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
5128                                                     SelectionDAG &DAG) const {
5129   SDValue Vec = Op.getOperand(0);
5130   MVT SubVecVT = Op.getSimpleValueType();
5131   MVT VecVT = Vec.getSimpleValueType();
5132 
5133   SDLoc DL(Op);
5134   MVT XLenVT = Subtarget.getXLenVT();
5135   unsigned OrigIdx = Op.getConstantOperandVal(1);
5136   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
5137 
5138   // We don't have the ability to slide mask vectors down indexed by their i1
5139   // elements; the smallest we can do is i8. Often we are able to bitcast to
5140   // equivalent i8 vectors. Note that when extracting a fixed-length vector
5141   // from a scalable one, we might not necessarily have enough scalable
5142   // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
5143   if (SubVecVT.getVectorElementType() == MVT::i1 && OrigIdx != 0) {
5144     if (VecVT.getVectorMinNumElements() >= 8 &&
5145         SubVecVT.getVectorMinNumElements() >= 8) {
5146       assert(OrigIdx % 8 == 0 && "Invalid index");
5147       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
5148              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
5149              "Unexpected mask vector lowering");
5150       OrigIdx /= 8;
5151       SubVecVT =
5152           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
5153                            SubVecVT.isScalableVector());
5154       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
5155                                VecVT.isScalableVector());
5156       Vec = DAG.getBitcast(VecVT, Vec);
5157     } else {
5158       // We can't slide this mask vector down, indexed by its i1 elements.
5159       // This poses a problem when we wish to extract a scalable vector which
5160       // can't be re-expressed as a larger type. Just choose the slow path and
5161       // extend to a larger type, then truncate back down.
5162       // TODO: We could probably improve this when extracting certain fixed
5163       // from fixed, where we can extract as i8 and shift the correct element
5164       // right to reach the desired subvector?
5165       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
5166       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
5167       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
5168       Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
5169                         Op.getOperand(1));
5170       SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
5171       return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
5172     }
5173   }
5174 
5175   // If the subvector vector is a fixed-length type, we cannot use subregister
5176   // manipulation to simplify the codegen; we don't know which register of a
5177   // LMUL group contains the specific subvector as we only know the minimum
5178   // register size. Therefore we must slide the vector group down the full
5179   // amount.
5180   if (SubVecVT.isFixedLengthVector()) {
5181     // With an index of 0 this is a cast-like subvector, which can be performed
5182     // with subregister operations.
5183     if (OrigIdx == 0)
5184       return Op;
5185     MVT ContainerVT = VecVT;
5186     if (VecVT.isFixedLengthVector()) {
5187       ContainerVT = getContainerForFixedLengthVector(VecVT);
5188       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5189     }
5190     SDValue Mask =
5191         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5192     // Set the vector length to only the number of elements we care about. This
5193     // avoids sliding down elements we're going to discard straight away.
5194     SDValue VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
5195     SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5196     SDValue Slidedown =
5197         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
5198                     DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
5199     // Now we can use a cast-like subvector extract to get the result.
5200     Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5201                             DAG.getConstant(0, DL, XLenVT));
5202     return DAG.getBitcast(Op.getValueType(), Slidedown);
5203   }
5204 
5205   unsigned SubRegIdx, RemIdx;
5206   std::tie(SubRegIdx, RemIdx) =
5207       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5208           VecVT, SubVecVT, OrigIdx, TRI);
5209 
5210   // If the Idx has been completely eliminated then this is a subvector extract
5211   // which naturally aligns to a vector register. These can easily be handled
5212   // using subregister manipulation.
5213   if (RemIdx == 0)
5214     return Op;
5215 
5216   // Else we must shift our vector register directly to extract the subvector.
5217   // Do this using VSLIDEDOWN.
5218 
5219   // If the vector type is an LMUL-group type, extract a subvector equal to the
5220   // nearest full vector register type. This should resolve to a EXTRACT_SUBREG
5221   // instruction.
5222   MVT InterSubVT = VecVT;
5223   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5224     InterSubVT = getLMUL1VT(VecVT);
5225     Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5226                       DAG.getConstant(OrigIdx - RemIdx, DL, XLenVT));
5227   }
5228 
5229   // Slide this vector register down by the desired number of elements in order
5230   // to place the desired subvector starting at element 0.
5231   SDValue SlidedownAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5232   // For scalable vectors this must be further multiplied by vscale.
5233   SlidedownAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlidedownAmt);
5234 
5235   SDValue Mask, VL;
5236   std::tie(Mask, VL) = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
5237   SDValue Slidedown =
5238       DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, InterSubVT,
5239                   DAG.getUNDEF(InterSubVT), Vec, SlidedownAmt, Mask, VL);
5240 
5241   // Now the vector is in the right position, extract our final subvector. This
5242   // should resolve to a COPY.
5243   Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5244                           DAG.getConstant(0, DL, XLenVT));
5245 
5246   // We might have bitcast from a mask type: cast back to the original type if
5247   // required.
5248   return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
5249 }
5250 
5251 // Lower step_vector to the vid instruction. Any non-identity step value must
5252 // be accounted for my manual expansion.
5253 SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
5254                                               SelectionDAG &DAG) const {
5255   SDLoc DL(Op);
5256   MVT VT = Op.getSimpleValueType();
5257   MVT XLenVT = Subtarget.getXLenVT();
5258   SDValue Mask, VL;
5259   std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
5260   SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
5261   uint64_t StepValImm = Op.getConstantOperandVal(0);
5262   if (StepValImm != 1) {
5263     if (isPowerOf2_64(StepValImm)) {
5264       SDValue StepVal =
5265           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
5266                       DAG.getConstant(Log2_64(StepValImm), DL, XLenVT));
5267       StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
5268     } else {
5269       SDValue StepVal = lowerScalarSplat(
5270           DAG.getConstant(StepValImm, DL, VT.getVectorElementType()), VL, VT,
5271           DL, DAG, Subtarget);
5272       StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
5273     }
5274   }
5275   return StepVec;
5276 }
5277 
5278 // Implement vector_reverse using vrgather.vv with indices determined by
5279 // subtracting the id of each element from (VLMAX-1). This will convert
5280 // the indices like so:
5281 // (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
5282 // TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
5283 SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
5284                                                  SelectionDAG &DAG) const {
5285   SDLoc DL(Op);
5286   MVT VecVT = Op.getSimpleValueType();
5287   unsigned EltSize = VecVT.getScalarSizeInBits();
5288   unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
5289 
5290   unsigned MaxVLMAX = 0;
5291   unsigned VectorBitsMax = Subtarget.getMaxRVVVectorSizeInBits();
5292   if (VectorBitsMax != 0)
5293     MaxVLMAX = ((VectorBitsMax / EltSize) * MinSize) / RISCV::RVVBitsPerBlock;
5294 
5295   unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
5296   MVT IntVT = VecVT.changeVectorElementTypeToInteger();
5297 
5298   // If this is SEW=8 and VLMAX is unknown or more than 256, we need
5299   // to use vrgatherei16.vv.
5300   // TODO: It's also possible to use vrgatherei16.vv for other types to
5301   // decrease register width for the index calculation.
5302   if ((MaxVLMAX == 0 || MaxVLMAX > 256) && EltSize == 8) {
5303     // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
5304     // Reverse each half, then reassemble them in reverse order.
5305     // NOTE: It's also possible that after splitting that VLMAX no longer
5306     // requires vrgatherei16.vv.
5307     if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
5308       SDValue Lo, Hi;
5309       std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
5310       EVT LoVT, HiVT;
5311       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
5312       Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
5313       Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
5314       // Reassemble the low and high pieces reversed.
5315       // FIXME: This is a CONCAT_VECTORS.
5316       SDValue Res =
5317           DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
5318                       DAG.getIntPtrConstant(0, DL));
5319       return DAG.getNode(
5320           ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo,
5321           DAG.getIntPtrConstant(LoVT.getVectorMinNumElements(), DL));
5322     }
5323 
5324     // Just promote the int type to i16 which will double the LMUL.
5325     IntVT = MVT::getVectorVT(MVT::i16, VecVT.getVectorElementCount());
5326     GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
5327   }
5328 
5329   MVT XLenVT = Subtarget.getXLenVT();
5330   SDValue Mask, VL;
5331   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5332 
5333   // Calculate VLMAX-1 for the desired SEW.
5334   unsigned MinElts = VecVT.getVectorMinNumElements();
5335   SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT,
5336                               DAG.getConstant(MinElts, DL, XLenVT));
5337   SDValue VLMinus1 =
5338       DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DAG.getConstant(1, DL, XLenVT));
5339 
5340   // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
5341   bool IsRV32E64 =
5342       !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
5343   SDValue SplatVL;
5344   if (!IsRV32E64)
5345     SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
5346   else
5347     SplatVL = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, IntVT, VLMinus1);
5348 
5349   SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
5350   SDValue Indices =
5351       DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, Mask, VL);
5352 
5353   return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices, Mask, VL);
5354 }
5355 
5356 SDValue
5357 RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
5358                                                      SelectionDAG &DAG) const {
5359   SDLoc DL(Op);
5360   auto *Load = cast<LoadSDNode>(Op);
5361 
5362   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5363                                         Load->getMemoryVT(),
5364                                         *Load->getMemOperand()) &&
5365          "Expecting a correctly-aligned load");
5366 
5367   MVT VT = Op.getSimpleValueType();
5368   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5369 
5370   SDValue VL =
5371       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5372 
5373   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5374   SDValue NewLoad = DAG.getMemIntrinsicNode(
5375       RISCVISD::VLE_VL, DL, VTs, {Load->getChain(), Load->getBasePtr(), VL},
5376       Load->getMemoryVT(), Load->getMemOperand());
5377 
5378   SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
5379   return DAG.getMergeValues({Result, Load->getChain()}, DL);
5380 }
5381 
5382 SDValue
5383 RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
5384                                                       SelectionDAG &DAG) const {
5385   SDLoc DL(Op);
5386   auto *Store = cast<StoreSDNode>(Op);
5387 
5388   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5389                                         Store->getMemoryVT(),
5390                                         *Store->getMemOperand()) &&
5391          "Expecting a correctly-aligned store");
5392 
5393   SDValue StoreVal = Store->getValue();
5394   MVT VT = StoreVal.getSimpleValueType();
5395 
5396   // If the size less than a byte, we need to pad with zeros to make a byte.
5397   if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
5398     VT = MVT::v8i1;
5399     StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
5400                            DAG.getConstant(0, DL, VT), StoreVal,
5401                            DAG.getIntPtrConstant(0, DL));
5402   }
5403 
5404   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5405 
5406   SDValue VL =
5407       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5408 
5409   SDValue NewValue =
5410       convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
5411   return DAG.getMemIntrinsicNode(
5412       RISCVISD::VSE_VL, DL, DAG.getVTList(MVT::Other),
5413       {Store->getChain(), NewValue, Store->getBasePtr(), VL},
5414       Store->getMemoryVT(), Store->getMemOperand());
5415 }
5416 
5417 SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
5418                                              SelectionDAG &DAG) const {
5419   SDLoc DL(Op);
5420   MVT VT = Op.getSimpleValueType();
5421 
5422   const auto *MemSD = cast<MemSDNode>(Op);
5423   EVT MemVT = MemSD->getMemoryVT();
5424   MachineMemOperand *MMO = MemSD->getMemOperand();
5425   SDValue Chain = MemSD->getChain();
5426   SDValue BasePtr = MemSD->getBasePtr();
5427 
5428   SDValue Mask, PassThru, VL;
5429   if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Op)) {
5430     Mask = VPLoad->getMask();
5431     PassThru = DAG.getUNDEF(VT);
5432     VL = VPLoad->getVectorLength();
5433   } else {
5434     const auto *MLoad = cast<MaskedLoadSDNode>(Op);
5435     Mask = MLoad->getMask();
5436     PassThru = MLoad->getPassThru();
5437   }
5438 
5439   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5440 
5441   MVT XLenVT = Subtarget.getXLenVT();
5442 
5443   MVT ContainerVT = VT;
5444   if (VT.isFixedLengthVector()) {
5445     ContainerVT = getContainerForFixedLengthVector(VT);
5446     PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5447     if (!IsUnmasked) {
5448       MVT MaskVT =
5449           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5450       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5451     }
5452   }
5453 
5454   if (!VL)
5455     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5456 
5457   unsigned IntID =
5458       IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
5459   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5460   if (IsUnmasked)
5461     Ops.push_back(DAG.getUNDEF(ContainerVT));
5462   else
5463     Ops.push_back(PassThru);
5464   Ops.push_back(BasePtr);
5465   if (!IsUnmasked)
5466     Ops.push_back(Mask);
5467   Ops.push_back(VL);
5468   if (!IsUnmasked)
5469     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5470 
5471   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5472 
5473   SDValue Result =
5474       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5475   Chain = Result.getValue(1);
5476 
5477   if (VT.isFixedLengthVector())
5478     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5479 
5480   return DAG.getMergeValues({Result, Chain}, DL);
5481 }
5482 
5483 SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
5484                                               SelectionDAG &DAG) const {
5485   SDLoc DL(Op);
5486 
5487   const auto *MemSD = cast<MemSDNode>(Op);
5488   EVT MemVT = MemSD->getMemoryVT();
5489   MachineMemOperand *MMO = MemSD->getMemOperand();
5490   SDValue Chain = MemSD->getChain();
5491   SDValue BasePtr = MemSD->getBasePtr();
5492   SDValue Val, Mask, VL;
5493 
5494   if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Op)) {
5495     Val = VPStore->getValue();
5496     Mask = VPStore->getMask();
5497     VL = VPStore->getVectorLength();
5498   } else {
5499     const auto *MStore = cast<MaskedStoreSDNode>(Op);
5500     Val = MStore->getValue();
5501     Mask = MStore->getMask();
5502   }
5503 
5504   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5505 
5506   MVT VT = Val.getSimpleValueType();
5507   MVT XLenVT = Subtarget.getXLenVT();
5508 
5509   MVT ContainerVT = VT;
5510   if (VT.isFixedLengthVector()) {
5511     ContainerVT = getContainerForFixedLengthVector(VT);
5512 
5513     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
5514     if (!IsUnmasked) {
5515       MVT MaskVT =
5516           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5517       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5518     }
5519   }
5520 
5521   if (!VL)
5522     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5523 
5524   unsigned IntID =
5525       IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
5526   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5527   Ops.push_back(Val);
5528   Ops.push_back(BasePtr);
5529   if (!IsUnmasked)
5530     Ops.push_back(Mask);
5531   Ops.push_back(VL);
5532 
5533   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
5534                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
5535 }
5536 
5537 SDValue
5538 RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(SDValue Op,
5539                                                       SelectionDAG &DAG) const {
5540   MVT InVT = Op.getOperand(0).getSimpleValueType();
5541   MVT ContainerVT = getContainerForFixedLengthVector(InVT);
5542 
5543   MVT VT = Op.getSimpleValueType();
5544 
5545   SDValue Op1 =
5546       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
5547   SDValue Op2 =
5548       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5549 
5550   SDLoc DL(Op);
5551   SDValue VL =
5552       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5553 
5554   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5555   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
5556 
5557   SDValue Cmp = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op1, Op2,
5558                             Op.getOperand(2), Mask, VL);
5559 
5560   return convertFromScalableVector(VT, Cmp, DAG, Subtarget);
5561 }
5562 
5563 SDValue RISCVTargetLowering::lowerFixedLengthVectorLogicOpToRVV(
5564     SDValue Op, SelectionDAG &DAG, unsigned MaskOpc, unsigned VecOpc) const {
5565   MVT VT = Op.getSimpleValueType();
5566 
5567   if (VT.getVectorElementType() == MVT::i1)
5568     return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMask*/ false);
5569 
5570   return lowerToScalableOp(Op, DAG, VecOpc, /*HasMask*/ true);
5571 }
5572 
5573 SDValue
5574 RISCVTargetLowering::lowerFixedLengthVectorShiftToRVV(SDValue Op,
5575                                                       SelectionDAG &DAG) const {
5576   unsigned Opc;
5577   switch (Op.getOpcode()) {
5578   default: llvm_unreachable("Unexpected opcode!");
5579   case ISD::SHL: Opc = RISCVISD::SHL_VL; break;
5580   case ISD::SRA: Opc = RISCVISD::SRA_VL; break;
5581   case ISD::SRL: Opc = RISCVISD::SRL_VL; break;
5582   }
5583 
5584   return lowerToScalableOp(Op, DAG, Opc);
5585 }
5586 
5587 // Lower vector ABS to smax(X, sub(0, X)).
5588 SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
5589   SDLoc DL(Op);
5590   MVT VT = Op.getSimpleValueType();
5591   SDValue X = Op.getOperand(0);
5592 
5593   assert(VT.isFixedLengthVector() && "Unexpected type");
5594 
5595   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5596   X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
5597 
5598   SDValue Mask, VL;
5599   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5600 
5601   SDValue SplatZero =
5602       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
5603                   DAG.getConstant(0, DL, Subtarget.getXLenVT()));
5604   SDValue NegX =
5605       DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, Mask, VL);
5606   SDValue Max =
5607       DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, Mask, VL);
5608 
5609   return convertFromScalableVector(VT, Max, DAG, Subtarget);
5610 }
5611 
5612 SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
5613     SDValue Op, SelectionDAG &DAG) const {
5614   SDLoc DL(Op);
5615   MVT VT = Op.getSimpleValueType();
5616   SDValue Mag = Op.getOperand(0);
5617   SDValue Sign = Op.getOperand(1);
5618   assert(Mag.getValueType() == Sign.getValueType() &&
5619          "Can only handle COPYSIGN with matching types.");
5620 
5621   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5622   Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget);
5623   Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget);
5624 
5625   SDValue Mask, VL;
5626   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5627 
5628   SDValue CopySign =
5629       DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag, Sign, Mask, VL);
5630 
5631   return convertFromScalableVector(VT, CopySign, DAG, Subtarget);
5632 }
5633 
5634 SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
5635     SDValue Op, SelectionDAG &DAG) const {
5636   MVT VT = Op.getSimpleValueType();
5637   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5638 
5639   MVT I1ContainerVT =
5640       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5641 
5642   SDValue CC =
5643       convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget);
5644   SDValue Op1 =
5645       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5646   SDValue Op2 =
5647       convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget);
5648 
5649   SDLoc DL(Op);
5650   SDValue Mask, VL;
5651   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5652 
5653   SDValue Select =
5654       DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC, Op1, Op2, VL);
5655 
5656   return convertFromScalableVector(VT, Select, DAG, Subtarget);
5657 }
5658 
5659 SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
5660                                                unsigned NewOpc,
5661                                                bool HasMask) const {
5662   MVT VT = Op.getSimpleValueType();
5663   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5664 
5665   // Create list of operands by converting existing ones to scalable types.
5666   SmallVector<SDValue, 6> Ops;
5667   for (const SDValue &V : Op->op_values()) {
5668     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5669 
5670     // Pass through non-vector operands.
5671     if (!V.getValueType().isVector()) {
5672       Ops.push_back(V);
5673       continue;
5674     }
5675 
5676     // "cast" fixed length vector to a scalable vector.
5677     assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
5678            "Only fixed length vectors are supported!");
5679     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5680   }
5681 
5682   SDLoc DL(Op);
5683   SDValue Mask, VL;
5684   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5685   if (HasMask)
5686     Ops.push_back(Mask);
5687   Ops.push_back(VL);
5688 
5689   SDValue ScalableRes = DAG.getNode(NewOpc, DL, ContainerVT, Ops);
5690   return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
5691 }
5692 
5693 // Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
5694 // * Operands of each node are assumed to be in the same order.
5695 // * The EVL operand is promoted from i32 to i64 on RV64.
5696 // * Fixed-length vectors are converted to their scalable-vector container
5697 //   types.
5698 SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
5699                                        unsigned RISCVISDOpc) const {
5700   SDLoc DL(Op);
5701   MVT VT = Op.getSimpleValueType();
5702   SmallVector<SDValue, 4> Ops;
5703 
5704   for (const auto &OpIdx : enumerate(Op->ops())) {
5705     SDValue V = OpIdx.value();
5706     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5707     // Pass through operands which aren't fixed-length vectors.
5708     if (!V.getValueType().isFixedLengthVector()) {
5709       Ops.push_back(V);
5710       continue;
5711     }
5712     // "cast" fixed length vector to a scalable vector.
5713     MVT OpVT = V.getSimpleValueType();
5714     MVT ContainerVT = getContainerForFixedLengthVector(OpVT);
5715     assert(useRVVForFixedLengthVectorVT(OpVT) &&
5716            "Only fixed length vectors are supported!");
5717     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5718   }
5719 
5720   if (!VT.isFixedLengthVector())
5721     return DAG.getNode(RISCVISDOpc, DL, VT, Ops);
5722 
5723   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5724 
5725   SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops);
5726 
5727   return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
5728 }
5729 
5730 SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op, SelectionDAG &DAG,
5731                                             unsigned MaskOpc,
5732                                             unsigned VecOpc) const {
5733   MVT VT = Op.getSimpleValueType();
5734   if (VT.getVectorElementType() != MVT::i1)
5735     return lowerVPOp(Op, DAG, VecOpc);
5736 
5737   // It is safe to drop mask parameter as masked-off elements are undef.
5738   SDValue Op1 = Op->getOperand(0);
5739   SDValue Op2 = Op->getOperand(1);
5740   SDValue VL = Op->getOperand(3);
5741 
5742   MVT ContainerVT = VT;
5743   const bool IsFixed = VT.isFixedLengthVector();
5744   if (IsFixed) {
5745     ContainerVT = getContainerForFixedLengthVector(VT);
5746     Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5747     Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
5748   }
5749 
5750   SDLoc DL(Op);
5751   SDValue Val = DAG.getNode(MaskOpc, DL, ContainerVT, Op1, Op2, VL);
5752   if (!IsFixed)
5753     return Val;
5754   return convertFromScalableVector(VT, Val, DAG, Subtarget);
5755 }
5756 
5757 // Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
5758 // matched to a RVV indexed load. The RVV indexed load instructions only
5759 // support the "unsigned unscaled" addressing mode; indices are implicitly
5760 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
5761 // signed or scaled indexing is extended to the XLEN value type and scaled
5762 // accordingly.
5763 SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
5764                                                SelectionDAG &DAG) const {
5765   SDLoc DL(Op);
5766   MVT VT = Op.getSimpleValueType();
5767 
5768   const auto *MemSD = cast<MemSDNode>(Op.getNode());
5769   EVT MemVT = MemSD->getMemoryVT();
5770   MachineMemOperand *MMO = MemSD->getMemOperand();
5771   SDValue Chain = MemSD->getChain();
5772   SDValue BasePtr = MemSD->getBasePtr();
5773 
5774   ISD::LoadExtType LoadExtType;
5775   SDValue Index, Mask, PassThru, VL;
5776 
5777   if (auto *VPGN = dyn_cast<VPGatherSDNode>(Op.getNode())) {
5778     Index = VPGN->getIndex();
5779     Mask = VPGN->getMask();
5780     PassThru = DAG.getUNDEF(VT);
5781     VL = VPGN->getVectorLength();
5782     // VP doesn't support extending loads.
5783     LoadExtType = ISD::NON_EXTLOAD;
5784   } else {
5785     // Else it must be a MGATHER.
5786     auto *MGN = cast<MaskedGatherSDNode>(Op.getNode());
5787     Index = MGN->getIndex();
5788     Mask = MGN->getMask();
5789     PassThru = MGN->getPassThru();
5790     LoadExtType = MGN->getExtensionType();
5791   }
5792 
5793   MVT IndexVT = Index.getSimpleValueType();
5794   MVT XLenVT = Subtarget.getXLenVT();
5795 
5796   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
5797          "Unexpected VTs!");
5798   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
5799   // Targets have to explicitly opt-in for extending vector loads.
5800   assert(LoadExtType == ISD::NON_EXTLOAD &&
5801          "Unexpected extending MGATHER/VP_GATHER");
5802   (void)LoadExtType;
5803 
5804   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
5805   // the selection of the masked intrinsics doesn't do this for us.
5806   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5807 
5808   MVT ContainerVT = VT;
5809   if (VT.isFixedLengthVector()) {
5810     // We need to use the larger of the result and index type to determine the
5811     // scalable type to use so we don't increase LMUL for any operand/result.
5812     if (VT.bitsGE(IndexVT)) {
5813       ContainerVT = getContainerForFixedLengthVector(VT);
5814       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
5815                                  ContainerVT.getVectorElementCount());
5816     } else {
5817       IndexVT = getContainerForFixedLengthVector(IndexVT);
5818       ContainerVT = MVT::getVectorVT(ContainerVT.getVectorElementType(),
5819                                      IndexVT.getVectorElementCount());
5820     }
5821 
5822     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
5823 
5824     if (!IsUnmasked) {
5825       MVT MaskVT =
5826           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5827       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5828       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5829     }
5830   }
5831 
5832   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
5833       IndexVT = IndexVT.changeVectorElementType(XLenVT);
5834       Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
5835   }
5836 
5837   if (!VL)
5838     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5839 
5840   unsigned IntID =
5841       IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
5842   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5843   if (IsUnmasked)
5844     Ops.push_back(DAG.getUNDEF(ContainerVT));
5845   else
5846     Ops.push_back(PassThru);
5847   Ops.push_back(BasePtr);
5848   Ops.push_back(Index);
5849   if (!IsUnmasked)
5850     Ops.push_back(Mask);
5851   Ops.push_back(VL);
5852   if (!IsUnmasked)
5853     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5854 
5855   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5856   SDValue Result =
5857       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5858   Chain = Result.getValue(1);
5859 
5860   if (VT.isFixedLengthVector())
5861     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5862 
5863   return DAG.getMergeValues({Result, Chain}, DL);
5864 }
5865 
5866 // Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
5867 // matched to a RVV indexed store. The RVV indexed store instructions only
5868 // support the "unsigned unscaled" addressing mode; indices are implicitly
5869 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
5870 // signed or scaled indexing is extended to the XLEN value type and scaled
5871 // accordingly.
5872 SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
5873                                                 SelectionDAG &DAG) const {
5874   SDLoc DL(Op);
5875   const auto *MemSD = cast<MemSDNode>(Op.getNode());
5876   EVT MemVT = MemSD->getMemoryVT();
5877   MachineMemOperand *MMO = MemSD->getMemOperand();
5878   SDValue Chain = MemSD->getChain();
5879   SDValue BasePtr = MemSD->getBasePtr();
5880 
5881   bool IsTruncatingStore = false;
5882   SDValue Index, Mask, Val, VL;
5883 
5884   if (auto *VPSN = dyn_cast<VPScatterSDNode>(Op.getNode())) {
5885     Index = VPSN->getIndex();
5886     Mask = VPSN->getMask();
5887     Val = VPSN->getValue();
5888     VL = VPSN->getVectorLength();
5889     // VP doesn't support truncating stores.
5890     IsTruncatingStore = false;
5891   } else {
5892     // Else it must be a MSCATTER.
5893     auto *MSN = cast<MaskedScatterSDNode>(Op.getNode());
5894     Index = MSN->getIndex();
5895     Mask = MSN->getMask();
5896     Val = MSN->getValue();
5897     IsTruncatingStore = MSN->isTruncatingStore();
5898   }
5899 
5900   MVT VT = Val.getSimpleValueType();
5901   MVT IndexVT = Index.getSimpleValueType();
5902   MVT XLenVT = Subtarget.getXLenVT();
5903 
5904   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
5905          "Unexpected VTs!");
5906   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
5907   // Targets have to explicitly opt-in for extending vector loads and
5908   // truncating vector stores.
5909   assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
5910   (void)IsTruncatingStore;
5911 
5912   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
5913   // the selection of the masked intrinsics doesn't do this for us.
5914   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5915 
5916   MVT ContainerVT = VT;
5917   if (VT.isFixedLengthVector()) {
5918     // We need to use the larger of the value and index type to determine the
5919     // scalable type to use so we don't increase LMUL for any operand/result.
5920     if (VT.bitsGE(IndexVT)) {
5921       ContainerVT = getContainerForFixedLengthVector(VT);
5922       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
5923                                  ContainerVT.getVectorElementCount());
5924     } else {
5925       IndexVT = getContainerForFixedLengthVector(IndexVT);
5926       ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
5927                                      IndexVT.getVectorElementCount());
5928     }
5929 
5930     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
5931     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
5932 
5933     if (!IsUnmasked) {
5934       MVT MaskVT =
5935           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5936       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5937     }
5938   }
5939 
5940   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
5941       IndexVT = IndexVT.changeVectorElementType(XLenVT);
5942       Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
5943   }
5944 
5945   if (!VL)
5946     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5947 
5948   unsigned IntID =
5949       IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
5950   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5951   Ops.push_back(Val);
5952   Ops.push_back(BasePtr);
5953   Ops.push_back(Index);
5954   if (!IsUnmasked)
5955     Ops.push_back(Mask);
5956   Ops.push_back(VL);
5957 
5958   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
5959                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
5960 }
5961 
5962 SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
5963                                                SelectionDAG &DAG) const {
5964   const MVT XLenVT = Subtarget.getXLenVT();
5965   SDLoc DL(Op);
5966   SDValue Chain = Op->getOperand(0);
5967   SDValue SysRegNo = DAG.getTargetConstant(
5968       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
5969   SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
5970   SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
5971 
5972   // Encoding used for rounding mode in RISCV differs from that used in
5973   // FLT_ROUNDS. To convert it the RISCV rounding mode is used as an index in a
5974   // table, which consists of a sequence of 4-bit fields, each representing
5975   // corresponding FLT_ROUNDS mode.
5976   static const int Table =
5977       (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
5978       (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
5979       (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
5980       (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
5981       (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
5982 
5983   SDValue Shift =
5984       DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
5985   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
5986                                 DAG.getConstant(Table, DL, XLenVT), Shift);
5987   SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
5988                                DAG.getConstant(7, DL, XLenVT));
5989 
5990   return DAG.getMergeValues({Masked, Chain}, DL);
5991 }
5992 
5993 SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
5994                                                SelectionDAG &DAG) const {
5995   const MVT XLenVT = Subtarget.getXLenVT();
5996   SDLoc DL(Op);
5997   SDValue Chain = Op->getOperand(0);
5998   SDValue RMValue = Op->getOperand(1);
5999   SDValue SysRegNo = DAG.getTargetConstant(
6000       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
6001 
6002   // Encoding used for rounding mode in RISCV differs from that used in
6003   // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
6004   // a table, which consists of a sequence of 4-bit fields, each representing
6005   // corresponding RISCV mode.
6006   static const unsigned Table =
6007       (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
6008       (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
6009       (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
6010       (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
6011       (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
6012 
6013   SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
6014                               DAG.getConstant(2, DL, XLenVT));
6015   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
6016                                 DAG.getConstant(Table, DL, XLenVT), Shift);
6017   RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
6018                         DAG.getConstant(0x7, DL, XLenVT));
6019   return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
6020                      RMValue);
6021 }
6022 
6023 static RISCVISD::NodeType getRISCVWOpcodeByIntr(unsigned IntNo) {
6024   switch (IntNo) {
6025   default:
6026     llvm_unreachable("Unexpected Intrinsic");
6027   case Intrinsic::riscv_grev:
6028     return RISCVISD::GREVW;
6029   case Intrinsic::riscv_gorc:
6030     return RISCVISD::GORCW;
6031   case Intrinsic::riscv_bcompress:
6032     return RISCVISD::BCOMPRESSW;
6033   case Intrinsic::riscv_bdecompress:
6034     return RISCVISD::BDECOMPRESSW;
6035   case Intrinsic::riscv_bfp:
6036     return RISCVISD::BFPW;
6037   case Intrinsic::riscv_fsl:
6038     return RISCVISD::FSLW;
6039   case Intrinsic::riscv_fsr:
6040     return RISCVISD::FSRW;
6041   }
6042 }
6043 
6044 // Converts the given intrinsic to a i64 operation with any extension.
6045 static SDValue customLegalizeToWOpByIntr(SDNode *N, SelectionDAG &DAG,
6046                                          unsigned IntNo) {
6047   SDLoc DL(N);
6048   RISCVISD::NodeType WOpcode = getRISCVWOpcodeByIntr(IntNo);
6049   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6050   SDValue NewOp2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6051   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp1, NewOp2);
6052   // ReplaceNodeResults requires we maintain the same type for the return value.
6053   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
6054 }
6055 
6056 // Returns the opcode of the target-specific SDNode that implements the 32-bit
6057 // form of the given Opcode.
6058 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
6059   switch (Opcode) {
6060   default:
6061     llvm_unreachable("Unexpected opcode");
6062   case ISD::SHL:
6063     return RISCVISD::SLLW;
6064   case ISD::SRA:
6065     return RISCVISD::SRAW;
6066   case ISD::SRL:
6067     return RISCVISD::SRLW;
6068   case ISD::SDIV:
6069     return RISCVISD::DIVW;
6070   case ISD::UDIV:
6071     return RISCVISD::DIVUW;
6072   case ISD::UREM:
6073     return RISCVISD::REMUW;
6074   case ISD::ROTL:
6075     return RISCVISD::ROLW;
6076   case ISD::ROTR:
6077     return RISCVISD::RORW;
6078   case RISCVISD::GREV:
6079     return RISCVISD::GREVW;
6080   case RISCVISD::GORC:
6081     return RISCVISD::GORCW;
6082   }
6083 }
6084 
6085 // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
6086 // node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
6087 // otherwise be promoted to i64, making it difficult to select the
6088 // SLLW/DIVUW/.../*W later one because the fact the operation was originally of
6089 // type i8/i16/i32 is lost.
6090 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
6091                                    unsigned ExtOpc = ISD::ANY_EXTEND) {
6092   SDLoc DL(N);
6093   RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6094   SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
6095   SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
6096   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6097   // ReplaceNodeResults requires we maintain the same type for the return value.
6098   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
6099 }
6100 
6101 // Converts the given 32-bit operation to a i64 operation with signed extension
6102 // semantic to reduce the signed extension instructions.
6103 static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
6104   SDLoc DL(N);
6105   SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6106   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6107   SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
6108   SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6109                                DAG.getValueType(MVT::i32));
6110   return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
6111 }
6112 
6113 void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
6114                                              SmallVectorImpl<SDValue> &Results,
6115                                              SelectionDAG &DAG) const {
6116   SDLoc DL(N);
6117   switch (N->getOpcode()) {
6118   default:
6119     llvm_unreachable("Don't know how to custom type legalize this operation!");
6120   case ISD::STRICT_FP_TO_SINT:
6121   case ISD::STRICT_FP_TO_UINT:
6122   case ISD::FP_TO_SINT:
6123   case ISD::FP_TO_UINT: {
6124     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6125            "Unexpected custom legalisation");
6126     bool IsStrict = N->isStrictFPOpcode();
6127     bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
6128                     N->getOpcode() == ISD::STRICT_FP_TO_SINT;
6129     SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
6130     if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
6131         TargetLowering::TypeSoftenFloat) {
6132       if (!isTypeLegal(Op0.getValueType()))
6133         return;
6134       if (IsStrict) {
6135         unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
6136                                 : RISCVISD::STRICT_FCVT_WU_RV64;
6137         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
6138         SDValue Res = DAG.getNode(
6139             Opc, DL, VTs, N->getOperand(0), Op0,
6140             DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
6141         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6142         Results.push_back(Res.getValue(1));
6143         return;
6144       }
6145       unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
6146       SDValue Res =
6147           DAG.getNode(Opc, DL, MVT::i64, Op0,
6148                       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
6149       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6150       return;
6151     }
6152     // If the FP type needs to be softened, emit a library call using the 'si'
6153     // version. If we left it to default legalization we'd end up with 'di'. If
6154     // the FP type doesn't need to be softened just let generic type
6155     // legalization promote the result type.
6156     RTLIB::Libcall LC;
6157     if (IsSigned)
6158       LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
6159     else
6160       LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
6161     MakeLibCallOptions CallOptions;
6162     EVT OpVT = Op0.getValueType();
6163     CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
6164     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
6165     SDValue Result;
6166     std::tie(Result, Chain) =
6167         makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
6168     Results.push_back(Result);
6169     if (IsStrict)
6170       Results.push_back(Chain);
6171     break;
6172   }
6173   case ISD::READCYCLECOUNTER: {
6174     assert(!Subtarget.is64Bit() &&
6175            "READCYCLECOUNTER only has custom type legalization on riscv32");
6176 
6177     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
6178     SDValue RCW =
6179         DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
6180 
6181     Results.push_back(
6182         DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
6183     Results.push_back(RCW.getValue(2));
6184     break;
6185   }
6186   case ISD::MUL: {
6187     unsigned Size = N->getSimpleValueType(0).getSizeInBits();
6188     unsigned XLen = Subtarget.getXLen();
6189     // This multiply needs to be expanded, try to use MULHSU+MUL if possible.
6190     if (Size > XLen) {
6191       assert(Size == (XLen * 2) && "Unexpected custom legalisation");
6192       SDValue LHS = N->getOperand(0);
6193       SDValue RHS = N->getOperand(1);
6194       APInt HighMask = APInt::getHighBitsSet(Size, XLen);
6195 
6196       bool LHSIsU = DAG.MaskedValueIsZero(LHS, HighMask);
6197       bool RHSIsU = DAG.MaskedValueIsZero(RHS, HighMask);
6198       // We need exactly one side to be unsigned.
6199       if (LHSIsU == RHSIsU)
6200         return;
6201 
6202       auto MakeMULPair = [&](SDValue S, SDValue U) {
6203         MVT XLenVT = Subtarget.getXLenVT();
6204         S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S);
6205         U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U);
6206         SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U);
6207         SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U);
6208         return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
6209       };
6210 
6211       bool LHSIsS = DAG.ComputeNumSignBits(LHS) > XLen;
6212       bool RHSIsS = DAG.ComputeNumSignBits(RHS) > XLen;
6213 
6214       // The other operand should be signed, but still prefer MULH when
6215       // possible.
6216       if (RHSIsU && LHSIsS && !RHSIsS)
6217         Results.push_back(MakeMULPair(LHS, RHS));
6218       else if (LHSIsU && RHSIsS && !LHSIsS)
6219         Results.push_back(MakeMULPair(RHS, LHS));
6220 
6221       return;
6222     }
6223     LLVM_FALLTHROUGH;
6224   }
6225   case ISD::ADD:
6226   case ISD::SUB:
6227     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6228            "Unexpected custom legalisation");
6229     Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
6230     break;
6231   case ISD::SHL:
6232   case ISD::SRA:
6233   case ISD::SRL:
6234     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6235            "Unexpected custom legalisation");
6236     if (N->getOperand(1).getOpcode() != ISD::Constant) {
6237       Results.push_back(customLegalizeToWOp(N, DAG));
6238       break;
6239     }
6240 
6241     // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
6242     // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
6243     // shift amount.
6244     if (N->getOpcode() == ISD::SHL) {
6245       SDLoc DL(N);
6246       SDValue NewOp0 =
6247           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6248       SDValue NewOp1 =
6249           DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
6250       SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
6251       SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6252                                    DAG.getValueType(MVT::i32));
6253       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6254     }
6255 
6256     break;
6257   case ISD::ROTL:
6258   case ISD::ROTR:
6259     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6260            "Unexpected custom legalisation");
6261     Results.push_back(customLegalizeToWOp(N, DAG));
6262     break;
6263   case ISD::CTTZ:
6264   case ISD::CTTZ_ZERO_UNDEF:
6265   case ISD::CTLZ:
6266   case ISD::CTLZ_ZERO_UNDEF: {
6267     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6268            "Unexpected custom legalisation");
6269 
6270     SDValue NewOp0 =
6271         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6272     bool IsCTZ =
6273         N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF;
6274     unsigned Opc = IsCTZ ? RISCVISD::CTZW : RISCVISD::CLZW;
6275     SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
6276     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6277     return;
6278   }
6279   case ISD::SDIV:
6280   case ISD::UDIV:
6281   case ISD::UREM: {
6282     MVT VT = N->getSimpleValueType(0);
6283     assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
6284            Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
6285            "Unexpected custom legalisation");
6286     // Don't promote division/remainder by constant since we should expand those
6287     // to multiply by magic constant.
6288     // FIXME: What if the expansion is disabled for minsize.
6289     if (N->getOperand(1).getOpcode() == ISD::Constant)
6290       return;
6291 
6292     // If the input is i32, use ANY_EXTEND since the W instructions don't read
6293     // the upper 32 bits. For other types we need to sign or zero extend
6294     // based on the opcode.
6295     unsigned ExtOpc = ISD::ANY_EXTEND;
6296     if (VT != MVT::i32)
6297       ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
6298                                            : ISD::ZERO_EXTEND;
6299 
6300     Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
6301     break;
6302   }
6303   case ISD::UADDO:
6304   case ISD::USUBO: {
6305     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6306            "Unexpected custom legalisation");
6307     bool IsAdd = N->getOpcode() == ISD::UADDO;
6308     // Create an ADDW or SUBW.
6309     SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6310     SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6311     SDValue Res =
6312         DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
6313     Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
6314                       DAG.getValueType(MVT::i32));
6315 
6316     // Sign extend the LHS and perform an unsigned compare with the ADDW result.
6317     // Since the inputs are sign extended from i32, this is equivalent to
6318     // comparing the lower 32 bits.
6319     LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6320     SDValue Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
6321                                     IsAdd ? ISD::SETULT : ISD::SETUGT);
6322 
6323     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6324     Results.push_back(Overflow);
6325     return;
6326   }
6327   case ISD::UADDSAT:
6328   case ISD::USUBSAT: {
6329     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6330            "Unexpected custom legalisation");
6331     if (Subtarget.hasStdExtZbb()) {
6332       // With Zbb we can sign extend and let LegalizeDAG use minu/maxu. Using
6333       // sign extend allows overflow of the lower 32 bits to be detected on
6334       // the promoted size.
6335       SDValue LHS =
6336           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6337       SDValue RHS =
6338           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
6339       SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
6340       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6341       return;
6342     }
6343 
6344     // Without Zbb, expand to UADDO/USUBO+select which will trigger our custom
6345     // promotion for UADDO/USUBO.
6346     Results.push_back(expandAddSubSat(N, DAG));
6347     return;
6348   }
6349   case ISD::BITCAST: {
6350     EVT VT = N->getValueType(0);
6351     assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
6352     SDValue Op0 = N->getOperand(0);
6353     EVT Op0VT = Op0.getValueType();
6354     MVT XLenVT = Subtarget.getXLenVT();
6355     if (VT == MVT::i16 && Op0VT == MVT::f16 && Subtarget.hasStdExtZfh()) {
6356       SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
6357       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
6358     } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
6359                Subtarget.hasStdExtF()) {
6360       SDValue FPConv =
6361           DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
6362       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
6363     } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
6364                isTypeLegal(Op0VT)) {
6365       // Custom-legalize bitcasts from fixed-length vector types to illegal
6366       // scalar types in order to improve codegen. Bitcast the vector to a
6367       // one-element vector type whose element type is the same as the result
6368       // type, and extract the first element.
6369       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
6370       if (isTypeLegal(BVT)) {
6371         SDValue BVec = DAG.getBitcast(BVT, Op0);
6372         Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
6373                                       DAG.getConstant(0, DL, XLenVT)));
6374       }
6375     }
6376     break;
6377   }
6378   case RISCVISD::GREV:
6379   case RISCVISD::GORC: {
6380     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6381            "Unexpected custom legalisation");
6382     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6383     // This is similar to customLegalizeToWOp, except that we pass the second
6384     // operand (a TargetConstant) straight through: it is already of type
6385     // XLenVT.
6386     RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6387     SDValue NewOp0 =
6388         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6389     SDValue NewOp1 =
6390         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6391     SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6392     // ReplaceNodeResults requires we maintain the same type for the return
6393     // value.
6394     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6395     break;
6396   }
6397   case RISCVISD::SHFL: {
6398     // There is no SHFLIW instruction, but we can just promote the operation.
6399     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6400            "Unexpected custom legalisation");
6401     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6402     SDValue NewOp0 =
6403         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6404     SDValue NewOp1 =
6405         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6406     SDValue NewRes = DAG.getNode(RISCVISD::SHFL, DL, MVT::i64, NewOp0, NewOp1);
6407     // ReplaceNodeResults requires we maintain the same type for the return
6408     // value.
6409     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6410     break;
6411   }
6412   case ISD::BSWAP:
6413   case ISD::BITREVERSE: {
6414     MVT VT = N->getSimpleValueType(0);
6415     MVT XLenVT = Subtarget.getXLenVT();
6416     assert((VT == MVT::i8 || VT == MVT::i16 ||
6417             (VT == MVT::i32 && Subtarget.is64Bit())) &&
6418            Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
6419     SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
6420     unsigned Imm = VT.getSizeInBits() - 1;
6421     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
6422     if (N->getOpcode() == ISD::BSWAP)
6423       Imm &= ~0x7U;
6424     unsigned Opc = Subtarget.is64Bit() ? RISCVISD::GREVW : RISCVISD::GREV;
6425     SDValue GREVI =
6426         DAG.getNode(Opc, DL, XLenVT, NewOp0, DAG.getConstant(Imm, DL, XLenVT));
6427     // ReplaceNodeResults requires we maintain the same type for the return
6428     // value.
6429     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, GREVI));
6430     break;
6431   }
6432   case ISD::FSHL:
6433   case ISD::FSHR: {
6434     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6435            Subtarget.hasStdExtZbt() && "Unexpected custom legalisation");
6436     SDValue NewOp0 =
6437         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6438     SDValue NewOp1 =
6439         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6440     SDValue NewShAmt =
6441         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6442     // FSLW/FSRW take a 6 bit shift amount but i32 FSHL/FSHR only use 5 bits.
6443     // Mask the shift amount to 5 bits to prevent accidentally setting bit 5.
6444     NewShAmt = DAG.getNode(ISD::AND, DL, MVT::i64, NewShAmt,
6445                            DAG.getConstant(0x1f, DL, MVT::i64));
6446     // fshl and fshr concatenate their operands in the same order. fsrw and fslw
6447     // instruction use different orders. fshl will return its first operand for
6448     // shift of zero, fshr will return its second operand. fsl and fsr both
6449     // return rs1 so the ISD nodes need to have different operand orders.
6450     // Shift amount is in rs2.
6451     unsigned Opc = RISCVISD::FSLW;
6452     if (N->getOpcode() == ISD::FSHR) {
6453       std::swap(NewOp0, NewOp1);
6454       Opc = RISCVISD::FSRW;
6455     }
6456     SDValue NewOp = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, NewShAmt);
6457     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewOp));
6458     break;
6459   }
6460   case ISD::EXTRACT_VECTOR_ELT: {
6461     // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
6462     // type is illegal (currently only vXi64 RV32).
6463     // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
6464     // transferred to the destination register. We issue two of these from the
6465     // upper- and lower- halves of the SEW-bit vector element, slid down to the
6466     // first element.
6467     SDValue Vec = N->getOperand(0);
6468     SDValue Idx = N->getOperand(1);
6469 
6470     // The vector type hasn't been legalized yet so we can't issue target
6471     // specific nodes if it needs legalization.
6472     // FIXME: We would manually legalize if it's important.
6473     if (!isTypeLegal(Vec.getValueType()))
6474       return;
6475 
6476     MVT VecVT = Vec.getSimpleValueType();
6477 
6478     assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
6479            VecVT.getVectorElementType() == MVT::i64 &&
6480            "Unexpected EXTRACT_VECTOR_ELT legalization");
6481 
6482     // If this is a fixed vector, we need to convert it to a scalable vector.
6483     MVT ContainerVT = VecVT;
6484     if (VecVT.isFixedLengthVector()) {
6485       ContainerVT = getContainerForFixedLengthVector(VecVT);
6486       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
6487     }
6488 
6489     MVT XLenVT = Subtarget.getXLenVT();
6490 
6491     // Use a VL of 1 to avoid processing more elements than we need.
6492     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
6493     SDValue VL = DAG.getConstant(1, DL, XLenVT);
6494     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6495 
6496     // Unless the index is known to be 0, we must slide the vector down to get
6497     // the desired element into index 0.
6498     if (!isNullConstant(Idx)) {
6499       Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
6500                         DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
6501     }
6502 
6503     // Extract the lower XLEN bits of the correct vector element.
6504     SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6505 
6506     // To extract the upper XLEN bits of the vector element, shift the first
6507     // element right by 32 bits and re-extract the lower XLEN bits.
6508     SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
6509                                      DAG.getConstant(32, DL, XLenVT), VL);
6510     SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec,
6511                                  ThirtyTwoV, Mask, VL);
6512 
6513     SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6514 
6515     Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6516     break;
6517   }
6518   case ISD::INTRINSIC_WO_CHAIN: {
6519     unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
6520     switch (IntNo) {
6521     default:
6522       llvm_unreachable(
6523           "Don't know how to custom type legalize this intrinsic!");
6524     case Intrinsic::riscv_grev:
6525     case Intrinsic::riscv_gorc:
6526     case Intrinsic::riscv_bcompress:
6527     case Intrinsic::riscv_bdecompress:
6528     case Intrinsic::riscv_bfp: {
6529       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6530              "Unexpected custom legalisation");
6531       Results.push_back(customLegalizeToWOpByIntr(N, DAG, IntNo));
6532       break;
6533     }
6534     case Intrinsic::riscv_fsl:
6535     case Intrinsic::riscv_fsr: {
6536       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6537              "Unexpected custom legalisation");
6538       SDValue NewOp1 =
6539           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6540       SDValue NewOp2 =
6541           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6542       SDValue NewOp3 =
6543           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3));
6544       unsigned Opc = getRISCVWOpcodeByIntr(IntNo);
6545       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2, NewOp3);
6546       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6547       break;
6548     }
6549     case Intrinsic::riscv_orc_b: {
6550       // Lower to the GORCI encoding for orc.b with the operand extended.
6551       SDValue NewOp =
6552           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6553       // If Zbp is enabled, use GORCIW which will sign extend the result.
6554       unsigned Opc =
6555           Subtarget.hasStdExtZbp() ? RISCVISD::GORCW : RISCVISD::GORC;
6556       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp,
6557                                 DAG.getConstant(7, DL, MVT::i64));
6558       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6559       return;
6560     }
6561     case Intrinsic::riscv_shfl:
6562     case Intrinsic::riscv_unshfl: {
6563       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6564              "Unexpected custom legalisation");
6565       SDValue NewOp1 =
6566           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6567       SDValue NewOp2 =
6568           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6569       unsigned Opc =
6570           IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFLW : RISCVISD::UNSHFLW;
6571       if (isa<ConstantSDNode>(N->getOperand(2))) {
6572         NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2,
6573                              DAG.getConstant(0xf, DL, MVT::i64));
6574         Opc =
6575             IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
6576       }
6577       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2);
6578       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6579       break;
6580     }
6581     case Intrinsic::riscv_vmv_x_s: {
6582       EVT VT = N->getValueType(0);
6583       MVT XLenVT = Subtarget.getXLenVT();
6584       if (VT.bitsLT(XLenVT)) {
6585         // Simple case just extract using vmv.x.s and truncate.
6586         SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
6587                                       Subtarget.getXLenVT(), N->getOperand(1));
6588         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
6589         return;
6590       }
6591 
6592       assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
6593              "Unexpected custom legalization");
6594 
6595       // We need to do the move in two steps.
6596       SDValue Vec = N->getOperand(1);
6597       MVT VecVT = Vec.getSimpleValueType();
6598 
6599       // First extract the lower XLEN bits of the element.
6600       SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6601 
6602       // To extract the upper XLEN bits of the vector element, shift the first
6603       // element right by 32 bits and re-extract the lower XLEN bits.
6604       SDValue VL = DAG.getConstant(1, DL, XLenVT);
6605       MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
6606       SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6607       SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT,
6608                                        DAG.getConstant(32, DL, XLenVT), VL);
6609       SDValue LShr32 =
6610           DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV, Mask, VL);
6611       SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6612 
6613       Results.push_back(
6614           DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6615       break;
6616     }
6617     }
6618     break;
6619   }
6620   case ISD::VECREDUCE_ADD:
6621   case ISD::VECREDUCE_AND:
6622   case ISD::VECREDUCE_OR:
6623   case ISD::VECREDUCE_XOR:
6624   case ISD::VECREDUCE_SMAX:
6625   case ISD::VECREDUCE_UMAX:
6626   case ISD::VECREDUCE_SMIN:
6627   case ISD::VECREDUCE_UMIN:
6628     if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
6629       Results.push_back(V);
6630     break;
6631   case ISD::VP_REDUCE_ADD:
6632   case ISD::VP_REDUCE_AND:
6633   case ISD::VP_REDUCE_OR:
6634   case ISD::VP_REDUCE_XOR:
6635   case ISD::VP_REDUCE_SMAX:
6636   case ISD::VP_REDUCE_UMAX:
6637   case ISD::VP_REDUCE_SMIN:
6638   case ISD::VP_REDUCE_UMIN:
6639     if (SDValue V = lowerVPREDUCE(SDValue(N, 0), DAG))
6640       Results.push_back(V);
6641     break;
6642   case ISD::FLT_ROUNDS_: {
6643     SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other);
6644     SDValue Res = DAG.getNode(ISD::FLT_ROUNDS_, DL, VTs, N->getOperand(0));
6645     Results.push_back(Res.getValue(0));
6646     Results.push_back(Res.getValue(1));
6647     break;
6648   }
6649   }
6650 }
6651 
6652 // A structure to hold one of the bit-manipulation patterns below. Together, a
6653 // SHL and non-SHL pattern may form a bit-manipulation pair on a single source:
6654 //   (or (and (shl x, 1), 0xAAAAAAAA),
6655 //       (and (srl x, 1), 0x55555555))
6656 struct RISCVBitmanipPat {
6657   SDValue Op;
6658   unsigned ShAmt;
6659   bool IsSHL;
6660 
6661   bool formsPairWith(const RISCVBitmanipPat &Other) const {
6662     return Op == Other.Op && ShAmt == Other.ShAmt && IsSHL != Other.IsSHL;
6663   }
6664 };
6665 
6666 // Matches patterns of the form
6667 //   (and (shl x, C2), (C1 << C2))
6668 //   (and (srl x, C2), C1)
6669 //   (shl (and x, C1), C2)
6670 //   (srl (and x, (C1 << C2)), C2)
6671 // Where C2 is a power of 2 and C1 has at least that many leading zeroes.
6672 // The expected masks for each shift amount are specified in BitmanipMasks where
6673 // BitmanipMasks[log2(C2)] specifies the expected C1 value.
6674 // The max allowed shift amount is either XLen/2 or XLen/4 determined by whether
6675 // BitmanipMasks contains 6 or 5 entries assuming that the maximum possible
6676 // XLen is 64.
6677 static Optional<RISCVBitmanipPat>
6678 matchRISCVBitmanipPat(SDValue Op, ArrayRef<uint64_t> BitmanipMasks) {
6679   assert((BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) &&
6680          "Unexpected number of masks");
6681   Optional<uint64_t> Mask;
6682   // Optionally consume a mask around the shift operation.
6683   if (Op.getOpcode() == ISD::AND && isa<ConstantSDNode>(Op.getOperand(1))) {
6684     Mask = Op.getConstantOperandVal(1);
6685     Op = Op.getOperand(0);
6686   }
6687   if (Op.getOpcode() != ISD::SHL && Op.getOpcode() != ISD::SRL)
6688     return None;
6689   bool IsSHL = Op.getOpcode() == ISD::SHL;
6690 
6691   if (!isa<ConstantSDNode>(Op.getOperand(1)))
6692     return None;
6693   uint64_t ShAmt = Op.getConstantOperandVal(1);
6694 
6695   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
6696   if (ShAmt >= Width || !isPowerOf2_64(ShAmt))
6697     return None;
6698   // If we don't have enough masks for 64 bit, then we must be trying to
6699   // match SHFL so we're only allowed to shift 1/4 of the width.
6700   if (BitmanipMasks.size() == 5 && ShAmt >= (Width / 2))
6701     return None;
6702 
6703   SDValue Src = Op.getOperand(0);
6704 
6705   // The expected mask is shifted left when the AND is found around SHL
6706   // patterns.
6707   //   ((x >> 1) & 0x55555555)
6708   //   ((x << 1) & 0xAAAAAAAA)
6709   bool SHLExpMask = IsSHL;
6710 
6711   if (!Mask) {
6712     // Sometimes LLVM keeps the mask as an operand of the shift, typically when
6713     // the mask is all ones: consume that now.
6714     if (Src.getOpcode() == ISD::AND && isa<ConstantSDNode>(Src.getOperand(1))) {
6715       Mask = Src.getConstantOperandVal(1);
6716       Src = Src.getOperand(0);
6717       // The expected mask is now in fact shifted left for SRL, so reverse the
6718       // decision.
6719       //   ((x & 0xAAAAAAAA) >> 1)
6720       //   ((x & 0x55555555) << 1)
6721       SHLExpMask = !SHLExpMask;
6722     } else {
6723       // Use a default shifted mask of all-ones if there's no AND, truncated
6724       // down to the expected width. This simplifies the logic later on.
6725       Mask = maskTrailingOnes<uint64_t>(Width);
6726       *Mask &= (IsSHL ? *Mask << ShAmt : *Mask >> ShAmt);
6727     }
6728   }
6729 
6730   unsigned MaskIdx = Log2_32(ShAmt);
6731   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
6732 
6733   if (SHLExpMask)
6734     ExpMask <<= ShAmt;
6735 
6736   if (Mask != ExpMask)
6737     return None;
6738 
6739   return RISCVBitmanipPat{Src, (unsigned)ShAmt, IsSHL};
6740 }
6741 
6742 // Matches any of the following bit-manipulation patterns:
6743 //   (and (shl x, 1), (0x55555555 << 1))
6744 //   (and (srl x, 1), 0x55555555)
6745 //   (shl (and x, 0x55555555), 1)
6746 //   (srl (and x, (0x55555555 << 1)), 1)
6747 // where the shift amount and mask may vary thus:
6748 //   [1]  = 0x55555555 / 0xAAAAAAAA
6749 //   [2]  = 0x33333333 / 0xCCCCCCCC
6750 //   [4]  = 0x0F0F0F0F / 0xF0F0F0F0
6751 //   [8]  = 0x00FF00FF / 0xFF00FF00
6752 //   [16] = 0x0000FFFF / 0xFFFFFFFF
6753 //   [32] = 0x00000000FFFFFFFF / 0xFFFFFFFF00000000 (for RV64)
6754 static Optional<RISCVBitmanipPat> matchGREVIPat(SDValue Op) {
6755   // These are the unshifted masks which we use to match bit-manipulation
6756   // patterns. They may be shifted left in certain circumstances.
6757   static const uint64_t BitmanipMasks[] = {
6758       0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
6759       0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
6760 
6761   return matchRISCVBitmanipPat(Op, BitmanipMasks);
6762 }
6763 
6764 // Match the following pattern as a GREVI(W) operation
6765 //   (or (BITMANIP_SHL x), (BITMANIP_SRL x))
6766 static SDValue combineORToGREV(SDValue Op, SelectionDAG &DAG,
6767                                const RISCVSubtarget &Subtarget) {
6768   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6769   EVT VT = Op.getValueType();
6770 
6771   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
6772     auto LHS = matchGREVIPat(Op.getOperand(0));
6773     auto RHS = matchGREVIPat(Op.getOperand(1));
6774     if (LHS && RHS && LHS->formsPairWith(*RHS)) {
6775       SDLoc DL(Op);
6776       return DAG.getNode(RISCVISD::GREV, DL, VT, LHS->Op,
6777                          DAG.getConstant(LHS->ShAmt, DL, VT));
6778     }
6779   }
6780   return SDValue();
6781 }
6782 
6783 // Matches any the following pattern as a GORCI(W) operation
6784 // 1.  (or (GREVI x, shamt), x) if shamt is a power of 2
6785 // 2.  (or x, (GREVI x, shamt)) if shamt is a power of 2
6786 // 3.  (or (or (BITMANIP_SHL x), x), (BITMANIP_SRL x))
6787 // Note that with the variant of 3.,
6788 //     (or (or (BITMANIP_SHL x), (BITMANIP_SRL x)), x)
6789 // the inner pattern will first be matched as GREVI and then the outer
6790 // pattern will be matched to GORC via the first rule above.
6791 // 4.  (or (rotl/rotr x, bitwidth/2), x)
6792 static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
6793                                const RISCVSubtarget &Subtarget) {
6794   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6795   EVT VT = Op.getValueType();
6796 
6797   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
6798     SDLoc DL(Op);
6799     SDValue Op0 = Op.getOperand(0);
6800     SDValue Op1 = Op.getOperand(1);
6801 
6802     auto MatchOROfReverse = [&](SDValue Reverse, SDValue X) {
6803       if (Reverse.getOpcode() == RISCVISD::GREV && Reverse.getOperand(0) == X &&
6804           isa<ConstantSDNode>(Reverse.getOperand(1)) &&
6805           isPowerOf2_32(Reverse.getConstantOperandVal(1)))
6806         return DAG.getNode(RISCVISD::GORC, DL, VT, X, Reverse.getOperand(1));
6807       // We can also form GORCI from ROTL/ROTR by half the bitwidth.
6808       if ((Reverse.getOpcode() == ISD::ROTL ||
6809            Reverse.getOpcode() == ISD::ROTR) &&
6810           Reverse.getOperand(0) == X &&
6811           isa<ConstantSDNode>(Reverse.getOperand(1))) {
6812         uint64_t RotAmt = Reverse.getConstantOperandVal(1);
6813         if (RotAmt == (VT.getSizeInBits() / 2))
6814           return DAG.getNode(RISCVISD::GORC, DL, VT, X,
6815                              DAG.getConstant(RotAmt, DL, VT));
6816       }
6817       return SDValue();
6818     };
6819 
6820     // Check for either commutable permutation of (or (GREVI x, shamt), x)
6821     if (SDValue V = MatchOROfReverse(Op0, Op1))
6822       return V;
6823     if (SDValue V = MatchOROfReverse(Op1, Op0))
6824       return V;
6825 
6826     // OR is commutable so canonicalize its OR operand to the left
6827     if (Op0.getOpcode() != ISD::OR && Op1.getOpcode() == ISD::OR)
6828       std::swap(Op0, Op1);
6829     if (Op0.getOpcode() != ISD::OR)
6830       return SDValue();
6831     SDValue OrOp0 = Op0.getOperand(0);
6832     SDValue OrOp1 = Op0.getOperand(1);
6833     auto LHS = matchGREVIPat(OrOp0);
6834     // OR is commutable so swap the operands and try again: x might have been
6835     // on the left
6836     if (!LHS) {
6837       std::swap(OrOp0, OrOp1);
6838       LHS = matchGREVIPat(OrOp0);
6839     }
6840     auto RHS = matchGREVIPat(Op1);
6841     if (LHS && RHS && LHS->formsPairWith(*RHS) && LHS->Op == OrOp1) {
6842       return DAG.getNode(RISCVISD::GORC, DL, VT, LHS->Op,
6843                          DAG.getConstant(LHS->ShAmt, DL, VT));
6844     }
6845   }
6846   return SDValue();
6847 }
6848 
6849 // Matches any of the following bit-manipulation patterns:
6850 //   (and (shl x, 1), (0x22222222 << 1))
6851 //   (and (srl x, 1), 0x22222222)
6852 //   (shl (and x, 0x22222222), 1)
6853 //   (srl (and x, (0x22222222 << 1)), 1)
6854 // where the shift amount and mask may vary thus:
6855 //   [1]  = 0x22222222 / 0x44444444
6856 //   [2]  = 0x0C0C0C0C / 0x3C3C3C3C
6857 //   [4]  = 0x00F000F0 / 0x0F000F00
6858 //   [8]  = 0x0000FF00 / 0x00FF0000
6859 //   [16] = 0x00000000FFFF0000 / 0x0000FFFF00000000 (for RV64)
6860 static Optional<RISCVBitmanipPat> matchSHFLPat(SDValue Op) {
6861   // These are the unshifted masks which we use to match bit-manipulation
6862   // patterns. They may be shifted left in certain circumstances.
6863   static const uint64_t BitmanipMasks[] = {
6864       0x2222222222222222ULL, 0x0C0C0C0C0C0C0C0CULL, 0x00F000F000F000F0ULL,
6865       0x0000FF000000FF00ULL, 0x00000000FFFF0000ULL};
6866 
6867   return matchRISCVBitmanipPat(Op, BitmanipMasks);
6868 }
6869 
6870 // Match (or (or (SHFL_SHL x), (SHFL_SHR x)), (SHFL_AND x)
6871 static SDValue combineORToSHFL(SDValue Op, SelectionDAG &DAG,
6872                                const RISCVSubtarget &Subtarget) {
6873   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6874   EVT VT = Op.getValueType();
6875 
6876   if (VT != MVT::i32 && VT != Subtarget.getXLenVT())
6877     return SDValue();
6878 
6879   SDValue Op0 = Op.getOperand(0);
6880   SDValue Op1 = Op.getOperand(1);
6881 
6882   // Or is commutable so canonicalize the second OR to the LHS.
6883   if (Op0.getOpcode() != ISD::OR)
6884     std::swap(Op0, Op1);
6885   if (Op0.getOpcode() != ISD::OR)
6886     return SDValue();
6887 
6888   // We found an inner OR, so our operands are the operands of the inner OR
6889   // and the other operand of the outer OR.
6890   SDValue A = Op0.getOperand(0);
6891   SDValue B = Op0.getOperand(1);
6892   SDValue C = Op1;
6893 
6894   auto Match1 = matchSHFLPat(A);
6895   auto Match2 = matchSHFLPat(B);
6896 
6897   // If neither matched, we failed.
6898   if (!Match1 && !Match2)
6899     return SDValue();
6900 
6901   // We had at least one match. if one failed, try the remaining C operand.
6902   if (!Match1) {
6903     std::swap(A, C);
6904     Match1 = matchSHFLPat(A);
6905     if (!Match1)
6906       return SDValue();
6907   } else if (!Match2) {
6908     std::swap(B, C);
6909     Match2 = matchSHFLPat(B);
6910     if (!Match2)
6911       return SDValue();
6912   }
6913   assert(Match1 && Match2);
6914 
6915   // Make sure our matches pair up.
6916   if (!Match1->formsPairWith(*Match2))
6917     return SDValue();
6918 
6919   // All the remains is to make sure C is an AND with the same input, that masks
6920   // out the bits that are being shuffled.
6921   if (C.getOpcode() != ISD::AND || !isa<ConstantSDNode>(C.getOperand(1)) ||
6922       C.getOperand(0) != Match1->Op)
6923     return SDValue();
6924 
6925   uint64_t Mask = C.getConstantOperandVal(1);
6926 
6927   static const uint64_t BitmanipMasks[] = {
6928       0x9999999999999999ULL, 0xC3C3C3C3C3C3C3C3ULL, 0xF00FF00FF00FF00FULL,
6929       0xFF0000FFFF0000FFULL, 0xFFFF00000000FFFFULL,
6930   };
6931 
6932   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
6933   unsigned MaskIdx = Log2_32(Match1->ShAmt);
6934   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
6935 
6936   if (Mask != ExpMask)
6937     return SDValue();
6938 
6939   SDLoc DL(Op);
6940   return DAG.getNode(RISCVISD::SHFL, DL, VT, Match1->Op,
6941                      DAG.getConstant(Match1->ShAmt, DL, VT));
6942 }
6943 
6944 // Optimize (add (shl x, c0), (shl y, c1)) ->
6945 //          (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
6946 static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
6947                                   const RISCVSubtarget &Subtarget) {
6948   // Perform this optimization only in the zba extension.
6949   if (!Subtarget.hasStdExtZba())
6950     return SDValue();
6951 
6952   // Skip for vector types and larger types.
6953   EVT VT = N->getValueType(0);
6954   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
6955     return SDValue();
6956 
6957   // The two operand nodes must be SHL and have no other use.
6958   SDValue N0 = N->getOperand(0);
6959   SDValue N1 = N->getOperand(1);
6960   if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
6961       !N0->hasOneUse() || !N1->hasOneUse())
6962     return SDValue();
6963 
6964   // Check c0 and c1.
6965   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
6966   auto *N1C = dyn_cast<ConstantSDNode>(N1->getOperand(1));
6967   if (!N0C || !N1C)
6968     return SDValue();
6969   int64_t C0 = N0C->getSExtValue();
6970   int64_t C1 = N1C->getSExtValue();
6971   if (C0 <= 0 || C1 <= 0)
6972     return SDValue();
6973 
6974   // Skip if SH1ADD/SH2ADD/SH3ADD are not applicable.
6975   int64_t Bits = std::min(C0, C1);
6976   int64_t Diff = std::abs(C0 - C1);
6977   if (Diff != 1 && Diff != 2 && Diff != 3)
6978     return SDValue();
6979 
6980   // Build nodes.
6981   SDLoc DL(N);
6982   SDValue NS = (C0 < C1) ? N0->getOperand(0) : N1->getOperand(0);
6983   SDValue NL = (C0 > C1) ? N0->getOperand(0) : N1->getOperand(0);
6984   SDValue NA0 =
6985       DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT));
6986   SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS);
6987   return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT));
6988 }
6989 
6990 // Combine (GREVI (GREVI x, C2), C1) -> (GREVI x, C1^C2) when C1^C2 is
6991 // non-zero, and to x when it is. Any repeated GREVI stage undoes itself.
6992 // Combine (GORCI (GORCI x, C2), C1) -> (GORCI x, C1|C2). Repeated stage does
6993 // not undo itself, but they are redundant.
6994 static SDValue combineGREVI_GORCI(SDNode *N, SelectionDAG &DAG) {
6995   SDValue Src = N->getOperand(0);
6996 
6997   if (Src.getOpcode() != N->getOpcode())
6998     return SDValue();
6999 
7000   if (!isa<ConstantSDNode>(N->getOperand(1)) ||
7001       !isa<ConstantSDNode>(Src.getOperand(1)))
7002     return SDValue();
7003 
7004   unsigned ShAmt1 = N->getConstantOperandVal(1);
7005   unsigned ShAmt2 = Src.getConstantOperandVal(1);
7006   Src = Src.getOperand(0);
7007 
7008   unsigned CombinedShAmt;
7009   if (N->getOpcode() == RISCVISD::GORC || N->getOpcode() == RISCVISD::GORCW)
7010     CombinedShAmt = ShAmt1 | ShAmt2;
7011   else
7012     CombinedShAmt = ShAmt1 ^ ShAmt2;
7013 
7014   if (CombinedShAmt == 0)
7015     return Src;
7016 
7017   SDLoc DL(N);
7018   return DAG.getNode(
7019       N->getOpcode(), DL, N->getValueType(0), Src,
7020       DAG.getConstant(CombinedShAmt, DL, N->getOperand(1).getValueType()));
7021 }
7022 
7023 // Combine a constant select operand into its use:
7024 //
7025 // (and (select cond, -1, c), x)
7026 //   -> (select cond, x, (and x, c))  [AllOnes=1]
7027 // (or  (select cond, 0, c), x)
7028 //   -> (select cond, x, (or x, c))  [AllOnes=0]
7029 // (xor (select cond, 0, c), x)
7030 //   -> (select cond, x, (xor x, c))  [AllOnes=0]
7031 // (add (select cond, 0, c), x)
7032 //   -> (select cond, x, (add x, c))  [AllOnes=0]
7033 // (sub x, (select cond, 0, c))
7034 //   -> (select cond, x, (sub x, c))  [AllOnes=0]
7035 static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
7036                                    SelectionDAG &DAG, bool AllOnes) {
7037   EVT VT = N->getValueType(0);
7038 
7039   // Skip vectors.
7040   if (VT.isVector())
7041     return SDValue();
7042 
7043   if ((Slct.getOpcode() != ISD::SELECT &&
7044        Slct.getOpcode() != RISCVISD::SELECT_CC) ||
7045       !Slct.hasOneUse())
7046     return SDValue();
7047 
7048   auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
7049     return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
7050   };
7051 
7052   bool SwapSelectOps;
7053   unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
7054   SDValue TrueVal = Slct.getOperand(1 + OpOffset);
7055   SDValue FalseVal = Slct.getOperand(2 + OpOffset);
7056   SDValue NonConstantVal;
7057   if (isZeroOrAllOnes(TrueVal, AllOnes)) {
7058     SwapSelectOps = false;
7059     NonConstantVal = FalseVal;
7060   } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
7061     SwapSelectOps = true;
7062     NonConstantVal = TrueVal;
7063   } else
7064     return SDValue();
7065 
7066   // Slct is now know to be the desired identity constant when CC is true.
7067   TrueVal = OtherOp;
7068   FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
7069   // Unless SwapSelectOps says the condition should be false.
7070   if (SwapSelectOps)
7071     std::swap(TrueVal, FalseVal);
7072 
7073   if (Slct.getOpcode() == RISCVISD::SELECT_CC)
7074     return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
7075                        {Slct.getOperand(0), Slct.getOperand(1),
7076                         Slct.getOperand(2), TrueVal, FalseVal});
7077 
7078   return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
7079                      {Slct.getOperand(0), TrueVal, FalseVal});
7080 }
7081 
7082 // Attempt combineSelectAndUse on each operand of a commutative operator N.
7083 static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
7084                                               bool AllOnes) {
7085   SDValue N0 = N->getOperand(0);
7086   SDValue N1 = N->getOperand(1);
7087   if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes))
7088     return Result;
7089   if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes))
7090     return Result;
7091   return SDValue();
7092 }
7093 
7094 // Transform (add (mul x, c0), c1) ->
7095 //           (add (mul (add x, c1/c0), c0), c1%c0).
7096 // if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
7097 // that should be excluded is when c0*(c1/c0) is simm12, which will lead
7098 // to an infinite loop in DAGCombine if transformed.
7099 // Or transform (add (mul x, c0), c1) ->
7100 //              (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
7101 // if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
7102 // case that should be excluded is when c0*(c1/c0+1) is simm12, which will
7103 // lead to an infinite loop in DAGCombine if transformed.
7104 // Or transform (add (mul x, c0), c1) ->
7105 //              (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
7106 // if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
7107 // case that should be excluded is when c0*(c1/c0-1) is simm12, which will
7108 // lead to an infinite loop in DAGCombine if transformed.
7109 // Or transform (add (mul x, c0), c1) ->
7110 //              (mul (add x, c1/c0), c0).
7111 // if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
7112 static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
7113                                      const RISCVSubtarget &Subtarget) {
7114   // Skip for vector types and larger types.
7115   EVT VT = N->getValueType(0);
7116   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
7117     return SDValue();
7118   // The first operand node must be a MUL and has no other use.
7119   SDValue N0 = N->getOperand(0);
7120   if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
7121     return SDValue();
7122   // Check if c0 and c1 match above conditions.
7123   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7124   auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
7125   if (!N0C || !N1C)
7126     return SDValue();
7127   int64_t C0 = N0C->getSExtValue();
7128   int64_t C1 = N1C->getSExtValue();
7129   int64_t CA, CB;
7130   if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
7131     return SDValue();
7132   // Search for proper CA (non-zero) and CB that both are simm12.
7133   if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
7134       !isInt<12>(C0 * (C1 / C0))) {
7135     CA = C1 / C0;
7136     CB = C1 % C0;
7137   } else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
7138              isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
7139     CA = C1 / C0 + 1;
7140     CB = C1 % C0 - C0;
7141   } else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
7142              isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
7143     CA = C1 / C0 - 1;
7144     CB = C1 % C0 + C0;
7145   } else
7146     return SDValue();
7147   // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
7148   SDLoc DL(N);
7149   SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
7150                              DAG.getConstant(CA, DL, VT));
7151   SDValue New1 =
7152       DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
7153   return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
7154 }
7155 
7156 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
7157                                  const RISCVSubtarget &Subtarget) {
7158   if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
7159     return V;
7160   if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
7161     return V;
7162   // fold (add (select lhs, rhs, cc, 0, y), x) ->
7163   //      (select lhs, rhs, cc, x, (add x, y))
7164   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7165 }
7166 
7167 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG) {
7168   // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
7169   //      (select lhs, rhs, cc, x, (sub x, y))
7170   SDValue N0 = N->getOperand(0);
7171   SDValue N1 = N->getOperand(1);
7172   return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false);
7173 }
7174 
7175 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG) {
7176   // fold (and (select lhs, rhs, cc, -1, y), x) ->
7177   //      (select lhs, rhs, cc, x, (and x, y))
7178   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true);
7179 }
7180 
7181 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
7182                                 const RISCVSubtarget &Subtarget) {
7183   if (Subtarget.hasStdExtZbp()) {
7184     if (auto GREV = combineORToGREV(SDValue(N, 0), DAG, Subtarget))
7185       return GREV;
7186     if (auto GORC = combineORToGORC(SDValue(N, 0), DAG, Subtarget))
7187       return GORC;
7188     if (auto SHFL = combineORToSHFL(SDValue(N, 0), DAG, Subtarget))
7189       return SHFL;
7190   }
7191 
7192   // fold (or (select cond, 0, y), x) ->
7193   //      (select cond, x, (or x, y))
7194   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7195 }
7196 
7197 static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG) {
7198   // fold (xor (select cond, 0, y), x) ->
7199   //      (select cond, x, (xor x, y))
7200   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7201 }
7202 
7203 // Attempt to turn ANY_EXTEND into SIGN_EXTEND if the input to the ANY_EXTEND
7204 // has users that require SIGN_EXTEND and the SIGN_EXTEND can be done for free
7205 // by an instruction like ADDW/SUBW/MULW. Without this the ANY_EXTEND would be
7206 // removed during type legalization leaving an ADD/SUB/MUL use that won't use
7207 // ADDW/SUBW/MULW.
7208 static SDValue performANY_EXTENDCombine(SDNode *N,
7209                                         TargetLowering::DAGCombinerInfo &DCI,
7210                                         const RISCVSubtarget &Subtarget) {
7211   if (!Subtarget.is64Bit())
7212     return SDValue();
7213 
7214   SelectionDAG &DAG = DCI.DAG;
7215 
7216   SDValue Src = N->getOperand(0);
7217   EVT VT = N->getValueType(0);
7218   if (VT != MVT::i64 || Src.getValueType() != MVT::i32)
7219     return SDValue();
7220 
7221   // The opcode must be one that can implicitly sign_extend.
7222   // FIXME: Additional opcodes.
7223   switch (Src.getOpcode()) {
7224   default:
7225     return SDValue();
7226   case ISD::MUL:
7227     if (!Subtarget.hasStdExtM())
7228       return SDValue();
7229     LLVM_FALLTHROUGH;
7230   case ISD::ADD:
7231   case ISD::SUB:
7232     break;
7233   }
7234 
7235   // Only handle cases where the result is used by a CopyToReg. That likely
7236   // means the value is a liveout of the basic block. This helps prevent
7237   // infinite combine loops like PR51206.
7238   if (none_of(N->uses(),
7239               [](SDNode *User) { return User->getOpcode() == ISD::CopyToReg; }))
7240     return SDValue();
7241 
7242   SmallVector<SDNode *, 4> SetCCs;
7243   for (SDNode::use_iterator UI = Src.getNode()->use_begin(),
7244                             UE = Src.getNode()->use_end();
7245        UI != UE; ++UI) {
7246     SDNode *User = *UI;
7247     if (User == N)
7248       continue;
7249     if (UI.getUse().getResNo() != Src.getResNo())
7250       continue;
7251     // All i32 setccs are legalized by sign extending operands.
7252     if (User->getOpcode() == ISD::SETCC) {
7253       SetCCs.push_back(User);
7254       continue;
7255     }
7256     // We don't know if we can extend this user.
7257     break;
7258   }
7259 
7260   // If we don't have any SetCCs, this isn't worthwhile.
7261   if (SetCCs.empty())
7262     return SDValue();
7263 
7264   SDLoc DL(N);
7265   SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Src);
7266   DCI.CombineTo(N, SExt);
7267 
7268   // Promote all the setccs.
7269   for (SDNode *SetCC : SetCCs) {
7270     SmallVector<SDValue, 4> Ops;
7271 
7272     for (unsigned j = 0; j != 2; ++j) {
7273       SDValue SOp = SetCC->getOperand(j);
7274       if (SOp == Src)
7275         Ops.push_back(SExt);
7276       else
7277         Ops.push_back(DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, SOp));
7278     }
7279 
7280     Ops.push_back(SetCC->getOperand(2));
7281     DCI.CombineTo(SetCC,
7282                   DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
7283   }
7284   return SDValue(N, 0);
7285 }
7286 
7287 // Try to form VWMUL or VWMULU.
7288 // FIXME: Support VWMULSU.
7289 static SDValue combineMUL_VLToVWMUL_VL(SDNode *N, SelectionDAG &DAG,
7290                                        bool Commute) {
7291   assert(N->getOpcode() == RISCVISD::MUL_VL && "Unexpected opcode");
7292   SDValue Op0 = N->getOperand(0);
7293   SDValue Op1 = N->getOperand(1);
7294   if (Commute)
7295     std::swap(Op0, Op1);
7296 
7297   bool IsSignExt = Op0.getOpcode() == RISCVISD::VSEXT_VL;
7298   bool IsZeroExt = Op0.getOpcode() == RISCVISD::VZEXT_VL;
7299   if ((!IsSignExt && !IsZeroExt) || !Op0.hasOneUse())
7300     return SDValue();
7301 
7302   SDValue Mask = N->getOperand(2);
7303   SDValue VL = N->getOperand(3);
7304 
7305   // Make sure the mask and VL match.
7306   if (Op0.getOperand(1) != Mask || Op0.getOperand(2) != VL)
7307     return SDValue();
7308 
7309   MVT VT = N->getSimpleValueType(0);
7310 
7311   // Determine the narrow size for a widening multiply.
7312   unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
7313   MVT NarrowVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowSize),
7314                                   VT.getVectorElementCount());
7315 
7316   SDLoc DL(N);
7317 
7318   // See if the other operand is the same opcode.
7319   if (Op0.getOpcode() == Op1.getOpcode()) {
7320     if (!Op1.hasOneUse())
7321       return SDValue();
7322 
7323     // Make sure the mask and VL match.
7324     if (Op1.getOperand(1) != Mask || Op1.getOperand(2) != VL)
7325       return SDValue();
7326 
7327     Op1 = Op1.getOperand(0);
7328   } else if (Op1.getOpcode() == RISCVISD::VMV_V_X_VL) {
7329     // The operand is a splat of a scalar.
7330 
7331     // The VL must be the same.
7332     if (Op1.getOperand(1) != VL)
7333       return SDValue();
7334 
7335     // Get the scalar value.
7336     Op1 = Op1.getOperand(0);
7337 
7338     // See if have enough sign bits or zero bits in the scalar to use a
7339     // widening multiply by splatting to smaller element size.
7340     unsigned EltBits = VT.getScalarSizeInBits();
7341     unsigned ScalarBits = Op1.getValueSizeInBits();
7342     // Make sure we're getting all element bits from the scalar register.
7343     // FIXME: Support implicit sign extension of vmv.v.x?
7344     if (ScalarBits < EltBits)
7345       return SDValue();
7346 
7347     if (IsSignExt) {
7348       if (DAG.ComputeNumSignBits(Op1) <= (ScalarBits - NarrowSize))
7349         return SDValue();
7350     } else {
7351       APInt Mask = APInt::getBitsSetFrom(ScalarBits, NarrowSize);
7352       if (!DAG.MaskedValueIsZero(Op1, Mask))
7353         return SDValue();
7354     }
7355 
7356     Op1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT, Op1, VL);
7357   } else
7358     return SDValue();
7359 
7360   Op0 = Op0.getOperand(0);
7361 
7362   // Re-introduce narrower extends if needed.
7363   unsigned ExtOpc = IsSignExt ? RISCVISD::VSEXT_VL : RISCVISD::VZEXT_VL;
7364   if (Op0.getValueType() != NarrowVT)
7365     Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
7366   if (Op1.getValueType() != NarrowVT)
7367     Op1 = DAG.getNode(ExtOpc, DL, NarrowVT, Op1, Mask, VL);
7368 
7369   unsigned WMulOpc = IsSignExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
7370   return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Mask, VL);
7371 }
7372 
7373 static RISCVFPRndMode::RoundingMode matchRoundingOp(SDValue Op) {
7374   switch (Op.getOpcode()) {
7375   case ISD::FROUNDEVEN: return RISCVFPRndMode::RNE;
7376   case ISD::FTRUNC:     return RISCVFPRndMode::RTZ;
7377   case ISD::FFLOOR:     return RISCVFPRndMode::RDN;
7378   case ISD::FCEIL:      return RISCVFPRndMode::RUP;
7379   case ISD::FROUND:     return RISCVFPRndMode::RMM;
7380   }
7381 
7382   return RISCVFPRndMode::Invalid;
7383 }
7384 
7385 // Fold
7386 //   (fp_to_int (froundeven X)) -> fcvt X, rne
7387 //   (fp_to_int (ftrunc X))     -> fcvt X, rtz
7388 //   (fp_to_int (ffloor X))     -> fcvt X, rdn
7389 //   (fp_to_int (fceil X))      -> fcvt X, rup
7390 //   (fp_to_int (fround X))     -> fcvt X, rmm
7391 static SDValue performFP_TO_INTCombine(SDNode *N,
7392                                        TargetLowering::DAGCombinerInfo &DCI,
7393                                        const RISCVSubtarget &Subtarget) {
7394   SelectionDAG &DAG = DCI.DAG;
7395   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7396   MVT XLenVT = Subtarget.getXLenVT();
7397 
7398   // Only handle XLen or i32 types. Other types narrower than XLen will
7399   // eventually be legalized to XLenVT.
7400   EVT VT = N->getValueType(0);
7401   if (VT != MVT::i32 && VT != XLenVT)
7402     return SDValue();
7403 
7404   SDValue Src = N->getOperand(0);
7405 
7406   // Ensure the FP type is also legal.
7407   if (!TLI.isTypeLegal(Src.getValueType()))
7408     return SDValue();
7409 
7410   // Don't do this for f16 with Zfhmin and not Zfh.
7411   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7412     return SDValue();
7413 
7414   RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src);
7415   if (FRM == RISCVFPRndMode::Invalid)
7416     return SDValue();
7417 
7418   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
7419 
7420   unsigned Opc;
7421   if (VT == XLenVT)
7422     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7423   else
7424     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7425 
7426   SDLoc DL(N);
7427   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src.getOperand(0),
7428                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7429   return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
7430 }
7431 
7432 // Fold
7433 //   (fp_to_int_sat (froundeven X)) -> (select X == nan, 0, (fcvt X, rne))
7434 //   (fp_to_int_sat (ftrunc X))     -> (select X == nan, 0, (fcvt X, rtz))
7435 //   (fp_to_int_sat (ffloor X))     -> (select X == nan, 0, (fcvt X, rdn))
7436 //   (fp_to_int_sat (fceil X))      -> (select X == nan, 0, (fcvt X, rup))
7437 //   (fp_to_int_sat (fround X))     -> (select X == nan, 0, (fcvt X, rmm))
7438 static SDValue performFP_TO_INT_SATCombine(SDNode *N,
7439                                        TargetLowering::DAGCombinerInfo &DCI,
7440                                        const RISCVSubtarget &Subtarget) {
7441   SelectionDAG &DAG = DCI.DAG;
7442   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7443   MVT XLenVT = Subtarget.getXLenVT();
7444 
7445   // Only handle XLen types. Other types narrower than XLen will eventually be
7446   // legalized to XLenVT.
7447   EVT DstVT = N->getValueType(0);
7448   if (DstVT != XLenVT)
7449     return SDValue();
7450 
7451   SDValue Src = N->getOperand(0);
7452 
7453   // Ensure the FP type is also legal.
7454   if (!TLI.isTypeLegal(Src.getValueType()))
7455     return SDValue();
7456 
7457   // Don't do this for f16 with Zfhmin and not Zfh.
7458   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7459     return SDValue();
7460 
7461   EVT SatVT = cast<VTSDNode>(N->getOperand(1))->getVT();
7462 
7463   RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src);
7464   if (FRM == RISCVFPRndMode::Invalid)
7465     return SDValue();
7466 
7467   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
7468 
7469   unsigned Opc;
7470   if (SatVT == DstVT)
7471     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7472   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
7473     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7474   else
7475     return SDValue();
7476   // FIXME: Support other SatVTs by clamping before or after the conversion.
7477 
7478   Src = Src.getOperand(0);
7479 
7480   SDLoc DL(N);
7481   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src,
7482                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7483 
7484   // RISCV FP-to-int conversions saturate to the destination register size, but
7485   // don't produce 0 for nan.
7486   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
7487   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
7488 }
7489 
7490 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
7491                                                DAGCombinerInfo &DCI) const {
7492   SelectionDAG &DAG = DCI.DAG;
7493 
7494   // Helper to call SimplifyDemandedBits on an operand of N where only some low
7495   // bits are demanded. N will be added to the Worklist if it was not deleted.
7496   // Caller should return SDValue(N, 0) if this returns true.
7497   auto SimplifyDemandedLowBitsHelper = [&](unsigned OpNo, unsigned LowBits) {
7498     SDValue Op = N->getOperand(OpNo);
7499     APInt Mask = APInt::getLowBitsSet(Op.getValueSizeInBits(), LowBits);
7500     if (!SimplifyDemandedBits(Op, Mask, DCI))
7501       return false;
7502 
7503     if (N->getOpcode() != ISD::DELETED_NODE)
7504       DCI.AddToWorklist(N);
7505     return true;
7506   };
7507 
7508   switch (N->getOpcode()) {
7509   default:
7510     break;
7511   case RISCVISD::SplitF64: {
7512     SDValue Op0 = N->getOperand(0);
7513     // If the input to SplitF64 is just BuildPairF64 then the operation is
7514     // redundant. Instead, use BuildPairF64's operands directly.
7515     if (Op0->getOpcode() == RISCVISD::BuildPairF64)
7516       return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
7517 
7518     SDLoc DL(N);
7519 
7520     // It's cheaper to materialise two 32-bit integers than to load a double
7521     // from the constant pool and transfer it to integer registers through the
7522     // stack.
7523     if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
7524       APInt V = C->getValueAPF().bitcastToAPInt();
7525       SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
7526       SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
7527       return DCI.CombineTo(N, Lo, Hi);
7528     }
7529 
7530     // This is a target-specific version of a DAGCombine performed in
7531     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7532     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7533     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7534     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7535         !Op0.getNode()->hasOneUse())
7536       break;
7537     SDValue NewSplitF64 =
7538         DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
7539                     Op0.getOperand(0));
7540     SDValue Lo = NewSplitF64.getValue(0);
7541     SDValue Hi = NewSplitF64.getValue(1);
7542     APInt SignBit = APInt::getSignMask(32);
7543     if (Op0.getOpcode() == ISD::FNEG) {
7544       SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
7545                                   DAG.getConstant(SignBit, DL, MVT::i32));
7546       return DCI.CombineTo(N, Lo, NewHi);
7547     }
7548     assert(Op0.getOpcode() == ISD::FABS);
7549     SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
7550                                 DAG.getConstant(~SignBit, DL, MVT::i32));
7551     return DCI.CombineTo(N, Lo, NewHi);
7552   }
7553   case RISCVISD::SLLW:
7554   case RISCVISD::SRAW:
7555   case RISCVISD::SRLW:
7556   case RISCVISD::ROLW:
7557   case RISCVISD::RORW: {
7558     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7559     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7560         SimplifyDemandedLowBitsHelper(1, 5))
7561       return SDValue(N, 0);
7562     break;
7563   }
7564   case RISCVISD::CLZW:
7565   case RISCVISD::CTZW: {
7566     // Only the lower 32 bits of the first operand are read
7567     if (SimplifyDemandedLowBitsHelper(0, 32))
7568       return SDValue(N, 0);
7569     break;
7570   }
7571   case RISCVISD::GREV:
7572   case RISCVISD::GORC: {
7573     // Only the lower log2(Bitwidth) bits of the the shift amount are read.
7574     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7575     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7576     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth)))
7577       return SDValue(N, 0);
7578 
7579     return combineGREVI_GORCI(N, DAG);
7580   }
7581   case RISCVISD::GREVW:
7582   case RISCVISD::GORCW: {
7583     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7584     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7585         SimplifyDemandedLowBitsHelper(1, 5))
7586       return SDValue(N, 0);
7587 
7588     return combineGREVI_GORCI(N, DAG);
7589   }
7590   case RISCVISD::SHFL:
7591   case RISCVISD::UNSHFL: {
7592     // Only the lower log2(Bitwidth)-1 bits of the the shift amount are read.
7593     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7594     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7595     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth) - 1))
7596       return SDValue(N, 0);
7597 
7598     break;
7599   }
7600   case RISCVISD::SHFLW:
7601   case RISCVISD::UNSHFLW: {
7602     // Only the lower 32 bits of LHS and lower 4 bits of RHS are read.
7603     SDValue LHS = N->getOperand(0);
7604     SDValue RHS = N->getOperand(1);
7605     APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
7606     APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 4);
7607     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7608         SimplifyDemandedLowBitsHelper(1, 4))
7609       return SDValue(N, 0);
7610 
7611     break;
7612   }
7613   case RISCVISD::BCOMPRESSW:
7614   case RISCVISD::BDECOMPRESSW: {
7615     // Only the lower 32 bits of LHS and RHS are read.
7616     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7617         SimplifyDemandedLowBitsHelper(1, 32))
7618       return SDValue(N, 0);
7619 
7620     break;
7621   }
7622   case RISCVISD::FMV_X_ANYEXTH:
7623   case RISCVISD::FMV_X_ANYEXTW_RV64: {
7624     SDLoc DL(N);
7625     SDValue Op0 = N->getOperand(0);
7626     MVT VT = N->getSimpleValueType(0);
7627     // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
7628     // conversion is unnecessary and can be replaced with the FMV_W_X_RV64
7629     // operand. Similar for FMV_X_ANYEXTH and FMV_H_X.
7630     if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
7631          Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
7632         (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
7633          Op0->getOpcode() == RISCVISD::FMV_H_X)) {
7634       assert(Op0.getOperand(0).getValueType() == VT &&
7635              "Unexpected value type!");
7636       return Op0.getOperand(0);
7637     }
7638 
7639     // This is a target-specific version of a DAGCombine performed in
7640     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7641     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7642     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7643     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7644         !Op0.getNode()->hasOneUse())
7645       break;
7646     SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
7647     unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
7648     APInt SignBit = APInt::getSignMask(FPBits).sextOrSelf(VT.getSizeInBits());
7649     if (Op0.getOpcode() == ISD::FNEG)
7650       return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
7651                          DAG.getConstant(SignBit, DL, VT));
7652 
7653     assert(Op0.getOpcode() == ISD::FABS);
7654     return DAG.getNode(ISD::AND, DL, VT, NewFMV,
7655                        DAG.getConstant(~SignBit, DL, VT));
7656   }
7657   case ISD::ADD:
7658     return performADDCombine(N, DAG, Subtarget);
7659   case ISD::SUB:
7660     return performSUBCombine(N, DAG);
7661   case ISD::AND:
7662     return performANDCombine(N, DAG);
7663   case ISD::OR:
7664     return performORCombine(N, DAG, Subtarget);
7665   case ISD::XOR:
7666     return performXORCombine(N, DAG);
7667   case ISD::ANY_EXTEND:
7668     return performANY_EXTENDCombine(N, DCI, Subtarget);
7669   case ISD::ZERO_EXTEND:
7670     // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
7671     // type legalization. This is safe because fp_to_uint produces poison if
7672     // it overflows.
7673     if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) {
7674       SDValue Src = N->getOperand(0);
7675       if (Src.getOpcode() == ISD::FP_TO_UINT &&
7676           isTypeLegal(Src.getOperand(0).getValueType()))
7677         return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
7678                            Src.getOperand(0));
7679       if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
7680           isTypeLegal(Src.getOperand(1).getValueType())) {
7681         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
7682         SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs,
7683                                   Src.getOperand(0), Src.getOperand(1));
7684         DCI.CombineTo(N, Res);
7685         DAG.ReplaceAllUsesOfValueWith(Src.getValue(1), Res.getValue(1));
7686         DCI.recursivelyDeleteUnusedNodes(Src.getNode());
7687         return SDValue(N, 0); // Return N so it doesn't get rechecked.
7688       }
7689     }
7690     return SDValue();
7691   case RISCVISD::SELECT_CC: {
7692     // Transform
7693     SDValue LHS = N->getOperand(0);
7694     SDValue RHS = N->getOperand(1);
7695     SDValue TrueV = N->getOperand(3);
7696     SDValue FalseV = N->getOperand(4);
7697 
7698     // If the True and False values are the same, we don't need a select_cc.
7699     if (TrueV == FalseV)
7700       return TrueV;
7701 
7702     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(2))->get();
7703     if (!ISD::isIntEqualitySetCC(CCVal))
7704       break;
7705 
7706     // Fold (select_cc (setlt X, Y), 0, ne, trueV, falseV) ->
7707     //      (select_cc X, Y, lt, trueV, falseV)
7708     // Sometimes the setcc is introduced after select_cc has been formed.
7709     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
7710         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
7711       // If we're looking for eq 0 instead of ne 0, we need to invert the
7712       // condition.
7713       bool Invert = CCVal == ISD::SETEQ;
7714       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
7715       if (Invert)
7716         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7717 
7718       SDLoc DL(N);
7719       RHS = LHS.getOperand(1);
7720       LHS = LHS.getOperand(0);
7721       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7722 
7723       SDValue TargetCC = DAG.getCondCode(CCVal);
7724       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
7725                          {LHS, RHS, TargetCC, TrueV, FalseV});
7726     }
7727 
7728     // Fold (select_cc (xor X, Y), 0, eq/ne, trueV, falseV) ->
7729     //      (select_cc X, Y, eq/ne, trueV, falseV)
7730     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
7731       return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), N->getValueType(0),
7732                          {LHS.getOperand(0), LHS.getOperand(1),
7733                           N->getOperand(2), TrueV, FalseV});
7734     // (select_cc X, 1, setne, trueV, falseV) ->
7735     // (select_cc X, 0, seteq, trueV, falseV) if we can prove X is 0/1.
7736     // This can occur when legalizing some floating point comparisons.
7737     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
7738     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
7739       SDLoc DL(N);
7740       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7741       SDValue TargetCC = DAG.getCondCode(CCVal);
7742       RHS = DAG.getConstant(0, DL, LHS.getValueType());
7743       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
7744                          {LHS, RHS, TargetCC, TrueV, FalseV});
7745     }
7746 
7747     break;
7748   }
7749   case RISCVISD::BR_CC: {
7750     SDValue LHS = N->getOperand(1);
7751     SDValue RHS = N->getOperand(2);
7752     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(3))->get();
7753     if (!ISD::isIntEqualitySetCC(CCVal))
7754       break;
7755 
7756     // Fold (br_cc (setlt X, Y), 0, ne, dest) ->
7757     //      (br_cc X, Y, lt, dest)
7758     // Sometimes the setcc is introduced after br_cc has been formed.
7759     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
7760         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
7761       // If we're looking for eq 0 instead of ne 0, we need to invert the
7762       // condition.
7763       bool Invert = CCVal == ISD::SETEQ;
7764       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
7765       if (Invert)
7766         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7767 
7768       SDLoc DL(N);
7769       RHS = LHS.getOperand(1);
7770       LHS = LHS.getOperand(0);
7771       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7772 
7773       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
7774                          N->getOperand(0), LHS, RHS, DAG.getCondCode(CCVal),
7775                          N->getOperand(4));
7776     }
7777 
7778     // Fold (br_cc (xor X, Y), 0, eq/ne, dest) ->
7779     //      (br_cc X, Y, eq/ne, trueV, falseV)
7780     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
7781       return DAG.getNode(RISCVISD::BR_CC, SDLoc(N), N->getValueType(0),
7782                          N->getOperand(0), LHS.getOperand(0), LHS.getOperand(1),
7783                          N->getOperand(3), N->getOperand(4));
7784 
7785     // (br_cc X, 1, setne, br_cc) ->
7786     // (br_cc X, 0, seteq, br_cc) if we can prove X is 0/1.
7787     // This can occur when legalizing some floating point comparisons.
7788     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
7789     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
7790       SDLoc DL(N);
7791       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7792       SDValue TargetCC = DAG.getCondCode(CCVal);
7793       RHS = DAG.getConstant(0, DL, LHS.getValueType());
7794       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
7795                          N->getOperand(0), LHS, RHS, TargetCC,
7796                          N->getOperand(4));
7797     }
7798     break;
7799   }
7800   case ISD::FP_TO_SINT:
7801   case ISD::FP_TO_UINT:
7802     return performFP_TO_INTCombine(N, DCI, Subtarget);
7803   case ISD::FP_TO_SINT_SAT:
7804   case ISD::FP_TO_UINT_SAT:
7805     return performFP_TO_INT_SATCombine(N, DCI, Subtarget);
7806   case ISD::FCOPYSIGN: {
7807     EVT VT = N->getValueType(0);
7808     if (!VT.isVector())
7809       break;
7810     // There is a form of VFSGNJ which injects the negated sign of its second
7811     // operand. Try and bubble any FNEG up after the extend/round to produce
7812     // this optimized pattern. Avoid modifying cases where FP_ROUND and
7813     // TRUNC=1.
7814     SDValue In2 = N->getOperand(1);
7815     // Avoid cases where the extend/round has multiple uses, as duplicating
7816     // those is typically more expensive than removing a fneg.
7817     if (!In2.hasOneUse())
7818       break;
7819     if (In2.getOpcode() != ISD::FP_EXTEND &&
7820         (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0))
7821       break;
7822     In2 = In2.getOperand(0);
7823     if (In2.getOpcode() != ISD::FNEG)
7824       break;
7825     SDLoc DL(N);
7826     SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
7827     return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
7828                        DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
7829   }
7830   case ISD::MGATHER:
7831   case ISD::MSCATTER:
7832   case ISD::VP_GATHER:
7833   case ISD::VP_SCATTER: {
7834     if (!DCI.isBeforeLegalize())
7835       break;
7836     SDValue Index, ScaleOp;
7837     bool IsIndexScaled = false;
7838     bool IsIndexSigned = false;
7839     if (const auto *VPGSN = dyn_cast<VPGatherScatterSDNode>(N)) {
7840       Index = VPGSN->getIndex();
7841       ScaleOp = VPGSN->getScale();
7842       IsIndexScaled = VPGSN->isIndexScaled();
7843       IsIndexSigned = VPGSN->isIndexSigned();
7844     } else {
7845       const auto *MGSN = cast<MaskedGatherScatterSDNode>(N);
7846       Index = MGSN->getIndex();
7847       ScaleOp = MGSN->getScale();
7848       IsIndexScaled = MGSN->isIndexScaled();
7849       IsIndexSigned = MGSN->isIndexSigned();
7850     }
7851     EVT IndexVT = Index.getValueType();
7852     MVT XLenVT = Subtarget.getXLenVT();
7853     // RISCV indexed loads only support the "unsigned unscaled" addressing
7854     // mode, so anything else must be manually legalized.
7855     bool NeedsIdxLegalization =
7856         IsIndexScaled ||
7857         (IsIndexSigned && IndexVT.getVectorElementType().bitsLT(XLenVT));
7858     if (!NeedsIdxLegalization)
7859       break;
7860 
7861     SDLoc DL(N);
7862 
7863     // Any index legalization should first promote to XLenVT, so we don't lose
7864     // bits when scaling. This may create an illegal index type so we let
7865     // LLVM's legalization take care of the splitting.
7866     // FIXME: LLVM can't split VP_GATHER or VP_SCATTER yet.
7867     if (IndexVT.getVectorElementType().bitsLT(XLenVT)) {
7868       IndexVT = IndexVT.changeVectorElementType(XLenVT);
7869       Index = DAG.getNode(IsIndexSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
7870                           DL, IndexVT, Index);
7871     }
7872 
7873     unsigned Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue();
7874     if (IsIndexScaled && Scale != 1) {
7875       // Manually scale the indices by the element size.
7876       // TODO: Sanitize the scale operand here?
7877       // TODO: For VP nodes, should we use VP_SHL here?
7878       assert(isPowerOf2_32(Scale) && "Expecting power-of-two types");
7879       SDValue SplatScale = DAG.getConstant(Log2_32(Scale), DL, IndexVT);
7880       Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, SplatScale);
7881     }
7882 
7883     ISD::MemIndexType NewIndexTy = ISD::UNSIGNED_UNSCALED;
7884     if (const auto *VPGN = dyn_cast<VPGatherSDNode>(N))
7885       return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
7886                              {VPGN->getChain(), VPGN->getBasePtr(), Index,
7887                               VPGN->getScale(), VPGN->getMask(),
7888                               VPGN->getVectorLength()},
7889                              VPGN->getMemOperand(), NewIndexTy);
7890     if (const auto *VPSN = dyn_cast<VPScatterSDNode>(N))
7891       return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
7892                               {VPSN->getChain(), VPSN->getValue(),
7893                                VPSN->getBasePtr(), Index, VPSN->getScale(),
7894                                VPSN->getMask(), VPSN->getVectorLength()},
7895                               VPSN->getMemOperand(), NewIndexTy);
7896     if (const auto *MGN = dyn_cast<MaskedGatherSDNode>(N))
7897       return DAG.getMaskedGather(
7898           N->getVTList(), MGN->getMemoryVT(), DL,
7899           {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
7900            MGN->getBasePtr(), Index, MGN->getScale()},
7901           MGN->getMemOperand(), NewIndexTy, MGN->getExtensionType());
7902     const auto *MSN = cast<MaskedScatterSDNode>(N);
7903     return DAG.getMaskedScatter(
7904         N->getVTList(), MSN->getMemoryVT(), DL,
7905         {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
7906          Index, MSN->getScale()},
7907         MSN->getMemOperand(), NewIndexTy, MSN->isTruncatingStore());
7908   }
7909   case RISCVISD::SRA_VL:
7910   case RISCVISD::SRL_VL:
7911   case RISCVISD::SHL_VL: {
7912     SDValue ShAmt = N->getOperand(1);
7913     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
7914       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
7915       SDLoc DL(N);
7916       SDValue VL = N->getOperand(3);
7917       EVT VT = N->getValueType(0);
7918       ShAmt =
7919           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, ShAmt.getOperand(0), VL);
7920       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
7921                          N->getOperand(2), N->getOperand(3));
7922     }
7923     break;
7924   }
7925   case ISD::SRA:
7926   case ISD::SRL:
7927   case ISD::SHL: {
7928     SDValue ShAmt = N->getOperand(1);
7929     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
7930       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
7931       SDLoc DL(N);
7932       EVT VT = N->getValueType(0);
7933       ShAmt =
7934           DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VT, ShAmt.getOperand(0));
7935       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
7936     }
7937     break;
7938   }
7939   case RISCVISD::MUL_VL:
7940     if (SDValue V = combineMUL_VLToVWMUL_VL(N, DAG, /*Commute*/ false))
7941       return V;
7942     // Mul is commutative.
7943     return combineMUL_VLToVWMUL_VL(N, DAG, /*Commute*/ true);
7944   case ISD::STORE: {
7945     auto *Store = cast<StoreSDNode>(N);
7946     SDValue Val = Store->getValue();
7947     // Combine store of vmv.x.s to vse with VL of 1.
7948     // FIXME: Support FP.
7949     if (Val.getOpcode() == RISCVISD::VMV_X_S) {
7950       SDValue Src = Val.getOperand(0);
7951       EVT VecVT = Src.getValueType();
7952       EVT MemVT = Store->getMemoryVT();
7953       // The memory VT and the element type must match.
7954       if (VecVT.getVectorElementType() == MemVT) {
7955         SDLoc DL(N);
7956         MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
7957         return DAG.getStoreVP(
7958             Store->getChain(), DL, Src, Store->getBasePtr(), Store->getOffset(),
7959             DAG.getConstant(1, DL, MaskVT),
7960             DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT,
7961             Store->getMemOperand(), Store->getAddressingMode(),
7962             Store->isTruncatingStore(), /*IsCompress*/ false);
7963       }
7964     }
7965 
7966     break;
7967   }
7968   }
7969 
7970   return SDValue();
7971 }
7972 
7973 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
7974     const SDNode *N, CombineLevel Level) const {
7975   // The following folds are only desirable if `(OP _, c1 << c2)` can be
7976   // materialised in fewer instructions than `(OP _, c1)`:
7977   //
7978   //   (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
7979   //   (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
7980   SDValue N0 = N->getOperand(0);
7981   EVT Ty = N0.getValueType();
7982   if (Ty.isScalarInteger() &&
7983       (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
7984     auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7985     auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
7986     if (C1 && C2) {
7987       const APInt &C1Int = C1->getAPIntValue();
7988       APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
7989 
7990       // We can materialise `c1 << c2` into an add immediate, so it's "free",
7991       // and the combine should happen, to potentially allow further combines
7992       // later.
7993       if (ShiftedC1Int.getMinSignedBits() <= 64 &&
7994           isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
7995         return true;
7996 
7997       // We can materialise `c1` in an add immediate, so it's "free", and the
7998       // combine should be prevented.
7999       if (C1Int.getMinSignedBits() <= 64 &&
8000           isLegalAddImmediate(C1Int.getSExtValue()))
8001         return false;
8002 
8003       // Neither constant will fit into an immediate, so find materialisation
8004       // costs.
8005       int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
8006                                               Subtarget.getFeatureBits(),
8007                                               /*CompressionCost*/true);
8008       int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
8009           ShiftedC1Int, Ty.getSizeInBits(), Subtarget.getFeatureBits(),
8010           /*CompressionCost*/true);
8011 
8012       // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
8013       // combine should be prevented.
8014       if (C1Cost < ShiftedC1Cost)
8015         return false;
8016     }
8017   }
8018   return true;
8019 }
8020 
8021 bool RISCVTargetLowering::targetShrinkDemandedConstant(
8022     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
8023     TargetLoweringOpt &TLO) const {
8024   // Delay this optimization as late as possible.
8025   if (!TLO.LegalOps)
8026     return false;
8027 
8028   EVT VT = Op.getValueType();
8029   if (VT.isVector())
8030     return false;
8031 
8032   // Only handle AND for now.
8033   if (Op.getOpcode() != ISD::AND)
8034     return false;
8035 
8036   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
8037   if (!C)
8038     return false;
8039 
8040   const APInt &Mask = C->getAPIntValue();
8041 
8042   // Clear all non-demanded bits initially.
8043   APInt ShrunkMask = Mask & DemandedBits;
8044 
8045   // Try to make a smaller immediate by setting undemanded bits.
8046 
8047   APInt ExpandedMask = Mask | ~DemandedBits;
8048 
8049   auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
8050     return ShrunkMask.isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
8051   };
8052   auto UseMask = [Mask, Op, VT, &TLO](const APInt &NewMask) -> bool {
8053     if (NewMask == Mask)
8054       return true;
8055     SDLoc DL(Op);
8056     SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
8057     SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
8058     return TLO.CombineTo(Op, NewOp);
8059   };
8060 
8061   // If the shrunk mask fits in sign extended 12 bits, let the target
8062   // independent code apply it.
8063   if (ShrunkMask.isSignedIntN(12))
8064     return false;
8065 
8066   // Preserve (and X, 0xffff) when zext.h is supported.
8067   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
8068     APInt NewMask = APInt(Mask.getBitWidth(), 0xffff);
8069     if (IsLegalMask(NewMask))
8070       return UseMask(NewMask);
8071   }
8072 
8073   // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
8074   if (VT == MVT::i64) {
8075     APInt NewMask = APInt(64, 0xffffffff);
8076     if (IsLegalMask(NewMask))
8077       return UseMask(NewMask);
8078   }
8079 
8080   // For the remaining optimizations, we need to be able to make a negative
8081   // number through a combination of mask and undemanded bits.
8082   if (!ExpandedMask.isNegative())
8083     return false;
8084 
8085   // What is the fewest number of bits we need to represent the negative number.
8086   unsigned MinSignedBits = ExpandedMask.getMinSignedBits();
8087 
8088   // Try to make a 12 bit negative immediate. If that fails try to make a 32
8089   // bit negative immediate unless the shrunk immediate already fits in 32 bits.
8090   APInt NewMask = ShrunkMask;
8091   if (MinSignedBits <= 12)
8092     NewMask.setBitsFrom(11);
8093   else if (MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(32))
8094     NewMask.setBitsFrom(31);
8095   else
8096     return false;
8097 
8098   // Check that our new mask is a subset of the demanded mask.
8099   assert(IsLegalMask(NewMask));
8100   return UseMask(NewMask);
8101 }
8102 
8103 static void computeGREV(APInt &Src, unsigned ShAmt) {
8104   ShAmt &= Src.getBitWidth() - 1;
8105   uint64_t x = Src.getZExtValue();
8106   if (ShAmt & 1)
8107     x = ((x & 0x5555555555555555LL) << 1) | ((x & 0xAAAAAAAAAAAAAAAALL) >> 1);
8108   if (ShAmt & 2)
8109     x = ((x & 0x3333333333333333LL) << 2) | ((x & 0xCCCCCCCCCCCCCCCCLL) >> 2);
8110   if (ShAmt & 4)
8111     x = ((x & 0x0F0F0F0F0F0F0F0FLL) << 4) | ((x & 0xF0F0F0F0F0F0F0F0LL) >> 4);
8112   if (ShAmt & 8)
8113     x = ((x & 0x00FF00FF00FF00FFLL) << 8) | ((x & 0xFF00FF00FF00FF00LL) >> 8);
8114   if (ShAmt & 16)
8115     x = ((x & 0x0000FFFF0000FFFFLL) << 16) | ((x & 0xFFFF0000FFFF0000LL) >> 16);
8116   if (ShAmt & 32)
8117     x = ((x & 0x00000000FFFFFFFFLL) << 32) | ((x & 0xFFFFFFFF00000000LL) >> 32);
8118   Src = x;
8119 }
8120 
8121 void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
8122                                                         KnownBits &Known,
8123                                                         const APInt &DemandedElts,
8124                                                         const SelectionDAG &DAG,
8125                                                         unsigned Depth) const {
8126   unsigned BitWidth = Known.getBitWidth();
8127   unsigned Opc = Op.getOpcode();
8128   assert((Opc >= ISD::BUILTIN_OP_END ||
8129           Opc == ISD::INTRINSIC_WO_CHAIN ||
8130           Opc == ISD::INTRINSIC_W_CHAIN ||
8131           Opc == ISD::INTRINSIC_VOID) &&
8132          "Should use MaskedValueIsZero if you don't know whether Op"
8133          " is a target node!");
8134 
8135   Known.resetAll();
8136   switch (Opc) {
8137   default: break;
8138   case RISCVISD::SELECT_CC: {
8139     Known = DAG.computeKnownBits(Op.getOperand(4), Depth + 1);
8140     // If we don't know any bits, early out.
8141     if (Known.isUnknown())
8142       break;
8143     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(3), Depth + 1);
8144 
8145     // Only known if known in both the LHS and RHS.
8146     Known = KnownBits::commonBits(Known, Known2);
8147     break;
8148   }
8149   case RISCVISD::REMUW: {
8150     KnownBits Known2;
8151     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
8152     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
8153     // We only care about the lower 32 bits.
8154     Known = KnownBits::urem(Known.trunc(32), Known2.trunc(32));
8155     // Restore the original width by sign extending.
8156     Known = Known.sext(BitWidth);
8157     break;
8158   }
8159   case RISCVISD::DIVUW: {
8160     KnownBits Known2;
8161     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
8162     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
8163     // We only care about the lower 32 bits.
8164     Known = KnownBits::udiv(Known.trunc(32), Known2.trunc(32));
8165     // Restore the original width by sign extending.
8166     Known = Known.sext(BitWidth);
8167     break;
8168   }
8169   case RISCVISD::CTZW: {
8170     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8171     unsigned PossibleTZ = Known2.trunc(32).countMaxTrailingZeros();
8172     unsigned LowBits = Log2_32(PossibleTZ) + 1;
8173     Known.Zero.setBitsFrom(LowBits);
8174     break;
8175   }
8176   case RISCVISD::CLZW: {
8177     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8178     unsigned PossibleLZ = Known2.trunc(32).countMaxLeadingZeros();
8179     unsigned LowBits = Log2_32(PossibleLZ) + 1;
8180     Known.Zero.setBitsFrom(LowBits);
8181     break;
8182   }
8183   case RISCVISD::GREV:
8184   case RISCVISD::GREVW: {
8185     if (auto *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
8186       Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8187       if (Opc == RISCVISD::GREVW)
8188         Known = Known.trunc(32);
8189       unsigned ShAmt = C->getZExtValue();
8190       computeGREV(Known.Zero, ShAmt);
8191       computeGREV(Known.One, ShAmt);
8192       if (Opc == RISCVISD::GREVW)
8193         Known = Known.sext(BitWidth);
8194     }
8195     break;
8196   }
8197   case RISCVISD::READ_VLENB:
8198     // We assume VLENB is at least 16 bytes.
8199     Known.Zero.setLowBits(4);
8200     // We assume VLENB is no more than 65536 / 8 bytes.
8201     Known.Zero.setBitsFrom(14);
8202     break;
8203   case ISD::INTRINSIC_W_CHAIN:
8204   case ISD::INTRINSIC_WO_CHAIN: {
8205     unsigned IntNo =
8206         Op.getConstantOperandVal(Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1);
8207     switch (IntNo) {
8208     default:
8209       // We can't do anything for most intrinsics.
8210       break;
8211     case Intrinsic::riscv_vsetvli:
8212     case Intrinsic::riscv_vsetvlimax:
8213     case Intrinsic::riscv_vsetvli_opt:
8214     case Intrinsic::riscv_vsetvlimax_opt:
8215       // Assume that VL output is positive and would fit in an int32_t.
8216       // TODO: VLEN might be capped at 16 bits in a future V spec update.
8217       if (BitWidth >= 32)
8218         Known.Zero.setBitsFrom(31);
8219       break;
8220     }
8221     break;
8222   }
8223   }
8224 }
8225 
8226 unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
8227     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
8228     unsigned Depth) const {
8229   switch (Op.getOpcode()) {
8230   default:
8231     break;
8232   case RISCVISD::SELECT_CC: {
8233     unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth + 1);
8234     if (Tmp == 1) return 1;  // Early out.
8235     unsigned Tmp2 = DAG.ComputeNumSignBits(Op.getOperand(4), DemandedElts, Depth + 1);
8236     return std::min(Tmp, Tmp2);
8237   }
8238   case RISCVISD::SLLW:
8239   case RISCVISD::SRAW:
8240   case RISCVISD::SRLW:
8241   case RISCVISD::DIVW:
8242   case RISCVISD::DIVUW:
8243   case RISCVISD::REMUW:
8244   case RISCVISD::ROLW:
8245   case RISCVISD::RORW:
8246   case RISCVISD::GREVW:
8247   case RISCVISD::GORCW:
8248   case RISCVISD::FSLW:
8249   case RISCVISD::FSRW:
8250   case RISCVISD::SHFLW:
8251   case RISCVISD::UNSHFLW:
8252   case RISCVISD::BCOMPRESSW:
8253   case RISCVISD::BDECOMPRESSW:
8254   case RISCVISD::BFPW:
8255   case RISCVISD::FCVT_W_RV64:
8256   case RISCVISD::FCVT_WU_RV64:
8257   case RISCVISD::STRICT_FCVT_W_RV64:
8258   case RISCVISD::STRICT_FCVT_WU_RV64:
8259     // TODO: As the result is sign-extended, this is conservatively correct. A
8260     // more precise answer could be calculated for SRAW depending on known
8261     // bits in the shift amount.
8262     return 33;
8263   case RISCVISD::SHFL:
8264   case RISCVISD::UNSHFL: {
8265     // There is no SHFLIW, but a i64 SHFLI with bit 4 of the control word
8266     // cleared doesn't affect bit 31. The upper 32 bits will be shuffled, but
8267     // will stay within the upper 32 bits. If there were more than 32 sign bits
8268     // before there will be at least 33 sign bits after.
8269     if (Op.getValueType() == MVT::i64 &&
8270         isa<ConstantSDNode>(Op.getOperand(1)) &&
8271         (Op.getConstantOperandVal(1) & 0x10) == 0) {
8272       unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
8273       if (Tmp > 32)
8274         return 33;
8275     }
8276     break;
8277   }
8278   case RISCVISD::VMV_X_S:
8279     // The number of sign bits of the scalar result is computed by obtaining the
8280     // element type of the input vector operand, subtracting its width from the
8281     // XLEN, and then adding one (sign bit within the element type). If the
8282     // element type is wider than XLen, the least-significant XLEN bits are
8283     // taken.
8284     if (Op.getOperand(0).getScalarValueSizeInBits() > Subtarget.getXLen())
8285       return 1;
8286     return Subtarget.getXLen() - Op.getOperand(0).getScalarValueSizeInBits() + 1;
8287   }
8288 
8289   return 1;
8290 }
8291 
8292 static MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
8293                                                   MachineBasicBlock *BB) {
8294   assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction");
8295 
8296   // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
8297   // Should the count have wrapped while it was being read, we need to try
8298   // again.
8299   // ...
8300   // read:
8301   // rdcycleh x3 # load high word of cycle
8302   // rdcycle  x2 # load low word of cycle
8303   // rdcycleh x4 # load high word of cycle
8304   // bne x3, x4, read # check if high word reads match, otherwise try again
8305   // ...
8306 
8307   MachineFunction &MF = *BB->getParent();
8308   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8309   MachineFunction::iterator It = ++BB->getIterator();
8310 
8311   MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8312   MF.insert(It, LoopMBB);
8313 
8314   MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8315   MF.insert(It, DoneMBB);
8316 
8317   // Transfer the remainder of BB and its successor edges to DoneMBB.
8318   DoneMBB->splice(DoneMBB->begin(), BB,
8319                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
8320   DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
8321 
8322   BB->addSuccessor(LoopMBB);
8323 
8324   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8325   Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
8326   Register LoReg = MI.getOperand(0).getReg();
8327   Register HiReg = MI.getOperand(1).getReg();
8328   DebugLoc DL = MI.getDebugLoc();
8329 
8330   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8331   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
8332       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8333       .addReg(RISCV::X0);
8334   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
8335       .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
8336       .addReg(RISCV::X0);
8337   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
8338       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8339       .addReg(RISCV::X0);
8340 
8341   BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
8342       .addReg(HiReg)
8343       .addReg(ReadAgainReg)
8344       .addMBB(LoopMBB);
8345 
8346   LoopMBB->addSuccessor(LoopMBB);
8347   LoopMBB->addSuccessor(DoneMBB);
8348 
8349   MI.eraseFromParent();
8350 
8351   return DoneMBB;
8352 }
8353 
8354 static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
8355                                              MachineBasicBlock *BB) {
8356   assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
8357 
8358   MachineFunction &MF = *BB->getParent();
8359   DebugLoc DL = MI.getDebugLoc();
8360   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8361   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8362   Register LoReg = MI.getOperand(0).getReg();
8363   Register HiReg = MI.getOperand(1).getReg();
8364   Register SrcReg = MI.getOperand(2).getReg();
8365   const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
8366   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8367 
8368   TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
8369                           RI);
8370   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8371   MachineMemOperand *MMOLo =
8372       MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 4, Align(8));
8373   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8374       MPI.getWithOffset(4), MachineMemOperand::MOLoad, 4, Align(8));
8375   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
8376       .addFrameIndex(FI)
8377       .addImm(0)
8378       .addMemOperand(MMOLo);
8379   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
8380       .addFrameIndex(FI)
8381       .addImm(4)
8382       .addMemOperand(MMOHi);
8383   MI.eraseFromParent(); // The pseudo instruction is gone now.
8384   return BB;
8385 }
8386 
8387 static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
8388                                                  MachineBasicBlock *BB) {
8389   assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
8390          "Unexpected instruction");
8391 
8392   MachineFunction &MF = *BB->getParent();
8393   DebugLoc DL = MI.getDebugLoc();
8394   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8395   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8396   Register DstReg = MI.getOperand(0).getReg();
8397   Register LoReg = MI.getOperand(1).getReg();
8398   Register HiReg = MI.getOperand(2).getReg();
8399   const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
8400   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8401 
8402   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8403   MachineMemOperand *MMOLo =
8404       MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Align(8));
8405   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8406       MPI.getWithOffset(4), MachineMemOperand::MOStore, 4, Align(8));
8407   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8408       .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
8409       .addFrameIndex(FI)
8410       .addImm(0)
8411       .addMemOperand(MMOLo);
8412   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8413       .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
8414       .addFrameIndex(FI)
8415       .addImm(4)
8416       .addMemOperand(MMOHi);
8417   TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
8418   MI.eraseFromParent(); // The pseudo instruction is gone now.
8419   return BB;
8420 }
8421 
8422 static bool isSelectPseudo(MachineInstr &MI) {
8423   switch (MI.getOpcode()) {
8424   default:
8425     return false;
8426   case RISCV::Select_GPR_Using_CC_GPR:
8427   case RISCV::Select_FPR16_Using_CC_GPR:
8428   case RISCV::Select_FPR32_Using_CC_GPR:
8429   case RISCV::Select_FPR64_Using_CC_GPR:
8430     return true;
8431   }
8432 }
8433 
8434 static MachineBasicBlock *emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB,
8435                                         unsigned RelOpcode, unsigned EqOpcode,
8436                                         const RISCVSubtarget &Subtarget) {
8437   DebugLoc DL = MI.getDebugLoc();
8438   Register DstReg = MI.getOperand(0).getReg();
8439   Register Src1Reg = MI.getOperand(1).getReg();
8440   Register Src2Reg = MI.getOperand(2).getReg();
8441   MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
8442   Register SavedFFlags = MRI.createVirtualRegister(&RISCV::GPRRegClass);
8443   const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
8444 
8445   // Save the current FFLAGS.
8446   BuildMI(*BB, MI, DL, TII.get(RISCV::ReadFFLAGS), SavedFFlags);
8447 
8448   auto MIB = BuildMI(*BB, MI, DL, TII.get(RelOpcode), DstReg)
8449                  .addReg(Src1Reg)
8450                  .addReg(Src2Reg);
8451   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8452     MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
8453 
8454   // Restore the FFLAGS.
8455   BuildMI(*BB, MI, DL, TII.get(RISCV::WriteFFLAGS))
8456       .addReg(SavedFFlags, RegState::Kill);
8457 
8458   // Issue a dummy FEQ opcode to raise exception for signaling NaNs.
8459   auto MIB2 = BuildMI(*BB, MI, DL, TII.get(EqOpcode), RISCV::X0)
8460                   .addReg(Src1Reg, getKillRegState(MI.getOperand(1).isKill()))
8461                   .addReg(Src2Reg, getKillRegState(MI.getOperand(2).isKill()));
8462   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8463     MIB2->setFlag(MachineInstr::MIFlag::NoFPExcept);
8464 
8465   // Erase the pseudoinstruction.
8466   MI.eraseFromParent();
8467   return BB;
8468 }
8469 
8470 static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
8471                                            MachineBasicBlock *BB,
8472                                            const RISCVSubtarget &Subtarget) {
8473   // To "insert" Select_* instructions, we actually have to insert the triangle
8474   // control-flow pattern.  The incoming instructions know the destination vreg
8475   // to set, the condition code register to branch on, the true/false values to
8476   // select between, and the condcode to use to select the appropriate branch.
8477   //
8478   // We produce the following control flow:
8479   //     HeadMBB
8480   //     |  \
8481   //     |  IfFalseMBB
8482   //     | /
8483   //    TailMBB
8484   //
8485   // When we find a sequence of selects we attempt to optimize their emission
8486   // by sharing the control flow. Currently we only handle cases where we have
8487   // multiple selects with the exact same condition (same LHS, RHS and CC).
8488   // The selects may be interleaved with other instructions if the other
8489   // instructions meet some requirements we deem safe:
8490   // - They are debug instructions. Otherwise,
8491   // - They do not have side-effects, do not access memory and their inputs do
8492   //   not depend on the results of the select pseudo-instructions.
8493   // The TrueV/FalseV operands of the selects cannot depend on the result of
8494   // previous selects in the sequence.
8495   // These conditions could be further relaxed. See the X86 target for a
8496   // related approach and more information.
8497   Register LHS = MI.getOperand(1).getReg();
8498   Register RHS = MI.getOperand(2).getReg();
8499   auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
8500 
8501   SmallVector<MachineInstr *, 4> SelectDebugValues;
8502   SmallSet<Register, 4> SelectDests;
8503   SelectDests.insert(MI.getOperand(0).getReg());
8504 
8505   MachineInstr *LastSelectPseudo = &MI;
8506 
8507   for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
8508        SequenceMBBI != E; ++SequenceMBBI) {
8509     if (SequenceMBBI->isDebugInstr())
8510       continue;
8511     else if (isSelectPseudo(*SequenceMBBI)) {
8512       if (SequenceMBBI->getOperand(1).getReg() != LHS ||
8513           SequenceMBBI->getOperand(2).getReg() != RHS ||
8514           SequenceMBBI->getOperand(3).getImm() != CC ||
8515           SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
8516           SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
8517         break;
8518       LastSelectPseudo = &*SequenceMBBI;
8519       SequenceMBBI->collectDebugValues(SelectDebugValues);
8520       SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
8521     } else {
8522       if (SequenceMBBI->hasUnmodeledSideEffects() ||
8523           SequenceMBBI->mayLoadOrStore())
8524         break;
8525       if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
8526             return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
8527           }))
8528         break;
8529     }
8530   }
8531 
8532   const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
8533   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8534   DebugLoc DL = MI.getDebugLoc();
8535   MachineFunction::iterator I = ++BB->getIterator();
8536 
8537   MachineBasicBlock *HeadMBB = BB;
8538   MachineFunction *F = BB->getParent();
8539   MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
8540   MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
8541 
8542   F->insert(I, IfFalseMBB);
8543   F->insert(I, TailMBB);
8544 
8545   // Transfer debug instructions associated with the selects to TailMBB.
8546   for (MachineInstr *DebugInstr : SelectDebugValues) {
8547     TailMBB->push_back(DebugInstr->removeFromParent());
8548   }
8549 
8550   // Move all instructions after the sequence to TailMBB.
8551   TailMBB->splice(TailMBB->end(), HeadMBB,
8552                   std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
8553   // Update machine-CFG edges by transferring all successors of the current
8554   // block to the new block which will contain the Phi nodes for the selects.
8555   TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
8556   // Set the successors for HeadMBB.
8557   HeadMBB->addSuccessor(IfFalseMBB);
8558   HeadMBB->addSuccessor(TailMBB);
8559 
8560   // Insert appropriate branch.
8561   BuildMI(HeadMBB, DL, TII.getBrCond(CC))
8562     .addReg(LHS)
8563     .addReg(RHS)
8564     .addMBB(TailMBB);
8565 
8566   // IfFalseMBB just falls through to TailMBB.
8567   IfFalseMBB->addSuccessor(TailMBB);
8568 
8569   // Create PHIs for all of the select pseudo-instructions.
8570   auto SelectMBBI = MI.getIterator();
8571   auto SelectEnd = std::next(LastSelectPseudo->getIterator());
8572   auto InsertionPoint = TailMBB->begin();
8573   while (SelectMBBI != SelectEnd) {
8574     auto Next = std::next(SelectMBBI);
8575     if (isSelectPseudo(*SelectMBBI)) {
8576       // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
8577       BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
8578               TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
8579           .addReg(SelectMBBI->getOperand(4).getReg())
8580           .addMBB(HeadMBB)
8581           .addReg(SelectMBBI->getOperand(5).getReg())
8582           .addMBB(IfFalseMBB);
8583       SelectMBBI->eraseFromParent();
8584     }
8585     SelectMBBI = Next;
8586   }
8587 
8588   F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
8589   return TailMBB;
8590 }
8591 
8592 MachineBasicBlock *
8593 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
8594                                                  MachineBasicBlock *BB) const {
8595   switch (MI.getOpcode()) {
8596   default:
8597     llvm_unreachable("Unexpected instr type to insert");
8598   case RISCV::ReadCycleWide:
8599     assert(!Subtarget.is64Bit() &&
8600            "ReadCycleWrite is only to be used on riscv32");
8601     return emitReadCycleWidePseudo(MI, BB);
8602   case RISCV::Select_GPR_Using_CC_GPR:
8603   case RISCV::Select_FPR16_Using_CC_GPR:
8604   case RISCV::Select_FPR32_Using_CC_GPR:
8605   case RISCV::Select_FPR64_Using_CC_GPR:
8606     return emitSelectPseudo(MI, BB, Subtarget);
8607   case RISCV::BuildPairF64Pseudo:
8608     return emitBuildPairF64Pseudo(MI, BB);
8609   case RISCV::SplitF64Pseudo:
8610     return emitSplitF64Pseudo(MI, BB);
8611   case RISCV::PseudoQuietFLE_H:
8612     return emitQuietFCMP(MI, BB, RISCV::FLE_H, RISCV::FEQ_H, Subtarget);
8613   case RISCV::PseudoQuietFLT_H:
8614     return emitQuietFCMP(MI, BB, RISCV::FLT_H, RISCV::FEQ_H, Subtarget);
8615   case RISCV::PseudoQuietFLE_S:
8616     return emitQuietFCMP(MI, BB, RISCV::FLE_S, RISCV::FEQ_S, Subtarget);
8617   case RISCV::PseudoQuietFLT_S:
8618     return emitQuietFCMP(MI, BB, RISCV::FLT_S, RISCV::FEQ_S, Subtarget);
8619   case RISCV::PseudoQuietFLE_D:
8620     return emitQuietFCMP(MI, BB, RISCV::FLE_D, RISCV::FEQ_D, Subtarget);
8621   case RISCV::PseudoQuietFLT_D:
8622     return emitQuietFCMP(MI, BB, RISCV::FLT_D, RISCV::FEQ_D, Subtarget);
8623   }
8624 }
8625 
8626 void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
8627                                                         SDNode *Node) const {
8628   // Add FRM dependency to any instructions with dynamic rounding mode.
8629   unsigned Opc = MI.getOpcode();
8630   auto Idx = RISCV::getNamedOperandIdx(Opc, RISCV::OpName::frm);
8631   if (Idx < 0)
8632     return;
8633   if (MI.getOperand(Idx).getImm() != RISCVFPRndMode::DYN)
8634     return;
8635   // If the instruction already reads FRM, don't add another read.
8636   if (MI.readsRegister(RISCV::FRM))
8637     return;
8638   MI.addOperand(
8639       MachineOperand::CreateReg(RISCV::FRM, /*isDef*/ false, /*isImp*/ true));
8640 }
8641 
8642 // Calling Convention Implementation.
8643 // The expectations for frontend ABI lowering vary from target to target.
8644 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
8645 // details, but this is a longer term goal. For now, we simply try to keep the
8646 // role of the frontend as simple and well-defined as possible. The rules can
8647 // be summarised as:
8648 // * Never split up large scalar arguments. We handle them here.
8649 // * If a hardfloat calling convention is being used, and the struct may be
8650 // passed in a pair of registers (fp+fp, int+fp), and both registers are
8651 // available, then pass as two separate arguments. If either the GPRs or FPRs
8652 // are exhausted, then pass according to the rule below.
8653 // * If a struct could never be passed in registers or directly in a stack
8654 // slot (as it is larger than 2*XLEN and the floating point rules don't
8655 // apply), then pass it using a pointer with the byval attribute.
8656 // * If a struct is less than 2*XLEN, then coerce to either a two-element
8657 // word-sized array or a 2*XLEN scalar (depending on alignment).
8658 // * The frontend can determine whether a struct is returned by reference or
8659 // not based on its size and fields. If it will be returned by reference, the
8660 // frontend must modify the prototype so a pointer with the sret annotation is
8661 // passed as the first argument. This is not necessary for large scalar
8662 // returns.
8663 // * Struct return values and varargs should be coerced to structs containing
8664 // register-size fields in the same situations they would be for fixed
8665 // arguments.
8666 
8667 static const MCPhysReg ArgGPRs[] = {
8668   RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
8669   RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
8670 };
8671 static const MCPhysReg ArgFPR16s[] = {
8672   RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
8673   RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
8674 };
8675 static const MCPhysReg ArgFPR32s[] = {
8676   RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
8677   RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
8678 };
8679 static const MCPhysReg ArgFPR64s[] = {
8680   RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
8681   RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
8682 };
8683 // This is an interim calling convention and it may be changed in the future.
8684 static const MCPhysReg ArgVRs[] = {
8685     RISCV::V8,  RISCV::V9,  RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
8686     RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
8687     RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
8688 static const MCPhysReg ArgVRM2s[] = {RISCV::V8M2,  RISCV::V10M2, RISCV::V12M2,
8689                                      RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
8690                                      RISCV::V20M2, RISCV::V22M2};
8691 static const MCPhysReg ArgVRM4s[] = {RISCV::V8M4, RISCV::V12M4, RISCV::V16M4,
8692                                      RISCV::V20M4};
8693 static const MCPhysReg ArgVRM8s[] = {RISCV::V8M8, RISCV::V16M8};
8694 
8695 // Pass a 2*XLEN argument that has been split into two XLEN values through
8696 // registers or the stack as necessary.
8697 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
8698                                 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
8699                                 MVT ValVT2, MVT LocVT2,
8700                                 ISD::ArgFlagsTy ArgFlags2) {
8701   unsigned XLenInBytes = XLen / 8;
8702   if (Register Reg = State.AllocateReg(ArgGPRs)) {
8703     // At least one half can be passed via register.
8704     State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
8705                                      VA1.getLocVT(), CCValAssign::Full));
8706   } else {
8707     // Both halves must be passed on the stack, with proper alignment.
8708     Align StackAlign =
8709         std::max(Align(XLenInBytes), ArgFlags1.getNonZeroOrigAlign());
8710     State.addLoc(
8711         CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
8712                             State.AllocateStack(XLenInBytes, StackAlign),
8713                             VA1.getLocVT(), CCValAssign::Full));
8714     State.addLoc(CCValAssign::getMem(
8715         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
8716         LocVT2, CCValAssign::Full));
8717     return false;
8718   }
8719 
8720   if (Register Reg = State.AllocateReg(ArgGPRs)) {
8721     // The second half can also be passed via register.
8722     State.addLoc(
8723         CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
8724   } else {
8725     // The second half is passed via the stack, without additional alignment.
8726     State.addLoc(CCValAssign::getMem(
8727         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
8728         LocVT2, CCValAssign::Full));
8729   }
8730 
8731   return false;
8732 }
8733 
8734 static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo,
8735                                Optional<unsigned> FirstMaskArgument,
8736                                CCState &State, const RISCVTargetLowering &TLI) {
8737   const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT);
8738   if (RC == &RISCV::VRRegClass) {
8739     // Assign the first mask argument to V0.
8740     // This is an interim calling convention and it may be changed in the
8741     // future.
8742     if (FirstMaskArgument.hasValue() && ValNo == FirstMaskArgument.getValue())
8743       return State.AllocateReg(RISCV::V0);
8744     return State.AllocateReg(ArgVRs);
8745   }
8746   if (RC == &RISCV::VRM2RegClass)
8747     return State.AllocateReg(ArgVRM2s);
8748   if (RC == &RISCV::VRM4RegClass)
8749     return State.AllocateReg(ArgVRM4s);
8750   if (RC == &RISCV::VRM8RegClass)
8751     return State.AllocateReg(ArgVRM8s);
8752   llvm_unreachable("Unhandled register class for ValueType");
8753 }
8754 
8755 // Implements the RISC-V calling convention. Returns true upon failure.
8756 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
8757                      MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
8758                      ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
8759                      bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
8760                      Optional<unsigned> FirstMaskArgument) {
8761   unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
8762   assert(XLen == 32 || XLen == 64);
8763   MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
8764 
8765   // Any return value split in to more than two values can't be returned
8766   // directly. Vectors are returned via the available vector registers.
8767   if (!LocVT.isVector() && IsRet && ValNo > 1)
8768     return true;
8769 
8770   // UseGPRForF16_F32 if targeting one of the soft-float ABIs, if passing a
8771   // variadic argument, or if no F16/F32 argument registers are available.
8772   bool UseGPRForF16_F32 = true;
8773   // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
8774   // variadic argument, or if no F64 argument registers are available.
8775   bool UseGPRForF64 = true;
8776 
8777   switch (ABI) {
8778   default:
8779     llvm_unreachable("Unexpected ABI");
8780   case RISCVABI::ABI_ILP32:
8781   case RISCVABI::ABI_LP64:
8782     break;
8783   case RISCVABI::ABI_ILP32F:
8784   case RISCVABI::ABI_LP64F:
8785     UseGPRForF16_F32 = !IsFixed;
8786     break;
8787   case RISCVABI::ABI_ILP32D:
8788   case RISCVABI::ABI_LP64D:
8789     UseGPRForF16_F32 = !IsFixed;
8790     UseGPRForF64 = !IsFixed;
8791     break;
8792   }
8793 
8794   // FPR16, FPR32, and FPR64 alias each other.
8795   if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s)) {
8796     UseGPRForF16_F32 = true;
8797     UseGPRForF64 = true;
8798   }
8799 
8800   // From this point on, rely on UseGPRForF16_F32, UseGPRForF64 and
8801   // similar local variables rather than directly checking against the target
8802   // ABI.
8803 
8804   if (UseGPRForF16_F32 && (ValVT == MVT::f16 || ValVT == MVT::f32)) {
8805     LocVT = XLenVT;
8806     LocInfo = CCValAssign::BCvt;
8807   } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
8808     LocVT = MVT::i64;
8809     LocInfo = CCValAssign::BCvt;
8810   }
8811 
8812   // If this is a variadic argument, the RISC-V calling convention requires
8813   // that it is assigned an 'even' or 'aligned' register if it has 8-byte
8814   // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
8815   // be used regardless of whether the original argument was split during
8816   // legalisation or not. The argument will not be passed by registers if the
8817   // original type is larger than 2*XLEN, so the register alignment rule does
8818   // not apply.
8819   unsigned TwoXLenInBytes = (2 * XLen) / 8;
8820   if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
8821       DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
8822     unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
8823     // Skip 'odd' register if necessary.
8824     if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
8825       State.AllocateReg(ArgGPRs);
8826   }
8827 
8828   SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
8829   SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
8830       State.getPendingArgFlags();
8831 
8832   assert(PendingLocs.size() == PendingArgFlags.size() &&
8833          "PendingLocs and PendingArgFlags out of sync");
8834 
8835   // Handle passing f64 on RV32D with a soft float ABI or when floating point
8836   // registers are exhausted.
8837   if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
8838     assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
8839            "Can't lower f64 if it is split");
8840     // Depending on available argument GPRS, f64 may be passed in a pair of
8841     // GPRs, split between a GPR and the stack, or passed completely on the
8842     // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
8843     // cases.
8844     Register Reg = State.AllocateReg(ArgGPRs);
8845     LocVT = MVT::i32;
8846     if (!Reg) {
8847       unsigned StackOffset = State.AllocateStack(8, Align(8));
8848       State.addLoc(
8849           CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
8850       return false;
8851     }
8852     if (!State.AllocateReg(ArgGPRs))
8853       State.AllocateStack(4, Align(4));
8854     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8855     return false;
8856   }
8857 
8858   // Fixed-length vectors are located in the corresponding scalable-vector
8859   // container types.
8860   if (ValVT.isFixedLengthVector())
8861     LocVT = TLI.getContainerForFixedLengthVector(LocVT);
8862 
8863   // Split arguments might be passed indirectly, so keep track of the pending
8864   // values. Split vectors are passed via a mix of registers and indirectly, so
8865   // treat them as we would any other argument.
8866   if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
8867     LocVT = XLenVT;
8868     LocInfo = CCValAssign::Indirect;
8869     PendingLocs.push_back(
8870         CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
8871     PendingArgFlags.push_back(ArgFlags);
8872     if (!ArgFlags.isSplitEnd()) {
8873       return false;
8874     }
8875   }
8876 
8877   // If the split argument only had two elements, it should be passed directly
8878   // in registers or on the stack.
8879   if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
8880       PendingLocs.size() <= 2) {
8881     assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
8882     // Apply the normal calling convention rules to the first half of the
8883     // split argument.
8884     CCValAssign VA = PendingLocs[0];
8885     ISD::ArgFlagsTy AF = PendingArgFlags[0];
8886     PendingLocs.clear();
8887     PendingArgFlags.clear();
8888     return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
8889                                ArgFlags);
8890   }
8891 
8892   // Allocate to a register if possible, or else a stack slot.
8893   Register Reg;
8894   unsigned StoreSizeBytes = XLen / 8;
8895   Align StackAlign = Align(XLen / 8);
8896 
8897   if (ValVT == MVT::f16 && !UseGPRForF16_F32)
8898     Reg = State.AllocateReg(ArgFPR16s);
8899   else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
8900     Reg = State.AllocateReg(ArgFPR32s);
8901   else if (ValVT == MVT::f64 && !UseGPRForF64)
8902     Reg = State.AllocateReg(ArgFPR64s);
8903   else if (ValVT.isVector()) {
8904     Reg = allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI);
8905     if (!Reg) {
8906       // For return values, the vector must be passed fully via registers or
8907       // via the stack.
8908       // FIXME: The proposed vector ABI only mandates v8-v15 for return values,
8909       // but we're using all of them.
8910       if (IsRet)
8911         return true;
8912       // Try using a GPR to pass the address
8913       if ((Reg = State.AllocateReg(ArgGPRs))) {
8914         LocVT = XLenVT;
8915         LocInfo = CCValAssign::Indirect;
8916       } else if (ValVT.isScalableVector()) {
8917         LocVT = XLenVT;
8918         LocInfo = CCValAssign::Indirect;
8919       } else {
8920         // Pass fixed-length vectors on the stack.
8921         LocVT = ValVT;
8922         StoreSizeBytes = ValVT.getStoreSize();
8923         // Align vectors to their element sizes, being careful for vXi1
8924         // vectors.
8925         StackAlign = MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
8926       }
8927     }
8928   } else {
8929     Reg = State.AllocateReg(ArgGPRs);
8930   }
8931 
8932   unsigned StackOffset =
8933       Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
8934 
8935   // If we reach this point and PendingLocs is non-empty, we must be at the
8936   // end of a split argument that must be passed indirectly.
8937   if (!PendingLocs.empty()) {
8938     assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
8939     assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
8940 
8941     for (auto &It : PendingLocs) {
8942       if (Reg)
8943         It.convertToReg(Reg);
8944       else
8945         It.convertToMem(StackOffset);
8946       State.addLoc(It);
8947     }
8948     PendingLocs.clear();
8949     PendingArgFlags.clear();
8950     return false;
8951   }
8952 
8953   assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
8954           (TLI.getSubtarget().hasVInstructions() && ValVT.isVector())) &&
8955          "Expected an XLenVT or vector types at this stage");
8956 
8957   if (Reg) {
8958     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8959     return false;
8960   }
8961 
8962   // When a floating-point value is passed on the stack, no bit-conversion is
8963   // needed.
8964   if (ValVT.isFloatingPoint()) {
8965     LocVT = ValVT;
8966     LocInfo = CCValAssign::Full;
8967   }
8968   State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
8969   return false;
8970 }
8971 
8972 template <typename ArgTy>
8973 static Optional<unsigned> preAssignMask(const ArgTy &Args) {
8974   for (const auto &ArgIdx : enumerate(Args)) {
8975     MVT ArgVT = ArgIdx.value().VT;
8976     if (ArgVT.isVector() && ArgVT.getVectorElementType() == MVT::i1)
8977       return ArgIdx.index();
8978   }
8979   return None;
8980 }
8981 
8982 void RISCVTargetLowering::analyzeInputArgs(
8983     MachineFunction &MF, CCState &CCInfo,
8984     const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
8985     RISCVCCAssignFn Fn) const {
8986   unsigned NumArgs = Ins.size();
8987   FunctionType *FType = MF.getFunction().getFunctionType();
8988 
8989   Optional<unsigned> FirstMaskArgument;
8990   if (Subtarget.hasVInstructions())
8991     FirstMaskArgument = preAssignMask(Ins);
8992 
8993   for (unsigned i = 0; i != NumArgs; ++i) {
8994     MVT ArgVT = Ins[i].VT;
8995     ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
8996 
8997     Type *ArgTy = nullptr;
8998     if (IsRet)
8999       ArgTy = FType->getReturnType();
9000     else if (Ins[i].isOrigArg())
9001       ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
9002 
9003     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9004     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
9005            ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this,
9006            FirstMaskArgument)) {
9007       LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
9008                         << EVT(ArgVT).getEVTString() << '\n');
9009       llvm_unreachable(nullptr);
9010     }
9011   }
9012 }
9013 
9014 void RISCVTargetLowering::analyzeOutputArgs(
9015     MachineFunction &MF, CCState &CCInfo,
9016     const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
9017     CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
9018   unsigned NumArgs = Outs.size();
9019 
9020   Optional<unsigned> FirstMaskArgument;
9021   if (Subtarget.hasVInstructions())
9022     FirstMaskArgument = preAssignMask(Outs);
9023 
9024   for (unsigned i = 0; i != NumArgs; i++) {
9025     MVT ArgVT = Outs[i].VT;
9026     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
9027     Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
9028 
9029     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9030     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
9031            ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this,
9032            FirstMaskArgument)) {
9033       LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
9034                         << EVT(ArgVT).getEVTString() << "\n");
9035       llvm_unreachable(nullptr);
9036     }
9037   }
9038 }
9039 
9040 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
9041 // values.
9042 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
9043                                    const CCValAssign &VA, const SDLoc &DL,
9044                                    const RISCVSubtarget &Subtarget) {
9045   switch (VA.getLocInfo()) {
9046   default:
9047     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9048   case CCValAssign::Full:
9049     if (VA.getValVT().isFixedLengthVector() && VA.getLocVT().isScalableVector())
9050       Val = convertFromScalableVector(VA.getValVT(), Val, DAG, Subtarget);
9051     break;
9052   case CCValAssign::BCvt:
9053     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
9054       Val = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, Val);
9055     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9056       Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
9057     else
9058       Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
9059     break;
9060   }
9061   return Val;
9062 }
9063 
9064 // The caller is responsible for loading the full value if the argument is
9065 // passed with CCValAssign::Indirect.
9066 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
9067                                 const CCValAssign &VA, const SDLoc &DL,
9068                                 const RISCVTargetLowering &TLI) {
9069   MachineFunction &MF = DAG.getMachineFunction();
9070   MachineRegisterInfo &RegInfo = MF.getRegInfo();
9071   EVT LocVT = VA.getLocVT();
9072   SDValue Val;
9073   const TargetRegisterClass *RC = TLI.getRegClassFor(LocVT.getSimpleVT());
9074   Register VReg = RegInfo.createVirtualRegister(RC);
9075   RegInfo.addLiveIn(VA.getLocReg(), VReg);
9076   Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
9077 
9078   if (VA.getLocInfo() == CCValAssign::Indirect)
9079     return Val;
9080 
9081   return convertLocVTToValVT(DAG, Val, VA, DL, TLI.getSubtarget());
9082 }
9083 
9084 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
9085                                    const CCValAssign &VA, const SDLoc &DL,
9086                                    const RISCVSubtarget &Subtarget) {
9087   EVT LocVT = VA.getLocVT();
9088 
9089   switch (VA.getLocInfo()) {
9090   default:
9091     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9092   case CCValAssign::Full:
9093     if (VA.getValVT().isFixedLengthVector() && LocVT.isScalableVector())
9094       Val = convertToScalableVector(LocVT, Val, DAG, Subtarget);
9095     break;
9096   case CCValAssign::BCvt:
9097     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
9098       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, VA.getLocVT(), Val);
9099     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9100       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
9101     else
9102       Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
9103     break;
9104   }
9105   return Val;
9106 }
9107 
9108 // The caller is responsible for loading the full value if the argument is
9109 // passed with CCValAssign::Indirect.
9110 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
9111                                 const CCValAssign &VA, const SDLoc &DL) {
9112   MachineFunction &MF = DAG.getMachineFunction();
9113   MachineFrameInfo &MFI = MF.getFrameInfo();
9114   EVT LocVT = VA.getLocVT();
9115   EVT ValVT = VA.getValVT();
9116   EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
9117   if (ValVT.isScalableVector()) {
9118     // When the value is a scalable vector, we save the pointer which points to
9119     // the scalable vector value in the stack. The ValVT will be the pointer
9120     // type, instead of the scalable vector type.
9121     ValVT = LocVT;
9122   }
9123   int FI = MFI.CreateFixedObject(ValVT.getStoreSize(), VA.getLocMemOffset(),
9124                                  /*IsImmutable=*/true);
9125   SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
9126   SDValue Val;
9127 
9128   ISD::LoadExtType ExtType;
9129   switch (VA.getLocInfo()) {
9130   default:
9131     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9132   case CCValAssign::Full:
9133   case CCValAssign::Indirect:
9134   case CCValAssign::BCvt:
9135     ExtType = ISD::NON_EXTLOAD;
9136     break;
9137   }
9138   Val = DAG.getExtLoad(
9139       ExtType, DL, LocVT, Chain, FIN,
9140       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
9141   return Val;
9142 }
9143 
9144 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
9145                                        const CCValAssign &VA, const SDLoc &DL) {
9146   assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
9147          "Unexpected VA");
9148   MachineFunction &MF = DAG.getMachineFunction();
9149   MachineFrameInfo &MFI = MF.getFrameInfo();
9150   MachineRegisterInfo &RegInfo = MF.getRegInfo();
9151 
9152   if (VA.isMemLoc()) {
9153     // f64 is passed on the stack.
9154     int FI =
9155         MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*IsImmutable=*/true);
9156     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
9157     return DAG.getLoad(MVT::f64, DL, Chain, FIN,
9158                        MachinePointerInfo::getFixedStack(MF, FI));
9159   }
9160 
9161   assert(VA.isRegLoc() && "Expected register VA assignment");
9162 
9163   Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
9164   RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
9165   SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
9166   SDValue Hi;
9167   if (VA.getLocReg() == RISCV::X17) {
9168     // Second half of f64 is passed on the stack.
9169     int FI = MFI.CreateFixedObject(4, 0, /*IsImmutable=*/true);
9170     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
9171     Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
9172                      MachinePointerInfo::getFixedStack(MF, FI));
9173   } else {
9174     // Second half of f64 is passed in another GPR.
9175     Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
9176     RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
9177     Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
9178   }
9179   return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
9180 }
9181 
9182 // FastCC has less than 1% performance improvement for some particular
9183 // benchmark. But theoretically, it may has benenfit for some cases.
9184 static bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
9185                             unsigned ValNo, MVT ValVT, MVT LocVT,
9186                             CCValAssign::LocInfo LocInfo,
9187                             ISD::ArgFlagsTy ArgFlags, CCState &State,
9188                             bool IsFixed, bool IsRet, Type *OrigTy,
9189                             const RISCVTargetLowering &TLI,
9190                             Optional<unsigned> FirstMaskArgument) {
9191 
9192   // X5 and X6 might be used for save-restore libcall.
9193   static const MCPhysReg GPRList[] = {
9194       RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
9195       RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7,  RISCV::X28,
9196       RISCV::X29, RISCV::X30, RISCV::X31};
9197 
9198   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9199     if (unsigned Reg = State.AllocateReg(GPRList)) {
9200       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9201       return false;
9202     }
9203   }
9204 
9205   if (LocVT == MVT::f16) {
9206     static const MCPhysReg FPR16List[] = {
9207         RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
9208         RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H,  RISCV::F1_H,
9209         RISCV::F2_H,  RISCV::F3_H,  RISCV::F4_H,  RISCV::F5_H,  RISCV::F6_H,
9210         RISCV::F7_H,  RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
9211     if (unsigned Reg = State.AllocateReg(FPR16List)) {
9212       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9213       return false;
9214     }
9215   }
9216 
9217   if (LocVT == MVT::f32) {
9218     static const MCPhysReg FPR32List[] = {
9219         RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
9220         RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F,  RISCV::F1_F,
9221         RISCV::F2_F,  RISCV::F3_F,  RISCV::F4_F,  RISCV::F5_F,  RISCV::F6_F,
9222         RISCV::F7_F,  RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
9223     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9224       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9225       return false;
9226     }
9227   }
9228 
9229   if (LocVT == MVT::f64) {
9230     static const MCPhysReg FPR64List[] = {
9231         RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
9232         RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D,  RISCV::F1_D,
9233         RISCV::F2_D,  RISCV::F3_D,  RISCV::F4_D,  RISCV::F5_D,  RISCV::F6_D,
9234         RISCV::F7_D,  RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
9235     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9236       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9237       return false;
9238     }
9239   }
9240 
9241   if (LocVT == MVT::i32 || LocVT == MVT::f32) {
9242     unsigned Offset4 = State.AllocateStack(4, Align(4));
9243     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset4, LocVT, LocInfo));
9244     return false;
9245   }
9246 
9247   if (LocVT == MVT::i64 || LocVT == MVT::f64) {
9248     unsigned Offset5 = State.AllocateStack(8, Align(8));
9249     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset5, LocVT, LocInfo));
9250     return false;
9251   }
9252 
9253   if (LocVT.isVector()) {
9254     if (unsigned Reg =
9255             allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI)) {
9256       // Fixed-length vectors are located in the corresponding scalable-vector
9257       // container types.
9258       if (ValVT.isFixedLengthVector())
9259         LocVT = TLI.getContainerForFixedLengthVector(LocVT);
9260       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9261     } else {
9262       // Try and pass the address via a "fast" GPR.
9263       if (unsigned GPRReg = State.AllocateReg(GPRList)) {
9264         LocInfo = CCValAssign::Indirect;
9265         LocVT = TLI.getSubtarget().getXLenVT();
9266         State.addLoc(CCValAssign::getReg(ValNo, ValVT, GPRReg, LocVT, LocInfo));
9267       } else if (ValVT.isFixedLengthVector()) {
9268         auto StackAlign =
9269             MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
9270         unsigned StackOffset =
9271             State.AllocateStack(ValVT.getStoreSize(), StackAlign);
9272         State.addLoc(
9273             CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
9274       } else {
9275         // Can't pass scalable vectors on the stack.
9276         return true;
9277       }
9278     }
9279 
9280     return false;
9281   }
9282 
9283   return true; // CC didn't match.
9284 }
9285 
9286 static bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
9287                          CCValAssign::LocInfo LocInfo,
9288                          ISD::ArgFlagsTy ArgFlags, CCState &State) {
9289 
9290   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9291     // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, R6, R7, SpLim
9292     //                        s1    s2  s3  s4  s5  s6  s7  s8  s9  s10 s11
9293     static const MCPhysReg GPRList[] = {
9294         RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
9295         RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
9296     if (unsigned Reg = State.AllocateReg(GPRList)) {
9297       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9298       return false;
9299     }
9300   }
9301 
9302   if (LocVT == MVT::f32) {
9303     // Pass in STG registers: F1, ..., F6
9304     //                        fs0 ... fs5
9305     static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
9306                                           RISCV::F18_F, RISCV::F19_F,
9307                                           RISCV::F20_F, RISCV::F21_F};
9308     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9309       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9310       return false;
9311     }
9312   }
9313 
9314   if (LocVT == MVT::f64) {
9315     // Pass in STG registers: D1, ..., D6
9316     //                        fs6 ... fs11
9317     static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
9318                                           RISCV::F24_D, RISCV::F25_D,
9319                                           RISCV::F26_D, RISCV::F27_D};
9320     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9321       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9322       return false;
9323     }
9324   }
9325 
9326   report_fatal_error("No registers left in GHC calling convention");
9327   return true;
9328 }
9329 
9330 // Transform physical registers into virtual registers.
9331 SDValue RISCVTargetLowering::LowerFormalArguments(
9332     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
9333     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
9334     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
9335 
9336   MachineFunction &MF = DAG.getMachineFunction();
9337 
9338   switch (CallConv) {
9339   default:
9340     report_fatal_error("Unsupported calling convention");
9341   case CallingConv::C:
9342   case CallingConv::Fast:
9343     break;
9344   case CallingConv::GHC:
9345     if (!MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtF] ||
9346         !MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtD])
9347       report_fatal_error(
9348         "GHC calling convention requires the F and D instruction set extensions");
9349   }
9350 
9351   const Function &Func = MF.getFunction();
9352   if (Func.hasFnAttribute("interrupt")) {
9353     if (!Func.arg_empty())
9354       report_fatal_error(
9355         "Functions with the interrupt attribute cannot have arguments!");
9356 
9357     StringRef Kind =
9358       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
9359 
9360     if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
9361       report_fatal_error(
9362         "Function interrupt attribute argument not supported!");
9363   }
9364 
9365   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9366   MVT XLenVT = Subtarget.getXLenVT();
9367   unsigned XLenInBytes = Subtarget.getXLen() / 8;
9368   // Used with vargs to acumulate store chains.
9369   std::vector<SDValue> OutChains;
9370 
9371   // Assign locations to all of the incoming arguments.
9372   SmallVector<CCValAssign, 16> ArgLocs;
9373   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9374 
9375   if (CallConv == CallingConv::GHC)
9376     CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_GHC);
9377   else
9378     analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false,
9379                      CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9380                                                    : CC_RISCV);
9381 
9382   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
9383     CCValAssign &VA = ArgLocs[i];
9384     SDValue ArgValue;
9385     // Passing f64 on RV32D with a soft float ABI must be handled as a special
9386     // case.
9387     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
9388       ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
9389     else if (VA.isRegLoc())
9390       ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, *this);
9391     else
9392       ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
9393 
9394     if (VA.getLocInfo() == CCValAssign::Indirect) {
9395       // If the original argument was split and passed by reference (e.g. i128
9396       // on RV32), we need to load all parts of it here (using the same
9397       // address). Vectors may be partly split to registers and partly to the
9398       // stack, in which case the base address is partly offset and subsequent
9399       // stores are relative to that.
9400       InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
9401                                    MachinePointerInfo()));
9402       unsigned ArgIndex = Ins[i].OrigArgIndex;
9403       unsigned ArgPartOffset = Ins[i].PartOffset;
9404       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9405       while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
9406         CCValAssign &PartVA = ArgLocs[i + 1];
9407         unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset;
9408         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9409         if (PartVA.getValVT().isScalableVector())
9410           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9411         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
9412         InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
9413                                      MachinePointerInfo()));
9414         ++i;
9415       }
9416       continue;
9417     }
9418     InVals.push_back(ArgValue);
9419   }
9420 
9421   if (IsVarArg) {
9422     ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
9423     unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
9424     const TargetRegisterClass *RC = &RISCV::GPRRegClass;
9425     MachineFrameInfo &MFI = MF.getFrameInfo();
9426     MachineRegisterInfo &RegInfo = MF.getRegInfo();
9427     RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
9428 
9429     // Offset of the first variable argument from stack pointer, and size of
9430     // the vararg save area. For now, the varargs save area is either zero or
9431     // large enough to hold a0-a7.
9432     int VaArgOffset, VarArgsSaveSize;
9433 
9434     // If all registers are allocated, then all varargs must be passed on the
9435     // stack and we don't need to save any argregs.
9436     if (ArgRegs.size() == Idx) {
9437       VaArgOffset = CCInfo.getNextStackOffset();
9438       VarArgsSaveSize = 0;
9439     } else {
9440       VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
9441       VaArgOffset = -VarArgsSaveSize;
9442     }
9443 
9444     // Record the frame index of the first variable argument
9445     // which is a value necessary to VASTART.
9446     int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9447     RVFI->setVarArgsFrameIndex(FI);
9448 
9449     // If saving an odd number of registers then create an extra stack slot to
9450     // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
9451     // offsets to even-numbered registered remain 2*XLEN-aligned.
9452     if (Idx % 2) {
9453       MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true);
9454       VarArgsSaveSize += XLenInBytes;
9455     }
9456 
9457     // Copy the integer registers that may have been used for passing varargs
9458     // to the vararg save area.
9459     for (unsigned I = Idx; I < ArgRegs.size();
9460          ++I, VaArgOffset += XLenInBytes) {
9461       const Register Reg = RegInfo.createVirtualRegister(RC);
9462       RegInfo.addLiveIn(ArgRegs[I], Reg);
9463       SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
9464       FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9465       SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9466       SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
9467                                    MachinePointerInfo::getFixedStack(MF, FI));
9468       cast<StoreSDNode>(Store.getNode())
9469           ->getMemOperand()
9470           ->setValue((Value *)nullptr);
9471       OutChains.push_back(Store);
9472     }
9473     RVFI->setVarArgsSaveSize(VarArgsSaveSize);
9474   }
9475 
9476   // All stores are grouped in one node to allow the matching between
9477   // the size of Ins and InVals. This only happens for vararg functions.
9478   if (!OutChains.empty()) {
9479     OutChains.push_back(Chain);
9480     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
9481   }
9482 
9483   return Chain;
9484 }
9485 
9486 /// isEligibleForTailCallOptimization - Check whether the call is eligible
9487 /// for tail call optimization.
9488 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
9489 bool RISCVTargetLowering::isEligibleForTailCallOptimization(
9490     CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
9491     const SmallVector<CCValAssign, 16> &ArgLocs) const {
9492 
9493   auto &Callee = CLI.Callee;
9494   auto CalleeCC = CLI.CallConv;
9495   auto &Outs = CLI.Outs;
9496   auto &Caller = MF.getFunction();
9497   auto CallerCC = Caller.getCallingConv();
9498 
9499   // Exception-handling functions need a special set of instructions to
9500   // indicate a return to the hardware. Tail-calling another function would
9501   // probably break this.
9502   // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
9503   // should be expanded as new function attributes are introduced.
9504   if (Caller.hasFnAttribute("interrupt"))
9505     return false;
9506 
9507   // Do not tail call opt if the stack is used to pass parameters.
9508   if (CCInfo.getNextStackOffset() != 0)
9509     return false;
9510 
9511   // Do not tail call opt if any parameters need to be passed indirectly.
9512   // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
9513   // passed indirectly. So the address of the value will be passed in a
9514   // register, or if not available, then the address is put on the stack. In
9515   // order to pass indirectly, space on the stack often needs to be allocated
9516   // in order to store the value. In this case the CCInfo.getNextStackOffset()
9517   // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
9518   // are passed CCValAssign::Indirect.
9519   for (auto &VA : ArgLocs)
9520     if (VA.getLocInfo() == CCValAssign::Indirect)
9521       return false;
9522 
9523   // Do not tail call opt if either caller or callee uses struct return
9524   // semantics.
9525   auto IsCallerStructRet = Caller.hasStructRetAttr();
9526   auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
9527   if (IsCallerStructRet || IsCalleeStructRet)
9528     return false;
9529 
9530   // Externally-defined functions with weak linkage should not be
9531   // tail-called. The behaviour of branch instructions in this situation (as
9532   // used for tail calls) is implementation-defined, so we cannot rely on the
9533   // linker replacing the tail call with a return.
9534   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
9535     const GlobalValue *GV = G->getGlobal();
9536     if (GV->hasExternalWeakLinkage())
9537       return false;
9538   }
9539 
9540   // The callee has to preserve all registers the caller needs to preserve.
9541   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
9542   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
9543   if (CalleeCC != CallerCC) {
9544     const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
9545     if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
9546       return false;
9547   }
9548 
9549   // Byval parameters hand the function a pointer directly into the stack area
9550   // we want to reuse during a tail call. Working around this *is* possible
9551   // but less efficient and uglier in LowerCall.
9552   for (auto &Arg : Outs)
9553     if (Arg.Flags.isByVal())
9554       return false;
9555 
9556   return true;
9557 }
9558 
9559 static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
9560   return DAG.getDataLayout().getPrefTypeAlign(
9561       VT.getTypeForEVT(*DAG.getContext()));
9562 }
9563 
9564 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
9565 // and output parameter nodes.
9566 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
9567                                        SmallVectorImpl<SDValue> &InVals) const {
9568   SelectionDAG &DAG = CLI.DAG;
9569   SDLoc &DL = CLI.DL;
9570   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
9571   SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
9572   SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
9573   SDValue Chain = CLI.Chain;
9574   SDValue Callee = CLI.Callee;
9575   bool &IsTailCall = CLI.IsTailCall;
9576   CallingConv::ID CallConv = CLI.CallConv;
9577   bool IsVarArg = CLI.IsVarArg;
9578   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9579   MVT XLenVT = Subtarget.getXLenVT();
9580 
9581   MachineFunction &MF = DAG.getMachineFunction();
9582 
9583   // Analyze the operands of the call, assigning locations to each operand.
9584   SmallVector<CCValAssign, 16> ArgLocs;
9585   CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9586 
9587   if (CallConv == CallingConv::GHC)
9588     ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_GHC);
9589   else
9590     analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI,
9591                       CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9592                                                     : CC_RISCV);
9593 
9594   // Check if it's really possible to do a tail call.
9595   if (IsTailCall)
9596     IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
9597 
9598   if (IsTailCall)
9599     ++NumTailCalls;
9600   else if (CLI.CB && CLI.CB->isMustTailCall())
9601     report_fatal_error("failed to perform tail call elimination on a call "
9602                        "site marked musttail");
9603 
9604   // Get a count of how many bytes are to be pushed on the stack.
9605   unsigned NumBytes = ArgCCInfo.getNextStackOffset();
9606 
9607   // Create local copies for byval args
9608   SmallVector<SDValue, 8> ByValArgs;
9609   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9610     ISD::ArgFlagsTy Flags = Outs[i].Flags;
9611     if (!Flags.isByVal())
9612       continue;
9613 
9614     SDValue Arg = OutVals[i];
9615     unsigned Size = Flags.getByValSize();
9616     Align Alignment = Flags.getNonZeroByValAlign();
9617 
9618     int FI =
9619         MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false);
9620     SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9621     SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
9622 
9623     Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
9624                           /*IsVolatile=*/false,
9625                           /*AlwaysInline=*/false, IsTailCall,
9626                           MachinePointerInfo(), MachinePointerInfo());
9627     ByValArgs.push_back(FIPtr);
9628   }
9629 
9630   if (!IsTailCall)
9631     Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
9632 
9633   // Copy argument values to their designated locations.
9634   SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
9635   SmallVector<SDValue, 8> MemOpChains;
9636   SDValue StackPtr;
9637   for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
9638     CCValAssign &VA = ArgLocs[i];
9639     SDValue ArgValue = OutVals[i];
9640     ISD::ArgFlagsTy Flags = Outs[i].Flags;
9641 
9642     // Handle passing f64 on RV32D with a soft float ABI as a special case.
9643     bool IsF64OnRV32DSoftABI =
9644         VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
9645     if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
9646       SDValue SplitF64 = DAG.getNode(
9647           RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
9648       SDValue Lo = SplitF64.getValue(0);
9649       SDValue Hi = SplitF64.getValue(1);
9650 
9651       Register RegLo = VA.getLocReg();
9652       RegsToPass.push_back(std::make_pair(RegLo, Lo));
9653 
9654       if (RegLo == RISCV::X17) {
9655         // Second half of f64 is passed on the stack.
9656         // Work out the address of the stack slot.
9657         if (!StackPtr.getNode())
9658           StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
9659         // Emit the store.
9660         MemOpChains.push_back(
9661             DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
9662       } else {
9663         // Second half of f64 is passed in another GPR.
9664         assert(RegLo < RISCV::X31 && "Invalid register pair");
9665         Register RegHigh = RegLo + 1;
9666         RegsToPass.push_back(std::make_pair(RegHigh, Hi));
9667       }
9668       continue;
9669     }
9670 
9671     // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
9672     // as any other MemLoc.
9673 
9674     // Promote the value if needed.
9675     // For now, only handle fully promoted and indirect arguments.
9676     if (VA.getLocInfo() == CCValAssign::Indirect) {
9677       // Store the argument in a stack slot and pass its address.
9678       Align StackAlign =
9679           std::max(getPrefTypeAlign(Outs[i].ArgVT, DAG),
9680                    getPrefTypeAlign(ArgValue.getValueType(), DAG));
9681       TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
9682       // If the original argument was split (e.g. i128), we need
9683       // to store the required parts of it here (and pass just one address).
9684       // Vectors may be partly split to registers and partly to the stack, in
9685       // which case the base address is partly offset and subsequent stores are
9686       // relative to that.
9687       unsigned ArgIndex = Outs[i].OrigArgIndex;
9688       unsigned ArgPartOffset = Outs[i].PartOffset;
9689       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9690       // Calculate the total size to store. We don't have access to what we're
9691       // actually storing other than performing the loop and collecting the
9692       // info.
9693       SmallVector<std::pair<SDValue, SDValue>> Parts;
9694       while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
9695         SDValue PartValue = OutVals[i + 1];
9696         unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset;
9697         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9698         EVT PartVT = PartValue.getValueType();
9699         if (PartVT.isScalableVector())
9700           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9701         StoredSize += PartVT.getStoreSize();
9702         StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
9703         Parts.push_back(std::make_pair(PartValue, Offset));
9704         ++i;
9705       }
9706       SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
9707       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
9708       MemOpChains.push_back(
9709           DAG.getStore(Chain, DL, ArgValue, SpillSlot,
9710                        MachinePointerInfo::getFixedStack(MF, FI)));
9711       for (const auto &Part : Parts) {
9712         SDValue PartValue = Part.first;
9713         SDValue PartOffset = Part.second;
9714         SDValue Address =
9715             DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
9716         MemOpChains.push_back(
9717             DAG.getStore(Chain, DL, PartValue, Address,
9718                          MachinePointerInfo::getFixedStack(MF, FI)));
9719       }
9720       ArgValue = SpillSlot;
9721     } else {
9722       ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget);
9723     }
9724 
9725     // Use local copy if it is a byval arg.
9726     if (Flags.isByVal())
9727       ArgValue = ByValArgs[j++];
9728 
9729     if (VA.isRegLoc()) {
9730       // Queue up the argument copies and emit them at the end.
9731       RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
9732     } else {
9733       assert(VA.isMemLoc() && "Argument not register or memory");
9734       assert(!IsTailCall && "Tail call not allowed if stack is used "
9735                             "for passing parameters");
9736 
9737       // Work out the address of the stack slot.
9738       if (!StackPtr.getNode())
9739         StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
9740       SDValue Address =
9741           DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
9742                       DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
9743 
9744       // Emit the store.
9745       MemOpChains.push_back(
9746           DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
9747     }
9748   }
9749 
9750   // Join the stores, which are independent of one another.
9751   if (!MemOpChains.empty())
9752     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
9753 
9754   SDValue Glue;
9755 
9756   // Build a sequence of copy-to-reg nodes, chained and glued together.
9757   for (auto &Reg : RegsToPass) {
9758     Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
9759     Glue = Chain.getValue(1);
9760   }
9761 
9762   // Validate that none of the argument registers have been marked as
9763   // reserved, if so report an error. Do the same for the return address if this
9764   // is not a tailcall.
9765   validateCCReservedRegs(RegsToPass, MF);
9766   if (!IsTailCall &&
9767       MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1))
9768     MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9769         MF.getFunction(),
9770         "Return address register required, but has been reserved."});
9771 
9772   // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
9773   // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
9774   // split it and then direct call can be matched by PseudoCALL.
9775   if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
9776     const GlobalValue *GV = S->getGlobal();
9777 
9778     unsigned OpFlags = RISCVII::MO_CALL;
9779     if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
9780       OpFlags = RISCVII::MO_PLT;
9781 
9782     Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
9783   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
9784     unsigned OpFlags = RISCVII::MO_CALL;
9785 
9786     if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
9787                                                  nullptr))
9788       OpFlags = RISCVII::MO_PLT;
9789 
9790     Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
9791   }
9792 
9793   // The first call operand is the chain and the second is the target address.
9794   SmallVector<SDValue, 8> Ops;
9795   Ops.push_back(Chain);
9796   Ops.push_back(Callee);
9797 
9798   // Add argument registers to the end of the list so that they are
9799   // known live into the call.
9800   for (auto &Reg : RegsToPass)
9801     Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
9802 
9803   if (!IsTailCall) {
9804     // Add a register mask operand representing the call-preserved registers.
9805     const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
9806     const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
9807     assert(Mask && "Missing call preserved mask for calling convention");
9808     Ops.push_back(DAG.getRegisterMask(Mask));
9809   }
9810 
9811   // Glue the call to the argument copies, if any.
9812   if (Glue.getNode())
9813     Ops.push_back(Glue);
9814 
9815   // Emit the call.
9816   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9817 
9818   if (IsTailCall) {
9819     MF.getFrameInfo().setHasTailCall();
9820     return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
9821   }
9822 
9823   Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
9824   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
9825   Glue = Chain.getValue(1);
9826 
9827   // Mark the end of the call, which is glued to the call itself.
9828   Chain = DAG.getCALLSEQ_END(Chain,
9829                              DAG.getConstant(NumBytes, DL, PtrVT, true),
9830                              DAG.getConstant(0, DL, PtrVT, true),
9831                              Glue, DL);
9832   Glue = Chain.getValue(1);
9833 
9834   // Assign locations to each value returned by this call.
9835   SmallVector<CCValAssign, 16> RVLocs;
9836   CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
9837   analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, CC_RISCV);
9838 
9839   // Copy all of the result registers out of their specified physreg.
9840   for (auto &VA : RVLocs) {
9841     // Copy the value out
9842     SDValue RetValue =
9843         DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
9844     // Glue the RetValue to the end of the call sequence
9845     Chain = RetValue.getValue(1);
9846     Glue = RetValue.getValue(2);
9847 
9848     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
9849       assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
9850       SDValue RetValue2 =
9851           DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
9852       Chain = RetValue2.getValue(1);
9853       Glue = RetValue2.getValue(2);
9854       RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
9855                              RetValue2);
9856     }
9857 
9858     RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget);
9859 
9860     InVals.push_back(RetValue);
9861   }
9862 
9863   return Chain;
9864 }
9865 
9866 bool RISCVTargetLowering::CanLowerReturn(
9867     CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
9868     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
9869   SmallVector<CCValAssign, 16> RVLocs;
9870   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
9871 
9872   Optional<unsigned> FirstMaskArgument;
9873   if (Subtarget.hasVInstructions())
9874     FirstMaskArgument = preAssignMask(Outs);
9875 
9876   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9877     MVT VT = Outs[i].VT;
9878     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
9879     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9880     if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
9881                  ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr,
9882                  *this, FirstMaskArgument))
9883       return false;
9884   }
9885   return true;
9886 }
9887 
9888 SDValue
9889 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
9890                                  bool IsVarArg,
9891                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
9892                                  const SmallVectorImpl<SDValue> &OutVals,
9893                                  const SDLoc &DL, SelectionDAG &DAG) const {
9894   const MachineFunction &MF = DAG.getMachineFunction();
9895   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
9896 
9897   // Stores the assignment of the return value to a location.
9898   SmallVector<CCValAssign, 16> RVLocs;
9899 
9900   // Info about the registers and stack slot.
9901   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
9902                  *DAG.getContext());
9903 
9904   analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
9905                     nullptr, CC_RISCV);
9906 
9907   if (CallConv == CallingConv::GHC && !RVLocs.empty())
9908     report_fatal_error("GHC functions return void only");
9909 
9910   SDValue Glue;
9911   SmallVector<SDValue, 4> RetOps(1, Chain);
9912 
9913   // Copy the result values into the output registers.
9914   for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
9915     SDValue Val = OutVals[i];
9916     CCValAssign &VA = RVLocs[i];
9917     assert(VA.isRegLoc() && "Can only return in registers!");
9918 
9919     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
9920       // Handle returning f64 on RV32D with a soft float ABI.
9921       assert(VA.isRegLoc() && "Expected return via registers");
9922       SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
9923                                      DAG.getVTList(MVT::i32, MVT::i32), Val);
9924       SDValue Lo = SplitF64.getValue(0);
9925       SDValue Hi = SplitF64.getValue(1);
9926       Register RegLo = VA.getLocReg();
9927       assert(RegLo < RISCV::X31 && "Invalid register pair");
9928       Register RegHi = RegLo + 1;
9929 
9930       if (STI.isRegisterReservedByUser(RegLo) ||
9931           STI.isRegisterReservedByUser(RegHi))
9932         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9933             MF.getFunction(),
9934             "Return value register required, but has been reserved."});
9935 
9936       Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
9937       Glue = Chain.getValue(1);
9938       RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
9939       Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
9940       Glue = Chain.getValue(1);
9941       RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
9942     } else {
9943       // Handle a 'normal' return.
9944       Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
9945       Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
9946 
9947       if (STI.isRegisterReservedByUser(VA.getLocReg()))
9948         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9949             MF.getFunction(),
9950             "Return value register required, but has been reserved."});
9951 
9952       // Guarantee that all emitted copies are stuck together.
9953       Glue = Chain.getValue(1);
9954       RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
9955     }
9956   }
9957 
9958   RetOps[0] = Chain; // Update chain.
9959 
9960   // Add the glue node if we have it.
9961   if (Glue.getNode()) {
9962     RetOps.push_back(Glue);
9963   }
9964 
9965   unsigned RetOpc = RISCVISD::RET_FLAG;
9966   // Interrupt service routines use different return instructions.
9967   const Function &Func = DAG.getMachineFunction().getFunction();
9968   if (Func.hasFnAttribute("interrupt")) {
9969     if (!Func.getReturnType()->isVoidTy())
9970       report_fatal_error(
9971           "Functions with the interrupt attribute must have void return type!");
9972 
9973     MachineFunction &MF = DAG.getMachineFunction();
9974     StringRef Kind =
9975       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
9976 
9977     if (Kind == "user")
9978       RetOpc = RISCVISD::URET_FLAG;
9979     else if (Kind == "supervisor")
9980       RetOpc = RISCVISD::SRET_FLAG;
9981     else
9982       RetOpc = RISCVISD::MRET_FLAG;
9983   }
9984 
9985   return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
9986 }
9987 
9988 void RISCVTargetLowering::validateCCReservedRegs(
9989     const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
9990     MachineFunction &MF) const {
9991   const Function &F = MF.getFunction();
9992   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
9993 
9994   if (llvm::any_of(Regs, [&STI](auto Reg) {
9995         return STI.isRegisterReservedByUser(Reg.first);
9996       }))
9997     F.getContext().diagnose(DiagnosticInfoUnsupported{
9998         F, "Argument register required, but has been reserved."});
9999 }
10000 
10001 bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
10002   return CI->isTailCall();
10003 }
10004 
10005 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
10006 #define NODE_NAME_CASE(NODE)                                                   \
10007   case RISCVISD::NODE:                                                         \
10008     return "RISCVISD::" #NODE;
10009   // clang-format off
10010   switch ((RISCVISD::NodeType)Opcode) {
10011   case RISCVISD::FIRST_NUMBER:
10012     break;
10013   NODE_NAME_CASE(RET_FLAG)
10014   NODE_NAME_CASE(URET_FLAG)
10015   NODE_NAME_CASE(SRET_FLAG)
10016   NODE_NAME_CASE(MRET_FLAG)
10017   NODE_NAME_CASE(CALL)
10018   NODE_NAME_CASE(SELECT_CC)
10019   NODE_NAME_CASE(BR_CC)
10020   NODE_NAME_CASE(BuildPairF64)
10021   NODE_NAME_CASE(SplitF64)
10022   NODE_NAME_CASE(TAIL)
10023   NODE_NAME_CASE(MULHSU)
10024   NODE_NAME_CASE(SLLW)
10025   NODE_NAME_CASE(SRAW)
10026   NODE_NAME_CASE(SRLW)
10027   NODE_NAME_CASE(DIVW)
10028   NODE_NAME_CASE(DIVUW)
10029   NODE_NAME_CASE(REMUW)
10030   NODE_NAME_CASE(ROLW)
10031   NODE_NAME_CASE(RORW)
10032   NODE_NAME_CASE(CLZW)
10033   NODE_NAME_CASE(CTZW)
10034   NODE_NAME_CASE(FSLW)
10035   NODE_NAME_CASE(FSRW)
10036   NODE_NAME_CASE(FSL)
10037   NODE_NAME_CASE(FSR)
10038   NODE_NAME_CASE(FMV_H_X)
10039   NODE_NAME_CASE(FMV_X_ANYEXTH)
10040   NODE_NAME_CASE(FMV_W_X_RV64)
10041   NODE_NAME_CASE(FMV_X_ANYEXTW_RV64)
10042   NODE_NAME_CASE(FCVT_X)
10043   NODE_NAME_CASE(FCVT_XU)
10044   NODE_NAME_CASE(FCVT_W_RV64)
10045   NODE_NAME_CASE(FCVT_WU_RV64)
10046   NODE_NAME_CASE(STRICT_FCVT_W_RV64)
10047   NODE_NAME_CASE(STRICT_FCVT_WU_RV64)
10048   NODE_NAME_CASE(READ_CYCLE_WIDE)
10049   NODE_NAME_CASE(GREV)
10050   NODE_NAME_CASE(GREVW)
10051   NODE_NAME_CASE(GORC)
10052   NODE_NAME_CASE(GORCW)
10053   NODE_NAME_CASE(SHFL)
10054   NODE_NAME_CASE(SHFLW)
10055   NODE_NAME_CASE(UNSHFL)
10056   NODE_NAME_CASE(UNSHFLW)
10057   NODE_NAME_CASE(BFP)
10058   NODE_NAME_CASE(BFPW)
10059   NODE_NAME_CASE(BCOMPRESS)
10060   NODE_NAME_CASE(BCOMPRESSW)
10061   NODE_NAME_CASE(BDECOMPRESS)
10062   NODE_NAME_CASE(BDECOMPRESSW)
10063   NODE_NAME_CASE(VMV_V_X_VL)
10064   NODE_NAME_CASE(VFMV_V_F_VL)
10065   NODE_NAME_CASE(VMV_X_S)
10066   NODE_NAME_CASE(VMV_S_X_VL)
10067   NODE_NAME_CASE(VFMV_S_F_VL)
10068   NODE_NAME_CASE(SPLAT_VECTOR_I64)
10069   NODE_NAME_CASE(SPLAT_VECTOR_SPLIT_I64_VL)
10070   NODE_NAME_CASE(READ_VLENB)
10071   NODE_NAME_CASE(TRUNCATE_VECTOR_VL)
10072   NODE_NAME_CASE(VSLIDEUP_VL)
10073   NODE_NAME_CASE(VSLIDE1UP_VL)
10074   NODE_NAME_CASE(VSLIDEDOWN_VL)
10075   NODE_NAME_CASE(VSLIDE1DOWN_VL)
10076   NODE_NAME_CASE(VID_VL)
10077   NODE_NAME_CASE(VFNCVT_ROD_VL)
10078   NODE_NAME_CASE(VECREDUCE_ADD_VL)
10079   NODE_NAME_CASE(VECREDUCE_UMAX_VL)
10080   NODE_NAME_CASE(VECREDUCE_SMAX_VL)
10081   NODE_NAME_CASE(VECREDUCE_UMIN_VL)
10082   NODE_NAME_CASE(VECREDUCE_SMIN_VL)
10083   NODE_NAME_CASE(VECREDUCE_AND_VL)
10084   NODE_NAME_CASE(VECREDUCE_OR_VL)
10085   NODE_NAME_CASE(VECREDUCE_XOR_VL)
10086   NODE_NAME_CASE(VECREDUCE_FADD_VL)
10087   NODE_NAME_CASE(VECREDUCE_SEQ_FADD_VL)
10088   NODE_NAME_CASE(VECREDUCE_FMIN_VL)
10089   NODE_NAME_CASE(VECREDUCE_FMAX_VL)
10090   NODE_NAME_CASE(ADD_VL)
10091   NODE_NAME_CASE(AND_VL)
10092   NODE_NAME_CASE(MUL_VL)
10093   NODE_NAME_CASE(OR_VL)
10094   NODE_NAME_CASE(SDIV_VL)
10095   NODE_NAME_CASE(SHL_VL)
10096   NODE_NAME_CASE(SREM_VL)
10097   NODE_NAME_CASE(SRA_VL)
10098   NODE_NAME_CASE(SRL_VL)
10099   NODE_NAME_CASE(SUB_VL)
10100   NODE_NAME_CASE(UDIV_VL)
10101   NODE_NAME_CASE(UREM_VL)
10102   NODE_NAME_CASE(XOR_VL)
10103   NODE_NAME_CASE(SADDSAT_VL)
10104   NODE_NAME_CASE(UADDSAT_VL)
10105   NODE_NAME_CASE(SSUBSAT_VL)
10106   NODE_NAME_CASE(USUBSAT_VL)
10107   NODE_NAME_CASE(FADD_VL)
10108   NODE_NAME_CASE(FSUB_VL)
10109   NODE_NAME_CASE(FMUL_VL)
10110   NODE_NAME_CASE(FDIV_VL)
10111   NODE_NAME_CASE(FNEG_VL)
10112   NODE_NAME_CASE(FABS_VL)
10113   NODE_NAME_CASE(FSQRT_VL)
10114   NODE_NAME_CASE(FMA_VL)
10115   NODE_NAME_CASE(FCOPYSIGN_VL)
10116   NODE_NAME_CASE(SMIN_VL)
10117   NODE_NAME_CASE(SMAX_VL)
10118   NODE_NAME_CASE(UMIN_VL)
10119   NODE_NAME_CASE(UMAX_VL)
10120   NODE_NAME_CASE(FMINNUM_VL)
10121   NODE_NAME_CASE(FMAXNUM_VL)
10122   NODE_NAME_CASE(MULHS_VL)
10123   NODE_NAME_CASE(MULHU_VL)
10124   NODE_NAME_CASE(FP_TO_SINT_VL)
10125   NODE_NAME_CASE(FP_TO_UINT_VL)
10126   NODE_NAME_CASE(SINT_TO_FP_VL)
10127   NODE_NAME_CASE(UINT_TO_FP_VL)
10128   NODE_NAME_CASE(FP_EXTEND_VL)
10129   NODE_NAME_CASE(FP_ROUND_VL)
10130   NODE_NAME_CASE(VWMUL_VL)
10131   NODE_NAME_CASE(VWMULU_VL)
10132   NODE_NAME_CASE(VWADDU_VL)
10133   NODE_NAME_CASE(SETCC_VL)
10134   NODE_NAME_CASE(VSELECT_VL)
10135   NODE_NAME_CASE(VP_MERGE_VL)
10136   NODE_NAME_CASE(VMAND_VL)
10137   NODE_NAME_CASE(VMOR_VL)
10138   NODE_NAME_CASE(VMXOR_VL)
10139   NODE_NAME_CASE(VMCLR_VL)
10140   NODE_NAME_CASE(VMSET_VL)
10141   NODE_NAME_CASE(VRGATHER_VX_VL)
10142   NODE_NAME_CASE(VRGATHER_VV_VL)
10143   NODE_NAME_CASE(VRGATHEREI16_VV_VL)
10144   NODE_NAME_CASE(VSEXT_VL)
10145   NODE_NAME_CASE(VZEXT_VL)
10146   NODE_NAME_CASE(VCPOP_VL)
10147   NODE_NAME_CASE(VLE_VL)
10148   NODE_NAME_CASE(VSE_VL)
10149   NODE_NAME_CASE(READ_CSR)
10150   NODE_NAME_CASE(WRITE_CSR)
10151   NODE_NAME_CASE(SWAP_CSR)
10152   }
10153   // clang-format on
10154   return nullptr;
10155 #undef NODE_NAME_CASE
10156 }
10157 
10158 /// getConstraintType - Given a constraint letter, return the type of
10159 /// constraint it is for this target.
10160 RISCVTargetLowering::ConstraintType
10161 RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
10162   if (Constraint.size() == 1) {
10163     switch (Constraint[0]) {
10164     default:
10165       break;
10166     case 'f':
10167       return C_RegisterClass;
10168     case 'I':
10169     case 'J':
10170     case 'K':
10171       return C_Immediate;
10172     case 'A':
10173       return C_Memory;
10174     case 'S': // A symbolic address
10175       return C_Other;
10176     }
10177   } else {
10178     if (Constraint == "vr" || Constraint == "vm")
10179       return C_RegisterClass;
10180   }
10181   return TargetLowering::getConstraintType(Constraint);
10182 }
10183 
10184 std::pair<unsigned, const TargetRegisterClass *>
10185 RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
10186                                                   StringRef Constraint,
10187                                                   MVT VT) const {
10188   // First, see if this is a constraint that directly corresponds to a
10189   // RISCV register class.
10190   if (Constraint.size() == 1) {
10191     switch (Constraint[0]) {
10192     case 'r':
10193       // TODO: Support fixed vectors up to XLen for P extension?
10194       if (VT.isVector())
10195         break;
10196       return std::make_pair(0U, &RISCV::GPRRegClass);
10197     case 'f':
10198       if (Subtarget.hasStdExtZfh() && VT == MVT::f16)
10199         return std::make_pair(0U, &RISCV::FPR16RegClass);
10200       if (Subtarget.hasStdExtF() && VT == MVT::f32)
10201         return std::make_pair(0U, &RISCV::FPR32RegClass);
10202       if (Subtarget.hasStdExtD() && VT == MVT::f64)
10203         return std::make_pair(0U, &RISCV::FPR64RegClass);
10204       break;
10205     default:
10206       break;
10207     }
10208   } else if (Constraint == "vr") {
10209     for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
10210                            &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10211       if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
10212         return std::make_pair(0U, RC);
10213     }
10214   } else if (Constraint == "vm") {
10215     if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
10216       return std::make_pair(0U, &RISCV::VMV0RegClass);
10217   }
10218 
10219   // Clang will correctly decode the usage of register name aliases into their
10220   // official names. However, other frontends like `rustc` do not. This allows
10221   // users of these frontends to use the ABI names for registers in LLVM-style
10222   // register constraints.
10223   unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
10224                                .Case("{zero}", RISCV::X0)
10225                                .Case("{ra}", RISCV::X1)
10226                                .Case("{sp}", RISCV::X2)
10227                                .Case("{gp}", RISCV::X3)
10228                                .Case("{tp}", RISCV::X4)
10229                                .Case("{t0}", RISCV::X5)
10230                                .Case("{t1}", RISCV::X6)
10231                                .Case("{t2}", RISCV::X7)
10232                                .Cases("{s0}", "{fp}", RISCV::X8)
10233                                .Case("{s1}", RISCV::X9)
10234                                .Case("{a0}", RISCV::X10)
10235                                .Case("{a1}", RISCV::X11)
10236                                .Case("{a2}", RISCV::X12)
10237                                .Case("{a3}", RISCV::X13)
10238                                .Case("{a4}", RISCV::X14)
10239                                .Case("{a5}", RISCV::X15)
10240                                .Case("{a6}", RISCV::X16)
10241                                .Case("{a7}", RISCV::X17)
10242                                .Case("{s2}", RISCV::X18)
10243                                .Case("{s3}", RISCV::X19)
10244                                .Case("{s4}", RISCV::X20)
10245                                .Case("{s5}", RISCV::X21)
10246                                .Case("{s6}", RISCV::X22)
10247                                .Case("{s7}", RISCV::X23)
10248                                .Case("{s8}", RISCV::X24)
10249                                .Case("{s9}", RISCV::X25)
10250                                .Case("{s10}", RISCV::X26)
10251                                .Case("{s11}", RISCV::X27)
10252                                .Case("{t3}", RISCV::X28)
10253                                .Case("{t4}", RISCV::X29)
10254                                .Case("{t5}", RISCV::X30)
10255                                .Case("{t6}", RISCV::X31)
10256                                .Default(RISCV::NoRegister);
10257   if (XRegFromAlias != RISCV::NoRegister)
10258     return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
10259 
10260   // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
10261   // TableGen record rather than the AsmName to choose registers for InlineAsm
10262   // constraints, plus we want to match those names to the widest floating point
10263   // register type available, manually select floating point registers here.
10264   //
10265   // The second case is the ABI name of the register, so that frontends can also
10266   // use the ABI names in register constraint lists.
10267   if (Subtarget.hasStdExtF()) {
10268     unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
10269                         .Cases("{f0}", "{ft0}", RISCV::F0_F)
10270                         .Cases("{f1}", "{ft1}", RISCV::F1_F)
10271                         .Cases("{f2}", "{ft2}", RISCV::F2_F)
10272                         .Cases("{f3}", "{ft3}", RISCV::F3_F)
10273                         .Cases("{f4}", "{ft4}", RISCV::F4_F)
10274                         .Cases("{f5}", "{ft5}", RISCV::F5_F)
10275                         .Cases("{f6}", "{ft6}", RISCV::F6_F)
10276                         .Cases("{f7}", "{ft7}", RISCV::F7_F)
10277                         .Cases("{f8}", "{fs0}", RISCV::F8_F)
10278                         .Cases("{f9}", "{fs1}", RISCV::F9_F)
10279                         .Cases("{f10}", "{fa0}", RISCV::F10_F)
10280                         .Cases("{f11}", "{fa1}", RISCV::F11_F)
10281                         .Cases("{f12}", "{fa2}", RISCV::F12_F)
10282                         .Cases("{f13}", "{fa3}", RISCV::F13_F)
10283                         .Cases("{f14}", "{fa4}", RISCV::F14_F)
10284                         .Cases("{f15}", "{fa5}", RISCV::F15_F)
10285                         .Cases("{f16}", "{fa6}", RISCV::F16_F)
10286                         .Cases("{f17}", "{fa7}", RISCV::F17_F)
10287                         .Cases("{f18}", "{fs2}", RISCV::F18_F)
10288                         .Cases("{f19}", "{fs3}", RISCV::F19_F)
10289                         .Cases("{f20}", "{fs4}", RISCV::F20_F)
10290                         .Cases("{f21}", "{fs5}", RISCV::F21_F)
10291                         .Cases("{f22}", "{fs6}", RISCV::F22_F)
10292                         .Cases("{f23}", "{fs7}", RISCV::F23_F)
10293                         .Cases("{f24}", "{fs8}", RISCV::F24_F)
10294                         .Cases("{f25}", "{fs9}", RISCV::F25_F)
10295                         .Cases("{f26}", "{fs10}", RISCV::F26_F)
10296                         .Cases("{f27}", "{fs11}", RISCV::F27_F)
10297                         .Cases("{f28}", "{ft8}", RISCV::F28_F)
10298                         .Cases("{f29}", "{ft9}", RISCV::F29_F)
10299                         .Cases("{f30}", "{ft10}", RISCV::F30_F)
10300                         .Cases("{f31}", "{ft11}", RISCV::F31_F)
10301                         .Default(RISCV::NoRegister);
10302     if (FReg != RISCV::NoRegister) {
10303       assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
10304       if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
10305         unsigned RegNo = FReg - RISCV::F0_F;
10306         unsigned DReg = RISCV::F0_D + RegNo;
10307         return std::make_pair(DReg, &RISCV::FPR64RegClass);
10308       }
10309       if (VT == MVT::f32 || VT == MVT::Other)
10310         return std::make_pair(FReg, &RISCV::FPR32RegClass);
10311       if (Subtarget.hasStdExtZfh() && VT == MVT::f16) {
10312         unsigned RegNo = FReg - RISCV::F0_F;
10313         unsigned HReg = RISCV::F0_H + RegNo;
10314         return std::make_pair(HReg, &RISCV::FPR16RegClass);
10315       }
10316     }
10317   }
10318 
10319   if (Subtarget.hasVInstructions()) {
10320     Register VReg = StringSwitch<Register>(Constraint.lower())
10321                         .Case("{v0}", RISCV::V0)
10322                         .Case("{v1}", RISCV::V1)
10323                         .Case("{v2}", RISCV::V2)
10324                         .Case("{v3}", RISCV::V3)
10325                         .Case("{v4}", RISCV::V4)
10326                         .Case("{v5}", RISCV::V5)
10327                         .Case("{v6}", RISCV::V6)
10328                         .Case("{v7}", RISCV::V7)
10329                         .Case("{v8}", RISCV::V8)
10330                         .Case("{v9}", RISCV::V9)
10331                         .Case("{v10}", RISCV::V10)
10332                         .Case("{v11}", RISCV::V11)
10333                         .Case("{v12}", RISCV::V12)
10334                         .Case("{v13}", RISCV::V13)
10335                         .Case("{v14}", RISCV::V14)
10336                         .Case("{v15}", RISCV::V15)
10337                         .Case("{v16}", RISCV::V16)
10338                         .Case("{v17}", RISCV::V17)
10339                         .Case("{v18}", RISCV::V18)
10340                         .Case("{v19}", RISCV::V19)
10341                         .Case("{v20}", RISCV::V20)
10342                         .Case("{v21}", RISCV::V21)
10343                         .Case("{v22}", RISCV::V22)
10344                         .Case("{v23}", RISCV::V23)
10345                         .Case("{v24}", RISCV::V24)
10346                         .Case("{v25}", RISCV::V25)
10347                         .Case("{v26}", RISCV::V26)
10348                         .Case("{v27}", RISCV::V27)
10349                         .Case("{v28}", RISCV::V28)
10350                         .Case("{v29}", RISCV::V29)
10351                         .Case("{v30}", RISCV::V30)
10352                         .Case("{v31}", RISCV::V31)
10353                         .Default(RISCV::NoRegister);
10354     if (VReg != RISCV::NoRegister) {
10355       if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
10356         return std::make_pair(VReg, &RISCV::VMRegClass);
10357       if (TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.SimpleTy))
10358         return std::make_pair(VReg, &RISCV::VRRegClass);
10359       for (const auto *RC :
10360            {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10361         if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) {
10362           VReg = TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
10363           return std::make_pair(VReg, RC);
10364         }
10365       }
10366     }
10367   }
10368 
10369   return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
10370 }
10371 
10372 unsigned
10373 RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
10374   // Currently only support length 1 constraints.
10375   if (ConstraintCode.size() == 1) {
10376     switch (ConstraintCode[0]) {
10377     case 'A':
10378       return InlineAsm::Constraint_A;
10379     default:
10380       break;
10381     }
10382   }
10383 
10384   return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
10385 }
10386 
10387 void RISCVTargetLowering::LowerAsmOperandForConstraint(
10388     SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
10389     SelectionDAG &DAG) const {
10390   // Currently only support length 1 constraints.
10391   if (Constraint.length() == 1) {
10392     switch (Constraint[0]) {
10393     case 'I':
10394       // Validate & create a 12-bit signed immediate operand.
10395       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10396         uint64_t CVal = C->getSExtValue();
10397         if (isInt<12>(CVal))
10398           Ops.push_back(
10399               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10400       }
10401       return;
10402     case 'J':
10403       // Validate & create an integer zero operand.
10404       if (auto *C = dyn_cast<ConstantSDNode>(Op))
10405         if (C->getZExtValue() == 0)
10406           Ops.push_back(
10407               DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
10408       return;
10409     case 'K':
10410       // Validate & create a 5-bit unsigned immediate operand.
10411       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10412         uint64_t CVal = C->getZExtValue();
10413         if (isUInt<5>(CVal))
10414           Ops.push_back(
10415               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10416       }
10417       return;
10418     case 'S':
10419       if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
10420         Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
10421                                                  GA->getValueType(0)));
10422       } else if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
10423         Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(),
10424                                                 BA->getValueType(0)));
10425       }
10426       return;
10427     default:
10428       break;
10429     }
10430   }
10431   TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
10432 }
10433 
10434 Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
10435                                                    Instruction *Inst,
10436                                                    AtomicOrdering Ord) const {
10437   if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
10438     return Builder.CreateFence(Ord);
10439   if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
10440     return Builder.CreateFence(AtomicOrdering::Release);
10441   return nullptr;
10442 }
10443 
10444 Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
10445                                                     Instruction *Inst,
10446                                                     AtomicOrdering Ord) const {
10447   if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
10448     return Builder.CreateFence(AtomicOrdering::Acquire);
10449   return nullptr;
10450 }
10451 
10452 TargetLowering::AtomicExpansionKind
10453 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
10454   // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
10455   // point operations can't be used in an lr/sc sequence without breaking the
10456   // forward-progress guarantee.
10457   if (AI->isFloatingPointOperation())
10458     return AtomicExpansionKind::CmpXChg;
10459 
10460   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
10461   if (Size == 8 || Size == 16)
10462     return AtomicExpansionKind::MaskedIntrinsic;
10463   return AtomicExpansionKind::None;
10464 }
10465 
10466 static Intrinsic::ID
10467 getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
10468   if (XLen == 32) {
10469     switch (BinOp) {
10470     default:
10471       llvm_unreachable("Unexpected AtomicRMW BinOp");
10472     case AtomicRMWInst::Xchg:
10473       return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
10474     case AtomicRMWInst::Add:
10475       return Intrinsic::riscv_masked_atomicrmw_add_i32;
10476     case AtomicRMWInst::Sub:
10477       return Intrinsic::riscv_masked_atomicrmw_sub_i32;
10478     case AtomicRMWInst::Nand:
10479       return Intrinsic::riscv_masked_atomicrmw_nand_i32;
10480     case AtomicRMWInst::Max:
10481       return Intrinsic::riscv_masked_atomicrmw_max_i32;
10482     case AtomicRMWInst::Min:
10483       return Intrinsic::riscv_masked_atomicrmw_min_i32;
10484     case AtomicRMWInst::UMax:
10485       return Intrinsic::riscv_masked_atomicrmw_umax_i32;
10486     case AtomicRMWInst::UMin:
10487       return Intrinsic::riscv_masked_atomicrmw_umin_i32;
10488     }
10489   }
10490 
10491   if (XLen == 64) {
10492     switch (BinOp) {
10493     default:
10494       llvm_unreachable("Unexpected AtomicRMW BinOp");
10495     case AtomicRMWInst::Xchg:
10496       return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
10497     case AtomicRMWInst::Add:
10498       return Intrinsic::riscv_masked_atomicrmw_add_i64;
10499     case AtomicRMWInst::Sub:
10500       return Intrinsic::riscv_masked_atomicrmw_sub_i64;
10501     case AtomicRMWInst::Nand:
10502       return Intrinsic::riscv_masked_atomicrmw_nand_i64;
10503     case AtomicRMWInst::Max:
10504       return Intrinsic::riscv_masked_atomicrmw_max_i64;
10505     case AtomicRMWInst::Min:
10506       return Intrinsic::riscv_masked_atomicrmw_min_i64;
10507     case AtomicRMWInst::UMax:
10508       return Intrinsic::riscv_masked_atomicrmw_umax_i64;
10509     case AtomicRMWInst::UMin:
10510       return Intrinsic::riscv_masked_atomicrmw_umin_i64;
10511     }
10512   }
10513 
10514   llvm_unreachable("Unexpected XLen\n");
10515 }
10516 
10517 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
10518     IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
10519     Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
10520   unsigned XLen = Subtarget.getXLen();
10521   Value *Ordering =
10522       Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
10523   Type *Tys[] = {AlignedAddr->getType()};
10524   Function *LrwOpScwLoop = Intrinsic::getDeclaration(
10525       AI->getModule(),
10526       getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
10527 
10528   if (XLen == 64) {
10529     Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
10530     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10531     ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
10532   }
10533 
10534   Value *Result;
10535 
10536   // Must pass the shift amount needed to sign extend the loaded value prior
10537   // to performing a signed comparison for min/max. ShiftAmt is the number of
10538   // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
10539   // is the number of bits to left+right shift the value in order to
10540   // sign-extend.
10541   if (AI->getOperation() == AtomicRMWInst::Min ||
10542       AI->getOperation() == AtomicRMWInst::Max) {
10543     const DataLayout &DL = AI->getModule()->getDataLayout();
10544     unsigned ValWidth =
10545         DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
10546     Value *SextShamt =
10547         Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
10548     Result = Builder.CreateCall(LrwOpScwLoop,
10549                                 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
10550   } else {
10551     Result =
10552         Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
10553   }
10554 
10555   if (XLen == 64)
10556     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10557   return Result;
10558 }
10559 
10560 TargetLowering::AtomicExpansionKind
10561 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
10562     AtomicCmpXchgInst *CI) const {
10563   unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
10564   if (Size == 8 || Size == 16)
10565     return AtomicExpansionKind::MaskedIntrinsic;
10566   return AtomicExpansionKind::None;
10567 }
10568 
10569 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
10570     IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
10571     Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
10572   unsigned XLen = Subtarget.getXLen();
10573   Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
10574   Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
10575   if (XLen == 64) {
10576     CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
10577     NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
10578     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10579     CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
10580   }
10581   Type *Tys[] = {AlignedAddr->getType()};
10582   Function *MaskedCmpXchg =
10583       Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
10584   Value *Result = Builder.CreateCall(
10585       MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
10586   if (XLen == 64)
10587     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10588   return Result;
10589 }
10590 
10591 bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
10592   return false;
10593 }
10594 
10595 bool RISCVTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
10596                                                EVT VT) const {
10597   if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
10598     return false;
10599 
10600   switch (FPVT.getSimpleVT().SimpleTy) {
10601   case MVT::f16:
10602     return Subtarget.hasStdExtZfh();
10603   case MVT::f32:
10604     return Subtarget.hasStdExtF();
10605   case MVT::f64:
10606     return Subtarget.hasStdExtD();
10607   default:
10608     return false;
10609   }
10610 }
10611 
10612 unsigned RISCVTargetLowering::getJumpTableEncoding() const {
10613   // If we are using the small code model, we can reduce size of jump table
10614   // entry to 4 bytes.
10615   if (Subtarget.is64Bit() && !isPositionIndependent() &&
10616       getTargetMachine().getCodeModel() == CodeModel::Small) {
10617     return MachineJumpTableInfo::EK_Custom32;
10618   }
10619   return TargetLowering::getJumpTableEncoding();
10620 }
10621 
10622 const MCExpr *RISCVTargetLowering::LowerCustomJumpTableEntry(
10623     const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
10624     unsigned uid, MCContext &Ctx) const {
10625   assert(Subtarget.is64Bit() && !isPositionIndependent() &&
10626          getTargetMachine().getCodeModel() == CodeModel::Small);
10627   return MCSymbolRefExpr::create(MBB->getSymbol(), Ctx);
10628 }
10629 
10630 bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
10631                                                      EVT VT) const {
10632   VT = VT.getScalarType();
10633 
10634   if (!VT.isSimple())
10635     return false;
10636 
10637   switch (VT.getSimpleVT().SimpleTy) {
10638   case MVT::f16:
10639     return Subtarget.hasStdExtZfh();
10640   case MVT::f32:
10641     return Subtarget.hasStdExtF();
10642   case MVT::f64:
10643     return Subtarget.hasStdExtD();
10644   default:
10645     break;
10646   }
10647 
10648   return false;
10649 }
10650 
10651 Register RISCVTargetLowering::getExceptionPointerRegister(
10652     const Constant *PersonalityFn) const {
10653   return RISCV::X10;
10654 }
10655 
10656 Register RISCVTargetLowering::getExceptionSelectorRegister(
10657     const Constant *PersonalityFn) const {
10658   return RISCV::X11;
10659 }
10660 
10661 bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
10662   // Return false to suppress the unnecessary extensions if the LibCall
10663   // arguments or return value is f32 type for LP64 ABI.
10664   RISCVABI::ABI ABI = Subtarget.getTargetABI();
10665   if (ABI == RISCVABI::ABI_LP64 && (Type == MVT::f32))
10666     return false;
10667 
10668   return true;
10669 }
10670 
10671 bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
10672   if (Subtarget.is64Bit() && Type == MVT::i32)
10673     return true;
10674 
10675   return IsSigned;
10676 }
10677 
10678 bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
10679                                                  SDValue C) const {
10680   // Check integral scalar types.
10681   if (VT.isScalarInteger()) {
10682     // Omit the optimization if the sub target has the M extension and the data
10683     // size exceeds XLen.
10684     if (Subtarget.hasStdExtM() && VT.getSizeInBits() > Subtarget.getXLen())
10685       return false;
10686     if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
10687       // Break the MUL to a SLLI and an ADD/SUB.
10688       const APInt &Imm = ConstNode->getAPIntValue();
10689       if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
10690           (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
10691         return true;
10692       // Optimize the MUL to (SH*ADD x, (SLLI x, bits)) if Imm is not simm12.
10693       if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
10694           ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
10695            (Imm - 8).isPowerOf2()))
10696         return true;
10697       // Omit the following optimization if the sub target has the M extension
10698       // and the data size >= XLen.
10699       if (Subtarget.hasStdExtM() && VT.getSizeInBits() >= Subtarget.getXLen())
10700         return false;
10701       // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
10702       // a pair of LUI/ADDI.
10703       if (!Imm.isSignedIntN(12) && Imm.countTrailingZeros() < 12) {
10704         APInt ImmS = Imm.ashr(Imm.countTrailingZeros());
10705         if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
10706             (1 - ImmS).isPowerOf2())
10707         return true;
10708       }
10709     }
10710   }
10711 
10712   return false;
10713 }
10714 
10715 bool RISCVTargetLowering::isMulAddWithConstProfitable(
10716     const SDValue &AddNode, const SDValue &ConstNode) const {
10717   // Let the DAGCombiner decide for vectors.
10718   EVT VT = AddNode.getValueType();
10719   if (VT.isVector())
10720     return true;
10721 
10722   // Let the DAGCombiner decide for larger types.
10723   if (VT.getScalarSizeInBits() > Subtarget.getXLen())
10724     return true;
10725 
10726   // It is worse if c1 is simm12 while c1*c2 is not.
10727   ConstantSDNode *C1Node = cast<ConstantSDNode>(AddNode.getOperand(1));
10728   ConstantSDNode *C2Node = cast<ConstantSDNode>(ConstNode);
10729   const APInt &C1 = C1Node->getAPIntValue();
10730   const APInt &C2 = C2Node->getAPIntValue();
10731   if (C1.isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
10732     return false;
10733 
10734   // Default to true and let the DAGCombiner decide.
10735   return true;
10736 }
10737 
10738 bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
10739     EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
10740     bool *Fast) const {
10741   if (!VT.isVector())
10742     return false;
10743 
10744   EVT ElemVT = VT.getVectorElementType();
10745   if (Alignment >= ElemVT.getStoreSize()) {
10746     if (Fast)
10747       *Fast = true;
10748     return true;
10749   }
10750 
10751   return false;
10752 }
10753 
10754 bool RISCVTargetLowering::splitValueIntoRegisterParts(
10755     SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
10756     unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {
10757   bool IsABIRegCopy = CC.hasValue();
10758   EVT ValueVT = Val.getValueType();
10759   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
10760     // Cast the f16 to i16, extend to i32, pad with ones to make a float nan,
10761     // and cast to f32.
10762     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
10763     Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
10764     Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
10765                       DAG.getConstant(0xFFFF0000, DL, MVT::i32));
10766     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
10767     Parts[0] = Val;
10768     return true;
10769   }
10770 
10771   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
10772     LLVMContext &Context = *DAG.getContext();
10773     EVT ValueEltVT = ValueVT.getVectorElementType();
10774     EVT PartEltVT = PartVT.getVectorElementType();
10775     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
10776     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
10777     if (PartVTBitSize % ValueVTBitSize == 0) {
10778       assert(PartVTBitSize >= ValueVTBitSize);
10779       // If the element types are different, bitcast to the same element type of
10780       // PartVT first.
10781       // Give an example here, we want copy a <vscale x 1 x i8> value to
10782       // <vscale x 4 x i16>.
10783       // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert
10784       // subvector, then we can bitcast to <vscale x 4 x i16>.
10785       if (ValueEltVT != PartEltVT) {
10786         if (PartVTBitSize > ValueVTBitSize) {
10787           unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
10788           assert(Count != 0 && "The number of element should not be zero.");
10789           EVT SameEltTypeVT =
10790               EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
10791           Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT,
10792                             DAG.getUNDEF(SameEltTypeVT), Val,
10793                             DAG.getVectorIdxConstant(0, DL));
10794         }
10795         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
10796       } else {
10797         Val =
10798             DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
10799                         Val, DAG.getVectorIdxConstant(0, DL));
10800       }
10801       Parts[0] = Val;
10802       return true;
10803     }
10804   }
10805   return false;
10806 }
10807 
10808 SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
10809     SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
10810     MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {
10811   bool IsABIRegCopy = CC.hasValue();
10812   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
10813     SDValue Val = Parts[0];
10814 
10815     // Cast the f32 to i32, truncate to i16, and cast back to f16.
10816     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
10817     Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
10818     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f16, Val);
10819     return Val;
10820   }
10821 
10822   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
10823     LLVMContext &Context = *DAG.getContext();
10824     SDValue Val = Parts[0];
10825     EVT ValueEltVT = ValueVT.getVectorElementType();
10826     EVT PartEltVT = PartVT.getVectorElementType();
10827     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
10828     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
10829     if (PartVTBitSize % ValueVTBitSize == 0) {
10830       assert(PartVTBitSize >= ValueVTBitSize);
10831       EVT SameEltTypeVT = ValueVT;
10832       // If the element types are different, convert it to the same element type
10833       // of PartVT.
10834       // Give an example here, we want copy a <vscale x 1 x i8> value from
10835       // <vscale x 4 x i16>.
10836       // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first,
10837       // then we can extract <vscale x 1 x i8>.
10838       if (ValueEltVT != PartEltVT) {
10839         unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
10840         assert(Count != 0 && "The number of element should not be zero.");
10841         SameEltTypeVT =
10842             EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
10843         Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
10844       }
10845       Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
10846                         DAG.getVectorIdxConstant(0, DL));
10847       return Val;
10848     }
10849   }
10850   return SDValue();
10851 }
10852 
10853 SDValue
10854 RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
10855                                    SelectionDAG &DAG,
10856                                    SmallVectorImpl<SDNode *> &Created) const {
10857   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
10858   if (isIntDivCheap(N->getValueType(0), Attr))
10859     return SDValue(N, 0); // Lower SDIV as SDIV
10860 
10861   assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
10862          "Unexpected divisor!");
10863 
10864   // Conditional move is needed, so do the transformation iff Zbt is enabled.
10865   if (!Subtarget.hasStdExtZbt())
10866     return SDValue();
10867 
10868   // When |Divisor| >= 2 ^ 12, it isn't profitable to do such transformation.
10869   // Besides, more critical path instructions will be generated when dividing
10870   // by 2. So we keep using the original DAGs for these cases.
10871   unsigned Lg2 = Divisor.countTrailingZeros();
10872   if (Lg2 == 1 || Lg2 >= 12)
10873     return SDValue();
10874 
10875   // fold (sdiv X, pow2)
10876   EVT VT = N->getValueType(0);
10877   if (VT != MVT::i32 && !(Subtarget.is64Bit() && VT == MVT::i64))
10878     return SDValue();
10879 
10880   SDLoc DL(N);
10881   SDValue N0 = N->getOperand(0);
10882   SDValue Zero = DAG.getConstant(0, DL, VT);
10883   SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
10884 
10885   // Add (N0 < 0) ? Pow2 - 1 : 0;
10886   SDValue Cmp = DAG.getSetCC(DL, VT, N0, Zero, ISD::SETLT);
10887   SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
10888   SDValue Sel = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
10889 
10890   Created.push_back(Cmp.getNode());
10891   Created.push_back(Add.getNode());
10892   Created.push_back(Sel.getNode());
10893 
10894   // Divide by pow2.
10895   SDValue SRA =
10896       DAG.getNode(ISD::SRA, DL, VT, Sel, DAG.getConstant(Lg2, DL, VT));
10897 
10898   // If we're dividing by a positive value, we're done.  Otherwise, we must
10899   // negate the result.
10900   if (Divisor.isNonNegative())
10901     return SRA;
10902 
10903   Created.push_back(SRA.getNode());
10904   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
10905 }
10906 
10907 #define GET_REGISTER_MATCHER
10908 #include "RISCVGenAsmMatcher.inc"
10909 
10910 Register
10911 RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
10912                                        const MachineFunction &MF) const {
10913   Register Reg = MatchRegisterAltName(RegName);
10914   if (Reg == RISCV::NoRegister)
10915     Reg = MatchRegisterName(RegName);
10916   if (Reg == RISCV::NoRegister)
10917     report_fatal_error(
10918         Twine("Invalid register name \"" + StringRef(RegName) + "\"."));
10919   BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
10920   if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg))
10921     report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
10922                              StringRef(RegName) + "\"."));
10923   return Reg;
10924 }
10925 
10926 namespace llvm {
10927 namespace RISCVVIntrinsicsTable {
10928 
10929 #define GET_RISCVVIntrinsicsTable_IMPL
10930 #include "RISCVGenSearchableTables.inc"
10931 
10932 } // namespace RISCVVIntrinsicsTable
10933 
10934 } // namespace llvm
10935