1 //===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation  ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the AArch64TargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64ISelLowering.h"
14 #include "AArch64CallingConvention.h"
15 #include "AArch64ExpandImm.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64PerfectShuffle.h"
18 #include "AArch64RegisterInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "Utils/AArch64BaseInfo.h"
22 #include "llvm/ADT/APFloat.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/STLExtras.h"
26 #include "llvm/ADT/SmallSet.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/Statistic.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/Analysis/ObjCARCUtil.h"
33 #include "llvm/Analysis/VectorUtils.h"
34 #include "llvm/CodeGen/CallingConvLower.h"
35 #include "llvm/CodeGen/MachineBasicBlock.h"
36 #include "llvm/CodeGen/MachineFrameInfo.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineInstrBuilder.h"
40 #include "llvm/CodeGen/MachineMemOperand.h"
41 #include "llvm/CodeGen/MachineRegisterInfo.h"
42 #include "llvm/CodeGen/RuntimeLibcalls.h"
43 #include "llvm/CodeGen/SelectionDAG.h"
44 #include "llvm/CodeGen/SelectionDAGNodes.h"
45 #include "llvm/CodeGen/TargetCallingConv.h"
46 #include "llvm/CodeGen/TargetInstrInfo.h"
47 #include "llvm/CodeGen/ValueTypes.h"
48 #include "llvm/IR/Attributes.h"
49 #include "llvm/IR/Constants.h"
50 #include "llvm/IR/DataLayout.h"
51 #include "llvm/IR/DebugLoc.h"
52 #include "llvm/IR/DerivedTypes.h"
53 #include "llvm/IR/Function.h"
54 #include "llvm/IR/GetElementPtrTypeIterator.h"
55 #include "llvm/IR/GlobalValue.h"
56 #include "llvm/IR/IRBuilder.h"
57 #include "llvm/IR/Instruction.h"
58 #include "llvm/IR/Instructions.h"
59 #include "llvm/IR/IntrinsicInst.h"
60 #include "llvm/IR/Intrinsics.h"
61 #include "llvm/IR/IntrinsicsAArch64.h"
62 #include "llvm/IR/Module.h"
63 #include "llvm/IR/OperandTraits.h"
64 #include "llvm/IR/PatternMatch.h"
65 #include "llvm/IR/Type.h"
66 #include "llvm/IR/Use.h"
67 #include "llvm/IR/Value.h"
68 #include "llvm/MC/MCRegisterInfo.h"
69 #include "llvm/Support/Casting.h"
70 #include "llvm/Support/CodeGen.h"
71 #include "llvm/Support/CommandLine.h"
72 #include "llvm/Support/Compiler.h"
73 #include "llvm/Support/Debug.h"
74 #include "llvm/Support/ErrorHandling.h"
75 #include "llvm/Support/KnownBits.h"
76 #include "llvm/Support/MachineValueType.h"
77 #include "llvm/Support/MathExtras.h"
78 #include "llvm/Support/raw_ostream.h"
79 #include "llvm/Target/TargetMachine.h"
80 #include "llvm/Target/TargetOptions.h"
81 #include <algorithm>
82 #include <bitset>
83 #include <cassert>
84 #include <cctype>
85 #include <cstdint>
86 #include <cstdlib>
87 #include <iterator>
88 #include <limits>
89 #include <tuple>
90 #include <utility>
91 #include <vector>
92 
93 using namespace llvm;
94 using namespace llvm::PatternMatch;
95 
96 #define DEBUG_TYPE "aarch64-lower"
97 
98 STATISTIC(NumTailCalls, "Number of tail calls");
99 STATISTIC(NumShiftInserts, "Number of vector shift inserts");
100 STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
101 
102 // FIXME: The necessary dtprel relocations don't seem to be supported
103 // well in the GNU bfd and gold linkers at the moment. Therefore, by
104 // default, for now, fall back to GeneralDynamic code generation.
105 cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration(
106     "aarch64-elf-ldtls-generation", cl::Hidden,
107     cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
108     cl::init(false));
109 
110 static cl::opt<bool>
111 EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
112                          cl::desc("Enable AArch64 logical imm instruction "
113                                   "optimization"),
114                          cl::init(true));
115 
116 // Temporary option added for the purpose of testing functionality added
117 // to DAGCombiner.cpp in D92230. It is expected that this can be removed
118 // in future when both implementations will be based off MGATHER rather
119 // than the GLD1 nodes added for the SVE gather load intrinsics.
120 static cl::opt<bool>
121 EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden,
122                                 cl::desc("Combine extends of AArch64 masked "
123                                          "gather intrinsics"),
124                                 cl::init(true));
125 
126 /// Value type used for condition codes.
127 static const MVT MVT_CC = MVT::i32;
128 
129 static inline EVT getPackedSVEVectorVT(EVT VT) {
130   switch (VT.getSimpleVT().SimpleTy) {
131   default:
132     llvm_unreachable("unexpected element type for vector");
133   case MVT::i8:
134     return MVT::nxv16i8;
135   case MVT::i16:
136     return MVT::nxv8i16;
137   case MVT::i32:
138     return MVT::nxv4i32;
139   case MVT::i64:
140     return MVT::nxv2i64;
141   case MVT::f16:
142     return MVT::nxv8f16;
143   case MVT::f32:
144     return MVT::nxv4f32;
145   case MVT::f64:
146     return MVT::nxv2f64;
147   case MVT::bf16:
148     return MVT::nxv8bf16;
149   }
150 }
151 
152 // NOTE: Currently there's only a need to return integer vector types. If this
153 // changes then just add an extra "type" parameter.
154 static inline EVT getPackedSVEVectorVT(ElementCount EC) {
155   switch (EC.getKnownMinValue()) {
156   default:
157     llvm_unreachable("unexpected element count for vector");
158   case 16:
159     return MVT::nxv16i8;
160   case 8:
161     return MVT::nxv8i16;
162   case 4:
163     return MVT::nxv4i32;
164   case 2:
165     return MVT::nxv2i64;
166   }
167 }
168 
169 static inline EVT getPromotedVTForPredicate(EVT VT) {
170   assert(VT.isScalableVector() && (VT.getVectorElementType() == MVT::i1) &&
171          "Expected scalable predicate vector type!");
172   switch (VT.getVectorMinNumElements()) {
173   default:
174     llvm_unreachable("unexpected element count for vector");
175   case 2:
176     return MVT::nxv2i64;
177   case 4:
178     return MVT::nxv4i32;
179   case 8:
180     return MVT::nxv8i16;
181   case 16:
182     return MVT::nxv16i8;
183   }
184 }
185 
186 /// Returns true if VT's elements occupy the lowest bit positions of its
187 /// associated register class without any intervening space.
188 ///
189 /// For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the
190 /// same register class, but only nxv8f16 can be treated as a packed vector.
191 static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) {
192   assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
193          "Expected legal vector type!");
194   return VT.isFixedLengthVector() ||
195          VT.getSizeInBits().getKnownMinSize() == AArch64::SVEBitsPerBlock;
196 }
197 
198 // Returns true for ####_MERGE_PASSTHRU opcodes, whose operands have a leading
199 // predicate and end with a passthru value matching the result type.
200 static bool isMergePassthruOpcode(unsigned Opc) {
201   switch (Opc) {
202   default:
203     return false;
204   case AArch64ISD::BITREVERSE_MERGE_PASSTHRU:
205   case AArch64ISD::BSWAP_MERGE_PASSTHRU:
206   case AArch64ISD::CTLZ_MERGE_PASSTHRU:
207   case AArch64ISD::CTPOP_MERGE_PASSTHRU:
208   case AArch64ISD::DUP_MERGE_PASSTHRU:
209   case AArch64ISD::ABS_MERGE_PASSTHRU:
210   case AArch64ISD::NEG_MERGE_PASSTHRU:
211   case AArch64ISD::FNEG_MERGE_PASSTHRU:
212   case AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU:
213   case AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU:
214   case AArch64ISD::FCEIL_MERGE_PASSTHRU:
215   case AArch64ISD::FFLOOR_MERGE_PASSTHRU:
216   case AArch64ISD::FNEARBYINT_MERGE_PASSTHRU:
217   case AArch64ISD::FRINT_MERGE_PASSTHRU:
218   case AArch64ISD::FROUND_MERGE_PASSTHRU:
219   case AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU:
220   case AArch64ISD::FTRUNC_MERGE_PASSTHRU:
221   case AArch64ISD::FP_ROUND_MERGE_PASSTHRU:
222   case AArch64ISD::FP_EXTEND_MERGE_PASSTHRU:
223   case AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU:
224   case AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU:
225   case AArch64ISD::FCVTZU_MERGE_PASSTHRU:
226   case AArch64ISD::FCVTZS_MERGE_PASSTHRU:
227   case AArch64ISD::FSQRT_MERGE_PASSTHRU:
228   case AArch64ISD::FRECPX_MERGE_PASSTHRU:
229   case AArch64ISD::FABS_MERGE_PASSTHRU:
230     return true;
231   }
232 }
233 
234 AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
235                                              const AArch64Subtarget &STI)
236     : TargetLowering(TM), Subtarget(&STI) {
237   // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
238   // we have to make something up. Arbitrarily, choose ZeroOrOne.
239   setBooleanContents(ZeroOrOneBooleanContent);
240   // When comparing vectors the result sets the different elements in the
241   // vector to all-one or all-zero.
242   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
243 
244   // Set up the register classes.
245   addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
246   addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
247 
248   if (Subtarget->hasFPARMv8()) {
249     addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
250     addRegisterClass(MVT::bf16, &AArch64::FPR16RegClass);
251     addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
252     addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
253     addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
254   }
255 
256   if (Subtarget->hasNEON()) {
257     addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
258     addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
259     // Someone set us up the NEON.
260     addDRTypeForNEON(MVT::v2f32);
261     addDRTypeForNEON(MVT::v8i8);
262     addDRTypeForNEON(MVT::v4i16);
263     addDRTypeForNEON(MVT::v2i32);
264     addDRTypeForNEON(MVT::v1i64);
265     addDRTypeForNEON(MVT::v1f64);
266     addDRTypeForNEON(MVT::v4f16);
267     if (Subtarget->hasBF16())
268       addDRTypeForNEON(MVT::v4bf16);
269 
270     addQRTypeForNEON(MVT::v4f32);
271     addQRTypeForNEON(MVT::v2f64);
272     addQRTypeForNEON(MVT::v16i8);
273     addQRTypeForNEON(MVT::v8i16);
274     addQRTypeForNEON(MVT::v4i32);
275     addQRTypeForNEON(MVT::v2i64);
276     addQRTypeForNEON(MVT::v8f16);
277     if (Subtarget->hasBF16())
278       addQRTypeForNEON(MVT::v8bf16);
279   }
280 
281   if (Subtarget->hasSVE()) {
282     // Add legal sve predicate types
283     addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
284     addRegisterClass(MVT::nxv4i1, &AArch64::PPRRegClass);
285     addRegisterClass(MVT::nxv8i1, &AArch64::PPRRegClass);
286     addRegisterClass(MVT::nxv16i1, &AArch64::PPRRegClass);
287 
288     // Add legal sve data types
289     addRegisterClass(MVT::nxv16i8, &AArch64::ZPRRegClass);
290     addRegisterClass(MVT::nxv8i16, &AArch64::ZPRRegClass);
291     addRegisterClass(MVT::nxv4i32, &AArch64::ZPRRegClass);
292     addRegisterClass(MVT::nxv2i64, &AArch64::ZPRRegClass);
293 
294     addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
295     addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
296     addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
297     addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
298     addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
299     addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
300 
301     if (Subtarget->hasBF16()) {
302       addRegisterClass(MVT::nxv2bf16, &AArch64::ZPRRegClass);
303       addRegisterClass(MVT::nxv4bf16, &AArch64::ZPRRegClass);
304       addRegisterClass(MVT::nxv8bf16, &AArch64::ZPRRegClass);
305     }
306 
307     if (Subtarget->useSVEForFixedLengthVectors()) {
308       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
309         if (useSVEForFixedLengthVectorVT(VT))
310           addRegisterClass(VT, &AArch64::ZPRRegClass);
311 
312       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
313         if (useSVEForFixedLengthVectorVT(VT))
314           addRegisterClass(VT, &AArch64::ZPRRegClass);
315     }
316 
317     for (auto VT : { MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64 }) {
318       setOperationAction(ISD::SADDSAT, VT, Legal);
319       setOperationAction(ISD::UADDSAT, VT, Legal);
320       setOperationAction(ISD::SSUBSAT, VT, Legal);
321       setOperationAction(ISD::USUBSAT, VT, Legal);
322       setOperationAction(ISD::UREM, VT, Expand);
323       setOperationAction(ISD::SREM, VT, Expand);
324       setOperationAction(ISD::SDIVREM, VT, Expand);
325       setOperationAction(ISD::UDIVREM, VT, Expand);
326     }
327 
328     for (auto VT :
329          { MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64, MVT::nxv4i8,
330            MVT::nxv4i16, MVT::nxv4i32, MVT::nxv8i8, MVT::nxv8i16 })
331       setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal);
332 
333     for (auto VT :
334          { MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32, MVT::nxv4f32,
335            MVT::nxv2f64 }) {
336       setCondCodeAction(ISD::SETO, VT, Expand);
337       setCondCodeAction(ISD::SETOLT, VT, Expand);
338       setCondCodeAction(ISD::SETLT, VT, Expand);
339       setCondCodeAction(ISD::SETOLE, VT, Expand);
340       setCondCodeAction(ISD::SETLE, VT, Expand);
341       setCondCodeAction(ISD::SETULT, VT, Expand);
342       setCondCodeAction(ISD::SETULE, VT, Expand);
343       setCondCodeAction(ISD::SETUGE, VT, Expand);
344       setCondCodeAction(ISD::SETUGT, VT, Expand);
345       setCondCodeAction(ISD::SETUEQ, VT, Expand);
346       setCondCodeAction(ISD::SETUNE, VT, Expand);
347 
348       setOperationAction(ISD::FREM, VT, Expand);
349       setOperationAction(ISD::FPOW, VT, Expand);
350       setOperationAction(ISD::FPOWI, VT, Expand);
351       setOperationAction(ISD::FCOS, VT, Expand);
352       setOperationAction(ISD::FSIN, VT, Expand);
353       setOperationAction(ISD::FSINCOS, VT, Expand);
354       setOperationAction(ISD::FEXP, VT, Expand);
355       setOperationAction(ISD::FEXP2, VT, Expand);
356       setOperationAction(ISD::FLOG, VT, Expand);
357       setOperationAction(ISD::FLOG2, VT, Expand);
358       setOperationAction(ISD::FLOG10, VT, Expand);
359     }
360   }
361 
362   // Compute derived properties from the register classes
363   computeRegisterProperties(Subtarget->getRegisterInfo());
364 
365   // Provide all sorts of operation actions
366   setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
367   setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
368   setOperationAction(ISD::SETCC, MVT::i32, Custom);
369   setOperationAction(ISD::SETCC, MVT::i64, Custom);
370   setOperationAction(ISD::SETCC, MVT::f16, Custom);
371   setOperationAction(ISD::SETCC, MVT::f32, Custom);
372   setOperationAction(ISD::SETCC, MVT::f64, Custom);
373   setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom);
374   setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom);
375   setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom);
376   setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom);
377   setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom);
378   setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom);
379   setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
380   setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
381   setOperationAction(ISD::BRCOND, MVT::Other, Expand);
382   setOperationAction(ISD::BR_CC, MVT::i32, Custom);
383   setOperationAction(ISD::BR_CC, MVT::i64, Custom);
384   setOperationAction(ISD::BR_CC, MVT::f16, Custom);
385   setOperationAction(ISD::BR_CC, MVT::f32, Custom);
386   setOperationAction(ISD::BR_CC, MVT::f64, Custom);
387   setOperationAction(ISD::SELECT, MVT::i32, Custom);
388   setOperationAction(ISD::SELECT, MVT::i64, Custom);
389   setOperationAction(ISD::SELECT, MVT::f16, Custom);
390   setOperationAction(ISD::SELECT, MVT::f32, Custom);
391   setOperationAction(ISD::SELECT, MVT::f64, Custom);
392   setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
393   setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
394   setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
395   setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
396   setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
397   setOperationAction(ISD::BR_JT, MVT::Other, Custom);
398   setOperationAction(ISD::JumpTable, MVT::i64, Custom);
399 
400   setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
401   setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
402   setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
403 
404   setOperationAction(ISD::FREM, MVT::f32, Expand);
405   setOperationAction(ISD::FREM, MVT::f64, Expand);
406   setOperationAction(ISD::FREM, MVT::f80, Expand);
407 
408   setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
409 
410   // Custom lowering hooks are needed for XOR
411   // to fold it into CSINC/CSINV.
412   setOperationAction(ISD::XOR, MVT::i32, Custom);
413   setOperationAction(ISD::XOR, MVT::i64, Custom);
414 
415   // Virtually no operation on f128 is legal, but LLVM can't expand them when
416   // there's a valid register class, so we need custom operations in most cases.
417   setOperationAction(ISD::FABS, MVT::f128, Expand);
418   setOperationAction(ISD::FADD, MVT::f128, LibCall);
419   setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
420   setOperationAction(ISD::FCOS, MVT::f128, Expand);
421   setOperationAction(ISD::FDIV, MVT::f128, LibCall);
422   setOperationAction(ISD::FMA, MVT::f128, Expand);
423   setOperationAction(ISD::FMUL, MVT::f128, LibCall);
424   setOperationAction(ISD::FNEG, MVT::f128, Expand);
425   setOperationAction(ISD::FPOW, MVT::f128, Expand);
426   setOperationAction(ISD::FREM, MVT::f128, Expand);
427   setOperationAction(ISD::FRINT, MVT::f128, Expand);
428   setOperationAction(ISD::FSIN, MVT::f128, Expand);
429   setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
430   setOperationAction(ISD::FSQRT, MVT::f128, Expand);
431   setOperationAction(ISD::FSUB, MVT::f128, LibCall);
432   setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
433   setOperationAction(ISD::SETCC, MVT::f128, Custom);
434   setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom);
435   setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom);
436   setOperationAction(ISD::BR_CC, MVT::f128, Custom);
437   setOperationAction(ISD::SELECT, MVT::f128, Custom);
438   setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
439   setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
440 
441   // Lowering for many of the conversions is actually specified by the non-f128
442   // type. The LowerXXX function will be trivial when f128 isn't involved.
443   setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
444   setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
445   setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
446   setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
447   setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
448   setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
449   setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
450   setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
451   setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
452   setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
453   setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
454   setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
455   setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
456   setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
457   setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
458   setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
459   setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
460   setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
461   setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
462   setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
463   setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
464   setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
465   setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
466   setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
467   setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
468   setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
469   setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
470   setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
471   setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
472   setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
473 
474   // Variable arguments.
475   setOperationAction(ISD::VASTART, MVT::Other, Custom);
476   setOperationAction(ISD::VAARG, MVT::Other, Custom);
477   setOperationAction(ISD::VACOPY, MVT::Other, Custom);
478   setOperationAction(ISD::VAEND, MVT::Other, Expand);
479 
480   // Variable-sized objects.
481   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
482   setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
483 
484   if (Subtarget->isTargetWindows())
485     setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
486   else
487     setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
488 
489   // Constant pool entries
490   setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
491 
492   // BlockAddress
493   setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
494 
495   // Add/Sub overflow ops with MVT::Glues are lowered to NZCV dependences.
496   setOperationAction(ISD::ADDC, MVT::i32, Custom);
497   setOperationAction(ISD::ADDE, MVT::i32, Custom);
498   setOperationAction(ISD::SUBC, MVT::i32, Custom);
499   setOperationAction(ISD::SUBE, MVT::i32, Custom);
500   setOperationAction(ISD::ADDC, MVT::i64, Custom);
501   setOperationAction(ISD::ADDE, MVT::i64, Custom);
502   setOperationAction(ISD::SUBC, MVT::i64, Custom);
503   setOperationAction(ISD::SUBE, MVT::i64, Custom);
504 
505   // AArch64 lacks both left-rotate and popcount instructions.
506   setOperationAction(ISD::ROTL, MVT::i32, Expand);
507   setOperationAction(ISD::ROTL, MVT::i64, Expand);
508   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
509     setOperationAction(ISD::ROTL, VT, Expand);
510     setOperationAction(ISD::ROTR, VT, Expand);
511   }
512 
513   // AArch64 doesn't have i32 MULH{S|U}.
514   setOperationAction(ISD::MULHU, MVT::i32, Expand);
515   setOperationAction(ISD::MULHS, MVT::i32, Expand);
516 
517   // AArch64 doesn't have {U|S}MUL_LOHI.
518   setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
519   setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
520 
521   setOperationAction(ISD::CTPOP, MVT::i32, Custom);
522   setOperationAction(ISD::CTPOP, MVT::i64, Custom);
523   setOperationAction(ISD::CTPOP, MVT::i128, Custom);
524 
525   setOperationAction(ISD::ABS, MVT::i32, Custom);
526   setOperationAction(ISD::ABS, MVT::i64, Custom);
527 
528   setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
529   setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
530   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
531     setOperationAction(ISD::SDIVREM, VT, Expand);
532     setOperationAction(ISD::UDIVREM, VT, Expand);
533   }
534   setOperationAction(ISD::SREM, MVT::i32, Expand);
535   setOperationAction(ISD::SREM, MVT::i64, Expand);
536   setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
537   setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
538   setOperationAction(ISD::UREM, MVT::i32, Expand);
539   setOperationAction(ISD::UREM, MVT::i64, Expand);
540 
541   // Custom lower Add/Sub/Mul with overflow.
542   setOperationAction(ISD::SADDO, MVT::i32, Custom);
543   setOperationAction(ISD::SADDO, MVT::i64, Custom);
544   setOperationAction(ISD::UADDO, MVT::i32, Custom);
545   setOperationAction(ISD::UADDO, MVT::i64, Custom);
546   setOperationAction(ISD::SSUBO, MVT::i32, Custom);
547   setOperationAction(ISD::SSUBO, MVT::i64, Custom);
548   setOperationAction(ISD::USUBO, MVT::i32, Custom);
549   setOperationAction(ISD::USUBO, MVT::i64, Custom);
550   setOperationAction(ISD::SMULO, MVT::i32, Custom);
551   setOperationAction(ISD::SMULO, MVT::i64, Custom);
552   setOperationAction(ISD::UMULO, MVT::i32, Custom);
553   setOperationAction(ISD::UMULO, MVT::i64, Custom);
554 
555   setOperationAction(ISD::FSIN, MVT::f32, Expand);
556   setOperationAction(ISD::FSIN, MVT::f64, Expand);
557   setOperationAction(ISD::FCOS, MVT::f32, Expand);
558   setOperationAction(ISD::FCOS, MVT::f64, Expand);
559   setOperationAction(ISD::FPOW, MVT::f32, Expand);
560   setOperationAction(ISD::FPOW, MVT::f64, Expand);
561   setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
562   setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
563   if (Subtarget->hasFullFP16())
564     setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
565   else
566     setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote);
567 
568   setOperationAction(ISD::FREM,    MVT::f16,   Promote);
569   setOperationAction(ISD::FREM,    MVT::v4f16, Expand);
570   setOperationAction(ISD::FREM,    MVT::v8f16, Expand);
571   setOperationAction(ISD::FPOW,    MVT::f16,   Promote);
572   setOperationAction(ISD::FPOW,    MVT::v4f16, Expand);
573   setOperationAction(ISD::FPOW,    MVT::v8f16, Expand);
574   setOperationAction(ISD::FPOWI,   MVT::f16,   Promote);
575   setOperationAction(ISD::FPOWI,   MVT::v4f16, Expand);
576   setOperationAction(ISD::FPOWI,   MVT::v8f16, Expand);
577   setOperationAction(ISD::FCOS,    MVT::f16,   Promote);
578   setOperationAction(ISD::FCOS,    MVT::v4f16, Expand);
579   setOperationAction(ISD::FCOS,    MVT::v8f16, Expand);
580   setOperationAction(ISD::FSIN,    MVT::f16,   Promote);
581   setOperationAction(ISD::FSIN,    MVT::v4f16, Expand);
582   setOperationAction(ISD::FSIN,    MVT::v8f16, Expand);
583   setOperationAction(ISD::FSINCOS, MVT::f16,   Promote);
584   setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand);
585   setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand);
586   setOperationAction(ISD::FEXP,    MVT::f16,   Promote);
587   setOperationAction(ISD::FEXP,    MVT::v4f16, Expand);
588   setOperationAction(ISD::FEXP,    MVT::v8f16, Expand);
589   setOperationAction(ISD::FEXP2,   MVT::f16,   Promote);
590   setOperationAction(ISD::FEXP2,   MVT::v4f16, Expand);
591   setOperationAction(ISD::FEXP2,   MVT::v8f16, Expand);
592   setOperationAction(ISD::FLOG,    MVT::f16,   Promote);
593   setOperationAction(ISD::FLOG,    MVT::v4f16, Expand);
594   setOperationAction(ISD::FLOG,    MVT::v8f16, Expand);
595   setOperationAction(ISD::FLOG2,   MVT::f16,   Promote);
596   setOperationAction(ISD::FLOG2,   MVT::v4f16, Expand);
597   setOperationAction(ISD::FLOG2,   MVT::v8f16, Expand);
598   setOperationAction(ISD::FLOG10,  MVT::f16,   Promote);
599   setOperationAction(ISD::FLOG10,  MVT::v4f16, Expand);
600   setOperationAction(ISD::FLOG10,  MVT::v8f16, Expand);
601 
602   if (!Subtarget->hasFullFP16()) {
603     setOperationAction(ISD::SELECT,      MVT::f16,  Promote);
604     setOperationAction(ISD::SELECT_CC,   MVT::f16,  Promote);
605     setOperationAction(ISD::SETCC,       MVT::f16,  Promote);
606     setOperationAction(ISD::BR_CC,       MVT::f16,  Promote);
607     setOperationAction(ISD::FADD,        MVT::f16,  Promote);
608     setOperationAction(ISD::FSUB,        MVT::f16,  Promote);
609     setOperationAction(ISD::FMUL,        MVT::f16,  Promote);
610     setOperationAction(ISD::FDIV,        MVT::f16,  Promote);
611     setOperationAction(ISD::FMA,         MVT::f16,  Promote);
612     setOperationAction(ISD::FNEG,        MVT::f16,  Promote);
613     setOperationAction(ISD::FABS,        MVT::f16,  Promote);
614     setOperationAction(ISD::FCEIL,       MVT::f16,  Promote);
615     setOperationAction(ISD::FSQRT,       MVT::f16,  Promote);
616     setOperationAction(ISD::FFLOOR,      MVT::f16,  Promote);
617     setOperationAction(ISD::FNEARBYINT,  MVT::f16,  Promote);
618     setOperationAction(ISD::FRINT,       MVT::f16,  Promote);
619     setOperationAction(ISD::FROUND,      MVT::f16,  Promote);
620     setOperationAction(ISD::FROUNDEVEN,  MVT::f16,  Promote);
621     setOperationAction(ISD::FTRUNC,      MVT::f16,  Promote);
622     setOperationAction(ISD::FMINNUM,     MVT::f16,  Promote);
623     setOperationAction(ISD::FMAXNUM,     MVT::f16,  Promote);
624     setOperationAction(ISD::FMINIMUM,    MVT::f16,  Promote);
625     setOperationAction(ISD::FMAXIMUM,    MVT::f16,  Promote);
626 
627     // promote v4f16 to v4f32 when that is known to be safe.
628     setOperationAction(ISD::FADD,        MVT::v4f16, Promote);
629     setOperationAction(ISD::FSUB,        MVT::v4f16, Promote);
630     setOperationAction(ISD::FMUL,        MVT::v4f16, Promote);
631     setOperationAction(ISD::FDIV,        MVT::v4f16, Promote);
632     AddPromotedToType(ISD::FADD,         MVT::v4f16, MVT::v4f32);
633     AddPromotedToType(ISD::FSUB,         MVT::v4f16, MVT::v4f32);
634     AddPromotedToType(ISD::FMUL,         MVT::v4f16, MVT::v4f32);
635     AddPromotedToType(ISD::FDIV,         MVT::v4f16, MVT::v4f32);
636 
637     setOperationAction(ISD::FABS,        MVT::v4f16, Expand);
638     setOperationAction(ISD::FNEG,        MVT::v4f16, Expand);
639     setOperationAction(ISD::FROUND,      MVT::v4f16, Expand);
640     setOperationAction(ISD::FROUNDEVEN,  MVT::v4f16, Expand);
641     setOperationAction(ISD::FMA,         MVT::v4f16, Expand);
642     setOperationAction(ISD::SETCC,       MVT::v4f16, Expand);
643     setOperationAction(ISD::BR_CC,       MVT::v4f16, Expand);
644     setOperationAction(ISD::SELECT,      MVT::v4f16, Expand);
645     setOperationAction(ISD::SELECT_CC,   MVT::v4f16, Expand);
646     setOperationAction(ISD::FTRUNC,      MVT::v4f16, Expand);
647     setOperationAction(ISD::FCOPYSIGN,   MVT::v4f16, Expand);
648     setOperationAction(ISD::FFLOOR,      MVT::v4f16, Expand);
649     setOperationAction(ISD::FCEIL,       MVT::v4f16, Expand);
650     setOperationAction(ISD::FRINT,       MVT::v4f16, Expand);
651     setOperationAction(ISD::FNEARBYINT,  MVT::v4f16, Expand);
652     setOperationAction(ISD::FSQRT,       MVT::v4f16, Expand);
653 
654     setOperationAction(ISD::FABS,        MVT::v8f16, Expand);
655     setOperationAction(ISD::FADD,        MVT::v8f16, Expand);
656     setOperationAction(ISD::FCEIL,       MVT::v8f16, Expand);
657     setOperationAction(ISD::FCOPYSIGN,   MVT::v8f16, Expand);
658     setOperationAction(ISD::FDIV,        MVT::v8f16, Expand);
659     setOperationAction(ISD::FFLOOR,      MVT::v8f16, Expand);
660     setOperationAction(ISD::FMA,         MVT::v8f16, Expand);
661     setOperationAction(ISD::FMUL,        MVT::v8f16, Expand);
662     setOperationAction(ISD::FNEARBYINT,  MVT::v8f16, Expand);
663     setOperationAction(ISD::FNEG,        MVT::v8f16, Expand);
664     setOperationAction(ISD::FROUND,      MVT::v8f16, Expand);
665     setOperationAction(ISD::FROUNDEVEN,  MVT::v8f16, Expand);
666     setOperationAction(ISD::FRINT,       MVT::v8f16, Expand);
667     setOperationAction(ISD::FSQRT,       MVT::v8f16, Expand);
668     setOperationAction(ISD::FSUB,        MVT::v8f16, Expand);
669     setOperationAction(ISD::FTRUNC,      MVT::v8f16, Expand);
670     setOperationAction(ISD::SETCC,       MVT::v8f16, Expand);
671     setOperationAction(ISD::BR_CC,       MVT::v8f16, Expand);
672     setOperationAction(ISD::SELECT,      MVT::v8f16, Expand);
673     setOperationAction(ISD::SELECT_CC,   MVT::v8f16, Expand);
674     setOperationAction(ISD::FP_EXTEND,   MVT::v8f16, Expand);
675   }
676 
677   // AArch64 has implementations of a lot of rounding-like FP operations.
678   for (MVT Ty : {MVT::f32, MVT::f64}) {
679     setOperationAction(ISD::FFLOOR, Ty, Legal);
680     setOperationAction(ISD::FNEARBYINT, Ty, Legal);
681     setOperationAction(ISD::FCEIL, Ty, Legal);
682     setOperationAction(ISD::FRINT, Ty, Legal);
683     setOperationAction(ISD::FTRUNC, Ty, Legal);
684     setOperationAction(ISD::FROUND, Ty, Legal);
685     setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
686     setOperationAction(ISD::FMINNUM, Ty, Legal);
687     setOperationAction(ISD::FMAXNUM, Ty, Legal);
688     setOperationAction(ISD::FMINIMUM, Ty, Legal);
689     setOperationAction(ISD::FMAXIMUM, Ty, Legal);
690     setOperationAction(ISD::LROUND, Ty, Legal);
691     setOperationAction(ISD::LLROUND, Ty, Legal);
692     setOperationAction(ISD::LRINT, Ty, Legal);
693     setOperationAction(ISD::LLRINT, Ty, Legal);
694   }
695 
696   if (Subtarget->hasFullFP16()) {
697     setOperationAction(ISD::FNEARBYINT, MVT::f16, Legal);
698     setOperationAction(ISD::FFLOOR,  MVT::f16, Legal);
699     setOperationAction(ISD::FCEIL,   MVT::f16, Legal);
700     setOperationAction(ISD::FRINT,   MVT::f16, Legal);
701     setOperationAction(ISD::FTRUNC,  MVT::f16, Legal);
702     setOperationAction(ISD::FROUND,  MVT::f16, Legal);
703     setOperationAction(ISD::FROUNDEVEN,  MVT::f16, Legal);
704     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
705     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
706     setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
707     setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
708   }
709 
710   setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
711 
712   setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
713   setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
714 
715   setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
716   setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
717   setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
718   setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
719   setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
720 
721   // Generate outline atomics library calls only if LSE was not specified for
722   // subtarget
723   if (Subtarget->outlineAtomics() && !Subtarget->hasLSE()) {
724     setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, LibCall);
725     setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, LibCall);
726     setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall);
727     setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, LibCall);
728     setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, LibCall);
729     setOperationAction(ISD::ATOMIC_SWAP, MVT::i8, LibCall);
730     setOperationAction(ISD::ATOMIC_SWAP, MVT::i16, LibCall);
731     setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall);
732     setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, LibCall);
733     setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i8, LibCall);
734     setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i16, LibCall);
735     setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall);
736     setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, LibCall);
737     setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i8, LibCall);
738     setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i16, LibCall);
739     setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall);
740     setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, LibCall);
741     setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i8, LibCall);
742     setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i16, LibCall);
743     setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i32, LibCall);
744     setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i64, LibCall);
745     setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i8, LibCall);
746     setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i16, LibCall);
747     setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall);
748     setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, LibCall);
749 #define LCALLNAMES(A, B, N)                                                    \
750   setLibcallName(A##N##_RELAX, #B #N "_relax");                                \
751   setLibcallName(A##N##_ACQ, #B #N "_acq");                                    \
752   setLibcallName(A##N##_REL, #B #N "_rel");                                    \
753   setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");
754 #define LCALLNAME4(A, B)                                                       \
755   LCALLNAMES(A, B, 1)                                                          \
756   LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
757 #define LCALLNAME5(A, B)                                                       \
758   LCALLNAMES(A, B, 1)                                                          \
759   LCALLNAMES(A, B, 2)                                                          \
760   LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
761     LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, __aarch64_cas)
762     LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, __aarch64_swp)
763     LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, __aarch64_ldadd)
764     LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, __aarch64_ldset)
765     LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, __aarch64_ldclr)
766     LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, __aarch64_ldeor)
767 #undef LCALLNAMES
768 #undef LCALLNAME4
769 #undef LCALLNAME5
770   }
771 
772   // 128-bit loads and stores can be done without expanding
773   setOperationAction(ISD::LOAD, MVT::i128, Custom);
774   setOperationAction(ISD::STORE, MVT::i128, Custom);
775 
776   // 256 bit non-temporal stores can be lowered to STNP. Do this as part of the
777   // custom lowering, as there are no un-paired non-temporal stores and
778   // legalization will break up 256 bit inputs.
779   setOperationAction(ISD::STORE, MVT::v32i8, Custom);
780   setOperationAction(ISD::STORE, MVT::v16i16, Custom);
781   setOperationAction(ISD::STORE, MVT::v16f16, Custom);
782   setOperationAction(ISD::STORE, MVT::v8i32, Custom);
783   setOperationAction(ISD::STORE, MVT::v8f32, Custom);
784   setOperationAction(ISD::STORE, MVT::v4f64, Custom);
785   setOperationAction(ISD::STORE, MVT::v4i64, Custom);
786 
787   // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0.
788   // This requires the Performance Monitors extension.
789   if (Subtarget->hasPerfMon())
790     setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
791 
792   if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
793       getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
794     // Issue __sincos_stret if available.
795     setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
796     setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
797   } else {
798     setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
799     setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
800   }
801 
802   if (Subtarget->getTargetTriple().isOSMSVCRT()) {
803     // MSVCRT doesn't have powi; fall back to pow
804     setLibcallName(RTLIB::POWI_F32, nullptr);
805     setLibcallName(RTLIB::POWI_F64, nullptr);
806   }
807 
808   // Make floating-point constants legal for the large code model, so they don't
809   // become loads from the constant pool.
810   if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) {
811     setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
812     setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
813   }
814 
815   // AArch64 does not have floating-point extending loads, i1 sign-extending
816   // load, floating-point truncating stores, or v2i32->v2i16 truncating store.
817   for (MVT VT : MVT::fp_valuetypes()) {
818     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
819     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
820     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand);
821     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
822   }
823   for (MVT VT : MVT::integer_valuetypes())
824     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand);
825 
826   setTruncStoreAction(MVT::f32, MVT::f16, Expand);
827   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
828   setTruncStoreAction(MVT::f64, MVT::f16, Expand);
829   setTruncStoreAction(MVT::f128, MVT::f80, Expand);
830   setTruncStoreAction(MVT::f128, MVT::f64, Expand);
831   setTruncStoreAction(MVT::f128, MVT::f32, Expand);
832   setTruncStoreAction(MVT::f128, MVT::f16, Expand);
833 
834   setOperationAction(ISD::BITCAST, MVT::i16, Custom);
835   setOperationAction(ISD::BITCAST, MVT::f16, Custom);
836   setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
837 
838   // Indexed loads and stores are supported.
839   for (unsigned im = (unsigned)ISD::PRE_INC;
840        im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
841     setIndexedLoadAction(im, MVT::i8, Legal);
842     setIndexedLoadAction(im, MVT::i16, Legal);
843     setIndexedLoadAction(im, MVT::i32, Legal);
844     setIndexedLoadAction(im, MVT::i64, Legal);
845     setIndexedLoadAction(im, MVT::f64, Legal);
846     setIndexedLoadAction(im, MVT::f32, Legal);
847     setIndexedLoadAction(im, MVT::f16, Legal);
848     setIndexedLoadAction(im, MVT::bf16, Legal);
849     setIndexedStoreAction(im, MVT::i8, Legal);
850     setIndexedStoreAction(im, MVT::i16, Legal);
851     setIndexedStoreAction(im, MVT::i32, Legal);
852     setIndexedStoreAction(im, MVT::i64, Legal);
853     setIndexedStoreAction(im, MVT::f64, Legal);
854     setIndexedStoreAction(im, MVT::f32, Legal);
855     setIndexedStoreAction(im, MVT::f16, Legal);
856     setIndexedStoreAction(im, MVT::bf16, Legal);
857   }
858 
859   // Trap.
860   setOperationAction(ISD::TRAP, MVT::Other, Legal);
861   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
862   setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
863 
864   // We combine OR nodes for bitfield operations.
865   setTargetDAGCombine(ISD::OR);
866   // Try to create BICs for vector ANDs.
867   setTargetDAGCombine(ISD::AND);
868 
869   // Vector add and sub nodes may conceal a high-half opportunity.
870   // Also, try to fold ADD into CSINC/CSINV..
871   setTargetDAGCombine(ISD::ADD);
872   setTargetDAGCombine(ISD::ABS);
873   setTargetDAGCombine(ISD::SUB);
874   setTargetDAGCombine(ISD::SRL);
875   setTargetDAGCombine(ISD::XOR);
876   setTargetDAGCombine(ISD::SINT_TO_FP);
877   setTargetDAGCombine(ISD::UINT_TO_FP);
878 
879   setTargetDAGCombine(ISD::FP_TO_SINT);
880   setTargetDAGCombine(ISD::FP_TO_UINT);
881   setTargetDAGCombine(ISD::FDIV);
882 
883   setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
884 
885   setTargetDAGCombine(ISD::ANY_EXTEND);
886   setTargetDAGCombine(ISD::ZERO_EXTEND);
887   setTargetDAGCombine(ISD::SIGN_EXTEND);
888   setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
889   setTargetDAGCombine(ISD::TRUNCATE);
890   setTargetDAGCombine(ISD::CONCAT_VECTORS);
891   setTargetDAGCombine(ISD::STORE);
892   if (Subtarget->supportsAddressTopByteIgnored())
893     setTargetDAGCombine(ISD::LOAD);
894 
895   setTargetDAGCombine(ISD::MUL);
896 
897   setTargetDAGCombine(ISD::SELECT);
898   setTargetDAGCombine(ISD::VSELECT);
899 
900   setTargetDAGCombine(ISD::INTRINSIC_VOID);
901   setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
902   setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
903   setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
904   setTargetDAGCombine(ISD::VECREDUCE_ADD);
905 
906   setTargetDAGCombine(ISD::GlobalAddress);
907 
908   // In case of strict alignment, avoid an excessive number of byte wide stores.
909   MaxStoresPerMemsetOptSize = 8;
910   MaxStoresPerMemset = Subtarget->requiresStrictAlign()
911                        ? MaxStoresPerMemsetOptSize : 32;
912 
913   MaxGluedStoresPerMemcpy = 4;
914   MaxStoresPerMemcpyOptSize = 4;
915   MaxStoresPerMemcpy = Subtarget->requiresStrictAlign()
916                        ? MaxStoresPerMemcpyOptSize : 16;
917 
918   MaxStoresPerMemmoveOptSize = MaxStoresPerMemmove = 4;
919 
920   MaxLoadsPerMemcmpOptSize = 4;
921   MaxLoadsPerMemcmp = Subtarget->requiresStrictAlign()
922                       ? MaxLoadsPerMemcmpOptSize : 8;
923 
924   setStackPointerRegisterToSaveRestore(AArch64::SP);
925 
926   setSchedulingPreference(Sched::Hybrid);
927 
928   EnableExtLdPromotion = true;
929 
930   // Set required alignment.
931   setMinFunctionAlignment(Align(4));
932   // Set preferred alignments.
933   setPrefLoopAlignment(Align(1ULL << STI.getPrefLoopLogAlignment()));
934   setPrefFunctionAlignment(Align(1ULL << STI.getPrefFunctionLogAlignment()));
935 
936   // Only change the limit for entries in a jump table if specified by
937   // the sub target, but not at the command line.
938   unsigned MaxJT = STI.getMaximumJumpTableSize();
939   if (MaxJT && getMaximumJumpTableSize() == UINT_MAX)
940     setMaximumJumpTableSize(MaxJT);
941 
942   setHasExtractBitsInsn(true);
943 
944   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
945 
946   if (Subtarget->hasNEON()) {
947     // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
948     // silliness like this:
949     setOperationAction(ISD::FABS, MVT::v1f64, Expand);
950     setOperationAction(ISD::FADD, MVT::v1f64, Expand);
951     setOperationAction(ISD::FCEIL, MVT::v1f64, Expand);
952     setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand);
953     setOperationAction(ISD::FCOS, MVT::v1f64, Expand);
954     setOperationAction(ISD::FDIV, MVT::v1f64, Expand);
955     setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand);
956     setOperationAction(ISD::FMA, MVT::v1f64, Expand);
957     setOperationAction(ISD::FMUL, MVT::v1f64, Expand);
958     setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand);
959     setOperationAction(ISD::FNEG, MVT::v1f64, Expand);
960     setOperationAction(ISD::FPOW, MVT::v1f64, Expand);
961     setOperationAction(ISD::FREM, MVT::v1f64, Expand);
962     setOperationAction(ISD::FROUND, MVT::v1f64, Expand);
963     setOperationAction(ISD::FROUNDEVEN, MVT::v1f64, Expand);
964     setOperationAction(ISD::FRINT, MVT::v1f64, Expand);
965     setOperationAction(ISD::FSIN, MVT::v1f64, Expand);
966     setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand);
967     setOperationAction(ISD::FSQRT, MVT::v1f64, Expand);
968     setOperationAction(ISD::FSUB, MVT::v1f64, Expand);
969     setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand);
970     setOperationAction(ISD::SETCC, MVT::v1f64, Expand);
971     setOperationAction(ISD::BR_CC, MVT::v1f64, Expand);
972     setOperationAction(ISD::SELECT, MVT::v1f64, Expand);
973     setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand);
974     setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand);
975 
976     setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand);
977     setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand);
978     setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand);
979     setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand);
980     setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand);
981 
982     setOperationAction(ISD::MUL, MVT::v1i64, Expand);
983 
984     // AArch64 doesn't have a direct vector ->f32 conversion instructions for
985     // elements smaller than i32, so promote the input to i32 first.
986     setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32);
987     setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32);
988     // i8 vector elements also need promotion to i32 for v8i8
989     setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32);
990     setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32);
991     // Similarly, there is no direct i32 -> f64 vector conversion instruction.
992     setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
993     setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
994     setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom);
995     setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom);
996     // Or, direct i32 -> f16 vector conversion.  Set it so custom, so the
997     // conversion happens in two steps: v4i32 -> v4f32 -> v4f16
998     setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
999     setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
1000 
1001     if (Subtarget->hasFullFP16()) {
1002       setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
1003       setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
1004       setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
1005       setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
1006     } else {
1007       // when AArch64 doesn't have fullfp16 support, promote the input
1008       // to i32 first.
1009       setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32);
1010       setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32);
1011       setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32);
1012       setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32);
1013     }
1014 
1015     setOperationAction(ISD::CTLZ,       MVT::v1i64, Expand);
1016     setOperationAction(ISD::CTLZ,       MVT::v2i64, Expand);
1017 
1018     // AArch64 doesn't have MUL.2d:
1019     setOperationAction(ISD::MUL, MVT::v2i64, Expand);
1020     // Custom handling for some quad-vector types to detect MULL.
1021     setOperationAction(ISD::MUL, MVT::v8i16, Custom);
1022     setOperationAction(ISD::MUL, MVT::v4i32, Custom);
1023     setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1024 
1025     // Saturates
1026     for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
1027                     MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1028       setOperationAction(ISD::SADDSAT, VT, Legal);
1029       setOperationAction(ISD::UADDSAT, VT, Legal);
1030       setOperationAction(ISD::SSUBSAT, VT, Legal);
1031       setOperationAction(ISD::USUBSAT, VT, Legal);
1032     }
1033 
1034     // Vector reductions
1035     for (MVT VT : { MVT::v4f16, MVT::v2f32,
1036                     MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
1037       if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) {
1038         setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1039         setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1040 
1041         setOperationAction(ISD::VECREDUCE_FADD, VT, Legal);
1042       }
1043     }
1044     for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
1045                     MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
1046       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1047       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1048       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1049       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1050       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1051     }
1052     setOperationAction(ISD::VECREDUCE_ADD, MVT::v2i64, Custom);
1053 
1054     setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
1055     setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
1056     // Likewise, narrowing and extending vector loads/stores aren't handled
1057     // directly.
1058     for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1059       setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
1060 
1061       if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
1062         setOperationAction(ISD::MULHS, VT, Legal);
1063         setOperationAction(ISD::MULHU, VT, Legal);
1064       } else {
1065         setOperationAction(ISD::MULHS, VT, Expand);
1066         setOperationAction(ISD::MULHU, VT, Expand);
1067       }
1068       setOperationAction(ISD::SMUL_LOHI, VT, Expand);
1069       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
1070 
1071       setOperationAction(ISD::BSWAP, VT, Expand);
1072       setOperationAction(ISD::CTTZ, VT, Expand);
1073 
1074       for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
1075         setTruncStoreAction(VT, InnerVT, Expand);
1076         setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
1077         setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
1078         setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1079       }
1080     }
1081 
1082     // AArch64 has implementations of a lot of rounding-like FP operations.
1083     for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) {
1084       setOperationAction(ISD::FFLOOR, Ty, Legal);
1085       setOperationAction(ISD::FNEARBYINT, Ty, Legal);
1086       setOperationAction(ISD::FCEIL, Ty, Legal);
1087       setOperationAction(ISD::FRINT, Ty, Legal);
1088       setOperationAction(ISD::FTRUNC, Ty, Legal);
1089       setOperationAction(ISD::FROUND, Ty, Legal);
1090       setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
1091     }
1092 
1093     if (Subtarget->hasFullFP16()) {
1094       for (MVT Ty : {MVT::v4f16, MVT::v8f16}) {
1095         setOperationAction(ISD::FFLOOR, Ty, Legal);
1096         setOperationAction(ISD::FNEARBYINT, Ty, Legal);
1097         setOperationAction(ISD::FCEIL, Ty, Legal);
1098         setOperationAction(ISD::FRINT, Ty, Legal);
1099         setOperationAction(ISD::FTRUNC, Ty, Legal);
1100         setOperationAction(ISD::FROUND, Ty, Legal);
1101         setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
1102       }
1103     }
1104 
1105     if (Subtarget->hasSVE())
1106       setOperationAction(ISD::VSCALE, MVT::i32, Custom);
1107 
1108     setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
1109   }
1110 
1111   if (Subtarget->hasSVE()) {
1112     // FIXME: Add custom lowering of MLOAD to handle different passthrus (not a
1113     // splat of 0 or undef) once vector selects supported in SVE codegen. See
1114     // D68877 for more details.
1115     for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
1116       setOperationAction(ISD::BITREVERSE, VT, Custom);
1117       setOperationAction(ISD::BSWAP, VT, Custom);
1118       setOperationAction(ISD::CTLZ, VT, Custom);
1119       setOperationAction(ISD::CTPOP, VT, Custom);
1120       setOperationAction(ISD::CTTZ, VT, Custom);
1121       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1122       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
1123       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
1124       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1125       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1126       setOperationAction(ISD::MGATHER, VT, Custom);
1127       setOperationAction(ISD::MSCATTER, VT, Custom);
1128       setOperationAction(ISD::MUL, VT, Custom);
1129       setOperationAction(ISD::MULHS, VT, Custom);
1130       setOperationAction(ISD::MULHU, VT, Custom);
1131       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1132       setOperationAction(ISD::SELECT, VT, Custom);
1133       setOperationAction(ISD::SETCC, VT, Custom);
1134       setOperationAction(ISD::SDIV, VT, Custom);
1135       setOperationAction(ISD::UDIV, VT, Custom);
1136       setOperationAction(ISD::SMIN, VT, Custom);
1137       setOperationAction(ISD::UMIN, VT, Custom);
1138       setOperationAction(ISD::SMAX, VT, Custom);
1139       setOperationAction(ISD::UMAX, VT, Custom);
1140       setOperationAction(ISD::SHL, VT, Custom);
1141       setOperationAction(ISD::SRL, VT, Custom);
1142       setOperationAction(ISD::SRA, VT, Custom);
1143       setOperationAction(ISD::ABS, VT, Custom);
1144       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1145       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1146       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1147       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1148       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1149       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1150       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1151       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1152       setOperationAction(ISD::STEP_VECTOR, VT, Custom);
1153 
1154       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
1155       setOperationAction(ISD::SMUL_LOHI, VT, Expand);
1156       setOperationAction(ISD::SELECT_CC, VT, Expand);
1157     }
1158 
1159     // Illegal unpacked integer vector types.
1160     for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) {
1161       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1162       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1163     }
1164 
1165     for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}) {
1166       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1167       setOperationAction(ISD::SELECT, VT, Custom);
1168       setOperationAction(ISD::SETCC, VT, Custom);
1169       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1170       setOperationAction(ISD::TRUNCATE, VT, Custom);
1171       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1172       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1173       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1174 
1175       setOperationAction(ISD::SELECT_CC, VT, Expand);
1176 
1177       // There are no legal MVT::nxv16f## based types.
1178       if (VT != MVT::nxv16i1) {
1179         setOperationAction(ISD::SINT_TO_FP, VT, Custom);
1180         setOperationAction(ISD::UINT_TO_FP, VT, Custom);
1181       }
1182     }
1183 
1184     for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
1185                     MVT::nxv4f32, MVT::nxv2f64}) {
1186       for (auto InnerVT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16,
1187                            MVT::nxv2f32, MVT::nxv4f32, MVT::nxv2f64}) {
1188         // Avoid marking truncating FP stores as legal to prevent the
1189         // DAGCombiner from creating unsupported truncating stores.
1190         setTruncStoreAction(VT, InnerVT, Expand);
1191       }
1192 
1193       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1194       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1195       setOperationAction(ISD::MGATHER, VT, Custom);
1196       setOperationAction(ISD::MSCATTER, VT, Custom);
1197       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1198       setOperationAction(ISD::SELECT, VT, Custom);
1199       setOperationAction(ISD::FADD, VT, Custom);
1200       setOperationAction(ISD::FDIV, VT, Custom);
1201       setOperationAction(ISD::FMA, VT, Custom);
1202       setOperationAction(ISD::FMAXIMUM, VT, Custom);
1203       setOperationAction(ISD::FMAXNUM, VT, Custom);
1204       setOperationAction(ISD::FMINIMUM, VT, Custom);
1205       setOperationAction(ISD::FMINNUM, VT, Custom);
1206       setOperationAction(ISD::FMUL, VT, Custom);
1207       setOperationAction(ISD::FNEG, VT, Custom);
1208       setOperationAction(ISD::FSUB, VT, Custom);
1209       setOperationAction(ISD::FCEIL, VT, Custom);
1210       setOperationAction(ISD::FFLOOR, VT, Custom);
1211       setOperationAction(ISD::FNEARBYINT, VT, Custom);
1212       setOperationAction(ISD::FRINT, VT, Custom);
1213       setOperationAction(ISD::FROUND, VT, Custom);
1214       setOperationAction(ISD::FROUNDEVEN, VT, Custom);
1215       setOperationAction(ISD::FTRUNC, VT, Custom);
1216       setOperationAction(ISD::FSQRT, VT, Custom);
1217       setOperationAction(ISD::FABS, VT, Custom);
1218       setOperationAction(ISD::FP_EXTEND, VT, Custom);
1219       setOperationAction(ISD::FP_ROUND, VT, Custom);
1220       setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1221       setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1222       setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1223       setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1224 
1225       setOperationAction(ISD::SELECT_CC, VT, Expand);
1226     }
1227 
1228     for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
1229       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1230       setOperationAction(ISD::MGATHER, VT, Custom);
1231       setOperationAction(ISD::MSCATTER, VT, Custom);
1232     }
1233 
1234     setOperationAction(ISD::SPLAT_VECTOR, MVT::nxv8bf16, Custom);
1235 
1236     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
1237     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
1238 
1239     // NOTE: Currently this has to happen after computeRegisterProperties rather
1240     // than the preferred option of combining it with the addRegisterClass call.
1241     if (Subtarget->useSVEForFixedLengthVectors()) {
1242       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
1243         if (useSVEForFixedLengthVectorVT(VT))
1244           addTypeForFixedLengthSVE(VT);
1245       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
1246         if (useSVEForFixedLengthVectorVT(VT))
1247           addTypeForFixedLengthSVE(VT);
1248 
1249       // 64bit results can mean a bigger than NEON input.
1250       for (auto VT : {MVT::v8i8, MVT::v4i16})
1251         setOperationAction(ISD::TRUNCATE, VT, Custom);
1252       setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
1253 
1254       // 128bit results imply a bigger than NEON input.
1255       for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
1256         setOperationAction(ISD::TRUNCATE, VT, Custom);
1257       for (auto VT : {MVT::v8f16, MVT::v4f32})
1258         setOperationAction(ISD::FP_ROUND, VT, Expand);
1259 
1260       // These operations are not supported on NEON but SVE can do them.
1261       setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
1262       setOperationAction(ISD::CTLZ, MVT::v1i64, Custom);
1263       setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
1264       setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
1265       setOperationAction(ISD::MUL, MVT::v1i64, Custom);
1266       setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1267       setOperationAction(ISD::MULHS, MVT::v1i64, Custom);
1268       setOperationAction(ISD::MULHS, MVT::v2i64, Custom);
1269       setOperationAction(ISD::MULHU, MVT::v1i64, Custom);
1270       setOperationAction(ISD::MULHU, MVT::v2i64, Custom);
1271       setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
1272       setOperationAction(ISD::SDIV, MVT::v16i8, Custom);
1273       setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
1274       setOperationAction(ISD::SDIV, MVT::v8i16, Custom);
1275       setOperationAction(ISD::SDIV, MVT::v2i32, Custom);
1276       setOperationAction(ISD::SDIV, MVT::v4i32, Custom);
1277       setOperationAction(ISD::SDIV, MVT::v1i64, Custom);
1278       setOperationAction(ISD::SDIV, MVT::v2i64, Custom);
1279       setOperationAction(ISD::SMAX, MVT::v1i64, Custom);
1280       setOperationAction(ISD::SMAX, MVT::v2i64, Custom);
1281       setOperationAction(ISD::SMIN, MVT::v1i64, Custom);
1282       setOperationAction(ISD::SMIN, MVT::v2i64, Custom);
1283       setOperationAction(ISD::UDIV, MVT::v8i8, Custom);
1284       setOperationAction(ISD::UDIV, MVT::v16i8, Custom);
1285       setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
1286       setOperationAction(ISD::UDIV, MVT::v8i16, Custom);
1287       setOperationAction(ISD::UDIV, MVT::v2i32, Custom);
1288       setOperationAction(ISD::UDIV, MVT::v4i32, Custom);
1289       setOperationAction(ISD::UDIV, MVT::v1i64, Custom);
1290       setOperationAction(ISD::UDIV, MVT::v2i64, Custom);
1291       setOperationAction(ISD::UMAX, MVT::v1i64, Custom);
1292       setOperationAction(ISD::UMAX, MVT::v2i64, Custom);
1293       setOperationAction(ISD::UMIN, MVT::v1i64, Custom);
1294       setOperationAction(ISD::UMIN, MVT::v2i64, Custom);
1295       setOperationAction(ISD::VECREDUCE_SMAX, MVT::v2i64, Custom);
1296       setOperationAction(ISD::VECREDUCE_SMIN, MVT::v2i64, Custom);
1297       setOperationAction(ISD::VECREDUCE_UMAX, MVT::v2i64, Custom);
1298       setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
1299 
1300       // Int operations with no NEON support.
1301       for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
1302                       MVT::v2i32, MVT::v4i32, MVT::v2i64}) {
1303         setOperationAction(ISD::BITREVERSE, VT, Custom);
1304         setOperationAction(ISD::CTTZ, VT, Custom);
1305         setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1306         setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1307         setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1308       }
1309 
1310       // FP operations with no NEON support.
1311       for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32,
1312                       MVT::v1f64, MVT::v2f64})
1313         setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1314 
1315       // Use SVE for vectors with more than 2 elements.
1316       for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
1317         setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1318     }
1319 
1320     setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv2i1, MVT::nxv2i64);
1321     setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv4i1, MVT::nxv4i32);
1322     setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv8i1, MVT::nxv8i16);
1323     setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv16i1, MVT::nxv16i8);
1324   }
1325 
1326   PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
1327 }
1328 
1329 void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
1330   assert(VT.isVector() && "VT should be a vector type");
1331 
1332   if (VT.isFloatingPoint()) {
1333     MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
1334     setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
1335     setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
1336   }
1337 
1338   // Mark vector float intrinsics as expand.
1339   if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
1340     setOperationAction(ISD::FSIN, VT, Expand);
1341     setOperationAction(ISD::FCOS, VT, Expand);
1342     setOperationAction(ISD::FPOW, VT, Expand);
1343     setOperationAction(ISD::FLOG, VT, Expand);
1344     setOperationAction(ISD::FLOG2, VT, Expand);
1345     setOperationAction(ISD::FLOG10, VT, Expand);
1346     setOperationAction(ISD::FEXP, VT, Expand);
1347     setOperationAction(ISD::FEXP2, VT, Expand);
1348 
1349     // But we do support custom-lowering for FCOPYSIGN.
1350     setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1351   }
1352 
1353   setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1354   setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1355   setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1356   setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1357   setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1358   setOperationAction(ISD::SRA, VT, Custom);
1359   setOperationAction(ISD::SRL, VT, Custom);
1360   setOperationAction(ISD::SHL, VT, Custom);
1361   setOperationAction(ISD::OR, VT, Custom);
1362   setOperationAction(ISD::SETCC, VT, Custom);
1363   setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
1364 
1365   setOperationAction(ISD::SELECT, VT, Expand);
1366   setOperationAction(ISD::SELECT_CC, VT, Expand);
1367   setOperationAction(ISD::VSELECT, VT, Expand);
1368   for (MVT InnerVT : MVT::all_valuetypes())
1369     setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
1370 
1371   // CNT supports only B element sizes, then use UADDLP to widen.
1372   if (VT != MVT::v8i8 && VT != MVT::v16i8)
1373     setOperationAction(ISD::CTPOP, VT, Custom);
1374 
1375   setOperationAction(ISD::UDIV, VT, Expand);
1376   setOperationAction(ISD::SDIV, VT, Expand);
1377   setOperationAction(ISD::UREM, VT, Expand);
1378   setOperationAction(ISD::SREM, VT, Expand);
1379   setOperationAction(ISD::FREM, VT, Expand);
1380 
1381   setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1382   setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1383 
1384   if (!VT.isFloatingPoint())
1385     setOperationAction(ISD::ABS, VT, Legal);
1386 
1387   // [SU][MIN|MAX] are available for all NEON types apart from i64.
1388   if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
1389     for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
1390       setOperationAction(Opcode, VT, Legal);
1391 
1392   // F[MIN|MAX][NUM|NAN] are available for all FP NEON types.
1393   if (VT.isFloatingPoint() &&
1394       VT.getVectorElementType() != MVT::bf16 &&
1395       (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()))
1396     for (unsigned Opcode :
1397          {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, ISD::FMAXNUM})
1398       setOperationAction(Opcode, VT, Legal);
1399 
1400   if (Subtarget->isLittleEndian()) {
1401     for (unsigned im = (unsigned)ISD::PRE_INC;
1402          im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
1403       setIndexedLoadAction(im, VT, Legal);
1404       setIndexedStoreAction(im, VT, Legal);
1405     }
1406   }
1407 }
1408 
1409 void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
1410   assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
1411 
1412   // By default everything must be expanded.
1413   for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1414     setOperationAction(Op, VT, Expand);
1415 
1416   // We use EXTRACT_SUBVECTOR to "cast" a scalable vector to a fixed length one.
1417   setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1418 
1419   if (VT.isFloatingPoint()) {
1420     setCondCodeAction(ISD::SETO, VT, Expand);
1421     setCondCodeAction(ISD::SETOLT, VT, Expand);
1422     setCondCodeAction(ISD::SETLT, VT, Expand);
1423     setCondCodeAction(ISD::SETOLE, VT, Expand);
1424     setCondCodeAction(ISD::SETLE, VT, Expand);
1425     setCondCodeAction(ISD::SETULT, VT, Expand);
1426     setCondCodeAction(ISD::SETULE, VT, Expand);
1427     setCondCodeAction(ISD::SETUGE, VT, Expand);
1428     setCondCodeAction(ISD::SETUGT, VT, Expand);
1429     setCondCodeAction(ISD::SETUEQ, VT, Expand);
1430     setCondCodeAction(ISD::SETUNE, VT, Expand);
1431   }
1432 
1433   // Lower fixed length vector operations to scalable equivalents.
1434   setOperationAction(ISD::ABS, VT, Custom);
1435   setOperationAction(ISD::ADD, VT, Custom);
1436   setOperationAction(ISD::AND, VT, Custom);
1437   setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1438   setOperationAction(ISD::BITREVERSE, VT, Custom);
1439   setOperationAction(ISD::BSWAP, VT, Custom);
1440   setOperationAction(ISD::CTLZ, VT, Custom);
1441   setOperationAction(ISD::CTPOP, VT, Custom);
1442   setOperationAction(ISD::CTTZ, VT, Custom);
1443   setOperationAction(ISD::FABS, VT, Custom);
1444   setOperationAction(ISD::FADD, VT, Custom);
1445   setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1446   setOperationAction(ISD::FCEIL, VT, Custom);
1447   setOperationAction(ISD::FDIV, VT, Custom);
1448   setOperationAction(ISD::FFLOOR, VT, Custom);
1449   setOperationAction(ISD::FMA, VT, Custom);
1450   setOperationAction(ISD::FMAXIMUM, VT, Custom);
1451   setOperationAction(ISD::FMAXNUM, VT, Custom);
1452   setOperationAction(ISD::FMINIMUM, VT, Custom);
1453   setOperationAction(ISD::FMINNUM, VT, Custom);
1454   setOperationAction(ISD::FMUL, VT, Custom);
1455   setOperationAction(ISD::FNEARBYINT, VT, Custom);
1456   setOperationAction(ISD::FNEG, VT, Custom);
1457   setOperationAction(ISD::FRINT, VT, Custom);
1458   setOperationAction(ISD::FROUND, VT, Custom);
1459   setOperationAction(ISD::FROUNDEVEN, VT, Custom);
1460   setOperationAction(ISD::FSQRT, VT, Custom);
1461   setOperationAction(ISD::FSUB, VT, Custom);
1462   setOperationAction(ISD::FTRUNC, VT, Custom);
1463   setOperationAction(ISD::LOAD, VT, Custom);
1464   setOperationAction(ISD::MUL, VT, Custom);
1465   setOperationAction(ISD::MULHS, VT, Custom);
1466   setOperationAction(ISD::MULHU, VT, Custom);
1467   setOperationAction(ISD::OR, VT, Custom);
1468   setOperationAction(ISD::SDIV, VT, Custom);
1469   setOperationAction(ISD::SELECT, VT, Custom);
1470   setOperationAction(ISD::SETCC, VT, Custom);
1471   setOperationAction(ISD::SHL, VT, Custom);
1472   setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1473   setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom);
1474   setOperationAction(ISD::SMAX, VT, Custom);
1475   setOperationAction(ISD::SMIN, VT, Custom);
1476   setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1477   setOperationAction(ISD::SRA, VT, Custom);
1478   setOperationAction(ISD::SRL, VT, Custom);
1479   setOperationAction(ISD::STORE, VT, Custom);
1480   setOperationAction(ISD::SUB, VT, Custom);
1481   setOperationAction(ISD::TRUNCATE, VT, Custom);
1482   setOperationAction(ISD::UDIV, VT, Custom);
1483   setOperationAction(ISD::UMAX, VT, Custom);
1484   setOperationAction(ISD::UMIN, VT, Custom);
1485   setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1486   setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1487   setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1488   setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1489   setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1490   setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1491   setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1492   setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1493   setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1494   setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1495   setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1496   setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1497   setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1498   setOperationAction(ISD::VSELECT, VT, Custom);
1499   setOperationAction(ISD::XOR, VT, Custom);
1500   setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1501 }
1502 
1503 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
1504   addRegisterClass(VT, &AArch64::FPR64RegClass);
1505   addTypeForNEON(VT, MVT::v2i32);
1506 }
1507 
1508 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
1509   addRegisterClass(VT, &AArch64::FPR128RegClass);
1510   addTypeForNEON(VT, MVT::v4i32);
1511 }
1512 
1513 EVT AArch64TargetLowering::getSetCCResultType(const DataLayout &,
1514                                               LLVMContext &C, EVT VT) const {
1515   if (!VT.isVector())
1516     return MVT::i32;
1517   if (VT.isScalableVector())
1518     return EVT::getVectorVT(C, MVT::i1, VT.getVectorElementCount());
1519   return VT.changeVectorElementTypeToInteger();
1520 }
1521 
1522 static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
1523                                const APInt &Demanded,
1524                                TargetLowering::TargetLoweringOpt &TLO,
1525                                unsigned NewOpc) {
1526   uint64_t OldImm = Imm, NewImm, Enc;
1527   uint64_t Mask = ((uint64_t)(-1LL) >> (64 - Size)), OrigMask = Mask;
1528 
1529   // Return if the immediate is already all zeros, all ones, a bimm32 or a
1530   // bimm64.
1531   if (Imm == 0 || Imm == Mask ||
1532       AArch64_AM::isLogicalImmediate(Imm & Mask, Size))
1533     return false;
1534 
1535   unsigned EltSize = Size;
1536   uint64_t DemandedBits = Demanded.getZExtValue();
1537 
1538   // Clear bits that are not demanded.
1539   Imm &= DemandedBits;
1540 
1541   while (true) {
1542     // The goal here is to set the non-demanded bits in a way that minimizes
1543     // the number of switching between 0 and 1. In order to achieve this goal,
1544     // we set the non-demanded bits to the value of the preceding demanded bits.
1545     // For example, if we have an immediate 0bx10xx0x1 ('x' indicates a
1546     // non-demanded bit), we copy bit0 (1) to the least significant 'x',
1547     // bit2 (0) to 'xx', and bit6 (1) to the most significant 'x'.
1548     // The final result is 0b11000011.
1549     uint64_t NonDemandedBits = ~DemandedBits;
1550     uint64_t InvertedImm = ~Imm & DemandedBits;
1551     uint64_t RotatedImm =
1552         ((InvertedImm << 1) | (InvertedImm >> (EltSize - 1) & 1)) &
1553         NonDemandedBits;
1554     uint64_t Sum = RotatedImm + NonDemandedBits;
1555     bool Carry = NonDemandedBits & ~Sum & (1ULL << (EltSize - 1));
1556     uint64_t Ones = (Sum + Carry) & NonDemandedBits;
1557     NewImm = (Imm | Ones) & Mask;
1558 
1559     // If NewImm or its bitwise NOT is a shifted mask, it is a bitmask immediate
1560     // or all-ones or all-zeros, in which case we can stop searching. Otherwise,
1561     // we halve the element size and continue the search.
1562     if (isShiftedMask_64(NewImm) || isShiftedMask_64(~(NewImm | ~Mask)))
1563       break;
1564 
1565     // We cannot shrink the element size any further if it is 2-bits.
1566     if (EltSize == 2)
1567       return false;
1568 
1569     EltSize /= 2;
1570     Mask >>= EltSize;
1571     uint64_t Hi = Imm >> EltSize, DemandedBitsHi = DemandedBits >> EltSize;
1572 
1573     // Return if there is mismatch in any of the demanded bits of Imm and Hi.
1574     if (((Imm ^ Hi) & (DemandedBits & DemandedBitsHi) & Mask) != 0)
1575       return false;
1576 
1577     // Merge the upper and lower halves of Imm and DemandedBits.
1578     Imm |= Hi;
1579     DemandedBits |= DemandedBitsHi;
1580   }
1581 
1582   ++NumOptimizedImms;
1583 
1584   // Replicate the element across the register width.
1585   while (EltSize < Size) {
1586     NewImm |= NewImm << EltSize;
1587     EltSize *= 2;
1588   }
1589 
1590   (void)OldImm;
1591   assert(((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
1592          "demanded bits should never be altered");
1593   assert(OldImm != NewImm && "the new imm shouldn't be equal to the old imm");
1594 
1595   // Create the new constant immediate node.
1596   EVT VT = Op.getValueType();
1597   SDLoc DL(Op);
1598   SDValue New;
1599 
1600   // If the new constant immediate is all-zeros or all-ones, let the target
1601   // independent DAG combine optimize this node.
1602   if (NewImm == 0 || NewImm == OrigMask) {
1603     New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
1604                           TLO.DAG.getConstant(NewImm, DL, VT));
1605   // Otherwise, create a machine node so that target independent DAG combine
1606   // doesn't undo this optimization.
1607   } else {
1608     Enc = AArch64_AM::encodeLogicalImmediate(NewImm, Size);
1609     SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
1610     New = SDValue(
1611         TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
1612   }
1613 
1614   return TLO.CombineTo(Op, New);
1615 }
1616 
1617 bool AArch64TargetLowering::targetShrinkDemandedConstant(
1618     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
1619     TargetLoweringOpt &TLO) const {
1620   // Delay this optimization to as late as possible.
1621   if (!TLO.LegalOps)
1622     return false;
1623 
1624   if (!EnableOptimizeLogicalImm)
1625     return false;
1626 
1627   EVT VT = Op.getValueType();
1628   if (VT.isVector())
1629     return false;
1630 
1631   unsigned Size = VT.getSizeInBits();
1632   assert((Size == 32 || Size == 64) &&
1633          "i32 or i64 is expected after legalization.");
1634 
1635   // Exit early if we demand all bits.
1636   if (DemandedBits.countPopulation() == Size)
1637     return false;
1638 
1639   unsigned NewOpc;
1640   switch (Op.getOpcode()) {
1641   default:
1642     return false;
1643   case ISD::AND:
1644     NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
1645     break;
1646   case ISD::OR:
1647     NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
1648     break;
1649   case ISD::XOR:
1650     NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
1651     break;
1652   }
1653   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
1654   if (!C)
1655     return false;
1656   uint64_t Imm = C->getZExtValue();
1657   return optimizeLogicalImm(Op, Size, Imm, DemandedBits, TLO, NewOpc);
1658 }
1659 
1660 /// computeKnownBitsForTargetNode - Determine which of the bits specified in
1661 /// Mask are known to be either zero or one and return them Known.
1662 void AArch64TargetLowering::computeKnownBitsForTargetNode(
1663     const SDValue Op, KnownBits &Known,
1664     const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
1665   switch (Op.getOpcode()) {
1666   default:
1667     break;
1668   case AArch64ISD::CSEL: {
1669     KnownBits Known2;
1670     Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
1671     Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
1672     Known = KnownBits::commonBits(Known, Known2);
1673     break;
1674   }
1675   case AArch64ISD::LOADgot:
1676   case AArch64ISD::ADDlow: {
1677     if (!Subtarget->isTargetILP32())
1678       break;
1679     // In ILP32 mode all valid pointers are in the low 4GB of the address-space.
1680     Known.Zero = APInt::getHighBitsSet(64, 32);
1681     break;
1682   }
1683   case ISD::INTRINSIC_W_CHAIN: {
1684     ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
1685     Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
1686     switch (IntID) {
1687     default: return;
1688     case Intrinsic::aarch64_ldaxr:
1689     case Intrinsic::aarch64_ldxr: {
1690       unsigned BitWidth = Known.getBitWidth();
1691       EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
1692       unsigned MemBits = VT.getScalarSizeInBits();
1693       Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
1694       return;
1695     }
1696     }
1697     break;
1698   }
1699   case ISD::INTRINSIC_WO_CHAIN:
1700   case ISD::INTRINSIC_VOID: {
1701     unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1702     switch (IntNo) {
1703     default:
1704       break;
1705     case Intrinsic::aarch64_neon_umaxv:
1706     case Intrinsic::aarch64_neon_uminv: {
1707       // Figure out the datatype of the vector operand. The UMINV instruction
1708       // will zero extend the result, so we can mark as known zero all the
1709       // bits larger than the element datatype. 32-bit or larget doesn't need
1710       // this as those are legal types and will be handled by isel directly.
1711       MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
1712       unsigned BitWidth = Known.getBitWidth();
1713       if (VT == MVT::v8i8 || VT == MVT::v16i8) {
1714         assert(BitWidth >= 8 && "Unexpected width!");
1715         APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
1716         Known.Zero |= Mask;
1717       } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
1718         assert(BitWidth >= 16 && "Unexpected width!");
1719         APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
1720         Known.Zero |= Mask;
1721       }
1722       break;
1723     } break;
1724     }
1725   }
1726   }
1727 }
1728 
1729 MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
1730                                                   EVT) const {
1731   return MVT::i64;
1732 }
1733 
1734 bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
1735     EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1736     bool *Fast) const {
1737   if (Subtarget->requiresStrictAlign())
1738     return false;
1739 
1740   if (Fast) {
1741     // Some CPUs are fine with unaligned stores except for 128-bit ones.
1742     *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 ||
1743             // See comments in performSTORECombine() for more details about
1744             // these conditions.
1745 
1746             // Code that uses clang vector extensions can mark that it
1747             // wants unaligned accesses to be treated as fast by
1748             // underspecifying alignment to be 1 or 2.
1749             Alignment <= 2 ||
1750 
1751             // Disregard v2i64. Memcpy lowering produces those and splitting
1752             // them regresses performance on micro-benchmarks and olden/bh.
1753             VT == MVT::v2i64;
1754   }
1755   return true;
1756 }
1757 
1758 // Same as above but handling LLTs instead.
1759 bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
1760     LLT Ty, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1761     bool *Fast) const {
1762   if (Subtarget->requiresStrictAlign())
1763     return false;
1764 
1765   if (Fast) {
1766     // Some CPUs are fine with unaligned stores except for 128-bit ones.
1767     *Fast = !Subtarget->isMisaligned128StoreSlow() ||
1768             Ty.getSizeInBytes() != 16 ||
1769             // See comments in performSTORECombine() for more details about
1770             // these conditions.
1771 
1772             // Code that uses clang vector extensions can mark that it
1773             // wants unaligned accesses to be treated as fast by
1774             // underspecifying alignment to be 1 or 2.
1775             Alignment <= 2 ||
1776 
1777             // Disregard v2i64. Memcpy lowering produces those and splitting
1778             // them regresses performance on micro-benchmarks and olden/bh.
1779             Ty == LLT::vector(2, 64);
1780   }
1781   return true;
1782 }
1783 
1784 FastISel *
1785 AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
1786                                       const TargetLibraryInfo *libInfo) const {
1787   return AArch64::createFastISel(funcInfo, libInfo);
1788 }
1789 
1790 const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
1791 #define MAKE_CASE(V)                                                           \
1792   case V:                                                                      \
1793     return #V;
1794   switch ((AArch64ISD::NodeType)Opcode) {
1795   case AArch64ISD::FIRST_NUMBER:
1796     break;
1797     MAKE_CASE(AArch64ISD::CALL)
1798     MAKE_CASE(AArch64ISD::ADRP)
1799     MAKE_CASE(AArch64ISD::ADR)
1800     MAKE_CASE(AArch64ISD::ADDlow)
1801     MAKE_CASE(AArch64ISD::LOADgot)
1802     MAKE_CASE(AArch64ISD::RET_FLAG)
1803     MAKE_CASE(AArch64ISD::BRCOND)
1804     MAKE_CASE(AArch64ISD::CSEL)
1805     MAKE_CASE(AArch64ISD::FCSEL)
1806     MAKE_CASE(AArch64ISD::CSINV)
1807     MAKE_CASE(AArch64ISD::CSNEG)
1808     MAKE_CASE(AArch64ISD::CSINC)
1809     MAKE_CASE(AArch64ISD::THREAD_POINTER)
1810     MAKE_CASE(AArch64ISD::TLSDESC_CALLSEQ)
1811     MAKE_CASE(AArch64ISD::ADD_PRED)
1812     MAKE_CASE(AArch64ISD::MUL_PRED)
1813     MAKE_CASE(AArch64ISD::MULHS_PRED)
1814     MAKE_CASE(AArch64ISD::MULHU_PRED)
1815     MAKE_CASE(AArch64ISD::SDIV_PRED)
1816     MAKE_CASE(AArch64ISD::SHL_PRED)
1817     MAKE_CASE(AArch64ISD::SMAX_PRED)
1818     MAKE_CASE(AArch64ISD::SMIN_PRED)
1819     MAKE_CASE(AArch64ISD::SRA_PRED)
1820     MAKE_CASE(AArch64ISD::SRL_PRED)
1821     MAKE_CASE(AArch64ISD::SUB_PRED)
1822     MAKE_CASE(AArch64ISD::UDIV_PRED)
1823     MAKE_CASE(AArch64ISD::UMAX_PRED)
1824     MAKE_CASE(AArch64ISD::UMIN_PRED)
1825     MAKE_CASE(AArch64ISD::FNEG_MERGE_PASSTHRU)
1826     MAKE_CASE(AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU)
1827     MAKE_CASE(AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU)
1828     MAKE_CASE(AArch64ISD::FCEIL_MERGE_PASSTHRU)
1829     MAKE_CASE(AArch64ISD::FFLOOR_MERGE_PASSTHRU)
1830     MAKE_CASE(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU)
1831     MAKE_CASE(AArch64ISD::FRINT_MERGE_PASSTHRU)
1832     MAKE_CASE(AArch64ISD::FROUND_MERGE_PASSTHRU)
1833     MAKE_CASE(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU)
1834     MAKE_CASE(AArch64ISD::FTRUNC_MERGE_PASSTHRU)
1835     MAKE_CASE(AArch64ISD::FP_ROUND_MERGE_PASSTHRU)
1836     MAKE_CASE(AArch64ISD::FP_EXTEND_MERGE_PASSTHRU)
1837     MAKE_CASE(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU)
1838     MAKE_CASE(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU)
1839     MAKE_CASE(AArch64ISD::FCVTZU_MERGE_PASSTHRU)
1840     MAKE_CASE(AArch64ISD::FCVTZS_MERGE_PASSTHRU)
1841     MAKE_CASE(AArch64ISD::FSQRT_MERGE_PASSTHRU)
1842     MAKE_CASE(AArch64ISD::FRECPX_MERGE_PASSTHRU)
1843     MAKE_CASE(AArch64ISD::FABS_MERGE_PASSTHRU)
1844     MAKE_CASE(AArch64ISD::ABS_MERGE_PASSTHRU)
1845     MAKE_CASE(AArch64ISD::NEG_MERGE_PASSTHRU)
1846     MAKE_CASE(AArch64ISD::SETCC_MERGE_ZERO)
1847     MAKE_CASE(AArch64ISD::ADC)
1848     MAKE_CASE(AArch64ISD::SBC)
1849     MAKE_CASE(AArch64ISD::ADDS)
1850     MAKE_CASE(AArch64ISD::SUBS)
1851     MAKE_CASE(AArch64ISD::ADCS)
1852     MAKE_CASE(AArch64ISD::SBCS)
1853     MAKE_CASE(AArch64ISD::ANDS)
1854     MAKE_CASE(AArch64ISD::CCMP)
1855     MAKE_CASE(AArch64ISD::CCMN)
1856     MAKE_CASE(AArch64ISD::FCCMP)
1857     MAKE_CASE(AArch64ISD::FCMP)
1858     MAKE_CASE(AArch64ISD::STRICT_FCMP)
1859     MAKE_CASE(AArch64ISD::STRICT_FCMPE)
1860     MAKE_CASE(AArch64ISD::DUP)
1861     MAKE_CASE(AArch64ISD::DUPLANE8)
1862     MAKE_CASE(AArch64ISD::DUPLANE16)
1863     MAKE_CASE(AArch64ISD::DUPLANE32)
1864     MAKE_CASE(AArch64ISD::DUPLANE64)
1865     MAKE_CASE(AArch64ISD::MOVI)
1866     MAKE_CASE(AArch64ISD::MOVIshift)
1867     MAKE_CASE(AArch64ISD::MOVIedit)
1868     MAKE_CASE(AArch64ISD::MOVImsl)
1869     MAKE_CASE(AArch64ISD::FMOV)
1870     MAKE_CASE(AArch64ISD::MVNIshift)
1871     MAKE_CASE(AArch64ISD::MVNImsl)
1872     MAKE_CASE(AArch64ISD::BICi)
1873     MAKE_CASE(AArch64ISD::ORRi)
1874     MAKE_CASE(AArch64ISD::BSP)
1875     MAKE_CASE(AArch64ISD::NEG)
1876     MAKE_CASE(AArch64ISD::EXTR)
1877     MAKE_CASE(AArch64ISD::ZIP1)
1878     MAKE_CASE(AArch64ISD::ZIP2)
1879     MAKE_CASE(AArch64ISD::UZP1)
1880     MAKE_CASE(AArch64ISD::UZP2)
1881     MAKE_CASE(AArch64ISD::TRN1)
1882     MAKE_CASE(AArch64ISD::TRN2)
1883     MAKE_CASE(AArch64ISD::REV16)
1884     MAKE_CASE(AArch64ISD::REV32)
1885     MAKE_CASE(AArch64ISD::REV64)
1886     MAKE_CASE(AArch64ISD::EXT)
1887     MAKE_CASE(AArch64ISD::VSHL)
1888     MAKE_CASE(AArch64ISD::VLSHR)
1889     MAKE_CASE(AArch64ISD::VASHR)
1890     MAKE_CASE(AArch64ISD::VSLI)
1891     MAKE_CASE(AArch64ISD::VSRI)
1892     MAKE_CASE(AArch64ISD::CMEQ)
1893     MAKE_CASE(AArch64ISD::CMGE)
1894     MAKE_CASE(AArch64ISD::CMGT)
1895     MAKE_CASE(AArch64ISD::CMHI)
1896     MAKE_CASE(AArch64ISD::CMHS)
1897     MAKE_CASE(AArch64ISD::FCMEQ)
1898     MAKE_CASE(AArch64ISD::FCMGE)
1899     MAKE_CASE(AArch64ISD::FCMGT)
1900     MAKE_CASE(AArch64ISD::CMEQz)
1901     MAKE_CASE(AArch64ISD::CMGEz)
1902     MAKE_CASE(AArch64ISD::CMGTz)
1903     MAKE_CASE(AArch64ISD::CMLEz)
1904     MAKE_CASE(AArch64ISD::CMLTz)
1905     MAKE_CASE(AArch64ISD::FCMEQz)
1906     MAKE_CASE(AArch64ISD::FCMGEz)
1907     MAKE_CASE(AArch64ISD::FCMGTz)
1908     MAKE_CASE(AArch64ISD::FCMLEz)
1909     MAKE_CASE(AArch64ISD::FCMLTz)
1910     MAKE_CASE(AArch64ISD::SADDV)
1911     MAKE_CASE(AArch64ISD::UADDV)
1912     MAKE_CASE(AArch64ISD::SRHADD)
1913     MAKE_CASE(AArch64ISD::URHADD)
1914     MAKE_CASE(AArch64ISD::SHADD)
1915     MAKE_CASE(AArch64ISD::UHADD)
1916     MAKE_CASE(AArch64ISD::SDOT)
1917     MAKE_CASE(AArch64ISD::UDOT)
1918     MAKE_CASE(AArch64ISD::SMINV)
1919     MAKE_CASE(AArch64ISD::UMINV)
1920     MAKE_CASE(AArch64ISD::SMAXV)
1921     MAKE_CASE(AArch64ISD::UMAXV)
1922     MAKE_CASE(AArch64ISD::SADDV_PRED)
1923     MAKE_CASE(AArch64ISD::UADDV_PRED)
1924     MAKE_CASE(AArch64ISD::SMAXV_PRED)
1925     MAKE_CASE(AArch64ISD::UMAXV_PRED)
1926     MAKE_CASE(AArch64ISD::SMINV_PRED)
1927     MAKE_CASE(AArch64ISD::UMINV_PRED)
1928     MAKE_CASE(AArch64ISD::ORV_PRED)
1929     MAKE_CASE(AArch64ISD::EORV_PRED)
1930     MAKE_CASE(AArch64ISD::ANDV_PRED)
1931     MAKE_CASE(AArch64ISD::CLASTA_N)
1932     MAKE_CASE(AArch64ISD::CLASTB_N)
1933     MAKE_CASE(AArch64ISD::LASTA)
1934     MAKE_CASE(AArch64ISD::LASTB)
1935     MAKE_CASE(AArch64ISD::REINTERPRET_CAST)
1936     MAKE_CASE(AArch64ISD::TBL)
1937     MAKE_CASE(AArch64ISD::FADD_PRED)
1938     MAKE_CASE(AArch64ISD::FADDA_PRED)
1939     MAKE_CASE(AArch64ISD::FADDV_PRED)
1940     MAKE_CASE(AArch64ISD::FDIV_PRED)
1941     MAKE_CASE(AArch64ISD::FMA_PRED)
1942     MAKE_CASE(AArch64ISD::FMAX_PRED)
1943     MAKE_CASE(AArch64ISD::FMAXV_PRED)
1944     MAKE_CASE(AArch64ISD::FMAXNM_PRED)
1945     MAKE_CASE(AArch64ISD::FMAXNMV_PRED)
1946     MAKE_CASE(AArch64ISD::FMIN_PRED)
1947     MAKE_CASE(AArch64ISD::FMINV_PRED)
1948     MAKE_CASE(AArch64ISD::FMINNM_PRED)
1949     MAKE_CASE(AArch64ISD::FMINNMV_PRED)
1950     MAKE_CASE(AArch64ISD::FMUL_PRED)
1951     MAKE_CASE(AArch64ISD::FSUB_PRED)
1952     MAKE_CASE(AArch64ISD::BIT)
1953     MAKE_CASE(AArch64ISD::CBZ)
1954     MAKE_CASE(AArch64ISD::CBNZ)
1955     MAKE_CASE(AArch64ISD::TBZ)
1956     MAKE_CASE(AArch64ISD::TBNZ)
1957     MAKE_CASE(AArch64ISD::TC_RETURN)
1958     MAKE_CASE(AArch64ISD::PREFETCH)
1959     MAKE_CASE(AArch64ISD::SITOF)
1960     MAKE_CASE(AArch64ISD::UITOF)
1961     MAKE_CASE(AArch64ISD::NVCAST)
1962     MAKE_CASE(AArch64ISD::MRS)
1963     MAKE_CASE(AArch64ISD::SQSHL_I)
1964     MAKE_CASE(AArch64ISD::UQSHL_I)
1965     MAKE_CASE(AArch64ISD::SRSHR_I)
1966     MAKE_CASE(AArch64ISD::URSHR_I)
1967     MAKE_CASE(AArch64ISD::SQSHLU_I)
1968     MAKE_CASE(AArch64ISD::WrapperLarge)
1969     MAKE_CASE(AArch64ISD::LD2post)
1970     MAKE_CASE(AArch64ISD::LD3post)
1971     MAKE_CASE(AArch64ISD::LD4post)
1972     MAKE_CASE(AArch64ISD::ST2post)
1973     MAKE_CASE(AArch64ISD::ST3post)
1974     MAKE_CASE(AArch64ISD::ST4post)
1975     MAKE_CASE(AArch64ISD::LD1x2post)
1976     MAKE_CASE(AArch64ISD::LD1x3post)
1977     MAKE_CASE(AArch64ISD::LD1x4post)
1978     MAKE_CASE(AArch64ISD::ST1x2post)
1979     MAKE_CASE(AArch64ISD::ST1x3post)
1980     MAKE_CASE(AArch64ISD::ST1x4post)
1981     MAKE_CASE(AArch64ISD::LD1DUPpost)
1982     MAKE_CASE(AArch64ISD::LD2DUPpost)
1983     MAKE_CASE(AArch64ISD::LD3DUPpost)
1984     MAKE_CASE(AArch64ISD::LD4DUPpost)
1985     MAKE_CASE(AArch64ISD::LD1LANEpost)
1986     MAKE_CASE(AArch64ISD::LD2LANEpost)
1987     MAKE_CASE(AArch64ISD::LD3LANEpost)
1988     MAKE_CASE(AArch64ISD::LD4LANEpost)
1989     MAKE_CASE(AArch64ISD::ST2LANEpost)
1990     MAKE_CASE(AArch64ISD::ST3LANEpost)
1991     MAKE_CASE(AArch64ISD::ST4LANEpost)
1992     MAKE_CASE(AArch64ISD::SMULL)
1993     MAKE_CASE(AArch64ISD::UMULL)
1994     MAKE_CASE(AArch64ISD::FRECPE)
1995     MAKE_CASE(AArch64ISD::FRECPS)
1996     MAKE_CASE(AArch64ISD::FRSQRTE)
1997     MAKE_CASE(AArch64ISD::FRSQRTS)
1998     MAKE_CASE(AArch64ISD::STG)
1999     MAKE_CASE(AArch64ISD::STZG)
2000     MAKE_CASE(AArch64ISD::ST2G)
2001     MAKE_CASE(AArch64ISD::STZ2G)
2002     MAKE_CASE(AArch64ISD::SUNPKHI)
2003     MAKE_CASE(AArch64ISD::SUNPKLO)
2004     MAKE_CASE(AArch64ISD::UUNPKHI)
2005     MAKE_CASE(AArch64ISD::UUNPKLO)
2006     MAKE_CASE(AArch64ISD::INSR)
2007     MAKE_CASE(AArch64ISD::PTEST)
2008     MAKE_CASE(AArch64ISD::PTRUE)
2009     MAKE_CASE(AArch64ISD::LD1_MERGE_ZERO)
2010     MAKE_CASE(AArch64ISD::LD1S_MERGE_ZERO)
2011     MAKE_CASE(AArch64ISD::LDNF1_MERGE_ZERO)
2012     MAKE_CASE(AArch64ISD::LDNF1S_MERGE_ZERO)
2013     MAKE_CASE(AArch64ISD::LDFF1_MERGE_ZERO)
2014     MAKE_CASE(AArch64ISD::LDFF1S_MERGE_ZERO)
2015     MAKE_CASE(AArch64ISD::LD1RQ_MERGE_ZERO)
2016     MAKE_CASE(AArch64ISD::LD1RO_MERGE_ZERO)
2017     MAKE_CASE(AArch64ISD::SVE_LD2_MERGE_ZERO)
2018     MAKE_CASE(AArch64ISD::SVE_LD3_MERGE_ZERO)
2019     MAKE_CASE(AArch64ISD::SVE_LD4_MERGE_ZERO)
2020     MAKE_CASE(AArch64ISD::GLD1_MERGE_ZERO)
2021     MAKE_CASE(AArch64ISD::GLD1_SCALED_MERGE_ZERO)
2022     MAKE_CASE(AArch64ISD::GLD1_SXTW_MERGE_ZERO)
2023     MAKE_CASE(AArch64ISD::GLD1_UXTW_MERGE_ZERO)
2024     MAKE_CASE(AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO)
2025     MAKE_CASE(AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO)
2026     MAKE_CASE(AArch64ISD::GLD1_IMM_MERGE_ZERO)
2027     MAKE_CASE(AArch64ISD::GLD1S_MERGE_ZERO)
2028     MAKE_CASE(AArch64ISD::GLD1S_SCALED_MERGE_ZERO)
2029     MAKE_CASE(AArch64ISD::GLD1S_SXTW_MERGE_ZERO)
2030     MAKE_CASE(AArch64ISD::GLD1S_UXTW_MERGE_ZERO)
2031     MAKE_CASE(AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO)
2032     MAKE_CASE(AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO)
2033     MAKE_CASE(AArch64ISD::GLD1S_IMM_MERGE_ZERO)
2034     MAKE_CASE(AArch64ISD::GLDFF1_MERGE_ZERO)
2035     MAKE_CASE(AArch64ISD::GLDFF1_SCALED_MERGE_ZERO)
2036     MAKE_CASE(AArch64ISD::GLDFF1_SXTW_MERGE_ZERO)
2037     MAKE_CASE(AArch64ISD::GLDFF1_UXTW_MERGE_ZERO)
2038     MAKE_CASE(AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO)
2039     MAKE_CASE(AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO)
2040     MAKE_CASE(AArch64ISD::GLDFF1_IMM_MERGE_ZERO)
2041     MAKE_CASE(AArch64ISD::GLDFF1S_MERGE_ZERO)
2042     MAKE_CASE(AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO)
2043     MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO)
2044     MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO)
2045     MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO)
2046     MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO)
2047     MAKE_CASE(AArch64ISD::GLDFF1S_IMM_MERGE_ZERO)
2048     MAKE_CASE(AArch64ISD::GLDNT1_MERGE_ZERO)
2049     MAKE_CASE(AArch64ISD::GLDNT1_INDEX_MERGE_ZERO)
2050     MAKE_CASE(AArch64ISD::GLDNT1S_MERGE_ZERO)
2051     MAKE_CASE(AArch64ISD::ST1_PRED)
2052     MAKE_CASE(AArch64ISD::SST1_PRED)
2053     MAKE_CASE(AArch64ISD::SST1_SCALED_PRED)
2054     MAKE_CASE(AArch64ISD::SST1_SXTW_PRED)
2055     MAKE_CASE(AArch64ISD::SST1_UXTW_PRED)
2056     MAKE_CASE(AArch64ISD::SST1_SXTW_SCALED_PRED)
2057     MAKE_CASE(AArch64ISD::SST1_UXTW_SCALED_PRED)
2058     MAKE_CASE(AArch64ISD::SST1_IMM_PRED)
2059     MAKE_CASE(AArch64ISD::SSTNT1_PRED)
2060     MAKE_CASE(AArch64ISD::SSTNT1_INDEX_PRED)
2061     MAKE_CASE(AArch64ISD::LDP)
2062     MAKE_CASE(AArch64ISD::STP)
2063     MAKE_CASE(AArch64ISD::STNP)
2064     MAKE_CASE(AArch64ISD::BITREVERSE_MERGE_PASSTHRU)
2065     MAKE_CASE(AArch64ISD::BSWAP_MERGE_PASSTHRU)
2066     MAKE_CASE(AArch64ISD::CTLZ_MERGE_PASSTHRU)
2067     MAKE_CASE(AArch64ISD::CTPOP_MERGE_PASSTHRU)
2068     MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU)
2069     MAKE_CASE(AArch64ISD::INDEX_VECTOR)
2070     MAKE_CASE(AArch64ISD::UABD)
2071     MAKE_CASE(AArch64ISD::SABD)
2072     MAKE_CASE(AArch64ISD::CALL_RVMARKER)
2073   }
2074 #undef MAKE_CASE
2075   return nullptr;
2076 }
2077 
2078 MachineBasicBlock *
2079 AArch64TargetLowering::EmitF128CSEL(MachineInstr &MI,
2080                                     MachineBasicBlock *MBB) const {
2081   // We materialise the F128CSEL pseudo-instruction as some control flow and a
2082   // phi node:
2083 
2084   // OrigBB:
2085   //     [... previous instrs leading to comparison ...]
2086   //     b.ne TrueBB
2087   //     b EndBB
2088   // TrueBB:
2089   //     ; Fallthrough
2090   // EndBB:
2091   //     Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB]
2092 
2093   MachineFunction *MF = MBB->getParent();
2094   const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2095   const BasicBlock *LLVM_BB = MBB->getBasicBlock();
2096   DebugLoc DL = MI.getDebugLoc();
2097   MachineFunction::iterator It = ++MBB->getIterator();
2098 
2099   Register DestReg = MI.getOperand(0).getReg();
2100   Register IfTrueReg = MI.getOperand(1).getReg();
2101   Register IfFalseReg = MI.getOperand(2).getReg();
2102   unsigned CondCode = MI.getOperand(3).getImm();
2103   bool NZCVKilled = MI.getOperand(4).isKill();
2104 
2105   MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB);
2106   MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB);
2107   MF->insert(It, TrueBB);
2108   MF->insert(It, EndBB);
2109 
2110   // Transfer rest of current basic-block to EndBB
2111   EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)),
2112                 MBB->end());
2113   EndBB->transferSuccessorsAndUpdatePHIs(MBB);
2114 
2115   BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
2116   BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
2117   MBB->addSuccessor(TrueBB);
2118   MBB->addSuccessor(EndBB);
2119 
2120   // TrueBB falls through to the end.
2121   TrueBB->addSuccessor(EndBB);
2122 
2123   if (!NZCVKilled) {
2124     TrueBB->addLiveIn(AArch64::NZCV);
2125     EndBB->addLiveIn(AArch64::NZCV);
2126   }
2127 
2128   BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
2129       .addReg(IfTrueReg)
2130       .addMBB(TrueBB)
2131       .addReg(IfFalseReg)
2132       .addMBB(MBB);
2133 
2134   MI.eraseFromParent();
2135   return EndBB;
2136 }
2137 
2138 MachineBasicBlock *AArch64TargetLowering::EmitLoweredCatchRet(
2139        MachineInstr &MI, MachineBasicBlock *BB) const {
2140   assert(!isAsynchronousEHPersonality(classifyEHPersonality(
2141              BB->getParent()->getFunction().getPersonalityFn())) &&
2142          "SEH does not use catchret!");
2143   return BB;
2144 }
2145 
2146 MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter(
2147     MachineInstr &MI, MachineBasicBlock *BB) const {
2148   switch (MI.getOpcode()) {
2149   default:
2150 #ifndef NDEBUG
2151     MI.dump();
2152 #endif
2153     llvm_unreachable("Unexpected instruction for custom inserter!");
2154 
2155   case AArch64::F128CSEL:
2156     return EmitF128CSEL(MI, BB);
2157 
2158   case TargetOpcode::STACKMAP:
2159   case TargetOpcode::PATCHPOINT:
2160   case TargetOpcode::STATEPOINT:
2161     return emitPatchPoint(MI, BB);
2162 
2163   case AArch64::CATCHRET:
2164     return EmitLoweredCatchRet(MI, BB);
2165   }
2166 }
2167 
2168 //===----------------------------------------------------------------------===//
2169 // AArch64 Lowering private implementation.
2170 //===----------------------------------------------------------------------===//
2171 
2172 //===----------------------------------------------------------------------===//
2173 // Lowering Code
2174 //===----------------------------------------------------------------------===//
2175 
2176 /// isZerosVector - Check whether SDNode N is a zero-filled vector.
2177 static bool isZerosVector(const SDNode *N) {
2178   // Look through a bit convert.
2179   while (N->getOpcode() == ISD::BITCAST)
2180     N = N->getOperand(0).getNode();
2181 
2182   if (ISD::isConstantSplatVectorAllZeros(N))
2183     return true;
2184 
2185   if (N->getOpcode() != AArch64ISD::DUP)
2186     return false;
2187 
2188   auto Opnd0 = N->getOperand(0);
2189   auto *CINT = dyn_cast<ConstantSDNode>(Opnd0);
2190   auto *CFP = dyn_cast<ConstantFPSDNode>(Opnd0);
2191   return (CINT && CINT->isNullValue()) || (CFP && CFP->isZero());
2192 }
2193 
2194 /// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
2195 /// CC
2196 static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
2197   switch (CC) {
2198   default:
2199     llvm_unreachable("Unknown condition code!");
2200   case ISD::SETNE:
2201     return AArch64CC::NE;
2202   case ISD::SETEQ:
2203     return AArch64CC::EQ;
2204   case ISD::SETGT:
2205     return AArch64CC::GT;
2206   case ISD::SETGE:
2207     return AArch64CC::GE;
2208   case ISD::SETLT:
2209     return AArch64CC::LT;
2210   case ISD::SETLE:
2211     return AArch64CC::LE;
2212   case ISD::SETUGT:
2213     return AArch64CC::HI;
2214   case ISD::SETUGE:
2215     return AArch64CC::HS;
2216   case ISD::SETULT:
2217     return AArch64CC::LO;
2218   case ISD::SETULE:
2219     return AArch64CC::LS;
2220   }
2221 }
2222 
2223 /// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
2224 static void changeFPCCToAArch64CC(ISD::CondCode CC,
2225                                   AArch64CC::CondCode &CondCode,
2226                                   AArch64CC::CondCode &CondCode2) {
2227   CondCode2 = AArch64CC::AL;
2228   switch (CC) {
2229   default:
2230     llvm_unreachable("Unknown FP condition!");
2231   case ISD::SETEQ:
2232   case ISD::SETOEQ:
2233     CondCode = AArch64CC::EQ;
2234     break;
2235   case ISD::SETGT:
2236   case ISD::SETOGT:
2237     CondCode = AArch64CC::GT;
2238     break;
2239   case ISD::SETGE:
2240   case ISD::SETOGE:
2241     CondCode = AArch64CC::GE;
2242     break;
2243   case ISD::SETOLT:
2244     CondCode = AArch64CC::MI;
2245     break;
2246   case ISD::SETOLE:
2247     CondCode = AArch64CC::LS;
2248     break;
2249   case ISD::SETONE:
2250     CondCode = AArch64CC::MI;
2251     CondCode2 = AArch64CC::GT;
2252     break;
2253   case ISD::SETO:
2254     CondCode = AArch64CC::VC;
2255     break;
2256   case ISD::SETUO:
2257     CondCode = AArch64CC::VS;
2258     break;
2259   case ISD::SETUEQ:
2260     CondCode = AArch64CC::EQ;
2261     CondCode2 = AArch64CC::VS;
2262     break;
2263   case ISD::SETUGT:
2264     CondCode = AArch64CC::HI;
2265     break;
2266   case ISD::SETUGE:
2267     CondCode = AArch64CC::PL;
2268     break;
2269   case ISD::SETLT:
2270   case ISD::SETULT:
2271     CondCode = AArch64CC::LT;
2272     break;
2273   case ISD::SETLE:
2274   case ISD::SETULE:
2275     CondCode = AArch64CC::LE;
2276     break;
2277   case ISD::SETNE:
2278   case ISD::SETUNE:
2279     CondCode = AArch64CC::NE;
2280     break;
2281   }
2282 }
2283 
2284 /// Convert a DAG fp condition code to an AArch64 CC.
2285 /// This differs from changeFPCCToAArch64CC in that it returns cond codes that
2286 /// should be AND'ed instead of OR'ed.
2287 static void changeFPCCToANDAArch64CC(ISD::CondCode CC,
2288                                      AArch64CC::CondCode &CondCode,
2289                                      AArch64CC::CondCode &CondCode2) {
2290   CondCode2 = AArch64CC::AL;
2291   switch (CC) {
2292   default:
2293     changeFPCCToAArch64CC(CC, CondCode, CondCode2);
2294     assert(CondCode2 == AArch64CC::AL);
2295     break;
2296   case ISD::SETONE:
2297     // (a one b)
2298     // == ((a olt b) || (a ogt b))
2299     // == ((a ord b) && (a une b))
2300     CondCode = AArch64CC::VC;
2301     CondCode2 = AArch64CC::NE;
2302     break;
2303   case ISD::SETUEQ:
2304     // (a ueq b)
2305     // == ((a uno b) || (a oeq b))
2306     // == ((a ule b) && (a uge b))
2307     CondCode = AArch64CC::PL;
2308     CondCode2 = AArch64CC::LE;
2309     break;
2310   }
2311 }
2312 
2313 /// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
2314 /// CC usable with the vector instructions. Fewer operations are available
2315 /// without a real NZCV register, so we have to use less efficient combinations
2316 /// to get the same effect.
2317 static void changeVectorFPCCToAArch64CC(ISD::CondCode CC,
2318                                         AArch64CC::CondCode &CondCode,
2319                                         AArch64CC::CondCode &CondCode2,
2320                                         bool &Invert) {
2321   Invert = false;
2322   switch (CC) {
2323   default:
2324     // Mostly the scalar mappings work fine.
2325     changeFPCCToAArch64CC(CC, CondCode, CondCode2);
2326     break;
2327   case ISD::SETUO:
2328     Invert = true;
2329     LLVM_FALLTHROUGH;
2330   case ISD::SETO:
2331     CondCode = AArch64CC::MI;
2332     CondCode2 = AArch64CC::GE;
2333     break;
2334   case ISD::SETUEQ:
2335   case ISD::SETULT:
2336   case ISD::SETULE:
2337   case ISD::SETUGT:
2338   case ISD::SETUGE:
2339     // All of the compare-mask comparisons are ordered, but we can switch
2340     // between the two by a double inversion. E.g. ULE == !OGT.
2341     Invert = true;
2342     changeFPCCToAArch64CC(getSetCCInverse(CC, /* FP inverse */ MVT::f32),
2343                           CondCode, CondCode2);
2344     break;
2345   }
2346 }
2347 
2348 static bool isLegalArithImmed(uint64_t C) {
2349   // Matches AArch64DAGToDAGISel::SelectArithImmed().
2350   bool IsLegal = (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0);
2351   LLVM_DEBUG(dbgs() << "Is imm " << C
2352                     << " legal: " << (IsLegal ? "yes\n" : "no\n"));
2353   return IsLegal;
2354 }
2355 
2356 // Can a (CMP op1, (sub 0, op2) be turned into a CMN instruction on
2357 // the grounds that "op1 - (-op2) == op1 + op2" ? Not always, the C and V flags
2358 // can be set differently by this operation. It comes down to whether
2359 // "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then
2360 // everything is fine. If not then the optimization is wrong. Thus general
2361 // comparisons are only valid if op2 != 0.
2362 //
2363 // So, finally, the only LLVM-native comparisons that don't mention C and V
2364 // are SETEQ and SETNE. They're the only ones we can safely use CMN for in
2365 // the absence of information about op2.
2366 static bool isCMN(SDValue Op, ISD::CondCode CC) {
2367   return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
2368          (CC == ISD::SETEQ || CC == ISD::SETNE);
2369 }
2370 
2371 static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
2372                                       SelectionDAG &DAG, SDValue Chain,
2373                                       bool IsSignaling) {
2374   EVT VT = LHS.getValueType();
2375   assert(VT != MVT::f128);
2376   assert(VT != MVT::f16 && "Lowering of strict fp16 not yet implemented");
2377   unsigned Opcode =
2378       IsSignaling ? AArch64ISD::STRICT_FCMPE : AArch64ISD::STRICT_FCMP;
2379   return DAG.getNode(Opcode, dl, {VT, MVT::Other}, {Chain, LHS, RHS});
2380 }
2381 
2382 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2383                               const SDLoc &dl, SelectionDAG &DAG) {
2384   EVT VT = LHS.getValueType();
2385   const bool FullFP16 =
2386     static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
2387 
2388   if (VT.isFloatingPoint()) {
2389     assert(VT != MVT::f128);
2390     if (VT == MVT::f16 && !FullFP16) {
2391       LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
2392       RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
2393       VT = MVT::f32;
2394     }
2395     return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS);
2396   }
2397 
2398   // The CMP instruction is just an alias for SUBS, and representing it as
2399   // SUBS means that it's possible to get CSE with subtract operations.
2400   // A later phase can perform the optimization of setting the destination
2401   // register to WZR/XZR if it ends up being unused.
2402   unsigned Opcode = AArch64ISD::SUBS;
2403 
2404   if (isCMN(RHS, CC)) {
2405     // Can we combine a (CMP op1, (sub 0, op2) into a CMN instruction ?
2406     Opcode = AArch64ISD::ADDS;
2407     RHS = RHS.getOperand(1);
2408   } else if (isCMN(LHS, CC)) {
2409     // As we are looking for EQ/NE compares, the operands can be commuted ; can
2410     // we combine a (CMP (sub 0, op1), op2) into a CMN instruction ?
2411     Opcode = AArch64ISD::ADDS;
2412     LHS = LHS.getOperand(1);
2413   } else if (isNullConstant(RHS) && !isUnsignedIntSetCC(CC)) {
2414     if (LHS.getOpcode() == ISD::AND) {
2415       // Similarly, (CMP (and X, Y), 0) can be implemented with a TST
2416       // (a.k.a. ANDS) except that the flags are only guaranteed to work for one
2417       // of the signed comparisons.
2418       const SDValue ANDSNode = DAG.getNode(AArch64ISD::ANDS, dl,
2419                                            DAG.getVTList(VT, MVT_CC),
2420                                            LHS.getOperand(0),
2421                                            LHS.getOperand(1));
2422       // Replace all users of (and X, Y) with newly generated (ands X, Y)
2423       DAG.ReplaceAllUsesWith(LHS, ANDSNode);
2424       return ANDSNode.getValue(1);
2425     } else if (LHS.getOpcode() == AArch64ISD::ANDS) {
2426       // Use result of ANDS
2427       return LHS.getValue(1);
2428     }
2429   }
2430 
2431   return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS)
2432       .getValue(1);
2433 }
2434 
2435 /// \defgroup AArch64CCMP CMP;CCMP matching
2436 ///
2437 /// These functions deal with the formation of CMP;CCMP;... sequences.
2438 /// The CCMP/CCMN/FCCMP/FCCMPE instructions allow the conditional execution of
2439 /// a comparison. They set the NZCV flags to a predefined value if their
2440 /// predicate is false. This allows to express arbitrary conjunctions, for
2441 /// example "cmp 0 (and (setCA (cmp A)) (setCB (cmp B)))"
2442 /// expressed as:
2443 ///   cmp A
2444 ///   ccmp B, inv(CB), CA
2445 ///   check for CB flags
2446 ///
2447 /// This naturally lets us implement chains of AND operations with SETCC
2448 /// operands. And we can even implement some other situations by transforming
2449 /// them:
2450 ///   - We can implement (NEG SETCC) i.e. negating a single comparison by
2451 ///     negating the flags used in a CCMP/FCCMP operations.
2452 ///   - We can negate the result of a whole chain of CMP/CCMP/FCCMP operations
2453 ///     by negating the flags we test for afterwards. i.e.
2454 ///     NEG (CMP CCMP CCCMP ...) can be implemented.
2455 ///   - Note that we can only ever negate all previously processed results.
2456 ///     What we can not implement by flipping the flags to test is a negation
2457 ///     of two sub-trees (because the negation affects all sub-trees emitted so
2458 ///     far, so the 2nd sub-tree we emit would also affect the first).
2459 /// With those tools we can implement some OR operations:
2460 ///   - (OR (SETCC A) (SETCC B)) can be implemented via:
2461 ///     NEG (AND (NEG (SETCC A)) (NEG (SETCC B)))
2462 ///   - After transforming OR to NEG/AND combinations we may be able to use NEG
2463 ///     elimination rules from earlier to implement the whole thing as a
2464 ///     CCMP/FCCMP chain.
2465 ///
2466 /// As complete example:
2467 ///     or (or (setCA (cmp A)) (setCB (cmp B)))
2468 ///        (and (setCC (cmp C)) (setCD (cmp D)))"
2469 /// can be reassociated to:
2470 ///     or (and (setCC (cmp C)) setCD (cmp D))
2471 //         (or (setCA (cmp A)) (setCB (cmp B)))
2472 /// can be transformed to:
2473 ///     not (and (not (and (setCC (cmp C)) (setCD (cmp D))))
2474 ///              (and (not (setCA (cmp A)) (not (setCB (cmp B))))))"
2475 /// which can be implemented as:
2476 ///   cmp C
2477 ///   ccmp D, inv(CD), CC
2478 ///   ccmp A, CA, inv(CD)
2479 ///   ccmp B, CB, inv(CA)
2480 ///   check for CB flags
2481 ///
2482 /// A counterexample is "or (and A B) (and C D)" which translates to
2483 /// not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we
2484 /// can only implement 1 of the inner (not) operations, but not both!
2485 /// @{
2486 
2487 /// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
2488 static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
2489                                          ISD::CondCode CC, SDValue CCOp,
2490                                          AArch64CC::CondCode Predicate,
2491                                          AArch64CC::CondCode OutCC,
2492                                          const SDLoc &DL, SelectionDAG &DAG) {
2493   unsigned Opcode = 0;
2494   const bool FullFP16 =
2495     static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
2496 
2497   if (LHS.getValueType().isFloatingPoint()) {
2498     assert(LHS.getValueType() != MVT::f128);
2499     if (LHS.getValueType() == MVT::f16 && !FullFP16) {
2500       LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
2501       RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
2502     }
2503     Opcode = AArch64ISD::FCCMP;
2504   } else if (RHS.getOpcode() == ISD::SUB) {
2505     SDValue SubOp0 = RHS.getOperand(0);
2506     if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
2507       // See emitComparison() on why we can only do this for SETEQ and SETNE.
2508       Opcode = AArch64ISD::CCMN;
2509       RHS = RHS.getOperand(1);
2510     }
2511   }
2512   if (Opcode == 0)
2513     Opcode = AArch64ISD::CCMP;
2514 
2515   SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
2516   AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC);
2517   unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
2518   SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
2519   return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp);
2520 }
2521 
2522 /// Returns true if @p Val is a tree of AND/OR/SETCC operations that can be
2523 /// expressed as a conjunction. See \ref AArch64CCMP.
2524 /// \param CanNegate    Set to true if we can negate the whole sub-tree just by
2525 ///                     changing the conditions on the SETCC tests.
2526 ///                     (this means we can call emitConjunctionRec() with
2527 ///                      Negate==true on this sub-tree)
2528 /// \param MustBeFirst  Set to true if this subtree needs to be negated and we
2529 ///                     cannot do the negation naturally. We are required to
2530 ///                     emit the subtree first in this case.
2531 /// \param WillNegate   Is true if are called when the result of this
2532 ///                     subexpression must be negated. This happens when the
2533 ///                     outer expression is an OR. We can use this fact to know
2534 ///                     that we have a double negation (or (or ...) ...) that
2535 ///                     can be implemented for free.
2536 static bool canEmitConjunction(const SDValue Val, bool &CanNegate,
2537                                bool &MustBeFirst, bool WillNegate,
2538                                unsigned Depth = 0) {
2539   if (!Val.hasOneUse())
2540     return false;
2541   unsigned Opcode = Val->getOpcode();
2542   if (Opcode == ISD::SETCC) {
2543     if (Val->getOperand(0).getValueType() == MVT::f128)
2544       return false;
2545     CanNegate = true;
2546     MustBeFirst = false;
2547     return true;
2548   }
2549   // Protect against exponential runtime and stack overflow.
2550   if (Depth > 6)
2551     return false;
2552   if (Opcode == ISD::AND || Opcode == ISD::OR) {
2553     bool IsOR = Opcode == ISD::OR;
2554     SDValue O0 = Val->getOperand(0);
2555     SDValue O1 = Val->getOperand(1);
2556     bool CanNegateL;
2557     bool MustBeFirstL;
2558     if (!canEmitConjunction(O0, CanNegateL, MustBeFirstL, IsOR, Depth+1))
2559       return false;
2560     bool CanNegateR;
2561     bool MustBeFirstR;
2562     if (!canEmitConjunction(O1, CanNegateR, MustBeFirstR, IsOR, Depth+1))
2563       return false;
2564 
2565     if (MustBeFirstL && MustBeFirstR)
2566       return false;
2567 
2568     if (IsOR) {
2569       // For an OR expression we need to be able to naturally negate at least
2570       // one side or we cannot do the transformation at all.
2571       if (!CanNegateL && !CanNegateR)
2572         return false;
2573       // If we the result of the OR will be negated and we can naturally negate
2574       // the leafs, then this sub-tree as a whole negates naturally.
2575       CanNegate = WillNegate && CanNegateL && CanNegateR;
2576       // If we cannot naturally negate the whole sub-tree, then this must be
2577       // emitted first.
2578       MustBeFirst = !CanNegate;
2579     } else {
2580       assert(Opcode == ISD::AND && "Must be OR or AND");
2581       // We cannot naturally negate an AND operation.
2582       CanNegate = false;
2583       MustBeFirst = MustBeFirstL || MustBeFirstR;
2584     }
2585     return true;
2586   }
2587   return false;
2588 }
2589 
2590 /// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain
2591 /// of CCMP/CFCMP ops. See @ref AArch64CCMP.
2592 /// Tries to transform the given i1 producing node @p Val to a series compare
2593 /// and conditional compare operations. @returns an NZCV flags producing node
2594 /// and sets @p OutCC to the flags that should be tested or returns SDValue() if
2595 /// transformation was not possible.
2596 /// \p Negate is true if we want this sub-tree being negated just by changing
2597 /// SETCC conditions.
2598 static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
2599     AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
2600     AArch64CC::CondCode Predicate) {
2601   // We're at a tree leaf, produce a conditional comparison operation.
2602   unsigned Opcode = Val->getOpcode();
2603   if (Opcode == ISD::SETCC) {
2604     SDValue LHS = Val->getOperand(0);
2605     SDValue RHS = Val->getOperand(1);
2606     ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get();
2607     bool isInteger = LHS.getValueType().isInteger();
2608     if (Negate)
2609       CC = getSetCCInverse(CC, LHS.getValueType());
2610     SDLoc DL(Val);
2611     // Determine OutCC and handle FP special case.
2612     if (isInteger) {
2613       OutCC = changeIntCCToAArch64CC(CC);
2614     } else {
2615       assert(LHS.getValueType().isFloatingPoint());
2616       AArch64CC::CondCode ExtraCC;
2617       changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC);
2618       // Some floating point conditions can't be tested with a single condition
2619       // code. Construct an additional comparison in this case.
2620       if (ExtraCC != AArch64CC::AL) {
2621         SDValue ExtraCmp;
2622         if (!CCOp.getNode())
2623           ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG);
2624         else
2625           ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate,
2626                                                ExtraCC, DL, DAG);
2627         CCOp = ExtraCmp;
2628         Predicate = ExtraCC;
2629       }
2630     }
2631 
2632     // Produce a normal comparison if we are first in the chain
2633     if (!CCOp)
2634       return emitComparison(LHS, RHS, CC, DL, DAG);
2635     // Otherwise produce a ccmp.
2636     return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL,
2637                                      DAG);
2638   }
2639   assert(Val->hasOneUse() && "Valid conjunction/disjunction tree");
2640 
2641   bool IsOR = Opcode == ISD::OR;
2642 
2643   SDValue LHS = Val->getOperand(0);
2644   bool CanNegateL;
2645   bool MustBeFirstL;
2646   bool ValidL = canEmitConjunction(LHS, CanNegateL, MustBeFirstL, IsOR);
2647   assert(ValidL && "Valid conjunction/disjunction tree");
2648   (void)ValidL;
2649 
2650   SDValue RHS = Val->getOperand(1);
2651   bool CanNegateR;
2652   bool MustBeFirstR;
2653   bool ValidR = canEmitConjunction(RHS, CanNegateR, MustBeFirstR, IsOR);
2654   assert(ValidR && "Valid conjunction/disjunction tree");
2655   (void)ValidR;
2656 
2657   // Swap sub-tree that must come first to the right side.
2658   if (MustBeFirstL) {
2659     assert(!MustBeFirstR && "Valid conjunction/disjunction tree");
2660     std::swap(LHS, RHS);
2661     std::swap(CanNegateL, CanNegateR);
2662     std::swap(MustBeFirstL, MustBeFirstR);
2663   }
2664 
2665   bool NegateR;
2666   bool NegateAfterR;
2667   bool NegateL;
2668   bool NegateAfterAll;
2669   if (Opcode == ISD::OR) {
2670     // Swap the sub-tree that we can negate naturally to the left.
2671     if (!CanNegateL) {
2672       assert(CanNegateR && "at least one side must be negatable");
2673       assert(!MustBeFirstR && "invalid conjunction/disjunction tree");
2674       assert(!Negate);
2675       std::swap(LHS, RHS);
2676       NegateR = false;
2677       NegateAfterR = true;
2678     } else {
2679       // Negate the left sub-tree if possible, otherwise negate the result.
2680       NegateR = CanNegateR;
2681       NegateAfterR = !CanNegateR;
2682     }
2683     NegateL = true;
2684     NegateAfterAll = !Negate;
2685   } else {
2686     assert(Opcode == ISD::AND && "Valid conjunction/disjunction tree");
2687     assert(!Negate && "Valid conjunction/disjunction tree");
2688 
2689     NegateL = false;
2690     NegateR = false;
2691     NegateAfterR = false;
2692     NegateAfterAll = false;
2693   }
2694 
2695   // Emit sub-trees.
2696   AArch64CC::CondCode RHSCC;
2697   SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
2698   if (NegateAfterR)
2699     RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
2700   SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
2701   if (NegateAfterAll)
2702     OutCC = AArch64CC::getInvertedCondCode(OutCC);
2703   return CmpL;
2704 }
2705 
2706 /// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
2707 /// In some cases this is even possible with OR operations in the expression.
2708 /// See \ref AArch64CCMP.
2709 /// \see emitConjunctionRec().
2710 static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
2711                                AArch64CC::CondCode &OutCC) {
2712   bool DummyCanNegate;
2713   bool DummyMustBeFirst;
2714   if (!canEmitConjunction(Val, DummyCanNegate, DummyMustBeFirst, false))
2715     return SDValue();
2716 
2717   return emitConjunctionRec(DAG, Val, OutCC, false, SDValue(), AArch64CC::AL);
2718 }
2719 
2720 /// @}
2721 
2722 /// Returns how profitable it is to fold a comparison's operand's shift and/or
2723 /// extension operations.
2724 static unsigned getCmpOperandFoldingProfit(SDValue Op) {
2725   auto isSupportedExtend = [&](SDValue V) {
2726     if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
2727       return true;
2728 
2729     if (V.getOpcode() == ISD::AND)
2730       if (ConstantSDNode *MaskCst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2731         uint64_t Mask = MaskCst->getZExtValue();
2732         return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
2733       }
2734 
2735     return false;
2736   };
2737 
2738   if (!Op.hasOneUse())
2739     return 0;
2740 
2741   if (isSupportedExtend(Op))
2742     return 1;
2743 
2744   unsigned Opc = Op.getOpcode();
2745   if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
2746     if (ConstantSDNode *ShiftCst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2747       uint64_t Shift = ShiftCst->getZExtValue();
2748       if (isSupportedExtend(Op.getOperand(0)))
2749         return (Shift <= 4) ? 2 : 1;
2750       EVT VT = Op.getValueType();
2751       if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63))
2752         return 1;
2753     }
2754 
2755   return 0;
2756 }
2757 
2758 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2759                              SDValue &AArch64cc, SelectionDAG &DAG,
2760                              const SDLoc &dl) {
2761   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
2762     EVT VT = RHS.getValueType();
2763     uint64_t C = RHSC->getZExtValue();
2764     if (!isLegalArithImmed(C)) {
2765       // Constant does not fit, try adjusting it by one?
2766       switch (CC) {
2767       default:
2768         break;
2769       case ISD::SETLT:
2770       case ISD::SETGE:
2771         if ((VT == MVT::i32 && C != 0x80000000 &&
2772              isLegalArithImmed((uint32_t)(C - 1))) ||
2773             (VT == MVT::i64 && C != 0x80000000ULL &&
2774              isLegalArithImmed(C - 1ULL))) {
2775           CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
2776           C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
2777           RHS = DAG.getConstant(C, dl, VT);
2778         }
2779         break;
2780       case ISD::SETULT:
2781       case ISD::SETUGE:
2782         if ((VT == MVT::i32 && C != 0 &&
2783              isLegalArithImmed((uint32_t)(C - 1))) ||
2784             (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
2785           CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
2786           C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
2787           RHS = DAG.getConstant(C, dl, VT);
2788         }
2789         break;
2790       case ISD::SETLE:
2791       case ISD::SETGT:
2792         if ((VT == MVT::i32 && C != INT32_MAX &&
2793              isLegalArithImmed((uint32_t)(C + 1))) ||
2794             (VT == MVT::i64 && C != INT64_MAX &&
2795              isLegalArithImmed(C + 1ULL))) {
2796           CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
2797           C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
2798           RHS = DAG.getConstant(C, dl, VT);
2799         }
2800         break;
2801       case ISD::SETULE:
2802       case ISD::SETUGT:
2803         if ((VT == MVT::i32 && C != UINT32_MAX &&
2804              isLegalArithImmed((uint32_t)(C + 1))) ||
2805             (VT == MVT::i64 && C != UINT64_MAX &&
2806              isLegalArithImmed(C + 1ULL))) {
2807           CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
2808           C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
2809           RHS = DAG.getConstant(C, dl, VT);
2810         }
2811         break;
2812       }
2813     }
2814   }
2815 
2816   // Comparisons are canonicalized so that the RHS operand is simpler than the
2817   // LHS one, the extreme case being when RHS is an immediate. However, AArch64
2818   // can fold some shift+extend operations on the RHS operand, so swap the
2819   // operands if that can be done.
2820   //
2821   // For example:
2822   //    lsl     w13, w11, #1
2823   //    cmp     w13, w12
2824   // can be turned into:
2825   //    cmp     w12, w11, lsl #1
2826   if (!isa<ConstantSDNode>(RHS) ||
2827       !isLegalArithImmed(cast<ConstantSDNode>(RHS)->getZExtValue())) {
2828     SDValue TheLHS = isCMN(LHS, CC) ? LHS.getOperand(1) : LHS;
2829 
2830     if (getCmpOperandFoldingProfit(TheLHS) > getCmpOperandFoldingProfit(RHS)) {
2831       std::swap(LHS, RHS);
2832       CC = ISD::getSetCCSwappedOperands(CC);
2833     }
2834   }
2835 
2836   SDValue Cmp;
2837   AArch64CC::CondCode AArch64CC;
2838   if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) {
2839     const ConstantSDNode *RHSC = cast<ConstantSDNode>(RHS);
2840 
2841     // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095.
2842     // For the i8 operand, the largest immediate is 255, so this can be easily
2843     // encoded in the compare instruction. For the i16 operand, however, the
2844     // largest immediate cannot be encoded in the compare.
2845     // Therefore, use a sign extending load and cmn to avoid materializing the
2846     // -1 constant. For example,
2847     // movz w1, #65535
2848     // ldrh w0, [x0, #0]
2849     // cmp w0, w1
2850     // >
2851     // ldrsh w0, [x0, #0]
2852     // cmn w0, #1
2853     // Fundamental, we're relying on the property that (zext LHS) == (zext RHS)
2854     // if and only if (sext LHS) == (sext RHS). The checks are in place to
2855     // ensure both the LHS and RHS are truly zero extended and to make sure the
2856     // transformation is profitable.
2857     if ((RHSC->getZExtValue() >> 16 == 0) && isa<LoadSDNode>(LHS) &&
2858         cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD &&
2859         cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 &&
2860         LHS.getNode()->hasNUsesOfValue(1, 0)) {
2861       int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue();
2862       if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) {
2863         SDValue SExt =
2864             DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
2865                         DAG.getValueType(MVT::i16));
2866         Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl,
2867                                                    RHS.getValueType()),
2868                              CC, dl, DAG);
2869         AArch64CC = changeIntCCToAArch64CC(CC);
2870       }
2871     }
2872 
2873     if (!Cmp && (RHSC->isNullValue() || RHSC->isOne())) {
2874       if ((Cmp = emitConjunction(DAG, LHS, AArch64CC))) {
2875         if ((CC == ISD::SETNE) ^ RHSC->isNullValue())
2876           AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC);
2877       }
2878     }
2879   }
2880 
2881   if (!Cmp) {
2882     Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
2883     AArch64CC = changeIntCCToAArch64CC(CC);
2884   }
2885   AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC);
2886   return Cmp;
2887 }
2888 
2889 static std::pair<SDValue, SDValue>
2890 getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
2891   assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) &&
2892          "Unsupported value type");
2893   SDValue Value, Overflow;
2894   SDLoc DL(Op);
2895   SDValue LHS = Op.getOperand(0);
2896   SDValue RHS = Op.getOperand(1);
2897   unsigned Opc = 0;
2898   switch (Op.getOpcode()) {
2899   default:
2900     llvm_unreachable("Unknown overflow instruction!");
2901   case ISD::SADDO:
2902     Opc = AArch64ISD::ADDS;
2903     CC = AArch64CC::VS;
2904     break;
2905   case ISD::UADDO:
2906     Opc = AArch64ISD::ADDS;
2907     CC = AArch64CC::HS;
2908     break;
2909   case ISD::SSUBO:
2910     Opc = AArch64ISD::SUBS;
2911     CC = AArch64CC::VS;
2912     break;
2913   case ISD::USUBO:
2914     Opc = AArch64ISD::SUBS;
2915     CC = AArch64CC::LO;
2916     break;
2917   // Multiply needs a little bit extra work.
2918   case ISD::SMULO:
2919   case ISD::UMULO: {
2920     CC = AArch64CC::NE;
2921     bool IsSigned = Op.getOpcode() == ISD::SMULO;
2922     if (Op.getValueType() == MVT::i32) {
2923       unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
2924       // For a 32 bit multiply with overflow check we want the instruction
2925       // selector to generate a widening multiply (SMADDL/UMADDL). For that we
2926       // need to generate the following pattern:
2927       // (i64 add 0, (i64 mul (i64 sext|zext i32 %a), (i64 sext|zext i32 %b))
2928       LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS);
2929       RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS);
2930       SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2931       SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
2932                                 DAG.getConstant(0, DL, MVT::i64));
2933       // On AArch64 the upper 32 bits are always zero extended for a 32 bit
2934       // operation. We need to clear out the upper 32 bits, because we used a
2935       // widening multiply that wrote all 64 bits. In the end this should be a
2936       // noop.
2937       Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add);
2938       if (IsSigned) {
2939         // The signed overflow check requires more than just a simple check for
2940         // any bit set in the upper 32 bits of the result. These bits could be
2941         // just the sign bits of a negative number. To perform the overflow
2942         // check we have to arithmetic shift right the 32nd bit of the result by
2943         // 31 bits. Then we compare the result to the upper 32 bits.
2944         SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
2945                                         DAG.getConstant(32, DL, MVT::i64));
2946         UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits);
2947         SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
2948                                         DAG.getConstant(31, DL, MVT::i64));
2949         // It is important that LowerBits is last, otherwise the arithmetic
2950         // shift will not be folded into the compare (SUBS).
2951         SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
2952         Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2953                        .getValue(1);
2954       } else {
2955         // The overflow check for unsigned multiply is easy. We only need to
2956         // check if any of the upper 32 bits are set. This can be done with a
2957         // CMP (shifted register). For that we need to generate the following
2958         // pattern:
2959         // (i64 AArch64ISD::SUBS i64 0, (i64 srl i64 %Mul, i64 32)
2960         SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2961                                         DAG.getConstant(32, DL, MVT::i64));
2962         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2963         Overflow =
2964             DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2965                         DAG.getConstant(0, DL, MVT::i64),
2966                         UpperBits).getValue(1);
2967       }
2968       break;
2969     }
2970     assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type");
2971     // For the 64 bit multiply
2972     Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2973     if (IsSigned) {
2974       SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
2975       SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
2976                                       DAG.getConstant(63, DL, MVT::i64));
2977       // It is important that LowerBits is last, otherwise the arithmetic
2978       // shift will not be folded into the compare (SUBS).
2979       SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2980       Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2981                      .getValue(1);
2982     } else {
2983       SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
2984       SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2985       Overflow =
2986           DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2987                       DAG.getConstant(0, DL, MVT::i64),
2988                       UpperBits).getValue(1);
2989     }
2990     break;
2991   }
2992   } // switch (...)
2993 
2994   if (Opc) {
2995     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
2996 
2997     // Emit the AArch64 operation with overflow check.
2998     Value = DAG.getNode(Opc, DL, VTs, LHS, RHS);
2999     Overflow = Value.getValue(1);
3000   }
3001   return std::make_pair(Value, Overflow);
3002 }
3003 
3004 SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
3005   if (useSVEForFixedLengthVectorVT(Op.getValueType()))
3006     return LowerToScalableOp(Op, DAG);
3007 
3008   SDValue Sel = Op.getOperand(0);
3009   SDValue Other = Op.getOperand(1);
3010   SDLoc dl(Sel);
3011 
3012   // If the operand is an overflow checking operation, invert the condition
3013   // code and kill the Not operation. I.e., transform:
3014   // (xor (overflow_op_bool, 1))
3015   //   -->
3016   // (csel 1, 0, invert(cc), overflow_op_bool)
3017   // ... which later gets transformed to just a cset instruction with an
3018   // inverted condition code, rather than a cset + eor sequence.
3019   if (isOneConstant(Other) && ISD::isOverflowIntrOpRes(Sel)) {
3020     // Only lower legal XALUO ops.
3021     if (!DAG.getTargetLoweringInfo().isTypeLegal(Sel->getValueType(0)))
3022       return SDValue();
3023 
3024     SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
3025     SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
3026     AArch64CC::CondCode CC;
3027     SDValue Value, Overflow;
3028     std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Sel.getValue(0), DAG);
3029     SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
3030     return DAG.getNode(AArch64ISD::CSEL, dl, Op.getValueType(), TVal, FVal,
3031                        CCVal, Overflow);
3032   }
3033   // If neither operand is a SELECT_CC, give up.
3034   if (Sel.getOpcode() != ISD::SELECT_CC)
3035     std::swap(Sel, Other);
3036   if (Sel.getOpcode() != ISD::SELECT_CC)
3037     return Op;
3038 
3039   // The folding we want to perform is:
3040   // (xor x, (select_cc a, b, cc, 0, -1) )
3041   //   -->
3042   // (csel x, (xor x, -1), cc ...)
3043   //
3044   // The latter will get matched to a CSINV instruction.
3045 
3046   ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get();
3047   SDValue LHS = Sel.getOperand(0);
3048   SDValue RHS = Sel.getOperand(1);
3049   SDValue TVal = Sel.getOperand(2);
3050   SDValue FVal = Sel.getOperand(3);
3051 
3052   // FIXME: This could be generalized to non-integer comparisons.
3053   if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
3054     return Op;
3055 
3056   ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
3057   ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
3058 
3059   // The values aren't constants, this isn't the pattern we're looking for.
3060   if (!CFVal || !CTVal)
3061     return Op;
3062 
3063   // We can commute the SELECT_CC by inverting the condition.  This
3064   // might be needed to make this fit into a CSINV pattern.
3065   if (CTVal->isAllOnesValue() && CFVal->isNullValue()) {
3066     std::swap(TVal, FVal);
3067     std::swap(CTVal, CFVal);
3068     CC = ISD::getSetCCInverse(CC, LHS.getValueType());
3069   }
3070 
3071   // If the constants line up, perform the transform!
3072   if (CTVal->isNullValue() && CFVal->isAllOnesValue()) {
3073     SDValue CCVal;
3074     SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
3075 
3076     FVal = Other;
3077     TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
3078                        DAG.getConstant(-1ULL, dl, Other.getValueType()));
3079 
3080     return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
3081                        CCVal, Cmp);
3082   }
3083 
3084   return Op;
3085 }
3086 
3087 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
3088   EVT VT = Op.getValueType();
3089 
3090   // Let legalize expand this if it isn't a legal type yet.
3091   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
3092     return SDValue();
3093 
3094   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
3095 
3096   unsigned Opc;
3097   bool ExtraOp = false;
3098   switch (Op.getOpcode()) {
3099   default:
3100     llvm_unreachable("Invalid code");
3101   case ISD::ADDC:
3102     Opc = AArch64ISD::ADDS;
3103     break;
3104   case ISD::SUBC:
3105     Opc = AArch64ISD::SUBS;
3106     break;
3107   case ISD::ADDE:
3108     Opc = AArch64ISD::ADCS;
3109     ExtraOp = true;
3110     break;
3111   case ISD::SUBE:
3112     Opc = AArch64ISD::SBCS;
3113     ExtraOp = true;
3114     break;
3115   }
3116 
3117   if (!ExtraOp)
3118     return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
3119   return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
3120                      Op.getOperand(2));
3121 }
3122 
3123 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
3124   // Let legalize expand this if it isn't a legal type yet.
3125   if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
3126     return SDValue();
3127 
3128   SDLoc dl(Op);
3129   AArch64CC::CondCode CC;
3130   // The actual operation that sets the overflow or carry flag.
3131   SDValue Value, Overflow;
3132   std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
3133 
3134   // We use 0 and 1 as false and true values.
3135   SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
3136   SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
3137 
3138   // We use an inverted condition, because the conditional select is inverted
3139   // too. This will allow it to be selected to a single instruction:
3140   // CSINC Wd, WZR, WZR, invert(cond).
3141   SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
3142   Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal,
3143                          CCVal, Overflow);
3144 
3145   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
3146   return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
3147 }
3148 
3149 // Prefetch operands are:
3150 // 1: Address to prefetch
3151 // 2: bool isWrite
3152 // 3: int locality (0 = no locality ... 3 = extreme locality)
3153 // 4: bool isDataCache
3154 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
3155   SDLoc DL(Op);
3156   unsigned IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
3157   unsigned Locality = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
3158   unsigned IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
3159 
3160   bool IsStream = !Locality;
3161   // When the locality number is set
3162   if (Locality) {
3163     // The front-end should have filtered out the out-of-range values
3164     assert(Locality <= 3 && "Prefetch locality out-of-range");
3165     // The locality degree is the opposite of the cache speed.
3166     // Put the number the other way around.
3167     // The encoding starts at 0 for level 1
3168     Locality = 3 - Locality;
3169   }
3170 
3171   // built the mask value encoding the expected behavior.
3172   unsigned PrfOp = (IsWrite << 4) |     // Load/Store bit
3173                    (!IsData << 3) |     // IsDataCache bit
3174                    (Locality << 1) |    // Cache level bits
3175                    (unsigned)IsStream;  // Stream bit
3176   return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
3177                      DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1));
3178 }
3179 
3180 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
3181                                               SelectionDAG &DAG) const {
3182   if (Op.getValueType().isScalableVector())
3183     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FP_EXTEND_MERGE_PASSTHRU);
3184 
3185   assert(Op.getValueType() == MVT::f128 && "Unexpected lowering");
3186   return SDValue();
3187 }
3188 
3189 SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
3190                                              SelectionDAG &DAG) const {
3191   if (Op.getValueType().isScalableVector())
3192     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FP_ROUND_MERGE_PASSTHRU);
3193 
3194   bool IsStrict = Op->isStrictFPOpcode();
3195   SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3196   EVT SrcVT = SrcVal.getValueType();
3197 
3198   if (SrcVT != MVT::f128) {
3199     // Expand cases where the input is a vector bigger than NEON.
3200     if (useSVEForFixedLengthVectorVT(SrcVT))
3201       return SDValue();
3202 
3203     // It's legal except when f128 is involved
3204     return Op;
3205   }
3206 
3207   return SDValue();
3208 }
3209 
3210 SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
3211                                                     SelectionDAG &DAG) const {
3212   // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
3213   // Any additional optimization in this function should be recorded
3214   // in the cost tables.
3215   EVT InVT = Op.getOperand(0).getValueType();
3216   EVT VT = Op.getValueType();
3217 
3218   if (VT.isScalableVector()) {
3219     unsigned Opcode = Op.getOpcode() == ISD::FP_TO_UINT
3220                           ? AArch64ISD::FCVTZU_MERGE_PASSTHRU
3221                           : AArch64ISD::FCVTZS_MERGE_PASSTHRU;
3222     return LowerToPredicatedOp(Op, DAG, Opcode);
3223   }
3224 
3225   unsigned NumElts = InVT.getVectorNumElements();
3226 
3227   // f16 conversions are promoted to f32 when full fp16 is not supported.
3228   if (InVT.getVectorElementType() == MVT::f16 &&
3229       !Subtarget->hasFullFP16()) {
3230     MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts);
3231     SDLoc dl(Op);
3232     return DAG.getNode(
3233         Op.getOpcode(), dl, Op.getValueType(),
3234         DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0)));
3235   }
3236 
3237   uint64_t VTSize = VT.getFixedSizeInBits();
3238   uint64_t InVTSize = InVT.getFixedSizeInBits();
3239   if (VTSize < InVTSize) {
3240     SDLoc dl(Op);
3241     SDValue Cv =
3242         DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(),
3243                     Op.getOperand(0));
3244     return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
3245   }
3246 
3247   if (VTSize > InVTSize) {
3248     SDLoc dl(Op);
3249     MVT ExtVT =
3250         MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()),
3251                          VT.getVectorNumElements());
3252     SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
3253     return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
3254   }
3255 
3256   // Type changing conversions are illegal.
3257   return Op;
3258 }
3259 
3260 SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
3261                                               SelectionDAG &DAG) const {
3262   bool IsStrict = Op->isStrictFPOpcode();
3263   SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3264 
3265   if (SrcVal.getValueType().isVector())
3266     return LowerVectorFP_TO_INT(Op, DAG);
3267 
3268   // f16 conversions are promoted to f32 when full fp16 is not supported.
3269   if (SrcVal.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) {
3270     assert(!IsStrict && "Lowering of strict fp16 not yet implemented");
3271     SDLoc dl(Op);
3272     return DAG.getNode(
3273         Op.getOpcode(), dl, Op.getValueType(),
3274         DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, SrcVal));
3275   }
3276 
3277   if (SrcVal.getValueType() != MVT::f128) {
3278     // It's legal except when f128 is involved
3279     return Op;
3280   }
3281 
3282   return SDValue();
3283 }
3284 
3285 SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
3286                                                     SelectionDAG &DAG) const {
3287   // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
3288   // Any additional optimization in this function should be recorded
3289   // in the cost tables.
3290   EVT VT = Op.getValueType();
3291   SDLoc dl(Op);
3292   SDValue In = Op.getOperand(0);
3293   EVT InVT = In.getValueType();
3294   unsigned Opc = Op.getOpcode();
3295   bool IsSigned = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP;
3296 
3297   if (VT.isScalableVector()) {
3298     if (InVT.getVectorElementType() == MVT::i1) {
3299       // We can't directly extend an SVE predicate; extend it first.
3300       unsigned CastOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3301       EVT CastVT = getPromotedVTForPredicate(InVT);
3302       In = DAG.getNode(CastOpc, dl, CastVT, In);
3303       return DAG.getNode(Opc, dl, VT, In);
3304     }
3305 
3306     unsigned Opcode = IsSigned ? AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU
3307                                : AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU;
3308     return LowerToPredicatedOp(Op, DAG, Opcode);
3309   }
3310 
3311   uint64_t VTSize = VT.getFixedSizeInBits();
3312   uint64_t InVTSize = InVT.getFixedSizeInBits();
3313   if (VTSize < InVTSize) {
3314     MVT CastVT =
3315         MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
3316                          InVT.getVectorNumElements());
3317     In = DAG.getNode(Opc, dl, CastVT, In);
3318     return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl));
3319   }
3320 
3321   if (VTSize > InVTSize) {
3322     unsigned CastOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3323     EVT CastVT = VT.changeVectorElementTypeToInteger();
3324     In = DAG.getNode(CastOpc, dl, CastVT, In);
3325     return DAG.getNode(Opc, dl, VT, In);
3326   }
3327 
3328   return Op;
3329 }
3330 
3331 SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
3332                                             SelectionDAG &DAG) const {
3333   if (Op.getValueType().isVector())
3334     return LowerVectorINT_TO_FP(Op, DAG);
3335 
3336   bool IsStrict = Op->isStrictFPOpcode();
3337   SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3338 
3339   // f16 conversions are promoted to f32 when full fp16 is not supported.
3340   if (Op.getValueType() == MVT::f16 &&
3341       !Subtarget->hasFullFP16()) {
3342     assert(!IsStrict && "Lowering of strict fp16 not yet implemented");
3343     SDLoc dl(Op);
3344     return DAG.getNode(
3345         ISD::FP_ROUND, dl, MVT::f16,
3346         DAG.getNode(Op.getOpcode(), dl, MVT::f32, SrcVal),
3347         DAG.getIntPtrConstant(0, dl));
3348   }
3349 
3350   // i128 conversions are libcalls.
3351   if (SrcVal.getValueType() == MVT::i128)
3352     return SDValue();
3353 
3354   // Other conversions are legal, unless it's to the completely software-based
3355   // fp128.
3356   if (Op.getValueType() != MVT::f128)
3357     return Op;
3358   return SDValue();
3359 }
3360 
3361 SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
3362                                             SelectionDAG &DAG) const {
3363   // For iOS, we want to call an alternative entry point: __sincos_stret,
3364   // which returns the values in two S / D registers.
3365   SDLoc dl(Op);
3366   SDValue Arg = Op.getOperand(0);
3367   EVT ArgVT = Arg.getValueType();
3368   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3369 
3370   ArgListTy Args;
3371   ArgListEntry Entry;
3372 
3373   Entry.Node = Arg;
3374   Entry.Ty = ArgTy;
3375   Entry.IsSExt = false;
3376   Entry.IsZExt = false;
3377   Args.push_back(Entry);
3378 
3379   RTLIB::Libcall LC = ArgVT == MVT::f64 ? RTLIB::SINCOS_STRET_F64
3380                                         : RTLIB::SINCOS_STRET_F32;
3381   const char *LibcallName = getLibcallName(LC);
3382   SDValue Callee =
3383       DAG.getExternalSymbol(LibcallName, getPointerTy(DAG.getDataLayout()));
3384 
3385   StructType *RetTy = StructType::get(ArgTy, ArgTy);
3386   TargetLowering::CallLoweringInfo CLI(DAG);
3387   CLI.setDebugLoc(dl)
3388       .setChain(DAG.getEntryNode())
3389       .setLibCallee(CallingConv::Fast, RetTy, Callee, std::move(Args));
3390 
3391   std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3392   return CallResult.first;
3393 }
3394 
3395 static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
3396   EVT OpVT = Op.getValueType();
3397   if (OpVT != MVT::f16 && OpVT != MVT::bf16)
3398     return SDValue();
3399 
3400   assert(Op.getOperand(0).getValueType() == MVT::i16);
3401   SDLoc DL(Op);
3402 
3403   Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0));
3404   Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op);
3405   return SDValue(
3406       DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, OpVT, Op,
3407                          DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
3408       0);
3409 }
3410 
3411 static EVT getExtensionTo64Bits(const EVT &OrigVT) {
3412   if (OrigVT.getSizeInBits() >= 64)
3413     return OrigVT;
3414 
3415   assert(OrigVT.isSimple() && "Expecting a simple value type");
3416 
3417   MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
3418   switch (OrigSimpleTy) {
3419   default: llvm_unreachable("Unexpected Vector Type");
3420   case MVT::v2i8:
3421   case MVT::v2i16:
3422      return MVT::v2i32;
3423   case MVT::v4i8:
3424     return  MVT::v4i16;
3425   }
3426 }
3427 
3428 static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG,
3429                                                  const EVT &OrigTy,
3430                                                  const EVT &ExtTy,
3431                                                  unsigned ExtOpcode) {
3432   // The vector originally had a size of OrigTy. It was then extended to ExtTy.
3433   // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
3434   // 64-bits we need to insert a new extension so that it will be 64-bits.
3435   assert(ExtTy.is128BitVector() && "Unexpected extension size");
3436   if (OrigTy.getSizeInBits() >= 64)
3437     return N;
3438 
3439   // Must extend size to at least 64 bits to be used as an operand for VMULL.
3440   EVT NewVT = getExtensionTo64Bits(OrigTy);
3441 
3442   return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
3443 }
3444 
3445 static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
3446                                    bool isSigned) {
3447   EVT VT = N->getValueType(0);
3448 
3449   if (N->getOpcode() != ISD::BUILD_VECTOR)
3450     return false;
3451 
3452   for (const SDValue &Elt : N->op_values()) {
3453     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
3454       unsigned EltSize = VT.getScalarSizeInBits();
3455       unsigned HalfSize = EltSize / 2;
3456       if (isSigned) {
3457         if (!isIntN(HalfSize, C->getSExtValue()))
3458           return false;
3459       } else {
3460         if (!isUIntN(HalfSize, C->getZExtValue()))
3461           return false;
3462       }
3463       continue;
3464     }
3465     return false;
3466   }
3467 
3468   return true;
3469 }
3470 
3471 static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) {
3472   if (N->getOpcode() == ISD::SIGN_EXTEND ||
3473       N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND)
3474     return addRequiredExtensionForVectorMULL(N->getOperand(0), DAG,
3475                                              N->getOperand(0)->getValueType(0),
3476                                              N->getValueType(0),
3477                                              N->getOpcode());
3478 
3479   assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
3480   EVT VT = N->getValueType(0);
3481   SDLoc dl(N);
3482   unsigned EltSize = VT.getScalarSizeInBits() / 2;
3483   unsigned NumElts = VT.getVectorNumElements();
3484   MVT TruncVT = MVT::getIntegerVT(EltSize);
3485   SmallVector<SDValue, 8> Ops;
3486   for (unsigned i = 0; i != NumElts; ++i) {
3487     ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i));
3488     const APInt &CInt = C->getAPIntValue();
3489     // Element types smaller than 32 bits are not legal, so use i32 elements.
3490     // The values are implicitly truncated so sext vs. zext doesn't matter.
3491     Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
3492   }
3493   return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops);
3494 }
3495 
3496 static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
3497   return N->getOpcode() == ISD::SIGN_EXTEND ||
3498          N->getOpcode() == ISD::ANY_EXTEND ||
3499          isExtendedBUILD_VECTOR(N, DAG, true);
3500 }
3501 
3502 static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
3503   return N->getOpcode() == ISD::ZERO_EXTEND ||
3504          N->getOpcode() == ISD::ANY_EXTEND ||
3505          isExtendedBUILD_VECTOR(N, DAG, false);
3506 }
3507 
3508 static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
3509   unsigned Opcode = N->getOpcode();
3510   if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
3511     SDNode *N0 = N->getOperand(0).getNode();
3512     SDNode *N1 = N->getOperand(1).getNode();
3513     return N0->hasOneUse() && N1->hasOneUse() &&
3514       isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
3515   }
3516   return false;
3517 }
3518 
3519 static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
3520   unsigned Opcode = N->getOpcode();
3521   if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
3522     SDNode *N0 = N->getOperand(0).getNode();
3523     SDNode *N1 = N->getOperand(1).getNode();
3524     return N0->hasOneUse() && N1->hasOneUse() &&
3525       isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
3526   }
3527   return false;
3528 }
3529 
3530 SDValue AArch64TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
3531                                                 SelectionDAG &DAG) const {
3532   // The rounding mode is in bits 23:22 of the FPSCR.
3533   // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
3534   // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
3535   // so that the shift + and get folded into a bitfield extract.
3536   SDLoc dl(Op);
3537 
3538   SDValue Chain = Op.getOperand(0);
3539   SDValue FPCR_64 = DAG.getNode(
3540       ISD::INTRINSIC_W_CHAIN, dl, {MVT::i64, MVT::Other},
3541       {Chain, DAG.getConstant(Intrinsic::aarch64_get_fpcr, dl, MVT::i64)});
3542   Chain = FPCR_64.getValue(1);
3543   SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64);
3544   SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
3545                                   DAG.getConstant(1U << 22, dl, MVT::i32));
3546   SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
3547                               DAG.getConstant(22, dl, MVT::i32));
3548   SDValue AND = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
3549                             DAG.getConstant(3, dl, MVT::i32));
3550   return DAG.getMergeValues({AND, Chain}, dl);
3551 }
3552 
3553 SDValue AArch64TargetLowering::LowerSET_ROUNDING(SDValue Op,
3554                                                  SelectionDAG &DAG) const {
3555   SDLoc DL(Op);
3556   SDValue Chain = Op->getOperand(0);
3557   SDValue RMValue = Op->getOperand(1);
3558 
3559   // The rounding mode is in bits 23:22 of the FPCR.
3560   // The llvm.set.rounding argument value to the rounding mode in FPCR mapping
3561   // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
3562   // ((arg - 1) & 3) << 22).
3563   //
3564   // The argument of llvm.set.rounding must be within the segment [0, 3], so
3565   // NearestTiesToAway (4) is not handled here. It is responsibility of the code
3566   // generated llvm.set.rounding to ensure this condition.
3567 
3568   // Calculate new value of FPCR[23:22].
3569   RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue,
3570                         DAG.getConstant(1, DL, MVT::i32));
3571   RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue,
3572                         DAG.getConstant(0x3, DL, MVT::i32));
3573   RMValue =
3574       DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue,
3575                   DAG.getConstant(AArch64::RoundingBitsPos, DL, MVT::i32));
3576   RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, RMValue);
3577 
3578   // Get current value of FPCR.
3579   SDValue Ops[] = {
3580       Chain, DAG.getTargetConstant(Intrinsic::aarch64_get_fpcr, DL, MVT::i64)};
3581   SDValue FPCR =
3582       DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i64, MVT::Other}, Ops);
3583   Chain = FPCR.getValue(1);
3584   FPCR = FPCR.getValue(0);
3585 
3586   // Put new rounding mode into FPSCR[23:22].
3587   const int RMMask = ~(AArch64::Rounding::rmMask << AArch64::RoundingBitsPos);
3588   FPCR = DAG.getNode(ISD::AND, DL, MVT::i64, FPCR,
3589                      DAG.getConstant(RMMask, DL, MVT::i64));
3590   FPCR = DAG.getNode(ISD::OR, DL, MVT::i64, FPCR, RMValue);
3591   SDValue Ops2[] = {
3592       Chain, DAG.getTargetConstant(Intrinsic::aarch64_set_fpcr, DL, MVT::i64),
3593       FPCR};
3594   return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
3595 }
3596 
3597 SDValue AArch64TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
3598   EVT VT = Op.getValueType();
3599 
3600   // If SVE is available then i64 vector multiplications can also be made legal.
3601   bool OverrideNEON = VT == MVT::v2i64 || VT == MVT::v1i64;
3602 
3603   if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT, OverrideNEON))
3604     return LowerToPredicatedOp(Op, DAG, AArch64ISD::MUL_PRED, OverrideNEON);
3605 
3606   // Multiplications are only custom-lowered for 128-bit vectors so that
3607   // VMULL can be detected.  Otherwise v2i64 multiplications are not legal.
3608   assert(VT.is128BitVector() && VT.isInteger() &&
3609          "unexpected type for custom-lowering ISD::MUL");
3610   SDNode *N0 = Op.getOperand(0).getNode();
3611   SDNode *N1 = Op.getOperand(1).getNode();
3612   unsigned NewOpc = 0;
3613   bool isMLA = false;
3614   bool isN0SExt = isSignExtended(N0, DAG);
3615   bool isN1SExt = isSignExtended(N1, DAG);
3616   if (isN0SExt && isN1SExt)
3617     NewOpc = AArch64ISD::SMULL;
3618   else {
3619     bool isN0ZExt = isZeroExtended(N0, DAG);
3620     bool isN1ZExt = isZeroExtended(N1, DAG);
3621     if (isN0ZExt && isN1ZExt)
3622       NewOpc = AArch64ISD::UMULL;
3623     else if (isN1SExt || isN1ZExt) {
3624       // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
3625       // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
3626       if (isN1SExt && isAddSubSExt(N0, DAG)) {
3627         NewOpc = AArch64ISD::SMULL;
3628         isMLA = true;
3629       } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
3630         NewOpc =  AArch64ISD::UMULL;
3631         isMLA = true;
3632       } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
3633         std::swap(N0, N1);
3634         NewOpc =  AArch64ISD::UMULL;
3635         isMLA = true;
3636       }
3637     }
3638 
3639     if (!NewOpc) {
3640       if (VT == MVT::v2i64)
3641         // Fall through to expand this.  It is not legal.
3642         return SDValue();
3643       else
3644         // Other vector multiplications are legal.
3645         return Op;
3646     }
3647   }
3648 
3649   // Legalize to a S/UMULL instruction
3650   SDLoc DL(Op);
3651   SDValue Op0;
3652   SDValue Op1 = skipExtensionForVectorMULL(N1, DAG);
3653   if (!isMLA) {
3654     Op0 = skipExtensionForVectorMULL(N0, DAG);
3655     assert(Op0.getValueType().is64BitVector() &&
3656            Op1.getValueType().is64BitVector() &&
3657            "unexpected types for extended operands to VMULL");
3658     return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
3659   }
3660   // Optimizing (zext A + zext B) * C, to (S/UMULL A, C) + (S/UMULL B, C) during
3661   // isel lowering to take advantage of no-stall back to back s/umul + s/umla.
3662   // This is true for CPUs with accumulate forwarding such as Cortex-A53/A57
3663   SDValue N00 = skipExtensionForVectorMULL(N0->getOperand(0).getNode(), DAG);
3664   SDValue N01 = skipExtensionForVectorMULL(N0->getOperand(1).getNode(), DAG);
3665   EVT Op1VT = Op1.getValueType();
3666   return DAG.getNode(N0->getOpcode(), DL, VT,
3667                      DAG.getNode(NewOpc, DL, VT,
3668                                DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
3669                      DAG.getNode(NewOpc, DL, VT,
3670                                DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
3671 }
3672 
3673 static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
3674                                int Pattern) {
3675   return DAG.getNode(AArch64ISD::PTRUE, DL, VT,
3676                      DAG.getTargetConstant(Pattern, DL, MVT::i32));
3677 }
3678 
3679 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
3680                                                      SelectionDAG &DAG) const {
3681   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3682   SDLoc dl(Op);
3683   switch (IntNo) {
3684   default: return SDValue();    // Don't custom lower most intrinsics.
3685   case Intrinsic::thread_pointer: {
3686     EVT PtrVT = getPointerTy(DAG.getDataLayout());
3687     return DAG.getNode(AArch64ISD::THREAD_POINTER, dl, PtrVT);
3688   }
3689   case Intrinsic::aarch64_neon_abs: {
3690     EVT Ty = Op.getValueType();
3691     if (Ty == MVT::i64) {
3692       SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64,
3693                                    Op.getOperand(1));
3694       Result = DAG.getNode(ISD::ABS, dl, MVT::v1i64, Result);
3695       return DAG.getNode(ISD::BITCAST, dl, MVT::i64, Result);
3696     } else if (Ty.isVector() && Ty.isInteger() && isTypeLegal(Ty)) {
3697       return DAG.getNode(ISD::ABS, dl, Ty, Op.getOperand(1));
3698     } else {
3699       report_fatal_error("Unexpected type for AArch64 NEON intrinic");
3700     }
3701   }
3702   case Intrinsic::aarch64_neon_smax:
3703     return DAG.getNode(ISD::SMAX, dl, Op.getValueType(),
3704                        Op.getOperand(1), Op.getOperand(2));
3705   case Intrinsic::aarch64_neon_umax:
3706     return DAG.getNode(ISD::UMAX, dl, Op.getValueType(),
3707                        Op.getOperand(1), Op.getOperand(2));
3708   case Intrinsic::aarch64_neon_smin:
3709     return DAG.getNode(ISD::SMIN, dl, Op.getValueType(),
3710                        Op.getOperand(1), Op.getOperand(2));
3711   case Intrinsic::aarch64_neon_umin:
3712     return DAG.getNode(ISD::UMIN, dl, Op.getValueType(),
3713                        Op.getOperand(1), Op.getOperand(2));
3714 
3715   case Intrinsic::aarch64_sve_sunpkhi:
3716     return DAG.getNode(AArch64ISD::SUNPKHI, dl, Op.getValueType(),
3717                        Op.getOperand(1));
3718   case Intrinsic::aarch64_sve_sunpklo:
3719     return DAG.getNode(AArch64ISD::SUNPKLO, dl, Op.getValueType(),
3720                        Op.getOperand(1));
3721   case Intrinsic::aarch64_sve_uunpkhi:
3722     return DAG.getNode(AArch64ISD::UUNPKHI, dl, Op.getValueType(),
3723                        Op.getOperand(1));
3724   case Intrinsic::aarch64_sve_uunpklo:
3725     return DAG.getNode(AArch64ISD::UUNPKLO, dl, Op.getValueType(),
3726                        Op.getOperand(1));
3727   case Intrinsic::aarch64_sve_clasta_n:
3728     return DAG.getNode(AArch64ISD::CLASTA_N, dl, Op.getValueType(),
3729                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
3730   case Intrinsic::aarch64_sve_clastb_n:
3731     return DAG.getNode(AArch64ISD::CLASTB_N, dl, Op.getValueType(),
3732                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
3733   case Intrinsic::aarch64_sve_lasta:
3734     return DAG.getNode(AArch64ISD::LASTA, dl, Op.getValueType(),
3735                        Op.getOperand(1), Op.getOperand(2));
3736   case Intrinsic::aarch64_sve_lastb:
3737     return DAG.getNode(AArch64ISD::LASTB, dl, Op.getValueType(),
3738                        Op.getOperand(1), Op.getOperand(2));
3739   case Intrinsic::aarch64_sve_rev:
3740     return DAG.getNode(ISD::VECTOR_REVERSE, dl, Op.getValueType(),
3741                        Op.getOperand(1));
3742   case Intrinsic::aarch64_sve_tbl:
3743     return DAG.getNode(AArch64ISD::TBL, dl, Op.getValueType(),
3744                        Op.getOperand(1), Op.getOperand(2));
3745   case Intrinsic::aarch64_sve_trn1:
3746     return DAG.getNode(AArch64ISD::TRN1, dl, Op.getValueType(),
3747                        Op.getOperand(1), Op.getOperand(2));
3748   case Intrinsic::aarch64_sve_trn2:
3749     return DAG.getNode(AArch64ISD::TRN2, dl, Op.getValueType(),
3750                        Op.getOperand(1), Op.getOperand(2));
3751   case Intrinsic::aarch64_sve_uzp1:
3752     return DAG.getNode(AArch64ISD::UZP1, dl, Op.getValueType(),
3753                        Op.getOperand(1), Op.getOperand(2));
3754   case Intrinsic::aarch64_sve_uzp2:
3755     return DAG.getNode(AArch64ISD::UZP2, dl, Op.getValueType(),
3756                        Op.getOperand(1), Op.getOperand(2));
3757   case Intrinsic::aarch64_sve_zip1:
3758     return DAG.getNode(AArch64ISD::ZIP1, dl, Op.getValueType(),
3759                        Op.getOperand(1), Op.getOperand(2));
3760   case Intrinsic::aarch64_sve_zip2:
3761     return DAG.getNode(AArch64ISD::ZIP2, dl, Op.getValueType(),
3762                        Op.getOperand(1), Op.getOperand(2));
3763   case Intrinsic::aarch64_sve_ptrue:
3764     return DAG.getNode(AArch64ISD::PTRUE, dl, Op.getValueType(),
3765                        Op.getOperand(1));
3766   case Intrinsic::aarch64_sve_clz:
3767     return DAG.getNode(AArch64ISD::CTLZ_MERGE_PASSTHRU, dl, Op.getValueType(),
3768                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3769   case Intrinsic::aarch64_sve_cnt: {
3770     SDValue Data = Op.getOperand(3);
3771     // CTPOP only supports integer operands.
3772     if (Data.getValueType().isFloatingPoint())
3773       Data = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Data);
3774     return DAG.getNode(AArch64ISD::CTPOP_MERGE_PASSTHRU, dl, Op.getValueType(),
3775                        Op.getOperand(2), Data, Op.getOperand(1));
3776   }
3777   case Intrinsic::aarch64_sve_dupq_lane:
3778     return LowerDUPQLane(Op, DAG);
3779   case Intrinsic::aarch64_sve_convert_from_svbool:
3780     return DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, Op.getValueType(),
3781                        Op.getOperand(1));
3782   case Intrinsic::aarch64_sve_fneg:
3783     return DAG.getNode(AArch64ISD::FNEG_MERGE_PASSTHRU, dl, Op.getValueType(),
3784                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3785   case Intrinsic::aarch64_sve_frintp:
3786     return DAG.getNode(AArch64ISD::FCEIL_MERGE_PASSTHRU, dl, Op.getValueType(),
3787                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3788   case Intrinsic::aarch64_sve_frintm:
3789     return DAG.getNode(AArch64ISD::FFLOOR_MERGE_PASSTHRU, dl, Op.getValueType(),
3790                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3791   case Intrinsic::aarch64_sve_frinti:
3792     return DAG.getNode(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, dl, Op.getValueType(),
3793                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3794   case Intrinsic::aarch64_sve_frintx:
3795     return DAG.getNode(AArch64ISD::FRINT_MERGE_PASSTHRU, dl, Op.getValueType(),
3796                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3797   case Intrinsic::aarch64_sve_frinta:
3798     return DAG.getNode(AArch64ISD::FROUND_MERGE_PASSTHRU, dl, Op.getValueType(),
3799                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3800   case Intrinsic::aarch64_sve_frintn:
3801     return DAG.getNode(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU, dl, Op.getValueType(),
3802                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3803   case Intrinsic::aarch64_sve_frintz:
3804     return DAG.getNode(AArch64ISD::FTRUNC_MERGE_PASSTHRU, dl, Op.getValueType(),
3805                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3806   case Intrinsic::aarch64_sve_ucvtf:
3807     return DAG.getNode(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU, dl,
3808                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
3809                        Op.getOperand(1));
3810   case Intrinsic::aarch64_sve_scvtf:
3811     return DAG.getNode(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU, dl,
3812                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
3813                        Op.getOperand(1));
3814   case Intrinsic::aarch64_sve_fcvtzu:
3815     return DAG.getNode(AArch64ISD::FCVTZU_MERGE_PASSTHRU, dl,
3816                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
3817                        Op.getOperand(1));
3818   case Intrinsic::aarch64_sve_fcvtzs:
3819     return DAG.getNode(AArch64ISD::FCVTZS_MERGE_PASSTHRU, dl,
3820                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
3821                        Op.getOperand(1));
3822   case Intrinsic::aarch64_sve_fsqrt:
3823     return DAG.getNode(AArch64ISD::FSQRT_MERGE_PASSTHRU, dl, Op.getValueType(),
3824                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3825   case Intrinsic::aarch64_sve_frecpx:
3826     return DAG.getNode(AArch64ISD::FRECPX_MERGE_PASSTHRU, dl, Op.getValueType(),
3827                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3828   case Intrinsic::aarch64_sve_fabs:
3829     return DAG.getNode(AArch64ISD::FABS_MERGE_PASSTHRU, dl, Op.getValueType(),
3830                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3831   case Intrinsic::aarch64_sve_abs:
3832     return DAG.getNode(AArch64ISD::ABS_MERGE_PASSTHRU, dl, Op.getValueType(),
3833                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3834   case Intrinsic::aarch64_sve_neg:
3835     return DAG.getNode(AArch64ISD::NEG_MERGE_PASSTHRU, dl, Op.getValueType(),
3836                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3837   case Intrinsic::aarch64_sve_convert_to_svbool: {
3838     EVT OutVT = Op.getValueType();
3839     EVT InVT = Op.getOperand(1).getValueType();
3840     // Return the operand if the cast isn't changing type,
3841     // i.e. <n x 16 x i1> -> <n x 16 x i1>
3842     if (InVT == OutVT)
3843       return Op.getOperand(1);
3844     // Otherwise, zero the newly introduced lanes.
3845     SDValue Reinterpret =
3846         DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, OutVT, Op.getOperand(1));
3847     SDValue Mask = getPTrue(DAG, dl, InVT, AArch64SVEPredPattern::all);
3848     SDValue MaskReinterpret =
3849         DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, OutVT, Mask);
3850     return DAG.getNode(ISD::AND, dl, OutVT, Reinterpret, MaskReinterpret);
3851   }
3852 
3853   case Intrinsic::aarch64_sve_insr: {
3854     SDValue Scalar = Op.getOperand(2);
3855     EVT ScalarTy = Scalar.getValueType();
3856     if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16))
3857       Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar);
3858 
3859     return DAG.getNode(AArch64ISD::INSR, dl, Op.getValueType(),
3860                        Op.getOperand(1), Scalar);
3861   }
3862   case Intrinsic::aarch64_sve_rbit:
3863     return DAG.getNode(AArch64ISD::BITREVERSE_MERGE_PASSTHRU, dl,
3864                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
3865                        Op.getOperand(1));
3866   case Intrinsic::aarch64_sve_revb:
3867     return DAG.getNode(AArch64ISD::BSWAP_MERGE_PASSTHRU, dl, Op.getValueType(),
3868                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3869   case Intrinsic::aarch64_sve_sxtb:
3870     return DAG.getNode(
3871         AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3872         Op.getOperand(2), Op.getOperand(3),
3873         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i8)),
3874         Op.getOperand(1));
3875   case Intrinsic::aarch64_sve_sxth:
3876     return DAG.getNode(
3877         AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3878         Op.getOperand(2), Op.getOperand(3),
3879         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i16)),
3880         Op.getOperand(1));
3881   case Intrinsic::aarch64_sve_sxtw:
3882     return DAG.getNode(
3883         AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3884         Op.getOperand(2), Op.getOperand(3),
3885         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i32)),
3886         Op.getOperand(1));
3887   case Intrinsic::aarch64_sve_uxtb:
3888     return DAG.getNode(
3889         AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3890         Op.getOperand(2), Op.getOperand(3),
3891         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i8)),
3892         Op.getOperand(1));
3893   case Intrinsic::aarch64_sve_uxth:
3894     return DAG.getNode(
3895         AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3896         Op.getOperand(2), Op.getOperand(3),
3897         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i16)),
3898         Op.getOperand(1));
3899   case Intrinsic::aarch64_sve_uxtw:
3900     return DAG.getNode(
3901         AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3902         Op.getOperand(2), Op.getOperand(3),
3903         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i32)),
3904         Op.getOperand(1));
3905 
3906   case Intrinsic::localaddress: {
3907     const auto &MF = DAG.getMachineFunction();
3908     const auto *RegInfo = Subtarget->getRegisterInfo();
3909     unsigned Reg = RegInfo->getLocalAddressRegister(MF);
3910     return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg,
3911                               Op.getSimpleValueType());
3912   }
3913 
3914   case Intrinsic::eh_recoverfp: {
3915     // FIXME: This needs to be implemented to correctly handle highly aligned
3916     // stack objects. For now we simply return the incoming FP. Refer D53541
3917     // for more details.
3918     SDValue FnOp = Op.getOperand(1);
3919     SDValue IncomingFPOp = Op.getOperand(2);
3920     GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
3921     auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
3922     if (!Fn)
3923       report_fatal_error(
3924           "llvm.eh.recoverfp must take a function as the first argument");
3925     return IncomingFPOp;
3926   }
3927 
3928   case Intrinsic::aarch64_neon_vsri:
3929   case Intrinsic::aarch64_neon_vsli: {
3930     EVT Ty = Op.getValueType();
3931 
3932     if (!Ty.isVector())
3933       report_fatal_error("Unexpected type for aarch64_neon_vsli");
3934 
3935     assert(Op.getConstantOperandVal(3) <= Ty.getScalarSizeInBits());
3936 
3937     bool IsShiftRight = IntNo == Intrinsic::aarch64_neon_vsri;
3938     unsigned Opcode = IsShiftRight ? AArch64ISD::VSRI : AArch64ISD::VSLI;
3939     return DAG.getNode(Opcode, dl, Ty, Op.getOperand(1), Op.getOperand(2),
3940                        Op.getOperand(3));
3941   }
3942 
3943   case Intrinsic::aarch64_neon_srhadd:
3944   case Intrinsic::aarch64_neon_urhadd:
3945   case Intrinsic::aarch64_neon_shadd:
3946   case Intrinsic::aarch64_neon_uhadd: {
3947     bool IsSignedAdd = (IntNo == Intrinsic::aarch64_neon_srhadd ||
3948                         IntNo == Intrinsic::aarch64_neon_shadd);
3949     bool IsRoundingAdd = (IntNo == Intrinsic::aarch64_neon_srhadd ||
3950                           IntNo == Intrinsic::aarch64_neon_urhadd);
3951     unsigned Opcode =
3952         IsSignedAdd ? (IsRoundingAdd ? AArch64ISD::SRHADD : AArch64ISD::SHADD)
3953                     : (IsRoundingAdd ? AArch64ISD::URHADD : AArch64ISD::UHADD);
3954     return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1),
3955                        Op.getOperand(2));
3956   }
3957   case Intrinsic::aarch64_neon_sabd:
3958   case Intrinsic::aarch64_neon_uabd: {
3959     unsigned Opcode = IntNo == Intrinsic::aarch64_neon_uabd ? AArch64ISD::UABD
3960                                                             : AArch64ISD::SABD;
3961     return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1),
3962                        Op.getOperand(2));
3963   }
3964   case Intrinsic::aarch64_neon_sdot:
3965   case Intrinsic::aarch64_neon_udot:
3966   case Intrinsic::aarch64_sve_sdot:
3967   case Intrinsic::aarch64_sve_udot: {
3968     unsigned Opcode = (IntNo == Intrinsic::aarch64_neon_udot ||
3969                        IntNo == Intrinsic::aarch64_sve_udot)
3970                           ? AArch64ISD::UDOT
3971                           : AArch64ISD::SDOT;
3972     return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1),
3973                        Op.getOperand(2), Op.getOperand(3));
3974   }
3975   }
3976 }
3977 
3978 bool AArch64TargetLowering::shouldExtendGSIndex(EVT VT, EVT &EltTy) const {
3979   if (VT.getVectorElementType() == MVT::i8 ||
3980       VT.getVectorElementType() == MVT::i16) {
3981     EltTy = MVT::i32;
3982     return true;
3983   }
3984   return false;
3985 }
3986 
3987 bool AArch64TargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
3988   if (VT.getVectorElementType() == MVT::i32 &&
3989       VT.getVectorElementCount().getKnownMinValue() >= 4)
3990     return true;
3991 
3992   return false;
3993 }
3994 
3995 bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
3996   return ExtVal.getValueType().isScalableVector();
3997 }
3998 
3999 unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
4000   std::map<std::tuple<bool, bool, bool>, unsigned> AddrModes = {
4001       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ false),
4002        AArch64ISD::GLD1_MERGE_ZERO},
4003       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ true),
4004        AArch64ISD::GLD1_UXTW_MERGE_ZERO},
4005       {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ false),
4006        AArch64ISD::GLD1_MERGE_ZERO},
4007       {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ true),
4008        AArch64ISD::GLD1_SXTW_MERGE_ZERO},
4009       {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ false),
4010        AArch64ISD::GLD1_SCALED_MERGE_ZERO},
4011       {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ true),
4012        AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO},
4013       {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ false),
4014        AArch64ISD::GLD1_SCALED_MERGE_ZERO},
4015       {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ true),
4016        AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO},
4017   };
4018   auto Key = std::make_tuple(IsScaled, IsSigned, NeedsExtend);
4019   return AddrModes.find(Key)->second;
4020 }
4021 
4022 unsigned getScatterVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
4023   std::map<std::tuple<bool, bool, bool>, unsigned> AddrModes = {
4024       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ false),
4025        AArch64ISD::SST1_PRED},
4026       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ true),
4027        AArch64ISD::SST1_UXTW_PRED},
4028       {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ false),
4029        AArch64ISD::SST1_PRED},
4030       {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ true),
4031        AArch64ISD::SST1_SXTW_PRED},
4032       {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ false),
4033        AArch64ISD::SST1_SCALED_PRED},
4034       {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ true),
4035        AArch64ISD::SST1_UXTW_SCALED_PRED},
4036       {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ false),
4037        AArch64ISD::SST1_SCALED_PRED},
4038       {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ true),
4039        AArch64ISD::SST1_SXTW_SCALED_PRED},
4040   };
4041   auto Key = std::make_tuple(IsScaled, IsSigned, NeedsExtend);
4042   return AddrModes.find(Key)->second;
4043 }
4044 
4045 unsigned getSignExtendedGatherOpcode(unsigned Opcode) {
4046   switch (Opcode) {
4047   default:
4048     llvm_unreachable("unimplemented opcode");
4049     return Opcode;
4050   case AArch64ISD::GLD1_MERGE_ZERO:
4051     return AArch64ISD::GLD1S_MERGE_ZERO;
4052   case AArch64ISD::GLD1_IMM_MERGE_ZERO:
4053     return AArch64ISD::GLD1S_IMM_MERGE_ZERO;
4054   case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
4055     return AArch64ISD::GLD1S_UXTW_MERGE_ZERO;
4056   case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
4057     return AArch64ISD::GLD1S_SXTW_MERGE_ZERO;
4058   case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
4059     return AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
4060   case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
4061     return AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO;
4062   case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
4063     return AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO;
4064   }
4065 }
4066 
4067 bool getGatherScatterIndexIsExtended(SDValue Index) {
4068   unsigned Opcode = Index.getOpcode();
4069   if (Opcode == ISD::SIGN_EXTEND_INREG)
4070     return true;
4071 
4072   if (Opcode == ISD::AND) {
4073     SDValue Splat = Index.getOperand(1);
4074     if (Splat.getOpcode() != ISD::SPLAT_VECTOR)
4075       return false;
4076     ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Splat.getOperand(0));
4077     if (!Mask || Mask->getZExtValue() != 0xFFFFFFFF)
4078       return false;
4079     return true;
4080   }
4081 
4082   return false;
4083 }
4084 
4085 // If the base pointer of a masked gather or scatter is null, we
4086 // may be able to swap BasePtr & Index and use the vector + register
4087 // or vector + immediate addressing mode, e.g.
4088 // VECTOR + REGISTER:
4089 //    getelementptr nullptr, <vscale x N x T> (splat(%offset)) + %indices)
4090 // -> getelementptr %offset, <vscale x N x T> %indices
4091 // VECTOR + IMMEDIATE:
4092 //    getelementptr nullptr, <vscale x N x T> (splat(#x)) + %indices)
4093 // -> getelementptr #x, <vscale x N x T> %indices
4094 void selectGatherScatterAddrMode(SDValue &BasePtr, SDValue &Index, EVT MemVT,
4095                                  unsigned &Opcode, bool IsGather,
4096                                  SelectionDAG &DAG) {
4097   if (!isNullConstant(BasePtr))
4098     return;
4099 
4100   ConstantSDNode *Offset = nullptr;
4101   if (Index.getOpcode() == ISD::ADD)
4102     if (auto SplatVal = DAG.getSplatValue(Index.getOperand(1))) {
4103       if (isa<ConstantSDNode>(SplatVal))
4104         Offset = cast<ConstantSDNode>(SplatVal);
4105       else {
4106         BasePtr = SplatVal;
4107         Index = Index->getOperand(0);
4108         return;
4109       }
4110     }
4111 
4112   unsigned NewOp =
4113       IsGather ? AArch64ISD::GLD1_IMM_MERGE_ZERO : AArch64ISD::SST1_IMM_PRED;
4114 
4115   if (!Offset) {
4116     std::swap(BasePtr, Index);
4117     Opcode = NewOp;
4118     return;
4119   }
4120 
4121   uint64_t OffsetVal = Offset->getZExtValue();
4122   unsigned ScalarSizeInBytes = MemVT.getScalarSizeInBits() / 8;
4123   auto ConstOffset = DAG.getConstant(OffsetVal, SDLoc(Index), MVT::i64);
4124 
4125   if (OffsetVal % ScalarSizeInBytes || OffsetVal / ScalarSizeInBytes > 31) {
4126     // Index is out of range for the immediate addressing mode
4127     BasePtr = ConstOffset;
4128     Index = Index->getOperand(0);
4129     return;
4130   }
4131 
4132   // Immediate is in range
4133   Opcode = NewOp;
4134   BasePtr = Index->getOperand(0);
4135   Index = ConstOffset;
4136 }
4137 
4138 SDValue AArch64TargetLowering::LowerMGATHER(SDValue Op,
4139                                             SelectionDAG &DAG) const {
4140   SDLoc DL(Op);
4141   MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(Op);
4142   assert(MGT && "Can only custom lower gather load nodes");
4143 
4144   SDValue Index = MGT->getIndex();
4145   SDValue Chain = MGT->getChain();
4146   SDValue PassThru = MGT->getPassThru();
4147   SDValue Mask = MGT->getMask();
4148   SDValue BasePtr = MGT->getBasePtr();
4149   ISD::LoadExtType ExtTy = MGT->getExtensionType();
4150 
4151   ISD::MemIndexType IndexType = MGT->getIndexType();
4152   bool IsScaled =
4153       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::UNSIGNED_SCALED;
4154   bool IsSigned =
4155       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::SIGNED_UNSCALED;
4156   bool IdxNeedsExtend =
4157       getGatherScatterIndexIsExtended(Index) ||
4158       Index.getSimpleValueType().getVectorElementType() == MVT::i32;
4159   bool ResNeedsSignExtend = ExtTy == ISD::EXTLOAD || ExtTy == ISD::SEXTLOAD;
4160 
4161   EVT VT = PassThru.getSimpleValueType();
4162   EVT MemVT = MGT->getMemoryVT();
4163   SDValue InputVT = DAG.getValueType(MemVT);
4164 
4165   if (VT.getVectorElementType() == MVT::bf16 &&
4166       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
4167     return SDValue();
4168 
4169   // Handle FP data by using an integer gather and casting the result.
4170   if (VT.isFloatingPoint()) {
4171     EVT PassThruVT = getPackedSVEVectorVT(VT.getVectorElementCount());
4172     PassThru = getSVESafeBitCast(PassThruVT, PassThru, DAG);
4173     InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger());
4174   }
4175 
4176   SDVTList VTs = DAG.getVTList(PassThru.getSimpleValueType(), MVT::Other);
4177 
4178   if (getGatherScatterIndexIsExtended(Index))
4179     Index = Index.getOperand(0);
4180 
4181   unsigned Opcode = getGatherVecOpcode(IsScaled, IsSigned, IdxNeedsExtend);
4182   selectGatherScatterAddrMode(BasePtr, Index, MemVT, Opcode,
4183                               /*isGather=*/true, DAG);
4184 
4185   if (ResNeedsSignExtend)
4186     Opcode = getSignExtendedGatherOpcode(Opcode);
4187 
4188   SDValue Ops[] = {Chain, Mask, BasePtr, Index, InputVT, PassThru};
4189   SDValue Gather = DAG.getNode(Opcode, DL, VTs, Ops);
4190 
4191   if (VT.isFloatingPoint()) {
4192     SDValue Cast = getSVESafeBitCast(VT, Gather, DAG);
4193     return DAG.getMergeValues({Cast, Gather.getValue(1)}, DL);
4194   }
4195 
4196   return Gather;
4197 }
4198 
4199 SDValue AArch64TargetLowering::LowerMSCATTER(SDValue Op,
4200                                              SelectionDAG &DAG) const {
4201   SDLoc DL(Op);
4202   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Op);
4203   assert(MSC && "Can only custom lower scatter store nodes");
4204 
4205   SDValue Index = MSC->getIndex();
4206   SDValue Chain = MSC->getChain();
4207   SDValue StoreVal = MSC->getValue();
4208   SDValue Mask = MSC->getMask();
4209   SDValue BasePtr = MSC->getBasePtr();
4210 
4211   ISD::MemIndexType IndexType = MSC->getIndexType();
4212   bool IsScaled =
4213       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::UNSIGNED_SCALED;
4214   bool IsSigned =
4215       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::SIGNED_UNSCALED;
4216   bool NeedsExtend =
4217       getGatherScatterIndexIsExtended(Index) ||
4218       Index.getSimpleValueType().getVectorElementType() == MVT::i32;
4219 
4220   EVT VT = StoreVal.getSimpleValueType();
4221   SDVTList VTs = DAG.getVTList(MVT::Other);
4222   EVT MemVT = MSC->getMemoryVT();
4223   SDValue InputVT = DAG.getValueType(MemVT);
4224 
4225   if (VT.getVectorElementType() == MVT::bf16 &&
4226       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
4227     return SDValue();
4228 
4229   // Handle FP data by casting the data so an integer scatter can be used.
4230   if (VT.isFloatingPoint()) {
4231     EVT StoreValVT = getPackedSVEVectorVT(VT.getVectorElementCount());
4232     StoreVal = getSVESafeBitCast(StoreValVT, StoreVal, DAG);
4233     InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger());
4234   }
4235 
4236   if (getGatherScatterIndexIsExtended(Index))
4237     Index = Index.getOperand(0);
4238 
4239   unsigned Opcode = getScatterVecOpcode(IsScaled, IsSigned, NeedsExtend);
4240   selectGatherScatterAddrMode(BasePtr, Index, MemVT, Opcode,
4241                               /*isGather=*/false, DAG);
4242 
4243   SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, InputVT};
4244   return DAG.getNode(Opcode, DL, VTs, Ops);
4245 }
4246 
4247 // Custom lower trunc store for v4i8 vectors, since it is promoted to v4i16.
4248 static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
4249                                         EVT VT, EVT MemVT,
4250                                         SelectionDAG &DAG) {
4251   assert(VT.isVector() && "VT should be a vector type");
4252   assert(MemVT == MVT::v4i8 && VT == MVT::v4i16);
4253 
4254   SDValue Value = ST->getValue();
4255 
4256   // It first extend the promoted v4i16 to v8i16, truncate to v8i8, and extract
4257   // the word lane which represent the v4i8 subvector.  It optimizes the store
4258   // to:
4259   //
4260   //   xtn  v0.8b, v0.8h
4261   //   str  s0, [x0]
4262 
4263   SDValue Undef = DAG.getUNDEF(MVT::i16);
4264   SDValue UndefVec = DAG.getBuildVector(MVT::v4i16, DL,
4265                                         {Undef, Undef, Undef, Undef});
4266 
4267   SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16,
4268                                  Value, UndefVec);
4269   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt);
4270 
4271   Trunc = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Trunc);
4272   SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
4273                                      Trunc, DAG.getConstant(0, DL, MVT::i64));
4274 
4275   return DAG.getStore(ST->getChain(), DL, ExtractTrunc,
4276                       ST->getBasePtr(), ST->getMemOperand());
4277 }
4278 
4279 // Custom lowering for any store, vector or scalar and/or default or with
4280 // a truncate operations.  Currently only custom lower truncate operation
4281 // from vector v4i16 to v4i8 or volatile stores of i128.
4282 SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
4283                                           SelectionDAG &DAG) const {
4284   SDLoc Dl(Op);
4285   StoreSDNode *StoreNode = cast<StoreSDNode>(Op);
4286   assert (StoreNode && "Can only custom lower store nodes");
4287 
4288   SDValue Value = StoreNode->getValue();
4289 
4290   EVT VT = Value.getValueType();
4291   EVT MemVT = StoreNode->getMemoryVT();
4292 
4293   if (VT.isVector()) {
4294     if (useSVEForFixedLengthVectorVT(VT))
4295       return LowerFixedLengthVectorStoreToSVE(Op, DAG);
4296 
4297     unsigned AS = StoreNode->getAddressSpace();
4298     Align Alignment = StoreNode->getAlign();
4299     if (Alignment < MemVT.getStoreSize() &&
4300         !allowsMisalignedMemoryAccesses(MemVT, AS, Alignment,
4301                                         StoreNode->getMemOperand()->getFlags(),
4302                                         nullptr)) {
4303       return scalarizeVectorStore(StoreNode, DAG);
4304     }
4305 
4306     if (StoreNode->isTruncatingStore()) {
4307       return LowerTruncateVectorStore(Dl, StoreNode, VT, MemVT, DAG);
4308     }
4309     // 256 bit non-temporal stores can be lowered to STNP. Do this as part of
4310     // the custom lowering, as there are no un-paired non-temporal stores and
4311     // legalization will break up 256 bit inputs.
4312     ElementCount EC = MemVT.getVectorElementCount();
4313     if (StoreNode->isNonTemporal() && MemVT.getSizeInBits() == 256u &&
4314         EC.isKnownEven() &&
4315         ((MemVT.getScalarSizeInBits() == 8u ||
4316           MemVT.getScalarSizeInBits() == 16u ||
4317           MemVT.getScalarSizeInBits() == 32u ||
4318           MemVT.getScalarSizeInBits() == 64u))) {
4319       SDValue Lo =
4320           DAG.getNode(ISD::EXTRACT_SUBVECTOR, Dl,
4321                       MemVT.getHalfNumVectorElementsVT(*DAG.getContext()),
4322                       StoreNode->getValue(), DAG.getConstant(0, Dl, MVT::i64));
4323       SDValue Hi =
4324           DAG.getNode(ISD::EXTRACT_SUBVECTOR, Dl,
4325                       MemVT.getHalfNumVectorElementsVT(*DAG.getContext()),
4326                       StoreNode->getValue(),
4327                       DAG.getConstant(EC.getKnownMinValue() / 2, Dl, MVT::i64));
4328       SDValue Result = DAG.getMemIntrinsicNode(
4329           AArch64ISD::STNP, Dl, DAG.getVTList(MVT::Other),
4330           {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
4331           StoreNode->getMemoryVT(), StoreNode->getMemOperand());
4332       return Result;
4333     }
4334   } else if (MemVT == MVT::i128 && StoreNode->isVolatile()) {
4335     assert(StoreNode->getValue()->getValueType(0) == MVT::i128);
4336     SDValue Lo =
4337         DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i64, StoreNode->getValue(),
4338                     DAG.getConstant(0, Dl, MVT::i64));
4339     SDValue Hi =
4340         DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i64, StoreNode->getValue(),
4341                     DAG.getConstant(1, Dl, MVT::i64));
4342     SDValue Result = DAG.getMemIntrinsicNode(
4343         AArch64ISD::STP, Dl, DAG.getVTList(MVT::Other),
4344         {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
4345         StoreNode->getMemoryVT(), StoreNode->getMemOperand());
4346     return Result;
4347   }
4348 
4349   return SDValue();
4350 }
4351 
4352 // Generate SUBS and CSEL for integer abs.
4353 SDValue AArch64TargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
4354   MVT VT = Op.getSimpleValueType();
4355 
4356   if (VT.isVector())
4357     return LowerToPredicatedOp(Op, DAG, AArch64ISD::ABS_MERGE_PASSTHRU);
4358 
4359   SDLoc DL(Op);
4360   SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
4361                             Op.getOperand(0));
4362   // Generate SUBS & CSEL.
4363   SDValue Cmp =
4364       DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32),
4365                   Op.getOperand(0), DAG.getConstant(0, DL, VT));
4366   return DAG.getNode(AArch64ISD::CSEL, DL, VT, Op.getOperand(0), Neg,
4367                      DAG.getConstant(AArch64CC::PL, DL, MVT::i32),
4368                      Cmp.getValue(1));
4369 }
4370 
4371 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
4372                                               SelectionDAG &DAG) const {
4373   LLVM_DEBUG(dbgs() << "Custom lowering: ");
4374   LLVM_DEBUG(Op.dump());
4375 
4376   switch (Op.getOpcode()) {
4377   default:
4378     llvm_unreachable("unimplemented operand");
4379     return SDValue();
4380   case ISD::BITCAST:
4381     return LowerBITCAST(Op, DAG);
4382   case ISD::GlobalAddress:
4383     return LowerGlobalAddress(Op, DAG);
4384   case ISD::GlobalTLSAddress:
4385     return LowerGlobalTLSAddress(Op, DAG);
4386   case ISD::SETCC:
4387   case ISD::STRICT_FSETCC:
4388   case ISD::STRICT_FSETCCS:
4389     return LowerSETCC(Op, DAG);
4390   case ISD::BR_CC:
4391     return LowerBR_CC(Op, DAG);
4392   case ISD::SELECT:
4393     return LowerSELECT(Op, DAG);
4394   case ISD::SELECT_CC:
4395     return LowerSELECT_CC(Op, DAG);
4396   case ISD::JumpTable:
4397     return LowerJumpTable(Op, DAG);
4398   case ISD::BR_JT:
4399     return LowerBR_JT(Op, DAG);
4400   case ISD::ConstantPool:
4401     return LowerConstantPool(Op, DAG);
4402   case ISD::BlockAddress:
4403     return LowerBlockAddress(Op, DAG);
4404   case ISD::VASTART:
4405     return LowerVASTART(Op, DAG);
4406   case ISD::VACOPY:
4407     return LowerVACOPY(Op, DAG);
4408   case ISD::VAARG:
4409     return LowerVAARG(Op, DAG);
4410   case ISD::ADDC:
4411   case ISD::ADDE:
4412   case ISD::SUBC:
4413   case ISD::SUBE:
4414     return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
4415   case ISD::SADDO:
4416   case ISD::UADDO:
4417   case ISD::SSUBO:
4418   case ISD::USUBO:
4419   case ISD::SMULO:
4420   case ISD::UMULO:
4421     return LowerXALUO(Op, DAG);
4422   case ISD::FADD:
4423     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FADD_PRED);
4424   case ISD::FSUB:
4425     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSUB_PRED);
4426   case ISD::FMUL:
4427     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMUL_PRED);
4428   case ISD::FMA:
4429     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMA_PRED);
4430   case ISD::FDIV:
4431     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FDIV_PRED);
4432   case ISD::FNEG:
4433     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FNEG_MERGE_PASSTHRU);
4434   case ISD::FCEIL:
4435     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FCEIL_MERGE_PASSTHRU);
4436   case ISD::FFLOOR:
4437     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FFLOOR_MERGE_PASSTHRU);
4438   case ISD::FNEARBYINT:
4439     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FNEARBYINT_MERGE_PASSTHRU);
4440   case ISD::FRINT:
4441     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FRINT_MERGE_PASSTHRU);
4442   case ISD::FROUND:
4443     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FROUND_MERGE_PASSTHRU);
4444   case ISD::FROUNDEVEN:
4445     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU);
4446   case ISD::FTRUNC:
4447     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FTRUNC_MERGE_PASSTHRU);
4448   case ISD::FSQRT:
4449     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSQRT_MERGE_PASSTHRU);
4450   case ISD::FABS:
4451     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FABS_MERGE_PASSTHRU);
4452   case ISD::FP_ROUND:
4453   case ISD::STRICT_FP_ROUND:
4454     return LowerFP_ROUND(Op, DAG);
4455   case ISD::FP_EXTEND:
4456     return LowerFP_EXTEND(Op, DAG);
4457   case ISD::FRAMEADDR:
4458     return LowerFRAMEADDR(Op, DAG);
4459   case ISD::SPONENTRY:
4460     return LowerSPONENTRY(Op, DAG);
4461   case ISD::RETURNADDR:
4462     return LowerRETURNADDR(Op, DAG);
4463   case ISD::ADDROFRETURNADDR:
4464     return LowerADDROFRETURNADDR(Op, DAG);
4465   case ISD::CONCAT_VECTORS:
4466     return LowerCONCAT_VECTORS(Op, DAG);
4467   case ISD::INSERT_VECTOR_ELT:
4468     return LowerINSERT_VECTOR_ELT(Op, DAG);
4469   case ISD::EXTRACT_VECTOR_ELT:
4470     return LowerEXTRACT_VECTOR_ELT(Op, DAG);
4471   case ISD::BUILD_VECTOR:
4472     return LowerBUILD_VECTOR(Op, DAG);
4473   case ISD::VECTOR_SHUFFLE:
4474     return LowerVECTOR_SHUFFLE(Op, DAG);
4475   case ISD::SPLAT_VECTOR:
4476     return LowerSPLAT_VECTOR(Op, DAG);
4477   case ISD::STEP_VECTOR:
4478     return LowerSTEP_VECTOR(Op, DAG);
4479   case ISD::EXTRACT_SUBVECTOR:
4480     return LowerEXTRACT_SUBVECTOR(Op, DAG);
4481   case ISD::INSERT_SUBVECTOR:
4482     return LowerINSERT_SUBVECTOR(Op, DAG);
4483   case ISD::SDIV:
4484   case ISD::UDIV:
4485     return LowerDIV(Op, DAG);
4486   case ISD::SMIN:
4487     return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMIN_PRED,
4488                                /*OverrideNEON=*/true);
4489   case ISD::UMIN:
4490     return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMIN_PRED,
4491                                /*OverrideNEON=*/true);
4492   case ISD::SMAX:
4493     return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMAX_PRED,
4494                                /*OverrideNEON=*/true);
4495   case ISD::UMAX:
4496     return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMAX_PRED,
4497                                /*OverrideNEON=*/true);
4498   case ISD::SRA:
4499   case ISD::SRL:
4500   case ISD::SHL:
4501     return LowerVectorSRA_SRL_SHL(Op, DAG);
4502   case ISD::SHL_PARTS:
4503     return LowerShiftLeftParts(Op, DAG);
4504   case ISD::SRL_PARTS:
4505   case ISD::SRA_PARTS:
4506     return LowerShiftRightParts(Op, DAG);
4507   case ISD::CTPOP:
4508     return LowerCTPOP(Op, DAG);
4509   case ISD::FCOPYSIGN:
4510     return LowerFCOPYSIGN(Op, DAG);
4511   case ISD::OR:
4512     return LowerVectorOR(Op, DAG);
4513   case ISD::XOR:
4514     return LowerXOR(Op, DAG);
4515   case ISD::PREFETCH:
4516     return LowerPREFETCH(Op, DAG);
4517   case ISD::SINT_TO_FP:
4518   case ISD::UINT_TO_FP:
4519   case ISD::STRICT_SINT_TO_FP:
4520   case ISD::STRICT_UINT_TO_FP:
4521     return LowerINT_TO_FP(Op, DAG);
4522   case ISD::FP_TO_SINT:
4523   case ISD::FP_TO_UINT:
4524   case ISD::STRICT_FP_TO_SINT:
4525   case ISD::STRICT_FP_TO_UINT:
4526     return LowerFP_TO_INT(Op, DAG);
4527   case ISD::FSINCOS:
4528     return LowerFSINCOS(Op, DAG);
4529   case ISD::FLT_ROUNDS_:
4530     return LowerFLT_ROUNDS_(Op, DAG);
4531   case ISD::SET_ROUNDING:
4532     return LowerSET_ROUNDING(Op, DAG);
4533   case ISD::MUL:
4534     return LowerMUL(Op, DAG);
4535   case ISD::MULHS:
4536     return LowerToPredicatedOp(Op, DAG, AArch64ISD::MULHS_PRED,
4537                                /*OverrideNEON=*/true);
4538   case ISD::MULHU:
4539     return LowerToPredicatedOp(Op, DAG, AArch64ISD::MULHU_PRED,
4540                                /*OverrideNEON=*/true);
4541   case ISD::INTRINSIC_WO_CHAIN:
4542     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
4543   case ISD::STORE:
4544     return LowerSTORE(Op, DAG);
4545   case ISD::MGATHER:
4546     return LowerMGATHER(Op, DAG);
4547   case ISD::MSCATTER:
4548     return LowerMSCATTER(Op, DAG);
4549   case ISD::VECREDUCE_SEQ_FADD:
4550     return LowerVECREDUCE_SEQ_FADD(Op, DAG);
4551   case ISD::VECREDUCE_ADD:
4552   case ISD::VECREDUCE_AND:
4553   case ISD::VECREDUCE_OR:
4554   case ISD::VECREDUCE_XOR:
4555   case ISD::VECREDUCE_SMAX:
4556   case ISD::VECREDUCE_SMIN:
4557   case ISD::VECREDUCE_UMAX:
4558   case ISD::VECREDUCE_UMIN:
4559   case ISD::VECREDUCE_FADD:
4560   case ISD::VECREDUCE_FMAX:
4561   case ISD::VECREDUCE_FMIN:
4562     return LowerVECREDUCE(Op, DAG);
4563   case ISD::ATOMIC_LOAD_SUB:
4564     return LowerATOMIC_LOAD_SUB(Op, DAG);
4565   case ISD::ATOMIC_LOAD_AND:
4566     return LowerATOMIC_LOAD_AND(Op, DAG);
4567   case ISD::DYNAMIC_STACKALLOC:
4568     return LowerDYNAMIC_STACKALLOC(Op, DAG);
4569   case ISD::VSCALE:
4570     return LowerVSCALE(Op, DAG);
4571   case ISD::ANY_EXTEND:
4572   case ISD::SIGN_EXTEND:
4573   case ISD::ZERO_EXTEND:
4574     return LowerFixedLengthVectorIntExtendToSVE(Op, DAG);
4575   case ISD::SIGN_EXTEND_INREG: {
4576     // Only custom lower when ExtraVT has a legal byte based element type.
4577     EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
4578     EVT ExtraEltVT = ExtraVT.getVectorElementType();
4579     if ((ExtraEltVT != MVT::i8) && (ExtraEltVT != MVT::i16) &&
4580         (ExtraEltVT != MVT::i32) && (ExtraEltVT != MVT::i64))
4581       return SDValue();
4582 
4583     return LowerToPredicatedOp(Op, DAG,
4584                                AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU);
4585   }
4586   case ISD::TRUNCATE:
4587     return LowerTRUNCATE(Op, DAG);
4588   case ISD::LOAD:
4589     if (useSVEForFixedLengthVectorVT(Op.getValueType()))
4590       return LowerFixedLengthVectorLoadToSVE(Op, DAG);
4591     llvm_unreachable("Unexpected request to lower ISD::LOAD");
4592   case ISD::ADD:
4593     return LowerToPredicatedOp(Op, DAG, AArch64ISD::ADD_PRED);
4594   case ISD::AND:
4595     return LowerToScalableOp(Op, DAG);
4596   case ISD::SUB:
4597     return LowerToPredicatedOp(Op, DAG, AArch64ISD::SUB_PRED);
4598   case ISD::FMAXIMUM:
4599     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMAX_PRED);
4600   case ISD::FMAXNUM:
4601     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMAXNM_PRED);
4602   case ISD::FMINIMUM:
4603     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMIN_PRED);
4604   case ISD::FMINNUM:
4605     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMINNM_PRED);
4606   case ISD::VSELECT:
4607     return LowerFixedLengthVectorSelectToSVE(Op, DAG);
4608   case ISD::ABS:
4609     return LowerABS(Op, DAG);
4610   case ISD::BITREVERSE:
4611     return LowerToPredicatedOp(Op, DAG, AArch64ISD::BITREVERSE_MERGE_PASSTHRU,
4612                                /*OverrideNEON=*/true);
4613   case ISD::BSWAP:
4614     return LowerToPredicatedOp(Op, DAG, AArch64ISD::BSWAP_MERGE_PASSTHRU);
4615   case ISD::CTLZ:
4616     return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTLZ_MERGE_PASSTHRU,
4617                                /*OverrideNEON=*/true);
4618   case ISD::CTTZ:
4619     return LowerCTTZ(Op, DAG);
4620   }
4621 }
4622 
4623 bool AArch64TargetLowering::mergeStoresAfterLegalization(EVT VT) const {
4624   return !Subtarget->useSVEForFixedLengthVectors();
4625 }
4626 
4627 bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(
4628     EVT VT, bool OverrideNEON) const {
4629   if (!Subtarget->useSVEForFixedLengthVectors())
4630     return false;
4631 
4632   if (!VT.isFixedLengthVector())
4633     return false;
4634 
4635   // Don't use SVE for vectors we cannot scalarize if required.
4636   switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
4637   // Fixed length predicates should be promoted to i8.
4638   // NOTE: This is consistent with how NEON (and thus 64/128bit vectors) work.
4639   case MVT::i1:
4640   default:
4641     return false;
4642   case MVT::i8:
4643   case MVT::i16:
4644   case MVT::i32:
4645   case MVT::i64:
4646   case MVT::f16:
4647   case MVT::f32:
4648   case MVT::f64:
4649     break;
4650   }
4651 
4652   // All SVE implementations support NEON sized vectors.
4653   if (OverrideNEON && (VT.is128BitVector() || VT.is64BitVector()))
4654     return true;
4655 
4656   // Ensure NEON MVTs only belong to a single register class.
4657   if (VT.getFixedSizeInBits() <= 128)
4658     return false;
4659 
4660   // Don't use SVE for types that don't fit.
4661   if (VT.getFixedSizeInBits() > Subtarget->getMinSVEVectorSizeInBits())
4662     return false;
4663 
4664   // TODO: Perhaps an artificial restriction, but worth having whilst getting
4665   // the base fixed length SVE support in place.
4666   if (!VT.isPow2VectorType())
4667     return false;
4668 
4669   return true;
4670 }
4671 
4672 //===----------------------------------------------------------------------===//
4673 //                      Calling Convention Implementation
4674 //===----------------------------------------------------------------------===//
4675 
4676 /// Selects the correct CCAssignFn for a given CallingConvention value.
4677 CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
4678                                                      bool IsVarArg) const {
4679   switch (CC) {
4680   default:
4681     report_fatal_error("Unsupported calling convention.");
4682   case CallingConv::WebKit_JS:
4683     return CC_AArch64_WebKit_JS;
4684   case CallingConv::GHC:
4685     return CC_AArch64_GHC;
4686   case CallingConv::C:
4687   case CallingConv::Fast:
4688   case CallingConv::PreserveMost:
4689   case CallingConv::CXX_FAST_TLS:
4690   case CallingConv::Swift:
4691     if (Subtarget->isTargetWindows() && IsVarArg)
4692       return CC_AArch64_Win64_VarArg;
4693     if (!Subtarget->isTargetDarwin())
4694       return CC_AArch64_AAPCS;
4695     if (!IsVarArg)
4696       return CC_AArch64_DarwinPCS;
4697     return Subtarget->isTargetILP32() ? CC_AArch64_DarwinPCS_ILP32_VarArg
4698                                       : CC_AArch64_DarwinPCS_VarArg;
4699    case CallingConv::Win64:
4700     return IsVarArg ? CC_AArch64_Win64_VarArg : CC_AArch64_AAPCS;
4701    case CallingConv::CFGuard_Check:
4702      return CC_AArch64_Win64_CFGuard_Check;
4703    case CallingConv::AArch64_VectorCall:
4704    case CallingConv::AArch64_SVE_VectorCall:
4705      return CC_AArch64_AAPCS;
4706   }
4707 }
4708 
4709 CCAssignFn *
4710 AArch64TargetLowering::CCAssignFnForReturn(CallingConv::ID CC) const {
4711   return CC == CallingConv::WebKit_JS ? RetCC_AArch64_WebKit_JS
4712                                       : RetCC_AArch64_AAPCS;
4713 }
4714 
4715 SDValue AArch64TargetLowering::LowerFormalArguments(
4716     SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4717     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
4718     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4719   MachineFunction &MF = DAG.getMachineFunction();
4720   MachineFrameInfo &MFI = MF.getFrameInfo();
4721   bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
4722 
4723   // Assign locations to all of the incoming arguments.
4724   SmallVector<CCValAssign, 16> ArgLocs;
4725   DenseMap<unsigned, SDValue> CopiedRegs;
4726   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4727                  *DAG.getContext());
4728 
4729   // At this point, Ins[].VT may already be promoted to i32. To correctly
4730   // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
4731   // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
4732   // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, here
4733   // we use a special version of AnalyzeFormalArguments to pass in ValVT and
4734   // LocVT.
4735   unsigned NumArgs = Ins.size();
4736   Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
4737   unsigned CurArgIdx = 0;
4738   for (unsigned i = 0; i != NumArgs; ++i) {
4739     MVT ValVT = Ins[i].VT;
4740     if (Ins[i].isOrigArg()) {
4741       std::advance(CurOrigArg, Ins[i].getOrigArgIndex() - CurArgIdx);
4742       CurArgIdx = Ins[i].getOrigArgIndex();
4743 
4744       // Get type of the original argument.
4745       EVT ActualVT = getValueType(DAG.getDataLayout(), CurOrigArg->getType(),
4746                                   /*AllowUnknown*/ true);
4747       MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other;
4748       // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
4749       if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
4750         ValVT = MVT::i8;
4751       else if (ActualMVT == MVT::i16)
4752         ValVT = MVT::i16;
4753     }
4754     bool UseVarArgCC = false;
4755     if (IsWin64)
4756       UseVarArgCC = isVarArg;
4757     CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, UseVarArgCC);
4758     bool Res =
4759         AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, CCInfo);
4760     assert(!Res && "Call operand has unhandled type");
4761     (void)Res;
4762   }
4763   SmallVector<SDValue, 16> ArgValues;
4764   unsigned ExtraArgLocs = 0;
4765   for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
4766     CCValAssign &VA = ArgLocs[i - ExtraArgLocs];
4767 
4768     if (Ins[i].Flags.isByVal()) {
4769       // Byval is used for HFAs in the PCS, but the system should work in a
4770       // non-compliant manner for larger structs.
4771       EVT PtrVT = getPointerTy(DAG.getDataLayout());
4772       int Size = Ins[i].Flags.getByValSize();
4773       unsigned NumRegs = (Size + 7) / 8;
4774 
4775       // FIXME: This works on big-endian for composite byvals, which are the common
4776       // case. It should also work for fundamental types too.
4777       unsigned FrameIdx =
4778         MFI.CreateFixedObject(8 * NumRegs, VA.getLocMemOffset(), false);
4779       SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT);
4780       InVals.push_back(FrameIdxN);
4781 
4782       continue;
4783     }
4784 
4785     SDValue ArgValue;
4786     if (VA.isRegLoc()) {
4787       // Arguments stored in registers.
4788       EVT RegVT = VA.getLocVT();
4789       const TargetRegisterClass *RC;
4790 
4791       if (RegVT == MVT::i32)
4792         RC = &AArch64::GPR32RegClass;
4793       else if (RegVT == MVT::i64)
4794         RC = &AArch64::GPR64RegClass;
4795       else if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4796         RC = &AArch64::FPR16RegClass;
4797       else if (RegVT == MVT::f32)
4798         RC = &AArch64::FPR32RegClass;
4799       else if (RegVT == MVT::f64 || RegVT.is64BitVector())
4800         RC = &AArch64::FPR64RegClass;
4801       else if (RegVT == MVT::f128 || RegVT.is128BitVector())
4802         RC = &AArch64::FPR128RegClass;
4803       else if (RegVT.isScalableVector() &&
4804                RegVT.getVectorElementType() == MVT::i1)
4805         RC = &AArch64::PPRRegClass;
4806       else if (RegVT.isScalableVector())
4807         RC = &AArch64::ZPRRegClass;
4808       else
4809         llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
4810 
4811       // Transform the arguments in physical registers into virtual ones.
4812       unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
4813       ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
4814 
4815       // If this is an 8, 16 or 32-bit value, it is really passed promoted
4816       // to 64 bits.  Insert an assert[sz]ext to capture this, then
4817       // truncate to the right size.
4818       switch (VA.getLocInfo()) {
4819       default:
4820         llvm_unreachable("Unknown loc info!");
4821       case CCValAssign::Full:
4822         break;
4823       case CCValAssign::Indirect:
4824         assert(VA.getValVT().isScalableVector() &&
4825                "Only scalable vectors can be passed indirectly");
4826         break;
4827       case CCValAssign::BCvt:
4828         ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue);
4829         break;
4830       case CCValAssign::AExt:
4831       case CCValAssign::SExt:
4832       case CCValAssign::ZExt:
4833         break;
4834       case CCValAssign::AExtUpper:
4835         ArgValue = DAG.getNode(ISD::SRL, DL, RegVT, ArgValue,
4836                                DAG.getConstant(32, DL, RegVT));
4837         ArgValue = DAG.getZExtOrTrunc(ArgValue, DL, VA.getValVT());
4838         break;
4839       }
4840     } else { // VA.isRegLoc()
4841       assert(VA.isMemLoc() && "CCValAssign is neither reg nor mem");
4842       unsigned ArgOffset = VA.getLocMemOffset();
4843       unsigned ArgSize = (VA.getLocInfo() == CCValAssign::Indirect
4844                               ? VA.getLocVT().getSizeInBits()
4845                               : VA.getValVT().getSizeInBits()) / 8;
4846 
4847       uint32_t BEAlign = 0;
4848       if (!Subtarget->isLittleEndian() && ArgSize < 8 &&
4849           !Ins[i].Flags.isInConsecutiveRegs())
4850         BEAlign = 8 - ArgSize;
4851 
4852       int FI = MFI.CreateFixedObject(ArgSize, ArgOffset + BEAlign, true);
4853 
4854       // Create load nodes to retrieve arguments from the stack.
4855       SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4856 
4857       // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
4858       ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
4859       MVT MemVT = VA.getValVT();
4860 
4861       switch (VA.getLocInfo()) {
4862       default:
4863         break;
4864       case CCValAssign::Trunc:
4865       case CCValAssign::BCvt:
4866         MemVT = VA.getLocVT();
4867         break;
4868       case CCValAssign::Indirect:
4869         assert(VA.getValVT().isScalableVector() &&
4870                "Only scalable vectors can be passed indirectly");
4871         MemVT = VA.getLocVT();
4872         break;
4873       case CCValAssign::SExt:
4874         ExtType = ISD::SEXTLOAD;
4875         break;
4876       case CCValAssign::ZExt:
4877         ExtType = ISD::ZEXTLOAD;
4878         break;
4879       case CCValAssign::AExt:
4880         ExtType = ISD::EXTLOAD;
4881         break;
4882       }
4883 
4884       ArgValue = DAG.getExtLoad(
4885           ExtType, DL, VA.getLocVT(), Chain, FIN,
4886           MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
4887           MemVT);
4888 
4889     }
4890 
4891     if (VA.getLocInfo() == CCValAssign::Indirect) {
4892       assert(VA.getValVT().isScalableVector() &&
4893            "Only scalable vectors can be passed indirectly");
4894 
4895       uint64_t PartSize = VA.getValVT().getStoreSize().getKnownMinSize();
4896       unsigned NumParts = 1;
4897       if (Ins[i].Flags.isInConsecutiveRegs()) {
4898         assert(!Ins[i].Flags.isInConsecutiveRegsLast());
4899         while (!Ins[i + NumParts - 1].Flags.isInConsecutiveRegsLast())
4900           ++NumParts;
4901       }
4902 
4903       MVT PartLoad = VA.getValVT();
4904       SDValue Ptr = ArgValue;
4905 
4906       // Ensure we generate all loads for each tuple part, whilst updating the
4907       // pointer after each load correctly using vscale.
4908       while (NumParts > 0) {
4909         ArgValue = DAG.getLoad(PartLoad, DL, Chain, Ptr, MachinePointerInfo());
4910         InVals.push_back(ArgValue);
4911         NumParts--;
4912         if (NumParts > 0) {
4913           SDValue BytesIncrement = DAG.getVScale(
4914               DL, Ptr.getValueType(),
4915               APInt(Ptr.getValueSizeInBits().getFixedSize(), PartSize));
4916           SDNodeFlags Flags;
4917           Flags.setNoUnsignedWrap(true);
4918           Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
4919                             BytesIncrement, Flags);
4920           ExtraArgLocs++;
4921           i++;
4922         }
4923       }
4924     } else {
4925       if (Subtarget->isTargetILP32() && Ins[i].Flags.isPointer())
4926         ArgValue = DAG.getNode(ISD::AssertZext, DL, ArgValue.getValueType(),
4927                                ArgValue, DAG.getValueType(MVT::i32));
4928       InVals.push_back(ArgValue);
4929     }
4930   }
4931   assert((ArgLocs.size() + ExtraArgLocs) == Ins.size());
4932 
4933   // varargs
4934   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
4935   if (isVarArg) {
4936     if (!Subtarget->isTargetDarwin() || IsWin64) {
4937       // The AAPCS variadic function ABI is identical to the non-variadic
4938       // one. As a result there may be more arguments in registers and we should
4939       // save them for future reference.
4940       // Win64 variadic functions also pass arguments in registers, but all float
4941       // arguments are passed in integer registers.
4942       saveVarArgRegisters(CCInfo, DAG, DL, Chain);
4943     }
4944 
4945     // This will point to the next argument passed via stack.
4946     unsigned StackOffset = CCInfo.getNextStackOffset();
4947     // We currently pass all varargs at 8-byte alignment, or 4 for ILP32
4948     StackOffset = alignTo(StackOffset, Subtarget->isTargetILP32() ? 4 : 8);
4949     FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
4950 
4951     if (MFI.hasMustTailInVarArgFunc()) {
4952       SmallVector<MVT, 2> RegParmTypes;
4953       RegParmTypes.push_back(MVT::i64);
4954       RegParmTypes.push_back(MVT::f128);
4955       // Compute the set of forwarded registers. The rest are scratch.
4956       SmallVectorImpl<ForwardedRegister> &Forwards =
4957                                        FuncInfo->getForwardedMustTailRegParms();
4958       CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes,
4959                                                CC_AArch64_AAPCS);
4960 
4961       // Conservatively forward X8, since it might be used for aggregate return.
4962       if (!CCInfo.isAllocated(AArch64::X8)) {
4963         unsigned X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
4964         Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
4965       }
4966     }
4967   }
4968 
4969   // On Windows, InReg pointers must be returned, so record the pointer in a
4970   // virtual register at the start of the function so it can be returned in the
4971   // epilogue.
4972   if (IsWin64) {
4973     for (unsigned I = 0, E = Ins.size(); I != E; ++I) {
4974       if (Ins[I].Flags.isInReg()) {
4975         assert(!FuncInfo->getSRetReturnReg());
4976 
4977         MVT PtrTy = getPointerTy(DAG.getDataLayout());
4978         Register Reg =
4979             MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
4980         FuncInfo->setSRetReturnReg(Reg);
4981 
4982         SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[I]);
4983         Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Copy, Chain);
4984         break;
4985       }
4986     }
4987   }
4988 
4989   unsigned StackArgSize = CCInfo.getNextStackOffset();
4990   bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4991   if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) {
4992     // This is a non-standard ABI so by fiat I say we're allowed to make full
4993     // use of the stack area to be popped, which must be aligned to 16 bytes in
4994     // any case:
4995     StackArgSize = alignTo(StackArgSize, 16);
4996 
4997     // If we're expected to restore the stack (e.g. fastcc) then we'll be adding
4998     // a multiple of 16.
4999     FuncInfo->setArgumentStackToRestore(StackArgSize);
5000 
5001     // This realignment carries over to the available bytes below. Our own
5002     // callers will guarantee the space is free by giving an aligned value to
5003     // CALLSEQ_START.
5004   }
5005   // Even if we're not expected to free up the space, it's useful to know how
5006   // much is there while considering tail calls (because we can reuse it).
5007   FuncInfo->setBytesInStackArgArea(StackArgSize);
5008 
5009   if (Subtarget->hasCustomCallingConv())
5010     Subtarget->getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
5011 
5012   return Chain;
5013 }
5014 
5015 void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
5016                                                 SelectionDAG &DAG,
5017                                                 const SDLoc &DL,
5018                                                 SDValue &Chain) const {
5019   MachineFunction &MF = DAG.getMachineFunction();
5020   MachineFrameInfo &MFI = MF.getFrameInfo();
5021   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
5022   auto PtrVT = getPointerTy(DAG.getDataLayout());
5023   bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
5024 
5025   SmallVector<SDValue, 8> MemOps;
5026 
5027   static const MCPhysReg GPRArgRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2,
5028                                           AArch64::X3, AArch64::X4, AArch64::X5,
5029                                           AArch64::X6, AArch64::X7 };
5030   static const unsigned NumGPRArgRegs = array_lengthof(GPRArgRegs);
5031   unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(GPRArgRegs);
5032 
5033   unsigned GPRSaveSize = 8 * (NumGPRArgRegs - FirstVariadicGPR);
5034   int GPRIdx = 0;
5035   if (GPRSaveSize != 0) {
5036     if (IsWin64) {
5037       GPRIdx = MFI.CreateFixedObject(GPRSaveSize, -(int)GPRSaveSize, false);
5038       if (GPRSaveSize & 15)
5039         // The extra size here, if triggered, will always be 8.
5040         MFI.CreateFixedObject(16 - (GPRSaveSize & 15), -(int)alignTo(GPRSaveSize, 16), false);
5041     } else
5042       GPRIdx = MFI.CreateStackObject(GPRSaveSize, Align(8), false);
5043 
5044     SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT);
5045 
5046     for (unsigned i = FirstVariadicGPR; i < NumGPRArgRegs; ++i) {
5047       unsigned VReg = MF.addLiveIn(GPRArgRegs[i], &AArch64::GPR64RegClass);
5048       SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
5049       SDValue Store = DAG.getStore(
5050           Val.getValue(1), DL, Val, FIN,
5051           IsWin64
5052               ? MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
5053                                                   GPRIdx,
5054                                                   (i - FirstVariadicGPR) * 8)
5055               : MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 8));
5056       MemOps.push_back(Store);
5057       FIN =
5058           DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT));
5059     }
5060   }
5061   FuncInfo->setVarArgsGPRIndex(GPRIdx);
5062   FuncInfo->setVarArgsGPRSize(GPRSaveSize);
5063 
5064   if (Subtarget->hasFPARMv8() && !IsWin64) {
5065     static const MCPhysReg FPRArgRegs[] = {
5066         AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3,
5067         AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7};
5068     static const unsigned NumFPRArgRegs = array_lengthof(FPRArgRegs);
5069     unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(FPRArgRegs);
5070 
5071     unsigned FPRSaveSize = 16 * (NumFPRArgRegs - FirstVariadicFPR);
5072     int FPRIdx = 0;
5073     if (FPRSaveSize != 0) {
5074       FPRIdx = MFI.CreateStackObject(FPRSaveSize, Align(16), false);
5075 
5076       SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
5077 
5078       for (unsigned i = FirstVariadicFPR; i < NumFPRArgRegs; ++i) {
5079         unsigned VReg = MF.addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass);
5080         SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
5081 
5082         SDValue Store = DAG.getStore(
5083             Val.getValue(1), DL, Val, FIN,
5084             MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 16));
5085         MemOps.push_back(Store);
5086         FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
5087                           DAG.getConstant(16, DL, PtrVT));
5088       }
5089     }
5090     FuncInfo->setVarArgsFPRIndex(FPRIdx);
5091     FuncInfo->setVarArgsFPRSize(FPRSaveSize);
5092   }
5093 
5094   if (!MemOps.empty()) {
5095     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
5096   }
5097 }
5098 
5099 /// LowerCallResult - Lower the result values of a call into the
5100 /// appropriate copies out of appropriate physical registers.
5101 SDValue AArch64TargetLowering::LowerCallResult(
5102     SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
5103     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
5104     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
5105     SDValue ThisVal) const {
5106   CCAssignFn *RetCC = CCAssignFnForReturn(CallConv);
5107   // Assign locations to each value returned by this call.
5108   SmallVector<CCValAssign, 16> RVLocs;
5109   DenseMap<unsigned, SDValue> CopiedRegs;
5110   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
5111                  *DAG.getContext());
5112   CCInfo.AnalyzeCallResult(Ins, RetCC);
5113 
5114   // Copy all of the result registers out of their specified physreg.
5115   for (unsigned i = 0; i != RVLocs.size(); ++i) {
5116     CCValAssign VA = RVLocs[i];
5117 
5118     // Pass 'this' value directly from the argument to return value, to avoid
5119     // reg unit interference
5120     if (i == 0 && isThisReturn) {
5121       assert(!VA.needsCustom() && VA.getLocVT() == MVT::i64 &&
5122              "unexpected return calling convention register assignment");
5123       InVals.push_back(ThisVal);
5124       continue;
5125     }
5126 
5127     // Avoid copying a physreg twice since RegAllocFast is incompetent and only
5128     // allows one use of a physreg per block.
5129     SDValue Val = CopiedRegs.lookup(VA.getLocReg());
5130     if (!Val) {
5131       Val =
5132           DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag);
5133       Chain = Val.getValue(1);
5134       InFlag = Val.getValue(2);
5135       CopiedRegs[VA.getLocReg()] = Val;
5136     }
5137 
5138     switch (VA.getLocInfo()) {
5139     default:
5140       llvm_unreachable("Unknown loc info!");
5141     case CCValAssign::Full:
5142       break;
5143     case CCValAssign::BCvt:
5144       Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
5145       break;
5146     case CCValAssign::AExtUpper:
5147       Val = DAG.getNode(ISD::SRL, DL, VA.getLocVT(), Val,
5148                         DAG.getConstant(32, DL, VA.getLocVT()));
5149       LLVM_FALLTHROUGH;
5150     case CCValAssign::AExt:
5151       LLVM_FALLTHROUGH;
5152     case CCValAssign::ZExt:
5153       Val = DAG.getZExtOrTrunc(Val, DL, VA.getValVT());
5154       break;
5155     }
5156 
5157     InVals.push_back(Val);
5158   }
5159 
5160   return Chain;
5161 }
5162 
5163 /// Return true if the calling convention is one that we can guarantee TCO for.
5164 static bool canGuaranteeTCO(CallingConv::ID CC) {
5165   return CC == CallingConv::Fast;
5166 }
5167 
5168 /// Return true if we might ever do TCO for calls with this calling convention.
5169 static bool mayTailCallThisCC(CallingConv::ID CC) {
5170   switch (CC) {
5171   case CallingConv::C:
5172   case CallingConv::AArch64_SVE_VectorCall:
5173   case CallingConv::PreserveMost:
5174   case CallingConv::Swift:
5175     return true;
5176   default:
5177     return canGuaranteeTCO(CC);
5178   }
5179 }
5180 
5181 bool AArch64TargetLowering::isEligibleForTailCallOptimization(
5182     SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
5183     const SmallVectorImpl<ISD::OutputArg> &Outs,
5184     const SmallVectorImpl<SDValue> &OutVals,
5185     const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
5186   if (!mayTailCallThisCC(CalleeCC))
5187     return false;
5188 
5189   MachineFunction &MF = DAG.getMachineFunction();
5190   const Function &CallerF = MF.getFunction();
5191   CallingConv::ID CallerCC = CallerF.getCallingConv();
5192 
5193   // Functions using the C or Fast calling convention that have an SVE signature
5194   // preserve more registers and should assume the SVE_VectorCall CC.
5195   // The check for matching callee-saved regs will determine whether it is
5196   // eligible for TCO.
5197   if ((CallerCC == CallingConv::C || CallerCC == CallingConv::Fast) &&
5198       AArch64RegisterInfo::hasSVEArgsOrReturn(&MF))
5199     CallerCC = CallingConv::AArch64_SVE_VectorCall;
5200 
5201   bool CCMatch = CallerCC == CalleeCC;
5202 
5203   // When using the Windows calling convention on a non-windows OS, we want
5204   // to back up and restore X18 in such functions; we can't do a tail call
5205   // from those functions.
5206   if (CallerCC == CallingConv::Win64 && !Subtarget->isTargetWindows() &&
5207       CalleeCC != CallingConv::Win64)
5208     return false;
5209 
5210   // Byval parameters hand the function a pointer directly into the stack area
5211   // we want to reuse during a tail call. Working around this *is* possible (see
5212   // X86) but less efficient and uglier in LowerCall.
5213   for (Function::const_arg_iterator i = CallerF.arg_begin(),
5214                                     e = CallerF.arg_end();
5215        i != e; ++i) {
5216     if (i->hasByValAttr())
5217       return false;
5218 
5219     // On Windows, "inreg" attributes signify non-aggregate indirect returns.
5220     // In this case, it is necessary to save/restore X0 in the callee. Tail
5221     // call opt interferes with this. So we disable tail call opt when the
5222     // caller has an argument with "inreg" attribute.
5223 
5224     // FIXME: Check whether the callee also has an "inreg" argument.
5225     if (i->hasInRegAttr())
5226       return false;
5227   }
5228 
5229   if (getTargetMachine().Options.GuaranteedTailCallOpt)
5230     return canGuaranteeTCO(CalleeCC) && CCMatch;
5231 
5232   // Externally-defined functions with weak linkage should not be
5233   // tail-called on AArch64 when the OS does not support dynamic
5234   // pre-emption of symbols, as the AAELF spec requires normal calls
5235   // to undefined weak functions to be replaced with a NOP or jump to the
5236   // next instruction. The behaviour of branch instructions in this
5237   // situation (as used for tail calls) is implementation-defined, so we
5238   // cannot rely on the linker replacing the tail call with a return.
5239   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
5240     const GlobalValue *GV = G->getGlobal();
5241     const Triple &TT = getTargetMachine().getTargetTriple();
5242     if (GV->hasExternalWeakLinkage() &&
5243         (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
5244       return false;
5245   }
5246 
5247   // Now we search for cases where we can use a tail call without changing the
5248   // ABI. Sibcall is used in some places (particularly gcc) to refer to this
5249   // concept.
5250 
5251   // I want anyone implementing a new calling convention to think long and hard
5252   // about this assert.
5253   assert((!isVarArg || CalleeCC == CallingConv::C) &&
5254          "Unexpected variadic calling convention");
5255 
5256   LLVMContext &C = *DAG.getContext();
5257   if (isVarArg && !Outs.empty()) {
5258     // At least two cases here: if caller is fastcc then we can't have any
5259     // memory arguments (we'd be expected to clean up the stack afterwards). If
5260     // caller is C then we could potentially use its argument area.
5261 
5262     // FIXME: for now we take the most conservative of these in both cases:
5263     // disallow all variadic memory operands.
5264     SmallVector<CCValAssign, 16> ArgLocs;
5265     CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
5266 
5267     CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, true));
5268     for (const CCValAssign &ArgLoc : ArgLocs)
5269       if (!ArgLoc.isRegLoc())
5270         return false;
5271   }
5272 
5273   // Check that the call results are passed in the same way.
5274   if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
5275                                   CCAssignFnForCall(CalleeCC, isVarArg),
5276                                   CCAssignFnForCall(CallerCC, isVarArg)))
5277     return false;
5278   // The callee has to preserve all registers the caller needs to preserve.
5279   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
5280   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
5281   if (!CCMatch) {
5282     const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
5283     if (Subtarget->hasCustomCallingConv()) {
5284       TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
5285       TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
5286     }
5287     if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
5288       return false;
5289   }
5290 
5291   // Nothing more to check if the callee is taking no arguments
5292   if (Outs.empty())
5293     return true;
5294 
5295   SmallVector<CCValAssign, 16> ArgLocs;
5296   CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
5297 
5298   CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg));
5299 
5300   const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
5301 
5302   // If any of the arguments is passed indirectly, it must be SVE, so the
5303   // 'getBytesInStackArgArea' is not sufficient to determine whether we need to
5304   // allocate space on the stack. That is why we determine this explicitly here
5305   // the call cannot be a tailcall.
5306   if (llvm::any_of(ArgLocs, [](CCValAssign &A) {
5307         assert((A.getLocInfo() != CCValAssign::Indirect ||
5308                 A.getValVT().isScalableVector()) &&
5309                "Expected value to be scalable");
5310         return A.getLocInfo() == CCValAssign::Indirect;
5311       }))
5312     return false;
5313 
5314   // If the stack arguments for this call do not fit into our own save area then
5315   // the call cannot be made tail.
5316   if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea())
5317     return false;
5318 
5319   const MachineRegisterInfo &MRI = MF.getRegInfo();
5320   if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
5321     return false;
5322 
5323   return true;
5324 }
5325 
5326 SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
5327                                                    SelectionDAG &DAG,
5328                                                    MachineFrameInfo &MFI,
5329                                                    int ClobberedFI) const {
5330   SmallVector<SDValue, 8> ArgChains;
5331   int64_t FirstByte = MFI.getObjectOffset(ClobberedFI);
5332   int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1;
5333 
5334   // Include the original chain at the beginning of the list. When this is
5335   // used by target LowerCall hooks, this helps legalize find the
5336   // CALLSEQ_BEGIN node.
5337   ArgChains.push_back(Chain);
5338 
5339   // Add a chain value for each stack argument corresponding
5340   for (SDNode::use_iterator U = DAG.getEntryNode().getNode()->use_begin(),
5341                             UE = DAG.getEntryNode().getNode()->use_end();
5342        U != UE; ++U)
5343     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
5344       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
5345         if (FI->getIndex() < 0) {
5346           int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex());
5347           int64_t InLastByte = InFirstByte;
5348           InLastByte += MFI.getObjectSize(FI->getIndex()) - 1;
5349 
5350           if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
5351               (FirstByte <= InFirstByte && InFirstByte <= LastByte))
5352             ArgChains.push_back(SDValue(L, 1));
5353         }
5354 
5355   // Build a tokenfactor for all the chains.
5356   return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
5357 }
5358 
5359 bool AArch64TargetLowering::DoesCalleeRestoreStack(CallingConv::ID CallCC,
5360                                                    bool TailCallOpt) const {
5361   return CallCC == CallingConv::Fast && TailCallOpt;
5362 }
5363 
5364 /// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain,
5365 /// and add input and output parameter nodes.
5366 SDValue
5367 AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
5368                                  SmallVectorImpl<SDValue> &InVals) const {
5369   SelectionDAG &DAG = CLI.DAG;
5370   SDLoc &DL = CLI.DL;
5371   SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
5372   SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
5373   SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
5374   SDValue Chain = CLI.Chain;
5375   SDValue Callee = CLI.Callee;
5376   bool &IsTailCall = CLI.IsTailCall;
5377   CallingConv::ID CallConv = CLI.CallConv;
5378   bool IsVarArg = CLI.IsVarArg;
5379 
5380   MachineFunction &MF = DAG.getMachineFunction();
5381   MachineFunction::CallSiteInfo CSInfo;
5382   bool IsThisReturn = false;
5383 
5384   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
5385   bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
5386   bool IsSibCall = false;
5387   bool IsCalleeWin64 = Subtarget->isCallingConvWin64(CallConv);
5388 
5389   // Check callee args/returns for SVE registers and set calling convention
5390   // accordingly.
5391   if (CallConv == CallingConv::C || CallConv == CallingConv::Fast) {
5392     bool CalleeOutSVE = any_of(Outs, [](ISD::OutputArg &Out){
5393       return Out.VT.isScalableVector();
5394     });
5395     bool CalleeInSVE = any_of(Ins, [](ISD::InputArg &In){
5396       return In.VT.isScalableVector();
5397     });
5398 
5399     if (CalleeInSVE || CalleeOutSVE)
5400       CallConv = CallingConv::AArch64_SVE_VectorCall;
5401   }
5402 
5403   if (IsTailCall) {
5404     // Check if it's really possible to do a tail call.
5405     IsTailCall = isEligibleForTailCallOptimization(
5406         Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
5407     if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall())
5408       report_fatal_error("failed to perform tail call elimination on a call "
5409                          "site marked musttail");
5410 
5411     // A sibling call is one where we're under the usual C ABI and not planning
5412     // to change that but can still do a tail call:
5413     if (!TailCallOpt && IsTailCall)
5414       IsSibCall = true;
5415 
5416     if (IsTailCall)
5417       ++NumTailCalls;
5418   }
5419 
5420   // Analyze operands of the call, assigning locations to each operand.
5421   SmallVector<CCValAssign, 16> ArgLocs;
5422   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs,
5423                  *DAG.getContext());
5424 
5425   if (IsVarArg) {
5426     // Handle fixed and variable vector arguments differently.
5427     // Variable vector arguments always go into memory.
5428     unsigned NumArgs = Outs.size();
5429 
5430     for (unsigned i = 0; i != NumArgs; ++i) {
5431       MVT ArgVT = Outs[i].VT;
5432       if (!Outs[i].IsFixed && ArgVT.isScalableVector())
5433         report_fatal_error("Passing SVE types to variadic functions is "
5434                            "currently not supported");
5435 
5436       ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
5437       bool UseVarArgCC = !Outs[i].IsFixed;
5438       // On Windows, the fixed arguments in a vararg call are passed in GPRs
5439       // too, so use the vararg CC to force them to integer registers.
5440       if (IsCalleeWin64)
5441         UseVarArgCC = true;
5442       CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, UseVarArgCC);
5443       bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
5444       assert(!Res && "Call operand has unhandled type");
5445       (void)Res;
5446     }
5447   } else {
5448     // At this point, Outs[].VT may already be promoted to i32. To correctly
5449     // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
5450     // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
5451     // Since AnalyzeCallOperands uses Ins[].VT for both ValVT and LocVT, here
5452     // we use a special version of AnalyzeCallOperands to pass in ValVT and
5453     // LocVT.
5454     unsigned NumArgs = Outs.size();
5455     for (unsigned i = 0; i != NumArgs; ++i) {
5456       MVT ValVT = Outs[i].VT;
5457       // Get type of the original argument.
5458       EVT ActualVT = getValueType(DAG.getDataLayout(),
5459                                   CLI.getArgs()[Outs[i].OrigArgIndex].Ty,
5460                                   /*AllowUnknown*/ true);
5461       MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT;
5462       ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
5463       // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
5464       if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
5465         ValVT = MVT::i8;
5466       else if (ActualMVT == MVT::i16)
5467         ValVT = MVT::i16;
5468 
5469       CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
5470       bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, ArgFlags, CCInfo);
5471       assert(!Res && "Call operand has unhandled type");
5472       (void)Res;
5473     }
5474   }
5475 
5476   // Get a count of how many bytes are to be pushed on the stack.
5477   unsigned NumBytes = CCInfo.getNextStackOffset();
5478 
5479   if (IsSibCall) {
5480     // Since we're not changing the ABI to make this a tail call, the memory
5481     // operands are already available in the caller's incoming argument space.
5482     NumBytes = 0;
5483   }
5484 
5485   // FPDiff is the byte offset of the call's argument area from the callee's.
5486   // Stores to callee stack arguments will be placed in FixedStackSlots offset
5487   // by this amount for a tail call. In a sibling call it must be 0 because the
5488   // caller will deallocate the entire stack and the callee still expects its
5489   // arguments to begin at SP+0. Completely unused for non-tail calls.
5490   int FPDiff = 0;
5491 
5492   if (IsTailCall && !IsSibCall) {
5493     unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
5494 
5495     // Since callee will pop argument stack as a tail call, we must keep the
5496     // popped size 16-byte aligned.
5497     NumBytes = alignTo(NumBytes, 16);
5498 
5499     // FPDiff will be negative if this tail call requires more space than we
5500     // would automatically have in our incoming argument space. Positive if we
5501     // can actually shrink the stack.
5502     FPDiff = NumReusableBytes - NumBytes;
5503 
5504     // The stack pointer must be 16-byte aligned at all times it's used for a
5505     // memory operation, which in practice means at *all* times and in
5506     // particular across call boundaries. Therefore our own arguments started at
5507     // a 16-byte aligned SP and the delta applied for the tail call should
5508     // satisfy the same constraint.
5509     assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
5510   }
5511 
5512   // Adjust the stack pointer for the new arguments...
5513   // These operations are automatically eliminated by the prolog/epilog pass
5514   if (!IsSibCall)
5515     Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
5516 
5517   SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
5518                                         getPointerTy(DAG.getDataLayout()));
5519 
5520   SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
5521   SmallSet<unsigned, 8> RegsUsed;
5522   SmallVector<SDValue, 8> MemOpChains;
5523   auto PtrVT = getPointerTy(DAG.getDataLayout());
5524 
5525   if (IsVarArg && CLI.CB && CLI.CB->isMustTailCall()) {
5526     const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
5527     for (const auto &F : Forwards) {
5528       SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT);
5529        RegsToPass.emplace_back(F.PReg, Val);
5530     }
5531   }
5532 
5533   // Walk the register/memloc assignments, inserting copies/loads.
5534   unsigned ExtraArgLocs = 0;
5535   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
5536     CCValAssign &VA = ArgLocs[i - ExtraArgLocs];
5537     SDValue Arg = OutVals[i];
5538     ISD::ArgFlagsTy Flags = Outs[i].Flags;
5539 
5540     // Promote the value if needed.
5541     switch (VA.getLocInfo()) {
5542     default:
5543       llvm_unreachable("Unknown loc info!");
5544     case CCValAssign::Full:
5545       break;
5546     case CCValAssign::SExt:
5547       Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
5548       break;
5549     case CCValAssign::ZExt:
5550       Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
5551       break;
5552     case CCValAssign::AExt:
5553       if (Outs[i].ArgVT == MVT::i1) {
5554         // AAPCS requires i1 to be zero-extended to 8-bits by the caller.
5555         Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
5556         Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg);
5557       }
5558       Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
5559       break;
5560     case CCValAssign::AExtUpper:
5561       assert(VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits");
5562       Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
5563       Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg,
5564                         DAG.getConstant(32, DL, VA.getLocVT()));
5565       break;
5566     case CCValAssign::BCvt:
5567       Arg = DAG.getBitcast(VA.getLocVT(), Arg);
5568       break;
5569     case CCValAssign::Trunc:
5570       Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
5571       break;
5572     case CCValAssign::FPExt:
5573       Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
5574       break;
5575     case CCValAssign::Indirect:
5576       assert(VA.getValVT().isScalableVector() &&
5577              "Only scalable vectors can be passed indirectly");
5578 
5579       uint64_t StoreSize = VA.getValVT().getStoreSize().getKnownMinSize();
5580       uint64_t PartSize = StoreSize;
5581       unsigned NumParts = 1;
5582       if (Outs[i].Flags.isInConsecutiveRegs()) {
5583         assert(!Outs[i].Flags.isInConsecutiveRegsLast());
5584         while (!Outs[i + NumParts - 1].Flags.isInConsecutiveRegsLast())
5585           ++NumParts;
5586         StoreSize *= NumParts;
5587       }
5588 
5589       MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5590       Type *Ty = EVT(VA.getValVT()).getTypeForEVT(*DAG.getContext());
5591       Align Alignment = DAG.getDataLayout().getPrefTypeAlign(Ty);
5592       int FI = MFI.CreateStackObject(StoreSize, Alignment, false);
5593       MFI.setStackID(FI, TargetStackID::ScalableVector);
5594 
5595       MachinePointerInfo MPI =
5596           MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
5597       SDValue Ptr = DAG.getFrameIndex(
5598           FI, DAG.getTargetLoweringInfo().getFrameIndexTy(DAG.getDataLayout()));
5599       SDValue SpillSlot = Ptr;
5600 
5601       // Ensure we generate all stores for each tuple part, whilst updating the
5602       // pointer after each store correctly using vscale.
5603       while (NumParts) {
5604         Chain = DAG.getStore(Chain, DL, OutVals[i], Ptr, MPI);
5605         NumParts--;
5606         if (NumParts > 0) {
5607           SDValue BytesIncrement = DAG.getVScale(
5608               DL, Ptr.getValueType(),
5609               APInt(Ptr.getValueSizeInBits().getFixedSize(), PartSize));
5610           SDNodeFlags Flags;
5611           Flags.setNoUnsignedWrap(true);
5612 
5613           MPI = MachinePointerInfo(MPI.getAddrSpace());
5614           Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
5615                             BytesIncrement, Flags);
5616           ExtraArgLocs++;
5617           i++;
5618         }
5619       }
5620 
5621       Arg = SpillSlot;
5622       break;
5623     }
5624 
5625     if (VA.isRegLoc()) {
5626       if (i == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
5627           Outs[0].VT == MVT::i64) {
5628         assert(VA.getLocVT() == MVT::i64 &&
5629                "unexpected calling convention register assignment");
5630         assert(!Ins.empty() && Ins[0].VT == MVT::i64 &&
5631                "unexpected use of 'returned'");
5632         IsThisReturn = true;
5633       }
5634       if (RegsUsed.count(VA.getLocReg())) {
5635         // If this register has already been used then we're trying to pack
5636         // parts of an [N x i32] into an X-register. The extension type will
5637         // take care of putting the two halves in the right place but we have to
5638         // combine them.
5639         SDValue &Bits =
5640             llvm::find_if(RegsToPass,
5641                           [=](const std::pair<unsigned, SDValue> &Elt) {
5642                             return Elt.first == VA.getLocReg();
5643                           })
5644                 ->second;
5645         Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg);
5646         // Call site info is used for function's parameter entry value
5647         // tracking. For now we track only simple cases when parameter
5648         // is transferred through whole register.
5649         llvm::erase_if(CSInfo, [&VA](MachineFunction::ArgRegPair ArgReg) {
5650           return ArgReg.Reg == VA.getLocReg();
5651         });
5652       } else {
5653         RegsToPass.emplace_back(VA.getLocReg(), Arg);
5654         RegsUsed.insert(VA.getLocReg());
5655         const TargetOptions &Options = DAG.getTarget().Options;
5656         if (Options.EmitCallSiteInfo)
5657           CSInfo.emplace_back(VA.getLocReg(), i);
5658       }
5659     } else {
5660       assert(VA.isMemLoc());
5661 
5662       SDValue DstAddr;
5663       MachinePointerInfo DstInfo;
5664 
5665       // FIXME: This works on big-endian for composite byvals, which are the
5666       // common case. It should also work for fundamental types too.
5667       uint32_t BEAlign = 0;
5668       unsigned OpSize;
5669       if (VA.getLocInfo() == CCValAssign::Indirect)
5670         OpSize = VA.getLocVT().getFixedSizeInBits();
5671       else
5672         OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
5673                                  : VA.getValVT().getSizeInBits();
5674       OpSize = (OpSize + 7) / 8;
5675       if (!Subtarget->isLittleEndian() && !Flags.isByVal() &&
5676           !Flags.isInConsecutiveRegs()) {
5677         if (OpSize < 8)
5678           BEAlign = 8 - OpSize;
5679       }
5680       unsigned LocMemOffset = VA.getLocMemOffset();
5681       int32_t Offset = LocMemOffset + BEAlign;
5682       SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
5683       PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
5684 
5685       if (IsTailCall) {
5686         Offset = Offset + FPDiff;
5687         int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
5688 
5689         DstAddr = DAG.getFrameIndex(FI, PtrVT);
5690         DstInfo =
5691             MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
5692 
5693         // Make sure any stack arguments overlapping with where we're storing
5694         // are loaded before this eventual operation. Otherwise they'll be
5695         // clobbered.
5696         Chain = addTokenForArgument(Chain, DAG, MF.getFrameInfo(), FI);
5697       } else {
5698         SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
5699 
5700         DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
5701         DstInfo = MachinePointerInfo::getStack(DAG.getMachineFunction(),
5702                                                LocMemOffset);
5703       }
5704 
5705       if (Outs[i].Flags.isByVal()) {
5706         SDValue SizeNode =
5707             DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i64);
5708         SDValue Cpy = DAG.getMemcpy(
5709             Chain, DL, DstAddr, Arg, SizeNode,
5710             Outs[i].Flags.getNonZeroByValAlign(),
5711             /*isVol = */ false, /*AlwaysInline = */ false,
5712             /*isTailCall = */ false, DstInfo, MachinePointerInfo());
5713 
5714         MemOpChains.push_back(Cpy);
5715       } else {
5716         // Since we pass i1/i8/i16 as i1/i8/i16 on stack and Arg is already
5717         // promoted to a legal register type i32, we should truncate Arg back to
5718         // i1/i8/i16.
5719         if (VA.getValVT() == MVT::i1 || VA.getValVT() == MVT::i8 ||
5720             VA.getValVT() == MVT::i16)
5721           Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg);
5722 
5723         SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo);
5724         MemOpChains.push_back(Store);
5725       }
5726     }
5727   }
5728 
5729   if (!MemOpChains.empty())
5730     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
5731 
5732   // Build a sequence of copy-to-reg nodes chained together with token chain
5733   // and flag operands which copy the outgoing args into the appropriate regs.
5734   SDValue InFlag;
5735   for (auto &RegToPass : RegsToPass) {
5736     Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
5737                              RegToPass.second, InFlag);
5738     InFlag = Chain.getValue(1);
5739   }
5740 
5741   // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
5742   // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
5743   // node so that legalize doesn't hack it.
5744   if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
5745     auto GV = G->getGlobal();
5746     unsigned OpFlags =
5747         Subtarget->classifyGlobalFunctionReference(GV, getTargetMachine());
5748     if (OpFlags & AArch64II::MO_GOT) {
5749       Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
5750       Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
5751     } else {
5752       const GlobalValue *GV = G->getGlobal();
5753       Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0);
5754     }
5755   } else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
5756     if (getTargetMachine().getCodeModel() == CodeModel::Large &&
5757         Subtarget->isTargetMachO()) {
5758       const char *Sym = S->getSymbol();
5759       Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, AArch64II::MO_GOT);
5760       Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
5761     } else {
5762       const char *Sym = S->getSymbol();
5763       Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, 0);
5764     }
5765   }
5766 
5767   // We don't usually want to end the call-sequence here because we would tidy
5768   // the frame up *after* the call, however in the ABI-changing tail-call case
5769   // we've carefully laid out the parameters so that when sp is reset they'll be
5770   // in the correct location.
5771   if (IsTailCall && !IsSibCall) {
5772     Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
5773                                DAG.getIntPtrConstant(0, DL, true), InFlag, DL);
5774     InFlag = Chain.getValue(1);
5775   }
5776 
5777   std::vector<SDValue> Ops;
5778   Ops.push_back(Chain);
5779   Ops.push_back(Callee);
5780 
5781   if (IsTailCall) {
5782     // Each tail call may have to adjust the stack by a different amount, so
5783     // this information must travel along with the operation for eventual
5784     // consumption by emitEpilogue.
5785     Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
5786   }
5787 
5788   // Add argument registers to the end of the list so that they are known live
5789   // into the call.
5790   for (auto &RegToPass : RegsToPass)
5791     Ops.push_back(DAG.getRegister(RegToPass.first,
5792                                   RegToPass.second.getValueType()));
5793 
5794   // Add a register mask operand representing the call-preserved registers.
5795   const uint32_t *Mask;
5796   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
5797   if (IsThisReturn) {
5798     // For 'this' returns, use the X0-preserving mask if applicable
5799     Mask = TRI->getThisReturnPreservedMask(MF, CallConv);
5800     if (!Mask) {
5801       IsThisReturn = false;
5802       Mask = TRI->getCallPreservedMask(MF, CallConv);
5803     }
5804   } else
5805     Mask = TRI->getCallPreservedMask(MF, CallConv);
5806 
5807   if (Subtarget->hasCustomCallingConv())
5808     TRI->UpdateCustomCallPreservedMask(MF, &Mask);
5809 
5810   if (TRI->isAnyArgRegReserved(MF))
5811     TRI->emitReservedArgRegCallError(MF);
5812 
5813   assert(Mask && "Missing call preserved mask for calling convention");
5814   Ops.push_back(DAG.getRegisterMask(Mask));
5815 
5816   if (InFlag.getNode())
5817     Ops.push_back(InFlag);
5818 
5819   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
5820 
5821   // If we're doing a tall call, use a TC_RETURN here rather than an
5822   // actual call instruction.
5823   if (IsTailCall) {
5824     MF.getFrameInfo().setHasTailCall();
5825     SDValue Ret = DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
5826     DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
5827     return Ret;
5828   }
5829 
5830   unsigned CallOpc = AArch64ISD::CALL;
5831   // Calls with operand bundle "clang.arc.attachedcall" are special. They should
5832   // be expanded to the call, directly followed by a special marker sequence.
5833   // Use the CALL_RVMARKER to do that.
5834   if (CLI.CB && objcarc::hasAttachedCallOpBundle(CLI.CB)) {
5835     assert(!IsTailCall &&
5836            "tail calls cannot be marked with clang.arc.attachedcall");
5837     CallOpc = AArch64ISD::CALL_RVMARKER;
5838   }
5839 
5840   // Returns a chain and a flag for retval copy to use.
5841   Chain = DAG.getNode(CallOpc, DL, NodeTys, Ops);
5842   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
5843   InFlag = Chain.getValue(1);
5844   DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
5845 
5846   uint64_t CalleePopBytes =
5847       DoesCalleeRestoreStack(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : 0;
5848 
5849   Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
5850                              DAG.getIntPtrConstant(CalleePopBytes, DL, true),
5851                              InFlag, DL);
5852   if (!Ins.empty())
5853     InFlag = Chain.getValue(1);
5854 
5855   // Handle result values, copying them out of physregs into vregs that we
5856   // return.
5857   return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
5858                          InVals, IsThisReturn,
5859                          IsThisReturn ? OutVals[0] : SDValue());
5860 }
5861 
5862 bool AArch64TargetLowering::CanLowerReturn(
5863     CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
5864     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
5865   CCAssignFn *RetCC = CCAssignFnForReturn(CallConv);
5866   SmallVector<CCValAssign, 16> RVLocs;
5867   CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
5868   return CCInfo.CheckReturn(Outs, RetCC);
5869 }
5870 
5871 SDValue
5872 AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
5873                                    bool isVarArg,
5874                                    const SmallVectorImpl<ISD::OutputArg> &Outs,
5875                                    const SmallVectorImpl<SDValue> &OutVals,
5876                                    const SDLoc &DL, SelectionDAG &DAG) const {
5877   auto &MF = DAG.getMachineFunction();
5878   auto *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
5879 
5880   CCAssignFn *RetCC = CCAssignFnForReturn(CallConv);
5881   SmallVector<CCValAssign, 16> RVLocs;
5882   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
5883                  *DAG.getContext());
5884   CCInfo.AnalyzeReturn(Outs, RetCC);
5885 
5886   // Copy the result values into the output registers.
5887   SDValue Flag;
5888   SmallVector<std::pair<unsigned, SDValue>, 4> RetVals;
5889   SmallSet<unsigned, 4> RegsUsed;
5890   for (unsigned i = 0, realRVLocIdx = 0; i != RVLocs.size();
5891        ++i, ++realRVLocIdx) {
5892     CCValAssign &VA = RVLocs[i];
5893     assert(VA.isRegLoc() && "Can only return in registers!");
5894     SDValue Arg = OutVals[realRVLocIdx];
5895 
5896     switch (VA.getLocInfo()) {
5897     default:
5898       llvm_unreachable("Unknown loc info!");
5899     case CCValAssign::Full:
5900       if (Outs[i].ArgVT == MVT::i1) {
5901         // AAPCS requires i1 to be zero-extended to i8 by the producer of the
5902         // value. This is strictly redundant on Darwin (which uses "zeroext
5903         // i1"), but will be optimised out before ISel.
5904         Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
5905         Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
5906       }
5907       break;
5908     case CCValAssign::BCvt:
5909       Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
5910       break;
5911     case CCValAssign::AExt:
5912     case CCValAssign::ZExt:
5913       Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
5914       break;
5915     case CCValAssign::AExtUpper:
5916       assert(VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits");
5917       Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
5918       Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg,
5919                         DAG.getConstant(32, DL, VA.getLocVT()));
5920       break;
5921     }
5922 
5923     if (RegsUsed.count(VA.getLocReg())) {
5924       SDValue &Bits =
5925           llvm::find_if(RetVals, [=](const std::pair<unsigned, SDValue> &Elt) {
5926             return Elt.first == VA.getLocReg();
5927           })->second;
5928       Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg);
5929     } else {
5930       RetVals.emplace_back(VA.getLocReg(), Arg);
5931       RegsUsed.insert(VA.getLocReg());
5932     }
5933   }
5934 
5935   SmallVector<SDValue, 4> RetOps(1, Chain);
5936   for (auto &RetVal : RetVals) {
5937     Chain = DAG.getCopyToReg(Chain, DL, RetVal.first, RetVal.second, Flag);
5938     Flag = Chain.getValue(1);
5939     RetOps.push_back(
5940         DAG.getRegister(RetVal.first, RetVal.second.getValueType()));
5941   }
5942 
5943   // Windows AArch64 ABIs require that for returning structs by value we copy
5944   // the sret argument into X0 for the return.
5945   // We saved the argument into a virtual register in the entry block,
5946   // so now we copy the value out and into X0.
5947   if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) {
5948     SDValue Val = DAG.getCopyFromReg(RetOps[0], DL, SRetReg,
5949                                      getPointerTy(MF.getDataLayout()));
5950 
5951     unsigned RetValReg = AArch64::X0;
5952     Chain = DAG.getCopyToReg(Chain, DL, RetValReg, Val, Flag);
5953     Flag = Chain.getValue(1);
5954 
5955     RetOps.push_back(
5956       DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout())));
5957   }
5958 
5959   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
5960   const MCPhysReg *I =
5961       TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
5962   if (I) {
5963     for (; *I; ++I) {
5964       if (AArch64::GPR64RegClass.contains(*I))
5965         RetOps.push_back(DAG.getRegister(*I, MVT::i64));
5966       else if (AArch64::FPR64RegClass.contains(*I))
5967         RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64)));
5968       else
5969         llvm_unreachable("Unexpected register class in CSRsViaCopy!");
5970     }
5971   }
5972 
5973   RetOps[0] = Chain; // Update chain.
5974 
5975   // Add the flag if we have it.
5976   if (Flag.getNode())
5977     RetOps.push_back(Flag);
5978 
5979   return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps);
5980 }
5981 
5982 //===----------------------------------------------------------------------===//
5983 //  Other Lowering Code
5984 //===----------------------------------------------------------------------===//
5985 
5986 SDValue AArch64TargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
5987                                              SelectionDAG &DAG,
5988                                              unsigned Flag) const {
5989   return DAG.getTargetGlobalAddress(N->getGlobal(), SDLoc(N), Ty,
5990                                     N->getOffset(), Flag);
5991 }
5992 
5993 SDValue AArch64TargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
5994                                              SelectionDAG &DAG,
5995                                              unsigned Flag) const {
5996   return DAG.getTargetJumpTable(N->getIndex(), Ty, Flag);
5997 }
5998 
5999 SDValue AArch64TargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
6000                                              SelectionDAG &DAG,
6001                                              unsigned Flag) const {
6002   return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
6003                                    N->getOffset(), Flag);
6004 }
6005 
6006 SDValue AArch64TargetLowering::getTargetNode(BlockAddressSDNode* N, EVT Ty,
6007                                              SelectionDAG &DAG,
6008                                              unsigned Flag) const {
6009   return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, 0, Flag);
6010 }
6011 
6012 // (loadGOT sym)
6013 template <class NodeTy>
6014 SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG,
6015                                       unsigned Flags) const {
6016   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getGOT\n");
6017   SDLoc DL(N);
6018   EVT Ty = getPointerTy(DAG.getDataLayout());
6019   SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT | Flags);
6020   // FIXME: Once remat is capable of dealing with instructions with register
6021   // operands, expand this into two nodes instead of using a wrapper node.
6022   return DAG.getNode(AArch64ISD::LOADgot, DL, Ty, GotAddr);
6023 }
6024 
6025 // (wrapper %highest(sym), %higher(sym), %hi(sym), %lo(sym))
6026 template <class NodeTy>
6027 SDValue AArch64TargetLowering::getAddrLarge(NodeTy *N, SelectionDAG &DAG,
6028                                             unsigned Flags) const {
6029   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddrLarge\n");
6030   SDLoc DL(N);
6031   EVT Ty = getPointerTy(DAG.getDataLayout());
6032   const unsigned char MO_NC = AArch64II::MO_NC;
6033   return DAG.getNode(
6034       AArch64ISD::WrapperLarge, DL, Ty,
6035       getTargetNode(N, Ty, DAG, AArch64II::MO_G3 | Flags),
6036       getTargetNode(N, Ty, DAG, AArch64II::MO_G2 | MO_NC | Flags),
6037       getTargetNode(N, Ty, DAG, AArch64II::MO_G1 | MO_NC | Flags),
6038       getTargetNode(N, Ty, DAG, AArch64II::MO_G0 | MO_NC | Flags));
6039 }
6040 
6041 // (addlow (adrp %hi(sym)) %lo(sym))
6042 template <class NodeTy>
6043 SDValue AArch64TargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
6044                                        unsigned Flags) const {
6045   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddr\n");
6046   SDLoc DL(N);
6047   EVT Ty = getPointerTy(DAG.getDataLayout());
6048   SDValue Hi = getTargetNode(N, Ty, DAG, AArch64II::MO_PAGE | Flags);
6049   SDValue Lo = getTargetNode(N, Ty, DAG,
6050                              AArch64II::MO_PAGEOFF | AArch64II::MO_NC | Flags);
6051   SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi);
6052   return DAG.getNode(AArch64ISD::ADDlow, DL, Ty, ADRP, Lo);
6053 }
6054 
6055 // (adr sym)
6056 template <class NodeTy>
6057 SDValue AArch64TargetLowering::getAddrTiny(NodeTy *N, SelectionDAG &DAG,
6058                                            unsigned Flags) const {
6059   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddrTiny\n");
6060   SDLoc DL(N);
6061   EVT Ty = getPointerTy(DAG.getDataLayout());
6062   SDValue Sym = getTargetNode(N, Ty, DAG, Flags);
6063   return DAG.getNode(AArch64ISD::ADR, DL, Ty, Sym);
6064 }
6065 
6066 SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
6067                                                   SelectionDAG &DAG) const {
6068   GlobalAddressSDNode *GN = cast<GlobalAddressSDNode>(Op);
6069   const GlobalValue *GV = GN->getGlobal();
6070   unsigned OpFlags = Subtarget->ClassifyGlobalReference(GV, getTargetMachine());
6071 
6072   if (OpFlags != AArch64II::MO_NO_FLAG)
6073     assert(cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&
6074            "unexpected offset in global node");
6075 
6076   // This also catches the large code model case for Darwin, and tiny code
6077   // model with got relocations.
6078   if ((OpFlags & AArch64II::MO_GOT) != 0) {
6079     return getGOT(GN, DAG, OpFlags);
6080   }
6081 
6082   SDValue Result;
6083   if (getTargetMachine().getCodeModel() == CodeModel::Large) {
6084     Result = getAddrLarge(GN, DAG, OpFlags);
6085   } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
6086     Result = getAddrTiny(GN, DAG, OpFlags);
6087   } else {
6088     Result = getAddr(GN, DAG, OpFlags);
6089   }
6090   EVT PtrVT = getPointerTy(DAG.getDataLayout());
6091   SDLoc DL(GN);
6092   if (OpFlags & (AArch64II::MO_DLLIMPORT | AArch64II::MO_COFFSTUB))
6093     Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
6094                          MachinePointerInfo::getGOT(DAG.getMachineFunction()));
6095   return Result;
6096 }
6097 
6098 /// Convert a TLS address reference into the correct sequence of loads
6099 /// and calls to compute the variable's address (for Darwin, currently) and
6100 /// return an SDValue containing the final node.
6101 
6102 /// Darwin only has one TLS scheme which must be capable of dealing with the
6103 /// fully general situation, in the worst case. This means:
6104 ///     + "extern __thread" declaration.
6105 ///     + Defined in a possibly unknown dynamic library.
6106 ///
6107 /// The general system is that each __thread variable has a [3 x i64] descriptor
6108 /// which contains information used by the runtime to calculate the address. The
6109 /// only part of this the compiler needs to know about is the first xword, which
6110 /// contains a function pointer that must be called with the address of the
6111 /// entire descriptor in "x0".
6112 ///
6113 /// Since this descriptor may be in a different unit, in general even the
6114 /// descriptor must be accessed via an indirect load. The "ideal" code sequence
6115 /// is:
6116 ///     adrp x0, _var@TLVPPAGE
6117 ///     ldr x0, [x0, _var@TLVPPAGEOFF]   ; x0 now contains address of descriptor
6118 ///     ldr x1, [x0]                     ; x1 contains 1st entry of descriptor,
6119 ///                                      ; the function pointer
6120 ///     blr x1                           ; Uses descriptor address in x0
6121 ///     ; Address of _var is now in x0.
6122 ///
6123 /// If the address of _var's descriptor *is* known to the linker, then it can
6124 /// change the first "ldr" instruction to an appropriate "add x0, x0, #imm" for
6125 /// a slight efficiency gain.
6126 SDValue
6127 AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
6128                                                    SelectionDAG &DAG) const {
6129   assert(Subtarget->isTargetDarwin() &&
6130          "This function expects a Darwin target");
6131 
6132   SDLoc DL(Op);
6133   MVT PtrVT = getPointerTy(DAG.getDataLayout());
6134   MVT PtrMemVT = getPointerMemTy(DAG.getDataLayout());
6135   const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
6136 
6137   SDValue TLVPAddr =
6138       DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
6139   SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
6140 
6141   // The first entry in the descriptor is a function pointer that we must call
6142   // to obtain the address of the variable.
6143   SDValue Chain = DAG.getEntryNode();
6144   SDValue FuncTLVGet = DAG.getLoad(
6145       PtrMemVT, DL, Chain, DescAddr,
6146       MachinePointerInfo::getGOT(DAG.getMachineFunction()),
6147       Align(PtrMemVT.getSizeInBits() / 8),
6148       MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
6149   Chain = FuncTLVGet.getValue(1);
6150 
6151   // Extend loaded pointer if necessary (i.e. if ILP32) to DAG pointer.
6152   FuncTLVGet = DAG.getZExtOrTrunc(FuncTLVGet, DL, PtrVT);
6153 
6154   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6155   MFI.setAdjustsStack(true);
6156 
6157   // TLS calls preserve all registers except those that absolutely must be
6158   // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
6159   // silly).
6160   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
6161   const uint32_t *Mask = TRI->getTLSCallPreservedMask();
6162   if (Subtarget->hasCustomCallingConv())
6163     TRI->UpdateCustomCallPreservedMask(DAG.getMachineFunction(), &Mask);
6164 
6165   // Finally, we can make the call. This is just a degenerate version of a
6166   // normal AArch64 call node: x0 takes the address of the descriptor, and
6167   // returns the address of the variable in this thread.
6168   Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
6169   Chain =
6170       DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
6171                   Chain, FuncTLVGet, DAG.getRegister(AArch64::X0, MVT::i64),
6172                   DAG.getRegisterMask(Mask), Chain.getValue(1));
6173   return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Chain.getValue(1));
6174 }
6175 
6176 /// Convert a thread-local variable reference into a sequence of instructions to
6177 /// compute the variable's address for the local exec TLS model of ELF targets.
6178 /// The sequence depends on the maximum TLS area size.
6179 SDValue AArch64TargetLowering::LowerELFTLSLocalExec(const GlobalValue *GV,
6180                                                     SDValue ThreadBase,
6181                                                     const SDLoc &DL,
6182                                                     SelectionDAG &DAG) const {
6183   EVT PtrVT = getPointerTy(DAG.getDataLayout());
6184   SDValue TPOff, Addr;
6185 
6186   switch (DAG.getTarget().Options.TLSSize) {
6187   default:
6188     llvm_unreachable("Unexpected TLS size");
6189 
6190   case 12: {
6191     // mrs   x0, TPIDR_EL0
6192     // add   x0, x0, :tprel_lo12:a
6193     SDValue Var = DAG.getTargetGlobalAddress(
6194         GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
6195     return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
6196                                       Var,
6197                                       DAG.getTargetConstant(0, DL, MVT::i32)),
6198                    0);
6199   }
6200 
6201   case 24: {
6202     // mrs   x0, TPIDR_EL0
6203     // add   x0, x0, :tprel_hi12:a
6204     // add   x0, x0, :tprel_lo12_nc:a
6205     SDValue HiVar = DAG.getTargetGlobalAddress(
6206         GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
6207     SDValue LoVar = DAG.getTargetGlobalAddress(
6208         GV, DL, PtrVT, 0,
6209         AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
6210     Addr = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
6211                                       HiVar,
6212                                       DAG.getTargetConstant(0, DL, MVT::i32)),
6213                    0);
6214     return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, Addr,
6215                                       LoVar,
6216                                       DAG.getTargetConstant(0, DL, MVT::i32)),
6217                    0);
6218   }
6219 
6220   case 32: {
6221     // mrs   x1, TPIDR_EL0
6222     // movz  x0, #:tprel_g1:a
6223     // movk  x0, #:tprel_g0_nc:a
6224     // add   x0, x1, x0
6225     SDValue HiVar = DAG.getTargetGlobalAddress(
6226         GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_G1);
6227     SDValue LoVar = DAG.getTargetGlobalAddress(
6228         GV, DL, PtrVT, 0,
6229         AArch64II::MO_TLS | AArch64II::MO_G0 | AArch64II::MO_NC);
6230     TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
6231                                        DAG.getTargetConstant(16, DL, MVT::i32)),
6232                     0);
6233     TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
6234                                        DAG.getTargetConstant(0, DL, MVT::i32)),
6235                     0);
6236     return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
6237   }
6238 
6239   case 48: {
6240     // mrs   x1, TPIDR_EL0
6241     // movz  x0, #:tprel_g2:a
6242     // movk  x0, #:tprel_g1_nc:a
6243     // movk  x0, #:tprel_g0_nc:a
6244     // add   x0, x1, x0
6245     SDValue HiVar = DAG.getTargetGlobalAddress(
6246         GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_G2);
6247     SDValue MiVar = DAG.getTargetGlobalAddress(
6248         GV, DL, PtrVT, 0,
6249         AArch64II::MO_TLS | AArch64II::MO_G1 | AArch64II::MO_NC);
6250     SDValue LoVar = DAG.getTargetGlobalAddress(
6251         GV, DL, PtrVT, 0,
6252         AArch64II::MO_TLS | AArch64II::MO_G0 | AArch64II::MO_NC);
6253     TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
6254                                        DAG.getTargetConstant(32, DL, MVT::i32)),
6255                     0);
6256     TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, MiVar,
6257                                        DAG.getTargetConstant(16, DL, MVT::i32)),
6258                     0);
6259     TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
6260                                        DAG.getTargetConstant(0, DL, MVT::i32)),
6261                     0);
6262     return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
6263   }
6264   }
6265 }
6266 
6267 /// When accessing thread-local variables under either the general-dynamic or
6268 /// local-dynamic system, we make a "TLS-descriptor" call. The variable will
6269 /// have a descriptor, accessible via a PC-relative ADRP, and whose first entry
6270 /// is a function pointer to carry out the resolution.
6271 ///
6272 /// The sequence is:
6273 ///    adrp  x0, :tlsdesc:var
6274 ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
6275 ///    add   x0, x0, #:tlsdesc_lo12:var
6276 ///    .tlsdesccall var
6277 ///    blr   x1
6278 ///    (TPIDR_EL0 offset now in x0)
6279 ///
6280 ///  The above sequence must be produced unscheduled, to enable the linker to
6281 ///  optimize/relax this sequence.
6282 ///  Therefore, a pseudo-instruction (TLSDESC_CALLSEQ) is used to represent the
6283 ///  above sequence, and expanded really late in the compilation flow, to ensure
6284 ///  the sequence is produced as per above.
6285 SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr,
6286                                                       const SDLoc &DL,
6287                                                       SelectionDAG &DAG) const {
6288   EVT PtrVT = getPointerTy(DAG.getDataLayout());
6289 
6290   SDValue Chain = DAG.getEntryNode();
6291   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
6292 
6293   Chain =
6294       DAG.getNode(AArch64ISD::TLSDESC_CALLSEQ, DL, NodeTys, {Chain, SymAddr});
6295   SDValue Glue = Chain.getValue(1);
6296 
6297   return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue);
6298 }
6299 
6300 SDValue
6301 AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
6302                                                 SelectionDAG &DAG) const {
6303   assert(Subtarget->isTargetELF() && "This function expects an ELF target");
6304 
6305   const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
6306 
6307   TLSModel::Model Model = getTargetMachine().getTLSModel(GA->getGlobal());
6308 
6309   if (!EnableAArch64ELFLocalDynamicTLSGeneration) {
6310     if (Model == TLSModel::LocalDynamic)
6311       Model = TLSModel::GeneralDynamic;
6312   }
6313 
6314   if (getTargetMachine().getCodeModel() == CodeModel::Large &&
6315       Model != TLSModel::LocalExec)
6316     report_fatal_error("ELF TLS only supported in small memory model or "
6317                        "in local exec TLS model");
6318   // Different choices can be made for the maximum size of the TLS area for a
6319   // module. For the small address model, the default TLS size is 16MiB and the
6320   // maximum TLS size is 4GiB.
6321   // FIXME: add tiny and large code model support for TLS access models other
6322   // than local exec. We currently generate the same code as small for tiny,
6323   // which may be larger than needed.
6324 
6325   SDValue TPOff;
6326   EVT PtrVT = getPointerTy(DAG.getDataLayout());
6327   SDLoc DL(Op);
6328   const GlobalValue *GV = GA->getGlobal();
6329 
6330   SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
6331 
6332   if (Model == TLSModel::LocalExec) {
6333     return LowerELFTLSLocalExec(GV, ThreadBase, DL, DAG);
6334   } else if (Model == TLSModel::InitialExec) {
6335     TPOff = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
6336     TPOff = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TPOff);
6337   } else if (Model == TLSModel::LocalDynamic) {
6338     // Local-dynamic accesses proceed in two phases. A general-dynamic TLS
6339     // descriptor call against the special symbol _TLS_MODULE_BASE_ to calculate
6340     // the beginning of the module's TLS region, followed by a DTPREL offset
6341     // calculation.
6342 
6343     // These accesses will need deduplicating if there's more than one.
6344     AArch64FunctionInfo *MFI =
6345         DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
6346     MFI->incNumLocalDynamicTLSAccesses();
6347 
6348     // The call needs a relocation too for linker relaxation. It doesn't make
6349     // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
6350     // the address.
6351     SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT,
6352                                                   AArch64II::MO_TLS);
6353 
6354     // Now we can calculate the offset from TPIDR_EL0 to this module's
6355     // thread-local area.
6356     TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
6357 
6358     // Now use :dtprel_whatever: operations to calculate this variable's offset
6359     // in its thread-storage area.
6360     SDValue HiVar = DAG.getTargetGlobalAddress(
6361         GV, DL, MVT::i64, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
6362     SDValue LoVar = DAG.getTargetGlobalAddress(
6363         GV, DL, MVT::i64, 0,
6364         AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
6365 
6366     TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
6367                                        DAG.getTargetConstant(0, DL, MVT::i32)),
6368                     0);
6369     TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
6370                                        DAG.getTargetConstant(0, DL, MVT::i32)),
6371                     0);
6372   } else if (Model == TLSModel::GeneralDynamic) {
6373     // The call needs a relocation too for linker relaxation. It doesn't make
6374     // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
6375     // the address.
6376     SDValue SymAddr =
6377         DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
6378 
6379     // Finally we can make a call to calculate the offset from tpidr_el0.
6380     TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
6381   } else
6382     llvm_unreachable("Unsupported ELF TLS access model");
6383 
6384   return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
6385 }
6386 
6387 SDValue
6388 AArch64TargetLowering::LowerWindowsGlobalTLSAddress(SDValue Op,
6389                                                     SelectionDAG &DAG) const {
6390   assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering");
6391 
6392   SDValue Chain = DAG.getEntryNode();
6393   EVT PtrVT = getPointerTy(DAG.getDataLayout());
6394   SDLoc DL(Op);
6395 
6396   SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
6397 
6398   // Load the ThreadLocalStoragePointer from the TEB
6399   // A pointer to the TLS array is located at offset 0x58 from the TEB.
6400   SDValue TLSArray =
6401       DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x58, DL));
6402   TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
6403   Chain = TLSArray.getValue(1);
6404 
6405   // Load the TLS index from the C runtime;
6406   // This does the same as getAddr(), but without having a GlobalAddressSDNode.
6407   // This also does the same as LOADgot, but using a generic i32 load,
6408   // while LOADgot only loads i64.
6409   SDValue TLSIndexHi =
6410       DAG.getTargetExternalSymbol("_tls_index", PtrVT, AArch64II::MO_PAGE);
6411   SDValue TLSIndexLo = DAG.getTargetExternalSymbol(
6412       "_tls_index", PtrVT, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
6413   SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi);
6414   SDValue TLSIndex =
6415       DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, TLSIndexLo);
6416   TLSIndex = DAG.getLoad(MVT::i32, DL, Chain, TLSIndex, MachinePointerInfo());
6417   Chain = TLSIndex.getValue(1);
6418 
6419   // The pointer to the thread's TLS data area is at the TLS Index scaled by 8
6420   // offset into the TLSArray.
6421   TLSIndex = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TLSIndex);
6422   SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
6423                              DAG.getConstant(3, DL, PtrVT));
6424   SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
6425                             DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
6426                             MachinePointerInfo());
6427   Chain = TLS.getValue(1);
6428 
6429   const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
6430   const GlobalValue *GV = GA->getGlobal();
6431   SDValue TGAHi = DAG.getTargetGlobalAddress(
6432       GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
6433   SDValue TGALo = DAG.getTargetGlobalAddress(
6434       GV, DL, PtrVT, 0,
6435       AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
6436 
6437   // Add the offset from the start of the .tls section (section base).
6438   SDValue Addr =
6439       SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
6440                                  DAG.getTargetConstant(0, DL, MVT::i32)),
6441               0);
6442   Addr = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, Addr, TGALo);
6443   return Addr;
6444 }
6445 
6446 SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
6447                                                      SelectionDAG &DAG) const {
6448   const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
6449   if (DAG.getTarget().useEmulatedTLS())
6450     return LowerToTLSEmulatedModel(GA, DAG);
6451 
6452   if (Subtarget->isTargetDarwin())
6453     return LowerDarwinGlobalTLSAddress(Op, DAG);
6454   if (Subtarget->isTargetELF())
6455     return LowerELFGlobalTLSAddress(Op, DAG);
6456   if (Subtarget->isTargetWindows())
6457     return LowerWindowsGlobalTLSAddress(Op, DAG);
6458 
6459   llvm_unreachable("Unexpected platform trying to use TLS");
6460 }
6461 
6462 // Looks through \param Val to determine the bit that can be used to
6463 // check the sign of the value. It returns the unextended value and
6464 // the sign bit position.
6465 std::pair<SDValue, uint64_t> lookThroughSignExtension(SDValue Val) {
6466   if (Val.getOpcode() == ISD::SIGN_EXTEND_INREG)
6467     return {Val.getOperand(0),
6468             cast<VTSDNode>(Val.getOperand(1))->getVT().getFixedSizeInBits() -
6469                 1};
6470 
6471   if (Val.getOpcode() == ISD::SIGN_EXTEND)
6472     return {Val.getOperand(0),
6473             Val.getOperand(0)->getValueType(0).getFixedSizeInBits() - 1};
6474 
6475   return {Val, Val.getValueSizeInBits() - 1};
6476 }
6477 
6478 SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
6479   SDValue Chain = Op.getOperand(0);
6480   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
6481   SDValue LHS = Op.getOperand(2);
6482   SDValue RHS = Op.getOperand(3);
6483   SDValue Dest = Op.getOperand(4);
6484   SDLoc dl(Op);
6485 
6486   MachineFunction &MF = DAG.getMachineFunction();
6487   // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z instructions
6488   // will not be produced, as they are conditional branch instructions that do
6489   // not set flags.
6490   bool ProduceNonFlagSettingCondBr =
6491       !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
6492 
6493   // Handle f128 first, since lowering it will result in comparing the return
6494   // value of a libcall against zero, which is just what the rest of LowerBR_CC
6495   // is expecting to deal with.
6496   if (LHS.getValueType() == MVT::f128) {
6497     softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
6498 
6499     // If softenSetCCOperands returned a scalar, we need to compare the result
6500     // against zero to select between true and false values.
6501     if (!RHS.getNode()) {
6502       RHS = DAG.getConstant(0, dl, LHS.getValueType());
6503       CC = ISD::SETNE;
6504     }
6505   }
6506 
6507   // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
6508   // instruction.
6509   if (ISD::isOverflowIntrOpRes(LHS) && isOneConstant(RHS) &&
6510       (CC == ISD::SETEQ || CC == ISD::SETNE)) {
6511     // Only lower legal XALUO ops.
6512     if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0)))
6513       return SDValue();
6514 
6515     // The actual operation with overflow check.
6516     AArch64CC::CondCode OFCC;
6517     SDValue Value, Overflow;
6518     std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, LHS.getValue(0), DAG);
6519 
6520     if (CC == ISD::SETNE)
6521       OFCC = getInvertedCondCode(OFCC);
6522     SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
6523 
6524     return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
6525                        Overflow);
6526   }
6527 
6528   if (LHS.getValueType().isInteger()) {
6529     assert((LHS.getValueType() == RHS.getValueType()) &&
6530            (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64));
6531 
6532     // If the RHS of the comparison is zero, we can potentially fold this
6533     // to a specialized branch.
6534     const ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
6535     if (RHSC && RHSC->getZExtValue() == 0 && ProduceNonFlagSettingCondBr) {
6536       if (CC == ISD::SETEQ) {
6537         // See if we can use a TBZ to fold in an AND as well.
6538         // TBZ has a smaller branch displacement than CBZ.  If the offset is
6539         // out of bounds, a late MI-layer pass rewrites branches.
6540         // 403.gcc is an example that hits this case.
6541         if (LHS.getOpcode() == ISD::AND &&
6542             isa<ConstantSDNode>(LHS.getOperand(1)) &&
6543             isPowerOf2_64(LHS.getConstantOperandVal(1))) {
6544           SDValue Test = LHS.getOperand(0);
6545           uint64_t Mask = LHS.getConstantOperandVal(1);
6546           return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test,
6547                              DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
6548                              Dest);
6549         }
6550 
6551         return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest);
6552       } else if (CC == ISD::SETNE) {
6553         // See if we can use a TBZ to fold in an AND as well.
6554         // TBZ has a smaller branch displacement than CBZ.  If the offset is
6555         // out of bounds, a late MI-layer pass rewrites branches.
6556         // 403.gcc is an example that hits this case.
6557         if (LHS.getOpcode() == ISD::AND &&
6558             isa<ConstantSDNode>(LHS.getOperand(1)) &&
6559             isPowerOf2_64(LHS.getConstantOperandVal(1))) {
6560           SDValue Test = LHS.getOperand(0);
6561           uint64_t Mask = LHS.getConstantOperandVal(1);
6562           return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test,
6563                              DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
6564                              Dest);
6565         }
6566 
6567         return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest);
6568       } else if (CC == ISD::SETLT && LHS.getOpcode() != ISD::AND) {
6569         // Don't combine AND since emitComparison converts the AND to an ANDS
6570         // (a.k.a. TST) and the test in the test bit and branch instruction
6571         // becomes redundant.  This would also increase register pressure.
6572         uint64_t SignBitPos;
6573         std::tie(LHS, SignBitPos) = lookThroughSignExtension(LHS);
6574         return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS,
6575                            DAG.getConstant(SignBitPos, dl, MVT::i64), Dest);
6576       }
6577     }
6578     if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT &&
6579         LHS.getOpcode() != ISD::AND && ProduceNonFlagSettingCondBr) {
6580       // Don't combine AND since emitComparison converts the AND to an ANDS
6581       // (a.k.a. TST) and the test in the test bit and branch instruction
6582       // becomes redundant.  This would also increase register pressure.
6583       uint64_t SignBitPos;
6584       std::tie(LHS, SignBitPos) = lookThroughSignExtension(LHS);
6585       return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS,
6586                          DAG.getConstant(SignBitPos, dl, MVT::i64), Dest);
6587     }
6588 
6589     SDValue CCVal;
6590     SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
6591     return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
6592                        Cmp);
6593   }
6594 
6595   assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::bf16 ||
6596          LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64);
6597 
6598   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
6599   // clean.  Some of them require two branches to implement.
6600   SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
6601   AArch64CC::CondCode CC1, CC2;
6602   changeFPCCToAArch64CC(CC, CC1, CC2);
6603   SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
6604   SDValue BR1 =
6605       DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp);
6606   if (CC2 != AArch64CC::AL) {
6607     SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
6608     return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val,
6609                        Cmp);
6610   }
6611 
6612   return BR1;
6613 }
6614 
6615 SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
6616                                               SelectionDAG &DAG) const {
6617   EVT VT = Op.getValueType();
6618   SDLoc DL(Op);
6619 
6620   SDValue In1 = Op.getOperand(0);
6621   SDValue In2 = Op.getOperand(1);
6622   EVT SrcVT = In2.getValueType();
6623 
6624   if (SrcVT.bitsLT(VT))
6625     In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2);
6626   else if (SrcVT.bitsGT(VT))
6627     In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL));
6628 
6629   EVT VecVT;
6630   uint64_t EltMask;
6631   SDValue VecVal1, VecVal2;
6632 
6633   auto setVecVal = [&] (int Idx) {
6634     if (!VT.isVector()) {
6635       VecVal1 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
6636                                           DAG.getUNDEF(VecVT), In1);
6637       VecVal2 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
6638                                           DAG.getUNDEF(VecVT), In2);
6639     } else {
6640       VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1);
6641       VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2);
6642     }
6643   };
6644 
6645   if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) {
6646     VecVT = (VT == MVT::v2f32 ? MVT::v2i32 : MVT::v4i32);
6647     EltMask = 0x80000000ULL;
6648     setVecVal(AArch64::ssub);
6649   } else if (VT == MVT::f64 || VT == MVT::v2f64) {
6650     VecVT = MVT::v2i64;
6651 
6652     // We want to materialize a mask with the high bit set, but the AdvSIMD
6653     // immediate moves cannot materialize that in a single instruction for
6654     // 64-bit elements. Instead, materialize zero and then negate it.
6655     EltMask = 0;
6656 
6657     setVecVal(AArch64::dsub);
6658   } else if (VT == MVT::f16 || VT == MVT::v4f16 || VT == MVT::v8f16) {
6659     VecVT = (VT == MVT::v4f16 ? MVT::v4i16 : MVT::v8i16);
6660     EltMask = 0x8000ULL;
6661     setVecVal(AArch64::hsub);
6662   } else {
6663     llvm_unreachable("Invalid type for copysign!");
6664   }
6665 
6666   SDValue BuildVec = DAG.getConstant(EltMask, DL, VecVT);
6667 
6668   // If we couldn't materialize the mask above, then the mask vector will be
6669   // the zero vector, and we need to negate it here.
6670   if (VT == MVT::f64 || VT == MVT::v2f64) {
6671     BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec);
6672     BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec);
6673     BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec);
6674   }
6675 
6676   SDValue Sel =
6677       DAG.getNode(AArch64ISD::BIT, DL, VecVT, VecVal1, VecVal2, BuildVec);
6678 
6679   if (VT == MVT::f16)
6680     return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, Sel);
6681   if (VT == MVT::f32)
6682     return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel);
6683   else if (VT == MVT::f64)
6684     return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel);
6685   else
6686     return DAG.getNode(ISD::BITCAST, DL, VT, Sel);
6687 }
6688 
6689 SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
6690   if (DAG.getMachineFunction().getFunction().hasFnAttribute(
6691           Attribute::NoImplicitFloat))
6692     return SDValue();
6693 
6694   if (!Subtarget->hasNEON())
6695     return SDValue();
6696 
6697   // While there is no integer popcount instruction, it can
6698   // be more efficiently lowered to the following sequence that uses
6699   // AdvSIMD registers/instructions as long as the copies to/from
6700   // the AdvSIMD registers are cheap.
6701   //  FMOV    D0, X0        // copy 64-bit int to vector, high bits zero'd
6702   //  CNT     V0.8B, V0.8B  // 8xbyte pop-counts
6703   //  ADDV    B0, V0.8B     // sum 8xbyte pop-counts
6704   //  UMOV    X0, V0.B[0]   // copy byte result back to integer reg
6705   SDValue Val = Op.getOperand(0);
6706   SDLoc DL(Op);
6707   EVT VT = Op.getValueType();
6708 
6709   if (VT == MVT::i32 || VT == MVT::i64) {
6710     if (VT == MVT::i32)
6711       Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val);
6712     Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val);
6713 
6714     SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
6715     SDValue UaddLV = DAG.getNode(
6716         ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
6717         DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
6718 
6719     if (VT == MVT::i64)
6720       UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV);
6721     return UaddLV;
6722   } else if (VT == MVT::i128) {
6723     Val = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Val);
6724 
6725     SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v16i8, Val);
6726     SDValue UaddLV = DAG.getNode(
6727         ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
6728         DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
6729 
6730     return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, UaddLV);
6731   }
6732 
6733   if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT))
6734     return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTPOP_MERGE_PASSTHRU);
6735 
6736   assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6737           VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6738          "Unexpected type for custom ctpop lowering");
6739 
6740   EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
6741   Val = DAG.getBitcast(VT8Bit, Val);
6742   Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val);
6743 
6744   // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
6745   unsigned EltSize = 8;
6746   unsigned NumElts = VT.is64BitVector() ? 8 : 16;
6747   while (EltSize != VT.getScalarSizeInBits()) {
6748     EltSize *= 2;
6749     NumElts /= 2;
6750     MVT WidenVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize), NumElts);
6751     Val = DAG.getNode(
6752         ISD::INTRINSIC_WO_CHAIN, DL, WidenVT,
6753         DAG.getConstant(Intrinsic::aarch64_neon_uaddlp, DL, MVT::i32), Val);
6754   }
6755 
6756   return Val;
6757 }
6758 
6759 SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
6760   EVT VT = Op.getValueType();
6761   assert(VT.isScalableVector() ||
6762          useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true));
6763 
6764   SDLoc DL(Op);
6765   SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
6766   return DAG.getNode(ISD::CTLZ, DL, VT, RBIT);
6767 }
6768 
6769 SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
6770 
6771   if (Op.getValueType().isVector())
6772     return LowerVSETCC(Op, DAG);
6773 
6774   bool IsStrict = Op->isStrictFPOpcode();
6775   bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
6776   unsigned OpNo = IsStrict ? 1 : 0;
6777   SDValue Chain;
6778   if (IsStrict)
6779     Chain = Op.getOperand(0);
6780   SDValue LHS = Op.getOperand(OpNo + 0);
6781   SDValue RHS = Op.getOperand(OpNo + 1);
6782   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(OpNo + 2))->get();
6783   SDLoc dl(Op);
6784 
6785   // We chose ZeroOrOneBooleanContents, so use zero and one.
6786   EVT VT = Op.getValueType();
6787   SDValue TVal = DAG.getConstant(1, dl, VT);
6788   SDValue FVal = DAG.getConstant(0, dl, VT);
6789 
6790   // Handle f128 first, since one possible outcome is a normal integer
6791   // comparison which gets picked up by the next if statement.
6792   if (LHS.getValueType() == MVT::f128) {
6793     softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS, Chain,
6794                         IsSignaling);
6795 
6796     // If softenSetCCOperands returned a scalar, use it.
6797     if (!RHS.getNode()) {
6798       assert(LHS.getValueType() == Op.getValueType() &&
6799              "Unexpected setcc expansion!");
6800       return IsStrict ? DAG.getMergeValues({LHS, Chain}, dl) : LHS;
6801     }
6802   }
6803 
6804   if (LHS.getValueType().isInteger()) {
6805     SDValue CCVal;
6806     SDValue Cmp = getAArch64Cmp(
6807         LHS, RHS, ISD::getSetCCInverse(CC, LHS.getValueType()), CCVal, DAG, dl);
6808 
6809     // Note that we inverted the condition above, so we reverse the order of
6810     // the true and false operands here.  This will allow the setcc to be
6811     // matched to a single CSINC instruction.
6812     SDValue Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
6813     return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
6814   }
6815 
6816   // Now we know we're dealing with FP values.
6817   assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||
6818          LHS.getValueType() == MVT::f64);
6819 
6820   // If that fails, we'll need to perform an FCMP + CSEL sequence.  Go ahead
6821   // and do the comparison.
6822   SDValue Cmp;
6823   if (IsStrict)
6824     Cmp = emitStrictFPComparison(LHS, RHS, dl, DAG, Chain, IsSignaling);
6825   else
6826     Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
6827 
6828   AArch64CC::CondCode CC1, CC2;
6829   changeFPCCToAArch64CC(CC, CC1, CC2);
6830   SDValue Res;
6831   if (CC2 == AArch64CC::AL) {
6832     changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, LHS.getValueType()), CC1,
6833                           CC2);
6834     SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
6835 
6836     // Note that we inverted the condition above, so we reverse the order of
6837     // the true and false operands here.  This will allow the setcc to be
6838     // matched to a single CSINC instruction.
6839     Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp);
6840   } else {
6841     // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
6842     // totally clean.  Some of them require two CSELs to implement.  As is in
6843     // this case, we emit the first CSEL and then emit a second using the output
6844     // of the first as the RHS.  We're effectively OR'ing the two CC's together.
6845 
6846     // FIXME: It would be nice if we could match the two CSELs to two CSINCs.
6847     SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
6848     SDValue CS1 =
6849         DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
6850 
6851     SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
6852     Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
6853   }
6854   return IsStrict ? DAG.getMergeValues({Res, Cmp.getValue(1)}, dl) : Res;
6855 }
6856 
6857 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
6858                                               SDValue RHS, SDValue TVal,
6859                                               SDValue FVal, const SDLoc &dl,
6860                                               SelectionDAG &DAG) const {
6861   // Handle f128 first, because it will result in a comparison of some RTLIB
6862   // call result against zero.
6863   if (LHS.getValueType() == MVT::f128) {
6864     softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
6865 
6866     // If softenSetCCOperands returned a scalar, we need to compare the result
6867     // against zero to select between true and false values.
6868     if (!RHS.getNode()) {
6869       RHS = DAG.getConstant(0, dl, LHS.getValueType());
6870       CC = ISD::SETNE;
6871     }
6872   }
6873 
6874   // Also handle f16, for which we need to do a f32 comparison.
6875   if (LHS.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) {
6876     LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
6877     RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
6878   }
6879 
6880   // Next, handle integers.
6881   if (LHS.getValueType().isInteger()) {
6882     assert((LHS.getValueType() == RHS.getValueType()) &&
6883            (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64));
6884 
6885     ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
6886     ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
6887     ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
6888     // Check for sign pattern (SELECT_CC setgt, iN lhs, -1, 1, -1) and transform
6889     // into (OR (ASR lhs, N-1), 1), which requires less instructions for the
6890     // supported types.
6891     if (CC == ISD::SETGT && RHSC && RHSC->isAllOnesValue() && CTVal && CFVal &&
6892         CTVal->isOne() && CFVal->isAllOnesValue() &&
6893         LHS.getValueType() == TVal.getValueType()) {
6894       EVT VT = LHS.getValueType();
6895       SDValue Shift =
6896           DAG.getNode(ISD::SRA, dl, VT, LHS,
6897                       DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
6898       return DAG.getNode(ISD::OR, dl, VT, Shift, DAG.getConstant(1, dl, VT));
6899     }
6900 
6901     unsigned Opcode = AArch64ISD::CSEL;
6902 
6903     // If both the TVal and the FVal are constants, see if we can swap them in
6904     // order to for a CSINV or CSINC out of them.
6905     if (CTVal && CFVal && CTVal->isAllOnesValue() && CFVal->isNullValue()) {
6906       std::swap(TVal, FVal);
6907       std::swap(CTVal, CFVal);
6908       CC = ISD::getSetCCInverse(CC, LHS.getValueType());
6909     } else if (CTVal && CFVal && CTVal->isOne() && CFVal->isNullValue()) {
6910       std::swap(TVal, FVal);
6911       std::swap(CTVal, CFVal);
6912       CC = ISD::getSetCCInverse(CC, LHS.getValueType());
6913     } else if (TVal.getOpcode() == ISD::XOR) {
6914       // If TVal is a NOT we want to swap TVal and FVal so that we can match
6915       // with a CSINV rather than a CSEL.
6916       if (isAllOnesConstant(TVal.getOperand(1))) {
6917         std::swap(TVal, FVal);
6918         std::swap(CTVal, CFVal);
6919         CC = ISD::getSetCCInverse(CC, LHS.getValueType());
6920       }
6921     } else if (TVal.getOpcode() == ISD::SUB) {
6922       // If TVal is a negation (SUB from 0) we want to swap TVal and FVal so
6923       // that we can match with a CSNEG rather than a CSEL.
6924       if (isNullConstant(TVal.getOperand(0))) {
6925         std::swap(TVal, FVal);
6926         std::swap(CTVal, CFVal);
6927         CC = ISD::getSetCCInverse(CC, LHS.getValueType());
6928       }
6929     } else if (CTVal && CFVal) {
6930       const int64_t TrueVal = CTVal->getSExtValue();
6931       const int64_t FalseVal = CFVal->getSExtValue();
6932       bool Swap = false;
6933 
6934       // If both TVal and FVal are constants, see if FVal is the
6935       // inverse/negation/increment of TVal and generate a CSINV/CSNEG/CSINC
6936       // instead of a CSEL in that case.
6937       if (TrueVal == ~FalseVal) {
6938         Opcode = AArch64ISD::CSINV;
6939       } else if (FalseVal > std::numeric_limits<int64_t>::min() &&
6940                  TrueVal == -FalseVal) {
6941         Opcode = AArch64ISD::CSNEG;
6942       } else if (TVal.getValueType() == MVT::i32) {
6943         // If our operands are only 32-bit wide, make sure we use 32-bit
6944         // arithmetic for the check whether we can use CSINC. This ensures that
6945         // the addition in the check will wrap around properly in case there is
6946         // an overflow (which would not be the case if we do the check with
6947         // 64-bit arithmetic).
6948         const uint32_t TrueVal32 = CTVal->getZExtValue();
6949         const uint32_t FalseVal32 = CFVal->getZExtValue();
6950 
6951         if ((TrueVal32 == FalseVal32 + 1) || (TrueVal32 + 1 == FalseVal32)) {
6952           Opcode = AArch64ISD::CSINC;
6953 
6954           if (TrueVal32 > FalseVal32) {
6955             Swap = true;
6956           }
6957         }
6958         // 64-bit check whether we can use CSINC.
6959       } else if ((TrueVal == FalseVal + 1) || (TrueVal + 1 == FalseVal)) {
6960         Opcode = AArch64ISD::CSINC;
6961 
6962         if (TrueVal > FalseVal) {
6963           Swap = true;
6964         }
6965       }
6966 
6967       // Swap TVal and FVal if necessary.
6968       if (Swap) {
6969         std::swap(TVal, FVal);
6970         std::swap(CTVal, CFVal);
6971         CC = ISD::getSetCCInverse(CC, LHS.getValueType());
6972       }
6973 
6974       if (Opcode != AArch64ISD::CSEL) {
6975         // Drop FVal since we can get its value by simply inverting/negating
6976         // TVal.
6977         FVal = TVal;
6978       }
6979     }
6980 
6981     // Avoid materializing a constant when possible by reusing a known value in
6982     // a register.  However, don't perform this optimization if the known value
6983     // is one, zero or negative one in the case of a CSEL.  We can always
6984     // materialize these values using CSINC, CSEL and CSINV with wzr/xzr as the
6985     // FVal, respectively.
6986     ConstantSDNode *RHSVal = dyn_cast<ConstantSDNode>(RHS);
6987     if (Opcode == AArch64ISD::CSEL && RHSVal && !RHSVal->isOne() &&
6988         !RHSVal->isNullValue() && !RHSVal->isAllOnesValue()) {
6989       AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
6990       // Transform "a == C ? C : x" to "a == C ? a : x" and "a != C ? x : C" to
6991       // "a != C ? x : a" to avoid materializing C.
6992       if (CTVal && CTVal == RHSVal && AArch64CC == AArch64CC::EQ)
6993         TVal = LHS;
6994       else if (CFVal && CFVal == RHSVal && AArch64CC == AArch64CC::NE)
6995         FVal = LHS;
6996     } else if (Opcode == AArch64ISD::CSNEG && RHSVal && RHSVal->isOne()) {
6997       assert (CTVal && CFVal && "Expected constant operands for CSNEG.");
6998       // Use a CSINV to transform "a == C ? 1 : -1" to "a == C ? a : -1" to
6999       // avoid materializing C.
7000       AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
7001       if (CTVal == RHSVal && AArch64CC == AArch64CC::EQ) {
7002         Opcode = AArch64ISD::CSINV;
7003         TVal = LHS;
7004         FVal = DAG.getConstant(0, dl, FVal.getValueType());
7005       }
7006     }
7007 
7008     SDValue CCVal;
7009     SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
7010     EVT VT = TVal.getValueType();
7011     return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp);
7012   }
7013 
7014   // Now we know we're dealing with FP values.
7015   assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||
7016          LHS.getValueType() == MVT::f64);
7017   assert(LHS.getValueType() == RHS.getValueType());
7018   EVT VT = TVal.getValueType();
7019   SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
7020 
7021   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
7022   // clean.  Some of them require two CSELs to implement.
7023   AArch64CC::CondCode CC1, CC2;
7024   changeFPCCToAArch64CC(CC, CC1, CC2);
7025 
7026   if (DAG.getTarget().Options.UnsafeFPMath) {
7027     // Transform "a == 0.0 ? 0.0 : x" to "a == 0.0 ? a : x" and
7028     // "a != 0.0 ? x : 0.0" to "a != 0.0 ? x : a" to avoid materializing 0.0.
7029     ConstantFPSDNode *RHSVal = dyn_cast<ConstantFPSDNode>(RHS);
7030     if (RHSVal && RHSVal->isZero()) {
7031       ConstantFPSDNode *CFVal = dyn_cast<ConstantFPSDNode>(FVal);
7032       ConstantFPSDNode *CTVal = dyn_cast<ConstantFPSDNode>(TVal);
7033 
7034       if ((CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
7035           CTVal && CTVal->isZero() && TVal.getValueType() == LHS.getValueType())
7036         TVal = LHS;
7037       else if ((CC == ISD::SETNE || CC == ISD::SETONE || CC == ISD::SETUNE) &&
7038                CFVal && CFVal->isZero() &&
7039                FVal.getValueType() == LHS.getValueType())
7040         FVal = LHS;
7041     }
7042   }
7043 
7044   // Emit first, and possibly only, CSEL.
7045   SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
7046   SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
7047 
7048   // If we need a second CSEL, emit it, using the output of the first as the
7049   // RHS.  We're effectively OR'ing the two CC's together.
7050   if (CC2 != AArch64CC::AL) {
7051     SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
7052     return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
7053   }
7054 
7055   // Otherwise, return the output of the first CSEL.
7056   return CS1;
7057 }
7058 
7059 SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
7060                                               SelectionDAG &DAG) const {
7061   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
7062   SDValue LHS = Op.getOperand(0);
7063   SDValue RHS = Op.getOperand(1);
7064   SDValue TVal = Op.getOperand(2);
7065   SDValue FVal = Op.getOperand(3);
7066   SDLoc DL(Op);
7067   return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
7068 }
7069 
7070 SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
7071                                            SelectionDAG &DAG) const {
7072   SDValue CCVal = Op->getOperand(0);
7073   SDValue TVal = Op->getOperand(1);
7074   SDValue FVal = Op->getOperand(2);
7075   SDLoc DL(Op);
7076 
7077   EVT Ty = Op.getValueType();
7078   if (Ty.isScalableVector()) {
7079     SDValue TruncCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, CCVal);
7080     MVT PredVT = MVT::getVectorVT(MVT::i1, Ty.getVectorElementCount());
7081     SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, TruncCC);
7082     return DAG.getNode(ISD::VSELECT, DL, Ty, SplatPred, TVal, FVal);
7083   }
7084 
7085   if (useSVEForFixedLengthVectorVT(Ty)) {
7086     // FIXME: Ideally this would be the same as above using i1 types, however
7087     // for the moment we can't deal with fixed i1 vector types properly, so
7088     // instead extend the predicate to a result type sized integer vector.
7089     MVT SplatValVT = MVT::getIntegerVT(Ty.getScalarSizeInBits());
7090     MVT PredVT = MVT::getVectorVT(SplatValVT, Ty.getVectorElementCount());
7091     SDValue SplatVal = DAG.getSExtOrTrunc(CCVal, DL, SplatValVT);
7092     SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, SplatVal);
7093     return DAG.getNode(ISD::VSELECT, DL, Ty, SplatPred, TVal, FVal);
7094   }
7095 
7096   // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select
7097   // instruction.
7098   if (ISD::isOverflowIntrOpRes(CCVal)) {
7099     // Only lower legal XALUO ops.
7100     if (!DAG.getTargetLoweringInfo().isTypeLegal(CCVal->getValueType(0)))
7101       return SDValue();
7102 
7103     AArch64CC::CondCode OFCC;
7104     SDValue Value, Overflow;
7105     std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, CCVal.getValue(0), DAG);
7106     SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
7107 
7108     return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
7109                        CCVal, Overflow);
7110   }
7111 
7112   // Lower it the same way as we would lower a SELECT_CC node.
7113   ISD::CondCode CC;
7114   SDValue LHS, RHS;
7115   if (CCVal.getOpcode() == ISD::SETCC) {
7116     LHS = CCVal.getOperand(0);
7117     RHS = CCVal.getOperand(1);
7118     CC = cast<CondCodeSDNode>(CCVal.getOperand(2))->get();
7119   } else {
7120     LHS = CCVal;
7121     RHS = DAG.getConstant(0, DL, CCVal.getValueType());
7122     CC = ISD::SETNE;
7123   }
7124   return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
7125 }
7126 
7127 SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
7128                                               SelectionDAG &DAG) const {
7129   // Jump table entries as PC relative offsets. No additional tweaking
7130   // is necessary here. Just get the address of the jump table.
7131   JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
7132 
7133   if (getTargetMachine().getCodeModel() == CodeModel::Large &&
7134       !Subtarget->isTargetMachO()) {
7135     return getAddrLarge(JT, DAG);
7136   } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
7137     return getAddrTiny(JT, DAG);
7138   }
7139   return getAddr(JT, DAG);
7140 }
7141 
7142 SDValue AArch64TargetLowering::LowerBR_JT(SDValue Op,
7143                                           SelectionDAG &DAG) const {
7144   // Jump table entries as PC relative offsets. No additional tweaking
7145   // is necessary here. Just get the address of the jump table.
7146   SDLoc DL(Op);
7147   SDValue JT = Op.getOperand(1);
7148   SDValue Entry = Op.getOperand(2);
7149   int JTI = cast<JumpTableSDNode>(JT.getNode())->getIndex();
7150 
7151   auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
7152   AFI->setJumpTableEntryInfo(JTI, 4, nullptr);
7153 
7154   SDNode *Dest =
7155       DAG.getMachineNode(AArch64::JumpTableDest32, DL, MVT::i64, MVT::i64, JT,
7156                          Entry, DAG.getTargetJumpTable(JTI, MVT::i32));
7157   return DAG.getNode(ISD::BRIND, DL, MVT::Other, Op.getOperand(0),
7158                      SDValue(Dest, 0));
7159 }
7160 
7161 SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
7162                                                  SelectionDAG &DAG) const {
7163   ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
7164 
7165   if (getTargetMachine().getCodeModel() == CodeModel::Large) {
7166     // Use the GOT for the large code model on iOS.
7167     if (Subtarget->isTargetMachO()) {
7168       return getGOT(CP, DAG);
7169     }
7170     return getAddrLarge(CP, DAG);
7171   } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
7172     return getAddrTiny(CP, DAG);
7173   } else {
7174     return getAddr(CP, DAG);
7175   }
7176 }
7177 
7178 SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
7179                                                SelectionDAG &DAG) const {
7180   BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Op);
7181   if (getTargetMachine().getCodeModel() == CodeModel::Large &&
7182       !Subtarget->isTargetMachO()) {
7183     return getAddrLarge(BA, DAG);
7184   } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
7185     return getAddrTiny(BA, DAG);
7186   }
7187   return getAddr(BA, DAG);
7188 }
7189 
7190 SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
7191                                                  SelectionDAG &DAG) const {
7192   AArch64FunctionInfo *FuncInfo =
7193       DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
7194 
7195   SDLoc DL(Op);
7196   SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(),
7197                                  getPointerTy(DAG.getDataLayout()));
7198   FR = DAG.getZExtOrTrunc(FR, DL, getPointerMemTy(DAG.getDataLayout()));
7199   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
7200   return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
7201                       MachinePointerInfo(SV));
7202 }
7203 
7204 SDValue AArch64TargetLowering::LowerWin64_VASTART(SDValue Op,
7205                                                   SelectionDAG &DAG) const {
7206   AArch64FunctionInfo *FuncInfo =
7207       DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
7208 
7209   SDLoc DL(Op);
7210   SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsGPRSize() > 0
7211                                      ? FuncInfo->getVarArgsGPRIndex()
7212                                      : FuncInfo->getVarArgsStackIndex(),
7213                                  getPointerTy(DAG.getDataLayout()));
7214   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
7215   return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
7216                       MachinePointerInfo(SV));
7217 }
7218 
7219 SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
7220                                                   SelectionDAG &DAG) const {
7221   // The layout of the va_list struct is specified in the AArch64 Procedure Call
7222   // Standard, section B.3.
7223   MachineFunction &MF = DAG.getMachineFunction();
7224   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
7225   unsigned PtrSize = Subtarget->isTargetILP32() ? 4 : 8;
7226   auto PtrMemVT = getPointerMemTy(DAG.getDataLayout());
7227   auto PtrVT = getPointerTy(DAG.getDataLayout());
7228   SDLoc DL(Op);
7229 
7230   SDValue Chain = Op.getOperand(0);
7231   SDValue VAList = Op.getOperand(1);
7232   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
7233   SmallVector<SDValue, 4> MemOps;
7234 
7235   // void *__stack at offset 0
7236   unsigned Offset = 0;
7237   SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT);
7238   Stack = DAG.getZExtOrTrunc(Stack, DL, PtrMemVT);
7239   MemOps.push_back(DAG.getStore(Chain, DL, Stack, VAList,
7240                                 MachinePointerInfo(SV), Align(PtrSize)));
7241 
7242   // void *__gr_top at offset 8 (4 on ILP32)
7243   Offset += PtrSize;
7244   int GPRSize = FuncInfo->getVarArgsGPRSize();
7245   if (GPRSize > 0) {
7246     SDValue GRTop, GRTopAddr;
7247 
7248     GRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7249                             DAG.getConstant(Offset, DL, PtrVT));
7250 
7251     GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), PtrVT);
7252     GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop,
7253                         DAG.getConstant(GPRSize, DL, PtrVT));
7254     GRTop = DAG.getZExtOrTrunc(GRTop, DL, PtrMemVT);
7255 
7256     MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr,
7257                                   MachinePointerInfo(SV, Offset),
7258                                   Align(PtrSize)));
7259   }
7260 
7261   // void *__vr_top at offset 16 (8 on ILP32)
7262   Offset += PtrSize;
7263   int FPRSize = FuncInfo->getVarArgsFPRSize();
7264   if (FPRSize > 0) {
7265     SDValue VRTop, VRTopAddr;
7266     VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7267                             DAG.getConstant(Offset, DL, PtrVT));
7268 
7269     VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), PtrVT);
7270     VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop,
7271                         DAG.getConstant(FPRSize, DL, PtrVT));
7272     VRTop = DAG.getZExtOrTrunc(VRTop, DL, PtrMemVT);
7273 
7274     MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr,
7275                                   MachinePointerInfo(SV, Offset),
7276                                   Align(PtrSize)));
7277   }
7278 
7279   // int __gr_offs at offset 24 (12 on ILP32)
7280   Offset += PtrSize;
7281   SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7282                                    DAG.getConstant(Offset, DL, PtrVT));
7283   MemOps.push_back(
7284       DAG.getStore(Chain, DL, DAG.getConstant(-GPRSize, DL, MVT::i32),
7285                    GROffsAddr, MachinePointerInfo(SV, Offset), Align(4)));
7286 
7287   // int __vr_offs at offset 28 (16 on ILP32)
7288   Offset += 4;
7289   SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7290                                    DAG.getConstant(Offset, DL, PtrVT));
7291   MemOps.push_back(
7292       DAG.getStore(Chain, DL, DAG.getConstant(-FPRSize, DL, MVT::i32),
7293                    VROffsAddr, MachinePointerInfo(SV, Offset), Align(4)));
7294 
7295   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
7296 }
7297 
7298 SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
7299                                             SelectionDAG &DAG) const {
7300   MachineFunction &MF = DAG.getMachineFunction();
7301 
7302   if (Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv()))
7303     return LowerWin64_VASTART(Op, DAG);
7304   else if (Subtarget->isTargetDarwin())
7305     return LowerDarwin_VASTART(Op, DAG);
7306   else
7307     return LowerAAPCS_VASTART(Op, DAG);
7308 }
7309 
7310 SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
7311                                            SelectionDAG &DAG) const {
7312   // AAPCS has three pointers and two ints (= 32 bytes), Darwin has single
7313   // pointer.
7314   SDLoc DL(Op);
7315   unsigned PtrSize = Subtarget->isTargetILP32() ? 4 : 8;
7316   unsigned VaListSize =
7317       (Subtarget->isTargetDarwin() || Subtarget->isTargetWindows())
7318           ? PtrSize
7319           : Subtarget->isTargetILP32() ? 20 : 32;
7320   const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
7321   const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
7322 
7323   return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1), Op.getOperand(2),
7324                        DAG.getConstant(VaListSize, DL, MVT::i32),
7325                        Align(PtrSize), false, false, false,
7326                        MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
7327 }
7328 
7329 SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
7330   assert(Subtarget->isTargetDarwin() &&
7331          "automatic va_arg instruction only works on Darwin");
7332 
7333   const Value *V = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
7334   EVT VT = Op.getValueType();
7335   SDLoc DL(Op);
7336   SDValue Chain = Op.getOperand(0);
7337   SDValue Addr = Op.getOperand(1);
7338   MaybeAlign Align(Op.getConstantOperandVal(3));
7339   unsigned MinSlotSize = Subtarget->isTargetILP32() ? 4 : 8;
7340   auto PtrVT = getPointerTy(DAG.getDataLayout());
7341   auto PtrMemVT = getPointerMemTy(DAG.getDataLayout());
7342   SDValue VAList =
7343       DAG.getLoad(PtrMemVT, DL, Chain, Addr, MachinePointerInfo(V));
7344   Chain = VAList.getValue(1);
7345   VAList = DAG.getZExtOrTrunc(VAList, DL, PtrVT);
7346 
7347   if (VT.isScalableVector())
7348     report_fatal_error("Passing SVE types to variadic functions is "
7349                        "currently not supported");
7350 
7351   if (Align && *Align > MinSlotSize) {
7352     VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7353                          DAG.getConstant(Align->value() - 1, DL, PtrVT));
7354     VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList,
7355                          DAG.getConstant(-(int64_t)Align->value(), DL, PtrVT));
7356   }
7357 
7358   Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
7359   unsigned ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
7360 
7361   // Scalar integer and FP values smaller than 64 bits are implicitly extended
7362   // up to 64 bits.  At the very least, we have to increase the striding of the
7363   // vaargs list to match this, and for FP values we need to introduce
7364   // FP_ROUND nodes as well.
7365   if (VT.isInteger() && !VT.isVector())
7366     ArgSize = std::max(ArgSize, MinSlotSize);
7367   bool NeedFPTrunc = false;
7368   if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) {
7369     ArgSize = 8;
7370     NeedFPTrunc = true;
7371   }
7372 
7373   // Increment the pointer, VAList, to the next vaarg
7374   SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7375                                DAG.getConstant(ArgSize, DL, PtrVT));
7376   VANext = DAG.getZExtOrTrunc(VANext, DL, PtrMemVT);
7377 
7378   // Store the incremented VAList to the legalized pointer
7379   SDValue APStore =
7380       DAG.getStore(Chain, DL, VANext, Addr, MachinePointerInfo(V));
7381 
7382   // Load the actual argument out of the pointer VAList
7383   if (NeedFPTrunc) {
7384     // Load the value as an f64.
7385     SDValue WideFP =
7386         DAG.getLoad(MVT::f64, DL, APStore, VAList, MachinePointerInfo());
7387     // Round the value down to an f32.
7388     SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
7389                                    DAG.getIntPtrConstant(1, DL));
7390     SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
7391     // Merge the rounded value with the chain output of the load.
7392     return DAG.getMergeValues(Ops, DL);
7393   }
7394 
7395   return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo());
7396 }
7397 
7398 SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
7399                                               SelectionDAG &DAG) const {
7400   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7401   MFI.setFrameAddressIsTaken(true);
7402 
7403   EVT VT = Op.getValueType();
7404   SDLoc DL(Op);
7405   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
7406   SDValue FrameAddr =
7407       DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, MVT::i64);
7408   while (Depth--)
7409     FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr,
7410                             MachinePointerInfo());
7411 
7412   if (Subtarget->isTargetILP32())
7413     FrameAddr = DAG.getNode(ISD::AssertZext, DL, MVT::i64, FrameAddr,
7414                             DAG.getValueType(VT));
7415 
7416   return FrameAddr;
7417 }
7418 
7419 SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
7420                                               SelectionDAG &DAG) const {
7421   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7422 
7423   EVT VT = getPointerTy(DAG.getDataLayout());
7424   SDLoc DL(Op);
7425   int FI = MFI.CreateFixedObject(4, 0, false);
7426   return DAG.getFrameIndex(FI, VT);
7427 }
7428 
7429 #define GET_REGISTER_MATCHER
7430 #include "AArch64GenAsmMatcher.inc"
7431 
7432 // FIXME? Maybe this could be a TableGen attribute on some registers and
7433 // this table could be generated automatically from RegInfo.
7434 Register AArch64TargetLowering::
7435 getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const {
7436   Register Reg = MatchRegisterName(RegName);
7437   if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
7438     const MCRegisterInfo *MRI = Subtarget->getRegisterInfo();
7439     unsigned DwarfRegNum = MRI->getDwarfRegNum(Reg, false);
7440     if (!Subtarget->isXRegisterReserved(DwarfRegNum))
7441       Reg = 0;
7442   }
7443   if (Reg)
7444     return Reg;
7445   report_fatal_error(Twine("Invalid register name \""
7446                               + StringRef(RegName)  + "\"."));
7447 }
7448 
7449 SDValue AArch64TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
7450                                                      SelectionDAG &DAG) const {
7451   DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
7452 
7453   EVT VT = Op.getValueType();
7454   SDLoc DL(Op);
7455 
7456   SDValue FrameAddr =
7457       DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
7458   SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
7459 
7460   return DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset);
7461 }
7462 
7463 SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
7464                                                SelectionDAG &DAG) const {
7465   MachineFunction &MF = DAG.getMachineFunction();
7466   MachineFrameInfo &MFI = MF.getFrameInfo();
7467   MFI.setReturnAddressIsTaken(true);
7468 
7469   EVT VT = Op.getValueType();
7470   SDLoc DL(Op);
7471   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
7472   SDValue ReturnAddress;
7473   if (Depth) {
7474     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
7475     SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
7476     ReturnAddress = DAG.getLoad(
7477         VT, DL, DAG.getEntryNode(),
7478         DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), MachinePointerInfo());
7479   } else {
7480     // Return LR, which contains the return address. Mark it an implicit
7481     // live-in.
7482     unsigned Reg = MF.addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
7483     ReturnAddress = DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
7484   }
7485 
7486   // The XPACLRI instruction assembles to a hint-space instruction before
7487   // Armv8.3-A therefore this instruction can be safely used for any pre
7488   // Armv8.3-A architectures. On Armv8.3-A and onwards XPACI is available so use
7489   // that instead.
7490   SDNode *St;
7491   if (Subtarget->hasPAuth()) {
7492     St = DAG.getMachineNode(AArch64::XPACI, DL, VT, ReturnAddress);
7493   } else {
7494     // XPACLRI operates on LR therefore we must move the operand accordingly.
7495     SDValue Chain =
7496         DAG.getCopyToReg(DAG.getEntryNode(), DL, AArch64::LR, ReturnAddress);
7497     St = DAG.getMachineNode(AArch64::XPACLRI, DL, VT, Chain);
7498   }
7499   return SDValue(St, 0);
7500 }
7501 
7502 /// LowerShiftRightParts - Lower SRA_PARTS, which returns two
7503 /// i64 values and take a 2 x i64 value to shift plus a shift amount.
7504 SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
7505                                                     SelectionDAG &DAG) const {
7506   assert(Op.getNumOperands() == 3 && "Not a double-shift!");
7507   EVT VT = Op.getValueType();
7508   unsigned VTBits = VT.getSizeInBits();
7509   SDLoc dl(Op);
7510   SDValue ShOpLo = Op.getOperand(0);
7511   SDValue ShOpHi = Op.getOperand(1);
7512   SDValue ShAmt = Op.getOperand(2);
7513   unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
7514 
7515   assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
7516 
7517   SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
7518                                  DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
7519   SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
7520 
7521   // Unfortunately, if ShAmt == 0, we just calculated "(SHL ShOpHi, 64)" which
7522   // is "undef". We wanted 0, so CSEL it directly.
7523   SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
7524                                ISD::SETEQ, dl, DAG);
7525   SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
7526   HiBitsForLo =
7527       DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64),
7528                   HiBitsForLo, CCVal, Cmp);
7529 
7530   SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
7531                                    DAG.getConstant(VTBits, dl, MVT::i64));
7532 
7533   SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
7534   SDValue LoForNormalShift =
7535       DAG.getNode(ISD::OR, dl, VT, LoBitsForLo, HiBitsForLo);
7536 
7537   Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE,
7538                        dl, DAG);
7539   CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
7540   SDValue LoForBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
7541   SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
7542                            LoForNormalShift, CCVal, Cmp);
7543 
7544   // AArch64 shifts larger than the register width are wrapped rather than
7545   // clamped, so we can't just emit "hi >> x".
7546   SDValue HiForNormalShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
7547   SDValue HiForBigShift =
7548       Opc == ISD::SRA
7549           ? DAG.getNode(Opc, dl, VT, ShOpHi,
7550                         DAG.getConstant(VTBits - 1, dl, MVT::i64))
7551           : DAG.getConstant(0, dl, VT);
7552   SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
7553                            HiForNormalShift, CCVal, Cmp);
7554 
7555   SDValue Ops[2] = { Lo, Hi };
7556   return DAG.getMergeValues(Ops, dl);
7557 }
7558 
7559 /// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
7560 /// i64 values and take a 2 x i64 value to shift plus a shift amount.
7561 SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
7562                                                    SelectionDAG &DAG) const {
7563   assert(Op.getNumOperands() == 3 && "Not a double-shift!");
7564   EVT VT = Op.getValueType();
7565   unsigned VTBits = VT.getSizeInBits();
7566   SDLoc dl(Op);
7567   SDValue ShOpLo = Op.getOperand(0);
7568   SDValue ShOpHi = Op.getOperand(1);
7569   SDValue ShAmt = Op.getOperand(2);
7570 
7571   assert(Op.getOpcode() == ISD::SHL_PARTS);
7572   SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
7573                                  DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
7574   SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
7575 
7576   // Unfortunately, if ShAmt == 0, we just calculated "(SRL ShOpLo, 64)" which
7577   // is "undef". We wanted 0, so CSEL it directly.
7578   SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
7579                                ISD::SETEQ, dl, DAG);
7580   SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
7581   LoBitsForHi =
7582       DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64),
7583                   LoBitsForHi, CCVal, Cmp);
7584 
7585   SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
7586                                    DAG.getConstant(VTBits, dl, MVT::i64));
7587   SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
7588   SDValue HiForNormalShift =
7589       DAG.getNode(ISD::OR, dl, VT, LoBitsForHi, HiBitsForHi);
7590 
7591   SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
7592 
7593   Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE,
7594                        dl, DAG);
7595   CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
7596   SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
7597                            HiForNormalShift, CCVal, Cmp);
7598 
7599   // AArch64 shifts of larger than register sizes are wrapped rather than
7600   // clamped, so we can't just emit "lo << a" if a is too big.
7601   SDValue LoForBigShift = DAG.getConstant(0, dl, VT);
7602   SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
7603   SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
7604                            LoForNormalShift, CCVal, Cmp);
7605 
7606   SDValue Ops[2] = { Lo, Hi };
7607   return DAG.getMergeValues(Ops, dl);
7608 }
7609 
7610 bool AArch64TargetLowering::isOffsetFoldingLegal(
7611     const GlobalAddressSDNode *GA) const {
7612   // Offsets are folded in the DAG combine rather than here so that we can
7613   // intelligently choose an offset based on the uses.
7614   return false;
7615 }
7616 
7617 bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
7618                                          bool OptForSize) const {
7619   bool IsLegal = false;
7620   // We can materialize #0.0 as fmov $Rd, XZR for 64-bit, 32-bit cases, and
7621   // 16-bit case when target has full fp16 support.
7622   // FIXME: We should be able to handle f128 as well with a clever lowering.
7623   const APInt ImmInt = Imm.bitcastToAPInt();
7624   if (VT == MVT::f64)
7625     IsLegal = AArch64_AM::getFP64Imm(ImmInt) != -1 || Imm.isPosZero();
7626   else if (VT == MVT::f32)
7627     IsLegal = AArch64_AM::getFP32Imm(ImmInt) != -1 || Imm.isPosZero();
7628   else if (VT == MVT::f16 && Subtarget->hasFullFP16())
7629     IsLegal = AArch64_AM::getFP16Imm(ImmInt) != -1 || Imm.isPosZero();
7630   // TODO: fmov h0, w0 is also legal, however on't have an isel pattern to
7631   //       generate that fmov.
7632 
7633   // If we can not materialize in immediate field for fmov, check if the
7634   // value can be encoded as the immediate operand of a logical instruction.
7635   // The immediate value will be created with either MOVZ, MOVN, or ORR.
7636   if (!IsLegal && (VT == MVT::f64 || VT == MVT::f32)) {
7637     // The cost is actually exactly the same for mov+fmov vs. adrp+ldr;
7638     // however the mov+fmov sequence is always better because of the reduced
7639     // cache pressure. The timings are still the same if you consider
7640     // movw+movk+fmov vs. adrp+ldr (it's one instruction longer, but the
7641     // movw+movk is fused). So we limit up to 2 instrdduction at most.
7642     SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
7643     AArch64_IMM::expandMOVImm(ImmInt.getZExtValue(), VT.getSizeInBits(),
7644 			      Insn);
7645     unsigned Limit = (OptForSize ? 1 : (Subtarget->hasFuseLiterals() ? 5 : 2));
7646     IsLegal = Insn.size() <= Limit;
7647   }
7648 
7649   LLVM_DEBUG(dbgs() << (IsLegal ? "Legal " : "Illegal ") << VT.getEVTString()
7650                     << " imm value: "; Imm.dump(););
7651   return IsLegal;
7652 }
7653 
7654 //===----------------------------------------------------------------------===//
7655 //                          AArch64 Optimization Hooks
7656 //===----------------------------------------------------------------------===//
7657 
7658 static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
7659                            SDValue Operand, SelectionDAG &DAG,
7660                            int &ExtraSteps) {
7661   EVT VT = Operand.getValueType();
7662   if (ST->hasNEON() &&
7663       (VT == MVT::f64 || VT == MVT::v1f64 || VT == MVT::v2f64 ||
7664        VT == MVT::f32 || VT == MVT::v1f32 ||
7665        VT == MVT::v2f32 || VT == MVT::v4f32)) {
7666     if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified)
7667       // For the reciprocal estimates, convergence is quadratic, so the number
7668       // of digits is doubled after each iteration.  In ARMv8, the accuracy of
7669       // the initial estimate is 2^-8.  Thus the number of extra steps to refine
7670       // the result for float (23 mantissa bits) is 2 and for double (52
7671       // mantissa bits) is 3.
7672       ExtraSteps = VT.getScalarType() == MVT::f64 ? 3 : 2;
7673 
7674     return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand);
7675   }
7676 
7677   return SDValue();
7678 }
7679 
7680 SDValue
7681 AArch64TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
7682                                         const DenormalMode &Mode) const {
7683   SDLoc DL(Op);
7684   EVT VT = Op.getValueType();
7685   EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7686   SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
7687   return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
7688 }
7689 
7690 SDValue
7691 AArch64TargetLowering::getSqrtResultForDenormInput(SDValue Op,
7692                                                    SelectionDAG &DAG) const {
7693   return Op;
7694 }
7695 
7696 SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
7697                                                SelectionDAG &DAG, int Enabled,
7698                                                int &ExtraSteps,
7699                                                bool &UseOneConst,
7700                                                bool Reciprocal) const {
7701   if (Enabled == ReciprocalEstimate::Enabled ||
7702       (Enabled == ReciprocalEstimate::Unspecified && Subtarget->useRSqrt()))
7703     if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRSQRTE, Operand,
7704                                        DAG, ExtraSteps)) {
7705       SDLoc DL(Operand);
7706       EVT VT = Operand.getValueType();
7707 
7708       SDNodeFlags Flags;
7709       Flags.setAllowReassociation(true);
7710 
7711       // Newton reciprocal square root iteration: E * 0.5 * (3 - X * E^2)
7712       // AArch64 reciprocal square root iteration instruction: 0.5 * (3 - M * N)
7713       for (int i = ExtraSteps; i > 0; --i) {
7714         SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
7715                                    Flags);
7716         Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, Flags);
7717         Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
7718       }
7719       if (!Reciprocal)
7720         Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags);
7721 
7722       ExtraSteps = 0;
7723       return Estimate;
7724     }
7725 
7726   return SDValue();
7727 }
7728 
7729 SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
7730                                                 SelectionDAG &DAG, int Enabled,
7731                                                 int &ExtraSteps) const {
7732   if (Enabled == ReciprocalEstimate::Enabled)
7733     if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRECPE, Operand,
7734                                        DAG, ExtraSteps)) {
7735       SDLoc DL(Operand);
7736       EVT VT = Operand.getValueType();
7737 
7738       SDNodeFlags Flags;
7739       Flags.setAllowReassociation(true);
7740 
7741       // Newton reciprocal iteration: E * (2 - X * E)
7742       // AArch64 reciprocal iteration instruction: (2 - M * N)
7743       for (int i = ExtraSteps; i > 0; --i) {
7744         SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
7745                                    Estimate, Flags);
7746         Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
7747       }
7748 
7749       ExtraSteps = 0;
7750       return Estimate;
7751     }
7752 
7753   return SDValue();
7754 }
7755 
7756 //===----------------------------------------------------------------------===//
7757 //                          AArch64 Inline Assembly Support
7758 //===----------------------------------------------------------------------===//
7759 
7760 // Table of Constraints
7761 // TODO: This is the current set of constraints supported by ARM for the
7762 // compiler, not all of them may make sense.
7763 //
7764 // r - A general register
7765 // w - An FP/SIMD register of some size in the range v0-v31
7766 // x - An FP/SIMD register of some size in the range v0-v15
7767 // I - Constant that can be used with an ADD instruction
7768 // J - Constant that can be used with a SUB instruction
7769 // K - Constant that can be used with a 32-bit logical instruction
7770 // L - Constant that can be used with a 64-bit logical instruction
7771 // M - Constant that can be used as a 32-bit MOV immediate
7772 // N - Constant that can be used as a 64-bit MOV immediate
7773 // Q - A memory reference with base register and no offset
7774 // S - A symbolic address
7775 // Y - Floating point constant zero
7776 // Z - Integer constant zero
7777 //
7778 //   Note that general register operands will be output using their 64-bit x
7779 // register name, whatever the size of the variable, unless the asm operand
7780 // is prefixed by the %w modifier. Floating-point and SIMD register operands
7781 // will be output with the v prefix unless prefixed by the %b, %h, %s, %d or
7782 // %q modifier.
7783 const char *AArch64TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
7784   // At this point, we have to lower this constraint to something else, so we
7785   // lower it to an "r" or "w". However, by doing this we will force the result
7786   // to be in register, while the X constraint is much more permissive.
7787   //
7788   // Although we are correct (we are free to emit anything, without
7789   // constraints), we might break use cases that would expect us to be more
7790   // efficient and emit something else.
7791   if (!Subtarget->hasFPARMv8())
7792     return "r";
7793 
7794   if (ConstraintVT.isFloatingPoint())
7795     return "w";
7796 
7797   if (ConstraintVT.isVector() &&
7798      (ConstraintVT.getSizeInBits() == 64 ||
7799       ConstraintVT.getSizeInBits() == 128))
7800     return "w";
7801 
7802   return "r";
7803 }
7804 
7805 enum PredicateConstraint {
7806   Upl,
7807   Upa,
7808   Invalid
7809 };
7810 
7811 static PredicateConstraint parsePredicateConstraint(StringRef Constraint) {
7812   PredicateConstraint P = PredicateConstraint::Invalid;
7813   if (Constraint == "Upa")
7814     P = PredicateConstraint::Upa;
7815   if (Constraint == "Upl")
7816     P = PredicateConstraint::Upl;
7817   return P;
7818 }
7819 
7820 /// getConstraintType - Given a constraint letter, return the type of
7821 /// constraint it is for this target.
7822 AArch64TargetLowering::ConstraintType
7823 AArch64TargetLowering::getConstraintType(StringRef Constraint) const {
7824   if (Constraint.size() == 1) {
7825     switch (Constraint[0]) {
7826     default:
7827       break;
7828     case 'x':
7829     case 'w':
7830     case 'y':
7831       return C_RegisterClass;
7832     // An address with a single base register. Due to the way we
7833     // currently handle addresses it is the same as 'r'.
7834     case 'Q':
7835       return C_Memory;
7836     case 'I':
7837     case 'J':
7838     case 'K':
7839     case 'L':
7840     case 'M':
7841     case 'N':
7842     case 'Y':
7843     case 'Z':
7844       return C_Immediate;
7845     case 'z':
7846     case 'S': // A symbolic address
7847       return C_Other;
7848     }
7849   } else if (parsePredicateConstraint(Constraint) !=
7850              PredicateConstraint::Invalid)
7851       return C_RegisterClass;
7852   return TargetLowering::getConstraintType(Constraint);
7853 }
7854 
7855 /// Examine constraint type and operand type and determine a weight value.
7856 /// This object must already have been set up with the operand type
7857 /// and the current alternative constraint selected.
7858 TargetLowering::ConstraintWeight
7859 AArch64TargetLowering::getSingleConstraintMatchWeight(
7860     AsmOperandInfo &info, const char *constraint) const {
7861   ConstraintWeight weight = CW_Invalid;
7862   Value *CallOperandVal = info.CallOperandVal;
7863   // If we don't have a value, we can't do a match,
7864   // but allow it at the lowest weight.
7865   if (!CallOperandVal)
7866     return CW_Default;
7867   Type *type = CallOperandVal->getType();
7868   // Look at the constraint type.
7869   switch (*constraint) {
7870   default:
7871     weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
7872     break;
7873   case 'x':
7874   case 'w':
7875   case 'y':
7876     if (type->isFloatingPointTy() || type->isVectorTy())
7877       weight = CW_Register;
7878     break;
7879   case 'z':
7880     weight = CW_Constant;
7881     break;
7882   case 'U':
7883     if (parsePredicateConstraint(constraint) != PredicateConstraint::Invalid)
7884       weight = CW_Register;
7885     break;
7886   }
7887   return weight;
7888 }
7889 
7890 std::pair<unsigned, const TargetRegisterClass *>
7891 AArch64TargetLowering::getRegForInlineAsmConstraint(
7892     const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
7893   if (Constraint.size() == 1) {
7894     switch (Constraint[0]) {
7895     case 'r':
7896       if (VT.isScalableVector())
7897         return std::make_pair(0U, nullptr);
7898       if (VT.getFixedSizeInBits() == 64)
7899         return std::make_pair(0U, &AArch64::GPR64commonRegClass);
7900       return std::make_pair(0U, &AArch64::GPR32commonRegClass);
7901     case 'w': {
7902       if (!Subtarget->hasFPARMv8())
7903         break;
7904       if (VT.isScalableVector()) {
7905         if (VT.getVectorElementType() != MVT::i1)
7906           return std::make_pair(0U, &AArch64::ZPRRegClass);
7907         return std::make_pair(0U, nullptr);
7908       }
7909       uint64_t VTSize = VT.getFixedSizeInBits();
7910       if (VTSize == 16)
7911         return std::make_pair(0U, &AArch64::FPR16RegClass);
7912       if (VTSize == 32)
7913         return std::make_pair(0U, &AArch64::FPR32RegClass);
7914       if (VTSize == 64)
7915         return std::make_pair(0U, &AArch64::FPR64RegClass);
7916       if (VTSize == 128)
7917         return std::make_pair(0U, &AArch64::FPR128RegClass);
7918       break;
7919     }
7920     // The instructions that this constraint is designed for can
7921     // only take 128-bit registers so just use that regclass.
7922     case 'x':
7923       if (!Subtarget->hasFPARMv8())
7924         break;
7925       if (VT.isScalableVector())
7926         return std::make_pair(0U, &AArch64::ZPR_4bRegClass);
7927       if (VT.getSizeInBits() == 128)
7928         return std::make_pair(0U, &AArch64::FPR128_loRegClass);
7929       break;
7930     case 'y':
7931       if (!Subtarget->hasFPARMv8())
7932         break;
7933       if (VT.isScalableVector())
7934         return std::make_pair(0U, &AArch64::ZPR_3bRegClass);
7935       break;
7936     }
7937   } else {
7938     PredicateConstraint PC = parsePredicateConstraint(Constraint);
7939     if (PC != PredicateConstraint::Invalid) {
7940       if (!VT.isScalableVector() || VT.getVectorElementType() != MVT::i1)
7941         return std::make_pair(0U, nullptr);
7942       bool restricted = (PC == PredicateConstraint::Upl);
7943       return restricted ? std::make_pair(0U, &AArch64::PPR_3bRegClass)
7944                         : std::make_pair(0U, &AArch64::PPRRegClass);
7945     }
7946   }
7947   if (StringRef("{cc}").equals_lower(Constraint))
7948     return std::make_pair(unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
7949 
7950   // Use the default implementation in TargetLowering to convert the register
7951   // constraint into a member of a register class.
7952   std::pair<unsigned, const TargetRegisterClass *> Res;
7953   Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
7954 
7955   // Not found as a standard register?
7956   if (!Res.second) {
7957     unsigned Size = Constraint.size();
7958     if ((Size == 4 || Size == 5) && Constraint[0] == '{' &&
7959         tolower(Constraint[1]) == 'v' && Constraint[Size - 1] == '}') {
7960       int RegNo;
7961       bool Failed = Constraint.slice(2, Size - 1).getAsInteger(10, RegNo);
7962       if (!Failed && RegNo >= 0 && RegNo <= 31) {
7963         // v0 - v31 are aliases of q0 - q31 or d0 - d31 depending on size.
7964         // By default we'll emit v0-v31 for this unless there's a modifier where
7965         // we'll emit the correct register as well.
7966         if (VT != MVT::Other && VT.getSizeInBits() == 64) {
7967           Res.first = AArch64::FPR64RegClass.getRegister(RegNo);
7968           Res.second = &AArch64::FPR64RegClass;
7969         } else {
7970           Res.first = AArch64::FPR128RegClass.getRegister(RegNo);
7971           Res.second = &AArch64::FPR128RegClass;
7972         }
7973       }
7974     }
7975   }
7976 
7977   if (Res.second && !Subtarget->hasFPARMv8() &&
7978       !AArch64::GPR32allRegClass.hasSubClassEq(Res.second) &&
7979       !AArch64::GPR64allRegClass.hasSubClassEq(Res.second))
7980     return std::make_pair(0U, nullptr);
7981 
7982   return Res;
7983 }
7984 
7985 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
7986 /// vector.  If it is invalid, don't add anything to Ops.
7987 void AArch64TargetLowering::LowerAsmOperandForConstraint(
7988     SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
7989     SelectionDAG &DAG) const {
7990   SDValue Result;
7991 
7992   // Currently only support length 1 constraints.
7993   if (Constraint.length() != 1)
7994     return;
7995 
7996   char ConstraintLetter = Constraint[0];
7997   switch (ConstraintLetter) {
7998   default:
7999     break;
8000 
8001   // This set of constraints deal with valid constants for various instructions.
8002   // Validate and return a target constant for them if we can.
8003   case 'z': {
8004     // 'z' maps to xzr or wzr so it needs an input of 0.
8005     if (!isNullConstant(Op))
8006       return;
8007 
8008     if (Op.getValueType() == MVT::i64)
8009       Result = DAG.getRegister(AArch64::XZR, MVT::i64);
8010     else
8011       Result = DAG.getRegister(AArch64::WZR, MVT::i32);
8012     break;
8013   }
8014   case 'S': {
8015     // An absolute symbolic address or label reference.
8016     if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
8017       Result = DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
8018                                           GA->getValueType(0));
8019     } else if (const BlockAddressSDNode *BA =
8020                    dyn_cast<BlockAddressSDNode>(Op)) {
8021       Result =
8022           DAG.getTargetBlockAddress(BA->getBlockAddress(), BA->getValueType(0));
8023     } else if (const ExternalSymbolSDNode *ES =
8024                    dyn_cast<ExternalSymbolSDNode>(Op)) {
8025       Result =
8026           DAG.getTargetExternalSymbol(ES->getSymbol(), ES->getValueType(0));
8027     } else
8028       return;
8029     break;
8030   }
8031 
8032   case 'I':
8033   case 'J':
8034   case 'K':
8035   case 'L':
8036   case 'M':
8037   case 'N':
8038     ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
8039     if (!C)
8040       return;
8041 
8042     // Grab the value and do some validation.
8043     uint64_t CVal = C->getZExtValue();
8044     switch (ConstraintLetter) {
8045     // The I constraint applies only to simple ADD or SUB immediate operands:
8046     // i.e. 0 to 4095 with optional shift by 12
8047     // The J constraint applies only to ADD or SUB immediates that would be
8048     // valid when negated, i.e. if [an add pattern] were to be output as a SUB
8049     // instruction [or vice versa], in other words -1 to -4095 with optional
8050     // left shift by 12.
8051     case 'I':
8052       if (isUInt<12>(CVal) || isShiftedUInt<12, 12>(CVal))
8053         break;
8054       return;
8055     case 'J': {
8056       uint64_t NVal = -C->getSExtValue();
8057       if (isUInt<12>(NVal) || isShiftedUInt<12, 12>(NVal)) {
8058         CVal = C->getSExtValue();
8059         break;
8060       }
8061       return;
8062     }
8063     // The K and L constraints apply *only* to logical immediates, including
8064     // what used to be the MOVI alias for ORR (though the MOVI alias has now
8065     // been removed and MOV should be used). So these constraints have to
8066     // distinguish between bit patterns that are valid 32-bit or 64-bit
8067     // "bitmask immediates": for example 0xaaaaaaaa is a valid bimm32 (K), but
8068     // not a valid bimm64 (L) where 0xaaaaaaaaaaaaaaaa would be valid, and vice
8069     // versa.
8070     case 'K':
8071       if (AArch64_AM::isLogicalImmediate(CVal, 32))
8072         break;
8073       return;
8074     case 'L':
8075       if (AArch64_AM::isLogicalImmediate(CVal, 64))
8076         break;
8077       return;
8078     // The M and N constraints are a superset of K and L respectively, for use
8079     // with the MOV (immediate) alias. As well as the logical immediates they
8080     // also match 32 or 64-bit immediates that can be loaded either using a
8081     // *single* MOVZ or MOVN , such as 32-bit 0x12340000, 0x00001234, 0xffffedca
8082     // (M) or 64-bit 0x1234000000000000 (N) etc.
8083     // As a note some of this code is liberally stolen from the asm parser.
8084     case 'M': {
8085       if (!isUInt<32>(CVal))
8086         return;
8087       if (AArch64_AM::isLogicalImmediate(CVal, 32))
8088         break;
8089       if ((CVal & 0xFFFF) == CVal)
8090         break;
8091       if ((CVal & 0xFFFF0000ULL) == CVal)
8092         break;
8093       uint64_t NCVal = ~(uint32_t)CVal;
8094       if ((NCVal & 0xFFFFULL) == NCVal)
8095         break;
8096       if ((NCVal & 0xFFFF0000ULL) == NCVal)
8097         break;
8098       return;
8099     }
8100     case 'N': {
8101       if (AArch64_AM::isLogicalImmediate(CVal, 64))
8102         break;
8103       if ((CVal & 0xFFFFULL) == CVal)
8104         break;
8105       if ((CVal & 0xFFFF0000ULL) == CVal)
8106         break;
8107       if ((CVal & 0xFFFF00000000ULL) == CVal)
8108         break;
8109       if ((CVal & 0xFFFF000000000000ULL) == CVal)
8110         break;
8111       uint64_t NCVal = ~CVal;
8112       if ((NCVal & 0xFFFFULL) == NCVal)
8113         break;
8114       if ((NCVal & 0xFFFF0000ULL) == NCVal)
8115         break;
8116       if ((NCVal & 0xFFFF00000000ULL) == NCVal)
8117         break;
8118       if ((NCVal & 0xFFFF000000000000ULL) == NCVal)
8119         break;
8120       return;
8121     }
8122     default:
8123       return;
8124     }
8125 
8126     // All assembler immediates are 64-bit integers.
8127     Result = DAG.getTargetConstant(CVal, SDLoc(Op), MVT::i64);
8128     break;
8129   }
8130 
8131   if (Result.getNode()) {
8132     Ops.push_back(Result);
8133     return;
8134   }
8135 
8136   return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
8137 }
8138 
8139 //===----------------------------------------------------------------------===//
8140 //                     AArch64 Advanced SIMD Support
8141 //===----------------------------------------------------------------------===//
8142 
8143 /// WidenVector - Given a value in the V64 register class, produce the
8144 /// equivalent value in the V128 register class.
8145 static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) {
8146   EVT VT = V64Reg.getValueType();
8147   unsigned NarrowSize = VT.getVectorNumElements();
8148   MVT EltTy = VT.getVectorElementType().getSimpleVT();
8149   MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize);
8150   SDLoc DL(V64Reg);
8151 
8152   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy),
8153                      V64Reg, DAG.getConstant(0, DL, MVT::i64));
8154 }
8155 
8156 /// getExtFactor - Determine the adjustment factor for the position when
8157 /// generating an "extract from vector registers" instruction.
8158 static unsigned getExtFactor(SDValue &V) {
8159   EVT EltType = V.getValueType().getVectorElementType();
8160   return EltType.getSizeInBits() / 8;
8161 }
8162 
8163 /// NarrowVector - Given a value in the V128 register class, produce the
8164 /// equivalent value in the V64 register class.
8165 static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
8166   EVT VT = V128Reg.getValueType();
8167   unsigned WideSize = VT.getVectorNumElements();
8168   MVT EltTy = VT.getVectorElementType().getSimpleVT();
8169   MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2);
8170   SDLoc DL(V128Reg);
8171 
8172   return DAG.getTargetExtractSubreg(AArch64::dsub, DL, NarrowTy, V128Reg);
8173 }
8174 
8175 // Gather data to see if the operation can be modelled as a
8176 // shuffle in combination with VEXTs.
8177 SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
8178                                                   SelectionDAG &DAG) const {
8179   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
8180   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::ReconstructShuffle\n");
8181   SDLoc dl(Op);
8182   EVT VT = Op.getValueType();
8183   assert(!VT.isScalableVector() &&
8184          "Scalable vectors cannot be used with ISD::BUILD_VECTOR");
8185   unsigned NumElts = VT.getVectorNumElements();
8186 
8187   struct ShuffleSourceInfo {
8188     SDValue Vec;
8189     unsigned MinElt;
8190     unsigned MaxElt;
8191 
8192     // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
8193     // be compatible with the shuffle we intend to construct. As a result
8194     // ShuffleVec will be some sliding window into the original Vec.
8195     SDValue ShuffleVec;
8196 
8197     // Code should guarantee that element i in Vec starts at element "WindowBase
8198     // + i * WindowScale in ShuffleVec".
8199     int WindowBase;
8200     int WindowScale;
8201 
8202     ShuffleSourceInfo(SDValue Vec)
8203       : Vec(Vec), MinElt(std::numeric_limits<unsigned>::max()), MaxElt(0),
8204           ShuffleVec(Vec), WindowBase(0), WindowScale(1) {}
8205 
8206     bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
8207   };
8208 
8209   // First gather all vectors used as an immediate source for this BUILD_VECTOR
8210   // node.
8211   SmallVector<ShuffleSourceInfo, 2> Sources;
8212   for (unsigned i = 0; i < NumElts; ++i) {
8213     SDValue V = Op.getOperand(i);
8214     if (V.isUndef())
8215       continue;
8216     else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
8217              !isa<ConstantSDNode>(V.getOperand(1))) {
8218       LLVM_DEBUG(
8219           dbgs() << "Reshuffle failed: "
8220                     "a shuffle can only come from building a vector from "
8221                     "various elements of other vectors, provided their "
8222                     "indices are constant\n");
8223       return SDValue();
8224     }
8225 
8226     // Add this element source to the list if it's not already there.
8227     SDValue SourceVec = V.getOperand(0);
8228     auto Source = find(Sources, SourceVec);
8229     if (Source == Sources.end())
8230       Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec));
8231 
8232     // Update the minimum and maximum lane number seen.
8233     unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue();
8234     Source->MinElt = std::min(Source->MinElt, EltNo);
8235     Source->MaxElt = std::max(Source->MaxElt, EltNo);
8236   }
8237 
8238   if (Sources.size() > 2) {
8239     LLVM_DEBUG(
8240         dbgs() << "Reshuffle failed: currently only do something sane when at "
8241                   "most two source vectors are involved\n");
8242     return SDValue();
8243   }
8244 
8245   // Find out the smallest element size among result and two sources, and use
8246   // it as element size to build the shuffle_vector.
8247   EVT SmallestEltTy = VT.getVectorElementType();
8248   for (auto &Source : Sources) {
8249     EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
8250     if (SrcEltTy.bitsLT(SmallestEltTy)) {
8251       SmallestEltTy = SrcEltTy;
8252     }
8253   }
8254   unsigned ResMultiplier =
8255       VT.getScalarSizeInBits() / SmallestEltTy.getFixedSizeInBits();
8256   uint64_t VTSize = VT.getFixedSizeInBits();
8257   NumElts = VTSize / SmallestEltTy.getFixedSizeInBits();
8258   EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts);
8259 
8260   // If the source vector is too wide or too narrow, we may nevertheless be able
8261   // to construct a compatible shuffle either by concatenating it with UNDEF or
8262   // extracting a suitable range of elements.
8263   for (auto &Src : Sources) {
8264     EVT SrcVT = Src.ShuffleVec.getValueType();
8265 
8266     uint64_t SrcVTSize = SrcVT.getFixedSizeInBits();
8267     if (SrcVTSize == VTSize)
8268       continue;
8269 
8270     // This stage of the search produces a source with the same element type as
8271     // the original, but with a total width matching the BUILD_VECTOR output.
8272     EVT EltVT = SrcVT.getVectorElementType();
8273     unsigned NumSrcElts = VTSize / EltVT.getFixedSizeInBits();
8274     EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
8275 
8276     if (SrcVTSize < VTSize) {
8277       assert(2 * SrcVTSize == VTSize);
8278       // We can pad out the smaller vector for free, so if it's part of a
8279       // shuffle...
8280       Src.ShuffleVec =
8281           DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
8282                       DAG.getUNDEF(Src.ShuffleVec.getValueType()));
8283       continue;
8284     }
8285 
8286     if (SrcVTSize != 2 * VTSize) {
8287       LLVM_DEBUG(
8288           dbgs() << "Reshuffle failed: result vector too small to extract\n");
8289       return SDValue();
8290     }
8291 
8292     if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8293       LLVM_DEBUG(
8294           dbgs() << "Reshuffle failed: span too large for a VEXT to cope\n");
8295       return SDValue();
8296     }
8297 
8298     if (Src.MinElt >= NumSrcElts) {
8299       // The extraction can just take the second half
8300       Src.ShuffleVec =
8301           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8302                       DAG.getConstant(NumSrcElts, dl, MVT::i64));
8303       Src.WindowBase = -NumSrcElts;
8304     } else if (Src.MaxElt < NumSrcElts) {
8305       // The extraction can just take the first half
8306       Src.ShuffleVec =
8307           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8308                       DAG.getConstant(0, dl, MVT::i64));
8309     } else {
8310       // An actual VEXT is needed
8311       SDValue VEXTSrc1 =
8312           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8313                       DAG.getConstant(0, dl, MVT::i64));
8314       SDValue VEXTSrc2 =
8315           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8316                       DAG.getConstant(NumSrcElts, dl, MVT::i64));
8317       unsigned Imm = Src.MinElt * getExtFactor(VEXTSrc1);
8318 
8319       if (!SrcVT.is64BitVector()) {
8320         LLVM_DEBUG(
8321           dbgs() << "Reshuffle failed: don't know how to lower AArch64ISD::EXT "
8322                     "for SVE vectors.");
8323         return SDValue();
8324       }
8325 
8326       Src.ShuffleVec = DAG.getNode(AArch64ISD::EXT, dl, DestVT, VEXTSrc1,
8327                                    VEXTSrc2,
8328                                    DAG.getConstant(Imm, dl, MVT::i32));
8329       Src.WindowBase = -Src.MinElt;
8330     }
8331   }
8332 
8333   // Another possible incompatibility occurs from the vector element types. We
8334   // can fix this by bitcasting the source vectors to the same type we intend
8335   // for the shuffle.
8336   for (auto &Src : Sources) {
8337     EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8338     if (SrcEltTy == SmallestEltTy)
8339       continue;
8340     assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
8341     Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec);
8342     Src.WindowScale =
8343         SrcEltTy.getFixedSizeInBits() / SmallestEltTy.getFixedSizeInBits();
8344     Src.WindowBase *= Src.WindowScale;
8345   }
8346 
8347   // Final sanity check before we try to actually produce a shuffle.
8348   LLVM_DEBUG(for (auto Src
8349                   : Sources)
8350                  assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8351 
8352   // The stars all align, our next step is to produce the mask for the shuffle.
8353   SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
8354   int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
8355   for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
8356     SDValue Entry = Op.getOperand(i);
8357     if (Entry.isUndef())
8358       continue;
8359 
8360     auto Src = find(Sources, Entry.getOperand(0));
8361     int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
8362 
8363     // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
8364     // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
8365     // segment.
8366     EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
8367     int BitsDefined = std::min(OrigEltTy.getScalarSizeInBits(),
8368                                VT.getScalarSizeInBits());
8369     int LanesDefined = BitsDefined / BitsPerShuffleLane;
8370 
8371     // This source is expected to fill ResMultiplier lanes of the final shuffle,
8372     // starting at the appropriate offset.
8373     int *LaneMask = &Mask[i * ResMultiplier];
8374 
8375     int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8376     ExtractBase += NumElts * (Src - Sources.begin());
8377     for (int j = 0; j < LanesDefined; ++j)
8378       LaneMask[j] = ExtractBase + j;
8379   }
8380 
8381   // Final check before we try to produce nonsense...
8382   if (!isShuffleMaskLegal(Mask, ShuffleVT)) {
8383     LLVM_DEBUG(dbgs() << "Reshuffle failed: illegal shuffle mask\n");
8384     return SDValue();
8385   }
8386 
8387   SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
8388   for (unsigned i = 0; i < Sources.size(); ++i)
8389     ShuffleOps[i] = Sources[i].ShuffleVec;
8390 
8391   SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
8392                                          ShuffleOps[1], Mask);
8393   SDValue V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
8394 
8395   LLVM_DEBUG(dbgs() << "Reshuffle, creating node: "; Shuffle.dump();
8396              dbgs() << "Reshuffle, creating node: "; V.dump(););
8397 
8398   return V;
8399 }
8400 
8401 // check if an EXT instruction can handle the shuffle mask when the
8402 // vector sources of the shuffle are the same.
8403 static bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
8404   unsigned NumElts = VT.getVectorNumElements();
8405 
8406   // Assume that the first shuffle index is not UNDEF.  Fail if it is.
8407   if (M[0] < 0)
8408     return false;
8409 
8410   Imm = M[0];
8411 
8412   // If this is a VEXT shuffle, the immediate value is the index of the first
8413   // element.  The other shuffle indices must be the successive elements after
8414   // the first one.
8415   unsigned ExpectedElt = Imm;
8416   for (unsigned i = 1; i < NumElts; ++i) {
8417     // Increment the expected index.  If it wraps around, just follow it
8418     // back to index zero and keep going.
8419     ++ExpectedElt;
8420     if (ExpectedElt == NumElts)
8421       ExpectedElt = 0;
8422 
8423     if (M[i] < 0)
8424       continue; // ignore UNDEF indices
8425     if (ExpectedElt != static_cast<unsigned>(M[i]))
8426       return false;
8427   }
8428 
8429   return true;
8430 }
8431 
8432 /// Check if a vector shuffle corresponds to a DUP instructions with a larger
8433 /// element width than the vector lane type. If that is the case the function
8434 /// returns true and writes the value of the DUP instruction lane operand into
8435 /// DupLaneOp
8436 static bool isWideDUPMask(ArrayRef<int> M, EVT VT, unsigned BlockSize,
8437                           unsigned &DupLaneOp) {
8438   assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) &&
8439          "Only possible block sizes for wide DUP are: 16, 32, 64");
8440 
8441   if (BlockSize <= VT.getScalarSizeInBits())
8442     return false;
8443   if (BlockSize % VT.getScalarSizeInBits() != 0)
8444     return false;
8445   if (VT.getSizeInBits() % BlockSize != 0)
8446     return false;
8447 
8448   size_t SingleVecNumElements = VT.getVectorNumElements();
8449   size_t NumEltsPerBlock = BlockSize / VT.getScalarSizeInBits();
8450   size_t NumBlocks = VT.getSizeInBits() / BlockSize;
8451 
8452   // We are looking for masks like
8453   // [0, 1, 0, 1] or [2, 3, 2, 3] or [4, 5, 6, 7, 4, 5, 6, 7] where any element
8454   // might be replaced by 'undefined'. BlockIndices will eventually contain
8455   // lane indices of the duplicated block (i.e. [0, 1], [2, 3] and [4, 5, 6, 7]
8456   // for the above examples)
8457   SmallVector<int, 8> BlockElts(NumEltsPerBlock, -1);
8458   for (size_t BlockIndex = 0; BlockIndex < NumBlocks; BlockIndex++)
8459     for (size_t I = 0; I < NumEltsPerBlock; I++) {
8460       int Elt = M[BlockIndex * NumEltsPerBlock + I];
8461       if (Elt < 0)
8462         continue;
8463       // For now we don't support shuffles that use the second operand
8464       if ((unsigned)Elt >= SingleVecNumElements)
8465         return false;
8466       if (BlockElts[I] < 0)
8467         BlockElts[I] = Elt;
8468       else if (BlockElts[I] != Elt)
8469         return false;
8470     }
8471 
8472   // We found a candidate block (possibly with some undefs). It must be a
8473   // sequence of consecutive integers starting with a value divisible by
8474   // NumEltsPerBlock with some values possibly replaced by undef-s.
8475 
8476   // Find first non-undef element
8477   auto FirstRealEltIter = find_if(BlockElts, [](int Elt) { return Elt >= 0; });
8478   assert(FirstRealEltIter != BlockElts.end() &&
8479          "Shuffle with all-undefs must have been caught by previous cases, "
8480          "e.g. isSplat()");
8481   if (FirstRealEltIter == BlockElts.end()) {
8482     DupLaneOp = 0;
8483     return true;
8484   }
8485 
8486   // Index of FirstRealElt in BlockElts
8487   size_t FirstRealIndex = FirstRealEltIter - BlockElts.begin();
8488 
8489   if ((unsigned)*FirstRealEltIter < FirstRealIndex)
8490     return false;
8491   // BlockElts[0] must have the following value if it isn't undef:
8492   size_t Elt0 = *FirstRealEltIter - FirstRealIndex;
8493 
8494   // Check the first element
8495   if (Elt0 % NumEltsPerBlock != 0)
8496     return false;
8497   // Check that the sequence indeed consists of consecutive integers (modulo
8498   // undefs)
8499   for (size_t I = 0; I < NumEltsPerBlock; I++)
8500     if (BlockElts[I] >= 0 && (unsigned)BlockElts[I] != Elt0 + I)
8501       return false;
8502 
8503   DupLaneOp = Elt0 / NumEltsPerBlock;
8504   return true;
8505 }
8506 
8507 // check if an EXT instruction can handle the shuffle mask when the
8508 // vector sources of the shuffle are different.
8509 static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
8510                       unsigned &Imm) {
8511   // Look for the first non-undef element.
8512   const int *FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
8513 
8514   // Benefit form APInt to handle overflow when calculating expected element.
8515   unsigned NumElts = VT.getVectorNumElements();
8516   unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
8517   APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1);
8518   // The following shuffle indices must be the successive elements after the
8519   // first real element.
8520   const int *FirstWrongElt = std::find_if(FirstRealElt + 1, M.end(),
8521       [&](int Elt) {return Elt != ExpectedElt++ && Elt != -1;});
8522   if (FirstWrongElt != M.end())
8523     return false;
8524 
8525   // The index of an EXT is the first element if it is not UNDEF.
8526   // Watch out for the beginning UNDEFs. The EXT index should be the expected
8527   // value of the first element.  E.g.
8528   // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.
8529   // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.
8530   // ExpectedElt is the last mask index plus 1.
8531   Imm = ExpectedElt.getZExtValue();
8532 
8533   // There are two difference cases requiring to reverse input vectors.
8534   // For example, for vector <4 x i32> we have the following cases,
8535   // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>)
8536   // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>)
8537   // For both cases, we finally use mask <5, 6, 7, 0>, which requires
8538   // to reverse two input vectors.
8539   if (Imm < NumElts)
8540     ReverseEXT = true;
8541   else
8542     Imm -= NumElts;
8543 
8544   return true;
8545 }
8546 
8547 /// isREVMask - Check if a vector shuffle corresponds to a REV
8548 /// instruction with the specified blocksize.  (The order of the elements
8549 /// within each block of the vector is reversed.)
8550 static bool isREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) {
8551   assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) &&
8552          "Only possible block sizes for REV are: 16, 32, 64");
8553 
8554   unsigned EltSz = VT.getScalarSizeInBits();
8555   if (EltSz == 64)
8556     return false;
8557 
8558   unsigned NumElts = VT.getVectorNumElements();
8559   unsigned BlockElts = M[0] + 1;
8560   // If the first shuffle index is UNDEF, be optimistic.
8561   if (M[0] < 0)
8562     BlockElts = BlockSize / EltSz;
8563 
8564   if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz)
8565     return false;
8566 
8567   for (unsigned i = 0; i < NumElts; ++i) {
8568     if (M[i] < 0)
8569       continue; // ignore UNDEF indices
8570     if ((unsigned)M[i] != (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts))
8571       return false;
8572   }
8573 
8574   return true;
8575 }
8576 
8577 static bool isZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8578   unsigned NumElts = VT.getVectorNumElements();
8579   if (NumElts % 2 != 0)
8580     return false;
8581   WhichResult = (M[0] == 0 ? 0 : 1);
8582   unsigned Idx = WhichResult * NumElts / 2;
8583   for (unsigned i = 0; i != NumElts; i += 2) {
8584     if ((M[i] >= 0 && (unsigned)M[i] != Idx) ||
8585         (M[i + 1] >= 0 && (unsigned)M[i + 1] != Idx + NumElts))
8586       return false;
8587     Idx += 1;
8588   }
8589 
8590   return true;
8591 }
8592 
8593 static bool isUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8594   unsigned NumElts = VT.getVectorNumElements();
8595   WhichResult = (M[0] == 0 ? 0 : 1);
8596   for (unsigned i = 0; i != NumElts; ++i) {
8597     if (M[i] < 0)
8598       continue; // ignore UNDEF indices
8599     if ((unsigned)M[i] != 2 * i + WhichResult)
8600       return false;
8601   }
8602 
8603   return true;
8604 }
8605 
8606 static bool isTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8607   unsigned NumElts = VT.getVectorNumElements();
8608   if (NumElts % 2 != 0)
8609     return false;
8610   WhichResult = (M[0] == 0 ? 0 : 1);
8611   for (unsigned i = 0; i < NumElts; i += 2) {
8612     if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) ||
8613         (M[i + 1] >= 0 && (unsigned)M[i + 1] != i + NumElts + WhichResult))
8614       return false;
8615   }
8616   return true;
8617 }
8618 
8619 /// isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of
8620 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
8621 /// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
8622 static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8623   unsigned NumElts = VT.getVectorNumElements();
8624   if (NumElts % 2 != 0)
8625     return false;
8626   WhichResult = (M[0] == 0 ? 0 : 1);
8627   unsigned Idx = WhichResult * NumElts / 2;
8628   for (unsigned i = 0; i != NumElts; i += 2) {
8629     if ((M[i] >= 0 && (unsigned)M[i] != Idx) ||
8630         (M[i + 1] >= 0 && (unsigned)M[i + 1] != Idx))
8631       return false;
8632     Idx += 1;
8633   }
8634 
8635   return true;
8636 }
8637 
8638 /// isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of
8639 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
8640 /// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
8641 static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8642   unsigned Half = VT.getVectorNumElements() / 2;
8643   WhichResult = (M[0] == 0 ? 0 : 1);
8644   for (unsigned j = 0; j != 2; ++j) {
8645     unsigned Idx = WhichResult;
8646     for (unsigned i = 0; i != Half; ++i) {
8647       int MIdx = M[i + j * Half];
8648       if (MIdx >= 0 && (unsigned)MIdx != Idx)
8649         return false;
8650       Idx += 2;
8651     }
8652   }
8653 
8654   return true;
8655 }
8656 
8657 /// isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of
8658 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
8659 /// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
8660 static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8661   unsigned NumElts = VT.getVectorNumElements();
8662   if (NumElts % 2 != 0)
8663     return false;
8664   WhichResult = (M[0] == 0 ? 0 : 1);
8665   for (unsigned i = 0; i < NumElts; i += 2) {
8666     if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) ||
8667         (M[i + 1] >= 0 && (unsigned)M[i + 1] != i + WhichResult))
8668       return false;
8669   }
8670   return true;
8671 }
8672 
8673 static bool isINSMask(ArrayRef<int> M, int NumInputElements,
8674                       bool &DstIsLeft, int &Anomaly) {
8675   if (M.size() != static_cast<size_t>(NumInputElements))
8676     return false;
8677 
8678   int NumLHSMatch = 0, NumRHSMatch = 0;
8679   int LastLHSMismatch = -1, LastRHSMismatch = -1;
8680 
8681   for (int i = 0; i < NumInputElements; ++i) {
8682     if (M[i] == -1) {
8683       ++NumLHSMatch;
8684       ++NumRHSMatch;
8685       continue;
8686     }
8687 
8688     if (M[i] == i)
8689       ++NumLHSMatch;
8690     else
8691       LastLHSMismatch = i;
8692 
8693     if (M[i] == i + NumInputElements)
8694       ++NumRHSMatch;
8695     else
8696       LastRHSMismatch = i;
8697   }
8698 
8699   if (NumLHSMatch == NumInputElements - 1) {
8700     DstIsLeft = true;
8701     Anomaly = LastLHSMismatch;
8702     return true;
8703   } else if (NumRHSMatch == NumInputElements - 1) {
8704     DstIsLeft = false;
8705     Anomaly = LastRHSMismatch;
8706     return true;
8707   }
8708 
8709   return false;
8710 }
8711 
8712 static bool isConcatMask(ArrayRef<int> Mask, EVT VT, bool SplitLHS) {
8713   if (VT.getSizeInBits() != 128)
8714     return false;
8715 
8716   unsigned NumElts = VT.getVectorNumElements();
8717 
8718   for (int I = 0, E = NumElts / 2; I != E; I++) {
8719     if (Mask[I] != I)
8720       return false;
8721   }
8722 
8723   int Offset = NumElts / 2;
8724   for (int I = NumElts / 2, E = NumElts; I != E; I++) {
8725     if (Mask[I] != I + SplitLHS * Offset)
8726       return false;
8727   }
8728 
8729   return true;
8730 }
8731 
8732 static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
8733   SDLoc DL(Op);
8734   EVT VT = Op.getValueType();
8735   SDValue V0 = Op.getOperand(0);
8736   SDValue V1 = Op.getOperand(1);
8737   ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask();
8738 
8739   if (VT.getVectorElementType() != V0.getValueType().getVectorElementType() ||
8740       VT.getVectorElementType() != V1.getValueType().getVectorElementType())
8741     return SDValue();
8742 
8743   bool SplitV0 = V0.getValueSizeInBits() == 128;
8744 
8745   if (!isConcatMask(Mask, VT, SplitV0))
8746     return SDValue();
8747 
8748   EVT CastVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
8749   if (SplitV0) {
8750     V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0,
8751                      DAG.getConstant(0, DL, MVT::i64));
8752   }
8753   if (V1.getValueSizeInBits() == 128) {
8754     V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1,
8755                      DAG.getConstant(0, DL, MVT::i64));
8756   }
8757   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1);
8758 }
8759 
8760 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
8761 /// the specified operations to build the shuffle.
8762 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
8763                                       SDValue RHS, SelectionDAG &DAG,
8764                                       const SDLoc &dl) {
8765   unsigned OpNum = (PFEntry >> 26) & 0x0F;
8766   unsigned LHSID = (PFEntry >> 13) & ((1 << 13) - 1);
8767   unsigned RHSID = (PFEntry >> 0) & ((1 << 13) - 1);
8768 
8769   enum {
8770     OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
8771     OP_VREV,
8772     OP_VDUP0,
8773     OP_VDUP1,
8774     OP_VDUP2,
8775     OP_VDUP3,
8776     OP_VEXT1,
8777     OP_VEXT2,
8778     OP_VEXT3,
8779     OP_VUZPL, // VUZP, left result
8780     OP_VUZPR, // VUZP, right result
8781     OP_VZIPL, // VZIP, left result
8782     OP_VZIPR, // VZIP, right result
8783     OP_VTRNL, // VTRN, left result
8784     OP_VTRNR  // VTRN, right result
8785   };
8786 
8787   if (OpNum == OP_COPY) {
8788     if (LHSID == (1 * 9 + 2) * 9 + 3)
8789       return LHS;
8790     assert(LHSID == ((4 * 9 + 5) * 9 + 6) * 9 + 7 && "Illegal OP_COPY!");
8791     return RHS;
8792   }
8793 
8794   SDValue OpLHS, OpRHS;
8795   OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
8796   OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
8797   EVT VT = OpLHS.getValueType();
8798 
8799   switch (OpNum) {
8800   default:
8801     llvm_unreachable("Unknown shuffle opcode!");
8802   case OP_VREV:
8803     // VREV divides the vector in half and swaps within the half.
8804     if (VT.getVectorElementType() == MVT::i32 ||
8805         VT.getVectorElementType() == MVT::f32)
8806       return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS);
8807     // vrev <4 x i16> -> REV32
8808     if (VT.getVectorElementType() == MVT::i16 ||
8809         VT.getVectorElementType() == MVT::f16 ||
8810         VT.getVectorElementType() == MVT::bf16)
8811       return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS);
8812     // vrev <4 x i8> -> REV16
8813     assert(VT.getVectorElementType() == MVT::i8);
8814     return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS);
8815   case OP_VDUP0:
8816   case OP_VDUP1:
8817   case OP_VDUP2:
8818   case OP_VDUP3: {
8819     EVT EltTy = VT.getVectorElementType();
8820     unsigned Opcode;
8821     if (EltTy == MVT::i8)
8822       Opcode = AArch64ISD::DUPLANE8;
8823     else if (EltTy == MVT::i16 || EltTy == MVT::f16 || EltTy == MVT::bf16)
8824       Opcode = AArch64ISD::DUPLANE16;
8825     else if (EltTy == MVT::i32 || EltTy == MVT::f32)
8826       Opcode = AArch64ISD::DUPLANE32;
8827     else if (EltTy == MVT::i64 || EltTy == MVT::f64)
8828       Opcode = AArch64ISD::DUPLANE64;
8829     else
8830       llvm_unreachable("Invalid vector element type?");
8831 
8832     if (VT.getSizeInBits() == 64)
8833       OpLHS = WidenVector(OpLHS, DAG);
8834     SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64);
8835     return DAG.getNode(Opcode, dl, VT, OpLHS, Lane);
8836   }
8837   case OP_VEXT1:
8838   case OP_VEXT2:
8839   case OP_VEXT3: {
8840     unsigned Imm = (OpNum - OP_VEXT1 + 1) * getExtFactor(OpLHS);
8841     return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS,
8842                        DAG.getConstant(Imm, dl, MVT::i32));
8843   }
8844   case OP_VUZPL:
8845     return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), OpLHS,
8846                        OpRHS);
8847   case OP_VUZPR:
8848     return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), OpLHS,
8849                        OpRHS);
8850   case OP_VZIPL:
8851     return DAG.getNode(AArch64ISD::ZIP1, dl, DAG.getVTList(VT, VT), OpLHS,
8852                        OpRHS);
8853   case OP_VZIPR:
8854     return DAG.getNode(AArch64ISD::ZIP2, dl, DAG.getVTList(VT, VT), OpLHS,
8855                        OpRHS);
8856   case OP_VTRNL:
8857     return DAG.getNode(AArch64ISD::TRN1, dl, DAG.getVTList(VT, VT), OpLHS,
8858                        OpRHS);
8859   case OP_VTRNR:
8860     return DAG.getNode(AArch64ISD::TRN2, dl, DAG.getVTList(VT, VT), OpLHS,
8861                        OpRHS);
8862   }
8863 }
8864 
8865 static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
8866                            SelectionDAG &DAG) {
8867   // Check to see if we can use the TBL instruction.
8868   SDValue V1 = Op.getOperand(0);
8869   SDValue V2 = Op.getOperand(1);
8870   SDLoc DL(Op);
8871 
8872   EVT EltVT = Op.getValueType().getVectorElementType();
8873   unsigned BytesPerElt = EltVT.getSizeInBits() / 8;
8874 
8875   SmallVector<SDValue, 8> TBLMask;
8876   for (int Val : ShuffleMask) {
8877     for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
8878       unsigned Offset = Byte + Val * BytesPerElt;
8879       TBLMask.push_back(DAG.getConstant(Offset, DL, MVT::i32));
8880     }
8881   }
8882 
8883   MVT IndexVT = MVT::v8i8;
8884   unsigned IndexLen = 8;
8885   if (Op.getValueSizeInBits() == 128) {
8886     IndexVT = MVT::v16i8;
8887     IndexLen = 16;
8888   }
8889 
8890   SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1);
8891   SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2);
8892 
8893   SDValue Shuffle;
8894   if (V2.getNode()->isUndef()) {
8895     if (IndexLen == 8)
8896       V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst);
8897     Shuffle = DAG.getNode(
8898         ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
8899         DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst,
8900         DAG.getBuildVector(IndexVT, DL,
8901                            makeArrayRef(TBLMask.data(), IndexLen)));
8902   } else {
8903     if (IndexLen == 8) {
8904       V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst);
8905       Shuffle = DAG.getNode(
8906           ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
8907           DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst,
8908           DAG.getBuildVector(IndexVT, DL,
8909                              makeArrayRef(TBLMask.data(), IndexLen)));
8910     } else {
8911       // FIXME: We cannot, for the moment, emit a TBL2 instruction because we
8912       // cannot currently represent the register constraints on the input
8913       // table registers.
8914       //  Shuffle = DAG.getNode(AArch64ISD::TBL2, DL, IndexVT, V1Cst, V2Cst,
8915       //                   DAG.getBuildVector(IndexVT, DL, &TBLMask[0],
8916       //                   IndexLen));
8917       Shuffle = DAG.getNode(
8918           ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
8919           DAG.getConstant(Intrinsic::aarch64_neon_tbl2, DL, MVT::i32), V1Cst,
8920           V2Cst, DAG.getBuildVector(IndexVT, DL,
8921                                     makeArrayRef(TBLMask.data(), IndexLen)));
8922     }
8923   }
8924   return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle);
8925 }
8926 
8927 static unsigned getDUPLANEOp(EVT EltType) {
8928   if (EltType == MVT::i8)
8929     return AArch64ISD::DUPLANE8;
8930   if (EltType == MVT::i16 || EltType == MVT::f16 || EltType == MVT::bf16)
8931     return AArch64ISD::DUPLANE16;
8932   if (EltType == MVT::i32 || EltType == MVT::f32)
8933     return AArch64ISD::DUPLANE32;
8934   if (EltType == MVT::i64 || EltType == MVT::f64)
8935     return AArch64ISD::DUPLANE64;
8936 
8937   llvm_unreachable("Invalid vector element type?");
8938 }
8939 
8940 static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT,
8941                             unsigned Opcode, SelectionDAG &DAG) {
8942   // Try to eliminate a bitcasted extract subvector before a DUPLANE.
8943   auto getScaledOffsetDup = [](SDValue BitCast, int &LaneC, MVT &CastVT) {
8944     // Match: dup (bitcast (extract_subv X, C)), LaneC
8945     if (BitCast.getOpcode() != ISD::BITCAST ||
8946         BitCast.getOperand(0).getOpcode() != ISD::EXTRACT_SUBVECTOR)
8947       return false;
8948 
8949     // The extract index must align in the destination type. That may not
8950     // happen if the bitcast is from narrow to wide type.
8951     SDValue Extract = BitCast.getOperand(0);
8952     unsigned ExtIdx = Extract.getConstantOperandVal(1);
8953     unsigned SrcEltBitWidth = Extract.getScalarValueSizeInBits();
8954     unsigned ExtIdxInBits = ExtIdx * SrcEltBitWidth;
8955     unsigned CastedEltBitWidth = BitCast.getScalarValueSizeInBits();
8956     if (ExtIdxInBits % CastedEltBitWidth != 0)
8957       return false;
8958 
8959     // Update the lane value by offsetting with the scaled extract index.
8960     LaneC += ExtIdxInBits / CastedEltBitWidth;
8961 
8962     // Determine the casted vector type of the wide vector input.
8963     // dup (bitcast (extract_subv X, C)), LaneC --> dup (bitcast X), LaneC'
8964     // Examples:
8965     // dup (bitcast (extract_subv v2f64 X, 1) to v2f32), 1 --> dup v4f32 X, 3
8966     // dup (bitcast (extract_subv v16i8 X, 8) to v4i16), 1 --> dup v8i16 X, 5
8967     unsigned SrcVecNumElts =
8968         Extract.getOperand(0).getValueSizeInBits() / CastedEltBitWidth;
8969     CastVT = MVT::getVectorVT(BitCast.getSimpleValueType().getScalarType(),
8970                               SrcVecNumElts);
8971     return true;
8972   };
8973   MVT CastVT;
8974   if (getScaledOffsetDup(V, Lane, CastVT)) {
8975     V = DAG.getBitcast(CastVT, V.getOperand(0).getOperand(0));
8976   } else if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
8977     // The lane is incremented by the index of the extract.
8978     // Example: dup v2f32 (extract v4f32 X, 2), 1 --> dup v4f32 X, 3
8979     Lane += V.getConstantOperandVal(1);
8980     V = V.getOperand(0);
8981   } else if (V.getOpcode() == ISD::CONCAT_VECTORS) {
8982     // The lane is decremented if we are splatting from the 2nd operand.
8983     // Example: dup v4i32 (concat v2i32 X, v2i32 Y), 3 --> dup v4i32 Y, 1
8984     unsigned Idx = Lane >= (int)VT.getVectorNumElements() / 2;
8985     Lane -= Idx * VT.getVectorNumElements() / 2;
8986     V = WidenVector(V.getOperand(Idx), DAG);
8987   } else if (VT.getSizeInBits() == 64) {
8988     // Widen the operand to 128-bit register with undef.
8989     V = WidenVector(V, DAG);
8990   }
8991   return DAG.getNode(Opcode, dl, VT, V, DAG.getConstant(Lane, dl, MVT::i64));
8992 }
8993 
8994 SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
8995                                                    SelectionDAG &DAG) const {
8996   SDLoc dl(Op);
8997   EVT VT = Op.getValueType();
8998 
8999   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
9000 
9001   // Convert shuffles that are directly supported on NEON to target-specific
9002   // DAG nodes, instead of keeping them as shuffles and matching them again
9003   // during code selection.  This is more efficient and avoids the possibility
9004   // of inconsistencies between legalization and selection.
9005   ArrayRef<int> ShuffleMask = SVN->getMask();
9006 
9007   SDValue V1 = Op.getOperand(0);
9008   SDValue V2 = Op.getOperand(1);
9009 
9010   assert(V1.getValueType() == VT && "Unexpected VECTOR_SHUFFLE type!");
9011   assert(ShuffleMask.size() == VT.getVectorNumElements() &&
9012          "Unexpected VECTOR_SHUFFLE mask size!");
9013 
9014   if (SVN->isSplat()) {
9015     int Lane = SVN->getSplatIndex();
9016     // If this is undef splat, generate it via "just" vdup, if possible.
9017     if (Lane == -1)
9018       Lane = 0;
9019 
9020     if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR)
9021       return DAG.getNode(AArch64ISD::DUP, dl, V1.getValueType(),
9022                          V1.getOperand(0));
9023     // Test if V1 is a BUILD_VECTOR and the lane being referenced is a non-
9024     // constant. If so, we can just reference the lane's definition directly.
9025     if (V1.getOpcode() == ISD::BUILD_VECTOR &&
9026         !isa<ConstantSDNode>(V1.getOperand(Lane)))
9027       return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane));
9028 
9029     // Otherwise, duplicate from the lane of the input vector.
9030     unsigned Opcode = getDUPLANEOp(V1.getValueType().getVectorElementType());
9031     return constructDup(V1, Lane, dl, VT, Opcode, DAG);
9032   }
9033 
9034   // Check if the mask matches a DUP for a wider element
9035   for (unsigned LaneSize : {64U, 32U, 16U}) {
9036     unsigned Lane = 0;
9037     if (isWideDUPMask(ShuffleMask, VT, LaneSize, Lane)) {
9038       unsigned Opcode = LaneSize == 64 ? AArch64ISD::DUPLANE64
9039                                        : LaneSize == 32 ? AArch64ISD::DUPLANE32
9040                                                         : AArch64ISD::DUPLANE16;
9041       // Cast V1 to an integer vector with required lane size
9042       MVT NewEltTy = MVT::getIntegerVT(LaneSize);
9043       unsigned NewEltCount = VT.getSizeInBits() / LaneSize;
9044       MVT NewVecTy = MVT::getVectorVT(NewEltTy, NewEltCount);
9045       V1 = DAG.getBitcast(NewVecTy, V1);
9046       // Constuct the DUP instruction
9047       V1 = constructDup(V1, Lane, dl, NewVecTy, Opcode, DAG);
9048       // Cast back to the original type
9049       return DAG.getBitcast(VT, V1);
9050     }
9051   }
9052 
9053   if (isREVMask(ShuffleMask, VT, 64))
9054     return DAG.getNode(AArch64ISD::REV64, dl, V1.getValueType(), V1, V2);
9055   if (isREVMask(ShuffleMask, VT, 32))
9056     return DAG.getNode(AArch64ISD::REV32, dl, V1.getValueType(), V1, V2);
9057   if (isREVMask(ShuffleMask, VT, 16))
9058     return DAG.getNode(AArch64ISD::REV16, dl, V1.getValueType(), V1, V2);
9059 
9060   if (((VT.getVectorNumElements() == 8 && VT.getScalarSizeInBits() == 16) ||
9061        (VT.getVectorNumElements() == 16 && VT.getScalarSizeInBits() == 8)) &&
9062       ShuffleVectorInst::isReverseMask(ShuffleMask)) {
9063     SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1);
9064     return DAG.getNode(AArch64ISD::EXT, dl, VT, Rev, Rev,
9065                        DAG.getConstant(8, dl, MVT::i32));
9066   }
9067 
9068   bool ReverseEXT = false;
9069   unsigned Imm;
9070   if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) {
9071     if (ReverseEXT)
9072       std::swap(V1, V2);
9073     Imm *= getExtFactor(V1);
9074     return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V2,
9075                        DAG.getConstant(Imm, dl, MVT::i32));
9076   } else if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, VT, Imm)) {
9077     Imm *= getExtFactor(V1);
9078     return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V1,
9079                        DAG.getConstant(Imm, dl, MVT::i32));
9080   }
9081 
9082   unsigned WhichResult;
9083   if (isZIPMask(ShuffleMask, VT, WhichResult)) {
9084     unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2;
9085     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
9086   }
9087   if (isUZPMask(ShuffleMask, VT, WhichResult)) {
9088     unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
9089     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
9090   }
9091   if (isTRNMask(ShuffleMask, VT, WhichResult)) {
9092     unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
9093     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
9094   }
9095 
9096   if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
9097     unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2;
9098     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
9099   }
9100   if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
9101     unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
9102     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
9103   }
9104   if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
9105     unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
9106     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
9107   }
9108 
9109   if (SDValue Concat = tryFormConcatFromShuffle(Op, DAG))
9110     return Concat;
9111 
9112   bool DstIsLeft;
9113   int Anomaly;
9114   int NumInputElements = V1.getValueType().getVectorNumElements();
9115   if (isINSMask(ShuffleMask, NumInputElements, DstIsLeft, Anomaly)) {
9116     SDValue DstVec = DstIsLeft ? V1 : V2;
9117     SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64);
9118 
9119     SDValue SrcVec = V1;
9120     int SrcLane = ShuffleMask[Anomaly];
9121     if (SrcLane >= NumInputElements) {
9122       SrcVec = V2;
9123       SrcLane -= VT.getVectorNumElements();
9124     }
9125     SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64);
9126 
9127     EVT ScalarVT = VT.getVectorElementType();
9128 
9129     if (ScalarVT.getFixedSizeInBits() < 32 && ScalarVT.isInteger())
9130       ScalarVT = MVT::i32;
9131 
9132     return DAG.getNode(
9133         ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
9134         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, SrcVec, SrcLaneV),
9135         DstLaneV);
9136   }
9137 
9138   // If the shuffle is not directly supported and it has 4 elements, use
9139   // the PerfectShuffle-generated table to synthesize it from other shuffles.
9140   unsigned NumElts = VT.getVectorNumElements();
9141   if (NumElts == 4) {
9142     unsigned PFIndexes[4];
9143     for (unsigned i = 0; i != 4; ++i) {
9144       if (ShuffleMask[i] < 0)
9145         PFIndexes[i] = 8;
9146       else
9147         PFIndexes[i] = ShuffleMask[i];
9148     }
9149 
9150     // Compute the index in the perfect shuffle table.
9151     unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
9152                             PFIndexes[2] * 9 + PFIndexes[3];
9153     unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
9154     unsigned Cost = (PFEntry >> 30);
9155 
9156     if (Cost <= 4)
9157       return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
9158   }
9159 
9160   return GenerateTBL(Op, ShuffleMask, DAG);
9161 }
9162 
9163 SDValue AArch64TargetLowering::LowerSTEP_VECTOR(SDValue Op,
9164                                                 SelectionDAG &DAG) const {
9165   SDLoc dl(Op);
9166   EVT VT = Op.getValueType();
9167   assert(VT.isScalableVector() &&
9168          "Only expect scalable vectors for STEP_VECTOR");
9169   assert(VT.getScalarType() != MVT::i1 &&
9170          "Vectors of i1 types not supported for STEP_VECTOR");
9171 
9172   SDValue StepVal = Op.getOperand(0);
9173   SDValue Zero = DAG.getConstant(0, dl, StepVal.getValueType());
9174   return DAG.getNode(AArch64ISD::INDEX_VECTOR, dl, VT, Zero, StepVal);
9175 }
9176 
9177 SDValue AArch64TargetLowering::LowerSPLAT_VECTOR(SDValue Op,
9178                                                  SelectionDAG &DAG) const {
9179   SDLoc dl(Op);
9180   EVT VT = Op.getValueType();
9181   EVT ElemVT = VT.getScalarType();
9182   SDValue SplatVal = Op.getOperand(0);
9183 
9184   if (useSVEForFixedLengthVectorVT(VT))
9185     return LowerToScalableOp(Op, DAG);
9186 
9187   // Extend input splat value where needed to fit into a GPR (32b or 64b only)
9188   // FPRs don't have this restriction.
9189   switch (ElemVT.getSimpleVT().SimpleTy) {
9190   case MVT::i1: {
9191     // The only legal i1 vectors are SVE vectors, so we can use SVE-specific
9192     // lowering code.
9193     if (auto *ConstVal = dyn_cast<ConstantSDNode>(SplatVal)) {
9194       if (ConstVal->isOne())
9195         return getPTrue(DAG, dl, VT, AArch64SVEPredPattern::all);
9196       // TODO: Add special case for constant false
9197     }
9198     // The general case of i1.  There isn't any natural way to do this,
9199     // so we use some trickery with whilelo.
9200     SplatVal = DAG.getAnyExtOrTrunc(SplatVal, dl, MVT::i64);
9201     SplatVal = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i64, SplatVal,
9202                            DAG.getValueType(MVT::i1));
9203     SDValue ID = DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo, dl,
9204                                        MVT::i64);
9205     return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, ID,
9206                        DAG.getConstant(0, dl, MVT::i64), SplatVal);
9207   }
9208   case MVT::i8:
9209   case MVT::i16:
9210   case MVT::i32:
9211     SplatVal = DAG.getAnyExtOrTrunc(SplatVal, dl, MVT::i32);
9212     break;
9213   case MVT::i64:
9214     SplatVal = DAG.getAnyExtOrTrunc(SplatVal, dl, MVT::i64);
9215     break;
9216   case MVT::f16:
9217   case MVT::bf16:
9218   case MVT::f32:
9219   case MVT::f64:
9220     // Fine as is
9221     break;
9222   default:
9223     report_fatal_error("Unsupported SPLAT_VECTOR input operand type");
9224   }
9225 
9226   return DAG.getNode(AArch64ISD::DUP, dl, VT, SplatVal);
9227 }
9228 
9229 SDValue AArch64TargetLowering::LowerDUPQLane(SDValue Op,
9230                                              SelectionDAG &DAG) const {
9231   SDLoc DL(Op);
9232 
9233   EVT VT = Op.getValueType();
9234   if (!isTypeLegal(VT) || !VT.isScalableVector())
9235     return SDValue();
9236 
9237   // Current lowering only supports the SVE-ACLE types.
9238   if (VT.getSizeInBits().getKnownMinSize() != AArch64::SVEBitsPerBlock)
9239     return SDValue();
9240 
9241   // The DUPQ operation is indepedent of element type so normalise to i64s.
9242   SDValue V = DAG.getNode(ISD::BITCAST, DL, MVT::nxv2i64, Op.getOperand(1));
9243   SDValue Idx128 = Op.getOperand(2);
9244 
9245   // DUPQ can be used when idx is in range.
9246   auto *CIdx = dyn_cast<ConstantSDNode>(Idx128);
9247   if (CIdx && (CIdx->getZExtValue() <= 3)) {
9248     SDValue CI = DAG.getTargetConstant(CIdx->getZExtValue(), DL, MVT::i64);
9249     SDNode *DUPQ =
9250         DAG.getMachineNode(AArch64::DUP_ZZI_Q, DL, MVT::nxv2i64, V, CI);
9251     return DAG.getNode(ISD::BITCAST, DL, VT, SDValue(DUPQ, 0));
9252   }
9253 
9254   // The ACLE says this must produce the same result as:
9255   //   svtbl(data, svadd_x(svptrue_b64(),
9256   //                       svand_x(svptrue_b64(), svindex_u64(0, 1), 1),
9257   //                       index * 2))
9258   SDValue One = DAG.getConstant(1, DL, MVT::i64);
9259   SDValue SplatOne = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, One);
9260 
9261   // create the vector 0,1,0,1,...
9262   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
9263   SDValue SV = DAG.getNode(AArch64ISD::INDEX_VECTOR,
9264                            DL, MVT::nxv2i64, Zero, One);
9265   SV = DAG.getNode(ISD::AND, DL, MVT::nxv2i64, SV, SplatOne);
9266 
9267   // create the vector idx64,idx64+1,idx64,idx64+1,...
9268   SDValue Idx64 = DAG.getNode(ISD::ADD, DL, MVT::i64, Idx128, Idx128);
9269   SDValue SplatIdx64 = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Idx64);
9270   SDValue ShuffleMask = DAG.getNode(ISD::ADD, DL, MVT::nxv2i64, SV, SplatIdx64);
9271 
9272   // create the vector Val[idx64],Val[idx64+1],Val[idx64],Val[idx64+1],...
9273   SDValue TBL = DAG.getNode(AArch64ISD::TBL, DL, MVT::nxv2i64, V, ShuffleMask);
9274   return DAG.getNode(ISD::BITCAST, DL, VT, TBL);
9275 }
9276 
9277 
9278 static bool resolveBuildVector(BuildVectorSDNode *BVN, APInt &CnstBits,
9279                                APInt &UndefBits) {
9280   EVT VT = BVN->getValueType(0);
9281   APInt SplatBits, SplatUndef;
9282   unsigned SplatBitSize;
9283   bool HasAnyUndefs;
9284   if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
9285     unsigned NumSplats = VT.getSizeInBits() / SplatBitSize;
9286 
9287     for (unsigned i = 0; i < NumSplats; ++i) {
9288       CnstBits <<= SplatBitSize;
9289       UndefBits <<= SplatBitSize;
9290       CnstBits |= SplatBits.zextOrTrunc(VT.getSizeInBits());
9291       UndefBits |= (SplatBits ^ SplatUndef).zextOrTrunc(VT.getSizeInBits());
9292     }
9293 
9294     return true;
9295   }
9296 
9297   return false;
9298 }
9299 
9300 // Try 64-bit splatted SIMD immediate.
9301 static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
9302                                  const APInt &Bits) {
9303   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9304     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9305     EVT VT = Op.getValueType();
9306     MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v2i64 : MVT::f64;
9307 
9308     if (AArch64_AM::isAdvSIMDModImmType10(Value)) {
9309       Value = AArch64_AM::encodeAdvSIMDModImmType10(Value);
9310 
9311       SDLoc dl(Op);
9312       SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
9313                                 DAG.getConstant(Value, dl, MVT::i32));
9314       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9315     }
9316   }
9317 
9318   return SDValue();
9319 }
9320 
9321 // Try 32-bit splatted SIMD immediate.
9322 static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
9323                                   const APInt &Bits,
9324                                   const SDValue *LHS = nullptr) {
9325   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9326     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9327     EVT VT = Op.getValueType();
9328     MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
9329     bool isAdvSIMDModImm = false;
9330     uint64_t Shift;
9331 
9332     if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType1(Value))) {
9333       Value = AArch64_AM::encodeAdvSIMDModImmType1(Value);
9334       Shift = 0;
9335     }
9336     else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType2(Value))) {
9337       Value = AArch64_AM::encodeAdvSIMDModImmType2(Value);
9338       Shift = 8;
9339     }
9340     else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType3(Value))) {
9341       Value = AArch64_AM::encodeAdvSIMDModImmType3(Value);
9342       Shift = 16;
9343     }
9344     else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType4(Value))) {
9345       Value = AArch64_AM::encodeAdvSIMDModImmType4(Value);
9346       Shift = 24;
9347     }
9348 
9349     if (isAdvSIMDModImm) {
9350       SDLoc dl(Op);
9351       SDValue Mov;
9352 
9353       if (LHS)
9354         Mov = DAG.getNode(NewOp, dl, MovTy, *LHS,
9355                           DAG.getConstant(Value, dl, MVT::i32),
9356                           DAG.getConstant(Shift, dl, MVT::i32));
9357       else
9358         Mov = DAG.getNode(NewOp, dl, MovTy,
9359                           DAG.getConstant(Value, dl, MVT::i32),
9360                           DAG.getConstant(Shift, dl, MVT::i32));
9361 
9362       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9363     }
9364   }
9365 
9366   return SDValue();
9367 }
9368 
9369 // Try 16-bit splatted SIMD immediate.
9370 static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
9371                                   const APInt &Bits,
9372                                   const SDValue *LHS = nullptr) {
9373   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9374     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9375     EVT VT = Op.getValueType();
9376     MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
9377     bool isAdvSIMDModImm = false;
9378     uint64_t Shift;
9379 
9380     if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType5(Value))) {
9381       Value = AArch64_AM::encodeAdvSIMDModImmType5(Value);
9382       Shift = 0;
9383     }
9384     else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType6(Value))) {
9385       Value = AArch64_AM::encodeAdvSIMDModImmType6(Value);
9386       Shift = 8;
9387     }
9388 
9389     if (isAdvSIMDModImm) {
9390       SDLoc dl(Op);
9391       SDValue Mov;
9392 
9393       if (LHS)
9394         Mov = DAG.getNode(NewOp, dl, MovTy, *LHS,
9395                           DAG.getConstant(Value, dl, MVT::i32),
9396                           DAG.getConstant(Shift, dl, MVT::i32));
9397       else
9398         Mov = DAG.getNode(NewOp, dl, MovTy,
9399                           DAG.getConstant(Value, dl, MVT::i32),
9400                           DAG.getConstant(Shift, dl, MVT::i32));
9401 
9402       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9403     }
9404   }
9405 
9406   return SDValue();
9407 }
9408 
9409 // Try 32-bit splatted SIMD immediate with shifted ones.
9410 static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op,
9411                                     SelectionDAG &DAG, const APInt &Bits) {
9412   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9413     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9414     EVT VT = Op.getValueType();
9415     MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
9416     bool isAdvSIMDModImm = false;
9417     uint64_t Shift;
9418 
9419     if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType7(Value))) {
9420       Value = AArch64_AM::encodeAdvSIMDModImmType7(Value);
9421       Shift = 264;
9422     }
9423     else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType8(Value))) {
9424       Value = AArch64_AM::encodeAdvSIMDModImmType8(Value);
9425       Shift = 272;
9426     }
9427 
9428     if (isAdvSIMDModImm) {
9429       SDLoc dl(Op);
9430       SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
9431                                 DAG.getConstant(Value, dl, MVT::i32),
9432                                 DAG.getConstant(Shift, dl, MVT::i32));
9433       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9434     }
9435   }
9436 
9437   return SDValue();
9438 }
9439 
9440 // Try 8-bit splatted SIMD immediate.
9441 static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
9442                                  const APInt &Bits) {
9443   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9444     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9445     EVT VT = Op.getValueType();
9446     MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8;
9447 
9448     if (AArch64_AM::isAdvSIMDModImmType9(Value)) {
9449       Value = AArch64_AM::encodeAdvSIMDModImmType9(Value);
9450 
9451       SDLoc dl(Op);
9452       SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
9453                                 DAG.getConstant(Value, dl, MVT::i32));
9454       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9455     }
9456   }
9457 
9458   return SDValue();
9459 }
9460 
9461 // Try FP splatted SIMD immediate.
9462 static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
9463                                   const APInt &Bits) {
9464   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9465     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9466     EVT VT = Op.getValueType();
9467     bool isWide = (VT.getSizeInBits() == 128);
9468     MVT MovTy;
9469     bool isAdvSIMDModImm = false;
9470 
9471     if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType11(Value))) {
9472       Value = AArch64_AM::encodeAdvSIMDModImmType11(Value);
9473       MovTy = isWide ? MVT::v4f32 : MVT::v2f32;
9474     }
9475     else if (isWide &&
9476              (isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType12(Value))) {
9477       Value = AArch64_AM::encodeAdvSIMDModImmType12(Value);
9478       MovTy = MVT::v2f64;
9479     }
9480 
9481     if (isAdvSIMDModImm) {
9482       SDLoc dl(Op);
9483       SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
9484                                 DAG.getConstant(Value, dl, MVT::i32));
9485       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9486     }
9487   }
9488 
9489   return SDValue();
9490 }
9491 
9492 // Specialized code to quickly find if PotentialBVec is a BuildVector that
9493 // consists of only the same constant int value, returned in reference arg
9494 // ConstVal
9495 static bool isAllConstantBuildVector(const SDValue &PotentialBVec,
9496                                      uint64_t &ConstVal) {
9497   BuildVectorSDNode *Bvec = dyn_cast<BuildVectorSDNode>(PotentialBVec);
9498   if (!Bvec)
9499     return false;
9500   ConstantSDNode *FirstElt = dyn_cast<ConstantSDNode>(Bvec->getOperand(0));
9501   if (!FirstElt)
9502     return false;
9503   EVT VT = Bvec->getValueType(0);
9504   unsigned NumElts = VT.getVectorNumElements();
9505   for (unsigned i = 1; i < NumElts; ++i)
9506     if (dyn_cast<ConstantSDNode>(Bvec->getOperand(i)) != FirstElt)
9507       return false;
9508   ConstVal = FirstElt->getZExtValue();
9509   return true;
9510 }
9511 
9512 static unsigned getIntrinsicID(const SDNode *N) {
9513   unsigned Opcode = N->getOpcode();
9514   switch (Opcode) {
9515   default:
9516     return Intrinsic::not_intrinsic;
9517   case ISD::INTRINSIC_WO_CHAIN: {
9518     unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
9519     if (IID < Intrinsic::num_intrinsics)
9520       return IID;
9521     return Intrinsic::not_intrinsic;
9522   }
9523   }
9524 }
9525 
9526 // Attempt to form a vector S[LR]I from (or (and X, BvecC1), (lsl Y, C2)),
9527 // to (SLI X, Y, C2), where X and Y have matching vector types, BvecC1 is a
9528 // BUILD_VECTORs with constant element C1, C2 is a constant, and:
9529 //   - for the SLI case: C1 == ~(Ones(ElemSizeInBits) << C2)
9530 //   - for the SRI case: C1 == ~(Ones(ElemSizeInBits) >> C2)
9531 // The (or (lsl Y, C2), (and X, BvecC1)) case is also handled.
9532 static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
9533   EVT VT = N->getValueType(0);
9534 
9535   if (!VT.isVector())
9536     return SDValue();
9537 
9538   SDLoc DL(N);
9539 
9540   SDValue And;
9541   SDValue Shift;
9542 
9543   SDValue FirstOp = N->getOperand(0);
9544   unsigned FirstOpc = FirstOp.getOpcode();
9545   SDValue SecondOp = N->getOperand(1);
9546   unsigned SecondOpc = SecondOp.getOpcode();
9547 
9548   // Is one of the operands an AND or a BICi? The AND may have been optimised to
9549   // a BICi in order to use an immediate instead of a register.
9550   // Is the other operand an shl or lshr? This will have been turned into:
9551   // AArch64ISD::VSHL vector, #shift or AArch64ISD::VLSHR vector, #shift.
9552   if ((FirstOpc == ISD::AND || FirstOpc == AArch64ISD::BICi) &&
9553       (SecondOpc == AArch64ISD::VSHL || SecondOpc == AArch64ISD::VLSHR)) {
9554     And = FirstOp;
9555     Shift = SecondOp;
9556 
9557   } else if ((SecondOpc == ISD::AND || SecondOpc == AArch64ISD::BICi) &&
9558              (FirstOpc == AArch64ISD::VSHL || FirstOpc == AArch64ISD::VLSHR)) {
9559     And = SecondOp;
9560     Shift = FirstOp;
9561   } else
9562     return SDValue();
9563 
9564   bool IsAnd = And.getOpcode() == ISD::AND;
9565   bool IsShiftRight = Shift.getOpcode() == AArch64ISD::VLSHR;
9566 
9567   // Is the shift amount constant?
9568   ConstantSDNode *C2node = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
9569   if (!C2node)
9570     return SDValue();
9571 
9572   uint64_t C1;
9573   if (IsAnd) {
9574     // Is the and mask vector all constant?
9575     if (!isAllConstantBuildVector(And.getOperand(1), C1))
9576       return SDValue();
9577   } else {
9578     // Reconstruct the corresponding AND immediate from the two BICi immediates.
9579     ConstantSDNode *C1nodeImm = dyn_cast<ConstantSDNode>(And.getOperand(1));
9580     ConstantSDNode *C1nodeShift = dyn_cast<ConstantSDNode>(And.getOperand(2));
9581     assert(C1nodeImm && C1nodeShift);
9582     C1 = ~(C1nodeImm->getZExtValue() << C1nodeShift->getZExtValue());
9583   }
9584 
9585   // Is C1 == ~(Ones(ElemSizeInBits) << C2) or
9586   // C1 == ~(Ones(ElemSizeInBits) >> C2), taking into account
9587   // how much one can shift elements of a particular size?
9588   uint64_t C2 = C2node->getZExtValue();
9589   unsigned ElemSizeInBits = VT.getScalarSizeInBits();
9590   if (C2 > ElemSizeInBits)
9591     return SDValue();
9592 
9593   APInt C1AsAPInt(ElemSizeInBits, C1);
9594   APInt RequiredC1 = IsShiftRight ? APInt::getHighBitsSet(ElemSizeInBits, C2)
9595                                   : APInt::getLowBitsSet(ElemSizeInBits, C2);
9596   if (C1AsAPInt != RequiredC1)
9597     return SDValue();
9598 
9599   SDValue X = And.getOperand(0);
9600   SDValue Y = Shift.getOperand(0);
9601 
9602   unsigned Inst = IsShiftRight ? AArch64ISD::VSRI : AArch64ISD::VSLI;
9603   SDValue ResultSLI = DAG.getNode(Inst, DL, VT, X, Y, Shift.getOperand(1));
9604 
9605   LLVM_DEBUG(dbgs() << "aarch64-lower: transformed: \n");
9606   LLVM_DEBUG(N->dump(&DAG));
9607   LLVM_DEBUG(dbgs() << "into: \n");
9608   LLVM_DEBUG(ResultSLI->dump(&DAG));
9609 
9610   ++NumShiftInserts;
9611   return ResultSLI;
9612 }
9613 
9614 SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
9615                                              SelectionDAG &DAG) const {
9616   if (useSVEForFixedLengthVectorVT(Op.getValueType()))
9617     return LowerToScalableOp(Op, DAG);
9618 
9619   // Attempt to form a vector S[LR]I from (or (and X, C1), (lsl Y, C2))
9620   if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG))
9621     return Res;
9622 
9623   EVT VT = Op.getValueType();
9624 
9625   SDValue LHS = Op.getOperand(0);
9626   BuildVectorSDNode *BVN =
9627       dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode());
9628   if (!BVN) {
9629     // OR commutes, so try swapping the operands.
9630     LHS = Op.getOperand(1);
9631     BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode());
9632   }
9633   if (!BVN)
9634     return Op;
9635 
9636   APInt DefBits(VT.getSizeInBits(), 0);
9637   APInt UndefBits(VT.getSizeInBits(), 0);
9638   if (resolveBuildVector(BVN, DefBits, UndefBits)) {
9639     SDValue NewOp;
9640 
9641     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::ORRi, Op, DAG,
9642                                     DefBits, &LHS)) ||
9643         (NewOp = tryAdvSIMDModImm16(AArch64ISD::ORRi, Op, DAG,
9644                                     DefBits, &LHS)))
9645       return NewOp;
9646 
9647     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::ORRi, Op, DAG,
9648                                     UndefBits, &LHS)) ||
9649         (NewOp = tryAdvSIMDModImm16(AArch64ISD::ORRi, Op, DAG,
9650                                     UndefBits, &LHS)))
9651       return NewOp;
9652   }
9653 
9654   // We can always fall back to a non-immediate OR.
9655   return Op;
9656 }
9657 
9658 // Normalize the operands of BUILD_VECTOR. The value of constant operands will
9659 // be truncated to fit element width.
9660 static SDValue NormalizeBuildVector(SDValue Op,
9661                                     SelectionDAG &DAG) {
9662   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
9663   SDLoc dl(Op);
9664   EVT VT = Op.getValueType();
9665   EVT EltTy= VT.getVectorElementType();
9666 
9667   if (EltTy.isFloatingPoint() || EltTy.getSizeInBits() > 16)
9668     return Op;
9669 
9670   SmallVector<SDValue, 16> Ops;
9671   for (SDValue Lane : Op->ops()) {
9672     // For integer vectors, type legalization would have promoted the
9673     // operands already. Otherwise, if Op is a floating-point splat
9674     // (with operands cast to integers), then the only possibilities
9675     // are constants and UNDEFs.
9676     if (auto *CstLane = dyn_cast<ConstantSDNode>(Lane)) {
9677       APInt LowBits(EltTy.getSizeInBits(),
9678                     CstLane->getZExtValue());
9679       Lane = DAG.getConstant(LowBits.getZExtValue(), dl, MVT::i32);
9680     } else if (Lane.getNode()->isUndef()) {
9681       Lane = DAG.getUNDEF(MVT::i32);
9682     } else {
9683       assert(Lane.getValueType() == MVT::i32 &&
9684              "Unexpected BUILD_VECTOR operand type");
9685     }
9686     Ops.push_back(Lane);
9687   }
9688   return DAG.getBuildVector(VT, dl, Ops);
9689 }
9690 
9691 static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG) {
9692   EVT VT = Op.getValueType();
9693 
9694   APInt DefBits(VT.getSizeInBits(), 0);
9695   APInt UndefBits(VT.getSizeInBits(), 0);
9696   BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
9697   if (resolveBuildVector(BVN, DefBits, UndefBits)) {
9698     SDValue NewOp;
9699     if ((NewOp = tryAdvSIMDModImm64(AArch64ISD::MOVIedit, Op, DAG, DefBits)) ||
9700         (NewOp = tryAdvSIMDModImm32(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
9701         (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MOVImsl, Op, DAG, DefBits)) ||
9702         (NewOp = tryAdvSIMDModImm16(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
9703         (NewOp = tryAdvSIMDModImm8(AArch64ISD::MOVI, Op, DAG, DefBits)) ||
9704         (NewOp = tryAdvSIMDModImmFP(AArch64ISD::FMOV, Op, DAG, DefBits)))
9705       return NewOp;
9706 
9707     DefBits = ~DefBits;
9708     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::MVNIshift, Op, DAG, DefBits)) ||
9709         (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MVNImsl, Op, DAG, DefBits)) ||
9710         (NewOp = tryAdvSIMDModImm16(AArch64ISD::MVNIshift, Op, DAG, DefBits)))
9711       return NewOp;
9712 
9713     DefBits = UndefBits;
9714     if ((NewOp = tryAdvSIMDModImm64(AArch64ISD::MOVIedit, Op, DAG, DefBits)) ||
9715         (NewOp = tryAdvSIMDModImm32(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
9716         (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MOVImsl, Op, DAG, DefBits)) ||
9717         (NewOp = tryAdvSIMDModImm16(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
9718         (NewOp = tryAdvSIMDModImm8(AArch64ISD::MOVI, Op, DAG, DefBits)) ||
9719         (NewOp = tryAdvSIMDModImmFP(AArch64ISD::FMOV, Op, DAG, DefBits)))
9720       return NewOp;
9721 
9722     DefBits = ~UndefBits;
9723     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::MVNIshift, Op, DAG, DefBits)) ||
9724         (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MVNImsl, Op, DAG, DefBits)) ||
9725         (NewOp = tryAdvSIMDModImm16(AArch64ISD::MVNIshift, Op, DAG, DefBits)))
9726       return NewOp;
9727   }
9728 
9729   return SDValue();
9730 }
9731 
9732 SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
9733                                                  SelectionDAG &DAG) const {
9734   EVT VT = Op.getValueType();
9735 
9736   // Try to build a simple constant vector.
9737   Op = NormalizeBuildVector(Op, DAG);
9738   if (VT.isInteger()) {
9739     // Certain vector constants, used to express things like logical NOT and
9740     // arithmetic NEG, are passed through unmodified.  This allows special
9741     // patterns for these operations to match, which will lower these constants
9742     // to whatever is proven necessary.
9743     BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
9744     if (BVN->isConstant())
9745       if (ConstantSDNode *Const = BVN->getConstantSplatNode()) {
9746         unsigned BitSize = VT.getVectorElementType().getSizeInBits();
9747         APInt Val(BitSize,
9748                   Const->getAPIntValue().zextOrTrunc(BitSize).getZExtValue());
9749         if (Val.isNullValue() || Val.isAllOnesValue())
9750           return Op;
9751       }
9752   }
9753 
9754   if (SDValue V = ConstantBuildVector(Op, DAG))
9755     return V;
9756 
9757   // Scan through the operands to find some interesting properties we can
9758   // exploit:
9759   //   1) If only one value is used, we can use a DUP, or
9760   //   2) if only the low element is not undef, we can just insert that, or
9761   //   3) if only one constant value is used (w/ some non-constant lanes),
9762   //      we can splat the constant value into the whole vector then fill
9763   //      in the non-constant lanes.
9764   //   4) FIXME: If different constant values are used, but we can intelligently
9765   //             select the values we'll be overwriting for the non-constant
9766   //             lanes such that we can directly materialize the vector
9767   //             some other way (MOVI, e.g.), we can be sneaky.
9768   //   5) if all operands are EXTRACT_VECTOR_ELT, check for VUZP.
9769   SDLoc dl(Op);
9770   unsigned NumElts = VT.getVectorNumElements();
9771   bool isOnlyLowElement = true;
9772   bool usesOnlyOneValue = true;
9773   bool usesOnlyOneConstantValue = true;
9774   bool isConstant = true;
9775   bool AllLanesExtractElt = true;
9776   unsigned NumConstantLanes = 0;
9777   unsigned NumDifferentLanes = 0;
9778   unsigned NumUndefLanes = 0;
9779   SDValue Value;
9780   SDValue ConstantValue;
9781   for (unsigned i = 0; i < NumElts; ++i) {
9782     SDValue V = Op.getOperand(i);
9783     if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
9784       AllLanesExtractElt = false;
9785     if (V.isUndef()) {
9786       ++NumUndefLanes;
9787       continue;
9788     }
9789     if (i > 0)
9790       isOnlyLowElement = false;
9791     if (!isIntOrFPConstant(V))
9792       isConstant = false;
9793 
9794     if (isIntOrFPConstant(V)) {
9795       ++NumConstantLanes;
9796       if (!ConstantValue.getNode())
9797         ConstantValue = V;
9798       else if (ConstantValue != V)
9799         usesOnlyOneConstantValue = false;
9800     }
9801 
9802     if (!Value.getNode())
9803       Value = V;
9804     else if (V != Value) {
9805       usesOnlyOneValue = false;
9806       ++NumDifferentLanes;
9807     }
9808   }
9809 
9810   if (!Value.getNode()) {
9811     LLVM_DEBUG(
9812         dbgs() << "LowerBUILD_VECTOR: value undefined, creating undef node\n");
9813     return DAG.getUNDEF(VT);
9814   }
9815 
9816   // Convert BUILD_VECTOR where all elements but the lowest are undef into
9817   // SCALAR_TO_VECTOR, except for when we have a single-element constant vector
9818   // as SimplifyDemandedBits will just turn that back into BUILD_VECTOR.
9819   if (isOnlyLowElement && !(NumElts == 1 && isIntOrFPConstant(Value))) {
9820     LLVM_DEBUG(dbgs() << "LowerBUILD_VECTOR: only low element used, creating 1 "
9821                          "SCALAR_TO_VECTOR node\n");
9822     return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
9823   }
9824 
9825   if (AllLanesExtractElt) {
9826     SDNode *Vector = nullptr;
9827     bool Even = false;
9828     bool Odd = false;
9829     // Check whether the extract elements match the Even pattern <0,2,4,...> or
9830     // the Odd pattern <1,3,5,...>.
9831     for (unsigned i = 0; i < NumElts; ++i) {
9832       SDValue V = Op.getOperand(i);
9833       const SDNode *N = V.getNode();
9834       if (!isa<ConstantSDNode>(N->getOperand(1)))
9835         break;
9836       SDValue N0 = N->getOperand(0);
9837 
9838       // All elements are extracted from the same vector.
9839       if (!Vector) {
9840         Vector = N0.getNode();
9841         // Check that the type of EXTRACT_VECTOR_ELT matches the type of
9842         // BUILD_VECTOR.
9843         if (VT.getVectorElementType() !=
9844             N0.getValueType().getVectorElementType())
9845           break;
9846       } else if (Vector != N0.getNode()) {
9847         Odd = false;
9848         Even = false;
9849         break;
9850       }
9851 
9852       // Extracted values are either at Even indices <0,2,4,...> or at Odd
9853       // indices <1,3,5,...>.
9854       uint64_t Val = N->getConstantOperandVal(1);
9855       if (Val == 2 * i) {
9856         Even = true;
9857         continue;
9858       }
9859       if (Val - 1 == 2 * i) {
9860         Odd = true;
9861         continue;
9862       }
9863 
9864       // Something does not match: abort.
9865       Odd = false;
9866       Even = false;
9867       break;
9868     }
9869     if (Even || Odd) {
9870       SDValue LHS =
9871           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
9872                       DAG.getConstant(0, dl, MVT::i64));
9873       SDValue RHS =
9874           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
9875                       DAG.getConstant(NumElts, dl, MVT::i64));
9876 
9877       if (Even && !Odd)
9878         return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), LHS,
9879                            RHS);
9880       if (Odd && !Even)
9881         return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), LHS,
9882                            RHS);
9883     }
9884   }
9885 
9886   // Use DUP for non-constant splats. For f32 constant splats, reduce to
9887   // i32 and try again.
9888   if (usesOnlyOneValue) {
9889     if (!isConstant) {
9890       if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9891           Value.getValueType() != VT) {
9892         LLVM_DEBUG(
9893             dbgs() << "LowerBUILD_VECTOR: use DUP for non-constant splats\n");
9894         return DAG.getNode(AArch64ISD::DUP, dl, VT, Value);
9895       }
9896 
9897       // This is actually a DUPLANExx operation, which keeps everything vectory.
9898 
9899       SDValue Lane = Value.getOperand(1);
9900       Value = Value.getOperand(0);
9901       if (Value.getValueSizeInBits() == 64) {
9902         LLVM_DEBUG(
9903             dbgs() << "LowerBUILD_VECTOR: DUPLANE works on 128-bit vectors, "
9904                       "widening it\n");
9905         Value = WidenVector(Value, DAG);
9906       }
9907 
9908       unsigned Opcode = getDUPLANEOp(VT.getVectorElementType());
9909       return DAG.getNode(Opcode, dl, VT, Value, Lane);
9910     }
9911 
9912     if (VT.getVectorElementType().isFloatingPoint()) {
9913       SmallVector<SDValue, 8> Ops;
9914       EVT EltTy = VT.getVectorElementType();
9915       assert ((EltTy == MVT::f16 || EltTy == MVT::bf16 || EltTy == MVT::f32 ||
9916                EltTy == MVT::f64) && "Unsupported floating-point vector type");
9917       LLVM_DEBUG(
9918           dbgs() << "LowerBUILD_VECTOR: float constant splats, creating int "
9919                     "BITCASTS, and try again\n");
9920       MVT NewType = MVT::getIntegerVT(EltTy.getSizeInBits());
9921       for (unsigned i = 0; i < NumElts; ++i)
9922         Ops.push_back(DAG.getNode(ISD::BITCAST, dl, NewType, Op.getOperand(i)));
9923       EVT VecVT = EVT::getVectorVT(*DAG.getContext(), NewType, NumElts);
9924       SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
9925       LLVM_DEBUG(dbgs() << "LowerBUILD_VECTOR: trying to lower new vector: ";
9926                  Val.dump(););
9927       Val = LowerBUILD_VECTOR(Val, DAG);
9928       if (Val.getNode())
9929         return DAG.getNode(ISD::BITCAST, dl, VT, Val);
9930     }
9931   }
9932 
9933   // If we need to insert a small number of different non-constant elements and
9934   // the vector width is sufficiently large, prefer using DUP with the common
9935   // value and INSERT_VECTOR_ELT for the different lanes. If DUP is preferred,
9936   // skip the constant lane handling below.
9937   bool PreferDUPAndInsert =
9938       !isConstant && NumDifferentLanes >= 1 &&
9939       NumDifferentLanes < ((NumElts - NumUndefLanes) / 2) &&
9940       NumDifferentLanes >= NumConstantLanes;
9941 
9942   // If there was only one constant value used and for more than one lane,
9943   // start by splatting that value, then replace the non-constant lanes. This
9944   // is better than the default, which will perform a separate initialization
9945   // for each lane.
9946   if (!PreferDUPAndInsert && NumConstantLanes > 0 && usesOnlyOneConstantValue) {
9947     // Firstly, try to materialize the splat constant.
9948     SDValue Vec = DAG.getSplatBuildVector(VT, dl, ConstantValue),
9949             Val = ConstantBuildVector(Vec, DAG);
9950     if (!Val) {
9951       // Otherwise, materialize the constant and splat it.
9952       Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue);
9953       DAG.ReplaceAllUsesWith(Vec.getNode(), &Val);
9954     }
9955 
9956     // Now insert the non-constant lanes.
9957     for (unsigned i = 0; i < NumElts; ++i) {
9958       SDValue V = Op.getOperand(i);
9959       SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
9960       if (!isIntOrFPConstant(V))
9961         // Note that type legalization likely mucked about with the VT of the
9962         // source operand, so we may have to convert it here before inserting.
9963         Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx);
9964     }
9965     return Val;
9966   }
9967 
9968   // This will generate a load from the constant pool.
9969   if (isConstant) {
9970     LLVM_DEBUG(
9971         dbgs() << "LowerBUILD_VECTOR: all elements are constant, use default "
9972                   "expansion\n");
9973     return SDValue();
9974   }
9975 
9976   // Empirical tests suggest this is rarely worth it for vectors of length <= 2.
9977   if (NumElts >= 4) {
9978     if (SDValue shuffle = ReconstructShuffle(Op, DAG))
9979       return shuffle;
9980   }
9981 
9982   if (PreferDUPAndInsert) {
9983     // First, build a constant vector with the common element.
9984     SmallVector<SDValue, 8> Ops(NumElts, Value);
9985     SDValue NewVector = LowerBUILD_VECTOR(DAG.getBuildVector(VT, dl, Ops), DAG);
9986     // Next, insert the elements that do not match the common value.
9987     for (unsigned I = 0; I < NumElts; ++I)
9988       if (Op.getOperand(I) != Value)
9989         NewVector =
9990             DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, NewVector,
9991                         Op.getOperand(I), DAG.getConstant(I, dl, MVT::i64));
9992 
9993     return NewVector;
9994   }
9995 
9996   // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
9997   // know the default expansion would otherwise fall back on something even
9998   // worse. For a vector with one or two non-undef values, that's
9999   // scalar_to_vector for the elements followed by a shuffle (provided the
10000   // shuffle is valid for the target) and materialization element by element
10001   // on the stack followed by a load for everything else.
10002   if (!isConstant && !usesOnlyOneValue) {
10003     LLVM_DEBUG(
10004         dbgs() << "LowerBUILD_VECTOR: alternatives failed, creating sequence "
10005                   "of INSERT_VECTOR_ELT\n");
10006 
10007     SDValue Vec = DAG.getUNDEF(VT);
10008     SDValue Op0 = Op.getOperand(0);
10009     unsigned i = 0;
10010 
10011     // Use SCALAR_TO_VECTOR for lane zero to
10012     // a) Avoid a RMW dependency on the full vector register, and
10013     // b) Allow the register coalescer to fold away the copy if the
10014     //    value is already in an S or D register, and we're forced to emit an
10015     //    INSERT_SUBREG that we can't fold anywhere.
10016     //
10017     // We also allow types like i8 and i16 which are illegal scalar but legal
10018     // vector element types. After type-legalization the inserted value is
10019     // extended (i32) and it is safe to cast them to the vector type by ignoring
10020     // the upper bits of the lowest lane (e.g. v8i8, v4i16).
10021     if (!Op0.isUndef()) {
10022       LLVM_DEBUG(dbgs() << "Creating node for op0, it is not undefined:\n");
10023       Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op0);
10024       ++i;
10025     }
10026     LLVM_DEBUG(if (i < NumElts) dbgs()
10027                    << "Creating nodes for the other vector elements:\n";);
10028     for (; i < NumElts; ++i) {
10029       SDValue V = Op.getOperand(i);
10030       if (V.isUndef())
10031         continue;
10032       SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
10033       Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
10034     }
10035     return Vec;
10036   }
10037 
10038   LLVM_DEBUG(
10039       dbgs() << "LowerBUILD_VECTOR: use default expansion, failed to find "
10040                 "better alternative\n");
10041   return SDValue();
10042 }
10043 
10044 SDValue AArch64TargetLowering::LowerCONCAT_VECTORS(SDValue Op,
10045                                                    SelectionDAG &DAG) const {
10046   assert(Op.getValueType().isScalableVector() &&
10047          isTypeLegal(Op.getValueType()) &&
10048          "Expected legal scalable vector type!");
10049 
10050   if (isTypeLegal(Op.getOperand(0).getValueType()) && Op.getNumOperands() == 2)
10051     return Op;
10052 
10053   return SDValue();
10054 }
10055 
10056 SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
10057                                                       SelectionDAG &DAG) const {
10058   assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Unknown opcode!");
10059 
10060   if (useSVEForFixedLengthVectorVT(Op.getValueType()))
10061     return LowerFixedLengthInsertVectorElt(Op, DAG);
10062 
10063   // Check for non-constant or out of range lane.
10064   EVT VT = Op.getOperand(0).getValueType();
10065   ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(2));
10066   if (!CI || CI->getZExtValue() >= VT.getVectorNumElements())
10067     return SDValue();
10068 
10069 
10070   // Insertion/extraction are legal for V128 types.
10071   if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
10072       VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
10073       VT == MVT::v8f16 || VT == MVT::v8bf16)
10074     return Op;
10075 
10076   if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 &&
10077       VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16 &&
10078       VT != MVT::v4bf16)
10079     return SDValue();
10080 
10081   // For V64 types, we perform insertion by expanding the value
10082   // to a V128 type and perform the insertion on that.
10083   SDLoc DL(Op);
10084   SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
10085   EVT WideTy = WideVec.getValueType();
10086 
10087   SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec,
10088                              Op.getOperand(1), Op.getOperand(2));
10089   // Re-narrow the resultant vector.
10090   return NarrowVector(Node, DAG);
10091 }
10092 
10093 SDValue
10094 AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
10095                                                SelectionDAG &DAG) const {
10096   assert(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unknown opcode!");
10097 
10098   EVT VT = Op.getOperand(0).getValueType();
10099   if (useSVEForFixedLengthVectorVT(VT))
10100     return LowerFixedLengthExtractVectorElt(Op, DAG);
10101 
10102   // Check for non-constant or out of range lane.
10103   ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(1));
10104   if (!CI || CI->getZExtValue() >= VT.getVectorNumElements())
10105     return SDValue();
10106 
10107 
10108   // Insertion/extraction are legal for V128 types.
10109   if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
10110       VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
10111       VT == MVT::v8f16 || VT == MVT::v8bf16)
10112     return Op;
10113 
10114   if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 &&
10115       VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16 &&
10116       VT != MVT::v4bf16)
10117     return SDValue();
10118 
10119   // For V64 types, we perform extraction by expanding the value
10120   // to a V128 type and perform the extraction on that.
10121   SDLoc DL(Op);
10122   SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
10123   EVT WideTy = WideVec.getValueType();
10124 
10125   EVT ExtrTy = WideTy.getVectorElementType();
10126   if (ExtrTy == MVT::i16 || ExtrTy == MVT::i8)
10127     ExtrTy = MVT::i32;
10128 
10129   // For extractions, we just return the result directly.
10130   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtrTy, WideVec,
10131                      Op.getOperand(1));
10132 }
10133 
10134 SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
10135                                                       SelectionDAG &DAG) const {
10136   assert(Op.getValueType().isFixedLengthVector() &&
10137          "Only cases that extract a fixed length vector are supported!");
10138 
10139   EVT InVT = Op.getOperand(0).getValueType();
10140   unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
10141   unsigned Size = Op.getValueSizeInBits();
10142 
10143   if (InVT.isScalableVector()) {
10144     // This will be matched by custom code during ISelDAGToDAG.
10145     if (Idx == 0 && isPackedVectorType(InVT, DAG))
10146       return Op;
10147 
10148     return SDValue();
10149   }
10150 
10151   // This will get lowered to an appropriate EXTRACT_SUBREG in ISel.
10152   if (Idx == 0 && InVT.getSizeInBits() <= 128)
10153     return Op;
10154 
10155   // If this is extracting the upper 64-bits of a 128-bit vector, we match
10156   // that directly.
10157   if (Size == 64 && Idx * InVT.getScalarSizeInBits() == 64 &&
10158       InVT.getSizeInBits() == 128)
10159     return Op;
10160 
10161   return SDValue();
10162 }
10163 
10164 SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
10165                                                      SelectionDAG &DAG) const {
10166   assert(Op.getValueType().isScalableVector() &&
10167          "Only expect to lower inserts into scalable vectors!");
10168 
10169   EVT InVT = Op.getOperand(1).getValueType();
10170   unsigned Idx = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
10171 
10172   if (InVT.isScalableVector()) {
10173     SDLoc DL(Op);
10174     EVT VT = Op.getValueType();
10175 
10176     if (!isTypeLegal(VT) || !VT.isInteger())
10177       return SDValue();
10178 
10179     SDValue Vec0 = Op.getOperand(0);
10180     SDValue Vec1 = Op.getOperand(1);
10181 
10182     // Ensure the subvector is half the size of the main vector.
10183     if (VT.getVectorElementCount() != (InVT.getVectorElementCount() * 2))
10184       return SDValue();
10185 
10186     // Extend elements of smaller vector...
10187     EVT WideVT = InVT.widenIntegerVectorElementType(*(DAG.getContext()));
10188     SDValue ExtVec = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Vec1);
10189 
10190     if (Idx == 0) {
10191       SDValue HiVec0 = DAG.getNode(AArch64ISD::UUNPKHI, DL, WideVT, Vec0);
10192       return DAG.getNode(AArch64ISD::UZP1, DL, VT, ExtVec, HiVec0);
10193     } else if (Idx == InVT.getVectorMinNumElements()) {
10194       SDValue LoVec0 = DAG.getNode(AArch64ISD::UUNPKLO, DL, WideVT, Vec0);
10195       return DAG.getNode(AArch64ISD::UZP1, DL, VT, LoVec0, ExtVec);
10196     }
10197 
10198     return SDValue();
10199   }
10200 
10201   // This will be matched by custom code during ISelDAGToDAG.
10202   if (Idx == 0 && isPackedVectorType(InVT, DAG) && Op.getOperand(0).isUndef())
10203     return Op;
10204 
10205   return SDValue();
10206 }
10207 
10208 SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
10209   EVT VT = Op.getValueType();
10210 
10211   if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))
10212     return LowerFixedLengthVectorIntDivideToSVE(Op, DAG);
10213 
10214   assert(VT.isScalableVector() && "Expected a scalable vector.");
10215 
10216   bool Signed = Op.getOpcode() == ISD::SDIV;
10217   unsigned PredOpcode = Signed ? AArch64ISD::SDIV_PRED : AArch64ISD::UDIV_PRED;
10218 
10219   if (VT == MVT::nxv4i32 || VT == MVT::nxv2i64)
10220     return LowerToPredicatedOp(Op, DAG, PredOpcode);
10221 
10222   // SVE doesn't have i8 and i16 DIV operations; widen them to 32-bit
10223   // operations, and truncate the result.
10224   EVT WidenedVT;
10225   if (VT == MVT::nxv16i8)
10226     WidenedVT = MVT::nxv8i16;
10227   else if (VT == MVT::nxv8i16)
10228     WidenedVT = MVT::nxv4i32;
10229   else
10230     llvm_unreachable("Unexpected Custom DIV operation");
10231 
10232   SDLoc dl(Op);
10233   unsigned UnpkLo = Signed ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO;
10234   unsigned UnpkHi = Signed ? AArch64ISD::SUNPKHI : AArch64ISD::UUNPKHI;
10235   SDValue Op0Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(0));
10236   SDValue Op1Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(1));
10237   SDValue Op0Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(0));
10238   SDValue Op1Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(1));
10239   SDValue ResultLo = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Lo, Op1Lo);
10240   SDValue ResultHi = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Hi, Op1Hi);
10241   return DAG.getNode(AArch64ISD::UZP1, dl, VT, ResultLo, ResultHi);
10242 }
10243 
10244 bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
10245   // Currently no fixed length shuffles that require SVE are legal.
10246   if (useSVEForFixedLengthVectorVT(VT))
10247     return false;
10248 
10249   if (VT.getVectorNumElements() == 4 &&
10250       (VT.is128BitVector() || VT.is64BitVector())) {
10251     unsigned PFIndexes[4];
10252     for (unsigned i = 0; i != 4; ++i) {
10253       if (M[i] < 0)
10254         PFIndexes[i] = 8;
10255       else
10256         PFIndexes[i] = M[i];
10257     }
10258 
10259     // Compute the index in the perfect shuffle table.
10260     unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
10261                             PFIndexes[2] * 9 + PFIndexes[3];
10262     unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
10263     unsigned Cost = (PFEntry >> 30);
10264 
10265     if (Cost <= 4)
10266       return true;
10267   }
10268 
10269   bool DummyBool;
10270   int DummyInt;
10271   unsigned DummyUnsigned;
10272 
10273   return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) || isREVMask(M, VT, 64) ||
10274           isREVMask(M, VT, 32) || isREVMask(M, VT, 16) ||
10275           isEXTMask(M, VT, DummyBool, DummyUnsigned) ||
10276           // isTBLMask(M, VT) || // FIXME: Port TBL support from ARM.
10277           isTRNMask(M, VT, DummyUnsigned) || isUZPMask(M, VT, DummyUnsigned) ||
10278           isZIPMask(M, VT, DummyUnsigned) ||
10279           isTRN_v_undef_Mask(M, VT, DummyUnsigned) ||
10280           isUZP_v_undef_Mask(M, VT, DummyUnsigned) ||
10281           isZIP_v_undef_Mask(M, VT, DummyUnsigned) ||
10282           isINSMask(M, VT.getVectorNumElements(), DummyBool, DummyInt) ||
10283           isConcatMask(M, VT, VT.getSizeInBits() == 128));
10284 }
10285 
10286 /// getVShiftImm - Check if this is a valid build_vector for the immediate
10287 /// operand of a vector shift operation, where all the elements of the
10288 /// build_vector must have the same constant integer value.
10289 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
10290   // Ignore bit_converts.
10291   while (Op.getOpcode() == ISD::BITCAST)
10292     Op = Op.getOperand(0);
10293   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
10294   APInt SplatBits, SplatUndef;
10295   unsigned SplatBitSize;
10296   bool HasAnyUndefs;
10297   if (!BVN || !BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
10298                                     HasAnyUndefs, ElementBits) ||
10299       SplatBitSize > ElementBits)
10300     return false;
10301   Cnt = SplatBits.getSExtValue();
10302   return true;
10303 }
10304 
10305 /// isVShiftLImm - Check if this is a valid build_vector for the immediate
10306 /// operand of a vector shift left operation.  That value must be in the range:
10307 ///   0 <= Value < ElementBits for a left shift; or
10308 ///   0 <= Value <= ElementBits for a long left shift.
10309 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
10310   assert(VT.isVector() && "vector shift count is not a vector type");
10311   int64_t ElementBits = VT.getScalarSizeInBits();
10312   if (!getVShiftImm(Op, ElementBits, Cnt))
10313     return false;
10314   return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
10315 }
10316 
10317 /// isVShiftRImm - Check if this is a valid build_vector for the immediate
10318 /// operand of a vector shift right operation. The value must be in the range:
10319 ///   1 <= Value <= ElementBits for a right shift; or
10320 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
10321   assert(VT.isVector() && "vector shift count is not a vector type");
10322   int64_t ElementBits = VT.getScalarSizeInBits();
10323   if (!getVShiftImm(Op, ElementBits, Cnt))
10324     return false;
10325   return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
10326 }
10327 
10328 SDValue AArch64TargetLowering::LowerTRUNCATE(SDValue Op,
10329                                              SelectionDAG &DAG) const {
10330   EVT VT = Op.getValueType();
10331 
10332   if (VT.getScalarType() == MVT::i1) {
10333     // Lower i1 truncate to `(x & 1) != 0`.
10334     SDLoc dl(Op);
10335     EVT OpVT = Op.getOperand(0).getValueType();
10336     SDValue Zero = DAG.getConstant(0, dl, OpVT);
10337     SDValue One = DAG.getConstant(1, dl, OpVT);
10338     SDValue And = DAG.getNode(ISD::AND, dl, OpVT, Op.getOperand(0), One);
10339     return DAG.getSetCC(dl, VT, And, Zero, ISD::SETNE);
10340   }
10341 
10342   if (!VT.isVector() || VT.isScalableVector())
10343     return SDValue();
10344 
10345   if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
10346     return LowerFixedLengthVectorTruncateToSVE(Op, DAG);
10347 
10348   return SDValue();
10349 }
10350 
10351 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
10352                                                       SelectionDAG &DAG) const {
10353   EVT VT = Op.getValueType();
10354   SDLoc DL(Op);
10355   int64_t Cnt;
10356 
10357   if (!Op.getOperand(1).getValueType().isVector())
10358     return Op;
10359   unsigned EltSize = VT.getScalarSizeInBits();
10360 
10361   switch (Op.getOpcode()) {
10362   default:
10363     llvm_unreachable("unexpected shift opcode");
10364 
10365   case ISD::SHL:
10366     if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT))
10367       return LowerToPredicatedOp(Op, DAG, AArch64ISD::SHL_PRED);
10368 
10369     if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize)
10370       return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0),
10371                          DAG.getConstant(Cnt, DL, MVT::i32));
10372     return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
10373                        DAG.getConstant(Intrinsic::aarch64_neon_ushl, DL,
10374                                        MVT::i32),
10375                        Op.getOperand(0), Op.getOperand(1));
10376   case ISD::SRA:
10377   case ISD::SRL:
10378     if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT)) {
10379       unsigned Opc = Op.getOpcode() == ISD::SRA ? AArch64ISD::SRA_PRED
10380                                                 : AArch64ISD::SRL_PRED;
10381       return LowerToPredicatedOp(Op, DAG, Opc);
10382     }
10383 
10384     // Right shift immediate
10385     if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) {
10386       unsigned Opc =
10387           (Op.getOpcode() == ISD::SRA) ? AArch64ISD::VASHR : AArch64ISD::VLSHR;
10388       return DAG.getNode(Opc, DL, VT, Op.getOperand(0),
10389                          DAG.getConstant(Cnt, DL, MVT::i32));
10390     }
10391 
10392     // Right shift register.  Note, there is not a shift right register
10393     // instruction, but the shift left register instruction takes a signed
10394     // value, where negative numbers specify a right shift.
10395     unsigned Opc = (Op.getOpcode() == ISD::SRA) ? Intrinsic::aarch64_neon_sshl
10396                                                 : Intrinsic::aarch64_neon_ushl;
10397     // negate the shift amount
10398     SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1));
10399     SDValue NegShiftLeft =
10400         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
10401                     DAG.getConstant(Opc, DL, MVT::i32), Op.getOperand(0),
10402                     NegShift);
10403     return NegShiftLeft;
10404   }
10405 
10406   return SDValue();
10407 }
10408 
10409 static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
10410                                     AArch64CC::CondCode CC, bool NoNans, EVT VT,
10411                                     const SDLoc &dl, SelectionDAG &DAG) {
10412   EVT SrcVT = LHS.getValueType();
10413   assert(VT.getSizeInBits() == SrcVT.getSizeInBits() &&
10414          "function only supposed to emit natural comparisons");
10415 
10416   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(RHS.getNode());
10417   APInt CnstBits(VT.getSizeInBits(), 0);
10418   APInt UndefBits(VT.getSizeInBits(), 0);
10419   bool IsCnst = BVN && resolveBuildVector(BVN, CnstBits, UndefBits);
10420   bool IsZero = IsCnst && (CnstBits == 0);
10421 
10422   if (SrcVT.getVectorElementType().isFloatingPoint()) {
10423     switch (CC) {
10424     default:
10425       return SDValue();
10426     case AArch64CC::NE: {
10427       SDValue Fcmeq;
10428       if (IsZero)
10429         Fcmeq = DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS);
10430       else
10431         Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
10432       return DAG.getNOT(dl, Fcmeq, VT);
10433     }
10434     case AArch64CC::EQ:
10435       if (IsZero)
10436         return DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS);
10437       return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
10438     case AArch64CC::GE:
10439       if (IsZero)
10440         return DAG.getNode(AArch64ISD::FCMGEz, dl, VT, LHS);
10441       return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS);
10442     case AArch64CC::GT:
10443       if (IsZero)
10444         return DAG.getNode(AArch64ISD::FCMGTz, dl, VT, LHS);
10445       return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS);
10446     case AArch64CC::LS:
10447       if (IsZero)
10448         return DAG.getNode(AArch64ISD::FCMLEz, dl, VT, LHS);
10449       return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS);
10450     case AArch64CC::LT:
10451       if (!NoNans)
10452         return SDValue();
10453       // If we ignore NaNs then we can use to the MI implementation.
10454       LLVM_FALLTHROUGH;
10455     case AArch64CC::MI:
10456       if (IsZero)
10457         return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS);
10458       return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS);
10459     }
10460   }
10461 
10462   switch (CC) {
10463   default:
10464     return SDValue();
10465   case AArch64CC::NE: {
10466     SDValue Cmeq;
10467     if (IsZero)
10468       Cmeq = DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS);
10469     else
10470       Cmeq = DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS);
10471     return DAG.getNOT(dl, Cmeq, VT);
10472   }
10473   case AArch64CC::EQ:
10474     if (IsZero)
10475       return DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS);
10476     return DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS);
10477   case AArch64CC::GE:
10478     if (IsZero)
10479       return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS);
10480     return DAG.getNode(AArch64ISD::CMGE, dl, VT, LHS, RHS);
10481   case AArch64CC::GT:
10482     if (IsZero)
10483       return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS);
10484     return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS);
10485   case AArch64CC::LE:
10486     if (IsZero)
10487       return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS);
10488     return DAG.getNode(AArch64ISD::CMGE, dl, VT, RHS, LHS);
10489   case AArch64CC::LS:
10490     return DAG.getNode(AArch64ISD::CMHS, dl, VT, RHS, LHS);
10491   case AArch64CC::LO:
10492     return DAG.getNode(AArch64ISD::CMHI, dl, VT, RHS, LHS);
10493   case AArch64CC::LT:
10494     if (IsZero)
10495       return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS);
10496     return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS);
10497   case AArch64CC::HI:
10498     return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS);
10499   case AArch64CC::HS:
10500     return DAG.getNode(AArch64ISD::CMHS, dl, VT, LHS, RHS);
10501   }
10502 }
10503 
10504 SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
10505                                            SelectionDAG &DAG) const {
10506   if (Op.getValueType().isScalableVector())
10507     return LowerToPredicatedOp(Op, DAG, AArch64ISD::SETCC_MERGE_ZERO);
10508 
10509   if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
10510     return LowerFixedLengthVectorSetccToSVE(Op, DAG);
10511 
10512   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
10513   SDValue LHS = Op.getOperand(0);
10514   SDValue RHS = Op.getOperand(1);
10515   EVT CmpVT = LHS.getValueType().changeVectorElementTypeToInteger();
10516   SDLoc dl(Op);
10517 
10518   if (LHS.getValueType().getVectorElementType().isInteger()) {
10519     assert(LHS.getValueType() == RHS.getValueType());
10520     AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
10521     SDValue Cmp =
10522         EmitVectorComparison(LHS, RHS, AArch64CC, false, CmpVT, dl, DAG);
10523     return DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType());
10524   }
10525 
10526   const bool FullFP16 =
10527     static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
10528 
10529   // Make v4f16 (only) fcmp operations utilise vector instructions
10530   // v8f16 support will be a litle more complicated
10531   if (!FullFP16 && LHS.getValueType().getVectorElementType() == MVT::f16) {
10532     if (LHS.getValueType().getVectorNumElements() == 4) {
10533       LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, LHS);
10534       RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, RHS);
10535       SDValue NewSetcc = DAG.getSetCC(dl, MVT::v4i16, LHS, RHS, CC);
10536       DAG.ReplaceAllUsesWith(Op, NewSetcc);
10537       CmpVT = MVT::v4i32;
10538     } else
10539       return SDValue();
10540   }
10541 
10542   assert((!FullFP16 && LHS.getValueType().getVectorElementType() != MVT::f16) ||
10543           LHS.getValueType().getVectorElementType() != MVT::f128);
10544 
10545   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
10546   // clean.  Some of them require two branches to implement.
10547   AArch64CC::CondCode CC1, CC2;
10548   bool ShouldInvert;
10549   changeVectorFPCCToAArch64CC(CC, CC1, CC2, ShouldInvert);
10550 
10551   bool NoNaNs = getTargetMachine().Options.NoNaNsFPMath;
10552   SDValue Cmp =
10553       EmitVectorComparison(LHS, RHS, CC1, NoNaNs, CmpVT, dl, DAG);
10554   if (!Cmp.getNode())
10555     return SDValue();
10556 
10557   if (CC2 != AArch64CC::AL) {
10558     SDValue Cmp2 =
10559         EmitVectorComparison(LHS, RHS, CC2, NoNaNs, CmpVT, dl, DAG);
10560     if (!Cmp2.getNode())
10561       return SDValue();
10562 
10563     Cmp = DAG.getNode(ISD::OR, dl, CmpVT, Cmp, Cmp2);
10564   }
10565 
10566   Cmp = DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType());
10567 
10568   if (ShouldInvert)
10569     Cmp = DAG.getNOT(dl, Cmp, Cmp.getValueType());
10570 
10571   return Cmp;
10572 }
10573 
10574 static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp,
10575                                   SelectionDAG &DAG) {
10576   SDValue VecOp = ScalarOp.getOperand(0);
10577   auto Rdx = DAG.getNode(Op, DL, VecOp.getSimpleValueType(), VecOp);
10578   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarOp.getValueType(), Rdx,
10579                      DAG.getConstant(0, DL, MVT::i64));
10580 }
10581 
10582 SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
10583                                               SelectionDAG &DAG) const {
10584   SDValue Src = Op.getOperand(0);
10585 
10586   // Try to lower fixed length reductions to SVE.
10587   EVT SrcVT = Src.getValueType();
10588   bool OverrideNEON = Op.getOpcode() == ISD::VECREDUCE_AND ||
10589                       Op.getOpcode() == ISD::VECREDUCE_OR ||
10590                       Op.getOpcode() == ISD::VECREDUCE_XOR ||
10591                       Op.getOpcode() == ISD::VECREDUCE_FADD ||
10592                       (Op.getOpcode() != ISD::VECREDUCE_ADD &&
10593                        SrcVT.getVectorElementType() == MVT::i64);
10594   if (SrcVT.isScalableVector() ||
10595       useSVEForFixedLengthVectorVT(SrcVT, OverrideNEON)) {
10596 
10597     if (SrcVT.getVectorElementType() == MVT::i1)
10598       return LowerPredReductionToSVE(Op, DAG);
10599 
10600     switch (Op.getOpcode()) {
10601     case ISD::VECREDUCE_ADD:
10602       return LowerReductionToSVE(AArch64ISD::UADDV_PRED, Op, DAG);
10603     case ISD::VECREDUCE_AND:
10604       return LowerReductionToSVE(AArch64ISD::ANDV_PRED, Op, DAG);
10605     case ISD::VECREDUCE_OR:
10606       return LowerReductionToSVE(AArch64ISD::ORV_PRED, Op, DAG);
10607     case ISD::VECREDUCE_SMAX:
10608       return LowerReductionToSVE(AArch64ISD::SMAXV_PRED, Op, DAG);
10609     case ISD::VECREDUCE_SMIN:
10610       return LowerReductionToSVE(AArch64ISD::SMINV_PRED, Op, DAG);
10611     case ISD::VECREDUCE_UMAX:
10612       return LowerReductionToSVE(AArch64ISD::UMAXV_PRED, Op, DAG);
10613     case ISD::VECREDUCE_UMIN:
10614       return LowerReductionToSVE(AArch64ISD::UMINV_PRED, Op, DAG);
10615     case ISD::VECREDUCE_XOR:
10616       return LowerReductionToSVE(AArch64ISD::EORV_PRED, Op, DAG);
10617     case ISD::VECREDUCE_FADD:
10618       return LowerReductionToSVE(AArch64ISD::FADDV_PRED, Op, DAG);
10619     case ISD::VECREDUCE_FMAX:
10620       return LowerReductionToSVE(AArch64ISD::FMAXNMV_PRED, Op, DAG);
10621     case ISD::VECREDUCE_FMIN:
10622       return LowerReductionToSVE(AArch64ISD::FMINNMV_PRED, Op, DAG);
10623     default:
10624       llvm_unreachable("Unhandled fixed length reduction");
10625     }
10626   }
10627 
10628   // Lower NEON reductions.
10629   SDLoc dl(Op);
10630   switch (Op.getOpcode()) {
10631   case ISD::VECREDUCE_ADD:
10632     return getReductionSDNode(AArch64ISD::UADDV, dl, Op, DAG);
10633   case ISD::VECREDUCE_SMAX:
10634     return getReductionSDNode(AArch64ISD::SMAXV, dl, Op, DAG);
10635   case ISD::VECREDUCE_SMIN:
10636     return getReductionSDNode(AArch64ISD::SMINV, dl, Op, DAG);
10637   case ISD::VECREDUCE_UMAX:
10638     return getReductionSDNode(AArch64ISD::UMAXV, dl, Op, DAG);
10639   case ISD::VECREDUCE_UMIN:
10640     return getReductionSDNode(AArch64ISD::UMINV, dl, Op, DAG);
10641   case ISD::VECREDUCE_FMAX: {
10642     return DAG.getNode(
10643         ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(),
10644         DAG.getConstant(Intrinsic::aarch64_neon_fmaxnmv, dl, MVT::i32),
10645         Src);
10646   }
10647   case ISD::VECREDUCE_FMIN: {
10648     return DAG.getNode(
10649         ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(),
10650         DAG.getConstant(Intrinsic::aarch64_neon_fminnmv, dl, MVT::i32),
10651         Src);
10652   }
10653   default:
10654     llvm_unreachable("Unhandled reduction");
10655   }
10656 }
10657 
10658 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_SUB(SDValue Op,
10659                                                     SelectionDAG &DAG) const {
10660   auto &Subtarget = static_cast<const AArch64Subtarget &>(DAG.getSubtarget());
10661   if (!Subtarget.hasLSE() && !Subtarget.outlineAtomics())
10662     return SDValue();
10663 
10664   // LSE has an atomic load-add instruction, but not a load-sub.
10665   SDLoc dl(Op);
10666   MVT VT = Op.getSimpleValueType();
10667   SDValue RHS = Op.getOperand(2);
10668   AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode());
10669   RHS = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), RHS);
10670   return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl, AN->getMemoryVT(),
10671                        Op.getOperand(0), Op.getOperand(1), RHS,
10672                        AN->getMemOperand());
10673 }
10674 
10675 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
10676                                                     SelectionDAG &DAG) const {
10677   auto &Subtarget = static_cast<const AArch64Subtarget &>(DAG.getSubtarget());
10678   if (!Subtarget.hasLSE() && !Subtarget.outlineAtomics())
10679     return SDValue();
10680 
10681   // LSE has an atomic load-clear instruction, but not a load-and.
10682   SDLoc dl(Op);
10683   MVT VT = Op.getSimpleValueType();
10684   SDValue RHS = Op.getOperand(2);
10685   AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode());
10686   RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getConstant(-1ULL, dl, VT), RHS);
10687   return DAG.getAtomic(ISD::ATOMIC_LOAD_CLR, dl, AN->getMemoryVT(),
10688                        Op.getOperand(0), Op.getOperand(1), RHS,
10689                        AN->getMemOperand());
10690 }
10691 
10692 SDValue AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(
10693     SDValue Op, SDValue Chain, SDValue &Size, SelectionDAG &DAG) const {
10694   SDLoc dl(Op);
10695   EVT PtrVT = getPointerTy(DAG.getDataLayout());
10696   SDValue Callee = DAG.getTargetExternalSymbol("__chkstk", PtrVT, 0);
10697 
10698   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
10699   const uint32_t *Mask = TRI->getWindowsStackProbePreservedMask();
10700   if (Subtarget->hasCustomCallingConv())
10701     TRI->UpdateCustomCallPreservedMask(DAG.getMachineFunction(), &Mask);
10702 
10703   Size = DAG.getNode(ISD::SRL, dl, MVT::i64, Size,
10704                      DAG.getConstant(4, dl, MVT::i64));
10705   Chain = DAG.getCopyToReg(Chain, dl, AArch64::X15, Size, SDValue());
10706   Chain =
10707       DAG.getNode(AArch64ISD::CALL, dl, DAG.getVTList(MVT::Other, MVT::Glue),
10708                   Chain, Callee, DAG.getRegister(AArch64::X15, MVT::i64),
10709                   DAG.getRegisterMask(Mask), Chain.getValue(1));
10710   // To match the actual intent better, we should read the output from X15 here
10711   // again (instead of potentially spilling it to the stack), but rereading Size
10712   // from X15 here doesn't work at -O0, since it thinks that X15 is undefined
10713   // here.
10714 
10715   Size = DAG.getNode(ISD::SHL, dl, MVT::i64, Size,
10716                      DAG.getConstant(4, dl, MVT::i64));
10717   return Chain;
10718 }
10719 
10720 SDValue
10721 AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
10722                                                SelectionDAG &DAG) const {
10723   assert(Subtarget->isTargetWindows() &&
10724          "Only Windows alloca probing supported");
10725   SDLoc dl(Op);
10726   // Get the inputs.
10727   SDNode *Node = Op.getNode();
10728   SDValue Chain = Op.getOperand(0);
10729   SDValue Size = Op.getOperand(1);
10730   MaybeAlign Align =
10731       cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue();
10732   EVT VT = Node->getValueType(0);
10733 
10734   if (DAG.getMachineFunction().getFunction().hasFnAttribute(
10735           "no-stack-arg-probe")) {
10736     SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
10737     Chain = SP.getValue(1);
10738     SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
10739     if (Align)
10740       SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
10741                        DAG.getConstant(-(uint64_t)Align->value(), dl, VT));
10742     Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
10743     SDValue Ops[2] = {SP, Chain};
10744     return DAG.getMergeValues(Ops, dl);
10745   }
10746 
10747   Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
10748 
10749   Chain = LowerWindowsDYNAMIC_STACKALLOC(Op, Chain, Size, DAG);
10750 
10751   SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
10752   Chain = SP.getValue(1);
10753   SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
10754   if (Align)
10755     SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
10756                      DAG.getConstant(-(uint64_t)Align->value(), dl, VT));
10757   Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
10758 
10759   Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
10760                              DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
10761 
10762   SDValue Ops[2] = {SP, Chain};
10763   return DAG.getMergeValues(Ops, dl);
10764 }
10765 
10766 SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op,
10767                                            SelectionDAG &DAG) const {
10768   EVT VT = Op.getValueType();
10769   assert(VT != MVT::i64 && "Expected illegal VSCALE node");
10770 
10771   SDLoc DL(Op);
10772   APInt MulImm = cast<ConstantSDNode>(Op.getOperand(0))->getAPIntValue();
10773   return DAG.getZExtOrTrunc(DAG.getVScale(DL, MVT::i64, MulImm.sextOrSelf(64)),
10774                             DL, VT);
10775 }
10776 
10777 /// Set the IntrinsicInfo for the `aarch64_sve_st<N>` intrinsics.
10778 template <unsigned NumVecs>
10779 static bool
10780 setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL,
10781               AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI) {
10782   Info.opc = ISD::INTRINSIC_VOID;
10783   // Retrieve EC from first vector argument.
10784   const EVT VT = TLI.getMemValueType(DL, CI.getArgOperand(0)->getType());
10785   ElementCount EC = VT.getVectorElementCount();
10786 #ifndef NDEBUG
10787   // Check the assumption that all input vectors are the same type.
10788   for (unsigned I = 0; I < NumVecs; ++I)
10789     assert(VT == TLI.getMemValueType(DL, CI.getArgOperand(I)->getType()) &&
10790            "Invalid type.");
10791 #endif
10792   // memVT is `NumVecs * VT`.
10793   Info.memVT = EVT::getVectorVT(CI.getType()->getContext(), VT.getScalarType(),
10794                                 EC * NumVecs);
10795   Info.ptrVal = CI.getArgOperand(CI.getNumArgOperands() - 1);
10796   Info.offset = 0;
10797   Info.align.reset();
10798   Info.flags = MachineMemOperand::MOStore;
10799   return true;
10800 }
10801 
10802 /// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
10803 /// MemIntrinsicNodes.  The associated MachineMemOperands record the alignment
10804 /// specified in the intrinsic calls.
10805 bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
10806                                                const CallInst &I,
10807                                                MachineFunction &MF,
10808                                                unsigned Intrinsic) const {
10809   auto &DL = I.getModule()->getDataLayout();
10810   switch (Intrinsic) {
10811   case Intrinsic::aarch64_sve_st2:
10812     return setInfoSVEStN<2>(*this, DL, Info, I);
10813   case Intrinsic::aarch64_sve_st3:
10814     return setInfoSVEStN<3>(*this, DL, Info, I);
10815   case Intrinsic::aarch64_sve_st4:
10816     return setInfoSVEStN<4>(*this, DL, Info, I);
10817   case Intrinsic::aarch64_neon_ld2:
10818   case Intrinsic::aarch64_neon_ld3:
10819   case Intrinsic::aarch64_neon_ld4:
10820   case Intrinsic::aarch64_neon_ld1x2:
10821   case Intrinsic::aarch64_neon_ld1x3:
10822   case Intrinsic::aarch64_neon_ld1x4:
10823   case Intrinsic::aarch64_neon_ld2lane:
10824   case Intrinsic::aarch64_neon_ld3lane:
10825   case Intrinsic::aarch64_neon_ld4lane:
10826   case Intrinsic::aarch64_neon_ld2r:
10827   case Intrinsic::aarch64_neon_ld3r:
10828   case Intrinsic::aarch64_neon_ld4r: {
10829     Info.opc = ISD::INTRINSIC_W_CHAIN;
10830     // Conservatively set memVT to the entire set of vectors loaded.
10831     uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
10832     Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
10833     Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1);
10834     Info.offset = 0;
10835     Info.align.reset();
10836     // volatile loads with NEON intrinsics not supported
10837     Info.flags = MachineMemOperand::MOLoad;
10838     return true;
10839   }
10840   case Intrinsic::aarch64_neon_st2:
10841   case Intrinsic::aarch64_neon_st3:
10842   case Intrinsic::aarch64_neon_st4:
10843   case Intrinsic::aarch64_neon_st1x2:
10844   case Intrinsic::aarch64_neon_st1x3:
10845   case Intrinsic::aarch64_neon_st1x4:
10846   case Intrinsic::aarch64_neon_st2lane:
10847   case Intrinsic::aarch64_neon_st3lane:
10848   case Intrinsic::aarch64_neon_st4lane: {
10849     Info.opc = ISD::INTRINSIC_VOID;
10850     // Conservatively set memVT to the entire set of vectors stored.
10851     unsigned NumElts = 0;
10852     for (unsigned ArgI = 0, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) {
10853       Type *ArgTy = I.getArgOperand(ArgI)->getType();
10854       if (!ArgTy->isVectorTy())
10855         break;
10856       NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
10857     }
10858     Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
10859     Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1);
10860     Info.offset = 0;
10861     Info.align.reset();
10862     // volatile stores with NEON intrinsics not supported
10863     Info.flags = MachineMemOperand::MOStore;
10864     return true;
10865   }
10866   case Intrinsic::aarch64_ldaxr:
10867   case Intrinsic::aarch64_ldxr: {
10868     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
10869     Info.opc = ISD::INTRINSIC_W_CHAIN;
10870     Info.memVT = MVT::getVT(PtrTy->getElementType());
10871     Info.ptrVal = I.getArgOperand(0);
10872     Info.offset = 0;
10873     Info.align = DL.getABITypeAlign(PtrTy->getElementType());
10874     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
10875     return true;
10876   }
10877   case Intrinsic::aarch64_stlxr:
10878   case Intrinsic::aarch64_stxr: {
10879     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
10880     Info.opc = ISD::INTRINSIC_W_CHAIN;
10881     Info.memVT = MVT::getVT(PtrTy->getElementType());
10882     Info.ptrVal = I.getArgOperand(1);
10883     Info.offset = 0;
10884     Info.align = DL.getABITypeAlign(PtrTy->getElementType());
10885     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
10886     return true;
10887   }
10888   case Intrinsic::aarch64_ldaxp:
10889   case Intrinsic::aarch64_ldxp:
10890     Info.opc = ISD::INTRINSIC_W_CHAIN;
10891     Info.memVT = MVT::i128;
10892     Info.ptrVal = I.getArgOperand(0);
10893     Info.offset = 0;
10894     Info.align = Align(16);
10895     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
10896     return true;
10897   case Intrinsic::aarch64_stlxp:
10898   case Intrinsic::aarch64_stxp:
10899     Info.opc = ISD::INTRINSIC_W_CHAIN;
10900     Info.memVT = MVT::i128;
10901     Info.ptrVal = I.getArgOperand(2);
10902     Info.offset = 0;
10903     Info.align = Align(16);
10904     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
10905     return true;
10906   case Intrinsic::aarch64_sve_ldnt1: {
10907     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
10908     Info.opc = ISD::INTRINSIC_W_CHAIN;
10909     Info.memVT = MVT::getVT(I.getType());
10910     Info.ptrVal = I.getArgOperand(1);
10911     Info.offset = 0;
10912     Info.align = DL.getABITypeAlign(PtrTy->getElementType());
10913     Info.flags = MachineMemOperand::MOLoad;
10914     if (Intrinsic == Intrinsic::aarch64_sve_ldnt1)
10915       Info.flags |= MachineMemOperand::MONonTemporal;
10916     return true;
10917   }
10918   case Intrinsic::aarch64_sve_stnt1: {
10919     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(2)->getType());
10920     Info.opc = ISD::INTRINSIC_W_CHAIN;
10921     Info.memVT = MVT::getVT(I.getOperand(0)->getType());
10922     Info.ptrVal = I.getArgOperand(2);
10923     Info.offset = 0;
10924     Info.align = DL.getABITypeAlign(PtrTy->getElementType());
10925     Info.flags = MachineMemOperand::MOStore;
10926     if (Intrinsic == Intrinsic::aarch64_sve_stnt1)
10927       Info.flags |= MachineMemOperand::MONonTemporal;
10928     return true;
10929   }
10930   default:
10931     break;
10932   }
10933 
10934   return false;
10935 }
10936 
10937 bool AArch64TargetLowering::shouldReduceLoadWidth(SDNode *Load,
10938                                                   ISD::LoadExtType ExtTy,
10939                                                   EVT NewVT) const {
10940   // TODO: This may be worth removing. Check regression tests for diffs.
10941   if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT))
10942     return false;
10943 
10944   // If we're reducing the load width in order to avoid having to use an extra
10945   // instruction to do extension then it's probably a good idea.
10946   if (ExtTy != ISD::NON_EXTLOAD)
10947     return true;
10948   // Don't reduce load width if it would prevent us from combining a shift into
10949   // the offset.
10950   MemSDNode *Mem = dyn_cast<MemSDNode>(Load);
10951   assert(Mem);
10952   const SDValue &Base = Mem->getBasePtr();
10953   if (Base.getOpcode() == ISD::ADD &&
10954       Base.getOperand(1).getOpcode() == ISD::SHL &&
10955       Base.getOperand(1).hasOneUse() &&
10956       Base.getOperand(1).getOperand(1).getOpcode() == ISD::Constant) {
10957     // The shift can be combined if it matches the size of the value being
10958     // loaded (and so reducing the width would make it not match).
10959     uint64_t ShiftAmount = Base.getOperand(1).getConstantOperandVal(1);
10960     uint64_t LoadBytes = Mem->getMemoryVT().getSizeInBits()/8;
10961     if (ShiftAmount == Log2_32(LoadBytes))
10962       return false;
10963   }
10964   // We have no reason to disallow reducing the load width, so allow it.
10965   return true;
10966 }
10967 
10968 // Truncations from 64-bit GPR to 32-bit GPR is free.
10969 bool AArch64TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
10970   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
10971     return false;
10972   uint64_t NumBits1 = Ty1->getPrimitiveSizeInBits().getFixedSize();
10973   uint64_t NumBits2 = Ty2->getPrimitiveSizeInBits().getFixedSize();
10974   return NumBits1 > NumBits2;
10975 }
10976 bool AArch64TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
10977   if (VT1.isVector() || VT2.isVector() || !VT1.isInteger() || !VT2.isInteger())
10978     return false;
10979   uint64_t NumBits1 = VT1.getFixedSizeInBits();
10980   uint64_t NumBits2 = VT2.getFixedSizeInBits();
10981   return NumBits1 > NumBits2;
10982 }
10983 
10984 /// Check if it is profitable to hoist instruction in then/else to if.
10985 /// Not profitable if I and it's user can form a FMA instruction
10986 /// because we prefer FMSUB/FMADD.
10987 bool AArch64TargetLowering::isProfitableToHoist(Instruction *I) const {
10988   if (I->getOpcode() != Instruction::FMul)
10989     return true;
10990 
10991   if (!I->hasOneUse())
10992     return true;
10993 
10994   Instruction *User = I->user_back();
10995 
10996   if (User &&
10997       !(User->getOpcode() == Instruction::FSub ||
10998         User->getOpcode() == Instruction::FAdd))
10999     return true;
11000 
11001   const TargetOptions &Options = getTargetMachine().Options;
11002   const Function *F = I->getFunction();
11003   const DataLayout &DL = F->getParent()->getDataLayout();
11004   Type *Ty = User->getOperand(0)->getType();
11005 
11006   return !(isFMAFasterThanFMulAndFAdd(*F, Ty) &&
11007            isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) &&
11008            (Options.AllowFPOpFusion == FPOpFusion::Fast ||
11009             Options.UnsafeFPMath));
11010 }
11011 
11012 // All 32-bit GPR operations implicitly zero the high-half of the corresponding
11013 // 64-bit GPR.
11014 bool AArch64TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
11015   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
11016     return false;
11017   unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
11018   unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
11019   return NumBits1 == 32 && NumBits2 == 64;
11020 }
11021 bool AArch64TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
11022   if (VT1.isVector() || VT2.isVector() || !VT1.isInteger() || !VT2.isInteger())
11023     return false;
11024   unsigned NumBits1 = VT1.getSizeInBits();
11025   unsigned NumBits2 = VT2.getSizeInBits();
11026   return NumBits1 == 32 && NumBits2 == 64;
11027 }
11028 
11029 bool AArch64TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
11030   EVT VT1 = Val.getValueType();
11031   if (isZExtFree(VT1, VT2)) {
11032     return true;
11033   }
11034 
11035   if (Val.getOpcode() != ISD::LOAD)
11036     return false;
11037 
11038   // 8-, 16-, and 32-bit integer loads all implicitly zero-extend.
11039   return (VT1.isSimple() && !VT1.isVector() && VT1.isInteger() &&
11040           VT2.isSimple() && !VT2.isVector() && VT2.isInteger() &&
11041           VT1.getSizeInBits() <= 32);
11042 }
11043 
11044 bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const {
11045   if (isa<FPExtInst>(Ext))
11046     return false;
11047 
11048   // Vector types are not free.
11049   if (Ext->getType()->isVectorTy())
11050     return false;
11051 
11052   for (const Use &U : Ext->uses()) {
11053     // The extension is free if we can fold it with a left shift in an
11054     // addressing mode or an arithmetic operation: add, sub, and cmp.
11055 
11056     // Is there a shift?
11057     const Instruction *Instr = cast<Instruction>(U.getUser());
11058 
11059     // Is this a constant shift?
11060     switch (Instr->getOpcode()) {
11061     case Instruction::Shl:
11062       if (!isa<ConstantInt>(Instr->getOperand(1)))
11063         return false;
11064       break;
11065     case Instruction::GetElementPtr: {
11066       gep_type_iterator GTI = gep_type_begin(Instr);
11067       auto &DL = Ext->getModule()->getDataLayout();
11068       std::advance(GTI, U.getOperandNo()-1);
11069       Type *IdxTy = GTI.getIndexedType();
11070       // This extension will end up with a shift because of the scaling factor.
11071       // 8-bit sized types have a scaling factor of 1, thus a shift amount of 0.
11072       // Get the shift amount based on the scaling factor:
11073       // log2(sizeof(IdxTy)) - log2(8).
11074       uint64_t ShiftAmt =
11075         countTrailingZeros(DL.getTypeStoreSizeInBits(IdxTy).getFixedSize()) - 3;
11076       // Is the constant foldable in the shift of the addressing mode?
11077       // I.e., shift amount is between 1 and 4 inclusive.
11078       if (ShiftAmt == 0 || ShiftAmt > 4)
11079         return false;
11080       break;
11081     }
11082     case Instruction::Trunc:
11083       // Check if this is a noop.
11084       // trunc(sext ty1 to ty2) to ty1.
11085       if (Instr->getType() == Ext->getOperand(0)->getType())
11086         continue;
11087       LLVM_FALLTHROUGH;
11088     default:
11089       return false;
11090     }
11091 
11092     // At this point we can use the bfm family, so this extension is free
11093     // for that use.
11094   }
11095   return true;
11096 }
11097 
11098 /// Check if both Op1 and Op2 are shufflevector extracts of either the lower
11099 /// or upper half of the vector elements.
11100 static bool areExtractShuffleVectors(Value *Op1, Value *Op2) {
11101   auto areTypesHalfed = [](Value *FullV, Value *HalfV) {
11102     auto *FullTy = FullV->getType();
11103     auto *HalfTy = HalfV->getType();
11104     return FullTy->getPrimitiveSizeInBits().getFixedSize() ==
11105            2 * HalfTy->getPrimitiveSizeInBits().getFixedSize();
11106   };
11107 
11108   auto extractHalf = [](Value *FullV, Value *HalfV) {
11109     auto *FullVT = cast<FixedVectorType>(FullV->getType());
11110     auto *HalfVT = cast<FixedVectorType>(HalfV->getType());
11111     return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
11112   };
11113 
11114   ArrayRef<int> M1, M2;
11115   Value *S1Op1, *S2Op1;
11116   if (!match(Op1, m_Shuffle(m_Value(S1Op1), m_Undef(), m_Mask(M1))) ||
11117       !match(Op2, m_Shuffle(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
11118     return false;
11119 
11120   // Check that the operands are half as wide as the result and we extract
11121   // half of the elements of the input vectors.
11122   if (!areTypesHalfed(S1Op1, Op1) || !areTypesHalfed(S2Op1, Op2) ||
11123       !extractHalf(S1Op1, Op1) || !extractHalf(S2Op1, Op2))
11124     return false;
11125 
11126   // Check the mask extracts either the lower or upper half of vector
11127   // elements.
11128   int M1Start = -1;
11129   int M2Start = -1;
11130   int NumElements = cast<FixedVectorType>(Op1->getType())->getNumElements() * 2;
11131   if (!ShuffleVectorInst::isExtractSubvectorMask(M1, NumElements, M1Start) ||
11132       !ShuffleVectorInst::isExtractSubvectorMask(M2, NumElements, M2Start) ||
11133       M1Start != M2Start || (M1Start != 0 && M2Start != (NumElements / 2)))
11134     return false;
11135 
11136   return true;
11137 }
11138 
11139 /// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
11140 /// of the vector elements.
11141 static bool areExtractExts(Value *Ext1, Value *Ext2) {
11142   auto areExtDoubled = [](Instruction *Ext) {
11143     return Ext->getType()->getScalarSizeInBits() ==
11144            2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
11145   };
11146 
11147   if (!match(Ext1, m_ZExtOrSExt(m_Value())) ||
11148       !match(Ext2, m_ZExtOrSExt(m_Value())) ||
11149       !areExtDoubled(cast<Instruction>(Ext1)) ||
11150       !areExtDoubled(cast<Instruction>(Ext2)))
11151     return false;
11152 
11153   return true;
11154 }
11155 
11156 /// Check if Op could be used with vmull_high_p64 intrinsic.
11157 static bool isOperandOfVmullHighP64(Value *Op) {
11158   Value *VectorOperand = nullptr;
11159   ConstantInt *ElementIndex = nullptr;
11160   return match(Op, m_ExtractElt(m_Value(VectorOperand),
11161                                 m_ConstantInt(ElementIndex))) &&
11162          ElementIndex->getValue() == 1 &&
11163          isa<FixedVectorType>(VectorOperand->getType()) &&
11164          cast<FixedVectorType>(VectorOperand->getType())->getNumElements() == 2;
11165 }
11166 
11167 /// Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
11168 static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2) {
11169   return isOperandOfVmullHighP64(Op1) && isOperandOfVmullHighP64(Op2);
11170 }
11171 
11172 /// Check if sinking \p I's operands to I's basic block is profitable, because
11173 /// the operands can be folded into a target instruction, e.g.
11174 /// shufflevectors extracts and/or sext/zext can be folded into (u,s)subl(2).
11175 bool AArch64TargetLowering::shouldSinkOperands(
11176     Instruction *I, SmallVectorImpl<Use *> &Ops) const {
11177   if (!I->getType()->isVectorTy())
11178     return false;
11179 
11180   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
11181     switch (II->getIntrinsicID()) {
11182     case Intrinsic::aarch64_neon_umull:
11183       if (!areExtractShuffleVectors(II->getOperand(0), II->getOperand(1)))
11184         return false;
11185       Ops.push_back(&II->getOperandUse(0));
11186       Ops.push_back(&II->getOperandUse(1));
11187       return true;
11188 
11189     case Intrinsic::aarch64_neon_pmull64:
11190       if (!areOperandsOfVmullHighP64(II->getArgOperand(0),
11191                                      II->getArgOperand(1)))
11192         return false;
11193       Ops.push_back(&II->getArgOperandUse(0));
11194       Ops.push_back(&II->getArgOperandUse(1));
11195       return true;
11196 
11197     default:
11198       return false;
11199     }
11200   }
11201 
11202   switch (I->getOpcode()) {
11203   case Instruction::Sub:
11204   case Instruction::Add: {
11205     if (!areExtractExts(I->getOperand(0), I->getOperand(1)))
11206       return false;
11207 
11208     // If the exts' operands extract either the lower or upper elements, we
11209     // can sink them too.
11210     auto Ext1 = cast<Instruction>(I->getOperand(0));
11211     auto Ext2 = cast<Instruction>(I->getOperand(1));
11212     if (areExtractShuffleVectors(Ext1, Ext2)) {
11213       Ops.push_back(&Ext1->getOperandUse(0));
11214       Ops.push_back(&Ext2->getOperandUse(0));
11215     }
11216 
11217     Ops.push_back(&I->getOperandUse(0));
11218     Ops.push_back(&I->getOperandUse(1));
11219 
11220     return true;
11221   }
11222   case Instruction::Mul: {
11223     bool IsProfitable = false;
11224     for (auto &Op : I->operands()) {
11225       // Make sure we are not already sinking this operand
11226       if (any_of(Ops, [&](Use *U) { return U->get() == Op; }))
11227         continue;
11228 
11229       ShuffleVectorInst *Shuffle = dyn_cast<ShuffleVectorInst>(Op);
11230       if (!Shuffle || !Shuffle->isZeroEltSplat())
11231         continue;
11232 
11233       Value *ShuffleOperand = Shuffle->getOperand(0);
11234       InsertElementInst *Insert = dyn_cast<InsertElementInst>(ShuffleOperand);
11235       if (!Insert)
11236         continue;
11237 
11238       Instruction *OperandInstr = dyn_cast<Instruction>(Insert->getOperand(1));
11239       if (!OperandInstr)
11240         continue;
11241 
11242       ConstantInt *ElementConstant =
11243           dyn_cast<ConstantInt>(Insert->getOperand(2));
11244       // Check that the insertelement is inserting into element 0
11245       if (!ElementConstant || ElementConstant->getZExtValue() != 0)
11246         continue;
11247 
11248       unsigned Opcode = OperandInstr->getOpcode();
11249       if (Opcode != Instruction::SExt && Opcode != Instruction::ZExt)
11250         continue;
11251 
11252       Ops.push_back(&Shuffle->getOperandUse(0));
11253       Ops.push_back(&Op);
11254       IsProfitable = true;
11255     }
11256 
11257     return IsProfitable;
11258   }
11259   default:
11260     return false;
11261   }
11262   return false;
11263 }
11264 
11265 bool AArch64TargetLowering::hasPairedLoad(EVT LoadedType,
11266                                           Align &RequiredAligment) const {
11267   if (!LoadedType.isSimple() ||
11268       (!LoadedType.isInteger() && !LoadedType.isFloatingPoint()))
11269     return false;
11270   // Cyclone supports unaligned accesses.
11271   RequiredAligment = Align(1);
11272   unsigned NumBits = LoadedType.getSizeInBits();
11273   return NumBits == 32 || NumBits == 64;
11274 }
11275 
11276 /// A helper function for determining the number of interleaved accesses we
11277 /// will generate when lowering accesses of the given type.
11278 unsigned
11279 AArch64TargetLowering::getNumInterleavedAccesses(VectorType *VecTy,
11280                                                  const DataLayout &DL) const {
11281   return (DL.getTypeSizeInBits(VecTy) + 127) / 128;
11282 }
11283 
11284 MachineMemOperand::Flags
11285 AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
11286   if (Subtarget->getProcFamily() == AArch64Subtarget::Falkor &&
11287       I.getMetadata(FALKOR_STRIDED_ACCESS_MD) != nullptr)
11288     return MOStridedAccess;
11289   return MachineMemOperand::MONone;
11290 }
11291 
11292 bool AArch64TargetLowering::isLegalInterleavedAccessType(
11293     VectorType *VecTy, const DataLayout &DL) const {
11294 
11295   unsigned VecSize = DL.getTypeSizeInBits(VecTy);
11296   unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
11297 
11298   // Ensure the number of vector elements is greater than 1.
11299   if (cast<FixedVectorType>(VecTy)->getNumElements() < 2)
11300     return false;
11301 
11302   // Ensure the element type is legal.
11303   if (ElSize != 8 && ElSize != 16 && ElSize != 32 && ElSize != 64)
11304     return false;
11305 
11306   // Ensure the total vector size is 64 or a multiple of 128. Types larger than
11307   // 128 will be split into multiple interleaved accesses.
11308   return VecSize == 64 || VecSize % 128 == 0;
11309 }
11310 
11311 /// Lower an interleaved load into a ldN intrinsic.
11312 ///
11313 /// E.g. Lower an interleaved load (Factor = 2):
11314 ///        %wide.vec = load <8 x i32>, <8 x i32>* %ptr
11315 ///        %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6>  ; Extract even elements
11316 ///        %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7>  ; Extract odd elements
11317 ///
11318 ///      Into:
11319 ///        %ld2 = { <4 x i32>, <4 x i32> } call llvm.aarch64.neon.ld2(%ptr)
11320 ///        %vec0 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 0
11321 ///        %vec1 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 1
11322 bool AArch64TargetLowering::lowerInterleavedLoad(
11323     LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles,
11324     ArrayRef<unsigned> Indices, unsigned Factor) const {
11325   assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
11326          "Invalid interleave factor");
11327   assert(!Shuffles.empty() && "Empty shufflevector input");
11328   assert(Shuffles.size() == Indices.size() &&
11329          "Unmatched number of shufflevectors and indices");
11330 
11331   const DataLayout &DL = LI->getModule()->getDataLayout();
11332 
11333   VectorType *VTy = Shuffles[0]->getType();
11334 
11335   // Skip if we do not have NEON and skip illegal vector types. We can
11336   // "legalize" wide vector types into multiple interleaved accesses as long as
11337   // the vector types are divisible by 128.
11338   if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(VTy, DL))
11339     return false;
11340 
11341   unsigned NumLoads = getNumInterleavedAccesses(VTy, DL);
11342 
11343   auto *FVTy = cast<FixedVectorType>(VTy);
11344 
11345   // A pointer vector can not be the return type of the ldN intrinsics. Need to
11346   // load integer vectors first and then convert to pointer vectors.
11347   Type *EltTy = FVTy->getElementType();
11348   if (EltTy->isPointerTy())
11349     FVTy =
11350         FixedVectorType::get(DL.getIntPtrType(EltTy), FVTy->getNumElements());
11351 
11352   IRBuilder<> Builder(LI);
11353 
11354   // The base address of the load.
11355   Value *BaseAddr = LI->getPointerOperand();
11356 
11357   if (NumLoads > 1) {
11358     // If we're going to generate more than one load, reset the sub-vector type
11359     // to something legal.
11360     FVTy = FixedVectorType::get(FVTy->getElementType(),
11361                                 FVTy->getNumElements() / NumLoads);
11362 
11363     // We will compute the pointer operand of each load from the original base
11364     // address using GEPs. Cast the base address to a pointer to the scalar
11365     // element type.
11366     BaseAddr = Builder.CreateBitCast(
11367         BaseAddr,
11368         FVTy->getElementType()->getPointerTo(LI->getPointerAddressSpace()));
11369   }
11370 
11371   Type *PtrTy = FVTy->getPointerTo(LI->getPointerAddressSpace());
11372   Type *Tys[2] = {FVTy, PtrTy};
11373   static const Intrinsic::ID LoadInts[3] = {Intrinsic::aarch64_neon_ld2,
11374                                             Intrinsic::aarch64_neon_ld3,
11375                                             Intrinsic::aarch64_neon_ld4};
11376   Function *LdNFunc =
11377       Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys);
11378 
11379   // Holds sub-vectors extracted from the load intrinsic return values. The
11380   // sub-vectors are associated with the shufflevector instructions they will
11381   // replace.
11382   DenseMap<ShuffleVectorInst *, SmallVector<Value *, 4>> SubVecs;
11383 
11384   for (unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
11385 
11386     // If we're generating more than one load, compute the base address of
11387     // subsequent loads as an offset from the previous.
11388     if (LoadCount > 0)
11389       BaseAddr = Builder.CreateConstGEP1_32(FVTy->getElementType(), BaseAddr,
11390                                             FVTy->getNumElements() * Factor);
11391 
11392     CallInst *LdN = Builder.CreateCall(
11393         LdNFunc, Builder.CreateBitCast(BaseAddr, PtrTy), "ldN");
11394 
11395     // Extract and store the sub-vectors returned by the load intrinsic.
11396     for (unsigned i = 0; i < Shuffles.size(); i++) {
11397       ShuffleVectorInst *SVI = Shuffles[i];
11398       unsigned Index = Indices[i];
11399 
11400       Value *SubVec = Builder.CreateExtractValue(LdN, Index);
11401 
11402       // Convert the integer vector to pointer vector if the element is pointer.
11403       if (EltTy->isPointerTy())
11404         SubVec = Builder.CreateIntToPtr(
11405             SubVec, FixedVectorType::get(SVI->getType()->getElementType(),
11406                                          FVTy->getNumElements()));
11407       SubVecs[SVI].push_back(SubVec);
11408     }
11409   }
11410 
11411   // Replace uses of the shufflevector instructions with the sub-vectors
11412   // returned by the load intrinsic. If a shufflevector instruction is
11413   // associated with more than one sub-vector, those sub-vectors will be
11414   // concatenated into a single wide vector.
11415   for (ShuffleVectorInst *SVI : Shuffles) {
11416     auto &SubVec = SubVecs[SVI];
11417     auto *WideVec =
11418         SubVec.size() > 1 ? concatenateVectors(Builder, SubVec) : SubVec[0];
11419     SVI->replaceAllUsesWith(WideVec);
11420   }
11421 
11422   return true;
11423 }
11424 
11425 /// Lower an interleaved store into a stN intrinsic.
11426 ///
11427 /// E.g. Lower an interleaved store (Factor = 3):
11428 ///        %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
11429 ///                 <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
11430 ///        store <12 x i32> %i.vec, <12 x i32>* %ptr
11431 ///
11432 ///      Into:
11433 ///        %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
11434 ///        %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
11435 ///        %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
11436 ///        call void llvm.aarch64.neon.st3(%sub.v0, %sub.v1, %sub.v2, %ptr)
11437 ///
11438 /// Note that the new shufflevectors will be removed and we'll only generate one
11439 /// st3 instruction in CodeGen.
11440 ///
11441 /// Example for a more general valid mask (Factor 3). Lower:
11442 ///        %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1,
11443 ///                 <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
11444 ///        store <12 x i32> %i.vec, <12 x i32>* %ptr
11445 ///
11446 ///      Into:
11447 ///        %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7>
11448 ///        %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35>
11449 ///        %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19>
11450 ///        call void llvm.aarch64.neon.st3(%sub.v0, %sub.v1, %sub.v2, %ptr)
11451 bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
11452                                                   ShuffleVectorInst *SVI,
11453                                                   unsigned Factor) const {
11454   assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
11455          "Invalid interleave factor");
11456 
11457   auto *VecTy = cast<FixedVectorType>(SVI->getType());
11458   assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store");
11459 
11460   unsigned LaneLen = VecTy->getNumElements() / Factor;
11461   Type *EltTy = VecTy->getElementType();
11462   auto *SubVecTy = FixedVectorType::get(EltTy, LaneLen);
11463 
11464   const DataLayout &DL = SI->getModule()->getDataLayout();
11465 
11466   // Skip if we do not have NEON and skip illegal vector types. We can
11467   // "legalize" wide vector types into multiple interleaved accesses as long as
11468   // the vector types are divisible by 128.
11469   if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(SubVecTy, DL))
11470     return false;
11471 
11472   unsigned NumStores = getNumInterleavedAccesses(SubVecTy, DL);
11473 
11474   Value *Op0 = SVI->getOperand(0);
11475   Value *Op1 = SVI->getOperand(1);
11476   IRBuilder<> Builder(SI);
11477 
11478   // StN intrinsics don't support pointer vectors as arguments. Convert pointer
11479   // vectors to integer vectors.
11480   if (EltTy->isPointerTy()) {
11481     Type *IntTy = DL.getIntPtrType(EltTy);
11482     unsigned NumOpElts =
11483         cast<FixedVectorType>(Op0->getType())->getNumElements();
11484 
11485     // Convert to the corresponding integer vector.
11486     auto *IntVecTy = FixedVectorType::get(IntTy, NumOpElts);
11487     Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
11488     Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
11489 
11490     SubVecTy = FixedVectorType::get(IntTy, LaneLen);
11491   }
11492 
11493   // The base address of the store.
11494   Value *BaseAddr = SI->getPointerOperand();
11495 
11496   if (NumStores > 1) {
11497     // If we're going to generate more than one store, reset the lane length
11498     // and sub-vector type to something legal.
11499     LaneLen /= NumStores;
11500     SubVecTy = FixedVectorType::get(SubVecTy->getElementType(), LaneLen);
11501 
11502     // We will compute the pointer operand of each store from the original base
11503     // address using GEPs. Cast the base address to a pointer to the scalar
11504     // element type.
11505     BaseAddr = Builder.CreateBitCast(
11506         BaseAddr,
11507         SubVecTy->getElementType()->getPointerTo(SI->getPointerAddressSpace()));
11508   }
11509 
11510   auto Mask = SVI->getShuffleMask();
11511 
11512   Type *PtrTy = SubVecTy->getPointerTo(SI->getPointerAddressSpace());
11513   Type *Tys[2] = {SubVecTy, PtrTy};
11514   static const Intrinsic::ID StoreInts[3] = {Intrinsic::aarch64_neon_st2,
11515                                              Intrinsic::aarch64_neon_st3,
11516                                              Intrinsic::aarch64_neon_st4};
11517   Function *StNFunc =
11518       Intrinsic::getDeclaration(SI->getModule(), StoreInts[Factor - 2], Tys);
11519 
11520   for (unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
11521 
11522     SmallVector<Value *, 5> Ops;
11523 
11524     // Split the shufflevector operands into sub vectors for the new stN call.
11525     for (unsigned i = 0; i < Factor; i++) {
11526       unsigned IdxI = StoreCount * LaneLen * Factor + i;
11527       if (Mask[IdxI] >= 0) {
11528         Ops.push_back(Builder.CreateShuffleVector(
11529             Op0, Op1, createSequentialMask(Mask[IdxI], LaneLen, 0)));
11530       } else {
11531         unsigned StartMask = 0;
11532         for (unsigned j = 1; j < LaneLen; j++) {
11533           unsigned IdxJ = StoreCount * LaneLen * Factor + j;
11534           if (Mask[IdxJ * Factor + IdxI] >= 0) {
11535             StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
11536             break;
11537           }
11538         }
11539         // Note: Filling undef gaps with random elements is ok, since
11540         // those elements were being written anyway (with undefs).
11541         // In the case of all undefs we're defaulting to using elems from 0
11542         // Note: StartMask cannot be negative, it's checked in
11543         // isReInterleaveMask
11544         Ops.push_back(Builder.CreateShuffleVector(
11545             Op0, Op1, createSequentialMask(StartMask, LaneLen, 0)));
11546       }
11547     }
11548 
11549     // If we generating more than one store, we compute the base address of
11550     // subsequent stores as an offset from the previous.
11551     if (StoreCount > 0)
11552       BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getElementType(),
11553                                             BaseAddr, LaneLen * Factor);
11554 
11555     Ops.push_back(Builder.CreateBitCast(BaseAddr, PtrTy));
11556     Builder.CreateCall(StNFunc, Ops);
11557   }
11558   return true;
11559 }
11560 
11561 // Lower an SVE structured load intrinsic returning a tuple type to target
11562 // specific intrinsic taking the same input but returning a multi-result value
11563 // of the split tuple type.
11564 //
11565 // E.g. Lowering an LD3:
11566 //
11567 //  call <vscale x 12 x i32> @llvm.aarch64.sve.ld3.nxv12i32(
11568 //                                                    <vscale x 4 x i1> %pred,
11569 //                                                    <vscale x 4 x i32>* %addr)
11570 //
11571 //  Output DAG:
11572 //
11573 //    t0: ch = EntryToken
11574 //        t2: nxv4i1,ch = CopyFromReg t0, Register:nxv4i1 %0
11575 //        t4: i64,ch = CopyFromReg t0, Register:i64 %1
11576 //    t5: nxv4i32,nxv4i32,nxv4i32,ch = AArch64ISD::SVE_LD3 t0, t2, t4
11577 //    t6: nxv12i32 = concat_vectors t5, t5:1, t5:2
11578 //
11579 // This is called pre-legalization to avoid widening/splitting issues with
11580 // non-power-of-2 tuple types used for LD3, such as nxv12i32.
11581 SDValue AArch64TargetLowering::LowerSVEStructLoad(unsigned Intrinsic,
11582                                                   ArrayRef<SDValue> LoadOps,
11583                                                   EVT VT, SelectionDAG &DAG,
11584                                                   const SDLoc &DL) const {
11585   assert(VT.isScalableVector() && "Can only lower scalable vectors");
11586 
11587   unsigned N, Opcode;
11588   static std::map<unsigned, std::pair<unsigned, unsigned>> IntrinsicMap = {
11589       {Intrinsic::aarch64_sve_ld2, {2, AArch64ISD::SVE_LD2_MERGE_ZERO}},
11590       {Intrinsic::aarch64_sve_ld3, {3, AArch64ISD::SVE_LD3_MERGE_ZERO}},
11591       {Intrinsic::aarch64_sve_ld4, {4, AArch64ISD::SVE_LD4_MERGE_ZERO}}};
11592 
11593   std::tie(N, Opcode) = IntrinsicMap[Intrinsic];
11594   assert(VT.getVectorElementCount().getKnownMinValue() % N == 0 &&
11595          "invalid tuple vector type!");
11596 
11597   EVT SplitVT =
11598       EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
11599                        VT.getVectorElementCount().divideCoefficientBy(N));
11600   assert(isTypeLegal(SplitVT));
11601 
11602   SmallVector<EVT, 5> VTs(N, SplitVT);
11603   VTs.push_back(MVT::Other); // Chain
11604   SDVTList NodeTys = DAG.getVTList(VTs);
11605 
11606   SDValue PseudoLoad = DAG.getNode(Opcode, DL, NodeTys, LoadOps);
11607   SmallVector<SDValue, 4> PseudoLoadOps;
11608   for (unsigned I = 0; I < N; ++I)
11609     PseudoLoadOps.push_back(SDValue(PseudoLoad.getNode(), I));
11610   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, PseudoLoadOps);
11611 }
11612 
11613 EVT AArch64TargetLowering::getOptimalMemOpType(
11614     const MemOp &Op, const AttributeList &FuncAttributes) const {
11615   bool CanImplicitFloat =
11616       !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat);
11617   bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat;
11618   bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat;
11619   // Only use AdvSIMD to implement memset of 32-byte and above. It would have
11620   // taken one instruction to materialize the v2i64 zero and one store (with
11621   // restrictive addressing mode). Just do i64 stores.
11622   bool IsSmallMemset = Op.isMemset() && Op.size() < 32;
11623   auto AlignmentIsAcceptable = [&](EVT VT, Align AlignCheck) {
11624     if (Op.isAligned(AlignCheck))
11625       return true;
11626     bool Fast;
11627     return allowsMisalignedMemoryAccesses(VT, 0, Align(1),
11628                                           MachineMemOperand::MONone, &Fast) &&
11629            Fast;
11630   };
11631 
11632   if (CanUseNEON && Op.isMemset() && !IsSmallMemset &&
11633       AlignmentIsAcceptable(MVT::v2i64, Align(16)))
11634     return MVT::v2i64;
11635   if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(MVT::f128, Align(16)))
11636     return MVT::f128;
11637   if (Op.size() >= 8 && AlignmentIsAcceptable(MVT::i64, Align(8)))
11638     return MVT::i64;
11639   if (Op.size() >= 4 && AlignmentIsAcceptable(MVT::i32, Align(4)))
11640     return MVT::i32;
11641   return MVT::Other;
11642 }
11643 
11644 LLT AArch64TargetLowering::getOptimalMemOpLLT(
11645     const MemOp &Op, const AttributeList &FuncAttributes) const {
11646   bool CanImplicitFloat =
11647       !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat);
11648   bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat;
11649   bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat;
11650   // Only use AdvSIMD to implement memset of 32-byte and above. It would have
11651   // taken one instruction to materialize the v2i64 zero and one store (with
11652   // restrictive addressing mode). Just do i64 stores.
11653   bool IsSmallMemset = Op.isMemset() && Op.size() < 32;
11654   auto AlignmentIsAcceptable = [&](EVT VT, Align AlignCheck) {
11655     if (Op.isAligned(AlignCheck))
11656       return true;
11657     bool Fast;
11658     return allowsMisalignedMemoryAccesses(VT, 0, Align(1),
11659                                           MachineMemOperand::MONone, &Fast) &&
11660            Fast;
11661   };
11662 
11663   if (CanUseNEON && Op.isMemset() && !IsSmallMemset &&
11664       AlignmentIsAcceptable(MVT::v2i64, Align(16)))
11665     return LLT::vector(2, 64);
11666   if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(MVT::f128, Align(16)))
11667     return LLT::scalar(128);
11668   if (Op.size() >= 8 && AlignmentIsAcceptable(MVT::i64, Align(8)))
11669     return LLT::scalar(64);
11670   if (Op.size() >= 4 && AlignmentIsAcceptable(MVT::i32, Align(4)))
11671     return LLT::scalar(32);
11672   return LLT();
11673 }
11674 
11675 // 12-bit optionally shifted immediates are legal for adds.
11676 bool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const {
11677   if (Immed == std::numeric_limits<int64_t>::min()) {
11678     LLVM_DEBUG(dbgs() << "Illegal add imm " << Immed
11679                       << ": avoid UB for INT64_MIN\n");
11680     return false;
11681   }
11682   // Same encoding for add/sub, just flip the sign.
11683   Immed = std::abs(Immed);
11684   bool IsLegal = ((Immed >> 12) == 0 ||
11685                   ((Immed & 0xfff) == 0 && Immed >> 24 == 0));
11686   LLVM_DEBUG(dbgs() << "Is " << Immed
11687                     << " legal add imm: " << (IsLegal ? "yes" : "no") << "\n");
11688   return IsLegal;
11689 }
11690 
11691 // Integer comparisons are implemented with ADDS/SUBS, so the range of valid
11692 // immediates is the same as for an add or a sub.
11693 bool AArch64TargetLowering::isLegalICmpImmediate(int64_t Immed) const {
11694   return isLegalAddImmediate(Immed);
11695 }
11696 
11697 /// isLegalAddressingMode - Return true if the addressing mode represented
11698 /// by AM is legal for this target, for a load/store of the specified type.
11699 bool AArch64TargetLowering::isLegalAddressingMode(const DataLayout &DL,
11700                                                   const AddrMode &AM, Type *Ty,
11701                                                   unsigned AS, Instruction *I) const {
11702   // AArch64 has five basic addressing modes:
11703   //  reg
11704   //  reg + 9-bit signed offset
11705   //  reg + SIZE_IN_BYTES * 12-bit unsigned offset
11706   //  reg1 + reg2
11707   //  reg + SIZE_IN_BYTES * reg
11708 
11709   // No global is ever allowed as a base.
11710   if (AM.BaseGV)
11711     return false;
11712 
11713   // No reg+reg+imm addressing.
11714   if (AM.HasBaseReg && AM.BaseOffs && AM.Scale)
11715     return false;
11716 
11717   // FIXME: Update this method to support scalable addressing modes.
11718   if (isa<ScalableVectorType>(Ty))
11719     return AM.HasBaseReg && !AM.BaseOffs && !AM.Scale;
11720 
11721   // check reg + imm case:
11722   // i.e., reg + 0, reg + imm9, reg + SIZE_IN_BYTES * uimm12
11723   uint64_t NumBytes = 0;
11724   if (Ty->isSized()) {
11725     uint64_t NumBits = DL.getTypeSizeInBits(Ty);
11726     NumBytes = NumBits / 8;
11727     if (!isPowerOf2_64(NumBits))
11728       NumBytes = 0;
11729   }
11730 
11731   if (!AM.Scale) {
11732     int64_t Offset = AM.BaseOffs;
11733 
11734     // 9-bit signed offset
11735     if (isInt<9>(Offset))
11736       return true;
11737 
11738     // 12-bit unsigned offset
11739     unsigned shift = Log2_64(NumBytes);
11740     if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 &&
11741         // Must be a multiple of NumBytes (NumBytes is a power of 2)
11742         (Offset >> shift) << shift == Offset)
11743       return true;
11744     return false;
11745   }
11746 
11747   // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2
11748 
11749   return AM.Scale == 1 || (AM.Scale > 0 && (uint64_t)AM.Scale == NumBytes);
11750 }
11751 
11752 bool AArch64TargetLowering::shouldConsiderGEPOffsetSplit() const {
11753   // Consider splitting large offset of struct or array.
11754   return true;
11755 }
11756 
11757 int AArch64TargetLowering::getScalingFactorCost(const DataLayout &DL,
11758                                                 const AddrMode &AM, Type *Ty,
11759                                                 unsigned AS) const {
11760   // Scaling factors are not free at all.
11761   // Operands                     | Rt Latency
11762   // -------------------------------------------
11763   // Rt, [Xn, Xm]                 | 4
11764   // -------------------------------------------
11765   // Rt, [Xn, Xm, lsl #imm]       | Rn: 4 Rm: 5
11766   // Rt, [Xn, Wm, <extend> #imm]  |
11767   if (isLegalAddressingMode(DL, AM, Ty, AS))
11768     // Scale represents reg2 * scale, thus account for 1 if
11769     // it is not equal to 0 or 1.
11770     return AM.Scale != 0 && AM.Scale != 1;
11771   return -1;
11772 }
11773 
11774 bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(
11775     const MachineFunction &MF, EVT VT) const {
11776   VT = VT.getScalarType();
11777 
11778   if (!VT.isSimple())
11779     return false;
11780 
11781   switch (VT.getSimpleVT().SimpleTy) {
11782   case MVT::f16:
11783     return Subtarget->hasFullFP16();
11784   case MVT::f32:
11785   case MVT::f64:
11786     return true;
11787   default:
11788     break;
11789   }
11790 
11791   return false;
11792 }
11793 
11794 bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
11795                                                        Type *Ty) const {
11796   switch (Ty->getScalarType()->getTypeID()) {
11797   case Type::FloatTyID:
11798   case Type::DoubleTyID:
11799     return true;
11800   default:
11801     return false;
11802   }
11803 }
11804 
11805 bool AArch64TargetLowering::generateFMAsInMachineCombiner(
11806     EVT VT, CodeGenOpt::Level OptLevel) const {
11807   return (OptLevel >= CodeGenOpt::Aggressive) && !VT.isScalableVector();
11808 }
11809 
11810 const MCPhysReg *
11811 AArch64TargetLowering::getScratchRegisters(CallingConv::ID) const {
11812   // LR is a callee-save register, but we must treat it as clobbered by any call
11813   // site. Hence we include LR in the scratch registers, which are in turn added
11814   // as implicit-defs for stackmaps and patchpoints.
11815   static const MCPhysReg ScratchRegs[] = {
11816     AArch64::X16, AArch64::X17, AArch64::LR, 0
11817   };
11818   return ScratchRegs;
11819 }
11820 
11821 bool
11822 AArch64TargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
11823                                                      CombineLevel Level) const {
11824   N = N->getOperand(0).getNode();
11825   EVT VT = N->getValueType(0);
11826     // If N is unsigned bit extraction: ((x >> C) & mask), then do not combine
11827     // it with shift to let it be lowered to UBFX.
11828   if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) &&
11829       isa<ConstantSDNode>(N->getOperand(1))) {
11830     uint64_t TruncMask = N->getConstantOperandVal(1);
11831     if (isMask_64(TruncMask) &&
11832       N->getOperand(0).getOpcode() == ISD::SRL &&
11833       isa<ConstantSDNode>(N->getOperand(0)->getOperand(1)))
11834       return false;
11835   }
11836   return true;
11837 }
11838 
11839 bool AArch64TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
11840                                                               Type *Ty) const {
11841   assert(Ty->isIntegerTy());
11842 
11843   unsigned BitSize = Ty->getPrimitiveSizeInBits();
11844   if (BitSize == 0)
11845     return false;
11846 
11847   int64_t Val = Imm.getSExtValue();
11848   if (Val == 0 || AArch64_AM::isLogicalImmediate(Val, BitSize))
11849     return true;
11850 
11851   if ((int64_t)Val < 0)
11852     Val = ~Val;
11853   if (BitSize == 32)
11854     Val &= (1LL << 32) - 1;
11855 
11856   unsigned LZ = countLeadingZeros((uint64_t)Val);
11857   unsigned Shift = (63 - LZ) / 16;
11858   // MOVZ is free so return true for one or fewer MOVK.
11859   return Shift < 3;
11860 }
11861 
11862 bool AArch64TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
11863                                                     unsigned Index) const {
11864   if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
11865     return false;
11866 
11867   return (Index == 0 || Index == ResVT.getVectorNumElements());
11868 }
11869 
11870 /// Turn vector tests of the signbit in the form of:
11871 ///   xor (sra X, elt_size(X)-1), -1
11872 /// into:
11873 ///   cmge X, X, #0
11874 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
11875                                          const AArch64Subtarget *Subtarget) {
11876   EVT VT = N->getValueType(0);
11877   if (!Subtarget->hasNEON() || !VT.isVector())
11878     return SDValue();
11879 
11880   // There must be a shift right algebraic before the xor, and the xor must be a
11881   // 'not' operation.
11882   SDValue Shift = N->getOperand(0);
11883   SDValue Ones = N->getOperand(1);
11884   if (Shift.getOpcode() != AArch64ISD::VASHR || !Shift.hasOneUse() ||
11885       !ISD::isBuildVectorAllOnes(Ones.getNode()))
11886     return SDValue();
11887 
11888   // The shift should be smearing the sign bit across each vector element.
11889   auto *ShiftAmt = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
11890   EVT ShiftEltTy = Shift.getValueType().getVectorElementType();
11891   if (!ShiftAmt || ShiftAmt->getZExtValue() != ShiftEltTy.getSizeInBits() - 1)
11892     return SDValue();
11893 
11894   return DAG.getNode(AArch64ISD::CMGEz, SDLoc(N), VT, Shift.getOperand(0));
11895 }
11896 
11897 // Turn a v8i8/v16i8 extended vecreduce into a udot/sdot and vecreduce
11898 //   vecreduce.add(ext(A)) to vecreduce.add(DOT(zero, A, one))
11899 //   vecreduce.add(mul(ext(A), ext(B))) to vecreduce.add(DOT(zero, A, B))
11900 static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
11901                                           const AArch64Subtarget *ST) {
11902   SDValue Op0 = N->getOperand(0);
11903   if (!ST->hasDotProd() || N->getValueType(0) != MVT::i32 ||
11904       Op0.getValueType().getVectorElementType() != MVT::i32)
11905     return SDValue();
11906 
11907   unsigned ExtOpcode = Op0.getOpcode();
11908   SDValue A = Op0;
11909   SDValue B;
11910   if (ExtOpcode == ISD::MUL) {
11911     A = Op0.getOperand(0);
11912     B = Op0.getOperand(1);
11913     if (A.getOpcode() != B.getOpcode() ||
11914         A.getOperand(0).getValueType() != B.getOperand(0).getValueType())
11915       return SDValue();
11916     ExtOpcode = A.getOpcode();
11917   }
11918   if (ExtOpcode != ISD::ZERO_EXTEND && ExtOpcode != ISD::SIGN_EXTEND)
11919     return SDValue();
11920 
11921   EVT Op0VT = A.getOperand(0).getValueType();
11922   if (Op0VT != MVT::v8i8 && Op0VT != MVT::v16i8)
11923     return SDValue();
11924 
11925   SDLoc DL(Op0);
11926   // For non-mla reductions B can be set to 1. For MLA we take the operand of
11927   // the extend B.
11928   if (!B)
11929     B = DAG.getConstant(1, DL, Op0VT);
11930   else
11931     B = B.getOperand(0);
11932 
11933   SDValue Zeros =
11934       DAG.getConstant(0, DL, Op0VT == MVT::v8i8 ? MVT::v2i32 : MVT::v4i32);
11935   auto DotOpcode =
11936       (ExtOpcode == ISD::ZERO_EXTEND) ? AArch64ISD::UDOT : AArch64ISD::SDOT;
11937   SDValue Dot = DAG.getNode(DotOpcode, DL, Zeros.getValueType(), Zeros,
11938                             A.getOperand(0), B);
11939   return DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0), Dot);
11940 }
11941 
11942 // Given a ABS node, detect the following pattern:
11943 // (ABS (SUB (EXTEND a), (EXTEND b))).
11944 // Generates UABD/SABD instruction.
11945 static SDValue performABSCombine(SDNode *N, SelectionDAG &DAG,
11946                                  TargetLowering::DAGCombinerInfo &DCI,
11947                                  const AArch64Subtarget *Subtarget) {
11948   SDValue AbsOp1 = N->getOperand(0);
11949   SDValue Op0, Op1;
11950 
11951   if (AbsOp1.getOpcode() != ISD::SUB)
11952     return SDValue();
11953 
11954   Op0 = AbsOp1.getOperand(0);
11955   Op1 = AbsOp1.getOperand(1);
11956 
11957   unsigned Opc0 = Op0.getOpcode();
11958   // Check if the operands of the sub are (zero|sign)-extended.
11959   if (Opc0 != Op1.getOpcode() ||
11960       (Opc0 != ISD::ZERO_EXTEND && Opc0 != ISD::SIGN_EXTEND))
11961     return SDValue();
11962 
11963   EVT VectorT1 = Op0.getOperand(0).getValueType();
11964   EVT VectorT2 = Op1.getOperand(0).getValueType();
11965   // Check if vectors are of same type and valid size.
11966   uint64_t Size = VectorT1.getFixedSizeInBits();
11967   if (VectorT1 != VectorT2 || (Size != 64 && Size != 128))
11968     return SDValue();
11969 
11970   // Check if vector element types are valid.
11971   EVT VT1 = VectorT1.getVectorElementType();
11972   if (VT1 != MVT::i8 && VT1 != MVT::i16 && VT1 != MVT::i32)
11973     return SDValue();
11974 
11975   Op0 = Op0.getOperand(0);
11976   Op1 = Op1.getOperand(0);
11977   unsigned ABDOpcode =
11978       (Opc0 == ISD::SIGN_EXTEND) ? AArch64ISD::SABD : AArch64ISD::UABD;
11979   SDValue ABD =
11980       DAG.getNode(ABDOpcode, SDLoc(N), Op0->getValueType(0), Op0, Op1);
11981   return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), ABD);
11982 }
11983 
11984 static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG,
11985                                  TargetLowering::DAGCombinerInfo &DCI,
11986                                  const AArch64Subtarget *Subtarget) {
11987   if (DCI.isBeforeLegalizeOps())
11988     return SDValue();
11989 
11990   return foldVectorXorShiftIntoCmp(N, DAG, Subtarget);
11991 }
11992 
11993 SDValue
11994 AArch64TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
11995                                      SelectionDAG &DAG,
11996                                      SmallVectorImpl<SDNode *> &Created) const {
11997   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
11998   if (isIntDivCheap(N->getValueType(0), Attr))
11999     return SDValue(N,0); // Lower SDIV as SDIV
12000 
12001   // fold (sdiv X, pow2)
12002   EVT VT = N->getValueType(0);
12003   if ((VT != MVT::i32 && VT != MVT::i64) ||
12004       !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2()))
12005     return SDValue();
12006 
12007   SDLoc DL(N);
12008   SDValue N0 = N->getOperand(0);
12009   unsigned Lg2 = Divisor.countTrailingZeros();
12010   SDValue Zero = DAG.getConstant(0, DL, VT);
12011   SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
12012 
12013   // Add (N0 < 0) ? Pow2 - 1 : 0;
12014   SDValue CCVal;
12015   SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETLT, CCVal, DAG, DL);
12016   SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
12017   SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp);
12018 
12019   Created.push_back(Cmp.getNode());
12020   Created.push_back(Add.getNode());
12021   Created.push_back(CSel.getNode());
12022 
12023   // Divide by pow2.
12024   SDValue SRA =
12025       DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64));
12026 
12027   // If we're dividing by a positive value, we're done.  Otherwise, we must
12028   // negate the result.
12029   if (Divisor.isNonNegative())
12030     return SRA;
12031 
12032   Created.push_back(SRA.getNode());
12033   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
12034 }
12035 
12036 static bool IsSVECntIntrinsic(SDValue S) {
12037   switch(getIntrinsicID(S.getNode())) {
12038   default:
12039     break;
12040   case Intrinsic::aarch64_sve_cntb:
12041   case Intrinsic::aarch64_sve_cnth:
12042   case Intrinsic::aarch64_sve_cntw:
12043   case Intrinsic::aarch64_sve_cntd:
12044     return true;
12045   }
12046   return false;
12047 }
12048 
12049 /// Calculates what the pre-extend type is, based on the extension
12050 /// operation node provided by \p Extend.
12051 ///
12052 /// In the case that \p Extend is a SIGN_EXTEND or a ZERO_EXTEND, the
12053 /// pre-extend type is pulled directly from the operand, while other extend
12054 /// operations need a bit more inspection to get this information.
12055 ///
12056 /// \param Extend The SDNode from the DAG that represents the extend operation
12057 /// \param DAG The SelectionDAG hosting the \p Extend node
12058 ///
12059 /// \returns The type representing the \p Extend source type, or \p MVT::Other
12060 /// if no valid type can be determined
12061 static EVT calculatePreExtendType(SDValue Extend, SelectionDAG &DAG) {
12062   switch (Extend.getOpcode()) {
12063   case ISD::SIGN_EXTEND:
12064   case ISD::ZERO_EXTEND:
12065     return Extend.getOperand(0).getValueType();
12066   case ISD::AssertSext:
12067   case ISD::AssertZext:
12068   case ISD::SIGN_EXTEND_INREG: {
12069     VTSDNode *TypeNode = dyn_cast<VTSDNode>(Extend.getOperand(1));
12070     if (!TypeNode)
12071       return MVT::Other;
12072     return TypeNode->getVT();
12073   }
12074   case ISD::AND: {
12075     ConstantSDNode *Constant =
12076         dyn_cast<ConstantSDNode>(Extend.getOperand(1).getNode());
12077     if (!Constant)
12078       return MVT::Other;
12079 
12080     uint32_t Mask = Constant->getZExtValue();
12081 
12082     if (Mask == UCHAR_MAX)
12083       return MVT::i8;
12084     else if (Mask == USHRT_MAX)
12085       return MVT::i16;
12086     else if (Mask == UINT_MAX)
12087       return MVT::i32;
12088 
12089     return MVT::Other;
12090   }
12091   default:
12092     return MVT::Other;
12093   }
12094 
12095   llvm_unreachable("Code path unhandled in calculatePreExtendType!");
12096 }
12097 
12098 /// Combines a dup(sext/zext) node pattern into sext/zext(dup)
12099 /// making use of the vector SExt/ZExt rather than the scalar SExt/ZExt
12100 static SDValue performCommonVectorExtendCombine(SDValue VectorShuffle,
12101                                                 SelectionDAG &DAG) {
12102 
12103   ShuffleVectorSDNode *ShuffleNode =
12104       dyn_cast<ShuffleVectorSDNode>(VectorShuffle.getNode());
12105   if (!ShuffleNode)
12106     return SDValue();
12107 
12108   // Ensuring the mask is zero before continuing
12109   if (!ShuffleNode->isSplat() || ShuffleNode->getSplatIndex() != 0)
12110     return SDValue();
12111 
12112   SDValue InsertVectorElt = VectorShuffle.getOperand(0);
12113 
12114   if (InsertVectorElt.getOpcode() != ISD::INSERT_VECTOR_ELT)
12115     return SDValue();
12116 
12117   SDValue InsertLane = InsertVectorElt.getOperand(2);
12118   ConstantSDNode *Constant = dyn_cast<ConstantSDNode>(InsertLane.getNode());
12119   // Ensures the insert is inserting into lane 0
12120   if (!Constant || Constant->getZExtValue() != 0)
12121     return SDValue();
12122 
12123   SDValue Extend = InsertVectorElt.getOperand(1);
12124   unsigned ExtendOpcode = Extend.getOpcode();
12125 
12126   bool IsSExt = ExtendOpcode == ISD::SIGN_EXTEND ||
12127                 ExtendOpcode == ISD::SIGN_EXTEND_INREG ||
12128                 ExtendOpcode == ISD::AssertSext;
12129   if (!IsSExt && ExtendOpcode != ISD::ZERO_EXTEND &&
12130       ExtendOpcode != ISD::AssertZext && ExtendOpcode != ISD::AND)
12131     return SDValue();
12132 
12133   EVT TargetType = VectorShuffle.getValueType();
12134   EVT PreExtendType = calculatePreExtendType(Extend, DAG);
12135 
12136   if ((TargetType != MVT::v8i16 && TargetType != MVT::v4i32 &&
12137        TargetType != MVT::v2i64) ||
12138       (PreExtendType == MVT::Other))
12139     return SDValue();
12140 
12141   // Restrict valid pre-extend data type
12142   if (PreExtendType != MVT::i8 && PreExtendType != MVT::i16 &&
12143       PreExtendType != MVT::i32)
12144     return SDValue();
12145 
12146   EVT PreExtendVT = TargetType.changeVectorElementType(PreExtendType);
12147 
12148   if (PreExtendVT.getVectorElementCount() != TargetType.getVectorElementCount())
12149     return SDValue();
12150 
12151   if (TargetType.getScalarSizeInBits() != PreExtendVT.getScalarSizeInBits() * 2)
12152     return SDValue();
12153 
12154   SDLoc DL(VectorShuffle);
12155 
12156   SDValue InsertVectorNode = DAG.getNode(
12157       InsertVectorElt.getOpcode(), DL, PreExtendVT, DAG.getUNDEF(PreExtendVT),
12158       DAG.getAnyExtOrTrunc(Extend.getOperand(0), DL, PreExtendType),
12159       DAG.getConstant(0, DL, MVT::i64));
12160 
12161   std::vector<int> ShuffleMask(TargetType.getVectorElementCount().getValue());
12162 
12163   SDValue VectorShuffleNode =
12164       DAG.getVectorShuffle(PreExtendVT, DL, InsertVectorNode,
12165                            DAG.getUNDEF(PreExtendVT), ShuffleMask);
12166 
12167   SDValue ExtendNode = DAG.getNode(IsSExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
12168                                    DL, TargetType, VectorShuffleNode);
12169 
12170   return ExtendNode;
12171 }
12172 
12173 /// Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup))
12174 /// making use of the vector SExt/ZExt rather than the scalar SExt/ZExt
12175 static SDValue performMulVectorExtendCombine(SDNode *Mul, SelectionDAG &DAG) {
12176   // If the value type isn't a vector, none of the operands are going to be dups
12177   if (!Mul->getValueType(0).isVector())
12178     return SDValue();
12179 
12180   SDValue Op0 = performCommonVectorExtendCombine(Mul->getOperand(0), DAG);
12181   SDValue Op1 = performCommonVectorExtendCombine(Mul->getOperand(1), DAG);
12182 
12183   // Neither operands have been changed, don't make any further changes
12184   if (!Op0 && !Op1)
12185     return SDValue();
12186 
12187   SDLoc DL(Mul);
12188   return DAG.getNode(Mul->getOpcode(), DL, Mul->getValueType(0),
12189                      Op0 ? Op0 : Mul->getOperand(0),
12190                      Op1 ? Op1 : Mul->getOperand(1));
12191 }
12192 
12193 static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
12194                                  TargetLowering::DAGCombinerInfo &DCI,
12195                                  const AArch64Subtarget *Subtarget) {
12196 
12197   if (SDValue Ext = performMulVectorExtendCombine(N, DAG))
12198     return Ext;
12199 
12200   if (DCI.isBeforeLegalizeOps())
12201     return SDValue();
12202 
12203   // The below optimizations require a constant RHS.
12204   if (!isa<ConstantSDNode>(N->getOperand(1)))
12205     return SDValue();
12206 
12207   SDValue N0 = N->getOperand(0);
12208   ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(1));
12209   const APInt &ConstValue = C->getAPIntValue();
12210 
12211   // Allow the scaling to be folded into the `cnt` instruction by preventing
12212   // the scaling to be obscured here. This makes it easier to pattern match.
12213   if (IsSVECntIntrinsic(N0) ||
12214      (N0->getOpcode() == ISD::TRUNCATE &&
12215       (IsSVECntIntrinsic(N0->getOperand(0)))))
12216        if (ConstValue.sge(1) && ConstValue.sle(16))
12217          return SDValue();
12218 
12219   // Multiplication of a power of two plus/minus one can be done more
12220   // cheaply as as shift+add/sub. For now, this is true unilaterally. If
12221   // future CPUs have a cheaper MADD instruction, this may need to be
12222   // gated on a subtarget feature. For Cyclone, 32-bit MADD is 4 cycles and
12223   // 64-bit is 5 cycles, so this is always a win.
12224   // More aggressively, some multiplications N0 * C can be lowered to
12225   // shift+add+shift if the constant C = A * B where A = 2^N + 1 and B = 2^M,
12226   // e.g. 6=3*2=(2+1)*2.
12227   // TODO: consider lowering more cases, e.g. C = 14, -6, -14 or even 45
12228   // which equals to (1+2)*16-(1+2).
12229   // TrailingZeroes is used to test if the mul can be lowered to
12230   // shift+add+shift.
12231   unsigned TrailingZeroes = ConstValue.countTrailingZeros();
12232   if (TrailingZeroes) {
12233     // Conservatively do not lower to shift+add+shift if the mul might be
12234     // folded into smul or umul.
12235     if (N0->hasOneUse() && (isSignExtended(N0.getNode(), DAG) ||
12236                             isZeroExtended(N0.getNode(), DAG)))
12237       return SDValue();
12238     // Conservatively do not lower to shift+add+shift if the mul might be
12239     // folded into madd or msub.
12240     if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ADD ||
12241                            N->use_begin()->getOpcode() == ISD::SUB))
12242       return SDValue();
12243   }
12244   // Use ShiftedConstValue instead of ConstValue to support both shift+add/sub
12245   // and shift+add+shift.
12246   APInt ShiftedConstValue = ConstValue.ashr(TrailingZeroes);
12247 
12248   unsigned ShiftAmt, AddSubOpc;
12249   // Is the shifted value the LHS operand of the add/sub?
12250   bool ShiftValUseIsN0 = true;
12251   // Do we need to negate the result?
12252   bool NegateResult = false;
12253 
12254   if (ConstValue.isNonNegative()) {
12255     // (mul x, 2^N + 1) => (add (shl x, N), x)
12256     // (mul x, 2^N - 1) => (sub (shl x, N), x)
12257     // (mul x, (2^N + 1) * 2^M) => (shl (add (shl x, N), x), M)
12258     APInt SCVMinus1 = ShiftedConstValue - 1;
12259     APInt CVPlus1 = ConstValue + 1;
12260     if (SCVMinus1.isPowerOf2()) {
12261       ShiftAmt = SCVMinus1.logBase2();
12262       AddSubOpc = ISD::ADD;
12263     } else if (CVPlus1.isPowerOf2()) {
12264       ShiftAmt = CVPlus1.logBase2();
12265       AddSubOpc = ISD::SUB;
12266     } else
12267       return SDValue();
12268   } else {
12269     // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
12270     // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
12271     APInt CVNegPlus1 = -ConstValue + 1;
12272     APInt CVNegMinus1 = -ConstValue - 1;
12273     if (CVNegPlus1.isPowerOf2()) {
12274       ShiftAmt = CVNegPlus1.logBase2();
12275       AddSubOpc = ISD::SUB;
12276       ShiftValUseIsN0 = false;
12277     } else if (CVNegMinus1.isPowerOf2()) {
12278       ShiftAmt = CVNegMinus1.logBase2();
12279       AddSubOpc = ISD::ADD;
12280       NegateResult = true;
12281     } else
12282       return SDValue();
12283   }
12284 
12285   SDLoc DL(N);
12286   EVT VT = N->getValueType(0);
12287   SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0,
12288                                    DAG.getConstant(ShiftAmt, DL, MVT::i64));
12289 
12290   SDValue AddSubN0 = ShiftValUseIsN0 ? ShiftedVal : N0;
12291   SDValue AddSubN1 = ShiftValUseIsN0 ? N0 : ShiftedVal;
12292   SDValue Res = DAG.getNode(AddSubOpc, DL, VT, AddSubN0, AddSubN1);
12293   assert(!(NegateResult && TrailingZeroes) &&
12294          "NegateResult and TrailingZeroes cannot both be true for now.");
12295   // Negate the result.
12296   if (NegateResult)
12297     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res);
12298   // Shift the result.
12299   if (TrailingZeroes)
12300     return DAG.getNode(ISD::SHL, DL, VT, Res,
12301                        DAG.getConstant(TrailingZeroes, DL, MVT::i64));
12302   return Res;
12303 }
12304 
12305 static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N,
12306                                                          SelectionDAG &DAG) {
12307   // Take advantage of vector comparisons producing 0 or -1 in each lane to
12308   // optimize away operation when it's from a constant.
12309   //
12310   // The general transformation is:
12311   //    UNARYOP(AND(VECTOR_CMP(x,y), constant)) -->
12312   //       AND(VECTOR_CMP(x,y), constant2)
12313   //    constant2 = UNARYOP(constant)
12314 
12315   // Early exit if this isn't a vector operation, the operand of the
12316   // unary operation isn't a bitwise AND, or if the sizes of the operations
12317   // aren't the same.
12318   EVT VT = N->getValueType(0);
12319   if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND ||
12320       N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC ||
12321       VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits())
12322     return SDValue();
12323 
12324   // Now check that the other operand of the AND is a constant. We could
12325   // make the transformation for non-constant splats as well, but it's unclear
12326   // that would be a benefit as it would not eliminate any operations, just
12327   // perform one more step in scalar code before moving to the vector unit.
12328   if (BuildVectorSDNode *BV =
12329           dyn_cast<BuildVectorSDNode>(N->getOperand(0)->getOperand(1))) {
12330     // Bail out if the vector isn't a constant.
12331     if (!BV->isConstant())
12332       return SDValue();
12333 
12334     // Everything checks out. Build up the new and improved node.
12335     SDLoc DL(N);
12336     EVT IntVT = BV->getValueType(0);
12337     // Create a new constant of the appropriate type for the transformed
12338     // DAG.
12339     SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
12340     // The AND node needs bitcasts to/from an integer vector type around it.
12341     SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
12342     SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
12343                                  N->getOperand(0)->getOperand(0), MaskConst);
12344     SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
12345     return Res;
12346   }
12347 
12348   return SDValue();
12349 }
12350 
12351 static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG,
12352                                      const AArch64Subtarget *Subtarget) {
12353   // First try to optimize away the conversion when it's conditionally from
12354   // a constant. Vectors only.
12355   if (SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG))
12356     return Res;
12357 
12358   EVT VT = N->getValueType(0);
12359   if (VT != MVT::f32 && VT != MVT::f64)
12360     return SDValue();
12361 
12362   // Only optimize when the source and destination types have the same width.
12363   if (VT.getSizeInBits() != N->getOperand(0).getValueSizeInBits())
12364     return SDValue();
12365 
12366   // If the result of an integer load is only used by an integer-to-float
12367   // conversion, use a fp load instead and a AdvSIMD scalar {S|U}CVTF instead.
12368   // This eliminates an "integer-to-vector-move" UOP and improves throughput.
12369   SDValue N0 = N->getOperand(0);
12370   if (Subtarget->hasNEON() && ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
12371       // Do not change the width of a volatile load.
12372       !cast<LoadSDNode>(N0)->isVolatile()) {
12373     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
12374     SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
12375                                LN0->getPointerInfo(), LN0->getAlignment(),
12376                                LN0->getMemOperand()->getFlags());
12377 
12378     // Make sure successors of the original load stay after it by updating them
12379     // to use the new Chain.
12380     DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), Load.getValue(1));
12381 
12382     unsigned Opcode =
12383         (N->getOpcode() == ISD::SINT_TO_FP) ? AArch64ISD::SITOF : AArch64ISD::UITOF;
12384     return DAG.getNode(Opcode, SDLoc(N), VT, Load);
12385   }
12386 
12387   return SDValue();
12388 }
12389 
12390 /// Fold a floating-point multiply by power of two into floating-point to
12391 /// fixed-point conversion.
12392 static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
12393                                      TargetLowering::DAGCombinerInfo &DCI,
12394                                      const AArch64Subtarget *Subtarget) {
12395   if (!Subtarget->hasNEON())
12396     return SDValue();
12397 
12398   if (!N->getValueType(0).isSimple())
12399     return SDValue();
12400 
12401   SDValue Op = N->getOperand(0);
12402   if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
12403       Op.getOpcode() != ISD::FMUL)
12404     return SDValue();
12405 
12406   SDValue ConstVec = Op->getOperand(1);
12407   if (!isa<BuildVectorSDNode>(ConstVec))
12408     return SDValue();
12409 
12410   MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
12411   uint32_t FloatBits = FloatTy.getSizeInBits();
12412   if (FloatBits != 32 && FloatBits != 64)
12413     return SDValue();
12414 
12415   MVT IntTy = N->getSimpleValueType(0).getVectorElementType();
12416   uint32_t IntBits = IntTy.getSizeInBits();
12417   if (IntBits != 16 && IntBits != 32 && IntBits != 64)
12418     return SDValue();
12419 
12420   // Avoid conversions where iN is larger than the float (e.g., float -> i64).
12421   if (IntBits > FloatBits)
12422     return SDValue();
12423 
12424   BitVector UndefElements;
12425   BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
12426   int32_t Bits = IntBits == 64 ? 64 : 32;
12427   int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, Bits + 1);
12428   if (C == -1 || C == 0 || C > Bits)
12429     return SDValue();
12430 
12431   MVT ResTy;
12432   unsigned NumLanes = Op.getValueType().getVectorNumElements();
12433   switch (NumLanes) {
12434   default:
12435     return SDValue();
12436   case 2:
12437     ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64;
12438     break;
12439   case 4:
12440     ResTy = FloatBits == 32 ? MVT::v4i32 : MVT::v4i64;
12441     break;
12442   }
12443 
12444   if (ResTy == MVT::v4i64 && DCI.isBeforeLegalizeOps())
12445     return SDValue();
12446 
12447   assert((ResTy != MVT::v4i64 || DCI.isBeforeLegalizeOps()) &&
12448          "Illegal vector type after legalization");
12449 
12450   SDLoc DL(N);
12451   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
12452   unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfp2fxs
12453                                       : Intrinsic::aarch64_neon_vcvtfp2fxu;
12454   SDValue FixConv =
12455       DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResTy,
12456                   DAG.getConstant(IntrinsicOpcode, DL, MVT::i32),
12457                   Op->getOperand(0), DAG.getConstant(C, DL, MVT::i32));
12458   // We can handle smaller integers by generating an extra trunc.
12459   if (IntBits < FloatBits)
12460     FixConv = DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), FixConv);
12461 
12462   return FixConv;
12463 }
12464 
12465 /// Fold a floating-point divide by power of two into fixed-point to
12466 /// floating-point conversion.
12467 static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG,
12468                                   TargetLowering::DAGCombinerInfo &DCI,
12469                                   const AArch64Subtarget *Subtarget) {
12470   if (!Subtarget->hasNEON())
12471     return SDValue();
12472 
12473   SDValue Op = N->getOperand(0);
12474   unsigned Opc = Op->getOpcode();
12475   if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
12476       !Op.getOperand(0).getValueType().isSimple() ||
12477       (Opc != ISD::SINT_TO_FP && Opc != ISD::UINT_TO_FP))
12478     return SDValue();
12479 
12480   SDValue ConstVec = N->getOperand(1);
12481   if (!isa<BuildVectorSDNode>(ConstVec))
12482     return SDValue();
12483 
12484   MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
12485   int32_t IntBits = IntTy.getSizeInBits();
12486   if (IntBits != 16 && IntBits != 32 && IntBits != 64)
12487     return SDValue();
12488 
12489   MVT FloatTy = N->getSimpleValueType(0).getVectorElementType();
12490   int32_t FloatBits = FloatTy.getSizeInBits();
12491   if (FloatBits != 32 && FloatBits != 64)
12492     return SDValue();
12493 
12494   // Avoid conversions where iN is larger than the float (e.g., i64 -> float).
12495   if (IntBits > FloatBits)
12496     return SDValue();
12497 
12498   BitVector UndefElements;
12499   BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
12500   int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, FloatBits + 1);
12501   if (C == -1 || C == 0 || C > FloatBits)
12502     return SDValue();
12503 
12504   MVT ResTy;
12505   unsigned NumLanes = Op.getValueType().getVectorNumElements();
12506   switch (NumLanes) {
12507   default:
12508     return SDValue();
12509   case 2:
12510     ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64;
12511     break;
12512   case 4:
12513     ResTy = FloatBits == 32 ? MVT::v4i32 : MVT::v4i64;
12514     break;
12515   }
12516 
12517   if (ResTy == MVT::v4i64 && DCI.isBeforeLegalizeOps())
12518     return SDValue();
12519 
12520   SDLoc DL(N);
12521   SDValue ConvInput = Op.getOperand(0);
12522   bool IsSigned = Opc == ISD::SINT_TO_FP;
12523   if (IntBits < FloatBits)
12524     ConvInput = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
12525                             ResTy, ConvInput);
12526 
12527   unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfxs2fp
12528                                       : Intrinsic::aarch64_neon_vcvtfxu2fp;
12529   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
12530                      DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), ConvInput,
12531                      DAG.getConstant(C, DL, MVT::i32));
12532 }
12533 
12534 /// An EXTR instruction is made up of two shifts, ORed together. This helper
12535 /// searches for and classifies those shifts.
12536 static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount,
12537                          bool &FromHi) {
12538   if (N.getOpcode() == ISD::SHL)
12539     FromHi = false;
12540   else if (N.getOpcode() == ISD::SRL)
12541     FromHi = true;
12542   else
12543     return false;
12544 
12545   if (!isa<ConstantSDNode>(N.getOperand(1)))
12546     return false;
12547 
12548   ShiftAmount = N->getConstantOperandVal(1);
12549   Src = N->getOperand(0);
12550   return true;
12551 }
12552 
12553 /// EXTR instruction extracts a contiguous chunk of bits from two existing
12554 /// registers viewed as a high/low pair. This function looks for the pattern:
12555 /// <tt>(or (shl VAL1, \#N), (srl VAL2, \#RegWidth-N))</tt> and replaces it
12556 /// with an EXTR. Can't quite be done in TableGen because the two immediates
12557 /// aren't independent.
12558 static SDValue tryCombineToEXTR(SDNode *N,
12559                                 TargetLowering::DAGCombinerInfo &DCI) {
12560   SelectionDAG &DAG = DCI.DAG;
12561   SDLoc DL(N);
12562   EVT VT = N->getValueType(0);
12563 
12564   assert(N->getOpcode() == ISD::OR && "Unexpected root");
12565 
12566   if (VT != MVT::i32 && VT != MVT::i64)
12567     return SDValue();
12568 
12569   SDValue LHS;
12570   uint32_t ShiftLHS = 0;
12571   bool LHSFromHi = false;
12572   if (!findEXTRHalf(N->getOperand(0), LHS, ShiftLHS, LHSFromHi))
12573     return SDValue();
12574 
12575   SDValue RHS;
12576   uint32_t ShiftRHS = 0;
12577   bool RHSFromHi = false;
12578   if (!findEXTRHalf(N->getOperand(1), RHS, ShiftRHS, RHSFromHi))
12579     return SDValue();
12580 
12581   // If they're both trying to come from the high part of the register, they're
12582   // not really an EXTR.
12583   if (LHSFromHi == RHSFromHi)
12584     return SDValue();
12585 
12586   if (ShiftLHS + ShiftRHS != VT.getSizeInBits())
12587     return SDValue();
12588 
12589   if (LHSFromHi) {
12590     std::swap(LHS, RHS);
12591     std::swap(ShiftLHS, ShiftRHS);
12592   }
12593 
12594   return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS,
12595                      DAG.getConstant(ShiftRHS, DL, MVT::i64));
12596 }
12597 
12598 static SDValue tryCombineToBSL(SDNode *N,
12599                                 TargetLowering::DAGCombinerInfo &DCI) {
12600   EVT VT = N->getValueType(0);
12601   SelectionDAG &DAG = DCI.DAG;
12602   SDLoc DL(N);
12603 
12604   if (!VT.isVector())
12605     return SDValue();
12606 
12607   // The combining code currently only works for NEON vectors. In particular,
12608   // it does not work for SVE when dealing with vectors wider than 128 bits.
12609   if (!VT.is64BitVector() && !VT.is128BitVector())
12610     return SDValue();
12611 
12612   SDValue N0 = N->getOperand(0);
12613   if (N0.getOpcode() != ISD::AND)
12614     return SDValue();
12615 
12616   SDValue N1 = N->getOperand(1);
12617   if (N1.getOpcode() != ISD::AND)
12618     return SDValue();
12619 
12620   // InstCombine does (not (neg a)) => (add a -1).
12621   // Try: (or (and (neg a) b) (and (add a -1) c)) => (bsl (neg a) b c)
12622   // Loop over all combinations of AND operands.
12623   for (int i = 1; i >= 0; --i) {
12624     for (int j = 1; j >= 0; --j) {
12625       SDValue O0 = N0->getOperand(i);
12626       SDValue O1 = N1->getOperand(j);
12627       SDValue Sub, Add, SubSibling, AddSibling;
12628 
12629       // Find a SUB and an ADD operand, one from each AND.
12630       if (O0.getOpcode() == ISD::SUB && O1.getOpcode() == ISD::ADD) {
12631         Sub = O0;
12632         Add = O1;
12633         SubSibling = N0->getOperand(1 - i);
12634         AddSibling = N1->getOperand(1 - j);
12635       } else if (O0.getOpcode() == ISD::ADD && O1.getOpcode() == ISD::SUB) {
12636         Add = O0;
12637         Sub = O1;
12638         AddSibling = N0->getOperand(1 - i);
12639         SubSibling = N1->getOperand(1 - j);
12640       } else
12641         continue;
12642 
12643       if (!ISD::isBuildVectorAllZeros(Sub.getOperand(0).getNode()))
12644         continue;
12645 
12646       // Constant ones is always righthand operand of the Add.
12647       if (!ISD::isBuildVectorAllOnes(Add.getOperand(1).getNode()))
12648         continue;
12649 
12650       if (Sub.getOperand(1) != Add.getOperand(0))
12651         continue;
12652 
12653       return DAG.getNode(AArch64ISD::BSP, DL, VT, Sub, SubSibling, AddSibling);
12654     }
12655   }
12656 
12657   // (or (and a b) (and (not a) c)) => (bsl a b c)
12658   // We only have to look for constant vectors here since the general, variable
12659   // case can be handled in TableGen.
12660   unsigned Bits = VT.getScalarSizeInBits();
12661   uint64_t BitMask = Bits == 64 ? -1ULL : ((1ULL << Bits) - 1);
12662   for (int i = 1; i >= 0; --i)
12663     for (int j = 1; j >= 0; --j) {
12664       BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(i));
12665       BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(j));
12666       if (!BVN0 || !BVN1)
12667         continue;
12668 
12669       bool FoundMatch = true;
12670       for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) {
12671         ConstantSDNode *CN0 = dyn_cast<ConstantSDNode>(BVN0->getOperand(k));
12672         ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(BVN1->getOperand(k));
12673         if (!CN0 || !CN1 ||
12674             CN0->getZExtValue() != (BitMask & ~CN1->getZExtValue())) {
12675           FoundMatch = false;
12676           break;
12677         }
12678       }
12679 
12680       if (FoundMatch)
12681         return DAG.getNode(AArch64ISD::BSP, DL, VT, SDValue(BVN0, 0),
12682                            N0->getOperand(1 - i), N1->getOperand(1 - j));
12683     }
12684 
12685   return SDValue();
12686 }
12687 
12688 static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
12689                                 const AArch64Subtarget *Subtarget) {
12690   // Attempt to form an EXTR from (or (shl VAL1, #N), (srl VAL2, #RegWidth-N))
12691   SelectionDAG &DAG = DCI.DAG;
12692   EVT VT = N->getValueType(0);
12693 
12694   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
12695     return SDValue();
12696 
12697   if (SDValue Res = tryCombineToEXTR(N, DCI))
12698     return Res;
12699 
12700   if (SDValue Res = tryCombineToBSL(N, DCI))
12701     return Res;
12702 
12703   return SDValue();
12704 }
12705 
12706 static bool isConstantSplatVectorMaskForType(SDNode *N, EVT MemVT) {
12707   if (!MemVT.getVectorElementType().isSimple())
12708     return false;
12709 
12710   uint64_t MaskForTy = 0ull;
12711   switch (MemVT.getVectorElementType().getSimpleVT().SimpleTy) {
12712   case MVT::i8:
12713     MaskForTy = 0xffull;
12714     break;
12715   case MVT::i16:
12716     MaskForTy = 0xffffull;
12717     break;
12718   case MVT::i32:
12719     MaskForTy = 0xffffffffull;
12720     break;
12721   default:
12722     return false;
12723     break;
12724   }
12725 
12726   if (N->getOpcode() == AArch64ISD::DUP || N->getOpcode() == ISD::SPLAT_VECTOR)
12727     if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0)))
12728       return Op0->getAPIntValue().getLimitedValue() == MaskForTy;
12729 
12730   return false;
12731 }
12732 
12733 static SDValue performSVEAndCombine(SDNode *N,
12734                                     TargetLowering::DAGCombinerInfo &DCI) {
12735   if (DCI.isBeforeLegalizeOps())
12736     return SDValue();
12737 
12738   SelectionDAG &DAG = DCI.DAG;
12739   SDValue Src = N->getOperand(0);
12740   unsigned Opc = Src->getOpcode();
12741 
12742   // Zero/any extend of an unsigned unpack
12743   if (Opc == AArch64ISD::UUNPKHI || Opc == AArch64ISD::UUNPKLO) {
12744     SDValue UnpkOp = Src->getOperand(0);
12745     SDValue Dup = N->getOperand(1);
12746 
12747     if (Dup.getOpcode() != AArch64ISD::DUP)
12748       return SDValue();
12749 
12750     SDLoc DL(N);
12751     ConstantSDNode *C = dyn_cast<ConstantSDNode>(Dup->getOperand(0));
12752     uint64_t ExtVal = C->getZExtValue();
12753 
12754     // If the mask is fully covered by the unpack, we don't need to push
12755     // a new AND onto the operand
12756     EVT EltTy = UnpkOp->getValueType(0).getVectorElementType();
12757     if ((ExtVal == 0xFF && EltTy == MVT::i8) ||
12758         (ExtVal == 0xFFFF && EltTy == MVT::i16) ||
12759         (ExtVal == 0xFFFFFFFF && EltTy == MVT::i32))
12760       return Src;
12761 
12762     // Truncate to prevent a DUP with an over wide constant
12763     APInt Mask = C->getAPIntValue().trunc(EltTy.getSizeInBits());
12764 
12765     // Otherwise, make sure we propagate the AND to the operand
12766     // of the unpack
12767     Dup = DAG.getNode(AArch64ISD::DUP, DL,
12768                       UnpkOp->getValueType(0),
12769                       DAG.getConstant(Mask.zextOrTrunc(32), DL, MVT::i32));
12770 
12771     SDValue And = DAG.getNode(ISD::AND, DL,
12772                               UnpkOp->getValueType(0), UnpkOp, Dup);
12773 
12774     return DAG.getNode(Opc, DL, N->getValueType(0), And);
12775   }
12776 
12777   if (!EnableCombineMGatherIntrinsics)
12778     return SDValue();
12779 
12780   SDValue Mask = N->getOperand(1);
12781 
12782   if (!Src.hasOneUse())
12783     return SDValue();
12784 
12785   EVT MemVT;
12786 
12787   // SVE load instructions perform an implicit zero-extend, which makes them
12788   // perfect candidates for combining.
12789   switch (Opc) {
12790   case AArch64ISD::LD1_MERGE_ZERO:
12791   case AArch64ISD::LDNF1_MERGE_ZERO:
12792   case AArch64ISD::LDFF1_MERGE_ZERO:
12793     MemVT = cast<VTSDNode>(Src->getOperand(3))->getVT();
12794     break;
12795   case AArch64ISD::GLD1_MERGE_ZERO:
12796   case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
12797   case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
12798   case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
12799   case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
12800   case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
12801   case AArch64ISD::GLD1_IMM_MERGE_ZERO:
12802   case AArch64ISD::GLDFF1_MERGE_ZERO:
12803   case AArch64ISD::GLDFF1_SCALED_MERGE_ZERO:
12804   case AArch64ISD::GLDFF1_SXTW_MERGE_ZERO:
12805   case AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO:
12806   case AArch64ISD::GLDFF1_UXTW_MERGE_ZERO:
12807   case AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO:
12808   case AArch64ISD::GLDFF1_IMM_MERGE_ZERO:
12809   case AArch64ISD::GLDNT1_MERGE_ZERO:
12810     MemVT = cast<VTSDNode>(Src->getOperand(4))->getVT();
12811     break;
12812   default:
12813     return SDValue();
12814   }
12815 
12816   if (isConstantSplatVectorMaskForType(Mask.getNode(), MemVT))
12817     return Src;
12818 
12819   return SDValue();
12820 }
12821 
12822 static SDValue performANDCombine(SDNode *N,
12823                                  TargetLowering::DAGCombinerInfo &DCI) {
12824   SelectionDAG &DAG = DCI.DAG;
12825   SDValue LHS = N->getOperand(0);
12826   EVT VT = N->getValueType(0);
12827   if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
12828     return SDValue();
12829 
12830   if (VT.isScalableVector())
12831     return performSVEAndCombine(N, DCI);
12832 
12833   // The combining code below works only for NEON vectors. In particular, it
12834   // does not work for SVE when dealing with vectors wider than 128 bits.
12835   if (!(VT.is64BitVector() || VT.is128BitVector()))
12836     return SDValue();
12837 
12838   BuildVectorSDNode *BVN =
12839       dyn_cast<BuildVectorSDNode>(N->getOperand(1).getNode());
12840   if (!BVN)
12841     return SDValue();
12842 
12843   // AND does not accept an immediate, so check if we can use a BIC immediate
12844   // instruction instead. We do this here instead of using a (and x, (mvni imm))
12845   // pattern in isel, because some immediates may be lowered to the preferred
12846   // (and x, (movi imm)) form, even though an mvni representation also exists.
12847   APInt DefBits(VT.getSizeInBits(), 0);
12848   APInt UndefBits(VT.getSizeInBits(), 0);
12849   if (resolveBuildVector(BVN, DefBits, UndefBits)) {
12850     SDValue NewOp;
12851 
12852     DefBits = ~DefBits;
12853     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
12854                                     DefBits, &LHS)) ||
12855         (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
12856                                     DefBits, &LHS)))
12857       return NewOp;
12858 
12859     UndefBits = ~UndefBits;
12860     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
12861                                     UndefBits, &LHS)) ||
12862         (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
12863                                     UndefBits, &LHS)))
12864       return NewOp;
12865   }
12866 
12867   return SDValue();
12868 }
12869 
12870 static SDValue performSRLCombine(SDNode *N,
12871                                  TargetLowering::DAGCombinerInfo &DCI) {
12872   SelectionDAG &DAG = DCI.DAG;
12873   EVT VT = N->getValueType(0);
12874   if (VT != MVT::i32 && VT != MVT::i64)
12875     return SDValue();
12876 
12877   // Canonicalize (srl (bswap i32 x), 16) to (rotr (bswap i32 x), 16), if the
12878   // high 16-bits of x are zero. Similarly, canonicalize (srl (bswap i64 x), 32)
12879   // to (rotr (bswap i64 x), 32), if the high 32-bits of x are zero.
12880   SDValue N0 = N->getOperand(0);
12881   if (N0.getOpcode() == ISD::BSWAP) {
12882     SDLoc DL(N);
12883     SDValue N1 = N->getOperand(1);
12884     SDValue N00 = N0.getOperand(0);
12885     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
12886       uint64_t ShiftAmt = C->getZExtValue();
12887       if (VT == MVT::i32 && ShiftAmt == 16 &&
12888           DAG.MaskedValueIsZero(N00, APInt::getHighBitsSet(32, 16)))
12889         return DAG.getNode(ISD::ROTR, DL, VT, N0, N1);
12890       if (VT == MVT::i64 && ShiftAmt == 32 &&
12891           DAG.MaskedValueIsZero(N00, APInt::getHighBitsSet(64, 32)))
12892         return DAG.getNode(ISD::ROTR, DL, VT, N0, N1);
12893     }
12894   }
12895   return SDValue();
12896 }
12897 
12898 // Attempt to form urhadd(OpA, OpB) from
12899 // truncate(vlshr(sub(zext(OpB), xor(zext(OpA), Ones(ElemSizeInBits))), 1))
12900 // or uhadd(OpA, OpB) from truncate(vlshr(add(zext(OpA), zext(OpB)), 1)).
12901 // The original form of the first expression is
12902 // truncate(srl(add(zext(OpB), add(zext(OpA), 1)), 1)) and the
12903 // (OpA + OpB + 1) subexpression will have been changed to (OpB - (~OpA)).
12904 // Before this function is called the srl will have been lowered to
12905 // AArch64ISD::VLSHR.
12906 // This pass can also recognize signed variants of the patterns that use sign
12907 // extension instead of zero extension and form a srhadd(OpA, OpB) or a
12908 // shadd(OpA, OpB) from them.
12909 static SDValue
12910 performVectorTruncateCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
12911                              SelectionDAG &DAG) {
12912   EVT VT = N->getValueType(0);
12913 
12914   // Since we are looking for a right shift by a constant value of 1 and we are
12915   // operating on types at least 16 bits in length (sign/zero extended OpA and
12916   // OpB, which are at least 8 bits), it follows that the truncate will always
12917   // discard the shifted-in bit and therefore the right shift will be logical
12918   // regardless of the signedness of OpA and OpB.
12919   SDValue Shift = N->getOperand(0);
12920   if (Shift.getOpcode() != AArch64ISD::VLSHR)
12921     return SDValue();
12922 
12923   // Is the right shift using an immediate value of 1?
12924   uint64_t ShiftAmount = Shift.getConstantOperandVal(1);
12925   if (ShiftAmount != 1)
12926     return SDValue();
12927 
12928   SDValue ExtendOpA, ExtendOpB;
12929   SDValue ShiftOp0 = Shift.getOperand(0);
12930   unsigned ShiftOp0Opc = ShiftOp0.getOpcode();
12931   if (ShiftOp0Opc == ISD::SUB) {
12932 
12933     SDValue Xor = ShiftOp0.getOperand(1);
12934     if (Xor.getOpcode() != ISD::XOR)
12935       return SDValue();
12936 
12937     // Is the XOR using a constant amount of all ones in the right hand side?
12938     uint64_t C;
12939     if (!isAllConstantBuildVector(Xor.getOperand(1), C))
12940       return SDValue();
12941 
12942     unsigned ElemSizeInBits = VT.getScalarSizeInBits();
12943     APInt CAsAPInt(ElemSizeInBits, C);
12944     if (CAsAPInt != APInt::getAllOnesValue(ElemSizeInBits))
12945       return SDValue();
12946 
12947     ExtendOpA = Xor.getOperand(0);
12948     ExtendOpB = ShiftOp0.getOperand(0);
12949   } else if (ShiftOp0Opc == ISD::ADD) {
12950     ExtendOpA = ShiftOp0.getOperand(0);
12951     ExtendOpB = ShiftOp0.getOperand(1);
12952   } else
12953     return SDValue();
12954 
12955   unsigned ExtendOpAOpc = ExtendOpA.getOpcode();
12956   unsigned ExtendOpBOpc = ExtendOpB.getOpcode();
12957   if (!(ExtendOpAOpc == ExtendOpBOpc &&
12958         (ExtendOpAOpc == ISD::ZERO_EXTEND || ExtendOpAOpc == ISD::SIGN_EXTEND)))
12959     return SDValue();
12960 
12961   // Is the result of the right shift being truncated to the same value type as
12962   // the original operands, OpA and OpB?
12963   SDValue OpA = ExtendOpA.getOperand(0);
12964   SDValue OpB = ExtendOpB.getOperand(0);
12965   EVT OpAVT = OpA.getValueType();
12966   assert(ExtendOpA.getValueType() == ExtendOpB.getValueType());
12967   if (!(VT == OpAVT && OpAVT == OpB.getValueType()))
12968     return SDValue();
12969 
12970   SDLoc DL(N);
12971   bool IsSignExtend = ExtendOpAOpc == ISD::SIGN_EXTEND;
12972   bool IsRHADD = ShiftOp0Opc == ISD::SUB;
12973   unsigned HADDOpc = IsSignExtend
12974                          ? (IsRHADD ? AArch64ISD::SRHADD : AArch64ISD::SHADD)
12975                          : (IsRHADD ? AArch64ISD::URHADD : AArch64ISD::UHADD);
12976   SDValue ResultHADD = DAG.getNode(HADDOpc, DL, VT, OpA, OpB);
12977 
12978   return ResultHADD;
12979 }
12980 
12981 static bool hasPairwiseAdd(unsigned Opcode, EVT VT, bool FullFP16) {
12982   switch (Opcode) {
12983   case ISD::FADD:
12984     return (FullFP16 && VT == MVT::f16) || VT == MVT::f32 || VT == MVT::f64;
12985   case ISD::ADD:
12986     return VT == MVT::i64;
12987   default:
12988     return false;
12989   }
12990 }
12991 
12992 static SDValue performExtractVectorEltCombine(SDNode *N, SelectionDAG &DAG) {
12993   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
12994   ConstantSDNode *ConstantN1 = dyn_cast<ConstantSDNode>(N1);
12995 
12996   EVT VT = N->getValueType(0);
12997   const bool FullFP16 =
12998       static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
12999 
13000   // Rewrite for pairwise fadd pattern
13001   //   (f32 (extract_vector_elt
13002   //           (fadd (vXf32 Other)
13003   //                 (vector_shuffle (vXf32 Other) undef <1,X,...> )) 0))
13004   // ->
13005   //   (f32 (fadd (extract_vector_elt (vXf32 Other) 0)
13006   //              (extract_vector_elt (vXf32 Other) 1))
13007   if (ConstantN1 && ConstantN1->getZExtValue() == 0 &&
13008       hasPairwiseAdd(N0->getOpcode(), VT, FullFP16)) {
13009     SDLoc DL(N0);
13010     SDValue N00 = N0->getOperand(0);
13011     SDValue N01 = N0->getOperand(1);
13012 
13013     ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(N01);
13014     SDValue Other = N00;
13015 
13016     // And handle the commutative case.
13017     if (!Shuffle) {
13018       Shuffle = dyn_cast<ShuffleVectorSDNode>(N00);
13019       Other = N01;
13020     }
13021 
13022     if (Shuffle && Shuffle->getMaskElt(0) == 1 &&
13023         Other == Shuffle->getOperand(0)) {
13024       return DAG.getNode(N0->getOpcode(), DL, VT,
13025                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other,
13026                                      DAG.getConstant(0, DL, MVT::i64)),
13027                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other,
13028                                      DAG.getConstant(1, DL, MVT::i64)));
13029     }
13030   }
13031 
13032   return SDValue();
13033 }
13034 
13035 static SDValue performConcatVectorsCombine(SDNode *N,
13036                                            TargetLowering::DAGCombinerInfo &DCI,
13037                                            SelectionDAG &DAG) {
13038   SDLoc dl(N);
13039   EVT VT = N->getValueType(0);
13040   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
13041   unsigned N0Opc = N0->getOpcode(), N1Opc = N1->getOpcode();
13042 
13043   // Optimize concat_vectors of truncated vectors, where the intermediate
13044   // type is illegal, to avoid said illegality,  e.g.,
13045   //   (v4i16 (concat_vectors (v2i16 (truncate (v2i64))),
13046   //                          (v2i16 (truncate (v2i64)))))
13047   // ->
13048   //   (v4i16 (truncate (vector_shuffle (v4i32 (bitcast (v2i64))),
13049   //                                    (v4i32 (bitcast (v2i64))),
13050   //                                    <0, 2, 4, 6>)))
13051   // This isn't really target-specific, but ISD::TRUNCATE legality isn't keyed
13052   // on both input and result type, so we might generate worse code.
13053   // On AArch64 we know it's fine for v2i64->v4i16 and v4i32->v8i8.
13054   if (N->getNumOperands() == 2 && N0Opc == ISD::TRUNCATE &&
13055       N1Opc == ISD::TRUNCATE) {
13056     SDValue N00 = N0->getOperand(0);
13057     SDValue N10 = N1->getOperand(0);
13058     EVT N00VT = N00.getValueType();
13059 
13060     if (N00VT == N10.getValueType() &&
13061         (N00VT == MVT::v2i64 || N00VT == MVT::v4i32) &&
13062         N00VT.getScalarSizeInBits() == 4 * VT.getScalarSizeInBits()) {
13063       MVT MidVT = (N00VT == MVT::v2i64 ? MVT::v4i32 : MVT::v8i16);
13064       SmallVector<int, 8> Mask(MidVT.getVectorNumElements());
13065       for (size_t i = 0; i < Mask.size(); ++i)
13066         Mask[i] = i * 2;
13067       return DAG.getNode(ISD::TRUNCATE, dl, VT,
13068                          DAG.getVectorShuffle(
13069                              MidVT, dl,
13070                              DAG.getNode(ISD::BITCAST, dl, MidVT, N00),
13071                              DAG.getNode(ISD::BITCAST, dl, MidVT, N10), Mask));
13072     }
13073   }
13074 
13075   // Wait 'til after everything is legalized to try this. That way we have
13076   // legal vector types and such.
13077   if (DCI.isBeforeLegalizeOps())
13078     return SDValue();
13079 
13080   // Optimise concat_vectors of two [us]rhadds or [us]hadds that use extracted
13081   // subvectors from the same original vectors. Combine these into a single
13082   // [us]rhadd or [us]hadd that operates on the two original vectors. Example:
13083   //  (v16i8 (concat_vectors (v8i8 (urhadd (extract_subvector (v16i8 OpA, <0>),
13084   //                                        extract_subvector (v16i8 OpB,
13085   //                                        <0>))),
13086   //                         (v8i8 (urhadd (extract_subvector (v16i8 OpA, <8>),
13087   //                                        extract_subvector (v16i8 OpB,
13088   //                                        <8>)))))
13089   // ->
13090   //  (v16i8(urhadd(v16i8 OpA, v16i8 OpB)))
13091   if (N->getNumOperands() == 2 && N0Opc == N1Opc &&
13092       (N0Opc == AArch64ISD::URHADD || N0Opc == AArch64ISD::SRHADD ||
13093        N0Opc == AArch64ISD::UHADD || N0Opc == AArch64ISD::SHADD)) {
13094     SDValue N00 = N0->getOperand(0);
13095     SDValue N01 = N0->getOperand(1);
13096     SDValue N10 = N1->getOperand(0);
13097     SDValue N11 = N1->getOperand(1);
13098 
13099     EVT N00VT = N00.getValueType();
13100     EVT N10VT = N10.getValueType();
13101 
13102     if (N00->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
13103         N01->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
13104         N10->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
13105         N11->getOpcode() == ISD::EXTRACT_SUBVECTOR && N00VT == N10VT) {
13106       SDValue N00Source = N00->getOperand(0);
13107       SDValue N01Source = N01->getOperand(0);
13108       SDValue N10Source = N10->getOperand(0);
13109       SDValue N11Source = N11->getOperand(0);
13110 
13111       if (N00Source == N10Source && N01Source == N11Source &&
13112           N00Source.getValueType() == VT && N01Source.getValueType() == VT) {
13113         assert(N0.getValueType() == N1.getValueType());
13114 
13115         uint64_t N00Index = N00.getConstantOperandVal(1);
13116         uint64_t N01Index = N01.getConstantOperandVal(1);
13117         uint64_t N10Index = N10.getConstantOperandVal(1);
13118         uint64_t N11Index = N11.getConstantOperandVal(1);
13119 
13120         if (N00Index == N01Index && N10Index == N11Index && N00Index == 0 &&
13121             N10Index == N00VT.getVectorNumElements())
13122           return DAG.getNode(N0Opc, dl, VT, N00Source, N01Source);
13123       }
13124     }
13125   }
13126 
13127   // If we see a (concat_vectors (v1x64 A), (v1x64 A)) it's really a vector
13128   // splat. The indexed instructions are going to be expecting a DUPLANE64, so
13129   // canonicalise to that.
13130   if (N0 == N1 && VT.getVectorNumElements() == 2) {
13131     assert(VT.getScalarSizeInBits() == 64);
13132     return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG),
13133                        DAG.getConstant(0, dl, MVT::i64));
13134   }
13135 
13136   // Canonicalise concat_vectors so that the right-hand vector has as few
13137   // bit-casts as possible before its real operation. The primary matching
13138   // destination for these operations will be the narrowing "2" instructions,
13139   // which depend on the operation being performed on this right-hand vector.
13140   // For example,
13141   //    (concat_vectors LHS,  (v1i64 (bitconvert (v4i16 RHS))))
13142   // becomes
13143   //    (bitconvert (concat_vectors (v4i16 (bitconvert LHS)), RHS))
13144 
13145   if (N1Opc != ISD::BITCAST)
13146     return SDValue();
13147   SDValue RHS = N1->getOperand(0);
13148   MVT RHSTy = RHS.getValueType().getSimpleVT();
13149   // If the RHS is not a vector, this is not the pattern we're looking for.
13150   if (!RHSTy.isVector())
13151     return SDValue();
13152 
13153   LLVM_DEBUG(
13154       dbgs() << "aarch64-lower: concat_vectors bitcast simplification\n");
13155 
13156   MVT ConcatTy = MVT::getVectorVT(RHSTy.getVectorElementType(),
13157                                   RHSTy.getVectorNumElements() * 2);
13158   return DAG.getNode(ISD::BITCAST, dl, VT,
13159                      DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatTy,
13160                                  DAG.getNode(ISD::BITCAST, dl, RHSTy, N0),
13161                                  RHS));
13162 }
13163 
13164 static SDValue tryCombineFixedPointConvert(SDNode *N,
13165                                            TargetLowering::DAGCombinerInfo &DCI,
13166                                            SelectionDAG &DAG) {
13167   // Wait until after everything is legalized to try this. That way we have
13168   // legal vector types and such.
13169   if (DCI.isBeforeLegalizeOps())
13170     return SDValue();
13171   // Transform a scalar conversion of a value from a lane extract into a
13172   // lane extract of a vector conversion. E.g., from foo1 to foo2:
13173   // double foo1(int64x2_t a) { return vcvtd_n_f64_s64(a[1], 9); }
13174   // double foo2(int64x2_t a) { return vcvtq_n_f64_s64(a, 9)[1]; }
13175   //
13176   // The second form interacts better with instruction selection and the
13177   // register allocator to avoid cross-class register copies that aren't
13178   // coalescable due to a lane reference.
13179 
13180   // Check the operand and see if it originates from a lane extract.
13181   SDValue Op1 = N->getOperand(1);
13182   if (Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
13183     // Yep, no additional predication needed. Perform the transform.
13184     SDValue IID = N->getOperand(0);
13185     SDValue Shift = N->getOperand(2);
13186     SDValue Vec = Op1.getOperand(0);
13187     SDValue Lane = Op1.getOperand(1);
13188     EVT ResTy = N->getValueType(0);
13189     EVT VecResTy;
13190     SDLoc DL(N);
13191 
13192     // The vector width should be 128 bits by the time we get here, even
13193     // if it started as 64 bits (the extract_vector handling will have
13194     // done so).
13195     assert(Vec.getValueSizeInBits() == 128 &&
13196            "unexpected vector size on extract_vector_elt!");
13197     if (Vec.getValueType() == MVT::v4i32)
13198       VecResTy = MVT::v4f32;
13199     else if (Vec.getValueType() == MVT::v2i64)
13200       VecResTy = MVT::v2f64;
13201     else
13202       llvm_unreachable("unexpected vector type!");
13203 
13204     SDValue Convert =
13205         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VecResTy, IID, Vec, Shift);
13206     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResTy, Convert, Lane);
13207   }
13208   return SDValue();
13209 }
13210 
13211 // AArch64 high-vector "long" operations are formed by performing the non-high
13212 // version on an extract_subvector of each operand which gets the high half:
13213 //
13214 //  (longop2 LHS, RHS) == (longop (extract_high LHS), (extract_high RHS))
13215 //
13216 // However, there are cases which don't have an extract_high explicitly, but
13217 // have another operation that can be made compatible with one for free. For
13218 // example:
13219 //
13220 //  (dupv64 scalar) --> (extract_high (dup128 scalar))
13221 //
13222 // This routine does the actual conversion of such DUPs, once outer routines
13223 // have determined that everything else is in order.
13224 // It also supports immediate DUP-like nodes (MOVI/MVNi), which we can fold
13225 // similarly here.
13226 static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
13227   switch (N.getOpcode()) {
13228   case AArch64ISD::DUP:
13229   case AArch64ISD::DUPLANE8:
13230   case AArch64ISD::DUPLANE16:
13231   case AArch64ISD::DUPLANE32:
13232   case AArch64ISD::DUPLANE64:
13233   case AArch64ISD::MOVI:
13234   case AArch64ISD::MOVIshift:
13235   case AArch64ISD::MOVIedit:
13236   case AArch64ISD::MOVImsl:
13237   case AArch64ISD::MVNIshift:
13238   case AArch64ISD::MVNImsl:
13239     break;
13240   default:
13241     // FMOV could be supported, but isn't very useful, as it would only occur
13242     // if you passed a bitcast' floating point immediate to an eligible long
13243     // integer op (addl, smull, ...).
13244     return SDValue();
13245   }
13246 
13247   MVT NarrowTy = N.getSimpleValueType();
13248   if (!NarrowTy.is64BitVector())
13249     return SDValue();
13250 
13251   MVT ElementTy = NarrowTy.getVectorElementType();
13252   unsigned NumElems = NarrowTy.getVectorNumElements();
13253   MVT NewVT = MVT::getVectorVT(ElementTy, NumElems * 2);
13254 
13255   SDLoc dl(N);
13256   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NarrowTy,
13257                      DAG.getNode(N->getOpcode(), dl, NewVT, N->ops()),
13258                      DAG.getConstant(NumElems, dl, MVT::i64));
13259 }
13260 
13261 static bool isEssentiallyExtractHighSubvector(SDValue N) {
13262   if (N.getOpcode() == ISD::BITCAST)
13263     N = N.getOperand(0);
13264   if (N.getOpcode() != ISD::EXTRACT_SUBVECTOR)
13265     return false;
13266   return cast<ConstantSDNode>(N.getOperand(1))->getAPIntValue() ==
13267          N.getOperand(0).getValueType().getVectorNumElements() / 2;
13268 }
13269 
13270 /// Helper structure to keep track of ISD::SET_CC operands.
13271 struct GenericSetCCInfo {
13272   const SDValue *Opnd0;
13273   const SDValue *Opnd1;
13274   ISD::CondCode CC;
13275 };
13276 
13277 /// Helper structure to keep track of a SET_CC lowered into AArch64 code.
13278 struct AArch64SetCCInfo {
13279   const SDValue *Cmp;
13280   AArch64CC::CondCode CC;
13281 };
13282 
13283 /// Helper structure to keep track of SetCC information.
13284 union SetCCInfo {
13285   GenericSetCCInfo Generic;
13286   AArch64SetCCInfo AArch64;
13287 };
13288 
13289 /// Helper structure to be able to read SetCC information.  If set to
13290 /// true, IsAArch64 field, Info is a AArch64SetCCInfo, otherwise Info is a
13291 /// GenericSetCCInfo.
13292 struct SetCCInfoAndKind {
13293   SetCCInfo Info;
13294   bool IsAArch64;
13295 };
13296 
13297 /// Check whether or not \p Op is a SET_CC operation, either a generic or
13298 /// an
13299 /// AArch64 lowered one.
13300 /// \p SetCCInfo is filled accordingly.
13301 /// \post SetCCInfo is meanginfull only when this function returns true.
13302 /// \return True when Op is a kind of SET_CC operation.
13303 static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo) {
13304   // If this is a setcc, this is straight forward.
13305   if (Op.getOpcode() == ISD::SETCC) {
13306     SetCCInfo.Info.Generic.Opnd0 = &Op.getOperand(0);
13307     SetCCInfo.Info.Generic.Opnd1 = &Op.getOperand(1);
13308     SetCCInfo.Info.Generic.CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
13309     SetCCInfo.IsAArch64 = false;
13310     return true;
13311   }
13312   // Otherwise, check if this is a matching csel instruction.
13313   // In other words:
13314   // - csel 1, 0, cc
13315   // - csel 0, 1, !cc
13316   if (Op.getOpcode() != AArch64ISD::CSEL)
13317     return false;
13318   // Set the information about the operands.
13319   // TODO: we want the operands of the Cmp not the csel
13320   SetCCInfo.Info.AArch64.Cmp = &Op.getOperand(3);
13321   SetCCInfo.IsAArch64 = true;
13322   SetCCInfo.Info.AArch64.CC = static_cast<AArch64CC::CondCode>(
13323       cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
13324 
13325   // Check that the operands matches the constraints:
13326   // (1) Both operands must be constants.
13327   // (2) One must be 1 and the other must be 0.
13328   ConstantSDNode *TValue = dyn_cast<ConstantSDNode>(Op.getOperand(0));
13329   ConstantSDNode *FValue = dyn_cast<ConstantSDNode>(Op.getOperand(1));
13330 
13331   // Check (1).
13332   if (!TValue || !FValue)
13333     return false;
13334 
13335   // Check (2).
13336   if (!TValue->isOne()) {
13337     // Update the comparison when we are interested in !cc.
13338     std::swap(TValue, FValue);
13339     SetCCInfo.Info.AArch64.CC =
13340         AArch64CC::getInvertedCondCode(SetCCInfo.Info.AArch64.CC);
13341   }
13342   return TValue->isOne() && FValue->isNullValue();
13343 }
13344 
13345 // Returns true if Op is setcc or zext of setcc.
13346 static bool isSetCCOrZExtSetCC(const SDValue& Op, SetCCInfoAndKind &Info) {
13347   if (isSetCC(Op, Info))
13348     return true;
13349   return ((Op.getOpcode() == ISD::ZERO_EXTEND) &&
13350     isSetCC(Op->getOperand(0), Info));
13351 }
13352 
13353 // The folding we want to perform is:
13354 // (add x, [zext] (setcc cc ...) )
13355 //   -->
13356 // (csel x, (add x, 1), !cc ...)
13357 //
13358 // The latter will get matched to a CSINC instruction.
13359 static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
13360   assert(Op && Op->getOpcode() == ISD::ADD && "Unexpected operation!");
13361   SDValue LHS = Op->getOperand(0);
13362   SDValue RHS = Op->getOperand(1);
13363   SetCCInfoAndKind InfoAndKind;
13364 
13365   // If both operands are a SET_CC, then we don't want to perform this
13366   // folding and create another csel as this results in more instructions
13367   // (and higher register usage).
13368   if (isSetCCOrZExtSetCC(LHS, InfoAndKind) &&
13369       isSetCCOrZExtSetCC(RHS, InfoAndKind))
13370     return SDValue();
13371 
13372   // If neither operand is a SET_CC, give up.
13373   if (!isSetCCOrZExtSetCC(LHS, InfoAndKind)) {
13374     std::swap(LHS, RHS);
13375     if (!isSetCCOrZExtSetCC(LHS, InfoAndKind))
13376       return SDValue();
13377   }
13378 
13379   // FIXME: This could be generatized to work for FP comparisons.
13380   EVT CmpVT = InfoAndKind.IsAArch64
13381                   ? InfoAndKind.Info.AArch64.Cmp->getOperand(0).getValueType()
13382                   : InfoAndKind.Info.Generic.Opnd0->getValueType();
13383   if (CmpVT != MVT::i32 && CmpVT != MVT::i64)
13384     return SDValue();
13385 
13386   SDValue CCVal;
13387   SDValue Cmp;
13388   SDLoc dl(Op);
13389   if (InfoAndKind.IsAArch64) {
13390     CCVal = DAG.getConstant(
13391         AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), dl,
13392         MVT::i32);
13393     Cmp = *InfoAndKind.Info.AArch64.Cmp;
13394   } else
13395     Cmp = getAArch64Cmp(
13396         *InfoAndKind.Info.Generic.Opnd0, *InfoAndKind.Info.Generic.Opnd1,
13397         ISD::getSetCCInverse(InfoAndKind.Info.Generic.CC, CmpVT), CCVal, DAG,
13398         dl);
13399 
13400   EVT VT = Op->getValueType(0);
13401   LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT));
13402   return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp);
13403 }
13404 
13405 // ADD(UADDV a, UADDV b) -->  UADDV(ADD a, b)
13406 static SDValue performUADDVCombine(SDNode *N, SelectionDAG &DAG) {
13407   EVT VT = N->getValueType(0);
13408   // Only scalar integer and vector types.
13409   if (N->getOpcode() != ISD::ADD || !VT.isScalarInteger())
13410     return SDValue();
13411 
13412   SDValue LHS = N->getOperand(0);
13413   SDValue RHS = N->getOperand(1);
13414   if (LHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
13415       RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT || LHS.getValueType() != VT)
13416     return SDValue();
13417 
13418   auto *LHSN1 = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
13419   auto *RHSN1 = dyn_cast<ConstantSDNode>(RHS->getOperand(1));
13420   if (!LHSN1 || LHSN1 != RHSN1 || !RHSN1->isNullValue())
13421     return SDValue();
13422 
13423   SDValue Op1 = LHS->getOperand(0);
13424   SDValue Op2 = RHS->getOperand(0);
13425   EVT OpVT1 = Op1.getValueType();
13426   EVT OpVT2 = Op2.getValueType();
13427   if (Op1.getOpcode() != AArch64ISD::UADDV || OpVT1 != OpVT2 ||
13428       Op2.getOpcode() != AArch64ISD::UADDV ||
13429       OpVT1.getVectorElementType() != VT)
13430     return SDValue();
13431 
13432   SDValue Val1 = Op1.getOperand(0);
13433   SDValue Val2 = Op2.getOperand(0);
13434   EVT ValVT = Val1->getValueType(0);
13435   SDLoc DL(N);
13436   SDValue AddVal = DAG.getNode(ISD::ADD, DL, ValVT, Val1, Val2);
13437   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
13438                      DAG.getNode(AArch64ISD::UADDV, DL, ValVT, AddVal),
13439                      DAG.getConstant(0, DL, MVT::i64));
13440 }
13441 
13442 // ADD(UDOT(zero, x, y), A) -->  UDOT(A, x, y)
13443 static SDValue performAddDotCombine(SDNode *N, SelectionDAG &DAG) {
13444   EVT VT = N->getValueType(0);
13445   if (N->getOpcode() != ISD::ADD)
13446     return SDValue();
13447 
13448   SDValue Dot = N->getOperand(0);
13449   SDValue A = N->getOperand(1);
13450   // Handle commutivity
13451   auto isZeroDot = [](SDValue Dot) {
13452     return (Dot.getOpcode() == AArch64ISD::UDOT ||
13453             Dot.getOpcode() == AArch64ISD::SDOT) &&
13454            isZerosVector(Dot.getOperand(0).getNode());
13455   };
13456   if (!isZeroDot(Dot))
13457     std::swap(Dot, A);
13458   if (!isZeroDot(Dot))
13459     return SDValue();
13460 
13461   return DAG.getNode(Dot.getOpcode(), SDLoc(N), VT, A, Dot.getOperand(1),
13462                      Dot.getOperand(2));
13463 }
13464 
13465 // The basic add/sub long vector instructions have variants with "2" on the end
13466 // which act on the high-half of their inputs. They are normally matched by
13467 // patterns like:
13468 //
13469 // (add (zeroext (extract_high LHS)),
13470 //      (zeroext (extract_high RHS)))
13471 // -> uaddl2 vD, vN, vM
13472 //
13473 // However, if one of the extracts is something like a duplicate, this
13474 // instruction can still be used profitably. This function puts the DAG into a
13475 // more appropriate form for those patterns to trigger.
13476 static SDValue performAddSubLongCombine(SDNode *N,
13477                                         TargetLowering::DAGCombinerInfo &DCI,
13478                                         SelectionDAG &DAG) {
13479   if (DCI.isBeforeLegalizeOps())
13480     return SDValue();
13481 
13482   MVT VT = N->getSimpleValueType(0);
13483   if (!VT.is128BitVector()) {
13484     if (N->getOpcode() == ISD::ADD)
13485       return performSetccAddFolding(N, DAG);
13486     return SDValue();
13487   }
13488 
13489   // Make sure both branches are extended in the same way.
13490   SDValue LHS = N->getOperand(0);
13491   SDValue RHS = N->getOperand(1);
13492   if ((LHS.getOpcode() != ISD::ZERO_EXTEND &&
13493        LHS.getOpcode() != ISD::SIGN_EXTEND) ||
13494       LHS.getOpcode() != RHS.getOpcode())
13495     return SDValue();
13496 
13497   unsigned ExtType = LHS.getOpcode();
13498 
13499   // It's not worth doing if at least one of the inputs isn't already an
13500   // extract, but we don't know which it'll be so we have to try both.
13501   if (isEssentiallyExtractHighSubvector(LHS.getOperand(0))) {
13502     RHS = tryExtendDUPToExtractHigh(RHS.getOperand(0), DAG);
13503     if (!RHS.getNode())
13504       return SDValue();
13505 
13506     RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS);
13507   } else if (isEssentiallyExtractHighSubvector(RHS.getOperand(0))) {
13508     LHS = tryExtendDUPToExtractHigh(LHS.getOperand(0), DAG);
13509     if (!LHS.getNode())
13510       return SDValue();
13511 
13512     LHS = DAG.getNode(ExtType, SDLoc(N), VT, LHS);
13513   }
13514 
13515   return DAG.getNode(N->getOpcode(), SDLoc(N), VT, LHS, RHS);
13516 }
13517 
13518 static SDValue performAddSubCombine(SDNode *N,
13519                                     TargetLowering::DAGCombinerInfo &DCI,
13520                                     SelectionDAG &DAG) {
13521   // Try to change sum of two reductions.
13522   if (SDValue Val = performUADDVCombine(N, DAG))
13523     return Val;
13524   if (SDValue Val = performAddDotCombine(N, DAG))
13525     return Val;
13526 
13527   return performAddSubLongCombine(N, DCI, DAG);
13528 }
13529 
13530 // Massage DAGs which we can use the high-half "long" operations on into
13531 // something isel will recognize better. E.g.
13532 //
13533 // (aarch64_neon_umull (extract_high vec) (dupv64 scalar)) -->
13534 //   (aarch64_neon_umull (extract_high (v2i64 vec)))
13535 //                     (extract_high (v2i64 (dup128 scalar)))))
13536 //
13537 static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N,
13538                                        TargetLowering::DAGCombinerInfo &DCI,
13539                                        SelectionDAG &DAG) {
13540   if (DCI.isBeforeLegalizeOps())
13541     return SDValue();
13542 
13543   SDValue LHS = N->getOperand((IID == Intrinsic::not_intrinsic) ? 0 : 1);
13544   SDValue RHS = N->getOperand((IID == Intrinsic::not_intrinsic) ? 1 : 2);
13545   assert(LHS.getValueType().is64BitVector() &&
13546          RHS.getValueType().is64BitVector() &&
13547          "unexpected shape for long operation");
13548 
13549   // Either node could be a DUP, but it's not worth doing both of them (you'd
13550   // just as well use the non-high version) so look for a corresponding extract
13551   // operation on the other "wing".
13552   if (isEssentiallyExtractHighSubvector(LHS)) {
13553     RHS = tryExtendDUPToExtractHigh(RHS, DAG);
13554     if (!RHS.getNode())
13555       return SDValue();
13556   } else if (isEssentiallyExtractHighSubvector(RHS)) {
13557     LHS = tryExtendDUPToExtractHigh(LHS, DAG);
13558     if (!LHS.getNode())
13559       return SDValue();
13560   }
13561 
13562   if (IID == Intrinsic::not_intrinsic)
13563     return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), LHS, RHS);
13564 
13565   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getValueType(0),
13566                      N->getOperand(0), LHS, RHS);
13567 }
13568 
13569 static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
13570   MVT ElemTy = N->getSimpleValueType(0).getScalarType();
13571   unsigned ElemBits = ElemTy.getSizeInBits();
13572 
13573   int64_t ShiftAmount;
13574   if (BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(2))) {
13575     APInt SplatValue, SplatUndef;
13576     unsigned SplatBitSize;
13577     bool HasAnyUndefs;
13578     if (!BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
13579                               HasAnyUndefs, ElemBits) ||
13580         SplatBitSize != ElemBits)
13581       return SDValue();
13582 
13583     ShiftAmount = SplatValue.getSExtValue();
13584   } else if (ConstantSDNode *CVN = dyn_cast<ConstantSDNode>(N->getOperand(2))) {
13585     ShiftAmount = CVN->getSExtValue();
13586   } else
13587     return SDValue();
13588 
13589   unsigned Opcode;
13590   bool IsRightShift;
13591   switch (IID) {
13592   default:
13593     llvm_unreachable("Unknown shift intrinsic");
13594   case Intrinsic::aarch64_neon_sqshl:
13595     Opcode = AArch64ISD::SQSHL_I;
13596     IsRightShift = false;
13597     break;
13598   case Intrinsic::aarch64_neon_uqshl:
13599     Opcode = AArch64ISD::UQSHL_I;
13600     IsRightShift = false;
13601     break;
13602   case Intrinsic::aarch64_neon_srshl:
13603     Opcode = AArch64ISD::SRSHR_I;
13604     IsRightShift = true;
13605     break;
13606   case Intrinsic::aarch64_neon_urshl:
13607     Opcode = AArch64ISD::URSHR_I;
13608     IsRightShift = true;
13609     break;
13610   case Intrinsic::aarch64_neon_sqshlu:
13611     Opcode = AArch64ISD::SQSHLU_I;
13612     IsRightShift = false;
13613     break;
13614   case Intrinsic::aarch64_neon_sshl:
13615   case Intrinsic::aarch64_neon_ushl:
13616     // For positive shift amounts we can use SHL, as ushl/sshl perform a regular
13617     // left shift for positive shift amounts. Below, we only replace the current
13618     // node with VSHL, if this condition is met.
13619     Opcode = AArch64ISD::VSHL;
13620     IsRightShift = false;
13621     break;
13622   }
13623 
13624   if (IsRightShift && ShiftAmount <= -1 && ShiftAmount >= -(int)ElemBits) {
13625     SDLoc dl(N);
13626     return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1),
13627                        DAG.getConstant(-ShiftAmount, dl, MVT::i32));
13628   } else if (!IsRightShift && ShiftAmount >= 0 && ShiftAmount < ElemBits) {
13629     SDLoc dl(N);
13630     return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1),
13631                        DAG.getConstant(ShiftAmount, dl, MVT::i32));
13632   }
13633 
13634   return SDValue();
13635 }
13636 
13637 // The CRC32[BH] instructions ignore the high bits of their data operand. Since
13638 // the intrinsics must be legal and take an i32, this means there's almost
13639 // certainly going to be a zext in the DAG which we can eliminate.
13640 static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG) {
13641   SDValue AndN = N->getOperand(2);
13642   if (AndN.getOpcode() != ISD::AND)
13643     return SDValue();
13644 
13645   ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(AndN.getOperand(1));
13646   if (!CMask || CMask->getZExtValue() != Mask)
13647     return SDValue();
13648 
13649   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), MVT::i32,
13650                      N->getOperand(0), N->getOperand(1), AndN.getOperand(0));
13651 }
13652 
13653 static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N,
13654                                            SelectionDAG &DAG) {
13655   SDLoc dl(N);
13656   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0),
13657                      DAG.getNode(Opc, dl,
13658                                  N->getOperand(1).getSimpleValueType(),
13659                                  N->getOperand(1)),
13660                      DAG.getConstant(0, dl, MVT::i64));
13661 }
13662 
13663 static SDValue LowerSVEIntrinsicIndex(SDNode *N, SelectionDAG &DAG) {
13664   SDLoc DL(N);
13665   SDValue Op1 = N->getOperand(1);
13666   SDValue Op2 = N->getOperand(2);
13667   EVT ScalarTy = Op1.getValueType();
13668 
13669   if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16)) {
13670     Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
13671     Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
13672   }
13673 
13674   return DAG.getNode(AArch64ISD::INDEX_VECTOR, DL, N->getValueType(0),
13675                      Op1, Op2);
13676 }
13677 
13678 static SDValue LowerSVEIntrinsicDUP(SDNode *N, SelectionDAG &DAG) {
13679   SDLoc dl(N);
13680   SDValue Scalar = N->getOperand(3);
13681   EVT ScalarTy = Scalar.getValueType();
13682 
13683   if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16))
13684     Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar);
13685 
13686   SDValue Passthru = N->getOperand(1);
13687   SDValue Pred = N->getOperand(2);
13688   return DAG.getNode(AArch64ISD::DUP_MERGE_PASSTHRU, dl, N->getValueType(0),
13689                      Pred, Scalar, Passthru);
13690 }
13691 
13692 static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG) {
13693   SDLoc dl(N);
13694   LLVMContext &Ctx = *DAG.getContext();
13695   EVT VT = N->getValueType(0);
13696 
13697   assert(VT.isScalableVector() && "Expected a scalable vector.");
13698 
13699   // Current lowering only supports the SVE-ACLE types.
13700   if (VT.getSizeInBits().getKnownMinSize() != AArch64::SVEBitsPerBlock)
13701     return SDValue();
13702 
13703   unsigned ElemSize = VT.getVectorElementType().getSizeInBits() / 8;
13704   unsigned ByteSize = VT.getSizeInBits().getKnownMinSize() / 8;
13705   EVT ByteVT =
13706       EVT::getVectorVT(Ctx, MVT::i8, ElementCount::getScalable(ByteSize));
13707 
13708   // Convert everything to the domain of EXT (i.e bytes).
13709   SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1));
13710   SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(2));
13711   SDValue Op2 = DAG.getNode(ISD::MUL, dl, MVT::i32, N->getOperand(3),
13712                             DAG.getConstant(ElemSize, dl, MVT::i32));
13713 
13714   SDValue EXT = DAG.getNode(AArch64ISD::EXT, dl, ByteVT, Op0, Op1, Op2);
13715   return DAG.getNode(ISD::BITCAST, dl, VT, EXT);
13716 }
13717 
13718 static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC,
13719                                         TargetLowering::DAGCombinerInfo &DCI,
13720                                         SelectionDAG &DAG) {
13721   if (DCI.isBeforeLegalize())
13722     return SDValue();
13723 
13724   SDValue Comparator = N->getOperand(3);
13725   if (Comparator.getOpcode() == AArch64ISD::DUP ||
13726       Comparator.getOpcode() == ISD::SPLAT_VECTOR) {
13727     unsigned IID = getIntrinsicID(N);
13728     EVT VT = N->getValueType(0);
13729     EVT CmpVT = N->getOperand(2).getValueType();
13730     SDValue Pred = N->getOperand(1);
13731     SDValue Imm;
13732     SDLoc DL(N);
13733 
13734     switch (IID) {
13735     default:
13736       llvm_unreachable("Called with wrong intrinsic!");
13737       break;
13738 
13739     // Signed comparisons
13740     case Intrinsic::aarch64_sve_cmpeq_wide:
13741     case Intrinsic::aarch64_sve_cmpne_wide:
13742     case Intrinsic::aarch64_sve_cmpge_wide:
13743     case Intrinsic::aarch64_sve_cmpgt_wide:
13744     case Intrinsic::aarch64_sve_cmplt_wide:
13745     case Intrinsic::aarch64_sve_cmple_wide: {
13746       if (auto *CN = dyn_cast<ConstantSDNode>(Comparator.getOperand(0))) {
13747         int64_t ImmVal = CN->getSExtValue();
13748         if (ImmVal >= -16 && ImmVal <= 15)
13749           Imm = DAG.getConstant(ImmVal, DL, MVT::i32);
13750         else
13751           return SDValue();
13752       }
13753       break;
13754     }
13755     // Unsigned comparisons
13756     case Intrinsic::aarch64_sve_cmphs_wide:
13757     case Intrinsic::aarch64_sve_cmphi_wide:
13758     case Intrinsic::aarch64_sve_cmplo_wide:
13759     case Intrinsic::aarch64_sve_cmpls_wide:  {
13760       if (auto *CN = dyn_cast<ConstantSDNode>(Comparator.getOperand(0))) {
13761         uint64_t ImmVal = CN->getZExtValue();
13762         if (ImmVal <= 127)
13763           Imm = DAG.getConstant(ImmVal, DL, MVT::i32);
13764         else
13765           return SDValue();
13766       }
13767       break;
13768     }
13769     }
13770 
13771     if (!Imm)
13772       return SDValue();
13773 
13774     SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, CmpVT, Imm);
13775     return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, VT, Pred,
13776                        N->getOperand(2), Splat, DAG.getCondCode(CC));
13777   }
13778 
13779   return SDValue();
13780 }
13781 
13782 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
13783                         AArch64CC::CondCode Cond) {
13784   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13785 
13786   SDLoc DL(Op);
13787   assert(Op.getValueType().isScalableVector() &&
13788          TLI.isTypeLegal(Op.getValueType()) &&
13789          "Expected legal scalable vector type!");
13790 
13791   // Ensure target specific opcodes are using legal type.
13792   EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
13793   SDValue TVal = DAG.getConstant(1, DL, OutVT);
13794   SDValue FVal = DAG.getConstant(0, DL, OutVT);
13795 
13796   // Set condition code (CC) flags.
13797   SDValue Test = DAG.getNode(AArch64ISD::PTEST, DL, MVT::Other, Pg, Op);
13798 
13799   // Convert CC to integer based on requested condition.
13800   // NOTE: Cond is inverted to promote CSEL's removal when it feeds a compare.
13801   SDValue CC = DAG.getConstant(getInvertedCondCode(Cond), DL, MVT::i32);
13802   SDValue Res = DAG.getNode(AArch64ISD::CSEL, DL, OutVT, FVal, TVal, CC, Test);
13803   return DAG.getZExtOrTrunc(Res, DL, VT);
13804 }
13805 
13806 static SDValue combineSVEReductionInt(SDNode *N, unsigned Opc,
13807                                       SelectionDAG &DAG) {
13808   SDLoc DL(N);
13809 
13810   SDValue Pred = N->getOperand(1);
13811   SDValue VecToReduce = N->getOperand(2);
13812 
13813   // NOTE: The integer reduction's result type is not always linked to the
13814   // operand's element type so we construct it from the intrinsic's result type.
13815   EVT ReduceVT = getPackedSVEVectorVT(N->getValueType(0));
13816   SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce);
13817 
13818   // SVE reductions set the whole vector register with the first element
13819   // containing the reduction result, which we'll now extract.
13820   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
13821   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce,
13822                      Zero);
13823 }
13824 
13825 static SDValue combineSVEReductionFP(SDNode *N, unsigned Opc,
13826                                      SelectionDAG &DAG) {
13827   SDLoc DL(N);
13828 
13829   SDValue Pred = N->getOperand(1);
13830   SDValue VecToReduce = N->getOperand(2);
13831 
13832   EVT ReduceVT = VecToReduce.getValueType();
13833   SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce);
13834 
13835   // SVE reductions set the whole vector register with the first element
13836   // containing the reduction result, which we'll now extract.
13837   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
13838   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce,
13839                      Zero);
13840 }
13841 
13842 static SDValue combineSVEReductionOrderedFP(SDNode *N, unsigned Opc,
13843                                             SelectionDAG &DAG) {
13844   SDLoc DL(N);
13845 
13846   SDValue Pred = N->getOperand(1);
13847   SDValue InitVal = N->getOperand(2);
13848   SDValue VecToReduce = N->getOperand(3);
13849   EVT ReduceVT = VecToReduce.getValueType();
13850 
13851   // Ordered reductions use the first lane of the result vector as the
13852   // reduction's initial value.
13853   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
13854   InitVal = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ReduceVT,
13855                         DAG.getUNDEF(ReduceVT), InitVal, Zero);
13856 
13857   SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, InitVal, VecToReduce);
13858 
13859   // SVE reductions set the whole vector register with the first element
13860   // containing the reduction result, which we'll now extract.
13861   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce,
13862                      Zero);
13863 }
13864 
13865 // If a merged operation has no inactive lanes we can relax it to a predicated
13866 // or unpredicated operation, which potentially allows better isel (perhaps
13867 // using immediate forms) or relaxing register reuse requirements.
13868 static SDValue convertMergedOpToPredOp(SDNode *N, unsigned PredOpc,
13869                                        SelectionDAG &DAG) {
13870   assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Expected intrinsic!");
13871   assert(N->getNumOperands() == 4 && "Expected 3 operand intrinsic!");
13872   SDValue Pg = N->getOperand(1);
13873 
13874   // ISD way to specify an all active predicate.
13875   if ((Pg.getOpcode() == AArch64ISD::PTRUE) &&
13876       (Pg.getConstantOperandVal(0) == AArch64SVEPredPattern::all))
13877     return DAG.getNode(PredOpc, SDLoc(N), N->getValueType(0), Pg,
13878                        N->getOperand(2), N->getOperand(3));
13879 
13880   // FUTURE: SplatVector(true)
13881   return SDValue();
13882 }
13883 
13884 static SDValue performIntrinsicCombine(SDNode *N,
13885                                        TargetLowering::DAGCombinerInfo &DCI,
13886                                        const AArch64Subtarget *Subtarget) {
13887   SelectionDAG &DAG = DCI.DAG;
13888   unsigned IID = getIntrinsicID(N);
13889   switch (IID) {
13890   default:
13891     break;
13892   case Intrinsic::aarch64_neon_vcvtfxs2fp:
13893   case Intrinsic::aarch64_neon_vcvtfxu2fp:
13894     return tryCombineFixedPointConvert(N, DCI, DAG);
13895   case Intrinsic::aarch64_neon_saddv:
13896     return combineAcrossLanesIntrinsic(AArch64ISD::SADDV, N, DAG);
13897   case Intrinsic::aarch64_neon_uaddv:
13898     return combineAcrossLanesIntrinsic(AArch64ISD::UADDV, N, DAG);
13899   case Intrinsic::aarch64_neon_sminv:
13900     return combineAcrossLanesIntrinsic(AArch64ISD::SMINV, N, DAG);
13901   case Intrinsic::aarch64_neon_uminv:
13902     return combineAcrossLanesIntrinsic(AArch64ISD::UMINV, N, DAG);
13903   case Intrinsic::aarch64_neon_smaxv:
13904     return combineAcrossLanesIntrinsic(AArch64ISD::SMAXV, N, DAG);
13905   case Intrinsic::aarch64_neon_umaxv:
13906     return combineAcrossLanesIntrinsic(AArch64ISD::UMAXV, N, DAG);
13907   case Intrinsic::aarch64_neon_fmax:
13908     return DAG.getNode(ISD::FMAXIMUM, SDLoc(N), N->getValueType(0),
13909                        N->getOperand(1), N->getOperand(2));
13910   case Intrinsic::aarch64_neon_fmin:
13911     return DAG.getNode(ISD::FMINIMUM, SDLoc(N), N->getValueType(0),
13912                        N->getOperand(1), N->getOperand(2));
13913   case Intrinsic::aarch64_neon_fmaxnm:
13914     return DAG.getNode(ISD::FMAXNUM, SDLoc(N), N->getValueType(0),
13915                        N->getOperand(1), N->getOperand(2));
13916   case Intrinsic::aarch64_neon_fminnm:
13917     return DAG.getNode(ISD::FMINNUM, SDLoc(N), N->getValueType(0),
13918                        N->getOperand(1), N->getOperand(2));
13919   case Intrinsic::aarch64_neon_smull:
13920   case Intrinsic::aarch64_neon_umull:
13921   case Intrinsic::aarch64_neon_pmull:
13922   case Intrinsic::aarch64_neon_sqdmull:
13923     return tryCombineLongOpWithDup(IID, N, DCI, DAG);
13924   case Intrinsic::aarch64_neon_sqshl:
13925   case Intrinsic::aarch64_neon_uqshl:
13926   case Intrinsic::aarch64_neon_sqshlu:
13927   case Intrinsic::aarch64_neon_srshl:
13928   case Intrinsic::aarch64_neon_urshl:
13929   case Intrinsic::aarch64_neon_sshl:
13930   case Intrinsic::aarch64_neon_ushl:
13931     return tryCombineShiftImm(IID, N, DAG);
13932   case Intrinsic::aarch64_crc32b:
13933   case Intrinsic::aarch64_crc32cb:
13934     return tryCombineCRC32(0xff, N, DAG);
13935   case Intrinsic::aarch64_crc32h:
13936   case Intrinsic::aarch64_crc32ch:
13937     return tryCombineCRC32(0xffff, N, DAG);
13938   case Intrinsic::aarch64_sve_saddv:
13939     // There is no i64 version of SADDV because the sign is irrelevant.
13940     if (N->getOperand(2)->getValueType(0).getVectorElementType() == MVT::i64)
13941       return combineSVEReductionInt(N, AArch64ISD::UADDV_PRED, DAG);
13942     else
13943       return combineSVEReductionInt(N, AArch64ISD::SADDV_PRED, DAG);
13944   case Intrinsic::aarch64_sve_uaddv:
13945     return combineSVEReductionInt(N, AArch64ISD::UADDV_PRED, DAG);
13946   case Intrinsic::aarch64_sve_smaxv:
13947     return combineSVEReductionInt(N, AArch64ISD::SMAXV_PRED, DAG);
13948   case Intrinsic::aarch64_sve_umaxv:
13949     return combineSVEReductionInt(N, AArch64ISD::UMAXV_PRED, DAG);
13950   case Intrinsic::aarch64_sve_sminv:
13951     return combineSVEReductionInt(N, AArch64ISD::SMINV_PRED, DAG);
13952   case Intrinsic::aarch64_sve_uminv:
13953     return combineSVEReductionInt(N, AArch64ISD::UMINV_PRED, DAG);
13954   case Intrinsic::aarch64_sve_orv:
13955     return combineSVEReductionInt(N, AArch64ISD::ORV_PRED, DAG);
13956   case Intrinsic::aarch64_sve_eorv:
13957     return combineSVEReductionInt(N, AArch64ISD::EORV_PRED, DAG);
13958   case Intrinsic::aarch64_sve_andv:
13959     return combineSVEReductionInt(N, AArch64ISD::ANDV_PRED, DAG);
13960   case Intrinsic::aarch64_sve_index:
13961     return LowerSVEIntrinsicIndex(N, DAG);
13962   case Intrinsic::aarch64_sve_dup:
13963     return LowerSVEIntrinsicDUP(N, DAG);
13964   case Intrinsic::aarch64_sve_dup_x:
13965     return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), N->getValueType(0),
13966                        N->getOperand(1));
13967   case Intrinsic::aarch64_sve_ext:
13968     return LowerSVEIntrinsicEXT(N, DAG);
13969   case Intrinsic::aarch64_sve_smin:
13970     return convertMergedOpToPredOp(N, AArch64ISD::SMIN_PRED, DAG);
13971   case Intrinsic::aarch64_sve_umin:
13972     return convertMergedOpToPredOp(N, AArch64ISD::UMIN_PRED, DAG);
13973   case Intrinsic::aarch64_sve_smax:
13974     return convertMergedOpToPredOp(N, AArch64ISD::SMAX_PRED, DAG);
13975   case Intrinsic::aarch64_sve_umax:
13976     return convertMergedOpToPredOp(N, AArch64ISD::UMAX_PRED, DAG);
13977   case Intrinsic::aarch64_sve_lsl:
13978     return convertMergedOpToPredOp(N, AArch64ISD::SHL_PRED, DAG);
13979   case Intrinsic::aarch64_sve_lsr:
13980     return convertMergedOpToPredOp(N, AArch64ISD::SRL_PRED, DAG);
13981   case Intrinsic::aarch64_sve_asr:
13982     return convertMergedOpToPredOp(N, AArch64ISD::SRA_PRED, DAG);
13983   case Intrinsic::aarch64_sve_cmphs:
13984     if (!N->getOperand(2).getValueType().isFloatingPoint())
13985       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
13986                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
13987                          N->getOperand(3), DAG.getCondCode(ISD::SETUGE));
13988     break;
13989   case Intrinsic::aarch64_sve_cmphi:
13990     if (!N->getOperand(2).getValueType().isFloatingPoint())
13991       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
13992                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
13993                          N->getOperand(3), DAG.getCondCode(ISD::SETUGT));
13994     break;
13995   case Intrinsic::aarch64_sve_cmpge:
13996     if (!N->getOperand(2).getValueType().isFloatingPoint())
13997       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
13998                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
13999                          N->getOperand(3), DAG.getCondCode(ISD::SETGE));
14000     break;
14001   case Intrinsic::aarch64_sve_cmpgt:
14002     if (!N->getOperand(2).getValueType().isFloatingPoint())
14003       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
14004                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
14005                          N->getOperand(3), DAG.getCondCode(ISD::SETGT));
14006     break;
14007   case Intrinsic::aarch64_sve_cmpeq:
14008     if (!N->getOperand(2).getValueType().isFloatingPoint())
14009       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
14010                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
14011                          N->getOperand(3), DAG.getCondCode(ISD::SETEQ));
14012     break;
14013   case Intrinsic::aarch64_sve_cmpne:
14014     if (!N->getOperand(2).getValueType().isFloatingPoint())
14015       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
14016                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
14017                          N->getOperand(3), DAG.getCondCode(ISD::SETNE));
14018     break;
14019   case Intrinsic::aarch64_sve_fadda:
14020     return combineSVEReductionOrderedFP(N, AArch64ISD::FADDA_PRED, DAG);
14021   case Intrinsic::aarch64_sve_faddv:
14022     return combineSVEReductionFP(N, AArch64ISD::FADDV_PRED, DAG);
14023   case Intrinsic::aarch64_sve_fmaxnmv:
14024     return combineSVEReductionFP(N, AArch64ISD::FMAXNMV_PRED, DAG);
14025   case Intrinsic::aarch64_sve_fmaxv:
14026     return combineSVEReductionFP(N, AArch64ISD::FMAXV_PRED, DAG);
14027   case Intrinsic::aarch64_sve_fminnmv:
14028     return combineSVEReductionFP(N, AArch64ISD::FMINNMV_PRED, DAG);
14029   case Intrinsic::aarch64_sve_fminv:
14030     return combineSVEReductionFP(N, AArch64ISD::FMINV_PRED, DAG);
14031   case Intrinsic::aarch64_sve_sel:
14032     return DAG.getNode(ISD::VSELECT, SDLoc(N), N->getValueType(0),
14033                        N->getOperand(1), N->getOperand(2), N->getOperand(3));
14034   case Intrinsic::aarch64_sve_cmpeq_wide:
14035     return tryConvertSVEWideCompare(N, ISD::SETEQ, DCI, DAG);
14036   case Intrinsic::aarch64_sve_cmpne_wide:
14037     return tryConvertSVEWideCompare(N, ISD::SETNE, DCI, DAG);
14038   case Intrinsic::aarch64_sve_cmpge_wide:
14039     return tryConvertSVEWideCompare(N, ISD::SETGE, DCI, DAG);
14040   case Intrinsic::aarch64_sve_cmpgt_wide:
14041     return tryConvertSVEWideCompare(N, ISD::SETGT, DCI, DAG);
14042   case Intrinsic::aarch64_sve_cmplt_wide:
14043     return tryConvertSVEWideCompare(N, ISD::SETLT, DCI, DAG);
14044   case Intrinsic::aarch64_sve_cmple_wide:
14045     return tryConvertSVEWideCompare(N, ISD::SETLE, DCI, DAG);
14046   case Intrinsic::aarch64_sve_cmphs_wide:
14047     return tryConvertSVEWideCompare(N, ISD::SETUGE, DCI, DAG);
14048   case Intrinsic::aarch64_sve_cmphi_wide:
14049     return tryConvertSVEWideCompare(N, ISD::SETUGT, DCI, DAG);
14050   case Intrinsic::aarch64_sve_cmplo_wide:
14051     return tryConvertSVEWideCompare(N, ISD::SETULT, DCI, DAG);
14052   case Intrinsic::aarch64_sve_cmpls_wide:
14053     return tryConvertSVEWideCompare(N, ISD::SETULE, DCI, DAG);
14054   case Intrinsic::aarch64_sve_ptest_any:
14055     return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
14056                     AArch64CC::ANY_ACTIVE);
14057   case Intrinsic::aarch64_sve_ptest_first:
14058     return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
14059                     AArch64CC::FIRST_ACTIVE);
14060   case Intrinsic::aarch64_sve_ptest_last:
14061     return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
14062                     AArch64CC::LAST_ACTIVE);
14063   }
14064   return SDValue();
14065 }
14066 
14067 static SDValue performExtendCombine(SDNode *N,
14068                                     TargetLowering::DAGCombinerInfo &DCI,
14069                                     SelectionDAG &DAG) {
14070   // If we see something like (zext (sabd (extract_high ...), (DUP ...))) then
14071   // we can convert that DUP into another extract_high (of a bigger DUP), which
14072   // helps the backend to decide that an sabdl2 would be useful, saving a real
14073   // extract_high operation.
14074   if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ZERO_EXTEND &&
14075       (N->getOperand(0).getOpcode() == AArch64ISD::UABD ||
14076        N->getOperand(0).getOpcode() == AArch64ISD::SABD)) {
14077     SDNode *ABDNode = N->getOperand(0).getNode();
14078     SDValue NewABD =
14079         tryCombineLongOpWithDup(Intrinsic::not_intrinsic, ABDNode, DCI, DAG);
14080     if (!NewABD.getNode())
14081       return SDValue();
14082 
14083     return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), NewABD);
14084   }
14085   return SDValue();
14086 }
14087 
14088 static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St,
14089                                SDValue SplatVal, unsigned NumVecElts) {
14090   assert(!St.isTruncatingStore() && "cannot split truncating vector store");
14091   unsigned OrigAlignment = St.getAlignment();
14092   unsigned EltOffset = SplatVal.getValueType().getSizeInBits() / 8;
14093 
14094   // Create scalar stores. This is at least as good as the code sequence for a
14095   // split unaligned store which is a dup.s, ext.b, and two stores.
14096   // Most of the time the three stores should be replaced by store pair
14097   // instructions (stp).
14098   SDLoc DL(&St);
14099   SDValue BasePtr = St.getBasePtr();
14100   uint64_t BaseOffset = 0;
14101 
14102   const MachinePointerInfo &PtrInfo = St.getPointerInfo();
14103   SDValue NewST1 =
14104       DAG.getStore(St.getChain(), DL, SplatVal, BasePtr, PtrInfo,
14105                    OrigAlignment, St.getMemOperand()->getFlags());
14106 
14107   // As this in ISel, we will not merge this add which may degrade results.
14108   if (BasePtr->getOpcode() == ISD::ADD &&
14109       isa<ConstantSDNode>(BasePtr->getOperand(1))) {
14110     BaseOffset = cast<ConstantSDNode>(BasePtr->getOperand(1))->getSExtValue();
14111     BasePtr = BasePtr->getOperand(0);
14112   }
14113 
14114   unsigned Offset = EltOffset;
14115   while (--NumVecElts) {
14116     unsigned Alignment = MinAlign(OrigAlignment, Offset);
14117     SDValue OffsetPtr =
14118         DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
14119                     DAG.getConstant(BaseOffset + Offset, DL, MVT::i64));
14120     NewST1 = DAG.getStore(NewST1.getValue(0), DL, SplatVal, OffsetPtr,
14121                           PtrInfo.getWithOffset(Offset), Alignment,
14122                           St.getMemOperand()->getFlags());
14123     Offset += EltOffset;
14124   }
14125   return NewST1;
14126 }
14127 
14128 // Returns an SVE type that ContentTy can be trivially sign or zero extended
14129 // into.
14130 static MVT getSVEContainerType(EVT ContentTy) {
14131   assert(ContentTy.isSimple() && "No SVE containers for extended types");
14132 
14133   switch (ContentTy.getSimpleVT().SimpleTy) {
14134   default:
14135     llvm_unreachable("No known SVE container for this MVT type");
14136   case MVT::nxv2i8:
14137   case MVT::nxv2i16:
14138   case MVT::nxv2i32:
14139   case MVT::nxv2i64:
14140   case MVT::nxv2f32:
14141   case MVT::nxv2f64:
14142     return MVT::nxv2i64;
14143   case MVT::nxv4i8:
14144   case MVT::nxv4i16:
14145   case MVT::nxv4i32:
14146   case MVT::nxv4f32:
14147     return MVT::nxv4i32;
14148   case MVT::nxv8i8:
14149   case MVT::nxv8i16:
14150   case MVT::nxv8f16:
14151   case MVT::nxv8bf16:
14152     return MVT::nxv8i16;
14153   case MVT::nxv16i8:
14154     return MVT::nxv16i8;
14155   }
14156 }
14157 
14158 static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc) {
14159   SDLoc DL(N);
14160   EVT VT = N->getValueType(0);
14161 
14162   if (VT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
14163     return SDValue();
14164 
14165   EVT ContainerVT = VT;
14166   if (ContainerVT.isInteger())
14167     ContainerVT = getSVEContainerType(ContainerVT);
14168 
14169   SDVTList VTs = DAG.getVTList(ContainerVT, MVT::Other);
14170   SDValue Ops[] = { N->getOperand(0), // Chain
14171                     N->getOperand(2), // Pg
14172                     N->getOperand(3), // Base
14173                     DAG.getValueType(VT) };
14174 
14175   SDValue Load = DAG.getNode(Opc, DL, VTs, Ops);
14176   SDValue LoadChain = SDValue(Load.getNode(), 1);
14177 
14178   if (ContainerVT.isInteger() && (VT != ContainerVT))
14179     Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0));
14180 
14181   return DAG.getMergeValues({ Load, LoadChain }, DL);
14182 }
14183 
14184 static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
14185   SDLoc DL(N);
14186   EVT VT = N->getValueType(0);
14187   EVT PtrTy = N->getOperand(3).getValueType();
14188 
14189   if (VT == MVT::nxv8bf16 &&
14190       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
14191     return SDValue();
14192 
14193   EVT LoadVT = VT;
14194   if (VT.isFloatingPoint())
14195     LoadVT = VT.changeTypeToInteger();
14196 
14197   auto *MINode = cast<MemIntrinsicSDNode>(N);
14198   SDValue PassThru = DAG.getConstant(0, DL, LoadVT);
14199   SDValue L = DAG.getMaskedLoad(LoadVT, DL, MINode->getChain(),
14200                                 MINode->getOperand(3), DAG.getUNDEF(PtrTy),
14201                                 MINode->getOperand(2), PassThru,
14202                                 MINode->getMemoryVT(), MINode->getMemOperand(),
14203                                 ISD::UNINDEXED, ISD::NON_EXTLOAD, false);
14204 
14205    if (VT.isFloatingPoint()) {
14206      SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) };
14207      return DAG.getMergeValues(Ops, DL);
14208    }
14209 
14210   return L;
14211 }
14212 
14213 template <unsigned Opcode>
14214 static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG) {
14215   static_assert(Opcode == AArch64ISD::LD1RQ_MERGE_ZERO ||
14216                     Opcode == AArch64ISD::LD1RO_MERGE_ZERO,
14217                 "Unsupported opcode.");
14218   SDLoc DL(N);
14219   EVT VT = N->getValueType(0);
14220   if (VT == MVT::nxv8bf16 &&
14221       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
14222     return SDValue();
14223 
14224   EVT LoadVT = VT;
14225   if (VT.isFloatingPoint())
14226     LoadVT = VT.changeTypeToInteger();
14227 
14228   SDValue Ops[] = {N->getOperand(0), N->getOperand(2), N->getOperand(3)};
14229   SDValue Load = DAG.getNode(Opcode, DL, {LoadVT, MVT::Other}, Ops);
14230   SDValue LoadChain = SDValue(Load.getNode(), 1);
14231 
14232   if (VT.isFloatingPoint())
14233     Load = DAG.getNode(ISD::BITCAST, DL, VT, Load.getValue(0));
14234 
14235   return DAG.getMergeValues({Load, LoadChain}, DL);
14236 }
14237 
14238 static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
14239   SDLoc DL(N);
14240   SDValue Data = N->getOperand(2);
14241   EVT DataVT = Data.getValueType();
14242   EVT HwSrcVt = getSVEContainerType(DataVT);
14243   SDValue InputVT = DAG.getValueType(DataVT);
14244 
14245   if (DataVT == MVT::nxv8bf16 &&
14246       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
14247     return SDValue();
14248 
14249   if (DataVT.isFloatingPoint())
14250     InputVT = DAG.getValueType(HwSrcVt);
14251 
14252   SDValue SrcNew;
14253   if (Data.getValueType().isFloatingPoint())
14254     SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Data);
14255   else
14256     SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Data);
14257 
14258   SDValue Ops[] = { N->getOperand(0), // Chain
14259                     SrcNew,
14260                     N->getOperand(4), // Base
14261                     N->getOperand(3), // Pg
14262                     InputVT
14263                   };
14264 
14265   return DAG.getNode(AArch64ISD::ST1_PRED, DL, N->getValueType(0), Ops);
14266 }
14267 
14268 static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG) {
14269   SDLoc DL(N);
14270 
14271   SDValue Data = N->getOperand(2);
14272   EVT DataVT = Data.getValueType();
14273   EVT PtrTy = N->getOperand(4).getValueType();
14274 
14275   if (DataVT == MVT::nxv8bf16 &&
14276       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
14277     return SDValue();
14278 
14279   if (DataVT.isFloatingPoint())
14280     Data = DAG.getNode(ISD::BITCAST, DL, DataVT.changeTypeToInteger(), Data);
14281 
14282   auto *MINode = cast<MemIntrinsicSDNode>(N);
14283   return DAG.getMaskedStore(MINode->getChain(), DL, Data, MINode->getOperand(4),
14284                             DAG.getUNDEF(PtrTy), MINode->getOperand(3),
14285                             MINode->getMemoryVT(), MINode->getMemOperand(),
14286                             ISD::UNINDEXED, false, false);
14287 }
14288 
14289 /// Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.  The
14290 /// load store optimizer pass will merge them to store pair stores.  This should
14291 /// be better than a movi to create the vector zero followed by a vector store
14292 /// if the zero constant is not re-used, since one instructions and one register
14293 /// live range will be removed.
14294 ///
14295 /// For example, the final generated code should be:
14296 ///
14297 ///   stp xzr, xzr, [x0]
14298 ///
14299 /// instead of:
14300 ///
14301 ///   movi v0.2d, #0
14302 ///   str q0, [x0]
14303 ///
14304 static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
14305   SDValue StVal = St.getValue();
14306   EVT VT = StVal.getValueType();
14307 
14308   // Avoid scalarizing zero splat stores for scalable vectors.
14309   if (VT.isScalableVector())
14310     return SDValue();
14311 
14312   // It is beneficial to scalarize a zero splat store for 2 or 3 i64 elements or
14313   // 2, 3 or 4 i32 elements.
14314   int NumVecElts = VT.getVectorNumElements();
14315   if (!(((NumVecElts == 2 || NumVecElts == 3) &&
14316          VT.getVectorElementType().getSizeInBits() == 64) ||
14317         ((NumVecElts == 2 || NumVecElts == 3 || NumVecElts == 4) &&
14318          VT.getVectorElementType().getSizeInBits() == 32)))
14319     return SDValue();
14320 
14321   if (StVal.getOpcode() != ISD::BUILD_VECTOR)
14322     return SDValue();
14323 
14324   // If the zero constant has more than one use then the vector store could be
14325   // better since the constant mov will be amortized and stp q instructions
14326   // should be able to be formed.
14327   if (!StVal.hasOneUse())
14328     return SDValue();
14329 
14330   // If the store is truncating then it's going down to i16 or smaller, which
14331   // means it can be implemented in a single store anyway.
14332   if (St.isTruncatingStore())
14333     return SDValue();
14334 
14335   // If the immediate offset of the address operand is too large for the stp
14336   // instruction, then bail out.
14337   if (DAG.isBaseWithConstantOffset(St.getBasePtr())) {
14338     int64_t Offset = St.getBasePtr()->getConstantOperandVal(1);
14339     if (Offset < -512 || Offset > 504)
14340       return SDValue();
14341   }
14342 
14343   for (int I = 0; I < NumVecElts; ++I) {
14344     SDValue EltVal = StVal.getOperand(I);
14345     if (!isNullConstant(EltVal) && !isNullFPConstant(EltVal))
14346       return SDValue();
14347   }
14348 
14349   // Use a CopyFromReg WZR/XZR here to prevent
14350   // DAGCombiner::MergeConsecutiveStores from undoing this transformation.
14351   SDLoc DL(&St);
14352   unsigned ZeroReg;
14353   EVT ZeroVT;
14354   if (VT.getVectorElementType().getSizeInBits() == 32) {
14355     ZeroReg = AArch64::WZR;
14356     ZeroVT = MVT::i32;
14357   } else {
14358     ZeroReg = AArch64::XZR;
14359     ZeroVT = MVT::i64;
14360   }
14361   SDValue SplatVal =
14362       DAG.getCopyFromReg(DAG.getEntryNode(), DL, ZeroReg, ZeroVT);
14363   return splitStoreSplat(DAG, St, SplatVal, NumVecElts);
14364 }
14365 
14366 /// Replace a splat of a scalar to a vector store by scalar stores of the scalar
14367 /// value. The load store optimizer pass will merge them to store pair stores.
14368 /// This has better performance than a splat of the scalar followed by a split
14369 /// vector store. Even if the stores are not merged it is four stores vs a dup,
14370 /// followed by an ext.b and two stores.
14371 static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
14372   SDValue StVal = St.getValue();
14373   EVT VT = StVal.getValueType();
14374 
14375   // Don't replace floating point stores, they possibly won't be transformed to
14376   // stp because of the store pair suppress pass.
14377   if (VT.isFloatingPoint())
14378     return SDValue();
14379 
14380   // We can express a splat as store pair(s) for 2 or 4 elements.
14381   unsigned NumVecElts = VT.getVectorNumElements();
14382   if (NumVecElts != 4 && NumVecElts != 2)
14383     return SDValue();
14384 
14385   // If the store is truncating then it's going down to i16 or smaller, which
14386   // means it can be implemented in a single store anyway.
14387   if (St.isTruncatingStore())
14388     return SDValue();
14389 
14390   // Check that this is a splat.
14391   // Make sure that each of the relevant vector element locations are inserted
14392   // to, i.e. 0 and 1 for v2i64 and 0, 1, 2, 3 for v4i32.
14393   std::bitset<4> IndexNotInserted((1 << NumVecElts) - 1);
14394   SDValue SplatVal;
14395   for (unsigned I = 0; I < NumVecElts; ++I) {
14396     // Check for insert vector elements.
14397     if (StVal.getOpcode() != ISD::INSERT_VECTOR_ELT)
14398       return SDValue();
14399 
14400     // Check that same value is inserted at each vector element.
14401     if (I == 0)
14402       SplatVal = StVal.getOperand(1);
14403     else if (StVal.getOperand(1) != SplatVal)
14404       return SDValue();
14405 
14406     // Check insert element index.
14407     ConstantSDNode *CIndex = dyn_cast<ConstantSDNode>(StVal.getOperand(2));
14408     if (!CIndex)
14409       return SDValue();
14410     uint64_t IndexVal = CIndex->getZExtValue();
14411     if (IndexVal >= NumVecElts)
14412       return SDValue();
14413     IndexNotInserted.reset(IndexVal);
14414 
14415     StVal = StVal.getOperand(0);
14416   }
14417   // Check that all vector element locations were inserted to.
14418   if (IndexNotInserted.any())
14419       return SDValue();
14420 
14421   return splitStoreSplat(DAG, St, SplatVal, NumVecElts);
14422 }
14423 
14424 static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
14425                            SelectionDAG &DAG,
14426                            const AArch64Subtarget *Subtarget) {
14427 
14428   StoreSDNode *S = cast<StoreSDNode>(N);
14429   if (S->isVolatile() || S->isIndexed())
14430     return SDValue();
14431 
14432   SDValue StVal = S->getValue();
14433   EVT VT = StVal.getValueType();
14434 
14435   if (!VT.isFixedLengthVector())
14436     return SDValue();
14437 
14438   // If we get a splat of zeros, convert this vector store to a store of
14439   // scalars. They will be merged into store pairs of xzr thereby removing one
14440   // instruction and one register.
14441   if (SDValue ReplacedZeroSplat = replaceZeroVectorStore(DAG, *S))
14442     return ReplacedZeroSplat;
14443 
14444   // FIXME: The logic for deciding if an unaligned store should be split should
14445   // be included in TLI.allowsMisalignedMemoryAccesses(), and there should be
14446   // a call to that function here.
14447 
14448   if (!Subtarget->isMisaligned128StoreSlow())
14449     return SDValue();
14450 
14451   // Don't split at -Oz.
14452   if (DAG.getMachineFunction().getFunction().hasMinSize())
14453     return SDValue();
14454 
14455   // Don't split v2i64 vectors. Memcpy lowering produces those and splitting
14456   // those up regresses performance on micro-benchmarks and olden/bh.
14457   if (VT.getVectorNumElements() < 2 || VT == MVT::v2i64)
14458     return SDValue();
14459 
14460   // Split unaligned 16B stores. They are terrible for performance.
14461   // Don't split stores with alignment of 1 or 2. Code that uses clang vector
14462   // extensions can use this to mark that it does not want splitting to happen
14463   // (by underspecifying alignment to be 1 or 2). Furthermore, the chance of
14464   // eliminating alignment hazards is only 1 in 8 for alignment of 2.
14465   if (VT.getSizeInBits() != 128 || S->getAlignment() >= 16 ||
14466       S->getAlignment() <= 2)
14467     return SDValue();
14468 
14469   // If we get a splat of a scalar convert this vector store to a store of
14470   // scalars. They will be merged into store pairs thereby removing two
14471   // instructions.
14472   if (SDValue ReplacedSplat = replaceSplatVectorStore(DAG, *S))
14473     return ReplacedSplat;
14474 
14475   SDLoc DL(S);
14476 
14477   // Split VT into two.
14478   EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
14479   unsigned NumElts = HalfVT.getVectorNumElements();
14480   SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
14481                                    DAG.getConstant(0, DL, MVT::i64));
14482   SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
14483                                    DAG.getConstant(NumElts, DL, MVT::i64));
14484   SDValue BasePtr = S->getBasePtr();
14485   SDValue NewST1 =
14486       DAG.getStore(S->getChain(), DL, SubVector0, BasePtr, S->getPointerInfo(),
14487                    S->getAlignment(), S->getMemOperand()->getFlags());
14488   SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
14489                                   DAG.getConstant(8, DL, MVT::i64));
14490   return DAG.getStore(NewST1.getValue(0), DL, SubVector1, OffsetPtr,
14491                       S->getPointerInfo(), S->getAlignment(),
14492                       S->getMemOperand()->getFlags());
14493 }
14494 
14495 static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG) {
14496   SDLoc DL(N);
14497   SDValue Op0 = N->getOperand(0);
14498   SDValue Op1 = N->getOperand(1);
14499   EVT ResVT = N->getValueType(0);
14500 
14501   // uzp1(unpklo(uzp1(x, y)), z) => uzp1(x, z)
14502   if (Op0.getOpcode() == AArch64ISD::UUNPKLO) {
14503     if (Op0.getOperand(0).getOpcode() == AArch64ISD::UZP1) {
14504       SDValue X = Op0.getOperand(0).getOperand(0);
14505       return DAG.getNode(AArch64ISD::UZP1, DL, ResVT, X, Op1);
14506     }
14507   }
14508 
14509   // uzp1(x, unpkhi(uzp1(y, z))) => uzp1(x, z)
14510   if (Op1.getOpcode() == AArch64ISD::UUNPKHI) {
14511     if (Op1.getOperand(0).getOpcode() == AArch64ISD::UZP1) {
14512       SDValue Z = Op1.getOperand(0).getOperand(1);
14513       return DAG.getNode(AArch64ISD::UZP1, DL, ResVT, Op0, Z);
14514     }
14515   }
14516 
14517   return SDValue();
14518 }
14519 
14520 static SDValue performGLD1Combine(SDNode *N, SelectionDAG &DAG) {
14521   unsigned Opc = N->getOpcode();
14522 
14523   assert(((Opc >= AArch64ISD::GLD1_MERGE_ZERO && // unsigned gather loads
14524            Opc <= AArch64ISD::GLD1_IMM_MERGE_ZERO) ||
14525           (Opc >= AArch64ISD::GLD1S_MERGE_ZERO && // signed gather loads
14526            Opc <= AArch64ISD::GLD1S_IMM_MERGE_ZERO)) &&
14527          "Invalid opcode.");
14528 
14529   const bool Scaled = Opc == AArch64ISD::GLD1_SCALED_MERGE_ZERO ||
14530                       Opc == AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
14531   const bool Signed = Opc == AArch64ISD::GLD1S_MERGE_ZERO ||
14532                       Opc == AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
14533   const bool Extended = Opc == AArch64ISD::GLD1_SXTW_MERGE_ZERO ||
14534                         Opc == AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO ||
14535                         Opc == AArch64ISD::GLD1_UXTW_MERGE_ZERO ||
14536                         Opc == AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO;
14537 
14538   SDLoc DL(N);
14539   SDValue Chain = N->getOperand(0);
14540   SDValue Pg = N->getOperand(1);
14541   SDValue Base = N->getOperand(2);
14542   SDValue Offset = N->getOperand(3);
14543   SDValue Ty = N->getOperand(4);
14544 
14545   EVT ResVT = N->getValueType(0);
14546 
14547   const auto OffsetOpc = Offset.getOpcode();
14548   const bool OffsetIsZExt =
14549       OffsetOpc == AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU;
14550   const bool OffsetIsSExt =
14551       OffsetOpc == AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU;
14552 
14553   // Fold sign/zero extensions of vector offsets into GLD1 nodes where possible.
14554   if (!Extended && (OffsetIsSExt || OffsetIsZExt)) {
14555     SDValue ExtPg = Offset.getOperand(0);
14556     VTSDNode *ExtFrom = cast<VTSDNode>(Offset.getOperand(2).getNode());
14557     EVT ExtFromEVT = ExtFrom->getVT().getVectorElementType();
14558 
14559     // If the predicate for the sign- or zero-extended offset is the
14560     // same as the predicate used for this load and the sign-/zero-extension
14561     // was from a 32-bits...
14562     if (ExtPg == Pg && ExtFromEVT == MVT::i32) {
14563       SDValue UnextendedOffset = Offset.getOperand(1);
14564 
14565       unsigned NewOpc = getGatherVecOpcode(Scaled, OffsetIsSExt, true);
14566       if (Signed)
14567         NewOpc = getSignExtendedGatherOpcode(NewOpc);
14568 
14569       return DAG.getNode(NewOpc, DL, {ResVT, MVT::Other},
14570                          {Chain, Pg, Base, UnextendedOffset, Ty});
14571     }
14572   }
14573 
14574   return SDValue();
14575 }
14576 
14577 /// Target-specific DAG combine function for post-increment LD1 (lane) and
14578 /// post-increment LD1R.
14579 static SDValue performPostLD1Combine(SDNode *N,
14580                                      TargetLowering::DAGCombinerInfo &DCI,
14581                                      bool IsLaneOp) {
14582   if (DCI.isBeforeLegalizeOps())
14583     return SDValue();
14584 
14585   SelectionDAG &DAG = DCI.DAG;
14586   EVT VT = N->getValueType(0);
14587 
14588   if (VT.isScalableVector())
14589     return SDValue();
14590 
14591   unsigned LoadIdx = IsLaneOp ? 1 : 0;
14592   SDNode *LD = N->getOperand(LoadIdx).getNode();
14593   // If it is not LOAD, can not do such combine.
14594   if (LD->getOpcode() != ISD::LOAD)
14595     return SDValue();
14596 
14597   // The vector lane must be a constant in the LD1LANE opcode.
14598   SDValue Lane;
14599   if (IsLaneOp) {
14600     Lane = N->getOperand(2);
14601     auto *LaneC = dyn_cast<ConstantSDNode>(Lane);
14602     if (!LaneC || LaneC->getZExtValue() >= VT.getVectorNumElements())
14603       return SDValue();
14604   }
14605 
14606   LoadSDNode *LoadSDN = cast<LoadSDNode>(LD);
14607   EVT MemVT = LoadSDN->getMemoryVT();
14608   // Check if memory operand is the same type as the vector element.
14609   if (MemVT != VT.getVectorElementType())
14610     return SDValue();
14611 
14612   // Check if there are other uses. If so, do not combine as it will introduce
14613   // an extra load.
14614   for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end(); UI != UE;
14615        ++UI) {
14616     if (UI.getUse().getResNo() == 1) // Ignore uses of the chain result.
14617       continue;
14618     if (*UI != N)
14619       return SDValue();
14620   }
14621 
14622   SDValue Addr = LD->getOperand(1);
14623   SDValue Vector = N->getOperand(0);
14624   // Search for a use of the address operand that is an increment.
14625   for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), UE =
14626        Addr.getNode()->use_end(); UI != UE; ++UI) {
14627     SDNode *User = *UI;
14628     if (User->getOpcode() != ISD::ADD
14629         || UI.getUse().getResNo() != Addr.getResNo())
14630       continue;
14631 
14632     // If the increment is a constant, it must match the memory ref size.
14633     SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
14634     if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) {
14635       uint32_t IncVal = CInc->getZExtValue();
14636       unsigned NumBytes = VT.getScalarSizeInBits() / 8;
14637       if (IncVal != NumBytes)
14638         continue;
14639       Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
14640     }
14641 
14642     // To avoid cycle construction make sure that neither the load nor the add
14643     // are predecessors to each other or the Vector.
14644     SmallPtrSet<const SDNode *, 32> Visited;
14645     SmallVector<const SDNode *, 16> Worklist;
14646     Visited.insert(Addr.getNode());
14647     Worklist.push_back(User);
14648     Worklist.push_back(LD);
14649     Worklist.push_back(Vector.getNode());
14650     if (SDNode::hasPredecessorHelper(LD, Visited, Worklist) ||
14651         SDNode::hasPredecessorHelper(User, Visited, Worklist))
14652       continue;
14653 
14654     SmallVector<SDValue, 8> Ops;
14655     Ops.push_back(LD->getOperand(0));  // Chain
14656     if (IsLaneOp) {
14657       Ops.push_back(Vector);           // The vector to be inserted
14658       Ops.push_back(Lane);             // The lane to be inserted in the vector
14659     }
14660     Ops.push_back(Addr);
14661     Ops.push_back(Inc);
14662 
14663     EVT Tys[3] = { VT, MVT::i64, MVT::Other };
14664     SDVTList SDTys = DAG.getVTList(Tys);
14665     unsigned NewOp = IsLaneOp ? AArch64ISD::LD1LANEpost : AArch64ISD::LD1DUPpost;
14666     SDValue UpdN = DAG.getMemIntrinsicNode(NewOp, SDLoc(N), SDTys, Ops,
14667                                            MemVT,
14668                                            LoadSDN->getMemOperand());
14669 
14670     // Update the uses.
14671     SDValue NewResults[] = {
14672         SDValue(LD, 0),            // The result of load
14673         SDValue(UpdN.getNode(), 2) // Chain
14674     };
14675     DCI.CombineTo(LD, NewResults);
14676     DCI.CombineTo(N, SDValue(UpdN.getNode(), 0));     // Dup/Inserted Result
14677     DCI.CombineTo(User, SDValue(UpdN.getNode(), 1));  // Write back register
14678 
14679     break;
14680   }
14681   return SDValue();
14682 }
14683 
14684 /// Simplify ``Addr`` given that the top byte of it is ignored by HW during
14685 /// address translation.
14686 static bool performTBISimplification(SDValue Addr,
14687                                      TargetLowering::DAGCombinerInfo &DCI,
14688                                      SelectionDAG &DAG) {
14689   APInt DemandedMask = APInt::getLowBitsSet(64, 56);
14690   KnownBits Known;
14691   TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
14692                                         !DCI.isBeforeLegalizeOps());
14693   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
14694   if (TLI.SimplifyDemandedBits(Addr, DemandedMask, Known, TLO)) {
14695     DCI.CommitTargetLoweringOpt(TLO);
14696     return true;
14697   }
14698   return false;
14699 }
14700 
14701 static SDValue performSTORECombine(SDNode *N,
14702                                    TargetLowering::DAGCombinerInfo &DCI,
14703                                    SelectionDAG &DAG,
14704                                    const AArch64Subtarget *Subtarget) {
14705   if (SDValue Split = splitStores(N, DCI, DAG, Subtarget))
14706     return Split;
14707 
14708   if (Subtarget->supportsAddressTopByteIgnored() &&
14709       performTBISimplification(N->getOperand(2), DCI, DAG))
14710     return SDValue(N, 0);
14711 
14712   return SDValue();
14713 }
14714 
14715 /// Target-specific DAG combine function for NEON load/store intrinsics
14716 /// to merge base address updates.
14717 static SDValue performNEONPostLDSTCombine(SDNode *N,
14718                                           TargetLowering::DAGCombinerInfo &DCI,
14719                                           SelectionDAG &DAG) {
14720   if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14721     return SDValue();
14722 
14723   unsigned AddrOpIdx = N->getNumOperands() - 1;
14724   SDValue Addr = N->getOperand(AddrOpIdx);
14725 
14726   // Search for a use of the address operand that is an increment.
14727   for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
14728        UE = Addr.getNode()->use_end(); UI != UE; ++UI) {
14729     SDNode *User = *UI;
14730     if (User->getOpcode() != ISD::ADD ||
14731         UI.getUse().getResNo() != Addr.getResNo())
14732       continue;
14733 
14734     // Check that the add is independent of the load/store.  Otherwise, folding
14735     // it would create a cycle.
14736     SmallPtrSet<const SDNode *, 32> Visited;
14737     SmallVector<const SDNode *, 16> Worklist;
14738     Visited.insert(Addr.getNode());
14739     Worklist.push_back(N);
14740     Worklist.push_back(User);
14741     if (SDNode::hasPredecessorHelper(N, Visited, Worklist) ||
14742         SDNode::hasPredecessorHelper(User, Visited, Worklist))
14743       continue;
14744 
14745     // Find the new opcode for the updating load/store.
14746     bool IsStore = false;
14747     bool IsLaneOp = false;
14748     bool IsDupOp = false;
14749     unsigned NewOpc = 0;
14750     unsigned NumVecs = 0;
14751     unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
14752     switch (IntNo) {
14753     default: llvm_unreachable("unexpected intrinsic for Neon base update");
14754     case Intrinsic::aarch64_neon_ld2:       NewOpc = AArch64ISD::LD2post;
14755       NumVecs = 2; break;
14756     case Intrinsic::aarch64_neon_ld3:       NewOpc = AArch64ISD::LD3post;
14757       NumVecs = 3; break;
14758     case Intrinsic::aarch64_neon_ld4:       NewOpc = AArch64ISD::LD4post;
14759       NumVecs = 4; break;
14760     case Intrinsic::aarch64_neon_st2:       NewOpc = AArch64ISD::ST2post;
14761       NumVecs = 2; IsStore = true; break;
14762     case Intrinsic::aarch64_neon_st3:       NewOpc = AArch64ISD::ST3post;
14763       NumVecs = 3; IsStore = true; break;
14764     case Intrinsic::aarch64_neon_st4:       NewOpc = AArch64ISD::ST4post;
14765       NumVecs = 4; IsStore = true; break;
14766     case Intrinsic::aarch64_neon_ld1x2:     NewOpc = AArch64ISD::LD1x2post;
14767       NumVecs = 2; break;
14768     case Intrinsic::aarch64_neon_ld1x3:     NewOpc = AArch64ISD::LD1x3post;
14769       NumVecs = 3; break;
14770     case Intrinsic::aarch64_neon_ld1x4:     NewOpc = AArch64ISD::LD1x4post;
14771       NumVecs = 4; break;
14772     case Intrinsic::aarch64_neon_st1x2:     NewOpc = AArch64ISD::ST1x2post;
14773       NumVecs = 2; IsStore = true; break;
14774     case Intrinsic::aarch64_neon_st1x3:     NewOpc = AArch64ISD::ST1x3post;
14775       NumVecs = 3; IsStore = true; break;
14776     case Intrinsic::aarch64_neon_st1x4:     NewOpc = AArch64ISD::ST1x4post;
14777       NumVecs = 4; IsStore = true; break;
14778     case Intrinsic::aarch64_neon_ld2r:      NewOpc = AArch64ISD::LD2DUPpost;
14779       NumVecs = 2; IsDupOp = true; break;
14780     case Intrinsic::aarch64_neon_ld3r:      NewOpc = AArch64ISD::LD3DUPpost;
14781       NumVecs = 3; IsDupOp = true; break;
14782     case Intrinsic::aarch64_neon_ld4r:      NewOpc = AArch64ISD::LD4DUPpost;
14783       NumVecs = 4; IsDupOp = true; break;
14784     case Intrinsic::aarch64_neon_ld2lane:   NewOpc = AArch64ISD::LD2LANEpost;
14785       NumVecs = 2; IsLaneOp = true; break;
14786     case Intrinsic::aarch64_neon_ld3lane:   NewOpc = AArch64ISD::LD3LANEpost;
14787       NumVecs = 3; IsLaneOp = true; break;
14788     case Intrinsic::aarch64_neon_ld4lane:   NewOpc = AArch64ISD::LD4LANEpost;
14789       NumVecs = 4; IsLaneOp = true; break;
14790     case Intrinsic::aarch64_neon_st2lane:   NewOpc = AArch64ISD::ST2LANEpost;
14791       NumVecs = 2; IsStore = true; IsLaneOp = true; break;
14792     case Intrinsic::aarch64_neon_st3lane:   NewOpc = AArch64ISD::ST3LANEpost;
14793       NumVecs = 3; IsStore = true; IsLaneOp = true; break;
14794     case Intrinsic::aarch64_neon_st4lane:   NewOpc = AArch64ISD::ST4LANEpost;
14795       NumVecs = 4; IsStore = true; IsLaneOp = true; break;
14796     }
14797 
14798     EVT VecTy;
14799     if (IsStore)
14800       VecTy = N->getOperand(2).getValueType();
14801     else
14802       VecTy = N->getValueType(0);
14803 
14804     // If the increment is a constant, it must match the memory ref size.
14805     SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
14806     if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) {
14807       uint32_t IncVal = CInc->getZExtValue();
14808       unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
14809       if (IsLaneOp || IsDupOp)
14810         NumBytes /= VecTy.getVectorNumElements();
14811       if (IncVal != NumBytes)
14812         continue;
14813       Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
14814     }
14815     SmallVector<SDValue, 8> Ops;
14816     Ops.push_back(N->getOperand(0)); // Incoming chain
14817     // Load lane and store have vector list as input.
14818     if (IsLaneOp || IsStore)
14819       for (unsigned i = 2; i < AddrOpIdx; ++i)
14820         Ops.push_back(N->getOperand(i));
14821     Ops.push_back(Addr); // Base register
14822     Ops.push_back(Inc);
14823 
14824     // Return Types.
14825     EVT Tys[6];
14826     unsigned NumResultVecs = (IsStore ? 0 : NumVecs);
14827     unsigned n;
14828     for (n = 0; n < NumResultVecs; ++n)
14829       Tys[n] = VecTy;
14830     Tys[n++] = MVT::i64;  // Type of write back register
14831     Tys[n] = MVT::Other;  // Type of the chain
14832     SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumResultVecs + 2));
14833 
14834     MemIntrinsicSDNode *MemInt = cast<MemIntrinsicSDNode>(N);
14835     SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, SDLoc(N), SDTys, Ops,
14836                                            MemInt->getMemoryVT(),
14837                                            MemInt->getMemOperand());
14838 
14839     // Update the uses.
14840     std::vector<SDValue> NewResults;
14841     for (unsigned i = 0; i < NumResultVecs; ++i) {
14842       NewResults.push_back(SDValue(UpdN.getNode(), i));
14843     }
14844     NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1));
14845     DCI.CombineTo(N, NewResults);
14846     DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
14847 
14848     break;
14849   }
14850   return SDValue();
14851 }
14852 
14853 // Checks to see if the value is the prescribed width and returns information
14854 // about its extension mode.
14855 static
14856 bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) {
14857   ExtType = ISD::NON_EXTLOAD;
14858   switch(V.getNode()->getOpcode()) {
14859   default:
14860     return false;
14861   case ISD::LOAD: {
14862     LoadSDNode *LoadNode = cast<LoadSDNode>(V.getNode());
14863     if ((LoadNode->getMemoryVT() == MVT::i8 && width == 8)
14864        || (LoadNode->getMemoryVT() == MVT::i16 && width == 16)) {
14865       ExtType = LoadNode->getExtensionType();
14866       return true;
14867     }
14868     return false;
14869   }
14870   case ISD::AssertSext: {
14871     VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
14872     if ((TypeNode->getVT() == MVT::i8 && width == 8)
14873        || (TypeNode->getVT() == MVT::i16 && width == 16)) {
14874       ExtType = ISD::SEXTLOAD;
14875       return true;
14876     }
14877     return false;
14878   }
14879   case ISD::AssertZext: {
14880     VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
14881     if ((TypeNode->getVT() == MVT::i8 && width == 8)
14882        || (TypeNode->getVT() == MVT::i16 && width == 16)) {
14883       ExtType = ISD::ZEXTLOAD;
14884       return true;
14885     }
14886     return false;
14887   }
14888   case ISD::Constant:
14889   case ISD::TargetConstant: {
14890     return std::abs(cast<ConstantSDNode>(V.getNode())->getSExtValue()) <
14891            1LL << (width - 1);
14892   }
14893   }
14894 
14895   return true;
14896 }
14897 
14898 // This function does a whole lot of voodoo to determine if the tests are
14899 // equivalent without and with a mask. Essentially what happens is that given a
14900 // DAG resembling:
14901 //
14902 //  +-------------+ +-------------+ +-------------+ +-------------+
14903 //  |    Input    | | AddConstant | | CompConstant| |     CC      |
14904 //  +-------------+ +-------------+ +-------------+ +-------------+
14905 //           |           |           |               |
14906 //           V           V           |    +----------+
14907 //          +-------------+  +----+  |    |
14908 //          |     ADD     |  |0xff|  |    |
14909 //          +-------------+  +----+  |    |
14910 //                  |           |    |    |
14911 //                  V           V    |    |
14912 //                 +-------------+   |    |
14913 //                 |     AND     |   |    |
14914 //                 +-------------+   |    |
14915 //                      |            |    |
14916 //                      +-----+      |    |
14917 //                            |      |    |
14918 //                            V      V    V
14919 //                           +-------------+
14920 //                           |     CMP     |
14921 //                           +-------------+
14922 //
14923 // The AND node may be safely removed for some combinations of inputs. In
14924 // particular we need to take into account the extension type of the Input,
14925 // the exact values of AddConstant, CompConstant, and CC, along with the nominal
14926 // width of the input (this can work for any width inputs, the above graph is
14927 // specific to 8 bits.
14928 //
14929 // The specific equations were worked out by generating output tables for each
14930 // AArch64CC value in terms of and AddConstant (w1), CompConstant(w2). The
14931 // problem was simplified by working with 4 bit inputs, which means we only
14932 // needed to reason about 24 distinct bit patterns: 8 patterns unique to zero
14933 // extension (8,15), 8 patterns unique to sign extensions (-8,-1), and 8
14934 // patterns present in both extensions (0,7). For every distinct set of
14935 // AddConstant and CompConstants bit patterns we can consider the masked and
14936 // unmasked versions to be equivalent if the result of this function is true for
14937 // all 16 distinct bit patterns of for the current extension type of Input (w0).
14938 //
14939 //   sub      w8, w0, w1
14940 //   and      w10, w8, #0x0f
14941 //   cmp      w8, w2
14942 //   cset     w9, AArch64CC
14943 //   cmp      w10, w2
14944 //   cset     w11, AArch64CC
14945 //   cmp      w9, w11
14946 //   cset     w0, eq
14947 //   ret
14948 //
14949 // Since the above function shows when the outputs are equivalent it defines
14950 // when it is safe to remove the AND. Unfortunately it only runs on AArch64 and
14951 // would be expensive to run during compiles. The equations below were written
14952 // in a test harness that confirmed they gave equivalent outputs to the above
14953 // for all inputs function, so they can be used determine if the removal is
14954 // legal instead.
14955 //
14956 // isEquivalentMaskless() is the code for testing if the AND can be removed
14957 // factored out of the DAG recognition as the DAG can take several forms.
14958 
14959 static bool isEquivalentMaskless(unsigned CC, unsigned width,
14960                                  ISD::LoadExtType ExtType, int AddConstant,
14961                                  int CompConstant) {
14962   // By being careful about our equations and only writing the in term
14963   // symbolic values and well known constants (0, 1, -1, MaxUInt) we can
14964   // make them generally applicable to all bit widths.
14965   int MaxUInt = (1 << width);
14966 
14967   // For the purposes of these comparisons sign extending the type is
14968   // equivalent to zero extending the add and displacing it by half the integer
14969   // width. Provided we are careful and make sure our equations are valid over
14970   // the whole range we can just adjust the input and avoid writing equations
14971   // for sign extended inputs.
14972   if (ExtType == ISD::SEXTLOAD)
14973     AddConstant -= (1 << (width-1));
14974 
14975   switch(CC) {
14976   case AArch64CC::LE:
14977   case AArch64CC::GT:
14978     if ((AddConstant == 0) ||
14979         (CompConstant == MaxUInt - 1 && AddConstant < 0) ||
14980         (AddConstant >= 0 && CompConstant < 0) ||
14981         (AddConstant <= 0 && CompConstant <= 0 && CompConstant < AddConstant))
14982       return true;
14983     break;
14984   case AArch64CC::LT:
14985   case AArch64CC::GE:
14986     if ((AddConstant == 0) ||
14987         (AddConstant >= 0 && CompConstant <= 0) ||
14988         (AddConstant <= 0 && CompConstant <= 0 && CompConstant <= AddConstant))
14989       return true;
14990     break;
14991   case AArch64CC::HI:
14992   case AArch64CC::LS:
14993     if ((AddConstant >= 0 && CompConstant < 0) ||
14994        (AddConstant <= 0 && CompConstant >= -1 &&
14995         CompConstant < AddConstant + MaxUInt))
14996       return true;
14997    break;
14998   case AArch64CC::PL:
14999   case AArch64CC::MI:
15000     if ((AddConstant == 0) ||
15001         (AddConstant > 0 && CompConstant <= 0) ||
15002         (AddConstant < 0 && CompConstant <= AddConstant))
15003       return true;
15004     break;
15005   case AArch64CC::LO:
15006   case AArch64CC::HS:
15007     if ((AddConstant >= 0 && CompConstant <= 0) ||
15008         (AddConstant <= 0 && CompConstant >= 0 &&
15009          CompConstant <= AddConstant + MaxUInt))
15010       return true;
15011     break;
15012   case AArch64CC::EQ:
15013   case AArch64CC::NE:
15014     if ((AddConstant > 0 && CompConstant < 0) ||
15015         (AddConstant < 0 && CompConstant >= 0 &&
15016          CompConstant < AddConstant + MaxUInt) ||
15017         (AddConstant >= 0 && CompConstant >= 0 &&
15018          CompConstant >= AddConstant) ||
15019         (AddConstant <= 0 && CompConstant < 0 && CompConstant < AddConstant))
15020       return true;
15021     break;
15022   case AArch64CC::VS:
15023   case AArch64CC::VC:
15024   case AArch64CC::AL:
15025   case AArch64CC::NV:
15026     return true;
15027   case AArch64CC::Invalid:
15028     break;
15029   }
15030 
15031   return false;
15032 }
15033 
15034 static
15035 SDValue performCONDCombine(SDNode *N,
15036                            TargetLowering::DAGCombinerInfo &DCI,
15037                            SelectionDAG &DAG, unsigned CCIndex,
15038                            unsigned CmpIndex) {
15039   unsigned CC = cast<ConstantSDNode>(N->getOperand(CCIndex))->getSExtValue();
15040   SDNode *SubsNode = N->getOperand(CmpIndex).getNode();
15041   unsigned CondOpcode = SubsNode->getOpcode();
15042 
15043   if (CondOpcode != AArch64ISD::SUBS)
15044     return SDValue();
15045 
15046   // There is a SUBS feeding this condition. Is it fed by a mask we can
15047   // use?
15048 
15049   SDNode *AndNode = SubsNode->getOperand(0).getNode();
15050   unsigned MaskBits = 0;
15051 
15052   if (AndNode->getOpcode() != ISD::AND)
15053     return SDValue();
15054 
15055   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(AndNode->getOperand(1))) {
15056     uint32_t CNV = CN->getZExtValue();
15057     if (CNV == 255)
15058       MaskBits = 8;
15059     else if (CNV == 65535)
15060       MaskBits = 16;
15061   }
15062 
15063   if (!MaskBits)
15064     return SDValue();
15065 
15066   SDValue AddValue = AndNode->getOperand(0);
15067 
15068   if (AddValue.getOpcode() != ISD::ADD)
15069     return SDValue();
15070 
15071   // The basic dag structure is correct, grab the inputs and validate them.
15072 
15073   SDValue AddInputValue1 = AddValue.getNode()->getOperand(0);
15074   SDValue AddInputValue2 = AddValue.getNode()->getOperand(1);
15075   SDValue SubsInputValue = SubsNode->getOperand(1);
15076 
15077   // The mask is present and the provenance of all the values is a smaller type,
15078   // lets see if the mask is superfluous.
15079 
15080   if (!isa<ConstantSDNode>(AddInputValue2.getNode()) ||
15081       !isa<ConstantSDNode>(SubsInputValue.getNode()))
15082     return SDValue();
15083 
15084   ISD::LoadExtType ExtType;
15085 
15086   if (!checkValueWidth(SubsInputValue, MaskBits, ExtType) ||
15087       !checkValueWidth(AddInputValue2, MaskBits, ExtType) ||
15088       !checkValueWidth(AddInputValue1, MaskBits, ExtType) )
15089     return SDValue();
15090 
15091   if(!isEquivalentMaskless(CC, MaskBits, ExtType,
15092                 cast<ConstantSDNode>(AddInputValue2.getNode())->getSExtValue(),
15093                 cast<ConstantSDNode>(SubsInputValue.getNode())->getSExtValue()))
15094     return SDValue();
15095 
15096   // The AND is not necessary, remove it.
15097 
15098   SDVTList VTs = DAG.getVTList(SubsNode->getValueType(0),
15099                                SubsNode->getValueType(1));
15100   SDValue Ops[] = { AddValue, SubsNode->getOperand(1) };
15101 
15102   SDValue NewValue = DAG.getNode(CondOpcode, SDLoc(SubsNode), VTs, Ops);
15103   DAG.ReplaceAllUsesWith(SubsNode, NewValue.getNode());
15104 
15105   return SDValue(N, 0);
15106 }
15107 
15108 // Optimize compare with zero and branch.
15109 static SDValue performBRCONDCombine(SDNode *N,
15110                                     TargetLowering::DAGCombinerInfo &DCI,
15111                                     SelectionDAG &DAG) {
15112   MachineFunction &MF = DAG.getMachineFunction();
15113   // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z instructions
15114   // will not be produced, as they are conditional branch instructions that do
15115   // not set flags.
15116   if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
15117     return SDValue();
15118 
15119   if (SDValue NV = performCONDCombine(N, DCI, DAG, 2, 3))
15120     N = NV.getNode();
15121   SDValue Chain = N->getOperand(0);
15122   SDValue Dest = N->getOperand(1);
15123   SDValue CCVal = N->getOperand(2);
15124   SDValue Cmp = N->getOperand(3);
15125 
15126   assert(isa<ConstantSDNode>(CCVal) && "Expected a ConstantSDNode here!");
15127   unsigned CC = cast<ConstantSDNode>(CCVal)->getZExtValue();
15128   if (CC != AArch64CC::EQ && CC != AArch64CC::NE)
15129     return SDValue();
15130 
15131   unsigned CmpOpc = Cmp.getOpcode();
15132   if (CmpOpc != AArch64ISD::ADDS && CmpOpc != AArch64ISD::SUBS)
15133     return SDValue();
15134 
15135   // Only attempt folding if there is only one use of the flag and no use of the
15136   // value.
15137   if (!Cmp->hasNUsesOfValue(0, 0) || !Cmp->hasNUsesOfValue(1, 1))
15138     return SDValue();
15139 
15140   SDValue LHS = Cmp.getOperand(0);
15141   SDValue RHS = Cmp.getOperand(1);
15142 
15143   assert(LHS.getValueType() == RHS.getValueType() &&
15144          "Expected the value type to be the same for both operands!");
15145   if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
15146     return SDValue();
15147 
15148   if (isNullConstant(LHS))
15149     std::swap(LHS, RHS);
15150 
15151   if (!isNullConstant(RHS))
15152     return SDValue();
15153 
15154   if (LHS.getOpcode() == ISD::SHL || LHS.getOpcode() == ISD::SRA ||
15155       LHS.getOpcode() == ISD::SRL)
15156     return SDValue();
15157 
15158   // Fold the compare into the branch instruction.
15159   SDValue BR;
15160   if (CC == AArch64CC::EQ)
15161     BR = DAG.getNode(AArch64ISD::CBZ, SDLoc(N), MVT::Other, Chain, LHS, Dest);
15162   else
15163     BR = DAG.getNode(AArch64ISD::CBNZ, SDLoc(N), MVT::Other, Chain, LHS, Dest);
15164 
15165   // Do not add new nodes to DAG combiner worklist.
15166   DCI.CombineTo(N, BR, false);
15167 
15168   return SDValue();
15169 }
15170 
15171 // Optimize some simple tbz/tbnz cases.  Returns the new operand and bit to test
15172 // as well as whether the test should be inverted.  This code is required to
15173 // catch these cases (as opposed to standard dag combines) because
15174 // AArch64ISD::TBZ is matched during legalization.
15175 static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert,
15176                                  SelectionDAG &DAG) {
15177 
15178   if (!Op->hasOneUse())
15179     return Op;
15180 
15181   // We don't handle undef/constant-fold cases below, as they should have
15182   // already been taken care of (e.g. and of 0, test of undefined shifted bits,
15183   // etc.)
15184 
15185   // (tbz (trunc x), b) -> (tbz x, b)
15186   // This case is just here to enable more of the below cases to be caught.
15187   if (Op->getOpcode() == ISD::TRUNCATE &&
15188       Bit < Op->getValueType(0).getSizeInBits()) {
15189     return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15190   }
15191 
15192   // (tbz (any_ext x), b) -> (tbz x, b) if we don't use the extended bits.
15193   if (Op->getOpcode() == ISD::ANY_EXTEND &&
15194       Bit < Op->getOperand(0).getValueSizeInBits()) {
15195     return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15196   }
15197 
15198   if (Op->getNumOperands() != 2)
15199     return Op;
15200 
15201   auto *C = dyn_cast<ConstantSDNode>(Op->getOperand(1));
15202   if (!C)
15203     return Op;
15204 
15205   switch (Op->getOpcode()) {
15206   default:
15207     return Op;
15208 
15209   // (tbz (and x, m), b) -> (tbz x, b)
15210   case ISD::AND:
15211     if ((C->getZExtValue() >> Bit) & 1)
15212       return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15213     return Op;
15214 
15215   // (tbz (shl x, c), b) -> (tbz x, b-c)
15216   case ISD::SHL:
15217     if (C->getZExtValue() <= Bit &&
15218         (Bit - C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
15219       Bit = Bit - C->getZExtValue();
15220       return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15221     }
15222     return Op;
15223 
15224   // (tbz (sra x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits in x
15225   case ISD::SRA:
15226     Bit = Bit + C->getZExtValue();
15227     if (Bit >= Op->getValueType(0).getSizeInBits())
15228       Bit = Op->getValueType(0).getSizeInBits() - 1;
15229     return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15230 
15231   // (tbz (srl x, c), b) -> (tbz x, b+c)
15232   case ISD::SRL:
15233     if ((Bit + C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
15234       Bit = Bit + C->getZExtValue();
15235       return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15236     }
15237     return Op;
15238 
15239   // (tbz (xor x, -1), b) -> (tbnz x, b)
15240   case ISD::XOR:
15241     if ((C->getZExtValue() >> Bit) & 1)
15242       Invert = !Invert;
15243     return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15244   }
15245 }
15246 
15247 // Optimize test single bit zero/non-zero and branch.
15248 static SDValue performTBZCombine(SDNode *N,
15249                                  TargetLowering::DAGCombinerInfo &DCI,
15250                                  SelectionDAG &DAG) {
15251   unsigned Bit = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
15252   bool Invert = false;
15253   SDValue TestSrc = N->getOperand(1);
15254   SDValue NewTestSrc = getTestBitOperand(TestSrc, Bit, Invert, DAG);
15255 
15256   if (TestSrc == NewTestSrc)
15257     return SDValue();
15258 
15259   unsigned NewOpc = N->getOpcode();
15260   if (Invert) {
15261     if (NewOpc == AArch64ISD::TBZ)
15262       NewOpc = AArch64ISD::TBNZ;
15263     else {
15264       assert(NewOpc == AArch64ISD::TBNZ);
15265       NewOpc = AArch64ISD::TBZ;
15266     }
15267   }
15268 
15269   SDLoc DL(N);
15270   return DAG.getNode(NewOpc, DL, MVT::Other, N->getOperand(0), NewTestSrc,
15271                      DAG.getConstant(Bit, DL, MVT::i64), N->getOperand(3));
15272 }
15273 
15274 // vselect (v1i1 setcc) ->
15275 //     vselect (v1iXX setcc)  (XX is the size of the compared operand type)
15276 // FIXME: Currently the type legalizer can't handle VSELECT having v1i1 as
15277 // condition. If it can legalize "VSELECT v1i1" correctly, no need to combine
15278 // such VSELECT.
15279 static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
15280   SDValue N0 = N->getOperand(0);
15281   EVT CCVT = N0.getValueType();
15282 
15283   // Check for sign pattern (VSELECT setgt, iN lhs, -1, 1, -1) and transform
15284   // into (OR (ASR lhs, N-1), 1), which requires less instructions for the
15285   // supported types.
15286   SDValue SetCC = N->getOperand(0);
15287   if (SetCC.getOpcode() == ISD::SETCC &&
15288       SetCC.getOperand(2) == DAG.getCondCode(ISD::SETGT)) {
15289     SDValue CmpLHS = SetCC.getOperand(0);
15290     EVT VT = CmpLHS.getValueType();
15291     SDNode *CmpRHS = SetCC.getOperand(1).getNode();
15292     SDNode *SplatLHS = N->getOperand(1).getNode();
15293     SDNode *SplatRHS = N->getOperand(2).getNode();
15294     APInt SplatLHSVal;
15295     if (CmpLHS.getValueType() == N->getOperand(1).getValueType() &&
15296         VT.isSimple() &&
15297         is_contained(
15298             makeArrayRef({MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
15299                           MVT::v2i32, MVT::v4i32, MVT::v2i64}),
15300             VT.getSimpleVT().SimpleTy) &&
15301         ISD::isConstantSplatVector(SplatLHS, SplatLHSVal) &&
15302         SplatLHSVal.isOneValue() && ISD::isConstantSplatVectorAllOnes(CmpRHS) &&
15303         ISD::isConstantSplatVectorAllOnes(SplatRHS)) {
15304       unsigned NumElts = VT.getVectorNumElements();
15305       SmallVector<SDValue, 8> Ops(
15306           NumElts, DAG.getConstant(VT.getScalarSizeInBits() - 1, SDLoc(N),
15307                                    VT.getScalarType()));
15308       SDValue Val = DAG.getBuildVector(VT, SDLoc(N), Ops);
15309 
15310       auto Shift = DAG.getNode(ISD::SRA, SDLoc(N), VT, CmpLHS, Val);
15311       auto Or = DAG.getNode(ISD::OR, SDLoc(N), VT, Shift, N->getOperand(1));
15312       return Or;
15313     }
15314   }
15315 
15316   if (N0.getOpcode() != ISD::SETCC ||
15317       CCVT.getVectorElementCount() != ElementCount::getFixed(1) ||
15318       CCVT.getVectorElementType() != MVT::i1)
15319     return SDValue();
15320 
15321   EVT ResVT = N->getValueType(0);
15322   EVT CmpVT = N0.getOperand(0).getValueType();
15323   // Only combine when the result type is of the same size as the compared
15324   // operands.
15325   if (ResVT.getSizeInBits() != CmpVT.getSizeInBits())
15326     return SDValue();
15327 
15328   SDValue IfTrue = N->getOperand(1);
15329   SDValue IfFalse = N->getOperand(2);
15330   SetCC = DAG.getSetCC(SDLoc(N), CmpVT.changeVectorElementTypeToInteger(),
15331                        N0.getOperand(0), N0.getOperand(1),
15332                        cast<CondCodeSDNode>(N0.getOperand(2))->get());
15333   return DAG.getNode(ISD::VSELECT, SDLoc(N), ResVT, SetCC,
15334                      IfTrue, IfFalse);
15335 }
15336 
15337 /// A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with
15338 /// the compare-mask instructions rather than going via NZCV, even if LHS and
15339 /// RHS are really scalar. This replaces any scalar setcc in the above pattern
15340 /// with a vector one followed by a DUP shuffle on the result.
15341 static SDValue performSelectCombine(SDNode *N,
15342                                     TargetLowering::DAGCombinerInfo &DCI) {
15343   SelectionDAG &DAG = DCI.DAG;
15344   SDValue N0 = N->getOperand(0);
15345   EVT ResVT = N->getValueType(0);
15346 
15347   if (N0.getOpcode() != ISD::SETCC)
15348     return SDValue();
15349 
15350   if (ResVT.isScalableVector())
15351     return SDValue();
15352 
15353   // Make sure the SETCC result is either i1 (initial DAG), or i32, the lowered
15354   // scalar SetCCResultType. We also don't expect vectors, because we assume
15355   // that selects fed by vector SETCCs are canonicalized to VSELECT.
15356   assert((N0.getValueType() == MVT::i1 || N0.getValueType() == MVT::i32) &&
15357          "Scalar-SETCC feeding SELECT has unexpected result type!");
15358 
15359   // If NumMaskElts == 0, the comparison is larger than select result. The
15360   // largest real NEON comparison is 64-bits per lane, which means the result is
15361   // at most 32-bits and an illegal vector. Just bail out for now.
15362   EVT SrcVT = N0.getOperand(0).getValueType();
15363 
15364   // Don't try to do this optimization when the setcc itself has i1 operands.
15365   // There are no legal vectors of i1, so this would be pointless.
15366   if (SrcVT == MVT::i1)
15367     return SDValue();
15368 
15369   int NumMaskElts = ResVT.getSizeInBits() / SrcVT.getSizeInBits();
15370   if (!ResVT.isVector() || NumMaskElts == 0)
15371     return SDValue();
15372 
15373   SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumMaskElts);
15374   EVT CCVT = SrcVT.changeVectorElementTypeToInteger();
15375 
15376   // Also bail out if the vector CCVT isn't the same size as ResVT.
15377   // This can happen if the SETCC operand size doesn't divide the ResVT size
15378   // (e.g., f64 vs v3f32).
15379   if (CCVT.getSizeInBits() != ResVT.getSizeInBits())
15380     return SDValue();
15381 
15382   // Make sure we didn't create illegal types, if we're not supposed to.
15383   assert(DCI.isBeforeLegalize() ||
15384          DAG.getTargetLoweringInfo().isTypeLegal(SrcVT));
15385 
15386   // First perform a vector comparison, where lane 0 is the one we're interested
15387   // in.
15388   SDLoc DL(N0);
15389   SDValue LHS =
15390       DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(0));
15391   SDValue RHS =
15392       DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(1));
15393   SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2));
15394 
15395   // Now duplicate the comparison mask we want across all other lanes.
15396   SmallVector<int, 8> DUPMask(CCVT.getVectorNumElements(), 0);
15397   SDValue Mask = DAG.getVectorShuffle(CCVT, DL, SetCC, SetCC, DUPMask);
15398   Mask = DAG.getNode(ISD::BITCAST, DL,
15399                      ResVT.changeVectorElementTypeToInteger(), Mask);
15400 
15401   return DAG.getSelect(DL, ResVT, Mask, N->getOperand(1), N->getOperand(2));
15402 }
15403 
15404 /// Get rid of unnecessary NVCASTs (that don't change the type).
15405 static SDValue performNVCASTCombine(SDNode *N) {
15406   if (N->getValueType(0) == N->getOperand(0).getValueType())
15407     return N->getOperand(0);
15408 
15409   return SDValue();
15410 }
15411 
15412 // If all users of the globaladdr are of the form (globaladdr + constant), find
15413 // the smallest constant, fold it into the globaladdr's offset and rewrite the
15414 // globaladdr as (globaladdr + constant) - constant.
15415 static SDValue performGlobalAddressCombine(SDNode *N, SelectionDAG &DAG,
15416                                            const AArch64Subtarget *Subtarget,
15417                                            const TargetMachine &TM) {
15418   auto *GN = cast<GlobalAddressSDNode>(N);
15419   if (Subtarget->ClassifyGlobalReference(GN->getGlobal(), TM) !=
15420       AArch64II::MO_NO_FLAG)
15421     return SDValue();
15422 
15423   uint64_t MinOffset = -1ull;
15424   for (SDNode *N : GN->uses()) {
15425     if (N->getOpcode() != ISD::ADD)
15426       return SDValue();
15427     auto *C = dyn_cast<ConstantSDNode>(N->getOperand(0));
15428     if (!C)
15429       C = dyn_cast<ConstantSDNode>(N->getOperand(1));
15430     if (!C)
15431       return SDValue();
15432     MinOffset = std::min(MinOffset, C->getZExtValue());
15433   }
15434   uint64_t Offset = MinOffset + GN->getOffset();
15435 
15436   // Require that the new offset is larger than the existing one. Otherwise, we
15437   // can end up oscillating between two possible DAGs, for example,
15438   // (add (add globaladdr + 10, -1), 1) and (add globaladdr + 9, 1).
15439   if (Offset <= uint64_t(GN->getOffset()))
15440     return SDValue();
15441 
15442   // Check whether folding this offset is legal. It must not go out of bounds of
15443   // the referenced object to avoid violating the code model, and must be
15444   // smaller than 2^21 because this is the largest offset expressible in all
15445   // object formats.
15446   //
15447   // This check also prevents us from folding negative offsets, which will end
15448   // up being treated in the same way as large positive ones. They could also
15449   // cause code model violations, and aren't really common enough to matter.
15450   if (Offset >= (1 << 21))
15451     return SDValue();
15452 
15453   const GlobalValue *GV = GN->getGlobal();
15454   Type *T = GV->getValueType();
15455   if (!T->isSized() ||
15456       Offset > GV->getParent()->getDataLayout().getTypeAllocSize(T))
15457     return SDValue();
15458 
15459   SDLoc DL(GN);
15460   SDValue Result = DAG.getGlobalAddress(GV, DL, MVT::i64, Offset);
15461   return DAG.getNode(ISD::SUB, DL, MVT::i64, Result,
15462                      DAG.getConstant(MinOffset, DL, MVT::i64));
15463 }
15464 
15465 // Turns the vector of indices into a vector of byte offstes by scaling Offset
15466 // by (BitWidth / 8).
15467 static SDValue getScaledOffsetForBitWidth(SelectionDAG &DAG, SDValue Offset,
15468                                           SDLoc DL, unsigned BitWidth) {
15469   assert(Offset.getValueType().isScalableVector() &&
15470          "This method is only for scalable vectors of offsets");
15471 
15472   SDValue Shift = DAG.getConstant(Log2_32(BitWidth / 8), DL, MVT::i64);
15473   SDValue SplatShift = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Shift);
15474 
15475   return DAG.getNode(ISD::SHL, DL, MVT::nxv2i64, Offset, SplatShift);
15476 }
15477 
15478 /// Check if the value of \p OffsetInBytes can be used as an immediate for
15479 /// the gather load/prefetch and scatter store instructions with vector base and
15480 /// immediate offset addressing mode:
15481 ///
15482 ///      [<Zn>.[S|D]{, #<imm>}]
15483 ///
15484 /// where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
15485 
15486 inline static bool isValidImmForSVEVecImmAddrMode(unsigned OffsetInBytes,
15487                                                   unsigned ScalarSizeInBytes) {
15488   // The immediate is not a multiple of the scalar size.
15489   if (OffsetInBytes % ScalarSizeInBytes)
15490     return false;
15491 
15492   // The immediate is out of range.
15493   if (OffsetInBytes / ScalarSizeInBytes > 31)
15494     return false;
15495 
15496   return true;
15497 }
15498 
15499 /// Check if the value of \p Offset represents a valid immediate for the SVE
15500 /// gather load/prefetch and scatter store instructiona with vector base and
15501 /// immediate offset addressing mode:
15502 ///
15503 ///      [<Zn>.[S|D]{, #<imm>}]
15504 ///
15505 /// where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
15506 static bool isValidImmForSVEVecImmAddrMode(SDValue Offset,
15507                                            unsigned ScalarSizeInBytes) {
15508   ConstantSDNode *OffsetConst = dyn_cast<ConstantSDNode>(Offset.getNode());
15509   return OffsetConst && isValidImmForSVEVecImmAddrMode(
15510                             OffsetConst->getZExtValue(), ScalarSizeInBytes);
15511 }
15512 
15513 static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG,
15514                                           unsigned Opcode,
15515                                           bool OnlyPackedOffsets = true) {
15516   const SDValue Src = N->getOperand(2);
15517   const EVT SrcVT = Src->getValueType(0);
15518   assert(SrcVT.isScalableVector() &&
15519          "Scatter stores are only possible for SVE vectors");
15520 
15521   SDLoc DL(N);
15522   MVT SrcElVT = SrcVT.getVectorElementType().getSimpleVT();
15523 
15524   // Make sure that source data will fit into an SVE register
15525   if (SrcVT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
15526     return SDValue();
15527 
15528   // For FPs, ACLE only supports _packed_ single and double precision types.
15529   if (SrcElVT.isFloatingPoint())
15530     if ((SrcVT != MVT::nxv4f32) && (SrcVT != MVT::nxv2f64))
15531       return SDValue();
15532 
15533   // Depending on the addressing mode, this is either a pointer or a vector of
15534   // pointers (that fits into one register)
15535   SDValue Base = N->getOperand(4);
15536   // Depending on the addressing mode, this is either a single offset or a
15537   // vector of offsets  (that fits into one register)
15538   SDValue Offset = N->getOperand(5);
15539 
15540   // For "scalar + vector of indices", just scale the indices. This only
15541   // applies to non-temporal scatters because there's no instruction that takes
15542   // indicies.
15543   if (Opcode == AArch64ISD::SSTNT1_INDEX_PRED) {
15544     Offset =
15545         getScaledOffsetForBitWidth(DAG, Offset, DL, SrcElVT.getSizeInBits());
15546     Opcode = AArch64ISD::SSTNT1_PRED;
15547   }
15548 
15549   // In the case of non-temporal gather loads there's only one SVE instruction
15550   // per data-size: "scalar + vector", i.e.
15551   //    * stnt1{b|h|w|d} { z0.s }, p0/z, [z0.s, x0]
15552   // Since we do have intrinsics that allow the arguments to be in a different
15553   // order, we may need to swap them to match the spec.
15554   if (Opcode == AArch64ISD::SSTNT1_PRED && Offset.getValueType().isVector())
15555     std::swap(Base, Offset);
15556 
15557   // SST1_IMM requires that the offset is an immediate that is:
15558   //    * a multiple of #SizeInBytes,
15559   //    * in the range [0, 31 x #SizeInBytes],
15560   // where #SizeInBytes is the size in bytes of the stored items. For
15561   // immediates outside that range and non-immediate scalar offsets use SST1 or
15562   // SST1_UXTW instead.
15563   if (Opcode == AArch64ISD::SST1_IMM_PRED) {
15564     if (!isValidImmForSVEVecImmAddrMode(Offset,
15565                                         SrcVT.getScalarSizeInBits() / 8)) {
15566       if (MVT::nxv4i32 == Base.getValueType().getSimpleVT().SimpleTy)
15567         Opcode = AArch64ISD::SST1_UXTW_PRED;
15568       else
15569         Opcode = AArch64ISD::SST1_PRED;
15570 
15571       std::swap(Base, Offset);
15572     }
15573   }
15574 
15575   auto &TLI = DAG.getTargetLoweringInfo();
15576   if (!TLI.isTypeLegal(Base.getValueType()))
15577     return SDValue();
15578 
15579   // Some scatter store variants allow unpacked offsets, but only as nxv2i32
15580   // vectors. These are implicitly sign (sxtw) or zero (zxtw) extend to
15581   // nxv2i64. Legalize accordingly.
15582   if (!OnlyPackedOffsets &&
15583       Offset.getValueType().getSimpleVT().SimpleTy == MVT::nxv2i32)
15584     Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0);
15585 
15586   if (!TLI.isTypeLegal(Offset.getValueType()))
15587     return SDValue();
15588 
15589   // Source value type that is representable in hardware
15590   EVT HwSrcVt = getSVEContainerType(SrcVT);
15591 
15592   // Keep the original type of the input data to store - this is needed to be
15593   // able to select the correct instruction, e.g. ST1B, ST1H, ST1W and ST1D. For
15594   // FP values we want the integer equivalent, so just use HwSrcVt.
15595   SDValue InputVT = DAG.getValueType(SrcVT);
15596   if (SrcVT.isFloatingPoint())
15597     InputVT = DAG.getValueType(HwSrcVt);
15598 
15599   SDVTList VTs = DAG.getVTList(MVT::Other);
15600   SDValue SrcNew;
15601 
15602   if (Src.getValueType().isFloatingPoint())
15603     SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Src);
15604   else
15605     SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Src);
15606 
15607   SDValue Ops[] = {N->getOperand(0), // Chain
15608                    SrcNew,
15609                    N->getOperand(3), // Pg
15610                    Base,
15611                    Offset,
15612                    InputVT};
15613 
15614   return DAG.getNode(Opcode, DL, VTs, Ops);
15615 }
15616 
15617 static SDValue performGatherLoadCombine(SDNode *N, SelectionDAG &DAG,
15618                                         unsigned Opcode,
15619                                         bool OnlyPackedOffsets = true) {
15620   const EVT RetVT = N->getValueType(0);
15621   assert(RetVT.isScalableVector() &&
15622          "Gather loads are only possible for SVE vectors");
15623 
15624   SDLoc DL(N);
15625 
15626   // Make sure that the loaded data will fit into an SVE register
15627   if (RetVT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
15628     return SDValue();
15629 
15630   // Depending on the addressing mode, this is either a pointer or a vector of
15631   // pointers (that fits into one register)
15632   SDValue Base = N->getOperand(3);
15633   // Depending on the addressing mode, this is either a single offset or a
15634   // vector of offsets  (that fits into one register)
15635   SDValue Offset = N->getOperand(4);
15636 
15637   // For "scalar + vector of indices", just scale the indices. This only
15638   // applies to non-temporal gathers because there's no instruction that takes
15639   // indicies.
15640   if (Opcode == AArch64ISD::GLDNT1_INDEX_MERGE_ZERO) {
15641     Offset = getScaledOffsetForBitWidth(DAG, Offset, DL,
15642                                         RetVT.getScalarSizeInBits());
15643     Opcode = AArch64ISD::GLDNT1_MERGE_ZERO;
15644   }
15645 
15646   // In the case of non-temporal gather loads there's only one SVE instruction
15647   // per data-size: "scalar + vector", i.e.
15648   //    * ldnt1{b|h|w|d} { z0.s }, p0/z, [z0.s, x0]
15649   // Since we do have intrinsics that allow the arguments to be in a different
15650   // order, we may need to swap them to match the spec.
15651   if (Opcode == AArch64ISD::GLDNT1_MERGE_ZERO &&
15652       Offset.getValueType().isVector())
15653     std::swap(Base, Offset);
15654 
15655   // GLD{FF}1_IMM requires that the offset is an immediate that is:
15656   //    * a multiple of #SizeInBytes,
15657   //    * in the range [0, 31 x #SizeInBytes],
15658   // where #SizeInBytes is the size in bytes of the loaded items. For
15659   // immediates outside that range and non-immediate scalar offsets use
15660   // GLD1_MERGE_ZERO or GLD1_UXTW_MERGE_ZERO instead.
15661   if (Opcode == AArch64ISD::GLD1_IMM_MERGE_ZERO ||
15662       Opcode == AArch64ISD::GLDFF1_IMM_MERGE_ZERO) {
15663     if (!isValidImmForSVEVecImmAddrMode(Offset,
15664                                         RetVT.getScalarSizeInBits() / 8)) {
15665       if (MVT::nxv4i32 == Base.getValueType().getSimpleVT().SimpleTy)
15666         Opcode = (Opcode == AArch64ISD::GLD1_IMM_MERGE_ZERO)
15667                      ? AArch64ISD::GLD1_UXTW_MERGE_ZERO
15668                      : AArch64ISD::GLDFF1_UXTW_MERGE_ZERO;
15669       else
15670         Opcode = (Opcode == AArch64ISD::GLD1_IMM_MERGE_ZERO)
15671                      ? AArch64ISD::GLD1_MERGE_ZERO
15672                      : AArch64ISD::GLDFF1_MERGE_ZERO;
15673 
15674       std::swap(Base, Offset);
15675     }
15676   }
15677 
15678   auto &TLI = DAG.getTargetLoweringInfo();
15679   if (!TLI.isTypeLegal(Base.getValueType()))
15680     return SDValue();
15681 
15682   // Some gather load variants allow unpacked offsets, but only as nxv2i32
15683   // vectors. These are implicitly sign (sxtw) or zero (zxtw) extend to
15684   // nxv2i64. Legalize accordingly.
15685   if (!OnlyPackedOffsets &&
15686       Offset.getValueType().getSimpleVT().SimpleTy == MVT::nxv2i32)
15687     Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0);
15688 
15689   // Return value type that is representable in hardware
15690   EVT HwRetVt = getSVEContainerType(RetVT);
15691 
15692   // Keep the original output value type around - this is needed to be able to
15693   // select the correct instruction, e.g. LD1B, LD1H, LD1W and LD1D. For FP
15694   // values we want the integer equivalent, so just use HwRetVT.
15695   SDValue OutVT = DAG.getValueType(RetVT);
15696   if (RetVT.isFloatingPoint())
15697     OutVT = DAG.getValueType(HwRetVt);
15698 
15699   SDVTList VTs = DAG.getVTList(HwRetVt, MVT::Other);
15700   SDValue Ops[] = {N->getOperand(0), // Chain
15701                    N->getOperand(2), // Pg
15702                    Base, Offset, OutVT};
15703 
15704   SDValue Load = DAG.getNode(Opcode, DL, VTs, Ops);
15705   SDValue LoadChain = SDValue(Load.getNode(), 1);
15706 
15707   if (RetVT.isInteger() && (RetVT != HwRetVt))
15708     Load = DAG.getNode(ISD::TRUNCATE, DL, RetVT, Load.getValue(0));
15709 
15710   // If the original return value was FP, bitcast accordingly. Doing it here
15711   // means that we can avoid adding TableGen patterns for FPs.
15712   if (RetVT.isFloatingPoint())
15713     Load = DAG.getNode(ISD::BITCAST, DL, RetVT, Load.getValue(0));
15714 
15715   return DAG.getMergeValues({Load, LoadChain}, DL);
15716 }
15717 
15718 static SDValue
15719 performSignExtendInRegCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
15720                               SelectionDAG &DAG) {
15721   SDLoc DL(N);
15722   SDValue Src = N->getOperand(0);
15723   unsigned Opc = Src->getOpcode();
15724 
15725   // Sign extend of an unsigned unpack -> signed unpack
15726   if (Opc == AArch64ISD::UUNPKHI || Opc == AArch64ISD::UUNPKLO) {
15727 
15728     unsigned SOpc = Opc == AArch64ISD::UUNPKHI ? AArch64ISD::SUNPKHI
15729                                                : AArch64ISD::SUNPKLO;
15730 
15731     // Push the sign extend to the operand of the unpack
15732     // This is necessary where, for example, the operand of the unpack
15733     // is another unpack:
15734     // 4i32 sign_extend_inreg (4i32 uunpklo(8i16 uunpklo (16i8 opnd)), from 4i8)
15735     // ->
15736     // 4i32 sunpklo (8i16 sign_extend_inreg(8i16 uunpklo (16i8 opnd), from 8i8)
15737     // ->
15738     // 4i32 sunpklo(8i16 sunpklo(16i8 opnd))
15739     SDValue ExtOp = Src->getOperand(0);
15740     auto VT = cast<VTSDNode>(N->getOperand(1))->getVT();
15741     EVT EltTy = VT.getVectorElementType();
15742     (void)EltTy;
15743 
15744     assert((EltTy == MVT::i8 || EltTy == MVT::i16 || EltTy == MVT::i32) &&
15745            "Sign extending from an invalid type");
15746 
15747     EVT ExtVT = VT.getDoubleNumVectorElementsVT(*DAG.getContext());
15748 
15749     SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ExtOp.getValueType(),
15750                               ExtOp, DAG.getValueType(ExtVT));
15751 
15752     return DAG.getNode(SOpc, DL, N->getValueType(0), Ext);
15753   }
15754 
15755   if (DCI.isBeforeLegalizeOps())
15756     return SDValue();
15757 
15758   if (!EnableCombineMGatherIntrinsics)
15759     return SDValue();
15760 
15761   // SVE load nodes (e.g. AArch64ISD::GLD1) are straightforward candidates
15762   // for DAG Combine with SIGN_EXTEND_INREG. Bail out for all other nodes.
15763   unsigned NewOpc;
15764   unsigned MemVTOpNum = 4;
15765   switch (Opc) {
15766   case AArch64ISD::LD1_MERGE_ZERO:
15767     NewOpc = AArch64ISD::LD1S_MERGE_ZERO;
15768     MemVTOpNum = 3;
15769     break;
15770   case AArch64ISD::LDNF1_MERGE_ZERO:
15771     NewOpc = AArch64ISD::LDNF1S_MERGE_ZERO;
15772     MemVTOpNum = 3;
15773     break;
15774   case AArch64ISD::LDFF1_MERGE_ZERO:
15775     NewOpc = AArch64ISD::LDFF1S_MERGE_ZERO;
15776     MemVTOpNum = 3;
15777     break;
15778   case AArch64ISD::GLD1_MERGE_ZERO:
15779     NewOpc = AArch64ISD::GLD1S_MERGE_ZERO;
15780     break;
15781   case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
15782     NewOpc = AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
15783     break;
15784   case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
15785     NewOpc = AArch64ISD::GLD1S_SXTW_MERGE_ZERO;
15786     break;
15787   case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
15788     NewOpc = AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO;
15789     break;
15790   case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
15791     NewOpc = AArch64ISD::GLD1S_UXTW_MERGE_ZERO;
15792     break;
15793   case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
15794     NewOpc = AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO;
15795     break;
15796   case AArch64ISD::GLD1_IMM_MERGE_ZERO:
15797     NewOpc = AArch64ISD::GLD1S_IMM_MERGE_ZERO;
15798     break;
15799   case AArch64ISD::GLDFF1_MERGE_ZERO:
15800     NewOpc = AArch64ISD::GLDFF1S_MERGE_ZERO;
15801     break;
15802   case AArch64ISD::GLDFF1_SCALED_MERGE_ZERO:
15803     NewOpc = AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO;
15804     break;
15805   case AArch64ISD::GLDFF1_SXTW_MERGE_ZERO:
15806     NewOpc = AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO;
15807     break;
15808   case AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO:
15809     NewOpc = AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO;
15810     break;
15811   case AArch64ISD::GLDFF1_UXTW_MERGE_ZERO:
15812     NewOpc = AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO;
15813     break;
15814   case AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO:
15815     NewOpc = AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO;
15816     break;
15817   case AArch64ISD::GLDFF1_IMM_MERGE_ZERO:
15818     NewOpc = AArch64ISD::GLDFF1S_IMM_MERGE_ZERO;
15819     break;
15820   case AArch64ISD::GLDNT1_MERGE_ZERO:
15821     NewOpc = AArch64ISD::GLDNT1S_MERGE_ZERO;
15822     break;
15823   default:
15824     return SDValue();
15825   }
15826 
15827   EVT SignExtSrcVT = cast<VTSDNode>(N->getOperand(1))->getVT();
15828   EVT SrcMemVT = cast<VTSDNode>(Src->getOperand(MemVTOpNum))->getVT();
15829 
15830   if ((SignExtSrcVT != SrcMemVT) || !Src.hasOneUse())
15831     return SDValue();
15832 
15833   EVT DstVT = N->getValueType(0);
15834   SDVTList VTs = DAG.getVTList(DstVT, MVT::Other);
15835 
15836   SmallVector<SDValue, 5> Ops;
15837   for (unsigned I = 0; I < Src->getNumOperands(); ++I)
15838     Ops.push_back(Src->getOperand(I));
15839 
15840   SDValue ExtLoad = DAG.getNode(NewOpc, SDLoc(N), VTs, Ops);
15841   DCI.CombineTo(N, ExtLoad);
15842   DCI.CombineTo(Src.getNode(), ExtLoad, ExtLoad.getValue(1));
15843 
15844   // Return N so it doesn't get rechecked
15845   return SDValue(N, 0);
15846 }
15847 
15848 /// Legalize the gather prefetch (scalar + vector addressing mode) when the
15849 /// offset vector is an unpacked 32-bit scalable vector. The other cases (Offset
15850 /// != nxv2i32) do not need legalization.
15851 static SDValue legalizeSVEGatherPrefetchOffsVec(SDNode *N, SelectionDAG &DAG) {
15852   const unsigned OffsetPos = 4;
15853   SDValue Offset = N->getOperand(OffsetPos);
15854 
15855   // Not an unpacked vector, bail out.
15856   if (Offset.getValueType().getSimpleVT().SimpleTy != MVT::nxv2i32)
15857     return SDValue();
15858 
15859   // Extend the unpacked offset vector to 64-bit lanes.
15860   SDLoc DL(N);
15861   Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset);
15862   SmallVector<SDValue, 5> Ops(N->op_begin(), N->op_end());
15863   // Replace the offset operand with the 64-bit one.
15864   Ops[OffsetPos] = Offset;
15865 
15866   return DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::Other), Ops);
15867 }
15868 
15869 /// Combines a node carrying the intrinsic
15870 /// `aarch64_sve_prf<T>_gather_scalar_offset` into a node that uses
15871 /// `aarch64_sve_prfb_gather_uxtw_index` when the scalar offset passed to
15872 /// `aarch64_sve_prf<T>_gather_scalar_offset` is not a valid immediate for the
15873 /// sve gather prefetch instruction with vector plus immediate addressing mode.
15874 static SDValue combineSVEPrefetchVecBaseImmOff(SDNode *N, SelectionDAG &DAG,
15875                                                unsigned ScalarSizeInBytes) {
15876   const unsigned ImmPos = 4, OffsetPos = 3;
15877   // No need to combine the node if the immediate is valid...
15878   if (isValidImmForSVEVecImmAddrMode(N->getOperand(ImmPos), ScalarSizeInBytes))
15879     return SDValue();
15880 
15881   // ...otherwise swap the offset base with the offset...
15882   SmallVector<SDValue, 5> Ops(N->op_begin(), N->op_end());
15883   std::swap(Ops[ImmPos], Ops[OffsetPos]);
15884   // ...and remap the intrinsic `aarch64_sve_prf<T>_gather_scalar_offset` to
15885   // `aarch64_sve_prfb_gather_uxtw_index`.
15886   SDLoc DL(N);
15887   Ops[1] = DAG.getConstant(Intrinsic::aarch64_sve_prfb_gather_uxtw_index, DL,
15888                            MVT::i64);
15889 
15890   return DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::Other), Ops);
15891 }
15892 
15893 SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
15894                                                  DAGCombinerInfo &DCI) const {
15895   SelectionDAG &DAG = DCI.DAG;
15896   switch (N->getOpcode()) {
15897   default:
15898     LLVM_DEBUG(dbgs() << "Custom combining: skipping\n");
15899     break;
15900   case ISD::ABS:
15901     return performABSCombine(N, DAG, DCI, Subtarget);
15902   case ISD::ADD:
15903   case ISD::SUB:
15904     return performAddSubCombine(N, DCI, DAG);
15905   case ISD::XOR:
15906     return performXorCombine(N, DAG, DCI, Subtarget);
15907   case ISD::MUL:
15908     return performMulCombine(N, DAG, DCI, Subtarget);
15909   case ISD::SINT_TO_FP:
15910   case ISD::UINT_TO_FP:
15911     return performIntToFpCombine(N, DAG, Subtarget);
15912   case ISD::FP_TO_SINT:
15913   case ISD::FP_TO_UINT:
15914     return performFpToIntCombine(N, DAG, DCI, Subtarget);
15915   case ISD::FDIV:
15916     return performFDivCombine(N, DAG, DCI, Subtarget);
15917   case ISD::OR:
15918     return performORCombine(N, DCI, Subtarget);
15919   case ISD::AND:
15920     return performANDCombine(N, DCI);
15921   case ISD::SRL:
15922     return performSRLCombine(N, DCI);
15923   case ISD::INTRINSIC_WO_CHAIN:
15924     return performIntrinsicCombine(N, DCI, Subtarget);
15925   case ISD::ANY_EXTEND:
15926   case ISD::ZERO_EXTEND:
15927   case ISD::SIGN_EXTEND:
15928     return performExtendCombine(N, DCI, DAG);
15929   case ISD::SIGN_EXTEND_INREG:
15930     return performSignExtendInRegCombine(N, DCI, DAG);
15931   case ISD::TRUNCATE:
15932     return performVectorTruncateCombine(N, DCI, DAG);
15933   case ISD::CONCAT_VECTORS:
15934     return performConcatVectorsCombine(N, DCI, DAG);
15935   case ISD::SELECT:
15936     return performSelectCombine(N, DCI);
15937   case ISD::VSELECT:
15938     return performVSelectCombine(N, DCI.DAG);
15939   case ISD::LOAD:
15940     if (performTBISimplification(N->getOperand(1), DCI, DAG))
15941       return SDValue(N, 0);
15942     break;
15943   case ISD::STORE:
15944     return performSTORECombine(N, DCI, DAG, Subtarget);
15945   case AArch64ISD::BRCOND:
15946     return performBRCONDCombine(N, DCI, DAG);
15947   case AArch64ISD::TBNZ:
15948   case AArch64ISD::TBZ:
15949     return performTBZCombine(N, DCI, DAG);
15950   case AArch64ISD::CSEL:
15951     return performCONDCombine(N, DCI, DAG, 2, 3);
15952   case AArch64ISD::DUP:
15953     return performPostLD1Combine(N, DCI, false);
15954   case AArch64ISD::NVCAST:
15955     return performNVCASTCombine(N);
15956   case AArch64ISD::UZP1:
15957     return performUzpCombine(N, DAG);
15958   case AArch64ISD::GLD1_MERGE_ZERO:
15959   case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
15960   case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
15961   case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
15962   case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
15963   case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
15964   case AArch64ISD::GLD1_IMM_MERGE_ZERO:
15965   case AArch64ISD::GLD1S_MERGE_ZERO:
15966   case AArch64ISD::GLD1S_SCALED_MERGE_ZERO:
15967   case AArch64ISD::GLD1S_UXTW_MERGE_ZERO:
15968   case AArch64ISD::GLD1S_SXTW_MERGE_ZERO:
15969   case AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO:
15970   case AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO:
15971   case AArch64ISD::GLD1S_IMM_MERGE_ZERO:
15972     return performGLD1Combine(N, DAG);
15973   case ISD::INSERT_VECTOR_ELT:
15974     return performPostLD1Combine(N, DCI, true);
15975   case ISD::EXTRACT_VECTOR_ELT:
15976     return performExtractVectorEltCombine(N, DAG);
15977   case ISD::VECREDUCE_ADD:
15978     return performVecReduceAddCombine(N, DCI.DAG, Subtarget);
15979   case ISD::INTRINSIC_VOID:
15980   case ISD::INTRINSIC_W_CHAIN:
15981     switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
15982     case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
15983       return combineSVEPrefetchVecBaseImmOff(N, DAG, 1 /*=ScalarSizeInBytes*/);
15984     case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
15985       return combineSVEPrefetchVecBaseImmOff(N, DAG, 2 /*=ScalarSizeInBytes*/);
15986     case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:
15987       return combineSVEPrefetchVecBaseImmOff(N, DAG, 4 /*=ScalarSizeInBytes*/);
15988     case Intrinsic::aarch64_sve_prfd_gather_scalar_offset:
15989       return combineSVEPrefetchVecBaseImmOff(N, DAG, 8 /*=ScalarSizeInBytes*/);
15990     case Intrinsic::aarch64_sve_prfb_gather_uxtw_index:
15991     case Intrinsic::aarch64_sve_prfb_gather_sxtw_index:
15992     case Intrinsic::aarch64_sve_prfh_gather_uxtw_index:
15993     case Intrinsic::aarch64_sve_prfh_gather_sxtw_index:
15994     case Intrinsic::aarch64_sve_prfw_gather_uxtw_index:
15995     case Intrinsic::aarch64_sve_prfw_gather_sxtw_index:
15996     case Intrinsic::aarch64_sve_prfd_gather_uxtw_index:
15997     case Intrinsic::aarch64_sve_prfd_gather_sxtw_index:
15998       return legalizeSVEGatherPrefetchOffsVec(N, DAG);
15999     case Intrinsic::aarch64_neon_ld2:
16000     case Intrinsic::aarch64_neon_ld3:
16001     case Intrinsic::aarch64_neon_ld4:
16002     case Intrinsic::aarch64_neon_ld1x2:
16003     case Intrinsic::aarch64_neon_ld1x3:
16004     case Intrinsic::aarch64_neon_ld1x4:
16005     case Intrinsic::aarch64_neon_ld2lane:
16006     case Intrinsic::aarch64_neon_ld3lane:
16007     case Intrinsic::aarch64_neon_ld4lane:
16008     case Intrinsic::aarch64_neon_ld2r:
16009     case Intrinsic::aarch64_neon_ld3r:
16010     case Intrinsic::aarch64_neon_ld4r:
16011     case Intrinsic::aarch64_neon_st2:
16012     case Intrinsic::aarch64_neon_st3:
16013     case Intrinsic::aarch64_neon_st4:
16014     case Intrinsic::aarch64_neon_st1x2:
16015     case Intrinsic::aarch64_neon_st1x3:
16016     case Intrinsic::aarch64_neon_st1x4:
16017     case Intrinsic::aarch64_neon_st2lane:
16018     case Intrinsic::aarch64_neon_st3lane:
16019     case Intrinsic::aarch64_neon_st4lane:
16020       return performNEONPostLDSTCombine(N, DCI, DAG);
16021     case Intrinsic::aarch64_sve_ldnt1:
16022       return performLDNT1Combine(N, DAG);
16023     case Intrinsic::aarch64_sve_ld1rq:
16024       return performLD1ReplicateCombine<AArch64ISD::LD1RQ_MERGE_ZERO>(N, DAG);
16025     case Intrinsic::aarch64_sve_ld1ro:
16026       return performLD1ReplicateCombine<AArch64ISD::LD1RO_MERGE_ZERO>(N, DAG);
16027     case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
16028       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_MERGE_ZERO);
16029     case Intrinsic::aarch64_sve_ldnt1_gather:
16030       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_MERGE_ZERO);
16031     case Intrinsic::aarch64_sve_ldnt1_gather_index:
16032       return performGatherLoadCombine(N, DAG,
16033                                       AArch64ISD::GLDNT1_INDEX_MERGE_ZERO);
16034     case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
16035       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_MERGE_ZERO);
16036     case Intrinsic::aarch64_sve_ld1:
16037       return performLD1Combine(N, DAG, AArch64ISD::LD1_MERGE_ZERO);
16038     case Intrinsic::aarch64_sve_ldnf1:
16039       return performLD1Combine(N, DAG, AArch64ISD::LDNF1_MERGE_ZERO);
16040     case Intrinsic::aarch64_sve_ldff1:
16041       return performLD1Combine(N, DAG, AArch64ISD::LDFF1_MERGE_ZERO);
16042     case Intrinsic::aarch64_sve_st1:
16043       return performST1Combine(N, DAG);
16044     case Intrinsic::aarch64_sve_stnt1:
16045       return performSTNT1Combine(N, DAG);
16046     case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
16047       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_PRED);
16048     case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
16049       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_PRED);
16050     case Intrinsic::aarch64_sve_stnt1_scatter:
16051       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_PRED);
16052     case Intrinsic::aarch64_sve_stnt1_scatter_index:
16053       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_INDEX_PRED);
16054     case Intrinsic::aarch64_sve_ld1_gather:
16055       return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_MERGE_ZERO);
16056     case Intrinsic::aarch64_sve_ld1_gather_index:
16057       return performGatherLoadCombine(N, DAG,
16058                                       AArch64ISD::GLD1_SCALED_MERGE_ZERO);
16059     case Intrinsic::aarch64_sve_ld1_gather_sxtw:
16060       return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_SXTW_MERGE_ZERO,
16061                                       /*OnlyPackedOffsets=*/false);
16062     case Intrinsic::aarch64_sve_ld1_gather_uxtw:
16063       return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_UXTW_MERGE_ZERO,
16064                                       /*OnlyPackedOffsets=*/false);
16065     case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
16066       return performGatherLoadCombine(N, DAG,
16067                                       AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO,
16068                                       /*OnlyPackedOffsets=*/false);
16069     case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
16070       return performGatherLoadCombine(N, DAG,
16071                                       AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO,
16072                                       /*OnlyPackedOffsets=*/false);
16073     case Intrinsic::aarch64_sve_ld1_gather_scalar_offset:
16074       return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_IMM_MERGE_ZERO);
16075     case Intrinsic::aarch64_sve_ldff1_gather:
16076       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDFF1_MERGE_ZERO);
16077     case Intrinsic::aarch64_sve_ldff1_gather_index:
16078       return performGatherLoadCombine(N, DAG,
16079                                       AArch64ISD::GLDFF1_SCALED_MERGE_ZERO);
16080     case Intrinsic::aarch64_sve_ldff1_gather_sxtw:
16081       return performGatherLoadCombine(N, DAG,
16082                                       AArch64ISD::GLDFF1_SXTW_MERGE_ZERO,
16083                                       /*OnlyPackedOffsets=*/false);
16084     case Intrinsic::aarch64_sve_ldff1_gather_uxtw:
16085       return performGatherLoadCombine(N, DAG,
16086                                       AArch64ISD::GLDFF1_UXTW_MERGE_ZERO,
16087                                       /*OnlyPackedOffsets=*/false);
16088     case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index:
16089       return performGatherLoadCombine(N, DAG,
16090                                       AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO,
16091                                       /*OnlyPackedOffsets=*/false);
16092     case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index:
16093       return performGatherLoadCombine(N, DAG,
16094                                       AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO,
16095                                       /*OnlyPackedOffsets=*/false);
16096     case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset:
16097       return performGatherLoadCombine(N, DAG,
16098                                       AArch64ISD::GLDFF1_IMM_MERGE_ZERO);
16099     case Intrinsic::aarch64_sve_st1_scatter:
16100       return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_PRED);
16101     case Intrinsic::aarch64_sve_st1_scatter_index:
16102       return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_SCALED_PRED);
16103     case Intrinsic::aarch64_sve_st1_scatter_sxtw:
16104       return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_SXTW_PRED,
16105                                         /*OnlyPackedOffsets=*/false);
16106     case Intrinsic::aarch64_sve_st1_scatter_uxtw:
16107       return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_UXTW_PRED,
16108                                         /*OnlyPackedOffsets=*/false);
16109     case Intrinsic::aarch64_sve_st1_scatter_sxtw_index:
16110       return performScatterStoreCombine(N, DAG,
16111                                         AArch64ISD::SST1_SXTW_SCALED_PRED,
16112                                         /*OnlyPackedOffsets=*/false);
16113     case Intrinsic::aarch64_sve_st1_scatter_uxtw_index:
16114       return performScatterStoreCombine(N, DAG,
16115                                         AArch64ISD::SST1_UXTW_SCALED_PRED,
16116                                         /*OnlyPackedOffsets=*/false);
16117     case Intrinsic::aarch64_sve_st1_scatter_scalar_offset:
16118       return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_IMM_PRED);
16119     case Intrinsic::aarch64_sve_tuple_get: {
16120       SDLoc DL(N);
16121       SDValue Chain = N->getOperand(0);
16122       SDValue Src1 = N->getOperand(2);
16123       SDValue Idx = N->getOperand(3);
16124 
16125       uint64_t IdxConst = cast<ConstantSDNode>(Idx)->getZExtValue();
16126       EVT ResVT = N->getValueType(0);
16127       uint64_t NumLanes = ResVT.getVectorElementCount().getKnownMinValue();
16128       SDValue ExtIdx = DAG.getVectorIdxConstant(IdxConst * NumLanes, DL);
16129       SDValue Val =
16130           DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, Src1, ExtIdx);
16131       return DAG.getMergeValues({Val, Chain}, DL);
16132     }
16133     case Intrinsic::aarch64_sve_tuple_set: {
16134       SDLoc DL(N);
16135       SDValue Chain = N->getOperand(0);
16136       SDValue Tuple = N->getOperand(2);
16137       SDValue Idx = N->getOperand(3);
16138       SDValue Vec = N->getOperand(4);
16139 
16140       EVT TupleVT = Tuple.getValueType();
16141       uint64_t TupleLanes = TupleVT.getVectorElementCount().getKnownMinValue();
16142 
16143       uint64_t IdxConst = cast<ConstantSDNode>(Idx)->getZExtValue();
16144       uint64_t NumLanes =
16145           Vec.getValueType().getVectorElementCount().getKnownMinValue();
16146 
16147       if ((TupleLanes % NumLanes) != 0)
16148         report_fatal_error("invalid tuple vector!");
16149 
16150       uint64_t NumVecs = TupleLanes / NumLanes;
16151 
16152       SmallVector<SDValue, 4> Opnds;
16153       for (unsigned I = 0; I < NumVecs; ++I) {
16154         if (I == IdxConst)
16155           Opnds.push_back(Vec);
16156         else {
16157           SDValue ExtIdx = DAG.getVectorIdxConstant(I * NumLanes, DL);
16158           Opnds.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
16159                                       Vec.getValueType(), Tuple, ExtIdx));
16160         }
16161       }
16162       SDValue Concat =
16163           DAG.getNode(ISD::CONCAT_VECTORS, DL, Tuple.getValueType(), Opnds);
16164       return DAG.getMergeValues({Concat, Chain}, DL);
16165     }
16166     case Intrinsic::aarch64_sve_tuple_create2:
16167     case Intrinsic::aarch64_sve_tuple_create3:
16168     case Intrinsic::aarch64_sve_tuple_create4: {
16169       SDLoc DL(N);
16170       SDValue Chain = N->getOperand(0);
16171 
16172       SmallVector<SDValue, 4> Opnds;
16173       for (unsigned I = 2; I < N->getNumOperands(); ++I)
16174         Opnds.push_back(N->getOperand(I));
16175 
16176       EVT VT = Opnds[0].getValueType();
16177       EVT EltVT = VT.getVectorElementType();
16178       EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
16179                                     VT.getVectorElementCount() *
16180                                         (N->getNumOperands() - 2));
16181       SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, DestVT, Opnds);
16182       return DAG.getMergeValues({Concat, Chain}, DL);
16183     }
16184     case Intrinsic::aarch64_sve_ld2:
16185     case Intrinsic::aarch64_sve_ld3:
16186     case Intrinsic::aarch64_sve_ld4: {
16187       SDLoc DL(N);
16188       SDValue Chain = N->getOperand(0);
16189       SDValue Mask = N->getOperand(2);
16190       SDValue BasePtr = N->getOperand(3);
16191       SDValue LoadOps[] = {Chain, Mask, BasePtr};
16192       unsigned IntrinsicID =
16193           cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
16194       SDValue Result =
16195           LowerSVEStructLoad(IntrinsicID, LoadOps, N->getValueType(0), DAG, DL);
16196       return DAG.getMergeValues({Result, Chain}, DL);
16197     }
16198     case Intrinsic::aarch64_rndr:
16199     case Intrinsic::aarch64_rndrrs: {
16200       unsigned IntrinsicID =
16201           cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
16202       auto Register =
16203           (IntrinsicID == Intrinsic::aarch64_rndr ? AArch64SysReg::RNDR
16204                                                   : AArch64SysReg::RNDRRS);
16205       SDLoc DL(N);
16206       SDValue A = DAG.getNode(
16207           AArch64ISD::MRS, DL, DAG.getVTList(MVT::i64, MVT::Glue, MVT::Other),
16208           N->getOperand(0), DAG.getConstant(Register, DL, MVT::i64));
16209       SDValue B = DAG.getNode(
16210           AArch64ISD::CSINC, DL, MVT::i32, DAG.getConstant(0, DL, MVT::i32),
16211           DAG.getConstant(0, DL, MVT::i32),
16212           DAG.getConstant(AArch64CC::NE, DL, MVT::i32), A.getValue(1));
16213       return DAG.getMergeValues(
16214           {A, DAG.getZExtOrTrunc(B, DL, MVT::i1), A.getValue(2)}, DL);
16215     }
16216     default:
16217       break;
16218     }
16219     break;
16220   case ISD::GlobalAddress:
16221     return performGlobalAddressCombine(N, DAG, Subtarget, getTargetMachine());
16222   }
16223   return SDValue();
16224 }
16225 
16226 // Check if the return value is used as only a return value, as otherwise
16227 // we can't perform a tail-call. In particular, we need to check for
16228 // target ISD nodes that are returns and any other "odd" constructs
16229 // that the generic analysis code won't necessarily catch.
16230 bool AArch64TargetLowering::isUsedByReturnOnly(SDNode *N,
16231                                                SDValue &Chain) const {
16232   if (N->getNumValues() != 1)
16233     return false;
16234   if (!N->hasNUsesOfValue(1, 0))
16235     return false;
16236 
16237   SDValue TCChain = Chain;
16238   SDNode *Copy = *N->use_begin();
16239   if (Copy->getOpcode() == ISD::CopyToReg) {
16240     // If the copy has a glue operand, we conservatively assume it isn't safe to
16241     // perform a tail call.
16242     if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() ==
16243         MVT::Glue)
16244       return false;
16245     TCChain = Copy->getOperand(0);
16246   } else if (Copy->getOpcode() != ISD::FP_EXTEND)
16247     return false;
16248 
16249   bool HasRet = false;
16250   for (SDNode *Node : Copy->uses()) {
16251     if (Node->getOpcode() != AArch64ISD::RET_FLAG)
16252       return false;
16253     HasRet = true;
16254   }
16255 
16256   if (!HasRet)
16257     return false;
16258 
16259   Chain = TCChain;
16260   return true;
16261 }
16262 
16263 // Return whether the an instruction can potentially be optimized to a tail
16264 // call. This will cause the optimizers to attempt to move, or duplicate,
16265 // return instructions to help enable tail call optimizations for this
16266 // instruction.
16267 bool AArch64TargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
16268   return CI->isTailCall();
16269 }
16270 
16271 bool AArch64TargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base,
16272                                                    SDValue &Offset,
16273                                                    ISD::MemIndexedMode &AM,
16274                                                    bool &IsInc,
16275                                                    SelectionDAG &DAG) const {
16276   if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)
16277     return false;
16278 
16279   Base = Op->getOperand(0);
16280   // All of the indexed addressing mode instructions take a signed
16281   // 9 bit immediate offset.
16282   if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1))) {
16283     int64_t RHSC = RHS->getSExtValue();
16284     if (Op->getOpcode() == ISD::SUB)
16285       RHSC = -(uint64_t)RHSC;
16286     if (!isInt<9>(RHSC))
16287       return false;
16288     IsInc = (Op->getOpcode() == ISD::ADD);
16289     Offset = Op->getOperand(1);
16290     return true;
16291   }
16292   return false;
16293 }
16294 
16295 bool AArch64TargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
16296                                                       SDValue &Offset,
16297                                                       ISD::MemIndexedMode &AM,
16298                                                       SelectionDAG &DAG) const {
16299   EVT VT;
16300   SDValue Ptr;
16301   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
16302     VT = LD->getMemoryVT();
16303     Ptr = LD->getBasePtr();
16304   } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
16305     VT = ST->getMemoryVT();
16306     Ptr = ST->getBasePtr();
16307   } else
16308     return false;
16309 
16310   bool IsInc;
16311   if (!getIndexedAddressParts(Ptr.getNode(), Base, Offset, AM, IsInc, DAG))
16312     return false;
16313   AM = IsInc ? ISD::PRE_INC : ISD::PRE_DEC;
16314   return true;
16315 }
16316 
16317 bool AArch64TargetLowering::getPostIndexedAddressParts(
16318     SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset,
16319     ISD::MemIndexedMode &AM, SelectionDAG &DAG) const {
16320   EVT VT;
16321   SDValue Ptr;
16322   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
16323     VT = LD->getMemoryVT();
16324     Ptr = LD->getBasePtr();
16325   } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
16326     VT = ST->getMemoryVT();
16327     Ptr = ST->getBasePtr();
16328   } else
16329     return false;
16330 
16331   bool IsInc;
16332   if (!getIndexedAddressParts(Op, Base, Offset, AM, IsInc, DAG))
16333     return false;
16334   // Post-indexing updates the base, so it's not a valid transform
16335   // if that's not the same as the load's pointer.
16336   if (Ptr != Base)
16337     return false;
16338   AM = IsInc ? ISD::POST_INC : ISD::POST_DEC;
16339   return true;
16340 }
16341 
16342 static void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
16343                                   SelectionDAG &DAG) {
16344   SDLoc DL(N);
16345   SDValue Op = N->getOperand(0);
16346 
16347   if (N->getValueType(0) != MVT::i16 ||
16348       (Op.getValueType() != MVT::f16 && Op.getValueType() != MVT::bf16))
16349     return;
16350 
16351   Op = SDValue(
16352       DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, MVT::f32,
16353                          DAG.getUNDEF(MVT::i32), Op,
16354                          DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
16355       0);
16356   Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op);
16357   Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op));
16358 }
16359 
16360 static void ReplaceReductionResults(SDNode *N,
16361                                     SmallVectorImpl<SDValue> &Results,
16362                                     SelectionDAG &DAG, unsigned InterOp,
16363                                     unsigned AcrossOp) {
16364   EVT LoVT, HiVT;
16365   SDValue Lo, Hi;
16366   SDLoc dl(N);
16367   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
16368   std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
16369   SDValue InterVal = DAG.getNode(InterOp, dl, LoVT, Lo, Hi);
16370   SDValue SplitVal = DAG.getNode(AcrossOp, dl, LoVT, InterVal);
16371   Results.push_back(SplitVal);
16372 }
16373 
16374 static std::pair<SDValue, SDValue> splitInt128(SDValue N, SelectionDAG &DAG) {
16375   SDLoc DL(N);
16376   SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, N);
16377   SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64,
16378                            DAG.getNode(ISD::SRL, DL, MVT::i128, N,
16379                                        DAG.getConstant(64, DL, MVT::i64)));
16380   return std::make_pair(Lo, Hi);
16381 }
16382 
16383 void AArch64TargetLowering::ReplaceExtractSubVectorResults(
16384     SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
16385   SDValue In = N->getOperand(0);
16386   EVT InVT = In.getValueType();
16387 
16388   // Common code will handle these just fine.
16389   if (!InVT.isScalableVector() || !InVT.isInteger())
16390     return;
16391 
16392   SDLoc DL(N);
16393   EVT VT = N->getValueType(0);
16394 
16395   // The following checks bail if this is not a halving operation.
16396 
16397   ElementCount ResEC = VT.getVectorElementCount();
16398 
16399   if (InVT.getVectorElementCount() != (ResEC * 2))
16400     return;
16401 
16402   auto *CIndex = dyn_cast<ConstantSDNode>(N->getOperand(1));
16403   if (!CIndex)
16404     return;
16405 
16406   unsigned Index = CIndex->getZExtValue();
16407   if ((Index != 0) && (Index != ResEC.getKnownMinValue()))
16408     return;
16409 
16410   unsigned Opcode = (Index == 0) ? AArch64ISD::UUNPKLO : AArch64ISD::UUNPKHI;
16411   EVT ExtendedHalfVT = VT.widenIntegerVectorElementType(*DAG.getContext());
16412 
16413   SDValue Half = DAG.getNode(Opcode, DL, ExtendedHalfVT, N->getOperand(0));
16414   Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Half));
16415 }
16416 
16417 // Create an even/odd pair of X registers holding integer value V.
16418 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
16419   SDLoc dl(V.getNode());
16420   SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i64);
16421   SDValue VHi = DAG.getAnyExtOrTrunc(
16422       DAG.getNode(ISD::SRL, dl, MVT::i128, V, DAG.getConstant(64, dl, MVT::i64)),
16423       dl, MVT::i64);
16424   if (DAG.getDataLayout().isBigEndian())
16425     std::swap (VLo, VHi);
16426   SDValue RegClass =
16427       DAG.getTargetConstant(AArch64::XSeqPairsClassRegClassID, dl, MVT::i32);
16428   SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, dl, MVT::i32);
16429   SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, dl, MVT::i32);
16430   const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
16431   return SDValue(
16432       DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
16433 }
16434 
16435 static void ReplaceCMP_SWAP_128Results(SDNode *N,
16436                                        SmallVectorImpl<SDValue> &Results,
16437                                        SelectionDAG &DAG,
16438                                        const AArch64Subtarget *Subtarget) {
16439   assert(N->getValueType(0) == MVT::i128 &&
16440          "AtomicCmpSwap on types less than 128 should be legal");
16441 
16442   if (Subtarget->hasLSE() || Subtarget->outlineAtomics()) {
16443     // LSE has a 128-bit compare and swap (CASP), but i128 is not a legal type,
16444     // so lower it here, wrapped in REG_SEQUENCE and EXTRACT_SUBREG.
16445     SDValue Ops[] = {
16446         createGPRPairNode(DAG, N->getOperand(2)), // Compare value
16447         createGPRPairNode(DAG, N->getOperand(3)), // Store value
16448         N->getOperand(1), // Ptr
16449         N->getOperand(0), // Chain in
16450     };
16451 
16452     MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
16453 
16454     unsigned Opcode;
16455     switch (MemOp->getOrdering()) {
16456     case AtomicOrdering::Monotonic:
16457       Opcode = AArch64::CASPX;
16458       break;
16459     case AtomicOrdering::Acquire:
16460       Opcode = AArch64::CASPAX;
16461       break;
16462     case AtomicOrdering::Release:
16463       Opcode = AArch64::CASPLX;
16464       break;
16465     case AtomicOrdering::AcquireRelease:
16466     case AtomicOrdering::SequentiallyConsistent:
16467       Opcode = AArch64::CASPALX;
16468       break;
16469     default:
16470       llvm_unreachable("Unexpected ordering!");
16471     }
16472 
16473     MachineSDNode *CmpSwap = DAG.getMachineNode(
16474         Opcode, SDLoc(N), DAG.getVTList(MVT::Untyped, MVT::Other), Ops);
16475     DAG.setNodeMemRefs(CmpSwap, {MemOp});
16476 
16477     unsigned SubReg1 = AArch64::sube64, SubReg2 = AArch64::subo64;
16478     if (DAG.getDataLayout().isBigEndian())
16479       std::swap(SubReg1, SubReg2);
16480     SDValue Lo = DAG.getTargetExtractSubreg(SubReg1, SDLoc(N), MVT::i64,
16481                                             SDValue(CmpSwap, 0));
16482     SDValue Hi = DAG.getTargetExtractSubreg(SubReg2, SDLoc(N), MVT::i64,
16483                                             SDValue(CmpSwap, 0));
16484     Results.push_back(
16485         DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128, Lo, Hi));
16486     Results.push_back(SDValue(CmpSwap, 1)); // Chain out
16487     return;
16488   }
16489 
16490   auto Desired = splitInt128(N->getOperand(2), DAG);
16491   auto New = splitInt128(N->getOperand(3), DAG);
16492   SDValue Ops[] = {N->getOperand(1), Desired.first, Desired.second,
16493                    New.first,        New.second,    N->getOperand(0)};
16494   SDNode *CmpSwap = DAG.getMachineNode(
16495       AArch64::CMP_SWAP_128, SDLoc(N),
16496       DAG.getVTList(MVT::i64, MVT::i64, MVT::i32, MVT::Other), Ops);
16497 
16498   MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
16499   DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
16500 
16501   Results.push_back(DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128,
16502                                 SDValue(CmpSwap, 0), SDValue(CmpSwap, 1)));
16503   Results.push_back(SDValue(CmpSwap, 3));
16504 }
16505 
16506 void AArch64TargetLowering::ReplaceNodeResults(
16507     SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
16508   switch (N->getOpcode()) {
16509   default:
16510     llvm_unreachable("Don't know how to custom expand this");
16511   case ISD::BITCAST:
16512     ReplaceBITCASTResults(N, Results, DAG);
16513     return;
16514   case ISD::VECREDUCE_ADD:
16515   case ISD::VECREDUCE_SMAX:
16516   case ISD::VECREDUCE_SMIN:
16517   case ISD::VECREDUCE_UMAX:
16518   case ISD::VECREDUCE_UMIN:
16519     Results.push_back(LowerVECREDUCE(SDValue(N, 0), DAG));
16520     return;
16521 
16522   case ISD::CTPOP:
16523     if (SDValue Result = LowerCTPOP(SDValue(N, 0), DAG))
16524       Results.push_back(Result);
16525     return;
16526   case AArch64ISD::SADDV:
16527     ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::SADDV);
16528     return;
16529   case AArch64ISD::UADDV:
16530     ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::UADDV);
16531     return;
16532   case AArch64ISD::SMINV:
16533     ReplaceReductionResults(N, Results, DAG, ISD::SMIN, AArch64ISD::SMINV);
16534     return;
16535   case AArch64ISD::UMINV:
16536     ReplaceReductionResults(N, Results, DAG, ISD::UMIN, AArch64ISD::UMINV);
16537     return;
16538   case AArch64ISD::SMAXV:
16539     ReplaceReductionResults(N, Results, DAG, ISD::SMAX, AArch64ISD::SMAXV);
16540     return;
16541   case AArch64ISD::UMAXV:
16542     ReplaceReductionResults(N, Results, DAG, ISD::UMAX, AArch64ISD::UMAXV);
16543     return;
16544   case ISD::FP_TO_UINT:
16545   case ISD::FP_TO_SINT:
16546     assert(N->getValueType(0) == MVT::i128 && "unexpected illegal conversion");
16547     // Let normal code take care of it by not adding anything to Results.
16548     return;
16549   case ISD::ATOMIC_CMP_SWAP:
16550     ReplaceCMP_SWAP_128Results(N, Results, DAG, Subtarget);
16551     return;
16552   case ISD::LOAD: {
16553     assert(SDValue(N, 0).getValueType() == MVT::i128 &&
16554            "unexpected load's value type");
16555     LoadSDNode *LoadNode = cast<LoadSDNode>(N);
16556     if (!LoadNode->isVolatile() || LoadNode->getMemoryVT() != MVT::i128) {
16557       // Non-volatile loads are optimized later in AArch64's load/store
16558       // optimizer.
16559       return;
16560     }
16561 
16562     SDValue Result = DAG.getMemIntrinsicNode(
16563         AArch64ISD::LDP, SDLoc(N),
16564         DAG.getVTList({MVT::i64, MVT::i64, MVT::Other}),
16565         {LoadNode->getChain(), LoadNode->getBasePtr()}, LoadNode->getMemoryVT(),
16566         LoadNode->getMemOperand());
16567 
16568     SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128,
16569                                Result.getValue(0), Result.getValue(1));
16570     Results.append({Pair, Result.getValue(2) /* Chain */});
16571     return;
16572   }
16573   case ISD::EXTRACT_SUBVECTOR:
16574     ReplaceExtractSubVectorResults(N, Results, DAG);
16575     return;
16576   case ISD::INTRINSIC_WO_CHAIN: {
16577     EVT VT = N->getValueType(0);
16578     assert((VT == MVT::i8 || VT == MVT::i16) &&
16579            "custom lowering for unexpected type");
16580 
16581     ConstantSDNode *CN = cast<ConstantSDNode>(N->getOperand(0));
16582     Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
16583     switch (IntID) {
16584     default:
16585       return;
16586     case Intrinsic::aarch64_sve_clasta_n: {
16587       SDLoc DL(N);
16588       auto Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, N->getOperand(2));
16589       auto V = DAG.getNode(AArch64ISD::CLASTA_N, DL, MVT::i32,
16590                            N->getOperand(1), Op2, N->getOperand(3));
16591       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
16592       return;
16593     }
16594     case Intrinsic::aarch64_sve_clastb_n: {
16595       SDLoc DL(N);
16596       auto Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, N->getOperand(2));
16597       auto V = DAG.getNode(AArch64ISD::CLASTB_N, DL, MVT::i32,
16598                            N->getOperand(1), Op2, N->getOperand(3));
16599       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
16600       return;
16601     }
16602     case Intrinsic::aarch64_sve_lasta: {
16603       SDLoc DL(N);
16604       auto V = DAG.getNode(AArch64ISD::LASTA, DL, MVT::i32,
16605                            N->getOperand(1), N->getOperand(2));
16606       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
16607       return;
16608     }
16609     case Intrinsic::aarch64_sve_lastb: {
16610       SDLoc DL(N);
16611       auto V = DAG.getNode(AArch64ISD::LASTB, DL, MVT::i32,
16612                            N->getOperand(1), N->getOperand(2));
16613       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
16614       return;
16615     }
16616     }
16617   }
16618   }
16619 }
16620 
16621 bool AArch64TargetLowering::useLoadStackGuardNode() const {
16622   if (Subtarget->isTargetAndroid() || Subtarget->isTargetFuchsia())
16623     return TargetLowering::useLoadStackGuardNode();
16624   return true;
16625 }
16626 
16627 unsigned AArch64TargetLowering::combineRepeatedFPDivisors() const {
16628   // Combine multiple FDIVs with the same divisor into multiple FMULs by the
16629   // reciprocal if there are three or more FDIVs.
16630   return 3;
16631 }
16632 
16633 TargetLoweringBase::LegalizeTypeAction
16634 AArch64TargetLowering::getPreferredVectorAction(MVT VT) const {
16635   // During type legalization, we prefer to widen v1i8, v1i16, v1i32  to v8i8,
16636   // v4i16, v2i32 instead of to promote.
16637   if (VT == MVT::v1i8 || VT == MVT::v1i16 || VT == MVT::v1i32 ||
16638       VT == MVT::v1f32)
16639     return TypeWidenVector;
16640 
16641   return TargetLoweringBase::getPreferredVectorAction(VT);
16642 }
16643 
16644 // Loads and stores less than 128-bits are already atomic; ones above that
16645 // are doomed anyway, so defer to the default libcall and blame the OS when
16646 // things go wrong.
16647 bool AArch64TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
16648   unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
16649   return Size == 128;
16650 }
16651 
16652 // Loads and stores less than 128-bits are already atomic; ones above that
16653 // are doomed anyway, so defer to the default libcall and blame the OS when
16654 // things go wrong.
16655 TargetLowering::AtomicExpansionKind
16656 AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
16657   unsigned Size = LI->getType()->getPrimitiveSizeInBits();
16658   return Size == 128 ? AtomicExpansionKind::LLSC : AtomicExpansionKind::None;
16659 }
16660 
16661 // For the real atomic operations, we have ldxr/stxr up to 128 bits,
16662 TargetLowering::AtomicExpansionKind
16663 AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
16664   if (AI->isFloatingPointOperation())
16665     return AtomicExpansionKind::CmpXChg;
16666 
16667   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
16668   if (Size > 128) return AtomicExpansionKind::None;
16669   // Nand not supported in LSE.
16670   if (AI->getOperation() == AtomicRMWInst::Nand) return AtomicExpansionKind::LLSC;
16671   // Leave 128 bits to LLSC.
16672   if (Subtarget->hasLSE() && Size < 128)
16673     return AtomicExpansionKind::None;
16674   if (Subtarget->outlineAtomics() && Size < 128) {
16675     // [U]Min/[U]Max RWM atomics are used in __sync_fetch_ libcalls so far.
16676     // Don't outline them unless
16677     // (1) high level <atomic> support approved:
16678     //   http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p0493r1.pdf
16679     // (2) low level libgcc and compiler-rt support implemented by:
16680     //   min/max outline atomics helpers
16681     if (AI->getOperation() != AtomicRMWInst::Min &&
16682         AI->getOperation() != AtomicRMWInst::Max &&
16683         AI->getOperation() != AtomicRMWInst::UMin &&
16684         AI->getOperation() != AtomicRMWInst::UMax) {
16685       return AtomicExpansionKind::None;
16686     }
16687   }
16688   return AtomicExpansionKind::LLSC;
16689 }
16690 
16691 TargetLowering::AtomicExpansionKind
16692 AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR(
16693     AtomicCmpXchgInst *AI) const {
16694   // If subtarget has LSE, leave cmpxchg intact for codegen.
16695   if (Subtarget->hasLSE() || Subtarget->outlineAtomics())
16696     return AtomicExpansionKind::None;
16697   // At -O0, fast-regalloc cannot cope with the live vregs necessary to
16698   // implement cmpxchg without spilling. If the address being exchanged is also
16699   // on the stack and close enough to the spill slot, this can lead to a
16700   // situation where the monitor always gets cleared and the atomic operation
16701   // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
16702   if (getTargetMachine().getOptLevel() == CodeGenOpt::None)
16703     return AtomicExpansionKind::None;
16704   return AtomicExpansionKind::LLSC;
16705 }
16706 
16707 Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
16708                                              AtomicOrdering Ord) const {
16709   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16710   Type *ValTy = cast<PointerType>(Addr->getType())->getElementType();
16711   bool IsAcquire = isAcquireOrStronger(Ord);
16712 
16713   // Since i128 isn't legal and intrinsics don't get type-lowered, the ldrexd
16714   // intrinsic must return {i64, i64} and we have to recombine them into a
16715   // single i128 here.
16716   if (ValTy->getPrimitiveSizeInBits() == 128) {
16717     Intrinsic::ID Int =
16718         IsAcquire ? Intrinsic::aarch64_ldaxp : Intrinsic::aarch64_ldxp;
16719     Function *Ldxr = Intrinsic::getDeclaration(M, Int);
16720 
16721     Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
16722     Value *LoHi = Builder.CreateCall(Ldxr, Addr, "lohi");
16723 
16724     Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
16725     Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
16726     Lo = Builder.CreateZExt(Lo, ValTy, "lo64");
16727     Hi = Builder.CreateZExt(Hi, ValTy, "hi64");
16728     return Builder.CreateOr(
16729         Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 64)), "val64");
16730   }
16731 
16732   Type *Tys[] = { Addr->getType() };
16733   Intrinsic::ID Int =
16734       IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
16735   Function *Ldxr = Intrinsic::getDeclaration(M, Int, Tys);
16736 
16737   Type *EltTy = cast<PointerType>(Addr->getType())->getElementType();
16738 
16739   const DataLayout &DL = M->getDataLayout();
16740   IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(EltTy));
16741   Value *Trunc = Builder.CreateTrunc(Builder.CreateCall(Ldxr, Addr), IntEltTy);
16742 
16743   return Builder.CreateBitCast(Trunc, EltTy);
16744 }
16745 
16746 void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
16747     IRBuilder<> &Builder) const {
16748   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16749   Builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::aarch64_clrex));
16750 }
16751 
16752 Value *AArch64TargetLowering::emitStoreConditional(IRBuilder<> &Builder,
16753                                                    Value *Val, Value *Addr,
16754                                                    AtomicOrdering Ord) const {
16755   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16756   bool IsRelease = isReleaseOrStronger(Ord);
16757 
16758   // Since the intrinsics must have legal type, the i128 intrinsics take two
16759   // parameters: "i64, i64". We must marshal Val into the appropriate form
16760   // before the call.
16761   if (Val->getType()->getPrimitiveSizeInBits() == 128) {
16762     Intrinsic::ID Int =
16763         IsRelease ? Intrinsic::aarch64_stlxp : Intrinsic::aarch64_stxp;
16764     Function *Stxr = Intrinsic::getDeclaration(M, Int);
16765     Type *Int64Ty = Type::getInt64Ty(M->getContext());
16766 
16767     Value *Lo = Builder.CreateTrunc(Val, Int64Ty, "lo");
16768     Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 64), Int64Ty, "hi");
16769     Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
16770     return Builder.CreateCall(Stxr, {Lo, Hi, Addr});
16771   }
16772 
16773   Intrinsic::ID Int =
16774       IsRelease ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr;
16775   Type *Tys[] = { Addr->getType() };
16776   Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys);
16777 
16778   const DataLayout &DL = M->getDataLayout();
16779   IntegerType *IntValTy = Builder.getIntNTy(DL.getTypeSizeInBits(Val->getType()));
16780   Val = Builder.CreateBitCast(Val, IntValTy);
16781 
16782   return Builder.CreateCall(Stxr,
16783                             {Builder.CreateZExtOrBitCast(
16784                                  Val, Stxr->getFunctionType()->getParamType(0)),
16785                              Addr});
16786 }
16787 
16788 bool AArch64TargetLowering::functionArgumentNeedsConsecutiveRegisters(
16789     Type *Ty, CallingConv::ID CallConv, bool isVarArg) const {
16790   if (Ty->isArrayTy())
16791     return true;
16792 
16793   const TypeSize &TySize = Ty->getPrimitiveSizeInBits();
16794   if (TySize.isScalable() && TySize.getKnownMinSize() > 128)
16795     return true;
16796 
16797   return false;
16798 }
16799 
16800 bool AArch64TargetLowering::shouldNormalizeToSelectSequence(LLVMContext &,
16801                                                             EVT) const {
16802   return false;
16803 }
16804 
16805 static Value *UseTlsOffset(IRBuilder<> &IRB, unsigned Offset) {
16806   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
16807   Function *ThreadPointerFunc =
16808       Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
16809   return IRB.CreatePointerCast(
16810       IRB.CreateConstGEP1_32(IRB.getInt8Ty(), IRB.CreateCall(ThreadPointerFunc),
16811                              Offset),
16812       IRB.getInt8PtrTy()->getPointerTo(0));
16813 }
16814 
16815 Value *AArch64TargetLowering::getIRStackGuard(IRBuilder<> &IRB) const {
16816   // Android provides a fixed TLS slot for the stack cookie. See the definition
16817   // of TLS_SLOT_STACK_GUARD in
16818   // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h
16819   if (Subtarget->isTargetAndroid())
16820     return UseTlsOffset(IRB, 0x28);
16821 
16822   // Fuchsia is similar.
16823   // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
16824   if (Subtarget->isTargetFuchsia())
16825     return UseTlsOffset(IRB, -0x10);
16826 
16827   return TargetLowering::getIRStackGuard(IRB);
16828 }
16829 
16830 void AArch64TargetLowering::insertSSPDeclarations(Module &M) const {
16831   // MSVC CRT provides functionalities for stack protection.
16832   if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment()) {
16833     // MSVC CRT has a global variable holding security cookie.
16834     M.getOrInsertGlobal("__security_cookie",
16835                         Type::getInt8PtrTy(M.getContext()));
16836 
16837     // MSVC CRT has a function to validate security cookie.
16838     FunctionCallee SecurityCheckCookie = M.getOrInsertFunction(
16839         "__security_check_cookie", Type::getVoidTy(M.getContext()),
16840         Type::getInt8PtrTy(M.getContext()));
16841     if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee())) {
16842       F->setCallingConv(CallingConv::Win64);
16843       F->addAttribute(1, Attribute::AttrKind::InReg);
16844     }
16845     return;
16846   }
16847   TargetLowering::insertSSPDeclarations(M);
16848 }
16849 
16850 Value *AArch64TargetLowering::getSDagStackGuard(const Module &M) const {
16851   // MSVC CRT has a global variable holding security cookie.
16852   if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
16853     return M.getGlobalVariable("__security_cookie");
16854   return TargetLowering::getSDagStackGuard(M);
16855 }
16856 
16857 Function *AArch64TargetLowering::getSSPStackGuardCheck(const Module &M) const {
16858   // MSVC CRT has a function to validate security cookie.
16859   if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
16860     return M.getFunction("__security_check_cookie");
16861   return TargetLowering::getSSPStackGuardCheck(M);
16862 }
16863 
16864 Value *AArch64TargetLowering::getSafeStackPointerLocation(IRBuilder<> &IRB) const {
16865   // Android provides a fixed TLS slot for the SafeStack pointer. See the
16866   // definition of TLS_SLOT_SAFESTACK in
16867   // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h
16868   if (Subtarget->isTargetAndroid())
16869     return UseTlsOffset(IRB, 0x48);
16870 
16871   // Fuchsia is similar.
16872   // <zircon/tls.h> defines ZX_TLS_UNSAFE_SP_OFFSET with this value.
16873   if (Subtarget->isTargetFuchsia())
16874     return UseTlsOffset(IRB, -0x8);
16875 
16876   return TargetLowering::getSafeStackPointerLocation(IRB);
16877 }
16878 
16879 bool AArch64TargetLowering::isMaskAndCmp0FoldingBeneficial(
16880     const Instruction &AndI) const {
16881   // Only sink 'and' mask to cmp use block if it is masking a single bit, since
16882   // this is likely to be fold the and/cmp/br into a single tbz instruction.  It
16883   // may be beneficial to sink in other cases, but we would have to check that
16884   // the cmp would not get folded into the br to form a cbz for these to be
16885   // beneficial.
16886   ConstantInt* Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
16887   if (!Mask)
16888     return false;
16889   return Mask->getValue().isPowerOf2();
16890 }
16891 
16892 bool AArch64TargetLowering::
16893     shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
16894         SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
16895         unsigned OldShiftOpcode, unsigned NewShiftOpcode,
16896         SelectionDAG &DAG) const {
16897   // Does baseline recommend not to perform the fold by default?
16898   if (!TargetLowering::shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
16899           X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG))
16900     return false;
16901   // Else, if this is a vector shift, prefer 'shl'.
16902   return X.getValueType().isScalarInteger() || NewShiftOpcode == ISD::SHL;
16903 }
16904 
16905 bool AArch64TargetLowering::shouldExpandShift(SelectionDAG &DAG,
16906                                               SDNode *N) const {
16907   if (DAG.getMachineFunction().getFunction().hasMinSize() &&
16908       !Subtarget->isTargetWindows() && !Subtarget->isTargetDarwin())
16909     return false;
16910   return true;
16911 }
16912 
16913 void AArch64TargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
16914   // Update IsSplitCSR in AArch64unctionInfo.
16915   AArch64FunctionInfo *AFI = Entry->getParent()->getInfo<AArch64FunctionInfo>();
16916   AFI->setIsSplitCSR(true);
16917 }
16918 
16919 void AArch64TargetLowering::insertCopiesSplitCSR(
16920     MachineBasicBlock *Entry,
16921     const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
16922   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
16923   const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
16924   if (!IStart)
16925     return;
16926 
16927   const TargetInstrInfo *TII = Subtarget->getInstrInfo();
16928   MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
16929   MachineBasicBlock::iterator MBBI = Entry->begin();
16930   for (const MCPhysReg *I = IStart; *I; ++I) {
16931     const TargetRegisterClass *RC = nullptr;
16932     if (AArch64::GPR64RegClass.contains(*I))
16933       RC = &AArch64::GPR64RegClass;
16934     else if (AArch64::FPR64RegClass.contains(*I))
16935       RC = &AArch64::FPR64RegClass;
16936     else
16937       llvm_unreachable("Unexpected register class in CSRsViaCopy!");
16938 
16939     Register NewVR = MRI->createVirtualRegister(RC);
16940     // Create copy from CSR to a virtual register.
16941     // FIXME: this currently does not emit CFI pseudo-instructions, it works
16942     // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
16943     // nounwind. If we want to generalize this later, we may need to emit
16944     // CFI pseudo-instructions.
16945     assert(Entry->getParent()->getFunction().hasFnAttribute(
16946                Attribute::NoUnwind) &&
16947            "Function should be nounwind in insertCopiesSplitCSR!");
16948     Entry->addLiveIn(*I);
16949     BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
16950         .addReg(*I);
16951 
16952     // Insert the copy-back instructions right before the terminator.
16953     for (auto *Exit : Exits)
16954       BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
16955               TII->get(TargetOpcode::COPY), *I)
16956           .addReg(NewVR);
16957   }
16958 }
16959 
16960 bool AArch64TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
16961   // Integer division on AArch64 is expensive. However, when aggressively
16962   // optimizing for code size, we prefer to use a div instruction, as it is
16963   // usually smaller than the alternative sequence.
16964   // The exception to this is vector division. Since AArch64 doesn't have vector
16965   // integer division, leaving the division as-is is a loss even in terms of
16966   // size, because it will have to be scalarized, while the alternative code
16967   // sequence can be performed in vector form.
16968   bool OptSize = Attr.hasFnAttribute(Attribute::MinSize);
16969   return OptSize && !VT.isVector();
16970 }
16971 
16972 bool AArch64TargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
16973   // We want inc-of-add for scalars and sub-of-not for vectors.
16974   return VT.isScalarInteger();
16975 }
16976 
16977 bool AArch64TargetLowering::enableAggressiveFMAFusion(EVT VT) const {
16978   return Subtarget->hasAggressiveFMA() && VT.isFloatingPoint();
16979 }
16980 
16981 unsigned
16982 AArch64TargetLowering::getVaListSizeInBits(const DataLayout &DL) const {
16983   if (Subtarget->isTargetDarwin() || Subtarget->isTargetWindows())
16984     return getPointerTy(DL).getSizeInBits();
16985 
16986   return 3 * getPointerTy(DL).getSizeInBits() + 2 * 32;
16987 }
16988 
16989 void AArch64TargetLowering::finalizeLowering(MachineFunction &MF) const {
16990   MF.getFrameInfo().computeMaxCallFrameSize(MF);
16991   TargetLoweringBase::finalizeLowering(MF);
16992 }
16993 
16994 // Unlike X86, we let frame lowering assign offsets to all catch objects.
16995 bool AArch64TargetLowering::needsFixedCatchObjects() const {
16996   return false;
16997 }
16998 
16999 bool AArch64TargetLowering::shouldLocalize(
17000     const MachineInstr &MI, const TargetTransformInfo *TTI) const {
17001   switch (MI.getOpcode()) {
17002   case TargetOpcode::G_GLOBAL_VALUE: {
17003     // On Darwin, TLS global vars get selected into function calls, which
17004     // we don't want localized, as they can get moved into the middle of a
17005     // another call sequence.
17006     const GlobalValue &GV = *MI.getOperand(1).getGlobal();
17007     if (GV.isThreadLocal() && Subtarget->isTargetMachO())
17008       return false;
17009     break;
17010   }
17011   // If we legalized G_GLOBAL_VALUE into ADRP + G_ADD_LOW, mark both as being
17012   // localizable.
17013   case AArch64::ADRP:
17014   case AArch64::G_ADD_LOW:
17015     return true;
17016   default:
17017     break;
17018   }
17019   return TargetLoweringBase::shouldLocalize(MI, TTI);
17020 }
17021 
17022 bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
17023   if (isa<ScalableVectorType>(Inst.getType()))
17024     return true;
17025 
17026   for (unsigned i = 0; i < Inst.getNumOperands(); ++i)
17027     if (isa<ScalableVectorType>(Inst.getOperand(i)->getType()))
17028       return true;
17029 
17030   if (const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
17031     if (isa<ScalableVectorType>(AI->getAllocatedType()))
17032       return true;
17033   }
17034 
17035   return false;
17036 }
17037 
17038 // Return the largest legal scalable vector type that matches VT's element type.
17039 static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT) {
17040   assert(VT.isFixedLengthVector() &&
17041          DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
17042          "Expected legal fixed length vector!");
17043   switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
17044   default:
17045     llvm_unreachable("unexpected element type for SVE container");
17046   case MVT::i8:
17047     return EVT(MVT::nxv16i8);
17048   case MVT::i16:
17049     return EVT(MVT::nxv8i16);
17050   case MVT::i32:
17051     return EVT(MVT::nxv4i32);
17052   case MVT::i64:
17053     return EVT(MVT::nxv2i64);
17054   case MVT::f16:
17055     return EVT(MVT::nxv8f16);
17056   case MVT::f32:
17057     return EVT(MVT::nxv4f32);
17058   case MVT::f64:
17059     return EVT(MVT::nxv2f64);
17060   }
17061 }
17062 
17063 // Return a PTRUE with active lanes corresponding to the extent of VT.
17064 static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
17065                                                 EVT VT) {
17066   assert(VT.isFixedLengthVector() &&
17067          DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
17068          "Expected legal fixed length vector!");
17069 
17070   int PgPattern;
17071   switch (VT.getVectorNumElements()) {
17072   default:
17073     llvm_unreachable("unexpected element count for SVE predicate");
17074   case 1:
17075     PgPattern = AArch64SVEPredPattern::vl1;
17076     break;
17077   case 2:
17078     PgPattern = AArch64SVEPredPattern::vl2;
17079     break;
17080   case 4:
17081     PgPattern = AArch64SVEPredPattern::vl4;
17082     break;
17083   case 8:
17084     PgPattern = AArch64SVEPredPattern::vl8;
17085     break;
17086   case 16:
17087     PgPattern = AArch64SVEPredPattern::vl16;
17088     break;
17089   case 32:
17090     PgPattern = AArch64SVEPredPattern::vl32;
17091     break;
17092   case 64:
17093     PgPattern = AArch64SVEPredPattern::vl64;
17094     break;
17095   case 128:
17096     PgPattern = AArch64SVEPredPattern::vl128;
17097     break;
17098   case 256:
17099     PgPattern = AArch64SVEPredPattern::vl256;
17100     break;
17101   }
17102 
17103   // TODO: For vectors that are exactly getMaxSVEVectorSizeInBits big, we can
17104   // use AArch64SVEPredPattern::all, which can enable the use of unpredicated
17105   // variants of instructions when available.
17106 
17107   MVT MaskVT;
17108   switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
17109   default:
17110     llvm_unreachable("unexpected element type for SVE predicate");
17111   case MVT::i8:
17112     MaskVT = MVT::nxv16i1;
17113     break;
17114   case MVT::i16:
17115   case MVT::f16:
17116     MaskVT = MVT::nxv8i1;
17117     break;
17118   case MVT::i32:
17119   case MVT::f32:
17120     MaskVT = MVT::nxv4i1;
17121     break;
17122   case MVT::i64:
17123   case MVT::f64:
17124     MaskVT = MVT::nxv2i1;
17125     break;
17126   }
17127 
17128   return DAG.getNode(AArch64ISD::PTRUE, DL, MaskVT,
17129                      DAG.getTargetConstant(PgPattern, DL, MVT::i64));
17130 }
17131 
17132 static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
17133                                              EVT VT) {
17134   assert(VT.isScalableVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
17135          "Expected legal scalable vector!");
17136   auto PredTy = VT.changeVectorElementType(MVT::i1);
17137   return getPTrue(DAG, DL, PredTy, AArch64SVEPredPattern::all);
17138 }
17139 
17140 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT) {
17141   if (VT.isFixedLengthVector())
17142     return getPredicateForFixedLengthVector(DAG, DL, VT);
17143 
17144   return getPredicateForScalableVector(DAG, DL, VT);
17145 }
17146 
17147 // Grow V to consume an entire SVE register.
17148 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
17149   assert(VT.isScalableVector() &&
17150          "Expected to convert into a scalable vector!");
17151   assert(V.getValueType().isFixedLengthVector() &&
17152          "Expected a fixed length vector operand!");
17153   SDLoc DL(V);
17154   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
17155   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
17156 }
17157 
17158 // Shrink V so it's just big enough to maintain a VT's worth of data.
17159 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
17160   assert(VT.isFixedLengthVector() &&
17161          "Expected to convert into a fixed length vector!");
17162   assert(V.getValueType().isScalableVector() &&
17163          "Expected a scalable vector operand!");
17164   SDLoc DL(V);
17165   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
17166   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
17167 }
17168 
17169 // Convert all fixed length vector loads larger than NEON to masked_loads.
17170 SDValue AArch64TargetLowering::LowerFixedLengthVectorLoadToSVE(
17171     SDValue Op, SelectionDAG &DAG) const {
17172   auto Load = cast<LoadSDNode>(Op);
17173 
17174   SDLoc DL(Op);
17175   EVT VT = Op.getValueType();
17176   EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
17177 
17178   auto NewLoad = DAG.getMaskedLoad(
17179       ContainerVT, DL, Load->getChain(), Load->getBasePtr(), Load->getOffset(),
17180       getPredicateForFixedLengthVector(DAG, DL, VT), DAG.getUNDEF(ContainerVT),
17181       Load->getMemoryVT(), Load->getMemOperand(), Load->getAddressingMode(),
17182       Load->getExtensionType());
17183 
17184   auto Result = convertFromScalableVector(DAG, VT, NewLoad);
17185   SDValue MergedValues[2] = {Result, Load->getChain()};
17186   return DAG.getMergeValues(MergedValues, DL);
17187 }
17188 
17189 // Convert all fixed length vector stores larger than NEON to masked_stores.
17190 SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
17191     SDValue Op, SelectionDAG &DAG) const {
17192   auto Store = cast<StoreSDNode>(Op);
17193 
17194   SDLoc DL(Op);
17195   EVT VT = Store->getValue().getValueType();
17196   EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
17197 
17198   auto NewValue = convertToScalableVector(DAG, ContainerVT, Store->getValue());
17199   return DAG.getMaskedStore(
17200       Store->getChain(), DL, NewValue, Store->getBasePtr(), Store->getOffset(),
17201       getPredicateForFixedLengthVector(DAG, DL, VT), Store->getMemoryVT(),
17202       Store->getMemOperand(), Store->getAddressingMode(),
17203       Store->isTruncatingStore());
17204 }
17205 
17206 SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
17207     SDValue Op, SelectionDAG &DAG) const {
17208   SDLoc dl(Op);
17209   EVT VT = Op.getValueType();
17210   EVT EltVT = VT.getVectorElementType();
17211 
17212   bool Signed = Op.getOpcode() == ISD::SDIV;
17213   unsigned PredOpcode = Signed ? AArch64ISD::SDIV_PRED : AArch64ISD::UDIV_PRED;
17214 
17215   // Scalable vector i32/i64 DIV is supported.
17216   if (EltVT == MVT::i32 || EltVT == MVT::i64)
17217     return LowerToPredicatedOp(Op, DAG, PredOpcode, /*OverrideNEON=*/true);
17218 
17219   // Scalable vector i8/i16 DIV is not supported. Promote it to i32.
17220   EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
17221   EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
17222   EVT FixedWidenedVT = HalfVT.widenIntegerVectorElementType(*DAG.getContext());
17223   EVT ScalableWidenedVT = getContainerForFixedLengthVector(DAG, FixedWidenedVT);
17224 
17225   // Convert the operands to scalable vectors.
17226   SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(0));
17227   SDValue Op1 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(1));
17228 
17229   // Extend the scalable operands.
17230   unsigned UnpkLo = Signed ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO;
17231   unsigned UnpkHi = Signed ? AArch64ISD::SUNPKHI : AArch64ISD::UUNPKHI;
17232   SDValue Op0Lo = DAG.getNode(UnpkLo, dl, ScalableWidenedVT, Op0);
17233   SDValue Op1Lo = DAG.getNode(UnpkLo, dl, ScalableWidenedVT, Op1);
17234   SDValue Op0Hi = DAG.getNode(UnpkHi, dl, ScalableWidenedVT, Op0);
17235   SDValue Op1Hi = DAG.getNode(UnpkHi, dl, ScalableWidenedVT, Op1);
17236 
17237   // Convert back to fixed vectors so the DIV can be further lowered.
17238   Op0Lo = convertFromScalableVector(DAG, FixedWidenedVT, Op0Lo);
17239   Op1Lo = convertFromScalableVector(DAG, FixedWidenedVT, Op1Lo);
17240   Op0Hi = convertFromScalableVector(DAG, FixedWidenedVT, Op0Hi);
17241   Op1Hi = convertFromScalableVector(DAG, FixedWidenedVT, Op1Hi);
17242   SDValue ResultLo = DAG.getNode(Op.getOpcode(), dl, FixedWidenedVT,
17243                                  Op0Lo, Op1Lo);
17244   SDValue ResultHi = DAG.getNode(Op.getOpcode(), dl, FixedWidenedVT,
17245                                  Op0Hi, Op1Hi);
17246 
17247   // Convert again to scalable vectors to truncate.
17248   ResultLo = convertToScalableVector(DAG, ScalableWidenedVT, ResultLo);
17249   ResultHi = convertToScalableVector(DAG, ScalableWidenedVT, ResultHi);
17250   SDValue ScalableResult = DAG.getNode(AArch64ISD::UZP1, dl, ContainerVT,
17251                                        ResultLo, ResultHi);
17252 
17253   return convertFromScalableVector(DAG, VT, ScalableResult);
17254 }
17255 
17256 SDValue AArch64TargetLowering::LowerFixedLengthVectorIntExtendToSVE(
17257     SDValue Op, SelectionDAG &DAG) const {
17258   EVT VT = Op.getValueType();
17259   assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
17260 
17261   SDLoc DL(Op);
17262   SDValue Val = Op.getOperand(0);
17263   EVT ContainerVT = getContainerForFixedLengthVector(DAG, Val.getValueType());
17264   Val = convertToScalableVector(DAG, ContainerVT, Val);
17265 
17266   bool Signed = Op.getOpcode() == ISD::SIGN_EXTEND;
17267   unsigned ExtendOpc = Signed ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO;
17268 
17269   // Repeatedly unpack Val until the result is of the desired element type.
17270   switch (ContainerVT.getSimpleVT().SimpleTy) {
17271   default:
17272     llvm_unreachable("unimplemented container type");
17273   case MVT::nxv16i8:
17274     Val = DAG.getNode(ExtendOpc, DL, MVT::nxv8i16, Val);
17275     if (VT.getVectorElementType() == MVT::i16)
17276       break;
17277     LLVM_FALLTHROUGH;
17278   case MVT::nxv8i16:
17279     Val = DAG.getNode(ExtendOpc, DL, MVT::nxv4i32, Val);
17280     if (VT.getVectorElementType() == MVT::i32)
17281       break;
17282     LLVM_FALLTHROUGH;
17283   case MVT::nxv4i32:
17284     Val = DAG.getNode(ExtendOpc, DL, MVT::nxv2i64, Val);
17285     assert(VT.getVectorElementType() == MVT::i64 && "Unexpected element type!");
17286     break;
17287   }
17288 
17289   return convertFromScalableVector(DAG, VT, Val);
17290 }
17291 
17292 SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
17293     SDValue Op, SelectionDAG &DAG) const {
17294   EVT VT = Op.getValueType();
17295   assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
17296 
17297   SDLoc DL(Op);
17298   SDValue Val = Op.getOperand(0);
17299   EVT ContainerVT = getContainerForFixedLengthVector(DAG, Val.getValueType());
17300   Val = convertToScalableVector(DAG, ContainerVT, Val);
17301 
17302   // Repeatedly truncate Val until the result is of the desired element type.
17303   switch (ContainerVT.getSimpleVT().SimpleTy) {
17304   default:
17305     llvm_unreachable("unimplemented container type");
17306   case MVT::nxv2i64:
17307     Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv4i32, Val);
17308     Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv4i32, Val, Val);
17309     if (VT.getVectorElementType() == MVT::i32)
17310       break;
17311     LLVM_FALLTHROUGH;
17312   case MVT::nxv4i32:
17313     Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv8i16, Val);
17314     Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv8i16, Val, Val);
17315     if (VT.getVectorElementType() == MVT::i16)
17316       break;
17317     LLVM_FALLTHROUGH;
17318   case MVT::nxv8i16:
17319     Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, Val);
17320     Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv16i8, Val, Val);
17321     assert(VT.getVectorElementType() == MVT::i8 && "Unexpected element type!");
17322     break;
17323   }
17324 
17325   return convertFromScalableVector(DAG, VT, Val);
17326 }
17327 
17328 SDValue AArch64TargetLowering::LowerFixedLengthExtractVectorElt(
17329     SDValue Op, SelectionDAG &DAG) const {
17330   EVT VT = Op.getValueType();
17331   EVT InVT = Op.getOperand(0).getValueType();
17332   assert(InVT.isFixedLengthVector() && "Expected fixed length vector type!");
17333 
17334   SDLoc DL(Op);
17335   EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
17336   SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(0));
17337 
17338   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Op.getOperand(1));
17339 }
17340 
17341 SDValue AArch64TargetLowering::LowerFixedLengthInsertVectorElt(
17342     SDValue Op, SelectionDAG &DAG) const {
17343   EVT VT = Op.getValueType();
17344   assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
17345 
17346   SDLoc DL(Op);
17347   EVT InVT = Op.getOperand(0).getValueType();
17348   EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
17349   SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(0));
17350 
17351   auto ScalableRes = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT, Op0,
17352                                  Op.getOperand(1), Op.getOperand(2));
17353 
17354   return convertFromScalableVector(DAG, VT, ScalableRes);
17355 }
17356 
17357 // Convert vector operation 'Op' to an equivalent predicated operation whereby
17358 // the original operation's type is used to construct a suitable predicate.
17359 // NOTE: The results for inactive lanes are undefined.
17360 SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
17361                                                    SelectionDAG &DAG,
17362                                                    unsigned NewOp,
17363                                                    bool OverrideNEON) const {
17364   EVT VT = Op.getValueType();
17365   SDLoc DL(Op);
17366   auto Pg = getPredicateForVector(DAG, DL, VT);
17367 
17368   if (useSVEForFixedLengthVectorVT(VT, OverrideNEON)) {
17369     EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
17370 
17371     // Create list of operands by converting existing ones to scalable types.
17372     SmallVector<SDValue, 4> Operands = {Pg};
17373     for (const SDValue &V : Op->op_values()) {
17374       if (isa<CondCodeSDNode>(V)) {
17375         Operands.push_back(V);
17376         continue;
17377       }
17378 
17379       if (const VTSDNode *VTNode = dyn_cast<VTSDNode>(V)) {
17380         EVT VTArg = VTNode->getVT().getVectorElementType();
17381         EVT NewVTArg = ContainerVT.changeVectorElementType(VTArg);
17382         Operands.push_back(DAG.getValueType(NewVTArg));
17383         continue;
17384       }
17385 
17386       assert(useSVEForFixedLengthVectorVT(V.getValueType(), OverrideNEON) &&
17387              "Only fixed length vectors are supported!");
17388       Operands.push_back(convertToScalableVector(DAG, ContainerVT, V));
17389     }
17390 
17391     if (isMergePassthruOpcode(NewOp))
17392       Operands.push_back(DAG.getUNDEF(ContainerVT));
17393 
17394     auto ScalableRes = DAG.getNode(NewOp, DL, ContainerVT, Operands);
17395     return convertFromScalableVector(DAG, VT, ScalableRes);
17396   }
17397 
17398   assert(VT.isScalableVector() && "Only expect to lower scalable vector op!");
17399 
17400   SmallVector<SDValue, 4> Operands = {Pg};
17401   for (const SDValue &V : Op->op_values()) {
17402     assert((!V.getValueType().isVector() ||
17403             V.getValueType().isScalableVector()) &&
17404            "Only scalable vectors are supported!");
17405     Operands.push_back(V);
17406   }
17407 
17408   if (isMergePassthruOpcode(NewOp))
17409     Operands.push_back(DAG.getUNDEF(VT));
17410 
17411   return DAG.getNode(NewOp, DL, VT, Operands);
17412 }
17413 
17414 // If a fixed length vector operation has no side effects when applied to
17415 // undefined elements, we can safely use scalable vectors to perform the same
17416 // operation without needing to worry about predication.
17417 SDValue AArch64TargetLowering::LowerToScalableOp(SDValue Op,
17418                                                  SelectionDAG &DAG) const {
17419   EVT VT = Op.getValueType();
17420   assert(useSVEForFixedLengthVectorVT(VT) &&
17421          "Only expected to lower fixed length vector operation!");
17422   EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
17423 
17424   // Create list of operands by converting existing ones to scalable types.
17425   SmallVector<SDValue, 4> Ops;
17426   for (const SDValue &V : Op->op_values()) {
17427     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
17428 
17429     // Pass through non-vector operands.
17430     if (!V.getValueType().isVector()) {
17431       Ops.push_back(V);
17432       continue;
17433     }
17434 
17435     // "cast" fixed length vector to a scalable vector.
17436     assert(useSVEForFixedLengthVectorVT(V.getValueType()) &&
17437            "Only fixed length vectors are supported!");
17438     Ops.push_back(convertToScalableVector(DAG, ContainerVT, V));
17439   }
17440 
17441   auto ScalableRes = DAG.getNode(Op.getOpcode(), SDLoc(Op), ContainerVT, Ops);
17442   return convertFromScalableVector(DAG, VT, ScalableRes);
17443 }
17444 
17445 SDValue AArch64TargetLowering::LowerVECREDUCE_SEQ_FADD(SDValue ScalarOp,
17446     SelectionDAG &DAG) const {
17447   SDLoc DL(ScalarOp);
17448   SDValue AccOp = ScalarOp.getOperand(0);
17449   SDValue VecOp = ScalarOp.getOperand(1);
17450   EVT SrcVT = VecOp.getValueType();
17451   EVT ResVT = SrcVT.getVectorElementType();
17452 
17453   EVT ContainerVT = SrcVT;
17454   if (SrcVT.isFixedLengthVector()) {
17455     ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT);
17456     VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
17457   }
17458 
17459   SDValue Pg = getPredicateForVector(DAG, DL, SrcVT);
17460   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
17461 
17462   // Convert operands to Scalable.
17463   AccOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT,
17464                       DAG.getUNDEF(ContainerVT), AccOp, Zero);
17465 
17466   // Perform reduction.
17467   SDValue Rdx = DAG.getNode(AArch64ISD::FADDA_PRED, DL, ContainerVT,
17468                             Pg, AccOp, VecOp);
17469 
17470   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Rdx, Zero);
17471 }
17472 
17473 SDValue AArch64TargetLowering::LowerPredReductionToSVE(SDValue ReduceOp,
17474                                                        SelectionDAG &DAG) const {
17475   SDLoc DL(ReduceOp);
17476   SDValue Op = ReduceOp.getOperand(0);
17477   EVT OpVT = Op.getValueType();
17478   EVT VT = ReduceOp.getValueType();
17479 
17480   if (!OpVT.isScalableVector() || OpVT.getVectorElementType() != MVT::i1)
17481     return SDValue();
17482 
17483   SDValue Pg = getPredicateForVector(DAG, DL, OpVT);
17484 
17485   switch (ReduceOp.getOpcode()) {
17486   default:
17487     return SDValue();
17488   case ISD::VECREDUCE_OR:
17489     return getPTest(DAG, VT, Pg, Op, AArch64CC::ANY_ACTIVE);
17490   case ISD::VECREDUCE_AND: {
17491     Op = DAG.getNode(ISD::XOR, DL, OpVT, Op, Pg);
17492     return getPTest(DAG, VT, Pg, Op, AArch64CC::NONE_ACTIVE);
17493   }
17494   case ISD::VECREDUCE_XOR: {
17495     SDValue ID =
17496         DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64);
17497     SDValue Cntp =
17498         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, ID, Pg, Op);
17499     return DAG.getAnyExtOrTrunc(Cntp, DL, VT);
17500   }
17501   }
17502 
17503   return SDValue();
17504 }
17505 
17506 SDValue AArch64TargetLowering::LowerReductionToSVE(unsigned Opcode,
17507                                                    SDValue ScalarOp,
17508                                                    SelectionDAG &DAG) const {
17509   SDLoc DL(ScalarOp);
17510   SDValue VecOp = ScalarOp.getOperand(0);
17511   EVT SrcVT = VecOp.getValueType();
17512 
17513   if (useSVEForFixedLengthVectorVT(SrcVT, true)) {
17514     EVT ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT);
17515     VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
17516   }
17517 
17518   // UADDV always returns an i64 result.
17519   EVT ResVT = (Opcode == AArch64ISD::UADDV_PRED) ? MVT::i64 :
17520                                                    SrcVT.getVectorElementType();
17521   EVT RdxVT = SrcVT;
17522   if (SrcVT.isFixedLengthVector() || Opcode == AArch64ISD::UADDV_PRED)
17523     RdxVT = getPackedSVEVectorVT(ResVT);
17524 
17525   SDValue Pg = getPredicateForVector(DAG, DL, SrcVT);
17526   SDValue Rdx = DAG.getNode(Opcode, DL, RdxVT, Pg, VecOp);
17527   SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT,
17528                             Rdx, DAG.getConstant(0, DL, MVT::i64));
17529 
17530   // The VEC_REDUCE nodes expect an element size result.
17531   if (ResVT != ScalarOp.getValueType())
17532     Res = DAG.getAnyExtOrTrunc(Res, DL, ScalarOp.getValueType());
17533 
17534   return Res;
17535 }
17536 
17537 SDValue
17538 AArch64TargetLowering::LowerFixedLengthVectorSelectToSVE(SDValue Op,
17539     SelectionDAG &DAG) const {
17540   EVT VT = Op.getValueType();
17541   SDLoc DL(Op);
17542 
17543   EVT InVT = Op.getOperand(1).getValueType();
17544   EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
17545   SDValue Op1 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(1));
17546   SDValue Op2 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(2));
17547 
17548   // Convert the mask to a predicated (NOTE: We don't need to worry about
17549   // inactive lanes since VSELECT is safe when given undefined elements).
17550   EVT MaskVT = Op.getOperand(0).getValueType();
17551   EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskVT);
17552   auto Mask = convertToScalableVector(DAG, MaskContainerVT, Op.getOperand(0));
17553   Mask = DAG.getNode(ISD::TRUNCATE, DL,
17554                      MaskContainerVT.changeVectorElementType(MVT::i1), Mask);
17555 
17556   auto ScalableRes = DAG.getNode(ISD::VSELECT, DL, ContainerVT,
17557                                 Mask, Op1, Op2);
17558 
17559   return convertFromScalableVector(DAG, VT, ScalableRes);
17560 }
17561 
17562 SDValue AArch64TargetLowering::LowerFixedLengthVectorSetccToSVE(
17563     SDValue Op, SelectionDAG &DAG) const {
17564   SDLoc DL(Op);
17565   EVT InVT = Op.getOperand(0).getValueType();
17566   EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
17567 
17568   assert(useSVEForFixedLengthVectorVT(InVT) &&
17569          "Only expected to lower fixed length vector operation!");
17570   assert(Op.getValueType() == InVT.changeTypeToInteger() &&
17571          "Expected integer result of the same bit length as the inputs!");
17572 
17573   auto Op1 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(0));
17574   auto Op2 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(1));
17575   auto Pg = getPredicateForFixedLengthVector(DAG, DL, InVT);
17576 
17577   EVT CmpVT = Pg.getValueType();
17578   auto Cmp = DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, CmpVT,
17579                          {Pg, Op1, Op2, Op.getOperand(2)});
17580 
17581   EVT PromoteVT = ContainerVT.changeTypeToInteger();
17582   auto Promote = DAG.getBoolExtOrTrunc(Cmp, DL, PromoteVT, InVT);
17583   return convertFromScalableVector(DAG, Op.getValueType(), Promote);
17584 }
17585 
17586 SDValue AArch64TargetLowering::getSVESafeBitCast(EVT VT, SDValue Op,
17587                                                  SelectionDAG &DAG) const {
17588   SDLoc DL(Op);
17589   EVT InVT = Op.getValueType();
17590   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17591   (void)TLI;
17592 
17593   assert(VT.isScalableVector() && TLI.isTypeLegal(VT) &&
17594          InVT.isScalableVector() && TLI.isTypeLegal(InVT) &&
17595          "Only expect to cast between legal scalable vector types!");
17596   assert((VT.getVectorElementType() == MVT::i1) ==
17597              (InVT.getVectorElementType() == MVT::i1) &&
17598          "Cannot cast between data and predicate scalable vector types!");
17599 
17600   if (InVT == VT)
17601     return Op;
17602 
17603   if (VT.getVectorElementType() == MVT::i1)
17604     return DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
17605 
17606   EVT PackedVT = getPackedSVEVectorVT(VT.getVectorElementType());
17607   EVT PackedInVT = getPackedSVEVectorVT(InVT.getVectorElementType());
17608   assert((VT == PackedVT || InVT == PackedInVT) &&
17609          "Cannot cast between unpacked scalable vector types!");
17610 
17611   // Pack input if required.
17612   if (InVT != PackedInVT)
17613     Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, PackedInVT, Op);
17614 
17615   Op = DAG.getNode(ISD::BITCAST, DL, PackedVT, Op);
17616 
17617   // Unpack result if required.
17618   if (VT != PackedVT)
17619     Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
17620 
17621   return Op;
17622 }
17623