1 //===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation  ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the AArch64TargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64ISelLowering.h"
14 #include "AArch64CallingConvention.h"
15 #include "AArch64ExpandImm.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64PerfectShuffle.h"
18 #include "AArch64RegisterInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "Utils/AArch64BaseInfo.h"
22 #include "llvm/ADT/APFloat.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/STLExtras.h"
26 #include "llvm/ADT/SmallSet.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/Statistic.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/Analysis/ObjCARCUtil.h"
33 #include "llvm/Analysis/VectorUtils.h"
34 #include "llvm/CodeGen/CallingConvLower.h"
35 #include "llvm/CodeGen/MachineBasicBlock.h"
36 #include "llvm/CodeGen/MachineFrameInfo.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineInstrBuilder.h"
40 #include "llvm/CodeGen/MachineMemOperand.h"
41 #include "llvm/CodeGen/MachineRegisterInfo.h"
42 #include "llvm/CodeGen/RuntimeLibcalls.h"
43 #include "llvm/CodeGen/SelectionDAG.h"
44 #include "llvm/CodeGen/SelectionDAGNodes.h"
45 #include "llvm/CodeGen/TargetCallingConv.h"
46 #include "llvm/CodeGen/TargetInstrInfo.h"
47 #include "llvm/CodeGen/ValueTypes.h"
48 #include "llvm/IR/Attributes.h"
49 #include "llvm/IR/Constants.h"
50 #include "llvm/IR/DataLayout.h"
51 #include "llvm/IR/DebugLoc.h"
52 #include "llvm/IR/DerivedTypes.h"
53 #include "llvm/IR/Function.h"
54 #include "llvm/IR/GetElementPtrTypeIterator.h"
55 #include "llvm/IR/GlobalValue.h"
56 #include "llvm/IR/IRBuilder.h"
57 #include "llvm/IR/Instruction.h"
58 #include "llvm/IR/Instructions.h"
59 #include "llvm/IR/IntrinsicInst.h"
60 #include "llvm/IR/Intrinsics.h"
61 #include "llvm/IR/IntrinsicsAArch64.h"
62 #include "llvm/IR/Module.h"
63 #include "llvm/IR/OperandTraits.h"
64 #include "llvm/IR/PatternMatch.h"
65 #include "llvm/IR/Type.h"
66 #include "llvm/IR/Use.h"
67 #include "llvm/IR/Value.h"
68 #include "llvm/MC/MCRegisterInfo.h"
69 #include "llvm/Support/Casting.h"
70 #include "llvm/Support/CodeGen.h"
71 #include "llvm/Support/CommandLine.h"
72 #include "llvm/Support/Compiler.h"
73 #include "llvm/Support/Debug.h"
74 #include "llvm/Support/ErrorHandling.h"
75 #include "llvm/Support/KnownBits.h"
76 #include "llvm/Support/MachineValueType.h"
77 #include "llvm/Support/MathExtras.h"
78 #include "llvm/Support/raw_ostream.h"
79 #include "llvm/Target/TargetMachine.h"
80 #include "llvm/Target/TargetOptions.h"
81 #include <algorithm>
82 #include <bitset>
83 #include <cassert>
84 #include <cctype>
85 #include <cstdint>
86 #include <cstdlib>
87 #include <iterator>
88 #include <limits>
89 #include <tuple>
90 #include <utility>
91 #include <vector>
92 
93 using namespace llvm;
94 using namespace llvm::PatternMatch;
95 
96 #define DEBUG_TYPE "aarch64-lower"
97 
98 STATISTIC(NumTailCalls, "Number of tail calls");
99 STATISTIC(NumShiftInserts, "Number of vector shift inserts");
100 STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
101 
102 // FIXME: The necessary dtprel relocations don't seem to be supported
103 // well in the GNU bfd and gold linkers at the moment. Therefore, by
104 // default, for now, fall back to GeneralDynamic code generation.
105 cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration(
106     "aarch64-elf-ldtls-generation", cl::Hidden,
107     cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
108     cl::init(false));
109 
110 static cl::opt<bool>
111 EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
112                          cl::desc("Enable AArch64 logical imm instruction "
113                                   "optimization"),
114                          cl::init(true));
115 
116 // Temporary option added for the purpose of testing functionality added
117 // to DAGCombiner.cpp in D92230. It is expected that this can be removed
118 // in future when both implementations will be based off MGATHER rather
119 // than the GLD1 nodes added for the SVE gather load intrinsics.
120 static cl::opt<bool>
121 EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden,
122                                 cl::desc("Combine extends of AArch64 masked "
123                                          "gather intrinsics"),
124                                 cl::init(true));
125 
126 /// Value type used for condition codes.
127 static const MVT MVT_CC = MVT::i32;
128 
129 static inline EVT getPackedSVEVectorVT(EVT VT) {
130   switch (VT.getSimpleVT().SimpleTy) {
131   default:
132     llvm_unreachable("unexpected element type for vector");
133   case MVT::i8:
134     return MVT::nxv16i8;
135   case MVT::i16:
136     return MVT::nxv8i16;
137   case MVT::i32:
138     return MVT::nxv4i32;
139   case MVT::i64:
140     return MVT::nxv2i64;
141   case MVT::f16:
142     return MVT::nxv8f16;
143   case MVT::f32:
144     return MVT::nxv4f32;
145   case MVT::f64:
146     return MVT::nxv2f64;
147   case MVT::bf16:
148     return MVT::nxv8bf16;
149   }
150 }
151 
152 // NOTE: Currently there's only a need to return integer vector types. If this
153 // changes then just add an extra "type" parameter.
154 static inline EVT getPackedSVEVectorVT(ElementCount EC) {
155   switch (EC.getKnownMinValue()) {
156   default:
157     llvm_unreachable("unexpected element count for vector");
158   case 16:
159     return MVT::nxv16i8;
160   case 8:
161     return MVT::nxv8i16;
162   case 4:
163     return MVT::nxv4i32;
164   case 2:
165     return MVT::nxv2i64;
166   }
167 }
168 
169 static inline EVT getPromotedVTForPredicate(EVT VT) {
170   assert(VT.isScalableVector() && (VT.getVectorElementType() == MVT::i1) &&
171          "Expected scalable predicate vector type!");
172   switch (VT.getVectorMinNumElements()) {
173   default:
174     llvm_unreachable("unexpected element count for vector");
175   case 2:
176     return MVT::nxv2i64;
177   case 4:
178     return MVT::nxv4i32;
179   case 8:
180     return MVT::nxv8i16;
181   case 16:
182     return MVT::nxv16i8;
183   }
184 }
185 
186 /// Returns true if VT's elements occupy the lowest bit positions of its
187 /// associated register class without any intervening space.
188 ///
189 /// For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the
190 /// same register class, but only nxv8f16 can be treated as a packed vector.
191 static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) {
192   assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
193          "Expected legal vector type!");
194   return VT.isFixedLengthVector() ||
195          VT.getSizeInBits().getKnownMinSize() == AArch64::SVEBitsPerBlock;
196 }
197 
198 // Returns true for ####_MERGE_PASSTHRU opcodes, whose operands have a leading
199 // predicate and end with a passthru value matching the result type.
200 static bool isMergePassthruOpcode(unsigned Opc) {
201   switch (Opc) {
202   default:
203     return false;
204   case AArch64ISD::BITREVERSE_MERGE_PASSTHRU:
205   case AArch64ISD::BSWAP_MERGE_PASSTHRU:
206   case AArch64ISD::CTLZ_MERGE_PASSTHRU:
207   case AArch64ISD::CTPOP_MERGE_PASSTHRU:
208   case AArch64ISD::DUP_MERGE_PASSTHRU:
209   case AArch64ISD::ABS_MERGE_PASSTHRU:
210   case AArch64ISD::NEG_MERGE_PASSTHRU:
211   case AArch64ISD::FNEG_MERGE_PASSTHRU:
212   case AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU:
213   case AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU:
214   case AArch64ISD::FCEIL_MERGE_PASSTHRU:
215   case AArch64ISD::FFLOOR_MERGE_PASSTHRU:
216   case AArch64ISD::FNEARBYINT_MERGE_PASSTHRU:
217   case AArch64ISD::FRINT_MERGE_PASSTHRU:
218   case AArch64ISD::FROUND_MERGE_PASSTHRU:
219   case AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU:
220   case AArch64ISD::FTRUNC_MERGE_PASSTHRU:
221   case AArch64ISD::FP_ROUND_MERGE_PASSTHRU:
222   case AArch64ISD::FP_EXTEND_MERGE_PASSTHRU:
223   case AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU:
224   case AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU:
225   case AArch64ISD::FCVTZU_MERGE_PASSTHRU:
226   case AArch64ISD::FCVTZS_MERGE_PASSTHRU:
227   case AArch64ISD::FSQRT_MERGE_PASSTHRU:
228   case AArch64ISD::FRECPX_MERGE_PASSTHRU:
229   case AArch64ISD::FABS_MERGE_PASSTHRU:
230     return true;
231   }
232 }
233 
234 AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
235                                              const AArch64Subtarget &STI)
236     : TargetLowering(TM), Subtarget(&STI) {
237   // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
238   // we have to make something up. Arbitrarily, choose ZeroOrOne.
239   setBooleanContents(ZeroOrOneBooleanContent);
240   // When comparing vectors the result sets the different elements in the
241   // vector to all-one or all-zero.
242   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
243 
244   // Set up the register classes.
245   addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
246   addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
247 
248   if (Subtarget->hasFPARMv8()) {
249     addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
250     addRegisterClass(MVT::bf16, &AArch64::FPR16RegClass);
251     addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
252     addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
253     addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
254   }
255 
256   if (Subtarget->hasNEON()) {
257     addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
258     addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
259     // Someone set us up the NEON.
260     addDRTypeForNEON(MVT::v2f32);
261     addDRTypeForNEON(MVT::v8i8);
262     addDRTypeForNEON(MVT::v4i16);
263     addDRTypeForNEON(MVT::v2i32);
264     addDRTypeForNEON(MVT::v1i64);
265     addDRTypeForNEON(MVT::v1f64);
266     addDRTypeForNEON(MVT::v4f16);
267     if (Subtarget->hasBF16())
268       addDRTypeForNEON(MVT::v4bf16);
269 
270     addQRTypeForNEON(MVT::v4f32);
271     addQRTypeForNEON(MVT::v2f64);
272     addQRTypeForNEON(MVT::v16i8);
273     addQRTypeForNEON(MVT::v8i16);
274     addQRTypeForNEON(MVT::v4i32);
275     addQRTypeForNEON(MVT::v2i64);
276     addQRTypeForNEON(MVT::v8f16);
277     if (Subtarget->hasBF16())
278       addQRTypeForNEON(MVT::v8bf16);
279   }
280 
281   if (Subtarget->hasSVE()) {
282     // Add legal sve predicate types
283     addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
284     addRegisterClass(MVT::nxv4i1, &AArch64::PPRRegClass);
285     addRegisterClass(MVT::nxv8i1, &AArch64::PPRRegClass);
286     addRegisterClass(MVT::nxv16i1, &AArch64::PPRRegClass);
287 
288     // Add legal sve data types
289     addRegisterClass(MVT::nxv16i8, &AArch64::ZPRRegClass);
290     addRegisterClass(MVT::nxv8i16, &AArch64::ZPRRegClass);
291     addRegisterClass(MVT::nxv4i32, &AArch64::ZPRRegClass);
292     addRegisterClass(MVT::nxv2i64, &AArch64::ZPRRegClass);
293 
294     addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
295     addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
296     addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
297     addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
298     addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
299     addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
300 
301     if (Subtarget->hasBF16()) {
302       addRegisterClass(MVT::nxv2bf16, &AArch64::ZPRRegClass);
303       addRegisterClass(MVT::nxv4bf16, &AArch64::ZPRRegClass);
304       addRegisterClass(MVT::nxv8bf16, &AArch64::ZPRRegClass);
305     }
306 
307     if (Subtarget->useSVEForFixedLengthVectors()) {
308       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
309         if (useSVEForFixedLengthVectorVT(VT))
310           addRegisterClass(VT, &AArch64::ZPRRegClass);
311 
312       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
313         if (useSVEForFixedLengthVectorVT(VT))
314           addRegisterClass(VT, &AArch64::ZPRRegClass);
315     }
316 
317     for (auto VT : { MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64 }) {
318       setOperationAction(ISD::SADDSAT, VT, Legal);
319       setOperationAction(ISD::UADDSAT, VT, Legal);
320       setOperationAction(ISD::SSUBSAT, VT, Legal);
321       setOperationAction(ISD::USUBSAT, VT, Legal);
322       setOperationAction(ISD::UREM, VT, Expand);
323       setOperationAction(ISD::SREM, VT, Expand);
324       setOperationAction(ISD::SDIVREM, VT, Expand);
325       setOperationAction(ISD::UDIVREM, VT, Expand);
326     }
327 
328     for (auto VT :
329          { MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64, MVT::nxv4i8,
330            MVT::nxv4i16, MVT::nxv4i32, MVT::nxv8i8, MVT::nxv8i16 })
331       setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal);
332 
333     for (auto VT :
334          { MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32, MVT::nxv4f32,
335            MVT::nxv2f64 }) {
336       setCondCodeAction(ISD::SETO, VT, Expand);
337       setCondCodeAction(ISD::SETOLT, VT, Expand);
338       setCondCodeAction(ISD::SETLT, VT, Expand);
339       setCondCodeAction(ISD::SETOLE, VT, Expand);
340       setCondCodeAction(ISD::SETLE, VT, Expand);
341       setCondCodeAction(ISD::SETULT, VT, Expand);
342       setCondCodeAction(ISD::SETULE, VT, Expand);
343       setCondCodeAction(ISD::SETUGE, VT, Expand);
344       setCondCodeAction(ISD::SETUGT, VT, Expand);
345       setCondCodeAction(ISD::SETUEQ, VT, Expand);
346       setCondCodeAction(ISD::SETUNE, VT, Expand);
347 
348       setOperationAction(ISD::FREM, VT, Expand);
349       setOperationAction(ISD::FPOW, VT, Expand);
350       setOperationAction(ISD::FPOWI, VT, Expand);
351       setOperationAction(ISD::FCOS, VT, Expand);
352       setOperationAction(ISD::FSIN, VT, Expand);
353       setOperationAction(ISD::FSINCOS, VT, Expand);
354       setOperationAction(ISD::FEXP, VT, Expand);
355       setOperationAction(ISD::FEXP2, VT, Expand);
356       setOperationAction(ISD::FLOG, VT, Expand);
357       setOperationAction(ISD::FLOG2, VT, Expand);
358       setOperationAction(ISD::FLOG10, VT, Expand);
359     }
360   }
361 
362   // Compute derived properties from the register classes
363   computeRegisterProperties(Subtarget->getRegisterInfo());
364 
365   // Provide all sorts of operation actions
366   setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
367   setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
368   setOperationAction(ISD::SETCC, MVT::i32, Custom);
369   setOperationAction(ISD::SETCC, MVT::i64, Custom);
370   setOperationAction(ISD::SETCC, MVT::f16, Custom);
371   setOperationAction(ISD::SETCC, MVT::f32, Custom);
372   setOperationAction(ISD::SETCC, MVT::f64, Custom);
373   setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom);
374   setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom);
375   setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom);
376   setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom);
377   setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom);
378   setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom);
379   setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
380   setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
381   setOperationAction(ISD::BRCOND, MVT::Other, Expand);
382   setOperationAction(ISD::BR_CC, MVT::i32, Custom);
383   setOperationAction(ISD::BR_CC, MVT::i64, Custom);
384   setOperationAction(ISD::BR_CC, MVT::f16, Custom);
385   setOperationAction(ISD::BR_CC, MVT::f32, Custom);
386   setOperationAction(ISD::BR_CC, MVT::f64, Custom);
387   setOperationAction(ISD::SELECT, MVT::i32, Custom);
388   setOperationAction(ISD::SELECT, MVT::i64, Custom);
389   setOperationAction(ISD::SELECT, MVT::f16, Custom);
390   setOperationAction(ISD::SELECT, MVT::f32, Custom);
391   setOperationAction(ISD::SELECT, MVT::f64, Custom);
392   setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
393   setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
394   setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
395   setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
396   setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
397   setOperationAction(ISD::BR_JT, MVT::Other, Custom);
398   setOperationAction(ISD::JumpTable, MVT::i64, Custom);
399 
400   setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
401   setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
402   setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
403 
404   setOperationAction(ISD::FREM, MVT::f32, Expand);
405   setOperationAction(ISD::FREM, MVT::f64, Expand);
406   setOperationAction(ISD::FREM, MVT::f80, Expand);
407 
408   setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
409 
410   // Custom lowering hooks are needed for XOR
411   // to fold it into CSINC/CSINV.
412   setOperationAction(ISD::XOR, MVT::i32, Custom);
413   setOperationAction(ISD::XOR, MVT::i64, Custom);
414 
415   // Virtually no operation on f128 is legal, but LLVM can't expand them when
416   // there's a valid register class, so we need custom operations in most cases.
417   setOperationAction(ISD::FABS, MVT::f128, Expand);
418   setOperationAction(ISD::FADD, MVT::f128, LibCall);
419   setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
420   setOperationAction(ISD::FCOS, MVT::f128, Expand);
421   setOperationAction(ISD::FDIV, MVT::f128, LibCall);
422   setOperationAction(ISD::FMA, MVT::f128, Expand);
423   setOperationAction(ISD::FMUL, MVT::f128, LibCall);
424   setOperationAction(ISD::FNEG, MVT::f128, Expand);
425   setOperationAction(ISD::FPOW, MVT::f128, Expand);
426   setOperationAction(ISD::FREM, MVT::f128, Expand);
427   setOperationAction(ISD::FRINT, MVT::f128, Expand);
428   setOperationAction(ISD::FSIN, MVT::f128, Expand);
429   setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
430   setOperationAction(ISD::FSQRT, MVT::f128, Expand);
431   setOperationAction(ISD::FSUB, MVT::f128, LibCall);
432   setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
433   setOperationAction(ISD::SETCC, MVT::f128, Custom);
434   setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom);
435   setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom);
436   setOperationAction(ISD::BR_CC, MVT::f128, Custom);
437   setOperationAction(ISD::SELECT, MVT::f128, Custom);
438   setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
439   setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
440 
441   // Lowering for many of the conversions is actually specified by the non-f128
442   // type. The LowerXXX function will be trivial when f128 isn't involved.
443   setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
444   setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
445   setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
446   setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
447   setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
448   setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
449   setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
450   setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
451   setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
452   setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
453   setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
454   setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
455   setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
456   setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
457   setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
458   setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
459   setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
460   setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
461   setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
462   setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
463   setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
464   setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
465   setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
466   setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
467   setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
468   setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
469   setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
470   setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
471   setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
472   setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
473 
474   // Variable arguments.
475   setOperationAction(ISD::VASTART, MVT::Other, Custom);
476   setOperationAction(ISD::VAARG, MVT::Other, Custom);
477   setOperationAction(ISD::VACOPY, MVT::Other, Custom);
478   setOperationAction(ISD::VAEND, MVT::Other, Expand);
479 
480   // Variable-sized objects.
481   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
482   setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
483 
484   if (Subtarget->isTargetWindows())
485     setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
486   else
487     setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
488 
489   // Constant pool entries
490   setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
491 
492   // BlockAddress
493   setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
494 
495   // Add/Sub overflow ops with MVT::Glues are lowered to NZCV dependences.
496   setOperationAction(ISD::ADDC, MVT::i32, Custom);
497   setOperationAction(ISD::ADDE, MVT::i32, Custom);
498   setOperationAction(ISD::SUBC, MVT::i32, Custom);
499   setOperationAction(ISD::SUBE, MVT::i32, Custom);
500   setOperationAction(ISD::ADDC, MVT::i64, Custom);
501   setOperationAction(ISD::ADDE, MVT::i64, Custom);
502   setOperationAction(ISD::SUBC, MVT::i64, Custom);
503   setOperationAction(ISD::SUBE, MVT::i64, Custom);
504 
505   // AArch64 lacks both left-rotate and popcount instructions.
506   setOperationAction(ISD::ROTL, MVT::i32, Expand);
507   setOperationAction(ISD::ROTL, MVT::i64, Expand);
508   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
509     setOperationAction(ISD::ROTL, VT, Expand);
510     setOperationAction(ISD::ROTR, VT, Expand);
511   }
512 
513   // AArch64 doesn't have i32 MULH{S|U}.
514   setOperationAction(ISD::MULHU, MVT::i32, Expand);
515   setOperationAction(ISD::MULHS, MVT::i32, Expand);
516 
517   // AArch64 doesn't have {U|S}MUL_LOHI.
518   setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
519   setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
520 
521   setOperationAction(ISD::CTPOP, MVT::i32, Custom);
522   setOperationAction(ISD::CTPOP, MVT::i64, Custom);
523   setOperationAction(ISD::CTPOP, MVT::i128, Custom);
524 
525   setOperationAction(ISD::ABS, MVT::i32, Custom);
526   setOperationAction(ISD::ABS, MVT::i64, Custom);
527 
528   setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
529   setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
530   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
531     setOperationAction(ISD::SDIVREM, VT, Expand);
532     setOperationAction(ISD::UDIVREM, VT, Expand);
533   }
534   setOperationAction(ISD::SREM, MVT::i32, Expand);
535   setOperationAction(ISD::SREM, MVT::i64, Expand);
536   setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
537   setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
538   setOperationAction(ISD::UREM, MVT::i32, Expand);
539   setOperationAction(ISD::UREM, MVT::i64, Expand);
540 
541   // Custom lower Add/Sub/Mul with overflow.
542   setOperationAction(ISD::SADDO, MVT::i32, Custom);
543   setOperationAction(ISD::SADDO, MVT::i64, Custom);
544   setOperationAction(ISD::UADDO, MVT::i32, Custom);
545   setOperationAction(ISD::UADDO, MVT::i64, Custom);
546   setOperationAction(ISD::SSUBO, MVT::i32, Custom);
547   setOperationAction(ISD::SSUBO, MVT::i64, Custom);
548   setOperationAction(ISD::USUBO, MVT::i32, Custom);
549   setOperationAction(ISD::USUBO, MVT::i64, Custom);
550   setOperationAction(ISD::SMULO, MVT::i32, Custom);
551   setOperationAction(ISD::SMULO, MVT::i64, Custom);
552   setOperationAction(ISD::UMULO, MVT::i32, Custom);
553   setOperationAction(ISD::UMULO, MVT::i64, Custom);
554 
555   setOperationAction(ISD::FSIN, MVT::f32, Expand);
556   setOperationAction(ISD::FSIN, MVT::f64, Expand);
557   setOperationAction(ISD::FCOS, MVT::f32, Expand);
558   setOperationAction(ISD::FCOS, MVT::f64, Expand);
559   setOperationAction(ISD::FPOW, MVT::f32, Expand);
560   setOperationAction(ISD::FPOW, MVT::f64, Expand);
561   setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
562   setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
563   if (Subtarget->hasFullFP16())
564     setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
565   else
566     setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote);
567 
568   setOperationAction(ISD::FREM,    MVT::f16,   Promote);
569   setOperationAction(ISD::FREM,    MVT::v4f16, Expand);
570   setOperationAction(ISD::FREM,    MVT::v8f16, Expand);
571   setOperationAction(ISD::FPOW,    MVT::f16,   Promote);
572   setOperationAction(ISD::FPOW,    MVT::v4f16, Expand);
573   setOperationAction(ISD::FPOW,    MVT::v8f16, Expand);
574   setOperationAction(ISD::FPOWI,   MVT::f16,   Promote);
575   setOperationAction(ISD::FPOWI,   MVT::v4f16, Expand);
576   setOperationAction(ISD::FPOWI,   MVT::v8f16, Expand);
577   setOperationAction(ISD::FCOS,    MVT::f16,   Promote);
578   setOperationAction(ISD::FCOS,    MVT::v4f16, Expand);
579   setOperationAction(ISD::FCOS,    MVT::v8f16, Expand);
580   setOperationAction(ISD::FSIN,    MVT::f16,   Promote);
581   setOperationAction(ISD::FSIN,    MVT::v4f16, Expand);
582   setOperationAction(ISD::FSIN,    MVT::v8f16, Expand);
583   setOperationAction(ISD::FSINCOS, MVT::f16,   Promote);
584   setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand);
585   setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand);
586   setOperationAction(ISD::FEXP,    MVT::f16,   Promote);
587   setOperationAction(ISD::FEXP,    MVT::v4f16, Expand);
588   setOperationAction(ISD::FEXP,    MVT::v8f16, Expand);
589   setOperationAction(ISD::FEXP2,   MVT::f16,   Promote);
590   setOperationAction(ISD::FEXP2,   MVT::v4f16, Expand);
591   setOperationAction(ISD::FEXP2,   MVT::v8f16, Expand);
592   setOperationAction(ISD::FLOG,    MVT::f16,   Promote);
593   setOperationAction(ISD::FLOG,    MVT::v4f16, Expand);
594   setOperationAction(ISD::FLOG,    MVT::v8f16, Expand);
595   setOperationAction(ISD::FLOG2,   MVT::f16,   Promote);
596   setOperationAction(ISD::FLOG2,   MVT::v4f16, Expand);
597   setOperationAction(ISD::FLOG2,   MVT::v8f16, Expand);
598   setOperationAction(ISD::FLOG10,  MVT::f16,   Promote);
599   setOperationAction(ISD::FLOG10,  MVT::v4f16, Expand);
600   setOperationAction(ISD::FLOG10,  MVT::v8f16, Expand);
601 
602   if (!Subtarget->hasFullFP16()) {
603     setOperationAction(ISD::SELECT,      MVT::f16,  Promote);
604     setOperationAction(ISD::SELECT_CC,   MVT::f16,  Promote);
605     setOperationAction(ISD::SETCC,       MVT::f16,  Promote);
606     setOperationAction(ISD::BR_CC,       MVT::f16,  Promote);
607     setOperationAction(ISD::FADD,        MVT::f16,  Promote);
608     setOperationAction(ISD::FSUB,        MVT::f16,  Promote);
609     setOperationAction(ISD::FMUL,        MVT::f16,  Promote);
610     setOperationAction(ISD::FDIV,        MVT::f16,  Promote);
611     setOperationAction(ISD::FMA,         MVT::f16,  Promote);
612     setOperationAction(ISD::FNEG,        MVT::f16,  Promote);
613     setOperationAction(ISD::FABS,        MVT::f16,  Promote);
614     setOperationAction(ISD::FCEIL,       MVT::f16,  Promote);
615     setOperationAction(ISD::FSQRT,       MVT::f16,  Promote);
616     setOperationAction(ISD::FFLOOR,      MVT::f16,  Promote);
617     setOperationAction(ISD::FNEARBYINT,  MVT::f16,  Promote);
618     setOperationAction(ISD::FRINT,       MVT::f16,  Promote);
619     setOperationAction(ISD::FROUND,      MVT::f16,  Promote);
620     setOperationAction(ISD::FROUNDEVEN,  MVT::f16,  Promote);
621     setOperationAction(ISD::FTRUNC,      MVT::f16,  Promote);
622     setOperationAction(ISD::FMINNUM,     MVT::f16,  Promote);
623     setOperationAction(ISD::FMAXNUM,     MVT::f16,  Promote);
624     setOperationAction(ISD::FMINIMUM,    MVT::f16,  Promote);
625     setOperationAction(ISD::FMAXIMUM,    MVT::f16,  Promote);
626 
627     // promote v4f16 to v4f32 when that is known to be safe.
628     setOperationAction(ISD::FADD,        MVT::v4f16, Promote);
629     setOperationAction(ISD::FSUB,        MVT::v4f16, Promote);
630     setOperationAction(ISD::FMUL,        MVT::v4f16, Promote);
631     setOperationAction(ISD::FDIV,        MVT::v4f16, Promote);
632     AddPromotedToType(ISD::FADD,         MVT::v4f16, MVT::v4f32);
633     AddPromotedToType(ISD::FSUB,         MVT::v4f16, MVT::v4f32);
634     AddPromotedToType(ISD::FMUL,         MVT::v4f16, MVT::v4f32);
635     AddPromotedToType(ISD::FDIV,         MVT::v4f16, MVT::v4f32);
636 
637     setOperationAction(ISD::FABS,        MVT::v4f16, Expand);
638     setOperationAction(ISD::FNEG,        MVT::v4f16, Expand);
639     setOperationAction(ISD::FROUND,      MVT::v4f16, Expand);
640     setOperationAction(ISD::FROUNDEVEN,  MVT::v4f16, Expand);
641     setOperationAction(ISD::FMA,         MVT::v4f16, Expand);
642     setOperationAction(ISD::SETCC,       MVT::v4f16, Expand);
643     setOperationAction(ISD::BR_CC,       MVT::v4f16, Expand);
644     setOperationAction(ISD::SELECT,      MVT::v4f16, Expand);
645     setOperationAction(ISD::SELECT_CC,   MVT::v4f16, Expand);
646     setOperationAction(ISD::FTRUNC,      MVT::v4f16, Expand);
647     setOperationAction(ISD::FCOPYSIGN,   MVT::v4f16, Expand);
648     setOperationAction(ISD::FFLOOR,      MVT::v4f16, Expand);
649     setOperationAction(ISD::FCEIL,       MVT::v4f16, Expand);
650     setOperationAction(ISD::FRINT,       MVT::v4f16, Expand);
651     setOperationAction(ISD::FNEARBYINT,  MVT::v4f16, Expand);
652     setOperationAction(ISD::FSQRT,       MVT::v4f16, Expand);
653 
654     setOperationAction(ISD::FABS,        MVT::v8f16, Expand);
655     setOperationAction(ISD::FADD,        MVT::v8f16, Expand);
656     setOperationAction(ISD::FCEIL,       MVT::v8f16, Expand);
657     setOperationAction(ISD::FCOPYSIGN,   MVT::v8f16, Expand);
658     setOperationAction(ISD::FDIV,        MVT::v8f16, Expand);
659     setOperationAction(ISD::FFLOOR,      MVT::v8f16, Expand);
660     setOperationAction(ISD::FMA,         MVT::v8f16, Expand);
661     setOperationAction(ISD::FMUL,        MVT::v8f16, Expand);
662     setOperationAction(ISD::FNEARBYINT,  MVT::v8f16, Expand);
663     setOperationAction(ISD::FNEG,        MVT::v8f16, Expand);
664     setOperationAction(ISD::FROUND,      MVT::v8f16, Expand);
665     setOperationAction(ISD::FROUNDEVEN,  MVT::v8f16, Expand);
666     setOperationAction(ISD::FRINT,       MVT::v8f16, Expand);
667     setOperationAction(ISD::FSQRT,       MVT::v8f16, Expand);
668     setOperationAction(ISD::FSUB,        MVT::v8f16, Expand);
669     setOperationAction(ISD::FTRUNC,      MVT::v8f16, Expand);
670     setOperationAction(ISD::SETCC,       MVT::v8f16, Expand);
671     setOperationAction(ISD::BR_CC,       MVT::v8f16, Expand);
672     setOperationAction(ISD::SELECT,      MVT::v8f16, Expand);
673     setOperationAction(ISD::SELECT_CC,   MVT::v8f16, Expand);
674     setOperationAction(ISD::FP_EXTEND,   MVT::v8f16, Expand);
675   }
676 
677   // AArch64 has implementations of a lot of rounding-like FP operations.
678   for (MVT Ty : {MVT::f32, MVT::f64}) {
679     setOperationAction(ISD::FFLOOR, Ty, Legal);
680     setOperationAction(ISD::FNEARBYINT, Ty, Legal);
681     setOperationAction(ISD::FCEIL, Ty, Legal);
682     setOperationAction(ISD::FRINT, Ty, Legal);
683     setOperationAction(ISD::FTRUNC, Ty, Legal);
684     setOperationAction(ISD::FROUND, Ty, Legal);
685     setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
686     setOperationAction(ISD::FMINNUM, Ty, Legal);
687     setOperationAction(ISD::FMAXNUM, Ty, Legal);
688     setOperationAction(ISD::FMINIMUM, Ty, Legal);
689     setOperationAction(ISD::FMAXIMUM, Ty, Legal);
690     setOperationAction(ISD::LROUND, Ty, Legal);
691     setOperationAction(ISD::LLROUND, Ty, Legal);
692     setOperationAction(ISD::LRINT, Ty, Legal);
693     setOperationAction(ISD::LLRINT, Ty, Legal);
694   }
695 
696   if (Subtarget->hasFullFP16()) {
697     setOperationAction(ISD::FNEARBYINT, MVT::f16, Legal);
698     setOperationAction(ISD::FFLOOR,  MVT::f16, Legal);
699     setOperationAction(ISD::FCEIL,   MVT::f16, Legal);
700     setOperationAction(ISD::FRINT,   MVT::f16, Legal);
701     setOperationAction(ISD::FTRUNC,  MVT::f16, Legal);
702     setOperationAction(ISD::FROUND,  MVT::f16, Legal);
703     setOperationAction(ISD::FROUNDEVEN,  MVT::f16, Legal);
704     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
705     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
706     setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
707     setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
708   }
709 
710   setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
711 
712   setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
713   setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
714 
715   setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
716   setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
717   setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
718   setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
719   setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
720 
721   // Generate outline atomics library calls only if LSE was not specified for
722   // subtarget
723   if (Subtarget->outlineAtomics() && !Subtarget->hasLSE()) {
724     setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, LibCall);
725     setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, LibCall);
726     setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall);
727     setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, LibCall);
728     setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, LibCall);
729     setOperationAction(ISD::ATOMIC_SWAP, MVT::i8, LibCall);
730     setOperationAction(ISD::ATOMIC_SWAP, MVT::i16, LibCall);
731     setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall);
732     setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, LibCall);
733     setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i8, LibCall);
734     setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i16, LibCall);
735     setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall);
736     setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, LibCall);
737     setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i8, LibCall);
738     setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i16, LibCall);
739     setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall);
740     setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, LibCall);
741     setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i8, LibCall);
742     setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i16, LibCall);
743     setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i32, LibCall);
744     setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i64, LibCall);
745     setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i8, LibCall);
746     setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i16, LibCall);
747     setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall);
748     setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, LibCall);
749 #define LCALLNAMES(A, B, N)                                                    \
750   setLibcallName(A##N##_RELAX, #B #N "_relax");                                \
751   setLibcallName(A##N##_ACQ, #B #N "_acq");                                    \
752   setLibcallName(A##N##_REL, #B #N "_rel");                                    \
753   setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");
754 #define LCALLNAME4(A, B)                                                       \
755   LCALLNAMES(A, B, 1)                                                          \
756   LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
757 #define LCALLNAME5(A, B)                                                       \
758   LCALLNAMES(A, B, 1)                                                          \
759   LCALLNAMES(A, B, 2)                                                          \
760   LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
761     LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, __aarch64_cas)
762     LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, __aarch64_swp)
763     LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, __aarch64_ldadd)
764     LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, __aarch64_ldset)
765     LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, __aarch64_ldclr)
766     LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, __aarch64_ldeor)
767 #undef LCALLNAMES
768 #undef LCALLNAME4
769 #undef LCALLNAME5
770   }
771 
772   // 128-bit loads and stores can be done without expanding
773   setOperationAction(ISD::LOAD, MVT::i128, Custom);
774   setOperationAction(ISD::STORE, MVT::i128, Custom);
775 
776   // 256 bit non-temporal stores can be lowered to STNP. Do this as part of the
777   // custom lowering, as there are no un-paired non-temporal stores and
778   // legalization will break up 256 bit inputs.
779   setOperationAction(ISD::STORE, MVT::v32i8, Custom);
780   setOperationAction(ISD::STORE, MVT::v16i16, Custom);
781   setOperationAction(ISD::STORE, MVT::v16f16, Custom);
782   setOperationAction(ISD::STORE, MVT::v8i32, Custom);
783   setOperationAction(ISD::STORE, MVT::v8f32, Custom);
784   setOperationAction(ISD::STORE, MVT::v4f64, Custom);
785   setOperationAction(ISD::STORE, MVT::v4i64, Custom);
786 
787   // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0.
788   // This requires the Performance Monitors extension.
789   if (Subtarget->hasPerfMon())
790     setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
791 
792   if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
793       getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
794     // Issue __sincos_stret if available.
795     setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
796     setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
797   } else {
798     setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
799     setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
800   }
801 
802   if (Subtarget->getTargetTriple().isOSMSVCRT()) {
803     // MSVCRT doesn't have powi; fall back to pow
804     setLibcallName(RTLIB::POWI_F32, nullptr);
805     setLibcallName(RTLIB::POWI_F64, nullptr);
806   }
807 
808   // Make floating-point constants legal for the large code model, so they don't
809   // become loads from the constant pool.
810   if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) {
811     setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
812     setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
813   }
814 
815   // AArch64 does not have floating-point extending loads, i1 sign-extending
816   // load, floating-point truncating stores, or v2i32->v2i16 truncating store.
817   for (MVT VT : MVT::fp_valuetypes()) {
818     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
819     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
820     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand);
821     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
822   }
823   for (MVT VT : MVT::integer_valuetypes())
824     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand);
825 
826   setTruncStoreAction(MVT::f32, MVT::f16, Expand);
827   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
828   setTruncStoreAction(MVT::f64, MVT::f16, Expand);
829   setTruncStoreAction(MVT::f128, MVT::f80, Expand);
830   setTruncStoreAction(MVT::f128, MVT::f64, Expand);
831   setTruncStoreAction(MVT::f128, MVT::f32, Expand);
832   setTruncStoreAction(MVT::f128, MVT::f16, Expand);
833 
834   setOperationAction(ISD::BITCAST, MVT::i16, Custom);
835   setOperationAction(ISD::BITCAST, MVT::f16, Custom);
836   setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
837 
838   // Indexed loads and stores are supported.
839   for (unsigned im = (unsigned)ISD::PRE_INC;
840        im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
841     setIndexedLoadAction(im, MVT::i8, Legal);
842     setIndexedLoadAction(im, MVT::i16, Legal);
843     setIndexedLoadAction(im, MVT::i32, Legal);
844     setIndexedLoadAction(im, MVT::i64, Legal);
845     setIndexedLoadAction(im, MVT::f64, Legal);
846     setIndexedLoadAction(im, MVT::f32, Legal);
847     setIndexedLoadAction(im, MVT::f16, Legal);
848     setIndexedLoadAction(im, MVT::bf16, Legal);
849     setIndexedStoreAction(im, MVT::i8, Legal);
850     setIndexedStoreAction(im, MVT::i16, Legal);
851     setIndexedStoreAction(im, MVT::i32, Legal);
852     setIndexedStoreAction(im, MVT::i64, Legal);
853     setIndexedStoreAction(im, MVT::f64, Legal);
854     setIndexedStoreAction(im, MVT::f32, Legal);
855     setIndexedStoreAction(im, MVT::f16, Legal);
856     setIndexedStoreAction(im, MVT::bf16, Legal);
857   }
858 
859   // Trap.
860   setOperationAction(ISD::TRAP, MVT::Other, Legal);
861   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
862   setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
863 
864   // We combine OR nodes for bitfield operations.
865   setTargetDAGCombine(ISD::OR);
866   // Try to create BICs for vector ANDs.
867   setTargetDAGCombine(ISD::AND);
868 
869   // Vector add and sub nodes may conceal a high-half opportunity.
870   // Also, try to fold ADD into CSINC/CSINV..
871   setTargetDAGCombine(ISD::ADD);
872   setTargetDAGCombine(ISD::ABS);
873   setTargetDAGCombine(ISD::SUB);
874   setTargetDAGCombine(ISD::SRL);
875   setTargetDAGCombine(ISD::XOR);
876   setTargetDAGCombine(ISD::SINT_TO_FP);
877   setTargetDAGCombine(ISD::UINT_TO_FP);
878 
879   setTargetDAGCombine(ISD::FP_TO_SINT);
880   setTargetDAGCombine(ISD::FP_TO_UINT);
881   setTargetDAGCombine(ISD::FDIV);
882 
883   setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
884 
885   setTargetDAGCombine(ISD::ANY_EXTEND);
886   setTargetDAGCombine(ISD::ZERO_EXTEND);
887   setTargetDAGCombine(ISD::SIGN_EXTEND);
888   setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
889   setTargetDAGCombine(ISD::TRUNCATE);
890   setTargetDAGCombine(ISD::CONCAT_VECTORS);
891   setTargetDAGCombine(ISD::STORE);
892   if (Subtarget->supportsAddressTopByteIgnored())
893     setTargetDAGCombine(ISD::LOAD);
894 
895   setTargetDAGCombine(ISD::MUL);
896 
897   setTargetDAGCombine(ISD::SELECT);
898   setTargetDAGCombine(ISD::VSELECT);
899 
900   setTargetDAGCombine(ISD::INTRINSIC_VOID);
901   setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
902   setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
903   setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
904   setTargetDAGCombine(ISD::VECREDUCE_ADD);
905 
906   setTargetDAGCombine(ISD::GlobalAddress);
907 
908   // In case of strict alignment, avoid an excessive number of byte wide stores.
909   MaxStoresPerMemsetOptSize = 8;
910   MaxStoresPerMemset = Subtarget->requiresStrictAlign()
911                        ? MaxStoresPerMemsetOptSize : 32;
912 
913   MaxGluedStoresPerMemcpy = 4;
914   MaxStoresPerMemcpyOptSize = 4;
915   MaxStoresPerMemcpy = Subtarget->requiresStrictAlign()
916                        ? MaxStoresPerMemcpyOptSize : 16;
917 
918   MaxStoresPerMemmoveOptSize = MaxStoresPerMemmove = 4;
919 
920   MaxLoadsPerMemcmpOptSize = 4;
921   MaxLoadsPerMemcmp = Subtarget->requiresStrictAlign()
922                       ? MaxLoadsPerMemcmpOptSize : 8;
923 
924   setStackPointerRegisterToSaveRestore(AArch64::SP);
925 
926   setSchedulingPreference(Sched::Hybrid);
927 
928   EnableExtLdPromotion = true;
929 
930   // Set required alignment.
931   setMinFunctionAlignment(Align(4));
932   // Set preferred alignments.
933   setPrefLoopAlignment(Align(1ULL << STI.getPrefLoopLogAlignment()));
934   setPrefFunctionAlignment(Align(1ULL << STI.getPrefFunctionLogAlignment()));
935 
936   // Only change the limit for entries in a jump table if specified by
937   // the sub target, but not at the command line.
938   unsigned MaxJT = STI.getMaximumJumpTableSize();
939   if (MaxJT && getMaximumJumpTableSize() == UINT_MAX)
940     setMaximumJumpTableSize(MaxJT);
941 
942   setHasExtractBitsInsn(true);
943 
944   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
945 
946   if (Subtarget->hasNEON()) {
947     // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
948     // silliness like this:
949     setOperationAction(ISD::FABS, MVT::v1f64, Expand);
950     setOperationAction(ISD::FADD, MVT::v1f64, Expand);
951     setOperationAction(ISD::FCEIL, MVT::v1f64, Expand);
952     setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand);
953     setOperationAction(ISD::FCOS, MVT::v1f64, Expand);
954     setOperationAction(ISD::FDIV, MVT::v1f64, Expand);
955     setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand);
956     setOperationAction(ISD::FMA, MVT::v1f64, Expand);
957     setOperationAction(ISD::FMUL, MVT::v1f64, Expand);
958     setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand);
959     setOperationAction(ISD::FNEG, MVT::v1f64, Expand);
960     setOperationAction(ISD::FPOW, MVT::v1f64, Expand);
961     setOperationAction(ISD::FREM, MVT::v1f64, Expand);
962     setOperationAction(ISD::FROUND, MVT::v1f64, Expand);
963     setOperationAction(ISD::FROUNDEVEN, MVT::v1f64, Expand);
964     setOperationAction(ISD::FRINT, MVT::v1f64, Expand);
965     setOperationAction(ISD::FSIN, MVT::v1f64, Expand);
966     setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand);
967     setOperationAction(ISD::FSQRT, MVT::v1f64, Expand);
968     setOperationAction(ISD::FSUB, MVT::v1f64, Expand);
969     setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand);
970     setOperationAction(ISD::SETCC, MVT::v1f64, Expand);
971     setOperationAction(ISD::BR_CC, MVT::v1f64, Expand);
972     setOperationAction(ISD::SELECT, MVT::v1f64, Expand);
973     setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand);
974     setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand);
975 
976     setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand);
977     setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand);
978     setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand);
979     setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand);
980     setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand);
981 
982     setOperationAction(ISD::MUL, MVT::v1i64, Expand);
983 
984     // AArch64 doesn't have a direct vector ->f32 conversion instructions for
985     // elements smaller than i32, so promote the input to i32 first.
986     setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32);
987     setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32);
988     // i8 vector elements also need promotion to i32 for v8i8
989     setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32);
990     setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32);
991     // Similarly, there is no direct i32 -> f64 vector conversion instruction.
992     setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
993     setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
994     setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom);
995     setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom);
996     // Or, direct i32 -> f16 vector conversion.  Set it so custom, so the
997     // conversion happens in two steps: v4i32 -> v4f32 -> v4f16
998     setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
999     setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
1000 
1001     if (Subtarget->hasFullFP16()) {
1002       setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
1003       setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
1004       setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
1005       setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
1006     } else {
1007       // when AArch64 doesn't have fullfp16 support, promote the input
1008       // to i32 first.
1009       setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32);
1010       setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32);
1011       setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32);
1012       setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32);
1013     }
1014 
1015     setOperationAction(ISD::CTLZ,       MVT::v1i64, Expand);
1016     setOperationAction(ISD::CTLZ,       MVT::v2i64, Expand);
1017 
1018     // AArch64 doesn't have MUL.2d:
1019     setOperationAction(ISD::MUL, MVT::v2i64, Expand);
1020     // Custom handling for some quad-vector types to detect MULL.
1021     setOperationAction(ISD::MUL, MVT::v8i16, Custom);
1022     setOperationAction(ISD::MUL, MVT::v4i32, Custom);
1023     setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1024 
1025     // Saturates
1026     for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
1027                     MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1028       setOperationAction(ISD::SADDSAT, VT, Legal);
1029       setOperationAction(ISD::UADDSAT, VT, Legal);
1030       setOperationAction(ISD::SSUBSAT, VT, Legal);
1031       setOperationAction(ISD::USUBSAT, VT, Legal);
1032     }
1033 
1034     // Vector reductions
1035     for (MVT VT : { MVT::v4f16, MVT::v2f32,
1036                     MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
1037       if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) {
1038         setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1039         setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1040 
1041         setOperationAction(ISD::VECREDUCE_FADD, VT, Legal);
1042       }
1043     }
1044     for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
1045                     MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
1046       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1047       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1048       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1049       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1050       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1051     }
1052     setOperationAction(ISD::VECREDUCE_ADD, MVT::v2i64, Custom);
1053 
1054     setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
1055     setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
1056     // Likewise, narrowing and extending vector loads/stores aren't handled
1057     // directly.
1058     for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1059       setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
1060 
1061       if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
1062         setOperationAction(ISD::MULHS, VT, Legal);
1063         setOperationAction(ISD::MULHU, VT, Legal);
1064       } else {
1065         setOperationAction(ISD::MULHS, VT, Expand);
1066         setOperationAction(ISD::MULHU, VT, Expand);
1067       }
1068       setOperationAction(ISD::SMUL_LOHI, VT, Expand);
1069       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
1070 
1071       setOperationAction(ISD::BSWAP, VT, Expand);
1072       setOperationAction(ISD::CTTZ, VT, Expand);
1073 
1074       for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
1075         setTruncStoreAction(VT, InnerVT, Expand);
1076         setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
1077         setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
1078         setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1079       }
1080     }
1081 
1082     // AArch64 has implementations of a lot of rounding-like FP operations.
1083     for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) {
1084       setOperationAction(ISD::FFLOOR, Ty, Legal);
1085       setOperationAction(ISD::FNEARBYINT, Ty, Legal);
1086       setOperationAction(ISD::FCEIL, Ty, Legal);
1087       setOperationAction(ISD::FRINT, Ty, Legal);
1088       setOperationAction(ISD::FTRUNC, Ty, Legal);
1089       setOperationAction(ISD::FROUND, Ty, Legal);
1090       setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
1091     }
1092 
1093     if (Subtarget->hasFullFP16()) {
1094       for (MVT Ty : {MVT::v4f16, MVT::v8f16}) {
1095         setOperationAction(ISD::FFLOOR, Ty, Legal);
1096         setOperationAction(ISD::FNEARBYINT, Ty, Legal);
1097         setOperationAction(ISD::FCEIL, Ty, Legal);
1098         setOperationAction(ISD::FRINT, Ty, Legal);
1099         setOperationAction(ISD::FTRUNC, Ty, Legal);
1100         setOperationAction(ISD::FROUND, Ty, Legal);
1101         setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
1102       }
1103     }
1104 
1105     if (Subtarget->hasSVE())
1106       setOperationAction(ISD::VSCALE, MVT::i32, Custom);
1107 
1108     setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
1109   }
1110 
1111   if (Subtarget->hasSVE()) {
1112     // FIXME: Add custom lowering of MLOAD to handle different passthrus (not a
1113     // splat of 0 or undef) once vector selects supported in SVE codegen. See
1114     // D68877 for more details.
1115     for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
1116       setOperationAction(ISD::BITREVERSE, VT, Custom);
1117       setOperationAction(ISD::BSWAP, VT, Custom);
1118       setOperationAction(ISD::CTLZ, VT, Custom);
1119       setOperationAction(ISD::CTPOP, VT, Custom);
1120       setOperationAction(ISD::CTTZ, VT, Custom);
1121       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1122       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
1123       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
1124       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1125       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1126       setOperationAction(ISD::MGATHER, VT, Custom);
1127       setOperationAction(ISD::MSCATTER, VT, Custom);
1128       setOperationAction(ISD::MUL, VT, Custom);
1129       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1130       setOperationAction(ISD::SELECT, VT, Custom);
1131       setOperationAction(ISD::SETCC, VT, Custom);
1132       setOperationAction(ISD::SDIV, VT, Custom);
1133       setOperationAction(ISD::UDIV, VT, Custom);
1134       setOperationAction(ISD::SMIN, VT, Custom);
1135       setOperationAction(ISD::UMIN, VT, Custom);
1136       setOperationAction(ISD::SMAX, VT, Custom);
1137       setOperationAction(ISD::UMAX, VT, Custom);
1138       setOperationAction(ISD::SHL, VT, Custom);
1139       setOperationAction(ISD::SRL, VT, Custom);
1140       setOperationAction(ISD::SRA, VT, Custom);
1141       setOperationAction(ISD::ABS, VT, Custom);
1142       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1143       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1144       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1145       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1146       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1147       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1148       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1149       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1150       setOperationAction(ISD::STEP_VECTOR, VT, Custom);
1151 
1152       setOperationAction(ISD::MULHU, VT, Expand);
1153       setOperationAction(ISD::MULHS, VT, Expand);
1154       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
1155       setOperationAction(ISD::SMUL_LOHI, VT, Expand);
1156     }
1157 
1158     // Illegal unpacked integer vector types.
1159     for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) {
1160       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1161       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1162     }
1163 
1164     for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}) {
1165       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1166       setOperationAction(ISD::SELECT, VT, Custom);
1167       setOperationAction(ISD::SETCC, VT, Custom);
1168       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1169       setOperationAction(ISD::TRUNCATE, VT, Custom);
1170       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1171       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1172       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1173 
1174       // There are no legal MVT::nxv16f## based types.
1175       if (VT != MVT::nxv16i1) {
1176         setOperationAction(ISD::SINT_TO_FP, VT, Custom);
1177         setOperationAction(ISD::UINT_TO_FP, VT, Custom);
1178       }
1179     }
1180 
1181     for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
1182                     MVT::nxv4f32, MVT::nxv2f64}) {
1183       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1184       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1185       setOperationAction(ISD::MGATHER, VT, Custom);
1186       setOperationAction(ISD::MSCATTER, VT, Custom);
1187       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1188       setOperationAction(ISD::SELECT, VT, Custom);
1189       setOperationAction(ISD::FADD, VT, Custom);
1190       setOperationAction(ISD::FDIV, VT, Custom);
1191       setOperationAction(ISD::FMA, VT, Custom);
1192       setOperationAction(ISD::FMAXIMUM, VT, Custom);
1193       setOperationAction(ISD::FMAXNUM, VT, Custom);
1194       setOperationAction(ISD::FMINIMUM, VT, Custom);
1195       setOperationAction(ISD::FMINNUM, VT, Custom);
1196       setOperationAction(ISD::FMUL, VT, Custom);
1197       setOperationAction(ISD::FNEG, VT, Custom);
1198       setOperationAction(ISD::FSUB, VT, Custom);
1199       setOperationAction(ISD::FCEIL, VT, Custom);
1200       setOperationAction(ISD::FFLOOR, VT, Custom);
1201       setOperationAction(ISD::FNEARBYINT, VT, Custom);
1202       setOperationAction(ISD::FRINT, VT, Custom);
1203       setOperationAction(ISD::FROUND, VT, Custom);
1204       setOperationAction(ISD::FROUNDEVEN, VT, Custom);
1205       setOperationAction(ISD::FTRUNC, VT, Custom);
1206       setOperationAction(ISD::FSQRT, VT, Custom);
1207       setOperationAction(ISD::FABS, VT, Custom);
1208       setOperationAction(ISD::FP_EXTEND, VT, Custom);
1209       setOperationAction(ISD::FP_ROUND, VT, Custom);
1210       setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1211       setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1212       setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1213       setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1214     }
1215 
1216     for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
1217       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1218       setOperationAction(ISD::MGATHER, VT, Custom);
1219       setOperationAction(ISD::MSCATTER, VT, Custom);
1220     }
1221 
1222     setOperationAction(ISD::SPLAT_VECTOR, MVT::nxv8bf16, Custom);
1223 
1224     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
1225     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
1226 
1227     // NOTE: Currently this has to happen after computeRegisterProperties rather
1228     // than the preferred option of combining it with the addRegisterClass call.
1229     if (Subtarget->useSVEForFixedLengthVectors()) {
1230       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
1231         if (useSVEForFixedLengthVectorVT(VT))
1232           addTypeForFixedLengthSVE(VT);
1233       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
1234         if (useSVEForFixedLengthVectorVT(VT))
1235           addTypeForFixedLengthSVE(VT);
1236 
1237       // 64bit results can mean a bigger than NEON input.
1238       for (auto VT : {MVT::v8i8, MVT::v4i16})
1239         setOperationAction(ISD::TRUNCATE, VT, Custom);
1240       setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
1241 
1242       // 128bit results imply a bigger than NEON input.
1243       for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
1244         setOperationAction(ISD::TRUNCATE, VT, Custom);
1245       for (auto VT : {MVT::v8f16, MVT::v4f32})
1246         setOperationAction(ISD::FP_ROUND, VT, Expand);
1247 
1248       // These operations are not supported on NEON but SVE can do them.
1249       setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
1250       setOperationAction(ISD::CTLZ, MVT::v1i64, Custom);
1251       setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
1252       setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
1253       setOperationAction(ISD::MUL, MVT::v1i64, Custom);
1254       setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1255       setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
1256       setOperationAction(ISD::SDIV, MVT::v16i8, Custom);
1257       setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
1258       setOperationAction(ISD::SDIV, MVT::v8i16, Custom);
1259       setOperationAction(ISD::SDIV, MVT::v2i32, Custom);
1260       setOperationAction(ISD::SDIV, MVT::v4i32, Custom);
1261       setOperationAction(ISD::SDIV, MVT::v1i64, Custom);
1262       setOperationAction(ISD::SDIV, MVT::v2i64, Custom);
1263       setOperationAction(ISD::SMAX, MVT::v1i64, Custom);
1264       setOperationAction(ISD::SMAX, MVT::v2i64, Custom);
1265       setOperationAction(ISD::SMIN, MVT::v1i64, Custom);
1266       setOperationAction(ISD::SMIN, MVT::v2i64, Custom);
1267       setOperationAction(ISD::UDIV, MVT::v8i8, Custom);
1268       setOperationAction(ISD::UDIV, MVT::v16i8, Custom);
1269       setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
1270       setOperationAction(ISD::UDIV, MVT::v8i16, Custom);
1271       setOperationAction(ISD::UDIV, MVT::v2i32, Custom);
1272       setOperationAction(ISD::UDIV, MVT::v4i32, Custom);
1273       setOperationAction(ISD::UDIV, MVT::v1i64, Custom);
1274       setOperationAction(ISD::UDIV, MVT::v2i64, Custom);
1275       setOperationAction(ISD::UMAX, MVT::v1i64, Custom);
1276       setOperationAction(ISD::UMAX, MVT::v2i64, Custom);
1277       setOperationAction(ISD::UMIN, MVT::v1i64, Custom);
1278       setOperationAction(ISD::UMIN, MVT::v2i64, Custom);
1279       setOperationAction(ISD::VECREDUCE_SMAX, MVT::v2i64, Custom);
1280       setOperationAction(ISD::VECREDUCE_SMIN, MVT::v2i64, Custom);
1281       setOperationAction(ISD::VECREDUCE_UMAX, MVT::v2i64, Custom);
1282       setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
1283 
1284       // Int operations with no NEON support.
1285       for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
1286                       MVT::v2i32, MVT::v4i32, MVT::v2i64}) {
1287         setOperationAction(ISD::BITREVERSE, VT, Custom);
1288         setOperationAction(ISD::CTTZ, VT, Custom);
1289         setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1290         setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1291         setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1292       }
1293 
1294       // FP operations with no NEON support.
1295       for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32,
1296                       MVT::v1f64, MVT::v2f64})
1297         setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1298 
1299       // Use SVE for vectors with more than 2 elements.
1300       for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
1301         setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1302     }
1303 
1304     setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv2i1, MVT::nxv2i64);
1305     setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv4i1, MVT::nxv4i32);
1306     setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv8i1, MVT::nxv8i16);
1307     setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv16i1, MVT::nxv16i8);
1308   }
1309 
1310   PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
1311 }
1312 
1313 void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
1314   assert(VT.isVector() && "VT should be a vector type");
1315 
1316   if (VT.isFloatingPoint()) {
1317     MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
1318     setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
1319     setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
1320   }
1321 
1322   // Mark vector float intrinsics as expand.
1323   if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
1324     setOperationAction(ISD::FSIN, VT, Expand);
1325     setOperationAction(ISD::FCOS, VT, Expand);
1326     setOperationAction(ISD::FPOW, VT, Expand);
1327     setOperationAction(ISD::FLOG, VT, Expand);
1328     setOperationAction(ISD::FLOG2, VT, Expand);
1329     setOperationAction(ISD::FLOG10, VT, Expand);
1330     setOperationAction(ISD::FEXP, VT, Expand);
1331     setOperationAction(ISD::FEXP2, VT, Expand);
1332 
1333     // But we do support custom-lowering for FCOPYSIGN.
1334     setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1335   }
1336 
1337   setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1338   setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1339   setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1340   setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1341   setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1342   setOperationAction(ISD::SRA, VT, Custom);
1343   setOperationAction(ISD::SRL, VT, Custom);
1344   setOperationAction(ISD::SHL, VT, Custom);
1345   setOperationAction(ISD::OR, VT, Custom);
1346   setOperationAction(ISD::SETCC, VT, Custom);
1347   setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
1348 
1349   setOperationAction(ISD::SELECT, VT, Expand);
1350   setOperationAction(ISD::SELECT_CC, VT, Expand);
1351   setOperationAction(ISD::VSELECT, VT, Expand);
1352   for (MVT InnerVT : MVT::all_valuetypes())
1353     setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
1354 
1355   // CNT supports only B element sizes, then use UADDLP to widen.
1356   if (VT != MVT::v8i8 && VT != MVT::v16i8)
1357     setOperationAction(ISD::CTPOP, VT, Custom);
1358 
1359   setOperationAction(ISD::UDIV, VT, Expand);
1360   setOperationAction(ISD::SDIV, VT, Expand);
1361   setOperationAction(ISD::UREM, VT, Expand);
1362   setOperationAction(ISD::SREM, VT, Expand);
1363   setOperationAction(ISD::FREM, VT, Expand);
1364 
1365   setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1366   setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1367 
1368   if (!VT.isFloatingPoint())
1369     setOperationAction(ISD::ABS, VT, Legal);
1370 
1371   // [SU][MIN|MAX] are available for all NEON types apart from i64.
1372   if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
1373     for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
1374       setOperationAction(Opcode, VT, Legal);
1375 
1376   // F[MIN|MAX][NUM|NAN] are available for all FP NEON types.
1377   if (VT.isFloatingPoint() &&
1378       VT.getVectorElementType() != MVT::bf16 &&
1379       (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()))
1380     for (unsigned Opcode :
1381          {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, ISD::FMAXNUM})
1382       setOperationAction(Opcode, VT, Legal);
1383 
1384   if (Subtarget->isLittleEndian()) {
1385     for (unsigned im = (unsigned)ISD::PRE_INC;
1386          im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
1387       setIndexedLoadAction(im, VT, Legal);
1388       setIndexedStoreAction(im, VT, Legal);
1389     }
1390   }
1391 }
1392 
1393 void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
1394   assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
1395 
1396   // By default everything must be expanded.
1397   for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1398     setOperationAction(Op, VT, Expand);
1399 
1400   // We use EXTRACT_SUBVECTOR to "cast" a scalable vector to a fixed length one.
1401   setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1402 
1403   if (VT.isFloatingPoint()) {
1404     setCondCodeAction(ISD::SETO, VT, Expand);
1405     setCondCodeAction(ISD::SETOLT, VT, Expand);
1406     setCondCodeAction(ISD::SETLT, VT, Expand);
1407     setCondCodeAction(ISD::SETOLE, VT, Expand);
1408     setCondCodeAction(ISD::SETLE, VT, Expand);
1409     setCondCodeAction(ISD::SETULT, VT, Expand);
1410     setCondCodeAction(ISD::SETULE, VT, Expand);
1411     setCondCodeAction(ISD::SETUGE, VT, Expand);
1412     setCondCodeAction(ISD::SETUGT, VT, Expand);
1413     setCondCodeAction(ISD::SETUEQ, VT, Expand);
1414     setCondCodeAction(ISD::SETUNE, VT, Expand);
1415   }
1416 
1417   // Lower fixed length vector operations to scalable equivalents.
1418   setOperationAction(ISD::ABS, VT, Custom);
1419   setOperationAction(ISD::ADD, VT, Custom);
1420   setOperationAction(ISD::AND, VT, Custom);
1421   setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1422   setOperationAction(ISD::BITREVERSE, VT, Custom);
1423   setOperationAction(ISD::BSWAP, VT, Custom);
1424   setOperationAction(ISD::CTLZ, VT, Custom);
1425   setOperationAction(ISD::CTPOP, VT, Custom);
1426   setOperationAction(ISD::CTTZ, VT, Custom);
1427   setOperationAction(ISD::FABS, VT, Custom);
1428   setOperationAction(ISD::FADD, VT, Custom);
1429   setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1430   setOperationAction(ISD::FCEIL, VT, Custom);
1431   setOperationAction(ISD::FDIV, VT, Custom);
1432   setOperationAction(ISD::FFLOOR, VT, Custom);
1433   setOperationAction(ISD::FMA, VT, Custom);
1434   setOperationAction(ISD::FMAXIMUM, VT, Custom);
1435   setOperationAction(ISD::FMAXNUM, VT, Custom);
1436   setOperationAction(ISD::FMINIMUM, VT, Custom);
1437   setOperationAction(ISD::FMINNUM, VT, Custom);
1438   setOperationAction(ISD::FMUL, VT, Custom);
1439   setOperationAction(ISD::FNEARBYINT, VT, Custom);
1440   setOperationAction(ISD::FNEG, VT, Custom);
1441   setOperationAction(ISD::FRINT, VT, Custom);
1442   setOperationAction(ISD::FROUND, VT, Custom);
1443   setOperationAction(ISD::FROUNDEVEN, VT, Custom);
1444   setOperationAction(ISD::FSQRT, VT, Custom);
1445   setOperationAction(ISD::FSUB, VT, Custom);
1446   setOperationAction(ISD::FTRUNC, VT, Custom);
1447   setOperationAction(ISD::LOAD, VT, Custom);
1448   setOperationAction(ISD::MUL, VT, Custom);
1449   setOperationAction(ISD::OR, VT, Custom);
1450   setOperationAction(ISD::SDIV, VT, Custom);
1451   setOperationAction(ISD::SETCC, VT, Custom);
1452   setOperationAction(ISD::SHL, VT, Custom);
1453   setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1454   setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom);
1455   setOperationAction(ISD::SMAX, VT, Custom);
1456   setOperationAction(ISD::SMIN, VT, Custom);
1457   setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1458   setOperationAction(ISD::SRA, VT, Custom);
1459   setOperationAction(ISD::SRL, VT, Custom);
1460   setOperationAction(ISD::STORE, VT, Custom);
1461   setOperationAction(ISD::SUB, VT, Custom);
1462   setOperationAction(ISD::TRUNCATE, VT, Custom);
1463   setOperationAction(ISD::UDIV, VT, Custom);
1464   setOperationAction(ISD::UMAX, VT, Custom);
1465   setOperationAction(ISD::UMIN, VT, Custom);
1466   setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1467   setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1468   setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1469   setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1470   setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1471   setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1472   setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1473   setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1474   setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1475   setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1476   setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1477   setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1478   setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1479   setOperationAction(ISD::VSELECT, VT, Custom);
1480   setOperationAction(ISD::XOR, VT, Custom);
1481   setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1482 }
1483 
1484 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
1485   addRegisterClass(VT, &AArch64::FPR64RegClass);
1486   addTypeForNEON(VT, MVT::v2i32);
1487 }
1488 
1489 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
1490   addRegisterClass(VT, &AArch64::FPR128RegClass);
1491   addTypeForNEON(VT, MVT::v4i32);
1492 }
1493 
1494 EVT AArch64TargetLowering::getSetCCResultType(const DataLayout &,
1495                                               LLVMContext &C, EVT VT) const {
1496   if (!VT.isVector())
1497     return MVT::i32;
1498   if (VT.isScalableVector())
1499     return EVT::getVectorVT(C, MVT::i1, VT.getVectorElementCount());
1500   return VT.changeVectorElementTypeToInteger();
1501 }
1502 
1503 static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
1504                                const APInt &Demanded,
1505                                TargetLowering::TargetLoweringOpt &TLO,
1506                                unsigned NewOpc) {
1507   uint64_t OldImm = Imm, NewImm, Enc;
1508   uint64_t Mask = ((uint64_t)(-1LL) >> (64 - Size)), OrigMask = Mask;
1509 
1510   // Return if the immediate is already all zeros, all ones, a bimm32 or a
1511   // bimm64.
1512   if (Imm == 0 || Imm == Mask ||
1513       AArch64_AM::isLogicalImmediate(Imm & Mask, Size))
1514     return false;
1515 
1516   unsigned EltSize = Size;
1517   uint64_t DemandedBits = Demanded.getZExtValue();
1518 
1519   // Clear bits that are not demanded.
1520   Imm &= DemandedBits;
1521 
1522   while (true) {
1523     // The goal here is to set the non-demanded bits in a way that minimizes
1524     // the number of switching between 0 and 1. In order to achieve this goal,
1525     // we set the non-demanded bits to the value of the preceding demanded bits.
1526     // For example, if we have an immediate 0bx10xx0x1 ('x' indicates a
1527     // non-demanded bit), we copy bit0 (1) to the least significant 'x',
1528     // bit2 (0) to 'xx', and bit6 (1) to the most significant 'x'.
1529     // The final result is 0b11000011.
1530     uint64_t NonDemandedBits = ~DemandedBits;
1531     uint64_t InvertedImm = ~Imm & DemandedBits;
1532     uint64_t RotatedImm =
1533         ((InvertedImm << 1) | (InvertedImm >> (EltSize - 1) & 1)) &
1534         NonDemandedBits;
1535     uint64_t Sum = RotatedImm + NonDemandedBits;
1536     bool Carry = NonDemandedBits & ~Sum & (1ULL << (EltSize - 1));
1537     uint64_t Ones = (Sum + Carry) & NonDemandedBits;
1538     NewImm = (Imm | Ones) & Mask;
1539 
1540     // If NewImm or its bitwise NOT is a shifted mask, it is a bitmask immediate
1541     // or all-ones or all-zeros, in which case we can stop searching. Otherwise,
1542     // we halve the element size and continue the search.
1543     if (isShiftedMask_64(NewImm) || isShiftedMask_64(~(NewImm | ~Mask)))
1544       break;
1545 
1546     // We cannot shrink the element size any further if it is 2-bits.
1547     if (EltSize == 2)
1548       return false;
1549 
1550     EltSize /= 2;
1551     Mask >>= EltSize;
1552     uint64_t Hi = Imm >> EltSize, DemandedBitsHi = DemandedBits >> EltSize;
1553 
1554     // Return if there is mismatch in any of the demanded bits of Imm and Hi.
1555     if (((Imm ^ Hi) & (DemandedBits & DemandedBitsHi) & Mask) != 0)
1556       return false;
1557 
1558     // Merge the upper and lower halves of Imm and DemandedBits.
1559     Imm |= Hi;
1560     DemandedBits |= DemandedBitsHi;
1561   }
1562 
1563   ++NumOptimizedImms;
1564 
1565   // Replicate the element across the register width.
1566   while (EltSize < Size) {
1567     NewImm |= NewImm << EltSize;
1568     EltSize *= 2;
1569   }
1570 
1571   (void)OldImm;
1572   assert(((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
1573          "demanded bits should never be altered");
1574   assert(OldImm != NewImm && "the new imm shouldn't be equal to the old imm");
1575 
1576   // Create the new constant immediate node.
1577   EVT VT = Op.getValueType();
1578   SDLoc DL(Op);
1579   SDValue New;
1580 
1581   // If the new constant immediate is all-zeros or all-ones, let the target
1582   // independent DAG combine optimize this node.
1583   if (NewImm == 0 || NewImm == OrigMask) {
1584     New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
1585                           TLO.DAG.getConstant(NewImm, DL, VT));
1586   // Otherwise, create a machine node so that target independent DAG combine
1587   // doesn't undo this optimization.
1588   } else {
1589     Enc = AArch64_AM::encodeLogicalImmediate(NewImm, Size);
1590     SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
1591     New = SDValue(
1592         TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
1593   }
1594 
1595   return TLO.CombineTo(Op, New);
1596 }
1597 
1598 bool AArch64TargetLowering::targetShrinkDemandedConstant(
1599     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
1600     TargetLoweringOpt &TLO) const {
1601   // Delay this optimization to as late as possible.
1602   if (!TLO.LegalOps)
1603     return false;
1604 
1605   if (!EnableOptimizeLogicalImm)
1606     return false;
1607 
1608   EVT VT = Op.getValueType();
1609   if (VT.isVector())
1610     return false;
1611 
1612   unsigned Size = VT.getSizeInBits();
1613   assert((Size == 32 || Size == 64) &&
1614          "i32 or i64 is expected after legalization.");
1615 
1616   // Exit early if we demand all bits.
1617   if (DemandedBits.countPopulation() == Size)
1618     return false;
1619 
1620   unsigned NewOpc;
1621   switch (Op.getOpcode()) {
1622   default:
1623     return false;
1624   case ISD::AND:
1625     NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
1626     break;
1627   case ISD::OR:
1628     NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
1629     break;
1630   case ISD::XOR:
1631     NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
1632     break;
1633   }
1634   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
1635   if (!C)
1636     return false;
1637   uint64_t Imm = C->getZExtValue();
1638   return optimizeLogicalImm(Op, Size, Imm, DemandedBits, TLO, NewOpc);
1639 }
1640 
1641 /// computeKnownBitsForTargetNode - Determine which of the bits specified in
1642 /// Mask are known to be either zero or one and return them Known.
1643 void AArch64TargetLowering::computeKnownBitsForTargetNode(
1644     const SDValue Op, KnownBits &Known,
1645     const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
1646   switch (Op.getOpcode()) {
1647   default:
1648     break;
1649   case AArch64ISD::CSEL: {
1650     KnownBits Known2;
1651     Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
1652     Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
1653     Known = KnownBits::commonBits(Known, Known2);
1654     break;
1655   }
1656   case AArch64ISD::LOADgot:
1657   case AArch64ISD::ADDlow: {
1658     if (!Subtarget->isTargetILP32())
1659       break;
1660     // In ILP32 mode all valid pointers are in the low 4GB of the address-space.
1661     Known.Zero = APInt::getHighBitsSet(64, 32);
1662     break;
1663   }
1664   case ISD::INTRINSIC_W_CHAIN: {
1665     ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
1666     Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
1667     switch (IntID) {
1668     default: return;
1669     case Intrinsic::aarch64_ldaxr:
1670     case Intrinsic::aarch64_ldxr: {
1671       unsigned BitWidth = Known.getBitWidth();
1672       EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
1673       unsigned MemBits = VT.getScalarSizeInBits();
1674       Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
1675       return;
1676     }
1677     }
1678     break;
1679   }
1680   case ISD::INTRINSIC_WO_CHAIN:
1681   case ISD::INTRINSIC_VOID: {
1682     unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1683     switch (IntNo) {
1684     default:
1685       break;
1686     case Intrinsic::aarch64_neon_umaxv:
1687     case Intrinsic::aarch64_neon_uminv: {
1688       // Figure out the datatype of the vector operand. The UMINV instruction
1689       // will zero extend the result, so we can mark as known zero all the
1690       // bits larger than the element datatype. 32-bit or larget doesn't need
1691       // this as those are legal types and will be handled by isel directly.
1692       MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
1693       unsigned BitWidth = Known.getBitWidth();
1694       if (VT == MVT::v8i8 || VT == MVT::v16i8) {
1695         assert(BitWidth >= 8 && "Unexpected width!");
1696         APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
1697         Known.Zero |= Mask;
1698       } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
1699         assert(BitWidth >= 16 && "Unexpected width!");
1700         APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
1701         Known.Zero |= Mask;
1702       }
1703       break;
1704     } break;
1705     }
1706   }
1707   }
1708 }
1709 
1710 MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
1711                                                   EVT) const {
1712   return MVT::i64;
1713 }
1714 
1715 bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
1716     EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1717     bool *Fast) const {
1718   if (Subtarget->requiresStrictAlign())
1719     return false;
1720 
1721   if (Fast) {
1722     // Some CPUs are fine with unaligned stores except for 128-bit ones.
1723     *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 ||
1724             // See comments in performSTORECombine() for more details about
1725             // these conditions.
1726 
1727             // Code that uses clang vector extensions can mark that it
1728             // wants unaligned accesses to be treated as fast by
1729             // underspecifying alignment to be 1 or 2.
1730             Alignment <= 2 ||
1731 
1732             // Disregard v2i64. Memcpy lowering produces those and splitting
1733             // them regresses performance on micro-benchmarks and olden/bh.
1734             VT == MVT::v2i64;
1735   }
1736   return true;
1737 }
1738 
1739 // Same as above but handling LLTs instead.
1740 bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
1741     LLT Ty, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1742     bool *Fast) const {
1743   if (Subtarget->requiresStrictAlign())
1744     return false;
1745 
1746   if (Fast) {
1747     // Some CPUs are fine with unaligned stores except for 128-bit ones.
1748     *Fast = !Subtarget->isMisaligned128StoreSlow() ||
1749             Ty.getSizeInBytes() != 16 ||
1750             // See comments in performSTORECombine() for more details about
1751             // these conditions.
1752 
1753             // Code that uses clang vector extensions can mark that it
1754             // wants unaligned accesses to be treated as fast by
1755             // underspecifying alignment to be 1 or 2.
1756             Alignment <= 2 ||
1757 
1758             // Disregard v2i64. Memcpy lowering produces those and splitting
1759             // them regresses performance on micro-benchmarks and olden/bh.
1760             Ty == LLT::vector(2, 64);
1761   }
1762   return true;
1763 }
1764 
1765 FastISel *
1766 AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
1767                                       const TargetLibraryInfo *libInfo) const {
1768   return AArch64::createFastISel(funcInfo, libInfo);
1769 }
1770 
1771 const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
1772 #define MAKE_CASE(V)                                                           \
1773   case V:                                                                      \
1774     return #V;
1775   switch ((AArch64ISD::NodeType)Opcode) {
1776   case AArch64ISD::FIRST_NUMBER:
1777     break;
1778     MAKE_CASE(AArch64ISD::CALL)
1779     MAKE_CASE(AArch64ISD::ADRP)
1780     MAKE_CASE(AArch64ISD::ADR)
1781     MAKE_CASE(AArch64ISD::ADDlow)
1782     MAKE_CASE(AArch64ISD::LOADgot)
1783     MAKE_CASE(AArch64ISD::RET_FLAG)
1784     MAKE_CASE(AArch64ISD::BRCOND)
1785     MAKE_CASE(AArch64ISD::CSEL)
1786     MAKE_CASE(AArch64ISD::FCSEL)
1787     MAKE_CASE(AArch64ISD::CSINV)
1788     MAKE_CASE(AArch64ISD::CSNEG)
1789     MAKE_CASE(AArch64ISD::CSINC)
1790     MAKE_CASE(AArch64ISD::THREAD_POINTER)
1791     MAKE_CASE(AArch64ISD::TLSDESC_CALLSEQ)
1792     MAKE_CASE(AArch64ISD::ADD_PRED)
1793     MAKE_CASE(AArch64ISD::MUL_PRED)
1794     MAKE_CASE(AArch64ISD::SDIV_PRED)
1795     MAKE_CASE(AArch64ISD::SHL_PRED)
1796     MAKE_CASE(AArch64ISD::SMAX_PRED)
1797     MAKE_CASE(AArch64ISD::SMIN_PRED)
1798     MAKE_CASE(AArch64ISD::SRA_PRED)
1799     MAKE_CASE(AArch64ISD::SRL_PRED)
1800     MAKE_CASE(AArch64ISD::SUB_PRED)
1801     MAKE_CASE(AArch64ISD::UDIV_PRED)
1802     MAKE_CASE(AArch64ISD::UMAX_PRED)
1803     MAKE_CASE(AArch64ISD::UMIN_PRED)
1804     MAKE_CASE(AArch64ISD::FNEG_MERGE_PASSTHRU)
1805     MAKE_CASE(AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU)
1806     MAKE_CASE(AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU)
1807     MAKE_CASE(AArch64ISD::FCEIL_MERGE_PASSTHRU)
1808     MAKE_CASE(AArch64ISD::FFLOOR_MERGE_PASSTHRU)
1809     MAKE_CASE(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU)
1810     MAKE_CASE(AArch64ISD::FRINT_MERGE_PASSTHRU)
1811     MAKE_CASE(AArch64ISD::FROUND_MERGE_PASSTHRU)
1812     MAKE_CASE(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU)
1813     MAKE_CASE(AArch64ISD::FTRUNC_MERGE_PASSTHRU)
1814     MAKE_CASE(AArch64ISD::FP_ROUND_MERGE_PASSTHRU)
1815     MAKE_CASE(AArch64ISD::FP_EXTEND_MERGE_PASSTHRU)
1816     MAKE_CASE(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU)
1817     MAKE_CASE(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU)
1818     MAKE_CASE(AArch64ISD::FCVTZU_MERGE_PASSTHRU)
1819     MAKE_CASE(AArch64ISD::FCVTZS_MERGE_PASSTHRU)
1820     MAKE_CASE(AArch64ISD::FSQRT_MERGE_PASSTHRU)
1821     MAKE_CASE(AArch64ISD::FRECPX_MERGE_PASSTHRU)
1822     MAKE_CASE(AArch64ISD::FABS_MERGE_PASSTHRU)
1823     MAKE_CASE(AArch64ISD::ABS_MERGE_PASSTHRU)
1824     MAKE_CASE(AArch64ISD::NEG_MERGE_PASSTHRU)
1825     MAKE_CASE(AArch64ISD::SETCC_MERGE_ZERO)
1826     MAKE_CASE(AArch64ISD::ADC)
1827     MAKE_CASE(AArch64ISD::SBC)
1828     MAKE_CASE(AArch64ISD::ADDS)
1829     MAKE_CASE(AArch64ISD::SUBS)
1830     MAKE_CASE(AArch64ISD::ADCS)
1831     MAKE_CASE(AArch64ISD::SBCS)
1832     MAKE_CASE(AArch64ISD::ANDS)
1833     MAKE_CASE(AArch64ISD::CCMP)
1834     MAKE_CASE(AArch64ISD::CCMN)
1835     MAKE_CASE(AArch64ISD::FCCMP)
1836     MAKE_CASE(AArch64ISD::FCMP)
1837     MAKE_CASE(AArch64ISD::STRICT_FCMP)
1838     MAKE_CASE(AArch64ISD::STRICT_FCMPE)
1839     MAKE_CASE(AArch64ISD::DUP)
1840     MAKE_CASE(AArch64ISD::DUPLANE8)
1841     MAKE_CASE(AArch64ISD::DUPLANE16)
1842     MAKE_CASE(AArch64ISD::DUPLANE32)
1843     MAKE_CASE(AArch64ISD::DUPLANE64)
1844     MAKE_CASE(AArch64ISD::MOVI)
1845     MAKE_CASE(AArch64ISD::MOVIshift)
1846     MAKE_CASE(AArch64ISD::MOVIedit)
1847     MAKE_CASE(AArch64ISD::MOVImsl)
1848     MAKE_CASE(AArch64ISD::FMOV)
1849     MAKE_CASE(AArch64ISD::MVNIshift)
1850     MAKE_CASE(AArch64ISD::MVNImsl)
1851     MAKE_CASE(AArch64ISD::BICi)
1852     MAKE_CASE(AArch64ISD::ORRi)
1853     MAKE_CASE(AArch64ISD::BSP)
1854     MAKE_CASE(AArch64ISD::NEG)
1855     MAKE_CASE(AArch64ISD::EXTR)
1856     MAKE_CASE(AArch64ISD::ZIP1)
1857     MAKE_CASE(AArch64ISD::ZIP2)
1858     MAKE_CASE(AArch64ISD::UZP1)
1859     MAKE_CASE(AArch64ISD::UZP2)
1860     MAKE_CASE(AArch64ISD::TRN1)
1861     MAKE_CASE(AArch64ISD::TRN2)
1862     MAKE_CASE(AArch64ISD::REV16)
1863     MAKE_CASE(AArch64ISD::REV32)
1864     MAKE_CASE(AArch64ISD::REV64)
1865     MAKE_CASE(AArch64ISD::EXT)
1866     MAKE_CASE(AArch64ISD::VSHL)
1867     MAKE_CASE(AArch64ISD::VLSHR)
1868     MAKE_CASE(AArch64ISD::VASHR)
1869     MAKE_CASE(AArch64ISD::VSLI)
1870     MAKE_CASE(AArch64ISD::VSRI)
1871     MAKE_CASE(AArch64ISD::CMEQ)
1872     MAKE_CASE(AArch64ISD::CMGE)
1873     MAKE_CASE(AArch64ISD::CMGT)
1874     MAKE_CASE(AArch64ISD::CMHI)
1875     MAKE_CASE(AArch64ISD::CMHS)
1876     MAKE_CASE(AArch64ISD::FCMEQ)
1877     MAKE_CASE(AArch64ISD::FCMGE)
1878     MAKE_CASE(AArch64ISD::FCMGT)
1879     MAKE_CASE(AArch64ISD::CMEQz)
1880     MAKE_CASE(AArch64ISD::CMGEz)
1881     MAKE_CASE(AArch64ISD::CMGTz)
1882     MAKE_CASE(AArch64ISD::CMLEz)
1883     MAKE_CASE(AArch64ISD::CMLTz)
1884     MAKE_CASE(AArch64ISD::FCMEQz)
1885     MAKE_CASE(AArch64ISD::FCMGEz)
1886     MAKE_CASE(AArch64ISD::FCMGTz)
1887     MAKE_CASE(AArch64ISD::FCMLEz)
1888     MAKE_CASE(AArch64ISD::FCMLTz)
1889     MAKE_CASE(AArch64ISD::SADDV)
1890     MAKE_CASE(AArch64ISD::UADDV)
1891     MAKE_CASE(AArch64ISD::SRHADD)
1892     MAKE_CASE(AArch64ISD::URHADD)
1893     MAKE_CASE(AArch64ISD::SHADD)
1894     MAKE_CASE(AArch64ISD::UHADD)
1895     MAKE_CASE(AArch64ISD::SDOT)
1896     MAKE_CASE(AArch64ISD::UDOT)
1897     MAKE_CASE(AArch64ISD::SMINV)
1898     MAKE_CASE(AArch64ISD::UMINV)
1899     MAKE_CASE(AArch64ISD::SMAXV)
1900     MAKE_CASE(AArch64ISD::UMAXV)
1901     MAKE_CASE(AArch64ISD::SADDV_PRED)
1902     MAKE_CASE(AArch64ISD::UADDV_PRED)
1903     MAKE_CASE(AArch64ISD::SMAXV_PRED)
1904     MAKE_CASE(AArch64ISD::UMAXV_PRED)
1905     MAKE_CASE(AArch64ISD::SMINV_PRED)
1906     MAKE_CASE(AArch64ISD::UMINV_PRED)
1907     MAKE_CASE(AArch64ISD::ORV_PRED)
1908     MAKE_CASE(AArch64ISD::EORV_PRED)
1909     MAKE_CASE(AArch64ISD::ANDV_PRED)
1910     MAKE_CASE(AArch64ISD::CLASTA_N)
1911     MAKE_CASE(AArch64ISD::CLASTB_N)
1912     MAKE_CASE(AArch64ISD::LASTA)
1913     MAKE_CASE(AArch64ISD::LASTB)
1914     MAKE_CASE(AArch64ISD::REINTERPRET_CAST)
1915     MAKE_CASE(AArch64ISD::TBL)
1916     MAKE_CASE(AArch64ISD::FADD_PRED)
1917     MAKE_CASE(AArch64ISD::FADDA_PRED)
1918     MAKE_CASE(AArch64ISD::FADDV_PRED)
1919     MAKE_CASE(AArch64ISD::FDIV_PRED)
1920     MAKE_CASE(AArch64ISD::FMA_PRED)
1921     MAKE_CASE(AArch64ISD::FMAX_PRED)
1922     MAKE_CASE(AArch64ISD::FMAXV_PRED)
1923     MAKE_CASE(AArch64ISD::FMAXNM_PRED)
1924     MAKE_CASE(AArch64ISD::FMAXNMV_PRED)
1925     MAKE_CASE(AArch64ISD::FMIN_PRED)
1926     MAKE_CASE(AArch64ISD::FMINV_PRED)
1927     MAKE_CASE(AArch64ISD::FMINNM_PRED)
1928     MAKE_CASE(AArch64ISD::FMINNMV_PRED)
1929     MAKE_CASE(AArch64ISD::FMUL_PRED)
1930     MAKE_CASE(AArch64ISD::FSUB_PRED)
1931     MAKE_CASE(AArch64ISD::BIT)
1932     MAKE_CASE(AArch64ISD::CBZ)
1933     MAKE_CASE(AArch64ISD::CBNZ)
1934     MAKE_CASE(AArch64ISD::TBZ)
1935     MAKE_CASE(AArch64ISD::TBNZ)
1936     MAKE_CASE(AArch64ISD::TC_RETURN)
1937     MAKE_CASE(AArch64ISD::PREFETCH)
1938     MAKE_CASE(AArch64ISD::SITOF)
1939     MAKE_CASE(AArch64ISD::UITOF)
1940     MAKE_CASE(AArch64ISD::NVCAST)
1941     MAKE_CASE(AArch64ISD::MRS)
1942     MAKE_CASE(AArch64ISD::SQSHL_I)
1943     MAKE_CASE(AArch64ISD::UQSHL_I)
1944     MAKE_CASE(AArch64ISD::SRSHR_I)
1945     MAKE_CASE(AArch64ISD::URSHR_I)
1946     MAKE_CASE(AArch64ISD::SQSHLU_I)
1947     MAKE_CASE(AArch64ISD::WrapperLarge)
1948     MAKE_CASE(AArch64ISD::LD2post)
1949     MAKE_CASE(AArch64ISD::LD3post)
1950     MAKE_CASE(AArch64ISD::LD4post)
1951     MAKE_CASE(AArch64ISD::ST2post)
1952     MAKE_CASE(AArch64ISD::ST3post)
1953     MAKE_CASE(AArch64ISD::ST4post)
1954     MAKE_CASE(AArch64ISD::LD1x2post)
1955     MAKE_CASE(AArch64ISD::LD1x3post)
1956     MAKE_CASE(AArch64ISD::LD1x4post)
1957     MAKE_CASE(AArch64ISD::ST1x2post)
1958     MAKE_CASE(AArch64ISD::ST1x3post)
1959     MAKE_CASE(AArch64ISD::ST1x4post)
1960     MAKE_CASE(AArch64ISD::LD1DUPpost)
1961     MAKE_CASE(AArch64ISD::LD2DUPpost)
1962     MAKE_CASE(AArch64ISD::LD3DUPpost)
1963     MAKE_CASE(AArch64ISD::LD4DUPpost)
1964     MAKE_CASE(AArch64ISD::LD1LANEpost)
1965     MAKE_CASE(AArch64ISD::LD2LANEpost)
1966     MAKE_CASE(AArch64ISD::LD3LANEpost)
1967     MAKE_CASE(AArch64ISD::LD4LANEpost)
1968     MAKE_CASE(AArch64ISD::ST2LANEpost)
1969     MAKE_CASE(AArch64ISD::ST3LANEpost)
1970     MAKE_CASE(AArch64ISD::ST4LANEpost)
1971     MAKE_CASE(AArch64ISD::SMULL)
1972     MAKE_CASE(AArch64ISD::UMULL)
1973     MAKE_CASE(AArch64ISD::FRECPE)
1974     MAKE_CASE(AArch64ISD::FRECPS)
1975     MAKE_CASE(AArch64ISD::FRSQRTE)
1976     MAKE_CASE(AArch64ISD::FRSQRTS)
1977     MAKE_CASE(AArch64ISD::STG)
1978     MAKE_CASE(AArch64ISD::STZG)
1979     MAKE_CASE(AArch64ISD::ST2G)
1980     MAKE_CASE(AArch64ISD::STZ2G)
1981     MAKE_CASE(AArch64ISD::SUNPKHI)
1982     MAKE_CASE(AArch64ISD::SUNPKLO)
1983     MAKE_CASE(AArch64ISD::UUNPKHI)
1984     MAKE_CASE(AArch64ISD::UUNPKLO)
1985     MAKE_CASE(AArch64ISD::INSR)
1986     MAKE_CASE(AArch64ISD::PTEST)
1987     MAKE_CASE(AArch64ISD::PTRUE)
1988     MAKE_CASE(AArch64ISD::LD1_MERGE_ZERO)
1989     MAKE_CASE(AArch64ISD::LD1S_MERGE_ZERO)
1990     MAKE_CASE(AArch64ISD::LDNF1_MERGE_ZERO)
1991     MAKE_CASE(AArch64ISD::LDNF1S_MERGE_ZERO)
1992     MAKE_CASE(AArch64ISD::LDFF1_MERGE_ZERO)
1993     MAKE_CASE(AArch64ISD::LDFF1S_MERGE_ZERO)
1994     MAKE_CASE(AArch64ISD::LD1RQ_MERGE_ZERO)
1995     MAKE_CASE(AArch64ISD::LD1RO_MERGE_ZERO)
1996     MAKE_CASE(AArch64ISD::SVE_LD2_MERGE_ZERO)
1997     MAKE_CASE(AArch64ISD::SVE_LD3_MERGE_ZERO)
1998     MAKE_CASE(AArch64ISD::SVE_LD4_MERGE_ZERO)
1999     MAKE_CASE(AArch64ISD::GLD1_MERGE_ZERO)
2000     MAKE_CASE(AArch64ISD::GLD1_SCALED_MERGE_ZERO)
2001     MAKE_CASE(AArch64ISD::GLD1_SXTW_MERGE_ZERO)
2002     MAKE_CASE(AArch64ISD::GLD1_UXTW_MERGE_ZERO)
2003     MAKE_CASE(AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO)
2004     MAKE_CASE(AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO)
2005     MAKE_CASE(AArch64ISD::GLD1_IMM_MERGE_ZERO)
2006     MAKE_CASE(AArch64ISD::GLD1S_MERGE_ZERO)
2007     MAKE_CASE(AArch64ISD::GLD1S_SCALED_MERGE_ZERO)
2008     MAKE_CASE(AArch64ISD::GLD1S_SXTW_MERGE_ZERO)
2009     MAKE_CASE(AArch64ISD::GLD1S_UXTW_MERGE_ZERO)
2010     MAKE_CASE(AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO)
2011     MAKE_CASE(AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO)
2012     MAKE_CASE(AArch64ISD::GLD1S_IMM_MERGE_ZERO)
2013     MAKE_CASE(AArch64ISD::GLDFF1_MERGE_ZERO)
2014     MAKE_CASE(AArch64ISD::GLDFF1_SCALED_MERGE_ZERO)
2015     MAKE_CASE(AArch64ISD::GLDFF1_SXTW_MERGE_ZERO)
2016     MAKE_CASE(AArch64ISD::GLDFF1_UXTW_MERGE_ZERO)
2017     MAKE_CASE(AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO)
2018     MAKE_CASE(AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO)
2019     MAKE_CASE(AArch64ISD::GLDFF1_IMM_MERGE_ZERO)
2020     MAKE_CASE(AArch64ISD::GLDFF1S_MERGE_ZERO)
2021     MAKE_CASE(AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO)
2022     MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO)
2023     MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO)
2024     MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO)
2025     MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO)
2026     MAKE_CASE(AArch64ISD::GLDFF1S_IMM_MERGE_ZERO)
2027     MAKE_CASE(AArch64ISD::GLDNT1_MERGE_ZERO)
2028     MAKE_CASE(AArch64ISD::GLDNT1_INDEX_MERGE_ZERO)
2029     MAKE_CASE(AArch64ISD::GLDNT1S_MERGE_ZERO)
2030     MAKE_CASE(AArch64ISD::ST1_PRED)
2031     MAKE_CASE(AArch64ISD::SST1_PRED)
2032     MAKE_CASE(AArch64ISD::SST1_SCALED_PRED)
2033     MAKE_CASE(AArch64ISD::SST1_SXTW_PRED)
2034     MAKE_CASE(AArch64ISD::SST1_UXTW_PRED)
2035     MAKE_CASE(AArch64ISD::SST1_SXTW_SCALED_PRED)
2036     MAKE_CASE(AArch64ISD::SST1_UXTW_SCALED_PRED)
2037     MAKE_CASE(AArch64ISD::SST1_IMM_PRED)
2038     MAKE_CASE(AArch64ISD::SSTNT1_PRED)
2039     MAKE_CASE(AArch64ISD::SSTNT1_INDEX_PRED)
2040     MAKE_CASE(AArch64ISD::LDP)
2041     MAKE_CASE(AArch64ISD::STP)
2042     MAKE_CASE(AArch64ISD::STNP)
2043     MAKE_CASE(AArch64ISD::BITREVERSE_MERGE_PASSTHRU)
2044     MAKE_CASE(AArch64ISD::BSWAP_MERGE_PASSTHRU)
2045     MAKE_CASE(AArch64ISD::CTLZ_MERGE_PASSTHRU)
2046     MAKE_CASE(AArch64ISD::CTPOP_MERGE_PASSTHRU)
2047     MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU)
2048     MAKE_CASE(AArch64ISD::INDEX_VECTOR)
2049     MAKE_CASE(AArch64ISD::UABD)
2050     MAKE_CASE(AArch64ISD::SABD)
2051     MAKE_CASE(AArch64ISD::CALL_RVMARKER)
2052   }
2053 #undef MAKE_CASE
2054   return nullptr;
2055 }
2056 
2057 MachineBasicBlock *
2058 AArch64TargetLowering::EmitF128CSEL(MachineInstr &MI,
2059                                     MachineBasicBlock *MBB) const {
2060   // We materialise the F128CSEL pseudo-instruction as some control flow and a
2061   // phi node:
2062 
2063   // OrigBB:
2064   //     [... previous instrs leading to comparison ...]
2065   //     b.ne TrueBB
2066   //     b EndBB
2067   // TrueBB:
2068   //     ; Fallthrough
2069   // EndBB:
2070   //     Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB]
2071 
2072   MachineFunction *MF = MBB->getParent();
2073   const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2074   const BasicBlock *LLVM_BB = MBB->getBasicBlock();
2075   DebugLoc DL = MI.getDebugLoc();
2076   MachineFunction::iterator It = ++MBB->getIterator();
2077 
2078   Register DestReg = MI.getOperand(0).getReg();
2079   Register IfTrueReg = MI.getOperand(1).getReg();
2080   Register IfFalseReg = MI.getOperand(2).getReg();
2081   unsigned CondCode = MI.getOperand(3).getImm();
2082   bool NZCVKilled = MI.getOperand(4).isKill();
2083 
2084   MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB);
2085   MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB);
2086   MF->insert(It, TrueBB);
2087   MF->insert(It, EndBB);
2088 
2089   // Transfer rest of current basic-block to EndBB
2090   EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)),
2091                 MBB->end());
2092   EndBB->transferSuccessorsAndUpdatePHIs(MBB);
2093 
2094   BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
2095   BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
2096   MBB->addSuccessor(TrueBB);
2097   MBB->addSuccessor(EndBB);
2098 
2099   // TrueBB falls through to the end.
2100   TrueBB->addSuccessor(EndBB);
2101 
2102   if (!NZCVKilled) {
2103     TrueBB->addLiveIn(AArch64::NZCV);
2104     EndBB->addLiveIn(AArch64::NZCV);
2105   }
2106 
2107   BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
2108       .addReg(IfTrueReg)
2109       .addMBB(TrueBB)
2110       .addReg(IfFalseReg)
2111       .addMBB(MBB);
2112 
2113   MI.eraseFromParent();
2114   return EndBB;
2115 }
2116 
2117 MachineBasicBlock *AArch64TargetLowering::EmitLoweredCatchRet(
2118        MachineInstr &MI, MachineBasicBlock *BB) const {
2119   assert(!isAsynchronousEHPersonality(classifyEHPersonality(
2120              BB->getParent()->getFunction().getPersonalityFn())) &&
2121          "SEH does not use catchret!");
2122   return BB;
2123 }
2124 
2125 MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter(
2126     MachineInstr &MI, MachineBasicBlock *BB) const {
2127   switch (MI.getOpcode()) {
2128   default:
2129 #ifndef NDEBUG
2130     MI.dump();
2131 #endif
2132     llvm_unreachable("Unexpected instruction for custom inserter!");
2133 
2134   case AArch64::F128CSEL:
2135     return EmitF128CSEL(MI, BB);
2136 
2137   case TargetOpcode::STACKMAP:
2138   case TargetOpcode::PATCHPOINT:
2139   case TargetOpcode::STATEPOINT:
2140     return emitPatchPoint(MI, BB);
2141 
2142   case AArch64::CATCHRET:
2143     return EmitLoweredCatchRet(MI, BB);
2144   }
2145 }
2146 
2147 //===----------------------------------------------------------------------===//
2148 // AArch64 Lowering private implementation.
2149 //===----------------------------------------------------------------------===//
2150 
2151 //===----------------------------------------------------------------------===//
2152 // Lowering Code
2153 //===----------------------------------------------------------------------===//
2154 
2155 /// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
2156 /// CC
2157 static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
2158   switch (CC) {
2159   default:
2160     llvm_unreachable("Unknown condition code!");
2161   case ISD::SETNE:
2162     return AArch64CC::NE;
2163   case ISD::SETEQ:
2164     return AArch64CC::EQ;
2165   case ISD::SETGT:
2166     return AArch64CC::GT;
2167   case ISD::SETGE:
2168     return AArch64CC::GE;
2169   case ISD::SETLT:
2170     return AArch64CC::LT;
2171   case ISD::SETLE:
2172     return AArch64CC::LE;
2173   case ISD::SETUGT:
2174     return AArch64CC::HI;
2175   case ISD::SETUGE:
2176     return AArch64CC::HS;
2177   case ISD::SETULT:
2178     return AArch64CC::LO;
2179   case ISD::SETULE:
2180     return AArch64CC::LS;
2181   }
2182 }
2183 
2184 /// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
2185 static void changeFPCCToAArch64CC(ISD::CondCode CC,
2186                                   AArch64CC::CondCode &CondCode,
2187                                   AArch64CC::CondCode &CondCode2) {
2188   CondCode2 = AArch64CC::AL;
2189   switch (CC) {
2190   default:
2191     llvm_unreachable("Unknown FP condition!");
2192   case ISD::SETEQ:
2193   case ISD::SETOEQ:
2194     CondCode = AArch64CC::EQ;
2195     break;
2196   case ISD::SETGT:
2197   case ISD::SETOGT:
2198     CondCode = AArch64CC::GT;
2199     break;
2200   case ISD::SETGE:
2201   case ISD::SETOGE:
2202     CondCode = AArch64CC::GE;
2203     break;
2204   case ISD::SETOLT:
2205     CondCode = AArch64CC::MI;
2206     break;
2207   case ISD::SETOLE:
2208     CondCode = AArch64CC::LS;
2209     break;
2210   case ISD::SETONE:
2211     CondCode = AArch64CC::MI;
2212     CondCode2 = AArch64CC::GT;
2213     break;
2214   case ISD::SETO:
2215     CondCode = AArch64CC::VC;
2216     break;
2217   case ISD::SETUO:
2218     CondCode = AArch64CC::VS;
2219     break;
2220   case ISD::SETUEQ:
2221     CondCode = AArch64CC::EQ;
2222     CondCode2 = AArch64CC::VS;
2223     break;
2224   case ISD::SETUGT:
2225     CondCode = AArch64CC::HI;
2226     break;
2227   case ISD::SETUGE:
2228     CondCode = AArch64CC::PL;
2229     break;
2230   case ISD::SETLT:
2231   case ISD::SETULT:
2232     CondCode = AArch64CC::LT;
2233     break;
2234   case ISD::SETLE:
2235   case ISD::SETULE:
2236     CondCode = AArch64CC::LE;
2237     break;
2238   case ISD::SETNE:
2239   case ISD::SETUNE:
2240     CondCode = AArch64CC::NE;
2241     break;
2242   }
2243 }
2244 
2245 /// Convert a DAG fp condition code to an AArch64 CC.
2246 /// This differs from changeFPCCToAArch64CC in that it returns cond codes that
2247 /// should be AND'ed instead of OR'ed.
2248 static void changeFPCCToANDAArch64CC(ISD::CondCode CC,
2249                                      AArch64CC::CondCode &CondCode,
2250                                      AArch64CC::CondCode &CondCode2) {
2251   CondCode2 = AArch64CC::AL;
2252   switch (CC) {
2253   default:
2254     changeFPCCToAArch64CC(CC, CondCode, CondCode2);
2255     assert(CondCode2 == AArch64CC::AL);
2256     break;
2257   case ISD::SETONE:
2258     // (a one b)
2259     // == ((a olt b) || (a ogt b))
2260     // == ((a ord b) && (a une b))
2261     CondCode = AArch64CC::VC;
2262     CondCode2 = AArch64CC::NE;
2263     break;
2264   case ISD::SETUEQ:
2265     // (a ueq b)
2266     // == ((a uno b) || (a oeq b))
2267     // == ((a ule b) && (a uge b))
2268     CondCode = AArch64CC::PL;
2269     CondCode2 = AArch64CC::LE;
2270     break;
2271   }
2272 }
2273 
2274 /// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
2275 /// CC usable with the vector instructions. Fewer operations are available
2276 /// without a real NZCV register, so we have to use less efficient combinations
2277 /// to get the same effect.
2278 static void changeVectorFPCCToAArch64CC(ISD::CondCode CC,
2279                                         AArch64CC::CondCode &CondCode,
2280                                         AArch64CC::CondCode &CondCode2,
2281                                         bool &Invert) {
2282   Invert = false;
2283   switch (CC) {
2284   default:
2285     // Mostly the scalar mappings work fine.
2286     changeFPCCToAArch64CC(CC, CondCode, CondCode2);
2287     break;
2288   case ISD::SETUO:
2289     Invert = true;
2290     LLVM_FALLTHROUGH;
2291   case ISD::SETO:
2292     CondCode = AArch64CC::MI;
2293     CondCode2 = AArch64CC::GE;
2294     break;
2295   case ISD::SETUEQ:
2296   case ISD::SETULT:
2297   case ISD::SETULE:
2298   case ISD::SETUGT:
2299   case ISD::SETUGE:
2300     // All of the compare-mask comparisons are ordered, but we can switch
2301     // between the two by a double inversion. E.g. ULE == !OGT.
2302     Invert = true;
2303     changeFPCCToAArch64CC(getSetCCInverse(CC, /* FP inverse */ MVT::f32),
2304                           CondCode, CondCode2);
2305     break;
2306   }
2307 }
2308 
2309 static bool isLegalArithImmed(uint64_t C) {
2310   // Matches AArch64DAGToDAGISel::SelectArithImmed().
2311   bool IsLegal = (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0);
2312   LLVM_DEBUG(dbgs() << "Is imm " << C
2313                     << " legal: " << (IsLegal ? "yes\n" : "no\n"));
2314   return IsLegal;
2315 }
2316 
2317 // Can a (CMP op1, (sub 0, op2) be turned into a CMN instruction on
2318 // the grounds that "op1 - (-op2) == op1 + op2" ? Not always, the C and V flags
2319 // can be set differently by this operation. It comes down to whether
2320 // "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then
2321 // everything is fine. If not then the optimization is wrong. Thus general
2322 // comparisons are only valid if op2 != 0.
2323 //
2324 // So, finally, the only LLVM-native comparisons that don't mention C and V
2325 // are SETEQ and SETNE. They're the only ones we can safely use CMN for in
2326 // the absence of information about op2.
2327 static bool isCMN(SDValue Op, ISD::CondCode CC) {
2328   return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
2329          (CC == ISD::SETEQ || CC == ISD::SETNE);
2330 }
2331 
2332 static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
2333                                       SelectionDAG &DAG, SDValue Chain,
2334                                       bool IsSignaling) {
2335   EVT VT = LHS.getValueType();
2336   assert(VT != MVT::f128);
2337   assert(VT != MVT::f16 && "Lowering of strict fp16 not yet implemented");
2338   unsigned Opcode =
2339       IsSignaling ? AArch64ISD::STRICT_FCMPE : AArch64ISD::STRICT_FCMP;
2340   return DAG.getNode(Opcode, dl, {VT, MVT::Other}, {Chain, LHS, RHS});
2341 }
2342 
2343 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2344                               const SDLoc &dl, SelectionDAG &DAG) {
2345   EVT VT = LHS.getValueType();
2346   const bool FullFP16 =
2347     static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
2348 
2349   if (VT.isFloatingPoint()) {
2350     assert(VT != MVT::f128);
2351     if (VT == MVT::f16 && !FullFP16) {
2352       LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
2353       RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
2354       VT = MVT::f32;
2355     }
2356     return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS);
2357   }
2358 
2359   // The CMP instruction is just an alias for SUBS, and representing it as
2360   // SUBS means that it's possible to get CSE with subtract operations.
2361   // A later phase can perform the optimization of setting the destination
2362   // register to WZR/XZR if it ends up being unused.
2363   unsigned Opcode = AArch64ISD::SUBS;
2364 
2365   if (isCMN(RHS, CC)) {
2366     // Can we combine a (CMP op1, (sub 0, op2) into a CMN instruction ?
2367     Opcode = AArch64ISD::ADDS;
2368     RHS = RHS.getOperand(1);
2369   } else if (isCMN(LHS, CC)) {
2370     // As we are looking for EQ/NE compares, the operands can be commuted ; can
2371     // we combine a (CMP (sub 0, op1), op2) into a CMN instruction ?
2372     Opcode = AArch64ISD::ADDS;
2373     LHS = LHS.getOperand(1);
2374   } else if (isNullConstant(RHS) && !isUnsignedIntSetCC(CC)) {
2375     if (LHS.getOpcode() == ISD::AND) {
2376       // Similarly, (CMP (and X, Y), 0) can be implemented with a TST
2377       // (a.k.a. ANDS) except that the flags are only guaranteed to work for one
2378       // of the signed comparisons.
2379       const SDValue ANDSNode = DAG.getNode(AArch64ISD::ANDS, dl,
2380                                            DAG.getVTList(VT, MVT_CC),
2381                                            LHS.getOperand(0),
2382                                            LHS.getOperand(1));
2383       // Replace all users of (and X, Y) with newly generated (ands X, Y)
2384       DAG.ReplaceAllUsesWith(LHS, ANDSNode);
2385       return ANDSNode.getValue(1);
2386     } else if (LHS.getOpcode() == AArch64ISD::ANDS) {
2387       // Use result of ANDS
2388       return LHS.getValue(1);
2389     }
2390   }
2391 
2392   return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS)
2393       .getValue(1);
2394 }
2395 
2396 /// \defgroup AArch64CCMP CMP;CCMP matching
2397 ///
2398 /// These functions deal with the formation of CMP;CCMP;... sequences.
2399 /// The CCMP/CCMN/FCCMP/FCCMPE instructions allow the conditional execution of
2400 /// a comparison. They set the NZCV flags to a predefined value if their
2401 /// predicate is false. This allows to express arbitrary conjunctions, for
2402 /// example "cmp 0 (and (setCA (cmp A)) (setCB (cmp B)))"
2403 /// expressed as:
2404 ///   cmp A
2405 ///   ccmp B, inv(CB), CA
2406 ///   check for CB flags
2407 ///
2408 /// This naturally lets us implement chains of AND operations with SETCC
2409 /// operands. And we can even implement some other situations by transforming
2410 /// them:
2411 ///   - We can implement (NEG SETCC) i.e. negating a single comparison by
2412 ///     negating the flags used in a CCMP/FCCMP operations.
2413 ///   - We can negate the result of a whole chain of CMP/CCMP/FCCMP operations
2414 ///     by negating the flags we test for afterwards. i.e.
2415 ///     NEG (CMP CCMP CCCMP ...) can be implemented.
2416 ///   - Note that we can only ever negate all previously processed results.
2417 ///     What we can not implement by flipping the flags to test is a negation
2418 ///     of two sub-trees (because the negation affects all sub-trees emitted so
2419 ///     far, so the 2nd sub-tree we emit would also affect the first).
2420 /// With those tools we can implement some OR operations:
2421 ///   - (OR (SETCC A) (SETCC B)) can be implemented via:
2422 ///     NEG (AND (NEG (SETCC A)) (NEG (SETCC B)))
2423 ///   - After transforming OR to NEG/AND combinations we may be able to use NEG
2424 ///     elimination rules from earlier to implement the whole thing as a
2425 ///     CCMP/FCCMP chain.
2426 ///
2427 /// As complete example:
2428 ///     or (or (setCA (cmp A)) (setCB (cmp B)))
2429 ///        (and (setCC (cmp C)) (setCD (cmp D)))"
2430 /// can be reassociated to:
2431 ///     or (and (setCC (cmp C)) setCD (cmp D))
2432 //         (or (setCA (cmp A)) (setCB (cmp B)))
2433 /// can be transformed to:
2434 ///     not (and (not (and (setCC (cmp C)) (setCD (cmp D))))
2435 ///              (and (not (setCA (cmp A)) (not (setCB (cmp B))))))"
2436 /// which can be implemented as:
2437 ///   cmp C
2438 ///   ccmp D, inv(CD), CC
2439 ///   ccmp A, CA, inv(CD)
2440 ///   ccmp B, CB, inv(CA)
2441 ///   check for CB flags
2442 ///
2443 /// A counterexample is "or (and A B) (and C D)" which translates to
2444 /// not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we
2445 /// can only implement 1 of the inner (not) operations, but not both!
2446 /// @{
2447 
2448 /// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
2449 static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
2450                                          ISD::CondCode CC, SDValue CCOp,
2451                                          AArch64CC::CondCode Predicate,
2452                                          AArch64CC::CondCode OutCC,
2453                                          const SDLoc &DL, SelectionDAG &DAG) {
2454   unsigned Opcode = 0;
2455   const bool FullFP16 =
2456     static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
2457 
2458   if (LHS.getValueType().isFloatingPoint()) {
2459     assert(LHS.getValueType() != MVT::f128);
2460     if (LHS.getValueType() == MVT::f16 && !FullFP16) {
2461       LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
2462       RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
2463     }
2464     Opcode = AArch64ISD::FCCMP;
2465   } else if (RHS.getOpcode() == ISD::SUB) {
2466     SDValue SubOp0 = RHS.getOperand(0);
2467     if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
2468       // See emitComparison() on why we can only do this for SETEQ and SETNE.
2469       Opcode = AArch64ISD::CCMN;
2470       RHS = RHS.getOperand(1);
2471     }
2472   }
2473   if (Opcode == 0)
2474     Opcode = AArch64ISD::CCMP;
2475 
2476   SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
2477   AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC);
2478   unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
2479   SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
2480   return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp);
2481 }
2482 
2483 /// Returns true if @p Val is a tree of AND/OR/SETCC operations that can be
2484 /// expressed as a conjunction. See \ref AArch64CCMP.
2485 /// \param CanNegate    Set to true if we can negate the whole sub-tree just by
2486 ///                     changing the conditions on the SETCC tests.
2487 ///                     (this means we can call emitConjunctionRec() with
2488 ///                      Negate==true on this sub-tree)
2489 /// \param MustBeFirst  Set to true if this subtree needs to be negated and we
2490 ///                     cannot do the negation naturally. We are required to
2491 ///                     emit the subtree first in this case.
2492 /// \param WillNegate   Is true if are called when the result of this
2493 ///                     subexpression must be negated. This happens when the
2494 ///                     outer expression is an OR. We can use this fact to know
2495 ///                     that we have a double negation (or (or ...) ...) that
2496 ///                     can be implemented for free.
2497 static bool canEmitConjunction(const SDValue Val, bool &CanNegate,
2498                                bool &MustBeFirst, bool WillNegate,
2499                                unsigned Depth = 0) {
2500   if (!Val.hasOneUse())
2501     return false;
2502   unsigned Opcode = Val->getOpcode();
2503   if (Opcode == ISD::SETCC) {
2504     if (Val->getOperand(0).getValueType() == MVT::f128)
2505       return false;
2506     CanNegate = true;
2507     MustBeFirst = false;
2508     return true;
2509   }
2510   // Protect against exponential runtime and stack overflow.
2511   if (Depth > 6)
2512     return false;
2513   if (Opcode == ISD::AND || Opcode == ISD::OR) {
2514     bool IsOR = Opcode == ISD::OR;
2515     SDValue O0 = Val->getOperand(0);
2516     SDValue O1 = Val->getOperand(1);
2517     bool CanNegateL;
2518     bool MustBeFirstL;
2519     if (!canEmitConjunction(O0, CanNegateL, MustBeFirstL, IsOR, Depth+1))
2520       return false;
2521     bool CanNegateR;
2522     bool MustBeFirstR;
2523     if (!canEmitConjunction(O1, CanNegateR, MustBeFirstR, IsOR, Depth+1))
2524       return false;
2525 
2526     if (MustBeFirstL && MustBeFirstR)
2527       return false;
2528 
2529     if (IsOR) {
2530       // For an OR expression we need to be able to naturally negate at least
2531       // one side or we cannot do the transformation at all.
2532       if (!CanNegateL && !CanNegateR)
2533         return false;
2534       // If we the result of the OR will be negated and we can naturally negate
2535       // the leafs, then this sub-tree as a whole negates naturally.
2536       CanNegate = WillNegate && CanNegateL && CanNegateR;
2537       // If we cannot naturally negate the whole sub-tree, then this must be
2538       // emitted first.
2539       MustBeFirst = !CanNegate;
2540     } else {
2541       assert(Opcode == ISD::AND && "Must be OR or AND");
2542       // We cannot naturally negate an AND operation.
2543       CanNegate = false;
2544       MustBeFirst = MustBeFirstL || MustBeFirstR;
2545     }
2546     return true;
2547   }
2548   return false;
2549 }
2550 
2551 /// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain
2552 /// of CCMP/CFCMP ops. See @ref AArch64CCMP.
2553 /// Tries to transform the given i1 producing node @p Val to a series compare
2554 /// and conditional compare operations. @returns an NZCV flags producing node
2555 /// and sets @p OutCC to the flags that should be tested or returns SDValue() if
2556 /// transformation was not possible.
2557 /// \p Negate is true if we want this sub-tree being negated just by changing
2558 /// SETCC conditions.
2559 static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
2560     AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
2561     AArch64CC::CondCode Predicate) {
2562   // We're at a tree leaf, produce a conditional comparison operation.
2563   unsigned Opcode = Val->getOpcode();
2564   if (Opcode == ISD::SETCC) {
2565     SDValue LHS = Val->getOperand(0);
2566     SDValue RHS = Val->getOperand(1);
2567     ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get();
2568     bool isInteger = LHS.getValueType().isInteger();
2569     if (Negate)
2570       CC = getSetCCInverse(CC, LHS.getValueType());
2571     SDLoc DL(Val);
2572     // Determine OutCC and handle FP special case.
2573     if (isInteger) {
2574       OutCC = changeIntCCToAArch64CC(CC);
2575     } else {
2576       assert(LHS.getValueType().isFloatingPoint());
2577       AArch64CC::CondCode ExtraCC;
2578       changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC);
2579       // Some floating point conditions can't be tested with a single condition
2580       // code. Construct an additional comparison in this case.
2581       if (ExtraCC != AArch64CC::AL) {
2582         SDValue ExtraCmp;
2583         if (!CCOp.getNode())
2584           ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG);
2585         else
2586           ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate,
2587                                                ExtraCC, DL, DAG);
2588         CCOp = ExtraCmp;
2589         Predicate = ExtraCC;
2590       }
2591     }
2592 
2593     // Produce a normal comparison if we are first in the chain
2594     if (!CCOp)
2595       return emitComparison(LHS, RHS, CC, DL, DAG);
2596     // Otherwise produce a ccmp.
2597     return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL,
2598                                      DAG);
2599   }
2600   assert(Val->hasOneUse() && "Valid conjunction/disjunction tree");
2601 
2602   bool IsOR = Opcode == ISD::OR;
2603 
2604   SDValue LHS = Val->getOperand(0);
2605   bool CanNegateL;
2606   bool MustBeFirstL;
2607   bool ValidL = canEmitConjunction(LHS, CanNegateL, MustBeFirstL, IsOR);
2608   assert(ValidL && "Valid conjunction/disjunction tree");
2609   (void)ValidL;
2610 
2611   SDValue RHS = Val->getOperand(1);
2612   bool CanNegateR;
2613   bool MustBeFirstR;
2614   bool ValidR = canEmitConjunction(RHS, CanNegateR, MustBeFirstR, IsOR);
2615   assert(ValidR && "Valid conjunction/disjunction tree");
2616   (void)ValidR;
2617 
2618   // Swap sub-tree that must come first to the right side.
2619   if (MustBeFirstL) {
2620     assert(!MustBeFirstR && "Valid conjunction/disjunction tree");
2621     std::swap(LHS, RHS);
2622     std::swap(CanNegateL, CanNegateR);
2623     std::swap(MustBeFirstL, MustBeFirstR);
2624   }
2625 
2626   bool NegateR;
2627   bool NegateAfterR;
2628   bool NegateL;
2629   bool NegateAfterAll;
2630   if (Opcode == ISD::OR) {
2631     // Swap the sub-tree that we can negate naturally to the left.
2632     if (!CanNegateL) {
2633       assert(CanNegateR && "at least one side must be negatable");
2634       assert(!MustBeFirstR && "invalid conjunction/disjunction tree");
2635       assert(!Negate);
2636       std::swap(LHS, RHS);
2637       NegateR = false;
2638       NegateAfterR = true;
2639     } else {
2640       // Negate the left sub-tree if possible, otherwise negate the result.
2641       NegateR = CanNegateR;
2642       NegateAfterR = !CanNegateR;
2643     }
2644     NegateL = true;
2645     NegateAfterAll = !Negate;
2646   } else {
2647     assert(Opcode == ISD::AND && "Valid conjunction/disjunction tree");
2648     assert(!Negate && "Valid conjunction/disjunction tree");
2649 
2650     NegateL = false;
2651     NegateR = false;
2652     NegateAfterR = false;
2653     NegateAfterAll = false;
2654   }
2655 
2656   // Emit sub-trees.
2657   AArch64CC::CondCode RHSCC;
2658   SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
2659   if (NegateAfterR)
2660     RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
2661   SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
2662   if (NegateAfterAll)
2663     OutCC = AArch64CC::getInvertedCondCode(OutCC);
2664   return CmpL;
2665 }
2666 
2667 /// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
2668 /// In some cases this is even possible with OR operations in the expression.
2669 /// See \ref AArch64CCMP.
2670 /// \see emitConjunctionRec().
2671 static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
2672                                AArch64CC::CondCode &OutCC) {
2673   bool DummyCanNegate;
2674   bool DummyMustBeFirst;
2675   if (!canEmitConjunction(Val, DummyCanNegate, DummyMustBeFirst, false))
2676     return SDValue();
2677 
2678   return emitConjunctionRec(DAG, Val, OutCC, false, SDValue(), AArch64CC::AL);
2679 }
2680 
2681 /// @}
2682 
2683 /// Returns how profitable it is to fold a comparison's operand's shift and/or
2684 /// extension operations.
2685 static unsigned getCmpOperandFoldingProfit(SDValue Op) {
2686   auto isSupportedExtend = [&](SDValue V) {
2687     if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
2688       return true;
2689 
2690     if (V.getOpcode() == ISD::AND)
2691       if (ConstantSDNode *MaskCst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2692         uint64_t Mask = MaskCst->getZExtValue();
2693         return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
2694       }
2695 
2696     return false;
2697   };
2698 
2699   if (!Op.hasOneUse())
2700     return 0;
2701 
2702   if (isSupportedExtend(Op))
2703     return 1;
2704 
2705   unsigned Opc = Op.getOpcode();
2706   if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
2707     if (ConstantSDNode *ShiftCst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2708       uint64_t Shift = ShiftCst->getZExtValue();
2709       if (isSupportedExtend(Op.getOperand(0)))
2710         return (Shift <= 4) ? 2 : 1;
2711       EVT VT = Op.getValueType();
2712       if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63))
2713         return 1;
2714     }
2715 
2716   return 0;
2717 }
2718 
2719 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2720                              SDValue &AArch64cc, SelectionDAG &DAG,
2721                              const SDLoc &dl) {
2722   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
2723     EVT VT = RHS.getValueType();
2724     uint64_t C = RHSC->getZExtValue();
2725     if (!isLegalArithImmed(C)) {
2726       // Constant does not fit, try adjusting it by one?
2727       switch (CC) {
2728       default:
2729         break;
2730       case ISD::SETLT:
2731       case ISD::SETGE:
2732         if ((VT == MVT::i32 && C != 0x80000000 &&
2733              isLegalArithImmed((uint32_t)(C - 1))) ||
2734             (VT == MVT::i64 && C != 0x80000000ULL &&
2735              isLegalArithImmed(C - 1ULL))) {
2736           CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
2737           C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
2738           RHS = DAG.getConstant(C, dl, VT);
2739         }
2740         break;
2741       case ISD::SETULT:
2742       case ISD::SETUGE:
2743         if ((VT == MVT::i32 && C != 0 &&
2744              isLegalArithImmed((uint32_t)(C - 1))) ||
2745             (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
2746           CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
2747           C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
2748           RHS = DAG.getConstant(C, dl, VT);
2749         }
2750         break;
2751       case ISD::SETLE:
2752       case ISD::SETGT:
2753         if ((VT == MVT::i32 && C != INT32_MAX &&
2754              isLegalArithImmed((uint32_t)(C + 1))) ||
2755             (VT == MVT::i64 && C != INT64_MAX &&
2756              isLegalArithImmed(C + 1ULL))) {
2757           CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
2758           C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
2759           RHS = DAG.getConstant(C, dl, VT);
2760         }
2761         break;
2762       case ISD::SETULE:
2763       case ISD::SETUGT:
2764         if ((VT == MVT::i32 && C != UINT32_MAX &&
2765              isLegalArithImmed((uint32_t)(C + 1))) ||
2766             (VT == MVT::i64 && C != UINT64_MAX &&
2767              isLegalArithImmed(C + 1ULL))) {
2768           CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
2769           C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
2770           RHS = DAG.getConstant(C, dl, VT);
2771         }
2772         break;
2773       }
2774     }
2775   }
2776 
2777   // Comparisons are canonicalized so that the RHS operand is simpler than the
2778   // LHS one, the extreme case being when RHS is an immediate. However, AArch64
2779   // can fold some shift+extend operations on the RHS operand, so swap the
2780   // operands if that can be done.
2781   //
2782   // For example:
2783   //    lsl     w13, w11, #1
2784   //    cmp     w13, w12
2785   // can be turned into:
2786   //    cmp     w12, w11, lsl #1
2787   if (!isa<ConstantSDNode>(RHS) ||
2788       !isLegalArithImmed(cast<ConstantSDNode>(RHS)->getZExtValue())) {
2789     SDValue TheLHS = isCMN(LHS, CC) ? LHS.getOperand(1) : LHS;
2790 
2791     if (getCmpOperandFoldingProfit(TheLHS) > getCmpOperandFoldingProfit(RHS)) {
2792       std::swap(LHS, RHS);
2793       CC = ISD::getSetCCSwappedOperands(CC);
2794     }
2795   }
2796 
2797   SDValue Cmp;
2798   AArch64CC::CondCode AArch64CC;
2799   if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) {
2800     const ConstantSDNode *RHSC = cast<ConstantSDNode>(RHS);
2801 
2802     // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095.
2803     // For the i8 operand, the largest immediate is 255, so this can be easily
2804     // encoded in the compare instruction. For the i16 operand, however, the
2805     // largest immediate cannot be encoded in the compare.
2806     // Therefore, use a sign extending load and cmn to avoid materializing the
2807     // -1 constant. For example,
2808     // movz w1, #65535
2809     // ldrh w0, [x0, #0]
2810     // cmp w0, w1
2811     // >
2812     // ldrsh w0, [x0, #0]
2813     // cmn w0, #1
2814     // Fundamental, we're relying on the property that (zext LHS) == (zext RHS)
2815     // if and only if (sext LHS) == (sext RHS). The checks are in place to
2816     // ensure both the LHS and RHS are truly zero extended and to make sure the
2817     // transformation is profitable.
2818     if ((RHSC->getZExtValue() >> 16 == 0) && isa<LoadSDNode>(LHS) &&
2819         cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD &&
2820         cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 &&
2821         LHS.getNode()->hasNUsesOfValue(1, 0)) {
2822       int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue();
2823       if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) {
2824         SDValue SExt =
2825             DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
2826                         DAG.getValueType(MVT::i16));
2827         Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl,
2828                                                    RHS.getValueType()),
2829                              CC, dl, DAG);
2830         AArch64CC = changeIntCCToAArch64CC(CC);
2831       }
2832     }
2833 
2834     if (!Cmp && (RHSC->isNullValue() || RHSC->isOne())) {
2835       if ((Cmp = emitConjunction(DAG, LHS, AArch64CC))) {
2836         if ((CC == ISD::SETNE) ^ RHSC->isNullValue())
2837           AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC);
2838       }
2839     }
2840   }
2841 
2842   if (!Cmp) {
2843     Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
2844     AArch64CC = changeIntCCToAArch64CC(CC);
2845   }
2846   AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC);
2847   return Cmp;
2848 }
2849 
2850 static std::pair<SDValue, SDValue>
2851 getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
2852   assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) &&
2853          "Unsupported value type");
2854   SDValue Value, Overflow;
2855   SDLoc DL(Op);
2856   SDValue LHS = Op.getOperand(0);
2857   SDValue RHS = Op.getOperand(1);
2858   unsigned Opc = 0;
2859   switch (Op.getOpcode()) {
2860   default:
2861     llvm_unreachable("Unknown overflow instruction!");
2862   case ISD::SADDO:
2863     Opc = AArch64ISD::ADDS;
2864     CC = AArch64CC::VS;
2865     break;
2866   case ISD::UADDO:
2867     Opc = AArch64ISD::ADDS;
2868     CC = AArch64CC::HS;
2869     break;
2870   case ISD::SSUBO:
2871     Opc = AArch64ISD::SUBS;
2872     CC = AArch64CC::VS;
2873     break;
2874   case ISD::USUBO:
2875     Opc = AArch64ISD::SUBS;
2876     CC = AArch64CC::LO;
2877     break;
2878   // Multiply needs a little bit extra work.
2879   case ISD::SMULO:
2880   case ISD::UMULO: {
2881     CC = AArch64CC::NE;
2882     bool IsSigned = Op.getOpcode() == ISD::SMULO;
2883     if (Op.getValueType() == MVT::i32) {
2884       unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
2885       // For a 32 bit multiply with overflow check we want the instruction
2886       // selector to generate a widening multiply (SMADDL/UMADDL). For that we
2887       // need to generate the following pattern:
2888       // (i64 add 0, (i64 mul (i64 sext|zext i32 %a), (i64 sext|zext i32 %b))
2889       LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS);
2890       RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS);
2891       SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2892       SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
2893                                 DAG.getConstant(0, DL, MVT::i64));
2894       // On AArch64 the upper 32 bits are always zero extended for a 32 bit
2895       // operation. We need to clear out the upper 32 bits, because we used a
2896       // widening multiply that wrote all 64 bits. In the end this should be a
2897       // noop.
2898       Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add);
2899       if (IsSigned) {
2900         // The signed overflow check requires more than just a simple check for
2901         // any bit set in the upper 32 bits of the result. These bits could be
2902         // just the sign bits of a negative number. To perform the overflow
2903         // check we have to arithmetic shift right the 32nd bit of the result by
2904         // 31 bits. Then we compare the result to the upper 32 bits.
2905         SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
2906                                         DAG.getConstant(32, DL, MVT::i64));
2907         UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits);
2908         SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
2909                                         DAG.getConstant(31, DL, MVT::i64));
2910         // It is important that LowerBits is last, otherwise the arithmetic
2911         // shift will not be folded into the compare (SUBS).
2912         SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
2913         Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2914                        .getValue(1);
2915       } else {
2916         // The overflow check for unsigned multiply is easy. We only need to
2917         // check if any of the upper 32 bits are set. This can be done with a
2918         // CMP (shifted register). For that we need to generate the following
2919         // pattern:
2920         // (i64 AArch64ISD::SUBS i64 0, (i64 srl i64 %Mul, i64 32)
2921         SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2922                                         DAG.getConstant(32, DL, MVT::i64));
2923         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2924         Overflow =
2925             DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2926                         DAG.getConstant(0, DL, MVT::i64),
2927                         UpperBits).getValue(1);
2928       }
2929       break;
2930     }
2931     assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type");
2932     // For the 64 bit multiply
2933     Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2934     if (IsSigned) {
2935       SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
2936       SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
2937                                       DAG.getConstant(63, DL, MVT::i64));
2938       // It is important that LowerBits is last, otherwise the arithmetic
2939       // shift will not be folded into the compare (SUBS).
2940       SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2941       Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2942                      .getValue(1);
2943     } else {
2944       SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
2945       SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2946       Overflow =
2947           DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2948                       DAG.getConstant(0, DL, MVT::i64),
2949                       UpperBits).getValue(1);
2950     }
2951     break;
2952   }
2953   } // switch (...)
2954 
2955   if (Opc) {
2956     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
2957 
2958     // Emit the AArch64 operation with overflow check.
2959     Value = DAG.getNode(Opc, DL, VTs, LHS, RHS);
2960     Overflow = Value.getValue(1);
2961   }
2962   return std::make_pair(Value, Overflow);
2963 }
2964 
2965 SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
2966   if (useSVEForFixedLengthVectorVT(Op.getValueType()))
2967     return LowerToScalableOp(Op, DAG);
2968 
2969   SDValue Sel = Op.getOperand(0);
2970   SDValue Other = Op.getOperand(1);
2971   SDLoc dl(Sel);
2972 
2973   // If the operand is an overflow checking operation, invert the condition
2974   // code and kill the Not operation. I.e., transform:
2975   // (xor (overflow_op_bool, 1))
2976   //   -->
2977   // (csel 1, 0, invert(cc), overflow_op_bool)
2978   // ... which later gets transformed to just a cset instruction with an
2979   // inverted condition code, rather than a cset + eor sequence.
2980   if (isOneConstant(Other) && ISD::isOverflowIntrOpRes(Sel)) {
2981     // Only lower legal XALUO ops.
2982     if (!DAG.getTargetLoweringInfo().isTypeLegal(Sel->getValueType(0)))
2983       return SDValue();
2984 
2985     SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2986     SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2987     AArch64CC::CondCode CC;
2988     SDValue Value, Overflow;
2989     std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Sel.getValue(0), DAG);
2990     SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2991     return DAG.getNode(AArch64ISD::CSEL, dl, Op.getValueType(), TVal, FVal,
2992                        CCVal, Overflow);
2993   }
2994   // If neither operand is a SELECT_CC, give up.
2995   if (Sel.getOpcode() != ISD::SELECT_CC)
2996     std::swap(Sel, Other);
2997   if (Sel.getOpcode() != ISD::SELECT_CC)
2998     return Op;
2999 
3000   // The folding we want to perform is:
3001   // (xor x, (select_cc a, b, cc, 0, -1) )
3002   //   -->
3003   // (csel x, (xor x, -1), cc ...)
3004   //
3005   // The latter will get matched to a CSINV instruction.
3006 
3007   ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get();
3008   SDValue LHS = Sel.getOperand(0);
3009   SDValue RHS = Sel.getOperand(1);
3010   SDValue TVal = Sel.getOperand(2);
3011   SDValue FVal = Sel.getOperand(3);
3012 
3013   // FIXME: This could be generalized to non-integer comparisons.
3014   if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
3015     return Op;
3016 
3017   ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
3018   ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
3019 
3020   // The values aren't constants, this isn't the pattern we're looking for.
3021   if (!CFVal || !CTVal)
3022     return Op;
3023 
3024   // We can commute the SELECT_CC by inverting the condition.  This
3025   // might be needed to make this fit into a CSINV pattern.
3026   if (CTVal->isAllOnesValue() && CFVal->isNullValue()) {
3027     std::swap(TVal, FVal);
3028     std::swap(CTVal, CFVal);
3029     CC = ISD::getSetCCInverse(CC, LHS.getValueType());
3030   }
3031 
3032   // If the constants line up, perform the transform!
3033   if (CTVal->isNullValue() && CFVal->isAllOnesValue()) {
3034     SDValue CCVal;
3035     SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
3036 
3037     FVal = Other;
3038     TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
3039                        DAG.getConstant(-1ULL, dl, Other.getValueType()));
3040 
3041     return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
3042                        CCVal, Cmp);
3043   }
3044 
3045   return Op;
3046 }
3047 
3048 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
3049   EVT VT = Op.getValueType();
3050 
3051   // Let legalize expand this if it isn't a legal type yet.
3052   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
3053     return SDValue();
3054 
3055   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
3056 
3057   unsigned Opc;
3058   bool ExtraOp = false;
3059   switch (Op.getOpcode()) {
3060   default:
3061     llvm_unreachable("Invalid code");
3062   case ISD::ADDC:
3063     Opc = AArch64ISD::ADDS;
3064     break;
3065   case ISD::SUBC:
3066     Opc = AArch64ISD::SUBS;
3067     break;
3068   case ISD::ADDE:
3069     Opc = AArch64ISD::ADCS;
3070     ExtraOp = true;
3071     break;
3072   case ISD::SUBE:
3073     Opc = AArch64ISD::SBCS;
3074     ExtraOp = true;
3075     break;
3076   }
3077 
3078   if (!ExtraOp)
3079     return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
3080   return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
3081                      Op.getOperand(2));
3082 }
3083 
3084 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
3085   // Let legalize expand this if it isn't a legal type yet.
3086   if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
3087     return SDValue();
3088 
3089   SDLoc dl(Op);
3090   AArch64CC::CondCode CC;
3091   // The actual operation that sets the overflow or carry flag.
3092   SDValue Value, Overflow;
3093   std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
3094 
3095   // We use 0 and 1 as false and true values.
3096   SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
3097   SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
3098 
3099   // We use an inverted condition, because the conditional select is inverted
3100   // too. This will allow it to be selected to a single instruction:
3101   // CSINC Wd, WZR, WZR, invert(cond).
3102   SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
3103   Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal,
3104                          CCVal, Overflow);
3105 
3106   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
3107   return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
3108 }
3109 
3110 // Prefetch operands are:
3111 // 1: Address to prefetch
3112 // 2: bool isWrite
3113 // 3: int locality (0 = no locality ... 3 = extreme locality)
3114 // 4: bool isDataCache
3115 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
3116   SDLoc DL(Op);
3117   unsigned IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
3118   unsigned Locality = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
3119   unsigned IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
3120 
3121   bool IsStream = !Locality;
3122   // When the locality number is set
3123   if (Locality) {
3124     // The front-end should have filtered out the out-of-range values
3125     assert(Locality <= 3 && "Prefetch locality out-of-range");
3126     // The locality degree is the opposite of the cache speed.
3127     // Put the number the other way around.
3128     // The encoding starts at 0 for level 1
3129     Locality = 3 - Locality;
3130   }
3131 
3132   // built the mask value encoding the expected behavior.
3133   unsigned PrfOp = (IsWrite << 4) |     // Load/Store bit
3134                    (!IsData << 3) |     // IsDataCache bit
3135                    (Locality << 1) |    // Cache level bits
3136                    (unsigned)IsStream;  // Stream bit
3137   return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
3138                      DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1));
3139 }
3140 
3141 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
3142                                               SelectionDAG &DAG) const {
3143   if (Op.getValueType().isScalableVector())
3144     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FP_EXTEND_MERGE_PASSTHRU);
3145 
3146   assert(Op.getValueType() == MVT::f128 && "Unexpected lowering");
3147   return SDValue();
3148 }
3149 
3150 SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
3151                                              SelectionDAG &DAG) const {
3152   if (Op.getValueType().isScalableVector())
3153     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FP_ROUND_MERGE_PASSTHRU);
3154 
3155   bool IsStrict = Op->isStrictFPOpcode();
3156   SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3157   EVT SrcVT = SrcVal.getValueType();
3158 
3159   if (SrcVT != MVT::f128) {
3160     // Expand cases where the input is a vector bigger than NEON.
3161     if (useSVEForFixedLengthVectorVT(SrcVT))
3162       return SDValue();
3163 
3164     // It's legal except when f128 is involved
3165     return Op;
3166   }
3167 
3168   return SDValue();
3169 }
3170 
3171 SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
3172                                                     SelectionDAG &DAG) const {
3173   // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
3174   // Any additional optimization in this function should be recorded
3175   // in the cost tables.
3176   EVT InVT = Op.getOperand(0).getValueType();
3177   EVT VT = Op.getValueType();
3178 
3179   if (VT.isScalableVector()) {
3180     unsigned Opcode = Op.getOpcode() == ISD::FP_TO_UINT
3181                           ? AArch64ISD::FCVTZU_MERGE_PASSTHRU
3182                           : AArch64ISD::FCVTZS_MERGE_PASSTHRU;
3183     return LowerToPredicatedOp(Op, DAG, Opcode);
3184   }
3185 
3186   unsigned NumElts = InVT.getVectorNumElements();
3187 
3188   // f16 conversions are promoted to f32 when full fp16 is not supported.
3189   if (InVT.getVectorElementType() == MVT::f16 &&
3190       !Subtarget->hasFullFP16()) {
3191     MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts);
3192     SDLoc dl(Op);
3193     return DAG.getNode(
3194         Op.getOpcode(), dl, Op.getValueType(),
3195         DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0)));
3196   }
3197 
3198   uint64_t VTSize = VT.getFixedSizeInBits();
3199   uint64_t InVTSize = InVT.getFixedSizeInBits();
3200   if (VTSize < InVTSize) {
3201     SDLoc dl(Op);
3202     SDValue Cv =
3203         DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(),
3204                     Op.getOperand(0));
3205     return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
3206   }
3207 
3208   if (VTSize > InVTSize) {
3209     SDLoc dl(Op);
3210     MVT ExtVT =
3211         MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()),
3212                          VT.getVectorNumElements());
3213     SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
3214     return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
3215   }
3216 
3217   // Type changing conversions are illegal.
3218   return Op;
3219 }
3220 
3221 SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
3222                                               SelectionDAG &DAG) const {
3223   bool IsStrict = Op->isStrictFPOpcode();
3224   SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3225 
3226   if (SrcVal.getValueType().isVector())
3227     return LowerVectorFP_TO_INT(Op, DAG);
3228 
3229   // f16 conversions are promoted to f32 when full fp16 is not supported.
3230   if (SrcVal.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) {
3231     assert(!IsStrict && "Lowering of strict fp16 not yet implemented");
3232     SDLoc dl(Op);
3233     return DAG.getNode(
3234         Op.getOpcode(), dl, Op.getValueType(),
3235         DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, SrcVal));
3236   }
3237 
3238   if (SrcVal.getValueType() != MVT::f128) {
3239     // It's legal except when f128 is involved
3240     return Op;
3241   }
3242 
3243   return SDValue();
3244 }
3245 
3246 SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
3247                                                     SelectionDAG &DAG) const {
3248   // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
3249   // Any additional optimization in this function should be recorded
3250   // in the cost tables.
3251   EVT VT = Op.getValueType();
3252   SDLoc dl(Op);
3253   SDValue In = Op.getOperand(0);
3254   EVT InVT = In.getValueType();
3255   unsigned Opc = Op.getOpcode();
3256   bool IsSigned = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP;
3257 
3258   if (VT.isScalableVector()) {
3259     if (InVT.getVectorElementType() == MVT::i1) {
3260       // We can't directly extend an SVE predicate; extend it first.
3261       unsigned CastOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3262       EVT CastVT = getPromotedVTForPredicate(InVT);
3263       In = DAG.getNode(CastOpc, dl, CastVT, In);
3264       return DAG.getNode(Opc, dl, VT, In);
3265     }
3266 
3267     unsigned Opcode = IsSigned ? AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU
3268                                : AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU;
3269     return LowerToPredicatedOp(Op, DAG, Opcode);
3270   }
3271 
3272   uint64_t VTSize = VT.getFixedSizeInBits();
3273   uint64_t InVTSize = InVT.getFixedSizeInBits();
3274   if (VTSize < InVTSize) {
3275     MVT CastVT =
3276         MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
3277                          InVT.getVectorNumElements());
3278     In = DAG.getNode(Opc, dl, CastVT, In);
3279     return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl));
3280   }
3281 
3282   if (VTSize > InVTSize) {
3283     unsigned CastOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3284     EVT CastVT = VT.changeVectorElementTypeToInteger();
3285     In = DAG.getNode(CastOpc, dl, CastVT, In);
3286     return DAG.getNode(Opc, dl, VT, In);
3287   }
3288 
3289   return Op;
3290 }
3291 
3292 SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
3293                                             SelectionDAG &DAG) const {
3294   if (Op.getValueType().isVector())
3295     return LowerVectorINT_TO_FP(Op, DAG);
3296 
3297   bool IsStrict = Op->isStrictFPOpcode();
3298   SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3299 
3300   // f16 conversions are promoted to f32 when full fp16 is not supported.
3301   if (Op.getValueType() == MVT::f16 &&
3302       !Subtarget->hasFullFP16()) {
3303     assert(!IsStrict && "Lowering of strict fp16 not yet implemented");
3304     SDLoc dl(Op);
3305     return DAG.getNode(
3306         ISD::FP_ROUND, dl, MVT::f16,
3307         DAG.getNode(Op.getOpcode(), dl, MVT::f32, SrcVal),
3308         DAG.getIntPtrConstant(0, dl));
3309   }
3310 
3311   // i128 conversions are libcalls.
3312   if (SrcVal.getValueType() == MVT::i128)
3313     return SDValue();
3314 
3315   // Other conversions are legal, unless it's to the completely software-based
3316   // fp128.
3317   if (Op.getValueType() != MVT::f128)
3318     return Op;
3319   return SDValue();
3320 }
3321 
3322 SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
3323                                             SelectionDAG &DAG) const {
3324   // For iOS, we want to call an alternative entry point: __sincos_stret,
3325   // which returns the values in two S / D registers.
3326   SDLoc dl(Op);
3327   SDValue Arg = Op.getOperand(0);
3328   EVT ArgVT = Arg.getValueType();
3329   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3330 
3331   ArgListTy Args;
3332   ArgListEntry Entry;
3333 
3334   Entry.Node = Arg;
3335   Entry.Ty = ArgTy;
3336   Entry.IsSExt = false;
3337   Entry.IsZExt = false;
3338   Args.push_back(Entry);
3339 
3340   RTLIB::Libcall LC = ArgVT == MVT::f64 ? RTLIB::SINCOS_STRET_F64
3341                                         : RTLIB::SINCOS_STRET_F32;
3342   const char *LibcallName = getLibcallName(LC);
3343   SDValue Callee =
3344       DAG.getExternalSymbol(LibcallName, getPointerTy(DAG.getDataLayout()));
3345 
3346   StructType *RetTy = StructType::get(ArgTy, ArgTy);
3347   TargetLowering::CallLoweringInfo CLI(DAG);
3348   CLI.setDebugLoc(dl)
3349       .setChain(DAG.getEntryNode())
3350       .setLibCallee(CallingConv::Fast, RetTy, Callee, std::move(Args));
3351 
3352   std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3353   return CallResult.first;
3354 }
3355 
3356 static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
3357   EVT OpVT = Op.getValueType();
3358   if (OpVT != MVT::f16 && OpVT != MVT::bf16)
3359     return SDValue();
3360 
3361   assert(Op.getOperand(0).getValueType() == MVT::i16);
3362   SDLoc DL(Op);
3363 
3364   Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0));
3365   Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op);
3366   return SDValue(
3367       DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, OpVT, Op,
3368                          DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
3369       0);
3370 }
3371 
3372 static EVT getExtensionTo64Bits(const EVT &OrigVT) {
3373   if (OrigVT.getSizeInBits() >= 64)
3374     return OrigVT;
3375 
3376   assert(OrigVT.isSimple() && "Expecting a simple value type");
3377 
3378   MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
3379   switch (OrigSimpleTy) {
3380   default: llvm_unreachable("Unexpected Vector Type");
3381   case MVT::v2i8:
3382   case MVT::v2i16:
3383      return MVT::v2i32;
3384   case MVT::v4i8:
3385     return  MVT::v4i16;
3386   }
3387 }
3388 
3389 static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG,
3390                                                  const EVT &OrigTy,
3391                                                  const EVT &ExtTy,
3392                                                  unsigned ExtOpcode) {
3393   // The vector originally had a size of OrigTy. It was then extended to ExtTy.
3394   // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
3395   // 64-bits we need to insert a new extension so that it will be 64-bits.
3396   assert(ExtTy.is128BitVector() && "Unexpected extension size");
3397   if (OrigTy.getSizeInBits() >= 64)
3398     return N;
3399 
3400   // Must extend size to at least 64 bits to be used as an operand for VMULL.
3401   EVT NewVT = getExtensionTo64Bits(OrigTy);
3402 
3403   return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
3404 }
3405 
3406 static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
3407                                    bool isSigned) {
3408   EVT VT = N->getValueType(0);
3409 
3410   if (N->getOpcode() != ISD::BUILD_VECTOR)
3411     return false;
3412 
3413   for (const SDValue &Elt : N->op_values()) {
3414     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
3415       unsigned EltSize = VT.getScalarSizeInBits();
3416       unsigned HalfSize = EltSize / 2;
3417       if (isSigned) {
3418         if (!isIntN(HalfSize, C->getSExtValue()))
3419           return false;
3420       } else {
3421         if (!isUIntN(HalfSize, C->getZExtValue()))
3422           return false;
3423       }
3424       continue;
3425     }
3426     return false;
3427   }
3428 
3429   return true;
3430 }
3431 
3432 static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) {
3433   if (N->getOpcode() == ISD::SIGN_EXTEND ||
3434       N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND)
3435     return addRequiredExtensionForVectorMULL(N->getOperand(0), DAG,
3436                                              N->getOperand(0)->getValueType(0),
3437                                              N->getValueType(0),
3438                                              N->getOpcode());
3439 
3440   assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
3441   EVT VT = N->getValueType(0);
3442   SDLoc dl(N);
3443   unsigned EltSize = VT.getScalarSizeInBits() / 2;
3444   unsigned NumElts = VT.getVectorNumElements();
3445   MVT TruncVT = MVT::getIntegerVT(EltSize);
3446   SmallVector<SDValue, 8> Ops;
3447   for (unsigned i = 0; i != NumElts; ++i) {
3448     ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i));
3449     const APInt &CInt = C->getAPIntValue();
3450     // Element types smaller than 32 bits are not legal, so use i32 elements.
3451     // The values are implicitly truncated so sext vs. zext doesn't matter.
3452     Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
3453   }
3454   return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops);
3455 }
3456 
3457 static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
3458   return N->getOpcode() == ISD::SIGN_EXTEND ||
3459          N->getOpcode() == ISD::ANY_EXTEND ||
3460          isExtendedBUILD_VECTOR(N, DAG, true);
3461 }
3462 
3463 static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
3464   return N->getOpcode() == ISD::ZERO_EXTEND ||
3465          N->getOpcode() == ISD::ANY_EXTEND ||
3466          isExtendedBUILD_VECTOR(N, DAG, false);
3467 }
3468 
3469 static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
3470   unsigned Opcode = N->getOpcode();
3471   if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
3472     SDNode *N0 = N->getOperand(0).getNode();
3473     SDNode *N1 = N->getOperand(1).getNode();
3474     return N0->hasOneUse() && N1->hasOneUse() &&
3475       isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
3476   }
3477   return false;
3478 }
3479 
3480 static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
3481   unsigned Opcode = N->getOpcode();
3482   if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
3483     SDNode *N0 = N->getOperand(0).getNode();
3484     SDNode *N1 = N->getOperand(1).getNode();
3485     return N0->hasOneUse() && N1->hasOneUse() &&
3486       isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
3487   }
3488   return false;
3489 }
3490 
3491 SDValue AArch64TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
3492                                                 SelectionDAG &DAG) const {
3493   // The rounding mode is in bits 23:22 of the FPSCR.
3494   // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
3495   // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
3496   // so that the shift + and get folded into a bitfield extract.
3497   SDLoc dl(Op);
3498 
3499   SDValue Chain = Op.getOperand(0);
3500   SDValue FPCR_64 = DAG.getNode(
3501       ISD::INTRINSIC_W_CHAIN, dl, {MVT::i64, MVT::Other},
3502       {Chain, DAG.getConstant(Intrinsic::aarch64_get_fpcr, dl, MVT::i64)});
3503   Chain = FPCR_64.getValue(1);
3504   SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64);
3505   SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
3506                                   DAG.getConstant(1U << 22, dl, MVT::i32));
3507   SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
3508                               DAG.getConstant(22, dl, MVT::i32));
3509   SDValue AND = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
3510                             DAG.getConstant(3, dl, MVT::i32));
3511   return DAG.getMergeValues({AND, Chain}, dl);
3512 }
3513 
3514 SDValue AArch64TargetLowering::LowerSET_ROUNDING(SDValue Op,
3515                                                  SelectionDAG &DAG) const {
3516   SDLoc DL(Op);
3517   SDValue Chain = Op->getOperand(0);
3518   SDValue RMValue = Op->getOperand(1);
3519 
3520   // The rounding mode is in bits 23:22 of the FPCR.
3521   // The llvm.set.rounding argument value to the rounding mode in FPCR mapping
3522   // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
3523   // ((arg - 1) & 3) << 22).
3524   //
3525   // The argument of llvm.set.rounding must be within the segment [0, 3], so
3526   // NearestTiesToAway (4) is not handled here. It is responsibility of the code
3527   // generated llvm.set.rounding to ensure this condition.
3528 
3529   // Calculate new value of FPCR[23:22].
3530   RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue,
3531                         DAG.getConstant(1, DL, MVT::i32));
3532   RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue,
3533                         DAG.getConstant(0x3, DL, MVT::i32));
3534   RMValue =
3535       DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue,
3536                   DAG.getConstant(AArch64::RoundingBitsPos, DL, MVT::i32));
3537   RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, RMValue);
3538 
3539   // Get current value of FPCR.
3540   SDValue Ops[] = {
3541       Chain, DAG.getTargetConstant(Intrinsic::aarch64_get_fpcr, DL, MVT::i64)};
3542   SDValue FPCR =
3543       DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i64, MVT::Other}, Ops);
3544   Chain = FPCR.getValue(1);
3545   FPCR = FPCR.getValue(0);
3546 
3547   // Put new rounding mode into FPSCR[23:22].
3548   const int RMMask = ~(AArch64::Rounding::rmMask << AArch64::RoundingBitsPos);
3549   FPCR = DAG.getNode(ISD::AND, DL, MVT::i64, FPCR,
3550                      DAG.getConstant(RMMask, DL, MVT::i64));
3551   FPCR = DAG.getNode(ISD::OR, DL, MVT::i64, FPCR, RMValue);
3552   SDValue Ops2[] = {
3553       Chain, DAG.getTargetConstant(Intrinsic::aarch64_set_fpcr, DL, MVT::i64),
3554       FPCR};
3555   return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
3556 }
3557 
3558 SDValue AArch64TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
3559   EVT VT = Op.getValueType();
3560 
3561   // If SVE is available then i64 vector multiplications can also be made legal.
3562   bool OverrideNEON = VT == MVT::v2i64 || VT == MVT::v1i64;
3563 
3564   if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT, OverrideNEON))
3565     return LowerToPredicatedOp(Op, DAG, AArch64ISD::MUL_PRED, OverrideNEON);
3566 
3567   // Multiplications are only custom-lowered for 128-bit vectors so that
3568   // VMULL can be detected.  Otherwise v2i64 multiplications are not legal.
3569   assert(VT.is128BitVector() && VT.isInteger() &&
3570          "unexpected type for custom-lowering ISD::MUL");
3571   SDNode *N0 = Op.getOperand(0).getNode();
3572   SDNode *N1 = Op.getOperand(1).getNode();
3573   unsigned NewOpc = 0;
3574   bool isMLA = false;
3575   bool isN0SExt = isSignExtended(N0, DAG);
3576   bool isN1SExt = isSignExtended(N1, DAG);
3577   if (isN0SExt && isN1SExt)
3578     NewOpc = AArch64ISD::SMULL;
3579   else {
3580     bool isN0ZExt = isZeroExtended(N0, DAG);
3581     bool isN1ZExt = isZeroExtended(N1, DAG);
3582     if (isN0ZExt && isN1ZExt)
3583       NewOpc = AArch64ISD::UMULL;
3584     else if (isN1SExt || isN1ZExt) {
3585       // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
3586       // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
3587       if (isN1SExt && isAddSubSExt(N0, DAG)) {
3588         NewOpc = AArch64ISD::SMULL;
3589         isMLA = true;
3590       } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
3591         NewOpc =  AArch64ISD::UMULL;
3592         isMLA = true;
3593       } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
3594         std::swap(N0, N1);
3595         NewOpc =  AArch64ISD::UMULL;
3596         isMLA = true;
3597       }
3598     }
3599 
3600     if (!NewOpc) {
3601       if (VT == MVT::v2i64)
3602         // Fall through to expand this.  It is not legal.
3603         return SDValue();
3604       else
3605         // Other vector multiplications are legal.
3606         return Op;
3607     }
3608   }
3609 
3610   // Legalize to a S/UMULL instruction
3611   SDLoc DL(Op);
3612   SDValue Op0;
3613   SDValue Op1 = skipExtensionForVectorMULL(N1, DAG);
3614   if (!isMLA) {
3615     Op0 = skipExtensionForVectorMULL(N0, DAG);
3616     assert(Op0.getValueType().is64BitVector() &&
3617            Op1.getValueType().is64BitVector() &&
3618            "unexpected types for extended operands to VMULL");
3619     return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
3620   }
3621   // Optimizing (zext A + zext B) * C, to (S/UMULL A, C) + (S/UMULL B, C) during
3622   // isel lowering to take advantage of no-stall back to back s/umul + s/umla.
3623   // This is true for CPUs with accumulate forwarding such as Cortex-A53/A57
3624   SDValue N00 = skipExtensionForVectorMULL(N0->getOperand(0).getNode(), DAG);
3625   SDValue N01 = skipExtensionForVectorMULL(N0->getOperand(1).getNode(), DAG);
3626   EVT Op1VT = Op1.getValueType();
3627   return DAG.getNode(N0->getOpcode(), DL, VT,
3628                      DAG.getNode(NewOpc, DL, VT,
3629                                DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
3630                      DAG.getNode(NewOpc, DL, VT,
3631                                DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
3632 }
3633 
3634 static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
3635                                int Pattern) {
3636   return DAG.getNode(AArch64ISD::PTRUE, DL, VT,
3637                      DAG.getTargetConstant(Pattern, DL, MVT::i32));
3638 }
3639 
3640 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
3641                                                      SelectionDAG &DAG) const {
3642   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3643   SDLoc dl(Op);
3644   switch (IntNo) {
3645   default: return SDValue();    // Don't custom lower most intrinsics.
3646   case Intrinsic::thread_pointer: {
3647     EVT PtrVT = getPointerTy(DAG.getDataLayout());
3648     return DAG.getNode(AArch64ISD::THREAD_POINTER, dl, PtrVT);
3649   }
3650   case Intrinsic::aarch64_neon_abs: {
3651     EVT Ty = Op.getValueType();
3652     if (Ty == MVT::i64) {
3653       SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64,
3654                                    Op.getOperand(1));
3655       Result = DAG.getNode(ISD::ABS, dl, MVT::v1i64, Result);
3656       return DAG.getNode(ISD::BITCAST, dl, MVT::i64, Result);
3657     } else if (Ty.isVector() && Ty.isInteger() && isTypeLegal(Ty)) {
3658       return DAG.getNode(ISD::ABS, dl, Ty, Op.getOperand(1));
3659     } else {
3660       report_fatal_error("Unexpected type for AArch64 NEON intrinic");
3661     }
3662   }
3663   case Intrinsic::aarch64_neon_smax:
3664     return DAG.getNode(ISD::SMAX, dl, Op.getValueType(),
3665                        Op.getOperand(1), Op.getOperand(2));
3666   case Intrinsic::aarch64_neon_umax:
3667     return DAG.getNode(ISD::UMAX, dl, Op.getValueType(),
3668                        Op.getOperand(1), Op.getOperand(2));
3669   case Intrinsic::aarch64_neon_smin:
3670     return DAG.getNode(ISD::SMIN, dl, Op.getValueType(),
3671                        Op.getOperand(1), Op.getOperand(2));
3672   case Intrinsic::aarch64_neon_umin:
3673     return DAG.getNode(ISD::UMIN, dl, Op.getValueType(),
3674                        Op.getOperand(1), Op.getOperand(2));
3675 
3676   case Intrinsic::aarch64_sve_sunpkhi:
3677     return DAG.getNode(AArch64ISD::SUNPKHI, dl, Op.getValueType(),
3678                        Op.getOperand(1));
3679   case Intrinsic::aarch64_sve_sunpklo:
3680     return DAG.getNode(AArch64ISD::SUNPKLO, dl, Op.getValueType(),
3681                        Op.getOperand(1));
3682   case Intrinsic::aarch64_sve_uunpkhi:
3683     return DAG.getNode(AArch64ISD::UUNPKHI, dl, Op.getValueType(),
3684                        Op.getOperand(1));
3685   case Intrinsic::aarch64_sve_uunpklo:
3686     return DAG.getNode(AArch64ISD::UUNPKLO, dl, Op.getValueType(),
3687                        Op.getOperand(1));
3688   case Intrinsic::aarch64_sve_clasta_n:
3689     return DAG.getNode(AArch64ISD::CLASTA_N, dl, Op.getValueType(),
3690                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
3691   case Intrinsic::aarch64_sve_clastb_n:
3692     return DAG.getNode(AArch64ISD::CLASTB_N, dl, Op.getValueType(),
3693                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
3694   case Intrinsic::aarch64_sve_lasta:
3695     return DAG.getNode(AArch64ISD::LASTA, dl, Op.getValueType(),
3696                        Op.getOperand(1), Op.getOperand(2));
3697   case Intrinsic::aarch64_sve_lastb:
3698     return DAG.getNode(AArch64ISD::LASTB, dl, Op.getValueType(),
3699                        Op.getOperand(1), Op.getOperand(2));
3700   case Intrinsic::aarch64_sve_rev:
3701     return DAG.getNode(ISD::VECTOR_REVERSE, dl, Op.getValueType(),
3702                        Op.getOperand(1));
3703   case Intrinsic::aarch64_sve_tbl:
3704     return DAG.getNode(AArch64ISD::TBL, dl, Op.getValueType(),
3705                        Op.getOperand(1), Op.getOperand(2));
3706   case Intrinsic::aarch64_sve_trn1:
3707     return DAG.getNode(AArch64ISD::TRN1, dl, Op.getValueType(),
3708                        Op.getOperand(1), Op.getOperand(2));
3709   case Intrinsic::aarch64_sve_trn2:
3710     return DAG.getNode(AArch64ISD::TRN2, dl, Op.getValueType(),
3711                        Op.getOperand(1), Op.getOperand(2));
3712   case Intrinsic::aarch64_sve_uzp1:
3713     return DAG.getNode(AArch64ISD::UZP1, dl, Op.getValueType(),
3714                        Op.getOperand(1), Op.getOperand(2));
3715   case Intrinsic::aarch64_sve_uzp2:
3716     return DAG.getNode(AArch64ISD::UZP2, dl, Op.getValueType(),
3717                        Op.getOperand(1), Op.getOperand(2));
3718   case Intrinsic::aarch64_sve_zip1:
3719     return DAG.getNode(AArch64ISD::ZIP1, dl, Op.getValueType(),
3720                        Op.getOperand(1), Op.getOperand(2));
3721   case Intrinsic::aarch64_sve_zip2:
3722     return DAG.getNode(AArch64ISD::ZIP2, dl, Op.getValueType(),
3723                        Op.getOperand(1), Op.getOperand(2));
3724   case Intrinsic::aarch64_sve_ptrue:
3725     return DAG.getNode(AArch64ISD::PTRUE, dl, Op.getValueType(),
3726                        Op.getOperand(1));
3727   case Intrinsic::aarch64_sve_clz:
3728     return DAG.getNode(AArch64ISD::CTLZ_MERGE_PASSTHRU, dl, Op.getValueType(),
3729                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3730   case Intrinsic::aarch64_sve_cnt: {
3731     SDValue Data = Op.getOperand(3);
3732     // CTPOP only supports integer operands.
3733     if (Data.getValueType().isFloatingPoint())
3734       Data = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Data);
3735     return DAG.getNode(AArch64ISD::CTPOP_MERGE_PASSTHRU, dl, Op.getValueType(),
3736                        Op.getOperand(2), Data, Op.getOperand(1));
3737   }
3738   case Intrinsic::aarch64_sve_dupq_lane:
3739     return LowerDUPQLane(Op, DAG);
3740   case Intrinsic::aarch64_sve_convert_from_svbool:
3741     return DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, Op.getValueType(),
3742                        Op.getOperand(1));
3743   case Intrinsic::aarch64_sve_fneg:
3744     return DAG.getNode(AArch64ISD::FNEG_MERGE_PASSTHRU, dl, Op.getValueType(),
3745                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3746   case Intrinsic::aarch64_sve_frintp:
3747     return DAG.getNode(AArch64ISD::FCEIL_MERGE_PASSTHRU, dl, Op.getValueType(),
3748                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3749   case Intrinsic::aarch64_sve_frintm:
3750     return DAG.getNode(AArch64ISD::FFLOOR_MERGE_PASSTHRU, dl, Op.getValueType(),
3751                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3752   case Intrinsic::aarch64_sve_frinti:
3753     return DAG.getNode(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, dl, Op.getValueType(),
3754                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3755   case Intrinsic::aarch64_sve_frintx:
3756     return DAG.getNode(AArch64ISD::FRINT_MERGE_PASSTHRU, dl, Op.getValueType(),
3757                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3758   case Intrinsic::aarch64_sve_frinta:
3759     return DAG.getNode(AArch64ISD::FROUND_MERGE_PASSTHRU, dl, Op.getValueType(),
3760                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3761   case Intrinsic::aarch64_sve_frintn:
3762     return DAG.getNode(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU, dl, Op.getValueType(),
3763                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3764   case Intrinsic::aarch64_sve_frintz:
3765     return DAG.getNode(AArch64ISD::FTRUNC_MERGE_PASSTHRU, dl, Op.getValueType(),
3766                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3767   case Intrinsic::aarch64_sve_ucvtf:
3768     return DAG.getNode(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU, dl,
3769                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
3770                        Op.getOperand(1));
3771   case Intrinsic::aarch64_sve_scvtf:
3772     return DAG.getNode(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU, dl,
3773                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
3774                        Op.getOperand(1));
3775   case Intrinsic::aarch64_sve_fcvtzu:
3776     return DAG.getNode(AArch64ISD::FCVTZU_MERGE_PASSTHRU, dl,
3777                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
3778                        Op.getOperand(1));
3779   case Intrinsic::aarch64_sve_fcvtzs:
3780     return DAG.getNode(AArch64ISD::FCVTZS_MERGE_PASSTHRU, dl,
3781                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
3782                        Op.getOperand(1));
3783   case Intrinsic::aarch64_sve_fsqrt:
3784     return DAG.getNode(AArch64ISD::FSQRT_MERGE_PASSTHRU, dl, Op.getValueType(),
3785                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3786   case Intrinsic::aarch64_sve_frecpx:
3787     return DAG.getNode(AArch64ISD::FRECPX_MERGE_PASSTHRU, dl, Op.getValueType(),
3788                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3789   case Intrinsic::aarch64_sve_fabs:
3790     return DAG.getNode(AArch64ISD::FABS_MERGE_PASSTHRU, dl, Op.getValueType(),
3791                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3792   case Intrinsic::aarch64_sve_abs:
3793     return DAG.getNode(AArch64ISD::ABS_MERGE_PASSTHRU, dl, Op.getValueType(),
3794                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3795   case Intrinsic::aarch64_sve_neg:
3796     return DAG.getNode(AArch64ISD::NEG_MERGE_PASSTHRU, dl, Op.getValueType(),
3797                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3798   case Intrinsic::aarch64_sve_convert_to_svbool: {
3799     EVT OutVT = Op.getValueType();
3800     EVT InVT = Op.getOperand(1).getValueType();
3801     // Return the operand if the cast isn't changing type,
3802     // i.e. <n x 16 x i1> -> <n x 16 x i1>
3803     if (InVT == OutVT)
3804       return Op.getOperand(1);
3805     // Otherwise, zero the newly introduced lanes.
3806     SDValue Reinterpret =
3807         DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, OutVT, Op.getOperand(1));
3808     SDValue Mask = getPTrue(DAG, dl, InVT, AArch64SVEPredPattern::all);
3809     SDValue MaskReinterpret =
3810         DAG.getNode(AArch64ISD::REINTERPRET_CAST, dl, OutVT, Mask);
3811     return DAG.getNode(ISD::AND, dl, OutVT, Reinterpret, MaskReinterpret);
3812   }
3813 
3814   case Intrinsic::aarch64_sve_insr: {
3815     SDValue Scalar = Op.getOperand(2);
3816     EVT ScalarTy = Scalar.getValueType();
3817     if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16))
3818       Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar);
3819 
3820     return DAG.getNode(AArch64ISD::INSR, dl, Op.getValueType(),
3821                        Op.getOperand(1), Scalar);
3822   }
3823   case Intrinsic::aarch64_sve_rbit:
3824     return DAG.getNode(AArch64ISD::BITREVERSE_MERGE_PASSTHRU, dl,
3825                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
3826                        Op.getOperand(1));
3827   case Intrinsic::aarch64_sve_revb:
3828     return DAG.getNode(AArch64ISD::BSWAP_MERGE_PASSTHRU, dl, Op.getValueType(),
3829                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
3830   case Intrinsic::aarch64_sve_sxtb:
3831     return DAG.getNode(
3832         AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3833         Op.getOperand(2), Op.getOperand(3),
3834         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i8)),
3835         Op.getOperand(1));
3836   case Intrinsic::aarch64_sve_sxth:
3837     return DAG.getNode(
3838         AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3839         Op.getOperand(2), Op.getOperand(3),
3840         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i16)),
3841         Op.getOperand(1));
3842   case Intrinsic::aarch64_sve_sxtw:
3843     return DAG.getNode(
3844         AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3845         Op.getOperand(2), Op.getOperand(3),
3846         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i32)),
3847         Op.getOperand(1));
3848   case Intrinsic::aarch64_sve_uxtb:
3849     return DAG.getNode(
3850         AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3851         Op.getOperand(2), Op.getOperand(3),
3852         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i8)),
3853         Op.getOperand(1));
3854   case Intrinsic::aarch64_sve_uxth:
3855     return DAG.getNode(
3856         AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3857         Op.getOperand(2), Op.getOperand(3),
3858         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i16)),
3859         Op.getOperand(1));
3860   case Intrinsic::aarch64_sve_uxtw:
3861     return DAG.getNode(
3862         AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
3863         Op.getOperand(2), Op.getOperand(3),
3864         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i32)),
3865         Op.getOperand(1));
3866 
3867   case Intrinsic::localaddress: {
3868     const auto &MF = DAG.getMachineFunction();
3869     const auto *RegInfo = Subtarget->getRegisterInfo();
3870     unsigned Reg = RegInfo->getLocalAddressRegister(MF);
3871     return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg,
3872                               Op.getSimpleValueType());
3873   }
3874 
3875   case Intrinsic::eh_recoverfp: {
3876     // FIXME: This needs to be implemented to correctly handle highly aligned
3877     // stack objects. For now we simply return the incoming FP. Refer D53541
3878     // for more details.
3879     SDValue FnOp = Op.getOperand(1);
3880     SDValue IncomingFPOp = Op.getOperand(2);
3881     GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
3882     auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
3883     if (!Fn)
3884       report_fatal_error(
3885           "llvm.eh.recoverfp must take a function as the first argument");
3886     return IncomingFPOp;
3887   }
3888 
3889   case Intrinsic::aarch64_neon_vsri:
3890   case Intrinsic::aarch64_neon_vsli: {
3891     EVT Ty = Op.getValueType();
3892 
3893     if (!Ty.isVector())
3894       report_fatal_error("Unexpected type for aarch64_neon_vsli");
3895 
3896     assert(Op.getConstantOperandVal(3) <= Ty.getScalarSizeInBits());
3897 
3898     bool IsShiftRight = IntNo == Intrinsic::aarch64_neon_vsri;
3899     unsigned Opcode = IsShiftRight ? AArch64ISD::VSRI : AArch64ISD::VSLI;
3900     return DAG.getNode(Opcode, dl, Ty, Op.getOperand(1), Op.getOperand(2),
3901                        Op.getOperand(3));
3902   }
3903 
3904   case Intrinsic::aarch64_neon_srhadd:
3905   case Intrinsic::aarch64_neon_urhadd:
3906   case Intrinsic::aarch64_neon_shadd:
3907   case Intrinsic::aarch64_neon_uhadd: {
3908     bool IsSignedAdd = (IntNo == Intrinsic::aarch64_neon_srhadd ||
3909                         IntNo == Intrinsic::aarch64_neon_shadd);
3910     bool IsRoundingAdd = (IntNo == Intrinsic::aarch64_neon_srhadd ||
3911                           IntNo == Intrinsic::aarch64_neon_urhadd);
3912     unsigned Opcode =
3913         IsSignedAdd ? (IsRoundingAdd ? AArch64ISD::SRHADD : AArch64ISD::SHADD)
3914                     : (IsRoundingAdd ? AArch64ISD::URHADD : AArch64ISD::UHADD);
3915     return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1),
3916                        Op.getOperand(2));
3917   }
3918   case Intrinsic::aarch64_neon_sabd:
3919   case Intrinsic::aarch64_neon_uabd: {
3920     unsigned Opcode = IntNo == Intrinsic::aarch64_neon_uabd ? AArch64ISD::UABD
3921                                                             : AArch64ISD::SABD;
3922     return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1),
3923                        Op.getOperand(2));
3924   }
3925   case Intrinsic::aarch64_neon_sdot:
3926   case Intrinsic::aarch64_neon_udot: {
3927     unsigned Opcode = IntNo == Intrinsic::aarch64_neon_udot ? AArch64ISD::UDOT
3928                                                             : AArch64ISD::SDOT;
3929     return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1),
3930                        Op.getOperand(2), Op.getOperand(3));
3931   }
3932   }
3933 }
3934 
3935 bool AArch64TargetLowering::shouldExtendGSIndex(EVT VT, EVT &EltTy) const {
3936   if (VT.getVectorElementType() == MVT::i8 ||
3937       VT.getVectorElementType() == MVT::i16) {
3938     EltTy = MVT::i32;
3939     return true;
3940   }
3941   return false;
3942 }
3943 
3944 bool AArch64TargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
3945   if (VT.getVectorElementType() == MVT::i32 &&
3946       VT.getVectorElementCount().getKnownMinValue() >= 4)
3947     return true;
3948 
3949   return false;
3950 }
3951 
3952 bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
3953   return ExtVal.getValueType().isScalableVector();
3954 }
3955 
3956 unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
3957   std::map<std::tuple<bool, bool, bool>, unsigned> AddrModes = {
3958       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ false),
3959        AArch64ISD::GLD1_MERGE_ZERO},
3960       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ true),
3961        AArch64ISD::GLD1_UXTW_MERGE_ZERO},
3962       {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ false),
3963        AArch64ISD::GLD1_MERGE_ZERO},
3964       {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ true),
3965        AArch64ISD::GLD1_SXTW_MERGE_ZERO},
3966       {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ false),
3967        AArch64ISD::GLD1_SCALED_MERGE_ZERO},
3968       {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ true),
3969        AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO},
3970       {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ false),
3971        AArch64ISD::GLD1_SCALED_MERGE_ZERO},
3972       {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ true),
3973        AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO},
3974   };
3975   auto Key = std::make_tuple(IsScaled, IsSigned, NeedsExtend);
3976   return AddrModes.find(Key)->second;
3977 }
3978 
3979 unsigned getScatterVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
3980   std::map<std::tuple<bool, bool, bool>, unsigned> AddrModes = {
3981       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ false),
3982        AArch64ISD::SST1_PRED},
3983       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ true),
3984        AArch64ISD::SST1_UXTW_PRED},
3985       {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ false),
3986        AArch64ISD::SST1_PRED},
3987       {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ true),
3988        AArch64ISD::SST1_SXTW_PRED},
3989       {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ false),
3990        AArch64ISD::SST1_SCALED_PRED},
3991       {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ true),
3992        AArch64ISD::SST1_UXTW_SCALED_PRED},
3993       {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ false),
3994        AArch64ISD::SST1_SCALED_PRED},
3995       {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ true),
3996        AArch64ISD::SST1_SXTW_SCALED_PRED},
3997   };
3998   auto Key = std::make_tuple(IsScaled, IsSigned, NeedsExtend);
3999   return AddrModes.find(Key)->second;
4000 }
4001 
4002 unsigned getSignExtendedGatherOpcode(unsigned Opcode) {
4003   switch (Opcode) {
4004   default:
4005     llvm_unreachable("unimplemented opcode");
4006     return Opcode;
4007   case AArch64ISD::GLD1_MERGE_ZERO:
4008     return AArch64ISD::GLD1S_MERGE_ZERO;
4009   case AArch64ISD::GLD1_IMM_MERGE_ZERO:
4010     return AArch64ISD::GLD1S_IMM_MERGE_ZERO;
4011   case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
4012     return AArch64ISD::GLD1S_UXTW_MERGE_ZERO;
4013   case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
4014     return AArch64ISD::GLD1S_SXTW_MERGE_ZERO;
4015   case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
4016     return AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
4017   case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
4018     return AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO;
4019   case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
4020     return AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO;
4021   }
4022 }
4023 
4024 bool getGatherScatterIndexIsExtended(SDValue Index) {
4025   unsigned Opcode = Index.getOpcode();
4026   if (Opcode == ISD::SIGN_EXTEND_INREG)
4027     return true;
4028 
4029   if (Opcode == ISD::AND) {
4030     SDValue Splat = Index.getOperand(1);
4031     if (Splat.getOpcode() != ISD::SPLAT_VECTOR)
4032       return false;
4033     ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Splat.getOperand(0));
4034     if (!Mask || Mask->getZExtValue() != 0xFFFFFFFF)
4035       return false;
4036     return true;
4037   }
4038 
4039   return false;
4040 }
4041 
4042 // If the base pointer of a masked gather or scatter is null, we
4043 // may be able to swap BasePtr & Index and use the vector + register
4044 // or vector + immediate addressing mode, e.g.
4045 // VECTOR + REGISTER:
4046 //    getelementptr nullptr, <vscale x N x T> (splat(%offset)) + %indices)
4047 // -> getelementptr %offset, <vscale x N x T> %indices
4048 // VECTOR + IMMEDIATE:
4049 //    getelementptr nullptr, <vscale x N x T> (splat(#x)) + %indices)
4050 // -> getelementptr #x, <vscale x N x T> %indices
4051 void selectGatherScatterAddrMode(SDValue &BasePtr, SDValue &Index, EVT MemVT,
4052                                  unsigned &Opcode, bool IsGather,
4053                                  SelectionDAG &DAG) {
4054   if (!isNullConstant(BasePtr))
4055     return;
4056 
4057   ConstantSDNode *Offset = nullptr;
4058   if (Index.getOpcode() == ISD::ADD)
4059     if (auto SplatVal = DAG.getSplatValue(Index.getOperand(1))) {
4060       if (isa<ConstantSDNode>(SplatVal))
4061         Offset = cast<ConstantSDNode>(SplatVal);
4062       else {
4063         BasePtr = SplatVal;
4064         Index = Index->getOperand(0);
4065         return;
4066       }
4067     }
4068 
4069   unsigned NewOp =
4070       IsGather ? AArch64ISD::GLD1_IMM_MERGE_ZERO : AArch64ISD::SST1_IMM_PRED;
4071 
4072   if (!Offset) {
4073     std::swap(BasePtr, Index);
4074     Opcode = NewOp;
4075     return;
4076   }
4077 
4078   uint64_t OffsetVal = Offset->getZExtValue();
4079   unsigned ScalarSizeInBytes = MemVT.getScalarSizeInBits() / 8;
4080   auto ConstOffset = DAG.getConstant(OffsetVal, SDLoc(Index), MVT::i64);
4081 
4082   if (OffsetVal % ScalarSizeInBytes || OffsetVal / ScalarSizeInBytes > 31) {
4083     // Index is out of range for the immediate addressing mode
4084     BasePtr = ConstOffset;
4085     Index = Index->getOperand(0);
4086     return;
4087   }
4088 
4089   // Immediate is in range
4090   Opcode = NewOp;
4091   BasePtr = Index->getOperand(0);
4092   Index = ConstOffset;
4093 }
4094 
4095 SDValue AArch64TargetLowering::LowerMGATHER(SDValue Op,
4096                                             SelectionDAG &DAG) const {
4097   SDLoc DL(Op);
4098   MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(Op);
4099   assert(MGT && "Can only custom lower gather load nodes");
4100 
4101   SDValue Index = MGT->getIndex();
4102   SDValue Chain = MGT->getChain();
4103   SDValue PassThru = MGT->getPassThru();
4104   SDValue Mask = MGT->getMask();
4105   SDValue BasePtr = MGT->getBasePtr();
4106   ISD::LoadExtType ExtTy = MGT->getExtensionType();
4107 
4108   ISD::MemIndexType IndexType = MGT->getIndexType();
4109   bool IsScaled =
4110       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::UNSIGNED_SCALED;
4111   bool IsSigned =
4112       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::SIGNED_UNSCALED;
4113   bool IdxNeedsExtend =
4114       getGatherScatterIndexIsExtended(Index) ||
4115       Index.getSimpleValueType().getVectorElementType() == MVT::i32;
4116   bool ResNeedsSignExtend = ExtTy == ISD::EXTLOAD || ExtTy == ISD::SEXTLOAD;
4117 
4118   EVT VT = PassThru.getSimpleValueType();
4119   EVT MemVT = MGT->getMemoryVT();
4120   SDValue InputVT = DAG.getValueType(MemVT);
4121 
4122   if (VT.getVectorElementType() == MVT::bf16 &&
4123       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
4124     return SDValue();
4125 
4126   // Handle FP data by using an integer gather and casting the result.
4127   if (VT.isFloatingPoint()) {
4128     EVT PassThruVT = getPackedSVEVectorVT(VT.getVectorElementCount());
4129     PassThru = getSVESafeBitCast(PassThruVT, PassThru, DAG);
4130     InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger());
4131   }
4132 
4133   SDVTList VTs = DAG.getVTList(PassThru.getSimpleValueType(), MVT::Other);
4134 
4135   if (getGatherScatterIndexIsExtended(Index))
4136     Index = Index.getOperand(0);
4137 
4138   unsigned Opcode = getGatherVecOpcode(IsScaled, IsSigned, IdxNeedsExtend);
4139   selectGatherScatterAddrMode(BasePtr, Index, MemVT, Opcode,
4140                               /*isGather=*/true, DAG);
4141 
4142   if (ResNeedsSignExtend)
4143     Opcode = getSignExtendedGatherOpcode(Opcode);
4144 
4145   SDValue Ops[] = {Chain, Mask, BasePtr, Index, InputVT, PassThru};
4146   SDValue Gather = DAG.getNode(Opcode, DL, VTs, Ops);
4147 
4148   if (VT.isFloatingPoint()) {
4149     SDValue Cast = getSVESafeBitCast(VT, Gather, DAG);
4150     return DAG.getMergeValues({Cast, Gather.getValue(1)}, DL);
4151   }
4152 
4153   return Gather;
4154 }
4155 
4156 SDValue AArch64TargetLowering::LowerMSCATTER(SDValue Op,
4157                                              SelectionDAG &DAG) const {
4158   SDLoc DL(Op);
4159   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Op);
4160   assert(MSC && "Can only custom lower scatter store nodes");
4161 
4162   SDValue Index = MSC->getIndex();
4163   SDValue Chain = MSC->getChain();
4164   SDValue StoreVal = MSC->getValue();
4165   SDValue Mask = MSC->getMask();
4166   SDValue BasePtr = MSC->getBasePtr();
4167 
4168   ISD::MemIndexType IndexType = MSC->getIndexType();
4169   bool IsScaled =
4170       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::UNSIGNED_SCALED;
4171   bool IsSigned =
4172       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::SIGNED_UNSCALED;
4173   bool NeedsExtend =
4174       getGatherScatterIndexIsExtended(Index) ||
4175       Index.getSimpleValueType().getVectorElementType() == MVT::i32;
4176 
4177   EVT VT = StoreVal.getSimpleValueType();
4178   SDVTList VTs = DAG.getVTList(MVT::Other);
4179   EVT MemVT = MSC->getMemoryVT();
4180   SDValue InputVT = DAG.getValueType(MemVT);
4181 
4182   if (VT.getVectorElementType() == MVT::bf16 &&
4183       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
4184     return SDValue();
4185 
4186   // Handle FP data by casting the data so an integer scatter can be used.
4187   if (VT.isFloatingPoint()) {
4188     EVT StoreValVT = getPackedSVEVectorVT(VT.getVectorElementCount());
4189     StoreVal = getSVESafeBitCast(StoreValVT, StoreVal, DAG);
4190     InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger());
4191   }
4192 
4193   if (getGatherScatterIndexIsExtended(Index))
4194     Index = Index.getOperand(0);
4195 
4196   unsigned Opcode = getScatterVecOpcode(IsScaled, IsSigned, NeedsExtend);
4197   selectGatherScatterAddrMode(BasePtr, Index, MemVT, Opcode,
4198                               /*isGather=*/false, DAG);
4199 
4200   SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, InputVT};
4201   return DAG.getNode(Opcode, DL, VTs, Ops);
4202 }
4203 
4204 // Custom lower trunc store for v4i8 vectors, since it is promoted to v4i16.
4205 static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
4206                                         EVT VT, EVT MemVT,
4207                                         SelectionDAG &DAG) {
4208   assert(VT.isVector() && "VT should be a vector type");
4209   assert(MemVT == MVT::v4i8 && VT == MVT::v4i16);
4210 
4211   SDValue Value = ST->getValue();
4212 
4213   // It first extend the promoted v4i16 to v8i16, truncate to v8i8, and extract
4214   // the word lane which represent the v4i8 subvector.  It optimizes the store
4215   // to:
4216   //
4217   //   xtn  v0.8b, v0.8h
4218   //   str  s0, [x0]
4219 
4220   SDValue Undef = DAG.getUNDEF(MVT::i16);
4221   SDValue UndefVec = DAG.getBuildVector(MVT::v4i16, DL,
4222                                         {Undef, Undef, Undef, Undef});
4223 
4224   SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16,
4225                                  Value, UndefVec);
4226   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt);
4227 
4228   Trunc = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Trunc);
4229   SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
4230                                      Trunc, DAG.getConstant(0, DL, MVT::i64));
4231 
4232   return DAG.getStore(ST->getChain(), DL, ExtractTrunc,
4233                       ST->getBasePtr(), ST->getMemOperand());
4234 }
4235 
4236 // Custom lowering for any store, vector or scalar and/or default or with
4237 // a truncate operations.  Currently only custom lower truncate operation
4238 // from vector v4i16 to v4i8 or volatile stores of i128.
4239 SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
4240                                           SelectionDAG &DAG) const {
4241   SDLoc Dl(Op);
4242   StoreSDNode *StoreNode = cast<StoreSDNode>(Op);
4243   assert (StoreNode && "Can only custom lower store nodes");
4244 
4245   SDValue Value = StoreNode->getValue();
4246 
4247   EVT VT = Value.getValueType();
4248   EVT MemVT = StoreNode->getMemoryVT();
4249 
4250   if (VT.isVector()) {
4251     if (useSVEForFixedLengthVectorVT(VT))
4252       return LowerFixedLengthVectorStoreToSVE(Op, DAG);
4253 
4254     unsigned AS = StoreNode->getAddressSpace();
4255     Align Alignment = StoreNode->getAlign();
4256     if (Alignment < MemVT.getStoreSize() &&
4257         !allowsMisalignedMemoryAccesses(MemVT, AS, Alignment,
4258                                         StoreNode->getMemOperand()->getFlags(),
4259                                         nullptr)) {
4260       return scalarizeVectorStore(StoreNode, DAG);
4261     }
4262 
4263     if (StoreNode->isTruncatingStore()) {
4264       return LowerTruncateVectorStore(Dl, StoreNode, VT, MemVT, DAG);
4265     }
4266     // 256 bit non-temporal stores can be lowered to STNP. Do this as part of
4267     // the custom lowering, as there are no un-paired non-temporal stores and
4268     // legalization will break up 256 bit inputs.
4269     ElementCount EC = MemVT.getVectorElementCount();
4270     if (StoreNode->isNonTemporal() && MemVT.getSizeInBits() == 256u &&
4271         EC.isKnownEven() &&
4272         ((MemVT.getScalarSizeInBits() == 8u ||
4273           MemVT.getScalarSizeInBits() == 16u ||
4274           MemVT.getScalarSizeInBits() == 32u ||
4275           MemVT.getScalarSizeInBits() == 64u))) {
4276       SDValue Lo =
4277           DAG.getNode(ISD::EXTRACT_SUBVECTOR, Dl,
4278                       MemVT.getHalfNumVectorElementsVT(*DAG.getContext()),
4279                       StoreNode->getValue(), DAG.getConstant(0, Dl, MVT::i64));
4280       SDValue Hi =
4281           DAG.getNode(ISD::EXTRACT_SUBVECTOR, Dl,
4282                       MemVT.getHalfNumVectorElementsVT(*DAG.getContext()),
4283                       StoreNode->getValue(),
4284                       DAG.getConstant(EC.getKnownMinValue() / 2, Dl, MVT::i64));
4285       SDValue Result = DAG.getMemIntrinsicNode(
4286           AArch64ISD::STNP, Dl, DAG.getVTList(MVT::Other),
4287           {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
4288           StoreNode->getMemoryVT(), StoreNode->getMemOperand());
4289       return Result;
4290     }
4291   } else if (MemVT == MVT::i128 && StoreNode->isVolatile()) {
4292     assert(StoreNode->getValue()->getValueType(0) == MVT::i128);
4293     SDValue Lo =
4294         DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i64, StoreNode->getValue(),
4295                     DAG.getConstant(0, Dl, MVT::i64));
4296     SDValue Hi =
4297         DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i64, StoreNode->getValue(),
4298                     DAG.getConstant(1, Dl, MVT::i64));
4299     SDValue Result = DAG.getMemIntrinsicNode(
4300         AArch64ISD::STP, Dl, DAG.getVTList(MVT::Other),
4301         {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
4302         StoreNode->getMemoryVT(), StoreNode->getMemOperand());
4303     return Result;
4304   }
4305 
4306   return SDValue();
4307 }
4308 
4309 // Generate SUBS and CSEL for integer abs.
4310 SDValue AArch64TargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
4311   MVT VT = Op.getSimpleValueType();
4312 
4313   if (VT.isVector())
4314     return LowerToPredicatedOp(Op, DAG, AArch64ISD::ABS_MERGE_PASSTHRU);
4315 
4316   SDLoc DL(Op);
4317   SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
4318                             Op.getOperand(0));
4319   // Generate SUBS & CSEL.
4320   SDValue Cmp =
4321       DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32),
4322                   Op.getOperand(0), DAG.getConstant(0, DL, VT));
4323   return DAG.getNode(AArch64ISD::CSEL, DL, VT, Op.getOperand(0), Neg,
4324                      DAG.getConstant(AArch64CC::PL, DL, MVT::i32),
4325                      Cmp.getValue(1));
4326 }
4327 
4328 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
4329                                               SelectionDAG &DAG) const {
4330   LLVM_DEBUG(dbgs() << "Custom lowering: ");
4331   LLVM_DEBUG(Op.dump());
4332 
4333   switch (Op.getOpcode()) {
4334   default:
4335     llvm_unreachable("unimplemented operand");
4336     return SDValue();
4337   case ISD::BITCAST:
4338     return LowerBITCAST(Op, DAG);
4339   case ISD::GlobalAddress:
4340     return LowerGlobalAddress(Op, DAG);
4341   case ISD::GlobalTLSAddress:
4342     return LowerGlobalTLSAddress(Op, DAG);
4343   case ISD::SETCC:
4344   case ISD::STRICT_FSETCC:
4345   case ISD::STRICT_FSETCCS:
4346     return LowerSETCC(Op, DAG);
4347   case ISD::BR_CC:
4348     return LowerBR_CC(Op, DAG);
4349   case ISD::SELECT:
4350     return LowerSELECT(Op, DAG);
4351   case ISD::SELECT_CC:
4352     return LowerSELECT_CC(Op, DAG);
4353   case ISD::JumpTable:
4354     return LowerJumpTable(Op, DAG);
4355   case ISD::BR_JT:
4356     return LowerBR_JT(Op, DAG);
4357   case ISD::ConstantPool:
4358     return LowerConstantPool(Op, DAG);
4359   case ISD::BlockAddress:
4360     return LowerBlockAddress(Op, DAG);
4361   case ISD::VASTART:
4362     return LowerVASTART(Op, DAG);
4363   case ISD::VACOPY:
4364     return LowerVACOPY(Op, DAG);
4365   case ISD::VAARG:
4366     return LowerVAARG(Op, DAG);
4367   case ISD::ADDC:
4368   case ISD::ADDE:
4369   case ISD::SUBC:
4370   case ISD::SUBE:
4371     return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
4372   case ISD::SADDO:
4373   case ISD::UADDO:
4374   case ISD::SSUBO:
4375   case ISD::USUBO:
4376   case ISD::SMULO:
4377   case ISD::UMULO:
4378     return LowerXALUO(Op, DAG);
4379   case ISD::FADD:
4380     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FADD_PRED);
4381   case ISD::FSUB:
4382     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSUB_PRED);
4383   case ISD::FMUL:
4384     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMUL_PRED);
4385   case ISD::FMA:
4386     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMA_PRED);
4387   case ISD::FDIV:
4388     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FDIV_PRED);
4389   case ISD::FNEG:
4390     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FNEG_MERGE_PASSTHRU);
4391   case ISD::FCEIL:
4392     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FCEIL_MERGE_PASSTHRU);
4393   case ISD::FFLOOR:
4394     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FFLOOR_MERGE_PASSTHRU);
4395   case ISD::FNEARBYINT:
4396     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FNEARBYINT_MERGE_PASSTHRU);
4397   case ISD::FRINT:
4398     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FRINT_MERGE_PASSTHRU);
4399   case ISD::FROUND:
4400     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FROUND_MERGE_PASSTHRU);
4401   case ISD::FROUNDEVEN:
4402     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU);
4403   case ISD::FTRUNC:
4404     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FTRUNC_MERGE_PASSTHRU);
4405   case ISD::FSQRT:
4406     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSQRT_MERGE_PASSTHRU);
4407   case ISD::FABS:
4408     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FABS_MERGE_PASSTHRU);
4409   case ISD::FP_ROUND:
4410   case ISD::STRICT_FP_ROUND:
4411     return LowerFP_ROUND(Op, DAG);
4412   case ISD::FP_EXTEND:
4413     return LowerFP_EXTEND(Op, DAG);
4414   case ISD::FRAMEADDR:
4415     return LowerFRAMEADDR(Op, DAG);
4416   case ISD::SPONENTRY:
4417     return LowerSPONENTRY(Op, DAG);
4418   case ISD::RETURNADDR:
4419     return LowerRETURNADDR(Op, DAG);
4420   case ISD::ADDROFRETURNADDR:
4421     return LowerADDROFRETURNADDR(Op, DAG);
4422   case ISD::CONCAT_VECTORS:
4423     return LowerCONCAT_VECTORS(Op, DAG);
4424   case ISD::INSERT_VECTOR_ELT:
4425     return LowerINSERT_VECTOR_ELT(Op, DAG);
4426   case ISD::EXTRACT_VECTOR_ELT:
4427     return LowerEXTRACT_VECTOR_ELT(Op, DAG);
4428   case ISD::BUILD_VECTOR:
4429     return LowerBUILD_VECTOR(Op, DAG);
4430   case ISD::VECTOR_SHUFFLE:
4431     return LowerVECTOR_SHUFFLE(Op, DAG);
4432   case ISD::SPLAT_VECTOR:
4433     return LowerSPLAT_VECTOR(Op, DAG);
4434   case ISD::STEP_VECTOR:
4435     return LowerSTEP_VECTOR(Op, DAG);
4436   case ISD::EXTRACT_SUBVECTOR:
4437     return LowerEXTRACT_SUBVECTOR(Op, DAG);
4438   case ISD::INSERT_SUBVECTOR:
4439     return LowerINSERT_SUBVECTOR(Op, DAG);
4440   case ISD::SDIV:
4441   case ISD::UDIV:
4442     return LowerDIV(Op, DAG);
4443   case ISD::SMIN:
4444     return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMIN_PRED,
4445                                /*OverrideNEON=*/true);
4446   case ISD::UMIN:
4447     return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMIN_PRED,
4448                                /*OverrideNEON=*/true);
4449   case ISD::SMAX:
4450     return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMAX_PRED,
4451                                /*OverrideNEON=*/true);
4452   case ISD::UMAX:
4453     return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMAX_PRED,
4454                                /*OverrideNEON=*/true);
4455   case ISD::SRA:
4456   case ISD::SRL:
4457   case ISD::SHL:
4458     return LowerVectorSRA_SRL_SHL(Op, DAG);
4459   case ISD::SHL_PARTS:
4460     return LowerShiftLeftParts(Op, DAG);
4461   case ISD::SRL_PARTS:
4462   case ISD::SRA_PARTS:
4463     return LowerShiftRightParts(Op, DAG);
4464   case ISD::CTPOP:
4465     return LowerCTPOP(Op, DAG);
4466   case ISD::FCOPYSIGN:
4467     return LowerFCOPYSIGN(Op, DAG);
4468   case ISD::OR:
4469     return LowerVectorOR(Op, DAG);
4470   case ISD::XOR:
4471     return LowerXOR(Op, DAG);
4472   case ISD::PREFETCH:
4473     return LowerPREFETCH(Op, DAG);
4474   case ISD::SINT_TO_FP:
4475   case ISD::UINT_TO_FP:
4476   case ISD::STRICT_SINT_TO_FP:
4477   case ISD::STRICT_UINT_TO_FP:
4478     return LowerINT_TO_FP(Op, DAG);
4479   case ISD::FP_TO_SINT:
4480   case ISD::FP_TO_UINT:
4481   case ISD::STRICT_FP_TO_SINT:
4482   case ISD::STRICT_FP_TO_UINT:
4483     return LowerFP_TO_INT(Op, DAG);
4484   case ISD::FSINCOS:
4485     return LowerFSINCOS(Op, DAG);
4486   case ISD::FLT_ROUNDS_:
4487     return LowerFLT_ROUNDS_(Op, DAG);
4488   case ISD::SET_ROUNDING:
4489     return LowerSET_ROUNDING(Op, DAG);
4490   case ISD::MUL:
4491     return LowerMUL(Op, DAG);
4492   case ISD::INTRINSIC_WO_CHAIN:
4493     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
4494   case ISD::STORE:
4495     return LowerSTORE(Op, DAG);
4496   case ISD::MGATHER:
4497     return LowerMGATHER(Op, DAG);
4498   case ISD::MSCATTER:
4499     return LowerMSCATTER(Op, DAG);
4500   case ISD::VECREDUCE_SEQ_FADD:
4501     return LowerVECREDUCE_SEQ_FADD(Op, DAG);
4502   case ISD::VECREDUCE_ADD:
4503   case ISD::VECREDUCE_AND:
4504   case ISD::VECREDUCE_OR:
4505   case ISD::VECREDUCE_XOR:
4506   case ISD::VECREDUCE_SMAX:
4507   case ISD::VECREDUCE_SMIN:
4508   case ISD::VECREDUCE_UMAX:
4509   case ISD::VECREDUCE_UMIN:
4510   case ISD::VECREDUCE_FADD:
4511   case ISD::VECREDUCE_FMAX:
4512   case ISD::VECREDUCE_FMIN:
4513     return LowerVECREDUCE(Op, DAG);
4514   case ISD::ATOMIC_LOAD_SUB:
4515     return LowerATOMIC_LOAD_SUB(Op, DAG);
4516   case ISD::ATOMIC_LOAD_AND:
4517     return LowerATOMIC_LOAD_AND(Op, DAG);
4518   case ISD::DYNAMIC_STACKALLOC:
4519     return LowerDYNAMIC_STACKALLOC(Op, DAG);
4520   case ISD::VSCALE:
4521     return LowerVSCALE(Op, DAG);
4522   case ISD::ANY_EXTEND:
4523   case ISD::SIGN_EXTEND:
4524   case ISD::ZERO_EXTEND:
4525     return LowerFixedLengthVectorIntExtendToSVE(Op, DAG);
4526   case ISD::SIGN_EXTEND_INREG: {
4527     // Only custom lower when ExtraVT has a legal byte based element type.
4528     EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
4529     EVT ExtraEltVT = ExtraVT.getVectorElementType();
4530     if ((ExtraEltVT != MVT::i8) && (ExtraEltVT != MVT::i16) &&
4531         (ExtraEltVT != MVT::i32) && (ExtraEltVT != MVT::i64))
4532       return SDValue();
4533 
4534     return LowerToPredicatedOp(Op, DAG,
4535                                AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU);
4536   }
4537   case ISD::TRUNCATE:
4538     return LowerTRUNCATE(Op, DAG);
4539   case ISD::LOAD:
4540     if (useSVEForFixedLengthVectorVT(Op.getValueType()))
4541       return LowerFixedLengthVectorLoadToSVE(Op, DAG);
4542     llvm_unreachable("Unexpected request to lower ISD::LOAD");
4543   case ISD::ADD:
4544     return LowerToPredicatedOp(Op, DAG, AArch64ISD::ADD_PRED);
4545   case ISD::AND:
4546     return LowerToScalableOp(Op, DAG);
4547   case ISD::SUB:
4548     return LowerToPredicatedOp(Op, DAG, AArch64ISD::SUB_PRED);
4549   case ISD::FMAXIMUM:
4550     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMAX_PRED);
4551   case ISD::FMAXNUM:
4552     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMAXNM_PRED);
4553   case ISD::FMINIMUM:
4554     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMIN_PRED);
4555   case ISD::FMINNUM:
4556     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMINNM_PRED);
4557   case ISD::VSELECT:
4558     return LowerFixedLengthVectorSelectToSVE(Op, DAG);
4559   case ISD::ABS:
4560     return LowerABS(Op, DAG);
4561   case ISD::BITREVERSE:
4562     return LowerToPredicatedOp(Op, DAG, AArch64ISD::BITREVERSE_MERGE_PASSTHRU,
4563                                /*OverrideNEON=*/true);
4564   case ISD::BSWAP:
4565     return LowerToPredicatedOp(Op, DAG, AArch64ISD::BSWAP_MERGE_PASSTHRU);
4566   case ISD::CTLZ:
4567     return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTLZ_MERGE_PASSTHRU,
4568                                /*OverrideNEON=*/true);
4569   case ISD::CTTZ:
4570     return LowerCTTZ(Op, DAG);
4571   }
4572 }
4573 
4574 bool AArch64TargetLowering::mergeStoresAfterLegalization(EVT VT) const {
4575   return !Subtarget->useSVEForFixedLengthVectors();
4576 }
4577 
4578 bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(
4579     EVT VT, bool OverrideNEON) const {
4580   if (!Subtarget->useSVEForFixedLengthVectors())
4581     return false;
4582 
4583   if (!VT.isFixedLengthVector())
4584     return false;
4585 
4586   // Don't use SVE for vectors we cannot scalarize if required.
4587   switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
4588   // Fixed length predicates should be promoted to i8.
4589   // NOTE: This is consistent with how NEON (and thus 64/128bit vectors) work.
4590   case MVT::i1:
4591   default:
4592     return false;
4593   case MVT::i8:
4594   case MVT::i16:
4595   case MVT::i32:
4596   case MVT::i64:
4597   case MVT::f16:
4598   case MVT::f32:
4599   case MVT::f64:
4600     break;
4601   }
4602 
4603   // All SVE implementations support NEON sized vectors.
4604   if (OverrideNEON && (VT.is128BitVector() || VT.is64BitVector()))
4605     return true;
4606 
4607   // Ensure NEON MVTs only belong to a single register class.
4608   if (VT.getFixedSizeInBits() <= 128)
4609     return false;
4610 
4611   // Don't use SVE for types that don't fit.
4612   if (VT.getFixedSizeInBits() > Subtarget->getMinSVEVectorSizeInBits())
4613     return false;
4614 
4615   // TODO: Perhaps an artificial restriction, but worth having whilst getting
4616   // the base fixed length SVE support in place.
4617   if (!VT.isPow2VectorType())
4618     return false;
4619 
4620   return true;
4621 }
4622 
4623 //===----------------------------------------------------------------------===//
4624 //                      Calling Convention Implementation
4625 //===----------------------------------------------------------------------===//
4626 
4627 /// Selects the correct CCAssignFn for a given CallingConvention value.
4628 CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
4629                                                      bool IsVarArg) const {
4630   switch (CC) {
4631   default:
4632     report_fatal_error("Unsupported calling convention.");
4633   case CallingConv::WebKit_JS:
4634     return CC_AArch64_WebKit_JS;
4635   case CallingConv::GHC:
4636     return CC_AArch64_GHC;
4637   case CallingConv::C:
4638   case CallingConv::Fast:
4639   case CallingConv::PreserveMost:
4640   case CallingConv::CXX_FAST_TLS:
4641   case CallingConv::Swift:
4642     if (Subtarget->isTargetWindows() && IsVarArg)
4643       return CC_AArch64_Win64_VarArg;
4644     if (!Subtarget->isTargetDarwin())
4645       return CC_AArch64_AAPCS;
4646     if (!IsVarArg)
4647       return CC_AArch64_DarwinPCS;
4648     return Subtarget->isTargetILP32() ? CC_AArch64_DarwinPCS_ILP32_VarArg
4649                                       : CC_AArch64_DarwinPCS_VarArg;
4650    case CallingConv::Win64:
4651     return IsVarArg ? CC_AArch64_Win64_VarArg : CC_AArch64_AAPCS;
4652    case CallingConv::CFGuard_Check:
4653      return CC_AArch64_Win64_CFGuard_Check;
4654    case CallingConv::AArch64_VectorCall:
4655    case CallingConv::AArch64_SVE_VectorCall:
4656      return CC_AArch64_AAPCS;
4657   }
4658 }
4659 
4660 CCAssignFn *
4661 AArch64TargetLowering::CCAssignFnForReturn(CallingConv::ID CC) const {
4662   return CC == CallingConv::WebKit_JS ? RetCC_AArch64_WebKit_JS
4663                                       : RetCC_AArch64_AAPCS;
4664 }
4665 
4666 SDValue AArch64TargetLowering::LowerFormalArguments(
4667     SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4668     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
4669     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4670   MachineFunction &MF = DAG.getMachineFunction();
4671   MachineFrameInfo &MFI = MF.getFrameInfo();
4672   bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
4673 
4674   // Assign locations to all of the incoming arguments.
4675   SmallVector<CCValAssign, 16> ArgLocs;
4676   DenseMap<unsigned, SDValue> CopiedRegs;
4677   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4678                  *DAG.getContext());
4679 
4680   // At this point, Ins[].VT may already be promoted to i32. To correctly
4681   // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
4682   // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
4683   // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, here
4684   // we use a special version of AnalyzeFormalArguments to pass in ValVT and
4685   // LocVT.
4686   unsigned NumArgs = Ins.size();
4687   Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
4688   unsigned CurArgIdx = 0;
4689   for (unsigned i = 0; i != NumArgs; ++i) {
4690     MVT ValVT = Ins[i].VT;
4691     if (Ins[i].isOrigArg()) {
4692       std::advance(CurOrigArg, Ins[i].getOrigArgIndex() - CurArgIdx);
4693       CurArgIdx = Ins[i].getOrigArgIndex();
4694 
4695       // Get type of the original argument.
4696       EVT ActualVT = getValueType(DAG.getDataLayout(), CurOrigArg->getType(),
4697                                   /*AllowUnknown*/ true);
4698       MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other;
4699       // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
4700       if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
4701         ValVT = MVT::i8;
4702       else if (ActualMVT == MVT::i16)
4703         ValVT = MVT::i16;
4704     }
4705     CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
4706     bool Res =
4707         AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, CCInfo);
4708     assert(!Res && "Call operand has unhandled type");
4709     (void)Res;
4710   }
4711   SmallVector<SDValue, 16> ArgValues;
4712   unsigned ExtraArgLocs = 0;
4713   for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
4714     CCValAssign &VA = ArgLocs[i - ExtraArgLocs];
4715 
4716     if (Ins[i].Flags.isByVal()) {
4717       // Byval is used for HFAs in the PCS, but the system should work in a
4718       // non-compliant manner for larger structs.
4719       EVT PtrVT = getPointerTy(DAG.getDataLayout());
4720       int Size = Ins[i].Flags.getByValSize();
4721       unsigned NumRegs = (Size + 7) / 8;
4722 
4723       // FIXME: This works on big-endian for composite byvals, which are the common
4724       // case. It should also work for fundamental types too.
4725       unsigned FrameIdx =
4726         MFI.CreateFixedObject(8 * NumRegs, VA.getLocMemOffset(), false);
4727       SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT);
4728       InVals.push_back(FrameIdxN);
4729 
4730       continue;
4731     }
4732 
4733     SDValue ArgValue;
4734     if (VA.isRegLoc()) {
4735       // Arguments stored in registers.
4736       EVT RegVT = VA.getLocVT();
4737       const TargetRegisterClass *RC;
4738 
4739       if (RegVT == MVT::i32)
4740         RC = &AArch64::GPR32RegClass;
4741       else if (RegVT == MVT::i64)
4742         RC = &AArch64::GPR64RegClass;
4743       else if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4744         RC = &AArch64::FPR16RegClass;
4745       else if (RegVT == MVT::f32)
4746         RC = &AArch64::FPR32RegClass;
4747       else if (RegVT == MVT::f64 || RegVT.is64BitVector())
4748         RC = &AArch64::FPR64RegClass;
4749       else if (RegVT == MVT::f128 || RegVT.is128BitVector())
4750         RC = &AArch64::FPR128RegClass;
4751       else if (RegVT.isScalableVector() &&
4752                RegVT.getVectorElementType() == MVT::i1)
4753         RC = &AArch64::PPRRegClass;
4754       else if (RegVT.isScalableVector())
4755         RC = &AArch64::ZPRRegClass;
4756       else
4757         llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
4758 
4759       // Transform the arguments in physical registers into virtual ones.
4760       unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
4761       ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
4762 
4763       // If this is an 8, 16 or 32-bit value, it is really passed promoted
4764       // to 64 bits.  Insert an assert[sz]ext to capture this, then
4765       // truncate to the right size.
4766       switch (VA.getLocInfo()) {
4767       default:
4768         llvm_unreachable("Unknown loc info!");
4769       case CCValAssign::Full:
4770         break;
4771       case CCValAssign::Indirect:
4772         assert(VA.getValVT().isScalableVector() &&
4773                "Only scalable vectors can be passed indirectly");
4774         break;
4775       case CCValAssign::BCvt:
4776         ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue);
4777         break;
4778       case CCValAssign::AExt:
4779       case CCValAssign::SExt:
4780       case CCValAssign::ZExt:
4781         break;
4782       case CCValAssign::AExtUpper:
4783         ArgValue = DAG.getNode(ISD::SRL, DL, RegVT, ArgValue,
4784                                DAG.getConstant(32, DL, RegVT));
4785         ArgValue = DAG.getZExtOrTrunc(ArgValue, DL, VA.getValVT());
4786         break;
4787       }
4788     } else { // VA.isRegLoc()
4789       assert(VA.isMemLoc() && "CCValAssign is neither reg nor mem");
4790       unsigned ArgOffset = VA.getLocMemOffset();
4791       unsigned ArgSize = (VA.getLocInfo() == CCValAssign::Indirect
4792                               ? VA.getLocVT().getSizeInBits()
4793                               : VA.getValVT().getSizeInBits()) / 8;
4794 
4795       uint32_t BEAlign = 0;
4796       if (!Subtarget->isLittleEndian() && ArgSize < 8 &&
4797           !Ins[i].Flags.isInConsecutiveRegs())
4798         BEAlign = 8 - ArgSize;
4799 
4800       int FI = MFI.CreateFixedObject(ArgSize, ArgOffset + BEAlign, true);
4801 
4802       // Create load nodes to retrieve arguments from the stack.
4803       SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4804 
4805       // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
4806       ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
4807       MVT MemVT = VA.getValVT();
4808 
4809       switch (VA.getLocInfo()) {
4810       default:
4811         break;
4812       case CCValAssign::Trunc:
4813       case CCValAssign::BCvt:
4814         MemVT = VA.getLocVT();
4815         break;
4816       case CCValAssign::Indirect:
4817         assert(VA.getValVT().isScalableVector() &&
4818                "Only scalable vectors can be passed indirectly");
4819         MemVT = VA.getLocVT();
4820         break;
4821       case CCValAssign::SExt:
4822         ExtType = ISD::SEXTLOAD;
4823         break;
4824       case CCValAssign::ZExt:
4825         ExtType = ISD::ZEXTLOAD;
4826         break;
4827       case CCValAssign::AExt:
4828         ExtType = ISD::EXTLOAD;
4829         break;
4830       }
4831 
4832       ArgValue = DAG.getExtLoad(
4833           ExtType, DL, VA.getLocVT(), Chain, FIN,
4834           MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
4835           MemVT);
4836 
4837     }
4838 
4839     if (VA.getLocInfo() == CCValAssign::Indirect) {
4840       assert(VA.getValVT().isScalableVector() &&
4841            "Only scalable vectors can be passed indirectly");
4842 
4843       uint64_t PartSize = VA.getValVT().getStoreSize().getKnownMinSize();
4844       unsigned NumParts = 1;
4845       if (Ins[i].Flags.isInConsecutiveRegs()) {
4846         assert(!Ins[i].Flags.isInConsecutiveRegsLast());
4847         while (!Ins[i + NumParts - 1].Flags.isInConsecutiveRegsLast())
4848           ++NumParts;
4849       }
4850 
4851       MVT PartLoad = VA.getValVT();
4852       SDValue Ptr = ArgValue;
4853 
4854       // Ensure we generate all loads for each tuple part, whilst updating the
4855       // pointer after each load correctly using vscale.
4856       while (NumParts > 0) {
4857         ArgValue = DAG.getLoad(PartLoad, DL, Chain, Ptr, MachinePointerInfo());
4858         InVals.push_back(ArgValue);
4859         NumParts--;
4860         if (NumParts > 0) {
4861           SDValue BytesIncrement = DAG.getVScale(
4862               DL, Ptr.getValueType(),
4863               APInt(Ptr.getValueSizeInBits().getFixedSize(), PartSize));
4864           SDNodeFlags Flags;
4865           Flags.setNoUnsignedWrap(true);
4866           Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
4867                             BytesIncrement, Flags);
4868           ExtraArgLocs++;
4869           i++;
4870         }
4871       }
4872     } else {
4873       if (Subtarget->isTargetILP32() && Ins[i].Flags.isPointer())
4874         ArgValue = DAG.getNode(ISD::AssertZext, DL, ArgValue.getValueType(),
4875                                ArgValue, DAG.getValueType(MVT::i32));
4876       InVals.push_back(ArgValue);
4877     }
4878   }
4879   assert((ArgLocs.size() + ExtraArgLocs) == Ins.size());
4880 
4881   // varargs
4882   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
4883   if (isVarArg) {
4884     if (!Subtarget->isTargetDarwin() || IsWin64) {
4885       // The AAPCS variadic function ABI is identical to the non-variadic
4886       // one. As a result there may be more arguments in registers and we should
4887       // save them for future reference.
4888       // Win64 variadic functions also pass arguments in registers, but all float
4889       // arguments are passed in integer registers.
4890       saveVarArgRegisters(CCInfo, DAG, DL, Chain);
4891     }
4892 
4893     // This will point to the next argument passed via stack.
4894     unsigned StackOffset = CCInfo.getNextStackOffset();
4895     // We currently pass all varargs at 8-byte alignment, or 4 for ILP32
4896     StackOffset = alignTo(StackOffset, Subtarget->isTargetILP32() ? 4 : 8);
4897     FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
4898 
4899     if (MFI.hasMustTailInVarArgFunc()) {
4900       SmallVector<MVT, 2> RegParmTypes;
4901       RegParmTypes.push_back(MVT::i64);
4902       RegParmTypes.push_back(MVT::f128);
4903       // Compute the set of forwarded registers. The rest are scratch.
4904       SmallVectorImpl<ForwardedRegister> &Forwards =
4905                                        FuncInfo->getForwardedMustTailRegParms();
4906       CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes,
4907                                                CC_AArch64_AAPCS);
4908 
4909       // Conservatively forward X8, since it might be used for aggregate return.
4910       if (!CCInfo.isAllocated(AArch64::X8)) {
4911         unsigned X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
4912         Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
4913       }
4914     }
4915   }
4916 
4917   // On Windows, InReg pointers must be returned, so record the pointer in a
4918   // virtual register at the start of the function so it can be returned in the
4919   // epilogue.
4920   if (IsWin64) {
4921     for (unsigned I = 0, E = Ins.size(); I != E; ++I) {
4922       if (Ins[I].Flags.isInReg()) {
4923         assert(!FuncInfo->getSRetReturnReg());
4924 
4925         MVT PtrTy = getPointerTy(DAG.getDataLayout());
4926         Register Reg =
4927             MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
4928         FuncInfo->setSRetReturnReg(Reg);
4929 
4930         SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[I]);
4931         Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Copy, Chain);
4932         break;
4933       }
4934     }
4935   }
4936 
4937   unsigned StackArgSize = CCInfo.getNextStackOffset();
4938   bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4939   if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) {
4940     // This is a non-standard ABI so by fiat I say we're allowed to make full
4941     // use of the stack area to be popped, which must be aligned to 16 bytes in
4942     // any case:
4943     StackArgSize = alignTo(StackArgSize, 16);
4944 
4945     // If we're expected to restore the stack (e.g. fastcc) then we'll be adding
4946     // a multiple of 16.
4947     FuncInfo->setArgumentStackToRestore(StackArgSize);
4948 
4949     // This realignment carries over to the available bytes below. Our own
4950     // callers will guarantee the space is free by giving an aligned value to
4951     // CALLSEQ_START.
4952   }
4953   // Even if we're not expected to free up the space, it's useful to know how
4954   // much is there while considering tail calls (because we can reuse it).
4955   FuncInfo->setBytesInStackArgArea(StackArgSize);
4956 
4957   if (Subtarget->hasCustomCallingConv())
4958     Subtarget->getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
4959 
4960   return Chain;
4961 }
4962 
4963 void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
4964                                                 SelectionDAG &DAG,
4965                                                 const SDLoc &DL,
4966                                                 SDValue &Chain) const {
4967   MachineFunction &MF = DAG.getMachineFunction();
4968   MachineFrameInfo &MFI = MF.getFrameInfo();
4969   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
4970   auto PtrVT = getPointerTy(DAG.getDataLayout());
4971   bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
4972 
4973   SmallVector<SDValue, 8> MemOps;
4974 
4975   static const MCPhysReg GPRArgRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2,
4976                                           AArch64::X3, AArch64::X4, AArch64::X5,
4977                                           AArch64::X6, AArch64::X7 };
4978   static const unsigned NumGPRArgRegs = array_lengthof(GPRArgRegs);
4979   unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(GPRArgRegs);
4980 
4981   unsigned GPRSaveSize = 8 * (NumGPRArgRegs - FirstVariadicGPR);
4982   int GPRIdx = 0;
4983   if (GPRSaveSize != 0) {
4984     if (IsWin64) {
4985       GPRIdx = MFI.CreateFixedObject(GPRSaveSize, -(int)GPRSaveSize, false);
4986       if (GPRSaveSize & 15)
4987         // The extra size here, if triggered, will always be 8.
4988         MFI.CreateFixedObject(16 - (GPRSaveSize & 15), -(int)alignTo(GPRSaveSize, 16), false);
4989     } else
4990       GPRIdx = MFI.CreateStackObject(GPRSaveSize, Align(8), false);
4991 
4992     SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT);
4993 
4994     for (unsigned i = FirstVariadicGPR; i < NumGPRArgRegs; ++i) {
4995       unsigned VReg = MF.addLiveIn(GPRArgRegs[i], &AArch64::GPR64RegClass);
4996       SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
4997       SDValue Store = DAG.getStore(
4998           Val.getValue(1), DL, Val, FIN,
4999           IsWin64
5000               ? MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
5001                                                   GPRIdx,
5002                                                   (i - FirstVariadicGPR) * 8)
5003               : MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 8));
5004       MemOps.push_back(Store);
5005       FIN =
5006           DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT));
5007     }
5008   }
5009   FuncInfo->setVarArgsGPRIndex(GPRIdx);
5010   FuncInfo->setVarArgsGPRSize(GPRSaveSize);
5011 
5012   if (Subtarget->hasFPARMv8() && !IsWin64) {
5013     static const MCPhysReg FPRArgRegs[] = {
5014         AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3,
5015         AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7};
5016     static const unsigned NumFPRArgRegs = array_lengthof(FPRArgRegs);
5017     unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(FPRArgRegs);
5018 
5019     unsigned FPRSaveSize = 16 * (NumFPRArgRegs - FirstVariadicFPR);
5020     int FPRIdx = 0;
5021     if (FPRSaveSize != 0) {
5022       FPRIdx = MFI.CreateStackObject(FPRSaveSize, Align(16), false);
5023 
5024       SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
5025 
5026       for (unsigned i = FirstVariadicFPR; i < NumFPRArgRegs; ++i) {
5027         unsigned VReg = MF.addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass);
5028         SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
5029 
5030         SDValue Store = DAG.getStore(
5031             Val.getValue(1), DL, Val, FIN,
5032             MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 16));
5033         MemOps.push_back(Store);
5034         FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
5035                           DAG.getConstant(16, DL, PtrVT));
5036       }
5037     }
5038     FuncInfo->setVarArgsFPRIndex(FPRIdx);
5039     FuncInfo->setVarArgsFPRSize(FPRSaveSize);
5040   }
5041 
5042   if (!MemOps.empty()) {
5043     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
5044   }
5045 }
5046 
5047 /// LowerCallResult - Lower the result values of a call into the
5048 /// appropriate copies out of appropriate physical registers.
5049 SDValue AArch64TargetLowering::LowerCallResult(
5050     SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
5051     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
5052     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
5053     SDValue ThisVal) const {
5054   CCAssignFn *RetCC = CCAssignFnForReturn(CallConv);
5055   // Assign locations to each value returned by this call.
5056   SmallVector<CCValAssign, 16> RVLocs;
5057   DenseMap<unsigned, SDValue> CopiedRegs;
5058   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
5059                  *DAG.getContext());
5060   CCInfo.AnalyzeCallResult(Ins, RetCC);
5061 
5062   // Copy all of the result registers out of their specified physreg.
5063   for (unsigned i = 0; i != RVLocs.size(); ++i) {
5064     CCValAssign VA = RVLocs[i];
5065 
5066     // Pass 'this' value directly from the argument to return value, to avoid
5067     // reg unit interference
5068     if (i == 0 && isThisReturn) {
5069       assert(!VA.needsCustom() && VA.getLocVT() == MVT::i64 &&
5070              "unexpected return calling convention register assignment");
5071       InVals.push_back(ThisVal);
5072       continue;
5073     }
5074 
5075     // Avoid copying a physreg twice since RegAllocFast is incompetent and only
5076     // allows one use of a physreg per block.
5077     SDValue Val = CopiedRegs.lookup(VA.getLocReg());
5078     if (!Val) {
5079       Val =
5080           DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag);
5081       Chain = Val.getValue(1);
5082       InFlag = Val.getValue(2);
5083       CopiedRegs[VA.getLocReg()] = Val;
5084     }
5085 
5086     switch (VA.getLocInfo()) {
5087     default:
5088       llvm_unreachable("Unknown loc info!");
5089     case CCValAssign::Full:
5090       break;
5091     case CCValAssign::BCvt:
5092       Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
5093       break;
5094     case CCValAssign::AExtUpper:
5095       Val = DAG.getNode(ISD::SRL, DL, VA.getLocVT(), Val,
5096                         DAG.getConstant(32, DL, VA.getLocVT()));
5097       LLVM_FALLTHROUGH;
5098     case CCValAssign::AExt:
5099       LLVM_FALLTHROUGH;
5100     case CCValAssign::ZExt:
5101       Val = DAG.getZExtOrTrunc(Val, DL, VA.getValVT());
5102       break;
5103     }
5104 
5105     InVals.push_back(Val);
5106   }
5107 
5108   return Chain;
5109 }
5110 
5111 /// Return true if the calling convention is one that we can guarantee TCO for.
5112 static bool canGuaranteeTCO(CallingConv::ID CC) {
5113   return CC == CallingConv::Fast;
5114 }
5115 
5116 /// Return true if we might ever do TCO for calls with this calling convention.
5117 static bool mayTailCallThisCC(CallingConv::ID CC) {
5118   switch (CC) {
5119   case CallingConv::C:
5120   case CallingConv::AArch64_SVE_VectorCall:
5121   case CallingConv::PreserveMost:
5122   case CallingConv::Swift:
5123     return true;
5124   default:
5125     return canGuaranteeTCO(CC);
5126   }
5127 }
5128 
5129 bool AArch64TargetLowering::isEligibleForTailCallOptimization(
5130     SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
5131     const SmallVectorImpl<ISD::OutputArg> &Outs,
5132     const SmallVectorImpl<SDValue> &OutVals,
5133     const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
5134   if (!mayTailCallThisCC(CalleeCC))
5135     return false;
5136 
5137   MachineFunction &MF = DAG.getMachineFunction();
5138   const Function &CallerF = MF.getFunction();
5139   CallingConv::ID CallerCC = CallerF.getCallingConv();
5140 
5141   // If this function uses the C calling convention but has an SVE signature,
5142   // then it preserves more registers and should assume the SVE_VectorCall CC.
5143   // The check for matching callee-saved regs will determine whether it is
5144   // eligible for TCO.
5145   if (CallerCC == CallingConv::C &&
5146       AArch64RegisterInfo::hasSVEArgsOrReturn(&MF))
5147     CallerCC = CallingConv::AArch64_SVE_VectorCall;
5148 
5149   bool CCMatch = CallerCC == CalleeCC;
5150 
5151   // When using the Windows calling convention on a non-windows OS, we want
5152   // to back up and restore X18 in such functions; we can't do a tail call
5153   // from those functions.
5154   if (CallerCC == CallingConv::Win64 && !Subtarget->isTargetWindows() &&
5155       CalleeCC != CallingConv::Win64)
5156     return false;
5157 
5158   // Byval parameters hand the function a pointer directly into the stack area
5159   // we want to reuse during a tail call. Working around this *is* possible (see
5160   // X86) but less efficient and uglier in LowerCall.
5161   for (Function::const_arg_iterator i = CallerF.arg_begin(),
5162                                     e = CallerF.arg_end();
5163        i != e; ++i) {
5164     if (i->hasByValAttr())
5165       return false;
5166 
5167     // On Windows, "inreg" attributes signify non-aggregate indirect returns.
5168     // In this case, it is necessary to save/restore X0 in the callee. Tail
5169     // call opt interferes with this. So we disable tail call opt when the
5170     // caller has an argument with "inreg" attribute.
5171 
5172     // FIXME: Check whether the callee also has an "inreg" argument.
5173     if (i->hasInRegAttr())
5174       return false;
5175   }
5176 
5177   if (getTargetMachine().Options.GuaranteedTailCallOpt)
5178     return canGuaranteeTCO(CalleeCC) && CCMatch;
5179 
5180   // Externally-defined functions with weak linkage should not be
5181   // tail-called on AArch64 when the OS does not support dynamic
5182   // pre-emption of symbols, as the AAELF spec requires normal calls
5183   // to undefined weak functions to be replaced with a NOP or jump to the
5184   // next instruction. The behaviour of branch instructions in this
5185   // situation (as used for tail calls) is implementation-defined, so we
5186   // cannot rely on the linker replacing the tail call with a return.
5187   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
5188     const GlobalValue *GV = G->getGlobal();
5189     const Triple &TT = getTargetMachine().getTargetTriple();
5190     if (GV->hasExternalWeakLinkage() &&
5191         (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
5192       return false;
5193   }
5194 
5195   // Now we search for cases where we can use a tail call without changing the
5196   // ABI. Sibcall is used in some places (particularly gcc) to refer to this
5197   // concept.
5198 
5199   // I want anyone implementing a new calling convention to think long and hard
5200   // about this assert.
5201   assert((!isVarArg || CalleeCC == CallingConv::C) &&
5202          "Unexpected variadic calling convention");
5203 
5204   LLVMContext &C = *DAG.getContext();
5205   if (isVarArg && !Outs.empty()) {
5206     // At least two cases here: if caller is fastcc then we can't have any
5207     // memory arguments (we'd be expected to clean up the stack afterwards). If
5208     // caller is C then we could potentially use its argument area.
5209 
5210     // FIXME: for now we take the most conservative of these in both cases:
5211     // disallow all variadic memory operands.
5212     SmallVector<CCValAssign, 16> ArgLocs;
5213     CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
5214 
5215     CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, true));
5216     for (const CCValAssign &ArgLoc : ArgLocs)
5217       if (!ArgLoc.isRegLoc())
5218         return false;
5219   }
5220 
5221   // Check that the call results are passed in the same way.
5222   if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
5223                                   CCAssignFnForCall(CalleeCC, isVarArg),
5224                                   CCAssignFnForCall(CallerCC, isVarArg)))
5225     return false;
5226   // The callee has to preserve all registers the caller needs to preserve.
5227   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
5228   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
5229   if (!CCMatch) {
5230     const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
5231     if (Subtarget->hasCustomCallingConv()) {
5232       TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
5233       TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
5234     }
5235     if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
5236       return false;
5237   }
5238 
5239   // Nothing more to check if the callee is taking no arguments
5240   if (Outs.empty())
5241     return true;
5242 
5243   SmallVector<CCValAssign, 16> ArgLocs;
5244   CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
5245 
5246   CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg));
5247 
5248   const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
5249 
5250   // If any of the arguments is passed indirectly, it must be SVE, so the
5251   // 'getBytesInStackArgArea' is not sufficient to determine whether we need to
5252   // allocate space on the stack. That is why we determine this explicitly here
5253   // the call cannot be a tailcall.
5254   if (llvm::any_of(ArgLocs, [](CCValAssign &A) {
5255         assert((A.getLocInfo() != CCValAssign::Indirect ||
5256                 A.getValVT().isScalableVector()) &&
5257                "Expected value to be scalable");
5258         return A.getLocInfo() == CCValAssign::Indirect;
5259       }))
5260     return false;
5261 
5262   // If the stack arguments for this call do not fit into our own save area then
5263   // the call cannot be made tail.
5264   if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea())
5265     return false;
5266 
5267   const MachineRegisterInfo &MRI = MF.getRegInfo();
5268   if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
5269     return false;
5270 
5271   return true;
5272 }
5273 
5274 SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
5275                                                    SelectionDAG &DAG,
5276                                                    MachineFrameInfo &MFI,
5277                                                    int ClobberedFI) const {
5278   SmallVector<SDValue, 8> ArgChains;
5279   int64_t FirstByte = MFI.getObjectOffset(ClobberedFI);
5280   int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1;
5281 
5282   // Include the original chain at the beginning of the list. When this is
5283   // used by target LowerCall hooks, this helps legalize find the
5284   // CALLSEQ_BEGIN node.
5285   ArgChains.push_back(Chain);
5286 
5287   // Add a chain value for each stack argument corresponding
5288   for (SDNode::use_iterator U = DAG.getEntryNode().getNode()->use_begin(),
5289                             UE = DAG.getEntryNode().getNode()->use_end();
5290        U != UE; ++U)
5291     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
5292       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
5293         if (FI->getIndex() < 0) {
5294           int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex());
5295           int64_t InLastByte = InFirstByte;
5296           InLastByte += MFI.getObjectSize(FI->getIndex()) - 1;
5297 
5298           if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
5299               (FirstByte <= InFirstByte && InFirstByte <= LastByte))
5300             ArgChains.push_back(SDValue(L, 1));
5301         }
5302 
5303   // Build a tokenfactor for all the chains.
5304   return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
5305 }
5306 
5307 bool AArch64TargetLowering::DoesCalleeRestoreStack(CallingConv::ID CallCC,
5308                                                    bool TailCallOpt) const {
5309   return CallCC == CallingConv::Fast && TailCallOpt;
5310 }
5311 
5312 /// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain,
5313 /// and add input and output parameter nodes.
5314 SDValue
5315 AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
5316                                  SmallVectorImpl<SDValue> &InVals) const {
5317   SelectionDAG &DAG = CLI.DAG;
5318   SDLoc &DL = CLI.DL;
5319   SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
5320   SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
5321   SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
5322   SDValue Chain = CLI.Chain;
5323   SDValue Callee = CLI.Callee;
5324   bool &IsTailCall = CLI.IsTailCall;
5325   CallingConv::ID CallConv = CLI.CallConv;
5326   bool IsVarArg = CLI.IsVarArg;
5327 
5328   MachineFunction &MF = DAG.getMachineFunction();
5329   MachineFunction::CallSiteInfo CSInfo;
5330   bool IsThisReturn = false;
5331 
5332   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
5333   bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
5334   bool IsSibCall = false;
5335 
5336   // Check callee args/returns for SVE registers and set calling convention
5337   // accordingly.
5338   if (CallConv == CallingConv::C) {
5339     bool CalleeOutSVE = any_of(Outs, [](ISD::OutputArg &Out){
5340       return Out.VT.isScalableVector();
5341     });
5342     bool CalleeInSVE = any_of(Ins, [](ISD::InputArg &In){
5343       return In.VT.isScalableVector();
5344     });
5345 
5346     if (CalleeInSVE || CalleeOutSVE)
5347       CallConv = CallingConv::AArch64_SVE_VectorCall;
5348   }
5349 
5350   if (IsTailCall) {
5351     // Check if it's really possible to do a tail call.
5352     IsTailCall = isEligibleForTailCallOptimization(
5353         Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
5354     if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall())
5355       report_fatal_error("failed to perform tail call elimination on a call "
5356                          "site marked musttail");
5357 
5358     // A sibling call is one where we're under the usual C ABI and not planning
5359     // to change that but can still do a tail call:
5360     if (!TailCallOpt && IsTailCall)
5361       IsSibCall = true;
5362 
5363     if (IsTailCall)
5364       ++NumTailCalls;
5365   }
5366 
5367   // Analyze operands of the call, assigning locations to each operand.
5368   SmallVector<CCValAssign, 16> ArgLocs;
5369   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs,
5370                  *DAG.getContext());
5371 
5372   if (IsVarArg) {
5373     // Handle fixed and variable vector arguments differently.
5374     // Variable vector arguments always go into memory.
5375     unsigned NumArgs = Outs.size();
5376 
5377     for (unsigned i = 0; i != NumArgs; ++i) {
5378       MVT ArgVT = Outs[i].VT;
5379       if (!Outs[i].IsFixed && ArgVT.isScalableVector())
5380         report_fatal_error("Passing SVE types to variadic functions is "
5381                            "currently not supported");
5382 
5383       ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
5384       CCAssignFn *AssignFn = CCAssignFnForCall(CallConv,
5385                                                /*IsVarArg=*/ !Outs[i].IsFixed);
5386       bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
5387       assert(!Res && "Call operand has unhandled type");
5388       (void)Res;
5389     }
5390   } else {
5391     // At this point, Outs[].VT may already be promoted to i32. To correctly
5392     // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
5393     // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
5394     // Since AnalyzeCallOperands uses Ins[].VT for both ValVT and LocVT, here
5395     // we use a special version of AnalyzeCallOperands to pass in ValVT and
5396     // LocVT.
5397     unsigned NumArgs = Outs.size();
5398     for (unsigned i = 0; i != NumArgs; ++i) {
5399       MVT ValVT = Outs[i].VT;
5400       // Get type of the original argument.
5401       EVT ActualVT = getValueType(DAG.getDataLayout(),
5402                                   CLI.getArgs()[Outs[i].OrigArgIndex].Ty,
5403                                   /*AllowUnknown*/ true);
5404       MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT;
5405       ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
5406       // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
5407       if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
5408         ValVT = MVT::i8;
5409       else if (ActualMVT == MVT::i16)
5410         ValVT = MVT::i16;
5411 
5412       CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
5413       bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, ArgFlags, CCInfo);
5414       assert(!Res && "Call operand has unhandled type");
5415       (void)Res;
5416     }
5417   }
5418 
5419   // Get a count of how many bytes are to be pushed on the stack.
5420   unsigned NumBytes = CCInfo.getNextStackOffset();
5421 
5422   if (IsSibCall) {
5423     // Since we're not changing the ABI to make this a tail call, the memory
5424     // operands are already available in the caller's incoming argument space.
5425     NumBytes = 0;
5426   }
5427 
5428   // FPDiff is the byte offset of the call's argument area from the callee's.
5429   // Stores to callee stack arguments will be placed in FixedStackSlots offset
5430   // by this amount for a tail call. In a sibling call it must be 0 because the
5431   // caller will deallocate the entire stack and the callee still expects its
5432   // arguments to begin at SP+0. Completely unused for non-tail calls.
5433   int FPDiff = 0;
5434 
5435   if (IsTailCall && !IsSibCall) {
5436     unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
5437 
5438     // Since callee will pop argument stack as a tail call, we must keep the
5439     // popped size 16-byte aligned.
5440     NumBytes = alignTo(NumBytes, 16);
5441 
5442     // FPDiff will be negative if this tail call requires more space than we
5443     // would automatically have in our incoming argument space. Positive if we
5444     // can actually shrink the stack.
5445     FPDiff = NumReusableBytes - NumBytes;
5446 
5447     // The stack pointer must be 16-byte aligned at all times it's used for a
5448     // memory operation, which in practice means at *all* times and in
5449     // particular across call boundaries. Therefore our own arguments started at
5450     // a 16-byte aligned SP and the delta applied for the tail call should
5451     // satisfy the same constraint.
5452     assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
5453   }
5454 
5455   // Adjust the stack pointer for the new arguments...
5456   // These operations are automatically eliminated by the prolog/epilog pass
5457   if (!IsSibCall)
5458     Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
5459 
5460   SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
5461                                         getPointerTy(DAG.getDataLayout()));
5462 
5463   SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
5464   SmallSet<unsigned, 8> RegsUsed;
5465   SmallVector<SDValue, 8> MemOpChains;
5466   auto PtrVT = getPointerTy(DAG.getDataLayout());
5467 
5468   if (IsVarArg && CLI.CB && CLI.CB->isMustTailCall()) {
5469     const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
5470     for (const auto &F : Forwards) {
5471       SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT);
5472        RegsToPass.emplace_back(F.PReg, Val);
5473     }
5474   }
5475 
5476   // Walk the register/memloc assignments, inserting copies/loads.
5477   unsigned ExtraArgLocs = 0;
5478   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
5479     CCValAssign &VA = ArgLocs[i - ExtraArgLocs];
5480     SDValue Arg = OutVals[i];
5481     ISD::ArgFlagsTy Flags = Outs[i].Flags;
5482 
5483     // Promote the value if needed.
5484     switch (VA.getLocInfo()) {
5485     default:
5486       llvm_unreachable("Unknown loc info!");
5487     case CCValAssign::Full:
5488       break;
5489     case CCValAssign::SExt:
5490       Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
5491       break;
5492     case CCValAssign::ZExt:
5493       Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
5494       break;
5495     case CCValAssign::AExt:
5496       if (Outs[i].ArgVT == MVT::i1) {
5497         // AAPCS requires i1 to be zero-extended to 8-bits by the caller.
5498         Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
5499         Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg);
5500       }
5501       Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
5502       break;
5503     case CCValAssign::AExtUpper:
5504       assert(VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits");
5505       Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
5506       Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg,
5507                         DAG.getConstant(32, DL, VA.getLocVT()));
5508       break;
5509     case CCValAssign::BCvt:
5510       Arg = DAG.getBitcast(VA.getLocVT(), Arg);
5511       break;
5512     case CCValAssign::Trunc:
5513       Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
5514       break;
5515     case CCValAssign::FPExt:
5516       Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
5517       break;
5518     case CCValAssign::Indirect:
5519       assert(VA.getValVT().isScalableVector() &&
5520              "Only scalable vectors can be passed indirectly");
5521 
5522       uint64_t StoreSize = VA.getValVT().getStoreSize().getKnownMinSize();
5523       uint64_t PartSize = StoreSize;
5524       unsigned NumParts = 1;
5525       if (Outs[i].Flags.isInConsecutiveRegs()) {
5526         assert(!Outs[i].Flags.isInConsecutiveRegsLast());
5527         while (!Outs[i + NumParts - 1].Flags.isInConsecutiveRegsLast())
5528           ++NumParts;
5529         StoreSize *= NumParts;
5530       }
5531 
5532       MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5533       Type *Ty = EVT(VA.getValVT()).getTypeForEVT(*DAG.getContext());
5534       Align Alignment = DAG.getDataLayout().getPrefTypeAlign(Ty);
5535       int FI = MFI.CreateStackObject(StoreSize, Alignment, false);
5536       MFI.setStackID(FI, TargetStackID::ScalableVector);
5537 
5538       MachinePointerInfo MPI =
5539           MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
5540       SDValue Ptr = DAG.getFrameIndex(
5541           FI, DAG.getTargetLoweringInfo().getFrameIndexTy(DAG.getDataLayout()));
5542       SDValue SpillSlot = Ptr;
5543 
5544       // Ensure we generate all stores for each tuple part, whilst updating the
5545       // pointer after each store correctly using vscale.
5546       while (NumParts) {
5547         Chain = DAG.getStore(Chain, DL, OutVals[i], Ptr, MPI);
5548         NumParts--;
5549         if (NumParts > 0) {
5550           SDValue BytesIncrement = DAG.getVScale(
5551               DL, Ptr.getValueType(),
5552               APInt(Ptr.getValueSizeInBits().getFixedSize(), PartSize));
5553           SDNodeFlags Flags;
5554           Flags.setNoUnsignedWrap(true);
5555 
5556           MPI = MachinePointerInfo(MPI.getAddrSpace());
5557           Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
5558                             BytesIncrement, Flags);
5559           ExtraArgLocs++;
5560           i++;
5561         }
5562       }
5563 
5564       Arg = SpillSlot;
5565       break;
5566     }
5567 
5568     if (VA.isRegLoc()) {
5569       if (i == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
5570           Outs[0].VT == MVT::i64) {
5571         assert(VA.getLocVT() == MVT::i64 &&
5572                "unexpected calling convention register assignment");
5573         assert(!Ins.empty() && Ins[0].VT == MVT::i64 &&
5574                "unexpected use of 'returned'");
5575         IsThisReturn = true;
5576       }
5577       if (RegsUsed.count(VA.getLocReg())) {
5578         // If this register has already been used then we're trying to pack
5579         // parts of an [N x i32] into an X-register. The extension type will
5580         // take care of putting the two halves in the right place but we have to
5581         // combine them.
5582         SDValue &Bits =
5583             llvm::find_if(RegsToPass,
5584                           [=](const std::pair<unsigned, SDValue> &Elt) {
5585                             return Elt.first == VA.getLocReg();
5586                           })
5587                 ->second;
5588         Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg);
5589         // Call site info is used for function's parameter entry value
5590         // tracking. For now we track only simple cases when parameter
5591         // is transferred through whole register.
5592         llvm::erase_if(CSInfo, [&VA](MachineFunction::ArgRegPair ArgReg) {
5593           return ArgReg.Reg == VA.getLocReg();
5594         });
5595       } else {
5596         RegsToPass.emplace_back(VA.getLocReg(), Arg);
5597         RegsUsed.insert(VA.getLocReg());
5598         const TargetOptions &Options = DAG.getTarget().Options;
5599         if (Options.EmitCallSiteInfo)
5600           CSInfo.emplace_back(VA.getLocReg(), i);
5601       }
5602     } else {
5603       assert(VA.isMemLoc());
5604 
5605       SDValue DstAddr;
5606       MachinePointerInfo DstInfo;
5607 
5608       // FIXME: This works on big-endian for composite byvals, which are the
5609       // common case. It should also work for fundamental types too.
5610       uint32_t BEAlign = 0;
5611       unsigned OpSize;
5612       if (VA.getLocInfo() == CCValAssign::Indirect)
5613         OpSize = VA.getLocVT().getFixedSizeInBits();
5614       else
5615         OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
5616                                  : VA.getValVT().getSizeInBits();
5617       OpSize = (OpSize + 7) / 8;
5618       if (!Subtarget->isLittleEndian() && !Flags.isByVal() &&
5619           !Flags.isInConsecutiveRegs()) {
5620         if (OpSize < 8)
5621           BEAlign = 8 - OpSize;
5622       }
5623       unsigned LocMemOffset = VA.getLocMemOffset();
5624       int32_t Offset = LocMemOffset + BEAlign;
5625       SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
5626       PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
5627 
5628       if (IsTailCall) {
5629         Offset = Offset + FPDiff;
5630         int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
5631 
5632         DstAddr = DAG.getFrameIndex(FI, PtrVT);
5633         DstInfo =
5634             MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
5635 
5636         // Make sure any stack arguments overlapping with where we're storing
5637         // are loaded before this eventual operation. Otherwise they'll be
5638         // clobbered.
5639         Chain = addTokenForArgument(Chain, DAG, MF.getFrameInfo(), FI);
5640       } else {
5641         SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
5642 
5643         DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
5644         DstInfo = MachinePointerInfo::getStack(DAG.getMachineFunction(),
5645                                                LocMemOffset);
5646       }
5647 
5648       if (Outs[i].Flags.isByVal()) {
5649         SDValue SizeNode =
5650             DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i64);
5651         SDValue Cpy = DAG.getMemcpy(
5652             Chain, DL, DstAddr, Arg, SizeNode,
5653             Outs[i].Flags.getNonZeroByValAlign(),
5654             /*isVol = */ false, /*AlwaysInline = */ false,
5655             /*isTailCall = */ false, DstInfo, MachinePointerInfo());
5656 
5657         MemOpChains.push_back(Cpy);
5658       } else {
5659         // Since we pass i1/i8/i16 as i1/i8/i16 on stack and Arg is already
5660         // promoted to a legal register type i32, we should truncate Arg back to
5661         // i1/i8/i16.
5662         if (VA.getValVT() == MVT::i1 || VA.getValVT() == MVT::i8 ||
5663             VA.getValVT() == MVT::i16)
5664           Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg);
5665 
5666         SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo);
5667         MemOpChains.push_back(Store);
5668       }
5669     }
5670   }
5671 
5672   if (!MemOpChains.empty())
5673     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
5674 
5675   // Build a sequence of copy-to-reg nodes chained together with token chain
5676   // and flag operands which copy the outgoing args into the appropriate regs.
5677   SDValue InFlag;
5678   for (auto &RegToPass : RegsToPass) {
5679     Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
5680                              RegToPass.second, InFlag);
5681     InFlag = Chain.getValue(1);
5682   }
5683 
5684   // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
5685   // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
5686   // node so that legalize doesn't hack it.
5687   if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
5688     auto GV = G->getGlobal();
5689     unsigned OpFlags =
5690         Subtarget->classifyGlobalFunctionReference(GV, getTargetMachine());
5691     if (OpFlags & AArch64II::MO_GOT) {
5692       Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
5693       Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
5694     } else {
5695       const GlobalValue *GV = G->getGlobal();
5696       Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0);
5697     }
5698   } else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
5699     if (getTargetMachine().getCodeModel() == CodeModel::Large &&
5700         Subtarget->isTargetMachO()) {
5701       const char *Sym = S->getSymbol();
5702       Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, AArch64II::MO_GOT);
5703       Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
5704     } else {
5705       const char *Sym = S->getSymbol();
5706       Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, 0);
5707     }
5708   }
5709 
5710   // We don't usually want to end the call-sequence here because we would tidy
5711   // the frame up *after* the call, however in the ABI-changing tail-call case
5712   // we've carefully laid out the parameters so that when sp is reset they'll be
5713   // in the correct location.
5714   if (IsTailCall && !IsSibCall) {
5715     Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
5716                                DAG.getIntPtrConstant(0, DL, true), InFlag, DL);
5717     InFlag = Chain.getValue(1);
5718   }
5719 
5720   std::vector<SDValue> Ops;
5721   Ops.push_back(Chain);
5722   Ops.push_back(Callee);
5723 
5724   if (IsTailCall) {
5725     // Each tail call may have to adjust the stack by a different amount, so
5726     // this information must travel along with the operation for eventual
5727     // consumption by emitEpilogue.
5728     Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
5729   }
5730 
5731   // Add argument registers to the end of the list so that they are known live
5732   // into the call.
5733   for (auto &RegToPass : RegsToPass)
5734     Ops.push_back(DAG.getRegister(RegToPass.first,
5735                                   RegToPass.second.getValueType()));
5736 
5737   // Add a register mask operand representing the call-preserved registers.
5738   const uint32_t *Mask;
5739   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
5740   if (IsThisReturn) {
5741     // For 'this' returns, use the X0-preserving mask if applicable
5742     Mask = TRI->getThisReturnPreservedMask(MF, CallConv);
5743     if (!Mask) {
5744       IsThisReturn = false;
5745       Mask = TRI->getCallPreservedMask(MF, CallConv);
5746     }
5747   } else
5748     Mask = TRI->getCallPreservedMask(MF, CallConv);
5749 
5750   if (Subtarget->hasCustomCallingConv())
5751     TRI->UpdateCustomCallPreservedMask(MF, &Mask);
5752 
5753   if (TRI->isAnyArgRegReserved(MF))
5754     TRI->emitReservedArgRegCallError(MF);
5755 
5756   assert(Mask && "Missing call preserved mask for calling convention");
5757   Ops.push_back(DAG.getRegisterMask(Mask));
5758 
5759   if (InFlag.getNode())
5760     Ops.push_back(InFlag);
5761 
5762   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
5763 
5764   // If we're doing a tall call, use a TC_RETURN here rather than an
5765   // actual call instruction.
5766   if (IsTailCall) {
5767     MF.getFrameInfo().setHasTailCall();
5768     SDValue Ret = DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
5769     DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
5770     return Ret;
5771   }
5772 
5773   unsigned CallOpc = AArch64ISD::CALL;
5774   // Calls with operand bundle "clang.arc.attachedcall" are special. They should
5775   // be expanded to the call, directly followed by a special marker sequence.
5776   // Use the CALL_RVMARKER to do that.
5777   if (CLI.CB && objcarc::hasAttachedCallOpBundle(CLI.CB)) {
5778     assert(!IsTailCall &&
5779            "tail calls cannot be marked with clang.arc.attachedcall");
5780     CallOpc = AArch64ISD::CALL_RVMARKER;
5781   }
5782 
5783   // Returns a chain and a flag for retval copy to use.
5784   Chain = DAG.getNode(CallOpc, DL, NodeTys, Ops);
5785   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
5786   InFlag = Chain.getValue(1);
5787   DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
5788 
5789   uint64_t CalleePopBytes =
5790       DoesCalleeRestoreStack(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : 0;
5791 
5792   Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
5793                              DAG.getIntPtrConstant(CalleePopBytes, DL, true),
5794                              InFlag, DL);
5795   if (!Ins.empty())
5796     InFlag = Chain.getValue(1);
5797 
5798   // Handle result values, copying them out of physregs into vregs that we
5799   // return.
5800   return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
5801                          InVals, IsThisReturn,
5802                          IsThisReturn ? OutVals[0] : SDValue());
5803 }
5804 
5805 bool AArch64TargetLowering::CanLowerReturn(
5806     CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
5807     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
5808   CCAssignFn *RetCC = CCAssignFnForReturn(CallConv);
5809   SmallVector<CCValAssign, 16> RVLocs;
5810   CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
5811   return CCInfo.CheckReturn(Outs, RetCC);
5812 }
5813 
5814 SDValue
5815 AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
5816                                    bool isVarArg,
5817                                    const SmallVectorImpl<ISD::OutputArg> &Outs,
5818                                    const SmallVectorImpl<SDValue> &OutVals,
5819                                    const SDLoc &DL, SelectionDAG &DAG) const {
5820   auto &MF = DAG.getMachineFunction();
5821   auto *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
5822 
5823   CCAssignFn *RetCC = CCAssignFnForReturn(CallConv);
5824   SmallVector<CCValAssign, 16> RVLocs;
5825   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
5826                  *DAG.getContext());
5827   CCInfo.AnalyzeReturn(Outs, RetCC);
5828 
5829   // Copy the result values into the output registers.
5830   SDValue Flag;
5831   SmallVector<std::pair<unsigned, SDValue>, 4> RetVals;
5832   SmallSet<unsigned, 4> RegsUsed;
5833   for (unsigned i = 0, realRVLocIdx = 0; i != RVLocs.size();
5834        ++i, ++realRVLocIdx) {
5835     CCValAssign &VA = RVLocs[i];
5836     assert(VA.isRegLoc() && "Can only return in registers!");
5837     SDValue Arg = OutVals[realRVLocIdx];
5838 
5839     switch (VA.getLocInfo()) {
5840     default:
5841       llvm_unreachable("Unknown loc info!");
5842     case CCValAssign::Full:
5843       if (Outs[i].ArgVT == MVT::i1) {
5844         // AAPCS requires i1 to be zero-extended to i8 by the producer of the
5845         // value. This is strictly redundant on Darwin (which uses "zeroext
5846         // i1"), but will be optimised out before ISel.
5847         Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
5848         Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
5849       }
5850       break;
5851     case CCValAssign::BCvt:
5852       Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
5853       break;
5854     case CCValAssign::AExt:
5855     case CCValAssign::ZExt:
5856       Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
5857       break;
5858     case CCValAssign::AExtUpper:
5859       assert(VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits");
5860       Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
5861       Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg,
5862                         DAG.getConstant(32, DL, VA.getLocVT()));
5863       break;
5864     }
5865 
5866     if (RegsUsed.count(VA.getLocReg())) {
5867       SDValue &Bits =
5868           llvm::find_if(RetVals, [=](const std::pair<unsigned, SDValue> &Elt) {
5869             return Elt.first == VA.getLocReg();
5870           })->second;
5871       Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg);
5872     } else {
5873       RetVals.emplace_back(VA.getLocReg(), Arg);
5874       RegsUsed.insert(VA.getLocReg());
5875     }
5876   }
5877 
5878   SmallVector<SDValue, 4> RetOps(1, Chain);
5879   for (auto &RetVal : RetVals) {
5880     Chain = DAG.getCopyToReg(Chain, DL, RetVal.first, RetVal.second, Flag);
5881     Flag = Chain.getValue(1);
5882     RetOps.push_back(
5883         DAG.getRegister(RetVal.first, RetVal.second.getValueType()));
5884   }
5885 
5886   // Windows AArch64 ABIs require that for returning structs by value we copy
5887   // the sret argument into X0 for the return.
5888   // We saved the argument into a virtual register in the entry block,
5889   // so now we copy the value out and into X0.
5890   if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) {
5891     SDValue Val = DAG.getCopyFromReg(RetOps[0], DL, SRetReg,
5892                                      getPointerTy(MF.getDataLayout()));
5893 
5894     unsigned RetValReg = AArch64::X0;
5895     Chain = DAG.getCopyToReg(Chain, DL, RetValReg, Val, Flag);
5896     Flag = Chain.getValue(1);
5897 
5898     RetOps.push_back(
5899       DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout())));
5900   }
5901 
5902   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
5903   const MCPhysReg *I =
5904       TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
5905   if (I) {
5906     for (; *I; ++I) {
5907       if (AArch64::GPR64RegClass.contains(*I))
5908         RetOps.push_back(DAG.getRegister(*I, MVT::i64));
5909       else if (AArch64::FPR64RegClass.contains(*I))
5910         RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64)));
5911       else
5912         llvm_unreachable("Unexpected register class in CSRsViaCopy!");
5913     }
5914   }
5915 
5916   RetOps[0] = Chain; // Update chain.
5917 
5918   // Add the flag if we have it.
5919   if (Flag.getNode())
5920     RetOps.push_back(Flag);
5921 
5922   return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps);
5923 }
5924 
5925 //===----------------------------------------------------------------------===//
5926 //  Other Lowering Code
5927 //===----------------------------------------------------------------------===//
5928 
5929 SDValue AArch64TargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
5930                                              SelectionDAG &DAG,
5931                                              unsigned Flag) const {
5932   return DAG.getTargetGlobalAddress(N->getGlobal(), SDLoc(N), Ty,
5933                                     N->getOffset(), Flag);
5934 }
5935 
5936 SDValue AArch64TargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
5937                                              SelectionDAG &DAG,
5938                                              unsigned Flag) const {
5939   return DAG.getTargetJumpTable(N->getIndex(), Ty, Flag);
5940 }
5941 
5942 SDValue AArch64TargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
5943                                              SelectionDAG &DAG,
5944                                              unsigned Flag) const {
5945   return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
5946                                    N->getOffset(), Flag);
5947 }
5948 
5949 SDValue AArch64TargetLowering::getTargetNode(BlockAddressSDNode* N, EVT Ty,
5950                                              SelectionDAG &DAG,
5951                                              unsigned Flag) const {
5952   return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, 0, Flag);
5953 }
5954 
5955 // (loadGOT sym)
5956 template <class NodeTy>
5957 SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG,
5958                                       unsigned Flags) const {
5959   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getGOT\n");
5960   SDLoc DL(N);
5961   EVT Ty = getPointerTy(DAG.getDataLayout());
5962   SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT | Flags);
5963   // FIXME: Once remat is capable of dealing with instructions with register
5964   // operands, expand this into two nodes instead of using a wrapper node.
5965   return DAG.getNode(AArch64ISD::LOADgot, DL, Ty, GotAddr);
5966 }
5967 
5968 // (wrapper %highest(sym), %higher(sym), %hi(sym), %lo(sym))
5969 template <class NodeTy>
5970 SDValue AArch64TargetLowering::getAddrLarge(NodeTy *N, SelectionDAG &DAG,
5971                                             unsigned Flags) const {
5972   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddrLarge\n");
5973   SDLoc DL(N);
5974   EVT Ty = getPointerTy(DAG.getDataLayout());
5975   const unsigned char MO_NC = AArch64II::MO_NC;
5976   return DAG.getNode(
5977       AArch64ISD::WrapperLarge, DL, Ty,
5978       getTargetNode(N, Ty, DAG, AArch64II::MO_G3 | Flags),
5979       getTargetNode(N, Ty, DAG, AArch64II::MO_G2 | MO_NC | Flags),
5980       getTargetNode(N, Ty, DAG, AArch64II::MO_G1 | MO_NC | Flags),
5981       getTargetNode(N, Ty, DAG, AArch64II::MO_G0 | MO_NC | Flags));
5982 }
5983 
5984 // (addlow (adrp %hi(sym)) %lo(sym))
5985 template <class NodeTy>
5986 SDValue AArch64TargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
5987                                        unsigned Flags) const {
5988   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddr\n");
5989   SDLoc DL(N);
5990   EVT Ty = getPointerTy(DAG.getDataLayout());
5991   SDValue Hi = getTargetNode(N, Ty, DAG, AArch64II::MO_PAGE | Flags);
5992   SDValue Lo = getTargetNode(N, Ty, DAG,
5993                              AArch64II::MO_PAGEOFF | AArch64II::MO_NC | Flags);
5994   SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi);
5995   return DAG.getNode(AArch64ISD::ADDlow, DL, Ty, ADRP, Lo);
5996 }
5997 
5998 // (adr sym)
5999 template <class NodeTy>
6000 SDValue AArch64TargetLowering::getAddrTiny(NodeTy *N, SelectionDAG &DAG,
6001                                            unsigned Flags) const {
6002   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddrTiny\n");
6003   SDLoc DL(N);
6004   EVT Ty = getPointerTy(DAG.getDataLayout());
6005   SDValue Sym = getTargetNode(N, Ty, DAG, Flags);
6006   return DAG.getNode(AArch64ISD::ADR, DL, Ty, Sym);
6007 }
6008 
6009 SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
6010                                                   SelectionDAG &DAG) const {
6011   GlobalAddressSDNode *GN = cast<GlobalAddressSDNode>(Op);
6012   const GlobalValue *GV = GN->getGlobal();
6013   unsigned OpFlags = Subtarget->ClassifyGlobalReference(GV, getTargetMachine());
6014 
6015   if (OpFlags != AArch64II::MO_NO_FLAG)
6016     assert(cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&
6017            "unexpected offset in global node");
6018 
6019   // This also catches the large code model case for Darwin, and tiny code
6020   // model with got relocations.
6021   if ((OpFlags & AArch64II::MO_GOT) != 0) {
6022     return getGOT(GN, DAG, OpFlags);
6023   }
6024 
6025   SDValue Result;
6026   if (getTargetMachine().getCodeModel() == CodeModel::Large) {
6027     Result = getAddrLarge(GN, DAG, OpFlags);
6028   } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
6029     Result = getAddrTiny(GN, DAG, OpFlags);
6030   } else {
6031     Result = getAddr(GN, DAG, OpFlags);
6032   }
6033   EVT PtrVT = getPointerTy(DAG.getDataLayout());
6034   SDLoc DL(GN);
6035   if (OpFlags & (AArch64II::MO_DLLIMPORT | AArch64II::MO_COFFSTUB))
6036     Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
6037                          MachinePointerInfo::getGOT(DAG.getMachineFunction()));
6038   return Result;
6039 }
6040 
6041 /// Convert a TLS address reference into the correct sequence of loads
6042 /// and calls to compute the variable's address (for Darwin, currently) and
6043 /// return an SDValue containing the final node.
6044 
6045 /// Darwin only has one TLS scheme which must be capable of dealing with the
6046 /// fully general situation, in the worst case. This means:
6047 ///     + "extern __thread" declaration.
6048 ///     + Defined in a possibly unknown dynamic library.
6049 ///
6050 /// The general system is that each __thread variable has a [3 x i64] descriptor
6051 /// which contains information used by the runtime to calculate the address. The
6052 /// only part of this the compiler needs to know about is the first xword, which
6053 /// contains a function pointer that must be called with the address of the
6054 /// entire descriptor in "x0".
6055 ///
6056 /// Since this descriptor may be in a different unit, in general even the
6057 /// descriptor must be accessed via an indirect load. The "ideal" code sequence
6058 /// is:
6059 ///     adrp x0, _var@TLVPPAGE
6060 ///     ldr x0, [x0, _var@TLVPPAGEOFF]   ; x0 now contains address of descriptor
6061 ///     ldr x1, [x0]                     ; x1 contains 1st entry of descriptor,
6062 ///                                      ; the function pointer
6063 ///     blr x1                           ; Uses descriptor address in x0
6064 ///     ; Address of _var is now in x0.
6065 ///
6066 /// If the address of _var's descriptor *is* known to the linker, then it can
6067 /// change the first "ldr" instruction to an appropriate "add x0, x0, #imm" for
6068 /// a slight efficiency gain.
6069 SDValue
6070 AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
6071                                                    SelectionDAG &DAG) const {
6072   assert(Subtarget->isTargetDarwin() &&
6073          "This function expects a Darwin target");
6074 
6075   SDLoc DL(Op);
6076   MVT PtrVT = getPointerTy(DAG.getDataLayout());
6077   MVT PtrMemVT = getPointerMemTy(DAG.getDataLayout());
6078   const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
6079 
6080   SDValue TLVPAddr =
6081       DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
6082   SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
6083 
6084   // The first entry in the descriptor is a function pointer that we must call
6085   // to obtain the address of the variable.
6086   SDValue Chain = DAG.getEntryNode();
6087   SDValue FuncTLVGet = DAG.getLoad(
6088       PtrMemVT, DL, Chain, DescAddr,
6089       MachinePointerInfo::getGOT(DAG.getMachineFunction()),
6090       Align(PtrMemVT.getSizeInBits() / 8),
6091       MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
6092   Chain = FuncTLVGet.getValue(1);
6093 
6094   // Extend loaded pointer if necessary (i.e. if ILP32) to DAG pointer.
6095   FuncTLVGet = DAG.getZExtOrTrunc(FuncTLVGet, DL, PtrVT);
6096 
6097   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6098   MFI.setAdjustsStack(true);
6099 
6100   // TLS calls preserve all registers except those that absolutely must be
6101   // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
6102   // silly).
6103   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
6104   const uint32_t *Mask = TRI->getTLSCallPreservedMask();
6105   if (Subtarget->hasCustomCallingConv())
6106     TRI->UpdateCustomCallPreservedMask(DAG.getMachineFunction(), &Mask);
6107 
6108   // Finally, we can make the call. This is just a degenerate version of a
6109   // normal AArch64 call node: x0 takes the address of the descriptor, and
6110   // returns the address of the variable in this thread.
6111   Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
6112   Chain =
6113       DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
6114                   Chain, FuncTLVGet, DAG.getRegister(AArch64::X0, MVT::i64),
6115                   DAG.getRegisterMask(Mask), Chain.getValue(1));
6116   return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Chain.getValue(1));
6117 }
6118 
6119 /// Convert a thread-local variable reference into a sequence of instructions to
6120 /// compute the variable's address for the local exec TLS model of ELF targets.
6121 /// The sequence depends on the maximum TLS area size.
6122 SDValue AArch64TargetLowering::LowerELFTLSLocalExec(const GlobalValue *GV,
6123                                                     SDValue ThreadBase,
6124                                                     const SDLoc &DL,
6125                                                     SelectionDAG &DAG) const {
6126   EVT PtrVT = getPointerTy(DAG.getDataLayout());
6127   SDValue TPOff, Addr;
6128 
6129   switch (DAG.getTarget().Options.TLSSize) {
6130   default:
6131     llvm_unreachable("Unexpected TLS size");
6132 
6133   case 12: {
6134     // mrs   x0, TPIDR_EL0
6135     // add   x0, x0, :tprel_lo12:a
6136     SDValue Var = DAG.getTargetGlobalAddress(
6137         GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
6138     return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
6139                                       Var,
6140                                       DAG.getTargetConstant(0, DL, MVT::i32)),
6141                    0);
6142   }
6143 
6144   case 24: {
6145     // mrs   x0, TPIDR_EL0
6146     // add   x0, x0, :tprel_hi12:a
6147     // add   x0, x0, :tprel_lo12_nc:a
6148     SDValue HiVar = DAG.getTargetGlobalAddress(
6149         GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
6150     SDValue LoVar = DAG.getTargetGlobalAddress(
6151         GV, DL, PtrVT, 0,
6152         AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
6153     Addr = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
6154                                       HiVar,
6155                                       DAG.getTargetConstant(0, DL, MVT::i32)),
6156                    0);
6157     return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, Addr,
6158                                       LoVar,
6159                                       DAG.getTargetConstant(0, DL, MVT::i32)),
6160                    0);
6161   }
6162 
6163   case 32: {
6164     // mrs   x1, TPIDR_EL0
6165     // movz  x0, #:tprel_g1:a
6166     // movk  x0, #:tprel_g0_nc:a
6167     // add   x0, x1, x0
6168     SDValue HiVar = DAG.getTargetGlobalAddress(
6169         GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_G1);
6170     SDValue LoVar = DAG.getTargetGlobalAddress(
6171         GV, DL, PtrVT, 0,
6172         AArch64II::MO_TLS | AArch64II::MO_G0 | AArch64II::MO_NC);
6173     TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
6174                                        DAG.getTargetConstant(16, DL, MVT::i32)),
6175                     0);
6176     TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
6177                                        DAG.getTargetConstant(0, DL, MVT::i32)),
6178                     0);
6179     return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
6180   }
6181 
6182   case 48: {
6183     // mrs   x1, TPIDR_EL0
6184     // movz  x0, #:tprel_g2:a
6185     // movk  x0, #:tprel_g1_nc:a
6186     // movk  x0, #:tprel_g0_nc:a
6187     // add   x0, x1, x0
6188     SDValue HiVar = DAG.getTargetGlobalAddress(
6189         GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_G2);
6190     SDValue MiVar = DAG.getTargetGlobalAddress(
6191         GV, DL, PtrVT, 0,
6192         AArch64II::MO_TLS | AArch64II::MO_G1 | AArch64II::MO_NC);
6193     SDValue LoVar = DAG.getTargetGlobalAddress(
6194         GV, DL, PtrVT, 0,
6195         AArch64II::MO_TLS | AArch64II::MO_G0 | AArch64II::MO_NC);
6196     TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
6197                                        DAG.getTargetConstant(32, DL, MVT::i32)),
6198                     0);
6199     TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, MiVar,
6200                                        DAG.getTargetConstant(16, DL, MVT::i32)),
6201                     0);
6202     TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
6203                                        DAG.getTargetConstant(0, DL, MVT::i32)),
6204                     0);
6205     return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
6206   }
6207   }
6208 }
6209 
6210 /// When accessing thread-local variables under either the general-dynamic or
6211 /// local-dynamic system, we make a "TLS-descriptor" call. The variable will
6212 /// have a descriptor, accessible via a PC-relative ADRP, and whose first entry
6213 /// is a function pointer to carry out the resolution.
6214 ///
6215 /// The sequence is:
6216 ///    adrp  x0, :tlsdesc:var
6217 ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
6218 ///    add   x0, x0, #:tlsdesc_lo12:var
6219 ///    .tlsdesccall var
6220 ///    blr   x1
6221 ///    (TPIDR_EL0 offset now in x0)
6222 ///
6223 ///  The above sequence must be produced unscheduled, to enable the linker to
6224 ///  optimize/relax this sequence.
6225 ///  Therefore, a pseudo-instruction (TLSDESC_CALLSEQ) is used to represent the
6226 ///  above sequence, and expanded really late in the compilation flow, to ensure
6227 ///  the sequence is produced as per above.
6228 SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr,
6229                                                       const SDLoc &DL,
6230                                                       SelectionDAG &DAG) const {
6231   EVT PtrVT = getPointerTy(DAG.getDataLayout());
6232 
6233   SDValue Chain = DAG.getEntryNode();
6234   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
6235 
6236   Chain =
6237       DAG.getNode(AArch64ISD::TLSDESC_CALLSEQ, DL, NodeTys, {Chain, SymAddr});
6238   SDValue Glue = Chain.getValue(1);
6239 
6240   return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue);
6241 }
6242 
6243 SDValue
6244 AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
6245                                                 SelectionDAG &DAG) const {
6246   assert(Subtarget->isTargetELF() && "This function expects an ELF target");
6247 
6248   const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
6249 
6250   TLSModel::Model Model = getTargetMachine().getTLSModel(GA->getGlobal());
6251 
6252   if (!EnableAArch64ELFLocalDynamicTLSGeneration) {
6253     if (Model == TLSModel::LocalDynamic)
6254       Model = TLSModel::GeneralDynamic;
6255   }
6256 
6257   if (getTargetMachine().getCodeModel() == CodeModel::Large &&
6258       Model != TLSModel::LocalExec)
6259     report_fatal_error("ELF TLS only supported in small memory model or "
6260                        "in local exec TLS model");
6261   // Different choices can be made for the maximum size of the TLS area for a
6262   // module. For the small address model, the default TLS size is 16MiB and the
6263   // maximum TLS size is 4GiB.
6264   // FIXME: add tiny and large code model support for TLS access models other
6265   // than local exec. We currently generate the same code as small for tiny,
6266   // which may be larger than needed.
6267 
6268   SDValue TPOff;
6269   EVT PtrVT = getPointerTy(DAG.getDataLayout());
6270   SDLoc DL(Op);
6271   const GlobalValue *GV = GA->getGlobal();
6272 
6273   SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
6274 
6275   if (Model == TLSModel::LocalExec) {
6276     return LowerELFTLSLocalExec(GV, ThreadBase, DL, DAG);
6277   } else if (Model == TLSModel::InitialExec) {
6278     TPOff = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
6279     TPOff = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TPOff);
6280   } else if (Model == TLSModel::LocalDynamic) {
6281     // Local-dynamic accesses proceed in two phases. A general-dynamic TLS
6282     // descriptor call against the special symbol _TLS_MODULE_BASE_ to calculate
6283     // the beginning of the module's TLS region, followed by a DTPREL offset
6284     // calculation.
6285 
6286     // These accesses will need deduplicating if there's more than one.
6287     AArch64FunctionInfo *MFI =
6288         DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
6289     MFI->incNumLocalDynamicTLSAccesses();
6290 
6291     // The call needs a relocation too for linker relaxation. It doesn't make
6292     // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
6293     // the address.
6294     SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT,
6295                                                   AArch64II::MO_TLS);
6296 
6297     // Now we can calculate the offset from TPIDR_EL0 to this module's
6298     // thread-local area.
6299     TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
6300 
6301     // Now use :dtprel_whatever: operations to calculate this variable's offset
6302     // in its thread-storage area.
6303     SDValue HiVar = DAG.getTargetGlobalAddress(
6304         GV, DL, MVT::i64, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
6305     SDValue LoVar = DAG.getTargetGlobalAddress(
6306         GV, DL, MVT::i64, 0,
6307         AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
6308 
6309     TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
6310                                        DAG.getTargetConstant(0, DL, MVT::i32)),
6311                     0);
6312     TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
6313                                        DAG.getTargetConstant(0, DL, MVT::i32)),
6314                     0);
6315   } else if (Model == TLSModel::GeneralDynamic) {
6316     // The call needs a relocation too for linker relaxation. It doesn't make
6317     // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
6318     // the address.
6319     SDValue SymAddr =
6320         DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
6321 
6322     // Finally we can make a call to calculate the offset from tpidr_el0.
6323     TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
6324   } else
6325     llvm_unreachable("Unsupported ELF TLS access model");
6326 
6327   return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
6328 }
6329 
6330 SDValue
6331 AArch64TargetLowering::LowerWindowsGlobalTLSAddress(SDValue Op,
6332                                                     SelectionDAG &DAG) const {
6333   assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering");
6334 
6335   SDValue Chain = DAG.getEntryNode();
6336   EVT PtrVT = getPointerTy(DAG.getDataLayout());
6337   SDLoc DL(Op);
6338 
6339   SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
6340 
6341   // Load the ThreadLocalStoragePointer from the TEB
6342   // A pointer to the TLS array is located at offset 0x58 from the TEB.
6343   SDValue TLSArray =
6344       DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x58, DL));
6345   TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
6346   Chain = TLSArray.getValue(1);
6347 
6348   // Load the TLS index from the C runtime;
6349   // This does the same as getAddr(), but without having a GlobalAddressSDNode.
6350   // This also does the same as LOADgot, but using a generic i32 load,
6351   // while LOADgot only loads i64.
6352   SDValue TLSIndexHi =
6353       DAG.getTargetExternalSymbol("_tls_index", PtrVT, AArch64II::MO_PAGE);
6354   SDValue TLSIndexLo = DAG.getTargetExternalSymbol(
6355       "_tls_index", PtrVT, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
6356   SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi);
6357   SDValue TLSIndex =
6358       DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, TLSIndexLo);
6359   TLSIndex = DAG.getLoad(MVT::i32, DL, Chain, TLSIndex, MachinePointerInfo());
6360   Chain = TLSIndex.getValue(1);
6361 
6362   // The pointer to the thread's TLS data area is at the TLS Index scaled by 8
6363   // offset into the TLSArray.
6364   TLSIndex = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TLSIndex);
6365   SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
6366                              DAG.getConstant(3, DL, PtrVT));
6367   SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
6368                             DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
6369                             MachinePointerInfo());
6370   Chain = TLS.getValue(1);
6371 
6372   const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
6373   const GlobalValue *GV = GA->getGlobal();
6374   SDValue TGAHi = DAG.getTargetGlobalAddress(
6375       GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
6376   SDValue TGALo = DAG.getTargetGlobalAddress(
6377       GV, DL, PtrVT, 0,
6378       AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
6379 
6380   // Add the offset from the start of the .tls section (section base).
6381   SDValue Addr =
6382       SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
6383                                  DAG.getTargetConstant(0, DL, MVT::i32)),
6384               0);
6385   Addr = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, Addr, TGALo);
6386   return Addr;
6387 }
6388 
6389 SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
6390                                                      SelectionDAG &DAG) const {
6391   const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
6392   if (DAG.getTarget().useEmulatedTLS())
6393     return LowerToTLSEmulatedModel(GA, DAG);
6394 
6395   if (Subtarget->isTargetDarwin())
6396     return LowerDarwinGlobalTLSAddress(Op, DAG);
6397   if (Subtarget->isTargetELF())
6398     return LowerELFGlobalTLSAddress(Op, DAG);
6399   if (Subtarget->isTargetWindows())
6400     return LowerWindowsGlobalTLSAddress(Op, DAG);
6401 
6402   llvm_unreachable("Unexpected platform trying to use TLS");
6403 }
6404 
6405 // Looks through \param Val to determine the bit that can be used to
6406 // check the sign of the value. It returns the unextended value and
6407 // the sign bit position.
6408 std::pair<SDValue, uint64_t> lookThroughSignExtension(SDValue Val) {
6409   if (Val.getOpcode() == ISD::SIGN_EXTEND_INREG)
6410     return {Val.getOperand(0),
6411             cast<VTSDNode>(Val.getOperand(1))->getVT().getFixedSizeInBits() -
6412                 1};
6413 
6414   if (Val.getOpcode() == ISD::SIGN_EXTEND)
6415     return {Val.getOperand(0),
6416             Val.getOperand(0)->getValueType(0).getFixedSizeInBits() - 1};
6417 
6418   return {Val, Val.getValueSizeInBits() - 1};
6419 }
6420 
6421 SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
6422   SDValue Chain = Op.getOperand(0);
6423   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
6424   SDValue LHS = Op.getOperand(2);
6425   SDValue RHS = Op.getOperand(3);
6426   SDValue Dest = Op.getOperand(4);
6427   SDLoc dl(Op);
6428 
6429   MachineFunction &MF = DAG.getMachineFunction();
6430   // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z instructions
6431   // will not be produced, as they are conditional branch instructions that do
6432   // not set flags.
6433   bool ProduceNonFlagSettingCondBr =
6434       !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
6435 
6436   // Handle f128 first, since lowering it will result in comparing the return
6437   // value of a libcall against zero, which is just what the rest of LowerBR_CC
6438   // is expecting to deal with.
6439   if (LHS.getValueType() == MVT::f128) {
6440     softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
6441 
6442     // If softenSetCCOperands returned a scalar, we need to compare the result
6443     // against zero to select between true and false values.
6444     if (!RHS.getNode()) {
6445       RHS = DAG.getConstant(0, dl, LHS.getValueType());
6446       CC = ISD::SETNE;
6447     }
6448   }
6449 
6450   // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
6451   // instruction.
6452   if (ISD::isOverflowIntrOpRes(LHS) && isOneConstant(RHS) &&
6453       (CC == ISD::SETEQ || CC == ISD::SETNE)) {
6454     // Only lower legal XALUO ops.
6455     if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0)))
6456       return SDValue();
6457 
6458     // The actual operation with overflow check.
6459     AArch64CC::CondCode OFCC;
6460     SDValue Value, Overflow;
6461     std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, LHS.getValue(0), DAG);
6462 
6463     if (CC == ISD::SETNE)
6464       OFCC = getInvertedCondCode(OFCC);
6465     SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
6466 
6467     return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
6468                        Overflow);
6469   }
6470 
6471   if (LHS.getValueType().isInteger()) {
6472     assert((LHS.getValueType() == RHS.getValueType()) &&
6473            (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64));
6474 
6475     // If the RHS of the comparison is zero, we can potentially fold this
6476     // to a specialized branch.
6477     const ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
6478     if (RHSC && RHSC->getZExtValue() == 0 && ProduceNonFlagSettingCondBr) {
6479       if (CC == ISD::SETEQ) {
6480         // See if we can use a TBZ to fold in an AND as well.
6481         // TBZ has a smaller branch displacement than CBZ.  If the offset is
6482         // out of bounds, a late MI-layer pass rewrites branches.
6483         // 403.gcc is an example that hits this case.
6484         if (LHS.getOpcode() == ISD::AND &&
6485             isa<ConstantSDNode>(LHS.getOperand(1)) &&
6486             isPowerOf2_64(LHS.getConstantOperandVal(1))) {
6487           SDValue Test = LHS.getOperand(0);
6488           uint64_t Mask = LHS.getConstantOperandVal(1);
6489           return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test,
6490                              DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
6491                              Dest);
6492         }
6493 
6494         return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest);
6495       } else if (CC == ISD::SETNE) {
6496         // See if we can use a TBZ to fold in an AND as well.
6497         // TBZ has a smaller branch displacement than CBZ.  If the offset is
6498         // out of bounds, a late MI-layer pass rewrites branches.
6499         // 403.gcc is an example that hits this case.
6500         if (LHS.getOpcode() == ISD::AND &&
6501             isa<ConstantSDNode>(LHS.getOperand(1)) &&
6502             isPowerOf2_64(LHS.getConstantOperandVal(1))) {
6503           SDValue Test = LHS.getOperand(0);
6504           uint64_t Mask = LHS.getConstantOperandVal(1);
6505           return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test,
6506                              DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
6507                              Dest);
6508         }
6509 
6510         return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest);
6511       } else if (CC == ISD::SETLT && LHS.getOpcode() != ISD::AND) {
6512         // Don't combine AND since emitComparison converts the AND to an ANDS
6513         // (a.k.a. TST) and the test in the test bit and branch instruction
6514         // becomes redundant.  This would also increase register pressure.
6515         uint64_t SignBitPos;
6516         std::tie(LHS, SignBitPos) = lookThroughSignExtension(LHS);
6517         return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS,
6518                            DAG.getConstant(SignBitPos, dl, MVT::i64), Dest);
6519       }
6520     }
6521     if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT &&
6522         LHS.getOpcode() != ISD::AND && ProduceNonFlagSettingCondBr) {
6523       // Don't combine AND since emitComparison converts the AND to an ANDS
6524       // (a.k.a. TST) and the test in the test bit and branch instruction
6525       // becomes redundant.  This would also increase register pressure.
6526       uint64_t SignBitPos;
6527       std::tie(LHS, SignBitPos) = lookThroughSignExtension(LHS);
6528       return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS,
6529                          DAG.getConstant(SignBitPos, dl, MVT::i64), Dest);
6530     }
6531 
6532     SDValue CCVal;
6533     SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
6534     return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
6535                        Cmp);
6536   }
6537 
6538   assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::bf16 ||
6539          LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64);
6540 
6541   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
6542   // clean.  Some of them require two branches to implement.
6543   SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
6544   AArch64CC::CondCode CC1, CC2;
6545   changeFPCCToAArch64CC(CC, CC1, CC2);
6546   SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
6547   SDValue BR1 =
6548       DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp);
6549   if (CC2 != AArch64CC::AL) {
6550     SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
6551     return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val,
6552                        Cmp);
6553   }
6554 
6555   return BR1;
6556 }
6557 
6558 SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
6559                                               SelectionDAG &DAG) const {
6560   EVT VT = Op.getValueType();
6561   SDLoc DL(Op);
6562 
6563   SDValue In1 = Op.getOperand(0);
6564   SDValue In2 = Op.getOperand(1);
6565   EVT SrcVT = In2.getValueType();
6566 
6567   if (SrcVT.bitsLT(VT))
6568     In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2);
6569   else if (SrcVT.bitsGT(VT))
6570     In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL));
6571 
6572   EVT VecVT;
6573   uint64_t EltMask;
6574   SDValue VecVal1, VecVal2;
6575 
6576   auto setVecVal = [&] (int Idx) {
6577     if (!VT.isVector()) {
6578       VecVal1 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
6579                                           DAG.getUNDEF(VecVT), In1);
6580       VecVal2 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
6581                                           DAG.getUNDEF(VecVT), In2);
6582     } else {
6583       VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1);
6584       VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2);
6585     }
6586   };
6587 
6588   if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) {
6589     VecVT = (VT == MVT::v2f32 ? MVT::v2i32 : MVT::v4i32);
6590     EltMask = 0x80000000ULL;
6591     setVecVal(AArch64::ssub);
6592   } else if (VT == MVT::f64 || VT == MVT::v2f64) {
6593     VecVT = MVT::v2i64;
6594 
6595     // We want to materialize a mask with the high bit set, but the AdvSIMD
6596     // immediate moves cannot materialize that in a single instruction for
6597     // 64-bit elements. Instead, materialize zero and then negate it.
6598     EltMask = 0;
6599 
6600     setVecVal(AArch64::dsub);
6601   } else if (VT == MVT::f16 || VT == MVT::v4f16 || VT == MVT::v8f16) {
6602     VecVT = (VT == MVT::v4f16 ? MVT::v4i16 : MVT::v8i16);
6603     EltMask = 0x8000ULL;
6604     setVecVal(AArch64::hsub);
6605   } else {
6606     llvm_unreachable("Invalid type for copysign!");
6607   }
6608 
6609   SDValue BuildVec = DAG.getConstant(EltMask, DL, VecVT);
6610 
6611   // If we couldn't materialize the mask above, then the mask vector will be
6612   // the zero vector, and we need to negate it here.
6613   if (VT == MVT::f64 || VT == MVT::v2f64) {
6614     BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec);
6615     BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec);
6616     BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec);
6617   }
6618 
6619   SDValue Sel =
6620       DAG.getNode(AArch64ISD::BIT, DL, VecVT, VecVal1, VecVal2, BuildVec);
6621 
6622   if (VT == MVT::f16)
6623     return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, Sel);
6624   if (VT == MVT::f32)
6625     return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel);
6626   else if (VT == MVT::f64)
6627     return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel);
6628   else
6629     return DAG.getNode(ISD::BITCAST, DL, VT, Sel);
6630 }
6631 
6632 SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
6633   if (DAG.getMachineFunction().getFunction().hasFnAttribute(
6634           Attribute::NoImplicitFloat))
6635     return SDValue();
6636 
6637   if (!Subtarget->hasNEON())
6638     return SDValue();
6639 
6640   // While there is no integer popcount instruction, it can
6641   // be more efficiently lowered to the following sequence that uses
6642   // AdvSIMD registers/instructions as long as the copies to/from
6643   // the AdvSIMD registers are cheap.
6644   //  FMOV    D0, X0        // copy 64-bit int to vector, high bits zero'd
6645   //  CNT     V0.8B, V0.8B  // 8xbyte pop-counts
6646   //  ADDV    B0, V0.8B     // sum 8xbyte pop-counts
6647   //  UMOV    X0, V0.B[0]   // copy byte result back to integer reg
6648   SDValue Val = Op.getOperand(0);
6649   SDLoc DL(Op);
6650   EVT VT = Op.getValueType();
6651 
6652   if (VT == MVT::i32 || VT == MVT::i64) {
6653     if (VT == MVT::i32)
6654       Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val);
6655     Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val);
6656 
6657     SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
6658     SDValue UaddLV = DAG.getNode(
6659         ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
6660         DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
6661 
6662     if (VT == MVT::i64)
6663       UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV);
6664     return UaddLV;
6665   } else if (VT == MVT::i128) {
6666     Val = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Val);
6667 
6668     SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v16i8, Val);
6669     SDValue UaddLV = DAG.getNode(
6670         ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
6671         DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
6672 
6673     return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, UaddLV);
6674   }
6675 
6676   if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT))
6677     return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTPOP_MERGE_PASSTHRU);
6678 
6679   assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6680           VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6681          "Unexpected type for custom ctpop lowering");
6682 
6683   EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
6684   Val = DAG.getBitcast(VT8Bit, Val);
6685   Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val);
6686 
6687   // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
6688   unsigned EltSize = 8;
6689   unsigned NumElts = VT.is64BitVector() ? 8 : 16;
6690   while (EltSize != VT.getScalarSizeInBits()) {
6691     EltSize *= 2;
6692     NumElts /= 2;
6693     MVT WidenVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize), NumElts);
6694     Val = DAG.getNode(
6695         ISD::INTRINSIC_WO_CHAIN, DL, WidenVT,
6696         DAG.getConstant(Intrinsic::aarch64_neon_uaddlp, DL, MVT::i32), Val);
6697   }
6698 
6699   return Val;
6700 }
6701 
6702 SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
6703   EVT VT = Op.getValueType();
6704   assert(VT.isScalableVector() ||
6705          useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true));
6706 
6707   SDLoc DL(Op);
6708   SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
6709   return DAG.getNode(ISD::CTLZ, DL, VT, RBIT);
6710 }
6711 
6712 SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
6713 
6714   if (Op.getValueType().isVector())
6715     return LowerVSETCC(Op, DAG);
6716 
6717   bool IsStrict = Op->isStrictFPOpcode();
6718   bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
6719   unsigned OpNo = IsStrict ? 1 : 0;
6720   SDValue Chain;
6721   if (IsStrict)
6722     Chain = Op.getOperand(0);
6723   SDValue LHS = Op.getOperand(OpNo + 0);
6724   SDValue RHS = Op.getOperand(OpNo + 1);
6725   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(OpNo + 2))->get();
6726   SDLoc dl(Op);
6727 
6728   // We chose ZeroOrOneBooleanContents, so use zero and one.
6729   EVT VT = Op.getValueType();
6730   SDValue TVal = DAG.getConstant(1, dl, VT);
6731   SDValue FVal = DAG.getConstant(0, dl, VT);
6732 
6733   // Handle f128 first, since one possible outcome is a normal integer
6734   // comparison which gets picked up by the next if statement.
6735   if (LHS.getValueType() == MVT::f128) {
6736     softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS, Chain,
6737                         IsSignaling);
6738 
6739     // If softenSetCCOperands returned a scalar, use it.
6740     if (!RHS.getNode()) {
6741       assert(LHS.getValueType() == Op.getValueType() &&
6742              "Unexpected setcc expansion!");
6743       return IsStrict ? DAG.getMergeValues({LHS, Chain}, dl) : LHS;
6744     }
6745   }
6746 
6747   if (LHS.getValueType().isInteger()) {
6748     SDValue CCVal;
6749     SDValue Cmp = getAArch64Cmp(
6750         LHS, RHS, ISD::getSetCCInverse(CC, LHS.getValueType()), CCVal, DAG, dl);
6751 
6752     // Note that we inverted the condition above, so we reverse the order of
6753     // the true and false operands here.  This will allow the setcc to be
6754     // matched to a single CSINC instruction.
6755     SDValue Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
6756     return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
6757   }
6758 
6759   // Now we know we're dealing with FP values.
6760   assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||
6761          LHS.getValueType() == MVT::f64);
6762 
6763   // If that fails, we'll need to perform an FCMP + CSEL sequence.  Go ahead
6764   // and do the comparison.
6765   SDValue Cmp;
6766   if (IsStrict)
6767     Cmp = emitStrictFPComparison(LHS, RHS, dl, DAG, Chain, IsSignaling);
6768   else
6769     Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
6770 
6771   AArch64CC::CondCode CC1, CC2;
6772   changeFPCCToAArch64CC(CC, CC1, CC2);
6773   SDValue Res;
6774   if (CC2 == AArch64CC::AL) {
6775     changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, LHS.getValueType()), CC1,
6776                           CC2);
6777     SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
6778 
6779     // Note that we inverted the condition above, so we reverse the order of
6780     // the true and false operands here.  This will allow the setcc to be
6781     // matched to a single CSINC instruction.
6782     Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp);
6783   } else {
6784     // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
6785     // totally clean.  Some of them require two CSELs to implement.  As is in
6786     // this case, we emit the first CSEL and then emit a second using the output
6787     // of the first as the RHS.  We're effectively OR'ing the two CC's together.
6788 
6789     // FIXME: It would be nice if we could match the two CSELs to two CSINCs.
6790     SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
6791     SDValue CS1 =
6792         DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
6793 
6794     SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
6795     Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
6796   }
6797   return IsStrict ? DAG.getMergeValues({Res, Cmp.getValue(1)}, dl) : Res;
6798 }
6799 
6800 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
6801                                               SDValue RHS, SDValue TVal,
6802                                               SDValue FVal, const SDLoc &dl,
6803                                               SelectionDAG &DAG) const {
6804   // Handle f128 first, because it will result in a comparison of some RTLIB
6805   // call result against zero.
6806   if (LHS.getValueType() == MVT::f128) {
6807     softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
6808 
6809     // If softenSetCCOperands returned a scalar, we need to compare the result
6810     // against zero to select between true and false values.
6811     if (!RHS.getNode()) {
6812       RHS = DAG.getConstant(0, dl, LHS.getValueType());
6813       CC = ISD::SETNE;
6814     }
6815   }
6816 
6817   // Also handle f16, for which we need to do a f32 comparison.
6818   if (LHS.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) {
6819     LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
6820     RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
6821   }
6822 
6823   // Next, handle integers.
6824   if (LHS.getValueType().isInteger()) {
6825     assert((LHS.getValueType() == RHS.getValueType()) &&
6826            (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64));
6827 
6828     ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
6829     ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
6830     ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
6831     // Check for sign pattern (SELECT_CC setgt, iN lhs, -1, 1, -1) and transform
6832     // into (OR (ASR lhs, N-1), 1), which requires less instructions for the
6833     // supported types.
6834     if (CC == ISD::SETGT && RHSC && RHSC->isAllOnesValue() && CTVal && CFVal &&
6835         CTVal->isOne() && CFVal->isAllOnesValue() &&
6836         LHS.getValueType() == TVal.getValueType()) {
6837       EVT VT = LHS.getValueType();
6838       SDValue Shift =
6839           DAG.getNode(ISD::SRA, dl, VT, LHS,
6840                       DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
6841       return DAG.getNode(ISD::OR, dl, VT, Shift, DAG.getConstant(1, dl, VT));
6842     }
6843 
6844     unsigned Opcode = AArch64ISD::CSEL;
6845 
6846     // If both the TVal and the FVal are constants, see if we can swap them in
6847     // order to for a CSINV or CSINC out of them.
6848     if (CTVal && CFVal && CTVal->isAllOnesValue() && CFVal->isNullValue()) {
6849       std::swap(TVal, FVal);
6850       std::swap(CTVal, CFVal);
6851       CC = ISD::getSetCCInverse(CC, LHS.getValueType());
6852     } else if (CTVal && CFVal && CTVal->isOne() && CFVal->isNullValue()) {
6853       std::swap(TVal, FVal);
6854       std::swap(CTVal, CFVal);
6855       CC = ISD::getSetCCInverse(CC, LHS.getValueType());
6856     } else if (TVal.getOpcode() == ISD::XOR) {
6857       // If TVal is a NOT we want to swap TVal and FVal so that we can match
6858       // with a CSINV rather than a CSEL.
6859       if (isAllOnesConstant(TVal.getOperand(1))) {
6860         std::swap(TVal, FVal);
6861         std::swap(CTVal, CFVal);
6862         CC = ISD::getSetCCInverse(CC, LHS.getValueType());
6863       }
6864     } else if (TVal.getOpcode() == ISD::SUB) {
6865       // If TVal is a negation (SUB from 0) we want to swap TVal and FVal so
6866       // that we can match with a CSNEG rather than a CSEL.
6867       if (isNullConstant(TVal.getOperand(0))) {
6868         std::swap(TVal, FVal);
6869         std::swap(CTVal, CFVal);
6870         CC = ISD::getSetCCInverse(CC, LHS.getValueType());
6871       }
6872     } else if (CTVal && CFVal) {
6873       const int64_t TrueVal = CTVal->getSExtValue();
6874       const int64_t FalseVal = CFVal->getSExtValue();
6875       bool Swap = false;
6876 
6877       // If both TVal and FVal are constants, see if FVal is the
6878       // inverse/negation/increment of TVal and generate a CSINV/CSNEG/CSINC
6879       // instead of a CSEL in that case.
6880       if (TrueVal == ~FalseVal) {
6881         Opcode = AArch64ISD::CSINV;
6882       } else if (FalseVal > std::numeric_limits<int64_t>::min() &&
6883                  TrueVal == -FalseVal) {
6884         Opcode = AArch64ISD::CSNEG;
6885       } else if (TVal.getValueType() == MVT::i32) {
6886         // If our operands are only 32-bit wide, make sure we use 32-bit
6887         // arithmetic for the check whether we can use CSINC. This ensures that
6888         // the addition in the check will wrap around properly in case there is
6889         // an overflow (which would not be the case if we do the check with
6890         // 64-bit arithmetic).
6891         const uint32_t TrueVal32 = CTVal->getZExtValue();
6892         const uint32_t FalseVal32 = CFVal->getZExtValue();
6893 
6894         if ((TrueVal32 == FalseVal32 + 1) || (TrueVal32 + 1 == FalseVal32)) {
6895           Opcode = AArch64ISD::CSINC;
6896 
6897           if (TrueVal32 > FalseVal32) {
6898             Swap = true;
6899           }
6900         }
6901         // 64-bit check whether we can use CSINC.
6902       } else if ((TrueVal == FalseVal + 1) || (TrueVal + 1 == FalseVal)) {
6903         Opcode = AArch64ISD::CSINC;
6904 
6905         if (TrueVal > FalseVal) {
6906           Swap = true;
6907         }
6908       }
6909 
6910       // Swap TVal and FVal if necessary.
6911       if (Swap) {
6912         std::swap(TVal, FVal);
6913         std::swap(CTVal, CFVal);
6914         CC = ISD::getSetCCInverse(CC, LHS.getValueType());
6915       }
6916 
6917       if (Opcode != AArch64ISD::CSEL) {
6918         // Drop FVal since we can get its value by simply inverting/negating
6919         // TVal.
6920         FVal = TVal;
6921       }
6922     }
6923 
6924     // Avoid materializing a constant when possible by reusing a known value in
6925     // a register.  However, don't perform this optimization if the known value
6926     // is one, zero or negative one in the case of a CSEL.  We can always
6927     // materialize these values using CSINC, CSEL and CSINV with wzr/xzr as the
6928     // FVal, respectively.
6929     ConstantSDNode *RHSVal = dyn_cast<ConstantSDNode>(RHS);
6930     if (Opcode == AArch64ISD::CSEL && RHSVal && !RHSVal->isOne() &&
6931         !RHSVal->isNullValue() && !RHSVal->isAllOnesValue()) {
6932       AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
6933       // Transform "a == C ? C : x" to "a == C ? a : x" and "a != C ? x : C" to
6934       // "a != C ? x : a" to avoid materializing C.
6935       if (CTVal && CTVal == RHSVal && AArch64CC == AArch64CC::EQ)
6936         TVal = LHS;
6937       else if (CFVal && CFVal == RHSVal && AArch64CC == AArch64CC::NE)
6938         FVal = LHS;
6939     } else if (Opcode == AArch64ISD::CSNEG && RHSVal && RHSVal->isOne()) {
6940       assert (CTVal && CFVal && "Expected constant operands for CSNEG.");
6941       // Use a CSINV to transform "a == C ? 1 : -1" to "a == C ? a : -1" to
6942       // avoid materializing C.
6943       AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
6944       if (CTVal == RHSVal && AArch64CC == AArch64CC::EQ) {
6945         Opcode = AArch64ISD::CSINV;
6946         TVal = LHS;
6947         FVal = DAG.getConstant(0, dl, FVal.getValueType());
6948       }
6949     }
6950 
6951     SDValue CCVal;
6952     SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
6953     EVT VT = TVal.getValueType();
6954     return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp);
6955   }
6956 
6957   // Now we know we're dealing with FP values.
6958   assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||
6959          LHS.getValueType() == MVT::f64);
6960   assert(LHS.getValueType() == RHS.getValueType());
6961   EVT VT = TVal.getValueType();
6962   SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
6963 
6964   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
6965   // clean.  Some of them require two CSELs to implement.
6966   AArch64CC::CondCode CC1, CC2;
6967   changeFPCCToAArch64CC(CC, CC1, CC2);
6968 
6969   if (DAG.getTarget().Options.UnsafeFPMath) {
6970     // Transform "a == 0.0 ? 0.0 : x" to "a == 0.0 ? a : x" and
6971     // "a != 0.0 ? x : 0.0" to "a != 0.0 ? x : a" to avoid materializing 0.0.
6972     ConstantFPSDNode *RHSVal = dyn_cast<ConstantFPSDNode>(RHS);
6973     if (RHSVal && RHSVal->isZero()) {
6974       ConstantFPSDNode *CFVal = dyn_cast<ConstantFPSDNode>(FVal);
6975       ConstantFPSDNode *CTVal = dyn_cast<ConstantFPSDNode>(TVal);
6976 
6977       if ((CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
6978           CTVal && CTVal->isZero() && TVal.getValueType() == LHS.getValueType())
6979         TVal = LHS;
6980       else if ((CC == ISD::SETNE || CC == ISD::SETONE || CC == ISD::SETUNE) &&
6981                CFVal && CFVal->isZero() &&
6982                FVal.getValueType() == LHS.getValueType())
6983         FVal = LHS;
6984     }
6985   }
6986 
6987   // Emit first, and possibly only, CSEL.
6988   SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
6989   SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
6990 
6991   // If we need a second CSEL, emit it, using the output of the first as the
6992   // RHS.  We're effectively OR'ing the two CC's together.
6993   if (CC2 != AArch64CC::AL) {
6994     SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
6995     return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
6996   }
6997 
6998   // Otherwise, return the output of the first CSEL.
6999   return CS1;
7000 }
7001 
7002 SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
7003                                               SelectionDAG &DAG) const {
7004   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
7005   SDValue LHS = Op.getOperand(0);
7006   SDValue RHS = Op.getOperand(1);
7007   SDValue TVal = Op.getOperand(2);
7008   SDValue FVal = Op.getOperand(3);
7009   SDLoc DL(Op);
7010   return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
7011 }
7012 
7013 SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
7014                                            SelectionDAG &DAG) const {
7015   SDValue CCVal = Op->getOperand(0);
7016   SDValue TVal = Op->getOperand(1);
7017   SDValue FVal = Op->getOperand(2);
7018   SDLoc DL(Op);
7019 
7020   EVT Ty = Op.getValueType();
7021   if (Ty.isScalableVector()) {
7022     SDValue TruncCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, CCVal);
7023     MVT PredVT = MVT::getVectorVT(MVT::i1, Ty.getVectorElementCount());
7024     SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, TruncCC);
7025     return DAG.getNode(ISD::VSELECT, DL, Ty, SplatPred, TVal, FVal);
7026   }
7027 
7028   // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select
7029   // instruction.
7030   if (ISD::isOverflowIntrOpRes(CCVal)) {
7031     // Only lower legal XALUO ops.
7032     if (!DAG.getTargetLoweringInfo().isTypeLegal(CCVal->getValueType(0)))
7033       return SDValue();
7034 
7035     AArch64CC::CondCode OFCC;
7036     SDValue Value, Overflow;
7037     std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, CCVal.getValue(0), DAG);
7038     SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
7039 
7040     return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
7041                        CCVal, Overflow);
7042   }
7043 
7044   // Lower it the same way as we would lower a SELECT_CC node.
7045   ISD::CondCode CC;
7046   SDValue LHS, RHS;
7047   if (CCVal.getOpcode() == ISD::SETCC) {
7048     LHS = CCVal.getOperand(0);
7049     RHS = CCVal.getOperand(1);
7050     CC = cast<CondCodeSDNode>(CCVal.getOperand(2))->get();
7051   } else {
7052     LHS = CCVal;
7053     RHS = DAG.getConstant(0, DL, CCVal.getValueType());
7054     CC = ISD::SETNE;
7055   }
7056   return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
7057 }
7058 
7059 SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
7060                                               SelectionDAG &DAG) const {
7061   // Jump table entries as PC relative offsets. No additional tweaking
7062   // is necessary here. Just get the address of the jump table.
7063   JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
7064 
7065   if (getTargetMachine().getCodeModel() == CodeModel::Large &&
7066       !Subtarget->isTargetMachO()) {
7067     return getAddrLarge(JT, DAG);
7068   } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
7069     return getAddrTiny(JT, DAG);
7070   }
7071   return getAddr(JT, DAG);
7072 }
7073 
7074 SDValue AArch64TargetLowering::LowerBR_JT(SDValue Op,
7075                                           SelectionDAG &DAG) const {
7076   // Jump table entries as PC relative offsets. No additional tweaking
7077   // is necessary here. Just get the address of the jump table.
7078   SDLoc DL(Op);
7079   SDValue JT = Op.getOperand(1);
7080   SDValue Entry = Op.getOperand(2);
7081   int JTI = cast<JumpTableSDNode>(JT.getNode())->getIndex();
7082 
7083   auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
7084   AFI->setJumpTableEntryInfo(JTI, 4, nullptr);
7085 
7086   SDNode *Dest =
7087       DAG.getMachineNode(AArch64::JumpTableDest32, DL, MVT::i64, MVT::i64, JT,
7088                          Entry, DAG.getTargetJumpTable(JTI, MVT::i32));
7089   return DAG.getNode(ISD::BRIND, DL, MVT::Other, Op.getOperand(0),
7090                      SDValue(Dest, 0));
7091 }
7092 
7093 SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
7094                                                  SelectionDAG &DAG) const {
7095   ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
7096 
7097   if (getTargetMachine().getCodeModel() == CodeModel::Large) {
7098     // Use the GOT for the large code model on iOS.
7099     if (Subtarget->isTargetMachO()) {
7100       return getGOT(CP, DAG);
7101     }
7102     return getAddrLarge(CP, DAG);
7103   } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
7104     return getAddrTiny(CP, DAG);
7105   } else {
7106     return getAddr(CP, DAG);
7107   }
7108 }
7109 
7110 SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
7111                                                SelectionDAG &DAG) const {
7112   BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Op);
7113   if (getTargetMachine().getCodeModel() == CodeModel::Large &&
7114       !Subtarget->isTargetMachO()) {
7115     return getAddrLarge(BA, DAG);
7116   } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
7117     return getAddrTiny(BA, DAG);
7118   }
7119   return getAddr(BA, DAG);
7120 }
7121 
7122 SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
7123                                                  SelectionDAG &DAG) const {
7124   AArch64FunctionInfo *FuncInfo =
7125       DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
7126 
7127   SDLoc DL(Op);
7128   SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(),
7129                                  getPointerTy(DAG.getDataLayout()));
7130   FR = DAG.getZExtOrTrunc(FR, DL, getPointerMemTy(DAG.getDataLayout()));
7131   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
7132   return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
7133                       MachinePointerInfo(SV));
7134 }
7135 
7136 SDValue AArch64TargetLowering::LowerWin64_VASTART(SDValue Op,
7137                                                   SelectionDAG &DAG) const {
7138   AArch64FunctionInfo *FuncInfo =
7139       DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
7140 
7141   SDLoc DL(Op);
7142   SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsGPRSize() > 0
7143                                      ? FuncInfo->getVarArgsGPRIndex()
7144                                      : FuncInfo->getVarArgsStackIndex(),
7145                                  getPointerTy(DAG.getDataLayout()));
7146   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
7147   return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
7148                       MachinePointerInfo(SV));
7149 }
7150 
7151 SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
7152                                                   SelectionDAG &DAG) const {
7153   // The layout of the va_list struct is specified in the AArch64 Procedure Call
7154   // Standard, section B.3.
7155   MachineFunction &MF = DAG.getMachineFunction();
7156   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
7157   unsigned PtrSize = Subtarget->isTargetILP32() ? 4 : 8;
7158   auto PtrMemVT = getPointerMemTy(DAG.getDataLayout());
7159   auto PtrVT = getPointerTy(DAG.getDataLayout());
7160   SDLoc DL(Op);
7161 
7162   SDValue Chain = Op.getOperand(0);
7163   SDValue VAList = Op.getOperand(1);
7164   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
7165   SmallVector<SDValue, 4> MemOps;
7166 
7167   // void *__stack at offset 0
7168   unsigned Offset = 0;
7169   SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT);
7170   Stack = DAG.getZExtOrTrunc(Stack, DL, PtrMemVT);
7171   MemOps.push_back(DAG.getStore(Chain, DL, Stack, VAList,
7172                                 MachinePointerInfo(SV), Align(PtrSize)));
7173 
7174   // void *__gr_top at offset 8 (4 on ILP32)
7175   Offset += PtrSize;
7176   int GPRSize = FuncInfo->getVarArgsGPRSize();
7177   if (GPRSize > 0) {
7178     SDValue GRTop, GRTopAddr;
7179 
7180     GRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7181                             DAG.getConstant(Offset, DL, PtrVT));
7182 
7183     GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), PtrVT);
7184     GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop,
7185                         DAG.getConstant(GPRSize, DL, PtrVT));
7186     GRTop = DAG.getZExtOrTrunc(GRTop, DL, PtrMemVT);
7187 
7188     MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr,
7189                                   MachinePointerInfo(SV, Offset),
7190                                   Align(PtrSize)));
7191   }
7192 
7193   // void *__vr_top at offset 16 (8 on ILP32)
7194   Offset += PtrSize;
7195   int FPRSize = FuncInfo->getVarArgsFPRSize();
7196   if (FPRSize > 0) {
7197     SDValue VRTop, VRTopAddr;
7198     VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7199                             DAG.getConstant(Offset, DL, PtrVT));
7200 
7201     VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), PtrVT);
7202     VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop,
7203                         DAG.getConstant(FPRSize, DL, PtrVT));
7204     VRTop = DAG.getZExtOrTrunc(VRTop, DL, PtrMemVT);
7205 
7206     MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr,
7207                                   MachinePointerInfo(SV, Offset),
7208                                   Align(PtrSize)));
7209   }
7210 
7211   // int __gr_offs at offset 24 (12 on ILP32)
7212   Offset += PtrSize;
7213   SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7214                                    DAG.getConstant(Offset, DL, PtrVT));
7215   MemOps.push_back(
7216       DAG.getStore(Chain, DL, DAG.getConstant(-GPRSize, DL, MVT::i32),
7217                    GROffsAddr, MachinePointerInfo(SV, Offset), Align(4)));
7218 
7219   // int __vr_offs at offset 28 (16 on ILP32)
7220   Offset += 4;
7221   SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7222                                    DAG.getConstant(Offset, DL, PtrVT));
7223   MemOps.push_back(
7224       DAG.getStore(Chain, DL, DAG.getConstant(-FPRSize, DL, MVT::i32),
7225                    VROffsAddr, MachinePointerInfo(SV, Offset), Align(4)));
7226 
7227   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
7228 }
7229 
7230 SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
7231                                             SelectionDAG &DAG) const {
7232   MachineFunction &MF = DAG.getMachineFunction();
7233 
7234   if (Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv()))
7235     return LowerWin64_VASTART(Op, DAG);
7236   else if (Subtarget->isTargetDarwin())
7237     return LowerDarwin_VASTART(Op, DAG);
7238   else
7239     return LowerAAPCS_VASTART(Op, DAG);
7240 }
7241 
7242 SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
7243                                            SelectionDAG &DAG) const {
7244   // AAPCS has three pointers and two ints (= 32 bytes), Darwin has single
7245   // pointer.
7246   SDLoc DL(Op);
7247   unsigned PtrSize = Subtarget->isTargetILP32() ? 4 : 8;
7248   unsigned VaListSize =
7249       (Subtarget->isTargetDarwin() || Subtarget->isTargetWindows())
7250           ? PtrSize
7251           : Subtarget->isTargetILP32() ? 20 : 32;
7252   const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
7253   const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
7254 
7255   return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1), Op.getOperand(2),
7256                        DAG.getConstant(VaListSize, DL, MVT::i32),
7257                        Align(PtrSize), false, false, false,
7258                        MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
7259 }
7260 
7261 SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
7262   assert(Subtarget->isTargetDarwin() &&
7263          "automatic va_arg instruction only works on Darwin");
7264 
7265   const Value *V = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
7266   EVT VT = Op.getValueType();
7267   SDLoc DL(Op);
7268   SDValue Chain = Op.getOperand(0);
7269   SDValue Addr = Op.getOperand(1);
7270   MaybeAlign Align(Op.getConstantOperandVal(3));
7271   unsigned MinSlotSize = Subtarget->isTargetILP32() ? 4 : 8;
7272   auto PtrVT = getPointerTy(DAG.getDataLayout());
7273   auto PtrMemVT = getPointerMemTy(DAG.getDataLayout());
7274   SDValue VAList =
7275       DAG.getLoad(PtrMemVT, DL, Chain, Addr, MachinePointerInfo(V));
7276   Chain = VAList.getValue(1);
7277   VAList = DAG.getZExtOrTrunc(VAList, DL, PtrVT);
7278 
7279   if (VT.isScalableVector())
7280     report_fatal_error("Passing SVE types to variadic functions is "
7281                        "currently not supported");
7282 
7283   if (Align && *Align > MinSlotSize) {
7284     VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7285                          DAG.getConstant(Align->value() - 1, DL, PtrVT));
7286     VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList,
7287                          DAG.getConstant(-(int64_t)Align->value(), DL, PtrVT));
7288   }
7289 
7290   Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
7291   unsigned ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
7292 
7293   // Scalar integer and FP values smaller than 64 bits are implicitly extended
7294   // up to 64 bits.  At the very least, we have to increase the striding of the
7295   // vaargs list to match this, and for FP values we need to introduce
7296   // FP_ROUND nodes as well.
7297   if (VT.isInteger() && !VT.isVector())
7298     ArgSize = std::max(ArgSize, MinSlotSize);
7299   bool NeedFPTrunc = false;
7300   if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) {
7301     ArgSize = 8;
7302     NeedFPTrunc = true;
7303   }
7304 
7305   // Increment the pointer, VAList, to the next vaarg
7306   SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
7307                                DAG.getConstant(ArgSize, DL, PtrVT));
7308   VANext = DAG.getZExtOrTrunc(VANext, DL, PtrMemVT);
7309 
7310   // Store the incremented VAList to the legalized pointer
7311   SDValue APStore =
7312       DAG.getStore(Chain, DL, VANext, Addr, MachinePointerInfo(V));
7313 
7314   // Load the actual argument out of the pointer VAList
7315   if (NeedFPTrunc) {
7316     // Load the value as an f64.
7317     SDValue WideFP =
7318         DAG.getLoad(MVT::f64, DL, APStore, VAList, MachinePointerInfo());
7319     // Round the value down to an f32.
7320     SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
7321                                    DAG.getIntPtrConstant(1, DL));
7322     SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
7323     // Merge the rounded value with the chain output of the load.
7324     return DAG.getMergeValues(Ops, DL);
7325   }
7326 
7327   return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo());
7328 }
7329 
7330 SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
7331                                               SelectionDAG &DAG) const {
7332   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7333   MFI.setFrameAddressIsTaken(true);
7334 
7335   EVT VT = Op.getValueType();
7336   SDLoc DL(Op);
7337   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
7338   SDValue FrameAddr =
7339       DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, MVT::i64);
7340   while (Depth--)
7341     FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr,
7342                             MachinePointerInfo());
7343 
7344   if (Subtarget->isTargetILP32())
7345     FrameAddr = DAG.getNode(ISD::AssertZext, DL, MVT::i64, FrameAddr,
7346                             DAG.getValueType(VT));
7347 
7348   return FrameAddr;
7349 }
7350 
7351 SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
7352                                               SelectionDAG &DAG) const {
7353   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7354 
7355   EVT VT = getPointerTy(DAG.getDataLayout());
7356   SDLoc DL(Op);
7357   int FI = MFI.CreateFixedObject(4, 0, false);
7358   return DAG.getFrameIndex(FI, VT);
7359 }
7360 
7361 #define GET_REGISTER_MATCHER
7362 #include "AArch64GenAsmMatcher.inc"
7363 
7364 // FIXME? Maybe this could be a TableGen attribute on some registers and
7365 // this table could be generated automatically from RegInfo.
7366 Register AArch64TargetLowering::
7367 getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const {
7368   Register Reg = MatchRegisterName(RegName);
7369   if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
7370     const MCRegisterInfo *MRI = Subtarget->getRegisterInfo();
7371     unsigned DwarfRegNum = MRI->getDwarfRegNum(Reg, false);
7372     if (!Subtarget->isXRegisterReserved(DwarfRegNum))
7373       Reg = 0;
7374   }
7375   if (Reg)
7376     return Reg;
7377   report_fatal_error(Twine("Invalid register name \""
7378                               + StringRef(RegName)  + "\"."));
7379 }
7380 
7381 SDValue AArch64TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
7382                                                      SelectionDAG &DAG) const {
7383   DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
7384 
7385   EVT VT = Op.getValueType();
7386   SDLoc DL(Op);
7387 
7388   SDValue FrameAddr =
7389       DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
7390   SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
7391 
7392   return DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset);
7393 }
7394 
7395 SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
7396                                                SelectionDAG &DAG) const {
7397   MachineFunction &MF = DAG.getMachineFunction();
7398   MachineFrameInfo &MFI = MF.getFrameInfo();
7399   MFI.setReturnAddressIsTaken(true);
7400 
7401   EVT VT = Op.getValueType();
7402   SDLoc DL(Op);
7403   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
7404   SDValue ReturnAddress;
7405   if (Depth) {
7406     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
7407     SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
7408     ReturnAddress = DAG.getLoad(
7409         VT, DL, DAG.getEntryNode(),
7410         DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), MachinePointerInfo());
7411   } else {
7412     // Return LR, which contains the return address. Mark it an implicit
7413     // live-in.
7414     unsigned Reg = MF.addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
7415     ReturnAddress = DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
7416   }
7417 
7418   // The XPACLRI instruction assembles to a hint-space instruction before
7419   // Armv8.3-A therefore this instruction can be safely used for any pre
7420   // Armv8.3-A architectures. On Armv8.3-A and onwards XPACI is available so use
7421   // that instead.
7422   SDNode *St;
7423   if (Subtarget->hasPAuth()) {
7424     St = DAG.getMachineNode(AArch64::XPACI, DL, VT, ReturnAddress);
7425   } else {
7426     // XPACLRI operates on LR therefore we must move the operand accordingly.
7427     SDValue Chain =
7428         DAG.getCopyToReg(DAG.getEntryNode(), DL, AArch64::LR, ReturnAddress);
7429     St = DAG.getMachineNode(AArch64::XPACLRI, DL, VT, Chain);
7430   }
7431   return SDValue(St, 0);
7432 }
7433 
7434 /// LowerShiftRightParts - Lower SRA_PARTS, which returns two
7435 /// i64 values and take a 2 x i64 value to shift plus a shift amount.
7436 SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
7437                                                     SelectionDAG &DAG) const {
7438   assert(Op.getNumOperands() == 3 && "Not a double-shift!");
7439   EVT VT = Op.getValueType();
7440   unsigned VTBits = VT.getSizeInBits();
7441   SDLoc dl(Op);
7442   SDValue ShOpLo = Op.getOperand(0);
7443   SDValue ShOpHi = Op.getOperand(1);
7444   SDValue ShAmt = Op.getOperand(2);
7445   unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
7446 
7447   assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
7448 
7449   SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
7450                                  DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
7451   SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
7452 
7453   // Unfortunately, if ShAmt == 0, we just calculated "(SHL ShOpHi, 64)" which
7454   // is "undef". We wanted 0, so CSEL it directly.
7455   SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
7456                                ISD::SETEQ, dl, DAG);
7457   SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
7458   HiBitsForLo =
7459       DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64),
7460                   HiBitsForLo, CCVal, Cmp);
7461 
7462   SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
7463                                    DAG.getConstant(VTBits, dl, MVT::i64));
7464 
7465   SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
7466   SDValue LoForNormalShift =
7467       DAG.getNode(ISD::OR, dl, VT, LoBitsForLo, HiBitsForLo);
7468 
7469   Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE,
7470                        dl, DAG);
7471   CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
7472   SDValue LoForBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
7473   SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
7474                            LoForNormalShift, CCVal, Cmp);
7475 
7476   // AArch64 shifts larger than the register width are wrapped rather than
7477   // clamped, so we can't just emit "hi >> x".
7478   SDValue HiForNormalShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
7479   SDValue HiForBigShift =
7480       Opc == ISD::SRA
7481           ? DAG.getNode(Opc, dl, VT, ShOpHi,
7482                         DAG.getConstant(VTBits - 1, dl, MVT::i64))
7483           : DAG.getConstant(0, dl, VT);
7484   SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
7485                            HiForNormalShift, CCVal, Cmp);
7486 
7487   SDValue Ops[2] = { Lo, Hi };
7488   return DAG.getMergeValues(Ops, dl);
7489 }
7490 
7491 /// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
7492 /// i64 values and take a 2 x i64 value to shift plus a shift amount.
7493 SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
7494                                                    SelectionDAG &DAG) const {
7495   assert(Op.getNumOperands() == 3 && "Not a double-shift!");
7496   EVT VT = Op.getValueType();
7497   unsigned VTBits = VT.getSizeInBits();
7498   SDLoc dl(Op);
7499   SDValue ShOpLo = Op.getOperand(0);
7500   SDValue ShOpHi = Op.getOperand(1);
7501   SDValue ShAmt = Op.getOperand(2);
7502 
7503   assert(Op.getOpcode() == ISD::SHL_PARTS);
7504   SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
7505                                  DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
7506   SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
7507 
7508   // Unfortunately, if ShAmt == 0, we just calculated "(SRL ShOpLo, 64)" which
7509   // is "undef". We wanted 0, so CSEL it directly.
7510   SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
7511                                ISD::SETEQ, dl, DAG);
7512   SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
7513   LoBitsForHi =
7514       DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64),
7515                   LoBitsForHi, CCVal, Cmp);
7516 
7517   SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
7518                                    DAG.getConstant(VTBits, dl, MVT::i64));
7519   SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
7520   SDValue HiForNormalShift =
7521       DAG.getNode(ISD::OR, dl, VT, LoBitsForHi, HiBitsForHi);
7522 
7523   SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
7524 
7525   Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE,
7526                        dl, DAG);
7527   CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
7528   SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
7529                            HiForNormalShift, CCVal, Cmp);
7530 
7531   // AArch64 shifts of larger than register sizes are wrapped rather than
7532   // clamped, so we can't just emit "lo << a" if a is too big.
7533   SDValue LoForBigShift = DAG.getConstant(0, dl, VT);
7534   SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
7535   SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
7536                            LoForNormalShift, CCVal, Cmp);
7537 
7538   SDValue Ops[2] = { Lo, Hi };
7539   return DAG.getMergeValues(Ops, dl);
7540 }
7541 
7542 bool AArch64TargetLowering::isOffsetFoldingLegal(
7543     const GlobalAddressSDNode *GA) const {
7544   // Offsets are folded in the DAG combine rather than here so that we can
7545   // intelligently choose an offset based on the uses.
7546   return false;
7547 }
7548 
7549 bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
7550                                          bool OptForSize) const {
7551   bool IsLegal = false;
7552   // We can materialize #0.0 as fmov $Rd, XZR for 64-bit, 32-bit cases, and
7553   // 16-bit case when target has full fp16 support.
7554   // FIXME: We should be able to handle f128 as well with a clever lowering.
7555   const APInt ImmInt = Imm.bitcastToAPInt();
7556   if (VT == MVT::f64)
7557     IsLegal = AArch64_AM::getFP64Imm(ImmInt) != -1 || Imm.isPosZero();
7558   else if (VT == MVT::f32)
7559     IsLegal = AArch64_AM::getFP32Imm(ImmInt) != -1 || Imm.isPosZero();
7560   else if (VT == MVT::f16 && Subtarget->hasFullFP16())
7561     IsLegal = AArch64_AM::getFP16Imm(ImmInt) != -1 || Imm.isPosZero();
7562   // TODO: fmov h0, w0 is also legal, however on't have an isel pattern to
7563   //       generate that fmov.
7564 
7565   // If we can not materialize in immediate field for fmov, check if the
7566   // value can be encoded as the immediate operand of a logical instruction.
7567   // The immediate value will be created with either MOVZ, MOVN, or ORR.
7568   if (!IsLegal && (VT == MVT::f64 || VT == MVT::f32)) {
7569     // The cost is actually exactly the same for mov+fmov vs. adrp+ldr;
7570     // however the mov+fmov sequence is always better because of the reduced
7571     // cache pressure. The timings are still the same if you consider
7572     // movw+movk+fmov vs. adrp+ldr (it's one instruction longer, but the
7573     // movw+movk is fused). So we limit up to 2 instrdduction at most.
7574     SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
7575     AArch64_IMM::expandMOVImm(ImmInt.getZExtValue(), VT.getSizeInBits(),
7576 			      Insn);
7577     unsigned Limit = (OptForSize ? 1 : (Subtarget->hasFuseLiterals() ? 5 : 2));
7578     IsLegal = Insn.size() <= Limit;
7579   }
7580 
7581   LLVM_DEBUG(dbgs() << (IsLegal ? "Legal " : "Illegal ") << VT.getEVTString()
7582                     << " imm value: "; Imm.dump(););
7583   return IsLegal;
7584 }
7585 
7586 //===----------------------------------------------------------------------===//
7587 //                          AArch64 Optimization Hooks
7588 //===----------------------------------------------------------------------===//
7589 
7590 static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
7591                            SDValue Operand, SelectionDAG &DAG,
7592                            int &ExtraSteps) {
7593   EVT VT = Operand.getValueType();
7594   if (ST->hasNEON() &&
7595       (VT == MVT::f64 || VT == MVT::v1f64 || VT == MVT::v2f64 ||
7596        VT == MVT::f32 || VT == MVT::v1f32 ||
7597        VT == MVT::v2f32 || VT == MVT::v4f32)) {
7598     if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified)
7599       // For the reciprocal estimates, convergence is quadratic, so the number
7600       // of digits is doubled after each iteration.  In ARMv8, the accuracy of
7601       // the initial estimate is 2^-8.  Thus the number of extra steps to refine
7602       // the result for float (23 mantissa bits) is 2 and for double (52
7603       // mantissa bits) is 3.
7604       ExtraSteps = VT.getScalarType() == MVT::f64 ? 3 : 2;
7605 
7606     return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand);
7607   }
7608 
7609   return SDValue();
7610 }
7611 
7612 SDValue
7613 AArch64TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
7614                                         const DenormalMode &Mode) const {
7615   SDLoc DL(Op);
7616   EVT VT = Op.getValueType();
7617   EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7618   SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
7619   return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
7620 }
7621 
7622 SDValue
7623 AArch64TargetLowering::getSqrtResultForDenormInput(SDValue Op,
7624                                                    SelectionDAG &DAG) const {
7625   return Op;
7626 }
7627 
7628 SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
7629                                                SelectionDAG &DAG, int Enabled,
7630                                                int &ExtraSteps,
7631                                                bool &UseOneConst,
7632                                                bool Reciprocal) const {
7633   if (Enabled == ReciprocalEstimate::Enabled ||
7634       (Enabled == ReciprocalEstimate::Unspecified && Subtarget->useRSqrt()))
7635     if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRSQRTE, Operand,
7636                                        DAG, ExtraSteps)) {
7637       SDLoc DL(Operand);
7638       EVT VT = Operand.getValueType();
7639 
7640       SDNodeFlags Flags;
7641       Flags.setAllowReassociation(true);
7642 
7643       // Newton reciprocal square root iteration: E * 0.5 * (3 - X * E^2)
7644       // AArch64 reciprocal square root iteration instruction: 0.5 * (3 - M * N)
7645       for (int i = ExtraSteps; i > 0; --i) {
7646         SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
7647                                    Flags);
7648         Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, Flags);
7649         Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
7650       }
7651       if (!Reciprocal)
7652         Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags);
7653 
7654       ExtraSteps = 0;
7655       return Estimate;
7656     }
7657 
7658   return SDValue();
7659 }
7660 
7661 SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
7662                                                 SelectionDAG &DAG, int Enabled,
7663                                                 int &ExtraSteps) const {
7664   if (Enabled == ReciprocalEstimate::Enabled)
7665     if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRECPE, Operand,
7666                                        DAG, ExtraSteps)) {
7667       SDLoc DL(Operand);
7668       EVT VT = Operand.getValueType();
7669 
7670       SDNodeFlags Flags;
7671       Flags.setAllowReassociation(true);
7672 
7673       // Newton reciprocal iteration: E * (2 - X * E)
7674       // AArch64 reciprocal iteration instruction: (2 - M * N)
7675       for (int i = ExtraSteps; i > 0; --i) {
7676         SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
7677                                    Estimate, Flags);
7678         Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
7679       }
7680 
7681       ExtraSteps = 0;
7682       return Estimate;
7683     }
7684 
7685   return SDValue();
7686 }
7687 
7688 //===----------------------------------------------------------------------===//
7689 //                          AArch64 Inline Assembly Support
7690 //===----------------------------------------------------------------------===//
7691 
7692 // Table of Constraints
7693 // TODO: This is the current set of constraints supported by ARM for the
7694 // compiler, not all of them may make sense.
7695 //
7696 // r - A general register
7697 // w - An FP/SIMD register of some size in the range v0-v31
7698 // x - An FP/SIMD register of some size in the range v0-v15
7699 // I - Constant that can be used with an ADD instruction
7700 // J - Constant that can be used with a SUB instruction
7701 // K - Constant that can be used with a 32-bit logical instruction
7702 // L - Constant that can be used with a 64-bit logical instruction
7703 // M - Constant that can be used as a 32-bit MOV immediate
7704 // N - Constant that can be used as a 64-bit MOV immediate
7705 // Q - A memory reference with base register and no offset
7706 // S - A symbolic address
7707 // Y - Floating point constant zero
7708 // Z - Integer constant zero
7709 //
7710 //   Note that general register operands will be output using their 64-bit x
7711 // register name, whatever the size of the variable, unless the asm operand
7712 // is prefixed by the %w modifier. Floating-point and SIMD register operands
7713 // will be output with the v prefix unless prefixed by the %b, %h, %s, %d or
7714 // %q modifier.
7715 const char *AArch64TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
7716   // At this point, we have to lower this constraint to something else, so we
7717   // lower it to an "r" or "w". However, by doing this we will force the result
7718   // to be in register, while the X constraint is much more permissive.
7719   //
7720   // Although we are correct (we are free to emit anything, without
7721   // constraints), we might break use cases that would expect us to be more
7722   // efficient and emit something else.
7723   if (!Subtarget->hasFPARMv8())
7724     return "r";
7725 
7726   if (ConstraintVT.isFloatingPoint())
7727     return "w";
7728 
7729   if (ConstraintVT.isVector() &&
7730      (ConstraintVT.getSizeInBits() == 64 ||
7731       ConstraintVT.getSizeInBits() == 128))
7732     return "w";
7733 
7734   return "r";
7735 }
7736 
7737 enum PredicateConstraint {
7738   Upl,
7739   Upa,
7740   Invalid
7741 };
7742 
7743 static PredicateConstraint parsePredicateConstraint(StringRef Constraint) {
7744   PredicateConstraint P = PredicateConstraint::Invalid;
7745   if (Constraint == "Upa")
7746     P = PredicateConstraint::Upa;
7747   if (Constraint == "Upl")
7748     P = PredicateConstraint::Upl;
7749   return P;
7750 }
7751 
7752 /// getConstraintType - Given a constraint letter, return the type of
7753 /// constraint it is for this target.
7754 AArch64TargetLowering::ConstraintType
7755 AArch64TargetLowering::getConstraintType(StringRef Constraint) const {
7756   if (Constraint.size() == 1) {
7757     switch (Constraint[0]) {
7758     default:
7759       break;
7760     case 'x':
7761     case 'w':
7762     case 'y':
7763       return C_RegisterClass;
7764     // An address with a single base register. Due to the way we
7765     // currently handle addresses it is the same as 'r'.
7766     case 'Q':
7767       return C_Memory;
7768     case 'I':
7769     case 'J':
7770     case 'K':
7771     case 'L':
7772     case 'M':
7773     case 'N':
7774     case 'Y':
7775     case 'Z':
7776       return C_Immediate;
7777     case 'z':
7778     case 'S': // A symbolic address
7779       return C_Other;
7780     }
7781   } else if (parsePredicateConstraint(Constraint) !=
7782              PredicateConstraint::Invalid)
7783       return C_RegisterClass;
7784   return TargetLowering::getConstraintType(Constraint);
7785 }
7786 
7787 /// Examine constraint type and operand type and determine a weight value.
7788 /// This object must already have been set up with the operand type
7789 /// and the current alternative constraint selected.
7790 TargetLowering::ConstraintWeight
7791 AArch64TargetLowering::getSingleConstraintMatchWeight(
7792     AsmOperandInfo &info, const char *constraint) const {
7793   ConstraintWeight weight = CW_Invalid;
7794   Value *CallOperandVal = info.CallOperandVal;
7795   // If we don't have a value, we can't do a match,
7796   // but allow it at the lowest weight.
7797   if (!CallOperandVal)
7798     return CW_Default;
7799   Type *type = CallOperandVal->getType();
7800   // Look at the constraint type.
7801   switch (*constraint) {
7802   default:
7803     weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
7804     break;
7805   case 'x':
7806   case 'w':
7807   case 'y':
7808     if (type->isFloatingPointTy() || type->isVectorTy())
7809       weight = CW_Register;
7810     break;
7811   case 'z':
7812     weight = CW_Constant;
7813     break;
7814   case 'U':
7815     if (parsePredicateConstraint(constraint) != PredicateConstraint::Invalid)
7816       weight = CW_Register;
7817     break;
7818   }
7819   return weight;
7820 }
7821 
7822 std::pair<unsigned, const TargetRegisterClass *>
7823 AArch64TargetLowering::getRegForInlineAsmConstraint(
7824     const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
7825   if (Constraint.size() == 1) {
7826     switch (Constraint[0]) {
7827     case 'r':
7828       if (VT.isScalableVector())
7829         return std::make_pair(0U, nullptr);
7830       if (VT.getFixedSizeInBits() == 64)
7831         return std::make_pair(0U, &AArch64::GPR64commonRegClass);
7832       return std::make_pair(0U, &AArch64::GPR32commonRegClass);
7833     case 'w': {
7834       if (!Subtarget->hasFPARMv8())
7835         break;
7836       if (VT.isScalableVector()) {
7837         if (VT.getVectorElementType() != MVT::i1)
7838           return std::make_pair(0U, &AArch64::ZPRRegClass);
7839         return std::make_pair(0U, nullptr);
7840       }
7841       uint64_t VTSize = VT.getFixedSizeInBits();
7842       if (VTSize == 16)
7843         return std::make_pair(0U, &AArch64::FPR16RegClass);
7844       if (VTSize == 32)
7845         return std::make_pair(0U, &AArch64::FPR32RegClass);
7846       if (VTSize == 64)
7847         return std::make_pair(0U, &AArch64::FPR64RegClass);
7848       if (VTSize == 128)
7849         return std::make_pair(0U, &AArch64::FPR128RegClass);
7850       break;
7851     }
7852     // The instructions that this constraint is designed for can
7853     // only take 128-bit registers so just use that regclass.
7854     case 'x':
7855       if (!Subtarget->hasFPARMv8())
7856         break;
7857       if (VT.isScalableVector())
7858         return std::make_pair(0U, &AArch64::ZPR_4bRegClass);
7859       if (VT.getSizeInBits() == 128)
7860         return std::make_pair(0U, &AArch64::FPR128_loRegClass);
7861       break;
7862     case 'y':
7863       if (!Subtarget->hasFPARMv8())
7864         break;
7865       if (VT.isScalableVector())
7866         return std::make_pair(0U, &AArch64::ZPR_3bRegClass);
7867       break;
7868     }
7869   } else {
7870     PredicateConstraint PC = parsePredicateConstraint(Constraint);
7871     if (PC != PredicateConstraint::Invalid) {
7872       if (!VT.isScalableVector() || VT.getVectorElementType() != MVT::i1)
7873         return std::make_pair(0U, nullptr);
7874       bool restricted = (PC == PredicateConstraint::Upl);
7875       return restricted ? std::make_pair(0U, &AArch64::PPR_3bRegClass)
7876                         : std::make_pair(0U, &AArch64::PPRRegClass);
7877     }
7878   }
7879   if (StringRef("{cc}").equals_lower(Constraint))
7880     return std::make_pair(unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
7881 
7882   // Use the default implementation in TargetLowering to convert the register
7883   // constraint into a member of a register class.
7884   std::pair<unsigned, const TargetRegisterClass *> Res;
7885   Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
7886 
7887   // Not found as a standard register?
7888   if (!Res.second) {
7889     unsigned Size = Constraint.size();
7890     if ((Size == 4 || Size == 5) && Constraint[0] == '{' &&
7891         tolower(Constraint[1]) == 'v' && Constraint[Size - 1] == '}') {
7892       int RegNo;
7893       bool Failed = Constraint.slice(2, Size - 1).getAsInteger(10, RegNo);
7894       if (!Failed && RegNo >= 0 && RegNo <= 31) {
7895         // v0 - v31 are aliases of q0 - q31 or d0 - d31 depending on size.
7896         // By default we'll emit v0-v31 for this unless there's a modifier where
7897         // we'll emit the correct register as well.
7898         if (VT != MVT::Other && VT.getSizeInBits() == 64) {
7899           Res.first = AArch64::FPR64RegClass.getRegister(RegNo);
7900           Res.second = &AArch64::FPR64RegClass;
7901         } else {
7902           Res.first = AArch64::FPR128RegClass.getRegister(RegNo);
7903           Res.second = &AArch64::FPR128RegClass;
7904         }
7905       }
7906     }
7907   }
7908 
7909   if (Res.second && !Subtarget->hasFPARMv8() &&
7910       !AArch64::GPR32allRegClass.hasSubClassEq(Res.second) &&
7911       !AArch64::GPR64allRegClass.hasSubClassEq(Res.second))
7912     return std::make_pair(0U, nullptr);
7913 
7914   return Res;
7915 }
7916 
7917 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
7918 /// vector.  If it is invalid, don't add anything to Ops.
7919 void AArch64TargetLowering::LowerAsmOperandForConstraint(
7920     SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
7921     SelectionDAG &DAG) const {
7922   SDValue Result;
7923 
7924   // Currently only support length 1 constraints.
7925   if (Constraint.length() != 1)
7926     return;
7927 
7928   char ConstraintLetter = Constraint[0];
7929   switch (ConstraintLetter) {
7930   default:
7931     break;
7932 
7933   // This set of constraints deal with valid constants for various instructions.
7934   // Validate and return a target constant for them if we can.
7935   case 'z': {
7936     // 'z' maps to xzr or wzr so it needs an input of 0.
7937     if (!isNullConstant(Op))
7938       return;
7939 
7940     if (Op.getValueType() == MVT::i64)
7941       Result = DAG.getRegister(AArch64::XZR, MVT::i64);
7942     else
7943       Result = DAG.getRegister(AArch64::WZR, MVT::i32);
7944     break;
7945   }
7946   case 'S': {
7947     // An absolute symbolic address or label reference.
7948     if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
7949       Result = DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
7950                                           GA->getValueType(0));
7951     } else if (const BlockAddressSDNode *BA =
7952                    dyn_cast<BlockAddressSDNode>(Op)) {
7953       Result =
7954           DAG.getTargetBlockAddress(BA->getBlockAddress(), BA->getValueType(0));
7955     } else if (const ExternalSymbolSDNode *ES =
7956                    dyn_cast<ExternalSymbolSDNode>(Op)) {
7957       Result =
7958           DAG.getTargetExternalSymbol(ES->getSymbol(), ES->getValueType(0));
7959     } else
7960       return;
7961     break;
7962   }
7963 
7964   case 'I':
7965   case 'J':
7966   case 'K':
7967   case 'L':
7968   case 'M':
7969   case 'N':
7970     ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
7971     if (!C)
7972       return;
7973 
7974     // Grab the value and do some validation.
7975     uint64_t CVal = C->getZExtValue();
7976     switch (ConstraintLetter) {
7977     // The I constraint applies only to simple ADD or SUB immediate operands:
7978     // i.e. 0 to 4095 with optional shift by 12
7979     // The J constraint applies only to ADD or SUB immediates that would be
7980     // valid when negated, i.e. if [an add pattern] were to be output as a SUB
7981     // instruction [or vice versa], in other words -1 to -4095 with optional
7982     // left shift by 12.
7983     case 'I':
7984       if (isUInt<12>(CVal) || isShiftedUInt<12, 12>(CVal))
7985         break;
7986       return;
7987     case 'J': {
7988       uint64_t NVal = -C->getSExtValue();
7989       if (isUInt<12>(NVal) || isShiftedUInt<12, 12>(NVal)) {
7990         CVal = C->getSExtValue();
7991         break;
7992       }
7993       return;
7994     }
7995     // The K and L constraints apply *only* to logical immediates, including
7996     // what used to be the MOVI alias for ORR (though the MOVI alias has now
7997     // been removed and MOV should be used). So these constraints have to
7998     // distinguish between bit patterns that are valid 32-bit or 64-bit
7999     // "bitmask immediates": for example 0xaaaaaaaa is a valid bimm32 (K), but
8000     // not a valid bimm64 (L) where 0xaaaaaaaaaaaaaaaa would be valid, and vice
8001     // versa.
8002     case 'K':
8003       if (AArch64_AM::isLogicalImmediate(CVal, 32))
8004         break;
8005       return;
8006     case 'L':
8007       if (AArch64_AM::isLogicalImmediate(CVal, 64))
8008         break;
8009       return;
8010     // The M and N constraints are a superset of K and L respectively, for use
8011     // with the MOV (immediate) alias. As well as the logical immediates they
8012     // also match 32 or 64-bit immediates that can be loaded either using a
8013     // *single* MOVZ or MOVN , such as 32-bit 0x12340000, 0x00001234, 0xffffedca
8014     // (M) or 64-bit 0x1234000000000000 (N) etc.
8015     // As a note some of this code is liberally stolen from the asm parser.
8016     case 'M': {
8017       if (!isUInt<32>(CVal))
8018         return;
8019       if (AArch64_AM::isLogicalImmediate(CVal, 32))
8020         break;
8021       if ((CVal & 0xFFFF) == CVal)
8022         break;
8023       if ((CVal & 0xFFFF0000ULL) == CVal)
8024         break;
8025       uint64_t NCVal = ~(uint32_t)CVal;
8026       if ((NCVal & 0xFFFFULL) == NCVal)
8027         break;
8028       if ((NCVal & 0xFFFF0000ULL) == NCVal)
8029         break;
8030       return;
8031     }
8032     case 'N': {
8033       if (AArch64_AM::isLogicalImmediate(CVal, 64))
8034         break;
8035       if ((CVal & 0xFFFFULL) == CVal)
8036         break;
8037       if ((CVal & 0xFFFF0000ULL) == CVal)
8038         break;
8039       if ((CVal & 0xFFFF00000000ULL) == CVal)
8040         break;
8041       if ((CVal & 0xFFFF000000000000ULL) == CVal)
8042         break;
8043       uint64_t NCVal = ~CVal;
8044       if ((NCVal & 0xFFFFULL) == NCVal)
8045         break;
8046       if ((NCVal & 0xFFFF0000ULL) == NCVal)
8047         break;
8048       if ((NCVal & 0xFFFF00000000ULL) == NCVal)
8049         break;
8050       if ((NCVal & 0xFFFF000000000000ULL) == NCVal)
8051         break;
8052       return;
8053     }
8054     default:
8055       return;
8056     }
8057 
8058     // All assembler immediates are 64-bit integers.
8059     Result = DAG.getTargetConstant(CVal, SDLoc(Op), MVT::i64);
8060     break;
8061   }
8062 
8063   if (Result.getNode()) {
8064     Ops.push_back(Result);
8065     return;
8066   }
8067 
8068   return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
8069 }
8070 
8071 //===----------------------------------------------------------------------===//
8072 //                     AArch64 Advanced SIMD Support
8073 //===----------------------------------------------------------------------===//
8074 
8075 /// WidenVector - Given a value in the V64 register class, produce the
8076 /// equivalent value in the V128 register class.
8077 static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) {
8078   EVT VT = V64Reg.getValueType();
8079   unsigned NarrowSize = VT.getVectorNumElements();
8080   MVT EltTy = VT.getVectorElementType().getSimpleVT();
8081   MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize);
8082   SDLoc DL(V64Reg);
8083 
8084   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy),
8085                      V64Reg, DAG.getConstant(0, DL, MVT::i64));
8086 }
8087 
8088 /// getExtFactor - Determine the adjustment factor for the position when
8089 /// generating an "extract from vector registers" instruction.
8090 static unsigned getExtFactor(SDValue &V) {
8091   EVT EltType = V.getValueType().getVectorElementType();
8092   return EltType.getSizeInBits() / 8;
8093 }
8094 
8095 /// NarrowVector - Given a value in the V128 register class, produce the
8096 /// equivalent value in the V64 register class.
8097 static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
8098   EVT VT = V128Reg.getValueType();
8099   unsigned WideSize = VT.getVectorNumElements();
8100   MVT EltTy = VT.getVectorElementType().getSimpleVT();
8101   MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2);
8102   SDLoc DL(V128Reg);
8103 
8104   return DAG.getTargetExtractSubreg(AArch64::dsub, DL, NarrowTy, V128Reg);
8105 }
8106 
8107 // Gather data to see if the operation can be modelled as a
8108 // shuffle in combination with VEXTs.
8109 SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
8110                                                   SelectionDAG &DAG) const {
8111   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
8112   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::ReconstructShuffle\n");
8113   SDLoc dl(Op);
8114   EVT VT = Op.getValueType();
8115   assert(!VT.isScalableVector() &&
8116          "Scalable vectors cannot be used with ISD::BUILD_VECTOR");
8117   unsigned NumElts = VT.getVectorNumElements();
8118 
8119   struct ShuffleSourceInfo {
8120     SDValue Vec;
8121     unsigned MinElt;
8122     unsigned MaxElt;
8123 
8124     // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
8125     // be compatible with the shuffle we intend to construct. As a result
8126     // ShuffleVec will be some sliding window into the original Vec.
8127     SDValue ShuffleVec;
8128 
8129     // Code should guarantee that element i in Vec starts at element "WindowBase
8130     // + i * WindowScale in ShuffleVec".
8131     int WindowBase;
8132     int WindowScale;
8133 
8134     ShuffleSourceInfo(SDValue Vec)
8135       : Vec(Vec), MinElt(std::numeric_limits<unsigned>::max()), MaxElt(0),
8136           ShuffleVec(Vec), WindowBase(0), WindowScale(1) {}
8137 
8138     bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
8139   };
8140 
8141   // First gather all vectors used as an immediate source for this BUILD_VECTOR
8142   // node.
8143   SmallVector<ShuffleSourceInfo, 2> Sources;
8144   for (unsigned i = 0; i < NumElts; ++i) {
8145     SDValue V = Op.getOperand(i);
8146     if (V.isUndef())
8147       continue;
8148     else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
8149              !isa<ConstantSDNode>(V.getOperand(1))) {
8150       LLVM_DEBUG(
8151           dbgs() << "Reshuffle failed: "
8152                     "a shuffle can only come from building a vector from "
8153                     "various elements of other vectors, provided their "
8154                     "indices are constant\n");
8155       return SDValue();
8156     }
8157 
8158     // Add this element source to the list if it's not already there.
8159     SDValue SourceVec = V.getOperand(0);
8160     auto Source = find(Sources, SourceVec);
8161     if (Source == Sources.end())
8162       Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec));
8163 
8164     // Update the minimum and maximum lane number seen.
8165     unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue();
8166     Source->MinElt = std::min(Source->MinElt, EltNo);
8167     Source->MaxElt = std::max(Source->MaxElt, EltNo);
8168   }
8169 
8170   if (Sources.size() > 2) {
8171     LLVM_DEBUG(
8172         dbgs() << "Reshuffle failed: currently only do something sane when at "
8173                   "most two source vectors are involved\n");
8174     return SDValue();
8175   }
8176 
8177   // Find out the smallest element size among result and two sources, and use
8178   // it as element size to build the shuffle_vector.
8179   EVT SmallestEltTy = VT.getVectorElementType();
8180   for (auto &Source : Sources) {
8181     EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
8182     if (SrcEltTy.bitsLT(SmallestEltTy)) {
8183       SmallestEltTy = SrcEltTy;
8184     }
8185   }
8186   unsigned ResMultiplier =
8187       VT.getScalarSizeInBits() / SmallestEltTy.getFixedSizeInBits();
8188   uint64_t VTSize = VT.getFixedSizeInBits();
8189   NumElts = VTSize / SmallestEltTy.getFixedSizeInBits();
8190   EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts);
8191 
8192   // If the source vector is too wide or too narrow, we may nevertheless be able
8193   // to construct a compatible shuffle either by concatenating it with UNDEF or
8194   // extracting a suitable range of elements.
8195   for (auto &Src : Sources) {
8196     EVT SrcVT = Src.ShuffleVec.getValueType();
8197 
8198     uint64_t SrcVTSize = SrcVT.getFixedSizeInBits();
8199     if (SrcVTSize == VTSize)
8200       continue;
8201 
8202     // This stage of the search produces a source with the same element type as
8203     // the original, but with a total width matching the BUILD_VECTOR output.
8204     EVT EltVT = SrcVT.getVectorElementType();
8205     unsigned NumSrcElts = VTSize / EltVT.getFixedSizeInBits();
8206     EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
8207 
8208     if (SrcVTSize < VTSize) {
8209       assert(2 * SrcVTSize == VTSize);
8210       // We can pad out the smaller vector for free, so if it's part of a
8211       // shuffle...
8212       Src.ShuffleVec =
8213           DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
8214                       DAG.getUNDEF(Src.ShuffleVec.getValueType()));
8215       continue;
8216     }
8217 
8218     if (SrcVTSize != 2 * VTSize) {
8219       LLVM_DEBUG(
8220           dbgs() << "Reshuffle failed: result vector too small to extract\n");
8221       return SDValue();
8222     }
8223 
8224     if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8225       LLVM_DEBUG(
8226           dbgs() << "Reshuffle failed: span too large for a VEXT to cope\n");
8227       return SDValue();
8228     }
8229 
8230     if (Src.MinElt >= NumSrcElts) {
8231       // The extraction can just take the second half
8232       Src.ShuffleVec =
8233           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8234                       DAG.getConstant(NumSrcElts, dl, MVT::i64));
8235       Src.WindowBase = -NumSrcElts;
8236     } else if (Src.MaxElt < NumSrcElts) {
8237       // The extraction can just take the first half
8238       Src.ShuffleVec =
8239           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8240                       DAG.getConstant(0, dl, MVT::i64));
8241     } else {
8242       // An actual VEXT is needed
8243       SDValue VEXTSrc1 =
8244           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8245                       DAG.getConstant(0, dl, MVT::i64));
8246       SDValue VEXTSrc2 =
8247           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8248                       DAG.getConstant(NumSrcElts, dl, MVT::i64));
8249       unsigned Imm = Src.MinElt * getExtFactor(VEXTSrc1);
8250 
8251       if (!SrcVT.is64BitVector()) {
8252         LLVM_DEBUG(
8253           dbgs() << "Reshuffle failed: don't know how to lower AArch64ISD::EXT "
8254                     "for SVE vectors.");
8255         return SDValue();
8256       }
8257 
8258       Src.ShuffleVec = DAG.getNode(AArch64ISD::EXT, dl, DestVT, VEXTSrc1,
8259                                    VEXTSrc2,
8260                                    DAG.getConstant(Imm, dl, MVT::i32));
8261       Src.WindowBase = -Src.MinElt;
8262     }
8263   }
8264 
8265   // Another possible incompatibility occurs from the vector element types. We
8266   // can fix this by bitcasting the source vectors to the same type we intend
8267   // for the shuffle.
8268   for (auto &Src : Sources) {
8269     EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8270     if (SrcEltTy == SmallestEltTy)
8271       continue;
8272     assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
8273     Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec);
8274     Src.WindowScale =
8275         SrcEltTy.getFixedSizeInBits() / SmallestEltTy.getFixedSizeInBits();
8276     Src.WindowBase *= Src.WindowScale;
8277   }
8278 
8279   // Final sanity check before we try to actually produce a shuffle.
8280   LLVM_DEBUG(for (auto Src
8281                   : Sources)
8282                  assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8283 
8284   // The stars all align, our next step is to produce the mask for the shuffle.
8285   SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
8286   int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
8287   for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
8288     SDValue Entry = Op.getOperand(i);
8289     if (Entry.isUndef())
8290       continue;
8291 
8292     auto Src = find(Sources, Entry.getOperand(0));
8293     int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
8294 
8295     // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
8296     // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
8297     // segment.
8298     EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
8299     int BitsDefined = std::min(OrigEltTy.getScalarSizeInBits(),
8300                                VT.getScalarSizeInBits());
8301     int LanesDefined = BitsDefined / BitsPerShuffleLane;
8302 
8303     // This source is expected to fill ResMultiplier lanes of the final shuffle,
8304     // starting at the appropriate offset.
8305     int *LaneMask = &Mask[i * ResMultiplier];
8306 
8307     int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8308     ExtractBase += NumElts * (Src - Sources.begin());
8309     for (int j = 0; j < LanesDefined; ++j)
8310       LaneMask[j] = ExtractBase + j;
8311   }
8312 
8313   // Final check before we try to produce nonsense...
8314   if (!isShuffleMaskLegal(Mask, ShuffleVT)) {
8315     LLVM_DEBUG(dbgs() << "Reshuffle failed: illegal shuffle mask\n");
8316     return SDValue();
8317   }
8318 
8319   SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
8320   for (unsigned i = 0; i < Sources.size(); ++i)
8321     ShuffleOps[i] = Sources[i].ShuffleVec;
8322 
8323   SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
8324                                          ShuffleOps[1], Mask);
8325   SDValue V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
8326 
8327   LLVM_DEBUG(dbgs() << "Reshuffle, creating node: "; Shuffle.dump();
8328              dbgs() << "Reshuffle, creating node: "; V.dump(););
8329 
8330   return V;
8331 }
8332 
8333 // check if an EXT instruction can handle the shuffle mask when the
8334 // vector sources of the shuffle are the same.
8335 static bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
8336   unsigned NumElts = VT.getVectorNumElements();
8337 
8338   // Assume that the first shuffle index is not UNDEF.  Fail if it is.
8339   if (M[0] < 0)
8340     return false;
8341 
8342   Imm = M[0];
8343 
8344   // If this is a VEXT shuffle, the immediate value is the index of the first
8345   // element.  The other shuffle indices must be the successive elements after
8346   // the first one.
8347   unsigned ExpectedElt = Imm;
8348   for (unsigned i = 1; i < NumElts; ++i) {
8349     // Increment the expected index.  If it wraps around, just follow it
8350     // back to index zero and keep going.
8351     ++ExpectedElt;
8352     if (ExpectedElt == NumElts)
8353       ExpectedElt = 0;
8354 
8355     if (M[i] < 0)
8356       continue; // ignore UNDEF indices
8357     if (ExpectedElt != static_cast<unsigned>(M[i]))
8358       return false;
8359   }
8360 
8361   return true;
8362 }
8363 
8364 /// Check if a vector shuffle corresponds to a DUP instructions with a larger
8365 /// element width than the vector lane type. If that is the case the function
8366 /// returns true and writes the value of the DUP instruction lane operand into
8367 /// DupLaneOp
8368 static bool isWideDUPMask(ArrayRef<int> M, EVT VT, unsigned BlockSize,
8369                           unsigned &DupLaneOp) {
8370   assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) &&
8371          "Only possible block sizes for wide DUP are: 16, 32, 64");
8372 
8373   if (BlockSize <= VT.getScalarSizeInBits())
8374     return false;
8375   if (BlockSize % VT.getScalarSizeInBits() != 0)
8376     return false;
8377   if (VT.getSizeInBits() % BlockSize != 0)
8378     return false;
8379 
8380   size_t SingleVecNumElements = VT.getVectorNumElements();
8381   size_t NumEltsPerBlock = BlockSize / VT.getScalarSizeInBits();
8382   size_t NumBlocks = VT.getSizeInBits() / BlockSize;
8383 
8384   // We are looking for masks like
8385   // [0, 1, 0, 1] or [2, 3, 2, 3] or [4, 5, 6, 7, 4, 5, 6, 7] where any element
8386   // might be replaced by 'undefined'. BlockIndices will eventually contain
8387   // lane indices of the duplicated block (i.e. [0, 1], [2, 3] and [4, 5, 6, 7]
8388   // for the above examples)
8389   SmallVector<int, 8> BlockElts(NumEltsPerBlock, -1);
8390   for (size_t BlockIndex = 0; BlockIndex < NumBlocks; BlockIndex++)
8391     for (size_t I = 0; I < NumEltsPerBlock; I++) {
8392       int Elt = M[BlockIndex * NumEltsPerBlock + I];
8393       if (Elt < 0)
8394         continue;
8395       // For now we don't support shuffles that use the second operand
8396       if ((unsigned)Elt >= SingleVecNumElements)
8397         return false;
8398       if (BlockElts[I] < 0)
8399         BlockElts[I] = Elt;
8400       else if (BlockElts[I] != Elt)
8401         return false;
8402     }
8403 
8404   // We found a candidate block (possibly with some undefs). It must be a
8405   // sequence of consecutive integers starting with a value divisible by
8406   // NumEltsPerBlock with some values possibly replaced by undef-s.
8407 
8408   // Find first non-undef element
8409   auto FirstRealEltIter = find_if(BlockElts, [](int Elt) { return Elt >= 0; });
8410   assert(FirstRealEltIter != BlockElts.end() &&
8411          "Shuffle with all-undefs must have been caught by previous cases, "
8412          "e.g. isSplat()");
8413   if (FirstRealEltIter == BlockElts.end()) {
8414     DupLaneOp = 0;
8415     return true;
8416   }
8417 
8418   // Index of FirstRealElt in BlockElts
8419   size_t FirstRealIndex = FirstRealEltIter - BlockElts.begin();
8420 
8421   if ((unsigned)*FirstRealEltIter < FirstRealIndex)
8422     return false;
8423   // BlockElts[0] must have the following value if it isn't undef:
8424   size_t Elt0 = *FirstRealEltIter - FirstRealIndex;
8425 
8426   // Check the first element
8427   if (Elt0 % NumEltsPerBlock != 0)
8428     return false;
8429   // Check that the sequence indeed consists of consecutive integers (modulo
8430   // undefs)
8431   for (size_t I = 0; I < NumEltsPerBlock; I++)
8432     if (BlockElts[I] >= 0 && (unsigned)BlockElts[I] != Elt0 + I)
8433       return false;
8434 
8435   DupLaneOp = Elt0 / NumEltsPerBlock;
8436   return true;
8437 }
8438 
8439 // check if an EXT instruction can handle the shuffle mask when the
8440 // vector sources of the shuffle are different.
8441 static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
8442                       unsigned &Imm) {
8443   // Look for the first non-undef element.
8444   const int *FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
8445 
8446   // Benefit form APInt to handle overflow when calculating expected element.
8447   unsigned NumElts = VT.getVectorNumElements();
8448   unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
8449   APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1);
8450   // The following shuffle indices must be the successive elements after the
8451   // first real element.
8452   const int *FirstWrongElt = std::find_if(FirstRealElt + 1, M.end(),
8453       [&](int Elt) {return Elt != ExpectedElt++ && Elt != -1;});
8454   if (FirstWrongElt != M.end())
8455     return false;
8456 
8457   // The index of an EXT is the first element if it is not UNDEF.
8458   // Watch out for the beginning UNDEFs. The EXT index should be the expected
8459   // value of the first element.  E.g.
8460   // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.
8461   // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.
8462   // ExpectedElt is the last mask index plus 1.
8463   Imm = ExpectedElt.getZExtValue();
8464 
8465   // There are two difference cases requiring to reverse input vectors.
8466   // For example, for vector <4 x i32> we have the following cases,
8467   // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>)
8468   // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>)
8469   // For both cases, we finally use mask <5, 6, 7, 0>, which requires
8470   // to reverse two input vectors.
8471   if (Imm < NumElts)
8472     ReverseEXT = true;
8473   else
8474     Imm -= NumElts;
8475 
8476   return true;
8477 }
8478 
8479 /// isREVMask - Check if a vector shuffle corresponds to a REV
8480 /// instruction with the specified blocksize.  (The order of the elements
8481 /// within each block of the vector is reversed.)
8482 static bool isREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) {
8483   assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) &&
8484          "Only possible block sizes for REV are: 16, 32, 64");
8485 
8486   unsigned EltSz = VT.getScalarSizeInBits();
8487   if (EltSz == 64)
8488     return false;
8489 
8490   unsigned NumElts = VT.getVectorNumElements();
8491   unsigned BlockElts = M[0] + 1;
8492   // If the first shuffle index is UNDEF, be optimistic.
8493   if (M[0] < 0)
8494     BlockElts = BlockSize / EltSz;
8495 
8496   if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz)
8497     return false;
8498 
8499   for (unsigned i = 0; i < NumElts; ++i) {
8500     if (M[i] < 0)
8501       continue; // ignore UNDEF indices
8502     if ((unsigned)M[i] != (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts))
8503       return false;
8504   }
8505 
8506   return true;
8507 }
8508 
8509 static bool isZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8510   unsigned NumElts = VT.getVectorNumElements();
8511   if (NumElts % 2 != 0)
8512     return false;
8513   WhichResult = (M[0] == 0 ? 0 : 1);
8514   unsigned Idx = WhichResult * NumElts / 2;
8515   for (unsigned i = 0; i != NumElts; i += 2) {
8516     if ((M[i] >= 0 && (unsigned)M[i] != Idx) ||
8517         (M[i + 1] >= 0 && (unsigned)M[i + 1] != Idx + NumElts))
8518       return false;
8519     Idx += 1;
8520   }
8521 
8522   return true;
8523 }
8524 
8525 static bool isUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8526   unsigned NumElts = VT.getVectorNumElements();
8527   WhichResult = (M[0] == 0 ? 0 : 1);
8528   for (unsigned i = 0; i != NumElts; ++i) {
8529     if (M[i] < 0)
8530       continue; // ignore UNDEF indices
8531     if ((unsigned)M[i] != 2 * i + WhichResult)
8532       return false;
8533   }
8534 
8535   return true;
8536 }
8537 
8538 static bool isTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8539   unsigned NumElts = VT.getVectorNumElements();
8540   if (NumElts % 2 != 0)
8541     return false;
8542   WhichResult = (M[0] == 0 ? 0 : 1);
8543   for (unsigned i = 0; i < NumElts; i += 2) {
8544     if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) ||
8545         (M[i + 1] >= 0 && (unsigned)M[i + 1] != i + NumElts + WhichResult))
8546       return false;
8547   }
8548   return true;
8549 }
8550 
8551 /// isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of
8552 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
8553 /// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
8554 static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8555   unsigned NumElts = VT.getVectorNumElements();
8556   if (NumElts % 2 != 0)
8557     return false;
8558   WhichResult = (M[0] == 0 ? 0 : 1);
8559   unsigned Idx = WhichResult * NumElts / 2;
8560   for (unsigned i = 0; i != NumElts; i += 2) {
8561     if ((M[i] >= 0 && (unsigned)M[i] != Idx) ||
8562         (M[i + 1] >= 0 && (unsigned)M[i + 1] != Idx))
8563       return false;
8564     Idx += 1;
8565   }
8566 
8567   return true;
8568 }
8569 
8570 /// isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of
8571 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
8572 /// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
8573 static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8574   unsigned Half = VT.getVectorNumElements() / 2;
8575   WhichResult = (M[0] == 0 ? 0 : 1);
8576   for (unsigned j = 0; j != 2; ++j) {
8577     unsigned Idx = WhichResult;
8578     for (unsigned i = 0; i != Half; ++i) {
8579       int MIdx = M[i + j * Half];
8580       if (MIdx >= 0 && (unsigned)MIdx != Idx)
8581         return false;
8582       Idx += 2;
8583     }
8584   }
8585 
8586   return true;
8587 }
8588 
8589 /// isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of
8590 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
8591 /// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
8592 static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
8593   unsigned NumElts = VT.getVectorNumElements();
8594   if (NumElts % 2 != 0)
8595     return false;
8596   WhichResult = (M[0] == 0 ? 0 : 1);
8597   for (unsigned i = 0; i < NumElts; i += 2) {
8598     if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) ||
8599         (M[i + 1] >= 0 && (unsigned)M[i + 1] != i + WhichResult))
8600       return false;
8601   }
8602   return true;
8603 }
8604 
8605 static bool isINSMask(ArrayRef<int> M, int NumInputElements,
8606                       bool &DstIsLeft, int &Anomaly) {
8607   if (M.size() != static_cast<size_t>(NumInputElements))
8608     return false;
8609 
8610   int NumLHSMatch = 0, NumRHSMatch = 0;
8611   int LastLHSMismatch = -1, LastRHSMismatch = -1;
8612 
8613   for (int i = 0; i < NumInputElements; ++i) {
8614     if (M[i] == -1) {
8615       ++NumLHSMatch;
8616       ++NumRHSMatch;
8617       continue;
8618     }
8619 
8620     if (M[i] == i)
8621       ++NumLHSMatch;
8622     else
8623       LastLHSMismatch = i;
8624 
8625     if (M[i] == i + NumInputElements)
8626       ++NumRHSMatch;
8627     else
8628       LastRHSMismatch = i;
8629   }
8630 
8631   if (NumLHSMatch == NumInputElements - 1) {
8632     DstIsLeft = true;
8633     Anomaly = LastLHSMismatch;
8634     return true;
8635   } else if (NumRHSMatch == NumInputElements - 1) {
8636     DstIsLeft = false;
8637     Anomaly = LastRHSMismatch;
8638     return true;
8639   }
8640 
8641   return false;
8642 }
8643 
8644 static bool isConcatMask(ArrayRef<int> Mask, EVT VT, bool SplitLHS) {
8645   if (VT.getSizeInBits() != 128)
8646     return false;
8647 
8648   unsigned NumElts = VT.getVectorNumElements();
8649 
8650   for (int I = 0, E = NumElts / 2; I != E; I++) {
8651     if (Mask[I] != I)
8652       return false;
8653   }
8654 
8655   int Offset = NumElts / 2;
8656   for (int I = NumElts / 2, E = NumElts; I != E; I++) {
8657     if (Mask[I] != I + SplitLHS * Offset)
8658       return false;
8659   }
8660 
8661   return true;
8662 }
8663 
8664 static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
8665   SDLoc DL(Op);
8666   EVT VT = Op.getValueType();
8667   SDValue V0 = Op.getOperand(0);
8668   SDValue V1 = Op.getOperand(1);
8669   ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask();
8670 
8671   if (VT.getVectorElementType() != V0.getValueType().getVectorElementType() ||
8672       VT.getVectorElementType() != V1.getValueType().getVectorElementType())
8673     return SDValue();
8674 
8675   bool SplitV0 = V0.getValueSizeInBits() == 128;
8676 
8677   if (!isConcatMask(Mask, VT, SplitV0))
8678     return SDValue();
8679 
8680   EVT CastVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
8681   if (SplitV0) {
8682     V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0,
8683                      DAG.getConstant(0, DL, MVT::i64));
8684   }
8685   if (V1.getValueSizeInBits() == 128) {
8686     V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1,
8687                      DAG.getConstant(0, DL, MVT::i64));
8688   }
8689   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1);
8690 }
8691 
8692 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
8693 /// the specified operations to build the shuffle.
8694 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
8695                                       SDValue RHS, SelectionDAG &DAG,
8696                                       const SDLoc &dl) {
8697   unsigned OpNum = (PFEntry >> 26) & 0x0F;
8698   unsigned LHSID = (PFEntry >> 13) & ((1 << 13) - 1);
8699   unsigned RHSID = (PFEntry >> 0) & ((1 << 13) - 1);
8700 
8701   enum {
8702     OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
8703     OP_VREV,
8704     OP_VDUP0,
8705     OP_VDUP1,
8706     OP_VDUP2,
8707     OP_VDUP3,
8708     OP_VEXT1,
8709     OP_VEXT2,
8710     OP_VEXT3,
8711     OP_VUZPL, // VUZP, left result
8712     OP_VUZPR, // VUZP, right result
8713     OP_VZIPL, // VZIP, left result
8714     OP_VZIPR, // VZIP, right result
8715     OP_VTRNL, // VTRN, left result
8716     OP_VTRNR  // VTRN, right result
8717   };
8718 
8719   if (OpNum == OP_COPY) {
8720     if (LHSID == (1 * 9 + 2) * 9 + 3)
8721       return LHS;
8722     assert(LHSID == ((4 * 9 + 5) * 9 + 6) * 9 + 7 && "Illegal OP_COPY!");
8723     return RHS;
8724   }
8725 
8726   SDValue OpLHS, OpRHS;
8727   OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
8728   OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
8729   EVT VT = OpLHS.getValueType();
8730 
8731   switch (OpNum) {
8732   default:
8733     llvm_unreachable("Unknown shuffle opcode!");
8734   case OP_VREV:
8735     // VREV divides the vector in half and swaps within the half.
8736     if (VT.getVectorElementType() == MVT::i32 ||
8737         VT.getVectorElementType() == MVT::f32)
8738       return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS);
8739     // vrev <4 x i16> -> REV32
8740     if (VT.getVectorElementType() == MVT::i16 ||
8741         VT.getVectorElementType() == MVT::f16 ||
8742         VT.getVectorElementType() == MVT::bf16)
8743       return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS);
8744     // vrev <4 x i8> -> REV16
8745     assert(VT.getVectorElementType() == MVT::i8);
8746     return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS);
8747   case OP_VDUP0:
8748   case OP_VDUP1:
8749   case OP_VDUP2:
8750   case OP_VDUP3: {
8751     EVT EltTy = VT.getVectorElementType();
8752     unsigned Opcode;
8753     if (EltTy == MVT::i8)
8754       Opcode = AArch64ISD::DUPLANE8;
8755     else if (EltTy == MVT::i16 || EltTy == MVT::f16 || EltTy == MVT::bf16)
8756       Opcode = AArch64ISD::DUPLANE16;
8757     else if (EltTy == MVT::i32 || EltTy == MVT::f32)
8758       Opcode = AArch64ISD::DUPLANE32;
8759     else if (EltTy == MVT::i64 || EltTy == MVT::f64)
8760       Opcode = AArch64ISD::DUPLANE64;
8761     else
8762       llvm_unreachable("Invalid vector element type?");
8763 
8764     if (VT.getSizeInBits() == 64)
8765       OpLHS = WidenVector(OpLHS, DAG);
8766     SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64);
8767     return DAG.getNode(Opcode, dl, VT, OpLHS, Lane);
8768   }
8769   case OP_VEXT1:
8770   case OP_VEXT2:
8771   case OP_VEXT3: {
8772     unsigned Imm = (OpNum - OP_VEXT1 + 1) * getExtFactor(OpLHS);
8773     return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS,
8774                        DAG.getConstant(Imm, dl, MVT::i32));
8775   }
8776   case OP_VUZPL:
8777     return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), OpLHS,
8778                        OpRHS);
8779   case OP_VUZPR:
8780     return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), OpLHS,
8781                        OpRHS);
8782   case OP_VZIPL:
8783     return DAG.getNode(AArch64ISD::ZIP1, dl, DAG.getVTList(VT, VT), OpLHS,
8784                        OpRHS);
8785   case OP_VZIPR:
8786     return DAG.getNode(AArch64ISD::ZIP2, dl, DAG.getVTList(VT, VT), OpLHS,
8787                        OpRHS);
8788   case OP_VTRNL:
8789     return DAG.getNode(AArch64ISD::TRN1, dl, DAG.getVTList(VT, VT), OpLHS,
8790                        OpRHS);
8791   case OP_VTRNR:
8792     return DAG.getNode(AArch64ISD::TRN2, dl, DAG.getVTList(VT, VT), OpLHS,
8793                        OpRHS);
8794   }
8795 }
8796 
8797 static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
8798                            SelectionDAG &DAG) {
8799   // Check to see if we can use the TBL instruction.
8800   SDValue V1 = Op.getOperand(0);
8801   SDValue V2 = Op.getOperand(1);
8802   SDLoc DL(Op);
8803 
8804   EVT EltVT = Op.getValueType().getVectorElementType();
8805   unsigned BytesPerElt = EltVT.getSizeInBits() / 8;
8806 
8807   SmallVector<SDValue, 8> TBLMask;
8808   for (int Val : ShuffleMask) {
8809     for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
8810       unsigned Offset = Byte + Val * BytesPerElt;
8811       TBLMask.push_back(DAG.getConstant(Offset, DL, MVT::i32));
8812     }
8813   }
8814 
8815   MVT IndexVT = MVT::v8i8;
8816   unsigned IndexLen = 8;
8817   if (Op.getValueSizeInBits() == 128) {
8818     IndexVT = MVT::v16i8;
8819     IndexLen = 16;
8820   }
8821 
8822   SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1);
8823   SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2);
8824 
8825   SDValue Shuffle;
8826   if (V2.getNode()->isUndef()) {
8827     if (IndexLen == 8)
8828       V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst);
8829     Shuffle = DAG.getNode(
8830         ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
8831         DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst,
8832         DAG.getBuildVector(IndexVT, DL,
8833                            makeArrayRef(TBLMask.data(), IndexLen)));
8834   } else {
8835     if (IndexLen == 8) {
8836       V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst);
8837       Shuffle = DAG.getNode(
8838           ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
8839           DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst,
8840           DAG.getBuildVector(IndexVT, DL,
8841                              makeArrayRef(TBLMask.data(), IndexLen)));
8842     } else {
8843       // FIXME: We cannot, for the moment, emit a TBL2 instruction because we
8844       // cannot currently represent the register constraints on the input
8845       // table registers.
8846       //  Shuffle = DAG.getNode(AArch64ISD::TBL2, DL, IndexVT, V1Cst, V2Cst,
8847       //                   DAG.getBuildVector(IndexVT, DL, &TBLMask[0],
8848       //                   IndexLen));
8849       Shuffle = DAG.getNode(
8850           ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
8851           DAG.getConstant(Intrinsic::aarch64_neon_tbl2, DL, MVT::i32), V1Cst,
8852           V2Cst, DAG.getBuildVector(IndexVT, DL,
8853                                     makeArrayRef(TBLMask.data(), IndexLen)));
8854     }
8855   }
8856   return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle);
8857 }
8858 
8859 static unsigned getDUPLANEOp(EVT EltType) {
8860   if (EltType == MVT::i8)
8861     return AArch64ISD::DUPLANE8;
8862   if (EltType == MVT::i16 || EltType == MVT::f16 || EltType == MVT::bf16)
8863     return AArch64ISD::DUPLANE16;
8864   if (EltType == MVT::i32 || EltType == MVT::f32)
8865     return AArch64ISD::DUPLANE32;
8866   if (EltType == MVT::i64 || EltType == MVT::f64)
8867     return AArch64ISD::DUPLANE64;
8868 
8869   llvm_unreachable("Invalid vector element type?");
8870 }
8871 
8872 static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT,
8873                             unsigned Opcode, SelectionDAG &DAG) {
8874   // Try to eliminate a bitcasted extract subvector before a DUPLANE.
8875   auto getScaledOffsetDup = [](SDValue BitCast, int &LaneC, MVT &CastVT) {
8876     // Match: dup (bitcast (extract_subv X, C)), LaneC
8877     if (BitCast.getOpcode() != ISD::BITCAST ||
8878         BitCast.getOperand(0).getOpcode() != ISD::EXTRACT_SUBVECTOR)
8879       return false;
8880 
8881     // The extract index must align in the destination type. That may not
8882     // happen if the bitcast is from narrow to wide type.
8883     SDValue Extract = BitCast.getOperand(0);
8884     unsigned ExtIdx = Extract.getConstantOperandVal(1);
8885     unsigned SrcEltBitWidth = Extract.getScalarValueSizeInBits();
8886     unsigned ExtIdxInBits = ExtIdx * SrcEltBitWidth;
8887     unsigned CastedEltBitWidth = BitCast.getScalarValueSizeInBits();
8888     if (ExtIdxInBits % CastedEltBitWidth != 0)
8889       return false;
8890 
8891     // Update the lane value by offsetting with the scaled extract index.
8892     LaneC += ExtIdxInBits / CastedEltBitWidth;
8893 
8894     // Determine the casted vector type of the wide vector input.
8895     // dup (bitcast (extract_subv X, C)), LaneC --> dup (bitcast X), LaneC'
8896     // Examples:
8897     // dup (bitcast (extract_subv v2f64 X, 1) to v2f32), 1 --> dup v4f32 X, 3
8898     // dup (bitcast (extract_subv v16i8 X, 8) to v4i16), 1 --> dup v8i16 X, 5
8899     unsigned SrcVecNumElts =
8900         Extract.getOperand(0).getValueSizeInBits() / CastedEltBitWidth;
8901     CastVT = MVT::getVectorVT(BitCast.getSimpleValueType().getScalarType(),
8902                               SrcVecNumElts);
8903     return true;
8904   };
8905   MVT CastVT;
8906   if (getScaledOffsetDup(V, Lane, CastVT)) {
8907     V = DAG.getBitcast(CastVT, V.getOperand(0).getOperand(0));
8908   } else if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
8909     // The lane is incremented by the index of the extract.
8910     // Example: dup v2f32 (extract v4f32 X, 2), 1 --> dup v4f32 X, 3
8911     Lane += V.getConstantOperandVal(1);
8912     V = V.getOperand(0);
8913   } else if (V.getOpcode() == ISD::CONCAT_VECTORS) {
8914     // The lane is decremented if we are splatting from the 2nd operand.
8915     // Example: dup v4i32 (concat v2i32 X, v2i32 Y), 3 --> dup v4i32 Y, 1
8916     unsigned Idx = Lane >= (int)VT.getVectorNumElements() / 2;
8917     Lane -= Idx * VT.getVectorNumElements() / 2;
8918     V = WidenVector(V.getOperand(Idx), DAG);
8919   } else if (VT.getSizeInBits() == 64) {
8920     // Widen the operand to 128-bit register with undef.
8921     V = WidenVector(V, DAG);
8922   }
8923   return DAG.getNode(Opcode, dl, VT, V, DAG.getConstant(Lane, dl, MVT::i64));
8924 }
8925 
8926 SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
8927                                                    SelectionDAG &DAG) const {
8928   SDLoc dl(Op);
8929   EVT VT = Op.getValueType();
8930 
8931   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
8932 
8933   // Convert shuffles that are directly supported on NEON to target-specific
8934   // DAG nodes, instead of keeping them as shuffles and matching them again
8935   // during code selection.  This is more efficient and avoids the possibility
8936   // of inconsistencies between legalization and selection.
8937   ArrayRef<int> ShuffleMask = SVN->getMask();
8938 
8939   SDValue V1 = Op.getOperand(0);
8940   SDValue V2 = Op.getOperand(1);
8941 
8942   if (SVN->isSplat()) {
8943     int Lane = SVN->getSplatIndex();
8944     // If this is undef splat, generate it via "just" vdup, if possible.
8945     if (Lane == -1)
8946       Lane = 0;
8947 
8948     if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR)
8949       return DAG.getNode(AArch64ISD::DUP, dl, V1.getValueType(),
8950                          V1.getOperand(0));
8951     // Test if V1 is a BUILD_VECTOR and the lane being referenced is a non-
8952     // constant. If so, we can just reference the lane's definition directly.
8953     if (V1.getOpcode() == ISD::BUILD_VECTOR &&
8954         !isa<ConstantSDNode>(V1.getOperand(Lane)))
8955       return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane));
8956 
8957     // Otherwise, duplicate from the lane of the input vector.
8958     unsigned Opcode = getDUPLANEOp(V1.getValueType().getVectorElementType());
8959     return constructDup(V1, Lane, dl, VT, Opcode, DAG);
8960   }
8961 
8962   // Check if the mask matches a DUP for a wider element
8963   for (unsigned LaneSize : {64U, 32U, 16U}) {
8964     unsigned Lane = 0;
8965     if (isWideDUPMask(ShuffleMask, VT, LaneSize, Lane)) {
8966       unsigned Opcode = LaneSize == 64 ? AArch64ISD::DUPLANE64
8967                                        : LaneSize == 32 ? AArch64ISD::DUPLANE32
8968                                                         : AArch64ISD::DUPLANE16;
8969       // Cast V1 to an integer vector with required lane size
8970       MVT NewEltTy = MVT::getIntegerVT(LaneSize);
8971       unsigned NewEltCount = VT.getSizeInBits() / LaneSize;
8972       MVT NewVecTy = MVT::getVectorVT(NewEltTy, NewEltCount);
8973       V1 = DAG.getBitcast(NewVecTy, V1);
8974       // Constuct the DUP instruction
8975       V1 = constructDup(V1, Lane, dl, NewVecTy, Opcode, DAG);
8976       // Cast back to the original type
8977       return DAG.getBitcast(VT, V1);
8978     }
8979   }
8980 
8981   if (isREVMask(ShuffleMask, VT, 64))
8982     return DAG.getNode(AArch64ISD::REV64, dl, V1.getValueType(), V1, V2);
8983   if (isREVMask(ShuffleMask, VT, 32))
8984     return DAG.getNode(AArch64ISD::REV32, dl, V1.getValueType(), V1, V2);
8985   if (isREVMask(ShuffleMask, VT, 16))
8986     return DAG.getNode(AArch64ISD::REV16, dl, V1.getValueType(), V1, V2);
8987 
8988   bool ReverseEXT = false;
8989   unsigned Imm;
8990   if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) {
8991     if (ReverseEXT)
8992       std::swap(V1, V2);
8993     Imm *= getExtFactor(V1);
8994     return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V2,
8995                        DAG.getConstant(Imm, dl, MVT::i32));
8996   } else if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, VT, Imm)) {
8997     Imm *= getExtFactor(V1);
8998     return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V1,
8999                        DAG.getConstant(Imm, dl, MVT::i32));
9000   }
9001 
9002   unsigned WhichResult;
9003   if (isZIPMask(ShuffleMask, VT, WhichResult)) {
9004     unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2;
9005     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
9006   }
9007   if (isUZPMask(ShuffleMask, VT, WhichResult)) {
9008     unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
9009     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
9010   }
9011   if (isTRNMask(ShuffleMask, VT, WhichResult)) {
9012     unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
9013     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
9014   }
9015 
9016   if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
9017     unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2;
9018     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
9019   }
9020   if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
9021     unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
9022     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
9023   }
9024   if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
9025     unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
9026     return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
9027   }
9028 
9029   if (SDValue Concat = tryFormConcatFromShuffle(Op, DAG))
9030     return Concat;
9031 
9032   bool DstIsLeft;
9033   int Anomaly;
9034   int NumInputElements = V1.getValueType().getVectorNumElements();
9035   if (isINSMask(ShuffleMask, NumInputElements, DstIsLeft, Anomaly)) {
9036     SDValue DstVec = DstIsLeft ? V1 : V2;
9037     SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64);
9038 
9039     SDValue SrcVec = V1;
9040     int SrcLane = ShuffleMask[Anomaly];
9041     if (SrcLane >= NumInputElements) {
9042       SrcVec = V2;
9043       SrcLane -= VT.getVectorNumElements();
9044     }
9045     SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64);
9046 
9047     EVT ScalarVT = VT.getVectorElementType();
9048 
9049     if (ScalarVT.getFixedSizeInBits() < 32 && ScalarVT.isInteger())
9050       ScalarVT = MVT::i32;
9051 
9052     return DAG.getNode(
9053         ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
9054         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, SrcVec, SrcLaneV),
9055         DstLaneV);
9056   }
9057 
9058   // If the shuffle is not directly supported and it has 4 elements, use
9059   // the PerfectShuffle-generated table to synthesize it from other shuffles.
9060   unsigned NumElts = VT.getVectorNumElements();
9061   if (NumElts == 4) {
9062     unsigned PFIndexes[4];
9063     for (unsigned i = 0; i != 4; ++i) {
9064       if (ShuffleMask[i] < 0)
9065         PFIndexes[i] = 8;
9066       else
9067         PFIndexes[i] = ShuffleMask[i];
9068     }
9069 
9070     // Compute the index in the perfect shuffle table.
9071     unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
9072                             PFIndexes[2] * 9 + PFIndexes[3];
9073     unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
9074     unsigned Cost = (PFEntry >> 30);
9075 
9076     if (Cost <= 4)
9077       return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
9078   }
9079 
9080   return GenerateTBL(Op, ShuffleMask, DAG);
9081 }
9082 
9083 SDValue AArch64TargetLowering::LowerSTEP_VECTOR(SDValue Op,
9084                                                 SelectionDAG &DAG) const {
9085   SDLoc dl(Op);
9086   EVT VT = Op.getValueType();
9087   assert(VT.isScalableVector() &&
9088          "Only expect scalable vectors for STEP_VECTOR");
9089   assert(VT.getScalarType() != MVT::i1 &&
9090          "Vectors of i1 types not supported for STEP_VECTOR");
9091 
9092   SDValue StepVal = Op.getOperand(0);
9093   SDValue Zero = DAG.getConstant(0, dl, StepVal.getValueType());
9094   return DAG.getNode(AArch64ISD::INDEX_VECTOR, dl, VT, Zero, StepVal);
9095 }
9096 
9097 SDValue AArch64TargetLowering::LowerSPLAT_VECTOR(SDValue Op,
9098                                                  SelectionDAG &DAG) const {
9099   SDLoc dl(Op);
9100   EVT VT = Op.getValueType();
9101   EVT ElemVT = VT.getScalarType();
9102   SDValue SplatVal = Op.getOperand(0);
9103 
9104   if (useSVEForFixedLengthVectorVT(VT))
9105     return LowerToScalableOp(Op, DAG);
9106 
9107   // Extend input splat value where needed to fit into a GPR (32b or 64b only)
9108   // FPRs don't have this restriction.
9109   switch (ElemVT.getSimpleVT().SimpleTy) {
9110   case MVT::i1: {
9111     // The only legal i1 vectors are SVE vectors, so we can use SVE-specific
9112     // lowering code.
9113     if (auto *ConstVal = dyn_cast<ConstantSDNode>(SplatVal)) {
9114       if (ConstVal->isOne())
9115         return getPTrue(DAG, dl, VT, AArch64SVEPredPattern::all);
9116       // TODO: Add special case for constant false
9117     }
9118     // The general case of i1.  There isn't any natural way to do this,
9119     // so we use some trickery with whilelo.
9120     SplatVal = DAG.getAnyExtOrTrunc(SplatVal, dl, MVT::i64);
9121     SplatVal = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i64, SplatVal,
9122                            DAG.getValueType(MVT::i1));
9123     SDValue ID = DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo, dl,
9124                                        MVT::i64);
9125     return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, ID,
9126                        DAG.getConstant(0, dl, MVT::i64), SplatVal);
9127   }
9128   case MVT::i8:
9129   case MVT::i16:
9130   case MVT::i32:
9131     SplatVal = DAG.getAnyExtOrTrunc(SplatVal, dl, MVT::i32);
9132     break;
9133   case MVT::i64:
9134     SplatVal = DAG.getAnyExtOrTrunc(SplatVal, dl, MVT::i64);
9135     break;
9136   case MVT::f16:
9137   case MVT::bf16:
9138   case MVT::f32:
9139   case MVT::f64:
9140     // Fine as is
9141     break;
9142   default:
9143     report_fatal_error("Unsupported SPLAT_VECTOR input operand type");
9144   }
9145 
9146   return DAG.getNode(AArch64ISD::DUP, dl, VT, SplatVal);
9147 }
9148 
9149 SDValue AArch64TargetLowering::LowerDUPQLane(SDValue Op,
9150                                              SelectionDAG &DAG) const {
9151   SDLoc DL(Op);
9152 
9153   EVT VT = Op.getValueType();
9154   if (!isTypeLegal(VT) || !VT.isScalableVector())
9155     return SDValue();
9156 
9157   // Current lowering only supports the SVE-ACLE types.
9158   if (VT.getSizeInBits().getKnownMinSize() != AArch64::SVEBitsPerBlock)
9159     return SDValue();
9160 
9161   // The DUPQ operation is indepedent of element type so normalise to i64s.
9162   SDValue V = DAG.getNode(ISD::BITCAST, DL, MVT::nxv2i64, Op.getOperand(1));
9163   SDValue Idx128 = Op.getOperand(2);
9164 
9165   // DUPQ can be used when idx is in range.
9166   auto *CIdx = dyn_cast<ConstantSDNode>(Idx128);
9167   if (CIdx && (CIdx->getZExtValue() <= 3)) {
9168     SDValue CI = DAG.getTargetConstant(CIdx->getZExtValue(), DL, MVT::i64);
9169     SDNode *DUPQ =
9170         DAG.getMachineNode(AArch64::DUP_ZZI_Q, DL, MVT::nxv2i64, V, CI);
9171     return DAG.getNode(ISD::BITCAST, DL, VT, SDValue(DUPQ, 0));
9172   }
9173 
9174   // The ACLE says this must produce the same result as:
9175   //   svtbl(data, svadd_x(svptrue_b64(),
9176   //                       svand_x(svptrue_b64(), svindex_u64(0, 1), 1),
9177   //                       index * 2))
9178   SDValue One = DAG.getConstant(1, DL, MVT::i64);
9179   SDValue SplatOne = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, One);
9180 
9181   // create the vector 0,1,0,1,...
9182   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
9183   SDValue SV = DAG.getNode(AArch64ISD::INDEX_VECTOR,
9184                            DL, MVT::nxv2i64, Zero, One);
9185   SV = DAG.getNode(ISD::AND, DL, MVT::nxv2i64, SV, SplatOne);
9186 
9187   // create the vector idx64,idx64+1,idx64,idx64+1,...
9188   SDValue Idx64 = DAG.getNode(ISD::ADD, DL, MVT::i64, Idx128, Idx128);
9189   SDValue SplatIdx64 = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Idx64);
9190   SDValue ShuffleMask = DAG.getNode(ISD::ADD, DL, MVT::nxv2i64, SV, SplatIdx64);
9191 
9192   // create the vector Val[idx64],Val[idx64+1],Val[idx64],Val[idx64+1],...
9193   SDValue TBL = DAG.getNode(AArch64ISD::TBL, DL, MVT::nxv2i64, V, ShuffleMask);
9194   return DAG.getNode(ISD::BITCAST, DL, VT, TBL);
9195 }
9196 
9197 
9198 static bool resolveBuildVector(BuildVectorSDNode *BVN, APInt &CnstBits,
9199                                APInt &UndefBits) {
9200   EVT VT = BVN->getValueType(0);
9201   APInt SplatBits, SplatUndef;
9202   unsigned SplatBitSize;
9203   bool HasAnyUndefs;
9204   if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
9205     unsigned NumSplats = VT.getSizeInBits() / SplatBitSize;
9206 
9207     for (unsigned i = 0; i < NumSplats; ++i) {
9208       CnstBits <<= SplatBitSize;
9209       UndefBits <<= SplatBitSize;
9210       CnstBits |= SplatBits.zextOrTrunc(VT.getSizeInBits());
9211       UndefBits |= (SplatBits ^ SplatUndef).zextOrTrunc(VT.getSizeInBits());
9212     }
9213 
9214     return true;
9215   }
9216 
9217   return false;
9218 }
9219 
9220 // Try 64-bit splatted SIMD immediate.
9221 static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
9222                                  const APInt &Bits) {
9223   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9224     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9225     EVT VT = Op.getValueType();
9226     MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v2i64 : MVT::f64;
9227 
9228     if (AArch64_AM::isAdvSIMDModImmType10(Value)) {
9229       Value = AArch64_AM::encodeAdvSIMDModImmType10(Value);
9230 
9231       SDLoc dl(Op);
9232       SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
9233                                 DAG.getConstant(Value, dl, MVT::i32));
9234       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9235     }
9236   }
9237 
9238   return SDValue();
9239 }
9240 
9241 // Try 32-bit splatted SIMD immediate.
9242 static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
9243                                   const APInt &Bits,
9244                                   const SDValue *LHS = nullptr) {
9245   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9246     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9247     EVT VT = Op.getValueType();
9248     MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
9249     bool isAdvSIMDModImm = false;
9250     uint64_t Shift;
9251 
9252     if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType1(Value))) {
9253       Value = AArch64_AM::encodeAdvSIMDModImmType1(Value);
9254       Shift = 0;
9255     }
9256     else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType2(Value))) {
9257       Value = AArch64_AM::encodeAdvSIMDModImmType2(Value);
9258       Shift = 8;
9259     }
9260     else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType3(Value))) {
9261       Value = AArch64_AM::encodeAdvSIMDModImmType3(Value);
9262       Shift = 16;
9263     }
9264     else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType4(Value))) {
9265       Value = AArch64_AM::encodeAdvSIMDModImmType4(Value);
9266       Shift = 24;
9267     }
9268 
9269     if (isAdvSIMDModImm) {
9270       SDLoc dl(Op);
9271       SDValue Mov;
9272 
9273       if (LHS)
9274         Mov = DAG.getNode(NewOp, dl, MovTy, *LHS,
9275                           DAG.getConstant(Value, dl, MVT::i32),
9276                           DAG.getConstant(Shift, dl, MVT::i32));
9277       else
9278         Mov = DAG.getNode(NewOp, dl, MovTy,
9279                           DAG.getConstant(Value, dl, MVT::i32),
9280                           DAG.getConstant(Shift, dl, MVT::i32));
9281 
9282       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9283     }
9284   }
9285 
9286   return SDValue();
9287 }
9288 
9289 // Try 16-bit splatted SIMD immediate.
9290 static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
9291                                   const APInt &Bits,
9292                                   const SDValue *LHS = nullptr) {
9293   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9294     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9295     EVT VT = Op.getValueType();
9296     MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
9297     bool isAdvSIMDModImm = false;
9298     uint64_t Shift;
9299 
9300     if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType5(Value))) {
9301       Value = AArch64_AM::encodeAdvSIMDModImmType5(Value);
9302       Shift = 0;
9303     }
9304     else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType6(Value))) {
9305       Value = AArch64_AM::encodeAdvSIMDModImmType6(Value);
9306       Shift = 8;
9307     }
9308 
9309     if (isAdvSIMDModImm) {
9310       SDLoc dl(Op);
9311       SDValue Mov;
9312 
9313       if (LHS)
9314         Mov = DAG.getNode(NewOp, dl, MovTy, *LHS,
9315                           DAG.getConstant(Value, dl, MVT::i32),
9316                           DAG.getConstant(Shift, dl, MVT::i32));
9317       else
9318         Mov = DAG.getNode(NewOp, dl, MovTy,
9319                           DAG.getConstant(Value, dl, MVT::i32),
9320                           DAG.getConstant(Shift, dl, MVT::i32));
9321 
9322       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9323     }
9324   }
9325 
9326   return SDValue();
9327 }
9328 
9329 // Try 32-bit splatted SIMD immediate with shifted ones.
9330 static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op,
9331                                     SelectionDAG &DAG, const APInt &Bits) {
9332   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9333     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9334     EVT VT = Op.getValueType();
9335     MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
9336     bool isAdvSIMDModImm = false;
9337     uint64_t Shift;
9338 
9339     if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType7(Value))) {
9340       Value = AArch64_AM::encodeAdvSIMDModImmType7(Value);
9341       Shift = 264;
9342     }
9343     else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType8(Value))) {
9344       Value = AArch64_AM::encodeAdvSIMDModImmType8(Value);
9345       Shift = 272;
9346     }
9347 
9348     if (isAdvSIMDModImm) {
9349       SDLoc dl(Op);
9350       SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
9351                                 DAG.getConstant(Value, dl, MVT::i32),
9352                                 DAG.getConstant(Shift, dl, MVT::i32));
9353       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9354     }
9355   }
9356 
9357   return SDValue();
9358 }
9359 
9360 // Try 8-bit splatted SIMD immediate.
9361 static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
9362                                  const APInt &Bits) {
9363   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9364     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9365     EVT VT = Op.getValueType();
9366     MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8;
9367 
9368     if (AArch64_AM::isAdvSIMDModImmType9(Value)) {
9369       Value = AArch64_AM::encodeAdvSIMDModImmType9(Value);
9370 
9371       SDLoc dl(Op);
9372       SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
9373                                 DAG.getConstant(Value, dl, MVT::i32));
9374       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9375     }
9376   }
9377 
9378   return SDValue();
9379 }
9380 
9381 // Try FP splatted SIMD immediate.
9382 static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
9383                                   const APInt &Bits) {
9384   if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9385     uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
9386     EVT VT = Op.getValueType();
9387     bool isWide = (VT.getSizeInBits() == 128);
9388     MVT MovTy;
9389     bool isAdvSIMDModImm = false;
9390 
9391     if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType11(Value))) {
9392       Value = AArch64_AM::encodeAdvSIMDModImmType11(Value);
9393       MovTy = isWide ? MVT::v4f32 : MVT::v2f32;
9394     }
9395     else if (isWide &&
9396              (isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType12(Value))) {
9397       Value = AArch64_AM::encodeAdvSIMDModImmType12(Value);
9398       MovTy = MVT::v2f64;
9399     }
9400 
9401     if (isAdvSIMDModImm) {
9402       SDLoc dl(Op);
9403       SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
9404                                 DAG.getConstant(Value, dl, MVT::i32));
9405       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
9406     }
9407   }
9408 
9409   return SDValue();
9410 }
9411 
9412 // Specialized code to quickly find if PotentialBVec is a BuildVector that
9413 // consists of only the same constant int value, returned in reference arg
9414 // ConstVal
9415 static bool isAllConstantBuildVector(const SDValue &PotentialBVec,
9416                                      uint64_t &ConstVal) {
9417   BuildVectorSDNode *Bvec = dyn_cast<BuildVectorSDNode>(PotentialBVec);
9418   if (!Bvec)
9419     return false;
9420   ConstantSDNode *FirstElt = dyn_cast<ConstantSDNode>(Bvec->getOperand(0));
9421   if (!FirstElt)
9422     return false;
9423   EVT VT = Bvec->getValueType(0);
9424   unsigned NumElts = VT.getVectorNumElements();
9425   for (unsigned i = 1; i < NumElts; ++i)
9426     if (dyn_cast<ConstantSDNode>(Bvec->getOperand(i)) != FirstElt)
9427       return false;
9428   ConstVal = FirstElt->getZExtValue();
9429   return true;
9430 }
9431 
9432 static unsigned getIntrinsicID(const SDNode *N) {
9433   unsigned Opcode = N->getOpcode();
9434   switch (Opcode) {
9435   default:
9436     return Intrinsic::not_intrinsic;
9437   case ISD::INTRINSIC_WO_CHAIN: {
9438     unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
9439     if (IID < Intrinsic::num_intrinsics)
9440       return IID;
9441     return Intrinsic::not_intrinsic;
9442   }
9443   }
9444 }
9445 
9446 // Attempt to form a vector S[LR]I from (or (and X, BvecC1), (lsl Y, C2)),
9447 // to (SLI X, Y, C2), where X and Y have matching vector types, BvecC1 is a
9448 // BUILD_VECTORs with constant element C1, C2 is a constant, and:
9449 //   - for the SLI case: C1 == ~(Ones(ElemSizeInBits) << C2)
9450 //   - for the SRI case: C1 == ~(Ones(ElemSizeInBits) >> C2)
9451 // The (or (lsl Y, C2), (and X, BvecC1)) case is also handled.
9452 static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
9453   EVT VT = N->getValueType(0);
9454 
9455   if (!VT.isVector())
9456     return SDValue();
9457 
9458   SDLoc DL(N);
9459 
9460   SDValue And;
9461   SDValue Shift;
9462 
9463   SDValue FirstOp = N->getOperand(0);
9464   unsigned FirstOpc = FirstOp.getOpcode();
9465   SDValue SecondOp = N->getOperand(1);
9466   unsigned SecondOpc = SecondOp.getOpcode();
9467 
9468   // Is one of the operands an AND or a BICi? The AND may have been optimised to
9469   // a BICi in order to use an immediate instead of a register.
9470   // Is the other operand an shl or lshr? This will have been turned into:
9471   // AArch64ISD::VSHL vector, #shift or AArch64ISD::VLSHR vector, #shift.
9472   if ((FirstOpc == ISD::AND || FirstOpc == AArch64ISD::BICi) &&
9473       (SecondOpc == AArch64ISD::VSHL || SecondOpc == AArch64ISD::VLSHR)) {
9474     And = FirstOp;
9475     Shift = SecondOp;
9476 
9477   } else if ((SecondOpc == ISD::AND || SecondOpc == AArch64ISD::BICi) &&
9478              (FirstOpc == AArch64ISD::VSHL || FirstOpc == AArch64ISD::VLSHR)) {
9479     And = SecondOp;
9480     Shift = FirstOp;
9481   } else
9482     return SDValue();
9483 
9484   bool IsAnd = And.getOpcode() == ISD::AND;
9485   bool IsShiftRight = Shift.getOpcode() == AArch64ISD::VLSHR;
9486 
9487   // Is the shift amount constant?
9488   ConstantSDNode *C2node = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
9489   if (!C2node)
9490     return SDValue();
9491 
9492   uint64_t C1;
9493   if (IsAnd) {
9494     // Is the and mask vector all constant?
9495     if (!isAllConstantBuildVector(And.getOperand(1), C1))
9496       return SDValue();
9497   } else {
9498     // Reconstruct the corresponding AND immediate from the two BICi immediates.
9499     ConstantSDNode *C1nodeImm = dyn_cast<ConstantSDNode>(And.getOperand(1));
9500     ConstantSDNode *C1nodeShift = dyn_cast<ConstantSDNode>(And.getOperand(2));
9501     assert(C1nodeImm && C1nodeShift);
9502     C1 = ~(C1nodeImm->getZExtValue() << C1nodeShift->getZExtValue());
9503   }
9504 
9505   // Is C1 == ~(Ones(ElemSizeInBits) << C2) or
9506   // C1 == ~(Ones(ElemSizeInBits) >> C2), taking into account
9507   // how much one can shift elements of a particular size?
9508   uint64_t C2 = C2node->getZExtValue();
9509   unsigned ElemSizeInBits = VT.getScalarSizeInBits();
9510   if (C2 > ElemSizeInBits)
9511     return SDValue();
9512 
9513   APInt C1AsAPInt(ElemSizeInBits, C1);
9514   APInt RequiredC1 = IsShiftRight ? APInt::getHighBitsSet(ElemSizeInBits, C2)
9515                                   : APInt::getLowBitsSet(ElemSizeInBits, C2);
9516   if (C1AsAPInt != RequiredC1)
9517     return SDValue();
9518 
9519   SDValue X = And.getOperand(0);
9520   SDValue Y = Shift.getOperand(0);
9521 
9522   unsigned Inst = IsShiftRight ? AArch64ISD::VSRI : AArch64ISD::VSLI;
9523   SDValue ResultSLI = DAG.getNode(Inst, DL, VT, X, Y, Shift.getOperand(1));
9524 
9525   LLVM_DEBUG(dbgs() << "aarch64-lower: transformed: \n");
9526   LLVM_DEBUG(N->dump(&DAG));
9527   LLVM_DEBUG(dbgs() << "into: \n");
9528   LLVM_DEBUG(ResultSLI->dump(&DAG));
9529 
9530   ++NumShiftInserts;
9531   return ResultSLI;
9532 }
9533 
9534 SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
9535                                              SelectionDAG &DAG) const {
9536   if (useSVEForFixedLengthVectorVT(Op.getValueType()))
9537     return LowerToScalableOp(Op, DAG);
9538 
9539   // Attempt to form a vector S[LR]I from (or (and X, C1), (lsl Y, C2))
9540   if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG))
9541     return Res;
9542 
9543   EVT VT = Op.getValueType();
9544 
9545   SDValue LHS = Op.getOperand(0);
9546   BuildVectorSDNode *BVN =
9547       dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode());
9548   if (!BVN) {
9549     // OR commutes, so try swapping the operands.
9550     LHS = Op.getOperand(1);
9551     BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode());
9552   }
9553   if (!BVN)
9554     return Op;
9555 
9556   APInt DefBits(VT.getSizeInBits(), 0);
9557   APInt UndefBits(VT.getSizeInBits(), 0);
9558   if (resolveBuildVector(BVN, DefBits, UndefBits)) {
9559     SDValue NewOp;
9560 
9561     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::ORRi, Op, DAG,
9562                                     DefBits, &LHS)) ||
9563         (NewOp = tryAdvSIMDModImm16(AArch64ISD::ORRi, Op, DAG,
9564                                     DefBits, &LHS)))
9565       return NewOp;
9566 
9567     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::ORRi, Op, DAG,
9568                                     UndefBits, &LHS)) ||
9569         (NewOp = tryAdvSIMDModImm16(AArch64ISD::ORRi, Op, DAG,
9570                                     UndefBits, &LHS)))
9571       return NewOp;
9572   }
9573 
9574   // We can always fall back to a non-immediate OR.
9575   return Op;
9576 }
9577 
9578 // Normalize the operands of BUILD_VECTOR. The value of constant operands will
9579 // be truncated to fit element width.
9580 static SDValue NormalizeBuildVector(SDValue Op,
9581                                     SelectionDAG &DAG) {
9582   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
9583   SDLoc dl(Op);
9584   EVT VT = Op.getValueType();
9585   EVT EltTy= VT.getVectorElementType();
9586 
9587   if (EltTy.isFloatingPoint() || EltTy.getSizeInBits() > 16)
9588     return Op;
9589 
9590   SmallVector<SDValue, 16> Ops;
9591   for (SDValue Lane : Op->ops()) {
9592     // For integer vectors, type legalization would have promoted the
9593     // operands already. Otherwise, if Op is a floating-point splat
9594     // (with operands cast to integers), then the only possibilities
9595     // are constants and UNDEFs.
9596     if (auto *CstLane = dyn_cast<ConstantSDNode>(Lane)) {
9597       APInt LowBits(EltTy.getSizeInBits(),
9598                     CstLane->getZExtValue());
9599       Lane = DAG.getConstant(LowBits.getZExtValue(), dl, MVT::i32);
9600     } else if (Lane.getNode()->isUndef()) {
9601       Lane = DAG.getUNDEF(MVT::i32);
9602     } else {
9603       assert(Lane.getValueType() == MVT::i32 &&
9604              "Unexpected BUILD_VECTOR operand type");
9605     }
9606     Ops.push_back(Lane);
9607   }
9608   return DAG.getBuildVector(VT, dl, Ops);
9609 }
9610 
9611 static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG) {
9612   EVT VT = Op.getValueType();
9613 
9614   APInt DefBits(VT.getSizeInBits(), 0);
9615   APInt UndefBits(VT.getSizeInBits(), 0);
9616   BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
9617   if (resolveBuildVector(BVN, DefBits, UndefBits)) {
9618     SDValue NewOp;
9619     if ((NewOp = tryAdvSIMDModImm64(AArch64ISD::MOVIedit, Op, DAG, DefBits)) ||
9620         (NewOp = tryAdvSIMDModImm32(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
9621         (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MOVImsl, Op, DAG, DefBits)) ||
9622         (NewOp = tryAdvSIMDModImm16(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
9623         (NewOp = tryAdvSIMDModImm8(AArch64ISD::MOVI, Op, DAG, DefBits)) ||
9624         (NewOp = tryAdvSIMDModImmFP(AArch64ISD::FMOV, Op, DAG, DefBits)))
9625       return NewOp;
9626 
9627     DefBits = ~DefBits;
9628     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::MVNIshift, Op, DAG, DefBits)) ||
9629         (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MVNImsl, Op, DAG, DefBits)) ||
9630         (NewOp = tryAdvSIMDModImm16(AArch64ISD::MVNIshift, Op, DAG, DefBits)))
9631       return NewOp;
9632 
9633     DefBits = UndefBits;
9634     if ((NewOp = tryAdvSIMDModImm64(AArch64ISD::MOVIedit, Op, DAG, DefBits)) ||
9635         (NewOp = tryAdvSIMDModImm32(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
9636         (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MOVImsl, Op, DAG, DefBits)) ||
9637         (NewOp = tryAdvSIMDModImm16(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
9638         (NewOp = tryAdvSIMDModImm8(AArch64ISD::MOVI, Op, DAG, DefBits)) ||
9639         (NewOp = tryAdvSIMDModImmFP(AArch64ISD::FMOV, Op, DAG, DefBits)))
9640       return NewOp;
9641 
9642     DefBits = ~UndefBits;
9643     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::MVNIshift, Op, DAG, DefBits)) ||
9644         (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MVNImsl, Op, DAG, DefBits)) ||
9645         (NewOp = tryAdvSIMDModImm16(AArch64ISD::MVNIshift, Op, DAG, DefBits)))
9646       return NewOp;
9647   }
9648 
9649   return SDValue();
9650 }
9651 
9652 SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
9653                                                  SelectionDAG &DAG) const {
9654   EVT VT = Op.getValueType();
9655 
9656   // Try to build a simple constant vector.
9657   Op = NormalizeBuildVector(Op, DAG);
9658   if (VT.isInteger()) {
9659     // Certain vector constants, used to express things like logical NOT and
9660     // arithmetic NEG, are passed through unmodified.  This allows special
9661     // patterns for these operations to match, which will lower these constants
9662     // to whatever is proven necessary.
9663     BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
9664     if (BVN->isConstant())
9665       if (ConstantSDNode *Const = BVN->getConstantSplatNode()) {
9666         unsigned BitSize = VT.getVectorElementType().getSizeInBits();
9667         APInt Val(BitSize,
9668                   Const->getAPIntValue().zextOrTrunc(BitSize).getZExtValue());
9669         if (Val.isNullValue() || Val.isAllOnesValue())
9670           return Op;
9671       }
9672   }
9673 
9674   if (SDValue V = ConstantBuildVector(Op, DAG))
9675     return V;
9676 
9677   // Scan through the operands to find some interesting properties we can
9678   // exploit:
9679   //   1) If only one value is used, we can use a DUP, or
9680   //   2) if only the low element is not undef, we can just insert that, or
9681   //   3) if only one constant value is used (w/ some non-constant lanes),
9682   //      we can splat the constant value into the whole vector then fill
9683   //      in the non-constant lanes.
9684   //   4) FIXME: If different constant values are used, but we can intelligently
9685   //             select the values we'll be overwriting for the non-constant
9686   //             lanes such that we can directly materialize the vector
9687   //             some other way (MOVI, e.g.), we can be sneaky.
9688   //   5) if all operands are EXTRACT_VECTOR_ELT, check for VUZP.
9689   SDLoc dl(Op);
9690   unsigned NumElts = VT.getVectorNumElements();
9691   bool isOnlyLowElement = true;
9692   bool usesOnlyOneValue = true;
9693   bool usesOnlyOneConstantValue = true;
9694   bool isConstant = true;
9695   bool AllLanesExtractElt = true;
9696   unsigned NumConstantLanes = 0;
9697   unsigned NumDifferentLanes = 0;
9698   unsigned NumUndefLanes = 0;
9699   SDValue Value;
9700   SDValue ConstantValue;
9701   for (unsigned i = 0; i < NumElts; ++i) {
9702     SDValue V = Op.getOperand(i);
9703     if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
9704       AllLanesExtractElt = false;
9705     if (V.isUndef()) {
9706       ++NumUndefLanes;
9707       continue;
9708     }
9709     if (i > 0)
9710       isOnlyLowElement = false;
9711     if (!isIntOrFPConstant(V))
9712       isConstant = false;
9713 
9714     if (isIntOrFPConstant(V)) {
9715       ++NumConstantLanes;
9716       if (!ConstantValue.getNode())
9717         ConstantValue = V;
9718       else if (ConstantValue != V)
9719         usesOnlyOneConstantValue = false;
9720     }
9721 
9722     if (!Value.getNode())
9723       Value = V;
9724     else if (V != Value) {
9725       usesOnlyOneValue = false;
9726       ++NumDifferentLanes;
9727     }
9728   }
9729 
9730   if (!Value.getNode()) {
9731     LLVM_DEBUG(
9732         dbgs() << "LowerBUILD_VECTOR: value undefined, creating undef node\n");
9733     return DAG.getUNDEF(VT);
9734   }
9735 
9736   // Convert BUILD_VECTOR where all elements but the lowest are undef into
9737   // SCALAR_TO_VECTOR, except for when we have a single-element constant vector
9738   // as SimplifyDemandedBits will just turn that back into BUILD_VECTOR.
9739   if (isOnlyLowElement && !(NumElts == 1 && isIntOrFPConstant(Value))) {
9740     LLVM_DEBUG(dbgs() << "LowerBUILD_VECTOR: only low element used, creating 1 "
9741                          "SCALAR_TO_VECTOR node\n");
9742     return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
9743   }
9744 
9745   if (AllLanesExtractElt) {
9746     SDNode *Vector = nullptr;
9747     bool Even = false;
9748     bool Odd = false;
9749     // Check whether the extract elements match the Even pattern <0,2,4,...> or
9750     // the Odd pattern <1,3,5,...>.
9751     for (unsigned i = 0; i < NumElts; ++i) {
9752       SDValue V = Op.getOperand(i);
9753       const SDNode *N = V.getNode();
9754       if (!isa<ConstantSDNode>(N->getOperand(1)))
9755         break;
9756       SDValue N0 = N->getOperand(0);
9757 
9758       // All elements are extracted from the same vector.
9759       if (!Vector) {
9760         Vector = N0.getNode();
9761         // Check that the type of EXTRACT_VECTOR_ELT matches the type of
9762         // BUILD_VECTOR.
9763         if (VT.getVectorElementType() !=
9764             N0.getValueType().getVectorElementType())
9765           break;
9766       } else if (Vector != N0.getNode()) {
9767         Odd = false;
9768         Even = false;
9769         break;
9770       }
9771 
9772       // Extracted values are either at Even indices <0,2,4,...> or at Odd
9773       // indices <1,3,5,...>.
9774       uint64_t Val = N->getConstantOperandVal(1);
9775       if (Val == 2 * i) {
9776         Even = true;
9777         continue;
9778       }
9779       if (Val - 1 == 2 * i) {
9780         Odd = true;
9781         continue;
9782       }
9783 
9784       // Something does not match: abort.
9785       Odd = false;
9786       Even = false;
9787       break;
9788     }
9789     if (Even || Odd) {
9790       SDValue LHS =
9791           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
9792                       DAG.getConstant(0, dl, MVT::i64));
9793       SDValue RHS =
9794           DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
9795                       DAG.getConstant(NumElts, dl, MVT::i64));
9796 
9797       if (Even && !Odd)
9798         return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), LHS,
9799                            RHS);
9800       if (Odd && !Even)
9801         return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), LHS,
9802                            RHS);
9803     }
9804   }
9805 
9806   // Use DUP for non-constant splats. For f32 constant splats, reduce to
9807   // i32 and try again.
9808   if (usesOnlyOneValue) {
9809     if (!isConstant) {
9810       if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9811           Value.getValueType() != VT) {
9812         LLVM_DEBUG(
9813             dbgs() << "LowerBUILD_VECTOR: use DUP for non-constant splats\n");
9814         return DAG.getNode(AArch64ISD::DUP, dl, VT, Value);
9815       }
9816 
9817       // This is actually a DUPLANExx operation, which keeps everything vectory.
9818 
9819       SDValue Lane = Value.getOperand(1);
9820       Value = Value.getOperand(0);
9821       if (Value.getValueSizeInBits() == 64) {
9822         LLVM_DEBUG(
9823             dbgs() << "LowerBUILD_VECTOR: DUPLANE works on 128-bit vectors, "
9824                       "widening it\n");
9825         Value = WidenVector(Value, DAG);
9826       }
9827 
9828       unsigned Opcode = getDUPLANEOp(VT.getVectorElementType());
9829       return DAG.getNode(Opcode, dl, VT, Value, Lane);
9830     }
9831 
9832     if (VT.getVectorElementType().isFloatingPoint()) {
9833       SmallVector<SDValue, 8> Ops;
9834       EVT EltTy = VT.getVectorElementType();
9835       assert ((EltTy == MVT::f16 || EltTy == MVT::bf16 || EltTy == MVT::f32 ||
9836                EltTy == MVT::f64) && "Unsupported floating-point vector type");
9837       LLVM_DEBUG(
9838           dbgs() << "LowerBUILD_VECTOR: float constant splats, creating int "
9839                     "BITCASTS, and try again\n");
9840       MVT NewType = MVT::getIntegerVT(EltTy.getSizeInBits());
9841       for (unsigned i = 0; i < NumElts; ++i)
9842         Ops.push_back(DAG.getNode(ISD::BITCAST, dl, NewType, Op.getOperand(i)));
9843       EVT VecVT = EVT::getVectorVT(*DAG.getContext(), NewType, NumElts);
9844       SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
9845       LLVM_DEBUG(dbgs() << "LowerBUILD_VECTOR: trying to lower new vector: ";
9846                  Val.dump(););
9847       Val = LowerBUILD_VECTOR(Val, DAG);
9848       if (Val.getNode())
9849         return DAG.getNode(ISD::BITCAST, dl, VT, Val);
9850     }
9851   }
9852 
9853   // If we need to insert a small number of different non-constant elements and
9854   // the vector width is sufficiently large, prefer using DUP with the common
9855   // value and INSERT_VECTOR_ELT for the different lanes. If DUP is preferred,
9856   // skip the constant lane handling below.
9857   bool PreferDUPAndInsert =
9858       !isConstant && NumDifferentLanes >= 1 &&
9859       NumDifferentLanes < ((NumElts - NumUndefLanes) / 2) &&
9860       NumDifferentLanes >= NumConstantLanes;
9861 
9862   // If there was only one constant value used and for more than one lane,
9863   // start by splatting that value, then replace the non-constant lanes. This
9864   // is better than the default, which will perform a separate initialization
9865   // for each lane.
9866   if (!PreferDUPAndInsert && NumConstantLanes > 0 && usesOnlyOneConstantValue) {
9867     // Firstly, try to materialize the splat constant.
9868     SDValue Vec = DAG.getSplatBuildVector(VT, dl, ConstantValue),
9869             Val = ConstantBuildVector(Vec, DAG);
9870     if (!Val) {
9871       // Otherwise, materialize the constant and splat it.
9872       Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue);
9873       DAG.ReplaceAllUsesWith(Vec.getNode(), &Val);
9874     }
9875 
9876     // Now insert the non-constant lanes.
9877     for (unsigned i = 0; i < NumElts; ++i) {
9878       SDValue V = Op.getOperand(i);
9879       SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
9880       if (!isIntOrFPConstant(V))
9881         // Note that type legalization likely mucked about with the VT of the
9882         // source operand, so we may have to convert it here before inserting.
9883         Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx);
9884     }
9885     return Val;
9886   }
9887 
9888   // This will generate a load from the constant pool.
9889   if (isConstant) {
9890     LLVM_DEBUG(
9891         dbgs() << "LowerBUILD_VECTOR: all elements are constant, use default "
9892                   "expansion\n");
9893     return SDValue();
9894   }
9895 
9896   // Empirical tests suggest this is rarely worth it for vectors of length <= 2.
9897   if (NumElts >= 4) {
9898     if (SDValue shuffle = ReconstructShuffle(Op, DAG))
9899       return shuffle;
9900   }
9901 
9902   if (PreferDUPAndInsert) {
9903     // First, build a constant vector with the common element.
9904     SmallVector<SDValue, 8> Ops(NumElts, Value);
9905     SDValue NewVector = LowerBUILD_VECTOR(DAG.getBuildVector(VT, dl, Ops), DAG);
9906     // Next, insert the elements that do not match the common value.
9907     for (unsigned I = 0; I < NumElts; ++I)
9908       if (Op.getOperand(I) != Value)
9909         NewVector =
9910             DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, NewVector,
9911                         Op.getOperand(I), DAG.getConstant(I, dl, MVT::i64));
9912 
9913     return NewVector;
9914   }
9915 
9916   // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
9917   // know the default expansion would otherwise fall back on something even
9918   // worse. For a vector with one or two non-undef values, that's
9919   // scalar_to_vector for the elements followed by a shuffle (provided the
9920   // shuffle is valid for the target) and materialization element by element
9921   // on the stack followed by a load for everything else.
9922   if (!isConstant && !usesOnlyOneValue) {
9923     LLVM_DEBUG(
9924         dbgs() << "LowerBUILD_VECTOR: alternatives failed, creating sequence "
9925                   "of INSERT_VECTOR_ELT\n");
9926 
9927     SDValue Vec = DAG.getUNDEF(VT);
9928     SDValue Op0 = Op.getOperand(0);
9929     unsigned i = 0;
9930 
9931     // Use SCALAR_TO_VECTOR for lane zero to
9932     // a) Avoid a RMW dependency on the full vector register, and
9933     // b) Allow the register coalescer to fold away the copy if the
9934     //    value is already in an S or D register, and we're forced to emit an
9935     //    INSERT_SUBREG that we can't fold anywhere.
9936     //
9937     // We also allow types like i8 and i16 which are illegal scalar but legal
9938     // vector element types. After type-legalization the inserted value is
9939     // extended (i32) and it is safe to cast them to the vector type by ignoring
9940     // the upper bits of the lowest lane (e.g. v8i8, v4i16).
9941     if (!Op0.isUndef()) {
9942       LLVM_DEBUG(dbgs() << "Creating node for op0, it is not undefined:\n");
9943       Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op0);
9944       ++i;
9945     }
9946     LLVM_DEBUG(if (i < NumElts) dbgs()
9947                    << "Creating nodes for the other vector elements:\n";);
9948     for (; i < NumElts; ++i) {
9949       SDValue V = Op.getOperand(i);
9950       if (V.isUndef())
9951         continue;
9952       SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
9953       Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
9954     }
9955     return Vec;
9956   }
9957 
9958   LLVM_DEBUG(
9959       dbgs() << "LowerBUILD_VECTOR: use default expansion, failed to find "
9960                 "better alternative\n");
9961   return SDValue();
9962 }
9963 
9964 SDValue AArch64TargetLowering::LowerCONCAT_VECTORS(SDValue Op,
9965                                                    SelectionDAG &DAG) const {
9966   assert(Op.getValueType().isScalableVector() &&
9967          isTypeLegal(Op.getValueType()) &&
9968          "Expected legal scalable vector type!");
9969 
9970   if (isTypeLegal(Op.getOperand(0).getValueType()) && Op.getNumOperands() == 2)
9971     return Op;
9972 
9973   return SDValue();
9974 }
9975 
9976 SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
9977                                                       SelectionDAG &DAG) const {
9978   assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Unknown opcode!");
9979 
9980   if (useSVEForFixedLengthVectorVT(Op.getValueType()))
9981     return LowerFixedLengthInsertVectorElt(Op, DAG);
9982 
9983   // Check for non-constant or out of range lane.
9984   EVT VT = Op.getOperand(0).getValueType();
9985   ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(2));
9986   if (!CI || CI->getZExtValue() >= VT.getVectorNumElements())
9987     return SDValue();
9988 
9989 
9990   // Insertion/extraction are legal for V128 types.
9991   if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
9992       VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
9993       VT == MVT::v8f16 || VT == MVT::v8bf16)
9994     return Op;
9995 
9996   if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 &&
9997       VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16 &&
9998       VT != MVT::v4bf16)
9999     return SDValue();
10000 
10001   // For V64 types, we perform insertion by expanding the value
10002   // to a V128 type and perform the insertion on that.
10003   SDLoc DL(Op);
10004   SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
10005   EVT WideTy = WideVec.getValueType();
10006 
10007   SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec,
10008                              Op.getOperand(1), Op.getOperand(2));
10009   // Re-narrow the resultant vector.
10010   return NarrowVector(Node, DAG);
10011 }
10012 
10013 SDValue
10014 AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
10015                                                SelectionDAG &DAG) const {
10016   assert(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unknown opcode!");
10017 
10018   EVT VT = Op.getOperand(0).getValueType();
10019   if (useSVEForFixedLengthVectorVT(VT))
10020     return LowerFixedLengthExtractVectorElt(Op, DAG);
10021 
10022   // Check for non-constant or out of range lane.
10023   ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(1));
10024   if (!CI || CI->getZExtValue() >= VT.getVectorNumElements())
10025     return SDValue();
10026 
10027 
10028   // Insertion/extraction are legal for V128 types.
10029   if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
10030       VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
10031       VT == MVT::v8f16 || VT == MVT::v8bf16)
10032     return Op;
10033 
10034   if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 &&
10035       VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16 &&
10036       VT != MVT::v4bf16)
10037     return SDValue();
10038 
10039   // For V64 types, we perform extraction by expanding the value
10040   // to a V128 type and perform the extraction on that.
10041   SDLoc DL(Op);
10042   SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
10043   EVT WideTy = WideVec.getValueType();
10044 
10045   EVT ExtrTy = WideTy.getVectorElementType();
10046   if (ExtrTy == MVT::i16 || ExtrTy == MVT::i8)
10047     ExtrTy = MVT::i32;
10048 
10049   // For extractions, we just return the result directly.
10050   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtrTy, WideVec,
10051                      Op.getOperand(1));
10052 }
10053 
10054 SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
10055                                                       SelectionDAG &DAG) const {
10056   assert(Op.getValueType().isFixedLengthVector() &&
10057          "Only cases that extract a fixed length vector are supported!");
10058 
10059   EVT InVT = Op.getOperand(0).getValueType();
10060   unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
10061   unsigned Size = Op.getValueSizeInBits();
10062 
10063   if (InVT.isScalableVector()) {
10064     // This will be matched by custom code during ISelDAGToDAG.
10065     if (Idx == 0 && isPackedVectorType(InVT, DAG))
10066       return Op;
10067 
10068     return SDValue();
10069   }
10070 
10071   // This will get lowered to an appropriate EXTRACT_SUBREG in ISel.
10072   if (Idx == 0 && InVT.getSizeInBits() <= 128)
10073     return Op;
10074 
10075   // If this is extracting the upper 64-bits of a 128-bit vector, we match
10076   // that directly.
10077   if (Size == 64 && Idx * InVT.getScalarSizeInBits() == 64 &&
10078       InVT.getSizeInBits() == 128)
10079     return Op;
10080 
10081   return SDValue();
10082 }
10083 
10084 SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
10085                                                      SelectionDAG &DAG) const {
10086   assert(Op.getValueType().isScalableVector() &&
10087          "Only expect to lower inserts into scalable vectors!");
10088 
10089   EVT InVT = Op.getOperand(1).getValueType();
10090   unsigned Idx = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
10091 
10092   if (InVT.isScalableVector()) {
10093     SDLoc DL(Op);
10094     EVT VT = Op.getValueType();
10095 
10096     if (!isTypeLegal(VT) || !VT.isInteger())
10097       return SDValue();
10098 
10099     SDValue Vec0 = Op.getOperand(0);
10100     SDValue Vec1 = Op.getOperand(1);
10101 
10102     // Ensure the subvector is half the size of the main vector.
10103     if (VT.getVectorElementCount() != (InVT.getVectorElementCount() * 2))
10104       return SDValue();
10105 
10106     // Extend elements of smaller vector...
10107     EVT WideVT = InVT.widenIntegerVectorElementType(*(DAG.getContext()));
10108     SDValue ExtVec = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Vec1);
10109 
10110     if (Idx == 0) {
10111       SDValue HiVec0 = DAG.getNode(AArch64ISD::UUNPKHI, DL, WideVT, Vec0);
10112       return DAG.getNode(AArch64ISD::UZP1, DL, VT, ExtVec, HiVec0);
10113     } else if (Idx == InVT.getVectorMinNumElements()) {
10114       SDValue LoVec0 = DAG.getNode(AArch64ISD::UUNPKLO, DL, WideVT, Vec0);
10115       return DAG.getNode(AArch64ISD::UZP1, DL, VT, LoVec0, ExtVec);
10116     }
10117 
10118     return SDValue();
10119   }
10120 
10121   // This will be matched by custom code during ISelDAGToDAG.
10122   if (Idx == 0 && isPackedVectorType(InVT, DAG) && Op.getOperand(0).isUndef())
10123     return Op;
10124 
10125   return SDValue();
10126 }
10127 
10128 SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
10129   EVT VT = Op.getValueType();
10130 
10131   if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))
10132     return LowerFixedLengthVectorIntDivideToSVE(Op, DAG);
10133 
10134   assert(VT.isScalableVector() && "Expected a scalable vector.");
10135 
10136   bool Signed = Op.getOpcode() == ISD::SDIV;
10137   unsigned PredOpcode = Signed ? AArch64ISD::SDIV_PRED : AArch64ISD::UDIV_PRED;
10138 
10139   if (VT == MVT::nxv4i32 || VT == MVT::nxv2i64)
10140     return LowerToPredicatedOp(Op, DAG, PredOpcode);
10141 
10142   // SVE doesn't have i8 and i16 DIV operations; widen them to 32-bit
10143   // operations, and truncate the result.
10144   EVT WidenedVT;
10145   if (VT == MVT::nxv16i8)
10146     WidenedVT = MVT::nxv8i16;
10147   else if (VT == MVT::nxv8i16)
10148     WidenedVT = MVT::nxv4i32;
10149   else
10150     llvm_unreachable("Unexpected Custom DIV operation");
10151 
10152   SDLoc dl(Op);
10153   unsigned UnpkLo = Signed ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO;
10154   unsigned UnpkHi = Signed ? AArch64ISD::SUNPKHI : AArch64ISD::UUNPKHI;
10155   SDValue Op0Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(0));
10156   SDValue Op1Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(1));
10157   SDValue Op0Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(0));
10158   SDValue Op1Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(1));
10159   SDValue ResultLo = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Lo, Op1Lo);
10160   SDValue ResultHi = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Hi, Op1Hi);
10161   return DAG.getNode(AArch64ISD::UZP1, dl, VT, ResultLo, ResultHi);
10162 }
10163 
10164 bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
10165   // Currently no fixed length shuffles that require SVE are legal.
10166   if (useSVEForFixedLengthVectorVT(VT))
10167     return false;
10168 
10169   if (VT.getVectorNumElements() == 4 &&
10170       (VT.is128BitVector() || VT.is64BitVector())) {
10171     unsigned PFIndexes[4];
10172     for (unsigned i = 0; i != 4; ++i) {
10173       if (M[i] < 0)
10174         PFIndexes[i] = 8;
10175       else
10176         PFIndexes[i] = M[i];
10177     }
10178 
10179     // Compute the index in the perfect shuffle table.
10180     unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
10181                             PFIndexes[2] * 9 + PFIndexes[3];
10182     unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
10183     unsigned Cost = (PFEntry >> 30);
10184 
10185     if (Cost <= 4)
10186       return true;
10187   }
10188 
10189   bool DummyBool;
10190   int DummyInt;
10191   unsigned DummyUnsigned;
10192 
10193   return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) || isREVMask(M, VT, 64) ||
10194           isREVMask(M, VT, 32) || isREVMask(M, VT, 16) ||
10195           isEXTMask(M, VT, DummyBool, DummyUnsigned) ||
10196           // isTBLMask(M, VT) || // FIXME: Port TBL support from ARM.
10197           isTRNMask(M, VT, DummyUnsigned) || isUZPMask(M, VT, DummyUnsigned) ||
10198           isZIPMask(M, VT, DummyUnsigned) ||
10199           isTRN_v_undef_Mask(M, VT, DummyUnsigned) ||
10200           isUZP_v_undef_Mask(M, VT, DummyUnsigned) ||
10201           isZIP_v_undef_Mask(M, VT, DummyUnsigned) ||
10202           isINSMask(M, VT.getVectorNumElements(), DummyBool, DummyInt) ||
10203           isConcatMask(M, VT, VT.getSizeInBits() == 128));
10204 }
10205 
10206 /// getVShiftImm - Check if this is a valid build_vector for the immediate
10207 /// operand of a vector shift operation, where all the elements of the
10208 /// build_vector must have the same constant integer value.
10209 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
10210   // Ignore bit_converts.
10211   while (Op.getOpcode() == ISD::BITCAST)
10212     Op = Op.getOperand(0);
10213   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
10214   APInt SplatBits, SplatUndef;
10215   unsigned SplatBitSize;
10216   bool HasAnyUndefs;
10217   if (!BVN || !BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
10218                                     HasAnyUndefs, ElementBits) ||
10219       SplatBitSize > ElementBits)
10220     return false;
10221   Cnt = SplatBits.getSExtValue();
10222   return true;
10223 }
10224 
10225 /// isVShiftLImm - Check if this is a valid build_vector for the immediate
10226 /// operand of a vector shift left operation.  That value must be in the range:
10227 ///   0 <= Value < ElementBits for a left shift; or
10228 ///   0 <= Value <= ElementBits for a long left shift.
10229 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
10230   assert(VT.isVector() && "vector shift count is not a vector type");
10231   int64_t ElementBits = VT.getScalarSizeInBits();
10232   if (!getVShiftImm(Op, ElementBits, Cnt))
10233     return false;
10234   return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
10235 }
10236 
10237 /// isVShiftRImm - Check if this is a valid build_vector for the immediate
10238 /// operand of a vector shift right operation. The value must be in the range:
10239 ///   1 <= Value <= ElementBits for a right shift; or
10240 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
10241   assert(VT.isVector() && "vector shift count is not a vector type");
10242   int64_t ElementBits = VT.getScalarSizeInBits();
10243   if (!getVShiftImm(Op, ElementBits, Cnt))
10244     return false;
10245   return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
10246 }
10247 
10248 SDValue AArch64TargetLowering::LowerTRUNCATE(SDValue Op,
10249                                              SelectionDAG &DAG) const {
10250   EVT VT = Op.getValueType();
10251 
10252   if (VT.getScalarType() == MVT::i1) {
10253     // Lower i1 truncate to `(x & 1) != 0`.
10254     SDLoc dl(Op);
10255     EVT OpVT = Op.getOperand(0).getValueType();
10256     SDValue Zero = DAG.getConstant(0, dl, OpVT);
10257     SDValue One = DAG.getConstant(1, dl, OpVT);
10258     SDValue And = DAG.getNode(ISD::AND, dl, OpVT, Op.getOperand(0), One);
10259     return DAG.getSetCC(dl, VT, And, Zero, ISD::SETNE);
10260   }
10261 
10262   if (!VT.isVector() || VT.isScalableVector())
10263     return SDValue();
10264 
10265   if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
10266     return LowerFixedLengthVectorTruncateToSVE(Op, DAG);
10267 
10268   return SDValue();
10269 }
10270 
10271 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
10272                                                       SelectionDAG &DAG) const {
10273   EVT VT = Op.getValueType();
10274   SDLoc DL(Op);
10275   int64_t Cnt;
10276 
10277   if (!Op.getOperand(1).getValueType().isVector())
10278     return Op;
10279   unsigned EltSize = VT.getScalarSizeInBits();
10280 
10281   switch (Op.getOpcode()) {
10282   default:
10283     llvm_unreachable("unexpected shift opcode");
10284 
10285   case ISD::SHL:
10286     if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT))
10287       return LowerToPredicatedOp(Op, DAG, AArch64ISD::SHL_PRED);
10288 
10289     if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize)
10290       return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0),
10291                          DAG.getConstant(Cnt, DL, MVT::i32));
10292     return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
10293                        DAG.getConstant(Intrinsic::aarch64_neon_ushl, DL,
10294                                        MVT::i32),
10295                        Op.getOperand(0), Op.getOperand(1));
10296   case ISD::SRA:
10297   case ISD::SRL:
10298     if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT)) {
10299       unsigned Opc = Op.getOpcode() == ISD::SRA ? AArch64ISD::SRA_PRED
10300                                                 : AArch64ISD::SRL_PRED;
10301       return LowerToPredicatedOp(Op, DAG, Opc);
10302     }
10303 
10304     // Right shift immediate
10305     if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) {
10306       unsigned Opc =
10307           (Op.getOpcode() == ISD::SRA) ? AArch64ISD::VASHR : AArch64ISD::VLSHR;
10308       return DAG.getNode(Opc, DL, VT, Op.getOperand(0),
10309                          DAG.getConstant(Cnt, DL, MVT::i32));
10310     }
10311 
10312     // Right shift register.  Note, there is not a shift right register
10313     // instruction, but the shift left register instruction takes a signed
10314     // value, where negative numbers specify a right shift.
10315     unsigned Opc = (Op.getOpcode() == ISD::SRA) ? Intrinsic::aarch64_neon_sshl
10316                                                 : Intrinsic::aarch64_neon_ushl;
10317     // negate the shift amount
10318     SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1));
10319     SDValue NegShiftLeft =
10320         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
10321                     DAG.getConstant(Opc, DL, MVT::i32), Op.getOperand(0),
10322                     NegShift);
10323     return NegShiftLeft;
10324   }
10325 
10326   return SDValue();
10327 }
10328 
10329 static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
10330                                     AArch64CC::CondCode CC, bool NoNans, EVT VT,
10331                                     const SDLoc &dl, SelectionDAG &DAG) {
10332   EVT SrcVT = LHS.getValueType();
10333   assert(VT.getSizeInBits() == SrcVT.getSizeInBits() &&
10334          "function only supposed to emit natural comparisons");
10335 
10336   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(RHS.getNode());
10337   APInt CnstBits(VT.getSizeInBits(), 0);
10338   APInt UndefBits(VT.getSizeInBits(), 0);
10339   bool IsCnst = BVN && resolveBuildVector(BVN, CnstBits, UndefBits);
10340   bool IsZero = IsCnst && (CnstBits == 0);
10341 
10342   if (SrcVT.getVectorElementType().isFloatingPoint()) {
10343     switch (CC) {
10344     default:
10345       return SDValue();
10346     case AArch64CC::NE: {
10347       SDValue Fcmeq;
10348       if (IsZero)
10349         Fcmeq = DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS);
10350       else
10351         Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
10352       return DAG.getNOT(dl, Fcmeq, VT);
10353     }
10354     case AArch64CC::EQ:
10355       if (IsZero)
10356         return DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS);
10357       return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
10358     case AArch64CC::GE:
10359       if (IsZero)
10360         return DAG.getNode(AArch64ISD::FCMGEz, dl, VT, LHS);
10361       return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS);
10362     case AArch64CC::GT:
10363       if (IsZero)
10364         return DAG.getNode(AArch64ISD::FCMGTz, dl, VT, LHS);
10365       return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS);
10366     case AArch64CC::LS:
10367       if (IsZero)
10368         return DAG.getNode(AArch64ISD::FCMLEz, dl, VT, LHS);
10369       return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS);
10370     case AArch64CC::LT:
10371       if (!NoNans)
10372         return SDValue();
10373       // If we ignore NaNs then we can use to the MI implementation.
10374       LLVM_FALLTHROUGH;
10375     case AArch64CC::MI:
10376       if (IsZero)
10377         return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS);
10378       return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS);
10379     }
10380   }
10381 
10382   switch (CC) {
10383   default:
10384     return SDValue();
10385   case AArch64CC::NE: {
10386     SDValue Cmeq;
10387     if (IsZero)
10388       Cmeq = DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS);
10389     else
10390       Cmeq = DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS);
10391     return DAG.getNOT(dl, Cmeq, VT);
10392   }
10393   case AArch64CC::EQ:
10394     if (IsZero)
10395       return DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS);
10396     return DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS);
10397   case AArch64CC::GE:
10398     if (IsZero)
10399       return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS);
10400     return DAG.getNode(AArch64ISD::CMGE, dl, VT, LHS, RHS);
10401   case AArch64CC::GT:
10402     if (IsZero)
10403       return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS);
10404     return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS);
10405   case AArch64CC::LE:
10406     if (IsZero)
10407       return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS);
10408     return DAG.getNode(AArch64ISD::CMGE, dl, VT, RHS, LHS);
10409   case AArch64CC::LS:
10410     return DAG.getNode(AArch64ISD::CMHS, dl, VT, RHS, LHS);
10411   case AArch64CC::LO:
10412     return DAG.getNode(AArch64ISD::CMHI, dl, VT, RHS, LHS);
10413   case AArch64CC::LT:
10414     if (IsZero)
10415       return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS);
10416     return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS);
10417   case AArch64CC::HI:
10418     return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS);
10419   case AArch64CC::HS:
10420     return DAG.getNode(AArch64ISD::CMHS, dl, VT, LHS, RHS);
10421   }
10422 }
10423 
10424 SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
10425                                            SelectionDAG &DAG) const {
10426   if (Op.getValueType().isScalableVector())
10427     return LowerToPredicatedOp(Op, DAG, AArch64ISD::SETCC_MERGE_ZERO);
10428 
10429   if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
10430     return LowerFixedLengthVectorSetccToSVE(Op, DAG);
10431 
10432   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
10433   SDValue LHS = Op.getOperand(0);
10434   SDValue RHS = Op.getOperand(1);
10435   EVT CmpVT = LHS.getValueType().changeVectorElementTypeToInteger();
10436   SDLoc dl(Op);
10437 
10438   if (LHS.getValueType().getVectorElementType().isInteger()) {
10439     assert(LHS.getValueType() == RHS.getValueType());
10440     AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
10441     SDValue Cmp =
10442         EmitVectorComparison(LHS, RHS, AArch64CC, false, CmpVT, dl, DAG);
10443     return DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType());
10444   }
10445 
10446   const bool FullFP16 =
10447     static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
10448 
10449   // Make v4f16 (only) fcmp operations utilise vector instructions
10450   // v8f16 support will be a litle more complicated
10451   if (!FullFP16 && LHS.getValueType().getVectorElementType() == MVT::f16) {
10452     if (LHS.getValueType().getVectorNumElements() == 4) {
10453       LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, LHS);
10454       RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, RHS);
10455       SDValue NewSetcc = DAG.getSetCC(dl, MVT::v4i16, LHS, RHS, CC);
10456       DAG.ReplaceAllUsesWith(Op, NewSetcc);
10457       CmpVT = MVT::v4i32;
10458     } else
10459       return SDValue();
10460   }
10461 
10462   assert((!FullFP16 && LHS.getValueType().getVectorElementType() != MVT::f16) ||
10463           LHS.getValueType().getVectorElementType() != MVT::f128);
10464 
10465   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
10466   // clean.  Some of them require two branches to implement.
10467   AArch64CC::CondCode CC1, CC2;
10468   bool ShouldInvert;
10469   changeVectorFPCCToAArch64CC(CC, CC1, CC2, ShouldInvert);
10470 
10471   bool NoNaNs = getTargetMachine().Options.NoNaNsFPMath;
10472   SDValue Cmp =
10473       EmitVectorComparison(LHS, RHS, CC1, NoNaNs, CmpVT, dl, DAG);
10474   if (!Cmp.getNode())
10475     return SDValue();
10476 
10477   if (CC2 != AArch64CC::AL) {
10478     SDValue Cmp2 =
10479         EmitVectorComparison(LHS, RHS, CC2, NoNaNs, CmpVT, dl, DAG);
10480     if (!Cmp2.getNode())
10481       return SDValue();
10482 
10483     Cmp = DAG.getNode(ISD::OR, dl, CmpVT, Cmp, Cmp2);
10484   }
10485 
10486   Cmp = DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType());
10487 
10488   if (ShouldInvert)
10489     Cmp = DAG.getNOT(dl, Cmp, Cmp.getValueType());
10490 
10491   return Cmp;
10492 }
10493 
10494 static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp,
10495                                   SelectionDAG &DAG) {
10496   SDValue VecOp = ScalarOp.getOperand(0);
10497   auto Rdx = DAG.getNode(Op, DL, VecOp.getSimpleValueType(), VecOp);
10498   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarOp.getValueType(), Rdx,
10499                      DAG.getConstant(0, DL, MVT::i64));
10500 }
10501 
10502 SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
10503                                               SelectionDAG &DAG) const {
10504   SDValue Src = Op.getOperand(0);
10505 
10506   // Try to lower fixed length reductions to SVE.
10507   EVT SrcVT = Src.getValueType();
10508   bool OverrideNEON = Op.getOpcode() == ISD::VECREDUCE_AND ||
10509                       Op.getOpcode() == ISD::VECREDUCE_OR ||
10510                       Op.getOpcode() == ISD::VECREDUCE_XOR ||
10511                       Op.getOpcode() == ISD::VECREDUCE_FADD ||
10512                       (Op.getOpcode() != ISD::VECREDUCE_ADD &&
10513                        SrcVT.getVectorElementType() == MVT::i64);
10514   if (SrcVT.isScalableVector() ||
10515       useSVEForFixedLengthVectorVT(SrcVT, OverrideNEON)) {
10516 
10517     if (SrcVT.getVectorElementType() == MVT::i1)
10518       return LowerPredReductionToSVE(Op, DAG);
10519 
10520     switch (Op.getOpcode()) {
10521     case ISD::VECREDUCE_ADD:
10522       return LowerReductionToSVE(AArch64ISD::UADDV_PRED, Op, DAG);
10523     case ISD::VECREDUCE_AND:
10524       return LowerReductionToSVE(AArch64ISD::ANDV_PRED, Op, DAG);
10525     case ISD::VECREDUCE_OR:
10526       return LowerReductionToSVE(AArch64ISD::ORV_PRED, Op, DAG);
10527     case ISD::VECREDUCE_SMAX:
10528       return LowerReductionToSVE(AArch64ISD::SMAXV_PRED, Op, DAG);
10529     case ISD::VECREDUCE_SMIN:
10530       return LowerReductionToSVE(AArch64ISD::SMINV_PRED, Op, DAG);
10531     case ISD::VECREDUCE_UMAX:
10532       return LowerReductionToSVE(AArch64ISD::UMAXV_PRED, Op, DAG);
10533     case ISD::VECREDUCE_UMIN:
10534       return LowerReductionToSVE(AArch64ISD::UMINV_PRED, Op, DAG);
10535     case ISD::VECREDUCE_XOR:
10536       return LowerReductionToSVE(AArch64ISD::EORV_PRED, Op, DAG);
10537     case ISD::VECREDUCE_FADD:
10538       return LowerReductionToSVE(AArch64ISD::FADDV_PRED, Op, DAG);
10539     case ISD::VECREDUCE_FMAX:
10540       return LowerReductionToSVE(AArch64ISD::FMAXNMV_PRED, Op, DAG);
10541     case ISD::VECREDUCE_FMIN:
10542       return LowerReductionToSVE(AArch64ISD::FMINNMV_PRED, Op, DAG);
10543     default:
10544       llvm_unreachable("Unhandled fixed length reduction");
10545     }
10546   }
10547 
10548   // Lower NEON reductions.
10549   SDLoc dl(Op);
10550   switch (Op.getOpcode()) {
10551   case ISD::VECREDUCE_ADD:
10552     return getReductionSDNode(AArch64ISD::UADDV, dl, Op, DAG);
10553   case ISD::VECREDUCE_SMAX:
10554     return getReductionSDNode(AArch64ISD::SMAXV, dl, Op, DAG);
10555   case ISD::VECREDUCE_SMIN:
10556     return getReductionSDNode(AArch64ISD::SMINV, dl, Op, DAG);
10557   case ISD::VECREDUCE_UMAX:
10558     return getReductionSDNode(AArch64ISD::UMAXV, dl, Op, DAG);
10559   case ISD::VECREDUCE_UMIN:
10560     return getReductionSDNode(AArch64ISD::UMINV, dl, Op, DAG);
10561   case ISD::VECREDUCE_FMAX: {
10562     return DAG.getNode(
10563         ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(),
10564         DAG.getConstant(Intrinsic::aarch64_neon_fmaxnmv, dl, MVT::i32),
10565         Src);
10566   }
10567   case ISD::VECREDUCE_FMIN: {
10568     return DAG.getNode(
10569         ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(),
10570         DAG.getConstant(Intrinsic::aarch64_neon_fminnmv, dl, MVT::i32),
10571         Src);
10572   }
10573   default:
10574     llvm_unreachable("Unhandled reduction");
10575   }
10576 }
10577 
10578 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_SUB(SDValue Op,
10579                                                     SelectionDAG &DAG) const {
10580   auto &Subtarget = static_cast<const AArch64Subtarget &>(DAG.getSubtarget());
10581   if (!Subtarget.hasLSE() && !Subtarget.outlineAtomics())
10582     return SDValue();
10583 
10584   // LSE has an atomic load-add instruction, but not a load-sub.
10585   SDLoc dl(Op);
10586   MVT VT = Op.getSimpleValueType();
10587   SDValue RHS = Op.getOperand(2);
10588   AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode());
10589   RHS = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), RHS);
10590   return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl, AN->getMemoryVT(),
10591                        Op.getOperand(0), Op.getOperand(1), RHS,
10592                        AN->getMemOperand());
10593 }
10594 
10595 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
10596                                                     SelectionDAG &DAG) const {
10597   auto &Subtarget = static_cast<const AArch64Subtarget &>(DAG.getSubtarget());
10598   if (!Subtarget.hasLSE() && !Subtarget.outlineAtomics())
10599     return SDValue();
10600 
10601   // LSE has an atomic load-clear instruction, but not a load-and.
10602   SDLoc dl(Op);
10603   MVT VT = Op.getSimpleValueType();
10604   SDValue RHS = Op.getOperand(2);
10605   AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode());
10606   RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getConstant(-1ULL, dl, VT), RHS);
10607   return DAG.getAtomic(ISD::ATOMIC_LOAD_CLR, dl, AN->getMemoryVT(),
10608                        Op.getOperand(0), Op.getOperand(1), RHS,
10609                        AN->getMemOperand());
10610 }
10611 
10612 SDValue AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(
10613     SDValue Op, SDValue Chain, SDValue &Size, SelectionDAG &DAG) const {
10614   SDLoc dl(Op);
10615   EVT PtrVT = getPointerTy(DAG.getDataLayout());
10616   SDValue Callee = DAG.getTargetExternalSymbol("__chkstk", PtrVT, 0);
10617 
10618   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
10619   const uint32_t *Mask = TRI->getWindowsStackProbePreservedMask();
10620   if (Subtarget->hasCustomCallingConv())
10621     TRI->UpdateCustomCallPreservedMask(DAG.getMachineFunction(), &Mask);
10622 
10623   Size = DAG.getNode(ISD::SRL, dl, MVT::i64, Size,
10624                      DAG.getConstant(4, dl, MVT::i64));
10625   Chain = DAG.getCopyToReg(Chain, dl, AArch64::X15, Size, SDValue());
10626   Chain =
10627       DAG.getNode(AArch64ISD::CALL, dl, DAG.getVTList(MVT::Other, MVT::Glue),
10628                   Chain, Callee, DAG.getRegister(AArch64::X15, MVT::i64),
10629                   DAG.getRegisterMask(Mask), Chain.getValue(1));
10630   // To match the actual intent better, we should read the output from X15 here
10631   // again (instead of potentially spilling it to the stack), but rereading Size
10632   // from X15 here doesn't work at -O0, since it thinks that X15 is undefined
10633   // here.
10634 
10635   Size = DAG.getNode(ISD::SHL, dl, MVT::i64, Size,
10636                      DAG.getConstant(4, dl, MVT::i64));
10637   return Chain;
10638 }
10639 
10640 SDValue
10641 AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
10642                                                SelectionDAG &DAG) const {
10643   assert(Subtarget->isTargetWindows() &&
10644          "Only Windows alloca probing supported");
10645   SDLoc dl(Op);
10646   // Get the inputs.
10647   SDNode *Node = Op.getNode();
10648   SDValue Chain = Op.getOperand(0);
10649   SDValue Size = Op.getOperand(1);
10650   MaybeAlign Align =
10651       cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue();
10652   EVT VT = Node->getValueType(0);
10653 
10654   if (DAG.getMachineFunction().getFunction().hasFnAttribute(
10655           "no-stack-arg-probe")) {
10656     SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
10657     Chain = SP.getValue(1);
10658     SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
10659     if (Align)
10660       SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
10661                        DAG.getConstant(-(uint64_t)Align->value(), dl, VT));
10662     Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
10663     SDValue Ops[2] = {SP, Chain};
10664     return DAG.getMergeValues(Ops, dl);
10665   }
10666 
10667   Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
10668 
10669   Chain = LowerWindowsDYNAMIC_STACKALLOC(Op, Chain, Size, DAG);
10670 
10671   SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
10672   Chain = SP.getValue(1);
10673   SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
10674   if (Align)
10675     SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
10676                      DAG.getConstant(-(uint64_t)Align->value(), dl, VT));
10677   Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
10678 
10679   Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
10680                              DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
10681 
10682   SDValue Ops[2] = {SP, Chain};
10683   return DAG.getMergeValues(Ops, dl);
10684 }
10685 
10686 SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op,
10687                                            SelectionDAG &DAG) const {
10688   EVT VT = Op.getValueType();
10689   assert(VT != MVT::i64 && "Expected illegal VSCALE node");
10690 
10691   SDLoc DL(Op);
10692   APInt MulImm = cast<ConstantSDNode>(Op.getOperand(0))->getAPIntValue();
10693   return DAG.getZExtOrTrunc(DAG.getVScale(DL, MVT::i64, MulImm.sextOrSelf(64)),
10694                             DL, VT);
10695 }
10696 
10697 /// Set the IntrinsicInfo for the `aarch64_sve_st<N>` intrinsics.
10698 template <unsigned NumVecs>
10699 static bool
10700 setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL,
10701               AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI) {
10702   Info.opc = ISD::INTRINSIC_VOID;
10703   // Retrieve EC from first vector argument.
10704   const EVT VT = TLI.getMemValueType(DL, CI.getArgOperand(0)->getType());
10705   ElementCount EC = VT.getVectorElementCount();
10706 #ifndef NDEBUG
10707   // Check the assumption that all input vectors are the same type.
10708   for (unsigned I = 0; I < NumVecs; ++I)
10709     assert(VT == TLI.getMemValueType(DL, CI.getArgOperand(I)->getType()) &&
10710            "Invalid type.");
10711 #endif
10712   // memVT is `NumVecs * VT`.
10713   Info.memVT = EVT::getVectorVT(CI.getType()->getContext(), VT.getScalarType(),
10714                                 EC * NumVecs);
10715   Info.ptrVal = CI.getArgOperand(CI.getNumArgOperands() - 1);
10716   Info.offset = 0;
10717   Info.align.reset();
10718   Info.flags = MachineMemOperand::MOStore;
10719   return true;
10720 }
10721 
10722 /// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
10723 /// MemIntrinsicNodes.  The associated MachineMemOperands record the alignment
10724 /// specified in the intrinsic calls.
10725 bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
10726                                                const CallInst &I,
10727                                                MachineFunction &MF,
10728                                                unsigned Intrinsic) const {
10729   auto &DL = I.getModule()->getDataLayout();
10730   switch (Intrinsic) {
10731   case Intrinsic::aarch64_sve_st2:
10732     return setInfoSVEStN<2>(*this, DL, Info, I);
10733   case Intrinsic::aarch64_sve_st3:
10734     return setInfoSVEStN<3>(*this, DL, Info, I);
10735   case Intrinsic::aarch64_sve_st4:
10736     return setInfoSVEStN<4>(*this, DL, Info, I);
10737   case Intrinsic::aarch64_neon_ld2:
10738   case Intrinsic::aarch64_neon_ld3:
10739   case Intrinsic::aarch64_neon_ld4:
10740   case Intrinsic::aarch64_neon_ld1x2:
10741   case Intrinsic::aarch64_neon_ld1x3:
10742   case Intrinsic::aarch64_neon_ld1x4:
10743   case Intrinsic::aarch64_neon_ld2lane:
10744   case Intrinsic::aarch64_neon_ld3lane:
10745   case Intrinsic::aarch64_neon_ld4lane:
10746   case Intrinsic::aarch64_neon_ld2r:
10747   case Intrinsic::aarch64_neon_ld3r:
10748   case Intrinsic::aarch64_neon_ld4r: {
10749     Info.opc = ISD::INTRINSIC_W_CHAIN;
10750     // Conservatively set memVT to the entire set of vectors loaded.
10751     uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
10752     Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
10753     Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1);
10754     Info.offset = 0;
10755     Info.align.reset();
10756     // volatile loads with NEON intrinsics not supported
10757     Info.flags = MachineMemOperand::MOLoad;
10758     return true;
10759   }
10760   case Intrinsic::aarch64_neon_st2:
10761   case Intrinsic::aarch64_neon_st3:
10762   case Intrinsic::aarch64_neon_st4:
10763   case Intrinsic::aarch64_neon_st1x2:
10764   case Intrinsic::aarch64_neon_st1x3:
10765   case Intrinsic::aarch64_neon_st1x4:
10766   case Intrinsic::aarch64_neon_st2lane:
10767   case Intrinsic::aarch64_neon_st3lane:
10768   case Intrinsic::aarch64_neon_st4lane: {
10769     Info.opc = ISD::INTRINSIC_VOID;
10770     // Conservatively set memVT to the entire set of vectors stored.
10771     unsigned NumElts = 0;
10772     for (unsigned ArgI = 0, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) {
10773       Type *ArgTy = I.getArgOperand(ArgI)->getType();
10774       if (!ArgTy->isVectorTy())
10775         break;
10776       NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
10777     }
10778     Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
10779     Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1);
10780     Info.offset = 0;
10781     Info.align.reset();
10782     // volatile stores with NEON intrinsics not supported
10783     Info.flags = MachineMemOperand::MOStore;
10784     return true;
10785   }
10786   case Intrinsic::aarch64_ldaxr:
10787   case Intrinsic::aarch64_ldxr: {
10788     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
10789     Info.opc = ISD::INTRINSIC_W_CHAIN;
10790     Info.memVT = MVT::getVT(PtrTy->getElementType());
10791     Info.ptrVal = I.getArgOperand(0);
10792     Info.offset = 0;
10793     Info.align = DL.getABITypeAlign(PtrTy->getElementType());
10794     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
10795     return true;
10796   }
10797   case Intrinsic::aarch64_stlxr:
10798   case Intrinsic::aarch64_stxr: {
10799     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
10800     Info.opc = ISD::INTRINSIC_W_CHAIN;
10801     Info.memVT = MVT::getVT(PtrTy->getElementType());
10802     Info.ptrVal = I.getArgOperand(1);
10803     Info.offset = 0;
10804     Info.align = DL.getABITypeAlign(PtrTy->getElementType());
10805     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
10806     return true;
10807   }
10808   case Intrinsic::aarch64_ldaxp:
10809   case Intrinsic::aarch64_ldxp:
10810     Info.opc = ISD::INTRINSIC_W_CHAIN;
10811     Info.memVT = MVT::i128;
10812     Info.ptrVal = I.getArgOperand(0);
10813     Info.offset = 0;
10814     Info.align = Align(16);
10815     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
10816     return true;
10817   case Intrinsic::aarch64_stlxp:
10818   case Intrinsic::aarch64_stxp:
10819     Info.opc = ISD::INTRINSIC_W_CHAIN;
10820     Info.memVT = MVT::i128;
10821     Info.ptrVal = I.getArgOperand(2);
10822     Info.offset = 0;
10823     Info.align = Align(16);
10824     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
10825     return true;
10826   case Intrinsic::aarch64_sve_ldnt1: {
10827     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
10828     Info.opc = ISD::INTRINSIC_W_CHAIN;
10829     Info.memVT = MVT::getVT(I.getType());
10830     Info.ptrVal = I.getArgOperand(1);
10831     Info.offset = 0;
10832     Info.align = DL.getABITypeAlign(PtrTy->getElementType());
10833     Info.flags = MachineMemOperand::MOLoad;
10834     if (Intrinsic == Intrinsic::aarch64_sve_ldnt1)
10835       Info.flags |= MachineMemOperand::MONonTemporal;
10836     return true;
10837   }
10838   case Intrinsic::aarch64_sve_stnt1: {
10839     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(2)->getType());
10840     Info.opc = ISD::INTRINSIC_W_CHAIN;
10841     Info.memVT = MVT::getVT(I.getOperand(0)->getType());
10842     Info.ptrVal = I.getArgOperand(2);
10843     Info.offset = 0;
10844     Info.align = DL.getABITypeAlign(PtrTy->getElementType());
10845     Info.flags = MachineMemOperand::MOStore;
10846     if (Intrinsic == Intrinsic::aarch64_sve_stnt1)
10847       Info.flags |= MachineMemOperand::MONonTemporal;
10848     return true;
10849   }
10850   default:
10851     break;
10852   }
10853 
10854   return false;
10855 }
10856 
10857 bool AArch64TargetLowering::shouldReduceLoadWidth(SDNode *Load,
10858                                                   ISD::LoadExtType ExtTy,
10859                                                   EVT NewVT) const {
10860   // TODO: This may be worth removing. Check regression tests for diffs.
10861   if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT))
10862     return false;
10863 
10864   // If we're reducing the load width in order to avoid having to use an extra
10865   // instruction to do extension then it's probably a good idea.
10866   if (ExtTy != ISD::NON_EXTLOAD)
10867     return true;
10868   // Don't reduce load width if it would prevent us from combining a shift into
10869   // the offset.
10870   MemSDNode *Mem = dyn_cast<MemSDNode>(Load);
10871   assert(Mem);
10872   const SDValue &Base = Mem->getBasePtr();
10873   if (Base.getOpcode() == ISD::ADD &&
10874       Base.getOperand(1).getOpcode() == ISD::SHL &&
10875       Base.getOperand(1).hasOneUse() &&
10876       Base.getOperand(1).getOperand(1).getOpcode() == ISD::Constant) {
10877     // The shift can be combined if it matches the size of the value being
10878     // loaded (and so reducing the width would make it not match).
10879     uint64_t ShiftAmount = Base.getOperand(1).getConstantOperandVal(1);
10880     uint64_t LoadBytes = Mem->getMemoryVT().getSizeInBits()/8;
10881     if (ShiftAmount == Log2_32(LoadBytes))
10882       return false;
10883   }
10884   // We have no reason to disallow reducing the load width, so allow it.
10885   return true;
10886 }
10887 
10888 // Truncations from 64-bit GPR to 32-bit GPR is free.
10889 bool AArch64TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
10890   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
10891     return false;
10892   uint64_t NumBits1 = Ty1->getPrimitiveSizeInBits().getFixedSize();
10893   uint64_t NumBits2 = Ty2->getPrimitiveSizeInBits().getFixedSize();
10894   return NumBits1 > NumBits2;
10895 }
10896 bool AArch64TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
10897   if (VT1.isVector() || VT2.isVector() || !VT1.isInteger() || !VT2.isInteger())
10898     return false;
10899   uint64_t NumBits1 = VT1.getFixedSizeInBits();
10900   uint64_t NumBits2 = VT2.getFixedSizeInBits();
10901   return NumBits1 > NumBits2;
10902 }
10903 
10904 /// Check if it is profitable to hoist instruction in then/else to if.
10905 /// Not profitable if I and it's user can form a FMA instruction
10906 /// because we prefer FMSUB/FMADD.
10907 bool AArch64TargetLowering::isProfitableToHoist(Instruction *I) const {
10908   if (I->getOpcode() != Instruction::FMul)
10909     return true;
10910 
10911   if (!I->hasOneUse())
10912     return true;
10913 
10914   Instruction *User = I->user_back();
10915 
10916   if (User &&
10917       !(User->getOpcode() == Instruction::FSub ||
10918         User->getOpcode() == Instruction::FAdd))
10919     return true;
10920 
10921   const TargetOptions &Options = getTargetMachine().Options;
10922   const Function *F = I->getFunction();
10923   const DataLayout &DL = F->getParent()->getDataLayout();
10924   Type *Ty = User->getOperand(0)->getType();
10925 
10926   return !(isFMAFasterThanFMulAndFAdd(*F, Ty) &&
10927            isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) &&
10928            (Options.AllowFPOpFusion == FPOpFusion::Fast ||
10929             Options.UnsafeFPMath));
10930 }
10931 
10932 // All 32-bit GPR operations implicitly zero the high-half of the corresponding
10933 // 64-bit GPR.
10934 bool AArch64TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
10935   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
10936     return false;
10937   unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
10938   unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
10939   return NumBits1 == 32 && NumBits2 == 64;
10940 }
10941 bool AArch64TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
10942   if (VT1.isVector() || VT2.isVector() || !VT1.isInteger() || !VT2.isInteger())
10943     return false;
10944   unsigned NumBits1 = VT1.getSizeInBits();
10945   unsigned NumBits2 = VT2.getSizeInBits();
10946   return NumBits1 == 32 && NumBits2 == 64;
10947 }
10948 
10949 bool AArch64TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
10950   EVT VT1 = Val.getValueType();
10951   if (isZExtFree(VT1, VT2)) {
10952     return true;
10953   }
10954 
10955   if (Val.getOpcode() != ISD::LOAD)
10956     return false;
10957 
10958   // 8-, 16-, and 32-bit integer loads all implicitly zero-extend.
10959   return (VT1.isSimple() && !VT1.isVector() && VT1.isInteger() &&
10960           VT2.isSimple() && !VT2.isVector() && VT2.isInteger() &&
10961           VT1.getSizeInBits() <= 32);
10962 }
10963 
10964 bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const {
10965   if (isa<FPExtInst>(Ext))
10966     return false;
10967 
10968   // Vector types are not free.
10969   if (Ext->getType()->isVectorTy())
10970     return false;
10971 
10972   for (const Use &U : Ext->uses()) {
10973     // The extension is free if we can fold it with a left shift in an
10974     // addressing mode or an arithmetic operation: add, sub, and cmp.
10975 
10976     // Is there a shift?
10977     const Instruction *Instr = cast<Instruction>(U.getUser());
10978 
10979     // Is this a constant shift?
10980     switch (Instr->getOpcode()) {
10981     case Instruction::Shl:
10982       if (!isa<ConstantInt>(Instr->getOperand(1)))
10983         return false;
10984       break;
10985     case Instruction::GetElementPtr: {
10986       gep_type_iterator GTI = gep_type_begin(Instr);
10987       auto &DL = Ext->getModule()->getDataLayout();
10988       std::advance(GTI, U.getOperandNo()-1);
10989       Type *IdxTy = GTI.getIndexedType();
10990       // This extension will end up with a shift because of the scaling factor.
10991       // 8-bit sized types have a scaling factor of 1, thus a shift amount of 0.
10992       // Get the shift amount based on the scaling factor:
10993       // log2(sizeof(IdxTy)) - log2(8).
10994       uint64_t ShiftAmt =
10995         countTrailingZeros(DL.getTypeStoreSizeInBits(IdxTy).getFixedSize()) - 3;
10996       // Is the constant foldable in the shift of the addressing mode?
10997       // I.e., shift amount is between 1 and 4 inclusive.
10998       if (ShiftAmt == 0 || ShiftAmt > 4)
10999         return false;
11000       break;
11001     }
11002     case Instruction::Trunc:
11003       // Check if this is a noop.
11004       // trunc(sext ty1 to ty2) to ty1.
11005       if (Instr->getType() == Ext->getOperand(0)->getType())
11006         continue;
11007       LLVM_FALLTHROUGH;
11008     default:
11009       return false;
11010     }
11011 
11012     // At this point we can use the bfm family, so this extension is free
11013     // for that use.
11014   }
11015   return true;
11016 }
11017 
11018 /// Check if both Op1 and Op2 are shufflevector extracts of either the lower
11019 /// or upper half of the vector elements.
11020 static bool areExtractShuffleVectors(Value *Op1, Value *Op2) {
11021   auto areTypesHalfed = [](Value *FullV, Value *HalfV) {
11022     auto *FullTy = FullV->getType();
11023     auto *HalfTy = HalfV->getType();
11024     return FullTy->getPrimitiveSizeInBits().getFixedSize() ==
11025            2 * HalfTy->getPrimitiveSizeInBits().getFixedSize();
11026   };
11027 
11028   auto extractHalf = [](Value *FullV, Value *HalfV) {
11029     auto *FullVT = cast<FixedVectorType>(FullV->getType());
11030     auto *HalfVT = cast<FixedVectorType>(HalfV->getType());
11031     return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
11032   };
11033 
11034   ArrayRef<int> M1, M2;
11035   Value *S1Op1, *S2Op1;
11036   if (!match(Op1, m_Shuffle(m_Value(S1Op1), m_Undef(), m_Mask(M1))) ||
11037       !match(Op2, m_Shuffle(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
11038     return false;
11039 
11040   // Check that the operands are half as wide as the result and we extract
11041   // half of the elements of the input vectors.
11042   if (!areTypesHalfed(S1Op1, Op1) || !areTypesHalfed(S2Op1, Op2) ||
11043       !extractHalf(S1Op1, Op1) || !extractHalf(S2Op1, Op2))
11044     return false;
11045 
11046   // Check the mask extracts either the lower or upper half of vector
11047   // elements.
11048   int M1Start = -1;
11049   int M2Start = -1;
11050   int NumElements = cast<FixedVectorType>(Op1->getType())->getNumElements() * 2;
11051   if (!ShuffleVectorInst::isExtractSubvectorMask(M1, NumElements, M1Start) ||
11052       !ShuffleVectorInst::isExtractSubvectorMask(M2, NumElements, M2Start) ||
11053       M1Start != M2Start || (M1Start != 0 && M2Start != (NumElements / 2)))
11054     return false;
11055 
11056   return true;
11057 }
11058 
11059 /// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
11060 /// of the vector elements.
11061 static bool areExtractExts(Value *Ext1, Value *Ext2) {
11062   auto areExtDoubled = [](Instruction *Ext) {
11063     return Ext->getType()->getScalarSizeInBits() ==
11064            2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
11065   };
11066 
11067   if (!match(Ext1, m_ZExtOrSExt(m_Value())) ||
11068       !match(Ext2, m_ZExtOrSExt(m_Value())) ||
11069       !areExtDoubled(cast<Instruction>(Ext1)) ||
11070       !areExtDoubled(cast<Instruction>(Ext2)))
11071     return false;
11072 
11073   return true;
11074 }
11075 
11076 /// Check if Op could be used with vmull_high_p64 intrinsic.
11077 static bool isOperandOfVmullHighP64(Value *Op) {
11078   Value *VectorOperand = nullptr;
11079   ConstantInt *ElementIndex = nullptr;
11080   return match(Op, m_ExtractElt(m_Value(VectorOperand),
11081                                 m_ConstantInt(ElementIndex))) &&
11082          ElementIndex->getValue() == 1 &&
11083          isa<FixedVectorType>(VectorOperand->getType()) &&
11084          cast<FixedVectorType>(VectorOperand->getType())->getNumElements() == 2;
11085 }
11086 
11087 /// Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
11088 static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2) {
11089   return isOperandOfVmullHighP64(Op1) && isOperandOfVmullHighP64(Op2);
11090 }
11091 
11092 /// Check if sinking \p I's operands to I's basic block is profitable, because
11093 /// the operands can be folded into a target instruction, e.g.
11094 /// shufflevectors extracts and/or sext/zext can be folded into (u,s)subl(2).
11095 bool AArch64TargetLowering::shouldSinkOperands(
11096     Instruction *I, SmallVectorImpl<Use *> &Ops) const {
11097   if (!I->getType()->isVectorTy())
11098     return false;
11099 
11100   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
11101     switch (II->getIntrinsicID()) {
11102     case Intrinsic::aarch64_neon_umull:
11103       if (!areExtractShuffleVectors(II->getOperand(0), II->getOperand(1)))
11104         return false;
11105       Ops.push_back(&II->getOperandUse(0));
11106       Ops.push_back(&II->getOperandUse(1));
11107       return true;
11108 
11109     case Intrinsic::aarch64_neon_pmull64:
11110       if (!areOperandsOfVmullHighP64(II->getArgOperand(0),
11111                                      II->getArgOperand(1)))
11112         return false;
11113       Ops.push_back(&II->getArgOperandUse(0));
11114       Ops.push_back(&II->getArgOperandUse(1));
11115       return true;
11116 
11117     default:
11118       return false;
11119     }
11120   }
11121 
11122   switch (I->getOpcode()) {
11123   case Instruction::Sub:
11124   case Instruction::Add: {
11125     if (!areExtractExts(I->getOperand(0), I->getOperand(1)))
11126       return false;
11127 
11128     // If the exts' operands extract either the lower or upper elements, we
11129     // can sink them too.
11130     auto Ext1 = cast<Instruction>(I->getOperand(0));
11131     auto Ext2 = cast<Instruction>(I->getOperand(1));
11132     if (areExtractShuffleVectors(Ext1, Ext2)) {
11133       Ops.push_back(&Ext1->getOperandUse(0));
11134       Ops.push_back(&Ext2->getOperandUse(0));
11135     }
11136 
11137     Ops.push_back(&I->getOperandUse(0));
11138     Ops.push_back(&I->getOperandUse(1));
11139 
11140     return true;
11141   }
11142   case Instruction::Mul: {
11143     bool IsProfitable = false;
11144     for (auto &Op : I->operands()) {
11145       // Make sure we are not already sinking this operand
11146       if (any_of(Ops, [&](Use *U) { return U->get() == Op; }))
11147         continue;
11148 
11149       ShuffleVectorInst *Shuffle = dyn_cast<ShuffleVectorInst>(Op);
11150       if (!Shuffle || !Shuffle->isZeroEltSplat())
11151         continue;
11152 
11153       Value *ShuffleOperand = Shuffle->getOperand(0);
11154       InsertElementInst *Insert = dyn_cast<InsertElementInst>(ShuffleOperand);
11155       if (!Insert)
11156         continue;
11157 
11158       Instruction *OperandInstr = dyn_cast<Instruction>(Insert->getOperand(1));
11159       if (!OperandInstr)
11160         continue;
11161 
11162       ConstantInt *ElementConstant =
11163           dyn_cast<ConstantInt>(Insert->getOperand(2));
11164       // Check that the insertelement is inserting into element 0
11165       if (!ElementConstant || ElementConstant->getZExtValue() != 0)
11166         continue;
11167 
11168       unsigned Opcode = OperandInstr->getOpcode();
11169       if (Opcode != Instruction::SExt && Opcode != Instruction::ZExt)
11170         continue;
11171 
11172       Ops.push_back(&Shuffle->getOperandUse(0));
11173       Ops.push_back(&Op);
11174       IsProfitable = true;
11175     }
11176 
11177     return IsProfitable;
11178   }
11179   default:
11180     return false;
11181   }
11182   return false;
11183 }
11184 
11185 bool AArch64TargetLowering::hasPairedLoad(EVT LoadedType,
11186                                           Align &RequiredAligment) const {
11187   if (!LoadedType.isSimple() ||
11188       (!LoadedType.isInteger() && !LoadedType.isFloatingPoint()))
11189     return false;
11190   // Cyclone supports unaligned accesses.
11191   RequiredAligment = Align(1);
11192   unsigned NumBits = LoadedType.getSizeInBits();
11193   return NumBits == 32 || NumBits == 64;
11194 }
11195 
11196 /// A helper function for determining the number of interleaved accesses we
11197 /// will generate when lowering accesses of the given type.
11198 unsigned
11199 AArch64TargetLowering::getNumInterleavedAccesses(VectorType *VecTy,
11200                                                  const DataLayout &DL) const {
11201   return (DL.getTypeSizeInBits(VecTy) + 127) / 128;
11202 }
11203 
11204 MachineMemOperand::Flags
11205 AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
11206   if (Subtarget->getProcFamily() == AArch64Subtarget::Falkor &&
11207       I.getMetadata(FALKOR_STRIDED_ACCESS_MD) != nullptr)
11208     return MOStridedAccess;
11209   return MachineMemOperand::MONone;
11210 }
11211 
11212 bool AArch64TargetLowering::isLegalInterleavedAccessType(
11213     VectorType *VecTy, const DataLayout &DL) const {
11214 
11215   unsigned VecSize = DL.getTypeSizeInBits(VecTy);
11216   unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
11217 
11218   // Ensure the number of vector elements is greater than 1.
11219   if (cast<FixedVectorType>(VecTy)->getNumElements() < 2)
11220     return false;
11221 
11222   // Ensure the element type is legal.
11223   if (ElSize != 8 && ElSize != 16 && ElSize != 32 && ElSize != 64)
11224     return false;
11225 
11226   // Ensure the total vector size is 64 or a multiple of 128. Types larger than
11227   // 128 will be split into multiple interleaved accesses.
11228   return VecSize == 64 || VecSize % 128 == 0;
11229 }
11230 
11231 /// Lower an interleaved load into a ldN intrinsic.
11232 ///
11233 /// E.g. Lower an interleaved load (Factor = 2):
11234 ///        %wide.vec = load <8 x i32>, <8 x i32>* %ptr
11235 ///        %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6>  ; Extract even elements
11236 ///        %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7>  ; Extract odd elements
11237 ///
11238 ///      Into:
11239 ///        %ld2 = { <4 x i32>, <4 x i32> } call llvm.aarch64.neon.ld2(%ptr)
11240 ///        %vec0 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 0
11241 ///        %vec1 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 1
11242 bool AArch64TargetLowering::lowerInterleavedLoad(
11243     LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles,
11244     ArrayRef<unsigned> Indices, unsigned Factor) const {
11245   assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
11246          "Invalid interleave factor");
11247   assert(!Shuffles.empty() && "Empty shufflevector input");
11248   assert(Shuffles.size() == Indices.size() &&
11249          "Unmatched number of shufflevectors and indices");
11250 
11251   const DataLayout &DL = LI->getModule()->getDataLayout();
11252 
11253   VectorType *VTy = Shuffles[0]->getType();
11254 
11255   // Skip if we do not have NEON and skip illegal vector types. We can
11256   // "legalize" wide vector types into multiple interleaved accesses as long as
11257   // the vector types are divisible by 128.
11258   if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(VTy, DL))
11259     return false;
11260 
11261   unsigned NumLoads = getNumInterleavedAccesses(VTy, DL);
11262 
11263   auto *FVTy = cast<FixedVectorType>(VTy);
11264 
11265   // A pointer vector can not be the return type of the ldN intrinsics. Need to
11266   // load integer vectors first and then convert to pointer vectors.
11267   Type *EltTy = FVTy->getElementType();
11268   if (EltTy->isPointerTy())
11269     FVTy =
11270         FixedVectorType::get(DL.getIntPtrType(EltTy), FVTy->getNumElements());
11271 
11272   IRBuilder<> Builder(LI);
11273 
11274   // The base address of the load.
11275   Value *BaseAddr = LI->getPointerOperand();
11276 
11277   if (NumLoads > 1) {
11278     // If we're going to generate more than one load, reset the sub-vector type
11279     // to something legal.
11280     FVTy = FixedVectorType::get(FVTy->getElementType(),
11281                                 FVTy->getNumElements() / NumLoads);
11282 
11283     // We will compute the pointer operand of each load from the original base
11284     // address using GEPs. Cast the base address to a pointer to the scalar
11285     // element type.
11286     BaseAddr = Builder.CreateBitCast(
11287         BaseAddr,
11288         FVTy->getElementType()->getPointerTo(LI->getPointerAddressSpace()));
11289   }
11290 
11291   Type *PtrTy = FVTy->getPointerTo(LI->getPointerAddressSpace());
11292   Type *Tys[2] = {FVTy, PtrTy};
11293   static const Intrinsic::ID LoadInts[3] = {Intrinsic::aarch64_neon_ld2,
11294                                             Intrinsic::aarch64_neon_ld3,
11295                                             Intrinsic::aarch64_neon_ld4};
11296   Function *LdNFunc =
11297       Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys);
11298 
11299   // Holds sub-vectors extracted from the load intrinsic return values. The
11300   // sub-vectors are associated with the shufflevector instructions they will
11301   // replace.
11302   DenseMap<ShuffleVectorInst *, SmallVector<Value *, 4>> SubVecs;
11303 
11304   for (unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
11305 
11306     // If we're generating more than one load, compute the base address of
11307     // subsequent loads as an offset from the previous.
11308     if (LoadCount > 0)
11309       BaseAddr = Builder.CreateConstGEP1_32(FVTy->getElementType(), BaseAddr,
11310                                             FVTy->getNumElements() * Factor);
11311 
11312     CallInst *LdN = Builder.CreateCall(
11313         LdNFunc, Builder.CreateBitCast(BaseAddr, PtrTy), "ldN");
11314 
11315     // Extract and store the sub-vectors returned by the load intrinsic.
11316     for (unsigned i = 0; i < Shuffles.size(); i++) {
11317       ShuffleVectorInst *SVI = Shuffles[i];
11318       unsigned Index = Indices[i];
11319 
11320       Value *SubVec = Builder.CreateExtractValue(LdN, Index);
11321 
11322       // Convert the integer vector to pointer vector if the element is pointer.
11323       if (EltTy->isPointerTy())
11324         SubVec = Builder.CreateIntToPtr(
11325             SubVec, FixedVectorType::get(SVI->getType()->getElementType(),
11326                                          FVTy->getNumElements()));
11327       SubVecs[SVI].push_back(SubVec);
11328     }
11329   }
11330 
11331   // Replace uses of the shufflevector instructions with the sub-vectors
11332   // returned by the load intrinsic. If a shufflevector instruction is
11333   // associated with more than one sub-vector, those sub-vectors will be
11334   // concatenated into a single wide vector.
11335   for (ShuffleVectorInst *SVI : Shuffles) {
11336     auto &SubVec = SubVecs[SVI];
11337     auto *WideVec =
11338         SubVec.size() > 1 ? concatenateVectors(Builder, SubVec) : SubVec[0];
11339     SVI->replaceAllUsesWith(WideVec);
11340   }
11341 
11342   return true;
11343 }
11344 
11345 /// Lower an interleaved store into a stN intrinsic.
11346 ///
11347 /// E.g. Lower an interleaved store (Factor = 3):
11348 ///        %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
11349 ///                 <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
11350 ///        store <12 x i32> %i.vec, <12 x i32>* %ptr
11351 ///
11352 ///      Into:
11353 ///        %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
11354 ///        %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
11355 ///        %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
11356 ///        call void llvm.aarch64.neon.st3(%sub.v0, %sub.v1, %sub.v2, %ptr)
11357 ///
11358 /// Note that the new shufflevectors will be removed and we'll only generate one
11359 /// st3 instruction in CodeGen.
11360 ///
11361 /// Example for a more general valid mask (Factor 3). Lower:
11362 ///        %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1,
11363 ///                 <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
11364 ///        store <12 x i32> %i.vec, <12 x i32>* %ptr
11365 ///
11366 ///      Into:
11367 ///        %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7>
11368 ///        %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35>
11369 ///        %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19>
11370 ///        call void llvm.aarch64.neon.st3(%sub.v0, %sub.v1, %sub.v2, %ptr)
11371 bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
11372                                                   ShuffleVectorInst *SVI,
11373                                                   unsigned Factor) const {
11374   assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
11375          "Invalid interleave factor");
11376 
11377   auto *VecTy = cast<FixedVectorType>(SVI->getType());
11378   assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store");
11379 
11380   unsigned LaneLen = VecTy->getNumElements() / Factor;
11381   Type *EltTy = VecTy->getElementType();
11382   auto *SubVecTy = FixedVectorType::get(EltTy, LaneLen);
11383 
11384   const DataLayout &DL = SI->getModule()->getDataLayout();
11385 
11386   // Skip if we do not have NEON and skip illegal vector types. We can
11387   // "legalize" wide vector types into multiple interleaved accesses as long as
11388   // the vector types are divisible by 128.
11389   if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(SubVecTy, DL))
11390     return false;
11391 
11392   unsigned NumStores = getNumInterleavedAccesses(SubVecTy, DL);
11393 
11394   Value *Op0 = SVI->getOperand(0);
11395   Value *Op1 = SVI->getOperand(1);
11396   IRBuilder<> Builder(SI);
11397 
11398   // StN intrinsics don't support pointer vectors as arguments. Convert pointer
11399   // vectors to integer vectors.
11400   if (EltTy->isPointerTy()) {
11401     Type *IntTy = DL.getIntPtrType(EltTy);
11402     unsigned NumOpElts =
11403         cast<FixedVectorType>(Op0->getType())->getNumElements();
11404 
11405     // Convert to the corresponding integer vector.
11406     auto *IntVecTy = FixedVectorType::get(IntTy, NumOpElts);
11407     Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
11408     Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
11409 
11410     SubVecTy = FixedVectorType::get(IntTy, LaneLen);
11411   }
11412 
11413   // The base address of the store.
11414   Value *BaseAddr = SI->getPointerOperand();
11415 
11416   if (NumStores > 1) {
11417     // If we're going to generate more than one store, reset the lane length
11418     // and sub-vector type to something legal.
11419     LaneLen /= NumStores;
11420     SubVecTy = FixedVectorType::get(SubVecTy->getElementType(), LaneLen);
11421 
11422     // We will compute the pointer operand of each store from the original base
11423     // address using GEPs. Cast the base address to a pointer to the scalar
11424     // element type.
11425     BaseAddr = Builder.CreateBitCast(
11426         BaseAddr,
11427         SubVecTy->getElementType()->getPointerTo(SI->getPointerAddressSpace()));
11428   }
11429 
11430   auto Mask = SVI->getShuffleMask();
11431 
11432   Type *PtrTy = SubVecTy->getPointerTo(SI->getPointerAddressSpace());
11433   Type *Tys[2] = {SubVecTy, PtrTy};
11434   static const Intrinsic::ID StoreInts[3] = {Intrinsic::aarch64_neon_st2,
11435                                              Intrinsic::aarch64_neon_st3,
11436                                              Intrinsic::aarch64_neon_st4};
11437   Function *StNFunc =
11438       Intrinsic::getDeclaration(SI->getModule(), StoreInts[Factor - 2], Tys);
11439 
11440   for (unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
11441 
11442     SmallVector<Value *, 5> Ops;
11443 
11444     // Split the shufflevector operands into sub vectors for the new stN call.
11445     for (unsigned i = 0; i < Factor; i++) {
11446       unsigned IdxI = StoreCount * LaneLen * Factor + i;
11447       if (Mask[IdxI] >= 0) {
11448         Ops.push_back(Builder.CreateShuffleVector(
11449             Op0, Op1, createSequentialMask(Mask[IdxI], LaneLen, 0)));
11450       } else {
11451         unsigned StartMask = 0;
11452         for (unsigned j = 1; j < LaneLen; j++) {
11453           unsigned IdxJ = StoreCount * LaneLen * Factor + j;
11454           if (Mask[IdxJ * Factor + IdxI] >= 0) {
11455             StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
11456             break;
11457           }
11458         }
11459         // Note: Filling undef gaps with random elements is ok, since
11460         // those elements were being written anyway (with undefs).
11461         // In the case of all undefs we're defaulting to using elems from 0
11462         // Note: StartMask cannot be negative, it's checked in
11463         // isReInterleaveMask
11464         Ops.push_back(Builder.CreateShuffleVector(
11465             Op0, Op1, createSequentialMask(StartMask, LaneLen, 0)));
11466       }
11467     }
11468 
11469     // If we generating more than one store, we compute the base address of
11470     // subsequent stores as an offset from the previous.
11471     if (StoreCount > 0)
11472       BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getElementType(),
11473                                             BaseAddr, LaneLen * Factor);
11474 
11475     Ops.push_back(Builder.CreateBitCast(BaseAddr, PtrTy));
11476     Builder.CreateCall(StNFunc, Ops);
11477   }
11478   return true;
11479 }
11480 
11481 // Lower an SVE structured load intrinsic returning a tuple type to target
11482 // specific intrinsic taking the same input but returning a multi-result value
11483 // of the split tuple type.
11484 //
11485 // E.g. Lowering an LD3:
11486 //
11487 //  call <vscale x 12 x i32> @llvm.aarch64.sve.ld3.nxv12i32(
11488 //                                                    <vscale x 4 x i1> %pred,
11489 //                                                    <vscale x 4 x i32>* %addr)
11490 //
11491 //  Output DAG:
11492 //
11493 //    t0: ch = EntryToken
11494 //        t2: nxv4i1,ch = CopyFromReg t0, Register:nxv4i1 %0
11495 //        t4: i64,ch = CopyFromReg t0, Register:i64 %1
11496 //    t5: nxv4i32,nxv4i32,nxv4i32,ch = AArch64ISD::SVE_LD3 t0, t2, t4
11497 //    t6: nxv12i32 = concat_vectors t5, t5:1, t5:2
11498 //
11499 // This is called pre-legalization to avoid widening/splitting issues with
11500 // non-power-of-2 tuple types used for LD3, such as nxv12i32.
11501 SDValue AArch64TargetLowering::LowerSVEStructLoad(unsigned Intrinsic,
11502                                                   ArrayRef<SDValue> LoadOps,
11503                                                   EVT VT, SelectionDAG &DAG,
11504                                                   const SDLoc &DL) const {
11505   assert(VT.isScalableVector() && "Can only lower scalable vectors");
11506 
11507   unsigned N, Opcode;
11508   static std::map<unsigned, std::pair<unsigned, unsigned>> IntrinsicMap = {
11509       {Intrinsic::aarch64_sve_ld2, {2, AArch64ISD::SVE_LD2_MERGE_ZERO}},
11510       {Intrinsic::aarch64_sve_ld3, {3, AArch64ISD::SVE_LD3_MERGE_ZERO}},
11511       {Intrinsic::aarch64_sve_ld4, {4, AArch64ISD::SVE_LD4_MERGE_ZERO}}};
11512 
11513   std::tie(N, Opcode) = IntrinsicMap[Intrinsic];
11514   assert(VT.getVectorElementCount().getKnownMinValue() % N == 0 &&
11515          "invalid tuple vector type!");
11516 
11517   EVT SplitVT =
11518       EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
11519                        VT.getVectorElementCount().divideCoefficientBy(N));
11520   assert(isTypeLegal(SplitVT));
11521 
11522   SmallVector<EVT, 5> VTs(N, SplitVT);
11523   VTs.push_back(MVT::Other); // Chain
11524   SDVTList NodeTys = DAG.getVTList(VTs);
11525 
11526   SDValue PseudoLoad = DAG.getNode(Opcode, DL, NodeTys, LoadOps);
11527   SmallVector<SDValue, 4> PseudoLoadOps;
11528   for (unsigned I = 0; I < N; ++I)
11529     PseudoLoadOps.push_back(SDValue(PseudoLoad.getNode(), I));
11530   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, PseudoLoadOps);
11531 }
11532 
11533 EVT AArch64TargetLowering::getOptimalMemOpType(
11534     const MemOp &Op, const AttributeList &FuncAttributes) const {
11535   bool CanImplicitFloat =
11536       !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat);
11537   bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat;
11538   bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat;
11539   // Only use AdvSIMD to implement memset of 32-byte and above. It would have
11540   // taken one instruction to materialize the v2i64 zero and one store (with
11541   // restrictive addressing mode). Just do i64 stores.
11542   bool IsSmallMemset = Op.isMemset() && Op.size() < 32;
11543   auto AlignmentIsAcceptable = [&](EVT VT, Align AlignCheck) {
11544     if (Op.isAligned(AlignCheck))
11545       return true;
11546     bool Fast;
11547     return allowsMisalignedMemoryAccesses(VT, 0, Align(1),
11548                                           MachineMemOperand::MONone, &Fast) &&
11549            Fast;
11550   };
11551 
11552   if (CanUseNEON && Op.isMemset() && !IsSmallMemset &&
11553       AlignmentIsAcceptable(MVT::v2i64, Align(16)))
11554     return MVT::v2i64;
11555   if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(MVT::f128, Align(16)))
11556     return MVT::f128;
11557   if (Op.size() >= 8 && AlignmentIsAcceptable(MVT::i64, Align(8)))
11558     return MVT::i64;
11559   if (Op.size() >= 4 && AlignmentIsAcceptable(MVT::i32, Align(4)))
11560     return MVT::i32;
11561   return MVT::Other;
11562 }
11563 
11564 LLT AArch64TargetLowering::getOptimalMemOpLLT(
11565     const MemOp &Op, const AttributeList &FuncAttributes) const {
11566   bool CanImplicitFloat =
11567       !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat);
11568   bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat;
11569   bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat;
11570   // Only use AdvSIMD to implement memset of 32-byte and above. It would have
11571   // taken one instruction to materialize the v2i64 zero and one store (with
11572   // restrictive addressing mode). Just do i64 stores.
11573   bool IsSmallMemset = Op.isMemset() && Op.size() < 32;
11574   auto AlignmentIsAcceptable = [&](EVT VT, Align AlignCheck) {
11575     if (Op.isAligned(AlignCheck))
11576       return true;
11577     bool Fast;
11578     return allowsMisalignedMemoryAccesses(VT, 0, Align(1),
11579                                           MachineMemOperand::MONone, &Fast) &&
11580            Fast;
11581   };
11582 
11583   if (CanUseNEON && Op.isMemset() && !IsSmallMemset &&
11584       AlignmentIsAcceptable(MVT::v2i64, Align(16)))
11585     return LLT::vector(2, 64);
11586   if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(MVT::f128, Align(16)))
11587     return LLT::scalar(128);
11588   if (Op.size() >= 8 && AlignmentIsAcceptable(MVT::i64, Align(8)))
11589     return LLT::scalar(64);
11590   if (Op.size() >= 4 && AlignmentIsAcceptable(MVT::i32, Align(4)))
11591     return LLT::scalar(32);
11592   return LLT();
11593 }
11594 
11595 // 12-bit optionally shifted immediates are legal for adds.
11596 bool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const {
11597   if (Immed == std::numeric_limits<int64_t>::min()) {
11598     LLVM_DEBUG(dbgs() << "Illegal add imm " << Immed
11599                       << ": avoid UB for INT64_MIN\n");
11600     return false;
11601   }
11602   // Same encoding for add/sub, just flip the sign.
11603   Immed = std::abs(Immed);
11604   bool IsLegal = ((Immed >> 12) == 0 ||
11605                   ((Immed & 0xfff) == 0 && Immed >> 24 == 0));
11606   LLVM_DEBUG(dbgs() << "Is " << Immed
11607                     << " legal add imm: " << (IsLegal ? "yes" : "no") << "\n");
11608   return IsLegal;
11609 }
11610 
11611 // Integer comparisons are implemented with ADDS/SUBS, so the range of valid
11612 // immediates is the same as for an add or a sub.
11613 bool AArch64TargetLowering::isLegalICmpImmediate(int64_t Immed) const {
11614   return isLegalAddImmediate(Immed);
11615 }
11616 
11617 /// isLegalAddressingMode - Return true if the addressing mode represented
11618 /// by AM is legal for this target, for a load/store of the specified type.
11619 bool AArch64TargetLowering::isLegalAddressingMode(const DataLayout &DL,
11620                                                   const AddrMode &AM, Type *Ty,
11621                                                   unsigned AS, Instruction *I) const {
11622   // AArch64 has five basic addressing modes:
11623   //  reg
11624   //  reg + 9-bit signed offset
11625   //  reg + SIZE_IN_BYTES * 12-bit unsigned offset
11626   //  reg1 + reg2
11627   //  reg + SIZE_IN_BYTES * reg
11628 
11629   // No global is ever allowed as a base.
11630   if (AM.BaseGV)
11631     return false;
11632 
11633   // No reg+reg+imm addressing.
11634   if (AM.HasBaseReg && AM.BaseOffs && AM.Scale)
11635     return false;
11636 
11637   // FIXME: Update this method to support scalable addressing modes.
11638   if (isa<ScalableVectorType>(Ty))
11639     return AM.HasBaseReg && !AM.BaseOffs && !AM.Scale;
11640 
11641   // check reg + imm case:
11642   // i.e., reg + 0, reg + imm9, reg + SIZE_IN_BYTES * uimm12
11643   uint64_t NumBytes = 0;
11644   if (Ty->isSized()) {
11645     uint64_t NumBits = DL.getTypeSizeInBits(Ty);
11646     NumBytes = NumBits / 8;
11647     if (!isPowerOf2_64(NumBits))
11648       NumBytes = 0;
11649   }
11650 
11651   if (!AM.Scale) {
11652     int64_t Offset = AM.BaseOffs;
11653 
11654     // 9-bit signed offset
11655     if (isInt<9>(Offset))
11656       return true;
11657 
11658     // 12-bit unsigned offset
11659     unsigned shift = Log2_64(NumBytes);
11660     if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 &&
11661         // Must be a multiple of NumBytes (NumBytes is a power of 2)
11662         (Offset >> shift) << shift == Offset)
11663       return true;
11664     return false;
11665   }
11666 
11667   // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2
11668 
11669   return AM.Scale == 1 || (AM.Scale > 0 && (uint64_t)AM.Scale == NumBytes);
11670 }
11671 
11672 bool AArch64TargetLowering::shouldConsiderGEPOffsetSplit() const {
11673   // Consider splitting large offset of struct or array.
11674   return true;
11675 }
11676 
11677 int AArch64TargetLowering::getScalingFactorCost(const DataLayout &DL,
11678                                                 const AddrMode &AM, Type *Ty,
11679                                                 unsigned AS) const {
11680   // Scaling factors are not free at all.
11681   // Operands                     | Rt Latency
11682   // -------------------------------------------
11683   // Rt, [Xn, Xm]                 | 4
11684   // -------------------------------------------
11685   // Rt, [Xn, Xm, lsl #imm]       | Rn: 4 Rm: 5
11686   // Rt, [Xn, Wm, <extend> #imm]  |
11687   if (isLegalAddressingMode(DL, AM, Ty, AS))
11688     // Scale represents reg2 * scale, thus account for 1 if
11689     // it is not equal to 0 or 1.
11690     return AM.Scale != 0 && AM.Scale != 1;
11691   return -1;
11692 }
11693 
11694 bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(
11695     const MachineFunction &MF, EVT VT) const {
11696   VT = VT.getScalarType();
11697 
11698   if (!VT.isSimple())
11699     return false;
11700 
11701   switch (VT.getSimpleVT().SimpleTy) {
11702   case MVT::f16:
11703     return Subtarget->hasFullFP16();
11704   case MVT::f32:
11705   case MVT::f64:
11706     return true;
11707   default:
11708     break;
11709   }
11710 
11711   return false;
11712 }
11713 
11714 bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
11715                                                        Type *Ty) const {
11716   switch (Ty->getScalarType()->getTypeID()) {
11717   case Type::FloatTyID:
11718   case Type::DoubleTyID:
11719     return true;
11720   default:
11721     return false;
11722   }
11723 }
11724 
11725 bool AArch64TargetLowering::generateFMAsInMachineCombiner(
11726     EVT VT, CodeGenOpt::Level OptLevel) const {
11727   return (OptLevel >= CodeGenOpt::Aggressive) && !VT.isScalableVector();
11728 }
11729 
11730 const MCPhysReg *
11731 AArch64TargetLowering::getScratchRegisters(CallingConv::ID) const {
11732   // LR is a callee-save register, but we must treat it as clobbered by any call
11733   // site. Hence we include LR in the scratch registers, which are in turn added
11734   // as implicit-defs for stackmaps and patchpoints.
11735   static const MCPhysReg ScratchRegs[] = {
11736     AArch64::X16, AArch64::X17, AArch64::LR, 0
11737   };
11738   return ScratchRegs;
11739 }
11740 
11741 bool
11742 AArch64TargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
11743                                                      CombineLevel Level) const {
11744   N = N->getOperand(0).getNode();
11745   EVT VT = N->getValueType(0);
11746     // If N is unsigned bit extraction: ((x >> C) & mask), then do not combine
11747     // it with shift to let it be lowered to UBFX.
11748   if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) &&
11749       isa<ConstantSDNode>(N->getOperand(1))) {
11750     uint64_t TruncMask = N->getConstantOperandVal(1);
11751     if (isMask_64(TruncMask) &&
11752       N->getOperand(0).getOpcode() == ISD::SRL &&
11753       isa<ConstantSDNode>(N->getOperand(0)->getOperand(1)))
11754       return false;
11755   }
11756   return true;
11757 }
11758 
11759 bool AArch64TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
11760                                                               Type *Ty) const {
11761   assert(Ty->isIntegerTy());
11762 
11763   unsigned BitSize = Ty->getPrimitiveSizeInBits();
11764   if (BitSize == 0)
11765     return false;
11766 
11767   int64_t Val = Imm.getSExtValue();
11768   if (Val == 0 || AArch64_AM::isLogicalImmediate(Val, BitSize))
11769     return true;
11770 
11771   if ((int64_t)Val < 0)
11772     Val = ~Val;
11773   if (BitSize == 32)
11774     Val &= (1LL << 32) - 1;
11775 
11776   unsigned LZ = countLeadingZeros((uint64_t)Val);
11777   unsigned Shift = (63 - LZ) / 16;
11778   // MOVZ is free so return true for one or fewer MOVK.
11779   return Shift < 3;
11780 }
11781 
11782 bool AArch64TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
11783                                                     unsigned Index) const {
11784   if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
11785     return false;
11786 
11787   return (Index == 0 || Index == ResVT.getVectorNumElements());
11788 }
11789 
11790 /// Turn vector tests of the signbit in the form of:
11791 ///   xor (sra X, elt_size(X)-1), -1
11792 /// into:
11793 ///   cmge X, X, #0
11794 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
11795                                          const AArch64Subtarget *Subtarget) {
11796   EVT VT = N->getValueType(0);
11797   if (!Subtarget->hasNEON() || !VT.isVector())
11798     return SDValue();
11799 
11800   // There must be a shift right algebraic before the xor, and the xor must be a
11801   // 'not' operation.
11802   SDValue Shift = N->getOperand(0);
11803   SDValue Ones = N->getOperand(1);
11804   if (Shift.getOpcode() != AArch64ISD::VASHR || !Shift.hasOneUse() ||
11805       !ISD::isBuildVectorAllOnes(Ones.getNode()))
11806     return SDValue();
11807 
11808   // The shift should be smearing the sign bit across each vector element.
11809   auto *ShiftAmt = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
11810   EVT ShiftEltTy = Shift.getValueType().getVectorElementType();
11811   if (!ShiftAmt || ShiftAmt->getZExtValue() != ShiftEltTy.getSizeInBits() - 1)
11812     return SDValue();
11813 
11814   return DAG.getNode(AArch64ISD::CMGEz, SDLoc(N), VT, Shift.getOperand(0));
11815 }
11816 
11817 // Turn a v8i8/v16i8 extended vecreduce into a udot/sdot and vecreduce
11818 //   vecreduce.add(ext(A)) to vecreduce.add(DOT(zero, A, one))
11819 //   vecreduce.add(mul(ext(A), ext(B))) to vecreduce.add(DOT(zero, A, B))
11820 static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
11821                                           const AArch64Subtarget *ST) {
11822   SDValue Op0 = N->getOperand(0);
11823   if (!ST->hasDotProd() || N->getValueType(0) != MVT::i32 ||
11824       Op0.getValueType().getVectorElementType() != MVT::i32)
11825     return SDValue();
11826 
11827   unsigned ExtOpcode = Op0.getOpcode();
11828   SDValue A = Op0;
11829   SDValue B;
11830   if (ExtOpcode == ISD::MUL) {
11831     A = Op0.getOperand(0);
11832     B = Op0.getOperand(1);
11833     if (A.getOpcode() != B.getOpcode() ||
11834         A.getOperand(0).getValueType() != B.getOperand(0).getValueType())
11835       return SDValue();
11836     ExtOpcode = A.getOpcode();
11837   }
11838   if (ExtOpcode != ISD::ZERO_EXTEND && ExtOpcode != ISD::SIGN_EXTEND)
11839     return SDValue();
11840 
11841   EVT Op0VT = A.getOperand(0).getValueType();
11842   if (Op0VT != MVT::v8i8 && Op0VT != MVT::v16i8)
11843     return SDValue();
11844 
11845   SDLoc DL(Op0);
11846   // For non-mla reductions B can be set to 1. For MLA we take the operand of
11847   // the extend B.
11848   if (!B)
11849     B = DAG.getConstant(1, DL, Op0VT);
11850   else
11851     B = B.getOperand(0);
11852 
11853   SDValue Zeros =
11854       DAG.getConstant(0, DL, Op0VT == MVT::v8i8 ? MVT::v2i32 : MVT::v4i32);
11855   auto DotOpcode =
11856       (ExtOpcode == ISD::ZERO_EXTEND) ? AArch64ISD::UDOT : AArch64ISD::SDOT;
11857   SDValue Dot = DAG.getNode(DotOpcode, DL, Zeros.getValueType(), Zeros,
11858                             A.getOperand(0), B);
11859   return DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0), Dot);
11860 }
11861 
11862 // Given a ABS node, detect the following pattern:
11863 // (ABS (SUB (EXTEND a), (EXTEND b))).
11864 // Generates UABD/SABD instruction.
11865 static SDValue performABSCombine(SDNode *N, SelectionDAG &DAG,
11866                                  TargetLowering::DAGCombinerInfo &DCI,
11867                                  const AArch64Subtarget *Subtarget) {
11868   SDValue AbsOp1 = N->getOperand(0);
11869   SDValue Op0, Op1;
11870 
11871   if (AbsOp1.getOpcode() != ISD::SUB)
11872     return SDValue();
11873 
11874   Op0 = AbsOp1.getOperand(0);
11875   Op1 = AbsOp1.getOperand(1);
11876 
11877   unsigned Opc0 = Op0.getOpcode();
11878   // Check if the operands of the sub are (zero|sign)-extended.
11879   if (Opc0 != Op1.getOpcode() ||
11880       (Opc0 != ISD::ZERO_EXTEND && Opc0 != ISD::SIGN_EXTEND))
11881     return SDValue();
11882 
11883   EVT VectorT1 = Op0.getOperand(0).getValueType();
11884   EVT VectorT2 = Op1.getOperand(0).getValueType();
11885   // Check if vectors are of same type and valid size.
11886   uint64_t Size = VectorT1.getFixedSizeInBits();
11887   if (VectorT1 != VectorT2 || (Size != 64 && Size != 128))
11888     return SDValue();
11889 
11890   // Check if vector element types are valid.
11891   EVT VT1 = VectorT1.getVectorElementType();
11892   if (VT1 != MVT::i8 && VT1 != MVT::i16 && VT1 != MVT::i32)
11893     return SDValue();
11894 
11895   Op0 = Op0.getOperand(0);
11896   Op1 = Op1.getOperand(0);
11897   unsigned ABDOpcode =
11898       (Opc0 == ISD::SIGN_EXTEND) ? AArch64ISD::SABD : AArch64ISD::UABD;
11899   SDValue ABD =
11900       DAG.getNode(ABDOpcode, SDLoc(N), Op0->getValueType(0), Op0, Op1);
11901   return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), ABD);
11902 }
11903 
11904 static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG,
11905                                  TargetLowering::DAGCombinerInfo &DCI,
11906                                  const AArch64Subtarget *Subtarget) {
11907   if (DCI.isBeforeLegalizeOps())
11908     return SDValue();
11909 
11910   return foldVectorXorShiftIntoCmp(N, DAG, Subtarget);
11911 }
11912 
11913 SDValue
11914 AArch64TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
11915                                      SelectionDAG &DAG,
11916                                      SmallVectorImpl<SDNode *> &Created) const {
11917   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
11918   if (isIntDivCheap(N->getValueType(0), Attr))
11919     return SDValue(N,0); // Lower SDIV as SDIV
11920 
11921   // fold (sdiv X, pow2)
11922   EVT VT = N->getValueType(0);
11923   if ((VT != MVT::i32 && VT != MVT::i64) ||
11924       !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2()))
11925     return SDValue();
11926 
11927   SDLoc DL(N);
11928   SDValue N0 = N->getOperand(0);
11929   unsigned Lg2 = Divisor.countTrailingZeros();
11930   SDValue Zero = DAG.getConstant(0, DL, VT);
11931   SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
11932 
11933   // Add (N0 < 0) ? Pow2 - 1 : 0;
11934   SDValue CCVal;
11935   SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETLT, CCVal, DAG, DL);
11936   SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
11937   SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp);
11938 
11939   Created.push_back(Cmp.getNode());
11940   Created.push_back(Add.getNode());
11941   Created.push_back(CSel.getNode());
11942 
11943   // Divide by pow2.
11944   SDValue SRA =
11945       DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64));
11946 
11947   // If we're dividing by a positive value, we're done.  Otherwise, we must
11948   // negate the result.
11949   if (Divisor.isNonNegative())
11950     return SRA;
11951 
11952   Created.push_back(SRA.getNode());
11953   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
11954 }
11955 
11956 static bool IsSVECntIntrinsic(SDValue S) {
11957   switch(getIntrinsicID(S.getNode())) {
11958   default:
11959     break;
11960   case Intrinsic::aarch64_sve_cntb:
11961   case Intrinsic::aarch64_sve_cnth:
11962   case Intrinsic::aarch64_sve_cntw:
11963   case Intrinsic::aarch64_sve_cntd:
11964     return true;
11965   }
11966   return false;
11967 }
11968 
11969 /// Calculates what the pre-extend type is, based on the extension
11970 /// operation node provided by \p Extend.
11971 ///
11972 /// In the case that \p Extend is a SIGN_EXTEND or a ZERO_EXTEND, the
11973 /// pre-extend type is pulled directly from the operand, while other extend
11974 /// operations need a bit more inspection to get this information.
11975 ///
11976 /// \param Extend The SDNode from the DAG that represents the extend operation
11977 /// \param DAG The SelectionDAG hosting the \p Extend node
11978 ///
11979 /// \returns The type representing the \p Extend source type, or \p MVT::Other
11980 /// if no valid type can be determined
11981 static EVT calculatePreExtendType(SDValue Extend, SelectionDAG &DAG) {
11982   switch (Extend.getOpcode()) {
11983   case ISD::SIGN_EXTEND:
11984   case ISD::ZERO_EXTEND:
11985     return Extend.getOperand(0).getValueType();
11986   case ISD::AssertSext:
11987   case ISD::AssertZext:
11988   case ISD::SIGN_EXTEND_INREG: {
11989     VTSDNode *TypeNode = dyn_cast<VTSDNode>(Extend.getOperand(1));
11990     if (!TypeNode)
11991       return MVT::Other;
11992     return TypeNode->getVT();
11993   }
11994   case ISD::AND: {
11995     ConstantSDNode *Constant =
11996         dyn_cast<ConstantSDNode>(Extend.getOperand(1).getNode());
11997     if (!Constant)
11998       return MVT::Other;
11999 
12000     uint32_t Mask = Constant->getZExtValue();
12001 
12002     if (Mask == UCHAR_MAX)
12003       return MVT::i8;
12004     else if (Mask == USHRT_MAX)
12005       return MVT::i16;
12006     else if (Mask == UINT_MAX)
12007       return MVT::i32;
12008 
12009     return MVT::Other;
12010   }
12011   default:
12012     return MVT::Other;
12013   }
12014 
12015   llvm_unreachable("Code path unhandled in calculatePreExtendType!");
12016 }
12017 
12018 /// Combines a dup(sext/zext) node pattern into sext/zext(dup)
12019 /// making use of the vector SExt/ZExt rather than the scalar SExt/ZExt
12020 static SDValue performCommonVectorExtendCombine(SDValue VectorShuffle,
12021                                                 SelectionDAG &DAG) {
12022 
12023   ShuffleVectorSDNode *ShuffleNode =
12024       dyn_cast<ShuffleVectorSDNode>(VectorShuffle.getNode());
12025   if (!ShuffleNode)
12026     return SDValue();
12027 
12028   // Ensuring the mask is zero before continuing
12029   if (!ShuffleNode->isSplat() || ShuffleNode->getSplatIndex() != 0)
12030     return SDValue();
12031 
12032   SDValue InsertVectorElt = VectorShuffle.getOperand(0);
12033 
12034   if (InsertVectorElt.getOpcode() != ISD::INSERT_VECTOR_ELT)
12035     return SDValue();
12036 
12037   SDValue InsertLane = InsertVectorElt.getOperand(2);
12038   ConstantSDNode *Constant = dyn_cast<ConstantSDNode>(InsertLane.getNode());
12039   // Ensures the insert is inserting into lane 0
12040   if (!Constant || Constant->getZExtValue() != 0)
12041     return SDValue();
12042 
12043   SDValue Extend = InsertVectorElt.getOperand(1);
12044   unsigned ExtendOpcode = Extend.getOpcode();
12045 
12046   bool IsSExt = ExtendOpcode == ISD::SIGN_EXTEND ||
12047                 ExtendOpcode == ISD::SIGN_EXTEND_INREG ||
12048                 ExtendOpcode == ISD::AssertSext;
12049   if (!IsSExt && ExtendOpcode != ISD::ZERO_EXTEND &&
12050       ExtendOpcode != ISD::AssertZext && ExtendOpcode != ISD::AND)
12051     return SDValue();
12052 
12053   EVT TargetType = VectorShuffle.getValueType();
12054   EVT PreExtendType = calculatePreExtendType(Extend, DAG);
12055 
12056   if ((TargetType != MVT::v8i16 && TargetType != MVT::v4i32 &&
12057        TargetType != MVT::v2i64) ||
12058       (PreExtendType == MVT::Other))
12059     return SDValue();
12060 
12061   // Restrict valid pre-extend data type
12062   if (PreExtendType != MVT::i8 && PreExtendType != MVT::i16 &&
12063       PreExtendType != MVT::i32)
12064     return SDValue();
12065 
12066   EVT PreExtendVT = TargetType.changeVectorElementType(PreExtendType);
12067 
12068   if (PreExtendVT.getVectorElementCount() != TargetType.getVectorElementCount())
12069     return SDValue();
12070 
12071   if (TargetType.getScalarSizeInBits() != PreExtendVT.getScalarSizeInBits() * 2)
12072     return SDValue();
12073 
12074   SDLoc DL(VectorShuffle);
12075 
12076   SDValue InsertVectorNode = DAG.getNode(
12077       InsertVectorElt.getOpcode(), DL, PreExtendVT, DAG.getUNDEF(PreExtendVT),
12078       DAG.getAnyExtOrTrunc(Extend.getOperand(0), DL, PreExtendType),
12079       DAG.getConstant(0, DL, MVT::i64));
12080 
12081   std::vector<int> ShuffleMask(TargetType.getVectorElementCount().getValue());
12082 
12083   SDValue VectorShuffleNode =
12084       DAG.getVectorShuffle(PreExtendVT, DL, InsertVectorNode,
12085                            DAG.getUNDEF(PreExtendVT), ShuffleMask);
12086 
12087   SDValue ExtendNode = DAG.getNode(IsSExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
12088                                    DL, TargetType, VectorShuffleNode);
12089 
12090   return ExtendNode;
12091 }
12092 
12093 /// Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup))
12094 /// making use of the vector SExt/ZExt rather than the scalar SExt/ZExt
12095 static SDValue performMulVectorExtendCombine(SDNode *Mul, SelectionDAG &DAG) {
12096   // If the value type isn't a vector, none of the operands are going to be dups
12097   if (!Mul->getValueType(0).isVector())
12098     return SDValue();
12099 
12100   SDValue Op0 = performCommonVectorExtendCombine(Mul->getOperand(0), DAG);
12101   SDValue Op1 = performCommonVectorExtendCombine(Mul->getOperand(1), DAG);
12102 
12103   // Neither operands have been changed, don't make any further changes
12104   if (!Op0 && !Op1)
12105     return SDValue();
12106 
12107   SDLoc DL(Mul);
12108   return DAG.getNode(Mul->getOpcode(), DL, Mul->getValueType(0),
12109                      Op0 ? Op0 : Mul->getOperand(0),
12110                      Op1 ? Op1 : Mul->getOperand(1));
12111 }
12112 
12113 static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
12114                                  TargetLowering::DAGCombinerInfo &DCI,
12115                                  const AArch64Subtarget *Subtarget) {
12116 
12117   if (SDValue Ext = performMulVectorExtendCombine(N, DAG))
12118     return Ext;
12119 
12120   if (DCI.isBeforeLegalizeOps())
12121     return SDValue();
12122 
12123   // The below optimizations require a constant RHS.
12124   if (!isa<ConstantSDNode>(N->getOperand(1)))
12125     return SDValue();
12126 
12127   SDValue N0 = N->getOperand(0);
12128   ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(1));
12129   const APInt &ConstValue = C->getAPIntValue();
12130 
12131   // Allow the scaling to be folded into the `cnt` instruction by preventing
12132   // the scaling to be obscured here. This makes it easier to pattern match.
12133   if (IsSVECntIntrinsic(N0) ||
12134      (N0->getOpcode() == ISD::TRUNCATE &&
12135       (IsSVECntIntrinsic(N0->getOperand(0)))))
12136        if (ConstValue.sge(1) && ConstValue.sle(16))
12137          return SDValue();
12138 
12139   // Multiplication of a power of two plus/minus one can be done more
12140   // cheaply as as shift+add/sub. For now, this is true unilaterally. If
12141   // future CPUs have a cheaper MADD instruction, this may need to be
12142   // gated on a subtarget feature. For Cyclone, 32-bit MADD is 4 cycles and
12143   // 64-bit is 5 cycles, so this is always a win.
12144   // More aggressively, some multiplications N0 * C can be lowered to
12145   // shift+add+shift if the constant C = A * B where A = 2^N + 1 and B = 2^M,
12146   // e.g. 6=3*2=(2+1)*2.
12147   // TODO: consider lowering more cases, e.g. C = 14, -6, -14 or even 45
12148   // which equals to (1+2)*16-(1+2).
12149   // TrailingZeroes is used to test if the mul can be lowered to
12150   // shift+add+shift.
12151   unsigned TrailingZeroes = ConstValue.countTrailingZeros();
12152   if (TrailingZeroes) {
12153     // Conservatively do not lower to shift+add+shift if the mul might be
12154     // folded into smul or umul.
12155     if (N0->hasOneUse() && (isSignExtended(N0.getNode(), DAG) ||
12156                             isZeroExtended(N0.getNode(), DAG)))
12157       return SDValue();
12158     // Conservatively do not lower to shift+add+shift if the mul might be
12159     // folded into madd or msub.
12160     if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ADD ||
12161                            N->use_begin()->getOpcode() == ISD::SUB))
12162       return SDValue();
12163   }
12164   // Use ShiftedConstValue instead of ConstValue to support both shift+add/sub
12165   // and shift+add+shift.
12166   APInt ShiftedConstValue = ConstValue.ashr(TrailingZeroes);
12167 
12168   unsigned ShiftAmt, AddSubOpc;
12169   // Is the shifted value the LHS operand of the add/sub?
12170   bool ShiftValUseIsN0 = true;
12171   // Do we need to negate the result?
12172   bool NegateResult = false;
12173 
12174   if (ConstValue.isNonNegative()) {
12175     // (mul x, 2^N + 1) => (add (shl x, N), x)
12176     // (mul x, 2^N - 1) => (sub (shl x, N), x)
12177     // (mul x, (2^N + 1) * 2^M) => (shl (add (shl x, N), x), M)
12178     APInt SCVMinus1 = ShiftedConstValue - 1;
12179     APInt CVPlus1 = ConstValue + 1;
12180     if (SCVMinus1.isPowerOf2()) {
12181       ShiftAmt = SCVMinus1.logBase2();
12182       AddSubOpc = ISD::ADD;
12183     } else if (CVPlus1.isPowerOf2()) {
12184       ShiftAmt = CVPlus1.logBase2();
12185       AddSubOpc = ISD::SUB;
12186     } else
12187       return SDValue();
12188   } else {
12189     // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
12190     // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
12191     APInt CVNegPlus1 = -ConstValue + 1;
12192     APInt CVNegMinus1 = -ConstValue - 1;
12193     if (CVNegPlus1.isPowerOf2()) {
12194       ShiftAmt = CVNegPlus1.logBase2();
12195       AddSubOpc = ISD::SUB;
12196       ShiftValUseIsN0 = false;
12197     } else if (CVNegMinus1.isPowerOf2()) {
12198       ShiftAmt = CVNegMinus1.logBase2();
12199       AddSubOpc = ISD::ADD;
12200       NegateResult = true;
12201     } else
12202       return SDValue();
12203   }
12204 
12205   SDLoc DL(N);
12206   EVT VT = N->getValueType(0);
12207   SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0,
12208                                    DAG.getConstant(ShiftAmt, DL, MVT::i64));
12209 
12210   SDValue AddSubN0 = ShiftValUseIsN0 ? ShiftedVal : N0;
12211   SDValue AddSubN1 = ShiftValUseIsN0 ? N0 : ShiftedVal;
12212   SDValue Res = DAG.getNode(AddSubOpc, DL, VT, AddSubN0, AddSubN1);
12213   assert(!(NegateResult && TrailingZeroes) &&
12214          "NegateResult and TrailingZeroes cannot both be true for now.");
12215   // Negate the result.
12216   if (NegateResult)
12217     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res);
12218   // Shift the result.
12219   if (TrailingZeroes)
12220     return DAG.getNode(ISD::SHL, DL, VT, Res,
12221                        DAG.getConstant(TrailingZeroes, DL, MVT::i64));
12222   return Res;
12223 }
12224 
12225 static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N,
12226                                                          SelectionDAG &DAG) {
12227   // Take advantage of vector comparisons producing 0 or -1 in each lane to
12228   // optimize away operation when it's from a constant.
12229   //
12230   // The general transformation is:
12231   //    UNARYOP(AND(VECTOR_CMP(x,y), constant)) -->
12232   //       AND(VECTOR_CMP(x,y), constant2)
12233   //    constant2 = UNARYOP(constant)
12234 
12235   // Early exit if this isn't a vector operation, the operand of the
12236   // unary operation isn't a bitwise AND, or if the sizes of the operations
12237   // aren't the same.
12238   EVT VT = N->getValueType(0);
12239   if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND ||
12240       N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC ||
12241       VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits())
12242     return SDValue();
12243 
12244   // Now check that the other operand of the AND is a constant. We could
12245   // make the transformation for non-constant splats as well, but it's unclear
12246   // that would be a benefit as it would not eliminate any operations, just
12247   // perform one more step in scalar code before moving to the vector unit.
12248   if (BuildVectorSDNode *BV =
12249           dyn_cast<BuildVectorSDNode>(N->getOperand(0)->getOperand(1))) {
12250     // Bail out if the vector isn't a constant.
12251     if (!BV->isConstant())
12252       return SDValue();
12253 
12254     // Everything checks out. Build up the new and improved node.
12255     SDLoc DL(N);
12256     EVT IntVT = BV->getValueType(0);
12257     // Create a new constant of the appropriate type for the transformed
12258     // DAG.
12259     SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
12260     // The AND node needs bitcasts to/from an integer vector type around it.
12261     SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
12262     SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
12263                                  N->getOperand(0)->getOperand(0), MaskConst);
12264     SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
12265     return Res;
12266   }
12267 
12268   return SDValue();
12269 }
12270 
12271 static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG,
12272                                      const AArch64Subtarget *Subtarget) {
12273   // First try to optimize away the conversion when it's conditionally from
12274   // a constant. Vectors only.
12275   if (SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG))
12276     return Res;
12277 
12278   EVT VT = N->getValueType(0);
12279   if (VT != MVT::f32 && VT != MVT::f64)
12280     return SDValue();
12281 
12282   // Only optimize when the source and destination types have the same width.
12283   if (VT.getSizeInBits() != N->getOperand(0).getValueSizeInBits())
12284     return SDValue();
12285 
12286   // If the result of an integer load is only used by an integer-to-float
12287   // conversion, use a fp load instead and a AdvSIMD scalar {S|U}CVTF instead.
12288   // This eliminates an "integer-to-vector-move" UOP and improves throughput.
12289   SDValue N0 = N->getOperand(0);
12290   if (Subtarget->hasNEON() && ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
12291       // Do not change the width of a volatile load.
12292       !cast<LoadSDNode>(N0)->isVolatile()) {
12293     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
12294     SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
12295                                LN0->getPointerInfo(), LN0->getAlignment(),
12296                                LN0->getMemOperand()->getFlags());
12297 
12298     // Make sure successors of the original load stay after it by updating them
12299     // to use the new Chain.
12300     DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), Load.getValue(1));
12301 
12302     unsigned Opcode =
12303         (N->getOpcode() == ISD::SINT_TO_FP) ? AArch64ISD::SITOF : AArch64ISD::UITOF;
12304     return DAG.getNode(Opcode, SDLoc(N), VT, Load);
12305   }
12306 
12307   return SDValue();
12308 }
12309 
12310 /// Fold a floating-point multiply by power of two into floating-point to
12311 /// fixed-point conversion.
12312 static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
12313                                      TargetLowering::DAGCombinerInfo &DCI,
12314                                      const AArch64Subtarget *Subtarget) {
12315   if (!Subtarget->hasNEON())
12316     return SDValue();
12317 
12318   if (!N->getValueType(0).isSimple())
12319     return SDValue();
12320 
12321   SDValue Op = N->getOperand(0);
12322   if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
12323       Op.getOpcode() != ISD::FMUL)
12324     return SDValue();
12325 
12326   SDValue ConstVec = Op->getOperand(1);
12327   if (!isa<BuildVectorSDNode>(ConstVec))
12328     return SDValue();
12329 
12330   MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
12331   uint32_t FloatBits = FloatTy.getSizeInBits();
12332   if (FloatBits != 32 && FloatBits != 64)
12333     return SDValue();
12334 
12335   MVT IntTy = N->getSimpleValueType(0).getVectorElementType();
12336   uint32_t IntBits = IntTy.getSizeInBits();
12337   if (IntBits != 16 && IntBits != 32 && IntBits != 64)
12338     return SDValue();
12339 
12340   // Avoid conversions where iN is larger than the float (e.g., float -> i64).
12341   if (IntBits > FloatBits)
12342     return SDValue();
12343 
12344   BitVector UndefElements;
12345   BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
12346   int32_t Bits = IntBits == 64 ? 64 : 32;
12347   int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, Bits + 1);
12348   if (C == -1 || C == 0 || C > Bits)
12349     return SDValue();
12350 
12351   MVT ResTy;
12352   unsigned NumLanes = Op.getValueType().getVectorNumElements();
12353   switch (NumLanes) {
12354   default:
12355     return SDValue();
12356   case 2:
12357     ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64;
12358     break;
12359   case 4:
12360     ResTy = FloatBits == 32 ? MVT::v4i32 : MVT::v4i64;
12361     break;
12362   }
12363 
12364   if (ResTy == MVT::v4i64 && DCI.isBeforeLegalizeOps())
12365     return SDValue();
12366 
12367   assert((ResTy != MVT::v4i64 || DCI.isBeforeLegalizeOps()) &&
12368          "Illegal vector type after legalization");
12369 
12370   SDLoc DL(N);
12371   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
12372   unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfp2fxs
12373                                       : Intrinsic::aarch64_neon_vcvtfp2fxu;
12374   SDValue FixConv =
12375       DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResTy,
12376                   DAG.getConstant(IntrinsicOpcode, DL, MVT::i32),
12377                   Op->getOperand(0), DAG.getConstant(C, DL, MVT::i32));
12378   // We can handle smaller integers by generating an extra trunc.
12379   if (IntBits < FloatBits)
12380     FixConv = DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), FixConv);
12381 
12382   return FixConv;
12383 }
12384 
12385 /// Fold a floating-point divide by power of two into fixed-point to
12386 /// floating-point conversion.
12387 static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG,
12388                                   TargetLowering::DAGCombinerInfo &DCI,
12389                                   const AArch64Subtarget *Subtarget) {
12390   if (!Subtarget->hasNEON())
12391     return SDValue();
12392 
12393   SDValue Op = N->getOperand(0);
12394   unsigned Opc = Op->getOpcode();
12395   if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
12396       !Op.getOperand(0).getValueType().isSimple() ||
12397       (Opc != ISD::SINT_TO_FP && Opc != ISD::UINT_TO_FP))
12398     return SDValue();
12399 
12400   SDValue ConstVec = N->getOperand(1);
12401   if (!isa<BuildVectorSDNode>(ConstVec))
12402     return SDValue();
12403 
12404   MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
12405   int32_t IntBits = IntTy.getSizeInBits();
12406   if (IntBits != 16 && IntBits != 32 && IntBits != 64)
12407     return SDValue();
12408 
12409   MVT FloatTy = N->getSimpleValueType(0).getVectorElementType();
12410   int32_t FloatBits = FloatTy.getSizeInBits();
12411   if (FloatBits != 32 && FloatBits != 64)
12412     return SDValue();
12413 
12414   // Avoid conversions where iN is larger than the float (e.g., i64 -> float).
12415   if (IntBits > FloatBits)
12416     return SDValue();
12417 
12418   BitVector UndefElements;
12419   BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
12420   int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, FloatBits + 1);
12421   if (C == -1 || C == 0 || C > FloatBits)
12422     return SDValue();
12423 
12424   MVT ResTy;
12425   unsigned NumLanes = Op.getValueType().getVectorNumElements();
12426   switch (NumLanes) {
12427   default:
12428     return SDValue();
12429   case 2:
12430     ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64;
12431     break;
12432   case 4:
12433     ResTy = FloatBits == 32 ? MVT::v4i32 : MVT::v4i64;
12434     break;
12435   }
12436 
12437   if (ResTy == MVT::v4i64 && DCI.isBeforeLegalizeOps())
12438     return SDValue();
12439 
12440   SDLoc DL(N);
12441   SDValue ConvInput = Op.getOperand(0);
12442   bool IsSigned = Opc == ISD::SINT_TO_FP;
12443   if (IntBits < FloatBits)
12444     ConvInput = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
12445                             ResTy, ConvInput);
12446 
12447   unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfxs2fp
12448                                       : Intrinsic::aarch64_neon_vcvtfxu2fp;
12449   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
12450                      DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), ConvInput,
12451                      DAG.getConstant(C, DL, MVT::i32));
12452 }
12453 
12454 /// An EXTR instruction is made up of two shifts, ORed together. This helper
12455 /// searches for and classifies those shifts.
12456 static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount,
12457                          bool &FromHi) {
12458   if (N.getOpcode() == ISD::SHL)
12459     FromHi = false;
12460   else if (N.getOpcode() == ISD::SRL)
12461     FromHi = true;
12462   else
12463     return false;
12464 
12465   if (!isa<ConstantSDNode>(N.getOperand(1)))
12466     return false;
12467 
12468   ShiftAmount = N->getConstantOperandVal(1);
12469   Src = N->getOperand(0);
12470   return true;
12471 }
12472 
12473 /// EXTR instruction extracts a contiguous chunk of bits from two existing
12474 /// registers viewed as a high/low pair. This function looks for the pattern:
12475 /// <tt>(or (shl VAL1, \#N), (srl VAL2, \#RegWidth-N))</tt> and replaces it
12476 /// with an EXTR. Can't quite be done in TableGen because the two immediates
12477 /// aren't independent.
12478 static SDValue tryCombineToEXTR(SDNode *N,
12479                                 TargetLowering::DAGCombinerInfo &DCI) {
12480   SelectionDAG &DAG = DCI.DAG;
12481   SDLoc DL(N);
12482   EVT VT = N->getValueType(0);
12483 
12484   assert(N->getOpcode() == ISD::OR && "Unexpected root");
12485 
12486   if (VT != MVT::i32 && VT != MVT::i64)
12487     return SDValue();
12488 
12489   SDValue LHS;
12490   uint32_t ShiftLHS = 0;
12491   bool LHSFromHi = false;
12492   if (!findEXTRHalf(N->getOperand(0), LHS, ShiftLHS, LHSFromHi))
12493     return SDValue();
12494 
12495   SDValue RHS;
12496   uint32_t ShiftRHS = 0;
12497   bool RHSFromHi = false;
12498   if (!findEXTRHalf(N->getOperand(1), RHS, ShiftRHS, RHSFromHi))
12499     return SDValue();
12500 
12501   // If they're both trying to come from the high part of the register, they're
12502   // not really an EXTR.
12503   if (LHSFromHi == RHSFromHi)
12504     return SDValue();
12505 
12506   if (ShiftLHS + ShiftRHS != VT.getSizeInBits())
12507     return SDValue();
12508 
12509   if (LHSFromHi) {
12510     std::swap(LHS, RHS);
12511     std::swap(ShiftLHS, ShiftRHS);
12512   }
12513 
12514   return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS,
12515                      DAG.getConstant(ShiftRHS, DL, MVT::i64));
12516 }
12517 
12518 static SDValue tryCombineToBSL(SDNode *N,
12519                                 TargetLowering::DAGCombinerInfo &DCI) {
12520   EVT VT = N->getValueType(0);
12521   SelectionDAG &DAG = DCI.DAG;
12522   SDLoc DL(N);
12523 
12524   if (!VT.isVector())
12525     return SDValue();
12526 
12527   SDValue N0 = N->getOperand(0);
12528   if (N0.getOpcode() != ISD::AND)
12529     return SDValue();
12530 
12531   SDValue N1 = N->getOperand(1);
12532   if (N1.getOpcode() != ISD::AND)
12533     return SDValue();
12534 
12535   // We only have to look for constant vectors here since the general, variable
12536   // case can be handled in TableGen.
12537   unsigned Bits = VT.getScalarSizeInBits();
12538   uint64_t BitMask = Bits == 64 ? -1ULL : ((1ULL << Bits) - 1);
12539   for (int i = 1; i >= 0; --i)
12540     for (int j = 1; j >= 0; --j) {
12541       BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(i));
12542       BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(j));
12543       if (!BVN0 || !BVN1)
12544         continue;
12545 
12546       bool FoundMatch = true;
12547       for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) {
12548         ConstantSDNode *CN0 = dyn_cast<ConstantSDNode>(BVN0->getOperand(k));
12549         ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(BVN1->getOperand(k));
12550         if (!CN0 || !CN1 ||
12551             CN0->getZExtValue() != (BitMask & ~CN1->getZExtValue())) {
12552           FoundMatch = false;
12553           break;
12554         }
12555       }
12556 
12557       if (FoundMatch)
12558         return DAG.getNode(AArch64ISD::BSP, DL, VT, SDValue(BVN0, 0),
12559                            N0->getOperand(1 - i), N1->getOperand(1 - j));
12560     }
12561 
12562   return SDValue();
12563 }
12564 
12565 static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
12566                                 const AArch64Subtarget *Subtarget) {
12567   // Attempt to form an EXTR from (or (shl VAL1, #N), (srl VAL2, #RegWidth-N))
12568   SelectionDAG &DAG = DCI.DAG;
12569   EVT VT = N->getValueType(0);
12570 
12571   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
12572     return SDValue();
12573 
12574   if (SDValue Res = tryCombineToEXTR(N, DCI))
12575     return Res;
12576 
12577   if (SDValue Res = tryCombineToBSL(N, DCI))
12578     return Res;
12579 
12580   return SDValue();
12581 }
12582 
12583 static bool isConstantSplatVectorMaskForType(SDNode *N, EVT MemVT) {
12584   if (!MemVT.getVectorElementType().isSimple())
12585     return false;
12586 
12587   uint64_t MaskForTy = 0ull;
12588   switch (MemVT.getVectorElementType().getSimpleVT().SimpleTy) {
12589   case MVT::i8:
12590     MaskForTy = 0xffull;
12591     break;
12592   case MVT::i16:
12593     MaskForTy = 0xffffull;
12594     break;
12595   case MVT::i32:
12596     MaskForTy = 0xffffffffull;
12597     break;
12598   default:
12599     return false;
12600     break;
12601   }
12602 
12603   if (N->getOpcode() == AArch64ISD::DUP || N->getOpcode() == ISD::SPLAT_VECTOR)
12604     if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0)))
12605       return Op0->getAPIntValue().getLimitedValue() == MaskForTy;
12606 
12607   return false;
12608 }
12609 
12610 static SDValue performSVEAndCombine(SDNode *N,
12611                                     TargetLowering::DAGCombinerInfo &DCI) {
12612   if (DCI.isBeforeLegalizeOps())
12613     return SDValue();
12614 
12615   SelectionDAG &DAG = DCI.DAG;
12616   SDValue Src = N->getOperand(0);
12617   unsigned Opc = Src->getOpcode();
12618 
12619   // Zero/any extend of an unsigned unpack
12620   if (Opc == AArch64ISD::UUNPKHI || Opc == AArch64ISD::UUNPKLO) {
12621     SDValue UnpkOp = Src->getOperand(0);
12622     SDValue Dup = N->getOperand(1);
12623 
12624     if (Dup.getOpcode() != AArch64ISD::DUP)
12625       return SDValue();
12626 
12627     SDLoc DL(N);
12628     ConstantSDNode *C = dyn_cast<ConstantSDNode>(Dup->getOperand(0));
12629     uint64_t ExtVal = C->getZExtValue();
12630 
12631     // If the mask is fully covered by the unpack, we don't need to push
12632     // a new AND onto the operand
12633     EVT EltTy = UnpkOp->getValueType(0).getVectorElementType();
12634     if ((ExtVal == 0xFF && EltTy == MVT::i8) ||
12635         (ExtVal == 0xFFFF && EltTy == MVT::i16) ||
12636         (ExtVal == 0xFFFFFFFF && EltTy == MVT::i32))
12637       return Src;
12638 
12639     // Truncate to prevent a DUP with an over wide constant
12640     APInt Mask = C->getAPIntValue().trunc(EltTy.getSizeInBits());
12641 
12642     // Otherwise, make sure we propagate the AND to the operand
12643     // of the unpack
12644     Dup = DAG.getNode(AArch64ISD::DUP, DL,
12645                       UnpkOp->getValueType(0),
12646                       DAG.getConstant(Mask.zextOrTrunc(32), DL, MVT::i32));
12647 
12648     SDValue And = DAG.getNode(ISD::AND, DL,
12649                               UnpkOp->getValueType(0), UnpkOp, Dup);
12650 
12651     return DAG.getNode(Opc, DL, N->getValueType(0), And);
12652   }
12653 
12654   if (!EnableCombineMGatherIntrinsics)
12655     return SDValue();
12656 
12657   SDValue Mask = N->getOperand(1);
12658 
12659   if (!Src.hasOneUse())
12660     return SDValue();
12661 
12662   EVT MemVT;
12663 
12664   // SVE load instructions perform an implicit zero-extend, which makes them
12665   // perfect candidates for combining.
12666   switch (Opc) {
12667   case AArch64ISD::LD1_MERGE_ZERO:
12668   case AArch64ISD::LDNF1_MERGE_ZERO:
12669   case AArch64ISD::LDFF1_MERGE_ZERO:
12670     MemVT = cast<VTSDNode>(Src->getOperand(3))->getVT();
12671     break;
12672   case AArch64ISD::GLD1_MERGE_ZERO:
12673   case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
12674   case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
12675   case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
12676   case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
12677   case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
12678   case AArch64ISD::GLD1_IMM_MERGE_ZERO:
12679   case AArch64ISD::GLDFF1_MERGE_ZERO:
12680   case AArch64ISD::GLDFF1_SCALED_MERGE_ZERO:
12681   case AArch64ISD::GLDFF1_SXTW_MERGE_ZERO:
12682   case AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO:
12683   case AArch64ISD::GLDFF1_UXTW_MERGE_ZERO:
12684   case AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO:
12685   case AArch64ISD::GLDFF1_IMM_MERGE_ZERO:
12686   case AArch64ISD::GLDNT1_MERGE_ZERO:
12687     MemVT = cast<VTSDNode>(Src->getOperand(4))->getVT();
12688     break;
12689   default:
12690     return SDValue();
12691   }
12692 
12693   if (isConstantSplatVectorMaskForType(Mask.getNode(), MemVT))
12694     return Src;
12695 
12696   return SDValue();
12697 }
12698 
12699 static SDValue performANDCombine(SDNode *N,
12700                                  TargetLowering::DAGCombinerInfo &DCI) {
12701   SelectionDAG &DAG = DCI.DAG;
12702   SDValue LHS = N->getOperand(0);
12703   EVT VT = N->getValueType(0);
12704   if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
12705     return SDValue();
12706 
12707   if (VT.isScalableVector())
12708     return performSVEAndCombine(N, DCI);
12709 
12710   // The combining code below works only for NEON vectors. In particular, it
12711   // does not work for SVE when dealing with vectors wider than 128 bits.
12712   if (!(VT.is64BitVector() || VT.is128BitVector()))
12713     return SDValue();
12714 
12715   BuildVectorSDNode *BVN =
12716       dyn_cast<BuildVectorSDNode>(N->getOperand(1).getNode());
12717   if (!BVN)
12718     return SDValue();
12719 
12720   // AND does not accept an immediate, so check if we can use a BIC immediate
12721   // instruction instead. We do this here instead of using a (and x, (mvni imm))
12722   // pattern in isel, because some immediates may be lowered to the preferred
12723   // (and x, (movi imm)) form, even though an mvni representation also exists.
12724   APInt DefBits(VT.getSizeInBits(), 0);
12725   APInt UndefBits(VT.getSizeInBits(), 0);
12726   if (resolveBuildVector(BVN, DefBits, UndefBits)) {
12727     SDValue NewOp;
12728 
12729     DefBits = ~DefBits;
12730     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
12731                                     DefBits, &LHS)) ||
12732         (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
12733                                     DefBits, &LHS)))
12734       return NewOp;
12735 
12736     UndefBits = ~UndefBits;
12737     if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
12738                                     UndefBits, &LHS)) ||
12739         (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
12740                                     UndefBits, &LHS)))
12741       return NewOp;
12742   }
12743 
12744   return SDValue();
12745 }
12746 
12747 static SDValue performSRLCombine(SDNode *N,
12748                                  TargetLowering::DAGCombinerInfo &DCI) {
12749   SelectionDAG &DAG = DCI.DAG;
12750   EVT VT = N->getValueType(0);
12751   if (VT != MVT::i32 && VT != MVT::i64)
12752     return SDValue();
12753 
12754   // Canonicalize (srl (bswap i32 x), 16) to (rotr (bswap i32 x), 16), if the
12755   // high 16-bits of x are zero. Similarly, canonicalize (srl (bswap i64 x), 32)
12756   // to (rotr (bswap i64 x), 32), if the high 32-bits of x are zero.
12757   SDValue N0 = N->getOperand(0);
12758   if (N0.getOpcode() == ISD::BSWAP) {
12759     SDLoc DL(N);
12760     SDValue N1 = N->getOperand(1);
12761     SDValue N00 = N0.getOperand(0);
12762     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
12763       uint64_t ShiftAmt = C->getZExtValue();
12764       if (VT == MVT::i32 && ShiftAmt == 16 &&
12765           DAG.MaskedValueIsZero(N00, APInt::getHighBitsSet(32, 16)))
12766         return DAG.getNode(ISD::ROTR, DL, VT, N0, N1);
12767       if (VT == MVT::i64 && ShiftAmt == 32 &&
12768           DAG.MaskedValueIsZero(N00, APInt::getHighBitsSet(64, 32)))
12769         return DAG.getNode(ISD::ROTR, DL, VT, N0, N1);
12770     }
12771   }
12772   return SDValue();
12773 }
12774 
12775 // Attempt to form urhadd(OpA, OpB) from
12776 // truncate(vlshr(sub(zext(OpB), xor(zext(OpA), Ones(ElemSizeInBits))), 1))
12777 // or uhadd(OpA, OpB) from truncate(vlshr(add(zext(OpA), zext(OpB)), 1)).
12778 // The original form of the first expression is
12779 // truncate(srl(add(zext(OpB), add(zext(OpA), 1)), 1)) and the
12780 // (OpA + OpB + 1) subexpression will have been changed to (OpB - (~OpA)).
12781 // Before this function is called the srl will have been lowered to
12782 // AArch64ISD::VLSHR.
12783 // This pass can also recognize signed variants of the patterns that use sign
12784 // extension instead of zero extension and form a srhadd(OpA, OpB) or a
12785 // shadd(OpA, OpB) from them.
12786 static SDValue
12787 performVectorTruncateCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
12788                              SelectionDAG &DAG) {
12789   EVT VT = N->getValueType(0);
12790 
12791   // Since we are looking for a right shift by a constant value of 1 and we are
12792   // operating on types at least 16 bits in length (sign/zero extended OpA and
12793   // OpB, which are at least 8 bits), it follows that the truncate will always
12794   // discard the shifted-in bit and therefore the right shift will be logical
12795   // regardless of the signedness of OpA and OpB.
12796   SDValue Shift = N->getOperand(0);
12797   if (Shift.getOpcode() != AArch64ISD::VLSHR)
12798     return SDValue();
12799 
12800   // Is the right shift using an immediate value of 1?
12801   uint64_t ShiftAmount = Shift.getConstantOperandVal(1);
12802   if (ShiftAmount != 1)
12803     return SDValue();
12804 
12805   SDValue ExtendOpA, ExtendOpB;
12806   SDValue ShiftOp0 = Shift.getOperand(0);
12807   unsigned ShiftOp0Opc = ShiftOp0.getOpcode();
12808   if (ShiftOp0Opc == ISD::SUB) {
12809 
12810     SDValue Xor = ShiftOp0.getOperand(1);
12811     if (Xor.getOpcode() != ISD::XOR)
12812       return SDValue();
12813 
12814     // Is the XOR using a constant amount of all ones in the right hand side?
12815     uint64_t C;
12816     if (!isAllConstantBuildVector(Xor.getOperand(1), C))
12817       return SDValue();
12818 
12819     unsigned ElemSizeInBits = VT.getScalarSizeInBits();
12820     APInt CAsAPInt(ElemSizeInBits, C);
12821     if (CAsAPInt != APInt::getAllOnesValue(ElemSizeInBits))
12822       return SDValue();
12823 
12824     ExtendOpA = Xor.getOperand(0);
12825     ExtendOpB = ShiftOp0.getOperand(0);
12826   } else if (ShiftOp0Opc == ISD::ADD) {
12827     ExtendOpA = ShiftOp0.getOperand(0);
12828     ExtendOpB = ShiftOp0.getOperand(1);
12829   } else
12830     return SDValue();
12831 
12832   unsigned ExtendOpAOpc = ExtendOpA.getOpcode();
12833   unsigned ExtendOpBOpc = ExtendOpB.getOpcode();
12834   if (!(ExtendOpAOpc == ExtendOpBOpc &&
12835         (ExtendOpAOpc == ISD::ZERO_EXTEND || ExtendOpAOpc == ISD::SIGN_EXTEND)))
12836     return SDValue();
12837 
12838   // Is the result of the right shift being truncated to the same value type as
12839   // the original operands, OpA and OpB?
12840   SDValue OpA = ExtendOpA.getOperand(0);
12841   SDValue OpB = ExtendOpB.getOperand(0);
12842   EVT OpAVT = OpA.getValueType();
12843   assert(ExtendOpA.getValueType() == ExtendOpB.getValueType());
12844   if (!(VT == OpAVT && OpAVT == OpB.getValueType()))
12845     return SDValue();
12846 
12847   SDLoc DL(N);
12848   bool IsSignExtend = ExtendOpAOpc == ISD::SIGN_EXTEND;
12849   bool IsRHADD = ShiftOp0Opc == ISD::SUB;
12850   unsigned HADDOpc = IsSignExtend
12851                          ? (IsRHADD ? AArch64ISD::SRHADD : AArch64ISD::SHADD)
12852                          : (IsRHADD ? AArch64ISD::URHADD : AArch64ISD::UHADD);
12853   SDValue ResultHADD = DAG.getNode(HADDOpc, DL, VT, OpA, OpB);
12854 
12855   return ResultHADD;
12856 }
12857 
12858 static bool hasPairwiseAdd(unsigned Opcode, EVT VT, bool FullFP16) {
12859   switch (Opcode) {
12860   case ISD::FADD:
12861     return (FullFP16 && VT == MVT::f16) || VT == MVT::f32 || VT == MVT::f64;
12862   case ISD::ADD:
12863     return VT == MVT::i64;
12864   default:
12865     return false;
12866   }
12867 }
12868 
12869 static SDValue performExtractVectorEltCombine(SDNode *N, SelectionDAG &DAG) {
12870   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
12871   ConstantSDNode *ConstantN1 = dyn_cast<ConstantSDNode>(N1);
12872 
12873   EVT VT = N->getValueType(0);
12874   const bool FullFP16 =
12875       static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
12876 
12877   // Rewrite for pairwise fadd pattern
12878   //   (f32 (extract_vector_elt
12879   //           (fadd (vXf32 Other)
12880   //                 (vector_shuffle (vXf32 Other) undef <1,X,...> )) 0))
12881   // ->
12882   //   (f32 (fadd (extract_vector_elt (vXf32 Other) 0)
12883   //              (extract_vector_elt (vXf32 Other) 1))
12884   if (ConstantN1 && ConstantN1->getZExtValue() == 0 &&
12885       hasPairwiseAdd(N0->getOpcode(), VT, FullFP16)) {
12886     SDLoc DL(N0);
12887     SDValue N00 = N0->getOperand(0);
12888     SDValue N01 = N0->getOperand(1);
12889 
12890     ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(N01);
12891     SDValue Other = N00;
12892 
12893     // And handle the commutative case.
12894     if (!Shuffle) {
12895       Shuffle = dyn_cast<ShuffleVectorSDNode>(N00);
12896       Other = N01;
12897     }
12898 
12899     if (Shuffle && Shuffle->getMaskElt(0) == 1 &&
12900         Other == Shuffle->getOperand(0)) {
12901       return DAG.getNode(N0->getOpcode(), DL, VT,
12902                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other,
12903                                      DAG.getConstant(0, DL, MVT::i64)),
12904                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other,
12905                                      DAG.getConstant(1, DL, MVT::i64)));
12906     }
12907   }
12908 
12909   return SDValue();
12910 }
12911 
12912 static SDValue performConcatVectorsCombine(SDNode *N,
12913                                            TargetLowering::DAGCombinerInfo &DCI,
12914                                            SelectionDAG &DAG) {
12915   SDLoc dl(N);
12916   EVT VT = N->getValueType(0);
12917   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
12918   unsigned N0Opc = N0->getOpcode(), N1Opc = N1->getOpcode();
12919 
12920   // Optimize concat_vectors of truncated vectors, where the intermediate
12921   // type is illegal, to avoid said illegality,  e.g.,
12922   //   (v4i16 (concat_vectors (v2i16 (truncate (v2i64))),
12923   //                          (v2i16 (truncate (v2i64)))))
12924   // ->
12925   //   (v4i16 (truncate (vector_shuffle (v4i32 (bitcast (v2i64))),
12926   //                                    (v4i32 (bitcast (v2i64))),
12927   //                                    <0, 2, 4, 6>)))
12928   // This isn't really target-specific, but ISD::TRUNCATE legality isn't keyed
12929   // on both input and result type, so we might generate worse code.
12930   // On AArch64 we know it's fine for v2i64->v4i16 and v4i32->v8i8.
12931   if (N->getNumOperands() == 2 && N0Opc == ISD::TRUNCATE &&
12932       N1Opc == ISD::TRUNCATE) {
12933     SDValue N00 = N0->getOperand(0);
12934     SDValue N10 = N1->getOperand(0);
12935     EVT N00VT = N00.getValueType();
12936 
12937     if (N00VT == N10.getValueType() &&
12938         (N00VT == MVT::v2i64 || N00VT == MVT::v4i32) &&
12939         N00VT.getScalarSizeInBits() == 4 * VT.getScalarSizeInBits()) {
12940       MVT MidVT = (N00VT == MVT::v2i64 ? MVT::v4i32 : MVT::v8i16);
12941       SmallVector<int, 8> Mask(MidVT.getVectorNumElements());
12942       for (size_t i = 0; i < Mask.size(); ++i)
12943         Mask[i] = i * 2;
12944       return DAG.getNode(ISD::TRUNCATE, dl, VT,
12945                          DAG.getVectorShuffle(
12946                              MidVT, dl,
12947                              DAG.getNode(ISD::BITCAST, dl, MidVT, N00),
12948                              DAG.getNode(ISD::BITCAST, dl, MidVT, N10), Mask));
12949     }
12950   }
12951 
12952   // Wait 'til after everything is legalized to try this. That way we have
12953   // legal vector types and such.
12954   if (DCI.isBeforeLegalizeOps())
12955     return SDValue();
12956 
12957   // Optimise concat_vectors of two [us]rhadds or [us]hadds that use extracted
12958   // subvectors from the same original vectors. Combine these into a single
12959   // [us]rhadd or [us]hadd that operates on the two original vectors. Example:
12960   //  (v16i8 (concat_vectors (v8i8 (urhadd (extract_subvector (v16i8 OpA, <0>),
12961   //                                        extract_subvector (v16i8 OpB,
12962   //                                        <0>))),
12963   //                         (v8i8 (urhadd (extract_subvector (v16i8 OpA, <8>),
12964   //                                        extract_subvector (v16i8 OpB,
12965   //                                        <8>)))))
12966   // ->
12967   //  (v16i8(urhadd(v16i8 OpA, v16i8 OpB)))
12968   if (N->getNumOperands() == 2 && N0Opc == N1Opc &&
12969       (N0Opc == AArch64ISD::URHADD || N0Opc == AArch64ISD::SRHADD ||
12970        N0Opc == AArch64ISD::UHADD || N0Opc == AArch64ISD::SHADD)) {
12971     SDValue N00 = N0->getOperand(0);
12972     SDValue N01 = N0->getOperand(1);
12973     SDValue N10 = N1->getOperand(0);
12974     SDValue N11 = N1->getOperand(1);
12975 
12976     EVT N00VT = N00.getValueType();
12977     EVT N10VT = N10.getValueType();
12978 
12979     if (N00->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
12980         N01->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
12981         N10->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
12982         N11->getOpcode() == ISD::EXTRACT_SUBVECTOR && N00VT == N10VT) {
12983       SDValue N00Source = N00->getOperand(0);
12984       SDValue N01Source = N01->getOperand(0);
12985       SDValue N10Source = N10->getOperand(0);
12986       SDValue N11Source = N11->getOperand(0);
12987 
12988       if (N00Source == N10Source && N01Source == N11Source &&
12989           N00Source.getValueType() == VT && N01Source.getValueType() == VT) {
12990         assert(N0.getValueType() == N1.getValueType());
12991 
12992         uint64_t N00Index = N00.getConstantOperandVal(1);
12993         uint64_t N01Index = N01.getConstantOperandVal(1);
12994         uint64_t N10Index = N10.getConstantOperandVal(1);
12995         uint64_t N11Index = N11.getConstantOperandVal(1);
12996 
12997         if (N00Index == N01Index && N10Index == N11Index && N00Index == 0 &&
12998             N10Index == N00VT.getVectorNumElements())
12999           return DAG.getNode(N0Opc, dl, VT, N00Source, N01Source);
13000       }
13001     }
13002   }
13003 
13004   // If we see a (concat_vectors (v1x64 A), (v1x64 A)) it's really a vector
13005   // splat. The indexed instructions are going to be expecting a DUPLANE64, so
13006   // canonicalise to that.
13007   if (N0 == N1 && VT.getVectorNumElements() == 2) {
13008     assert(VT.getScalarSizeInBits() == 64);
13009     return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG),
13010                        DAG.getConstant(0, dl, MVT::i64));
13011   }
13012 
13013   // Canonicalise concat_vectors so that the right-hand vector has as few
13014   // bit-casts as possible before its real operation. The primary matching
13015   // destination for these operations will be the narrowing "2" instructions,
13016   // which depend on the operation being performed on this right-hand vector.
13017   // For example,
13018   //    (concat_vectors LHS,  (v1i64 (bitconvert (v4i16 RHS))))
13019   // becomes
13020   //    (bitconvert (concat_vectors (v4i16 (bitconvert LHS)), RHS))
13021 
13022   if (N1Opc != ISD::BITCAST)
13023     return SDValue();
13024   SDValue RHS = N1->getOperand(0);
13025   MVT RHSTy = RHS.getValueType().getSimpleVT();
13026   // If the RHS is not a vector, this is not the pattern we're looking for.
13027   if (!RHSTy.isVector())
13028     return SDValue();
13029 
13030   LLVM_DEBUG(
13031       dbgs() << "aarch64-lower: concat_vectors bitcast simplification\n");
13032 
13033   MVT ConcatTy = MVT::getVectorVT(RHSTy.getVectorElementType(),
13034                                   RHSTy.getVectorNumElements() * 2);
13035   return DAG.getNode(ISD::BITCAST, dl, VT,
13036                      DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatTy,
13037                                  DAG.getNode(ISD::BITCAST, dl, RHSTy, N0),
13038                                  RHS));
13039 }
13040 
13041 static SDValue tryCombineFixedPointConvert(SDNode *N,
13042                                            TargetLowering::DAGCombinerInfo &DCI,
13043                                            SelectionDAG &DAG) {
13044   // Wait until after everything is legalized to try this. That way we have
13045   // legal vector types and such.
13046   if (DCI.isBeforeLegalizeOps())
13047     return SDValue();
13048   // Transform a scalar conversion of a value from a lane extract into a
13049   // lane extract of a vector conversion. E.g., from foo1 to foo2:
13050   // double foo1(int64x2_t a) { return vcvtd_n_f64_s64(a[1], 9); }
13051   // double foo2(int64x2_t a) { return vcvtq_n_f64_s64(a, 9)[1]; }
13052   //
13053   // The second form interacts better with instruction selection and the
13054   // register allocator to avoid cross-class register copies that aren't
13055   // coalescable due to a lane reference.
13056 
13057   // Check the operand and see if it originates from a lane extract.
13058   SDValue Op1 = N->getOperand(1);
13059   if (Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
13060     // Yep, no additional predication needed. Perform the transform.
13061     SDValue IID = N->getOperand(0);
13062     SDValue Shift = N->getOperand(2);
13063     SDValue Vec = Op1.getOperand(0);
13064     SDValue Lane = Op1.getOperand(1);
13065     EVT ResTy = N->getValueType(0);
13066     EVT VecResTy;
13067     SDLoc DL(N);
13068 
13069     // The vector width should be 128 bits by the time we get here, even
13070     // if it started as 64 bits (the extract_vector handling will have
13071     // done so).
13072     assert(Vec.getValueSizeInBits() == 128 &&
13073            "unexpected vector size on extract_vector_elt!");
13074     if (Vec.getValueType() == MVT::v4i32)
13075       VecResTy = MVT::v4f32;
13076     else if (Vec.getValueType() == MVT::v2i64)
13077       VecResTy = MVT::v2f64;
13078     else
13079       llvm_unreachable("unexpected vector type!");
13080 
13081     SDValue Convert =
13082         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VecResTy, IID, Vec, Shift);
13083     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResTy, Convert, Lane);
13084   }
13085   return SDValue();
13086 }
13087 
13088 // AArch64 high-vector "long" operations are formed by performing the non-high
13089 // version on an extract_subvector of each operand which gets the high half:
13090 //
13091 //  (longop2 LHS, RHS) == (longop (extract_high LHS), (extract_high RHS))
13092 //
13093 // However, there are cases which don't have an extract_high explicitly, but
13094 // have another operation that can be made compatible with one for free. For
13095 // example:
13096 //
13097 //  (dupv64 scalar) --> (extract_high (dup128 scalar))
13098 //
13099 // This routine does the actual conversion of such DUPs, once outer routines
13100 // have determined that everything else is in order.
13101 // It also supports immediate DUP-like nodes (MOVI/MVNi), which we can fold
13102 // similarly here.
13103 static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
13104   switch (N.getOpcode()) {
13105   case AArch64ISD::DUP:
13106   case AArch64ISD::DUPLANE8:
13107   case AArch64ISD::DUPLANE16:
13108   case AArch64ISD::DUPLANE32:
13109   case AArch64ISD::DUPLANE64:
13110   case AArch64ISD::MOVI:
13111   case AArch64ISD::MOVIshift:
13112   case AArch64ISD::MOVIedit:
13113   case AArch64ISD::MOVImsl:
13114   case AArch64ISD::MVNIshift:
13115   case AArch64ISD::MVNImsl:
13116     break;
13117   default:
13118     // FMOV could be supported, but isn't very useful, as it would only occur
13119     // if you passed a bitcast' floating point immediate to an eligible long
13120     // integer op (addl, smull, ...).
13121     return SDValue();
13122   }
13123 
13124   MVT NarrowTy = N.getSimpleValueType();
13125   if (!NarrowTy.is64BitVector())
13126     return SDValue();
13127 
13128   MVT ElementTy = NarrowTy.getVectorElementType();
13129   unsigned NumElems = NarrowTy.getVectorNumElements();
13130   MVT NewVT = MVT::getVectorVT(ElementTy, NumElems * 2);
13131 
13132   SDLoc dl(N);
13133   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NarrowTy,
13134                      DAG.getNode(N->getOpcode(), dl, NewVT, N->ops()),
13135                      DAG.getConstant(NumElems, dl, MVT::i64));
13136 }
13137 
13138 static bool isEssentiallyExtractHighSubvector(SDValue N) {
13139   if (N.getOpcode() == ISD::BITCAST)
13140     N = N.getOperand(0);
13141   if (N.getOpcode() != ISD::EXTRACT_SUBVECTOR)
13142     return false;
13143   return cast<ConstantSDNode>(N.getOperand(1))->getAPIntValue() ==
13144          N.getOperand(0).getValueType().getVectorNumElements() / 2;
13145 }
13146 
13147 /// Helper structure to keep track of ISD::SET_CC operands.
13148 struct GenericSetCCInfo {
13149   const SDValue *Opnd0;
13150   const SDValue *Opnd1;
13151   ISD::CondCode CC;
13152 };
13153 
13154 /// Helper structure to keep track of a SET_CC lowered into AArch64 code.
13155 struct AArch64SetCCInfo {
13156   const SDValue *Cmp;
13157   AArch64CC::CondCode CC;
13158 };
13159 
13160 /// Helper structure to keep track of SetCC information.
13161 union SetCCInfo {
13162   GenericSetCCInfo Generic;
13163   AArch64SetCCInfo AArch64;
13164 };
13165 
13166 /// Helper structure to be able to read SetCC information.  If set to
13167 /// true, IsAArch64 field, Info is a AArch64SetCCInfo, otherwise Info is a
13168 /// GenericSetCCInfo.
13169 struct SetCCInfoAndKind {
13170   SetCCInfo Info;
13171   bool IsAArch64;
13172 };
13173 
13174 /// Check whether or not \p Op is a SET_CC operation, either a generic or
13175 /// an
13176 /// AArch64 lowered one.
13177 /// \p SetCCInfo is filled accordingly.
13178 /// \post SetCCInfo is meanginfull only when this function returns true.
13179 /// \return True when Op is a kind of SET_CC operation.
13180 static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo) {
13181   // If this is a setcc, this is straight forward.
13182   if (Op.getOpcode() == ISD::SETCC) {
13183     SetCCInfo.Info.Generic.Opnd0 = &Op.getOperand(0);
13184     SetCCInfo.Info.Generic.Opnd1 = &Op.getOperand(1);
13185     SetCCInfo.Info.Generic.CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
13186     SetCCInfo.IsAArch64 = false;
13187     return true;
13188   }
13189   // Otherwise, check if this is a matching csel instruction.
13190   // In other words:
13191   // - csel 1, 0, cc
13192   // - csel 0, 1, !cc
13193   if (Op.getOpcode() != AArch64ISD::CSEL)
13194     return false;
13195   // Set the information about the operands.
13196   // TODO: we want the operands of the Cmp not the csel
13197   SetCCInfo.Info.AArch64.Cmp = &Op.getOperand(3);
13198   SetCCInfo.IsAArch64 = true;
13199   SetCCInfo.Info.AArch64.CC = static_cast<AArch64CC::CondCode>(
13200       cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
13201 
13202   // Check that the operands matches the constraints:
13203   // (1) Both operands must be constants.
13204   // (2) One must be 1 and the other must be 0.
13205   ConstantSDNode *TValue = dyn_cast<ConstantSDNode>(Op.getOperand(0));
13206   ConstantSDNode *FValue = dyn_cast<ConstantSDNode>(Op.getOperand(1));
13207 
13208   // Check (1).
13209   if (!TValue || !FValue)
13210     return false;
13211 
13212   // Check (2).
13213   if (!TValue->isOne()) {
13214     // Update the comparison when we are interested in !cc.
13215     std::swap(TValue, FValue);
13216     SetCCInfo.Info.AArch64.CC =
13217         AArch64CC::getInvertedCondCode(SetCCInfo.Info.AArch64.CC);
13218   }
13219   return TValue->isOne() && FValue->isNullValue();
13220 }
13221 
13222 // Returns true if Op is setcc or zext of setcc.
13223 static bool isSetCCOrZExtSetCC(const SDValue& Op, SetCCInfoAndKind &Info) {
13224   if (isSetCC(Op, Info))
13225     return true;
13226   return ((Op.getOpcode() == ISD::ZERO_EXTEND) &&
13227     isSetCC(Op->getOperand(0), Info));
13228 }
13229 
13230 // The folding we want to perform is:
13231 // (add x, [zext] (setcc cc ...) )
13232 //   -->
13233 // (csel x, (add x, 1), !cc ...)
13234 //
13235 // The latter will get matched to a CSINC instruction.
13236 static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
13237   assert(Op && Op->getOpcode() == ISD::ADD && "Unexpected operation!");
13238   SDValue LHS = Op->getOperand(0);
13239   SDValue RHS = Op->getOperand(1);
13240   SetCCInfoAndKind InfoAndKind;
13241 
13242   // If both operands are a SET_CC, then we don't want to perform this
13243   // folding and create another csel as this results in more instructions
13244   // (and higher register usage).
13245   if (isSetCCOrZExtSetCC(LHS, InfoAndKind) &&
13246       isSetCCOrZExtSetCC(RHS, InfoAndKind))
13247     return SDValue();
13248 
13249   // If neither operand is a SET_CC, give up.
13250   if (!isSetCCOrZExtSetCC(LHS, InfoAndKind)) {
13251     std::swap(LHS, RHS);
13252     if (!isSetCCOrZExtSetCC(LHS, InfoAndKind))
13253       return SDValue();
13254   }
13255 
13256   // FIXME: This could be generatized to work for FP comparisons.
13257   EVT CmpVT = InfoAndKind.IsAArch64
13258                   ? InfoAndKind.Info.AArch64.Cmp->getOperand(0).getValueType()
13259                   : InfoAndKind.Info.Generic.Opnd0->getValueType();
13260   if (CmpVT != MVT::i32 && CmpVT != MVT::i64)
13261     return SDValue();
13262 
13263   SDValue CCVal;
13264   SDValue Cmp;
13265   SDLoc dl(Op);
13266   if (InfoAndKind.IsAArch64) {
13267     CCVal = DAG.getConstant(
13268         AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), dl,
13269         MVT::i32);
13270     Cmp = *InfoAndKind.Info.AArch64.Cmp;
13271   } else
13272     Cmp = getAArch64Cmp(
13273         *InfoAndKind.Info.Generic.Opnd0, *InfoAndKind.Info.Generic.Opnd1,
13274         ISD::getSetCCInverse(InfoAndKind.Info.Generic.CC, CmpVT), CCVal, DAG,
13275         dl);
13276 
13277   EVT VT = Op->getValueType(0);
13278   LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT));
13279   return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp);
13280 }
13281 
13282 // ADD(UADDV a, UADDV b) -->  UADDV(ADD a, b)
13283 static SDValue performUADDVCombine(SDNode *N, SelectionDAG &DAG) {
13284   EVT VT = N->getValueType(0);
13285   // Only scalar integer and vector types.
13286   if (N->getOpcode() != ISD::ADD || !VT.isScalarInteger())
13287     return SDValue();
13288 
13289   SDValue LHS = N->getOperand(0);
13290   SDValue RHS = N->getOperand(1);
13291   if (LHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
13292       RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT || LHS.getValueType() != VT)
13293     return SDValue();
13294 
13295   auto *LHSN1 = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
13296   auto *RHSN1 = dyn_cast<ConstantSDNode>(RHS->getOperand(1));
13297   if (!LHSN1 || LHSN1 != RHSN1 || !RHSN1->isNullValue())
13298     return SDValue();
13299 
13300   SDValue Op1 = LHS->getOperand(0);
13301   SDValue Op2 = RHS->getOperand(0);
13302   EVT OpVT1 = Op1.getValueType();
13303   EVT OpVT2 = Op2.getValueType();
13304   if (Op1.getOpcode() != AArch64ISD::UADDV || OpVT1 != OpVT2 ||
13305       Op2.getOpcode() != AArch64ISD::UADDV ||
13306       OpVT1.getVectorElementType() != VT)
13307     return SDValue();
13308 
13309   SDValue Val1 = Op1.getOperand(0);
13310   SDValue Val2 = Op2.getOperand(0);
13311   EVT ValVT = Val1->getValueType(0);
13312   SDLoc DL(N);
13313   SDValue AddVal = DAG.getNode(ISD::ADD, DL, ValVT, Val1, Val2);
13314   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
13315                      DAG.getNode(AArch64ISD::UADDV, DL, ValVT, AddVal),
13316                      DAG.getConstant(0, DL, MVT::i64));
13317 }
13318 
13319 // ADD(UDOT(zero, x, y), A) -->  UDOT(A, x, y)
13320 static SDValue performAddDotCombine(SDNode *N, SelectionDAG &DAG) {
13321   EVT VT = N->getValueType(0);
13322   if (N->getOpcode() != ISD::ADD)
13323     return SDValue();
13324 
13325   SDValue Dot = N->getOperand(0);
13326   SDValue A = N->getOperand(1);
13327   // Handle commutivity
13328   auto isZeroDot = [](SDValue Dot) {
13329     return (Dot.getOpcode() == AArch64ISD::UDOT ||
13330             Dot.getOpcode() == AArch64ISD::SDOT) &&
13331            ISD::isBuildVectorAllZeros(Dot.getOperand(0).getNode());
13332   };
13333   if (!isZeroDot(Dot))
13334     std::swap(Dot, A);
13335   if (!isZeroDot(Dot))
13336     return SDValue();
13337 
13338   return DAG.getNode(Dot.getOpcode(), SDLoc(N), VT, A, Dot.getOperand(1),
13339                      Dot.getOperand(2));
13340 }
13341 
13342 // The basic add/sub long vector instructions have variants with "2" on the end
13343 // which act on the high-half of their inputs. They are normally matched by
13344 // patterns like:
13345 //
13346 // (add (zeroext (extract_high LHS)),
13347 //      (zeroext (extract_high RHS)))
13348 // -> uaddl2 vD, vN, vM
13349 //
13350 // However, if one of the extracts is something like a duplicate, this
13351 // instruction can still be used profitably. This function puts the DAG into a
13352 // more appropriate form for those patterns to trigger.
13353 static SDValue performAddSubLongCombine(SDNode *N,
13354                                         TargetLowering::DAGCombinerInfo &DCI,
13355                                         SelectionDAG &DAG) {
13356   if (DCI.isBeforeLegalizeOps())
13357     return SDValue();
13358 
13359   MVT VT = N->getSimpleValueType(0);
13360   if (!VT.is128BitVector()) {
13361     if (N->getOpcode() == ISD::ADD)
13362       return performSetccAddFolding(N, DAG);
13363     return SDValue();
13364   }
13365 
13366   // Make sure both branches are extended in the same way.
13367   SDValue LHS = N->getOperand(0);
13368   SDValue RHS = N->getOperand(1);
13369   if ((LHS.getOpcode() != ISD::ZERO_EXTEND &&
13370        LHS.getOpcode() != ISD::SIGN_EXTEND) ||
13371       LHS.getOpcode() != RHS.getOpcode())
13372     return SDValue();
13373 
13374   unsigned ExtType = LHS.getOpcode();
13375 
13376   // It's not worth doing if at least one of the inputs isn't already an
13377   // extract, but we don't know which it'll be so we have to try both.
13378   if (isEssentiallyExtractHighSubvector(LHS.getOperand(0))) {
13379     RHS = tryExtendDUPToExtractHigh(RHS.getOperand(0), DAG);
13380     if (!RHS.getNode())
13381       return SDValue();
13382 
13383     RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS);
13384   } else if (isEssentiallyExtractHighSubvector(RHS.getOperand(0))) {
13385     LHS = tryExtendDUPToExtractHigh(LHS.getOperand(0), DAG);
13386     if (!LHS.getNode())
13387       return SDValue();
13388 
13389     LHS = DAG.getNode(ExtType, SDLoc(N), VT, LHS);
13390   }
13391 
13392   return DAG.getNode(N->getOpcode(), SDLoc(N), VT, LHS, RHS);
13393 }
13394 
13395 static SDValue performAddSubCombine(SDNode *N,
13396                                     TargetLowering::DAGCombinerInfo &DCI,
13397                                     SelectionDAG &DAG) {
13398   // Try to change sum of two reductions.
13399   if (SDValue Val = performUADDVCombine(N, DAG))
13400     return Val;
13401   if (SDValue Val = performAddDotCombine(N, DAG))
13402     return Val;
13403 
13404   return performAddSubLongCombine(N, DCI, DAG);
13405 }
13406 
13407 // Massage DAGs which we can use the high-half "long" operations on into
13408 // something isel will recognize better. E.g.
13409 //
13410 // (aarch64_neon_umull (extract_high vec) (dupv64 scalar)) -->
13411 //   (aarch64_neon_umull (extract_high (v2i64 vec)))
13412 //                     (extract_high (v2i64 (dup128 scalar)))))
13413 //
13414 static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N,
13415                                        TargetLowering::DAGCombinerInfo &DCI,
13416                                        SelectionDAG &DAG) {
13417   if (DCI.isBeforeLegalizeOps())
13418     return SDValue();
13419 
13420   SDValue LHS = N->getOperand((IID == Intrinsic::not_intrinsic) ? 0 : 1);
13421   SDValue RHS = N->getOperand((IID == Intrinsic::not_intrinsic) ? 1 : 2);
13422   assert(LHS.getValueType().is64BitVector() &&
13423          RHS.getValueType().is64BitVector() &&
13424          "unexpected shape for long operation");
13425 
13426   // Either node could be a DUP, but it's not worth doing both of them (you'd
13427   // just as well use the non-high version) so look for a corresponding extract
13428   // operation on the other "wing".
13429   if (isEssentiallyExtractHighSubvector(LHS)) {
13430     RHS = tryExtendDUPToExtractHigh(RHS, DAG);
13431     if (!RHS.getNode())
13432       return SDValue();
13433   } else if (isEssentiallyExtractHighSubvector(RHS)) {
13434     LHS = tryExtendDUPToExtractHigh(LHS, DAG);
13435     if (!LHS.getNode())
13436       return SDValue();
13437   }
13438 
13439   if (IID == Intrinsic::not_intrinsic)
13440     return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), LHS, RHS);
13441 
13442   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getValueType(0),
13443                      N->getOperand(0), LHS, RHS);
13444 }
13445 
13446 static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
13447   MVT ElemTy = N->getSimpleValueType(0).getScalarType();
13448   unsigned ElemBits = ElemTy.getSizeInBits();
13449 
13450   int64_t ShiftAmount;
13451   if (BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(2))) {
13452     APInt SplatValue, SplatUndef;
13453     unsigned SplatBitSize;
13454     bool HasAnyUndefs;
13455     if (!BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
13456                               HasAnyUndefs, ElemBits) ||
13457         SplatBitSize != ElemBits)
13458       return SDValue();
13459 
13460     ShiftAmount = SplatValue.getSExtValue();
13461   } else if (ConstantSDNode *CVN = dyn_cast<ConstantSDNode>(N->getOperand(2))) {
13462     ShiftAmount = CVN->getSExtValue();
13463   } else
13464     return SDValue();
13465 
13466   unsigned Opcode;
13467   bool IsRightShift;
13468   switch (IID) {
13469   default:
13470     llvm_unreachable("Unknown shift intrinsic");
13471   case Intrinsic::aarch64_neon_sqshl:
13472     Opcode = AArch64ISD::SQSHL_I;
13473     IsRightShift = false;
13474     break;
13475   case Intrinsic::aarch64_neon_uqshl:
13476     Opcode = AArch64ISD::UQSHL_I;
13477     IsRightShift = false;
13478     break;
13479   case Intrinsic::aarch64_neon_srshl:
13480     Opcode = AArch64ISD::SRSHR_I;
13481     IsRightShift = true;
13482     break;
13483   case Intrinsic::aarch64_neon_urshl:
13484     Opcode = AArch64ISD::URSHR_I;
13485     IsRightShift = true;
13486     break;
13487   case Intrinsic::aarch64_neon_sqshlu:
13488     Opcode = AArch64ISD::SQSHLU_I;
13489     IsRightShift = false;
13490     break;
13491   case Intrinsic::aarch64_neon_sshl:
13492   case Intrinsic::aarch64_neon_ushl:
13493     // For positive shift amounts we can use SHL, as ushl/sshl perform a regular
13494     // left shift for positive shift amounts. Below, we only replace the current
13495     // node with VSHL, if this condition is met.
13496     Opcode = AArch64ISD::VSHL;
13497     IsRightShift = false;
13498     break;
13499   }
13500 
13501   if (IsRightShift && ShiftAmount <= -1 && ShiftAmount >= -(int)ElemBits) {
13502     SDLoc dl(N);
13503     return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1),
13504                        DAG.getConstant(-ShiftAmount, dl, MVT::i32));
13505   } else if (!IsRightShift && ShiftAmount >= 0 && ShiftAmount < ElemBits) {
13506     SDLoc dl(N);
13507     return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1),
13508                        DAG.getConstant(ShiftAmount, dl, MVT::i32));
13509   }
13510 
13511   return SDValue();
13512 }
13513 
13514 // The CRC32[BH] instructions ignore the high bits of their data operand. Since
13515 // the intrinsics must be legal and take an i32, this means there's almost
13516 // certainly going to be a zext in the DAG which we can eliminate.
13517 static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG) {
13518   SDValue AndN = N->getOperand(2);
13519   if (AndN.getOpcode() != ISD::AND)
13520     return SDValue();
13521 
13522   ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(AndN.getOperand(1));
13523   if (!CMask || CMask->getZExtValue() != Mask)
13524     return SDValue();
13525 
13526   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), MVT::i32,
13527                      N->getOperand(0), N->getOperand(1), AndN.getOperand(0));
13528 }
13529 
13530 static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N,
13531                                            SelectionDAG &DAG) {
13532   SDLoc dl(N);
13533   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0),
13534                      DAG.getNode(Opc, dl,
13535                                  N->getOperand(1).getSimpleValueType(),
13536                                  N->getOperand(1)),
13537                      DAG.getConstant(0, dl, MVT::i64));
13538 }
13539 
13540 static SDValue LowerSVEIntrinsicIndex(SDNode *N, SelectionDAG &DAG) {
13541   SDLoc DL(N);
13542   SDValue Op1 = N->getOperand(1);
13543   SDValue Op2 = N->getOperand(2);
13544   EVT ScalarTy = Op1.getValueType();
13545 
13546   if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16)) {
13547     Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
13548     Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
13549   }
13550 
13551   return DAG.getNode(AArch64ISD::INDEX_VECTOR, DL, N->getValueType(0),
13552                      Op1, Op2);
13553 }
13554 
13555 static SDValue LowerSVEIntrinsicDUP(SDNode *N, SelectionDAG &DAG) {
13556   SDLoc dl(N);
13557   SDValue Scalar = N->getOperand(3);
13558   EVT ScalarTy = Scalar.getValueType();
13559 
13560   if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16))
13561     Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar);
13562 
13563   SDValue Passthru = N->getOperand(1);
13564   SDValue Pred = N->getOperand(2);
13565   return DAG.getNode(AArch64ISD::DUP_MERGE_PASSTHRU, dl, N->getValueType(0),
13566                      Pred, Scalar, Passthru);
13567 }
13568 
13569 static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG) {
13570   SDLoc dl(N);
13571   LLVMContext &Ctx = *DAG.getContext();
13572   EVT VT = N->getValueType(0);
13573 
13574   assert(VT.isScalableVector() && "Expected a scalable vector.");
13575 
13576   // Current lowering only supports the SVE-ACLE types.
13577   if (VT.getSizeInBits().getKnownMinSize() != AArch64::SVEBitsPerBlock)
13578     return SDValue();
13579 
13580   unsigned ElemSize = VT.getVectorElementType().getSizeInBits() / 8;
13581   unsigned ByteSize = VT.getSizeInBits().getKnownMinSize() / 8;
13582   EVT ByteVT =
13583       EVT::getVectorVT(Ctx, MVT::i8, ElementCount::getScalable(ByteSize));
13584 
13585   // Convert everything to the domain of EXT (i.e bytes).
13586   SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1));
13587   SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(2));
13588   SDValue Op2 = DAG.getNode(ISD::MUL, dl, MVT::i32, N->getOperand(3),
13589                             DAG.getConstant(ElemSize, dl, MVT::i32));
13590 
13591   SDValue EXT = DAG.getNode(AArch64ISD::EXT, dl, ByteVT, Op0, Op1, Op2);
13592   return DAG.getNode(ISD::BITCAST, dl, VT, EXT);
13593 }
13594 
13595 static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC,
13596                                         TargetLowering::DAGCombinerInfo &DCI,
13597                                         SelectionDAG &DAG) {
13598   if (DCI.isBeforeLegalize())
13599     return SDValue();
13600 
13601   SDValue Comparator = N->getOperand(3);
13602   if (Comparator.getOpcode() == AArch64ISD::DUP ||
13603       Comparator.getOpcode() == ISD::SPLAT_VECTOR) {
13604     unsigned IID = getIntrinsicID(N);
13605     EVT VT = N->getValueType(0);
13606     EVT CmpVT = N->getOperand(2).getValueType();
13607     SDValue Pred = N->getOperand(1);
13608     SDValue Imm;
13609     SDLoc DL(N);
13610 
13611     switch (IID) {
13612     default:
13613       llvm_unreachable("Called with wrong intrinsic!");
13614       break;
13615 
13616     // Signed comparisons
13617     case Intrinsic::aarch64_sve_cmpeq_wide:
13618     case Intrinsic::aarch64_sve_cmpne_wide:
13619     case Intrinsic::aarch64_sve_cmpge_wide:
13620     case Intrinsic::aarch64_sve_cmpgt_wide:
13621     case Intrinsic::aarch64_sve_cmplt_wide:
13622     case Intrinsic::aarch64_sve_cmple_wide: {
13623       if (auto *CN = dyn_cast<ConstantSDNode>(Comparator.getOperand(0))) {
13624         int64_t ImmVal = CN->getSExtValue();
13625         if (ImmVal >= -16 && ImmVal <= 15)
13626           Imm = DAG.getConstant(ImmVal, DL, MVT::i32);
13627         else
13628           return SDValue();
13629       }
13630       break;
13631     }
13632     // Unsigned comparisons
13633     case Intrinsic::aarch64_sve_cmphs_wide:
13634     case Intrinsic::aarch64_sve_cmphi_wide:
13635     case Intrinsic::aarch64_sve_cmplo_wide:
13636     case Intrinsic::aarch64_sve_cmpls_wide:  {
13637       if (auto *CN = dyn_cast<ConstantSDNode>(Comparator.getOperand(0))) {
13638         uint64_t ImmVal = CN->getZExtValue();
13639         if (ImmVal <= 127)
13640           Imm = DAG.getConstant(ImmVal, DL, MVT::i32);
13641         else
13642           return SDValue();
13643       }
13644       break;
13645     }
13646     }
13647 
13648     if (!Imm)
13649       return SDValue();
13650 
13651     SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, CmpVT, Imm);
13652     return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, VT, Pred,
13653                        N->getOperand(2), Splat, DAG.getCondCode(CC));
13654   }
13655 
13656   return SDValue();
13657 }
13658 
13659 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
13660                         AArch64CC::CondCode Cond) {
13661   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13662 
13663   SDLoc DL(Op);
13664   assert(Op.getValueType().isScalableVector() &&
13665          TLI.isTypeLegal(Op.getValueType()) &&
13666          "Expected legal scalable vector type!");
13667 
13668   // Ensure target specific opcodes are using legal type.
13669   EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
13670   SDValue TVal = DAG.getConstant(1, DL, OutVT);
13671   SDValue FVal = DAG.getConstant(0, DL, OutVT);
13672 
13673   // Set condition code (CC) flags.
13674   SDValue Test = DAG.getNode(AArch64ISD::PTEST, DL, MVT::Other, Pg, Op);
13675 
13676   // Convert CC to integer based on requested condition.
13677   // NOTE: Cond is inverted to promote CSEL's removal when it feeds a compare.
13678   SDValue CC = DAG.getConstant(getInvertedCondCode(Cond), DL, MVT::i32);
13679   SDValue Res = DAG.getNode(AArch64ISD::CSEL, DL, OutVT, FVal, TVal, CC, Test);
13680   return DAG.getZExtOrTrunc(Res, DL, VT);
13681 }
13682 
13683 static SDValue combineSVEReductionInt(SDNode *N, unsigned Opc,
13684                                       SelectionDAG &DAG) {
13685   SDLoc DL(N);
13686 
13687   SDValue Pred = N->getOperand(1);
13688   SDValue VecToReduce = N->getOperand(2);
13689 
13690   // NOTE: The integer reduction's result type is not always linked to the
13691   // operand's element type so we construct it from the intrinsic's result type.
13692   EVT ReduceVT = getPackedSVEVectorVT(N->getValueType(0));
13693   SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce);
13694 
13695   // SVE reductions set the whole vector register with the first element
13696   // containing the reduction result, which we'll now extract.
13697   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
13698   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce,
13699                      Zero);
13700 }
13701 
13702 static SDValue combineSVEReductionFP(SDNode *N, unsigned Opc,
13703                                      SelectionDAG &DAG) {
13704   SDLoc DL(N);
13705 
13706   SDValue Pred = N->getOperand(1);
13707   SDValue VecToReduce = N->getOperand(2);
13708 
13709   EVT ReduceVT = VecToReduce.getValueType();
13710   SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce);
13711 
13712   // SVE reductions set the whole vector register with the first element
13713   // containing the reduction result, which we'll now extract.
13714   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
13715   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce,
13716                      Zero);
13717 }
13718 
13719 static SDValue combineSVEReductionOrderedFP(SDNode *N, unsigned Opc,
13720                                             SelectionDAG &DAG) {
13721   SDLoc DL(N);
13722 
13723   SDValue Pred = N->getOperand(1);
13724   SDValue InitVal = N->getOperand(2);
13725   SDValue VecToReduce = N->getOperand(3);
13726   EVT ReduceVT = VecToReduce.getValueType();
13727 
13728   // Ordered reductions use the first lane of the result vector as the
13729   // reduction's initial value.
13730   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
13731   InitVal = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ReduceVT,
13732                         DAG.getUNDEF(ReduceVT), InitVal, Zero);
13733 
13734   SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, InitVal, VecToReduce);
13735 
13736   // SVE reductions set the whole vector register with the first element
13737   // containing the reduction result, which we'll now extract.
13738   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce,
13739                      Zero);
13740 }
13741 
13742 // If a merged operation has no inactive lanes we can relax it to a predicated
13743 // or unpredicated operation, which potentially allows better isel (perhaps
13744 // using immediate forms) or relaxing register reuse requirements.
13745 static SDValue convertMergedOpToPredOp(SDNode *N, unsigned PredOpc,
13746                                        SelectionDAG &DAG) {
13747   assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Expected intrinsic!");
13748   assert(N->getNumOperands() == 4 && "Expected 3 operand intrinsic!");
13749   SDValue Pg = N->getOperand(1);
13750 
13751   // ISD way to specify an all active predicate.
13752   if ((Pg.getOpcode() == AArch64ISD::PTRUE) &&
13753       (Pg.getConstantOperandVal(0) == AArch64SVEPredPattern::all))
13754     return DAG.getNode(PredOpc, SDLoc(N), N->getValueType(0), Pg,
13755                        N->getOperand(2), N->getOperand(3));
13756 
13757   // FUTURE: SplatVector(true)
13758   return SDValue();
13759 }
13760 
13761 static SDValue performIntrinsicCombine(SDNode *N,
13762                                        TargetLowering::DAGCombinerInfo &DCI,
13763                                        const AArch64Subtarget *Subtarget) {
13764   SelectionDAG &DAG = DCI.DAG;
13765   unsigned IID = getIntrinsicID(N);
13766   switch (IID) {
13767   default:
13768     break;
13769   case Intrinsic::aarch64_neon_vcvtfxs2fp:
13770   case Intrinsic::aarch64_neon_vcvtfxu2fp:
13771     return tryCombineFixedPointConvert(N, DCI, DAG);
13772   case Intrinsic::aarch64_neon_saddv:
13773     return combineAcrossLanesIntrinsic(AArch64ISD::SADDV, N, DAG);
13774   case Intrinsic::aarch64_neon_uaddv:
13775     return combineAcrossLanesIntrinsic(AArch64ISD::UADDV, N, DAG);
13776   case Intrinsic::aarch64_neon_sminv:
13777     return combineAcrossLanesIntrinsic(AArch64ISD::SMINV, N, DAG);
13778   case Intrinsic::aarch64_neon_uminv:
13779     return combineAcrossLanesIntrinsic(AArch64ISD::UMINV, N, DAG);
13780   case Intrinsic::aarch64_neon_smaxv:
13781     return combineAcrossLanesIntrinsic(AArch64ISD::SMAXV, N, DAG);
13782   case Intrinsic::aarch64_neon_umaxv:
13783     return combineAcrossLanesIntrinsic(AArch64ISD::UMAXV, N, DAG);
13784   case Intrinsic::aarch64_neon_fmax:
13785     return DAG.getNode(ISD::FMAXIMUM, SDLoc(N), N->getValueType(0),
13786                        N->getOperand(1), N->getOperand(2));
13787   case Intrinsic::aarch64_neon_fmin:
13788     return DAG.getNode(ISD::FMINIMUM, SDLoc(N), N->getValueType(0),
13789                        N->getOperand(1), N->getOperand(2));
13790   case Intrinsic::aarch64_neon_fmaxnm:
13791     return DAG.getNode(ISD::FMAXNUM, SDLoc(N), N->getValueType(0),
13792                        N->getOperand(1), N->getOperand(2));
13793   case Intrinsic::aarch64_neon_fminnm:
13794     return DAG.getNode(ISD::FMINNUM, SDLoc(N), N->getValueType(0),
13795                        N->getOperand(1), N->getOperand(2));
13796   case Intrinsic::aarch64_neon_smull:
13797   case Intrinsic::aarch64_neon_umull:
13798   case Intrinsic::aarch64_neon_pmull:
13799   case Intrinsic::aarch64_neon_sqdmull:
13800     return tryCombineLongOpWithDup(IID, N, DCI, DAG);
13801   case Intrinsic::aarch64_neon_sqshl:
13802   case Intrinsic::aarch64_neon_uqshl:
13803   case Intrinsic::aarch64_neon_sqshlu:
13804   case Intrinsic::aarch64_neon_srshl:
13805   case Intrinsic::aarch64_neon_urshl:
13806   case Intrinsic::aarch64_neon_sshl:
13807   case Intrinsic::aarch64_neon_ushl:
13808     return tryCombineShiftImm(IID, N, DAG);
13809   case Intrinsic::aarch64_crc32b:
13810   case Intrinsic::aarch64_crc32cb:
13811     return tryCombineCRC32(0xff, N, DAG);
13812   case Intrinsic::aarch64_crc32h:
13813   case Intrinsic::aarch64_crc32ch:
13814     return tryCombineCRC32(0xffff, N, DAG);
13815   case Intrinsic::aarch64_sve_saddv:
13816     // There is no i64 version of SADDV because the sign is irrelevant.
13817     if (N->getOperand(2)->getValueType(0).getVectorElementType() == MVT::i64)
13818       return combineSVEReductionInt(N, AArch64ISD::UADDV_PRED, DAG);
13819     else
13820       return combineSVEReductionInt(N, AArch64ISD::SADDV_PRED, DAG);
13821   case Intrinsic::aarch64_sve_uaddv:
13822     return combineSVEReductionInt(N, AArch64ISD::UADDV_PRED, DAG);
13823   case Intrinsic::aarch64_sve_smaxv:
13824     return combineSVEReductionInt(N, AArch64ISD::SMAXV_PRED, DAG);
13825   case Intrinsic::aarch64_sve_umaxv:
13826     return combineSVEReductionInt(N, AArch64ISD::UMAXV_PRED, DAG);
13827   case Intrinsic::aarch64_sve_sminv:
13828     return combineSVEReductionInt(N, AArch64ISD::SMINV_PRED, DAG);
13829   case Intrinsic::aarch64_sve_uminv:
13830     return combineSVEReductionInt(N, AArch64ISD::UMINV_PRED, DAG);
13831   case Intrinsic::aarch64_sve_orv:
13832     return combineSVEReductionInt(N, AArch64ISD::ORV_PRED, DAG);
13833   case Intrinsic::aarch64_sve_eorv:
13834     return combineSVEReductionInt(N, AArch64ISD::EORV_PRED, DAG);
13835   case Intrinsic::aarch64_sve_andv:
13836     return combineSVEReductionInt(N, AArch64ISD::ANDV_PRED, DAG);
13837   case Intrinsic::aarch64_sve_index:
13838     return LowerSVEIntrinsicIndex(N, DAG);
13839   case Intrinsic::aarch64_sve_dup:
13840     return LowerSVEIntrinsicDUP(N, DAG);
13841   case Intrinsic::aarch64_sve_dup_x:
13842     return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), N->getValueType(0),
13843                        N->getOperand(1));
13844   case Intrinsic::aarch64_sve_ext:
13845     return LowerSVEIntrinsicEXT(N, DAG);
13846   case Intrinsic::aarch64_sve_smin:
13847     return convertMergedOpToPredOp(N, AArch64ISD::SMIN_PRED, DAG);
13848   case Intrinsic::aarch64_sve_umin:
13849     return convertMergedOpToPredOp(N, AArch64ISD::UMIN_PRED, DAG);
13850   case Intrinsic::aarch64_sve_smax:
13851     return convertMergedOpToPredOp(N, AArch64ISD::SMAX_PRED, DAG);
13852   case Intrinsic::aarch64_sve_umax:
13853     return convertMergedOpToPredOp(N, AArch64ISD::UMAX_PRED, DAG);
13854   case Intrinsic::aarch64_sve_lsl:
13855     return convertMergedOpToPredOp(N, AArch64ISD::SHL_PRED, DAG);
13856   case Intrinsic::aarch64_sve_lsr:
13857     return convertMergedOpToPredOp(N, AArch64ISD::SRL_PRED, DAG);
13858   case Intrinsic::aarch64_sve_asr:
13859     return convertMergedOpToPredOp(N, AArch64ISD::SRA_PRED, DAG);
13860   case Intrinsic::aarch64_sve_cmphs:
13861     if (!N->getOperand(2).getValueType().isFloatingPoint())
13862       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
13863                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
13864                          N->getOperand(3), DAG.getCondCode(ISD::SETUGE));
13865     break;
13866   case Intrinsic::aarch64_sve_cmphi:
13867     if (!N->getOperand(2).getValueType().isFloatingPoint())
13868       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
13869                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
13870                          N->getOperand(3), DAG.getCondCode(ISD::SETUGT));
13871     break;
13872   case Intrinsic::aarch64_sve_cmpge:
13873     if (!N->getOperand(2).getValueType().isFloatingPoint())
13874       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
13875                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
13876                          N->getOperand(3), DAG.getCondCode(ISD::SETGE));
13877     break;
13878   case Intrinsic::aarch64_sve_cmpgt:
13879     if (!N->getOperand(2).getValueType().isFloatingPoint())
13880       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
13881                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
13882                          N->getOperand(3), DAG.getCondCode(ISD::SETGT));
13883     break;
13884   case Intrinsic::aarch64_sve_cmpeq:
13885     if (!N->getOperand(2).getValueType().isFloatingPoint())
13886       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
13887                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
13888                          N->getOperand(3), DAG.getCondCode(ISD::SETEQ));
13889     break;
13890   case Intrinsic::aarch64_sve_cmpne:
13891     if (!N->getOperand(2).getValueType().isFloatingPoint())
13892       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
13893                          N->getValueType(0), N->getOperand(1), N->getOperand(2),
13894                          N->getOperand(3), DAG.getCondCode(ISD::SETNE));
13895     break;
13896   case Intrinsic::aarch64_sve_fadda:
13897     return combineSVEReductionOrderedFP(N, AArch64ISD::FADDA_PRED, DAG);
13898   case Intrinsic::aarch64_sve_faddv:
13899     return combineSVEReductionFP(N, AArch64ISD::FADDV_PRED, DAG);
13900   case Intrinsic::aarch64_sve_fmaxnmv:
13901     return combineSVEReductionFP(N, AArch64ISD::FMAXNMV_PRED, DAG);
13902   case Intrinsic::aarch64_sve_fmaxv:
13903     return combineSVEReductionFP(N, AArch64ISD::FMAXV_PRED, DAG);
13904   case Intrinsic::aarch64_sve_fminnmv:
13905     return combineSVEReductionFP(N, AArch64ISD::FMINNMV_PRED, DAG);
13906   case Intrinsic::aarch64_sve_fminv:
13907     return combineSVEReductionFP(N, AArch64ISD::FMINV_PRED, DAG);
13908   case Intrinsic::aarch64_sve_sel:
13909     return DAG.getNode(ISD::VSELECT, SDLoc(N), N->getValueType(0),
13910                        N->getOperand(1), N->getOperand(2), N->getOperand(3));
13911   case Intrinsic::aarch64_sve_cmpeq_wide:
13912     return tryConvertSVEWideCompare(N, ISD::SETEQ, DCI, DAG);
13913   case Intrinsic::aarch64_sve_cmpne_wide:
13914     return tryConvertSVEWideCompare(N, ISD::SETNE, DCI, DAG);
13915   case Intrinsic::aarch64_sve_cmpge_wide:
13916     return tryConvertSVEWideCompare(N, ISD::SETGE, DCI, DAG);
13917   case Intrinsic::aarch64_sve_cmpgt_wide:
13918     return tryConvertSVEWideCompare(N, ISD::SETGT, DCI, DAG);
13919   case Intrinsic::aarch64_sve_cmplt_wide:
13920     return tryConvertSVEWideCompare(N, ISD::SETLT, DCI, DAG);
13921   case Intrinsic::aarch64_sve_cmple_wide:
13922     return tryConvertSVEWideCompare(N, ISD::SETLE, DCI, DAG);
13923   case Intrinsic::aarch64_sve_cmphs_wide:
13924     return tryConvertSVEWideCompare(N, ISD::SETUGE, DCI, DAG);
13925   case Intrinsic::aarch64_sve_cmphi_wide:
13926     return tryConvertSVEWideCompare(N, ISD::SETUGT, DCI, DAG);
13927   case Intrinsic::aarch64_sve_cmplo_wide:
13928     return tryConvertSVEWideCompare(N, ISD::SETULT, DCI, DAG);
13929   case Intrinsic::aarch64_sve_cmpls_wide:
13930     return tryConvertSVEWideCompare(N, ISD::SETULE, DCI, DAG);
13931   case Intrinsic::aarch64_sve_ptest_any:
13932     return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
13933                     AArch64CC::ANY_ACTIVE);
13934   case Intrinsic::aarch64_sve_ptest_first:
13935     return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
13936                     AArch64CC::FIRST_ACTIVE);
13937   case Intrinsic::aarch64_sve_ptest_last:
13938     return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
13939                     AArch64CC::LAST_ACTIVE);
13940   }
13941   return SDValue();
13942 }
13943 
13944 static SDValue performExtendCombine(SDNode *N,
13945                                     TargetLowering::DAGCombinerInfo &DCI,
13946                                     SelectionDAG &DAG) {
13947   // If we see something like (zext (sabd (extract_high ...), (DUP ...))) then
13948   // we can convert that DUP into another extract_high (of a bigger DUP), which
13949   // helps the backend to decide that an sabdl2 would be useful, saving a real
13950   // extract_high operation.
13951   if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ZERO_EXTEND &&
13952       (N->getOperand(0).getOpcode() == AArch64ISD::UABD ||
13953        N->getOperand(0).getOpcode() == AArch64ISD::SABD)) {
13954     SDNode *ABDNode = N->getOperand(0).getNode();
13955     SDValue NewABD =
13956         tryCombineLongOpWithDup(Intrinsic::not_intrinsic, ABDNode, DCI, DAG);
13957     if (!NewABD.getNode())
13958       return SDValue();
13959 
13960     return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), NewABD);
13961   }
13962   return SDValue();
13963 }
13964 
13965 static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St,
13966                                SDValue SplatVal, unsigned NumVecElts) {
13967   assert(!St.isTruncatingStore() && "cannot split truncating vector store");
13968   unsigned OrigAlignment = St.getAlignment();
13969   unsigned EltOffset = SplatVal.getValueType().getSizeInBits() / 8;
13970 
13971   // Create scalar stores. This is at least as good as the code sequence for a
13972   // split unaligned store which is a dup.s, ext.b, and two stores.
13973   // Most of the time the three stores should be replaced by store pair
13974   // instructions (stp).
13975   SDLoc DL(&St);
13976   SDValue BasePtr = St.getBasePtr();
13977   uint64_t BaseOffset = 0;
13978 
13979   const MachinePointerInfo &PtrInfo = St.getPointerInfo();
13980   SDValue NewST1 =
13981       DAG.getStore(St.getChain(), DL, SplatVal, BasePtr, PtrInfo,
13982                    OrigAlignment, St.getMemOperand()->getFlags());
13983 
13984   // As this in ISel, we will not merge this add which may degrade results.
13985   if (BasePtr->getOpcode() == ISD::ADD &&
13986       isa<ConstantSDNode>(BasePtr->getOperand(1))) {
13987     BaseOffset = cast<ConstantSDNode>(BasePtr->getOperand(1))->getSExtValue();
13988     BasePtr = BasePtr->getOperand(0);
13989   }
13990 
13991   unsigned Offset = EltOffset;
13992   while (--NumVecElts) {
13993     unsigned Alignment = MinAlign(OrigAlignment, Offset);
13994     SDValue OffsetPtr =
13995         DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
13996                     DAG.getConstant(BaseOffset + Offset, DL, MVT::i64));
13997     NewST1 = DAG.getStore(NewST1.getValue(0), DL, SplatVal, OffsetPtr,
13998                           PtrInfo.getWithOffset(Offset), Alignment,
13999                           St.getMemOperand()->getFlags());
14000     Offset += EltOffset;
14001   }
14002   return NewST1;
14003 }
14004 
14005 // Returns an SVE type that ContentTy can be trivially sign or zero extended
14006 // into.
14007 static MVT getSVEContainerType(EVT ContentTy) {
14008   assert(ContentTy.isSimple() && "No SVE containers for extended types");
14009 
14010   switch (ContentTy.getSimpleVT().SimpleTy) {
14011   default:
14012     llvm_unreachable("No known SVE container for this MVT type");
14013   case MVT::nxv2i8:
14014   case MVT::nxv2i16:
14015   case MVT::nxv2i32:
14016   case MVT::nxv2i64:
14017   case MVT::nxv2f32:
14018   case MVT::nxv2f64:
14019     return MVT::nxv2i64;
14020   case MVT::nxv4i8:
14021   case MVT::nxv4i16:
14022   case MVT::nxv4i32:
14023   case MVT::nxv4f32:
14024     return MVT::nxv4i32;
14025   case MVT::nxv8i8:
14026   case MVT::nxv8i16:
14027   case MVT::nxv8f16:
14028   case MVT::nxv8bf16:
14029     return MVT::nxv8i16;
14030   case MVT::nxv16i8:
14031     return MVT::nxv16i8;
14032   }
14033 }
14034 
14035 static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc) {
14036   SDLoc DL(N);
14037   EVT VT = N->getValueType(0);
14038 
14039   if (VT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
14040     return SDValue();
14041 
14042   EVT ContainerVT = VT;
14043   if (ContainerVT.isInteger())
14044     ContainerVT = getSVEContainerType(ContainerVT);
14045 
14046   SDVTList VTs = DAG.getVTList(ContainerVT, MVT::Other);
14047   SDValue Ops[] = { N->getOperand(0), // Chain
14048                     N->getOperand(2), // Pg
14049                     N->getOperand(3), // Base
14050                     DAG.getValueType(VT) };
14051 
14052   SDValue Load = DAG.getNode(Opc, DL, VTs, Ops);
14053   SDValue LoadChain = SDValue(Load.getNode(), 1);
14054 
14055   if (ContainerVT.isInteger() && (VT != ContainerVT))
14056     Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0));
14057 
14058   return DAG.getMergeValues({ Load, LoadChain }, DL);
14059 }
14060 
14061 static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
14062   SDLoc DL(N);
14063   EVT VT = N->getValueType(0);
14064   EVT PtrTy = N->getOperand(3).getValueType();
14065 
14066   if (VT == MVT::nxv8bf16 &&
14067       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
14068     return SDValue();
14069 
14070   EVT LoadVT = VT;
14071   if (VT.isFloatingPoint())
14072     LoadVT = VT.changeTypeToInteger();
14073 
14074   auto *MINode = cast<MemIntrinsicSDNode>(N);
14075   SDValue PassThru = DAG.getConstant(0, DL, LoadVT);
14076   SDValue L = DAG.getMaskedLoad(LoadVT, DL, MINode->getChain(),
14077                                 MINode->getOperand(3), DAG.getUNDEF(PtrTy),
14078                                 MINode->getOperand(2), PassThru,
14079                                 MINode->getMemoryVT(), MINode->getMemOperand(),
14080                                 ISD::UNINDEXED, ISD::NON_EXTLOAD, false);
14081 
14082    if (VT.isFloatingPoint()) {
14083      SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) };
14084      return DAG.getMergeValues(Ops, DL);
14085    }
14086 
14087   return L;
14088 }
14089 
14090 template <unsigned Opcode>
14091 static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG) {
14092   static_assert(Opcode == AArch64ISD::LD1RQ_MERGE_ZERO ||
14093                     Opcode == AArch64ISD::LD1RO_MERGE_ZERO,
14094                 "Unsupported opcode.");
14095   SDLoc DL(N);
14096   EVT VT = N->getValueType(0);
14097   if (VT == MVT::nxv8bf16 &&
14098       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
14099     return SDValue();
14100 
14101   EVT LoadVT = VT;
14102   if (VT.isFloatingPoint())
14103     LoadVT = VT.changeTypeToInteger();
14104 
14105   SDValue Ops[] = {N->getOperand(0), N->getOperand(2), N->getOperand(3)};
14106   SDValue Load = DAG.getNode(Opcode, DL, {LoadVT, MVT::Other}, Ops);
14107   SDValue LoadChain = SDValue(Load.getNode(), 1);
14108 
14109   if (VT.isFloatingPoint())
14110     Load = DAG.getNode(ISD::BITCAST, DL, VT, Load.getValue(0));
14111 
14112   return DAG.getMergeValues({Load, LoadChain}, DL);
14113 }
14114 
14115 static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
14116   SDLoc DL(N);
14117   SDValue Data = N->getOperand(2);
14118   EVT DataVT = Data.getValueType();
14119   EVT HwSrcVt = getSVEContainerType(DataVT);
14120   SDValue InputVT = DAG.getValueType(DataVT);
14121 
14122   if (DataVT == MVT::nxv8bf16 &&
14123       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
14124     return SDValue();
14125 
14126   if (DataVT.isFloatingPoint())
14127     InputVT = DAG.getValueType(HwSrcVt);
14128 
14129   SDValue SrcNew;
14130   if (Data.getValueType().isFloatingPoint())
14131     SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Data);
14132   else
14133     SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Data);
14134 
14135   SDValue Ops[] = { N->getOperand(0), // Chain
14136                     SrcNew,
14137                     N->getOperand(4), // Base
14138                     N->getOperand(3), // Pg
14139                     InputVT
14140                   };
14141 
14142   return DAG.getNode(AArch64ISD::ST1_PRED, DL, N->getValueType(0), Ops);
14143 }
14144 
14145 static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG) {
14146   SDLoc DL(N);
14147 
14148   SDValue Data = N->getOperand(2);
14149   EVT DataVT = Data.getValueType();
14150   EVT PtrTy = N->getOperand(4).getValueType();
14151 
14152   if (DataVT == MVT::nxv8bf16 &&
14153       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
14154     return SDValue();
14155 
14156   if (DataVT.isFloatingPoint())
14157     Data = DAG.getNode(ISD::BITCAST, DL, DataVT.changeTypeToInteger(), Data);
14158 
14159   auto *MINode = cast<MemIntrinsicSDNode>(N);
14160   return DAG.getMaskedStore(MINode->getChain(), DL, Data, MINode->getOperand(4),
14161                             DAG.getUNDEF(PtrTy), MINode->getOperand(3),
14162                             MINode->getMemoryVT(), MINode->getMemOperand(),
14163                             ISD::UNINDEXED, false, false);
14164 }
14165 
14166 /// Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.  The
14167 /// load store optimizer pass will merge them to store pair stores.  This should
14168 /// be better than a movi to create the vector zero followed by a vector store
14169 /// if the zero constant is not re-used, since one instructions and one register
14170 /// live range will be removed.
14171 ///
14172 /// For example, the final generated code should be:
14173 ///
14174 ///   stp xzr, xzr, [x0]
14175 ///
14176 /// instead of:
14177 ///
14178 ///   movi v0.2d, #0
14179 ///   str q0, [x0]
14180 ///
14181 static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
14182   SDValue StVal = St.getValue();
14183   EVT VT = StVal.getValueType();
14184 
14185   // Avoid scalarizing zero splat stores for scalable vectors.
14186   if (VT.isScalableVector())
14187     return SDValue();
14188 
14189   // It is beneficial to scalarize a zero splat store for 2 or 3 i64 elements or
14190   // 2, 3 or 4 i32 elements.
14191   int NumVecElts = VT.getVectorNumElements();
14192   if (!(((NumVecElts == 2 || NumVecElts == 3) &&
14193          VT.getVectorElementType().getSizeInBits() == 64) ||
14194         ((NumVecElts == 2 || NumVecElts == 3 || NumVecElts == 4) &&
14195          VT.getVectorElementType().getSizeInBits() == 32)))
14196     return SDValue();
14197 
14198   if (StVal.getOpcode() != ISD::BUILD_VECTOR)
14199     return SDValue();
14200 
14201   // If the zero constant has more than one use then the vector store could be
14202   // better since the constant mov will be amortized and stp q instructions
14203   // should be able to be formed.
14204   if (!StVal.hasOneUse())
14205     return SDValue();
14206 
14207   // If the store is truncating then it's going down to i16 or smaller, which
14208   // means it can be implemented in a single store anyway.
14209   if (St.isTruncatingStore())
14210     return SDValue();
14211 
14212   // If the immediate offset of the address operand is too large for the stp
14213   // instruction, then bail out.
14214   if (DAG.isBaseWithConstantOffset(St.getBasePtr())) {
14215     int64_t Offset = St.getBasePtr()->getConstantOperandVal(1);
14216     if (Offset < -512 || Offset > 504)
14217       return SDValue();
14218   }
14219 
14220   for (int I = 0; I < NumVecElts; ++I) {
14221     SDValue EltVal = StVal.getOperand(I);
14222     if (!isNullConstant(EltVal) && !isNullFPConstant(EltVal))
14223       return SDValue();
14224   }
14225 
14226   // Use a CopyFromReg WZR/XZR here to prevent
14227   // DAGCombiner::MergeConsecutiveStores from undoing this transformation.
14228   SDLoc DL(&St);
14229   unsigned ZeroReg;
14230   EVT ZeroVT;
14231   if (VT.getVectorElementType().getSizeInBits() == 32) {
14232     ZeroReg = AArch64::WZR;
14233     ZeroVT = MVT::i32;
14234   } else {
14235     ZeroReg = AArch64::XZR;
14236     ZeroVT = MVT::i64;
14237   }
14238   SDValue SplatVal =
14239       DAG.getCopyFromReg(DAG.getEntryNode(), DL, ZeroReg, ZeroVT);
14240   return splitStoreSplat(DAG, St, SplatVal, NumVecElts);
14241 }
14242 
14243 /// Replace a splat of a scalar to a vector store by scalar stores of the scalar
14244 /// value. The load store optimizer pass will merge them to store pair stores.
14245 /// This has better performance than a splat of the scalar followed by a split
14246 /// vector store. Even if the stores are not merged it is four stores vs a dup,
14247 /// followed by an ext.b and two stores.
14248 static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
14249   SDValue StVal = St.getValue();
14250   EVT VT = StVal.getValueType();
14251 
14252   // Don't replace floating point stores, they possibly won't be transformed to
14253   // stp because of the store pair suppress pass.
14254   if (VT.isFloatingPoint())
14255     return SDValue();
14256 
14257   // We can express a splat as store pair(s) for 2 or 4 elements.
14258   unsigned NumVecElts = VT.getVectorNumElements();
14259   if (NumVecElts != 4 && NumVecElts != 2)
14260     return SDValue();
14261 
14262   // If the store is truncating then it's going down to i16 or smaller, which
14263   // means it can be implemented in a single store anyway.
14264   if (St.isTruncatingStore())
14265     return SDValue();
14266 
14267   // Check that this is a splat.
14268   // Make sure that each of the relevant vector element locations are inserted
14269   // to, i.e. 0 and 1 for v2i64 and 0, 1, 2, 3 for v4i32.
14270   std::bitset<4> IndexNotInserted((1 << NumVecElts) - 1);
14271   SDValue SplatVal;
14272   for (unsigned I = 0; I < NumVecElts; ++I) {
14273     // Check for insert vector elements.
14274     if (StVal.getOpcode() != ISD::INSERT_VECTOR_ELT)
14275       return SDValue();
14276 
14277     // Check that same value is inserted at each vector element.
14278     if (I == 0)
14279       SplatVal = StVal.getOperand(1);
14280     else if (StVal.getOperand(1) != SplatVal)
14281       return SDValue();
14282 
14283     // Check insert element index.
14284     ConstantSDNode *CIndex = dyn_cast<ConstantSDNode>(StVal.getOperand(2));
14285     if (!CIndex)
14286       return SDValue();
14287     uint64_t IndexVal = CIndex->getZExtValue();
14288     if (IndexVal >= NumVecElts)
14289       return SDValue();
14290     IndexNotInserted.reset(IndexVal);
14291 
14292     StVal = StVal.getOperand(0);
14293   }
14294   // Check that all vector element locations were inserted to.
14295   if (IndexNotInserted.any())
14296       return SDValue();
14297 
14298   return splitStoreSplat(DAG, St, SplatVal, NumVecElts);
14299 }
14300 
14301 static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
14302                            SelectionDAG &DAG,
14303                            const AArch64Subtarget *Subtarget) {
14304 
14305   StoreSDNode *S = cast<StoreSDNode>(N);
14306   if (S->isVolatile() || S->isIndexed())
14307     return SDValue();
14308 
14309   SDValue StVal = S->getValue();
14310   EVT VT = StVal.getValueType();
14311 
14312   if (!VT.isFixedLengthVector())
14313     return SDValue();
14314 
14315   // If we get a splat of zeros, convert this vector store to a store of
14316   // scalars. They will be merged into store pairs of xzr thereby removing one
14317   // instruction and one register.
14318   if (SDValue ReplacedZeroSplat = replaceZeroVectorStore(DAG, *S))
14319     return ReplacedZeroSplat;
14320 
14321   // FIXME: The logic for deciding if an unaligned store should be split should
14322   // be included in TLI.allowsMisalignedMemoryAccesses(), and there should be
14323   // a call to that function here.
14324 
14325   if (!Subtarget->isMisaligned128StoreSlow())
14326     return SDValue();
14327 
14328   // Don't split at -Oz.
14329   if (DAG.getMachineFunction().getFunction().hasMinSize())
14330     return SDValue();
14331 
14332   // Don't split v2i64 vectors. Memcpy lowering produces those and splitting
14333   // those up regresses performance on micro-benchmarks and olden/bh.
14334   if (VT.getVectorNumElements() < 2 || VT == MVT::v2i64)
14335     return SDValue();
14336 
14337   // Split unaligned 16B stores. They are terrible for performance.
14338   // Don't split stores with alignment of 1 or 2. Code that uses clang vector
14339   // extensions can use this to mark that it does not want splitting to happen
14340   // (by underspecifying alignment to be 1 or 2). Furthermore, the chance of
14341   // eliminating alignment hazards is only 1 in 8 for alignment of 2.
14342   if (VT.getSizeInBits() != 128 || S->getAlignment() >= 16 ||
14343       S->getAlignment() <= 2)
14344     return SDValue();
14345 
14346   // If we get a splat of a scalar convert this vector store to a store of
14347   // scalars. They will be merged into store pairs thereby removing two
14348   // instructions.
14349   if (SDValue ReplacedSplat = replaceSplatVectorStore(DAG, *S))
14350     return ReplacedSplat;
14351 
14352   SDLoc DL(S);
14353 
14354   // Split VT into two.
14355   EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
14356   unsigned NumElts = HalfVT.getVectorNumElements();
14357   SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
14358                                    DAG.getConstant(0, DL, MVT::i64));
14359   SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
14360                                    DAG.getConstant(NumElts, DL, MVT::i64));
14361   SDValue BasePtr = S->getBasePtr();
14362   SDValue NewST1 =
14363       DAG.getStore(S->getChain(), DL, SubVector0, BasePtr, S->getPointerInfo(),
14364                    S->getAlignment(), S->getMemOperand()->getFlags());
14365   SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
14366                                   DAG.getConstant(8, DL, MVT::i64));
14367   return DAG.getStore(NewST1.getValue(0), DL, SubVector1, OffsetPtr,
14368                       S->getPointerInfo(), S->getAlignment(),
14369                       S->getMemOperand()->getFlags());
14370 }
14371 
14372 static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG) {
14373   SDLoc DL(N);
14374   SDValue Op0 = N->getOperand(0);
14375   SDValue Op1 = N->getOperand(1);
14376   EVT ResVT = N->getValueType(0);
14377 
14378   // uzp1(unpklo(uzp1(x, y)), z) => uzp1(x, z)
14379   if (Op0.getOpcode() == AArch64ISD::UUNPKLO) {
14380     if (Op0.getOperand(0).getOpcode() == AArch64ISD::UZP1) {
14381       SDValue X = Op0.getOperand(0).getOperand(0);
14382       return DAG.getNode(AArch64ISD::UZP1, DL, ResVT, X, Op1);
14383     }
14384   }
14385 
14386   // uzp1(x, unpkhi(uzp1(y, z))) => uzp1(x, z)
14387   if (Op1.getOpcode() == AArch64ISD::UUNPKHI) {
14388     if (Op1.getOperand(0).getOpcode() == AArch64ISD::UZP1) {
14389       SDValue Z = Op1.getOperand(0).getOperand(1);
14390       return DAG.getNode(AArch64ISD::UZP1, DL, ResVT, Op0, Z);
14391     }
14392   }
14393 
14394   return SDValue();
14395 }
14396 
14397 static SDValue performGLD1Combine(SDNode *N, SelectionDAG &DAG) {
14398   unsigned Opc = N->getOpcode();
14399 
14400   assert(((Opc >= AArch64ISD::GLD1_MERGE_ZERO && // unsigned gather loads
14401            Opc <= AArch64ISD::GLD1_IMM_MERGE_ZERO) ||
14402           (Opc >= AArch64ISD::GLD1S_MERGE_ZERO && // signed gather loads
14403            Opc <= AArch64ISD::GLD1S_IMM_MERGE_ZERO)) &&
14404          "Invalid opcode.");
14405 
14406   const bool Scaled = Opc == AArch64ISD::GLD1_SCALED_MERGE_ZERO ||
14407                       Opc == AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
14408   const bool Signed = Opc == AArch64ISD::GLD1S_MERGE_ZERO ||
14409                       Opc == AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
14410   const bool Extended = Opc == AArch64ISD::GLD1_SXTW_MERGE_ZERO ||
14411                         Opc == AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO ||
14412                         Opc == AArch64ISD::GLD1_UXTW_MERGE_ZERO ||
14413                         Opc == AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO;
14414 
14415   SDLoc DL(N);
14416   SDValue Chain = N->getOperand(0);
14417   SDValue Pg = N->getOperand(1);
14418   SDValue Base = N->getOperand(2);
14419   SDValue Offset = N->getOperand(3);
14420   SDValue Ty = N->getOperand(4);
14421 
14422   EVT ResVT = N->getValueType(0);
14423 
14424   const auto OffsetOpc = Offset.getOpcode();
14425   const bool OffsetIsZExt =
14426       OffsetOpc == AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU;
14427   const bool OffsetIsSExt =
14428       OffsetOpc == AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU;
14429 
14430   // Fold sign/zero extensions of vector offsets into GLD1 nodes where possible.
14431   if (!Extended && (OffsetIsSExt || OffsetIsZExt)) {
14432     SDValue ExtPg = Offset.getOperand(0);
14433     VTSDNode *ExtFrom = cast<VTSDNode>(Offset.getOperand(2).getNode());
14434     EVT ExtFromEVT = ExtFrom->getVT().getVectorElementType();
14435 
14436     // If the predicate for the sign- or zero-extended offset is the
14437     // same as the predicate used for this load and the sign-/zero-extension
14438     // was from a 32-bits...
14439     if (ExtPg == Pg && ExtFromEVT == MVT::i32) {
14440       SDValue UnextendedOffset = Offset.getOperand(1);
14441 
14442       unsigned NewOpc = getGatherVecOpcode(Scaled, OffsetIsSExt, true);
14443       if (Signed)
14444         NewOpc = getSignExtendedGatherOpcode(NewOpc);
14445 
14446       return DAG.getNode(NewOpc, DL, {ResVT, MVT::Other},
14447                          {Chain, Pg, Base, UnextendedOffset, Ty});
14448     }
14449   }
14450 
14451   return SDValue();
14452 }
14453 
14454 /// Target-specific DAG combine function for post-increment LD1 (lane) and
14455 /// post-increment LD1R.
14456 static SDValue performPostLD1Combine(SDNode *N,
14457                                      TargetLowering::DAGCombinerInfo &DCI,
14458                                      bool IsLaneOp) {
14459   if (DCI.isBeforeLegalizeOps())
14460     return SDValue();
14461 
14462   SelectionDAG &DAG = DCI.DAG;
14463   EVT VT = N->getValueType(0);
14464 
14465   if (VT.isScalableVector())
14466     return SDValue();
14467 
14468   unsigned LoadIdx = IsLaneOp ? 1 : 0;
14469   SDNode *LD = N->getOperand(LoadIdx).getNode();
14470   // If it is not LOAD, can not do such combine.
14471   if (LD->getOpcode() != ISD::LOAD)
14472     return SDValue();
14473 
14474   // The vector lane must be a constant in the LD1LANE opcode.
14475   SDValue Lane;
14476   if (IsLaneOp) {
14477     Lane = N->getOperand(2);
14478     auto *LaneC = dyn_cast<ConstantSDNode>(Lane);
14479     if (!LaneC || LaneC->getZExtValue() >= VT.getVectorNumElements())
14480       return SDValue();
14481   }
14482 
14483   LoadSDNode *LoadSDN = cast<LoadSDNode>(LD);
14484   EVT MemVT = LoadSDN->getMemoryVT();
14485   // Check if memory operand is the same type as the vector element.
14486   if (MemVT != VT.getVectorElementType())
14487     return SDValue();
14488 
14489   // Check if there are other uses. If so, do not combine as it will introduce
14490   // an extra load.
14491   for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end(); UI != UE;
14492        ++UI) {
14493     if (UI.getUse().getResNo() == 1) // Ignore uses of the chain result.
14494       continue;
14495     if (*UI != N)
14496       return SDValue();
14497   }
14498 
14499   SDValue Addr = LD->getOperand(1);
14500   SDValue Vector = N->getOperand(0);
14501   // Search for a use of the address operand that is an increment.
14502   for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), UE =
14503        Addr.getNode()->use_end(); UI != UE; ++UI) {
14504     SDNode *User = *UI;
14505     if (User->getOpcode() != ISD::ADD
14506         || UI.getUse().getResNo() != Addr.getResNo())
14507       continue;
14508 
14509     // If the increment is a constant, it must match the memory ref size.
14510     SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
14511     if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) {
14512       uint32_t IncVal = CInc->getZExtValue();
14513       unsigned NumBytes = VT.getScalarSizeInBits() / 8;
14514       if (IncVal != NumBytes)
14515         continue;
14516       Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
14517     }
14518 
14519     // To avoid cycle construction make sure that neither the load nor the add
14520     // are predecessors to each other or the Vector.
14521     SmallPtrSet<const SDNode *, 32> Visited;
14522     SmallVector<const SDNode *, 16> Worklist;
14523     Visited.insert(Addr.getNode());
14524     Worklist.push_back(User);
14525     Worklist.push_back(LD);
14526     Worklist.push_back(Vector.getNode());
14527     if (SDNode::hasPredecessorHelper(LD, Visited, Worklist) ||
14528         SDNode::hasPredecessorHelper(User, Visited, Worklist))
14529       continue;
14530 
14531     SmallVector<SDValue, 8> Ops;
14532     Ops.push_back(LD->getOperand(0));  // Chain
14533     if (IsLaneOp) {
14534       Ops.push_back(Vector);           // The vector to be inserted
14535       Ops.push_back(Lane);             // The lane to be inserted in the vector
14536     }
14537     Ops.push_back(Addr);
14538     Ops.push_back(Inc);
14539 
14540     EVT Tys[3] = { VT, MVT::i64, MVT::Other };
14541     SDVTList SDTys = DAG.getVTList(Tys);
14542     unsigned NewOp = IsLaneOp ? AArch64ISD::LD1LANEpost : AArch64ISD::LD1DUPpost;
14543     SDValue UpdN = DAG.getMemIntrinsicNode(NewOp, SDLoc(N), SDTys, Ops,
14544                                            MemVT,
14545                                            LoadSDN->getMemOperand());
14546 
14547     // Update the uses.
14548     SDValue NewResults[] = {
14549         SDValue(LD, 0),            // The result of load
14550         SDValue(UpdN.getNode(), 2) // Chain
14551     };
14552     DCI.CombineTo(LD, NewResults);
14553     DCI.CombineTo(N, SDValue(UpdN.getNode(), 0));     // Dup/Inserted Result
14554     DCI.CombineTo(User, SDValue(UpdN.getNode(), 1));  // Write back register
14555 
14556     break;
14557   }
14558   return SDValue();
14559 }
14560 
14561 /// Simplify ``Addr`` given that the top byte of it is ignored by HW during
14562 /// address translation.
14563 static bool performTBISimplification(SDValue Addr,
14564                                      TargetLowering::DAGCombinerInfo &DCI,
14565                                      SelectionDAG &DAG) {
14566   APInt DemandedMask = APInt::getLowBitsSet(64, 56);
14567   KnownBits Known;
14568   TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
14569                                         !DCI.isBeforeLegalizeOps());
14570   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
14571   if (TLI.SimplifyDemandedBits(Addr, DemandedMask, Known, TLO)) {
14572     DCI.CommitTargetLoweringOpt(TLO);
14573     return true;
14574   }
14575   return false;
14576 }
14577 
14578 static SDValue performSTORECombine(SDNode *N,
14579                                    TargetLowering::DAGCombinerInfo &DCI,
14580                                    SelectionDAG &DAG,
14581                                    const AArch64Subtarget *Subtarget) {
14582   if (SDValue Split = splitStores(N, DCI, DAG, Subtarget))
14583     return Split;
14584 
14585   if (Subtarget->supportsAddressTopByteIgnored() &&
14586       performTBISimplification(N->getOperand(2), DCI, DAG))
14587     return SDValue(N, 0);
14588 
14589   return SDValue();
14590 }
14591 
14592 /// Target-specific DAG combine function for NEON load/store intrinsics
14593 /// to merge base address updates.
14594 static SDValue performNEONPostLDSTCombine(SDNode *N,
14595                                           TargetLowering::DAGCombinerInfo &DCI,
14596                                           SelectionDAG &DAG) {
14597   if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14598     return SDValue();
14599 
14600   unsigned AddrOpIdx = N->getNumOperands() - 1;
14601   SDValue Addr = N->getOperand(AddrOpIdx);
14602 
14603   // Search for a use of the address operand that is an increment.
14604   for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
14605        UE = Addr.getNode()->use_end(); UI != UE; ++UI) {
14606     SDNode *User = *UI;
14607     if (User->getOpcode() != ISD::ADD ||
14608         UI.getUse().getResNo() != Addr.getResNo())
14609       continue;
14610 
14611     // Check that the add is independent of the load/store.  Otherwise, folding
14612     // it would create a cycle.
14613     SmallPtrSet<const SDNode *, 32> Visited;
14614     SmallVector<const SDNode *, 16> Worklist;
14615     Visited.insert(Addr.getNode());
14616     Worklist.push_back(N);
14617     Worklist.push_back(User);
14618     if (SDNode::hasPredecessorHelper(N, Visited, Worklist) ||
14619         SDNode::hasPredecessorHelper(User, Visited, Worklist))
14620       continue;
14621 
14622     // Find the new opcode for the updating load/store.
14623     bool IsStore = false;
14624     bool IsLaneOp = false;
14625     bool IsDupOp = false;
14626     unsigned NewOpc = 0;
14627     unsigned NumVecs = 0;
14628     unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
14629     switch (IntNo) {
14630     default: llvm_unreachable("unexpected intrinsic for Neon base update");
14631     case Intrinsic::aarch64_neon_ld2:       NewOpc = AArch64ISD::LD2post;
14632       NumVecs = 2; break;
14633     case Intrinsic::aarch64_neon_ld3:       NewOpc = AArch64ISD::LD3post;
14634       NumVecs = 3; break;
14635     case Intrinsic::aarch64_neon_ld4:       NewOpc = AArch64ISD::LD4post;
14636       NumVecs = 4; break;
14637     case Intrinsic::aarch64_neon_st2:       NewOpc = AArch64ISD::ST2post;
14638       NumVecs = 2; IsStore = true; break;
14639     case Intrinsic::aarch64_neon_st3:       NewOpc = AArch64ISD::ST3post;
14640       NumVecs = 3; IsStore = true; break;
14641     case Intrinsic::aarch64_neon_st4:       NewOpc = AArch64ISD::ST4post;
14642       NumVecs = 4; IsStore = true; break;
14643     case Intrinsic::aarch64_neon_ld1x2:     NewOpc = AArch64ISD::LD1x2post;
14644       NumVecs = 2; break;
14645     case Intrinsic::aarch64_neon_ld1x3:     NewOpc = AArch64ISD::LD1x3post;
14646       NumVecs = 3; break;
14647     case Intrinsic::aarch64_neon_ld1x4:     NewOpc = AArch64ISD::LD1x4post;
14648       NumVecs = 4; break;
14649     case Intrinsic::aarch64_neon_st1x2:     NewOpc = AArch64ISD::ST1x2post;
14650       NumVecs = 2; IsStore = true; break;
14651     case Intrinsic::aarch64_neon_st1x3:     NewOpc = AArch64ISD::ST1x3post;
14652       NumVecs = 3; IsStore = true; break;
14653     case Intrinsic::aarch64_neon_st1x4:     NewOpc = AArch64ISD::ST1x4post;
14654       NumVecs = 4; IsStore = true; break;
14655     case Intrinsic::aarch64_neon_ld2r:      NewOpc = AArch64ISD::LD2DUPpost;
14656       NumVecs = 2; IsDupOp = true; break;
14657     case Intrinsic::aarch64_neon_ld3r:      NewOpc = AArch64ISD::LD3DUPpost;
14658       NumVecs = 3; IsDupOp = true; break;
14659     case Intrinsic::aarch64_neon_ld4r:      NewOpc = AArch64ISD::LD4DUPpost;
14660       NumVecs = 4; IsDupOp = true; break;
14661     case Intrinsic::aarch64_neon_ld2lane:   NewOpc = AArch64ISD::LD2LANEpost;
14662       NumVecs = 2; IsLaneOp = true; break;
14663     case Intrinsic::aarch64_neon_ld3lane:   NewOpc = AArch64ISD::LD3LANEpost;
14664       NumVecs = 3; IsLaneOp = true; break;
14665     case Intrinsic::aarch64_neon_ld4lane:   NewOpc = AArch64ISD::LD4LANEpost;
14666       NumVecs = 4; IsLaneOp = true; break;
14667     case Intrinsic::aarch64_neon_st2lane:   NewOpc = AArch64ISD::ST2LANEpost;
14668       NumVecs = 2; IsStore = true; IsLaneOp = true; break;
14669     case Intrinsic::aarch64_neon_st3lane:   NewOpc = AArch64ISD::ST3LANEpost;
14670       NumVecs = 3; IsStore = true; IsLaneOp = true; break;
14671     case Intrinsic::aarch64_neon_st4lane:   NewOpc = AArch64ISD::ST4LANEpost;
14672       NumVecs = 4; IsStore = true; IsLaneOp = true; break;
14673     }
14674 
14675     EVT VecTy;
14676     if (IsStore)
14677       VecTy = N->getOperand(2).getValueType();
14678     else
14679       VecTy = N->getValueType(0);
14680 
14681     // If the increment is a constant, it must match the memory ref size.
14682     SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
14683     if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) {
14684       uint32_t IncVal = CInc->getZExtValue();
14685       unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
14686       if (IsLaneOp || IsDupOp)
14687         NumBytes /= VecTy.getVectorNumElements();
14688       if (IncVal != NumBytes)
14689         continue;
14690       Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
14691     }
14692     SmallVector<SDValue, 8> Ops;
14693     Ops.push_back(N->getOperand(0)); // Incoming chain
14694     // Load lane and store have vector list as input.
14695     if (IsLaneOp || IsStore)
14696       for (unsigned i = 2; i < AddrOpIdx; ++i)
14697         Ops.push_back(N->getOperand(i));
14698     Ops.push_back(Addr); // Base register
14699     Ops.push_back(Inc);
14700 
14701     // Return Types.
14702     EVT Tys[6];
14703     unsigned NumResultVecs = (IsStore ? 0 : NumVecs);
14704     unsigned n;
14705     for (n = 0; n < NumResultVecs; ++n)
14706       Tys[n] = VecTy;
14707     Tys[n++] = MVT::i64;  // Type of write back register
14708     Tys[n] = MVT::Other;  // Type of the chain
14709     SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumResultVecs + 2));
14710 
14711     MemIntrinsicSDNode *MemInt = cast<MemIntrinsicSDNode>(N);
14712     SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, SDLoc(N), SDTys, Ops,
14713                                            MemInt->getMemoryVT(),
14714                                            MemInt->getMemOperand());
14715 
14716     // Update the uses.
14717     std::vector<SDValue> NewResults;
14718     for (unsigned i = 0; i < NumResultVecs; ++i) {
14719       NewResults.push_back(SDValue(UpdN.getNode(), i));
14720     }
14721     NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1));
14722     DCI.CombineTo(N, NewResults);
14723     DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
14724 
14725     break;
14726   }
14727   return SDValue();
14728 }
14729 
14730 // Checks to see if the value is the prescribed width and returns information
14731 // about its extension mode.
14732 static
14733 bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) {
14734   ExtType = ISD::NON_EXTLOAD;
14735   switch(V.getNode()->getOpcode()) {
14736   default:
14737     return false;
14738   case ISD::LOAD: {
14739     LoadSDNode *LoadNode = cast<LoadSDNode>(V.getNode());
14740     if ((LoadNode->getMemoryVT() == MVT::i8 && width == 8)
14741        || (LoadNode->getMemoryVT() == MVT::i16 && width == 16)) {
14742       ExtType = LoadNode->getExtensionType();
14743       return true;
14744     }
14745     return false;
14746   }
14747   case ISD::AssertSext: {
14748     VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
14749     if ((TypeNode->getVT() == MVT::i8 && width == 8)
14750        || (TypeNode->getVT() == MVT::i16 && width == 16)) {
14751       ExtType = ISD::SEXTLOAD;
14752       return true;
14753     }
14754     return false;
14755   }
14756   case ISD::AssertZext: {
14757     VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
14758     if ((TypeNode->getVT() == MVT::i8 && width == 8)
14759        || (TypeNode->getVT() == MVT::i16 && width == 16)) {
14760       ExtType = ISD::ZEXTLOAD;
14761       return true;
14762     }
14763     return false;
14764   }
14765   case ISD::Constant:
14766   case ISD::TargetConstant: {
14767     return std::abs(cast<ConstantSDNode>(V.getNode())->getSExtValue()) <
14768            1LL << (width - 1);
14769   }
14770   }
14771 
14772   return true;
14773 }
14774 
14775 // This function does a whole lot of voodoo to determine if the tests are
14776 // equivalent without and with a mask. Essentially what happens is that given a
14777 // DAG resembling:
14778 //
14779 //  +-------------+ +-------------+ +-------------+ +-------------+
14780 //  |    Input    | | AddConstant | | CompConstant| |     CC      |
14781 //  +-------------+ +-------------+ +-------------+ +-------------+
14782 //           |           |           |               |
14783 //           V           V           |    +----------+
14784 //          +-------------+  +----+  |    |
14785 //          |     ADD     |  |0xff|  |    |
14786 //          +-------------+  +----+  |    |
14787 //                  |           |    |    |
14788 //                  V           V    |    |
14789 //                 +-------------+   |    |
14790 //                 |     AND     |   |    |
14791 //                 +-------------+   |    |
14792 //                      |            |    |
14793 //                      +-----+      |    |
14794 //                            |      |    |
14795 //                            V      V    V
14796 //                           +-------------+
14797 //                           |     CMP     |
14798 //                           +-------------+
14799 //
14800 // The AND node may be safely removed for some combinations of inputs. In
14801 // particular we need to take into account the extension type of the Input,
14802 // the exact values of AddConstant, CompConstant, and CC, along with the nominal
14803 // width of the input (this can work for any width inputs, the above graph is
14804 // specific to 8 bits.
14805 //
14806 // The specific equations were worked out by generating output tables for each
14807 // AArch64CC value in terms of and AddConstant (w1), CompConstant(w2). The
14808 // problem was simplified by working with 4 bit inputs, which means we only
14809 // needed to reason about 24 distinct bit patterns: 8 patterns unique to zero
14810 // extension (8,15), 8 patterns unique to sign extensions (-8,-1), and 8
14811 // patterns present in both extensions (0,7). For every distinct set of
14812 // AddConstant and CompConstants bit patterns we can consider the masked and
14813 // unmasked versions to be equivalent if the result of this function is true for
14814 // all 16 distinct bit patterns of for the current extension type of Input (w0).
14815 //
14816 //   sub      w8, w0, w1
14817 //   and      w10, w8, #0x0f
14818 //   cmp      w8, w2
14819 //   cset     w9, AArch64CC
14820 //   cmp      w10, w2
14821 //   cset     w11, AArch64CC
14822 //   cmp      w9, w11
14823 //   cset     w0, eq
14824 //   ret
14825 //
14826 // Since the above function shows when the outputs are equivalent it defines
14827 // when it is safe to remove the AND. Unfortunately it only runs on AArch64 and
14828 // would be expensive to run during compiles. The equations below were written
14829 // in a test harness that confirmed they gave equivalent outputs to the above
14830 // for all inputs function, so they can be used determine if the removal is
14831 // legal instead.
14832 //
14833 // isEquivalentMaskless() is the code for testing if the AND can be removed
14834 // factored out of the DAG recognition as the DAG can take several forms.
14835 
14836 static bool isEquivalentMaskless(unsigned CC, unsigned width,
14837                                  ISD::LoadExtType ExtType, int AddConstant,
14838                                  int CompConstant) {
14839   // By being careful about our equations and only writing the in term
14840   // symbolic values and well known constants (0, 1, -1, MaxUInt) we can
14841   // make them generally applicable to all bit widths.
14842   int MaxUInt = (1 << width);
14843 
14844   // For the purposes of these comparisons sign extending the type is
14845   // equivalent to zero extending the add and displacing it by half the integer
14846   // width. Provided we are careful and make sure our equations are valid over
14847   // the whole range we can just adjust the input and avoid writing equations
14848   // for sign extended inputs.
14849   if (ExtType == ISD::SEXTLOAD)
14850     AddConstant -= (1 << (width-1));
14851 
14852   switch(CC) {
14853   case AArch64CC::LE:
14854   case AArch64CC::GT:
14855     if ((AddConstant == 0) ||
14856         (CompConstant == MaxUInt - 1 && AddConstant < 0) ||
14857         (AddConstant >= 0 && CompConstant < 0) ||
14858         (AddConstant <= 0 && CompConstant <= 0 && CompConstant < AddConstant))
14859       return true;
14860     break;
14861   case AArch64CC::LT:
14862   case AArch64CC::GE:
14863     if ((AddConstant == 0) ||
14864         (AddConstant >= 0 && CompConstant <= 0) ||
14865         (AddConstant <= 0 && CompConstant <= 0 && CompConstant <= AddConstant))
14866       return true;
14867     break;
14868   case AArch64CC::HI:
14869   case AArch64CC::LS:
14870     if ((AddConstant >= 0 && CompConstant < 0) ||
14871        (AddConstant <= 0 && CompConstant >= -1 &&
14872         CompConstant < AddConstant + MaxUInt))
14873       return true;
14874    break;
14875   case AArch64CC::PL:
14876   case AArch64CC::MI:
14877     if ((AddConstant == 0) ||
14878         (AddConstant > 0 && CompConstant <= 0) ||
14879         (AddConstant < 0 && CompConstant <= AddConstant))
14880       return true;
14881     break;
14882   case AArch64CC::LO:
14883   case AArch64CC::HS:
14884     if ((AddConstant >= 0 && CompConstant <= 0) ||
14885         (AddConstant <= 0 && CompConstant >= 0 &&
14886          CompConstant <= AddConstant + MaxUInt))
14887       return true;
14888     break;
14889   case AArch64CC::EQ:
14890   case AArch64CC::NE:
14891     if ((AddConstant > 0 && CompConstant < 0) ||
14892         (AddConstant < 0 && CompConstant >= 0 &&
14893          CompConstant < AddConstant + MaxUInt) ||
14894         (AddConstant >= 0 && CompConstant >= 0 &&
14895          CompConstant >= AddConstant) ||
14896         (AddConstant <= 0 && CompConstant < 0 && CompConstant < AddConstant))
14897       return true;
14898     break;
14899   case AArch64CC::VS:
14900   case AArch64CC::VC:
14901   case AArch64CC::AL:
14902   case AArch64CC::NV:
14903     return true;
14904   case AArch64CC::Invalid:
14905     break;
14906   }
14907 
14908   return false;
14909 }
14910 
14911 static
14912 SDValue performCONDCombine(SDNode *N,
14913                            TargetLowering::DAGCombinerInfo &DCI,
14914                            SelectionDAG &DAG, unsigned CCIndex,
14915                            unsigned CmpIndex) {
14916   unsigned CC = cast<ConstantSDNode>(N->getOperand(CCIndex))->getSExtValue();
14917   SDNode *SubsNode = N->getOperand(CmpIndex).getNode();
14918   unsigned CondOpcode = SubsNode->getOpcode();
14919 
14920   if (CondOpcode != AArch64ISD::SUBS)
14921     return SDValue();
14922 
14923   // There is a SUBS feeding this condition. Is it fed by a mask we can
14924   // use?
14925 
14926   SDNode *AndNode = SubsNode->getOperand(0).getNode();
14927   unsigned MaskBits = 0;
14928 
14929   if (AndNode->getOpcode() != ISD::AND)
14930     return SDValue();
14931 
14932   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(AndNode->getOperand(1))) {
14933     uint32_t CNV = CN->getZExtValue();
14934     if (CNV == 255)
14935       MaskBits = 8;
14936     else if (CNV == 65535)
14937       MaskBits = 16;
14938   }
14939 
14940   if (!MaskBits)
14941     return SDValue();
14942 
14943   SDValue AddValue = AndNode->getOperand(0);
14944 
14945   if (AddValue.getOpcode() != ISD::ADD)
14946     return SDValue();
14947 
14948   // The basic dag structure is correct, grab the inputs and validate them.
14949 
14950   SDValue AddInputValue1 = AddValue.getNode()->getOperand(0);
14951   SDValue AddInputValue2 = AddValue.getNode()->getOperand(1);
14952   SDValue SubsInputValue = SubsNode->getOperand(1);
14953 
14954   // The mask is present and the provenance of all the values is a smaller type,
14955   // lets see if the mask is superfluous.
14956 
14957   if (!isa<ConstantSDNode>(AddInputValue2.getNode()) ||
14958       !isa<ConstantSDNode>(SubsInputValue.getNode()))
14959     return SDValue();
14960 
14961   ISD::LoadExtType ExtType;
14962 
14963   if (!checkValueWidth(SubsInputValue, MaskBits, ExtType) ||
14964       !checkValueWidth(AddInputValue2, MaskBits, ExtType) ||
14965       !checkValueWidth(AddInputValue1, MaskBits, ExtType) )
14966     return SDValue();
14967 
14968   if(!isEquivalentMaskless(CC, MaskBits, ExtType,
14969                 cast<ConstantSDNode>(AddInputValue2.getNode())->getSExtValue(),
14970                 cast<ConstantSDNode>(SubsInputValue.getNode())->getSExtValue()))
14971     return SDValue();
14972 
14973   // The AND is not necessary, remove it.
14974 
14975   SDVTList VTs = DAG.getVTList(SubsNode->getValueType(0),
14976                                SubsNode->getValueType(1));
14977   SDValue Ops[] = { AddValue, SubsNode->getOperand(1) };
14978 
14979   SDValue NewValue = DAG.getNode(CondOpcode, SDLoc(SubsNode), VTs, Ops);
14980   DAG.ReplaceAllUsesWith(SubsNode, NewValue.getNode());
14981 
14982   return SDValue(N, 0);
14983 }
14984 
14985 // Optimize compare with zero and branch.
14986 static SDValue performBRCONDCombine(SDNode *N,
14987                                     TargetLowering::DAGCombinerInfo &DCI,
14988                                     SelectionDAG &DAG) {
14989   MachineFunction &MF = DAG.getMachineFunction();
14990   // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z instructions
14991   // will not be produced, as they are conditional branch instructions that do
14992   // not set flags.
14993   if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
14994     return SDValue();
14995 
14996   if (SDValue NV = performCONDCombine(N, DCI, DAG, 2, 3))
14997     N = NV.getNode();
14998   SDValue Chain = N->getOperand(0);
14999   SDValue Dest = N->getOperand(1);
15000   SDValue CCVal = N->getOperand(2);
15001   SDValue Cmp = N->getOperand(3);
15002 
15003   assert(isa<ConstantSDNode>(CCVal) && "Expected a ConstantSDNode here!");
15004   unsigned CC = cast<ConstantSDNode>(CCVal)->getZExtValue();
15005   if (CC != AArch64CC::EQ && CC != AArch64CC::NE)
15006     return SDValue();
15007 
15008   unsigned CmpOpc = Cmp.getOpcode();
15009   if (CmpOpc != AArch64ISD::ADDS && CmpOpc != AArch64ISD::SUBS)
15010     return SDValue();
15011 
15012   // Only attempt folding if there is only one use of the flag and no use of the
15013   // value.
15014   if (!Cmp->hasNUsesOfValue(0, 0) || !Cmp->hasNUsesOfValue(1, 1))
15015     return SDValue();
15016 
15017   SDValue LHS = Cmp.getOperand(0);
15018   SDValue RHS = Cmp.getOperand(1);
15019 
15020   assert(LHS.getValueType() == RHS.getValueType() &&
15021          "Expected the value type to be the same for both operands!");
15022   if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
15023     return SDValue();
15024 
15025   if (isNullConstant(LHS))
15026     std::swap(LHS, RHS);
15027 
15028   if (!isNullConstant(RHS))
15029     return SDValue();
15030 
15031   if (LHS.getOpcode() == ISD::SHL || LHS.getOpcode() == ISD::SRA ||
15032       LHS.getOpcode() == ISD::SRL)
15033     return SDValue();
15034 
15035   // Fold the compare into the branch instruction.
15036   SDValue BR;
15037   if (CC == AArch64CC::EQ)
15038     BR = DAG.getNode(AArch64ISD::CBZ, SDLoc(N), MVT::Other, Chain, LHS, Dest);
15039   else
15040     BR = DAG.getNode(AArch64ISD::CBNZ, SDLoc(N), MVT::Other, Chain, LHS, Dest);
15041 
15042   // Do not add new nodes to DAG combiner worklist.
15043   DCI.CombineTo(N, BR, false);
15044 
15045   return SDValue();
15046 }
15047 
15048 // Optimize some simple tbz/tbnz cases.  Returns the new operand and bit to test
15049 // as well as whether the test should be inverted.  This code is required to
15050 // catch these cases (as opposed to standard dag combines) because
15051 // AArch64ISD::TBZ is matched during legalization.
15052 static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert,
15053                                  SelectionDAG &DAG) {
15054 
15055   if (!Op->hasOneUse())
15056     return Op;
15057 
15058   // We don't handle undef/constant-fold cases below, as they should have
15059   // already been taken care of (e.g. and of 0, test of undefined shifted bits,
15060   // etc.)
15061 
15062   // (tbz (trunc x), b) -> (tbz x, b)
15063   // This case is just here to enable more of the below cases to be caught.
15064   if (Op->getOpcode() == ISD::TRUNCATE &&
15065       Bit < Op->getValueType(0).getSizeInBits()) {
15066     return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15067   }
15068 
15069   // (tbz (any_ext x), b) -> (tbz x, b) if we don't use the extended bits.
15070   if (Op->getOpcode() == ISD::ANY_EXTEND &&
15071       Bit < Op->getOperand(0).getValueSizeInBits()) {
15072     return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15073   }
15074 
15075   if (Op->getNumOperands() != 2)
15076     return Op;
15077 
15078   auto *C = dyn_cast<ConstantSDNode>(Op->getOperand(1));
15079   if (!C)
15080     return Op;
15081 
15082   switch (Op->getOpcode()) {
15083   default:
15084     return Op;
15085 
15086   // (tbz (and x, m), b) -> (tbz x, b)
15087   case ISD::AND:
15088     if ((C->getZExtValue() >> Bit) & 1)
15089       return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15090     return Op;
15091 
15092   // (tbz (shl x, c), b) -> (tbz x, b-c)
15093   case ISD::SHL:
15094     if (C->getZExtValue() <= Bit &&
15095         (Bit - C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
15096       Bit = Bit - C->getZExtValue();
15097       return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15098     }
15099     return Op;
15100 
15101   // (tbz (sra x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits in x
15102   case ISD::SRA:
15103     Bit = Bit + C->getZExtValue();
15104     if (Bit >= Op->getValueType(0).getSizeInBits())
15105       Bit = Op->getValueType(0).getSizeInBits() - 1;
15106     return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15107 
15108   // (tbz (srl x, c), b) -> (tbz x, b+c)
15109   case ISD::SRL:
15110     if ((Bit + C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
15111       Bit = Bit + C->getZExtValue();
15112       return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15113     }
15114     return Op;
15115 
15116   // (tbz (xor x, -1), b) -> (tbnz x, b)
15117   case ISD::XOR:
15118     if ((C->getZExtValue() >> Bit) & 1)
15119       Invert = !Invert;
15120     return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
15121   }
15122 }
15123 
15124 // Optimize test single bit zero/non-zero and branch.
15125 static SDValue performTBZCombine(SDNode *N,
15126                                  TargetLowering::DAGCombinerInfo &DCI,
15127                                  SelectionDAG &DAG) {
15128   unsigned Bit = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
15129   bool Invert = false;
15130   SDValue TestSrc = N->getOperand(1);
15131   SDValue NewTestSrc = getTestBitOperand(TestSrc, Bit, Invert, DAG);
15132 
15133   if (TestSrc == NewTestSrc)
15134     return SDValue();
15135 
15136   unsigned NewOpc = N->getOpcode();
15137   if (Invert) {
15138     if (NewOpc == AArch64ISD::TBZ)
15139       NewOpc = AArch64ISD::TBNZ;
15140     else {
15141       assert(NewOpc == AArch64ISD::TBNZ);
15142       NewOpc = AArch64ISD::TBZ;
15143     }
15144   }
15145 
15146   SDLoc DL(N);
15147   return DAG.getNode(NewOpc, DL, MVT::Other, N->getOperand(0), NewTestSrc,
15148                      DAG.getConstant(Bit, DL, MVT::i64), N->getOperand(3));
15149 }
15150 
15151 // vselect (v1i1 setcc) ->
15152 //     vselect (v1iXX setcc)  (XX is the size of the compared operand type)
15153 // FIXME: Currently the type legalizer can't handle VSELECT having v1i1 as
15154 // condition. If it can legalize "VSELECT v1i1" correctly, no need to combine
15155 // such VSELECT.
15156 static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
15157   SDValue N0 = N->getOperand(0);
15158   EVT CCVT = N0.getValueType();
15159 
15160   // Check for sign pattern (VSELECT setgt, iN lhs, -1, 1, -1) and transform
15161   // into (OR (ASR lhs, N-1), 1), which requires less instructions for the
15162   // supported types.
15163   SDValue SetCC = N->getOperand(0);
15164   if (SetCC.getOpcode() == ISD::SETCC &&
15165       SetCC.getOperand(2) == DAG.getCondCode(ISD::SETGT)) {
15166     SDValue CmpLHS = SetCC.getOperand(0);
15167     EVT VT = CmpLHS.getValueType();
15168     SDNode *CmpRHS = SetCC.getOperand(1).getNode();
15169     SDNode *SplatLHS = N->getOperand(1).getNode();
15170     SDNode *SplatRHS = N->getOperand(2).getNode();
15171     APInt SplatLHSVal;
15172     if (CmpLHS.getValueType() == N->getOperand(1).getValueType() &&
15173         VT.isSimple() &&
15174         is_contained(
15175             makeArrayRef({MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
15176                           MVT::v2i32, MVT::v4i32, MVT::v2i64}),
15177             VT.getSimpleVT().SimpleTy) &&
15178         ISD::isConstantSplatVector(SplatLHS, SplatLHSVal) &&
15179         SplatLHSVal.isOneValue() && ISD::isConstantSplatVectorAllOnes(CmpRHS) &&
15180         ISD::isConstantSplatVectorAllOnes(SplatRHS)) {
15181       unsigned NumElts = VT.getVectorNumElements();
15182       SmallVector<SDValue, 8> Ops(
15183           NumElts, DAG.getConstant(VT.getScalarSizeInBits() - 1, SDLoc(N),
15184                                    VT.getScalarType()));
15185       SDValue Val = DAG.getBuildVector(VT, SDLoc(N), Ops);
15186 
15187       auto Shift = DAG.getNode(ISD::SRA, SDLoc(N), VT, CmpLHS, Val);
15188       auto Or = DAG.getNode(ISD::OR, SDLoc(N), VT, Shift, N->getOperand(1));
15189       return Or;
15190     }
15191   }
15192 
15193   if (N0.getOpcode() != ISD::SETCC ||
15194       CCVT.getVectorElementCount() != ElementCount::getFixed(1) ||
15195       CCVT.getVectorElementType() != MVT::i1)
15196     return SDValue();
15197 
15198   EVT ResVT = N->getValueType(0);
15199   EVT CmpVT = N0.getOperand(0).getValueType();
15200   // Only combine when the result type is of the same size as the compared
15201   // operands.
15202   if (ResVT.getSizeInBits() != CmpVT.getSizeInBits())
15203     return SDValue();
15204 
15205   SDValue IfTrue = N->getOperand(1);
15206   SDValue IfFalse = N->getOperand(2);
15207   SetCC = DAG.getSetCC(SDLoc(N), CmpVT.changeVectorElementTypeToInteger(),
15208                        N0.getOperand(0), N0.getOperand(1),
15209                        cast<CondCodeSDNode>(N0.getOperand(2))->get());
15210   return DAG.getNode(ISD::VSELECT, SDLoc(N), ResVT, SetCC,
15211                      IfTrue, IfFalse);
15212 }
15213 
15214 /// A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with
15215 /// the compare-mask instructions rather than going via NZCV, even if LHS and
15216 /// RHS are really scalar. This replaces any scalar setcc in the above pattern
15217 /// with a vector one followed by a DUP shuffle on the result.
15218 static SDValue performSelectCombine(SDNode *N,
15219                                     TargetLowering::DAGCombinerInfo &DCI) {
15220   SelectionDAG &DAG = DCI.DAG;
15221   SDValue N0 = N->getOperand(0);
15222   EVT ResVT = N->getValueType(0);
15223 
15224   if (N0.getOpcode() != ISD::SETCC)
15225     return SDValue();
15226 
15227   // Make sure the SETCC result is either i1 (initial DAG), or i32, the lowered
15228   // scalar SetCCResultType. We also don't expect vectors, because we assume
15229   // that selects fed by vector SETCCs are canonicalized to VSELECT.
15230   assert((N0.getValueType() == MVT::i1 || N0.getValueType() == MVT::i32) &&
15231          "Scalar-SETCC feeding SELECT has unexpected result type!");
15232 
15233   // If NumMaskElts == 0, the comparison is larger than select result. The
15234   // largest real NEON comparison is 64-bits per lane, which means the result is
15235   // at most 32-bits and an illegal vector. Just bail out for now.
15236   EVT SrcVT = N0.getOperand(0).getValueType();
15237 
15238   // Don't try to do this optimization when the setcc itself has i1 operands.
15239   // There are no legal vectors of i1, so this would be pointless.
15240   if (SrcVT == MVT::i1)
15241     return SDValue();
15242 
15243   int NumMaskElts = ResVT.getSizeInBits() / SrcVT.getSizeInBits();
15244   if (!ResVT.isVector() || NumMaskElts == 0)
15245     return SDValue();
15246 
15247   SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumMaskElts);
15248   EVT CCVT = SrcVT.changeVectorElementTypeToInteger();
15249 
15250   // Also bail out if the vector CCVT isn't the same size as ResVT.
15251   // This can happen if the SETCC operand size doesn't divide the ResVT size
15252   // (e.g., f64 vs v3f32).
15253   if (CCVT.getSizeInBits() != ResVT.getSizeInBits())
15254     return SDValue();
15255 
15256   // Make sure we didn't create illegal types, if we're not supposed to.
15257   assert(DCI.isBeforeLegalize() ||
15258          DAG.getTargetLoweringInfo().isTypeLegal(SrcVT));
15259 
15260   // First perform a vector comparison, where lane 0 is the one we're interested
15261   // in.
15262   SDLoc DL(N0);
15263   SDValue LHS =
15264       DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(0));
15265   SDValue RHS =
15266       DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(1));
15267   SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2));
15268 
15269   // Now duplicate the comparison mask we want across all other lanes.
15270   SmallVector<int, 8> DUPMask(CCVT.getVectorNumElements(), 0);
15271   SDValue Mask = DAG.getVectorShuffle(CCVT, DL, SetCC, SetCC, DUPMask);
15272   Mask = DAG.getNode(ISD::BITCAST, DL,
15273                      ResVT.changeVectorElementTypeToInteger(), Mask);
15274 
15275   return DAG.getSelect(DL, ResVT, Mask, N->getOperand(1), N->getOperand(2));
15276 }
15277 
15278 /// Get rid of unnecessary NVCASTs (that don't change the type).
15279 static SDValue performNVCASTCombine(SDNode *N) {
15280   if (N->getValueType(0) == N->getOperand(0).getValueType())
15281     return N->getOperand(0);
15282 
15283   return SDValue();
15284 }
15285 
15286 // If all users of the globaladdr are of the form (globaladdr + constant), find
15287 // the smallest constant, fold it into the globaladdr's offset and rewrite the
15288 // globaladdr as (globaladdr + constant) - constant.
15289 static SDValue performGlobalAddressCombine(SDNode *N, SelectionDAG &DAG,
15290                                            const AArch64Subtarget *Subtarget,
15291                                            const TargetMachine &TM) {
15292   auto *GN = cast<GlobalAddressSDNode>(N);
15293   if (Subtarget->ClassifyGlobalReference(GN->getGlobal(), TM) !=
15294       AArch64II::MO_NO_FLAG)
15295     return SDValue();
15296 
15297   uint64_t MinOffset = -1ull;
15298   for (SDNode *N : GN->uses()) {
15299     if (N->getOpcode() != ISD::ADD)
15300       return SDValue();
15301     auto *C = dyn_cast<ConstantSDNode>(N->getOperand(0));
15302     if (!C)
15303       C = dyn_cast<ConstantSDNode>(N->getOperand(1));
15304     if (!C)
15305       return SDValue();
15306     MinOffset = std::min(MinOffset, C->getZExtValue());
15307   }
15308   uint64_t Offset = MinOffset + GN->getOffset();
15309 
15310   // Require that the new offset is larger than the existing one. Otherwise, we
15311   // can end up oscillating between two possible DAGs, for example,
15312   // (add (add globaladdr + 10, -1), 1) and (add globaladdr + 9, 1).
15313   if (Offset <= uint64_t(GN->getOffset()))
15314     return SDValue();
15315 
15316   // Check whether folding this offset is legal. It must not go out of bounds of
15317   // the referenced object to avoid violating the code model, and must be
15318   // smaller than 2^21 because this is the largest offset expressible in all
15319   // object formats.
15320   //
15321   // This check also prevents us from folding negative offsets, which will end
15322   // up being treated in the same way as large positive ones. They could also
15323   // cause code model violations, and aren't really common enough to matter.
15324   if (Offset >= (1 << 21))
15325     return SDValue();
15326 
15327   const GlobalValue *GV = GN->getGlobal();
15328   Type *T = GV->getValueType();
15329   if (!T->isSized() ||
15330       Offset > GV->getParent()->getDataLayout().getTypeAllocSize(T))
15331     return SDValue();
15332 
15333   SDLoc DL(GN);
15334   SDValue Result = DAG.getGlobalAddress(GV, DL, MVT::i64, Offset);
15335   return DAG.getNode(ISD::SUB, DL, MVT::i64, Result,
15336                      DAG.getConstant(MinOffset, DL, MVT::i64));
15337 }
15338 
15339 // Turns the vector of indices into a vector of byte offstes by scaling Offset
15340 // by (BitWidth / 8).
15341 static SDValue getScaledOffsetForBitWidth(SelectionDAG &DAG, SDValue Offset,
15342                                           SDLoc DL, unsigned BitWidth) {
15343   assert(Offset.getValueType().isScalableVector() &&
15344          "This method is only for scalable vectors of offsets");
15345 
15346   SDValue Shift = DAG.getConstant(Log2_32(BitWidth / 8), DL, MVT::i64);
15347   SDValue SplatShift = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Shift);
15348 
15349   return DAG.getNode(ISD::SHL, DL, MVT::nxv2i64, Offset, SplatShift);
15350 }
15351 
15352 /// Check if the value of \p OffsetInBytes can be used as an immediate for
15353 /// the gather load/prefetch and scatter store instructions with vector base and
15354 /// immediate offset addressing mode:
15355 ///
15356 ///      [<Zn>.[S|D]{, #<imm>}]
15357 ///
15358 /// where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
15359 
15360 inline static bool isValidImmForSVEVecImmAddrMode(unsigned OffsetInBytes,
15361                                                   unsigned ScalarSizeInBytes) {
15362   // The immediate is not a multiple of the scalar size.
15363   if (OffsetInBytes % ScalarSizeInBytes)
15364     return false;
15365 
15366   // The immediate is out of range.
15367   if (OffsetInBytes / ScalarSizeInBytes > 31)
15368     return false;
15369 
15370   return true;
15371 }
15372 
15373 /// Check if the value of \p Offset represents a valid immediate for the SVE
15374 /// gather load/prefetch and scatter store instructiona with vector base and
15375 /// immediate offset addressing mode:
15376 ///
15377 ///      [<Zn>.[S|D]{, #<imm>}]
15378 ///
15379 /// where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
15380 static bool isValidImmForSVEVecImmAddrMode(SDValue Offset,
15381                                            unsigned ScalarSizeInBytes) {
15382   ConstantSDNode *OffsetConst = dyn_cast<ConstantSDNode>(Offset.getNode());
15383   return OffsetConst && isValidImmForSVEVecImmAddrMode(
15384                             OffsetConst->getZExtValue(), ScalarSizeInBytes);
15385 }
15386 
15387 static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG,
15388                                           unsigned Opcode,
15389                                           bool OnlyPackedOffsets = true) {
15390   const SDValue Src = N->getOperand(2);
15391   const EVT SrcVT = Src->getValueType(0);
15392   assert(SrcVT.isScalableVector() &&
15393          "Scatter stores are only possible for SVE vectors");
15394 
15395   SDLoc DL(N);
15396   MVT SrcElVT = SrcVT.getVectorElementType().getSimpleVT();
15397 
15398   // Make sure that source data will fit into an SVE register
15399   if (SrcVT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
15400     return SDValue();
15401 
15402   // For FPs, ACLE only supports _packed_ single and double precision types.
15403   if (SrcElVT.isFloatingPoint())
15404     if ((SrcVT != MVT::nxv4f32) && (SrcVT != MVT::nxv2f64))
15405       return SDValue();
15406 
15407   // Depending on the addressing mode, this is either a pointer or a vector of
15408   // pointers (that fits into one register)
15409   SDValue Base = N->getOperand(4);
15410   // Depending on the addressing mode, this is either a single offset or a
15411   // vector of offsets  (that fits into one register)
15412   SDValue Offset = N->getOperand(5);
15413 
15414   // For "scalar + vector of indices", just scale the indices. This only
15415   // applies to non-temporal scatters because there's no instruction that takes
15416   // indicies.
15417   if (Opcode == AArch64ISD::SSTNT1_INDEX_PRED) {
15418     Offset =
15419         getScaledOffsetForBitWidth(DAG, Offset, DL, SrcElVT.getSizeInBits());
15420     Opcode = AArch64ISD::SSTNT1_PRED;
15421   }
15422 
15423   // In the case of non-temporal gather loads there's only one SVE instruction
15424   // per data-size: "scalar + vector", i.e.
15425   //    * stnt1{b|h|w|d} { z0.s }, p0/z, [z0.s, x0]
15426   // Since we do have intrinsics that allow the arguments to be in a different
15427   // order, we may need to swap them to match the spec.
15428   if (Opcode == AArch64ISD::SSTNT1_PRED && Offset.getValueType().isVector())
15429     std::swap(Base, Offset);
15430 
15431   // SST1_IMM requires that the offset is an immediate that is:
15432   //    * a multiple of #SizeInBytes,
15433   //    * in the range [0, 31 x #SizeInBytes],
15434   // where #SizeInBytes is the size in bytes of the stored items. For
15435   // immediates outside that range and non-immediate scalar offsets use SST1 or
15436   // SST1_UXTW instead.
15437   if (Opcode == AArch64ISD::SST1_IMM_PRED) {
15438     if (!isValidImmForSVEVecImmAddrMode(Offset,
15439                                         SrcVT.getScalarSizeInBits() / 8)) {
15440       if (MVT::nxv4i32 == Base.getValueType().getSimpleVT().SimpleTy)
15441         Opcode = AArch64ISD::SST1_UXTW_PRED;
15442       else
15443         Opcode = AArch64ISD::SST1_PRED;
15444 
15445       std::swap(Base, Offset);
15446     }
15447   }
15448 
15449   auto &TLI = DAG.getTargetLoweringInfo();
15450   if (!TLI.isTypeLegal(Base.getValueType()))
15451     return SDValue();
15452 
15453   // Some scatter store variants allow unpacked offsets, but only as nxv2i32
15454   // vectors. These are implicitly sign (sxtw) or zero (zxtw) extend to
15455   // nxv2i64. Legalize accordingly.
15456   if (!OnlyPackedOffsets &&
15457       Offset.getValueType().getSimpleVT().SimpleTy == MVT::nxv2i32)
15458     Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0);
15459 
15460   if (!TLI.isTypeLegal(Offset.getValueType()))
15461     return SDValue();
15462 
15463   // Source value type that is representable in hardware
15464   EVT HwSrcVt = getSVEContainerType(SrcVT);
15465 
15466   // Keep the original type of the input data to store - this is needed to be
15467   // able to select the correct instruction, e.g. ST1B, ST1H, ST1W and ST1D. For
15468   // FP values we want the integer equivalent, so just use HwSrcVt.
15469   SDValue InputVT = DAG.getValueType(SrcVT);
15470   if (SrcVT.isFloatingPoint())
15471     InputVT = DAG.getValueType(HwSrcVt);
15472 
15473   SDVTList VTs = DAG.getVTList(MVT::Other);
15474   SDValue SrcNew;
15475 
15476   if (Src.getValueType().isFloatingPoint())
15477     SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Src);
15478   else
15479     SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Src);
15480 
15481   SDValue Ops[] = {N->getOperand(0), // Chain
15482                    SrcNew,
15483                    N->getOperand(3), // Pg
15484                    Base,
15485                    Offset,
15486                    InputVT};
15487 
15488   return DAG.getNode(Opcode, DL, VTs, Ops);
15489 }
15490 
15491 static SDValue performGatherLoadCombine(SDNode *N, SelectionDAG &DAG,
15492                                         unsigned Opcode,
15493                                         bool OnlyPackedOffsets = true) {
15494   const EVT RetVT = N->getValueType(0);
15495   assert(RetVT.isScalableVector() &&
15496          "Gather loads are only possible for SVE vectors");
15497 
15498   SDLoc DL(N);
15499 
15500   // Make sure that the loaded data will fit into an SVE register
15501   if (RetVT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
15502     return SDValue();
15503 
15504   // Depending on the addressing mode, this is either a pointer or a vector of
15505   // pointers (that fits into one register)
15506   SDValue Base = N->getOperand(3);
15507   // Depending on the addressing mode, this is either a single offset or a
15508   // vector of offsets  (that fits into one register)
15509   SDValue Offset = N->getOperand(4);
15510 
15511   // For "scalar + vector of indices", just scale the indices. This only
15512   // applies to non-temporal gathers because there's no instruction that takes
15513   // indicies.
15514   if (Opcode == AArch64ISD::GLDNT1_INDEX_MERGE_ZERO) {
15515     Offset = getScaledOffsetForBitWidth(DAG, Offset, DL,
15516                                         RetVT.getScalarSizeInBits());
15517     Opcode = AArch64ISD::GLDNT1_MERGE_ZERO;
15518   }
15519 
15520   // In the case of non-temporal gather loads there's only one SVE instruction
15521   // per data-size: "scalar + vector", i.e.
15522   //    * ldnt1{b|h|w|d} { z0.s }, p0/z, [z0.s, x0]
15523   // Since we do have intrinsics that allow the arguments to be in a different
15524   // order, we may need to swap them to match the spec.
15525   if (Opcode == AArch64ISD::GLDNT1_MERGE_ZERO &&
15526       Offset.getValueType().isVector())
15527     std::swap(Base, Offset);
15528 
15529   // GLD{FF}1_IMM requires that the offset is an immediate that is:
15530   //    * a multiple of #SizeInBytes,
15531   //    * in the range [0, 31 x #SizeInBytes],
15532   // where #SizeInBytes is the size in bytes of the loaded items. For
15533   // immediates outside that range and non-immediate scalar offsets use
15534   // GLD1_MERGE_ZERO or GLD1_UXTW_MERGE_ZERO instead.
15535   if (Opcode == AArch64ISD::GLD1_IMM_MERGE_ZERO ||
15536       Opcode == AArch64ISD::GLDFF1_IMM_MERGE_ZERO) {
15537     if (!isValidImmForSVEVecImmAddrMode(Offset,
15538                                         RetVT.getScalarSizeInBits() / 8)) {
15539       if (MVT::nxv4i32 == Base.getValueType().getSimpleVT().SimpleTy)
15540         Opcode = (Opcode == AArch64ISD::GLD1_IMM_MERGE_ZERO)
15541                      ? AArch64ISD::GLD1_UXTW_MERGE_ZERO
15542                      : AArch64ISD::GLDFF1_UXTW_MERGE_ZERO;
15543       else
15544         Opcode = (Opcode == AArch64ISD::GLD1_IMM_MERGE_ZERO)
15545                      ? AArch64ISD::GLD1_MERGE_ZERO
15546                      : AArch64ISD::GLDFF1_MERGE_ZERO;
15547 
15548       std::swap(Base, Offset);
15549     }
15550   }
15551 
15552   auto &TLI = DAG.getTargetLoweringInfo();
15553   if (!TLI.isTypeLegal(Base.getValueType()))
15554     return SDValue();
15555 
15556   // Some gather load variants allow unpacked offsets, but only as nxv2i32
15557   // vectors. These are implicitly sign (sxtw) or zero (zxtw) extend to
15558   // nxv2i64. Legalize accordingly.
15559   if (!OnlyPackedOffsets &&
15560       Offset.getValueType().getSimpleVT().SimpleTy == MVT::nxv2i32)
15561     Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0);
15562 
15563   // Return value type that is representable in hardware
15564   EVT HwRetVt = getSVEContainerType(RetVT);
15565 
15566   // Keep the original output value type around - this is needed to be able to
15567   // select the correct instruction, e.g. LD1B, LD1H, LD1W and LD1D. For FP
15568   // values we want the integer equivalent, so just use HwRetVT.
15569   SDValue OutVT = DAG.getValueType(RetVT);
15570   if (RetVT.isFloatingPoint())
15571     OutVT = DAG.getValueType(HwRetVt);
15572 
15573   SDVTList VTs = DAG.getVTList(HwRetVt, MVT::Other);
15574   SDValue Ops[] = {N->getOperand(0), // Chain
15575                    N->getOperand(2), // Pg
15576                    Base, Offset, OutVT};
15577 
15578   SDValue Load = DAG.getNode(Opcode, DL, VTs, Ops);
15579   SDValue LoadChain = SDValue(Load.getNode(), 1);
15580 
15581   if (RetVT.isInteger() && (RetVT != HwRetVt))
15582     Load = DAG.getNode(ISD::TRUNCATE, DL, RetVT, Load.getValue(0));
15583 
15584   // If the original return value was FP, bitcast accordingly. Doing it here
15585   // means that we can avoid adding TableGen patterns for FPs.
15586   if (RetVT.isFloatingPoint())
15587     Load = DAG.getNode(ISD::BITCAST, DL, RetVT, Load.getValue(0));
15588 
15589   return DAG.getMergeValues({Load, LoadChain}, DL);
15590 }
15591 
15592 static SDValue
15593 performSignExtendInRegCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
15594                               SelectionDAG &DAG) {
15595   SDLoc DL(N);
15596   SDValue Src = N->getOperand(0);
15597   unsigned Opc = Src->getOpcode();
15598 
15599   // Sign extend of an unsigned unpack -> signed unpack
15600   if (Opc == AArch64ISD::UUNPKHI || Opc == AArch64ISD::UUNPKLO) {
15601 
15602     unsigned SOpc = Opc == AArch64ISD::UUNPKHI ? AArch64ISD::SUNPKHI
15603                                                : AArch64ISD::SUNPKLO;
15604 
15605     // Push the sign extend to the operand of the unpack
15606     // This is necessary where, for example, the operand of the unpack
15607     // is another unpack:
15608     // 4i32 sign_extend_inreg (4i32 uunpklo(8i16 uunpklo (16i8 opnd)), from 4i8)
15609     // ->
15610     // 4i32 sunpklo (8i16 sign_extend_inreg(8i16 uunpklo (16i8 opnd), from 8i8)
15611     // ->
15612     // 4i32 sunpklo(8i16 sunpklo(16i8 opnd))
15613     SDValue ExtOp = Src->getOperand(0);
15614     auto VT = cast<VTSDNode>(N->getOperand(1))->getVT();
15615     EVT EltTy = VT.getVectorElementType();
15616     (void)EltTy;
15617 
15618     assert((EltTy == MVT::i8 || EltTy == MVT::i16 || EltTy == MVT::i32) &&
15619            "Sign extending from an invalid type");
15620 
15621     EVT ExtVT = VT.getDoubleNumVectorElementsVT(*DAG.getContext());
15622 
15623     SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ExtOp.getValueType(),
15624                               ExtOp, DAG.getValueType(ExtVT));
15625 
15626     return DAG.getNode(SOpc, DL, N->getValueType(0), Ext);
15627   }
15628 
15629   if (DCI.isBeforeLegalizeOps())
15630     return SDValue();
15631 
15632   if (!EnableCombineMGatherIntrinsics)
15633     return SDValue();
15634 
15635   // SVE load nodes (e.g. AArch64ISD::GLD1) are straightforward candidates
15636   // for DAG Combine with SIGN_EXTEND_INREG. Bail out for all other nodes.
15637   unsigned NewOpc;
15638   unsigned MemVTOpNum = 4;
15639   switch (Opc) {
15640   case AArch64ISD::LD1_MERGE_ZERO:
15641     NewOpc = AArch64ISD::LD1S_MERGE_ZERO;
15642     MemVTOpNum = 3;
15643     break;
15644   case AArch64ISD::LDNF1_MERGE_ZERO:
15645     NewOpc = AArch64ISD::LDNF1S_MERGE_ZERO;
15646     MemVTOpNum = 3;
15647     break;
15648   case AArch64ISD::LDFF1_MERGE_ZERO:
15649     NewOpc = AArch64ISD::LDFF1S_MERGE_ZERO;
15650     MemVTOpNum = 3;
15651     break;
15652   case AArch64ISD::GLD1_MERGE_ZERO:
15653     NewOpc = AArch64ISD::GLD1S_MERGE_ZERO;
15654     break;
15655   case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
15656     NewOpc = AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
15657     break;
15658   case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
15659     NewOpc = AArch64ISD::GLD1S_SXTW_MERGE_ZERO;
15660     break;
15661   case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
15662     NewOpc = AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO;
15663     break;
15664   case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
15665     NewOpc = AArch64ISD::GLD1S_UXTW_MERGE_ZERO;
15666     break;
15667   case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
15668     NewOpc = AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO;
15669     break;
15670   case AArch64ISD::GLD1_IMM_MERGE_ZERO:
15671     NewOpc = AArch64ISD::GLD1S_IMM_MERGE_ZERO;
15672     break;
15673   case AArch64ISD::GLDFF1_MERGE_ZERO:
15674     NewOpc = AArch64ISD::GLDFF1S_MERGE_ZERO;
15675     break;
15676   case AArch64ISD::GLDFF1_SCALED_MERGE_ZERO:
15677     NewOpc = AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO;
15678     break;
15679   case AArch64ISD::GLDFF1_SXTW_MERGE_ZERO:
15680     NewOpc = AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO;
15681     break;
15682   case AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO:
15683     NewOpc = AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO;
15684     break;
15685   case AArch64ISD::GLDFF1_UXTW_MERGE_ZERO:
15686     NewOpc = AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO;
15687     break;
15688   case AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO:
15689     NewOpc = AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO;
15690     break;
15691   case AArch64ISD::GLDFF1_IMM_MERGE_ZERO:
15692     NewOpc = AArch64ISD::GLDFF1S_IMM_MERGE_ZERO;
15693     break;
15694   case AArch64ISD::GLDNT1_MERGE_ZERO:
15695     NewOpc = AArch64ISD::GLDNT1S_MERGE_ZERO;
15696     break;
15697   default:
15698     return SDValue();
15699   }
15700 
15701   EVT SignExtSrcVT = cast<VTSDNode>(N->getOperand(1))->getVT();
15702   EVT SrcMemVT = cast<VTSDNode>(Src->getOperand(MemVTOpNum))->getVT();
15703 
15704   if ((SignExtSrcVT != SrcMemVT) || !Src.hasOneUse())
15705     return SDValue();
15706 
15707   EVT DstVT = N->getValueType(0);
15708   SDVTList VTs = DAG.getVTList(DstVT, MVT::Other);
15709 
15710   SmallVector<SDValue, 5> Ops;
15711   for (unsigned I = 0; I < Src->getNumOperands(); ++I)
15712     Ops.push_back(Src->getOperand(I));
15713 
15714   SDValue ExtLoad = DAG.getNode(NewOpc, SDLoc(N), VTs, Ops);
15715   DCI.CombineTo(N, ExtLoad);
15716   DCI.CombineTo(Src.getNode(), ExtLoad, ExtLoad.getValue(1));
15717 
15718   // Return N so it doesn't get rechecked
15719   return SDValue(N, 0);
15720 }
15721 
15722 /// Legalize the gather prefetch (scalar + vector addressing mode) when the
15723 /// offset vector is an unpacked 32-bit scalable vector. The other cases (Offset
15724 /// != nxv2i32) do not need legalization.
15725 static SDValue legalizeSVEGatherPrefetchOffsVec(SDNode *N, SelectionDAG &DAG) {
15726   const unsigned OffsetPos = 4;
15727   SDValue Offset = N->getOperand(OffsetPos);
15728 
15729   // Not an unpacked vector, bail out.
15730   if (Offset.getValueType().getSimpleVT().SimpleTy != MVT::nxv2i32)
15731     return SDValue();
15732 
15733   // Extend the unpacked offset vector to 64-bit lanes.
15734   SDLoc DL(N);
15735   Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset);
15736   SmallVector<SDValue, 5> Ops(N->op_begin(), N->op_end());
15737   // Replace the offset operand with the 64-bit one.
15738   Ops[OffsetPos] = Offset;
15739 
15740   return DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::Other), Ops);
15741 }
15742 
15743 /// Combines a node carrying the intrinsic
15744 /// `aarch64_sve_prf<T>_gather_scalar_offset` into a node that uses
15745 /// `aarch64_sve_prfb_gather_uxtw_index` when the scalar offset passed to
15746 /// `aarch64_sve_prf<T>_gather_scalar_offset` is not a valid immediate for the
15747 /// sve gather prefetch instruction with vector plus immediate addressing mode.
15748 static SDValue combineSVEPrefetchVecBaseImmOff(SDNode *N, SelectionDAG &DAG,
15749                                                unsigned ScalarSizeInBytes) {
15750   const unsigned ImmPos = 4, OffsetPos = 3;
15751   // No need to combine the node if the immediate is valid...
15752   if (isValidImmForSVEVecImmAddrMode(N->getOperand(ImmPos), ScalarSizeInBytes))
15753     return SDValue();
15754 
15755   // ...otherwise swap the offset base with the offset...
15756   SmallVector<SDValue, 5> Ops(N->op_begin(), N->op_end());
15757   std::swap(Ops[ImmPos], Ops[OffsetPos]);
15758   // ...and remap the intrinsic `aarch64_sve_prf<T>_gather_scalar_offset` to
15759   // `aarch64_sve_prfb_gather_uxtw_index`.
15760   SDLoc DL(N);
15761   Ops[1] = DAG.getConstant(Intrinsic::aarch64_sve_prfb_gather_uxtw_index, DL,
15762                            MVT::i64);
15763 
15764   return DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::Other), Ops);
15765 }
15766 
15767 SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
15768                                                  DAGCombinerInfo &DCI) const {
15769   SelectionDAG &DAG = DCI.DAG;
15770   switch (N->getOpcode()) {
15771   default:
15772     LLVM_DEBUG(dbgs() << "Custom combining: skipping\n");
15773     break;
15774   case ISD::ABS:
15775     return performABSCombine(N, DAG, DCI, Subtarget);
15776   case ISD::ADD:
15777   case ISD::SUB:
15778     return performAddSubCombine(N, DCI, DAG);
15779   case ISD::XOR:
15780     return performXorCombine(N, DAG, DCI, Subtarget);
15781   case ISD::MUL:
15782     return performMulCombine(N, DAG, DCI, Subtarget);
15783   case ISD::SINT_TO_FP:
15784   case ISD::UINT_TO_FP:
15785     return performIntToFpCombine(N, DAG, Subtarget);
15786   case ISD::FP_TO_SINT:
15787   case ISD::FP_TO_UINT:
15788     return performFpToIntCombine(N, DAG, DCI, Subtarget);
15789   case ISD::FDIV:
15790     return performFDivCombine(N, DAG, DCI, Subtarget);
15791   case ISD::OR:
15792     return performORCombine(N, DCI, Subtarget);
15793   case ISD::AND:
15794     return performANDCombine(N, DCI);
15795   case ISD::SRL:
15796     return performSRLCombine(N, DCI);
15797   case ISD::INTRINSIC_WO_CHAIN:
15798     return performIntrinsicCombine(N, DCI, Subtarget);
15799   case ISD::ANY_EXTEND:
15800   case ISD::ZERO_EXTEND:
15801   case ISD::SIGN_EXTEND:
15802     return performExtendCombine(N, DCI, DAG);
15803   case ISD::SIGN_EXTEND_INREG:
15804     return performSignExtendInRegCombine(N, DCI, DAG);
15805   case ISD::TRUNCATE:
15806     return performVectorTruncateCombine(N, DCI, DAG);
15807   case ISD::CONCAT_VECTORS:
15808     return performConcatVectorsCombine(N, DCI, DAG);
15809   case ISD::SELECT:
15810     return performSelectCombine(N, DCI);
15811   case ISD::VSELECT:
15812     return performVSelectCombine(N, DCI.DAG);
15813   case ISD::LOAD:
15814     if (performTBISimplification(N->getOperand(1), DCI, DAG))
15815       return SDValue(N, 0);
15816     break;
15817   case ISD::STORE:
15818     return performSTORECombine(N, DCI, DAG, Subtarget);
15819   case AArch64ISD::BRCOND:
15820     return performBRCONDCombine(N, DCI, DAG);
15821   case AArch64ISD::TBNZ:
15822   case AArch64ISD::TBZ:
15823     return performTBZCombine(N, DCI, DAG);
15824   case AArch64ISD::CSEL:
15825     return performCONDCombine(N, DCI, DAG, 2, 3);
15826   case AArch64ISD::DUP:
15827     return performPostLD1Combine(N, DCI, false);
15828   case AArch64ISD::NVCAST:
15829     return performNVCASTCombine(N);
15830   case AArch64ISD::UZP1:
15831     return performUzpCombine(N, DAG);
15832   case AArch64ISD::GLD1_MERGE_ZERO:
15833   case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
15834   case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
15835   case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
15836   case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
15837   case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
15838   case AArch64ISD::GLD1_IMM_MERGE_ZERO:
15839   case AArch64ISD::GLD1S_MERGE_ZERO:
15840   case AArch64ISD::GLD1S_SCALED_MERGE_ZERO:
15841   case AArch64ISD::GLD1S_UXTW_MERGE_ZERO:
15842   case AArch64ISD::GLD1S_SXTW_MERGE_ZERO:
15843   case AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO:
15844   case AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO:
15845   case AArch64ISD::GLD1S_IMM_MERGE_ZERO:
15846     return performGLD1Combine(N, DAG);
15847   case ISD::INSERT_VECTOR_ELT:
15848     return performPostLD1Combine(N, DCI, true);
15849   case ISD::EXTRACT_VECTOR_ELT:
15850     return performExtractVectorEltCombine(N, DAG);
15851   case ISD::VECREDUCE_ADD:
15852     return performVecReduceAddCombine(N, DCI.DAG, Subtarget);
15853   case ISD::INTRINSIC_VOID:
15854   case ISD::INTRINSIC_W_CHAIN:
15855     switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
15856     case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
15857       return combineSVEPrefetchVecBaseImmOff(N, DAG, 1 /*=ScalarSizeInBytes*/);
15858     case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
15859       return combineSVEPrefetchVecBaseImmOff(N, DAG, 2 /*=ScalarSizeInBytes*/);
15860     case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:
15861       return combineSVEPrefetchVecBaseImmOff(N, DAG, 4 /*=ScalarSizeInBytes*/);
15862     case Intrinsic::aarch64_sve_prfd_gather_scalar_offset:
15863       return combineSVEPrefetchVecBaseImmOff(N, DAG, 8 /*=ScalarSizeInBytes*/);
15864     case Intrinsic::aarch64_sve_prfb_gather_uxtw_index:
15865     case Intrinsic::aarch64_sve_prfb_gather_sxtw_index:
15866     case Intrinsic::aarch64_sve_prfh_gather_uxtw_index:
15867     case Intrinsic::aarch64_sve_prfh_gather_sxtw_index:
15868     case Intrinsic::aarch64_sve_prfw_gather_uxtw_index:
15869     case Intrinsic::aarch64_sve_prfw_gather_sxtw_index:
15870     case Intrinsic::aarch64_sve_prfd_gather_uxtw_index:
15871     case Intrinsic::aarch64_sve_prfd_gather_sxtw_index:
15872       return legalizeSVEGatherPrefetchOffsVec(N, DAG);
15873     case Intrinsic::aarch64_neon_ld2:
15874     case Intrinsic::aarch64_neon_ld3:
15875     case Intrinsic::aarch64_neon_ld4:
15876     case Intrinsic::aarch64_neon_ld1x2:
15877     case Intrinsic::aarch64_neon_ld1x3:
15878     case Intrinsic::aarch64_neon_ld1x4:
15879     case Intrinsic::aarch64_neon_ld2lane:
15880     case Intrinsic::aarch64_neon_ld3lane:
15881     case Intrinsic::aarch64_neon_ld4lane:
15882     case Intrinsic::aarch64_neon_ld2r:
15883     case Intrinsic::aarch64_neon_ld3r:
15884     case Intrinsic::aarch64_neon_ld4r:
15885     case Intrinsic::aarch64_neon_st2:
15886     case Intrinsic::aarch64_neon_st3:
15887     case Intrinsic::aarch64_neon_st4:
15888     case Intrinsic::aarch64_neon_st1x2:
15889     case Intrinsic::aarch64_neon_st1x3:
15890     case Intrinsic::aarch64_neon_st1x4:
15891     case Intrinsic::aarch64_neon_st2lane:
15892     case Intrinsic::aarch64_neon_st3lane:
15893     case Intrinsic::aarch64_neon_st4lane:
15894       return performNEONPostLDSTCombine(N, DCI, DAG);
15895     case Intrinsic::aarch64_sve_ldnt1:
15896       return performLDNT1Combine(N, DAG);
15897     case Intrinsic::aarch64_sve_ld1rq:
15898       return performLD1ReplicateCombine<AArch64ISD::LD1RQ_MERGE_ZERO>(N, DAG);
15899     case Intrinsic::aarch64_sve_ld1ro:
15900       return performLD1ReplicateCombine<AArch64ISD::LD1RO_MERGE_ZERO>(N, DAG);
15901     case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
15902       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_MERGE_ZERO);
15903     case Intrinsic::aarch64_sve_ldnt1_gather:
15904       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_MERGE_ZERO);
15905     case Intrinsic::aarch64_sve_ldnt1_gather_index:
15906       return performGatherLoadCombine(N, DAG,
15907                                       AArch64ISD::GLDNT1_INDEX_MERGE_ZERO);
15908     case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
15909       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_MERGE_ZERO);
15910     case Intrinsic::aarch64_sve_ld1:
15911       return performLD1Combine(N, DAG, AArch64ISD::LD1_MERGE_ZERO);
15912     case Intrinsic::aarch64_sve_ldnf1:
15913       return performLD1Combine(N, DAG, AArch64ISD::LDNF1_MERGE_ZERO);
15914     case Intrinsic::aarch64_sve_ldff1:
15915       return performLD1Combine(N, DAG, AArch64ISD::LDFF1_MERGE_ZERO);
15916     case Intrinsic::aarch64_sve_st1:
15917       return performST1Combine(N, DAG);
15918     case Intrinsic::aarch64_sve_stnt1:
15919       return performSTNT1Combine(N, DAG);
15920     case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
15921       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_PRED);
15922     case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
15923       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_PRED);
15924     case Intrinsic::aarch64_sve_stnt1_scatter:
15925       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_PRED);
15926     case Intrinsic::aarch64_sve_stnt1_scatter_index:
15927       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_INDEX_PRED);
15928     case Intrinsic::aarch64_sve_ld1_gather:
15929       return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_MERGE_ZERO);
15930     case Intrinsic::aarch64_sve_ld1_gather_index:
15931       return performGatherLoadCombine(N, DAG,
15932                                       AArch64ISD::GLD1_SCALED_MERGE_ZERO);
15933     case Intrinsic::aarch64_sve_ld1_gather_sxtw:
15934       return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_SXTW_MERGE_ZERO,
15935                                       /*OnlyPackedOffsets=*/false);
15936     case Intrinsic::aarch64_sve_ld1_gather_uxtw:
15937       return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_UXTW_MERGE_ZERO,
15938                                       /*OnlyPackedOffsets=*/false);
15939     case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
15940       return performGatherLoadCombine(N, DAG,
15941                                       AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO,
15942                                       /*OnlyPackedOffsets=*/false);
15943     case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
15944       return performGatherLoadCombine(N, DAG,
15945                                       AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO,
15946                                       /*OnlyPackedOffsets=*/false);
15947     case Intrinsic::aarch64_sve_ld1_gather_scalar_offset:
15948       return performGatherLoadCombine(N, DAG, AArch64ISD::GLD1_IMM_MERGE_ZERO);
15949     case Intrinsic::aarch64_sve_ldff1_gather:
15950       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDFF1_MERGE_ZERO);
15951     case Intrinsic::aarch64_sve_ldff1_gather_index:
15952       return performGatherLoadCombine(N, DAG,
15953                                       AArch64ISD::GLDFF1_SCALED_MERGE_ZERO);
15954     case Intrinsic::aarch64_sve_ldff1_gather_sxtw:
15955       return performGatherLoadCombine(N, DAG,
15956                                       AArch64ISD::GLDFF1_SXTW_MERGE_ZERO,
15957                                       /*OnlyPackedOffsets=*/false);
15958     case Intrinsic::aarch64_sve_ldff1_gather_uxtw:
15959       return performGatherLoadCombine(N, DAG,
15960                                       AArch64ISD::GLDFF1_UXTW_MERGE_ZERO,
15961                                       /*OnlyPackedOffsets=*/false);
15962     case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index:
15963       return performGatherLoadCombine(N, DAG,
15964                                       AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO,
15965                                       /*OnlyPackedOffsets=*/false);
15966     case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index:
15967       return performGatherLoadCombine(N, DAG,
15968                                       AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO,
15969                                       /*OnlyPackedOffsets=*/false);
15970     case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset:
15971       return performGatherLoadCombine(N, DAG,
15972                                       AArch64ISD::GLDFF1_IMM_MERGE_ZERO);
15973     case Intrinsic::aarch64_sve_st1_scatter:
15974       return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_PRED);
15975     case Intrinsic::aarch64_sve_st1_scatter_index:
15976       return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_SCALED_PRED);
15977     case Intrinsic::aarch64_sve_st1_scatter_sxtw:
15978       return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_SXTW_PRED,
15979                                         /*OnlyPackedOffsets=*/false);
15980     case Intrinsic::aarch64_sve_st1_scatter_uxtw:
15981       return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_UXTW_PRED,
15982                                         /*OnlyPackedOffsets=*/false);
15983     case Intrinsic::aarch64_sve_st1_scatter_sxtw_index:
15984       return performScatterStoreCombine(N, DAG,
15985                                         AArch64ISD::SST1_SXTW_SCALED_PRED,
15986                                         /*OnlyPackedOffsets=*/false);
15987     case Intrinsic::aarch64_sve_st1_scatter_uxtw_index:
15988       return performScatterStoreCombine(N, DAG,
15989                                         AArch64ISD::SST1_UXTW_SCALED_PRED,
15990                                         /*OnlyPackedOffsets=*/false);
15991     case Intrinsic::aarch64_sve_st1_scatter_scalar_offset:
15992       return performScatterStoreCombine(N, DAG, AArch64ISD::SST1_IMM_PRED);
15993     case Intrinsic::aarch64_sve_tuple_get: {
15994       SDLoc DL(N);
15995       SDValue Chain = N->getOperand(0);
15996       SDValue Src1 = N->getOperand(2);
15997       SDValue Idx = N->getOperand(3);
15998 
15999       uint64_t IdxConst = cast<ConstantSDNode>(Idx)->getZExtValue();
16000       EVT ResVT = N->getValueType(0);
16001       uint64_t NumLanes = ResVT.getVectorElementCount().getKnownMinValue();
16002       SDValue ExtIdx = DAG.getVectorIdxConstant(IdxConst * NumLanes, DL);
16003       SDValue Val =
16004           DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, Src1, ExtIdx);
16005       return DAG.getMergeValues({Val, Chain}, DL);
16006     }
16007     case Intrinsic::aarch64_sve_tuple_set: {
16008       SDLoc DL(N);
16009       SDValue Chain = N->getOperand(0);
16010       SDValue Tuple = N->getOperand(2);
16011       SDValue Idx = N->getOperand(3);
16012       SDValue Vec = N->getOperand(4);
16013 
16014       EVT TupleVT = Tuple.getValueType();
16015       uint64_t TupleLanes = TupleVT.getVectorElementCount().getKnownMinValue();
16016 
16017       uint64_t IdxConst = cast<ConstantSDNode>(Idx)->getZExtValue();
16018       uint64_t NumLanes =
16019           Vec.getValueType().getVectorElementCount().getKnownMinValue();
16020 
16021       if ((TupleLanes % NumLanes) != 0)
16022         report_fatal_error("invalid tuple vector!");
16023 
16024       uint64_t NumVecs = TupleLanes / NumLanes;
16025 
16026       SmallVector<SDValue, 4> Opnds;
16027       for (unsigned I = 0; I < NumVecs; ++I) {
16028         if (I == IdxConst)
16029           Opnds.push_back(Vec);
16030         else {
16031           SDValue ExtIdx = DAG.getVectorIdxConstant(I * NumLanes, DL);
16032           Opnds.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
16033                                       Vec.getValueType(), Tuple, ExtIdx));
16034         }
16035       }
16036       SDValue Concat =
16037           DAG.getNode(ISD::CONCAT_VECTORS, DL, Tuple.getValueType(), Opnds);
16038       return DAG.getMergeValues({Concat, Chain}, DL);
16039     }
16040     case Intrinsic::aarch64_sve_tuple_create2:
16041     case Intrinsic::aarch64_sve_tuple_create3:
16042     case Intrinsic::aarch64_sve_tuple_create4: {
16043       SDLoc DL(N);
16044       SDValue Chain = N->getOperand(0);
16045 
16046       SmallVector<SDValue, 4> Opnds;
16047       for (unsigned I = 2; I < N->getNumOperands(); ++I)
16048         Opnds.push_back(N->getOperand(I));
16049 
16050       EVT VT = Opnds[0].getValueType();
16051       EVT EltVT = VT.getVectorElementType();
16052       EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
16053                                     VT.getVectorElementCount() *
16054                                         (N->getNumOperands() - 2));
16055       SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, DestVT, Opnds);
16056       return DAG.getMergeValues({Concat, Chain}, DL);
16057     }
16058     case Intrinsic::aarch64_sve_ld2:
16059     case Intrinsic::aarch64_sve_ld3:
16060     case Intrinsic::aarch64_sve_ld4: {
16061       SDLoc DL(N);
16062       SDValue Chain = N->getOperand(0);
16063       SDValue Mask = N->getOperand(2);
16064       SDValue BasePtr = N->getOperand(3);
16065       SDValue LoadOps[] = {Chain, Mask, BasePtr};
16066       unsigned IntrinsicID =
16067           cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
16068       SDValue Result =
16069           LowerSVEStructLoad(IntrinsicID, LoadOps, N->getValueType(0), DAG, DL);
16070       return DAG.getMergeValues({Result, Chain}, DL);
16071     }
16072     case Intrinsic::aarch64_rndr:
16073     case Intrinsic::aarch64_rndrrs: {
16074       unsigned IntrinsicID =
16075           cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
16076       auto Register =
16077           (IntrinsicID == Intrinsic::aarch64_rndr ? AArch64SysReg::RNDR
16078                                                   : AArch64SysReg::RNDRRS);
16079       SDLoc DL(N);
16080       SDValue A = DAG.getNode(
16081           AArch64ISD::MRS, DL, DAG.getVTList(MVT::i64, MVT::Glue, MVT::Other),
16082           N->getOperand(0), DAG.getConstant(Register, DL, MVT::i64));
16083       SDValue B = DAG.getNode(
16084           AArch64ISD::CSINC, DL, MVT::i32, DAG.getConstant(0, DL, MVT::i32),
16085           DAG.getConstant(0, DL, MVT::i32),
16086           DAG.getConstant(AArch64CC::NE, DL, MVT::i32), A.getValue(1));
16087       return DAG.getMergeValues(
16088           {A, DAG.getZExtOrTrunc(B, DL, MVT::i1), A.getValue(2)}, DL);
16089     }
16090     default:
16091       break;
16092     }
16093     break;
16094   case ISD::GlobalAddress:
16095     return performGlobalAddressCombine(N, DAG, Subtarget, getTargetMachine());
16096   }
16097   return SDValue();
16098 }
16099 
16100 // Check if the return value is used as only a return value, as otherwise
16101 // we can't perform a tail-call. In particular, we need to check for
16102 // target ISD nodes that are returns and any other "odd" constructs
16103 // that the generic analysis code won't necessarily catch.
16104 bool AArch64TargetLowering::isUsedByReturnOnly(SDNode *N,
16105                                                SDValue &Chain) const {
16106   if (N->getNumValues() != 1)
16107     return false;
16108   if (!N->hasNUsesOfValue(1, 0))
16109     return false;
16110 
16111   SDValue TCChain = Chain;
16112   SDNode *Copy = *N->use_begin();
16113   if (Copy->getOpcode() == ISD::CopyToReg) {
16114     // If the copy has a glue operand, we conservatively assume it isn't safe to
16115     // perform a tail call.
16116     if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() ==
16117         MVT::Glue)
16118       return false;
16119     TCChain = Copy->getOperand(0);
16120   } else if (Copy->getOpcode() != ISD::FP_EXTEND)
16121     return false;
16122 
16123   bool HasRet = false;
16124   for (SDNode *Node : Copy->uses()) {
16125     if (Node->getOpcode() != AArch64ISD::RET_FLAG)
16126       return false;
16127     HasRet = true;
16128   }
16129 
16130   if (!HasRet)
16131     return false;
16132 
16133   Chain = TCChain;
16134   return true;
16135 }
16136 
16137 // Return whether the an instruction can potentially be optimized to a tail
16138 // call. This will cause the optimizers to attempt to move, or duplicate,
16139 // return instructions to help enable tail call optimizations for this
16140 // instruction.
16141 bool AArch64TargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
16142   return CI->isTailCall();
16143 }
16144 
16145 bool AArch64TargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base,
16146                                                    SDValue &Offset,
16147                                                    ISD::MemIndexedMode &AM,
16148                                                    bool &IsInc,
16149                                                    SelectionDAG &DAG) const {
16150   if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)
16151     return false;
16152 
16153   Base = Op->getOperand(0);
16154   // All of the indexed addressing mode instructions take a signed
16155   // 9 bit immediate offset.
16156   if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1))) {
16157     int64_t RHSC = RHS->getSExtValue();
16158     if (Op->getOpcode() == ISD::SUB)
16159       RHSC = -(uint64_t)RHSC;
16160     if (!isInt<9>(RHSC))
16161       return false;
16162     IsInc = (Op->getOpcode() == ISD::ADD);
16163     Offset = Op->getOperand(1);
16164     return true;
16165   }
16166   return false;
16167 }
16168 
16169 bool AArch64TargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
16170                                                       SDValue &Offset,
16171                                                       ISD::MemIndexedMode &AM,
16172                                                       SelectionDAG &DAG) const {
16173   EVT VT;
16174   SDValue Ptr;
16175   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
16176     VT = LD->getMemoryVT();
16177     Ptr = LD->getBasePtr();
16178   } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
16179     VT = ST->getMemoryVT();
16180     Ptr = ST->getBasePtr();
16181   } else
16182     return false;
16183 
16184   bool IsInc;
16185   if (!getIndexedAddressParts(Ptr.getNode(), Base, Offset, AM, IsInc, DAG))
16186     return false;
16187   AM = IsInc ? ISD::PRE_INC : ISD::PRE_DEC;
16188   return true;
16189 }
16190 
16191 bool AArch64TargetLowering::getPostIndexedAddressParts(
16192     SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset,
16193     ISD::MemIndexedMode &AM, SelectionDAG &DAG) const {
16194   EVT VT;
16195   SDValue Ptr;
16196   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
16197     VT = LD->getMemoryVT();
16198     Ptr = LD->getBasePtr();
16199   } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
16200     VT = ST->getMemoryVT();
16201     Ptr = ST->getBasePtr();
16202   } else
16203     return false;
16204 
16205   bool IsInc;
16206   if (!getIndexedAddressParts(Op, Base, Offset, AM, IsInc, DAG))
16207     return false;
16208   // Post-indexing updates the base, so it's not a valid transform
16209   // if that's not the same as the load's pointer.
16210   if (Ptr != Base)
16211     return false;
16212   AM = IsInc ? ISD::POST_INC : ISD::POST_DEC;
16213   return true;
16214 }
16215 
16216 static void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
16217                                   SelectionDAG &DAG) {
16218   SDLoc DL(N);
16219   SDValue Op = N->getOperand(0);
16220 
16221   if (N->getValueType(0) != MVT::i16 ||
16222       (Op.getValueType() != MVT::f16 && Op.getValueType() != MVT::bf16))
16223     return;
16224 
16225   Op = SDValue(
16226       DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, MVT::f32,
16227                          DAG.getUNDEF(MVT::i32), Op,
16228                          DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
16229       0);
16230   Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op);
16231   Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op));
16232 }
16233 
16234 static void ReplaceReductionResults(SDNode *N,
16235                                     SmallVectorImpl<SDValue> &Results,
16236                                     SelectionDAG &DAG, unsigned InterOp,
16237                                     unsigned AcrossOp) {
16238   EVT LoVT, HiVT;
16239   SDValue Lo, Hi;
16240   SDLoc dl(N);
16241   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
16242   std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
16243   SDValue InterVal = DAG.getNode(InterOp, dl, LoVT, Lo, Hi);
16244   SDValue SplitVal = DAG.getNode(AcrossOp, dl, LoVT, InterVal);
16245   Results.push_back(SplitVal);
16246 }
16247 
16248 static std::pair<SDValue, SDValue> splitInt128(SDValue N, SelectionDAG &DAG) {
16249   SDLoc DL(N);
16250   SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, N);
16251   SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64,
16252                            DAG.getNode(ISD::SRL, DL, MVT::i128, N,
16253                                        DAG.getConstant(64, DL, MVT::i64)));
16254   return std::make_pair(Lo, Hi);
16255 }
16256 
16257 void AArch64TargetLowering::ReplaceExtractSubVectorResults(
16258     SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
16259   SDValue In = N->getOperand(0);
16260   EVT InVT = In.getValueType();
16261 
16262   // Common code will handle these just fine.
16263   if (!InVT.isScalableVector() || !InVT.isInteger())
16264     return;
16265 
16266   SDLoc DL(N);
16267   EVT VT = N->getValueType(0);
16268 
16269   // The following checks bail if this is not a halving operation.
16270 
16271   ElementCount ResEC = VT.getVectorElementCount();
16272 
16273   if (InVT.getVectorElementCount() != (ResEC * 2))
16274     return;
16275 
16276   auto *CIndex = dyn_cast<ConstantSDNode>(N->getOperand(1));
16277   if (!CIndex)
16278     return;
16279 
16280   unsigned Index = CIndex->getZExtValue();
16281   if ((Index != 0) && (Index != ResEC.getKnownMinValue()))
16282     return;
16283 
16284   unsigned Opcode = (Index == 0) ? AArch64ISD::UUNPKLO : AArch64ISD::UUNPKHI;
16285   EVT ExtendedHalfVT = VT.widenIntegerVectorElementType(*DAG.getContext());
16286 
16287   SDValue Half = DAG.getNode(Opcode, DL, ExtendedHalfVT, N->getOperand(0));
16288   Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Half));
16289 }
16290 
16291 // Create an even/odd pair of X registers holding integer value V.
16292 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
16293   SDLoc dl(V.getNode());
16294   SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i64);
16295   SDValue VHi = DAG.getAnyExtOrTrunc(
16296       DAG.getNode(ISD::SRL, dl, MVT::i128, V, DAG.getConstant(64, dl, MVT::i64)),
16297       dl, MVT::i64);
16298   if (DAG.getDataLayout().isBigEndian())
16299     std::swap (VLo, VHi);
16300   SDValue RegClass =
16301       DAG.getTargetConstant(AArch64::XSeqPairsClassRegClassID, dl, MVT::i32);
16302   SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, dl, MVT::i32);
16303   SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, dl, MVT::i32);
16304   const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
16305   return SDValue(
16306       DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
16307 }
16308 
16309 static void ReplaceCMP_SWAP_128Results(SDNode *N,
16310                                        SmallVectorImpl<SDValue> &Results,
16311                                        SelectionDAG &DAG,
16312                                        const AArch64Subtarget *Subtarget) {
16313   assert(N->getValueType(0) == MVT::i128 &&
16314          "AtomicCmpSwap on types less than 128 should be legal");
16315 
16316   if (Subtarget->hasLSE() || Subtarget->outlineAtomics()) {
16317     // LSE has a 128-bit compare and swap (CASP), but i128 is not a legal type,
16318     // so lower it here, wrapped in REG_SEQUENCE and EXTRACT_SUBREG.
16319     SDValue Ops[] = {
16320         createGPRPairNode(DAG, N->getOperand(2)), // Compare value
16321         createGPRPairNode(DAG, N->getOperand(3)), // Store value
16322         N->getOperand(1), // Ptr
16323         N->getOperand(0), // Chain in
16324     };
16325 
16326     MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
16327 
16328     unsigned Opcode;
16329     switch (MemOp->getOrdering()) {
16330     case AtomicOrdering::Monotonic:
16331       Opcode = AArch64::CASPX;
16332       break;
16333     case AtomicOrdering::Acquire:
16334       Opcode = AArch64::CASPAX;
16335       break;
16336     case AtomicOrdering::Release:
16337       Opcode = AArch64::CASPLX;
16338       break;
16339     case AtomicOrdering::AcquireRelease:
16340     case AtomicOrdering::SequentiallyConsistent:
16341       Opcode = AArch64::CASPALX;
16342       break;
16343     default:
16344       llvm_unreachable("Unexpected ordering!");
16345     }
16346 
16347     MachineSDNode *CmpSwap = DAG.getMachineNode(
16348         Opcode, SDLoc(N), DAG.getVTList(MVT::Untyped, MVT::Other), Ops);
16349     DAG.setNodeMemRefs(CmpSwap, {MemOp});
16350 
16351     unsigned SubReg1 = AArch64::sube64, SubReg2 = AArch64::subo64;
16352     if (DAG.getDataLayout().isBigEndian())
16353       std::swap(SubReg1, SubReg2);
16354     SDValue Lo = DAG.getTargetExtractSubreg(SubReg1, SDLoc(N), MVT::i64,
16355                                             SDValue(CmpSwap, 0));
16356     SDValue Hi = DAG.getTargetExtractSubreg(SubReg2, SDLoc(N), MVT::i64,
16357                                             SDValue(CmpSwap, 0));
16358     Results.push_back(
16359         DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128, Lo, Hi));
16360     Results.push_back(SDValue(CmpSwap, 1)); // Chain out
16361     return;
16362   }
16363 
16364   auto Desired = splitInt128(N->getOperand(2), DAG);
16365   auto New = splitInt128(N->getOperand(3), DAG);
16366   SDValue Ops[] = {N->getOperand(1), Desired.first, Desired.second,
16367                    New.first,        New.second,    N->getOperand(0)};
16368   SDNode *CmpSwap = DAG.getMachineNode(
16369       AArch64::CMP_SWAP_128, SDLoc(N),
16370       DAG.getVTList(MVT::i64, MVT::i64, MVT::i32, MVT::Other), Ops);
16371 
16372   MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
16373   DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
16374 
16375   Results.push_back(DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128,
16376                                 SDValue(CmpSwap, 0), SDValue(CmpSwap, 1)));
16377   Results.push_back(SDValue(CmpSwap, 3));
16378 }
16379 
16380 void AArch64TargetLowering::ReplaceNodeResults(
16381     SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
16382   switch (N->getOpcode()) {
16383   default:
16384     llvm_unreachable("Don't know how to custom expand this");
16385   case ISD::BITCAST:
16386     ReplaceBITCASTResults(N, Results, DAG);
16387     return;
16388   case ISD::VECREDUCE_ADD:
16389   case ISD::VECREDUCE_SMAX:
16390   case ISD::VECREDUCE_SMIN:
16391   case ISD::VECREDUCE_UMAX:
16392   case ISD::VECREDUCE_UMIN:
16393     Results.push_back(LowerVECREDUCE(SDValue(N, 0), DAG));
16394     return;
16395 
16396   case ISD::CTPOP:
16397     if (SDValue Result = LowerCTPOP(SDValue(N, 0), DAG))
16398       Results.push_back(Result);
16399     return;
16400   case AArch64ISD::SADDV:
16401     ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::SADDV);
16402     return;
16403   case AArch64ISD::UADDV:
16404     ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::UADDV);
16405     return;
16406   case AArch64ISD::SMINV:
16407     ReplaceReductionResults(N, Results, DAG, ISD::SMIN, AArch64ISD::SMINV);
16408     return;
16409   case AArch64ISD::UMINV:
16410     ReplaceReductionResults(N, Results, DAG, ISD::UMIN, AArch64ISD::UMINV);
16411     return;
16412   case AArch64ISD::SMAXV:
16413     ReplaceReductionResults(N, Results, DAG, ISD::SMAX, AArch64ISD::SMAXV);
16414     return;
16415   case AArch64ISD::UMAXV:
16416     ReplaceReductionResults(N, Results, DAG, ISD::UMAX, AArch64ISD::UMAXV);
16417     return;
16418   case ISD::FP_TO_UINT:
16419   case ISD::FP_TO_SINT:
16420     assert(N->getValueType(0) == MVT::i128 && "unexpected illegal conversion");
16421     // Let normal code take care of it by not adding anything to Results.
16422     return;
16423   case ISD::ATOMIC_CMP_SWAP:
16424     ReplaceCMP_SWAP_128Results(N, Results, DAG, Subtarget);
16425     return;
16426   case ISD::LOAD: {
16427     assert(SDValue(N, 0).getValueType() == MVT::i128 &&
16428            "unexpected load's value type");
16429     LoadSDNode *LoadNode = cast<LoadSDNode>(N);
16430     if (!LoadNode->isVolatile() || LoadNode->getMemoryVT() != MVT::i128) {
16431       // Non-volatile loads are optimized later in AArch64's load/store
16432       // optimizer.
16433       return;
16434     }
16435 
16436     SDValue Result = DAG.getMemIntrinsicNode(
16437         AArch64ISD::LDP, SDLoc(N),
16438         DAG.getVTList({MVT::i64, MVT::i64, MVT::Other}),
16439         {LoadNode->getChain(), LoadNode->getBasePtr()}, LoadNode->getMemoryVT(),
16440         LoadNode->getMemOperand());
16441 
16442     SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128,
16443                                Result.getValue(0), Result.getValue(1));
16444     Results.append({Pair, Result.getValue(2) /* Chain */});
16445     return;
16446   }
16447   case ISD::EXTRACT_SUBVECTOR:
16448     ReplaceExtractSubVectorResults(N, Results, DAG);
16449     return;
16450   case ISD::INTRINSIC_WO_CHAIN: {
16451     EVT VT = N->getValueType(0);
16452     assert((VT == MVT::i8 || VT == MVT::i16) &&
16453            "custom lowering for unexpected type");
16454 
16455     ConstantSDNode *CN = cast<ConstantSDNode>(N->getOperand(0));
16456     Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
16457     switch (IntID) {
16458     default:
16459       return;
16460     case Intrinsic::aarch64_sve_clasta_n: {
16461       SDLoc DL(N);
16462       auto Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, N->getOperand(2));
16463       auto V = DAG.getNode(AArch64ISD::CLASTA_N, DL, MVT::i32,
16464                            N->getOperand(1), Op2, N->getOperand(3));
16465       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
16466       return;
16467     }
16468     case Intrinsic::aarch64_sve_clastb_n: {
16469       SDLoc DL(N);
16470       auto Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, N->getOperand(2));
16471       auto V = DAG.getNode(AArch64ISD::CLASTB_N, DL, MVT::i32,
16472                            N->getOperand(1), Op2, N->getOperand(3));
16473       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
16474       return;
16475     }
16476     case Intrinsic::aarch64_sve_lasta: {
16477       SDLoc DL(N);
16478       auto V = DAG.getNode(AArch64ISD::LASTA, DL, MVT::i32,
16479                            N->getOperand(1), N->getOperand(2));
16480       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
16481       return;
16482     }
16483     case Intrinsic::aarch64_sve_lastb: {
16484       SDLoc DL(N);
16485       auto V = DAG.getNode(AArch64ISD::LASTB, DL, MVT::i32,
16486                            N->getOperand(1), N->getOperand(2));
16487       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
16488       return;
16489     }
16490     }
16491   }
16492   }
16493 }
16494 
16495 bool AArch64TargetLowering::useLoadStackGuardNode() const {
16496   if (Subtarget->isTargetAndroid() || Subtarget->isTargetFuchsia())
16497     return TargetLowering::useLoadStackGuardNode();
16498   return true;
16499 }
16500 
16501 unsigned AArch64TargetLowering::combineRepeatedFPDivisors() const {
16502   // Combine multiple FDIVs with the same divisor into multiple FMULs by the
16503   // reciprocal if there are three or more FDIVs.
16504   return 3;
16505 }
16506 
16507 TargetLoweringBase::LegalizeTypeAction
16508 AArch64TargetLowering::getPreferredVectorAction(MVT VT) const {
16509   // During type legalization, we prefer to widen v1i8, v1i16, v1i32  to v8i8,
16510   // v4i16, v2i32 instead of to promote.
16511   if (VT == MVT::v1i8 || VT == MVT::v1i16 || VT == MVT::v1i32 ||
16512       VT == MVT::v1f32)
16513     return TypeWidenVector;
16514 
16515   return TargetLoweringBase::getPreferredVectorAction(VT);
16516 }
16517 
16518 // Loads and stores less than 128-bits are already atomic; ones above that
16519 // are doomed anyway, so defer to the default libcall and blame the OS when
16520 // things go wrong.
16521 bool AArch64TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
16522   unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
16523   return Size == 128;
16524 }
16525 
16526 // Loads and stores less than 128-bits are already atomic; ones above that
16527 // are doomed anyway, so defer to the default libcall and blame the OS when
16528 // things go wrong.
16529 TargetLowering::AtomicExpansionKind
16530 AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
16531   unsigned Size = LI->getType()->getPrimitiveSizeInBits();
16532   return Size == 128 ? AtomicExpansionKind::LLSC : AtomicExpansionKind::None;
16533 }
16534 
16535 // For the real atomic operations, we have ldxr/stxr up to 128 bits,
16536 TargetLowering::AtomicExpansionKind
16537 AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
16538   if (AI->isFloatingPointOperation())
16539     return AtomicExpansionKind::CmpXChg;
16540 
16541   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
16542   if (Size > 128) return AtomicExpansionKind::None;
16543   // Nand not supported in LSE.
16544   if (AI->getOperation() == AtomicRMWInst::Nand) return AtomicExpansionKind::LLSC;
16545   // Leave 128 bits to LLSC.
16546   if (Subtarget->hasLSE() && Size < 128)
16547     return AtomicExpansionKind::None;
16548   if (Subtarget->outlineAtomics() && Size < 128) {
16549     // [U]Min/[U]Max RWM atomics are used in __sync_fetch_ libcalls so far.
16550     // Don't outline them unless
16551     // (1) high level <atomic> support approved:
16552     //   http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p0493r1.pdf
16553     // (2) low level libgcc and compiler-rt support implemented by:
16554     //   min/max outline atomics helpers
16555     if (AI->getOperation() != AtomicRMWInst::Min &&
16556         AI->getOperation() != AtomicRMWInst::Max &&
16557         AI->getOperation() != AtomicRMWInst::UMin &&
16558         AI->getOperation() != AtomicRMWInst::UMax) {
16559       return AtomicExpansionKind::None;
16560     }
16561   }
16562   return AtomicExpansionKind::LLSC;
16563 }
16564 
16565 TargetLowering::AtomicExpansionKind
16566 AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR(
16567     AtomicCmpXchgInst *AI) const {
16568   // If subtarget has LSE, leave cmpxchg intact for codegen.
16569   if (Subtarget->hasLSE() || Subtarget->outlineAtomics())
16570     return AtomicExpansionKind::None;
16571   // At -O0, fast-regalloc cannot cope with the live vregs necessary to
16572   // implement cmpxchg without spilling. If the address being exchanged is also
16573   // on the stack and close enough to the spill slot, this can lead to a
16574   // situation where the monitor always gets cleared and the atomic operation
16575   // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
16576   if (getTargetMachine().getOptLevel() == CodeGenOpt::None)
16577     return AtomicExpansionKind::None;
16578   return AtomicExpansionKind::LLSC;
16579 }
16580 
16581 Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
16582                                              AtomicOrdering Ord) const {
16583   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16584   Type *ValTy = cast<PointerType>(Addr->getType())->getElementType();
16585   bool IsAcquire = isAcquireOrStronger(Ord);
16586 
16587   // Since i128 isn't legal and intrinsics don't get type-lowered, the ldrexd
16588   // intrinsic must return {i64, i64} and we have to recombine them into a
16589   // single i128 here.
16590   if (ValTy->getPrimitiveSizeInBits() == 128) {
16591     Intrinsic::ID Int =
16592         IsAcquire ? Intrinsic::aarch64_ldaxp : Intrinsic::aarch64_ldxp;
16593     Function *Ldxr = Intrinsic::getDeclaration(M, Int);
16594 
16595     Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
16596     Value *LoHi = Builder.CreateCall(Ldxr, Addr, "lohi");
16597 
16598     Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
16599     Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
16600     Lo = Builder.CreateZExt(Lo, ValTy, "lo64");
16601     Hi = Builder.CreateZExt(Hi, ValTy, "hi64");
16602     return Builder.CreateOr(
16603         Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 64)), "val64");
16604   }
16605 
16606   Type *Tys[] = { Addr->getType() };
16607   Intrinsic::ID Int =
16608       IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
16609   Function *Ldxr = Intrinsic::getDeclaration(M, Int, Tys);
16610 
16611   Type *EltTy = cast<PointerType>(Addr->getType())->getElementType();
16612 
16613   const DataLayout &DL = M->getDataLayout();
16614   IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(EltTy));
16615   Value *Trunc = Builder.CreateTrunc(Builder.CreateCall(Ldxr, Addr), IntEltTy);
16616 
16617   return Builder.CreateBitCast(Trunc, EltTy);
16618 }
16619 
16620 void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
16621     IRBuilder<> &Builder) const {
16622   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16623   Builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::aarch64_clrex));
16624 }
16625 
16626 Value *AArch64TargetLowering::emitStoreConditional(IRBuilder<> &Builder,
16627                                                    Value *Val, Value *Addr,
16628                                                    AtomicOrdering Ord) const {
16629   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16630   bool IsRelease = isReleaseOrStronger(Ord);
16631 
16632   // Since the intrinsics must have legal type, the i128 intrinsics take two
16633   // parameters: "i64, i64". We must marshal Val into the appropriate form
16634   // before the call.
16635   if (Val->getType()->getPrimitiveSizeInBits() == 128) {
16636     Intrinsic::ID Int =
16637         IsRelease ? Intrinsic::aarch64_stlxp : Intrinsic::aarch64_stxp;
16638     Function *Stxr = Intrinsic::getDeclaration(M, Int);
16639     Type *Int64Ty = Type::getInt64Ty(M->getContext());
16640 
16641     Value *Lo = Builder.CreateTrunc(Val, Int64Ty, "lo");
16642     Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 64), Int64Ty, "hi");
16643     Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
16644     return Builder.CreateCall(Stxr, {Lo, Hi, Addr});
16645   }
16646 
16647   Intrinsic::ID Int =
16648       IsRelease ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr;
16649   Type *Tys[] = { Addr->getType() };
16650   Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys);
16651 
16652   const DataLayout &DL = M->getDataLayout();
16653   IntegerType *IntValTy = Builder.getIntNTy(DL.getTypeSizeInBits(Val->getType()));
16654   Val = Builder.CreateBitCast(Val, IntValTy);
16655 
16656   return Builder.CreateCall(Stxr,
16657                             {Builder.CreateZExtOrBitCast(
16658                                  Val, Stxr->getFunctionType()->getParamType(0)),
16659                              Addr});
16660 }
16661 
16662 bool AArch64TargetLowering::functionArgumentNeedsConsecutiveRegisters(
16663     Type *Ty, CallingConv::ID CallConv, bool isVarArg) const {
16664   if (Ty->isArrayTy())
16665     return true;
16666 
16667   const TypeSize &TySize = Ty->getPrimitiveSizeInBits();
16668   if (TySize.isScalable() && TySize.getKnownMinSize() > 128)
16669     return true;
16670 
16671   return false;
16672 }
16673 
16674 bool AArch64TargetLowering::shouldNormalizeToSelectSequence(LLVMContext &,
16675                                                             EVT) const {
16676   return false;
16677 }
16678 
16679 static Value *UseTlsOffset(IRBuilder<> &IRB, unsigned Offset) {
16680   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
16681   Function *ThreadPointerFunc =
16682       Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
16683   return IRB.CreatePointerCast(
16684       IRB.CreateConstGEP1_32(IRB.getInt8Ty(), IRB.CreateCall(ThreadPointerFunc),
16685                              Offset),
16686       IRB.getInt8PtrTy()->getPointerTo(0));
16687 }
16688 
16689 Value *AArch64TargetLowering::getIRStackGuard(IRBuilder<> &IRB) const {
16690   // Android provides a fixed TLS slot for the stack cookie. See the definition
16691   // of TLS_SLOT_STACK_GUARD in
16692   // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h
16693   if (Subtarget->isTargetAndroid())
16694     return UseTlsOffset(IRB, 0x28);
16695 
16696   // Fuchsia is similar.
16697   // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
16698   if (Subtarget->isTargetFuchsia())
16699     return UseTlsOffset(IRB, -0x10);
16700 
16701   return TargetLowering::getIRStackGuard(IRB);
16702 }
16703 
16704 void AArch64TargetLowering::insertSSPDeclarations(Module &M) const {
16705   // MSVC CRT provides functionalities for stack protection.
16706   if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment()) {
16707     // MSVC CRT has a global variable holding security cookie.
16708     M.getOrInsertGlobal("__security_cookie",
16709                         Type::getInt8PtrTy(M.getContext()));
16710 
16711     // MSVC CRT has a function to validate security cookie.
16712     FunctionCallee SecurityCheckCookie = M.getOrInsertFunction(
16713         "__security_check_cookie", Type::getVoidTy(M.getContext()),
16714         Type::getInt8PtrTy(M.getContext()));
16715     if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee())) {
16716       F->setCallingConv(CallingConv::Win64);
16717       F->addAttribute(1, Attribute::AttrKind::InReg);
16718     }
16719     return;
16720   }
16721   TargetLowering::insertSSPDeclarations(M);
16722 }
16723 
16724 Value *AArch64TargetLowering::getSDagStackGuard(const Module &M) const {
16725   // MSVC CRT has a global variable holding security cookie.
16726   if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
16727     return M.getGlobalVariable("__security_cookie");
16728   return TargetLowering::getSDagStackGuard(M);
16729 }
16730 
16731 Function *AArch64TargetLowering::getSSPStackGuardCheck(const Module &M) const {
16732   // MSVC CRT has a function to validate security cookie.
16733   if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
16734     return M.getFunction("__security_check_cookie");
16735   return TargetLowering::getSSPStackGuardCheck(M);
16736 }
16737 
16738 Value *AArch64TargetLowering::getSafeStackPointerLocation(IRBuilder<> &IRB) const {
16739   // Android provides a fixed TLS slot for the SafeStack pointer. See the
16740   // definition of TLS_SLOT_SAFESTACK in
16741   // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h
16742   if (Subtarget->isTargetAndroid())
16743     return UseTlsOffset(IRB, 0x48);
16744 
16745   // Fuchsia is similar.
16746   // <zircon/tls.h> defines ZX_TLS_UNSAFE_SP_OFFSET with this value.
16747   if (Subtarget->isTargetFuchsia())
16748     return UseTlsOffset(IRB, -0x8);
16749 
16750   return TargetLowering::getSafeStackPointerLocation(IRB);
16751 }
16752 
16753 bool AArch64TargetLowering::isMaskAndCmp0FoldingBeneficial(
16754     const Instruction &AndI) const {
16755   // Only sink 'and' mask to cmp use block if it is masking a single bit, since
16756   // this is likely to be fold the and/cmp/br into a single tbz instruction.  It
16757   // may be beneficial to sink in other cases, but we would have to check that
16758   // the cmp would not get folded into the br to form a cbz for these to be
16759   // beneficial.
16760   ConstantInt* Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
16761   if (!Mask)
16762     return false;
16763   return Mask->getValue().isPowerOf2();
16764 }
16765 
16766 bool AArch64TargetLowering::
16767     shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
16768         SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
16769         unsigned OldShiftOpcode, unsigned NewShiftOpcode,
16770         SelectionDAG &DAG) const {
16771   // Does baseline recommend not to perform the fold by default?
16772   if (!TargetLowering::shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
16773           X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG))
16774     return false;
16775   // Else, if this is a vector shift, prefer 'shl'.
16776   return X.getValueType().isScalarInteger() || NewShiftOpcode == ISD::SHL;
16777 }
16778 
16779 bool AArch64TargetLowering::shouldExpandShift(SelectionDAG &DAG,
16780                                               SDNode *N) const {
16781   if (DAG.getMachineFunction().getFunction().hasMinSize() &&
16782       !Subtarget->isTargetWindows() && !Subtarget->isTargetDarwin())
16783     return false;
16784   return true;
16785 }
16786 
16787 void AArch64TargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
16788   // Update IsSplitCSR in AArch64unctionInfo.
16789   AArch64FunctionInfo *AFI = Entry->getParent()->getInfo<AArch64FunctionInfo>();
16790   AFI->setIsSplitCSR(true);
16791 }
16792 
16793 void AArch64TargetLowering::insertCopiesSplitCSR(
16794     MachineBasicBlock *Entry,
16795     const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
16796   const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
16797   const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
16798   if (!IStart)
16799     return;
16800 
16801   const TargetInstrInfo *TII = Subtarget->getInstrInfo();
16802   MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
16803   MachineBasicBlock::iterator MBBI = Entry->begin();
16804   for (const MCPhysReg *I = IStart; *I; ++I) {
16805     const TargetRegisterClass *RC = nullptr;
16806     if (AArch64::GPR64RegClass.contains(*I))
16807       RC = &AArch64::GPR64RegClass;
16808     else if (AArch64::FPR64RegClass.contains(*I))
16809       RC = &AArch64::FPR64RegClass;
16810     else
16811       llvm_unreachable("Unexpected register class in CSRsViaCopy!");
16812 
16813     Register NewVR = MRI->createVirtualRegister(RC);
16814     // Create copy from CSR to a virtual register.
16815     // FIXME: this currently does not emit CFI pseudo-instructions, it works
16816     // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
16817     // nounwind. If we want to generalize this later, we may need to emit
16818     // CFI pseudo-instructions.
16819     assert(Entry->getParent()->getFunction().hasFnAttribute(
16820                Attribute::NoUnwind) &&
16821            "Function should be nounwind in insertCopiesSplitCSR!");
16822     Entry->addLiveIn(*I);
16823     BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
16824         .addReg(*I);
16825 
16826     // Insert the copy-back instructions right before the terminator.
16827     for (auto *Exit : Exits)
16828       BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
16829               TII->get(TargetOpcode::COPY), *I)
16830           .addReg(NewVR);
16831   }
16832 }
16833 
16834 bool AArch64TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
16835   // Integer division on AArch64 is expensive. However, when aggressively
16836   // optimizing for code size, we prefer to use a div instruction, as it is
16837   // usually smaller than the alternative sequence.
16838   // The exception to this is vector division. Since AArch64 doesn't have vector
16839   // integer division, leaving the division as-is is a loss even in terms of
16840   // size, because it will have to be scalarized, while the alternative code
16841   // sequence can be performed in vector form.
16842   bool OptSize = Attr.hasFnAttribute(Attribute::MinSize);
16843   return OptSize && !VT.isVector();
16844 }
16845 
16846 bool AArch64TargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
16847   // We want inc-of-add for scalars and sub-of-not for vectors.
16848   return VT.isScalarInteger();
16849 }
16850 
16851 bool AArch64TargetLowering::enableAggressiveFMAFusion(EVT VT) const {
16852   return Subtarget->hasAggressiveFMA() && VT.isFloatingPoint();
16853 }
16854 
16855 unsigned
16856 AArch64TargetLowering::getVaListSizeInBits(const DataLayout &DL) const {
16857   if (Subtarget->isTargetDarwin() || Subtarget->isTargetWindows())
16858     return getPointerTy(DL).getSizeInBits();
16859 
16860   return 3 * getPointerTy(DL).getSizeInBits() + 2 * 32;
16861 }
16862 
16863 void AArch64TargetLowering::finalizeLowering(MachineFunction &MF) const {
16864   MF.getFrameInfo().computeMaxCallFrameSize(MF);
16865   TargetLoweringBase::finalizeLowering(MF);
16866 }
16867 
16868 // Unlike X86, we let frame lowering assign offsets to all catch objects.
16869 bool AArch64TargetLowering::needsFixedCatchObjects() const {
16870   return false;
16871 }
16872 
16873 bool AArch64TargetLowering::shouldLocalize(
16874     const MachineInstr &MI, const TargetTransformInfo *TTI) const {
16875   switch (MI.getOpcode()) {
16876   case TargetOpcode::G_GLOBAL_VALUE: {
16877     // On Darwin, TLS global vars get selected into function calls, which
16878     // we don't want localized, as they can get moved into the middle of a
16879     // another call sequence.
16880     const GlobalValue &GV = *MI.getOperand(1).getGlobal();
16881     if (GV.isThreadLocal() && Subtarget->isTargetMachO())
16882       return false;
16883     break;
16884   }
16885   // If we legalized G_GLOBAL_VALUE into ADRP + G_ADD_LOW, mark both as being
16886   // localizable.
16887   case AArch64::ADRP:
16888   case AArch64::G_ADD_LOW:
16889     return true;
16890   default:
16891     break;
16892   }
16893   return TargetLoweringBase::shouldLocalize(MI, TTI);
16894 }
16895 
16896 bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
16897   if (isa<ScalableVectorType>(Inst.getType()))
16898     return true;
16899 
16900   for (unsigned i = 0; i < Inst.getNumOperands(); ++i)
16901     if (isa<ScalableVectorType>(Inst.getOperand(i)->getType()))
16902       return true;
16903 
16904   if (const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
16905     if (isa<ScalableVectorType>(AI->getAllocatedType()))
16906       return true;
16907   }
16908 
16909   return false;
16910 }
16911 
16912 // Return the largest legal scalable vector type that matches VT's element type.
16913 static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT) {
16914   assert(VT.isFixedLengthVector() &&
16915          DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
16916          "Expected legal fixed length vector!");
16917   switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
16918   default:
16919     llvm_unreachable("unexpected element type for SVE container");
16920   case MVT::i8:
16921     return EVT(MVT::nxv16i8);
16922   case MVT::i16:
16923     return EVT(MVT::nxv8i16);
16924   case MVT::i32:
16925     return EVT(MVT::nxv4i32);
16926   case MVT::i64:
16927     return EVT(MVT::nxv2i64);
16928   case MVT::f16:
16929     return EVT(MVT::nxv8f16);
16930   case MVT::f32:
16931     return EVT(MVT::nxv4f32);
16932   case MVT::f64:
16933     return EVT(MVT::nxv2f64);
16934   }
16935 }
16936 
16937 // Return a PTRUE with active lanes corresponding to the extent of VT.
16938 static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
16939                                                 EVT VT) {
16940   assert(VT.isFixedLengthVector() &&
16941          DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
16942          "Expected legal fixed length vector!");
16943 
16944   int PgPattern;
16945   switch (VT.getVectorNumElements()) {
16946   default:
16947     llvm_unreachable("unexpected element count for SVE predicate");
16948   case 1:
16949     PgPattern = AArch64SVEPredPattern::vl1;
16950     break;
16951   case 2:
16952     PgPattern = AArch64SVEPredPattern::vl2;
16953     break;
16954   case 4:
16955     PgPattern = AArch64SVEPredPattern::vl4;
16956     break;
16957   case 8:
16958     PgPattern = AArch64SVEPredPattern::vl8;
16959     break;
16960   case 16:
16961     PgPattern = AArch64SVEPredPattern::vl16;
16962     break;
16963   case 32:
16964     PgPattern = AArch64SVEPredPattern::vl32;
16965     break;
16966   case 64:
16967     PgPattern = AArch64SVEPredPattern::vl64;
16968     break;
16969   case 128:
16970     PgPattern = AArch64SVEPredPattern::vl128;
16971     break;
16972   case 256:
16973     PgPattern = AArch64SVEPredPattern::vl256;
16974     break;
16975   }
16976 
16977   // TODO: For vectors that are exactly getMaxSVEVectorSizeInBits big, we can
16978   // use AArch64SVEPredPattern::all, which can enable the use of unpredicated
16979   // variants of instructions when available.
16980 
16981   MVT MaskVT;
16982   switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
16983   default:
16984     llvm_unreachable("unexpected element type for SVE predicate");
16985   case MVT::i8:
16986     MaskVT = MVT::nxv16i1;
16987     break;
16988   case MVT::i16:
16989   case MVT::f16:
16990     MaskVT = MVT::nxv8i1;
16991     break;
16992   case MVT::i32:
16993   case MVT::f32:
16994     MaskVT = MVT::nxv4i1;
16995     break;
16996   case MVT::i64:
16997   case MVT::f64:
16998     MaskVT = MVT::nxv2i1;
16999     break;
17000   }
17001 
17002   return DAG.getNode(AArch64ISD::PTRUE, DL, MaskVT,
17003                      DAG.getTargetConstant(PgPattern, DL, MVT::i64));
17004 }
17005 
17006 static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
17007                                              EVT VT) {
17008   assert(VT.isScalableVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
17009          "Expected legal scalable vector!");
17010   auto PredTy = VT.changeVectorElementType(MVT::i1);
17011   return getPTrue(DAG, DL, PredTy, AArch64SVEPredPattern::all);
17012 }
17013 
17014 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT) {
17015   if (VT.isFixedLengthVector())
17016     return getPredicateForFixedLengthVector(DAG, DL, VT);
17017 
17018   return getPredicateForScalableVector(DAG, DL, VT);
17019 }
17020 
17021 // Grow V to consume an entire SVE register.
17022 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
17023   assert(VT.isScalableVector() &&
17024          "Expected to convert into a scalable vector!");
17025   assert(V.getValueType().isFixedLengthVector() &&
17026          "Expected a fixed length vector operand!");
17027   SDLoc DL(V);
17028   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
17029   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
17030 }
17031 
17032 // Shrink V so it's just big enough to maintain a VT's worth of data.
17033 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
17034   assert(VT.isFixedLengthVector() &&
17035          "Expected to convert into a fixed length vector!");
17036   assert(V.getValueType().isScalableVector() &&
17037          "Expected a scalable vector operand!");
17038   SDLoc DL(V);
17039   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
17040   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
17041 }
17042 
17043 // Convert all fixed length vector loads larger than NEON to masked_loads.
17044 SDValue AArch64TargetLowering::LowerFixedLengthVectorLoadToSVE(
17045     SDValue Op, SelectionDAG &DAG) const {
17046   auto Load = cast<LoadSDNode>(Op);
17047 
17048   SDLoc DL(Op);
17049   EVT VT = Op.getValueType();
17050   EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
17051 
17052   auto NewLoad = DAG.getMaskedLoad(
17053       ContainerVT, DL, Load->getChain(), Load->getBasePtr(), Load->getOffset(),
17054       getPredicateForFixedLengthVector(DAG, DL, VT), DAG.getUNDEF(ContainerVT),
17055       Load->getMemoryVT(), Load->getMemOperand(), Load->getAddressingMode(),
17056       Load->getExtensionType());
17057 
17058   auto Result = convertFromScalableVector(DAG, VT, NewLoad);
17059   SDValue MergedValues[2] = {Result, Load->getChain()};
17060   return DAG.getMergeValues(MergedValues, DL);
17061 }
17062 
17063 // Convert all fixed length vector stores larger than NEON to masked_stores.
17064 SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
17065     SDValue Op, SelectionDAG &DAG) const {
17066   auto Store = cast<StoreSDNode>(Op);
17067 
17068   SDLoc DL(Op);
17069   EVT VT = Store->getValue().getValueType();
17070   EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
17071 
17072   auto NewValue = convertToScalableVector(DAG, ContainerVT, Store->getValue());
17073   return DAG.getMaskedStore(
17074       Store->getChain(), DL, NewValue, Store->getBasePtr(), Store->getOffset(),
17075       getPredicateForFixedLengthVector(DAG, DL, VT), Store->getMemoryVT(),
17076       Store->getMemOperand(), Store->getAddressingMode(),
17077       Store->isTruncatingStore());
17078 }
17079 
17080 SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
17081     SDValue Op, SelectionDAG &DAG) const {
17082   SDLoc dl(Op);
17083   EVT VT = Op.getValueType();
17084   EVT EltVT = VT.getVectorElementType();
17085 
17086   bool Signed = Op.getOpcode() == ISD::SDIV;
17087   unsigned PredOpcode = Signed ? AArch64ISD::SDIV_PRED : AArch64ISD::UDIV_PRED;
17088 
17089   // Scalable vector i32/i64 DIV is supported.
17090   if (EltVT == MVT::i32 || EltVT == MVT::i64)
17091     return LowerToPredicatedOp(Op, DAG, PredOpcode, /*OverrideNEON=*/true);
17092 
17093   // Scalable vector i8/i16 DIV is not supported. Promote it to i32.
17094   EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
17095   EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
17096   EVT FixedWidenedVT = HalfVT.widenIntegerVectorElementType(*DAG.getContext());
17097   EVT ScalableWidenedVT = getContainerForFixedLengthVector(DAG, FixedWidenedVT);
17098 
17099   // Convert the operands to scalable vectors.
17100   SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(0));
17101   SDValue Op1 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(1));
17102 
17103   // Extend the scalable operands.
17104   unsigned UnpkLo = Signed ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO;
17105   unsigned UnpkHi = Signed ? AArch64ISD::SUNPKHI : AArch64ISD::UUNPKHI;
17106   SDValue Op0Lo = DAG.getNode(UnpkLo, dl, ScalableWidenedVT, Op0);
17107   SDValue Op1Lo = DAG.getNode(UnpkLo, dl, ScalableWidenedVT, Op1);
17108   SDValue Op0Hi = DAG.getNode(UnpkHi, dl, ScalableWidenedVT, Op0);
17109   SDValue Op1Hi = DAG.getNode(UnpkHi, dl, ScalableWidenedVT, Op1);
17110 
17111   // Convert back to fixed vectors so the DIV can be further lowered.
17112   Op0Lo = convertFromScalableVector(DAG, FixedWidenedVT, Op0Lo);
17113   Op1Lo = convertFromScalableVector(DAG, FixedWidenedVT, Op1Lo);
17114   Op0Hi = convertFromScalableVector(DAG, FixedWidenedVT, Op0Hi);
17115   Op1Hi = convertFromScalableVector(DAG, FixedWidenedVT, Op1Hi);
17116   SDValue ResultLo = DAG.getNode(Op.getOpcode(), dl, FixedWidenedVT,
17117                                  Op0Lo, Op1Lo);
17118   SDValue ResultHi = DAG.getNode(Op.getOpcode(), dl, FixedWidenedVT,
17119                                  Op0Hi, Op1Hi);
17120 
17121   // Convert again to scalable vectors to truncate.
17122   ResultLo = convertToScalableVector(DAG, ScalableWidenedVT, ResultLo);
17123   ResultHi = convertToScalableVector(DAG, ScalableWidenedVT, ResultHi);
17124   SDValue ScalableResult = DAG.getNode(AArch64ISD::UZP1, dl, ContainerVT,
17125                                        ResultLo, ResultHi);
17126 
17127   return convertFromScalableVector(DAG, VT, ScalableResult);
17128 }
17129 
17130 SDValue AArch64TargetLowering::LowerFixedLengthVectorIntExtendToSVE(
17131     SDValue Op, SelectionDAG &DAG) const {
17132   EVT VT = Op.getValueType();
17133   assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
17134 
17135   SDLoc DL(Op);
17136   SDValue Val = Op.getOperand(0);
17137   EVT ContainerVT = getContainerForFixedLengthVector(DAG, Val.getValueType());
17138   Val = convertToScalableVector(DAG, ContainerVT, Val);
17139 
17140   bool Signed = Op.getOpcode() == ISD::SIGN_EXTEND;
17141   unsigned ExtendOpc = Signed ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO;
17142 
17143   // Repeatedly unpack Val until the result is of the desired element type.
17144   switch (ContainerVT.getSimpleVT().SimpleTy) {
17145   default:
17146     llvm_unreachable("unimplemented container type");
17147   case MVT::nxv16i8:
17148     Val = DAG.getNode(ExtendOpc, DL, MVT::nxv8i16, Val);
17149     if (VT.getVectorElementType() == MVT::i16)
17150       break;
17151     LLVM_FALLTHROUGH;
17152   case MVT::nxv8i16:
17153     Val = DAG.getNode(ExtendOpc, DL, MVT::nxv4i32, Val);
17154     if (VT.getVectorElementType() == MVT::i32)
17155       break;
17156     LLVM_FALLTHROUGH;
17157   case MVT::nxv4i32:
17158     Val = DAG.getNode(ExtendOpc, DL, MVT::nxv2i64, Val);
17159     assert(VT.getVectorElementType() == MVT::i64 && "Unexpected element type!");
17160     break;
17161   }
17162 
17163   return convertFromScalableVector(DAG, VT, Val);
17164 }
17165 
17166 SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
17167     SDValue Op, SelectionDAG &DAG) const {
17168   EVT VT = Op.getValueType();
17169   assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
17170 
17171   SDLoc DL(Op);
17172   SDValue Val = Op.getOperand(0);
17173   EVT ContainerVT = getContainerForFixedLengthVector(DAG, Val.getValueType());
17174   Val = convertToScalableVector(DAG, ContainerVT, Val);
17175 
17176   // Repeatedly truncate Val until the result is of the desired element type.
17177   switch (ContainerVT.getSimpleVT().SimpleTy) {
17178   default:
17179     llvm_unreachable("unimplemented container type");
17180   case MVT::nxv2i64:
17181     Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv4i32, Val);
17182     Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv4i32, Val, Val);
17183     if (VT.getVectorElementType() == MVT::i32)
17184       break;
17185     LLVM_FALLTHROUGH;
17186   case MVT::nxv4i32:
17187     Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv8i16, Val);
17188     Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv8i16, Val, Val);
17189     if (VT.getVectorElementType() == MVT::i16)
17190       break;
17191     LLVM_FALLTHROUGH;
17192   case MVT::nxv8i16:
17193     Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, Val);
17194     Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv16i8, Val, Val);
17195     assert(VT.getVectorElementType() == MVT::i8 && "Unexpected element type!");
17196     break;
17197   }
17198 
17199   return convertFromScalableVector(DAG, VT, Val);
17200 }
17201 
17202 SDValue AArch64TargetLowering::LowerFixedLengthExtractVectorElt(
17203     SDValue Op, SelectionDAG &DAG) const {
17204   EVT VT = Op.getValueType();
17205   EVT InVT = Op.getOperand(0).getValueType();
17206   assert(InVT.isFixedLengthVector() && "Expected fixed length vector type!");
17207 
17208   SDLoc DL(Op);
17209   EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
17210   SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(0));
17211 
17212   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Op.getOperand(1));
17213 }
17214 
17215 SDValue AArch64TargetLowering::LowerFixedLengthInsertVectorElt(
17216     SDValue Op, SelectionDAG &DAG) const {
17217   EVT VT = Op.getValueType();
17218   assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
17219 
17220   SDLoc DL(Op);
17221   EVT InVT = Op.getOperand(0).getValueType();
17222   EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
17223   SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(0));
17224 
17225   auto ScalableRes = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT, Op0,
17226                                  Op.getOperand(1), Op.getOperand(2));
17227 
17228   return convertFromScalableVector(DAG, VT, ScalableRes);
17229 }
17230 
17231 // Convert vector operation 'Op' to an equivalent predicated operation whereby
17232 // the original operation's type is used to construct a suitable predicate.
17233 // NOTE: The results for inactive lanes are undefined.
17234 SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
17235                                                    SelectionDAG &DAG,
17236                                                    unsigned NewOp,
17237                                                    bool OverrideNEON) const {
17238   EVT VT = Op.getValueType();
17239   SDLoc DL(Op);
17240   auto Pg = getPredicateForVector(DAG, DL, VT);
17241 
17242   if (useSVEForFixedLengthVectorVT(VT, OverrideNEON)) {
17243     EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
17244 
17245     // Create list of operands by converting existing ones to scalable types.
17246     SmallVector<SDValue, 4> Operands = {Pg};
17247     for (const SDValue &V : Op->op_values()) {
17248       if (isa<CondCodeSDNode>(V)) {
17249         Operands.push_back(V);
17250         continue;
17251       }
17252 
17253       if (const VTSDNode *VTNode = dyn_cast<VTSDNode>(V)) {
17254         EVT VTArg = VTNode->getVT().getVectorElementType();
17255         EVT NewVTArg = ContainerVT.changeVectorElementType(VTArg);
17256         Operands.push_back(DAG.getValueType(NewVTArg));
17257         continue;
17258       }
17259 
17260       assert(useSVEForFixedLengthVectorVT(V.getValueType(), OverrideNEON) &&
17261              "Only fixed length vectors are supported!");
17262       Operands.push_back(convertToScalableVector(DAG, ContainerVT, V));
17263     }
17264 
17265     if (isMergePassthruOpcode(NewOp))
17266       Operands.push_back(DAG.getUNDEF(ContainerVT));
17267 
17268     auto ScalableRes = DAG.getNode(NewOp, DL, ContainerVT, Operands);
17269     return convertFromScalableVector(DAG, VT, ScalableRes);
17270   }
17271 
17272   assert(VT.isScalableVector() && "Only expect to lower scalable vector op!");
17273 
17274   SmallVector<SDValue, 4> Operands = {Pg};
17275   for (const SDValue &V : Op->op_values()) {
17276     assert((!V.getValueType().isVector() ||
17277             V.getValueType().isScalableVector()) &&
17278            "Only scalable vectors are supported!");
17279     Operands.push_back(V);
17280   }
17281 
17282   if (isMergePassthruOpcode(NewOp))
17283     Operands.push_back(DAG.getUNDEF(VT));
17284 
17285   return DAG.getNode(NewOp, DL, VT, Operands);
17286 }
17287 
17288 // If a fixed length vector operation has no side effects when applied to
17289 // undefined elements, we can safely use scalable vectors to perform the same
17290 // operation without needing to worry about predication.
17291 SDValue AArch64TargetLowering::LowerToScalableOp(SDValue Op,
17292                                                  SelectionDAG &DAG) const {
17293   EVT VT = Op.getValueType();
17294   assert(useSVEForFixedLengthVectorVT(VT) &&
17295          "Only expected to lower fixed length vector operation!");
17296   EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
17297 
17298   // Create list of operands by converting existing ones to scalable types.
17299   SmallVector<SDValue, 4> Ops;
17300   for (const SDValue &V : Op->op_values()) {
17301     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
17302 
17303     // Pass through non-vector operands.
17304     if (!V.getValueType().isVector()) {
17305       Ops.push_back(V);
17306       continue;
17307     }
17308 
17309     // "cast" fixed length vector to a scalable vector.
17310     assert(useSVEForFixedLengthVectorVT(V.getValueType()) &&
17311            "Only fixed length vectors are supported!");
17312     Ops.push_back(convertToScalableVector(DAG, ContainerVT, V));
17313   }
17314 
17315   auto ScalableRes = DAG.getNode(Op.getOpcode(), SDLoc(Op), ContainerVT, Ops);
17316   return convertFromScalableVector(DAG, VT, ScalableRes);
17317 }
17318 
17319 SDValue AArch64TargetLowering::LowerVECREDUCE_SEQ_FADD(SDValue ScalarOp,
17320     SelectionDAG &DAG) const {
17321   SDLoc DL(ScalarOp);
17322   SDValue AccOp = ScalarOp.getOperand(0);
17323   SDValue VecOp = ScalarOp.getOperand(1);
17324   EVT SrcVT = VecOp.getValueType();
17325   EVT ResVT = SrcVT.getVectorElementType();
17326 
17327   EVT ContainerVT = SrcVT;
17328   if (SrcVT.isFixedLengthVector()) {
17329     ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT);
17330     VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
17331   }
17332 
17333   SDValue Pg = getPredicateForVector(DAG, DL, SrcVT);
17334   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
17335 
17336   // Convert operands to Scalable.
17337   AccOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT,
17338                       DAG.getUNDEF(ContainerVT), AccOp, Zero);
17339 
17340   // Perform reduction.
17341   SDValue Rdx = DAG.getNode(AArch64ISD::FADDA_PRED, DL, ContainerVT,
17342                             Pg, AccOp, VecOp);
17343 
17344   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Rdx, Zero);
17345 }
17346 
17347 SDValue AArch64TargetLowering::LowerPredReductionToSVE(SDValue ReduceOp,
17348                                                        SelectionDAG &DAG) const {
17349   SDLoc DL(ReduceOp);
17350   SDValue Op = ReduceOp.getOperand(0);
17351   EVT OpVT = Op.getValueType();
17352   EVT VT = ReduceOp.getValueType();
17353 
17354   if (!OpVT.isScalableVector() || OpVT.getVectorElementType() != MVT::i1)
17355     return SDValue();
17356 
17357   SDValue Pg = getPredicateForVector(DAG, DL, OpVT);
17358 
17359   switch (ReduceOp.getOpcode()) {
17360   default:
17361     return SDValue();
17362   case ISD::VECREDUCE_OR:
17363     return getPTest(DAG, VT, Pg, Op, AArch64CC::ANY_ACTIVE);
17364   case ISD::VECREDUCE_AND: {
17365     Op = DAG.getNode(ISD::XOR, DL, OpVT, Op, Pg);
17366     return getPTest(DAG, VT, Pg, Op, AArch64CC::NONE_ACTIVE);
17367   }
17368   case ISD::VECREDUCE_XOR: {
17369     SDValue ID =
17370         DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64);
17371     SDValue Cntp =
17372         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, ID, Pg, Op);
17373     return DAG.getAnyExtOrTrunc(Cntp, DL, VT);
17374   }
17375   }
17376 
17377   return SDValue();
17378 }
17379 
17380 SDValue AArch64TargetLowering::LowerReductionToSVE(unsigned Opcode,
17381                                                    SDValue ScalarOp,
17382                                                    SelectionDAG &DAG) const {
17383   SDLoc DL(ScalarOp);
17384   SDValue VecOp = ScalarOp.getOperand(0);
17385   EVT SrcVT = VecOp.getValueType();
17386 
17387   if (useSVEForFixedLengthVectorVT(SrcVT, true)) {
17388     EVT ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT);
17389     VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
17390   }
17391 
17392   // UADDV always returns an i64 result.
17393   EVT ResVT = (Opcode == AArch64ISD::UADDV_PRED) ? MVT::i64 :
17394                                                    SrcVT.getVectorElementType();
17395   EVT RdxVT = SrcVT;
17396   if (SrcVT.isFixedLengthVector() || Opcode == AArch64ISD::UADDV_PRED)
17397     RdxVT = getPackedSVEVectorVT(ResVT);
17398 
17399   SDValue Pg = getPredicateForVector(DAG, DL, SrcVT);
17400   SDValue Rdx = DAG.getNode(Opcode, DL, RdxVT, Pg, VecOp);
17401   SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT,
17402                             Rdx, DAG.getConstant(0, DL, MVT::i64));
17403 
17404   // The VEC_REDUCE nodes expect an element size result.
17405   if (ResVT != ScalarOp.getValueType())
17406     Res = DAG.getAnyExtOrTrunc(Res, DL, ScalarOp.getValueType());
17407 
17408   return Res;
17409 }
17410 
17411 SDValue
17412 AArch64TargetLowering::LowerFixedLengthVectorSelectToSVE(SDValue Op,
17413     SelectionDAG &DAG) const {
17414   EVT VT = Op.getValueType();
17415   SDLoc DL(Op);
17416 
17417   EVT InVT = Op.getOperand(1).getValueType();
17418   EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
17419   SDValue Op1 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(1));
17420   SDValue Op2 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(2));
17421 
17422   // Convert the mask to a predicated (NOTE: We don't need to worry about
17423   // inactive lanes since VSELECT is safe when given undefined elements).
17424   EVT MaskVT = Op.getOperand(0).getValueType();
17425   EVT MaskContainerVT = getContainerForFixedLengthVector(DAG, MaskVT);
17426   auto Mask = convertToScalableVector(DAG, MaskContainerVT, Op.getOperand(0));
17427   Mask = DAG.getNode(ISD::TRUNCATE, DL,
17428                      MaskContainerVT.changeVectorElementType(MVT::i1), Mask);
17429 
17430   auto ScalableRes = DAG.getNode(ISD::VSELECT, DL, ContainerVT,
17431                                 Mask, Op1, Op2);
17432 
17433   return convertFromScalableVector(DAG, VT, ScalableRes);
17434 }
17435 
17436 SDValue AArch64TargetLowering::LowerFixedLengthVectorSetccToSVE(
17437     SDValue Op, SelectionDAG &DAG) const {
17438   SDLoc DL(Op);
17439   EVT InVT = Op.getOperand(0).getValueType();
17440   EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
17441 
17442   assert(useSVEForFixedLengthVectorVT(InVT) &&
17443          "Only expected to lower fixed length vector operation!");
17444   assert(Op.getValueType() == InVT.changeTypeToInteger() &&
17445          "Expected integer result of the same bit length as the inputs!");
17446 
17447   auto Op1 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(0));
17448   auto Op2 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(1));
17449   auto Pg = getPredicateForFixedLengthVector(DAG, DL, InVT);
17450 
17451   EVT CmpVT = Pg.getValueType();
17452   auto Cmp = DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, CmpVT,
17453                          {Pg, Op1, Op2, Op.getOperand(2)});
17454 
17455   EVT PromoteVT = ContainerVT.changeTypeToInteger();
17456   auto Promote = DAG.getBoolExtOrTrunc(Cmp, DL, PromoteVT, InVT);
17457   return convertFromScalableVector(DAG, Op.getValueType(), Promote);
17458 }
17459 
17460 SDValue AArch64TargetLowering::getSVESafeBitCast(EVT VT, SDValue Op,
17461                                                  SelectionDAG &DAG) const {
17462   SDLoc DL(Op);
17463   EVT InVT = Op.getValueType();
17464   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17465   (void)TLI;
17466 
17467   assert(VT.isScalableVector() && TLI.isTypeLegal(VT) &&
17468          InVT.isScalableVector() && TLI.isTypeLegal(InVT) &&
17469          "Only expect to cast between legal scalable vector types!");
17470   assert((VT.getVectorElementType() == MVT::i1) ==
17471              (InVT.getVectorElementType() == MVT::i1) &&
17472          "Cannot cast between data and predicate scalable vector types!");
17473 
17474   if (InVT == VT)
17475     return Op;
17476 
17477   if (VT.getVectorElementType() == MVT::i1)
17478     return DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
17479 
17480   EVT PackedVT = getPackedSVEVectorVT(VT.getVectorElementType());
17481   EVT PackedInVT = getPackedSVEVectorVT(InVT.getVectorElementType());
17482   assert((VT == PackedVT || InVT == PackedInVT) &&
17483          "Cannot cast between unpacked scalable vector types!");
17484 
17485   // Pack input if required.
17486   if (InVT != PackedInVT)
17487     Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, PackedInVT, Op);
17488 
17489   Op = DAG.getNode(ISD::BITCAST, DL, PackedVT, Op);
17490 
17491   // Unpack result if required.
17492   if (VT != PackedVT)
17493     Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
17494 
17495   return Op;
17496 }
17497