1 //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the SelectionDAG class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/CodeGen/SelectionDAG.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/APSInt.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/BitVector.h"
20 #include "llvm/ADT/FoldingSet.h"
21 #include "llvm/ADT/None.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallPtrSet.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/Triple.h"
26 #include "llvm/ADT/Twine.h"
27 #include "llvm/Analysis/BlockFrequencyInfo.h"
28 #include "llvm/Analysis/MemoryLocation.h"
29 #include "llvm/Analysis/ProfileSummaryInfo.h"
30 #include "llvm/Analysis/ValueTracking.h"
31 #include "llvm/CodeGen/FunctionLoweringInfo.h"
32 #include "llvm/CodeGen/ISDOpcodes.h"
33 #include "llvm/CodeGen/MachineBasicBlock.h"
34 #include "llvm/CodeGen/MachineConstantPool.h"
35 #include "llvm/CodeGen/MachineFrameInfo.h"
36 #include "llvm/CodeGen/MachineFunction.h"
37 #include "llvm/CodeGen/MachineMemOperand.h"
38 #include "llvm/CodeGen/RuntimeLibcalls.h"
39 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
40 #include "llvm/CodeGen/SelectionDAGNodes.h"
41 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
42 #include "llvm/CodeGen/TargetFrameLowering.h"
43 #include "llvm/CodeGen/TargetLowering.h"
44 #include "llvm/CodeGen/TargetRegisterInfo.h"
45 #include "llvm/CodeGen/TargetSubtargetInfo.h"
46 #include "llvm/CodeGen/ValueTypes.h"
47 #include "llvm/IR/Constant.h"
48 #include "llvm/IR/Constants.h"
49 #include "llvm/IR/DataLayout.h"
50 #include "llvm/IR/DebugInfoMetadata.h"
51 #include "llvm/IR/DebugLoc.h"
52 #include "llvm/IR/DerivedTypes.h"
53 #include "llvm/IR/Function.h"
54 #include "llvm/IR/GlobalValue.h"
55 #include "llvm/IR/Metadata.h"
56 #include "llvm/IR/Type.h"
57 #include "llvm/IR/Value.h"
58 #include "llvm/Support/Casting.h"
59 #include "llvm/Support/CodeGen.h"
60 #include "llvm/Support/Compiler.h"
61 #include "llvm/Support/Debug.h"
62 #include "llvm/Support/ErrorHandling.h"
63 #include "llvm/Support/KnownBits.h"
64 #include "llvm/Support/MachineValueType.h"
65 #include "llvm/Support/ManagedStatic.h"
66 #include "llvm/Support/MathExtras.h"
67 #include "llvm/Support/Mutex.h"
68 #include "llvm/Support/raw_ostream.h"
69 #include "llvm/Target/TargetMachine.h"
70 #include "llvm/Target/TargetOptions.h"
71 #include "llvm/Transforms/Utils/SizeOpts.h"
72 #include <algorithm>
73 #include <cassert>
74 #include <cstdint>
75 #include <cstdlib>
76 #include <limits>
77 #include <set>
78 #include <string>
79 #include <utility>
80 #include <vector>
81 
82 using namespace llvm;
83 
84 /// makeVTList - Return an instance of the SDVTList struct initialized with the
85 /// specified members.
86 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
87   SDVTList Res = {VTs, NumVTs};
88   return Res;
89 }
90 
91 // Default null implementations of the callbacks.
92 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
93 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
94 void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
95 
96 void SelectionDAG::DAGNodeDeletedListener::anchor() {}
97 
98 #define DEBUG_TYPE "selectiondag"
99 
100 static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
101        cl::Hidden, cl::init(true),
102        cl::desc("Gang up loads and stores generated by inlining of memcpy"));
103 
104 static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
105        cl::desc("Number limit for gluing ld/st of memcpy."),
106        cl::Hidden, cl::init(0));
107 
108 static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
109   LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
110 }
111 
112 //===----------------------------------------------------------------------===//
113 //                              ConstantFPSDNode Class
114 //===----------------------------------------------------------------------===//
115 
116 /// isExactlyValue - We don't rely on operator== working on double values, as
117 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
118 /// As such, this method can be used to do an exact bit-for-bit comparison of
119 /// two floating point values.
120 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
121   return getValueAPF().bitwiseIsEqual(V);
122 }
123 
124 bool ConstantFPSDNode::isValueValidForType(EVT VT,
125                                            const APFloat& Val) {
126   assert(VT.isFloatingPoint() && "Can only convert between FP types");
127 
128   // convert modifies in place, so make a copy.
129   APFloat Val2 = APFloat(Val);
130   bool losesInfo;
131   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
132                       APFloat::rmNearestTiesToEven,
133                       &losesInfo);
134   return !losesInfo;
135 }
136 
137 //===----------------------------------------------------------------------===//
138 //                              ISD Namespace
139 //===----------------------------------------------------------------------===//
140 
141 bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
142   auto *BV = dyn_cast<BuildVectorSDNode>(N);
143   if (!BV)
144     return false;
145 
146   APInt SplatUndef;
147   unsigned SplatBitSize;
148   bool HasUndefs;
149   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
150   return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
151                              EltSize) &&
152          EltSize == SplatBitSize;
153 }
154 
155 // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
156 // specializations of the more general isConstantSplatVector()?
157 
158 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
159   // Look through a bit convert.
160   while (N->getOpcode() == ISD::BITCAST)
161     N = N->getOperand(0).getNode();
162 
163   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
164 
165   unsigned i = 0, e = N->getNumOperands();
166 
167   // Skip over all of the undef values.
168   while (i != e && N->getOperand(i).isUndef())
169     ++i;
170 
171   // Do not accept an all-undef vector.
172   if (i == e) return false;
173 
174   // Do not accept build_vectors that aren't all constants or which have non-~0
175   // elements. We have to be a bit careful here, as the type of the constant
176   // may not be the same as the type of the vector elements due to type
177   // legalization (the elements are promoted to a legal type for the target and
178   // a vector of a type may be legal when the base element type is not).
179   // We only want to check enough bits to cover the vector elements, because
180   // we care if the resultant vector is all ones, not whether the individual
181   // constants are.
182   SDValue NotZero = N->getOperand(i);
183   unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
184   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
185     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
186       return false;
187   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
188     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
189       return false;
190   } else
191     return false;
192 
193   // Okay, we have at least one ~0 value, check to see if the rest match or are
194   // undefs. Even with the above element type twiddling, this should be OK, as
195   // the same type legalization should have applied to all the elements.
196   for (++i; i != e; ++i)
197     if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
198       return false;
199   return true;
200 }
201 
202 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
203   // Look through a bit convert.
204   while (N->getOpcode() == ISD::BITCAST)
205     N = N->getOperand(0).getNode();
206 
207   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
208 
209   bool IsAllUndef = true;
210   for (const SDValue &Op : N->op_values()) {
211     if (Op.isUndef())
212       continue;
213     IsAllUndef = false;
214     // Do not accept build_vectors that aren't all constants or which have non-0
215     // elements. We have to be a bit careful here, as the type of the constant
216     // may not be the same as the type of the vector elements due to type
217     // legalization (the elements are promoted to a legal type for the target
218     // and a vector of a type may be legal when the base element type is not).
219     // We only want to check enough bits to cover the vector elements, because
220     // we care if the resultant vector is all zeros, not whether the individual
221     // constants are.
222     unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
223     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
224       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
225         return false;
226     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
227       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
228         return false;
229     } else
230       return false;
231   }
232 
233   // Do not accept an all-undef vector.
234   if (IsAllUndef)
235     return false;
236   return true;
237 }
238 
239 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
240   if (N->getOpcode() != ISD::BUILD_VECTOR)
241     return false;
242 
243   for (const SDValue &Op : N->op_values()) {
244     if (Op.isUndef())
245       continue;
246     if (!isa<ConstantSDNode>(Op))
247       return false;
248   }
249   return true;
250 }
251 
252 bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
253   if (N->getOpcode() != ISD::BUILD_VECTOR)
254     return false;
255 
256   for (const SDValue &Op : N->op_values()) {
257     if (Op.isUndef())
258       continue;
259     if (!isa<ConstantFPSDNode>(Op))
260       return false;
261   }
262   return true;
263 }
264 
265 bool ISD::allOperandsUndef(const SDNode *N) {
266   // Return false if the node has no operands.
267   // This is "logically inconsistent" with the definition of "all" but
268   // is probably the desired behavior.
269   if (N->getNumOperands() == 0)
270     return false;
271   return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
272 }
273 
274 bool ISD::matchUnaryPredicate(SDValue Op,
275                               std::function<bool(ConstantSDNode *)> Match,
276                               bool AllowUndefs) {
277   // FIXME: Add support for scalar UNDEF cases?
278   if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
279     return Match(Cst);
280 
281   // FIXME: Add support for vector UNDEF cases?
282   if (ISD::BUILD_VECTOR != Op.getOpcode())
283     return false;
284 
285   EVT SVT = Op.getValueType().getScalarType();
286   for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
287     if (AllowUndefs && Op.getOperand(i).isUndef()) {
288       if (!Match(nullptr))
289         return false;
290       continue;
291     }
292 
293     auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
294     if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
295       return false;
296   }
297   return true;
298 }
299 
300 bool ISD::matchBinaryPredicate(
301     SDValue LHS, SDValue RHS,
302     std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
303     bool AllowUndefs, bool AllowTypeMismatch) {
304   if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
305     return false;
306 
307   // TODO: Add support for scalar UNDEF cases?
308   if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
309     if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
310       return Match(LHSCst, RHSCst);
311 
312   // TODO: Add support for vector UNDEF cases?
313   if (ISD::BUILD_VECTOR != LHS.getOpcode() ||
314       ISD::BUILD_VECTOR != RHS.getOpcode())
315     return false;
316 
317   EVT SVT = LHS.getValueType().getScalarType();
318   for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
319     SDValue LHSOp = LHS.getOperand(i);
320     SDValue RHSOp = RHS.getOperand(i);
321     bool LHSUndef = AllowUndefs && LHSOp.isUndef();
322     bool RHSUndef = AllowUndefs && RHSOp.isUndef();
323     auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
324     auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
325     if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
326       return false;
327     if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
328                                LHSOp.getValueType() != RHSOp.getValueType()))
329       return false;
330     if (!Match(LHSCst, RHSCst))
331       return false;
332   }
333   return true;
334 }
335 
336 ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
337   switch (VecReduceOpcode) {
338   default:
339     llvm_unreachable("Expected VECREDUCE opcode");
340   case ISD::VECREDUCE_FADD:
341   case ISD::VECREDUCE_SEQ_FADD:
342     return ISD::FADD;
343   case ISD::VECREDUCE_FMUL:
344   case ISD::VECREDUCE_SEQ_FMUL:
345     return ISD::FMUL;
346   case ISD::VECREDUCE_ADD:
347     return ISD::ADD;
348   case ISD::VECREDUCE_MUL:
349     return ISD::MUL;
350   case ISD::VECREDUCE_AND:
351     return ISD::AND;
352   case ISD::VECREDUCE_OR:
353     return ISD::OR;
354   case ISD::VECREDUCE_XOR:
355     return ISD::XOR;
356   case ISD::VECREDUCE_SMAX:
357     return ISD::SMAX;
358   case ISD::VECREDUCE_SMIN:
359     return ISD::SMIN;
360   case ISD::VECREDUCE_UMAX:
361     return ISD::UMAX;
362   case ISD::VECREDUCE_UMIN:
363     return ISD::UMIN;
364   case ISD::VECREDUCE_FMAX:
365     return ISD::FMAXNUM;
366   case ISD::VECREDUCE_FMIN:
367     return ISD::FMINNUM;
368   }
369 }
370 
371 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
372   switch (ExtType) {
373   case ISD::EXTLOAD:
374     return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
375   case ISD::SEXTLOAD:
376     return ISD::SIGN_EXTEND;
377   case ISD::ZEXTLOAD:
378     return ISD::ZERO_EXTEND;
379   default:
380     break;
381   }
382 
383   llvm_unreachable("Invalid LoadExtType");
384 }
385 
386 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
387   // To perform this operation, we just need to swap the L and G bits of the
388   // operation.
389   unsigned OldL = (Operation >> 2) & 1;
390   unsigned OldG = (Operation >> 1) & 1;
391   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
392                        (OldL << 1) |       // New G bit
393                        (OldG << 2));       // New L bit.
394 }
395 
396 static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
397   unsigned Operation = Op;
398   if (isIntegerLike)
399     Operation ^= 7;   // Flip L, G, E bits, but not U.
400   else
401     Operation ^= 15;  // Flip all of the condition bits.
402 
403   if (Operation > ISD::SETTRUE2)
404     Operation &= ~8;  // Don't let N and U bits get set.
405 
406   return ISD::CondCode(Operation);
407 }
408 
409 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
410   return getSetCCInverseImpl(Op, Type.isInteger());
411 }
412 
413 ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
414                                                bool isIntegerLike) {
415   return getSetCCInverseImpl(Op, isIntegerLike);
416 }
417 
418 /// For an integer comparison, return 1 if the comparison is a signed operation
419 /// and 2 if the result is an unsigned comparison. Return zero if the operation
420 /// does not depend on the sign of the input (setne and seteq).
421 static int isSignedOp(ISD::CondCode Opcode) {
422   switch (Opcode) {
423   default: llvm_unreachable("Illegal integer setcc operation!");
424   case ISD::SETEQ:
425   case ISD::SETNE: return 0;
426   case ISD::SETLT:
427   case ISD::SETLE:
428   case ISD::SETGT:
429   case ISD::SETGE: return 1;
430   case ISD::SETULT:
431   case ISD::SETULE:
432   case ISD::SETUGT:
433   case ISD::SETUGE: return 2;
434   }
435 }
436 
437 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
438                                        EVT Type) {
439   bool IsInteger = Type.isInteger();
440   if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
441     // Cannot fold a signed integer setcc with an unsigned integer setcc.
442     return ISD::SETCC_INVALID;
443 
444   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
445 
446   // If the N and U bits get set, then the resultant comparison DOES suddenly
447   // care about orderedness, and it is true when ordered.
448   if (Op > ISD::SETTRUE2)
449     Op &= ~16;     // Clear the U bit if the N bit is set.
450 
451   // Canonicalize illegal integer setcc's.
452   if (IsInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
453     Op = ISD::SETNE;
454 
455   return ISD::CondCode(Op);
456 }
457 
458 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
459                                         EVT Type) {
460   bool IsInteger = Type.isInteger();
461   if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
462     // Cannot fold a signed setcc with an unsigned setcc.
463     return ISD::SETCC_INVALID;
464 
465   // Combine all of the condition bits.
466   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
467 
468   // Canonicalize illegal integer setcc's.
469   if (IsInteger) {
470     switch (Result) {
471     default: break;
472     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
473     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
474     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
475     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
476     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
477     }
478   }
479 
480   return Result;
481 }
482 
483 //===----------------------------------------------------------------------===//
484 //                           SDNode Profile Support
485 //===----------------------------------------------------------------------===//
486 
487 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
488 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
489   ID.AddInteger(OpC);
490 }
491 
492 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
493 /// solely with their pointer.
494 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
495   ID.AddPointer(VTList.VTs);
496 }
497 
498 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
499 static void AddNodeIDOperands(FoldingSetNodeID &ID,
500                               ArrayRef<SDValue> Ops) {
501   for (auto& Op : Ops) {
502     ID.AddPointer(Op.getNode());
503     ID.AddInteger(Op.getResNo());
504   }
505 }
506 
507 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
508 static void AddNodeIDOperands(FoldingSetNodeID &ID,
509                               ArrayRef<SDUse> Ops) {
510   for (auto& Op : Ops) {
511     ID.AddPointer(Op.getNode());
512     ID.AddInteger(Op.getResNo());
513   }
514 }
515 
516 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
517                           SDVTList VTList, ArrayRef<SDValue> OpList) {
518   AddNodeIDOpcode(ID, OpC);
519   AddNodeIDValueTypes(ID, VTList);
520   AddNodeIDOperands(ID, OpList);
521 }
522 
523 /// If this is an SDNode with special info, add this info to the NodeID data.
524 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
525   switch (N->getOpcode()) {
526   case ISD::TargetExternalSymbol:
527   case ISD::ExternalSymbol:
528   case ISD::MCSymbol:
529     llvm_unreachable("Should only be used on nodes with operands");
530   default: break;  // Normal nodes don't need extra info.
531   case ISD::TargetConstant:
532   case ISD::Constant: {
533     const ConstantSDNode *C = cast<ConstantSDNode>(N);
534     ID.AddPointer(C->getConstantIntValue());
535     ID.AddBoolean(C->isOpaque());
536     break;
537   }
538   case ISD::TargetConstantFP:
539   case ISD::ConstantFP:
540     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
541     break;
542   case ISD::TargetGlobalAddress:
543   case ISD::GlobalAddress:
544   case ISD::TargetGlobalTLSAddress:
545   case ISD::GlobalTLSAddress: {
546     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
547     ID.AddPointer(GA->getGlobal());
548     ID.AddInteger(GA->getOffset());
549     ID.AddInteger(GA->getTargetFlags());
550     break;
551   }
552   case ISD::BasicBlock:
553     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
554     break;
555   case ISD::Register:
556     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
557     break;
558   case ISD::RegisterMask:
559     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
560     break;
561   case ISD::SRCVALUE:
562     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
563     break;
564   case ISD::FrameIndex:
565   case ISD::TargetFrameIndex:
566     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
567     break;
568   case ISD::LIFETIME_START:
569   case ISD::LIFETIME_END:
570     if (cast<LifetimeSDNode>(N)->hasOffset()) {
571       ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
572       ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
573     }
574     break;
575   case ISD::JumpTable:
576   case ISD::TargetJumpTable:
577     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
578     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
579     break;
580   case ISD::ConstantPool:
581   case ISD::TargetConstantPool: {
582     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
583     ID.AddInteger(CP->getAlign().value());
584     ID.AddInteger(CP->getOffset());
585     if (CP->isMachineConstantPoolEntry())
586       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
587     else
588       ID.AddPointer(CP->getConstVal());
589     ID.AddInteger(CP->getTargetFlags());
590     break;
591   }
592   case ISD::TargetIndex: {
593     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
594     ID.AddInteger(TI->getIndex());
595     ID.AddInteger(TI->getOffset());
596     ID.AddInteger(TI->getTargetFlags());
597     break;
598   }
599   case ISD::LOAD: {
600     const LoadSDNode *LD = cast<LoadSDNode>(N);
601     ID.AddInteger(LD->getMemoryVT().getRawBits());
602     ID.AddInteger(LD->getRawSubclassData());
603     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
604     break;
605   }
606   case ISD::STORE: {
607     const StoreSDNode *ST = cast<StoreSDNode>(N);
608     ID.AddInteger(ST->getMemoryVT().getRawBits());
609     ID.AddInteger(ST->getRawSubclassData());
610     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
611     break;
612   }
613   case ISD::MLOAD: {
614     const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
615     ID.AddInteger(MLD->getMemoryVT().getRawBits());
616     ID.AddInteger(MLD->getRawSubclassData());
617     ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
618     break;
619   }
620   case ISD::MSTORE: {
621     const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
622     ID.AddInteger(MST->getMemoryVT().getRawBits());
623     ID.AddInteger(MST->getRawSubclassData());
624     ID.AddInteger(MST->getPointerInfo().getAddrSpace());
625     break;
626   }
627   case ISD::MGATHER: {
628     const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
629     ID.AddInteger(MG->getMemoryVT().getRawBits());
630     ID.AddInteger(MG->getRawSubclassData());
631     ID.AddInteger(MG->getPointerInfo().getAddrSpace());
632     break;
633   }
634   case ISD::MSCATTER: {
635     const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
636     ID.AddInteger(MS->getMemoryVT().getRawBits());
637     ID.AddInteger(MS->getRawSubclassData());
638     ID.AddInteger(MS->getPointerInfo().getAddrSpace());
639     break;
640   }
641   case ISD::ATOMIC_CMP_SWAP:
642   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
643   case ISD::ATOMIC_SWAP:
644   case ISD::ATOMIC_LOAD_ADD:
645   case ISD::ATOMIC_LOAD_SUB:
646   case ISD::ATOMIC_LOAD_AND:
647   case ISD::ATOMIC_LOAD_CLR:
648   case ISD::ATOMIC_LOAD_OR:
649   case ISD::ATOMIC_LOAD_XOR:
650   case ISD::ATOMIC_LOAD_NAND:
651   case ISD::ATOMIC_LOAD_MIN:
652   case ISD::ATOMIC_LOAD_MAX:
653   case ISD::ATOMIC_LOAD_UMIN:
654   case ISD::ATOMIC_LOAD_UMAX:
655   case ISD::ATOMIC_LOAD:
656   case ISD::ATOMIC_STORE: {
657     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
658     ID.AddInteger(AT->getMemoryVT().getRawBits());
659     ID.AddInteger(AT->getRawSubclassData());
660     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
661     break;
662   }
663   case ISD::PREFETCH: {
664     const MemSDNode *PF = cast<MemSDNode>(N);
665     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
666     break;
667   }
668   case ISD::VECTOR_SHUFFLE: {
669     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
670     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
671          i != e; ++i)
672       ID.AddInteger(SVN->getMaskElt(i));
673     break;
674   }
675   case ISD::TargetBlockAddress:
676   case ISD::BlockAddress: {
677     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
678     ID.AddPointer(BA->getBlockAddress());
679     ID.AddInteger(BA->getOffset());
680     ID.AddInteger(BA->getTargetFlags());
681     break;
682   }
683   } // end switch (N->getOpcode())
684 
685   // Target specific memory nodes could also have address spaces to check.
686   if (N->isTargetMemoryOpcode())
687     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
688 }
689 
690 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
691 /// data.
692 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
693   AddNodeIDOpcode(ID, N->getOpcode());
694   // Add the return value info.
695   AddNodeIDValueTypes(ID, N->getVTList());
696   // Add the operand info.
697   AddNodeIDOperands(ID, N->ops());
698 
699   // Handle SDNode leafs with special info.
700   AddNodeIDCustom(ID, N);
701 }
702 
703 //===----------------------------------------------------------------------===//
704 //                              SelectionDAG Class
705 //===----------------------------------------------------------------------===//
706 
707 /// doNotCSE - Return true if CSE should not be performed for this node.
708 static bool doNotCSE(SDNode *N) {
709   if (N->getValueType(0) == MVT::Glue)
710     return true; // Never CSE anything that produces a flag.
711 
712   switch (N->getOpcode()) {
713   default: break;
714   case ISD::HANDLENODE:
715   case ISD::EH_LABEL:
716     return true;   // Never CSE these nodes.
717   }
718 
719   // Check that remaining values produced are not flags.
720   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
721     if (N->getValueType(i) == MVT::Glue)
722       return true; // Never CSE anything that produces a flag.
723 
724   return false;
725 }
726 
727 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
728 /// SelectionDAG.
729 void SelectionDAG::RemoveDeadNodes() {
730   // Create a dummy node (which is not added to allnodes), that adds a reference
731   // to the root node, preventing it from being deleted.
732   HandleSDNode Dummy(getRoot());
733 
734   SmallVector<SDNode*, 128> DeadNodes;
735 
736   // Add all obviously-dead nodes to the DeadNodes worklist.
737   for (SDNode &Node : allnodes())
738     if (Node.use_empty())
739       DeadNodes.push_back(&Node);
740 
741   RemoveDeadNodes(DeadNodes);
742 
743   // If the root changed (e.g. it was a dead load, update the root).
744   setRoot(Dummy.getValue());
745 }
746 
747 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
748 /// given list, and any nodes that become unreachable as a result.
749 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
750 
751   // Process the worklist, deleting the nodes and adding their uses to the
752   // worklist.
753   while (!DeadNodes.empty()) {
754     SDNode *N = DeadNodes.pop_back_val();
755     // Skip to next node if we've already managed to delete the node. This could
756     // happen if replacing a node causes a node previously added to the node to
757     // be deleted.
758     if (N->getOpcode() == ISD::DELETED_NODE)
759       continue;
760 
761     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
762       DUL->NodeDeleted(N, nullptr);
763 
764     // Take the node out of the appropriate CSE map.
765     RemoveNodeFromCSEMaps(N);
766 
767     // Next, brutally remove the operand list.  This is safe to do, as there are
768     // no cycles in the graph.
769     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
770       SDUse &Use = *I++;
771       SDNode *Operand = Use.getNode();
772       Use.set(SDValue());
773 
774       // Now that we removed this operand, see if there are no uses of it left.
775       if (Operand->use_empty())
776         DeadNodes.push_back(Operand);
777     }
778 
779     DeallocateNode(N);
780   }
781 }
782 
783 void SelectionDAG::RemoveDeadNode(SDNode *N){
784   SmallVector<SDNode*, 16> DeadNodes(1, N);
785 
786   // Create a dummy node that adds a reference to the root node, preventing
787   // it from being deleted.  (This matters if the root is an operand of the
788   // dead node.)
789   HandleSDNode Dummy(getRoot());
790 
791   RemoveDeadNodes(DeadNodes);
792 }
793 
794 void SelectionDAG::DeleteNode(SDNode *N) {
795   // First take this out of the appropriate CSE map.
796   RemoveNodeFromCSEMaps(N);
797 
798   // Finally, remove uses due to operands of this node, remove from the
799   // AllNodes list, and delete the node.
800   DeleteNodeNotInCSEMaps(N);
801 }
802 
803 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
804   assert(N->getIterator() != AllNodes.begin() &&
805          "Cannot delete the entry node!");
806   assert(N->use_empty() && "Cannot delete a node that is not dead!");
807 
808   // Drop all of the operands and decrement used node's use counts.
809   N->DropOperands();
810 
811   DeallocateNode(N);
812 }
813 
814 void SDDbgInfo::erase(const SDNode *Node) {
815   DbgValMapType::iterator I = DbgValMap.find(Node);
816   if (I == DbgValMap.end())
817     return;
818   for (auto &Val: I->second)
819     Val->setIsInvalidated();
820   DbgValMap.erase(I);
821 }
822 
823 void SelectionDAG::DeallocateNode(SDNode *N) {
824   // If we have operands, deallocate them.
825   removeOperands(N);
826 
827   NodeAllocator.Deallocate(AllNodes.remove(N));
828 
829   // Set the opcode to DELETED_NODE to help catch bugs when node
830   // memory is reallocated.
831   // FIXME: There are places in SDag that have grown a dependency on the opcode
832   // value in the released node.
833   __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
834   N->NodeType = ISD::DELETED_NODE;
835 
836   // If any of the SDDbgValue nodes refer to this SDNode, invalidate
837   // them and forget about that node.
838   DbgInfo->erase(N);
839 }
840 
841 #ifndef NDEBUG
842 /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
843 static void VerifySDNode(SDNode *N) {
844   switch (N->getOpcode()) {
845   default:
846     break;
847   case ISD::BUILD_PAIR: {
848     EVT VT = N->getValueType(0);
849     assert(N->getNumValues() == 1 && "Too many results!");
850     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
851            "Wrong return type!");
852     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
853     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
854            "Mismatched operand types!");
855     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
856            "Wrong operand type!");
857     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
858            "Wrong return type size");
859     break;
860   }
861   case ISD::BUILD_VECTOR: {
862     assert(N->getNumValues() == 1 && "Too many results!");
863     assert(N->getValueType(0).isVector() && "Wrong return type!");
864     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
865            "Wrong number of operands!");
866     EVT EltVT = N->getValueType(0).getVectorElementType();
867     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
868       assert((I->getValueType() == EltVT ||
869              (EltVT.isInteger() && I->getValueType().isInteger() &&
870               EltVT.bitsLE(I->getValueType()))) &&
871             "Wrong operand type!");
872       assert(I->getValueType() == N->getOperand(0).getValueType() &&
873              "Operands must all have the same type");
874     }
875     break;
876   }
877   }
878 }
879 #endif // NDEBUG
880 
881 /// Insert a newly allocated node into the DAG.
882 ///
883 /// Handles insertion into the all nodes list and CSE map, as well as
884 /// verification and other common operations when a new node is allocated.
885 void SelectionDAG::InsertNode(SDNode *N) {
886   AllNodes.push_back(N);
887 #ifndef NDEBUG
888   N->PersistentId = NextPersistentId++;
889   VerifySDNode(N);
890 #endif
891   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
892     DUL->NodeInserted(N);
893 }
894 
895 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
896 /// correspond to it.  This is useful when we're about to delete or repurpose
897 /// the node.  We don't want future request for structurally identical nodes
898 /// to return N anymore.
899 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
900   bool Erased = false;
901   switch (N->getOpcode()) {
902   case ISD::HANDLENODE: return false;  // noop.
903   case ISD::CONDCODE:
904     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
905            "Cond code doesn't exist!");
906     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
907     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
908     break;
909   case ISD::ExternalSymbol:
910     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
911     break;
912   case ISD::TargetExternalSymbol: {
913     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
914     Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
915         ESN->getSymbol(), ESN->getTargetFlags()));
916     break;
917   }
918   case ISD::MCSymbol: {
919     auto *MCSN = cast<MCSymbolSDNode>(N);
920     Erased = MCSymbols.erase(MCSN->getMCSymbol());
921     break;
922   }
923   case ISD::VALUETYPE: {
924     EVT VT = cast<VTSDNode>(N)->getVT();
925     if (VT.isExtended()) {
926       Erased = ExtendedValueTypeNodes.erase(VT);
927     } else {
928       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
929       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
930     }
931     break;
932   }
933   default:
934     // Remove it from the CSE Map.
935     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
936     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
937     Erased = CSEMap.RemoveNode(N);
938     break;
939   }
940 #ifndef NDEBUG
941   // Verify that the node was actually in one of the CSE maps, unless it has a
942   // flag result (which cannot be CSE'd) or is one of the special cases that are
943   // not subject to CSE.
944   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
945       !N->isMachineOpcode() && !doNotCSE(N)) {
946     N->dump(this);
947     dbgs() << "\n";
948     llvm_unreachable("Node is not in map!");
949   }
950 #endif
951   return Erased;
952 }
953 
954 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
955 /// maps and modified in place. Add it back to the CSE maps, unless an identical
956 /// node already exists, in which case transfer all its users to the existing
957 /// node. This transfer can potentially trigger recursive merging.
958 void
959 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
960   // For node types that aren't CSE'd, just act as if no identical node
961   // already exists.
962   if (!doNotCSE(N)) {
963     SDNode *Existing = CSEMap.GetOrInsertNode(N);
964     if (Existing != N) {
965       // If there was already an existing matching node, use ReplaceAllUsesWith
966       // to replace the dead one with the existing one.  This can cause
967       // recursive merging of other unrelated nodes down the line.
968       ReplaceAllUsesWith(N, Existing);
969 
970       // N is now dead. Inform the listeners and delete it.
971       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
972         DUL->NodeDeleted(N, Existing);
973       DeleteNodeNotInCSEMaps(N);
974       return;
975     }
976   }
977 
978   // If the node doesn't already exist, we updated it.  Inform listeners.
979   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
980     DUL->NodeUpdated(N);
981 }
982 
983 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
984 /// were replaced with those specified.  If this node is never memoized,
985 /// return null, otherwise return a pointer to the slot it would take.  If a
986 /// node already exists with these operands, the slot will be non-null.
987 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
988                                            void *&InsertPos) {
989   if (doNotCSE(N))
990     return nullptr;
991 
992   SDValue Ops[] = { Op };
993   FoldingSetNodeID ID;
994   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
995   AddNodeIDCustom(ID, N);
996   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
997   if (Node)
998     Node->intersectFlagsWith(N->getFlags());
999   return Node;
1000 }
1001 
1002 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1003 /// were replaced with those specified.  If this node is never memoized,
1004 /// return null, otherwise return a pointer to the slot it would take.  If a
1005 /// node already exists with these operands, the slot will be non-null.
1006 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1007                                            SDValue Op1, SDValue Op2,
1008                                            void *&InsertPos) {
1009   if (doNotCSE(N))
1010     return nullptr;
1011 
1012   SDValue Ops[] = { Op1, Op2 };
1013   FoldingSetNodeID ID;
1014   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1015   AddNodeIDCustom(ID, N);
1016   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1017   if (Node)
1018     Node->intersectFlagsWith(N->getFlags());
1019   return Node;
1020 }
1021 
1022 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1023 /// were replaced with those specified.  If this node is never memoized,
1024 /// return null, otherwise return a pointer to the slot it would take.  If a
1025 /// node already exists with these operands, the slot will be non-null.
1026 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1027                                            void *&InsertPos) {
1028   if (doNotCSE(N))
1029     return nullptr;
1030 
1031   FoldingSetNodeID ID;
1032   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1033   AddNodeIDCustom(ID, N);
1034   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1035   if (Node)
1036     Node->intersectFlagsWith(N->getFlags());
1037   return Node;
1038 }
1039 
1040 Align SelectionDAG::getEVTAlign(EVT VT) const {
1041   Type *Ty = VT == MVT::iPTR ?
1042                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
1043                    VT.getTypeForEVT(*getContext());
1044 
1045   return getDataLayout().getABITypeAlign(Ty);
1046 }
1047 
1048 // EntryNode could meaningfully have debug info if we can find it...
1049 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
1050     : TM(tm), OptLevel(OL),
1051       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
1052       Root(getEntryNode()) {
1053   InsertNode(&EntryNode);
1054   DbgInfo = new SDDbgInfo();
1055 }
1056 
1057 void SelectionDAG::init(MachineFunction &NewMF,
1058                         OptimizationRemarkEmitter &NewORE,
1059                         Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
1060                         LegacyDivergenceAnalysis * Divergence,
1061                         ProfileSummaryInfo *PSIin,
1062                         BlockFrequencyInfo *BFIin) {
1063   MF = &NewMF;
1064   SDAGISelPass = PassPtr;
1065   ORE = &NewORE;
1066   TLI = getSubtarget().getTargetLowering();
1067   TSI = getSubtarget().getSelectionDAGInfo();
1068   LibInfo = LibraryInfo;
1069   Context = &MF->getFunction().getContext();
1070   DA = Divergence;
1071   PSI = PSIin;
1072   BFI = BFIin;
1073 }
1074 
1075 SelectionDAG::~SelectionDAG() {
1076   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1077   allnodes_clear();
1078   OperandRecycler.clear(OperandAllocator);
1079   delete DbgInfo;
1080 }
1081 
1082 bool SelectionDAG::shouldOptForSize() const {
1083   return MF->getFunction().hasOptSize() ||
1084       llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
1085 }
1086 
1087 void SelectionDAG::allnodes_clear() {
1088   assert(&*AllNodes.begin() == &EntryNode);
1089   AllNodes.remove(AllNodes.begin());
1090   while (!AllNodes.empty())
1091     DeallocateNode(&AllNodes.front());
1092 #ifndef NDEBUG
1093   NextPersistentId = 0;
1094 #endif
1095 }
1096 
1097 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1098                                           void *&InsertPos) {
1099   SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1100   if (N) {
1101     switch (N->getOpcode()) {
1102     default: break;
1103     case ISD::Constant:
1104     case ISD::ConstantFP:
1105       llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1106                        "debug location.  Use another overload.");
1107     }
1108   }
1109   return N;
1110 }
1111 
1112 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1113                                           const SDLoc &DL, void *&InsertPos) {
1114   SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1115   if (N) {
1116     switch (N->getOpcode()) {
1117     case ISD::Constant:
1118     case ISD::ConstantFP:
1119       // Erase debug location from the node if the node is used at several
1120       // different places. Do not propagate one location to all uses as it
1121       // will cause a worse single stepping debugging experience.
1122       if (N->getDebugLoc() != DL.getDebugLoc())
1123         N->setDebugLoc(DebugLoc());
1124       break;
1125     default:
1126       // When the node's point of use is located earlier in the instruction
1127       // sequence than its prior point of use, update its debug info to the
1128       // earlier location.
1129       if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1130         N->setDebugLoc(DL.getDebugLoc());
1131       break;
1132     }
1133   }
1134   return N;
1135 }
1136 
1137 void SelectionDAG::clear() {
1138   allnodes_clear();
1139   OperandRecycler.clear(OperandAllocator);
1140   OperandAllocator.Reset();
1141   CSEMap.clear();
1142 
1143   ExtendedValueTypeNodes.clear();
1144   ExternalSymbols.clear();
1145   TargetExternalSymbols.clear();
1146   MCSymbols.clear();
1147   SDCallSiteDbgInfo.clear();
1148   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1149             static_cast<CondCodeSDNode*>(nullptr));
1150   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1151             static_cast<SDNode*>(nullptr));
1152 
1153   EntryNode.UseList = nullptr;
1154   InsertNode(&EntryNode);
1155   Root = getEntryNode();
1156   DbgInfo->clear();
1157 }
1158 
1159 SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1160   return VT.bitsGT(Op.getValueType())
1161              ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1162              : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL));
1163 }
1164 
1165 std::pair<SDValue, SDValue>
1166 SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1167                                        const SDLoc &DL, EVT VT) {
1168   assert(!VT.bitsEq(Op.getValueType()) &&
1169          "Strict no-op FP extend/round not allowed.");
1170   SDValue Res =
1171       VT.bitsGT(Op.getValueType())
1172           ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1173           : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1174                     {Chain, Op, getIntPtrConstant(0, DL)});
1175 
1176   return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1177 }
1178 
1179 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1180   return VT.bitsGT(Op.getValueType()) ?
1181     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1182     getNode(ISD::TRUNCATE, DL, VT, Op);
1183 }
1184 
1185 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1186   return VT.bitsGT(Op.getValueType()) ?
1187     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1188     getNode(ISD::TRUNCATE, DL, VT, Op);
1189 }
1190 
1191 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1192   return VT.bitsGT(Op.getValueType()) ?
1193     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1194     getNode(ISD::TRUNCATE, DL, VT, Op);
1195 }
1196 
1197 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1198                                         EVT OpVT) {
1199   if (VT.bitsLE(Op.getValueType()))
1200     return getNode(ISD::TRUNCATE, SL, VT, Op);
1201 
1202   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1203   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1204 }
1205 
1206 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1207   EVT OpVT = Op.getValueType();
1208   assert(VT.isInteger() && OpVT.isInteger() &&
1209          "Cannot getZeroExtendInReg FP types");
1210   assert(VT.isVector() == OpVT.isVector() &&
1211          "getZeroExtendInReg type should be vector iff the operand "
1212          "type is vector!");
1213   assert((!VT.isVector() ||
1214           VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1215          "Vector element counts must match in getZeroExtendInReg");
1216   assert(VT.bitsLE(OpVT) && "Not extending!");
1217   if (OpVT == VT)
1218     return Op;
1219   APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(),
1220                                    VT.getScalarSizeInBits());
1221   return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));
1222 }
1223 
1224 SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1225   // Only unsigned pointer semantics are supported right now. In the future this
1226   // might delegate to TLI to check pointer signedness.
1227   return getZExtOrTrunc(Op, DL, VT);
1228 }
1229 
1230 SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1231   // Only unsigned pointer semantics are supported right now. In the future this
1232   // might delegate to TLI to check pointer signedness.
1233   return getZeroExtendInReg(Op, DL, VT);
1234 }
1235 
1236 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1237 SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1238   EVT EltVT = VT.getScalarType();
1239   SDValue NegOne =
1240     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
1241   return getNode(ISD::XOR, DL, VT, Val, NegOne);
1242 }
1243 
1244 SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1245   SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1246   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1247 }
1248 
1249 SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1250                                       EVT OpVT) {
1251   if (!V)
1252     return getConstant(0, DL, VT);
1253 
1254   switch (TLI->getBooleanContents(OpVT)) {
1255   case TargetLowering::ZeroOrOneBooleanContent:
1256   case TargetLowering::UndefinedBooleanContent:
1257     return getConstant(1, DL, VT);
1258   case TargetLowering::ZeroOrNegativeOneBooleanContent:
1259     return getAllOnesConstant(DL, VT);
1260   }
1261   llvm_unreachable("Unexpected boolean content enum!");
1262 }
1263 
1264 SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1265                                   bool isT, bool isO) {
1266   EVT EltVT = VT.getScalarType();
1267   assert((EltVT.getSizeInBits() >= 64 ||
1268          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1269          "getConstant with a uint64_t value that doesn't fit in the type!");
1270   return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1271 }
1272 
1273 SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1274                                   bool isT, bool isO) {
1275   return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1276 }
1277 
1278 SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1279                                   EVT VT, bool isT, bool isO) {
1280   assert(VT.isInteger() && "Cannot create FP integer constant!");
1281 
1282   EVT EltVT = VT.getScalarType();
1283   const ConstantInt *Elt = &Val;
1284 
1285   // In some cases the vector type is legal but the element type is illegal and
1286   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
1287   // inserted value (the type does not need to match the vector element type).
1288   // Any extra bits introduced will be truncated away.
1289   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1290       TargetLowering::TypePromoteInteger) {
1291    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1292    APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1293    Elt = ConstantInt::get(*getContext(), NewVal);
1294   }
1295   // In other cases the element type is illegal and needs to be expanded, for
1296   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1297   // the value into n parts and use a vector type with n-times the elements.
1298   // Then bitcast to the type requested.
1299   // Legalizing constants too early makes the DAGCombiner's job harder so we
1300   // only legalize if the DAG tells us we must produce legal types.
1301   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1302            TLI->getTypeAction(*getContext(), EltVT) ==
1303            TargetLowering::TypeExpandInteger) {
1304     const APInt &NewVal = Elt->getValue();
1305     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1306     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1307     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1308     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1309 
1310     // Check the temporary vector is the correct size. If this fails then
1311     // getTypeToTransformTo() probably returned a type whose size (in bits)
1312     // isn't a power-of-2 factor of the requested type size.
1313     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1314 
1315     SmallVector<SDValue, 2> EltParts;
1316     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
1317       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
1318                                            .zextOrTrunc(ViaEltSizeInBits), DL,
1319                                      ViaEltVT, isT, isO));
1320     }
1321 
1322     // EltParts is currently in little endian order. If we actually want
1323     // big-endian order then reverse it now.
1324     if (getDataLayout().isBigEndian())
1325       std::reverse(EltParts.begin(), EltParts.end());
1326 
1327     // The elements must be reversed when the element order is different
1328     // to the endianness of the elements (because the BITCAST is itself a
1329     // vector shuffle in this situation). However, we do not need any code to
1330     // perform this reversal because getConstant() is producing a vector
1331     // splat.
1332     // This situation occurs in MIPS MSA.
1333 
1334     SmallVector<SDValue, 8> Ops;
1335     for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1336       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
1337 
1338     SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1339     return V;
1340   }
1341 
1342   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1343          "APInt size does not match type size!");
1344   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1345   FoldingSetNodeID ID;
1346   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1347   ID.AddPointer(Elt);
1348   ID.AddBoolean(isO);
1349   void *IP = nullptr;
1350   SDNode *N = nullptr;
1351   if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1352     if (!VT.isVector())
1353       return SDValue(N, 0);
1354 
1355   if (!N) {
1356     N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1357     CSEMap.InsertNode(N, IP);
1358     InsertNode(N);
1359     NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1360   }
1361 
1362   SDValue Result(N, 0);
1363   if (VT.isScalableVector())
1364     Result = getSplatVector(VT, DL, Result);
1365   else if (VT.isVector())
1366     Result = getSplatBuildVector(VT, DL, Result);
1367 
1368   return Result;
1369 }
1370 
1371 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1372                                         bool isTarget) {
1373   return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1374 }
1375 
1376 SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1377                                              const SDLoc &DL, bool LegalTypes) {
1378   assert(VT.isInteger() && "Shift amount is not an integer type!");
1379   EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1380   return getConstant(Val, DL, ShiftVT);
1381 }
1382 
1383 SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1384                                            bool isTarget) {
1385   return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);
1386 }
1387 
1388 SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1389                                     bool isTarget) {
1390   return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1391 }
1392 
1393 SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1394                                     EVT VT, bool isTarget) {
1395   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1396 
1397   EVT EltVT = VT.getScalarType();
1398 
1399   // Do the map lookup using the actual bit pattern for the floating point
1400   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1401   // we don't have issues with SNANs.
1402   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1403   FoldingSetNodeID ID;
1404   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1405   ID.AddPointer(&V);
1406   void *IP = nullptr;
1407   SDNode *N = nullptr;
1408   if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1409     if (!VT.isVector())
1410       return SDValue(N, 0);
1411 
1412   if (!N) {
1413     N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1414     CSEMap.InsertNode(N, IP);
1415     InsertNode(N);
1416   }
1417 
1418   SDValue Result(N, 0);
1419   if (VT.isScalableVector())
1420     Result = getSplatVector(VT, DL, Result);
1421   else if (VT.isVector())
1422     Result = getSplatBuildVector(VT, DL, Result);
1423   NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1424   return Result;
1425 }
1426 
1427 SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1428                                     bool isTarget) {
1429   EVT EltVT = VT.getScalarType();
1430   if (EltVT == MVT::f32)
1431     return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1432   else if (EltVT == MVT::f64)
1433     return getConstantFP(APFloat(Val), DL, VT, isTarget);
1434   else if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1435            EltVT == MVT::f16 || EltVT == MVT::bf16) {
1436     bool Ignored;
1437     APFloat APF = APFloat(Val);
1438     APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1439                 &Ignored);
1440     return getConstantFP(APF, DL, VT, isTarget);
1441   } else
1442     llvm_unreachable("Unsupported type in getConstantFP");
1443 }
1444 
1445 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1446                                        EVT VT, int64_t Offset, bool isTargetGA,
1447                                        unsigned TargetFlags) {
1448   assert((TargetFlags == 0 || isTargetGA) &&
1449          "Cannot set target flags on target-independent globals");
1450 
1451   // Truncate (with sign-extension) the offset value to the pointer size.
1452   unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1453   if (BitWidth < 64)
1454     Offset = SignExtend64(Offset, BitWidth);
1455 
1456   unsigned Opc;
1457   if (GV->isThreadLocal())
1458     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1459   else
1460     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1461 
1462   FoldingSetNodeID ID;
1463   AddNodeIDNode(ID, Opc, getVTList(VT), None);
1464   ID.AddPointer(GV);
1465   ID.AddInteger(Offset);
1466   ID.AddInteger(TargetFlags);
1467   void *IP = nullptr;
1468   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1469     return SDValue(E, 0);
1470 
1471   auto *N = newSDNode<GlobalAddressSDNode>(
1472       Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1473   CSEMap.InsertNode(N, IP);
1474     InsertNode(N);
1475   return SDValue(N, 0);
1476 }
1477 
1478 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1479   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1480   FoldingSetNodeID ID;
1481   AddNodeIDNode(ID, Opc, getVTList(VT), None);
1482   ID.AddInteger(FI);
1483   void *IP = nullptr;
1484   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1485     return SDValue(E, 0);
1486 
1487   auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1488   CSEMap.InsertNode(N, IP);
1489   InsertNode(N);
1490   return SDValue(N, 0);
1491 }
1492 
1493 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1494                                    unsigned TargetFlags) {
1495   assert((TargetFlags == 0 || isTarget) &&
1496          "Cannot set target flags on target-independent jump tables");
1497   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1498   FoldingSetNodeID ID;
1499   AddNodeIDNode(ID, Opc, getVTList(VT), None);
1500   ID.AddInteger(JTI);
1501   ID.AddInteger(TargetFlags);
1502   void *IP = nullptr;
1503   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1504     return SDValue(E, 0);
1505 
1506   auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1507   CSEMap.InsertNode(N, IP);
1508   InsertNode(N);
1509   return SDValue(N, 0);
1510 }
1511 
1512 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1513                                       MaybeAlign Alignment, int Offset,
1514                                       bool isTarget, unsigned TargetFlags) {
1515   assert((TargetFlags == 0 || isTarget) &&
1516          "Cannot set target flags on target-independent globals");
1517   if (!Alignment)
1518     Alignment = shouldOptForSize()
1519                     ? getDataLayout().getABITypeAlign(C->getType())
1520                     : getDataLayout().getPrefTypeAlign(C->getType());
1521   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1522   FoldingSetNodeID ID;
1523   AddNodeIDNode(ID, Opc, getVTList(VT), None);
1524   ID.AddInteger(Alignment->value());
1525   ID.AddInteger(Offset);
1526   ID.AddPointer(C);
1527   ID.AddInteger(TargetFlags);
1528   void *IP = nullptr;
1529   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1530     return SDValue(E, 0);
1531 
1532   auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1533                                           TargetFlags);
1534   CSEMap.InsertNode(N, IP);
1535   InsertNode(N);
1536   SDValue V = SDValue(N, 0);
1537   NewSDValueDbgMsg(V, "Creating new constant pool: ", this);
1538   return V;
1539 }
1540 
1541 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1542                                       MaybeAlign Alignment, int Offset,
1543                                       bool isTarget, unsigned TargetFlags) {
1544   assert((TargetFlags == 0 || isTarget) &&
1545          "Cannot set target flags on target-independent globals");
1546   if (!Alignment)
1547     Alignment = getDataLayout().getPrefTypeAlign(C->getType());
1548   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1549   FoldingSetNodeID ID;
1550   AddNodeIDNode(ID, Opc, getVTList(VT), None);
1551   ID.AddInteger(Alignment->value());
1552   ID.AddInteger(Offset);
1553   C->addSelectionDAGCSEId(ID);
1554   ID.AddInteger(TargetFlags);
1555   void *IP = nullptr;
1556   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1557     return SDValue(E, 0);
1558 
1559   auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1560                                           TargetFlags);
1561   CSEMap.InsertNode(N, IP);
1562   InsertNode(N);
1563   return SDValue(N, 0);
1564 }
1565 
1566 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1567                                      unsigned TargetFlags) {
1568   FoldingSetNodeID ID;
1569   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1570   ID.AddInteger(Index);
1571   ID.AddInteger(Offset);
1572   ID.AddInteger(TargetFlags);
1573   void *IP = nullptr;
1574   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1575     return SDValue(E, 0);
1576 
1577   auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
1578   CSEMap.InsertNode(N, IP);
1579   InsertNode(N);
1580   return SDValue(N, 0);
1581 }
1582 
1583 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1584   FoldingSetNodeID ID;
1585   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1586   ID.AddPointer(MBB);
1587   void *IP = nullptr;
1588   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1589     return SDValue(E, 0);
1590 
1591   auto *N = newSDNode<BasicBlockSDNode>(MBB);
1592   CSEMap.InsertNode(N, IP);
1593   InsertNode(N);
1594   return SDValue(N, 0);
1595 }
1596 
1597 SDValue SelectionDAG::getValueType(EVT VT) {
1598   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1599       ValueTypeNodes.size())
1600     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1601 
1602   SDNode *&N = VT.isExtended() ?
1603     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1604 
1605   if (N) return SDValue(N, 0);
1606   N = newSDNode<VTSDNode>(VT);
1607   InsertNode(N);
1608   return SDValue(N, 0);
1609 }
1610 
1611 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1612   SDNode *&N = ExternalSymbols[Sym];
1613   if (N) return SDValue(N, 0);
1614   N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1615   InsertNode(N);
1616   return SDValue(N, 0);
1617 }
1618 
1619 SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1620   SDNode *&N = MCSymbols[Sym];
1621   if (N)
1622     return SDValue(N, 0);
1623   N = newSDNode<MCSymbolSDNode>(Sym, VT);
1624   InsertNode(N);
1625   return SDValue(N, 0);
1626 }
1627 
1628 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1629                                               unsigned TargetFlags) {
1630   SDNode *&N =
1631       TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1632   if (N) return SDValue(N, 0);
1633   N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1634   InsertNode(N);
1635   return SDValue(N, 0);
1636 }
1637 
1638 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1639   if ((unsigned)Cond >= CondCodeNodes.size())
1640     CondCodeNodes.resize(Cond+1);
1641 
1642   if (!CondCodeNodes[Cond]) {
1643     auto *N = newSDNode<CondCodeSDNode>(Cond);
1644     CondCodeNodes[Cond] = N;
1645     InsertNode(N);
1646   }
1647 
1648   return SDValue(CondCodeNodes[Cond], 0);
1649 }
1650 
1651 /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
1652 /// point at N1 to point at N2 and indices that point at N2 to point at N1.
1653 static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1654   std::swap(N1, N2);
1655   ShuffleVectorSDNode::commuteMask(M);
1656 }
1657 
1658 SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1659                                        SDValue N2, ArrayRef<int> Mask) {
1660   assert(VT.getVectorNumElements() == Mask.size() &&
1661            "Must have the same number of vector elements as mask elements!");
1662   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
1663          "Invalid VECTOR_SHUFFLE");
1664 
1665   // Canonicalize shuffle undef, undef -> undef
1666   if (N1.isUndef() && N2.isUndef())
1667     return getUNDEF(VT);
1668 
1669   // Validate that all indices in Mask are within the range of the elements
1670   // input to the shuffle.
1671   int NElts = Mask.size();
1672   assert(llvm::all_of(Mask,
1673                       [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
1674          "Index out of range");
1675 
1676   // Copy the mask so we can do any needed cleanup.
1677   SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
1678 
1679   // Canonicalize shuffle v, v -> v, undef
1680   if (N1 == N2) {
1681     N2 = getUNDEF(VT);
1682     for (int i = 0; i != NElts; ++i)
1683       if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
1684   }
1685 
1686   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
1687   if (N1.isUndef())
1688     commuteShuffle(N1, N2, MaskVec);
1689 
1690   if (TLI->hasVectorBlend()) {
1691     // If shuffling a splat, try to blend the splat instead. We do this here so
1692     // that even when this arises during lowering we don't have to re-handle it.
1693     auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1694       BitVector UndefElements;
1695       SDValue Splat = BV->getSplatValue(&UndefElements);
1696       if (!Splat)
1697         return;
1698 
1699       for (int i = 0; i < NElts; ++i) {
1700         if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
1701           continue;
1702 
1703         // If this input comes from undef, mark it as such.
1704         if (UndefElements[MaskVec[i] - Offset]) {
1705           MaskVec[i] = -1;
1706           continue;
1707         }
1708 
1709         // If we can blend a non-undef lane, use that instead.
1710         if (!UndefElements[i])
1711           MaskVec[i] = i + Offset;
1712       }
1713     };
1714     if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1715       BlendSplat(N1BV, 0);
1716     if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1717       BlendSplat(N2BV, NElts);
1718   }
1719 
1720   // Canonicalize all index into lhs, -> shuffle lhs, undef
1721   // Canonicalize all index into rhs, -> shuffle rhs, undef
1722   bool AllLHS = true, AllRHS = true;
1723   bool N2Undef = N2.isUndef();
1724   for (int i = 0; i != NElts; ++i) {
1725     if (MaskVec[i] >= NElts) {
1726       if (N2Undef)
1727         MaskVec[i] = -1;
1728       else
1729         AllLHS = false;
1730     } else if (MaskVec[i] >= 0) {
1731       AllRHS = false;
1732     }
1733   }
1734   if (AllLHS && AllRHS)
1735     return getUNDEF(VT);
1736   if (AllLHS && !N2Undef)
1737     N2 = getUNDEF(VT);
1738   if (AllRHS) {
1739     N1 = getUNDEF(VT);
1740     commuteShuffle(N1, N2, MaskVec);
1741   }
1742   // Reset our undef status after accounting for the mask.
1743   N2Undef = N2.isUndef();
1744   // Re-check whether both sides ended up undef.
1745   if (N1.isUndef() && N2Undef)
1746     return getUNDEF(VT);
1747 
1748   // If Identity shuffle return that node.
1749   bool Identity = true, AllSame = true;
1750   for (int i = 0; i != NElts; ++i) {
1751     if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
1752     if (MaskVec[i] != MaskVec[0]) AllSame = false;
1753   }
1754   if (Identity && NElts)
1755     return N1;
1756 
1757   // Shuffling a constant splat doesn't change the result.
1758   if (N2Undef) {
1759     SDValue V = N1;
1760 
1761     // Look through any bitcasts. We check that these don't change the number
1762     // (and size) of elements and just changes their types.
1763     while (V.getOpcode() == ISD::BITCAST)
1764       V = V->getOperand(0);
1765 
1766     // A splat should always show up as a build vector node.
1767     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1768       BitVector UndefElements;
1769       SDValue Splat = BV->getSplatValue(&UndefElements);
1770       // If this is a splat of an undef, shuffling it is also undef.
1771       if (Splat && Splat.isUndef())
1772         return getUNDEF(VT);
1773 
1774       bool SameNumElts =
1775           V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
1776 
1777       // We only have a splat which can skip shuffles if there is a splatted
1778       // value and no undef lanes rearranged by the shuffle.
1779       if (Splat && UndefElements.none()) {
1780         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1781         // number of elements match or the value splatted is a zero constant.
1782         if (SameNumElts)
1783           return N1;
1784         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1785           if (C->isNullValue())
1786             return N1;
1787       }
1788 
1789       // If the shuffle itself creates a splat, build the vector directly.
1790       if (AllSame && SameNumElts) {
1791         EVT BuildVT = BV->getValueType(0);
1792         const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1793         SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
1794 
1795         // We may have jumped through bitcasts, so the type of the
1796         // BUILD_VECTOR may not match the type of the shuffle.
1797         if (BuildVT != VT)
1798           NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1799         return NewBV;
1800       }
1801     }
1802   }
1803 
1804   FoldingSetNodeID ID;
1805   SDValue Ops[2] = { N1, N2 };
1806   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1807   for (int i = 0; i != NElts; ++i)
1808     ID.AddInteger(MaskVec[i]);
1809 
1810   void* IP = nullptr;
1811   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1812     return SDValue(E, 0);
1813 
1814   // Allocate the mask array for the node out of the BumpPtrAllocator, since
1815   // SDNode doesn't have access to it.  This memory will be "leaked" when
1816   // the node is deallocated, but recovered when the NodeAllocator is released.
1817   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1818   llvm::copy(MaskVec, MaskAlloc);
1819 
1820   auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
1821                                            dl.getDebugLoc(), MaskAlloc);
1822   createOperands(N, Ops);
1823 
1824   CSEMap.InsertNode(N, IP);
1825   InsertNode(N);
1826   SDValue V = SDValue(N, 0);
1827   NewSDValueDbgMsg(V, "Creating new node: ", this);
1828   return V;
1829 }
1830 
1831 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1832   EVT VT = SV.getValueType(0);
1833   SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1834   ShuffleVectorSDNode::commuteMask(MaskVec);
1835 
1836   SDValue Op0 = SV.getOperand(0);
1837   SDValue Op1 = SV.getOperand(1);
1838   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
1839 }
1840 
1841 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1842   FoldingSetNodeID ID;
1843   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1844   ID.AddInteger(RegNo);
1845   void *IP = nullptr;
1846   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1847     return SDValue(E, 0);
1848 
1849   auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
1850   N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA);
1851   CSEMap.InsertNode(N, IP);
1852   InsertNode(N);
1853   return SDValue(N, 0);
1854 }
1855 
1856 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1857   FoldingSetNodeID ID;
1858   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
1859   ID.AddPointer(RegMask);
1860   void *IP = nullptr;
1861   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1862     return SDValue(E, 0);
1863 
1864   auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
1865   CSEMap.InsertNode(N, IP);
1866   InsertNode(N);
1867   return SDValue(N, 0);
1868 }
1869 
1870 SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
1871                                  MCSymbol *Label) {
1872   return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
1873 }
1874 
1875 SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
1876                                    SDValue Root, MCSymbol *Label) {
1877   FoldingSetNodeID ID;
1878   SDValue Ops[] = { Root };
1879   AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
1880   ID.AddPointer(Label);
1881   void *IP = nullptr;
1882   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1883     return SDValue(E, 0);
1884 
1885   auto *N =
1886       newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
1887   createOperands(N, Ops);
1888 
1889   CSEMap.InsertNode(N, IP);
1890   InsertNode(N);
1891   return SDValue(N, 0);
1892 }
1893 
1894 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
1895                                       int64_t Offset, bool isTarget,
1896                                       unsigned TargetFlags) {
1897   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
1898 
1899   FoldingSetNodeID ID;
1900   AddNodeIDNode(ID, Opc, getVTList(VT), None);
1901   ID.AddPointer(BA);
1902   ID.AddInteger(Offset);
1903   ID.AddInteger(TargetFlags);
1904   void *IP = nullptr;
1905   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1906     return SDValue(E, 0);
1907 
1908   auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
1909   CSEMap.InsertNode(N, IP);
1910   InsertNode(N);
1911   return SDValue(N, 0);
1912 }
1913 
1914 SDValue SelectionDAG::getSrcValue(const Value *V) {
1915   FoldingSetNodeID ID;
1916   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
1917   ID.AddPointer(V);
1918 
1919   void *IP = nullptr;
1920   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1921     return SDValue(E, 0);
1922 
1923   auto *N = newSDNode<SrcValueSDNode>(V);
1924   CSEMap.InsertNode(N, IP);
1925   InsertNode(N);
1926   return SDValue(N, 0);
1927 }
1928 
1929 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1930   FoldingSetNodeID ID;
1931   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
1932   ID.AddPointer(MD);
1933 
1934   void *IP = nullptr;
1935   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1936     return SDValue(E, 0);
1937 
1938   auto *N = newSDNode<MDNodeSDNode>(MD);
1939   CSEMap.InsertNode(N, IP);
1940   InsertNode(N);
1941   return SDValue(N, 0);
1942 }
1943 
1944 SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
1945   if (VT == V.getValueType())
1946     return V;
1947 
1948   return getNode(ISD::BITCAST, SDLoc(V), VT, V);
1949 }
1950 
1951 SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
1952                                        unsigned SrcAS, unsigned DestAS) {
1953   SDValue Ops[] = {Ptr};
1954   FoldingSetNodeID ID;
1955   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
1956   ID.AddInteger(SrcAS);
1957   ID.AddInteger(DestAS);
1958 
1959   void *IP = nullptr;
1960   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1961     return SDValue(E, 0);
1962 
1963   auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
1964                                            VT, SrcAS, DestAS);
1965   createOperands(N, Ops);
1966 
1967   CSEMap.InsertNode(N, IP);
1968   InsertNode(N);
1969   return SDValue(N, 0);
1970 }
1971 
1972 SDValue SelectionDAG::getFreeze(SDValue V) {
1973   return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
1974 }
1975 
1976 /// getShiftAmountOperand - Return the specified value casted to
1977 /// the target's desired shift amount type.
1978 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1979   EVT OpTy = Op.getValueType();
1980   EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
1981   if (OpTy == ShTy || OpTy.isVector()) return Op;
1982 
1983   return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
1984 }
1985 
1986 SDValue SelectionDAG::expandVAArg(SDNode *Node) {
1987   SDLoc dl(Node);
1988   const TargetLowering &TLI = getTargetLoweringInfo();
1989   const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
1990   EVT VT = Node->getValueType(0);
1991   SDValue Tmp1 = Node->getOperand(0);
1992   SDValue Tmp2 = Node->getOperand(1);
1993   const MaybeAlign MA(Node->getConstantOperandVal(3));
1994 
1995   SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
1996                                Tmp2, MachinePointerInfo(V));
1997   SDValue VAList = VAListLoad;
1998 
1999   if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2000     VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2001                      getConstant(MA->value() - 1, dl, VAList.getValueType()));
2002 
2003     VAList =
2004         getNode(ISD::AND, dl, VAList.getValueType(), VAList,
2005                 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
2006   }
2007 
2008   // Increment the pointer, VAList, to the next vaarg
2009   Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2010                  getConstant(getDataLayout().getTypeAllocSize(
2011                                                VT.getTypeForEVT(*getContext())),
2012                              dl, VAList.getValueType()));
2013   // Store the incremented VAList to the legalized pointer
2014   Tmp1 =
2015       getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
2016   // Load the actual argument out of the pointer VAList
2017   return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
2018 }
2019 
2020 SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2021   SDLoc dl(Node);
2022   const TargetLowering &TLI = getTargetLoweringInfo();
2023   // This defaults to loading a pointer from the input and storing it to the
2024   // output, returning the chain.
2025   const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2026   const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2027   SDValue Tmp1 =
2028       getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
2029               Node->getOperand(2), MachinePointerInfo(VS));
2030   return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2031                   MachinePointerInfo(VD));
2032 }
2033 
2034 Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2035   const DataLayout &DL = getDataLayout();
2036   Type *Ty = VT.getTypeForEVT(*getContext());
2037   Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2038 
2039   if (TLI->isTypeLegal(VT) || !VT.isVector())
2040     return RedAlign;
2041 
2042   const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2043   const Align StackAlign = TFI->getStackAlign();
2044 
2045   // See if we can choose a smaller ABI alignment in cases where it's an
2046   // illegal vector type that will get broken down.
2047   if (RedAlign > StackAlign) {
2048     EVT IntermediateVT;
2049     MVT RegisterVT;
2050     unsigned NumIntermediates;
2051     TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,
2052                                 NumIntermediates, RegisterVT);
2053     Ty = IntermediateVT.getTypeForEVT(*getContext());
2054     Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2055     if (RedAlign2 < RedAlign)
2056       RedAlign = RedAlign2;
2057   }
2058 
2059   return RedAlign;
2060 }
2061 
2062 SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2063   MachineFrameInfo &MFI = MF->getFrameInfo();
2064   const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2065   int StackID = 0;
2066   if (Bytes.isScalable())
2067     StackID = TFI->getStackIDForScalableVectors();
2068   // The stack id gives an indication of whether the object is scalable or
2069   // not, so it's safe to pass in the minimum size here.
2070   int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinSize(), Alignment,
2071                                        false, nullptr, StackID);
2072   return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2073 }
2074 
2075 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2076   Type *Ty = VT.getTypeForEVT(*getContext());
2077   Align StackAlign =
2078       std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));
2079   return CreateStackTemporary(VT.getStoreSize(), StackAlign);
2080 }
2081 
2082 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2083   TypeSize VT1Size = VT1.getStoreSize();
2084   TypeSize VT2Size = VT2.getStoreSize();
2085   assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2086          "Don't know how to choose the maximum size when creating a stack "
2087          "temporary");
2088   TypeSize Bytes =
2089       VT1Size.getKnownMinSize() > VT2Size.getKnownMinSize() ? VT1Size : VT2Size;
2090 
2091   Type *Ty1 = VT1.getTypeForEVT(*getContext());
2092   Type *Ty2 = VT2.getTypeForEVT(*getContext());
2093   const DataLayout &DL = getDataLayout();
2094   Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));
2095   return CreateStackTemporary(Bytes, Align);
2096 }
2097 
2098 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2099                                 ISD::CondCode Cond, const SDLoc &dl) {
2100   EVT OpVT = N1.getValueType();
2101 
2102   // These setcc operations always fold.
2103   switch (Cond) {
2104   default: break;
2105   case ISD::SETFALSE:
2106   case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2107   case ISD::SETTRUE:
2108   case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2109 
2110   case ISD::SETOEQ:
2111   case ISD::SETOGT:
2112   case ISD::SETOGE:
2113   case ISD::SETOLT:
2114   case ISD::SETOLE:
2115   case ISD::SETONE:
2116   case ISD::SETO:
2117   case ISD::SETUO:
2118   case ISD::SETUEQ:
2119   case ISD::SETUNE:
2120     assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2121     break;
2122   }
2123 
2124   if (OpVT.isInteger()) {
2125     // For EQ and NE, we can always pick a value for the undef to make the
2126     // predicate pass or fail, so we can return undef.
2127     // Matches behavior in llvm::ConstantFoldCompareInstruction.
2128     // icmp eq/ne X, undef -> undef.
2129     if ((N1.isUndef() || N2.isUndef()) &&
2130         (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2131       return getUNDEF(VT);
2132 
2133     // If both operands are undef, we can return undef for int comparison.
2134     // icmp undef, undef -> undef.
2135     if (N1.isUndef() && N2.isUndef())
2136       return getUNDEF(VT);
2137 
2138     // icmp X, X -> true/false
2139     // icmp X, undef -> true/false because undef could be X.
2140     if (N1 == N2)
2141       return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2142   }
2143 
2144   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2145     const APInt &C2 = N2C->getAPIntValue();
2146     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2147       const APInt &C1 = N1C->getAPIntValue();
2148 
2149       switch (Cond) {
2150       default: llvm_unreachable("Unknown integer setcc!");
2151       case ISD::SETEQ:  return getBoolConstant(C1 == C2, dl, VT, OpVT);
2152       case ISD::SETNE:  return getBoolConstant(C1 != C2, dl, VT, OpVT);
2153       case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT);
2154       case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT);
2155       case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT);
2156       case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT);
2157       case ISD::SETLT:  return getBoolConstant(C1.slt(C2), dl, VT, OpVT);
2158       case ISD::SETGT:  return getBoolConstant(C1.sgt(C2), dl, VT, OpVT);
2159       case ISD::SETLE:  return getBoolConstant(C1.sle(C2), dl, VT, OpVT);
2160       case ISD::SETGE:  return getBoolConstant(C1.sge(C2), dl, VT, OpVT);
2161       }
2162     }
2163   }
2164 
2165   auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2166   auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2167 
2168   if (N1CFP && N2CFP) {
2169     APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2170     switch (Cond) {
2171     default: break;
2172     case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
2173                         return getUNDEF(VT);
2174                       LLVM_FALLTHROUGH;
2175     case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2176                                              OpVT);
2177     case ISD::SETNE:  if (R==APFloat::cmpUnordered)
2178                         return getUNDEF(VT);
2179                       LLVM_FALLTHROUGH;
2180     case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2181                                              R==APFloat::cmpLessThan, dl, VT,
2182                                              OpVT);
2183     case ISD::SETLT:  if (R==APFloat::cmpUnordered)
2184                         return getUNDEF(VT);
2185                       LLVM_FALLTHROUGH;
2186     case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2187                                              OpVT);
2188     case ISD::SETGT:  if (R==APFloat::cmpUnordered)
2189                         return getUNDEF(VT);
2190                       LLVM_FALLTHROUGH;
2191     case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
2192                                              VT, OpVT);
2193     case ISD::SETLE:  if (R==APFloat::cmpUnordered)
2194                         return getUNDEF(VT);
2195                       LLVM_FALLTHROUGH;
2196     case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
2197                                              R==APFloat::cmpEqual, dl, VT,
2198                                              OpVT);
2199     case ISD::SETGE:  if (R==APFloat::cmpUnordered)
2200                         return getUNDEF(VT);
2201                       LLVM_FALLTHROUGH;
2202     case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2203                                          R==APFloat::cmpEqual, dl, VT, OpVT);
2204     case ISD::SETO:   return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2205                                              OpVT);
2206     case ISD::SETUO:  return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2207                                              OpVT);
2208     case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
2209                                              R==APFloat::cmpEqual, dl, VT,
2210                                              OpVT);
2211     case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2212                                              OpVT);
2213     case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
2214                                              R==APFloat::cmpLessThan, dl, VT,
2215                                              OpVT);
2216     case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2217                                              R==APFloat::cmpUnordered, dl, VT,
2218                                              OpVT);
2219     case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
2220                                              VT, OpVT);
2221     case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2222                                              OpVT);
2223     }
2224   } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2225     // Ensure that the constant occurs on the RHS.
2226     ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2227     if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2228       return SDValue();
2229     return getSetCC(dl, VT, N2, N1, SwappedCond);
2230   } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2231              (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2232     // If an operand is known to be a nan (or undef that could be a nan), we can
2233     // fold it.
2234     // Choosing NaN for the undef will always make unordered comparison succeed
2235     // and ordered comparison fails.
2236     // Matches behavior in llvm::ConstantFoldCompareInstruction.
2237     switch (ISD::getUnorderedFlavor(Cond)) {
2238     default:
2239       llvm_unreachable("Unknown flavor!");
2240     case 0: // Known false.
2241       return getBoolConstant(false, dl, VT, OpVT);
2242     case 1: // Known true.
2243       return getBoolConstant(true, dl, VT, OpVT);
2244     case 2: // Undefined.
2245       return getUNDEF(VT);
2246     }
2247   }
2248 
2249   // Could not fold it.
2250   return SDValue();
2251 }
2252 
2253 /// See if the specified operand can be simplified with the knowledge that only
2254 /// the bits specified by DemandedBits are used.
2255 /// TODO: really we should be making this into the DAG equivalent of
2256 /// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2257 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) {
2258   EVT VT = V.getValueType();
2259 
2260   if (VT.isScalableVector())
2261     return SDValue();
2262 
2263   APInt DemandedElts = VT.isVector()
2264                            ? APInt::getAllOnesValue(VT.getVectorNumElements())
2265                            : APInt(1, 1);
2266   return GetDemandedBits(V, DemandedBits, DemandedElts);
2267 }
2268 
2269 /// See if the specified operand can be simplified with the knowledge that only
2270 /// the bits specified by DemandedBits are used in the elements specified by
2271 /// DemandedElts.
2272 /// TODO: really we should be making this into the DAG equivalent of
2273 /// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2274 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits,
2275                                       const APInt &DemandedElts) {
2276   switch (V.getOpcode()) {
2277   default:
2278     return TLI->SimplifyMultipleUseDemandedBits(V, DemandedBits, DemandedElts,
2279                                                 *this, 0);
2280   case ISD::Constant: {
2281     const APInt &CVal = cast<ConstantSDNode>(V)->getAPIntValue();
2282     APInt NewVal = CVal & DemandedBits;
2283     if (NewVal != CVal)
2284       return getConstant(NewVal, SDLoc(V), V.getValueType());
2285     break;
2286   }
2287   case ISD::SRL:
2288     // Only look at single-use SRLs.
2289     if (!V.getNode()->hasOneUse())
2290       break;
2291     if (auto *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2292       // See if we can recursively simplify the LHS.
2293       unsigned Amt = RHSC->getZExtValue();
2294 
2295       // Watch out for shift count overflow though.
2296       if (Amt >= DemandedBits.getBitWidth())
2297         break;
2298       APInt SrcDemandedBits = DemandedBits << Amt;
2299       if (SDValue SimplifyLHS =
2300               GetDemandedBits(V.getOperand(0), SrcDemandedBits))
2301         return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
2302                        V.getOperand(1));
2303     }
2304     break;
2305   }
2306   return SDValue();
2307 }
2308 
2309 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
2310 /// use this predicate to simplify operations downstream.
2311 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2312   unsigned BitWidth = Op.getScalarValueSizeInBits();
2313   return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2314 }
2315 
2316 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
2317 /// this predicate to simplify operations downstream.  Mask is known to be zero
2318 /// for bits that V cannot have.
2319 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2320                                      unsigned Depth) const {
2321   return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);
2322 }
2323 
2324 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2325 /// DemandedElts.  We use this predicate to simplify operations downstream.
2326 /// Mask is known to be zero for bits that V cannot have.
2327 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2328                                      const APInt &DemandedElts,
2329                                      unsigned Depth) const {
2330   return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2331 }
2332 
2333 /// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2334 bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2335                                         unsigned Depth) const {
2336   return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2337 }
2338 
2339 /// isSplatValue - Return true if the vector V has the same value
2340 /// across all DemandedElts. For scalable vectors it does not make
2341 /// sense to specify which elements are demanded or undefined, therefore
2342 /// they are simply ignored.
2343 bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2344                                 APInt &UndefElts) {
2345   EVT VT = V.getValueType();
2346   assert(VT.isVector() && "Vector type expected");
2347 
2348   if (!VT.isScalableVector() && !DemandedElts)
2349     return false; // No demanded elts, better to assume we don't know anything.
2350 
2351   // Deal with some common cases here that work for both fixed and scalable
2352   // vector types.
2353   switch (V.getOpcode()) {
2354   case ISD::SPLAT_VECTOR:
2355     UndefElts = V.getOperand(0).isUndef()
2356                     ? APInt::getAllOnesValue(DemandedElts.getBitWidth())
2357                     : APInt(DemandedElts.getBitWidth(), 0);
2358     return true;
2359   case ISD::ADD:
2360   case ISD::SUB:
2361   case ISD::AND: {
2362     APInt UndefLHS, UndefRHS;
2363     SDValue LHS = V.getOperand(0);
2364     SDValue RHS = V.getOperand(1);
2365     if (isSplatValue(LHS, DemandedElts, UndefLHS) &&
2366         isSplatValue(RHS, DemandedElts, UndefRHS)) {
2367       UndefElts = UndefLHS | UndefRHS;
2368       return true;
2369     }
2370     break;
2371   }
2372   case ISD::TRUNCATE:
2373   case ISD::SIGN_EXTEND:
2374   case ISD::ZERO_EXTEND:
2375     return isSplatValue(V.getOperand(0), DemandedElts, UndefElts);
2376   }
2377 
2378   // We don't support other cases than those above for scalable vectors at
2379   // the moment.
2380   if (VT.isScalableVector())
2381     return false;
2382 
2383   unsigned NumElts = VT.getVectorNumElements();
2384   assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
2385   UndefElts = APInt::getNullValue(NumElts);
2386 
2387   switch (V.getOpcode()) {
2388   case ISD::BUILD_VECTOR: {
2389     SDValue Scl;
2390     for (unsigned i = 0; i != NumElts; ++i) {
2391       SDValue Op = V.getOperand(i);
2392       if (Op.isUndef()) {
2393         UndefElts.setBit(i);
2394         continue;
2395       }
2396       if (!DemandedElts[i])
2397         continue;
2398       if (Scl && Scl != Op)
2399         return false;
2400       Scl = Op;
2401     }
2402     return true;
2403   }
2404   case ISD::VECTOR_SHUFFLE: {
2405     // Check if this is a shuffle node doing a splat.
2406     // TODO: Do we need to handle shuffle(splat, undef, mask)?
2407     int SplatIndex = -1;
2408     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2409     for (int i = 0; i != (int)NumElts; ++i) {
2410       int M = Mask[i];
2411       if (M < 0) {
2412         UndefElts.setBit(i);
2413         continue;
2414       }
2415       if (!DemandedElts[i])
2416         continue;
2417       if (0 <= SplatIndex && SplatIndex != M)
2418         return false;
2419       SplatIndex = M;
2420     }
2421     return true;
2422   }
2423   case ISD::EXTRACT_SUBVECTOR: {
2424     // Offset the demanded elts by the subvector index.
2425     SDValue Src = V.getOperand(0);
2426     uint64_t Idx = V.getConstantOperandVal(1);
2427     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2428     APInt UndefSrcElts;
2429     APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2430     if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts)) {
2431       UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2432       return true;
2433     }
2434     break;
2435   }
2436   }
2437 
2438   return false;
2439 }
2440 
2441 /// Helper wrapper to main isSplatValue function.
2442 bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) {
2443   EVT VT = V.getValueType();
2444   assert(VT.isVector() && "Vector type expected");
2445 
2446   APInt UndefElts;
2447   APInt DemandedElts;
2448 
2449   // For now we don't support this with scalable vectors.
2450   if (!VT.isScalableVector())
2451     DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
2452   return isSplatValue(V, DemandedElts, UndefElts) &&
2453          (AllowUndefs || !UndefElts);
2454 }
2455 
2456 SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2457   V = peekThroughExtractSubvectors(V);
2458 
2459   EVT VT = V.getValueType();
2460   unsigned Opcode = V.getOpcode();
2461   switch (Opcode) {
2462   default: {
2463     APInt UndefElts;
2464     APInt DemandedElts;
2465 
2466     if (!VT.isScalableVector())
2467       DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
2468 
2469     if (isSplatValue(V, DemandedElts, UndefElts)) {
2470       if (VT.isScalableVector()) {
2471         // DemandedElts and UndefElts are ignored for scalable vectors, since
2472         // the only supported cases are SPLAT_VECTOR nodes.
2473         SplatIdx = 0;
2474       } else {
2475         // Handle case where all demanded elements are UNDEF.
2476         if (DemandedElts.isSubsetOf(UndefElts)) {
2477           SplatIdx = 0;
2478           return getUNDEF(VT);
2479         }
2480         SplatIdx = (UndefElts & DemandedElts).countTrailingOnes();
2481       }
2482       return V;
2483     }
2484     break;
2485   }
2486   case ISD::SPLAT_VECTOR:
2487     SplatIdx = 0;
2488     return V;
2489   case ISD::VECTOR_SHUFFLE: {
2490     if (VT.isScalableVector())
2491       return SDValue();
2492 
2493     // Check if this is a shuffle node doing a splat.
2494     // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2495     // getTargetVShiftNode currently struggles without the splat source.
2496     auto *SVN = cast<ShuffleVectorSDNode>(V);
2497     if (!SVN->isSplat())
2498       break;
2499     int Idx = SVN->getSplatIndex();
2500     int NumElts = V.getValueType().getVectorNumElements();
2501     SplatIdx = Idx % NumElts;
2502     return V.getOperand(Idx / NumElts);
2503   }
2504   }
2505 
2506   return SDValue();
2507 }
2508 
2509 SDValue SelectionDAG::getSplatValue(SDValue V) {
2510   int SplatIdx;
2511   if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx))
2512     return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V),
2513                    SrcVector.getValueType().getScalarType(), SrcVector,
2514                    getVectorIdxConstant(SplatIdx, SDLoc(V)));
2515   return SDValue();
2516 }
2517 
2518 const APInt *
2519 SelectionDAG::getValidShiftAmountConstant(SDValue V,
2520                                           const APInt &DemandedElts) const {
2521   assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2522           V.getOpcode() == ISD::SRA) &&
2523          "Unknown shift node");
2524   unsigned BitWidth = V.getScalarValueSizeInBits();
2525   if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) {
2526     // Shifting more than the bitwidth is not valid.
2527     const APInt &ShAmt = SA->getAPIntValue();
2528     if (ShAmt.ult(BitWidth))
2529       return &ShAmt;
2530   }
2531   return nullptr;
2532 }
2533 
2534 const APInt *SelectionDAG::getValidMinimumShiftAmountConstant(
2535     SDValue V, const APInt &DemandedElts) const {
2536   assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2537           V.getOpcode() == ISD::SRA) &&
2538          "Unknown shift node");
2539   if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2540     return ValidAmt;
2541   unsigned BitWidth = V.getScalarValueSizeInBits();
2542   auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2543   if (!BV)
2544     return nullptr;
2545   const APInt *MinShAmt = nullptr;
2546   for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
2547     if (!DemandedElts[i])
2548       continue;
2549     auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
2550     if (!SA)
2551       return nullptr;
2552     // Shifting more than the bitwidth is not valid.
2553     const APInt &ShAmt = SA->getAPIntValue();
2554     if (ShAmt.uge(BitWidth))
2555       return nullptr;
2556     if (MinShAmt && MinShAmt->ule(ShAmt))
2557       continue;
2558     MinShAmt = &ShAmt;
2559   }
2560   return MinShAmt;
2561 }
2562 
2563 const APInt *SelectionDAG::getValidMaximumShiftAmountConstant(
2564     SDValue V, const APInt &DemandedElts) const {
2565   assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2566           V.getOpcode() == ISD::SRA) &&
2567          "Unknown shift node");
2568   if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2569     return ValidAmt;
2570   unsigned BitWidth = V.getScalarValueSizeInBits();
2571   auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2572   if (!BV)
2573     return nullptr;
2574   const APInt *MaxShAmt = nullptr;
2575   for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
2576     if (!DemandedElts[i])
2577       continue;
2578     auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
2579     if (!SA)
2580       return nullptr;
2581     // Shifting more than the bitwidth is not valid.
2582     const APInt &ShAmt = SA->getAPIntValue();
2583     if (ShAmt.uge(BitWidth))
2584       return nullptr;
2585     if (MaxShAmt && MaxShAmt->uge(ShAmt))
2586       continue;
2587     MaxShAmt = &ShAmt;
2588   }
2589   return MaxShAmt;
2590 }
2591 
2592 /// Determine which bits of Op are known to be either zero or one and return
2593 /// them in Known. For vectors, the known bits are those that are shared by
2594 /// every vector element.
2595 KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
2596   EVT VT = Op.getValueType();
2597 
2598   // TOOD: Until we have a plan for how to represent demanded elements for
2599   // scalable vectors, we can just bail out for now.
2600   if (Op.getValueType().isScalableVector()) {
2601     unsigned BitWidth = Op.getScalarValueSizeInBits();
2602     return KnownBits(BitWidth);
2603   }
2604 
2605   APInt DemandedElts = VT.isVector()
2606                            ? APInt::getAllOnesValue(VT.getVectorNumElements())
2607                            : APInt(1, 1);
2608   return computeKnownBits(Op, DemandedElts, Depth);
2609 }
2610 
2611 /// Determine which bits of Op are known to be either zero or one and return
2612 /// them in Known. The DemandedElts argument allows us to only collect the known
2613 /// bits that are shared by the requested vector elements.
2614 KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
2615                                          unsigned Depth) const {
2616   unsigned BitWidth = Op.getScalarValueSizeInBits();
2617 
2618   KnownBits Known(BitWidth);   // Don't know anything.
2619 
2620   // TOOD: Until we have a plan for how to represent demanded elements for
2621   // scalable vectors, we can just bail out for now.
2622   if (Op.getValueType().isScalableVector())
2623     return Known;
2624 
2625   if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2626     // We know all of the bits for a constant!
2627     return KnownBits::makeConstant(C->getAPIntValue());
2628   }
2629   if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
2630     // We know all of the bits for a constant fp!
2631     return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt());
2632   }
2633 
2634   if (Depth >= MaxRecursionDepth)
2635     return Known;  // Limit search depth.
2636 
2637   KnownBits Known2;
2638   unsigned NumElts = DemandedElts.getBitWidth();
2639   assert((!Op.getValueType().isVector() ||
2640           NumElts == Op.getValueType().getVectorNumElements()) &&
2641          "Unexpected vector size");
2642 
2643   if (!DemandedElts)
2644     return Known;  // No demanded elts, better to assume we don't know anything.
2645 
2646   unsigned Opcode = Op.getOpcode();
2647   switch (Opcode) {
2648   case ISD::BUILD_VECTOR:
2649     // Collect the known bits that are shared by every demanded vector element.
2650     Known.Zero.setAllBits(); Known.One.setAllBits();
2651     for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
2652       if (!DemandedElts[i])
2653         continue;
2654 
2655       SDValue SrcOp = Op.getOperand(i);
2656       Known2 = computeKnownBits(SrcOp, Depth + 1);
2657 
2658       // BUILD_VECTOR can implicitly truncate sources, we must handle this.
2659       if (SrcOp.getValueSizeInBits() != BitWidth) {
2660         assert(SrcOp.getValueSizeInBits() > BitWidth &&
2661                "Expected BUILD_VECTOR implicit truncation");
2662         Known2 = Known2.trunc(BitWidth);
2663       }
2664 
2665       // Known bits are the values that are shared by every demanded element.
2666       Known = KnownBits::commonBits(Known, Known2);
2667 
2668       // If we don't know any bits, early out.
2669       if (Known.isUnknown())
2670         break;
2671     }
2672     break;
2673   case ISD::VECTOR_SHUFFLE: {
2674     // Collect the known bits that are shared by every vector element referenced
2675     // by the shuffle.
2676     APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
2677     Known.Zero.setAllBits(); Known.One.setAllBits();
2678     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
2679     assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
2680     for (unsigned i = 0; i != NumElts; ++i) {
2681       if (!DemandedElts[i])
2682         continue;
2683 
2684       int M = SVN->getMaskElt(i);
2685       if (M < 0) {
2686         // For UNDEF elements, we don't know anything about the common state of
2687         // the shuffle result.
2688         Known.resetAll();
2689         DemandedLHS.clearAllBits();
2690         DemandedRHS.clearAllBits();
2691         break;
2692       }
2693 
2694       if ((unsigned)M < NumElts)
2695         DemandedLHS.setBit((unsigned)M % NumElts);
2696       else
2697         DemandedRHS.setBit((unsigned)M % NumElts);
2698     }
2699     // Known bits are the values that are shared by every demanded element.
2700     if (!!DemandedLHS) {
2701       SDValue LHS = Op.getOperand(0);
2702       Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
2703       Known = KnownBits::commonBits(Known, Known2);
2704     }
2705     // If we don't know any bits, early out.
2706     if (Known.isUnknown())
2707       break;
2708     if (!!DemandedRHS) {
2709       SDValue RHS = Op.getOperand(1);
2710       Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
2711       Known = KnownBits::commonBits(Known, Known2);
2712     }
2713     break;
2714   }
2715   case ISD::CONCAT_VECTORS: {
2716     // Split DemandedElts and test each of the demanded subvectors.
2717     Known.Zero.setAllBits(); Known.One.setAllBits();
2718     EVT SubVectorVT = Op.getOperand(0).getValueType();
2719     unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
2720     unsigned NumSubVectors = Op.getNumOperands();
2721     for (unsigned i = 0; i != NumSubVectors; ++i) {
2722       APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
2723       DemandedSub = DemandedSub.trunc(NumSubVectorElts);
2724       if (!!DemandedSub) {
2725         SDValue Sub = Op.getOperand(i);
2726         Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
2727         Known = KnownBits::commonBits(Known, Known2);
2728       }
2729       // If we don't know any bits, early out.
2730       if (Known.isUnknown())
2731         break;
2732     }
2733     break;
2734   }
2735   case ISD::INSERT_SUBVECTOR: {
2736     // Demand any elements from the subvector and the remainder from the src its
2737     // inserted into.
2738     SDValue Src = Op.getOperand(0);
2739     SDValue Sub = Op.getOperand(1);
2740     uint64_t Idx = Op.getConstantOperandVal(2);
2741     unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2742     APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2743     APInt DemandedSrcElts = DemandedElts;
2744     DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
2745 
2746     Known.One.setAllBits();
2747     Known.Zero.setAllBits();
2748     if (!!DemandedSubElts) {
2749       Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
2750       if (Known.isUnknown())
2751         break; // early-out.
2752     }
2753     if (!!DemandedSrcElts) {
2754       Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2755       Known = KnownBits::commonBits(Known, Known2);
2756     }
2757     break;
2758   }
2759   case ISD::EXTRACT_SUBVECTOR: {
2760     // Offset the demanded elts by the subvector index.
2761     SDValue Src = Op.getOperand(0);
2762     // Bail until we can represent demanded elements for scalable vectors.
2763     if (Src.getValueType().isScalableVector())
2764       break;
2765     uint64_t Idx = Op.getConstantOperandVal(1);
2766     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2767     APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2768     Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2769     break;
2770   }
2771   case ISD::SCALAR_TO_VECTOR: {
2772     // We know about scalar_to_vector as much as we know about it source,
2773     // which becomes the first element of otherwise unknown vector.
2774     if (DemandedElts != 1)
2775       break;
2776 
2777     SDValue N0 = Op.getOperand(0);
2778     Known = computeKnownBits(N0, Depth + 1);
2779     if (N0.getValueSizeInBits() != BitWidth)
2780       Known = Known.trunc(BitWidth);
2781 
2782     break;
2783   }
2784   case ISD::BITCAST: {
2785     SDValue N0 = Op.getOperand(0);
2786     EVT SubVT = N0.getValueType();
2787     unsigned SubBitWidth = SubVT.getScalarSizeInBits();
2788 
2789     // Ignore bitcasts from unsupported types.
2790     if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
2791       break;
2792 
2793     // Fast handling of 'identity' bitcasts.
2794     if (BitWidth == SubBitWidth) {
2795       Known = computeKnownBits(N0, DemandedElts, Depth + 1);
2796       break;
2797     }
2798 
2799     bool IsLE = getDataLayout().isLittleEndian();
2800 
2801     // Bitcast 'small element' vector to 'large element' scalar/vector.
2802     if ((BitWidth % SubBitWidth) == 0) {
2803       assert(N0.getValueType().isVector() && "Expected bitcast from vector");
2804 
2805       // Collect known bits for the (larger) output by collecting the known
2806       // bits from each set of sub elements and shift these into place.
2807       // We need to separately call computeKnownBits for each set of
2808       // sub elements as the knownbits for each is likely to be different.
2809       unsigned SubScale = BitWidth / SubBitWidth;
2810       APInt SubDemandedElts(NumElts * SubScale, 0);
2811       for (unsigned i = 0; i != NumElts; ++i)
2812         if (DemandedElts[i])
2813           SubDemandedElts.setBit(i * SubScale);
2814 
2815       for (unsigned i = 0; i != SubScale; ++i) {
2816         Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
2817                          Depth + 1);
2818         unsigned Shifts = IsLE ? i : SubScale - 1 - i;
2819         Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * Shifts);
2820         Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * Shifts);
2821       }
2822     }
2823 
2824     // Bitcast 'large element' scalar/vector to 'small element' vector.
2825     if ((SubBitWidth % BitWidth) == 0) {
2826       assert(Op.getValueType().isVector() && "Expected bitcast to vector");
2827 
2828       // Collect known bits for the (smaller) output by collecting the known
2829       // bits from the overlapping larger input elements and extracting the
2830       // sub sections we actually care about.
2831       unsigned SubScale = SubBitWidth / BitWidth;
2832       APInt SubDemandedElts(NumElts / SubScale, 0);
2833       for (unsigned i = 0; i != NumElts; ++i)
2834         if (DemandedElts[i])
2835           SubDemandedElts.setBit(i / SubScale);
2836 
2837       Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
2838 
2839       Known.Zero.setAllBits(); Known.One.setAllBits();
2840       for (unsigned i = 0; i != NumElts; ++i)
2841         if (DemandedElts[i]) {
2842           unsigned Shifts = IsLE ? i : NumElts - 1 - i;
2843           unsigned Offset = (Shifts % SubScale) * BitWidth;
2844           Known.One &= Known2.One.lshr(Offset).trunc(BitWidth);
2845           Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth);
2846           // If we don't know any bits, early out.
2847           if (Known.isUnknown())
2848             break;
2849         }
2850     }
2851     break;
2852   }
2853   case ISD::AND:
2854     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2855     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2856 
2857     Known &= Known2;
2858     break;
2859   case ISD::OR:
2860     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2861     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2862 
2863     Known |= Known2;
2864     break;
2865   case ISD::XOR:
2866     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2867     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2868 
2869     Known ^= Known2;
2870     break;
2871   case ISD::MUL: {
2872     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2873     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2874     Known = KnownBits::computeForMul(Known, Known2);
2875     break;
2876   }
2877   case ISD::UDIV: {
2878     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2879     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2880     Known = KnownBits::udiv(Known, Known2);
2881     break;
2882   }
2883   case ISD::SELECT:
2884   case ISD::VSELECT:
2885     Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
2886     // If we don't know any bits, early out.
2887     if (Known.isUnknown())
2888       break;
2889     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
2890 
2891     // Only known if known in both the LHS and RHS.
2892     Known = KnownBits::commonBits(Known, Known2);
2893     break;
2894   case ISD::SELECT_CC:
2895     Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
2896     // If we don't know any bits, early out.
2897     if (Known.isUnknown())
2898       break;
2899     Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
2900 
2901     // Only known if known in both the LHS and RHS.
2902     Known = KnownBits::commonBits(Known, Known2);
2903     break;
2904   case ISD::SMULO:
2905   case ISD::UMULO:
2906   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
2907     if (Op.getResNo() != 1)
2908       break;
2909     // The boolean result conforms to getBooleanContents.
2910     // If we know the result of a setcc has the top bits zero, use this info.
2911     // We know that we have an integer-based boolean since these operations
2912     // are only available for integer.
2913     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2914             TargetLowering::ZeroOrOneBooleanContent &&
2915         BitWidth > 1)
2916       Known.Zero.setBitsFrom(1);
2917     break;
2918   case ISD::SETCC:
2919   case ISD::STRICT_FSETCC:
2920   case ISD::STRICT_FSETCCS: {
2921     unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
2922     // If we know the result of a setcc has the top bits zero, use this info.
2923     if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
2924             TargetLowering::ZeroOrOneBooleanContent &&
2925         BitWidth > 1)
2926       Known.Zero.setBitsFrom(1);
2927     break;
2928   }
2929   case ISD::SHL:
2930     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2931     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2932     Known = KnownBits::shl(Known, Known2);
2933 
2934     // Minimum shift low bits are known zero.
2935     if (const APInt *ShMinAmt =
2936             getValidMinimumShiftAmountConstant(Op, DemandedElts))
2937       Known.Zero.setLowBits(ShMinAmt->getZExtValue());
2938     break;
2939   case ISD::SRL:
2940     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2941     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2942     Known = KnownBits::lshr(Known, Known2);
2943 
2944     // Minimum shift high bits are known zero.
2945     if (const APInt *ShMinAmt =
2946             getValidMinimumShiftAmountConstant(Op, DemandedElts))
2947       Known.Zero.setHighBits(ShMinAmt->getZExtValue());
2948     break;
2949   case ISD::SRA:
2950     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2951     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2952     Known = KnownBits::ashr(Known, Known2);
2953     // TODO: Add minimum shift high known sign bits.
2954     break;
2955   case ISD::FSHL:
2956   case ISD::FSHR:
2957     if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
2958       unsigned Amt = C->getAPIntValue().urem(BitWidth);
2959 
2960       // For fshl, 0-shift returns the 1st arg.
2961       // For fshr, 0-shift returns the 2nd arg.
2962       if (Amt == 0) {
2963         Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
2964                                  DemandedElts, Depth + 1);
2965         break;
2966       }
2967 
2968       // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
2969       // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
2970       Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2971       Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2972       if (Opcode == ISD::FSHL) {
2973         Known.One <<= Amt;
2974         Known.Zero <<= Amt;
2975         Known2.One.lshrInPlace(BitWidth - Amt);
2976         Known2.Zero.lshrInPlace(BitWidth - Amt);
2977       } else {
2978         Known.One <<= BitWidth - Amt;
2979         Known.Zero <<= BitWidth - Amt;
2980         Known2.One.lshrInPlace(Amt);
2981         Known2.Zero.lshrInPlace(Amt);
2982       }
2983       Known.One |= Known2.One;
2984       Known.Zero |= Known2.Zero;
2985     }
2986     break;
2987   case ISD::SIGN_EXTEND_INREG: {
2988     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2989     unsigned EBits = EVT.getScalarSizeInBits();
2990 
2991     // Sign extension.  Compute the demanded bits in the result that are not
2992     // present in the input.
2993     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
2994 
2995     APInt InSignMask = APInt::getSignMask(EBits);
2996     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
2997 
2998     // If the sign extended bits are demanded, we know that the sign
2999     // bit is demanded.
3000     InSignMask = InSignMask.zext(BitWidth);
3001     if (NewBits.getBoolValue())
3002       InputDemandedBits |= InSignMask;
3003 
3004     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3005     Known.One &= InputDemandedBits;
3006     Known.Zero &= InputDemandedBits;
3007 
3008     // If the sign bit of the input is known set or clear, then we know the
3009     // top bits of the result.
3010     if (Known.Zero.intersects(InSignMask)) {        // Input sign bit known clear
3011       Known.Zero |= NewBits;
3012       Known.One  &= ~NewBits;
3013     } else if (Known.One.intersects(InSignMask)) {  // Input sign bit known set
3014       Known.One  |= NewBits;
3015       Known.Zero &= ~NewBits;
3016     } else {                              // Input sign bit unknown
3017       Known.Zero &= ~NewBits;
3018       Known.One  &= ~NewBits;
3019     }
3020     break;
3021   }
3022   case ISD::CTTZ:
3023   case ISD::CTTZ_ZERO_UNDEF: {
3024     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3025     // If we have a known 1, its position is our upper bound.
3026     unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3027     unsigned LowBits = Log2_32(PossibleTZ) + 1;
3028     Known.Zero.setBitsFrom(LowBits);
3029     break;
3030   }
3031   case ISD::CTLZ:
3032   case ISD::CTLZ_ZERO_UNDEF: {
3033     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3034     // If we have a known 1, its position is our upper bound.
3035     unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3036     unsigned LowBits = Log2_32(PossibleLZ) + 1;
3037     Known.Zero.setBitsFrom(LowBits);
3038     break;
3039   }
3040   case ISD::CTPOP: {
3041     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3042     // If we know some of the bits are zero, they can't be one.
3043     unsigned PossibleOnes = Known2.countMaxPopulation();
3044     Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
3045     break;
3046   }
3047   case ISD::PARITY: {
3048     // Parity returns 0 everywhere but the LSB.
3049     Known.Zero.setBitsFrom(1);
3050     break;
3051   }
3052   case ISD::LOAD: {
3053     LoadSDNode *LD = cast<LoadSDNode>(Op);
3054     const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3055     if (ISD::isNON_EXTLoad(LD) && Cst) {
3056       // Determine any common known bits from the loaded constant pool value.
3057       Type *CstTy = Cst->getType();
3058       if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits()) {
3059         // If its a vector splat, then we can (quickly) reuse the scalar path.
3060         // NOTE: We assume all elements match and none are UNDEF.
3061         if (CstTy->isVectorTy()) {
3062           if (const Constant *Splat = Cst->getSplatValue()) {
3063             Cst = Splat;
3064             CstTy = Cst->getType();
3065           }
3066         }
3067         // TODO - do we need to handle different bitwidths?
3068         if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3069           // Iterate across all vector elements finding common known bits.
3070           Known.One.setAllBits();
3071           Known.Zero.setAllBits();
3072           for (unsigned i = 0; i != NumElts; ++i) {
3073             if (!DemandedElts[i])
3074               continue;
3075             if (Constant *Elt = Cst->getAggregateElement(i)) {
3076               if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3077                 const APInt &Value = CInt->getValue();
3078                 Known.One &= Value;
3079                 Known.Zero &= ~Value;
3080                 continue;
3081               }
3082               if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3083                 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3084                 Known.One &= Value;
3085                 Known.Zero &= ~Value;
3086                 continue;
3087               }
3088             }
3089             Known.One.clearAllBits();
3090             Known.Zero.clearAllBits();
3091             break;
3092           }
3093         } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3094           if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3095             const APInt &Value = CInt->getValue();
3096             Known.One = Value;
3097             Known.Zero = ~Value;
3098           } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3099             APInt Value = CFP->getValueAPF().bitcastToAPInt();
3100             Known.One = Value;
3101             Known.Zero = ~Value;
3102           }
3103         }
3104       }
3105     } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
3106       // If this is a ZEXTLoad and we are looking at the loaded value.
3107       EVT VT = LD->getMemoryVT();
3108       unsigned MemBits = VT.getScalarSizeInBits();
3109       Known.Zero.setBitsFrom(MemBits);
3110     } else if (const MDNode *Ranges = LD->getRanges()) {
3111       if (LD->getExtensionType() == ISD::NON_EXTLOAD)
3112         computeKnownBitsFromRangeMetadata(*Ranges, Known);
3113     }
3114     break;
3115   }
3116   case ISD::ZERO_EXTEND_VECTOR_INREG: {
3117     EVT InVT = Op.getOperand(0).getValueType();
3118     APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3119     Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3120     Known = Known.zext(BitWidth);
3121     break;
3122   }
3123   case ISD::ZERO_EXTEND: {
3124     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3125     Known = Known.zext(BitWidth);
3126     break;
3127   }
3128   case ISD::SIGN_EXTEND_VECTOR_INREG: {
3129     EVT InVT = Op.getOperand(0).getValueType();
3130     APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3131     Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3132     // If the sign bit is known to be zero or one, then sext will extend
3133     // it to the top bits, else it will just zext.
3134     Known = Known.sext(BitWidth);
3135     break;
3136   }
3137   case ISD::SIGN_EXTEND: {
3138     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3139     // If the sign bit is known to be zero or one, then sext will extend
3140     // it to the top bits, else it will just zext.
3141     Known = Known.sext(BitWidth);
3142     break;
3143   }
3144   case ISD::ANY_EXTEND_VECTOR_INREG: {
3145     EVT InVT = Op.getOperand(0).getValueType();
3146     APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3147     Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3148     Known = Known.anyext(BitWidth);
3149     break;
3150   }
3151   case ISD::ANY_EXTEND: {
3152     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3153     Known = Known.anyext(BitWidth);
3154     break;
3155   }
3156   case ISD::TRUNCATE: {
3157     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3158     Known = Known.trunc(BitWidth);
3159     break;
3160   }
3161   case ISD::AssertZext: {
3162     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3163     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
3164     Known = computeKnownBits(Op.getOperand(0), Depth+1);
3165     Known.Zero |= (~InMask);
3166     Known.One  &= (~Known.Zero);
3167     break;
3168   }
3169   case ISD::AssertAlign: {
3170     unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
3171     assert(LogOfAlign != 0);
3172     // If a node is guaranteed to be aligned, set low zero bits accordingly as
3173     // well as clearing one bits.
3174     Known.Zero.setLowBits(LogOfAlign);
3175     Known.One.clearLowBits(LogOfAlign);
3176     break;
3177   }
3178   case ISD::FGETSIGN:
3179     // All bits are zero except the low bit.
3180     Known.Zero.setBitsFrom(1);
3181     break;
3182   case ISD::USUBO:
3183   case ISD::SSUBO:
3184     if (Op.getResNo() == 1) {
3185       // If we know the result of a setcc has the top bits zero, use this info.
3186       if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3187               TargetLowering::ZeroOrOneBooleanContent &&
3188           BitWidth > 1)
3189         Known.Zero.setBitsFrom(1);
3190       break;
3191     }
3192     LLVM_FALLTHROUGH;
3193   case ISD::SUB:
3194   case ISD::SUBC: {
3195     assert(Op.getResNo() == 0 &&
3196            "We only compute knownbits for the difference here.");
3197 
3198     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3199     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3200     Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false,
3201                                         Known, Known2);
3202     break;
3203   }
3204   case ISD::UADDO:
3205   case ISD::SADDO:
3206   case ISD::ADDCARRY:
3207     if (Op.getResNo() == 1) {
3208       // If we know the result of a setcc has the top bits zero, use this info.
3209       if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3210               TargetLowering::ZeroOrOneBooleanContent &&
3211           BitWidth > 1)
3212         Known.Zero.setBitsFrom(1);
3213       break;
3214     }
3215     LLVM_FALLTHROUGH;
3216   case ISD::ADD:
3217   case ISD::ADDC:
3218   case ISD::ADDE: {
3219     assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
3220 
3221     // With ADDE and ADDCARRY, a carry bit may be added in.
3222     KnownBits Carry(1);
3223     if (Opcode == ISD::ADDE)
3224       // Can't track carry from glue, set carry to unknown.
3225       Carry.resetAll();
3226     else if (Opcode == ISD::ADDCARRY)
3227       // TODO: Compute known bits for the carry operand. Not sure if it is worth
3228       // the trouble (how often will we find a known carry bit). And I haven't
3229       // tested this very much yet, but something like this might work:
3230       //   Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3231       //   Carry = Carry.zextOrTrunc(1, false);
3232       Carry.resetAll();
3233     else
3234       Carry.setAllZero();
3235 
3236     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3237     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3238     Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3239     break;
3240   }
3241   case ISD::SREM: {
3242     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3243     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3244     Known = KnownBits::srem(Known, Known2);
3245     break;
3246   }
3247   case ISD::UREM: {
3248     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3249     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3250     Known = KnownBits::urem(Known, Known2);
3251     break;
3252   }
3253   case ISD::EXTRACT_ELEMENT: {
3254     Known = computeKnownBits(Op.getOperand(0), Depth+1);
3255     const unsigned Index = Op.getConstantOperandVal(1);
3256     const unsigned EltBitWidth = Op.getValueSizeInBits();
3257 
3258     // Remove low part of known bits mask
3259     Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3260     Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3261 
3262     // Remove high part of known bit mask
3263     Known = Known.trunc(EltBitWidth);
3264     break;
3265   }
3266   case ISD::EXTRACT_VECTOR_ELT: {
3267     SDValue InVec = Op.getOperand(0);
3268     SDValue EltNo = Op.getOperand(1);
3269     EVT VecVT = InVec.getValueType();
3270     // computeKnownBits not yet implemented for scalable vectors.
3271     if (VecVT.isScalableVector())
3272       break;
3273     const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3274     const unsigned NumSrcElts = VecVT.getVectorNumElements();
3275 
3276     // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3277     // anything about the extended bits.
3278     if (BitWidth > EltBitWidth)
3279       Known = Known.trunc(EltBitWidth);
3280 
3281     // If we know the element index, just demand that vector element, else for
3282     // an unknown element index, ignore DemandedElts and demand them all.
3283     APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
3284     auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3285     if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3286       DemandedSrcElts =
3287           APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3288 
3289     Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);
3290     if (BitWidth > EltBitWidth)
3291       Known = Known.anyext(BitWidth);
3292     break;
3293   }
3294   case ISD::INSERT_VECTOR_ELT: {
3295     // If we know the element index, split the demand between the
3296     // source vector and the inserted element, otherwise assume we need
3297     // the original demanded vector elements and the value.
3298     SDValue InVec = Op.getOperand(0);
3299     SDValue InVal = Op.getOperand(1);
3300     SDValue EltNo = Op.getOperand(2);
3301     bool DemandedVal = true;
3302     APInt DemandedVecElts = DemandedElts;
3303     auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3304     if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3305       unsigned EltIdx = CEltNo->getZExtValue();
3306       DemandedVal = !!DemandedElts[EltIdx];
3307       DemandedVecElts.clearBit(EltIdx);
3308     }
3309     Known.One.setAllBits();
3310     Known.Zero.setAllBits();
3311     if (DemandedVal) {
3312       Known2 = computeKnownBits(InVal, Depth + 1);
3313       Known = KnownBits::commonBits(Known, Known2.zextOrTrunc(BitWidth));
3314     }
3315     if (!!DemandedVecElts) {
3316       Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);
3317       Known = KnownBits::commonBits(Known, Known2);
3318     }
3319     break;
3320   }
3321   case ISD::BITREVERSE: {
3322     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3323     Known = Known2.reverseBits();
3324     break;
3325   }
3326   case ISD::BSWAP: {
3327     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3328     Known = Known2.byteSwap();
3329     break;
3330   }
3331   case ISD::ABS: {
3332     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3333     Known = Known2.abs();
3334     break;
3335   }
3336   case ISD::UMIN: {
3337     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3338     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3339     Known = KnownBits::umin(Known, Known2);
3340     break;
3341   }
3342   case ISD::UMAX: {
3343     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3344     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3345     Known = KnownBits::umax(Known, Known2);
3346     break;
3347   }
3348   case ISD::SMIN:
3349   case ISD::SMAX: {
3350     // If we have a clamp pattern, we know that the number of sign bits will be
3351     // the minimum of the clamp min/max range.
3352     bool IsMax = (Opcode == ISD::SMAX);
3353     ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3354     if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3355       if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3356         CstHigh =
3357             isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3358     if (CstLow && CstHigh) {
3359       if (!IsMax)
3360         std::swap(CstLow, CstHigh);
3361 
3362       const APInt &ValueLow = CstLow->getAPIntValue();
3363       const APInt &ValueHigh = CstHigh->getAPIntValue();
3364       if (ValueLow.sle(ValueHigh)) {
3365         unsigned LowSignBits = ValueLow.getNumSignBits();
3366         unsigned HighSignBits = ValueHigh.getNumSignBits();
3367         unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3368         if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3369           Known.One.setHighBits(MinSignBits);
3370           break;
3371         }
3372         if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3373           Known.Zero.setHighBits(MinSignBits);
3374           break;
3375         }
3376       }
3377     }
3378 
3379     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3380     if (Known.isUnknown()) break; // Early-out
3381     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3382     if (IsMax)
3383       Known = KnownBits::smax(Known, Known2);
3384     else
3385       Known = KnownBits::smin(Known, Known2);
3386     break;
3387   }
3388   case ISD::FrameIndex:
3389   case ISD::TargetFrameIndex:
3390     TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),
3391                                        Known, getMachineFunction());
3392     break;
3393 
3394   default:
3395     if (Opcode < ISD::BUILTIN_OP_END)
3396       break;
3397     LLVM_FALLTHROUGH;
3398   case ISD::INTRINSIC_WO_CHAIN:
3399   case ISD::INTRINSIC_W_CHAIN:
3400   case ISD::INTRINSIC_VOID:
3401     // Allow the target to implement this method for its nodes.
3402     TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
3403     break;
3404   }
3405 
3406   assert(!Known.hasConflict() && "Bits known to be one AND zero?");
3407   return Known;
3408 }
3409 
3410 SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
3411                                                              SDValue N1) const {
3412   // X + 0 never overflow
3413   if (isNullConstant(N1))
3414     return OFK_Never;
3415 
3416   KnownBits N1Known = computeKnownBits(N1);
3417   if (N1Known.Zero.getBoolValue()) {
3418     KnownBits N0Known = computeKnownBits(N0);
3419 
3420     bool overflow;
3421     (void)N0Known.getMaxValue().uadd_ov(N1Known.getMaxValue(), overflow);
3422     if (!overflow)
3423       return OFK_Never;
3424   }
3425 
3426   // mulhi + 1 never overflow
3427   if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
3428       (N1Known.getMaxValue() & 0x01) == N1Known.getMaxValue())
3429     return OFK_Never;
3430 
3431   if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) {
3432     KnownBits N0Known = computeKnownBits(N0);
3433 
3434     if ((N0Known.getMaxValue() & 0x01) == N0Known.getMaxValue())
3435       return OFK_Never;
3436   }
3437 
3438   return OFK_Sometime;
3439 }
3440 
3441 bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
3442   EVT OpVT = Val.getValueType();
3443   unsigned BitWidth = OpVT.getScalarSizeInBits();
3444 
3445   // Is the constant a known power of 2?
3446   if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
3447     return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3448 
3449   // A left-shift of a constant one will have exactly one bit set because
3450   // shifting the bit off the end is undefined.
3451   if (Val.getOpcode() == ISD::SHL) {
3452     auto *C = isConstOrConstSplat(Val.getOperand(0));
3453     if (C && C->getAPIntValue() == 1)
3454       return true;
3455   }
3456 
3457   // Similarly, a logical right-shift of a constant sign-bit will have exactly
3458   // one bit set.
3459   if (Val.getOpcode() == ISD::SRL) {
3460     auto *C = isConstOrConstSplat(Val.getOperand(0));
3461     if (C && C->getAPIntValue().isSignMask())
3462       return true;
3463   }
3464 
3465   // Are all operands of a build vector constant powers of two?
3466   if (Val.getOpcode() == ISD::BUILD_VECTOR)
3467     if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
3468           if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
3469             return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3470           return false;
3471         }))
3472       return true;
3473 
3474   // More could be done here, though the above checks are enough
3475   // to handle some common cases.
3476 
3477   // Fall back to computeKnownBits to catch other known cases.
3478   KnownBits Known = computeKnownBits(Val);
3479   return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
3480 }
3481 
3482 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
3483   EVT VT = Op.getValueType();
3484 
3485   // TODO: Assume we don't know anything for now.
3486   if (VT.isScalableVector())
3487     return 1;
3488 
3489   APInt DemandedElts = VT.isVector()
3490                            ? APInt::getAllOnesValue(VT.getVectorNumElements())
3491                            : APInt(1, 1);
3492   return ComputeNumSignBits(Op, DemandedElts, Depth);
3493 }
3494 
3495 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
3496                                           unsigned Depth) const {
3497   EVT VT = Op.getValueType();
3498   assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
3499   unsigned VTBits = VT.getScalarSizeInBits();
3500   unsigned NumElts = DemandedElts.getBitWidth();
3501   unsigned Tmp, Tmp2;
3502   unsigned FirstAnswer = 1;
3503 
3504   if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3505     const APInt &Val = C->getAPIntValue();
3506     return Val.getNumSignBits();
3507   }
3508 
3509   if (Depth >= MaxRecursionDepth)
3510     return 1;  // Limit search depth.
3511 
3512   if (!DemandedElts || VT.isScalableVector())
3513     return 1;  // No demanded elts, better to assume we don't know anything.
3514 
3515   unsigned Opcode = Op.getOpcode();
3516   switch (Opcode) {
3517   default: break;
3518   case ISD::AssertSext:
3519     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3520     return VTBits-Tmp+1;
3521   case ISD::AssertZext:
3522     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3523     return VTBits-Tmp;
3524 
3525   case ISD::BUILD_VECTOR:
3526     Tmp = VTBits;
3527     for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
3528       if (!DemandedElts[i])
3529         continue;
3530 
3531       SDValue SrcOp = Op.getOperand(i);
3532       Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);
3533 
3534       // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3535       if (SrcOp.getValueSizeInBits() != VTBits) {
3536         assert(SrcOp.getValueSizeInBits() > VTBits &&
3537                "Expected BUILD_VECTOR implicit truncation");
3538         unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
3539         Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
3540       }
3541       Tmp = std::min(Tmp, Tmp2);
3542     }
3543     return Tmp;
3544 
3545   case ISD::VECTOR_SHUFFLE: {
3546     // Collect the minimum number of sign bits that are shared by every vector
3547     // element referenced by the shuffle.
3548     APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
3549     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3550     assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3551     for (unsigned i = 0; i != NumElts; ++i) {
3552       int M = SVN->getMaskElt(i);
3553       if (!DemandedElts[i])
3554         continue;
3555       // For UNDEF elements, we don't know anything about the common state of
3556       // the shuffle result.
3557       if (M < 0)
3558         return 1;
3559       if ((unsigned)M < NumElts)
3560         DemandedLHS.setBit((unsigned)M % NumElts);
3561       else
3562         DemandedRHS.setBit((unsigned)M % NumElts);
3563     }
3564     Tmp = std::numeric_limits<unsigned>::max();
3565     if (!!DemandedLHS)
3566       Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
3567     if (!!DemandedRHS) {
3568       Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
3569       Tmp = std::min(Tmp, Tmp2);
3570     }
3571     // If we don't know anything, early out and try computeKnownBits fall-back.
3572     if (Tmp == 1)
3573       break;
3574     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3575     return Tmp;
3576   }
3577 
3578   case ISD::BITCAST: {
3579     SDValue N0 = Op.getOperand(0);
3580     EVT SrcVT = N0.getValueType();
3581     unsigned SrcBits = SrcVT.getScalarSizeInBits();
3582 
3583     // Ignore bitcasts from unsupported types..
3584     if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
3585       break;
3586 
3587     // Fast handling of 'identity' bitcasts.
3588     if (VTBits == SrcBits)
3589       return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
3590 
3591     bool IsLE = getDataLayout().isLittleEndian();
3592 
3593     // Bitcast 'large element' scalar/vector to 'small element' vector.
3594     if ((SrcBits % VTBits) == 0) {
3595       assert(VT.isVector() && "Expected bitcast to vector");
3596 
3597       unsigned Scale = SrcBits / VTBits;
3598       APInt SrcDemandedElts(NumElts / Scale, 0);
3599       for (unsigned i = 0; i != NumElts; ++i)
3600         if (DemandedElts[i])
3601           SrcDemandedElts.setBit(i / Scale);
3602 
3603       // Fast case - sign splat can be simply split across the small elements.
3604       Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
3605       if (Tmp == SrcBits)
3606         return VTBits;
3607 
3608       // Slow case - determine how far the sign extends into each sub-element.
3609       Tmp2 = VTBits;
3610       for (unsigned i = 0; i != NumElts; ++i)
3611         if (DemandedElts[i]) {
3612           unsigned SubOffset = i % Scale;
3613           SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
3614           SubOffset = SubOffset * VTBits;
3615           if (Tmp <= SubOffset)
3616             return 1;
3617           Tmp2 = std::min(Tmp2, Tmp - SubOffset);
3618         }
3619       return Tmp2;
3620     }
3621     break;
3622   }
3623 
3624   case ISD::SIGN_EXTEND:
3625     Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
3626     return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
3627   case ISD::SIGN_EXTEND_INREG:
3628     // Max of the input and what this extends.
3629     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
3630     Tmp = VTBits-Tmp+1;
3631     Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3632     return std::max(Tmp, Tmp2);
3633   case ISD::SIGN_EXTEND_VECTOR_INREG: {
3634     SDValue Src = Op.getOperand(0);
3635     EVT SrcVT = Src.getValueType();
3636     APInt DemandedSrcElts = DemandedElts.zextOrSelf(SrcVT.getVectorNumElements());
3637     Tmp = VTBits - SrcVT.getScalarSizeInBits();
3638     return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
3639   }
3640   case ISD::SRA:
3641     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3642     // SRA X, C -> adds C sign bits.
3643     if (const APInt *ShAmt =
3644             getValidMinimumShiftAmountConstant(Op, DemandedElts))
3645       Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits);
3646     return Tmp;
3647   case ISD::SHL:
3648     if (const APInt *ShAmt =
3649             getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
3650       // shl destroys sign bits, ensure it doesn't shift out all sign bits.
3651       Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3652       if (ShAmt->ult(Tmp))
3653         return Tmp - ShAmt->getZExtValue();
3654     }
3655     break;
3656   case ISD::AND:
3657   case ISD::OR:
3658   case ISD::XOR:    // NOT is handled here.
3659     // Logical binary ops preserve the number of sign bits at the worst.
3660     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3661     if (Tmp != 1) {
3662       Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3663       FirstAnswer = std::min(Tmp, Tmp2);
3664       // We computed what we know about the sign bits as our first
3665       // answer. Now proceed to the generic code that uses
3666       // computeKnownBits, and pick whichever answer is better.
3667     }
3668     break;
3669 
3670   case ISD::SELECT:
3671   case ISD::VSELECT:
3672     Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3673     if (Tmp == 1) return 1;  // Early out.
3674     Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3675     return std::min(Tmp, Tmp2);
3676   case ISD::SELECT_CC:
3677     Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3678     if (Tmp == 1) return 1;  // Early out.
3679     Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
3680     return std::min(Tmp, Tmp2);
3681 
3682   case ISD::SMIN:
3683   case ISD::SMAX: {
3684     // If we have a clamp pattern, we know that the number of sign bits will be
3685     // the minimum of the clamp min/max range.
3686     bool IsMax = (Opcode == ISD::SMAX);
3687     ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3688     if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3689       if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3690         CstHigh =
3691             isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3692     if (CstLow && CstHigh) {
3693       if (!IsMax)
3694         std::swap(CstLow, CstHigh);
3695       if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
3696         Tmp = CstLow->getAPIntValue().getNumSignBits();
3697         Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
3698         return std::min(Tmp, Tmp2);
3699       }
3700     }
3701 
3702     // Fallback - just get the minimum number of sign bits of the operands.
3703     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3704     if (Tmp == 1)
3705       return 1;  // Early out.
3706     Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3707     return std::min(Tmp, Tmp2);
3708   }
3709   case ISD::UMIN:
3710   case ISD::UMAX:
3711     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3712     if (Tmp == 1)
3713       return 1;  // Early out.
3714     Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3715     return std::min(Tmp, Tmp2);
3716   case ISD::SADDO:
3717   case ISD::UADDO:
3718   case ISD::SSUBO:
3719   case ISD::USUBO:
3720   case ISD::SMULO:
3721   case ISD::UMULO:
3722     if (Op.getResNo() != 1)
3723       break;
3724     // The boolean result conforms to getBooleanContents.  Fall through.
3725     // If setcc returns 0/-1, all bits are sign bits.
3726     // We know that we have an integer-based boolean since these operations
3727     // are only available for integer.
3728     if (TLI->getBooleanContents(VT.isVector(), false) ==
3729         TargetLowering::ZeroOrNegativeOneBooleanContent)
3730       return VTBits;
3731     break;
3732   case ISD::SETCC:
3733   case ISD::STRICT_FSETCC:
3734   case ISD::STRICT_FSETCCS: {
3735     unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3736     // If setcc returns 0/-1, all bits are sign bits.
3737     if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3738         TargetLowering::ZeroOrNegativeOneBooleanContent)
3739       return VTBits;
3740     break;
3741   }
3742   case ISD::ROTL:
3743   case ISD::ROTR:
3744     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3745 
3746     // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
3747     if (Tmp == VTBits)
3748       return VTBits;
3749 
3750     if (ConstantSDNode *C =
3751             isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3752       unsigned RotAmt = C->getAPIntValue().urem(VTBits);
3753 
3754       // Handle rotate right by N like a rotate left by 32-N.
3755       if (Opcode == ISD::ROTR)
3756         RotAmt = (VTBits - RotAmt) % VTBits;
3757 
3758       // If we aren't rotating out all of the known-in sign bits, return the
3759       // number that are left.  This handles rotl(sext(x), 1) for example.
3760       if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
3761     }
3762     break;
3763   case ISD::ADD:
3764   case ISD::ADDC:
3765     // Add can have at most one carry bit.  Thus we know that the output
3766     // is, at worst, one more bit than the inputs.
3767     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3768     if (Tmp == 1) return 1; // Early out.
3769 
3770     // Special case decrementing a value (ADD X, -1):
3771     if (ConstantSDNode *CRHS =
3772             isConstOrConstSplat(Op.getOperand(1), DemandedElts))
3773       if (CRHS->isAllOnesValue()) {
3774         KnownBits Known =
3775             computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3776 
3777         // If the input is known to be 0 or 1, the output is 0/-1, which is all
3778         // sign bits set.
3779         if ((Known.Zero | 1).isAllOnesValue())
3780           return VTBits;
3781 
3782         // If we are subtracting one from a positive number, there is no carry
3783         // out of the result.
3784         if (Known.isNonNegative())
3785           return Tmp;
3786       }
3787 
3788     Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3789     if (Tmp2 == 1) return 1; // Early out.
3790     return std::min(Tmp, Tmp2) - 1;
3791   case ISD::SUB:
3792     Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3793     if (Tmp2 == 1) return 1; // Early out.
3794 
3795     // Handle NEG.
3796     if (ConstantSDNode *CLHS =
3797             isConstOrConstSplat(Op.getOperand(0), DemandedElts))
3798       if (CLHS->isNullValue()) {
3799         KnownBits Known =
3800             computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3801         // If the input is known to be 0 or 1, the output is 0/-1, which is all
3802         // sign bits set.
3803         if ((Known.Zero | 1).isAllOnesValue())
3804           return VTBits;
3805 
3806         // If the input is known to be positive (the sign bit is known clear),
3807         // the output of the NEG has the same number of sign bits as the input.
3808         if (Known.isNonNegative())
3809           return Tmp2;
3810 
3811         // Otherwise, we treat this like a SUB.
3812       }
3813 
3814     // Sub can have at most one carry bit.  Thus we know that the output
3815     // is, at worst, one more bit than the inputs.
3816     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3817     if (Tmp == 1) return 1; // Early out.
3818     return std::min(Tmp, Tmp2) - 1;
3819   case ISD::MUL: {
3820     // The output of the Mul can be at most twice the valid bits in the inputs.
3821     unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3822     if (SignBitsOp0 == 1)
3823       break;
3824     unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3825     if (SignBitsOp1 == 1)
3826       break;
3827     unsigned OutValidBits =
3828         (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
3829     return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
3830   }
3831   case ISD::TRUNCATE: {
3832     // Check if the sign bits of source go down as far as the truncated value.
3833     unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
3834     unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3835     if (NumSrcSignBits > (NumSrcBits - VTBits))
3836       return NumSrcSignBits - (NumSrcBits - VTBits);
3837     break;
3838   }
3839   case ISD::EXTRACT_ELEMENT: {
3840     const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3841     const int BitWidth = Op.getValueSizeInBits();
3842     const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
3843 
3844     // Get reverse index (starting from 1), Op1 value indexes elements from
3845     // little end. Sign starts at big end.
3846     const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
3847 
3848     // If the sign portion ends in our element the subtraction gives correct
3849     // result. Otherwise it gives either negative or > bitwidth result
3850     return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
3851   }
3852   case ISD::INSERT_VECTOR_ELT: {
3853     // If we know the element index, split the demand between the
3854     // source vector and the inserted element, otherwise assume we need
3855     // the original demanded vector elements and the value.
3856     SDValue InVec = Op.getOperand(0);
3857     SDValue InVal = Op.getOperand(1);
3858     SDValue EltNo = Op.getOperand(2);
3859     bool DemandedVal = true;
3860     APInt DemandedVecElts = DemandedElts;
3861     auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3862     if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3863       unsigned EltIdx = CEltNo->getZExtValue();
3864       DemandedVal = !!DemandedElts[EltIdx];
3865       DemandedVecElts.clearBit(EltIdx);
3866     }
3867     Tmp = std::numeric_limits<unsigned>::max();
3868     if (DemandedVal) {
3869       // TODO - handle implicit truncation of inserted elements.
3870       if (InVal.getScalarValueSizeInBits() != VTBits)
3871         break;
3872       Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
3873       Tmp = std::min(Tmp, Tmp2);
3874     }
3875     if (!!DemandedVecElts) {
3876       Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);
3877       Tmp = std::min(Tmp, Tmp2);
3878     }
3879     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3880     return Tmp;
3881   }
3882   case ISD::EXTRACT_VECTOR_ELT: {
3883     SDValue InVec = Op.getOperand(0);
3884     SDValue EltNo = Op.getOperand(1);
3885     EVT VecVT = InVec.getValueType();
3886     const unsigned BitWidth = Op.getValueSizeInBits();
3887     const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
3888     const unsigned NumSrcElts = VecVT.getVectorNumElements();
3889 
3890     // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
3891     // anything about sign bits. But if the sizes match we can derive knowledge
3892     // about sign bits from the vector operand.
3893     if (BitWidth != EltBitWidth)
3894       break;
3895 
3896     // If we know the element index, just demand that vector element, else for
3897     // an unknown element index, ignore DemandedElts and demand them all.
3898     APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
3899     auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3900     if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3901       DemandedSrcElts =
3902           APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3903 
3904     return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
3905   }
3906   case ISD::EXTRACT_SUBVECTOR: {
3907     // Offset the demanded elts by the subvector index.
3908     SDValue Src = Op.getOperand(0);
3909     // Bail until we can represent demanded elements for scalable vectors.
3910     if (Src.getValueType().isScalableVector())
3911       break;
3912     uint64_t Idx = Op.getConstantOperandVal(1);
3913     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3914     APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
3915     return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
3916   }
3917   case ISD::CONCAT_VECTORS: {
3918     // Determine the minimum number of sign bits across all demanded
3919     // elts of the input vectors. Early out if the result is already 1.
3920     Tmp = std::numeric_limits<unsigned>::max();
3921     EVT SubVectorVT = Op.getOperand(0).getValueType();
3922     unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3923     unsigned NumSubVectors = Op.getNumOperands();
3924     for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
3925       APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
3926       DemandedSub = DemandedSub.trunc(NumSubVectorElts);
3927       if (!DemandedSub)
3928         continue;
3929       Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
3930       Tmp = std::min(Tmp, Tmp2);
3931     }
3932     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3933     return Tmp;
3934   }
3935   case ISD::INSERT_SUBVECTOR: {
3936     // Demand any elements from the subvector and the remainder from the src its
3937     // inserted into.
3938     SDValue Src = Op.getOperand(0);
3939     SDValue Sub = Op.getOperand(1);
3940     uint64_t Idx = Op.getConstantOperandVal(2);
3941     unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3942     APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3943     APInt DemandedSrcElts = DemandedElts;
3944     DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
3945 
3946     Tmp = std::numeric_limits<unsigned>::max();
3947     if (!!DemandedSubElts) {
3948       Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
3949       if (Tmp == 1)
3950         return 1; // early-out
3951     }
3952     if (!!DemandedSrcElts) {
3953       Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
3954       Tmp = std::min(Tmp, Tmp2);
3955     }
3956     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3957     return Tmp;
3958   }
3959   }
3960 
3961   // If we are looking at the loaded value of the SDNode.
3962   if (Op.getResNo() == 0) {
3963     // Handle LOADX separately here. EXTLOAD case will fallthrough.
3964     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
3965       unsigned ExtType = LD->getExtensionType();
3966       switch (ExtType) {
3967       default: break;
3968       case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
3969         Tmp = LD->getMemoryVT().getScalarSizeInBits();
3970         return VTBits - Tmp + 1;
3971       case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
3972         Tmp = LD->getMemoryVT().getScalarSizeInBits();
3973         return VTBits - Tmp;
3974       case ISD::NON_EXTLOAD:
3975         if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
3976           // We only need to handle vectors - computeKnownBits should handle
3977           // scalar cases.
3978           Type *CstTy = Cst->getType();
3979           if (CstTy->isVectorTy() &&
3980               (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits()) {
3981             Tmp = VTBits;
3982             for (unsigned i = 0; i != NumElts; ++i) {
3983               if (!DemandedElts[i])
3984                 continue;
3985               if (Constant *Elt = Cst->getAggregateElement(i)) {
3986                 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3987                   const APInt &Value = CInt->getValue();
3988                   Tmp = std::min(Tmp, Value.getNumSignBits());
3989                   continue;
3990                 }
3991                 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3992                   APInt Value = CFP->getValueAPF().bitcastToAPInt();
3993                   Tmp = std::min(Tmp, Value.getNumSignBits());
3994                   continue;
3995                 }
3996               }
3997               // Unknown type. Conservatively assume no bits match sign bit.
3998               return 1;
3999             }
4000             return Tmp;
4001           }
4002         }
4003         break;
4004       }
4005     }
4006   }
4007 
4008   // Allow the target to implement this method for its nodes.
4009   if (Opcode >= ISD::BUILTIN_OP_END ||
4010       Opcode == ISD::INTRINSIC_WO_CHAIN ||
4011       Opcode == ISD::INTRINSIC_W_CHAIN ||
4012       Opcode == ISD::INTRINSIC_VOID) {
4013     unsigned NumBits =
4014         TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
4015     if (NumBits > 1)
4016       FirstAnswer = std::max(FirstAnswer, NumBits);
4017   }
4018 
4019   // Finally, if we can prove that the top bits of the result are 0's or 1's,
4020   // use this information.
4021   KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4022 
4023   APInt Mask;
4024   if (Known.isNonNegative()) {        // sign bit is 0
4025     Mask = Known.Zero;
4026   } else if (Known.isNegative()) {  // sign bit is 1;
4027     Mask = Known.One;
4028   } else {
4029     // Nothing known.
4030     return FirstAnswer;
4031   }
4032 
4033   // Okay, we know that the sign bit in Mask is set.  Use CLO to determine
4034   // the number of identical bits in the top of the input value.
4035   Mask <<= Mask.getBitWidth()-VTBits;
4036   return std::max(FirstAnswer, Mask.countLeadingOnes());
4037 }
4038 
4039 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
4040   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
4041       !isa<ConstantSDNode>(Op.getOperand(1)))
4042     return false;
4043 
4044   if (Op.getOpcode() == ISD::OR &&
4045       !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
4046     return false;
4047 
4048   return true;
4049 }
4050 
4051 bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
4052   // If we're told that NaNs won't happen, assume they won't.
4053   if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
4054     return true;
4055 
4056   if (Depth >= MaxRecursionDepth)
4057     return false; // Limit search depth.
4058 
4059   // TODO: Handle vectors.
4060   // If the value is a constant, we can obviously see if it is a NaN or not.
4061   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
4062     return !C->getValueAPF().isNaN() ||
4063            (SNaN && !C->getValueAPF().isSignaling());
4064   }
4065 
4066   unsigned Opcode = Op.getOpcode();
4067   switch (Opcode) {
4068   case ISD::FADD:
4069   case ISD::FSUB:
4070   case ISD::FMUL:
4071   case ISD::FDIV:
4072   case ISD::FREM:
4073   case ISD::FSIN:
4074   case ISD::FCOS: {
4075     if (SNaN)
4076       return true;
4077     // TODO: Need isKnownNeverInfinity
4078     return false;
4079   }
4080   case ISD::FCANONICALIZE:
4081   case ISD::FEXP:
4082   case ISD::FEXP2:
4083   case ISD::FTRUNC:
4084   case ISD::FFLOOR:
4085   case ISD::FCEIL:
4086   case ISD::FROUND:
4087   case ISD::FROUNDEVEN:
4088   case ISD::FRINT:
4089   case ISD::FNEARBYINT: {
4090     if (SNaN)
4091       return true;
4092     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4093   }
4094   case ISD::FABS:
4095   case ISD::FNEG:
4096   case ISD::FCOPYSIGN: {
4097     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4098   }
4099   case ISD::SELECT:
4100     return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4101            isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4102   case ISD::FP_EXTEND:
4103   case ISD::FP_ROUND: {
4104     if (SNaN)
4105       return true;
4106     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4107   }
4108   case ISD::SINT_TO_FP:
4109   case ISD::UINT_TO_FP:
4110     return true;
4111   case ISD::FMA:
4112   case ISD::FMAD: {
4113     if (SNaN)
4114       return true;
4115     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4116            isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4117            isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4118   }
4119   case ISD::FSQRT: // Need is known positive
4120   case ISD::FLOG:
4121   case ISD::FLOG2:
4122   case ISD::FLOG10:
4123   case ISD::FPOWI:
4124   case ISD::FPOW: {
4125     if (SNaN)
4126       return true;
4127     // TODO: Refine on operand
4128     return false;
4129   }
4130   case ISD::FMINNUM:
4131   case ISD::FMAXNUM: {
4132     // Only one needs to be known not-nan, since it will be returned if the
4133     // other ends up being one.
4134     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
4135            isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4136   }
4137   case ISD::FMINNUM_IEEE:
4138   case ISD::FMAXNUM_IEEE: {
4139     if (SNaN)
4140       return true;
4141     // This can return a NaN if either operand is an sNaN, or if both operands
4142     // are NaN.
4143     return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
4144             isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
4145            (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
4146             isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
4147   }
4148   case ISD::FMINIMUM:
4149   case ISD::FMAXIMUM: {
4150     // TODO: Does this quiet or return the origina NaN as-is?
4151     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4152            isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4153   }
4154   case ISD::EXTRACT_VECTOR_ELT: {
4155     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4156   }
4157   default:
4158     if (Opcode >= ISD::BUILTIN_OP_END ||
4159         Opcode == ISD::INTRINSIC_WO_CHAIN ||
4160         Opcode == ISD::INTRINSIC_W_CHAIN ||
4161         Opcode == ISD::INTRINSIC_VOID) {
4162       return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
4163     }
4164 
4165     return false;
4166   }
4167 }
4168 
4169 bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
4170   assert(Op.getValueType().isFloatingPoint() &&
4171          "Floating point type expected");
4172 
4173   // If the value is a constant, we can obviously see if it is a zero or not.
4174   // TODO: Add BuildVector support.
4175   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
4176     return !C->isZero();
4177   return false;
4178 }
4179 
4180 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
4181   assert(!Op.getValueType().isFloatingPoint() &&
4182          "Floating point types unsupported - use isKnownNeverZeroFloat");
4183 
4184   // If the value is a constant, we can obviously see if it is a zero or not.
4185   if (ISD::matchUnaryPredicate(
4186           Op, [](ConstantSDNode *C) { return !C->isNullValue(); }))
4187     return true;
4188 
4189   // TODO: Recognize more cases here.
4190   switch (Op.getOpcode()) {
4191   default: break;
4192   case ISD::OR:
4193     if (isKnownNeverZero(Op.getOperand(1)) ||
4194         isKnownNeverZero(Op.getOperand(0)))
4195       return true;
4196     break;
4197   }
4198 
4199   return false;
4200 }
4201 
4202 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
4203   // Check the obvious case.
4204   if (A == B) return true;
4205 
4206   // For for negative and positive zero.
4207   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
4208     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
4209       if (CA->isZero() && CB->isZero()) return true;
4210 
4211   // Otherwise they may not be equal.
4212   return false;
4213 }
4214 
4215 // FIXME: unify with llvm::haveNoCommonBitsSet.
4216 // FIXME: could also handle masked merge pattern (X & ~M) op (Y & M)
4217 bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
4218   assert(A.getValueType() == B.getValueType() &&
4219          "Values must have the same type");
4220   return (computeKnownBits(A).Zero | computeKnownBits(B).Zero).isAllOnesValue();
4221 }
4222 
4223 static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
4224                                 ArrayRef<SDValue> Ops,
4225                                 SelectionDAG &DAG) {
4226   int NumOps = Ops.size();
4227   assert(NumOps != 0 && "Can't build an empty vector!");
4228   assert(!VT.isScalableVector() &&
4229          "BUILD_VECTOR cannot be used with scalable types");
4230   assert(VT.getVectorNumElements() == (unsigned)NumOps &&
4231          "Incorrect element count in BUILD_VECTOR!");
4232 
4233   // BUILD_VECTOR of UNDEFs is UNDEF.
4234   if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4235     return DAG.getUNDEF(VT);
4236 
4237   // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
4238   SDValue IdentitySrc;
4239   bool IsIdentity = true;
4240   for (int i = 0; i != NumOps; ++i) {
4241     if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
4242         Ops[i].getOperand(0).getValueType() != VT ||
4243         (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
4244         !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
4245         cast<ConstantSDNode>(Ops[i].getOperand(1))->getAPIntValue() != i) {
4246       IsIdentity = false;
4247       break;
4248     }
4249     IdentitySrc = Ops[i].getOperand(0);
4250   }
4251   if (IsIdentity)
4252     return IdentitySrc;
4253 
4254   return SDValue();
4255 }
4256 
4257 /// Try to simplify vector concatenation to an input value, undef, or build
4258 /// vector.
4259 static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
4260                                   ArrayRef<SDValue> Ops,
4261                                   SelectionDAG &DAG) {
4262   assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
4263   assert(llvm::all_of(Ops,
4264                       [Ops](SDValue Op) {
4265                         return Ops[0].getValueType() == Op.getValueType();
4266                       }) &&
4267          "Concatenation of vectors with inconsistent value types!");
4268   assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
4269              VT.getVectorElementCount() &&
4270          "Incorrect element count in vector concatenation!");
4271 
4272   if (Ops.size() == 1)
4273     return Ops[0];
4274 
4275   // Concat of UNDEFs is UNDEF.
4276   if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4277     return DAG.getUNDEF(VT);
4278 
4279   // Scan the operands and look for extract operations from a single source
4280   // that correspond to insertion at the same location via this concatenation:
4281   // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
4282   SDValue IdentitySrc;
4283   bool IsIdentity = true;
4284   for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
4285     SDValue Op = Ops[i];
4286     unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
4287     if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
4288         Op.getOperand(0).getValueType() != VT ||
4289         (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
4290         Op.getConstantOperandVal(1) != IdentityIndex) {
4291       IsIdentity = false;
4292       break;
4293     }
4294     assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
4295            "Unexpected identity source vector for concat of extracts");
4296     IdentitySrc = Op.getOperand(0);
4297   }
4298   if (IsIdentity) {
4299     assert(IdentitySrc && "Failed to set source vector of extracts");
4300     return IdentitySrc;
4301   }
4302 
4303   // The code below this point is only designed to work for fixed width
4304   // vectors, so we bail out for now.
4305   if (VT.isScalableVector())
4306     return SDValue();
4307 
4308   // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
4309   // simplified to one big BUILD_VECTOR.
4310   // FIXME: Add support for SCALAR_TO_VECTOR as well.
4311   EVT SVT = VT.getScalarType();
4312   SmallVector<SDValue, 16> Elts;
4313   for (SDValue Op : Ops) {
4314     EVT OpVT = Op.getValueType();
4315     if (Op.isUndef())
4316       Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
4317     else if (Op.getOpcode() == ISD::BUILD_VECTOR)
4318       Elts.append(Op->op_begin(), Op->op_end());
4319     else
4320       return SDValue();
4321   }
4322 
4323   // BUILD_VECTOR requires all inputs to be of the same type, find the
4324   // maximum type and extend them all.
4325   for (SDValue Op : Elts)
4326     SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
4327 
4328   if (SVT.bitsGT(VT.getScalarType())) {
4329     for (SDValue &Op : Elts) {
4330       if (Op.isUndef())
4331         Op = DAG.getUNDEF(SVT);
4332       else
4333         Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
4334                  ? DAG.getZExtOrTrunc(Op, DL, SVT)
4335                  : DAG.getSExtOrTrunc(Op, DL, SVT);
4336     }
4337   }
4338 
4339   SDValue V = DAG.getBuildVector(VT, DL, Elts);
4340   NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
4341   return V;
4342 }
4343 
4344 /// Gets or creates the specified node.
4345 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
4346   FoldingSetNodeID ID;
4347   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
4348   void *IP = nullptr;
4349   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
4350     return SDValue(E, 0);
4351 
4352   auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4353                               getVTList(VT));
4354   CSEMap.InsertNode(N, IP);
4355 
4356   InsertNode(N);
4357   SDValue V = SDValue(N, 0);
4358   NewSDValueDbgMsg(V, "Creating new node: ", this);
4359   return V;
4360 }
4361 
4362 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4363                               SDValue Operand) {
4364   SDNodeFlags Flags;
4365   if (Inserter)
4366     Flags = Inserter->getFlags();
4367   return getNode(Opcode, DL, VT, Operand, Flags);
4368 }
4369 
4370 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4371                               SDValue Operand, const SDNodeFlags Flags) {
4372   // Constant fold unary operations with an integer constant operand. Even
4373   // opaque constant will be folded, because the folding of unary operations
4374   // doesn't create new constants with different values. Nevertheless, the
4375   // opaque flag is preserved during folding to prevent future folding with
4376   // other constants.
4377   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
4378     const APInt &Val = C->getAPIntValue();
4379     switch (Opcode) {
4380     default: break;
4381     case ISD::SIGN_EXTEND:
4382       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
4383                          C->isTargetOpcode(), C->isOpaque());
4384     case ISD::TRUNCATE:
4385       if (C->isOpaque())
4386         break;
4387       LLVM_FALLTHROUGH;
4388     case ISD::ANY_EXTEND:
4389     case ISD::ZERO_EXTEND:
4390       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
4391                          C->isTargetOpcode(), C->isOpaque());
4392     case ISD::UINT_TO_FP:
4393     case ISD::SINT_TO_FP: {
4394       APFloat apf(EVTToAPFloatSemantics(VT),
4395                   APInt::getNullValue(VT.getSizeInBits()));
4396       (void)apf.convertFromAPInt(Val,
4397                                  Opcode==ISD::SINT_TO_FP,
4398                                  APFloat::rmNearestTiesToEven);
4399       return getConstantFP(apf, DL, VT);
4400     }
4401     case ISD::BITCAST:
4402       if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
4403         return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
4404       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
4405         return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
4406       if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
4407         return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
4408       if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
4409         return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
4410       break;
4411     case ISD::ABS:
4412       return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
4413                          C->isOpaque());
4414     case ISD::BITREVERSE:
4415       return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
4416                          C->isOpaque());
4417     case ISD::BSWAP:
4418       return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
4419                          C->isOpaque());
4420     case ISD::CTPOP:
4421       return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
4422                          C->isOpaque());
4423     case ISD::CTLZ:
4424     case ISD::CTLZ_ZERO_UNDEF:
4425       return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
4426                          C->isOpaque());
4427     case ISD::CTTZ:
4428     case ISD::CTTZ_ZERO_UNDEF:
4429       return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
4430                          C->isOpaque());
4431     case ISD::FP16_TO_FP: {
4432       bool Ignored;
4433       APFloat FPV(APFloat::IEEEhalf(),
4434                   (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
4435 
4436       // This can return overflow, underflow, or inexact; we don't care.
4437       // FIXME need to be more flexible about rounding mode.
4438       (void)FPV.convert(EVTToAPFloatSemantics(VT),
4439                         APFloat::rmNearestTiesToEven, &Ignored);
4440       return getConstantFP(FPV, DL, VT);
4441     }
4442     }
4443   }
4444 
4445   // Constant fold unary operations with a floating point constant operand.
4446   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
4447     APFloat V = C->getValueAPF();    // make copy
4448     switch (Opcode) {
4449     case ISD::FNEG:
4450       V.changeSign();
4451       return getConstantFP(V, DL, VT);
4452     case ISD::FABS:
4453       V.clearSign();
4454       return getConstantFP(V, DL, VT);
4455     case ISD::FCEIL: {
4456       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
4457       if (fs == APFloat::opOK || fs == APFloat::opInexact)
4458         return getConstantFP(V, DL, VT);
4459       break;
4460     }
4461     case ISD::FTRUNC: {
4462       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
4463       if (fs == APFloat::opOK || fs == APFloat::opInexact)
4464         return getConstantFP(V, DL, VT);
4465       break;
4466     }
4467     case ISD::FFLOOR: {
4468       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
4469       if (fs == APFloat::opOK || fs == APFloat::opInexact)
4470         return getConstantFP(V, DL, VT);
4471       break;
4472     }
4473     case ISD::FP_EXTEND: {
4474       bool ignored;
4475       // This can return overflow, underflow, or inexact; we don't care.
4476       // FIXME need to be more flexible about rounding mode.
4477       (void)V.convert(EVTToAPFloatSemantics(VT),
4478                       APFloat::rmNearestTiesToEven, &ignored);
4479       return getConstantFP(V, DL, VT);
4480     }
4481     case ISD::FP_TO_SINT:
4482     case ISD::FP_TO_UINT: {
4483       bool ignored;
4484       APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
4485       // FIXME need to be more flexible about rounding mode.
4486       APFloat::opStatus s =
4487           V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
4488       if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
4489         break;
4490       return getConstant(IntVal, DL, VT);
4491     }
4492     case ISD::BITCAST:
4493       if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
4494         return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4495       else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
4496         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4497       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
4498         return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4499       break;
4500     case ISD::FP_TO_FP16: {
4501       bool Ignored;
4502       // This can return overflow, underflow, or inexact; we don't care.
4503       // FIXME need to be more flexible about rounding mode.
4504       (void)V.convert(APFloat::IEEEhalf(),
4505                       APFloat::rmNearestTiesToEven, &Ignored);
4506       return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4507     }
4508     }
4509   }
4510 
4511   // Constant fold unary operations with a vector integer or float operand.
4512   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
4513     if (BV->isConstant()) {
4514       switch (Opcode) {
4515       default:
4516         // FIXME: Entirely reasonable to perform folding of other unary
4517         // operations here as the need arises.
4518         break;
4519       case ISD::FNEG:
4520       case ISD::FABS:
4521       case ISD::FCEIL:
4522       case ISD::FTRUNC:
4523       case ISD::FFLOOR:
4524       case ISD::FP_EXTEND:
4525       case ISD::FP_TO_SINT:
4526       case ISD::FP_TO_UINT:
4527       case ISD::TRUNCATE:
4528       case ISD::ANY_EXTEND:
4529       case ISD::ZERO_EXTEND:
4530       case ISD::SIGN_EXTEND:
4531       case ISD::UINT_TO_FP:
4532       case ISD::SINT_TO_FP:
4533       case ISD::ABS:
4534       case ISD::BITREVERSE:
4535       case ISD::BSWAP:
4536       case ISD::CTLZ:
4537       case ISD::CTLZ_ZERO_UNDEF:
4538       case ISD::CTTZ:
4539       case ISD::CTTZ_ZERO_UNDEF:
4540       case ISD::CTPOP: {
4541         SDValue Ops = { Operand };
4542         if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
4543           return Fold;
4544       }
4545       }
4546     }
4547   }
4548 
4549   unsigned OpOpcode = Operand.getNode()->getOpcode();
4550   switch (Opcode) {
4551   case ISD::FREEZE:
4552     assert(VT == Operand.getValueType() && "Unexpected VT!");
4553     break;
4554   case ISD::TokenFactor:
4555   case ISD::MERGE_VALUES:
4556   case ISD::CONCAT_VECTORS:
4557     return Operand;         // Factor, merge or concat of one node?  No need.
4558   case ISD::BUILD_VECTOR: {
4559     // Attempt to simplify BUILD_VECTOR.
4560     SDValue Ops[] = {Operand};
4561     if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
4562       return V;
4563     break;
4564   }
4565   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
4566   case ISD::FP_EXTEND:
4567     assert(VT.isFloatingPoint() &&
4568            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
4569     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
4570     assert((!VT.isVector() ||
4571             VT.getVectorElementCount() ==
4572             Operand.getValueType().getVectorElementCount()) &&
4573            "Vector element count mismatch!");
4574     assert(Operand.getValueType().bitsLT(VT) &&
4575            "Invalid fpext node, dst < src!");
4576     if (Operand.isUndef())
4577       return getUNDEF(VT);
4578     break;
4579   case ISD::FP_TO_SINT:
4580   case ISD::FP_TO_UINT:
4581     if (Operand.isUndef())
4582       return getUNDEF(VT);
4583     break;
4584   case ISD::SINT_TO_FP:
4585   case ISD::UINT_TO_FP:
4586     // [us]itofp(undef) = 0, because the result value is bounded.
4587     if (Operand.isUndef())
4588       return getConstantFP(0.0, DL, VT);
4589     break;
4590   case ISD::SIGN_EXTEND:
4591     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4592            "Invalid SIGN_EXTEND!");
4593     assert(VT.isVector() == Operand.getValueType().isVector() &&
4594            "SIGN_EXTEND result type type should be vector iff the operand "
4595            "type is vector!");
4596     if (Operand.getValueType() == VT) return Operand;   // noop extension
4597     assert((!VT.isVector() ||
4598             VT.getVectorElementCount() ==
4599                 Operand.getValueType().getVectorElementCount()) &&
4600            "Vector element count mismatch!");
4601     assert(Operand.getValueType().bitsLT(VT) &&
4602            "Invalid sext node, dst < src!");
4603     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
4604       return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4605     else if (OpOpcode == ISD::UNDEF)
4606       // sext(undef) = 0, because the top bits will all be the same.
4607       return getConstant(0, DL, VT);
4608     break;
4609   case ISD::ZERO_EXTEND:
4610     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4611            "Invalid ZERO_EXTEND!");
4612     assert(VT.isVector() == Operand.getValueType().isVector() &&
4613            "ZERO_EXTEND result type type should be vector iff the operand "
4614            "type is vector!");
4615     if (Operand.getValueType() == VT) return Operand;   // noop extension
4616     assert((!VT.isVector() ||
4617             VT.getVectorElementCount() ==
4618                 Operand.getValueType().getVectorElementCount()) &&
4619            "Vector element count mismatch!");
4620     assert(Operand.getValueType().bitsLT(VT) &&
4621            "Invalid zext node, dst < src!");
4622     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
4623       return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
4624     else if (OpOpcode == ISD::UNDEF)
4625       // zext(undef) = 0, because the top bits will be zero.
4626       return getConstant(0, DL, VT);
4627     break;
4628   case ISD::ANY_EXTEND:
4629     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4630            "Invalid ANY_EXTEND!");
4631     assert(VT.isVector() == Operand.getValueType().isVector() &&
4632            "ANY_EXTEND result type type should be vector iff the operand "
4633            "type is vector!");
4634     if (Operand.getValueType() == VT) return Operand;   // noop extension
4635     assert((!VT.isVector() ||
4636             VT.getVectorElementCount() ==
4637                 Operand.getValueType().getVectorElementCount()) &&
4638            "Vector element count mismatch!");
4639     assert(Operand.getValueType().bitsLT(VT) &&
4640            "Invalid anyext node, dst < src!");
4641 
4642     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4643         OpOpcode == ISD::ANY_EXTEND)
4644       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
4645       return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4646     else if (OpOpcode == ISD::UNDEF)
4647       return getUNDEF(VT);
4648 
4649     // (ext (trunc x)) -> x
4650     if (OpOpcode == ISD::TRUNCATE) {
4651       SDValue OpOp = Operand.getOperand(0);
4652       if (OpOp.getValueType() == VT) {
4653         transferDbgValues(Operand, OpOp);
4654         return OpOp;
4655       }
4656     }
4657     break;
4658   case ISD::TRUNCATE:
4659     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4660            "Invalid TRUNCATE!");
4661     assert(VT.isVector() == Operand.getValueType().isVector() &&
4662            "TRUNCATE result type type should be vector iff the operand "
4663            "type is vector!");
4664     if (Operand.getValueType() == VT) return Operand;   // noop truncate
4665     assert((!VT.isVector() ||
4666             VT.getVectorElementCount() ==
4667                 Operand.getValueType().getVectorElementCount()) &&
4668            "Vector element count mismatch!");
4669     assert(Operand.getValueType().bitsGT(VT) &&
4670            "Invalid truncate node, src < dst!");
4671     if (OpOpcode == ISD::TRUNCATE)
4672       return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4673     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4674         OpOpcode == ISD::ANY_EXTEND) {
4675       // If the source is smaller than the dest, we still need an extend.
4676       if (Operand.getOperand(0).getValueType().getScalarType()
4677             .bitsLT(VT.getScalarType()))
4678         return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4679       if (Operand.getOperand(0).getValueType().bitsGT(VT))
4680         return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4681       return Operand.getOperand(0);
4682     }
4683     if (OpOpcode == ISD::UNDEF)
4684       return getUNDEF(VT);
4685     break;
4686   case ISD::ANY_EXTEND_VECTOR_INREG:
4687   case ISD::ZERO_EXTEND_VECTOR_INREG:
4688   case ISD::SIGN_EXTEND_VECTOR_INREG:
4689     assert(VT.isVector() && "This DAG node is restricted to vector types.");
4690     assert(Operand.getValueType().bitsLE(VT) &&
4691            "The input must be the same size or smaller than the result.");
4692     assert(VT.getVectorNumElements() <
4693              Operand.getValueType().getVectorNumElements() &&
4694            "The destination vector type must have fewer lanes than the input.");
4695     break;
4696   case ISD::ABS:
4697     assert(VT.isInteger() && VT == Operand.getValueType() &&
4698            "Invalid ABS!");
4699     if (OpOpcode == ISD::UNDEF)
4700       return getUNDEF(VT);
4701     break;
4702   case ISD::BSWAP:
4703     assert(VT.isInteger() && VT == Operand.getValueType() &&
4704            "Invalid BSWAP!");
4705     assert((VT.getScalarSizeInBits() % 16 == 0) &&
4706            "BSWAP types must be a multiple of 16 bits!");
4707     if (OpOpcode == ISD::UNDEF)
4708       return getUNDEF(VT);
4709     break;
4710   case ISD::BITREVERSE:
4711     assert(VT.isInteger() && VT == Operand.getValueType() &&
4712            "Invalid BITREVERSE!");
4713     if (OpOpcode == ISD::UNDEF)
4714       return getUNDEF(VT);
4715     break;
4716   case ISD::BITCAST:
4717     // Basic sanity checking.
4718     assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&
4719            "Cannot BITCAST between types of different sizes!");
4720     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
4721     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
4722       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
4723     if (OpOpcode == ISD::UNDEF)
4724       return getUNDEF(VT);
4725     break;
4726   case ISD::SCALAR_TO_VECTOR:
4727     assert(VT.isVector() && !Operand.getValueType().isVector() &&
4728            (VT.getVectorElementType() == Operand.getValueType() ||
4729             (VT.getVectorElementType().isInteger() &&
4730              Operand.getValueType().isInteger() &&
4731              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
4732            "Illegal SCALAR_TO_VECTOR node!");
4733     if (OpOpcode == ISD::UNDEF)
4734       return getUNDEF(VT);
4735     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
4736     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
4737         isa<ConstantSDNode>(Operand.getOperand(1)) &&
4738         Operand.getConstantOperandVal(1) == 0 &&
4739         Operand.getOperand(0).getValueType() == VT)
4740       return Operand.getOperand(0);
4741     break;
4742   case ISD::FNEG:
4743     // Negation of an unknown bag of bits is still completely undefined.
4744     if (OpOpcode == ISD::UNDEF)
4745       return getUNDEF(VT);
4746 
4747     if (OpOpcode == ISD::FNEG)  // --X -> X
4748       return Operand.getOperand(0);
4749     break;
4750   case ISD::FABS:
4751     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
4752       return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
4753     break;
4754   case ISD::VSCALE:
4755     assert(VT == Operand.getValueType() && "Unexpected VT!");
4756     break;
4757   case ISD::VECREDUCE_SMIN:
4758   case ISD::VECREDUCE_UMAX:
4759     if (Operand.getValueType().getScalarType() == MVT::i1)
4760       return getNode(ISD::VECREDUCE_OR, DL, VT, Operand);
4761     break;
4762   case ISD::VECREDUCE_SMAX:
4763   case ISD::VECREDUCE_UMIN:
4764     if (Operand.getValueType().getScalarType() == MVT::i1)
4765       return getNode(ISD::VECREDUCE_AND, DL, VT, Operand);
4766     break;
4767   }
4768 
4769   SDNode *N;
4770   SDVTList VTs = getVTList(VT);
4771   SDValue Ops[] = {Operand};
4772   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
4773     FoldingSetNodeID ID;
4774     AddNodeIDNode(ID, Opcode, VTs, Ops);
4775     void *IP = nullptr;
4776     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
4777       E->intersectFlagsWith(Flags);
4778       return SDValue(E, 0);
4779     }
4780 
4781     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4782     N->setFlags(Flags);
4783     createOperands(N, Ops);
4784     CSEMap.InsertNode(N, IP);
4785   } else {
4786     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4787     createOperands(N, Ops);
4788   }
4789 
4790   InsertNode(N);
4791   SDValue V = SDValue(N, 0);
4792   NewSDValueDbgMsg(V, "Creating new node: ", this);
4793   return V;
4794 }
4795 
4796 static llvm::Optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
4797                                        const APInt &C2) {
4798   switch (Opcode) {
4799   case ISD::ADD:  return C1 + C2;
4800   case ISD::SUB:  return C1 - C2;
4801   case ISD::MUL:  return C1 * C2;
4802   case ISD::AND:  return C1 & C2;
4803   case ISD::OR:   return C1 | C2;
4804   case ISD::XOR:  return C1 ^ C2;
4805   case ISD::SHL:  return C1 << C2;
4806   case ISD::SRL:  return C1.lshr(C2);
4807   case ISD::SRA:  return C1.ashr(C2);
4808   case ISD::ROTL: return C1.rotl(C2);
4809   case ISD::ROTR: return C1.rotr(C2);
4810   case ISD::SMIN: return C1.sle(C2) ? C1 : C2;
4811   case ISD::SMAX: return C1.sge(C2) ? C1 : C2;
4812   case ISD::UMIN: return C1.ule(C2) ? C1 : C2;
4813   case ISD::UMAX: return C1.uge(C2) ? C1 : C2;
4814   case ISD::SADDSAT: return C1.sadd_sat(C2);
4815   case ISD::UADDSAT: return C1.uadd_sat(C2);
4816   case ISD::SSUBSAT: return C1.ssub_sat(C2);
4817   case ISD::USUBSAT: return C1.usub_sat(C2);
4818   case ISD::UDIV:
4819     if (!C2.getBoolValue())
4820       break;
4821     return C1.udiv(C2);
4822   case ISD::UREM:
4823     if (!C2.getBoolValue())
4824       break;
4825     return C1.urem(C2);
4826   case ISD::SDIV:
4827     if (!C2.getBoolValue())
4828       break;
4829     return C1.sdiv(C2);
4830   case ISD::SREM:
4831     if (!C2.getBoolValue())
4832       break;
4833     return C1.srem(C2);
4834   }
4835   return llvm::None;
4836 }
4837 
4838 SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
4839                                        const GlobalAddressSDNode *GA,
4840                                        const SDNode *N2) {
4841   if (GA->getOpcode() != ISD::GlobalAddress)
4842     return SDValue();
4843   if (!TLI->isOffsetFoldingLegal(GA))
4844     return SDValue();
4845   auto *C2 = dyn_cast<ConstantSDNode>(N2);
4846   if (!C2)
4847     return SDValue();
4848   int64_t Offset = C2->getSExtValue();
4849   switch (Opcode) {
4850   case ISD::ADD: break;
4851   case ISD::SUB: Offset = -uint64_t(Offset); break;
4852   default: return SDValue();
4853   }
4854   return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
4855                           GA->getOffset() + uint64_t(Offset));
4856 }
4857 
4858 bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
4859   switch (Opcode) {
4860   case ISD::SDIV:
4861   case ISD::UDIV:
4862   case ISD::SREM:
4863   case ISD::UREM: {
4864     // If a divisor is zero/undef or any element of a divisor vector is
4865     // zero/undef, the whole op is undef.
4866     assert(Ops.size() == 2 && "Div/rem should have 2 operands");
4867     SDValue Divisor = Ops[1];
4868     if (Divisor.isUndef() || isNullConstant(Divisor))
4869       return true;
4870 
4871     return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
4872            llvm::any_of(Divisor->op_values(),
4873                         [](SDValue V) { return V.isUndef() ||
4874                                         isNullConstant(V); });
4875     // TODO: Handle signed overflow.
4876   }
4877   // TODO: Handle oversized shifts.
4878   default:
4879     return false;
4880   }
4881 }
4882 
4883 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4884                                              EVT VT, ArrayRef<SDValue> Ops) {
4885   // If the opcode is a target-specific ISD node, there's nothing we can
4886   // do here and the operand rules may not line up with the below, so
4887   // bail early.
4888   if (Opcode >= ISD::BUILTIN_OP_END)
4889     return SDValue();
4890 
4891   // For now, the array Ops should only contain two values.
4892   // This enforcement will be removed once this function is merged with
4893   // FoldConstantVectorArithmetic
4894   if (Ops.size() != 2)
4895     return SDValue();
4896 
4897   if (isUndef(Opcode, Ops))
4898     return getUNDEF(VT);
4899 
4900   SDNode *N1 = Ops[0].getNode();
4901   SDNode *N2 = Ops[1].getNode();
4902 
4903   // Handle the case of two scalars.
4904   if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) {
4905     if (auto *C2 = dyn_cast<ConstantSDNode>(N2)) {
4906       if (C1->isOpaque() || C2->isOpaque())
4907         return SDValue();
4908 
4909       Optional<APInt> FoldAttempt =
4910           FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
4911       if (!FoldAttempt)
4912         return SDValue();
4913 
4914       SDValue Folded = getConstant(FoldAttempt.getValue(), DL, VT);
4915       assert((!Folded || !VT.isVector()) &&
4916              "Can't fold vectors ops with scalar operands");
4917       return Folded;
4918     }
4919   }
4920 
4921   // fold (add Sym, c) -> Sym+c
4922   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N1))
4923     return FoldSymbolOffset(Opcode, VT, GA, N2);
4924   if (TLI->isCommutativeBinOp(Opcode))
4925     if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2))
4926       return FoldSymbolOffset(Opcode, VT, GA, N1);
4927 
4928   // TODO: All the folds below are performed lane-by-lane and assume a fixed
4929   // vector width, however we should be able to do constant folds involving
4930   // splat vector nodes too.
4931   if (VT.isScalableVector())
4932     return SDValue();
4933 
4934   // For fixed width vectors, extract each constant element and fold them
4935   // individually. Either input may be an undef value.
4936   auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
4937   if (!BV1 && !N1->isUndef())
4938     return SDValue();
4939   auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
4940   if (!BV2 && !N2->isUndef())
4941     return SDValue();
4942   // If both operands are undef, that's handled the same way as scalars.
4943   if (!BV1 && !BV2)
4944     return SDValue();
4945 
4946   assert((!BV1 || !BV2 || BV1->getNumOperands() == BV2->getNumOperands()) &&
4947          "Vector binop with different number of elements in operands?");
4948 
4949   EVT SVT = VT.getScalarType();
4950   EVT LegalSVT = SVT;
4951   if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
4952     LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4953     if (LegalSVT.bitsLT(SVT))
4954       return SDValue();
4955   }
4956   SmallVector<SDValue, 4> Outputs;
4957   unsigned NumOps = BV1 ? BV1->getNumOperands() : BV2->getNumOperands();
4958   for (unsigned I = 0; I != NumOps; ++I) {
4959     SDValue V1 = BV1 ? BV1->getOperand(I) : getUNDEF(SVT);
4960     SDValue V2 = BV2 ? BV2->getOperand(I) : getUNDEF(SVT);
4961     if (SVT.isInteger()) {
4962       if (V1->getValueType(0).bitsGT(SVT))
4963         V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
4964       if (V2->getValueType(0).bitsGT(SVT))
4965         V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
4966     }
4967 
4968     if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
4969       return SDValue();
4970 
4971     // Fold one vector element.
4972     SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
4973     if (LegalSVT != SVT)
4974       ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
4975 
4976     // Scalar folding only succeeded if the result is a constant or UNDEF.
4977     if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
4978         ScalarResult.getOpcode() != ISD::ConstantFP)
4979       return SDValue();
4980     Outputs.push_back(ScalarResult);
4981   }
4982 
4983   assert(VT.getVectorNumElements() == Outputs.size() &&
4984          "Vector size mismatch!");
4985 
4986   // We may have a vector type but a scalar result. Create a splat.
4987   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
4988 
4989   // Build a big vector out of the scalar elements we generated.
4990   return getBuildVector(VT, SDLoc(), Outputs);
4991 }
4992 
4993 // TODO: Merge with FoldConstantArithmetic
4994 SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
4995                                                    const SDLoc &DL, EVT VT,
4996                                                    ArrayRef<SDValue> Ops,
4997                                                    const SDNodeFlags Flags) {
4998   // If the opcode is a target-specific ISD node, there's nothing we can
4999   // do here and the operand rules may not line up with the below, so
5000   // bail early.
5001   if (Opcode >= ISD::BUILTIN_OP_END)
5002     return SDValue();
5003 
5004   if (isUndef(Opcode, Ops))
5005     return getUNDEF(VT);
5006 
5007   // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
5008   if (!VT.isVector())
5009     return SDValue();
5010 
5011   // TODO: All the folds below are performed lane-by-lane and assume a fixed
5012   // vector width, however we should be able to do constant folds involving
5013   // splat vector nodes too.
5014   if (VT.isScalableVector())
5015     return SDValue();
5016 
5017   // From this point onwards all vectors are assumed to be fixed width.
5018   unsigned NumElts = VT.getVectorNumElements();
5019 
5020   auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
5021     return !Op.getValueType().isVector() ||
5022            Op.getValueType().getVectorNumElements() == NumElts;
5023   };
5024 
5025   auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
5026     BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
5027     return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) ||
5028            (BV && BV->isConstant());
5029   };
5030 
5031   // All operands must be vector types with the same number of elements as
5032   // the result type and must be either UNDEF or a build vector of constant
5033   // or UNDEF scalars.
5034   if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) ||
5035       !llvm::all_of(Ops, IsScalarOrSameVectorSize))
5036     return SDValue();
5037 
5038   // If we are comparing vectors, then the result needs to be a i1 boolean
5039   // that is then sign-extended back to the legal result type.
5040   EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
5041 
5042   // Find legal integer scalar type for constant promotion and
5043   // ensure that its scalar size is at least as large as source.
5044   EVT LegalSVT = VT.getScalarType();
5045   if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
5046     LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
5047     if (LegalSVT.bitsLT(VT.getScalarType()))
5048       return SDValue();
5049   }
5050 
5051   // Constant fold each scalar lane separately.
5052   SmallVector<SDValue, 4> ScalarResults;
5053   for (unsigned i = 0; i != NumElts; i++) {
5054     SmallVector<SDValue, 4> ScalarOps;
5055     for (SDValue Op : Ops) {
5056       EVT InSVT = Op.getValueType().getScalarType();
5057       BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
5058       if (!InBV) {
5059         // We've checked that this is UNDEF or a constant of some kind.
5060         if (Op.isUndef())
5061           ScalarOps.push_back(getUNDEF(InSVT));
5062         else
5063           ScalarOps.push_back(Op);
5064         continue;
5065       }
5066 
5067       SDValue ScalarOp = InBV->getOperand(i);
5068       EVT ScalarVT = ScalarOp.getValueType();
5069 
5070       // Build vector (integer) scalar operands may need implicit
5071       // truncation - do this before constant folding.
5072       if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
5073         ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
5074 
5075       ScalarOps.push_back(ScalarOp);
5076     }
5077 
5078     // Constant fold the scalar operands.
5079     SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
5080 
5081     // Legalize the (integer) scalar constant if necessary.
5082     if (LegalSVT != SVT)
5083       ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
5084 
5085     // Scalar folding only succeeded if the result is a constant or UNDEF.
5086     if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
5087         ScalarResult.getOpcode() != ISD::ConstantFP)
5088       return SDValue();
5089     ScalarResults.push_back(ScalarResult);
5090   }
5091 
5092   SDValue V = getBuildVector(VT, DL, ScalarResults);
5093   NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
5094   return V;
5095 }
5096 
5097 SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
5098                                          EVT VT, SDValue N1, SDValue N2) {
5099   // TODO: We don't do any constant folding for strict FP opcodes here, but we
5100   //       should. That will require dealing with a potentially non-default
5101   //       rounding mode, checking the "opStatus" return value from the APFloat
5102   //       math calculations, and possibly other variations.
5103   auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
5104   auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
5105   if (N1CFP && N2CFP) {
5106     APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF();
5107     switch (Opcode) {
5108     case ISD::FADD:
5109       C1.add(C2, APFloat::rmNearestTiesToEven);
5110       return getConstantFP(C1, DL, VT);
5111     case ISD::FSUB:
5112       C1.subtract(C2, APFloat::rmNearestTiesToEven);
5113       return getConstantFP(C1, DL, VT);
5114     case ISD::FMUL:
5115       C1.multiply(C2, APFloat::rmNearestTiesToEven);
5116       return getConstantFP(C1, DL, VT);
5117     case ISD::FDIV:
5118       C1.divide(C2, APFloat::rmNearestTiesToEven);
5119       return getConstantFP(C1, DL, VT);
5120     case ISD::FREM:
5121       C1.mod(C2);
5122       return getConstantFP(C1, DL, VT);
5123     case ISD::FCOPYSIGN:
5124       C1.copySign(C2);
5125       return getConstantFP(C1, DL, VT);
5126     default: break;
5127     }
5128   }
5129   if (N1CFP && Opcode == ISD::FP_ROUND) {
5130     APFloat C1 = N1CFP->getValueAPF();    // make copy
5131     bool Unused;
5132     // This can return overflow, underflow, or inexact; we don't care.
5133     // FIXME need to be more flexible about rounding mode.
5134     (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
5135                       &Unused);
5136     return getConstantFP(C1, DL, VT);
5137   }
5138 
5139   switch (Opcode) {
5140   case ISD::FSUB:
5141     // -0.0 - undef --> undef (consistent with "fneg undef")
5142     if (N1CFP && N1CFP->getValueAPF().isNegZero() && N2.isUndef())
5143       return getUNDEF(VT);
5144     LLVM_FALLTHROUGH;
5145 
5146   case ISD::FADD:
5147   case ISD::FMUL:
5148   case ISD::FDIV:
5149   case ISD::FREM:
5150     // If both operands are undef, the result is undef. If 1 operand is undef,
5151     // the result is NaN. This should match the behavior of the IR optimizer.
5152     if (N1.isUndef() && N2.isUndef())
5153       return getUNDEF(VT);
5154     if (N1.isUndef() || N2.isUndef())
5155       return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT);
5156   }
5157   return SDValue();
5158 }
5159 
5160 SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
5161   assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
5162 
5163   // There's no need to assert on a byte-aligned pointer. All pointers are at
5164   // least byte aligned.
5165   if (A == Align(1))
5166     return Val;
5167 
5168   FoldingSetNodeID ID;
5169   AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val});
5170   ID.AddInteger(A.value());
5171 
5172   void *IP = nullptr;
5173   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
5174     return SDValue(E, 0);
5175 
5176   auto *N = newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(),
5177                                          Val.getValueType(), A);
5178   createOperands(N, {Val});
5179 
5180   CSEMap.InsertNode(N, IP);
5181   InsertNode(N);
5182 
5183   SDValue V(N, 0);
5184   NewSDValueDbgMsg(V, "Creating new node: ", this);
5185   return V;
5186 }
5187 
5188 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5189                               SDValue N1, SDValue N2) {
5190   SDNodeFlags Flags;
5191   if (Inserter)
5192     Flags = Inserter->getFlags();
5193   return getNode(Opcode, DL, VT, N1, N2, Flags);
5194 }
5195 
5196 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5197                               SDValue N1, SDValue N2, const SDNodeFlags Flags) {
5198   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
5199   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
5200   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
5201   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
5202 
5203   // Canonicalize constant to RHS if commutative.
5204   if (TLI->isCommutativeBinOp(Opcode)) {
5205     if (N1C && !N2C) {
5206       std::swap(N1C, N2C);
5207       std::swap(N1, N2);
5208     } else if (N1CFP && !N2CFP) {
5209       std::swap(N1CFP, N2CFP);
5210       std::swap(N1, N2);
5211     }
5212   }
5213 
5214   switch (Opcode) {
5215   default: break;
5216   case ISD::TokenFactor:
5217     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
5218            N2.getValueType() == MVT::Other && "Invalid token factor!");
5219     // Fold trivial token factors.
5220     if (N1.getOpcode() == ISD::EntryToken) return N2;
5221     if (N2.getOpcode() == ISD::EntryToken) return N1;
5222     if (N1 == N2) return N1;
5223     break;
5224   case ISD::BUILD_VECTOR: {
5225     // Attempt to simplify BUILD_VECTOR.
5226     SDValue Ops[] = {N1, N2};
5227     if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5228       return V;
5229     break;
5230   }
5231   case ISD::CONCAT_VECTORS: {
5232     SDValue Ops[] = {N1, N2};
5233     if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
5234       return V;
5235     break;
5236   }
5237   case ISD::AND:
5238     assert(VT.isInteger() && "This operator does not apply to FP types!");
5239     assert(N1.getValueType() == N2.getValueType() &&
5240            N1.getValueType() == VT && "Binary operator types must match!");
5241     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
5242     // worth handling here.
5243     if (N2C && N2C->isNullValue())
5244       return N2;
5245     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
5246       return N1;
5247     break;
5248   case ISD::OR:
5249   case ISD::XOR:
5250   case ISD::ADD:
5251   case ISD::SUB:
5252     assert(VT.isInteger() && "This operator does not apply to FP types!");
5253     assert(N1.getValueType() == N2.getValueType() &&
5254            N1.getValueType() == VT && "Binary operator types must match!");
5255     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
5256     // it's worth handling here.
5257     if (N2C && N2C->isNullValue())
5258       return N1;
5259     break;
5260   case ISD::MUL:
5261     assert(VT.isInteger() && "This operator does not apply to FP types!");
5262     assert(N1.getValueType() == N2.getValueType() &&
5263            N1.getValueType() == VT && "Binary operator types must match!");
5264     if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
5265       APInt MulImm = cast<ConstantSDNode>(N1->getOperand(0))->getAPIntValue();
5266       APInt N2CImm = N2C->getAPIntValue();
5267       return getVScale(DL, VT, MulImm * N2CImm);
5268     }
5269     break;
5270   case ISD::UDIV:
5271   case ISD::UREM:
5272   case ISD::MULHU:
5273   case ISD::MULHS:
5274   case ISD::SDIV:
5275   case ISD::SREM:
5276   case ISD::SADDSAT:
5277   case ISD::SSUBSAT:
5278   case ISD::UADDSAT:
5279   case ISD::USUBSAT:
5280     assert(VT.isInteger() && "This operator does not apply to FP types!");
5281     assert(N1.getValueType() == N2.getValueType() &&
5282            N1.getValueType() == VT && "Binary operator types must match!");
5283     break;
5284   case ISD::SMIN:
5285   case ISD::UMAX:
5286     assert(VT.isInteger() && "This operator does not apply to FP types!");
5287     assert(N1.getValueType() == N2.getValueType() &&
5288            N1.getValueType() == VT && "Binary operator types must match!");
5289     if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
5290       return getNode(ISD::OR, DL, VT, N1, N2);
5291     break;
5292   case ISD::SMAX:
5293   case ISD::UMIN:
5294     assert(VT.isInteger() && "This operator does not apply to FP types!");
5295     assert(N1.getValueType() == N2.getValueType() &&
5296            N1.getValueType() == VT && "Binary operator types must match!");
5297     if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
5298       return getNode(ISD::AND, DL, VT, N1, N2);
5299     break;
5300   case ISD::FADD:
5301   case ISD::FSUB:
5302   case ISD::FMUL:
5303   case ISD::FDIV:
5304   case ISD::FREM:
5305     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
5306     assert(N1.getValueType() == N2.getValueType() &&
5307            N1.getValueType() == VT && "Binary operator types must match!");
5308     if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags))
5309       return V;
5310     break;
5311   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
5312     assert(N1.getValueType() == VT &&
5313            N1.getValueType().isFloatingPoint() &&
5314            N2.getValueType().isFloatingPoint() &&
5315            "Invalid FCOPYSIGN!");
5316     break;
5317   case ISD::SHL:
5318     if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
5319       APInt MulImm = cast<ConstantSDNode>(N1->getOperand(0))->getAPIntValue();
5320       APInt ShiftImm = N2C->getAPIntValue();
5321       return getVScale(DL, VT, MulImm << ShiftImm);
5322     }
5323     LLVM_FALLTHROUGH;
5324   case ISD::SRA:
5325   case ISD::SRL:
5326     if (SDValue V = simplifyShift(N1, N2))
5327       return V;
5328     LLVM_FALLTHROUGH;
5329   case ISD::ROTL:
5330   case ISD::ROTR:
5331     assert(VT == N1.getValueType() &&
5332            "Shift operators return type must be the same as their first arg");
5333     assert(VT.isInteger() && N2.getValueType().isInteger() &&
5334            "Shifts only work on integers");
5335     assert((!VT.isVector() || VT == N2.getValueType()) &&
5336            "Vector shift amounts must be in the same as their first arg");
5337     // Verify that the shift amount VT is big enough to hold valid shift
5338     // amounts.  This catches things like trying to shift an i1024 value by an
5339     // i8, which is easy to fall into in generic code that uses
5340     // TLI.getShiftAmount().
5341     assert(N2.getValueType().getScalarSizeInBits() >=
5342                Log2_32_Ceil(VT.getScalarSizeInBits()) &&
5343            "Invalid use of small shift amount with oversized value!");
5344 
5345     // Always fold shifts of i1 values so the code generator doesn't need to
5346     // handle them.  Since we know the size of the shift has to be less than the
5347     // size of the value, the shift/rotate count is guaranteed to be zero.
5348     if (VT == MVT::i1)
5349       return N1;
5350     if (N2C && N2C->isNullValue())
5351       return N1;
5352     break;
5353   case ISD::FP_ROUND:
5354     assert(VT.isFloatingPoint() &&
5355            N1.getValueType().isFloatingPoint() &&
5356            VT.bitsLE(N1.getValueType()) &&
5357            N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
5358            "Invalid FP_ROUND!");
5359     if (N1.getValueType() == VT) return N1;  // noop conversion.
5360     break;
5361   case ISD::AssertSext:
5362   case ISD::AssertZext: {
5363     EVT EVT = cast<VTSDNode>(N2)->getVT();
5364     assert(VT == N1.getValueType() && "Not an inreg extend!");
5365     assert(VT.isInteger() && EVT.isInteger() &&
5366            "Cannot *_EXTEND_INREG FP types");
5367     assert(!EVT.isVector() &&
5368            "AssertSExt/AssertZExt type should be the vector element type "
5369            "rather than the vector type!");
5370     assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
5371     if (VT.getScalarType() == EVT) return N1; // noop assertion.
5372     break;
5373   }
5374   case ISD::SIGN_EXTEND_INREG: {
5375     EVT EVT = cast<VTSDNode>(N2)->getVT();
5376     assert(VT == N1.getValueType() && "Not an inreg extend!");
5377     assert(VT.isInteger() && EVT.isInteger() &&
5378            "Cannot *_EXTEND_INREG FP types");
5379     assert(EVT.isVector() == VT.isVector() &&
5380            "SIGN_EXTEND_INREG type should be vector iff the operand "
5381            "type is vector!");
5382     assert((!EVT.isVector() ||
5383             EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
5384            "Vector element counts must match in SIGN_EXTEND_INREG");
5385     assert(EVT.bitsLE(VT) && "Not extending!");
5386     if (EVT == VT) return N1;  // Not actually extending
5387 
5388     auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
5389       unsigned FromBits = EVT.getScalarSizeInBits();
5390       Val <<= Val.getBitWidth() - FromBits;
5391       Val.ashrInPlace(Val.getBitWidth() - FromBits);
5392       return getConstant(Val, DL, ConstantVT);
5393     };
5394 
5395     if (N1C) {
5396       const APInt &Val = N1C->getAPIntValue();
5397       return SignExtendInReg(Val, VT);
5398     }
5399     if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
5400       SmallVector<SDValue, 8> Ops;
5401       llvm::EVT OpVT = N1.getOperand(0).getValueType();
5402       for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
5403         SDValue Op = N1.getOperand(i);
5404         if (Op.isUndef()) {
5405           Ops.push_back(getUNDEF(OpVT));
5406           continue;
5407         }
5408         ConstantSDNode *C = cast<ConstantSDNode>(Op);
5409         APInt Val = C->getAPIntValue();
5410         Ops.push_back(SignExtendInReg(Val, OpVT));
5411       }
5412       return getBuildVector(VT, DL, Ops);
5413     }
5414     break;
5415   }
5416   case ISD::EXTRACT_VECTOR_ELT:
5417     assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
5418            "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
5419              element type of the vector.");
5420 
5421     // Extract from an undefined value or using an undefined index is undefined.
5422     if (N1.isUndef() || N2.isUndef())
5423       return getUNDEF(VT);
5424 
5425     // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
5426     // vectors. For scalable vectors we will provide appropriate support for
5427     // dealing with arbitrary indices.
5428     if (N2C && N1.getValueType().isFixedLengthVector() &&
5429         N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
5430       return getUNDEF(VT);
5431 
5432     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
5433     // expanding copies of large vectors from registers. This only works for
5434     // fixed length vectors, since we need to know the exact number of
5435     // elements.
5436     if (N2C && N1.getOperand(0).getValueType().isFixedLengthVector() &&
5437         N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0) {
5438       unsigned Factor =
5439         N1.getOperand(0).getValueType().getVectorNumElements();
5440       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
5441                      N1.getOperand(N2C->getZExtValue() / Factor),
5442                      getVectorIdxConstant(N2C->getZExtValue() % Factor, DL));
5443     }
5444 
5445     // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
5446     // lowering is expanding large vector constants.
5447     if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
5448                 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
5449       assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
5450               N1.getValueType().isFixedLengthVector()) &&
5451              "BUILD_VECTOR used for scalable vectors");
5452       unsigned Index =
5453           N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
5454       SDValue Elt = N1.getOperand(Index);
5455 
5456       if (VT != Elt.getValueType())
5457         // If the vector element type is not legal, the BUILD_VECTOR operands
5458         // are promoted and implicitly truncated, and the result implicitly
5459         // extended. Make that explicit here.
5460         Elt = getAnyExtOrTrunc(Elt, DL, VT);
5461 
5462       return Elt;
5463     }
5464 
5465     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
5466     // operations are lowered to scalars.
5467     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
5468       // If the indices are the same, return the inserted element else
5469       // if the indices are known different, extract the element from
5470       // the original vector.
5471       SDValue N1Op2 = N1.getOperand(2);
5472       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
5473 
5474       if (N1Op2C && N2C) {
5475         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
5476           if (VT == N1.getOperand(1).getValueType())
5477             return N1.getOperand(1);
5478           else
5479             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
5480         }
5481 
5482         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
5483       }
5484     }
5485 
5486     // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
5487     // when vector types are scalarized and v1iX is legal.
5488     // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
5489     // Here we are completely ignoring the extract element index (N2),
5490     // which is fine for fixed width vectors, since any index other than 0
5491     // is undefined anyway. However, this cannot be ignored for scalable
5492     // vectors - in theory we could support this, but we don't want to do this
5493     // without a profitability check.
5494     if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5495         N1.getValueType().isFixedLengthVector() &&
5496         N1.getValueType().getVectorNumElements() == 1) {
5497       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
5498                      N1.getOperand(1));
5499     }
5500     break;
5501   case ISD::EXTRACT_ELEMENT:
5502     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
5503     assert(!N1.getValueType().isVector() && !VT.isVector() &&
5504            (N1.getValueType().isInteger() == VT.isInteger()) &&
5505            N1.getValueType() != VT &&
5506            "Wrong types for EXTRACT_ELEMENT!");
5507 
5508     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
5509     // 64-bit integers into 32-bit parts.  Instead of building the extract of
5510     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
5511     if (N1.getOpcode() == ISD::BUILD_PAIR)
5512       return N1.getOperand(N2C->getZExtValue());
5513 
5514     // EXTRACT_ELEMENT of a constant int is also very common.
5515     if (N1C) {
5516       unsigned ElementSize = VT.getSizeInBits();
5517       unsigned Shift = ElementSize * N2C->getZExtValue();
5518       APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift);
5519       return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
5520     }
5521     break;
5522   case ISD::EXTRACT_SUBVECTOR:
5523     EVT N1VT = N1.getValueType();
5524     assert(VT.isVector() && N1VT.isVector() &&
5525            "Extract subvector VTs must be vectors!");
5526     assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
5527            "Extract subvector VTs must have the same element type!");
5528     assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
5529            "Cannot extract a scalable vector from a fixed length vector!");
5530     assert((VT.isScalableVector() != N1VT.isScalableVector() ||
5531             VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
5532            "Extract subvector must be from larger vector to smaller vector!");
5533     assert(N2C && "Extract subvector index must be a constant");
5534     assert((VT.isScalableVector() != N1VT.isScalableVector() ||
5535             (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
5536                 N1VT.getVectorMinNumElements()) &&
5537            "Extract subvector overflow!");
5538     assert(N2C->getAPIntValue().getBitWidth() ==
5539                TLI->getVectorIdxTy(getDataLayout())
5540                    .getSizeInBits()
5541                    .getFixedSize() &&
5542            "Constant index for EXTRACT_SUBVECTOR has an invalid size");
5543 
5544     // Trivial extraction.
5545     if (VT == N1VT)
5546       return N1;
5547 
5548     // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
5549     if (N1.isUndef())
5550       return getUNDEF(VT);
5551 
5552     // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
5553     // the concat have the same type as the extract.
5554     if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 &&
5555         VT == N1.getOperand(0).getValueType()) {
5556       unsigned Factor = VT.getVectorMinNumElements();
5557       return N1.getOperand(N2C->getZExtValue() / Factor);
5558     }
5559 
5560     // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
5561     // during shuffle legalization.
5562     if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
5563         VT == N1.getOperand(1).getValueType())
5564       return N1.getOperand(1);
5565     break;
5566   }
5567 
5568   // Perform trivial constant folding.
5569   if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
5570     return SV;
5571 
5572   if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2))
5573     return V;
5574 
5575   // Canonicalize an UNDEF to the RHS, even over a constant.
5576   if (N1.isUndef()) {
5577     if (TLI->isCommutativeBinOp(Opcode)) {
5578       std::swap(N1, N2);
5579     } else {
5580       switch (Opcode) {
5581       case ISD::SIGN_EXTEND_INREG:
5582       case ISD::SUB:
5583         return getUNDEF(VT);     // fold op(undef, arg2) -> undef
5584       case ISD::UDIV:
5585       case ISD::SDIV:
5586       case ISD::UREM:
5587       case ISD::SREM:
5588       case ISD::SSUBSAT:
5589       case ISD::USUBSAT:
5590         return getConstant(0, DL, VT);    // fold op(undef, arg2) -> 0
5591       }
5592     }
5593   }
5594 
5595   // Fold a bunch of operators when the RHS is undef.
5596   if (N2.isUndef()) {
5597     switch (Opcode) {
5598     case ISD::XOR:
5599       if (N1.isUndef())
5600         // Handle undef ^ undef -> 0 special case. This is a common
5601         // idiom (misuse).
5602         return getConstant(0, DL, VT);
5603       LLVM_FALLTHROUGH;
5604     case ISD::ADD:
5605     case ISD::SUB:
5606     case ISD::UDIV:
5607     case ISD::SDIV:
5608     case ISD::UREM:
5609     case ISD::SREM:
5610       return getUNDEF(VT);       // fold op(arg1, undef) -> undef
5611     case ISD::MUL:
5612     case ISD::AND:
5613     case ISD::SSUBSAT:
5614     case ISD::USUBSAT:
5615       return getConstant(0, DL, VT);  // fold op(arg1, undef) -> 0
5616     case ISD::OR:
5617     case ISD::SADDSAT:
5618     case ISD::UADDSAT:
5619       return getAllOnesConstant(DL, VT);
5620     }
5621   }
5622 
5623   // Memoize this node if possible.
5624   SDNode *N;
5625   SDVTList VTs = getVTList(VT);
5626   SDValue Ops[] = {N1, N2};
5627   if (VT != MVT::Glue) {
5628     FoldingSetNodeID ID;
5629     AddNodeIDNode(ID, Opcode, VTs, Ops);
5630     void *IP = nullptr;
5631     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5632       E->intersectFlagsWith(Flags);
5633       return SDValue(E, 0);
5634     }
5635 
5636     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5637     N->setFlags(Flags);
5638     createOperands(N, Ops);
5639     CSEMap.InsertNode(N, IP);
5640   } else {
5641     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5642     createOperands(N, Ops);
5643   }
5644 
5645   InsertNode(N);
5646   SDValue V = SDValue(N, 0);
5647   NewSDValueDbgMsg(V, "Creating new node: ", this);
5648   return V;
5649 }
5650 
5651 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5652                               SDValue N1, SDValue N2, SDValue N3) {
5653   SDNodeFlags Flags;
5654   if (Inserter)
5655     Flags = Inserter->getFlags();
5656   return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
5657 }
5658 
5659 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5660                               SDValue N1, SDValue N2, SDValue N3,
5661                               const SDNodeFlags Flags) {
5662   // Perform various simplifications.
5663   switch (Opcode) {
5664   case ISD::FMA: {
5665     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
5666     assert(N1.getValueType() == VT && N2.getValueType() == VT &&
5667            N3.getValueType() == VT && "FMA types must match!");
5668     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
5669     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
5670     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
5671     if (N1CFP && N2CFP && N3CFP) {
5672       APFloat  V1 = N1CFP->getValueAPF();
5673       const APFloat &V2 = N2CFP->getValueAPF();
5674       const APFloat &V3 = N3CFP->getValueAPF();
5675       V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
5676       return getConstantFP(V1, DL, VT);
5677     }
5678     break;
5679   }
5680   case ISD::BUILD_VECTOR: {
5681     // Attempt to simplify BUILD_VECTOR.
5682     SDValue Ops[] = {N1, N2, N3};
5683     if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5684       return V;
5685     break;
5686   }
5687   case ISD::CONCAT_VECTORS: {
5688     SDValue Ops[] = {N1, N2, N3};
5689     if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
5690       return V;
5691     break;
5692   }
5693   case ISD::SETCC: {
5694     assert(VT.isInteger() && "SETCC result type must be an integer!");
5695     assert(N1.getValueType() == N2.getValueType() &&
5696            "SETCC operands must have the same type!");
5697     assert(VT.isVector() == N1.getValueType().isVector() &&
5698            "SETCC type should be vector iff the operand type is vector!");
5699     assert((!VT.isVector() || VT.getVectorElementCount() ==
5700                                   N1.getValueType().getVectorElementCount()) &&
5701            "SETCC vector element counts must match!");
5702     // Use FoldSetCC to simplify SETCC's.
5703     if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
5704       return V;
5705     // Vector constant folding.
5706     SDValue Ops[] = {N1, N2, N3};
5707     if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) {
5708       NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
5709       return V;
5710     }
5711     break;
5712   }
5713   case ISD::SELECT:
5714   case ISD::VSELECT:
5715     if (SDValue V = simplifySelect(N1, N2, N3))
5716       return V;
5717     break;
5718   case ISD::VECTOR_SHUFFLE:
5719     llvm_unreachable("should use getVectorShuffle constructor!");
5720   case ISD::INSERT_VECTOR_ELT: {
5721     ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
5722     // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
5723     // for scalable vectors where we will generate appropriate code to
5724     // deal with out-of-bounds cases correctly.
5725     if (N3C && N1.getValueType().isFixedLengthVector() &&
5726         N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
5727       return getUNDEF(VT);
5728 
5729     // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
5730     if (N3.isUndef())
5731       return getUNDEF(VT);
5732 
5733     // If the inserted element is an UNDEF, just use the input vector.
5734     if (N2.isUndef())
5735       return N1;
5736 
5737     break;
5738   }
5739   case ISD::INSERT_SUBVECTOR: {
5740     // Inserting undef into undef is still undef.
5741     if (N1.isUndef() && N2.isUndef())
5742       return getUNDEF(VT);
5743 
5744     EVT N2VT = N2.getValueType();
5745     assert(VT == N1.getValueType() &&
5746            "Dest and insert subvector source types must match!");
5747     assert(VT.isVector() && N2VT.isVector() &&
5748            "Insert subvector VTs must be vectors!");
5749     assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
5750            "Cannot insert a scalable vector into a fixed length vector!");
5751     assert((VT.isScalableVector() != N2VT.isScalableVector() ||
5752             VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
5753            "Insert subvector must be from smaller vector to larger vector!");
5754     assert(isa<ConstantSDNode>(N3) &&
5755            "Insert subvector index must be constant");
5756     assert((VT.isScalableVector() != N2VT.isScalableVector() ||
5757             (N2VT.getVectorMinNumElements() +
5758              cast<ConstantSDNode>(N3)->getZExtValue()) <=
5759                 VT.getVectorMinNumElements()) &&
5760            "Insert subvector overflow!");
5761 
5762     // Trivial insertion.
5763     if (VT == N2VT)
5764       return N2;
5765 
5766     // If this is an insert of an extracted vector into an undef vector, we
5767     // can just use the input to the extract.
5768     if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5769         N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT)
5770       return N2.getOperand(0);
5771     break;
5772   }
5773   case ISD::BITCAST:
5774     // Fold bit_convert nodes from a type to themselves.
5775     if (N1.getValueType() == VT)
5776       return N1;
5777     break;
5778   }
5779 
5780   // Memoize node if it doesn't produce a flag.
5781   SDNode *N;
5782   SDVTList VTs = getVTList(VT);
5783   SDValue Ops[] = {N1, N2, N3};
5784   if (VT != MVT::Glue) {
5785     FoldingSetNodeID ID;
5786     AddNodeIDNode(ID, Opcode, VTs, Ops);
5787     void *IP = nullptr;
5788     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5789       E->intersectFlagsWith(Flags);
5790       return SDValue(E, 0);
5791     }
5792 
5793     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5794     N->setFlags(Flags);
5795     createOperands(N, Ops);
5796     CSEMap.InsertNode(N, IP);
5797   } else {
5798     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5799     createOperands(N, Ops);
5800   }
5801 
5802   InsertNode(N);
5803   SDValue V = SDValue(N, 0);
5804   NewSDValueDbgMsg(V, "Creating new node: ", this);
5805   return V;
5806 }
5807 
5808 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5809                               SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
5810   SDValue Ops[] = { N1, N2, N3, N4 };
5811   return getNode(Opcode, DL, VT, Ops);
5812 }
5813 
5814 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5815                               SDValue N1, SDValue N2, SDValue N3, SDValue N4,
5816                               SDValue N5) {
5817   SDValue Ops[] = { N1, N2, N3, N4, N5 };
5818   return getNode(Opcode, DL, VT, Ops);
5819 }
5820 
5821 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
5822 /// the incoming stack arguments to be loaded from the stack.
5823 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
5824   SmallVector<SDValue, 8> ArgChains;
5825 
5826   // Include the original chain at the beginning of the list. When this is
5827   // used by target LowerCall hooks, this helps legalize find the
5828   // CALLSEQ_BEGIN node.
5829   ArgChains.push_back(Chain);
5830 
5831   // Add a chain value for each stack argument.
5832   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
5833        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
5834     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
5835       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
5836         if (FI->getIndex() < 0)
5837           ArgChains.push_back(SDValue(L, 1));
5838 
5839   // Build a tokenfactor for all the chains.
5840   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
5841 }
5842 
5843 /// getMemsetValue - Vectorized representation of the memset value
5844 /// operand.
5845 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
5846                               const SDLoc &dl) {
5847   assert(!Value.isUndef());
5848 
5849   unsigned NumBits = VT.getScalarSizeInBits();
5850   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
5851     assert(C->getAPIntValue().getBitWidth() == 8);
5852     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
5853     if (VT.isInteger()) {
5854       bool IsOpaque = VT.getSizeInBits() > 64 ||
5855           !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());
5856       return DAG.getConstant(Val, dl, VT, false, IsOpaque);
5857     }
5858     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
5859                              VT);
5860   }
5861 
5862   assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
5863   EVT IntVT = VT.getScalarType();
5864   if (!IntVT.isInteger())
5865     IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
5866 
5867   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
5868   if (NumBits > 8) {
5869     // Use a multiplication with 0x010101... to extend the input to the
5870     // required length.
5871     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
5872     Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
5873                         DAG.getConstant(Magic, dl, IntVT));
5874   }
5875 
5876   if (VT != Value.getValueType() && !VT.isInteger())
5877     Value = DAG.getBitcast(VT.getScalarType(), Value);
5878   if (VT != Value.getValueType())
5879     Value = DAG.getSplatBuildVector(VT, dl, Value);
5880 
5881   return Value;
5882 }
5883 
5884 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
5885 /// used when a memcpy is turned into a memset when the source is a constant
5886 /// string ptr.
5887 static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
5888                                   const TargetLowering &TLI,
5889                                   const ConstantDataArraySlice &Slice) {
5890   // Handle vector with all elements zero.
5891   if (Slice.Array == nullptr) {
5892     if (VT.isInteger())
5893       return DAG.getConstant(0, dl, VT);
5894     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
5895       return DAG.getConstantFP(0.0, dl, VT);
5896     else if (VT.isVector()) {
5897       unsigned NumElts = VT.getVectorNumElements();
5898       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
5899       return DAG.getNode(ISD::BITCAST, dl, VT,
5900                          DAG.getConstant(0, dl,
5901                                          EVT::getVectorVT(*DAG.getContext(),
5902                                                           EltVT, NumElts)));
5903     } else
5904       llvm_unreachable("Expected type!");
5905   }
5906 
5907   assert(!VT.isVector() && "Can't handle vector type here!");
5908   unsigned NumVTBits = VT.getSizeInBits();
5909   unsigned NumVTBytes = NumVTBits / 8;
5910   unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
5911 
5912   APInt Val(NumVTBits, 0);
5913   if (DAG.getDataLayout().isLittleEndian()) {
5914     for (unsigned i = 0; i != NumBytes; ++i)
5915       Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
5916   } else {
5917     for (unsigned i = 0; i != NumBytes; ++i)
5918       Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
5919   }
5920 
5921   // If the "cost" of materializing the integer immediate is less than the cost
5922   // of a load, then it is cost effective to turn the load into the immediate.
5923   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
5924   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
5925     return DAG.getConstant(Val, dl, VT);
5926   return SDValue(nullptr, 0);
5927 }
5928 
5929 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
5930                                            const SDLoc &DL,
5931                                            const SDNodeFlags Flags) {
5932   EVT VT = Base.getValueType();
5933   SDValue Index;
5934 
5935   if (Offset.isScalable())
5936     Index = getVScale(DL, Base.getValueType(),
5937                       APInt(Base.getValueSizeInBits().getFixedSize(),
5938                             Offset.getKnownMinSize()));
5939   else
5940     Index = getConstant(Offset.getFixedSize(), DL, VT);
5941 
5942   return getMemBasePlusOffset(Base, Index, DL, Flags);
5943 }
5944 
5945 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
5946                                            const SDLoc &DL,
5947                                            const SDNodeFlags Flags) {
5948   assert(Offset.getValueType().isInteger());
5949   EVT BasePtrVT = Ptr.getValueType();
5950   return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags);
5951 }
5952 
5953 /// Returns true if memcpy source is constant data.
5954 static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
5955   uint64_t SrcDelta = 0;
5956   GlobalAddressSDNode *G = nullptr;
5957   if (Src.getOpcode() == ISD::GlobalAddress)
5958     G = cast<GlobalAddressSDNode>(Src);
5959   else if (Src.getOpcode() == ISD::ADD &&
5960            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
5961            Src.getOperand(1).getOpcode() == ISD::Constant) {
5962     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
5963     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
5964   }
5965   if (!G)
5966     return false;
5967 
5968   return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
5969                                   SrcDelta + G->getOffset());
5970 }
5971 
5972 static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
5973                                       SelectionDAG &DAG) {
5974   // On Darwin, -Os means optimize for size without hurting performance, so
5975   // only really optimize for size when -Oz (MinSize) is used.
5976   if (MF.getTarget().getTargetTriple().isOSDarwin())
5977     return MF.getFunction().hasMinSize();
5978   return DAG.shouldOptForSize();
5979 }
5980 
5981 static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
5982                           SmallVector<SDValue, 32> &OutChains, unsigned From,
5983                           unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
5984                           SmallVector<SDValue, 16> &OutStoreChains) {
5985   assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
5986   assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
5987   SmallVector<SDValue, 16> GluedLoadChains;
5988   for (unsigned i = From; i < To; ++i) {
5989     OutChains.push_back(OutLoadChains[i]);
5990     GluedLoadChains.push_back(OutLoadChains[i]);
5991   }
5992 
5993   // Chain for all loads.
5994   SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5995                                   GluedLoadChains);
5996 
5997   for (unsigned i = From; i < To; ++i) {
5998     StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
5999     SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
6000                                   ST->getBasePtr(), ST->getMemoryVT(),
6001                                   ST->getMemOperand());
6002     OutChains.push_back(NewStore);
6003   }
6004 }
6005 
6006 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
6007                                        SDValue Chain, SDValue Dst, SDValue Src,
6008                                        uint64_t Size, Align Alignment,
6009                                        bool isVol, bool AlwaysInline,
6010                                        MachinePointerInfo DstPtrInfo,
6011                                        MachinePointerInfo SrcPtrInfo) {
6012   // Turn a memcpy of undef to nop.
6013   // FIXME: We need to honor volatile even is Src is undef.
6014   if (Src.isUndef())
6015     return Chain;
6016 
6017   // Expand memcpy to a series of load and store ops if the size operand falls
6018   // below a certain threshold.
6019   // TODO: In the AlwaysInline case, if the size is big then generate a loop
6020   // rather than maybe a humongous number of loads and stores.
6021   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6022   const DataLayout &DL = DAG.getDataLayout();
6023   LLVMContext &C = *DAG.getContext();
6024   std::vector<EVT> MemOps;
6025   bool DstAlignCanChange = false;
6026   MachineFunction &MF = DAG.getMachineFunction();
6027   MachineFrameInfo &MFI = MF.getFrameInfo();
6028   bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
6029   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6030   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6031     DstAlignCanChange = true;
6032   MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
6033   if (!SrcAlign || Alignment > *SrcAlign)
6034     SrcAlign = Alignment;
6035   assert(SrcAlign && "SrcAlign must be set");
6036   ConstantDataArraySlice Slice;
6037   // If marked as volatile, perform a copy even when marked as constant.
6038   bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
6039   bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
6040   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
6041   const MemOp Op = isZeroConstant
6042                        ? MemOp::Set(Size, DstAlignCanChange, Alignment,
6043                                     /*IsZeroMemset*/ true, isVol)
6044                        : MemOp::Copy(Size, DstAlignCanChange, Alignment,
6045                                      *SrcAlign, isVol, CopyFromConstant);
6046   if (!TLI.findOptimalMemOpLowering(
6047           MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),
6048           SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))
6049     return SDValue();
6050 
6051   if (DstAlignCanChange) {
6052     Type *Ty = MemOps[0].getTypeForEVT(C);
6053     Align NewAlign = DL.getABITypeAlign(Ty);
6054 
6055     // Don't promote to an alignment that would require dynamic stack
6056     // realignment.
6057     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6058     if (!TRI->needsStackRealignment(MF))
6059       while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
6060         NewAlign = NewAlign / 2;
6061 
6062     if (NewAlign > Alignment) {
6063       // Give the stack frame object a larger alignment if needed.
6064       if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
6065         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6066       Alignment = NewAlign;
6067     }
6068   }
6069 
6070   MachineMemOperand::Flags MMOFlags =
6071       isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
6072   SmallVector<SDValue, 16> OutLoadChains;
6073   SmallVector<SDValue, 16> OutStoreChains;
6074   SmallVector<SDValue, 32> OutChains;
6075   unsigned NumMemOps = MemOps.size();
6076   uint64_t SrcOff = 0, DstOff = 0;
6077   for (unsigned i = 0; i != NumMemOps; ++i) {
6078     EVT VT = MemOps[i];
6079     unsigned VTSize = VT.getSizeInBits() / 8;
6080     SDValue Value, Store;
6081 
6082     if (VTSize > Size) {
6083       // Issuing an unaligned load / store pair  that overlaps with the previous
6084       // pair. Adjust the offset accordingly.
6085       assert(i == NumMemOps-1 && i != 0);
6086       SrcOff -= VTSize - Size;
6087       DstOff -= VTSize - Size;
6088     }
6089 
6090     if (CopyFromConstant &&
6091         (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
6092       // It's unlikely a store of a vector immediate can be done in a single
6093       // instruction. It would require a load from a constantpool first.
6094       // We only handle zero vectors here.
6095       // FIXME: Handle other cases where store of vector immediate is done in
6096       // a single instruction.
6097       ConstantDataArraySlice SubSlice;
6098       if (SrcOff < Slice.Length) {
6099         SubSlice = Slice;
6100         SubSlice.move(SrcOff);
6101       } else {
6102         // This is an out-of-bounds access and hence UB. Pretend we read zero.
6103         SubSlice.Array = nullptr;
6104         SubSlice.Offset = 0;
6105         SubSlice.Length = VTSize;
6106       }
6107       Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
6108       if (Value.getNode()) {
6109         Store = DAG.getStore(
6110             Chain, dl, Value,
6111             DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6112             DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags);
6113         OutChains.push_back(Store);
6114       }
6115     }
6116 
6117     if (!Store.getNode()) {
6118       // The type might not be legal for the target.  This should only happen
6119       // if the type is smaller than a legal type, as on PPC, so the right
6120       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
6121       // to Load/Store if NVT==VT.
6122       // FIXME does the case above also need this?
6123       EVT NVT = TLI.getTypeToTransformTo(C, VT);
6124       assert(NVT.bitsGE(VT));
6125 
6126       bool isDereferenceable =
6127         SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
6128       MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
6129       if (isDereferenceable)
6130         SrcMMOFlags |= MachineMemOperand::MODereferenceable;
6131 
6132       Value = DAG.getExtLoad(
6133           ISD::EXTLOAD, dl, NVT, Chain,
6134           DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl),
6135           SrcPtrInfo.getWithOffset(SrcOff), VT,
6136           commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags);
6137       OutLoadChains.push_back(Value.getValue(1));
6138 
6139       Store = DAG.getTruncStore(
6140           Chain, dl, Value,
6141           DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6142           DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags);
6143       OutStoreChains.push_back(Store);
6144     }
6145     SrcOff += VTSize;
6146     DstOff += VTSize;
6147     Size -= VTSize;
6148   }
6149 
6150   unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
6151                                 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
6152   unsigned NumLdStInMemcpy = OutStoreChains.size();
6153 
6154   if (NumLdStInMemcpy) {
6155     // It may be that memcpy might be converted to memset if it's memcpy
6156     // of constants. In such a case, we won't have loads and stores, but
6157     // just stores. In the absence of loads, there is nothing to gang up.
6158     if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
6159       // If target does not care, just leave as it.
6160       for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
6161         OutChains.push_back(OutLoadChains[i]);
6162         OutChains.push_back(OutStoreChains[i]);
6163       }
6164     } else {
6165       // Ld/St less than/equal limit set by target.
6166       if (NumLdStInMemcpy <= GluedLdStLimit) {
6167           chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
6168                                         NumLdStInMemcpy, OutLoadChains,
6169                                         OutStoreChains);
6170       } else {
6171         unsigned NumberLdChain =  NumLdStInMemcpy / GluedLdStLimit;
6172         unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
6173         unsigned GlueIter = 0;
6174 
6175         for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
6176           unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
6177           unsigned IndexTo   = NumLdStInMemcpy - GlueIter;
6178 
6179           chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
6180                                        OutLoadChains, OutStoreChains);
6181           GlueIter += GluedLdStLimit;
6182         }
6183 
6184         // Residual ld/st.
6185         if (RemainingLdStInMemcpy) {
6186           chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
6187                                         RemainingLdStInMemcpy, OutLoadChains,
6188                                         OutStoreChains);
6189         }
6190       }
6191     }
6192   }
6193   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6194 }
6195 
6196 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
6197                                         SDValue Chain, SDValue Dst, SDValue Src,
6198                                         uint64_t Size, Align Alignment,
6199                                         bool isVol, bool AlwaysInline,
6200                                         MachinePointerInfo DstPtrInfo,
6201                                         MachinePointerInfo SrcPtrInfo) {
6202   // Turn a memmove of undef to nop.
6203   // FIXME: We need to honor volatile even is Src is undef.
6204   if (Src.isUndef())
6205     return Chain;
6206 
6207   // Expand memmove to a series of load and store ops if the size operand falls
6208   // below a certain threshold.
6209   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6210   const DataLayout &DL = DAG.getDataLayout();
6211   LLVMContext &C = *DAG.getContext();
6212   std::vector<EVT> MemOps;
6213   bool DstAlignCanChange = false;
6214   MachineFunction &MF = DAG.getMachineFunction();
6215   MachineFrameInfo &MFI = MF.getFrameInfo();
6216   bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
6217   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6218   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6219     DstAlignCanChange = true;
6220   MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
6221   if (!SrcAlign || Alignment > *SrcAlign)
6222     SrcAlign = Alignment;
6223   assert(SrcAlign && "SrcAlign must be set");
6224   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
6225   if (!TLI.findOptimalMemOpLowering(
6226           MemOps, Limit,
6227           MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign,
6228                       /*IsVolatile*/ true),
6229           DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
6230           MF.getFunction().getAttributes()))
6231     return SDValue();
6232 
6233   if (DstAlignCanChange) {
6234     Type *Ty = MemOps[0].getTypeForEVT(C);
6235     Align NewAlign = DL.getABITypeAlign(Ty);
6236     if (NewAlign > Alignment) {
6237       // Give the stack frame object a larger alignment if needed.
6238       if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
6239         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6240       Alignment = NewAlign;
6241     }
6242   }
6243 
6244   MachineMemOperand::Flags MMOFlags =
6245       isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
6246   uint64_t SrcOff = 0, DstOff = 0;
6247   SmallVector<SDValue, 8> LoadValues;
6248   SmallVector<SDValue, 8> LoadChains;
6249   SmallVector<SDValue, 8> OutChains;
6250   unsigned NumMemOps = MemOps.size();
6251   for (unsigned i = 0; i < NumMemOps; i++) {
6252     EVT VT = MemOps[i];
6253     unsigned VTSize = VT.getSizeInBits() / 8;
6254     SDValue Value;
6255 
6256     bool isDereferenceable =
6257       SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
6258     MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
6259     if (isDereferenceable)
6260       SrcMMOFlags |= MachineMemOperand::MODereferenceable;
6261 
6262     Value =
6263         DAG.getLoad(VT, dl, Chain,
6264                     DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl),
6265                     SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags);
6266     LoadValues.push_back(Value);
6267     LoadChains.push_back(Value.getValue(1));
6268     SrcOff += VTSize;
6269   }
6270   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
6271   OutChains.clear();
6272   for (unsigned i = 0; i < NumMemOps; i++) {
6273     EVT VT = MemOps[i];
6274     unsigned VTSize = VT.getSizeInBits() / 8;
6275     SDValue Store;
6276 
6277     Store =
6278         DAG.getStore(Chain, dl, LoadValues[i],
6279                      DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6280                      DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags);
6281     OutChains.push_back(Store);
6282     DstOff += VTSize;
6283   }
6284 
6285   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6286 }
6287 
6288 /// Lower the call to 'memset' intrinsic function into a series of store
6289 /// operations.
6290 ///
6291 /// \param DAG Selection DAG where lowered code is placed.
6292 /// \param dl Link to corresponding IR location.
6293 /// \param Chain Control flow dependency.
6294 /// \param Dst Pointer to destination memory location.
6295 /// \param Src Value of byte to write into the memory.
6296 /// \param Size Number of bytes to write.
6297 /// \param Alignment Alignment of the destination in bytes.
6298 /// \param isVol True if destination is volatile.
6299 /// \param DstPtrInfo IR information on the memory pointer.
6300 /// \returns New head in the control flow, if lowering was successful, empty
6301 /// SDValue otherwise.
6302 ///
6303 /// The function tries to replace 'llvm.memset' intrinsic with several store
6304 /// operations and value calculation code. This is usually profitable for small
6305 /// memory size.
6306 static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
6307                                SDValue Chain, SDValue Dst, SDValue Src,
6308                                uint64_t Size, Align Alignment, bool isVol,
6309                                MachinePointerInfo DstPtrInfo) {
6310   // Turn a memset of undef to nop.
6311   // FIXME: We need to honor volatile even is Src is undef.
6312   if (Src.isUndef())
6313     return Chain;
6314 
6315   // Expand memset to a series of load/store ops if the size operand
6316   // falls below a certain threshold.
6317   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6318   std::vector<EVT> MemOps;
6319   bool DstAlignCanChange = false;
6320   MachineFunction &MF = DAG.getMachineFunction();
6321   MachineFrameInfo &MFI = MF.getFrameInfo();
6322   bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
6323   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6324   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6325     DstAlignCanChange = true;
6326   bool IsZeroVal =
6327     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
6328   if (!TLI.findOptimalMemOpLowering(
6329           MemOps, TLI.getMaxStoresPerMemset(OptSize),
6330           MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
6331           DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))
6332     return SDValue();
6333 
6334   if (DstAlignCanChange) {
6335     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
6336     Align NewAlign = DAG.getDataLayout().getABITypeAlign(Ty);
6337     if (NewAlign > Alignment) {
6338       // Give the stack frame object a larger alignment if needed.
6339       if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
6340         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6341       Alignment = NewAlign;
6342     }
6343   }
6344 
6345   SmallVector<SDValue, 8> OutChains;
6346   uint64_t DstOff = 0;
6347   unsigned NumMemOps = MemOps.size();
6348 
6349   // Find the largest store and generate the bit pattern for it.
6350   EVT LargestVT = MemOps[0];
6351   for (unsigned i = 1; i < NumMemOps; i++)
6352     if (MemOps[i].bitsGT(LargestVT))
6353       LargestVT = MemOps[i];
6354   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
6355 
6356   for (unsigned i = 0; i < NumMemOps; i++) {
6357     EVT VT = MemOps[i];
6358     unsigned VTSize = VT.getSizeInBits() / 8;
6359     if (VTSize > Size) {
6360       // Issuing an unaligned load / store pair  that overlaps with the previous
6361       // pair. Adjust the offset accordingly.
6362       assert(i == NumMemOps-1 && i != 0);
6363       DstOff -= VTSize - Size;
6364     }
6365 
6366     // If this store is smaller than the largest store see whether we can get
6367     // the smaller value for free with a truncate.
6368     SDValue Value = MemSetValue;
6369     if (VT.bitsLT(LargestVT)) {
6370       if (!LargestVT.isVector() && !VT.isVector() &&
6371           TLI.isTruncateFree(LargestVT, VT))
6372         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
6373       else
6374         Value = getMemsetValue(Src, VT, DAG, dl);
6375     }
6376     assert(Value.getValueType() == VT && "Value with wrong type.");
6377     SDValue Store = DAG.getStore(
6378         Chain, dl, Value,
6379         DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6380         DstPtrInfo.getWithOffset(DstOff), Alignment,
6381         isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone);
6382     OutChains.push_back(Store);
6383     DstOff += VT.getSizeInBits() / 8;
6384     Size -= VTSize;
6385   }
6386 
6387   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6388 }
6389 
6390 static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
6391                                             unsigned AS) {
6392   // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
6393   // pointer operands can be losslessly bitcasted to pointers of address space 0
6394   if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) {
6395     report_fatal_error("cannot lower memory intrinsic in address space " +
6396                        Twine(AS));
6397   }
6398 }
6399 
6400 SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
6401                                 SDValue Src, SDValue Size, Align Alignment,
6402                                 bool isVol, bool AlwaysInline, bool isTailCall,
6403                                 MachinePointerInfo DstPtrInfo,
6404                                 MachinePointerInfo SrcPtrInfo) {
6405   // Check to see if we should lower the memcpy to loads and stores first.
6406   // For cases within the target-specified limits, this is the best choice.
6407   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
6408   if (ConstantSize) {
6409     // Memcpy with size zero? Just return the original chain.
6410     if (ConstantSize->isNullValue())
6411       return Chain;
6412 
6413     SDValue Result = getMemcpyLoadsAndStores(
6414         *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
6415         isVol, false, DstPtrInfo, SrcPtrInfo);
6416     if (Result.getNode())
6417       return Result;
6418   }
6419 
6420   // Then check to see if we should lower the memcpy with target-specific
6421   // code. If the target chooses to do this, this is the next best.
6422   if (TSI) {
6423     SDValue Result = TSI->EmitTargetCodeForMemcpy(
6424         *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,
6425         DstPtrInfo, SrcPtrInfo);
6426     if (Result.getNode())
6427       return Result;
6428   }
6429 
6430   // If we really need inline code and the target declined to provide it,
6431   // use a (potentially long) sequence of loads and stores.
6432   if (AlwaysInline) {
6433     assert(ConstantSize && "AlwaysInline requires a constant size!");
6434     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
6435                                    ConstantSize->getZExtValue(), Alignment,
6436                                    isVol, true, DstPtrInfo, SrcPtrInfo);
6437   }
6438 
6439   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6440   checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
6441 
6442   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
6443   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
6444   // respect volatile, so they may do things like read or write memory
6445   // beyond the given memory regions. But fixing this isn't easy, and most
6446   // people don't care.
6447 
6448   // Emit a library call.
6449   TargetLowering::ArgListTy Args;
6450   TargetLowering::ArgListEntry Entry;
6451   Entry.Ty = Type::getInt8PtrTy(*getContext());
6452   Entry.Node = Dst; Args.push_back(Entry);
6453   Entry.Node = Src; Args.push_back(Entry);
6454 
6455   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6456   Entry.Node = Size; Args.push_back(Entry);
6457   // FIXME: pass in SDLoc
6458   TargetLowering::CallLoweringInfo CLI(*this);
6459   CLI.setDebugLoc(dl)
6460       .setChain(Chain)
6461       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
6462                     Dst.getValueType().getTypeForEVT(*getContext()),
6463                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
6464                                       TLI->getPointerTy(getDataLayout())),
6465                     std::move(Args))
6466       .setDiscardResult()
6467       .setTailCall(isTailCall);
6468 
6469   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6470   return CallResult.second;
6471 }
6472 
6473 SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
6474                                       SDValue Dst, unsigned DstAlign,
6475                                       SDValue Src, unsigned SrcAlign,
6476                                       SDValue Size, Type *SizeTy,
6477                                       unsigned ElemSz, bool isTailCall,
6478                                       MachinePointerInfo DstPtrInfo,
6479                                       MachinePointerInfo SrcPtrInfo) {
6480   // Emit a library call.
6481   TargetLowering::ArgListTy Args;
6482   TargetLowering::ArgListEntry Entry;
6483   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6484   Entry.Node = Dst;
6485   Args.push_back(Entry);
6486 
6487   Entry.Node = Src;
6488   Args.push_back(Entry);
6489 
6490   Entry.Ty = SizeTy;
6491   Entry.Node = Size;
6492   Args.push_back(Entry);
6493 
6494   RTLIB::Libcall LibraryCall =
6495       RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz);
6496   if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
6497     report_fatal_error("Unsupported element size");
6498 
6499   TargetLowering::CallLoweringInfo CLI(*this);
6500   CLI.setDebugLoc(dl)
6501       .setChain(Chain)
6502       .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
6503                     Type::getVoidTy(*getContext()),
6504                     getExternalSymbol(TLI->getLibcallName(LibraryCall),
6505                                       TLI->getPointerTy(getDataLayout())),
6506                     std::move(Args))
6507       .setDiscardResult()
6508       .setTailCall(isTailCall);
6509 
6510   std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6511   return CallResult.second;
6512 }
6513 
6514 SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
6515                                  SDValue Src, SDValue Size, Align Alignment,
6516                                  bool isVol, bool isTailCall,
6517                                  MachinePointerInfo DstPtrInfo,
6518                                  MachinePointerInfo SrcPtrInfo) {
6519   // Check to see if we should lower the memmove to loads and stores first.
6520   // For cases within the target-specified limits, this is the best choice.
6521   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
6522   if (ConstantSize) {
6523     // Memmove with size zero? Just return the original chain.
6524     if (ConstantSize->isNullValue())
6525       return Chain;
6526 
6527     SDValue Result = getMemmoveLoadsAndStores(
6528         *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
6529         isVol, false, DstPtrInfo, SrcPtrInfo);
6530     if (Result.getNode())
6531       return Result;
6532   }
6533 
6534   // Then check to see if we should lower the memmove with target-specific
6535   // code. If the target chooses to do this, this is the next best.
6536   if (TSI) {
6537     SDValue Result =
6538         TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size,
6539                                       Alignment, isVol, DstPtrInfo, SrcPtrInfo);
6540     if (Result.getNode())
6541       return Result;
6542   }
6543 
6544   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6545   checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
6546 
6547   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
6548   // not be safe.  See memcpy above for more details.
6549 
6550   // Emit a library call.
6551   TargetLowering::ArgListTy Args;
6552   TargetLowering::ArgListEntry Entry;
6553   Entry.Ty = Type::getInt8PtrTy(*getContext());
6554   Entry.Node = Dst; Args.push_back(Entry);
6555   Entry.Node = Src; Args.push_back(Entry);
6556 
6557   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6558   Entry.Node = Size; Args.push_back(Entry);
6559   // FIXME:  pass in SDLoc
6560   TargetLowering::CallLoweringInfo CLI(*this);
6561   CLI.setDebugLoc(dl)
6562       .setChain(Chain)
6563       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
6564                     Dst.getValueType().getTypeForEVT(*getContext()),
6565                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
6566                                       TLI->getPointerTy(getDataLayout())),
6567                     std::move(Args))
6568       .setDiscardResult()
6569       .setTailCall(isTailCall);
6570 
6571   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6572   return CallResult.second;
6573 }
6574 
6575 SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
6576                                        SDValue Dst, unsigned DstAlign,
6577                                        SDValue Src, unsigned SrcAlign,
6578                                        SDValue Size, Type *SizeTy,
6579                                        unsigned ElemSz, bool isTailCall,
6580                                        MachinePointerInfo DstPtrInfo,
6581                                        MachinePointerInfo SrcPtrInfo) {
6582   // Emit a library call.
6583   TargetLowering::ArgListTy Args;
6584   TargetLowering::ArgListEntry Entry;
6585   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6586   Entry.Node = Dst;
6587   Args.push_back(Entry);
6588 
6589   Entry.Node = Src;
6590   Args.push_back(Entry);
6591 
6592   Entry.Ty = SizeTy;
6593   Entry.Node = Size;
6594   Args.push_back(Entry);
6595 
6596   RTLIB::Libcall LibraryCall =
6597       RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz);
6598   if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
6599     report_fatal_error("Unsupported element size");
6600 
6601   TargetLowering::CallLoweringInfo CLI(*this);
6602   CLI.setDebugLoc(dl)
6603       .setChain(Chain)
6604       .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
6605                     Type::getVoidTy(*getContext()),
6606                     getExternalSymbol(TLI->getLibcallName(LibraryCall),
6607                                       TLI->getPointerTy(getDataLayout())),
6608                     std::move(Args))
6609       .setDiscardResult()
6610       .setTailCall(isTailCall);
6611 
6612   std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6613   return CallResult.second;
6614 }
6615 
6616 SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
6617                                 SDValue Src, SDValue Size, Align Alignment,
6618                                 bool isVol, bool isTailCall,
6619                                 MachinePointerInfo DstPtrInfo) {
6620   // Check to see if we should lower the memset to stores first.
6621   // For cases within the target-specified limits, this is the best choice.
6622   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
6623   if (ConstantSize) {
6624     // Memset with size zero? Just return the original chain.
6625     if (ConstantSize->isNullValue())
6626       return Chain;
6627 
6628     SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
6629                                      ConstantSize->getZExtValue(), Alignment,
6630                                      isVol, DstPtrInfo);
6631 
6632     if (Result.getNode())
6633       return Result;
6634   }
6635 
6636   // Then check to see if we should lower the memset with target-specific
6637   // code. If the target chooses to do this, this is the next best.
6638   if (TSI) {
6639     SDValue Result = TSI->EmitTargetCodeForMemset(
6640         *this, dl, Chain, Dst, Src, Size, Alignment, isVol, DstPtrInfo);
6641     if (Result.getNode())
6642       return Result;
6643   }
6644 
6645   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6646 
6647   // Emit a library call.
6648   TargetLowering::ArgListTy Args;
6649   TargetLowering::ArgListEntry Entry;
6650   Entry.Node = Dst; Entry.Ty = Type::getInt8PtrTy(*getContext());
6651   Args.push_back(Entry);
6652   Entry.Node = Src;
6653   Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
6654   Args.push_back(Entry);
6655   Entry.Node = Size;
6656   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6657   Args.push_back(Entry);
6658 
6659   // FIXME: pass in SDLoc
6660   TargetLowering::CallLoweringInfo CLI(*this);
6661   CLI.setDebugLoc(dl)
6662       .setChain(Chain)
6663       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
6664                     Dst.getValueType().getTypeForEVT(*getContext()),
6665                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
6666                                       TLI->getPointerTy(getDataLayout())),
6667                     std::move(Args))
6668       .setDiscardResult()
6669       .setTailCall(isTailCall);
6670 
6671   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6672   return CallResult.second;
6673 }
6674 
6675 SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
6676                                       SDValue Dst, unsigned DstAlign,
6677                                       SDValue Value, SDValue Size, Type *SizeTy,
6678                                       unsigned ElemSz, bool isTailCall,
6679                                       MachinePointerInfo DstPtrInfo) {
6680   // Emit a library call.
6681   TargetLowering::ArgListTy Args;
6682   TargetLowering::ArgListEntry Entry;
6683   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6684   Entry.Node = Dst;
6685   Args.push_back(Entry);
6686 
6687   Entry.Ty = Type::getInt8Ty(*getContext());
6688   Entry.Node = Value;
6689   Args.push_back(Entry);
6690 
6691   Entry.Ty = SizeTy;
6692   Entry.Node = Size;
6693   Args.push_back(Entry);
6694 
6695   RTLIB::Libcall LibraryCall =
6696       RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz);
6697   if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
6698     report_fatal_error("Unsupported element size");
6699 
6700   TargetLowering::CallLoweringInfo CLI(*this);
6701   CLI.setDebugLoc(dl)
6702       .setChain(Chain)
6703       .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
6704                     Type::getVoidTy(*getContext()),
6705                     getExternalSymbol(TLI->getLibcallName(LibraryCall),
6706                                       TLI->getPointerTy(getDataLayout())),
6707                     std::move(Args))
6708       .setDiscardResult()
6709       .setTailCall(isTailCall);
6710 
6711   std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6712   return CallResult.second;
6713 }
6714 
6715 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6716                                 SDVTList VTList, ArrayRef<SDValue> Ops,
6717                                 MachineMemOperand *MMO) {
6718   FoldingSetNodeID ID;
6719   ID.AddInteger(MemVT.getRawBits());
6720   AddNodeIDNode(ID, Opcode, VTList, Ops);
6721   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6722   void* IP = nullptr;
6723   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6724     cast<AtomicSDNode>(E)->refineAlignment(MMO);
6725     return SDValue(E, 0);
6726   }
6727 
6728   auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
6729                                     VTList, MemVT, MMO);
6730   createOperands(N, Ops);
6731 
6732   CSEMap.InsertNode(N, IP);
6733   InsertNode(N);
6734   return SDValue(N, 0);
6735 }
6736 
6737 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
6738                                        EVT MemVT, SDVTList VTs, SDValue Chain,
6739                                        SDValue Ptr, SDValue Cmp, SDValue Swp,
6740                                        MachineMemOperand *MMO) {
6741   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
6742          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
6743   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
6744 
6745   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
6746   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
6747 }
6748 
6749 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6750                                 SDValue Chain, SDValue Ptr, SDValue Val,
6751                                 MachineMemOperand *MMO) {
6752   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
6753           Opcode == ISD::ATOMIC_LOAD_SUB ||
6754           Opcode == ISD::ATOMIC_LOAD_AND ||
6755           Opcode == ISD::ATOMIC_LOAD_CLR ||
6756           Opcode == ISD::ATOMIC_LOAD_OR ||
6757           Opcode == ISD::ATOMIC_LOAD_XOR ||
6758           Opcode == ISD::ATOMIC_LOAD_NAND ||
6759           Opcode == ISD::ATOMIC_LOAD_MIN ||
6760           Opcode == ISD::ATOMIC_LOAD_MAX ||
6761           Opcode == ISD::ATOMIC_LOAD_UMIN ||
6762           Opcode == ISD::ATOMIC_LOAD_UMAX ||
6763           Opcode == ISD::ATOMIC_LOAD_FADD ||
6764           Opcode == ISD::ATOMIC_LOAD_FSUB ||
6765           Opcode == ISD::ATOMIC_SWAP ||
6766           Opcode == ISD::ATOMIC_STORE) &&
6767          "Invalid Atomic Op");
6768 
6769   EVT VT = Val.getValueType();
6770 
6771   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
6772                                                getVTList(VT, MVT::Other);
6773   SDValue Ops[] = {Chain, Ptr, Val};
6774   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
6775 }
6776 
6777 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6778                                 EVT VT, SDValue Chain, SDValue Ptr,
6779                                 MachineMemOperand *MMO) {
6780   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
6781 
6782   SDVTList VTs = getVTList(VT, MVT::Other);
6783   SDValue Ops[] = {Chain, Ptr};
6784   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
6785 }
6786 
6787 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
6788 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
6789   if (Ops.size() == 1)
6790     return Ops[0];
6791 
6792   SmallVector<EVT, 4> VTs;
6793   VTs.reserve(Ops.size());
6794   for (unsigned i = 0; i < Ops.size(); ++i)
6795     VTs.push_back(Ops[i].getValueType());
6796   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
6797 }
6798 
6799 SDValue SelectionDAG::getMemIntrinsicNode(
6800     unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
6801     EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
6802     MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
6803   if (!Size && MemVT.isScalableVector())
6804     Size = MemoryLocation::UnknownSize;
6805   else if (!Size)
6806     Size = MemVT.getStoreSize();
6807 
6808   MachineFunction &MF = getMachineFunction();
6809   MachineMemOperand *MMO =
6810       MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);
6811 
6812   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
6813 }
6814 
6815 SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
6816                                           SDVTList VTList,
6817                                           ArrayRef<SDValue> Ops, EVT MemVT,
6818                                           MachineMemOperand *MMO) {
6819   assert((Opcode == ISD::INTRINSIC_VOID ||
6820           Opcode == ISD::INTRINSIC_W_CHAIN ||
6821           Opcode == ISD::PREFETCH ||
6822           ((int)Opcode <= std::numeric_limits<int>::max() &&
6823            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
6824          "Opcode is not a memory-accessing opcode!");
6825 
6826   // Memoize the node unless it returns a flag.
6827   MemIntrinsicSDNode *N;
6828   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
6829     FoldingSetNodeID ID;
6830     AddNodeIDNode(ID, Opcode, VTList, Ops);
6831     ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
6832         Opcode, dl.getIROrder(), VTList, MemVT, MMO));
6833     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6834     void *IP = nullptr;
6835     if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6836       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
6837       return SDValue(E, 0);
6838     }
6839 
6840     N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
6841                                       VTList, MemVT, MMO);
6842     createOperands(N, Ops);
6843 
6844   CSEMap.InsertNode(N, IP);
6845   } else {
6846     N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
6847                                       VTList, MemVT, MMO);
6848     createOperands(N, Ops);
6849   }
6850   InsertNode(N);
6851   SDValue V(N, 0);
6852   NewSDValueDbgMsg(V, "Creating new node: ", this);
6853   return V;
6854 }
6855 
6856 SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
6857                                       SDValue Chain, int FrameIndex,
6858                                       int64_t Size, int64_t Offset) {
6859   const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
6860   const auto VTs = getVTList(MVT::Other);
6861   SDValue Ops[2] = {
6862       Chain,
6863       getFrameIndex(FrameIndex,
6864                     getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),
6865                     true)};
6866 
6867   FoldingSetNodeID ID;
6868   AddNodeIDNode(ID, Opcode, VTs, Ops);
6869   ID.AddInteger(FrameIndex);
6870   ID.AddInteger(Size);
6871   ID.AddInteger(Offset);
6872   void *IP = nullptr;
6873   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
6874     return SDValue(E, 0);
6875 
6876   LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
6877       Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
6878   createOperands(N, Ops);
6879   CSEMap.InsertNode(N, IP);
6880   InsertNode(N);
6881   SDValue V(N, 0);
6882   NewSDValueDbgMsg(V, "Creating new node: ", this);
6883   return V;
6884 }
6885 
6886 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
6887 /// MachinePointerInfo record from it.  This is particularly useful because the
6888 /// code generator has many cases where it doesn't bother passing in a
6889 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
6890 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
6891                                            SelectionDAG &DAG, SDValue Ptr,
6892                                            int64_t Offset = 0) {
6893   // If this is FI+Offset, we can model it.
6894   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
6895     return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
6896                                              FI->getIndex(), Offset);
6897 
6898   // If this is (FI+Offset1)+Offset2, we can model it.
6899   if (Ptr.getOpcode() != ISD::ADD ||
6900       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
6901       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
6902     return Info;
6903 
6904   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
6905   return MachinePointerInfo::getFixedStack(
6906       DAG.getMachineFunction(), FI,
6907       Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
6908 }
6909 
6910 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
6911 /// MachinePointerInfo record from it.  This is particularly useful because the
6912 /// code generator has many cases where it doesn't bother passing in a
6913 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
6914 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
6915                                            SelectionDAG &DAG, SDValue Ptr,
6916                                            SDValue OffsetOp) {
6917   // If the 'Offset' value isn't a constant, we can't handle this.
6918   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
6919     return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
6920   if (OffsetOp.isUndef())
6921     return InferPointerInfo(Info, DAG, Ptr);
6922   return Info;
6923 }
6924 
6925 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
6926                               EVT VT, const SDLoc &dl, SDValue Chain,
6927                               SDValue Ptr, SDValue Offset,
6928                               MachinePointerInfo PtrInfo, EVT MemVT,
6929                               Align Alignment,
6930                               MachineMemOperand::Flags MMOFlags,
6931                               const AAMDNodes &AAInfo, const MDNode *Ranges) {
6932   assert(Chain.getValueType() == MVT::Other &&
6933         "Invalid chain type");
6934 
6935   MMOFlags |= MachineMemOperand::MOLoad;
6936   assert((MMOFlags & MachineMemOperand::MOStore) == 0);
6937   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
6938   // clients.
6939   if (PtrInfo.V.isNull())
6940     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
6941 
6942   uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
6943   MachineFunction &MF = getMachineFunction();
6944   MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
6945                                                    Alignment, AAInfo, Ranges);
6946   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
6947 }
6948 
6949 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
6950                               EVT VT, const SDLoc &dl, SDValue Chain,
6951                               SDValue Ptr, SDValue Offset, EVT MemVT,
6952                               MachineMemOperand *MMO) {
6953   if (VT == MemVT) {
6954     ExtType = ISD::NON_EXTLOAD;
6955   } else if (ExtType == ISD::NON_EXTLOAD) {
6956     assert(VT == MemVT && "Non-extending load from different memory type!");
6957   } else {
6958     // Extending load.
6959     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
6960            "Should only be an extending load, not truncating!");
6961     assert(VT.isInteger() == MemVT.isInteger() &&
6962            "Cannot convert from FP to Int or Int -> FP!");
6963     assert(VT.isVector() == MemVT.isVector() &&
6964            "Cannot use an ext load to convert to or from a vector!");
6965     assert((!VT.isVector() ||
6966             VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
6967            "Cannot use an ext load to change the number of vector elements!");
6968   }
6969 
6970   bool Indexed = AM != ISD::UNINDEXED;
6971   assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
6972 
6973   SDVTList VTs = Indexed ?
6974     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
6975   SDValue Ops[] = { Chain, Ptr, Offset };
6976   FoldingSetNodeID ID;
6977   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
6978   ID.AddInteger(MemVT.getRawBits());
6979   ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
6980       dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
6981   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6982   void *IP = nullptr;
6983   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6984     cast<LoadSDNode>(E)->refineAlignment(MMO);
6985     return SDValue(E, 0);
6986   }
6987   auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
6988                                   ExtType, MemVT, MMO);
6989   createOperands(N, Ops);
6990 
6991   CSEMap.InsertNode(N, IP);
6992   InsertNode(N);
6993   SDValue V(N, 0);
6994   NewSDValueDbgMsg(V, "Creating new node: ", this);
6995   return V;
6996 }
6997 
6998 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6999                               SDValue Ptr, MachinePointerInfo PtrInfo,
7000                               MaybeAlign Alignment,
7001                               MachineMemOperand::Flags MMOFlags,
7002                               const AAMDNodes &AAInfo, const MDNode *Ranges) {
7003   SDValue Undef = getUNDEF(Ptr.getValueType());
7004   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
7005                  PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
7006 }
7007 
7008 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
7009                               SDValue Ptr, MachineMemOperand *MMO) {
7010   SDValue Undef = getUNDEF(Ptr.getValueType());
7011   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
7012                  VT, MMO);
7013 }
7014 
7015 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
7016                                  EVT VT, SDValue Chain, SDValue Ptr,
7017                                  MachinePointerInfo PtrInfo, EVT MemVT,
7018                                  MaybeAlign Alignment,
7019                                  MachineMemOperand::Flags MMOFlags,
7020                                  const AAMDNodes &AAInfo) {
7021   SDValue Undef = getUNDEF(Ptr.getValueType());
7022   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
7023                  MemVT, Alignment, MMOFlags, AAInfo);
7024 }
7025 
7026 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
7027                                  EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
7028                                  MachineMemOperand *MMO) {
7029   SDValue Undef = getUNDEF(Ptr.getValueType());
7030   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
7031                  MemVT, MMO);
7032 }
7033 
7034 SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
7035                                      SDValue Base, SDValue Offset,
7036                                      ISD::MemIndexedMode AM) {
7037   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
7038   assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
7039   // Don't propagate the invariant or dereferenceable flags.
7040   auto MMOFlags =
7041       LD->getMemOperand()->getFlags() &
7042       ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
7043   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
7044                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
7045                  LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
7046 }
7047 
7048 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7049                                SDValue Ptr, MachinePointerInfo PtrInfo,
7050                                Align Alignment,
7051                                MachineMemOperand::Flags MMOFlags,
7052                                const AAMDNodes &AAInfo) {
7053   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
7054 
7055   MMOFlags |= MachineMemOperand::MOStore;
7056   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
7057 
7058   if (PtrInfo.V.isNull())
7059     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
7060 
7061   MachineFunction &MF = getMachineFunction();
7062   uint64_t Size =
7063       MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize());
7064   MachineMemOperand *MMO =
7065       MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
7066   return getStore(Chain, dl, Val, Ptr, MMO);
7067 }
7068 
7069 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7070                                SDValue Ptr, MachineMemOperand *MMO) {
7071   assert(Chain.getValueType() == MVT::Other &&
7072         "Invalid chain type");
7073   EVT VT = Val.getValueType();
7074   SDVTList VTs = getVTList(MVT::Other);
7075   SDValue Undef = getUNDEF(Ptr.getValueType());
7076   SDValue Ops[] = { Chain, Val, Ptr, Undef };
7077   FoldingSetNodeID ID;
7078   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7079   ID.AddInteger(VT.getRawBits());
7080   ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
7081       dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
7082   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7083   void *IP = nullptr;
7084   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7085     cast<StoreSDNode>(E)->refineAlignment(MMO);
7086     return SDValue(E, 0);
7087   }
7088   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7089                                    ISD::UNINDEXED, false, VT, MMO);
7090   createOperands(N, Ops);
7091 
7092   CSEMap.InsertNode(N, IP);
7093   InsertNode(N);
7094   SDValue V(N, 0);
7095   NewSDValueDbgMsg(V, "Creating new node: ", this);
7096   return V;
7097 }
7098 
7099 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7100                                     SDValue Ptr, MachinePointerInfo PtrInfo,
7101                                     EVT SVT, Align Alignment,
7102                                     MachineMemOperand::Flags MMOFlags,
7103                                     const AAMDNodes &AAInfo) {
7104   assert(Chain.getValueType() == MVT::Other &&
7105         "Invalid chain type");
7106 
7107   MMOFlags |= MachineMemOperand::MOStore;
7108   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
7109 
7110   if (PtrInfo.V.isNull())
7111     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
7112 
7113   MachineFunction &MF = getMachineFunction();
7114   MachineMemOperand *MMO = MF.getMachineMemOperand(
7115       PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
7116       Alignment, AAInfo);
7117   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
7118 }
7119 
7120 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7121                                     SDValue Ptr, EVT SVT,
7122                                     MachineMemOperand *MMO) {
7123   EVT VT = Val.getValueType();
7124 
7125   assert(Chain.getValueType() == MVT::Other &&
7126         "Invalid chain type");
7127   if (VT == SVT)
7128     return getStore(Chain, dl, Val, Ptr, MMO);
7129 
7130   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
7131          "Should only be a truncating store, not extending!");
7132   assert(VT.isInteger() == SVT.isInteger() &&
7133          "Can't do FP-INT conversion!");
7134   assert(VT.isVector() == SVT.isVector() &&
7135          "Cannot use trunc store to convert to or from a vector!");
7136   assert((!VT.isVector() ||
7137           VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
7138          "Cannot use trunc store to change the number of vector elements!");
7139 
7140   SDVTList VTs = getVTList(MVT::Other);
7141   SDValue Undef = getUNDEF(Ptr.getValueType());
7142   SDValue Ops[] = { Chain, Val, Ptr, Undef };
7143   FoldingSetNodeID ID;
7144   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7145   ID.AddInteger(SVT.getRawBits());
7146   ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
7147       dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
7148   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7149   void *IP = nullptr;
7150   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7151     cast<StoreSDNode>(E)->refineAlignment(MMO);
7152     return SDValue(E, 0);
7153   }
7154   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7155                                    ISD::UNINDEXED, true, SVT, MMO);
7156   createOperands(N, Ops);
7157 
7158   CSEMap.InsertNode(N, IP);
7159   InsertNode(N);
7160   SDValue V(N, 0);
7161   NewSDValueDbgMsg(V, "Creating new node: ", this);
7162   return V;
7163 }
7164 
7165 SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
7166                                       SDValue Base, SDValue Offset,
7167                                       ISD::MemIndexedMode AM) {
7168   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
7169   assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
7170   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
7171   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
7172   FoldingSetNodeID ID;
7173   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7174   ID.AddInteger(ST->getMemoryVT().getRawBits());
7175   ID.AddInteger(ST->getRawSubclassData());
7176   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
7177   void *IP = nullptr;
7178   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
7179     return SDValue(E, 0);
7180 
7181   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
7182                                    ST->isTruncatingStore(), ST->getMemoryVT(),
7183                                    ST->getMemOperand());
7184   createOperands(N, Ops);
7185 
7186   CSEMap.InsertNode(N, IP);
7187   InsertNode(N);
7188   SDValue V(N, 0);
7189   NewSDValueDbgMsg(V, "Creating new node: ", this);
7190   return V;
7191 }
7192 
7193 SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
7194                                     SDValue Base, SDValue Offset, SDValue Mask,
7195                                     SDValue PassThru, EVT MemVT,
7196                                     MachineMemOperand *MMO,
7197                                     ISD::MemIndexedMode AM,
7198                                     ISD::LoadExtType ExtTy, bool isExpanding) {
7199   bool Indexed = AM != ISD::UNINDEXED;
7200   assert((Indexed || Offset.isUndef()) &&
7201          "Unindexed masked load with an offset!");
7202   SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
7203                          : getVTList(VT, MVT::Other);
7204   SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
7205   FoldingSetNodeID ID;
7206   AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
7207   ID.AddInteger(MemVT.getRawBits());
7208   ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
7209       dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
7210   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7211   void *IP = nullptr;
7212   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7213     cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
7214     return SDValue(E, 0);
7215   }
7216   auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7217                                         AM, ExtTy, isExpanding, MemVT, MMO);
7218   createOperands(N, Ops);
7219 
7220   CSEMap.InsertNode(N, IP);
7221   InsertNode(N);
7222   SDValue V(N, 0);
7223   NewSDValueDbgMsg(V, "Creating new node: ", this);
7224   return V;
7225 }
7226 
7227 SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
7228                                            SDValue Base, SDValue Offset,
7229                                            ISD::MemIndexedMode AM) {
7230   MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad);
7231   assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
7232   return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,
7233                        Offset, LD->getMask(), LD->getPassThru(),
7234                        LD->getMemoryVT(), LD->getMemOperand(), AM,
7235                        LD->getExtensionType(), LD->isExpandingLoad());
7236 }
7237 
7238 SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
7239                                      SDValue Val, SDValue Base, SDValue Offset,
7240                                      SDValue Mask, EVT MemVT,
7241                                      MachineMemOperand *MMO,
7242                                      ISD::MemIndexedMode AM, bool IsTruncating,
7243                                      bool IsCompressing) {
7244   assert(Chain.getValueType() == MVT::Other &&
7245         "Invalid chain type");
7246   bool Indexed = AM != ISD::UNINDEXED;
7247   assert((Indexed || Offset.isUndef()) &&
7248          "Unindexed masked store with an offset!");
7249   SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
7250                          : getVTList(MVT::Other);
7251   SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
7252   FoldingSetNodeID ID;
7253   AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
7254   ID.AddInteger(MemVT.getRawBits());
7255   ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
7256       dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
7257   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7258   void *IP = nullptr;
7259   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7260     cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
7261     return SDValue(E, 0);
7262   }
7263   auto *N =
7264       newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
7265                                    IsTruncating, IsCompressing, MemVT, MMO);
7266   createOperands(N, Ops);
7267 
7268   CSEMap.InsertNode(N, IP);
7269   InsertNode(N);
7270   SDValue V(N, 0);
7271   NewSDValueDbgMsg(V, "Creating new node: ", this);
7272   return V;
7273 }
7274 
7275 SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
7276                                             SDValue Base, SDValue Offset,
7277                                             ISD::MemIndexedMode AM) {
7278   MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore);
7279   assert(ST->getOffset().isUndef() &&
7280          "Masked store is already a indexed store!");
7281   return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
7282                         ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
7283                         AM, ST->isTruncatingStore(), ST->isCompressingStore());
7284 }
7285 
7286 SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
7287                                       ArrayRef<SDValue> Ops,
7288                                       MachineMemOperand *MMO,
7289                                       ISD::MemIndexType IndexType) {
7290   assert(Ops.size() == 6 && "Incompatible number of operands");
7291 
7292   FoldingSetNodeID ID;
7293   AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
7294   ID.AddInteger(VT.getRawBits());
7295   ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
7296       dl.getIROrder(), VTs, VT, MMO, IndexType));
7297   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7298   void *IP = nullptr;
7299   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7300     cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
7301     return SDValue(E, 0);
7302   }
7303 
7304   auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
7305                                           VTs, VT, MMO, IndexType);
7306   createOperands(N, Ops);
7307 
7308   assert(N->getPassThru().getValueType() == N->getValueType(0) &&
7309          "Incompatible type of the PassThru value in MaskedGatherSDNode");
7310   assert(N->getMask().getValueType().getVectorNumElements() ==
7311              N->getValueType(0).getVectorNumElements() &&
7312          "Vector width mismatch between mask and data");
7313   assert(N->getIndex().getValueType().getVectorNumElements() >=
7314              N->getValueType(0).getVectorNumElements() &&
7315          "Vector width mismatch between index and data");
7316   assert(isa<ConstantSDNode>(N->getScale()) &&
7317          cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
7318          "Scale should be a constant power of 2");
7319 
7320   CSEMap.InsertNode(N, IP);
7321   InsertNode(N);
7322   SDValue V(N, 0);
7323   NewSDValueDbgMsg(V, "Creating new node: ", this);
7324   return V;
7325 }
7326 
7327 SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
7328                                        ArrayRef<SDValue> Ops,
7329                                        MachineMemOperand *MMO,
7330                                        ISD::MemIndexType IndexType,
7331                                        bool IsTrunc) {
7332   assert(Ops.size() == 6 && "Incompatible number of operands");
7333 
7334   FoldingSetNodeID ID;
7335   AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
7336   ID.AddInteger(VT.getRawBits());
7337   ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
7338       dl.getIROrder(), VTs, VT, MMO, IndexType, IsTrunc));
7339   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7340   void *IP = nullptr;
7341   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7342     cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
7343     return SDValue(E, 0);
7344   }
7345 
7346   IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]);
7347   auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
7348                                            VTs, VT, MMO, IndexType, IsTrunc);
7349   createOperands(N, Ops);
7350 
7351   assert(N->getMask().getValueType().getVectorElementCount() ==
7352              N->getValue().getValueType().getVectorElementCount() &&
7353          "Vector width mismatch between mask and data");
7354   assert(
7355       N->getIndex().getValueType().getVectorElementCount().isScalable() ==
7356           N->getValue().getValueType().getVectorElementCount().isScalable() &&
7357       "Scalable flags of index and data do not match");
7358   assert(ElementCount::isKnownGE(
7359              N->getIndex().getValueType().getVectorElementCount(),
7360              N->getValue().getValueType().getVectorElementCount()) &&
7361          "Vector width mismatch between index and data");
7362   assert(isa<ConstantSDNode>(N->getScale()) &&
7363          cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
7364          "Scale should be a constant power of 2");
7365 
7366   CSEMap.InsertNode(N, IP);
7367   InsertNode(N);
7368   SDValue V(N, 0);
7369   NewSDValueDbgMsg(V, "Creating new node: ", this);
7370   return V;
7371 }
7372 
7373 SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
7374   // select undef, T, F --> T (if T is a constant), otherwise F
7375   // select, ?, undef, F --> F
7376   // select, ?, T, undef --> T
7377   if (Cond.isUndef())
7378     return isConstantValueOfAnyType(T) ? T : F;
7379   if (T.isUndef())
7380     return F;
7381   if (F.isUndef())
7382     return T;
7383 
7384   // select true, T, F --> T
7385   // select false, T, F --> F
7386   if (auto *CondC = dyn_cast<ConstantSDNode>(Cond))
7387     return CondC->isNullValue() ? F : T;
7388 
7389   // TODO: This should simplify VSELECT with constant condition using something
7390   // like this (but check boolean contents to be complete?):
7391   //  if (ISD::isBuildVectorAllOnes(Cond.getNode()))
7392   //    return T;
7393   //  if (ISD::isBuildVectorAllZeros(Cond.getNode()))
7394   //    return F;
7395 
7396   // select ?, T, T --> T
7397   if (T == F)
7398     return T;
7399 
7400   return SDValue();
7401 }
7402 
7403 SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
7404   // shift undef, Y --> 0 (can always assume that the undef value is 0)
7405   if (X.isUndef())
7406     return getConstant(0, SDLoc(X.getNode()), X.getValueType());
7407   // shift X, undef --> undef (because it may shift by the bitwidth)
7408   if (Y.isUndef())
7409     return getUNDEF(X.getValueType());
7410 
7411   // shift 0, Y --> 0
7412   // shift X, 0 --> X
7413   if (isNullOrNullSplat(X) || isNullOrNullSplat(Y))
7414     return X;
7415 
7416   // shift X, C >= bitwidth(X) --> undef
7417   // All vector elements must be too big (or undef) to avoid partial undefs.
7418   auto isShiftTooBig = [X](ConstantSDNode *Val) {
7419     return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
7420   };
7421   if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
7422     return getUNDEF(X.getValueType());
7423 
7424   return SDValue();
7425 }
7426 
7427 SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
7428                                       SDNodeFlags Flags) {
7429   // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
7430   // (an undef operand can be chosen to be Nan/Inf), then the result of this
7431   // operation is poison. That result can be relaxed to undef.
7432   ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true);
7433   ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);
7434   bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
7435                 (YC && YC->getValueAPF().isNaN());
7436   bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
7437                 (YC && YC->getValueAPF().isInfinity());
7438 
7439   if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
7440     return getUNDEF(X.getValueType());
7441 
7442   if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
7443     return getUNDEF(X.getValueType());
7444 
7445   if (!YC)
7446     return SDValue();
7447 
7448   // X + -0.0 --> X
7449   if (Opcode == ISD::FADD)
7450     if (YC->getValueAPF().isNegZero())
7451       return X;
7452 
7453   // X - +0.0 --> X
7454   if (Opcode == ISD::FSUB)
7455     if (YC->getValueAPF().isPosZero())
7456       return X;
7457 
7458   // X * 1.0 --> X
7459   // X / 1.0 --> X
7460   if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
7461     if (YC->getValueAPF().isExactlyValue(1.0))
7462       return X;
7463 
7464   // X * 0.0 --> 0.0
7465   if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
7466     if (YC->getValueAPF().isZero())
7467       return getConstantFP(0.0, SDLoc(Y), Y.getValueType());
7468 
7469   return SDValue();
7470 }
7471 
7472 SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
7473                                SDValue Ptr, SDValue SV, unsigned Align) {
7474   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
7475   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
7476 }
7477 
7478 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7479                               ArrayRef<SDUse> Ops) {
7480   switch (Ops.size()) {
7481   case 0: return getNode(Opcode, DL, VT);
7482   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
7483   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
7484   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
7485   default: break;
7486   }
7487 
7488   // Copy from an SDUse array into an SDValue array for use with
7489   // the regular getNode logic.
7490   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
7491   return getNode(Opcode, DL, VT, NewOps);
7492 }
7493 
7494 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7495                               ArrayRef<SDValue> Ops) {
7496   SDNodeFlags Flags;
7497   if (Inserter)
7498     Flags = Inserter->getFlags();
7499   return getNode(Opcode, DL, VT, Ops, Flags);
7500 }
7501 
7502 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7503                               ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
7504   unsigned NumOps = Ops.size();
7505   switch (NumOps) {
7506   case 0: return getNode(Opcode, DL, VT);
7507   case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
7508   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
7509   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
7510   default: break;
7511   }
7512 
7513   switch (Opcode) {
7514   default: break;
7515   case ISD::BUILD_VECTOR:
7516     // Attempt to simplify BUILD_VECTOR.
7517     if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7518       return V;
7519     break;
7520   case ISD::CONCAT_VECTORS:
7521     if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7522       return V;
7523     break;
7524   case ISD::SELECT_CC:
7525     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
7526     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
7527            "LHS and RHS of condition must have same type!");
7528     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
7529            "True and False arms of SelectCC must have same type!");
7530     assert(Ops[2].getValueType() == VT &&
7531            "select_cc node must be of same type as true and false value!");
7532     break;
7533   case ISD::BR_CC:
7534     assert(NumOps == 5 && "BR_CC takes 5 operands!");
7535     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
7536            "LHS/RHS of comparison should match types!");
7537     break;
7538   }
7539 
7540   // Memoize nodes.
7541   SDNode *N;
7542   SDVTList VTs = getVTList(VT);
7543 
7544   if (VT != MVT::Glue) {
7545     FoldingSetNodeID ID;
7546     AddNodeIDNode(ID, Opcode, VTs, Ops);
7547     void *IP = nullptr;
7548 
7549     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
7550       return SDValue(E, 0);
7551 
7552     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7553     createOperands(N, Ops);
7554 
7555     CSEMap.InsertNode(N, IP);
7556   } else {
7557     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7558     createOperands(N, Ops);
7559   }
7560 
7561   N->setFlags(Flags);
7562   InsertNode(N);
7563   SDValue V(N, 0);
7564   NewSDValueDbgMsg(V, "Creating new node: ", this);
7565   return V;
7566 }
7567 
7568 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
7569                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
7570   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
7571 }
7572 
7573 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7574                               ArrayRef<SDValue> Ops) {
7575   SDNodeFlags Flags;
7576   if (Inserter)
7577     Flags = Inserter->getFlags();
7578   return getNode(Opcode, DL, VTList, Ops, Flags);
7579 }
7580 
7581 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7582                               ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
7583   if (VTList.NumVTs == 1)
7584     return getNode(Opcode, DL, VTList.VTs[0], Ops);
7585 
7586   switch (Opcode) {
7587   case ISD::STRICT_FP_EXTEND:
7588     assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
7589            "Invalid STRICT_FP_EXTEND!");
7590     assert(VTList.VTs[0].isFloatingPoint() &&
7591            Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
7592     assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
7593            "STRICT_FP_EXTEND result type should be vector iff the operand "
7594            "type is vector!");
7595     assert((!VTList.VTs[0].isVector() ||
7596             VTList.VTs[0].getVectorNumElements() ==
7597             Ops[1].getValueType().getVectorNumElements()) &&
7598            "Vector element count mismatch!");
7599     assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
7600            "Invalid fpext node, dst <= src!");
7601     break;
7602   case ISD::STRICT_FP_ROUND:
7603     assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
7604     assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
7605            "STRICT_FP_ROUND result type should be vector iff the operand "
7606            "type is vector!");
7607     assert((!VTList.VTs[0].isVector() ||
7608             VTList.VTs[0].getVectorNumElements() ==
7609             Ops[1].getValueType().getVectorNumElements()) &&
7610            "Vector element count mismatch!");
7611     assert(VTList.VTs[0].isFloatingPoint() &&
7612            Ops[1].getValueType().isFloatingPoint() &&
7613            VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
7614            isa<ConstantSDNode>(Ops[2]) &&
7615            (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 ||
7616             cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) &&
7617            "Invalid STRICT_FP_ROUND!");
7618     break;
7619 #if 0
7620   // FIXME: figure out how to safely handle things like
7621   // int foo(int x) { return 1 << (x & 255); }
7622   // int bar() { return foo(256); }
7623   case ISD::SRA_PARTS:
7624   case ISD::SRL_PARTS:
7625   case ISD::SHL_PARTS:
7626     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
7627         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
7628       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
7629     else if (N3.getOpcode() == ISD::AND)
7630       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
7631         // If the and is only masking out bits that cannot effect the shift,
7632         // eliminate the and.
7633         unsigned NumBits = VT.getScalarSizeInBits()*2;
7634         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
7635           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
7636       }
7637     break;
7638 #endif
7639   }
7640 
7641   // Memoize the node unless it returns a flag.
7642   SDNode *N;
7643   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
7644     FoldingSetNodeID ID;
7645     AddNodeIDNode(ID, Opcode, VTList, Ops);
7646     void *IP = nullptr;
7647     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
7648       return SDValue(E, 0);
7649 
7650     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
7651     createOperands(N, Ops);
7652     CSEMap.InsertNode(N, IP);
7653   } else {
7654     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
7655     createOperands(N, Ops);
7656   }
7657 
7658   N->setFlags(Flags);
7659   InsertNode(N);
7660   SDValue V(N, 0);
7661   NewSDValueDbgMsg(V, "Creating new node: ", this);
7662   return V;
7663 }
7664 
7665 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
7666                               SDVTList VTList) {
7667   return getNode(Opcode, DL, VTList, None);
7668 }
7669 
7670 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7671                               SDValue N1) {
7672   SDValue Ops[] = { N1 };
7673   return getNode(Opcode, DL, VTList, Ops);
7674 }
7675 
7676 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7677                               SDValue N1, SDValue N2) {
7678   SDValue Ops[] = { N1, N2 };
7679   return getNode(Opcode, DL, VTList, Ops);
7680 }
7681 
7682 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7683                               SDValue N1, SDValue N2, SDValue N3) {
7684   SDValue Ops[] = { N1, N2, N3 };
7685   return getNode(Opcode, DL, VTList, Ops);
7686 }
7687 
7688 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7689                               SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7690   SDValue Ops[] = { N1, N2, N3, N4 };
7691   return getNode(Opcode, DL, VTList, Ops);
7692 }
7693 
7694 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7695                               SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7696                               SDValue N5) {
7697   SDValue Ops[] = { N1, N2, N3, N4, N5 };
7698   return getNode(Opcode, DL, VTList, Ops);
7699 }
7700 
7701 SDVTList SelectionDAG::getVTList(EVT VT) {
7702   return makeVTList(SDNode::getValueTypeList(VT), 1);
7703 }
7704 
7705 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
7706   FoldingSetNodeID ID;
7707   ID.AddInteger(2U);
7708   ID.AddInteger(VT1.getRawBits());
7709   ID.AddInteger(VT2.getRawBits());
7710 
7711   void *IP = nullptr;
7712   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
7713   if (!Result) {
7714     EVT *Array = Allocator.Allocate<EVT>(2);
7715     Array[0] = VT1;
7716     Array[1] = VT2;
7717     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
7718     VTListMap.InsertNode(Result, IP);
7719   }
7720   return Result->getSDVTList();
7721 }
7722 
7723 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
7724   FoldingSetNodeID ID;
7725   ID.AddInteger(3U);
7726   ID.AddInteger(VT1.getRawBits());
7727   ID.AddInteger(VT2.getRawBits());
7728   ID.AddInteger(VT3.getRawBits());
7729 
7730   void *IP = nullptr;
7731   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
7732   if (!Result) {
7733     EVT *Array = Allocator.Allocate<EVT>(3);
7734     Array[0] = VT1;
7735     Array[1] = VT2;
7736     Array[2] = VT3;
7737     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
7738     VTListMap.InsertNode(Result, IP);
7739   }
7740   return Result->getSDVTList();
7741 }
7742 
7743 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
7744   FoldingSetNodeID ID;
7745   ID.AddInteger(4U);
7746   ID.AddInteger(VT1.getRawBits());
7747   ID.AddInteger(VT2.getRawBits());
7748   ID.AddInteger(VT3.getRawBits());
7749   ID.AddInteger(VT4.getRawBits());
7750 
7751   void *IP = nullptr;
7752   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
7753   if (!Result) {
7754     EVT *Array = Allocator.Allocate<EVT>(4);
7755     Array[0] = VT1;
7756     Array[1] = VT2;
7757     Array[2] = VT3;
7758     Array[3] = VT4;
7759     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
7760     VTListMap.InsertNode(Result, IP);
7761   }
7762   return Result->getSDVTList();
7763 }
7764 
7765 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
7766   unsigned NumVTs = VTs.size();
7767   FoldingSetNodeID ID;
7768   ID.AddInteger(NumVTs);
7769   for (unsigned index = 0; index < NumVTs; index++) {
7770     ID.AddInteger(VTs[index].getRawBits());
7771   }
7772 
7773   void *IP = nullptr;
7774   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
7775   if (!Result) {
7776     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
7777     llvm::copy(VTs, Array);
7778     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
7779     VTListMap.InsertNode(Result, IP);
7780   }
7781   return Result->getSDVTList();
7782 }
7783 
7784 
7785 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
7786 /// specified operands.  If the resultant node already exists in the DAG,
7787 /// this does not modify the specified node, instead it returns the node that
7788 /// already exists.  If the resultant node does not exist in the DAG, the
7789 /// input node is returned.  As a degenerate case, if you specify the same
7790 /// input operands as the node already has, the input node is returned.
7791 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
7792   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
7793 
7794   // Check to see if there is no change.
7795   if (Op == N->getOperand(0)) return N;
7796 
7797   // See if the modified node already exists.
7798   void *InsertPos = nullptr;
7799   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
7800     return Existing;
7801 
7802   // Nope it doesn't.  Remove the node from its current place in the maps.
7803   if (InsertPos)
7804     if (!RemoveNodeFromCSEMaps(N))
7805       InsertPos = nullptr;
7806 
7807   // Now we update the operands.
7808   N->OperandList[0].set(Op);
7809 
7810   updateDivergence(N);
7811   // If this gets put into a CSE map, add it.
7812   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
7813   return N;
7814 }
7815 
7816 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
7817   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
7818 
7819   // Check to see if there is no change.
7820   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
7821     return N;   // No operands changed, just return the input node.
7822 
7823   // See if the modified node already exists.
7824   void *InsertPos = nullptr;
7825   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
7826     return Existing;
7827 
7828   // Nope it doesn't.  Remove the node from its current place in the maps.
7829   if (InsertPos)
7830     if (!RemoveNodeFromCSEMaps(N))
7831       InsertPos = nullptr;
7832 
7833   // Now we update the operands.
7834   if (N->OperandList[0] != Op1)
7835     N->OperandList[0].set(Op1);
7836   if (N->OperandList[1] != Op2)
7837     N->OperandList[1].set(Op2);
7838 
7839   updateDivergence(N);
7840   // If this gets put into a CSE map, add it.
7841   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
7842   return N;
7843 }
7844 
7845 SDNode *SelectionDAG::
7846 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
7847   SDValue Ops[] = { Op1, Op2, Op3 };
7848   return UpdateNodeOperands(N, Ops);
7849 }
7850 
7851 SDNode *SelectionDAG::
7852 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
7853                    SDValue Op3, SDValue Op4) {
7854   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
7855   return UpdateNodeOperands(N, Ops);
7856 }
7857 
7858 SDNode *SelectionDAG::
7859 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
7860                    SDValue Op3, SDValue Op4, SDValue Op5) {
7861   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
7862   return UpdateNodeOperands(N, Ops);
7863 }
7864 
7865 SDNode *SelectionDAG::
7866 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
7867   unsigned NumOps = Ops.size();
7868   assert(N->getNumOperands() == NumOps &&
7869          "Update with wrong number of operands");
7870 
7871   // If no operands changed just return the input node.
7872   if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
7873     return N;
7874 
7875   // See if the modified node already exists.
7876   void *InsertPos = nullptr;
7877   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
7878     return Existing;
7879 
7880   // Nope it doesn't.  Remove the node from its current place in the maps.
7881   if (InsertPos)
7882     if (!RemoveNodeFromCSEMaps(N))
7883       InsertPos = nullptr;
7884 
7885   // Now we update the operands.
7886   for (unsigned i = 0; i != NumOps; ++i)
7887     if (N->OperandList[i] != Ops[i])
7888       N->OperandList[i].set(Ops[i]);
7889 
7890   updateDivergence(N);
7891   // If this gets put into a CSE map, add it.
7892   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
7893   return N;
7894 }
7895 
7896 /// DropOperands - Release the operands and set this node to have
7897 /// zero operands.
7898 void SDNode::DropOperands() {
7899   // Unlike the code in MorphNodeTo that does this, we don't need to
7900   // watch for dead nodes here.
7901   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
7902     SDUse &Use = *I++;
7903     Use.set(SDValue());
7904   }
7905 }
7906 
7907 void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
7908                                   ArrayRef<MachineMemOperand *> NewMemRefs) {
7909   if (NewMemRefs.empty()) {
7910     N->clearMemRefs();
7911     return;
7912   }
7913 
7914   // Check if we can avoid allocating by storing a single reference directly.
7915   if (NewMemRefs.size() == 1) {
7916     N->MemRefs = NewMemRefs[0];
7917     N->NumMemRefs = 1;
7918     return;
7919   }
7920 
7921   MachineMemOperand **MemRefsBuffer =
7922       Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
7923   llvm::copy(NewMemRefs, MemRefsBuffer);
7924   N->MemRefs = MemRefsBuffer;
7925   N->NumMemRefs = static_cast<int>(NewMemRefs.size());
7926 }
7927 
7928 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
7929 /// machine opcode.
7930 ///
7931 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7932                                    EVT VT) {
7933   SDVTList VTs = getVTList(VT);
7934   return SelectNodeTo(N, MachineOpc, VTs, None);
7935 }
7936 
7937 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7938                                    EVT VT, SDValue Op1) {
7939   SDVTList VTs = getVTList(VT);
7940   SDValue Ops[] = { Op1 };
7941   return SelectNodeTo(N, MachineOpc, VTs, Ops);
7942 }
7943 
7944 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7945                                    EVT VT, SDValue Op1,
7946                                    SDValue Op2) {
7947   SDVTList VTs = getVTList(VT);
7948   SDValue Ops[] = { Op1, Op2 };
7949   return SelectNodeTo(N, MachineOpc, VTs, Ops);
7950 }
7951 
7952 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7953                                    EVT VT, SDValue Op1,
7954                                    SDValue Op2, SDValue Op3) {
7955   SDVTList VTs = getVTList(VT);
7956   SDValue Ops[] = { Op1, Op2, Op3 };
7957   return SelectNodeTo(N, MachineOpc, VTs, Ops);
7958 }
7959 
7960 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7961                                    EVT VT, ArrayRef<SDValue> Ops) {
7962   SDVTList VTs = getVTList(VT);
7963   return SelectNodeTo(N, MachineOpc, VTs, Ops);
7964 }
7965 
7966 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7967                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
7968   SDVTList VTs = getVTList(VT1, VT2);
7969   return SelectNodeTo(N, MachineOpc, VTs, Ops);
7970 }
7971 
7972 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7973                                    EVT VT1, EVT VT2) {
7974   SDVTList VTs = getVTList(VT1, VT2);
7975   return SelectNodeTo(N, MachineOpc, VTs, None);
7976 }
7977 
7978 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7979                                    EVT VT1, EVT VT2, EVT VT3,
7980                                    ArrayRef<SDValue> Ops) {
7981   SDVTList VTs = getVTList(VT1, VT2, VT3);
7982   return SelectNodeTo(N, MachineOpc, VTs, Ops);
7983 }
7984 
7985 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7986                                    EVT VT1, EVT VT2,
7987                                    SDValue Op1, SDValue Op2) {
7988   SDVTList VTs = getVTList(VT1, VT2);
7989   SDValue Ops[] = { Op1, Op2 };
7990   return SelectNodeTo(N, MachineOpc, VTs, Ops);
7991 }
7992 
7993 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7994                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
7995   SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
7996   // Reset the NodeID to -1.
7997   New->setNodeId(-1);
7998   if (New != N) {
7999     ReplaceAllUsesWith(N, New);
8000     RemoveDeadNode(N);
8001   }
8002   return New;
8003 }
8004 
8005 /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
8006 /// the line number information on the merged node since it is not possible to
8007 /// preserve the information that operation is associated with multiple lines.
8008 /// This will make the debugger working better at -O0, were there is a higher
8009 /// probability having other instructions associated with that line.
8010 ///
8011 /// For IROrder, we keep the smaller of the two
8012 SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
8013   DebugLoc NLoc = N->getDebugLoc();
8014   if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
8015     N->setDebugLoc(DebugLoc());
8016   }
8017   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
8018   N->setIROrder(Order);
8019   return N;
8020 }
8021 
8022 /// MorphNodeTo - This *mutates* the specified node to have the specified
8023 /// return type, opcode, and operands.
8024 ///
8025 /// Note that MorphNodeTo returns the resultant node.  If there is already a
8026 /// node of the specified opcode and operands, it returns that node instead of
8027 /// the current one.  Note that the SDLoc need not be the same.
8028 ///
8029 /// Using MorphNodeTo is faster than creating a new node and swapping it in
8030 /// with ReplaceAllUsesWith both because it often avoids allocating a new
8031 /// node, and because it doesn't require CSE recalculation for any of
8032 /// the node's users.
8033 ///
8034 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
8035 /// As a consequence it isn't appropriate to use from within the DAG combiner or
8036 /// the legalizer which maintain worklists that would need to be updated when
8037 /// deleting things.
8038 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
8039                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
8040   // If an identical node already exists, use it.
8041   void *IP = nullptr;
8042   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
8043     FoldingSetNodeID ID;
8044     AddNodeIDNode(ID, Opc, VTs, Ops);
8045     if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
8046       return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
8047   }
8048 
8049   if (!RemoveNodeFromCSEMaps(N))
8050     IP = nullptr;
8051 
8052   // Start the morphing.
8053   N->NodeType = Opc;
8054   N->ValueList = VTs.VTs;
8055   N->NumValues = VTs.NumVTs;
8056 
8057   // Clear the operands list, updating used nodes to remove this from their
8058   // use list.  Keep track of any operands that become dead as a result.
8059   SmallPtrSet<SDNode*, 16> DeadNodeSet;
8060   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
8061     SDUse &Use = *I++;
8062     SDNode *Used = Use.getNode();
8063     Use.set(SDValue());
8064     if (Used->use_empty())
8065       DeadNodeSet.insert(Used);
8066   }
8067 
8068   // For MachineNode, initialize the memory references information.
8069   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
8070     MN->clearMemRefs();
8071 
8072   // Swap for an appropriately sized array from the recycler.
8073   removeOperands(N);
8074   createOperands(N, Ops);
8075 
8076   // Delete any nodes that are still dead after adding the uses for the
8077   // new operands.
8078   if (!DeadNodeSet.empty()) {
8079     SmallVector<SDNode *, 16> DeadNodes;
8080     for (SDNode *N : DeadNodeSet)
8081       if (N->use_empty())
8082         DeadNodes.push_back(N);
8083     RemoveDeadNodes(DeadNodes);
8084   }
8085 
8086   if (IP)
8087     CSEMap.InsertNode(N, IP);   // Memoize the new node.
8088   return N;
8089 }
8090 
8091 SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
8092   unsigned OrigOpc = Node->getOpcode();
8093   unsigned NewOpc;
8094   switch (OrigOpc) {
8095   default:
8096     llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
8097 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
8098   case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
8099 #define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
8100   case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
8101 #include "llvm/IR/ConstrainedOps.def"
8102   }
8103 
8104   assert(Node->getNumValues() == 2 && "Unexpected number of results!");
8105 
8106   // We're taking this node out of the chain, so we need to re-link things.
8107   SDValue InputChain = Node->getOperand(0);
8108   SDValue OutputChain = SDValue(Node, 1);
8109   ReplaceAllUsesOfValueWith(OutputChain, InputChain);
8110 
8111   SmallVector<SDValue, 3> Ops;
8112   for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
8113     Ops.push_back(Node->getOperand(i));
8114 
8115   SDVTList VTs = getVTList(Node->getValueType(0));
8116   SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);
8117 
8118   // MorphNodeTo can operate in two ways: if an existing node with the
8119   // specified operands exists, it can just return it.  Otherwise, it
8120   // updates the node in place to have the requested operands.
8121   if (Res == Node) {
8122     // If we updated the node in place, reset the node ID.  To the isel,
8123     // this should be just like a newly allocated machine node.
8124     Res->setNodeId(-1);
8125   } else {
8126     ReplaceAllUsesWith(Node, Res);
8127     RemoveDeadNode(Node);
8128   }
8129 
8130   return Res;
8131 }
8132 
8133 /// getMachineNode - These are used for target selectors to create a new node
8134 /// with specified return type(s), MachineInstr opcode, and operands.
8135 ///
8136 /// Note that getMachineNode returns the resultant node.  If there is already a
8137 /// node of the specified opcode and operands, it returns that node instead of
8138 /// the current one.
8139 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8140                                             EVT VT) {
8141   SDVTList VTs = getVTList(VT);
8142   return getMachineNode(Opcode, dl, VTs, None);
8143 }
8144 
8145 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8146                                             EVT VT, SDValue Op1) {
8147   SDVTList VTs = getVTList(VT);
8148   SDValue Ops[] = { Op1 };
8149   return getMachineNode(Opcode, dl, VTs, Ops);
8150 }
8151 
8152 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8153                                             EVT VT, SDValue Op1, SDValue Op2) {
8154   SDVTList VTs = getVTList(VT);
8155   SDValue Ops[] = { Op1, Op2 };
8156   return getMachineNode(Opcode, dl, VTs, Ops);
8157 }
8158 
8159 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8160                                             EVT VT, SDValue Op1, SDValue Op2,
8161                                             SDValue Op3) {
8162   SDVTList VTs = getVTList(VT);
8163   SDValue Ops[] = { Op1, Op2, Op3 };
8164   return getMachineNode(Opcode, dl, VTs, Ops);
8165 }
8166 
8167 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8168                                             EVT VT, ArrayRef<SDValue> Ops) {
8169   SDVTList VTs = getVTList(VT);
8170   return getMachineNode(Opcode, dl, VTs, Ops);
8171 }
8172 
8173 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8174                                             EVT VT1, EVT VT2, SDValue Op1,
8175                                             SDValue Op2) {
8176   SDVTList VTs = getVTList(VT1, VT2);
8177   SDValue Ops[] = { Op1, Op2 };
8178   return getMachineNode(Opcode, dl, VTs, Ops);
8179 }
8180 
8181 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8182                                             EVT VT1, EVT VT2, SDValue Op1,
8183                                             SDValue Op2, SDValue Op3) {
8184   SDVTList VTs = getVTList(VT1, VT2);
8185   SDValue Ops[] = { Op1, Op2, Op3 };
8186   return getMachineNode(Opcode, dl, VTs, Ops);
8187 }
8188 
8189 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8190                                             EVT VT1, EVT VT2,
8191                                             ArrayRef<SDValue> Ops) {
8192   SDVTList VTs = getVTList(VT1, VT2);
8193   return getMachineNode(Opcode, dl, VTs, Ops);
8194 }
8195 
8196 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8197                                             EVT VT1, EVT VT2, EVT VT3,
8198                                             SDValue Op1, SDValue Op2) {
8199   SDVTList VTs = getVTList(VT1, VT2, VT3);
8200   SDValue Ops[] = { Op1, Op2 };
8201   return getMachineNode(Opcode, dl, VTs, Ops);
8202 }
8203 
8204 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8205                                             EVT VT1, EVT VT2, EVT VT3,
8206                                             SDValue Op1, SDValue Op2,
8207                                             SDValue Op3) {
8208   SDVTList VTs = getVTList(VT1, VT2, VT3);
8209   SDValue Ops[] = { Op1, Op2, Op3 };
8210   return getMachineNode(Opcode, dl, VTs, Ops);
8211 }
8212 
8213 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8214                                             EVT VT1, EVT VT2, EVT VT3,
8215                                             ArrayRef<SDValue> Ops) {
8216   SDVTList VTs = getVTList(VT1, VT2, VT3);
8217   return getMachineNode(Opcode, dl, VTs, Ops);
8218 }
8219 
8220 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8221                                             ArrayRef<EVT> ResultTys,
8222                                             ArrayRef<SDValue> Ops) {
8223   SDVTList VTs = getVTList(ResultTys);
8224   return getMachineNode(Opcode, dl, VTs, Ops);
8225 }
8226 
8227 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
8228                                             SDVTList VTs,
8229                                             ArrayRef<SDValue> Ops) {
8230   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
8231   MachineSDNode *N;
8232   void *IP = nullptr;
8233 
8234   if (DoCSE) {
8235     FoldingSetNodeID ID;
8236     AddNodeIDNode(ID, ~Opcode, VTs, Ops);
8237     IP = nullptr;
8238     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
8239       return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
8240     }
8241   }
8242 
8243   // Allocate a new MachineSDNode.
8244   N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
8245   createOperands(N, Ops);
8246 
8247   if (DoCSE)
8248     CSEMap.InsertNode(N, IP);
8249 
8250   InsertNode(N);
8251   NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this);
8252   return N;
8253 }
8254 
8255 /// getTargetExtractSubreg - A convenience function for creating
8256 /// TargetOpcode::EXTRACT_SUBREG nodes.
8257 SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
8258                                              SDValue Operand) {
8259   SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
8260   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
8261                                   VT, Operand, SRIdxVal);
8262   return SDValue(Subreg, 0);
8263 }
8264 
8265 /// getTargetInsertSubreg - A convenience function for creating
8266 /// TargetOpcode::INSERT_SUBREG nodes.
8267 SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
8268                                             SDValue Operand, SDValue Subreg) {
8269   SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
8270   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
8271                                   VT, Operand, Subreg, SRIdxVal);
8272   return SDValue(Result, 0);
8273 }
8274 
8275 /// getNodeIfExists - Get the specified node if it's already available, or
8276 /// else return NULL.
8277 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
8278                                       ArrayRef<SDValue> Ops) {
8279   SDNodeFlags Flags;
8280   if (Inserter)
8281     Flags = Inserter->getFlags();
8282   return getNodeIfExists(Opcode, VTList, Ops, Flags);
8283 }
8284 
8285 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
8286                                       ArrayRef<SDValue> Ops,
8287                                       const SDNodeFlags Flags) {
8288   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
8289     FoldingSetNodeID ID;
8290     AddNodeIDNode(ID, Opcode, VTList, Ops);
8291     void *IP = nullptr;
8292     if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
8293       E->intersectFlagsWith(Flags);
8294       return E;
8295     }
8296   }
8297   return nullptr;
8298 }
8299 
8300 /// getDbgValue - Creates a SDDbgValue node.
8301 ///
8302 /// SDNode
8303 SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
8304                                       SDNode *N, unsigned R, bool IsIndirect,
8305                                       const DebugLoc &DL, unsigned O) {
8306   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
8307          "Expected inlined-at fields to agree");
8308   return new (DbgInfo->getAlloc())
8309       SDDbgValue(Var, Expr, N, R, IsIndirect, DL, O);
8310 }
8311 
8312 /// Constant
8313 SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
8314                                               DIExpression *Expr,
8315                                               const Value *C,
8316                                               const DebugLoc &DL, unsigned O) {
8317   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
8318          "Expected inlined-at fields to agree");
8319   return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, DL, O);
8320 }
8321 
8322 /// FrameIndex
8323 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
8324                                                 DIExpression *Expr, unsigned FI,
8325                                                 bool IsIndirect,
8326                                                 const DebugLoc &DL,
8327                                                 unsigned O) {
8328   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
8329          "Expected inlined-at fields to agree");
8330   return new (DbgInfo->getAlloc())
8331       SDDbgValue(Var, Expr, FI, IsIndirect, DL, O, SDDbgValue::FRAMEIX);
8332 }
8333 
8334 /// VReg
8335 SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var,
8336                                           DIExpression *Expr,
8337                                           unsigned VReg, bool IsIndirect,
8338                                           const DebugLoc &DL, unsigned O) {
8339   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
8340          "Expected inlined-at fields to agree");
8341   return new (DbgInfo->getAlloc())
8342       SDDbgValue(Var, Expr, VReg, IsIndirect, DL, O, SDDbgValue::VREG);
8343 }
8344 
8345 void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
8346                                      unsigned OffsetInBits, unsigned SizeInBits,
8347                                      bool InvalidateDbg) {
8348   SDNode *FromNode = From.getNode();
8349   SDNode *ToNode = To.getNode();
8350   assert(FromNode && ToNode && "Can't modify dbg values");
8351 
8352   // PR35338
8353   // TODO: assert(From != To && "Redundant dbg value transfer");
8354   // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
8355   if (From == To || FromNode == ToNode)
8356     return;
8357 
8358   if (!FromNode->getHasDebugValue())
8359     return;
8360 
8361   SmallVector<SDDbgValue *, 2> ClonedDVs;
8362   for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
8363     if (Dbg->getKind() != SDDbgValue::SDNODE || Dbg->isInvalidated())
8364       continue;
8365 
8366     // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
8367 
8368     // Just transfer the dbg value attached to From.
8369     if (Dbg->getResNo() != From.getResNo())
8370       continue;
8371 
8372     DIVariable *Var = Dbg->getVariable();
8373     auto *Expr = Dbg->getExpression();
8374     // If a fragment is requested, update the expression.
8375     if (SizeInBits) {
8376       // When splitting a larger (e.g., sign-extended) value whose
8377       // lower bits are described with an SDDbgValue, do not attempt
8378       // to transfer the SDDbgValue to the upper bits.
8379       if (auto FI = Expr->getFragmentInfo())
8380         if (OffsetInBits + SizeInBits > FI->SizeInBits)
8381           continue;
8382       auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
8383                                                              SizeInBits);
8384       if (!Fragment)
8385         continue;
8386       Expr = *Fragment;
8387     }
8388     // Clone the SDDbgValue and move it to To.
8389     SDDbgValue *Clone = getDbgValue(
8390         Var, Expr, ToNode, To.getResNo(), Dbg->isIndirect(), Dbg->getDebugLoc(),
8391         std::max(ToNode->getIROrder(), Dbg->getOrder()));
8392     ClonedDVs.push_back(Clone);
8393 
8394     if (InvalidateDbg) {
8395       // Invalidate value and indicate the SDDbgValue should not be emitted.
8396       Dbg->setIsInvalidated();
8397       Dbg->setIsEmitted();
8398     }
8399   }
8400 
8401   for (SDDbgValue *Dbg : ClonedDVs)
8402     AddDbgValue(Dbg, ToNode, false);
8403 }
8404 
8405 void SelectionDAG::salvageDebugInfo(SDNode &N) {
8406   if (!N.getHasDebugValue())
8407     return;
8408 
8409   SmallVector<SDDbgValue *, 2> ClonedDVs;
8410   for (auto DV : GetDbgValues(&N)) {
8411     if (DV->isInvalidated())
8412       continue;
8413     switch (N.getOpcode()) {
8414     default:
8415       break;
8416     case ISD::ADD:
8417       SDValue N0 = N.getOperand(0);
8418       SDValue N1 = N.getOperand(1);
8419       if (!isConstantIntBuildVectorOrConstantInt(N0) &&
8420           isConstantIntBuildVectorOrConstantInt(N1)) {
8421         uint64_t Offset = N.getConstantOperandVal(1);
8422         // Rewrite an ADD constant node into a DIExpression. Since we are
8423         // performing arithmetic to compute the variable's *value* in the
8424         // DIExpression, we need to mark the expression with a
8425         // DW_OP_stack_value.
8426         auto *DIExpr = DV->getExpression();
8427         DIExpr =
8428             DIExpression::prepend(DIExpr, DIExpression::StackValue, Offset);
8429         SDDbgValue *Clone =
8430             getDbgValue(DV->getVariable(), DIExpr, N0.getNode(), N0.getResNo(),
8431                         DV->isIndirect(), DV->getDebugLoc(), DV->getOrder());
8432         ClonedDVs.push_back(Clone);
8433         DV->setIsInvalidated();
8434         DV->setIsEmitted();
8435         LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
8436                    N0.getNode()->dumprFull(this);
8437                    dbgs() << " into " << *DIExpr << '\n');
8438       }
8439     }
8440   }
8441 
8442   for (SDDbgValue *Dbg : ClonedDVs)
8443     AddDbgValue(Dbg, Dbg->getSDNode(), false);
8444 }
8445 
8446 /// Creates a SDDbgLabel node.
8447 SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
8448                                       const DebugLoc &DL, unsigned O) {
8449   assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
8450          "Expected inlined-at fields to agree");
8451   return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
8452 }
8453 
8454 namespace {
8455 
8456 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
8457 /// pointed to by a use iterator is deleted, increment the use iterator
8458 /// so that it doesn't dangle.
8459 ///
8460 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
8461   SDNode::use_iterator &UI;
8462   SDNode::use_iterator &UE;
8463 
8464   void NodeDeleted(SDNode *N, SDNode *E) override {
8465     // Increment the iterator as needed.
8466     while (UI != UE && N == *UI)
8467       ++UI;
8468   }
8469 
8470 public:
8471   RAUWUpdateListener(SelectionDAG &d,
8472                      SDNode::use_iterator &ui,
8473                      SDNode::use_iterator &ue)
8474     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
8475 };
8476 
8477 } // end anonymous namespace
8478 
8479 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
8480 /// This can cause recursive merging of nodes in the DAG.
8481 ///
8482 /// This version assumes From has a single result value.
8483 ///
8484 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
8485   SDNode *From = FromN.getNode();
8486   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
8487          "Cannot replace with this method!");
8488   assert(From != To.getNode() && "Cannot replace uses of with self");
8489 
8490   // Preserve Debug Values
8491   transferDbgValues(FromN, To);
8492 
8493   // Iterate over all the existing uses of From. New uses will be added
8494   // to the beginning of the use list, which we avoid visiting.
8495   // This specifically avoids visiting uses of From that arise while the
8496   // replacement is happening, because any such uses would be the result
8497   // of CSE: If an existing node looks like From after one of its operands
8498   // is replaced by To, we don't want to replace of all its users with To
8499   // too. See PR3018 for more info.
8500   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
8501   RAUWUpdateListener Listener(*this, UI, UE);
8502   while (UI != UE) {
8503     SDNode *User = *UI;
8504 
8505     // This node is about to morph, remove its old self from the CSE maps.
8506     RemoveNodeFromCSEMaps(User);
8507 
8508     // A user can appear in a use list multiple times, and when this
8509     // happens the uses are usually next to each other in the list.
8510     // To help reduce the number of CSE recomputations, process all
8511     // the uses of this user that we can find this way.
8512     do {
8513       SDUse &Use = UI.getUse();
8514       ++UI;
8515       Use.set(To);
8516       if (To->isDivergent() != From->isDivergent())
8517         updateDivergence(User);
8518     } while (UI != UE && *UI == User);
8519     // Now that we have modified User, add it back to the CSE maps.  If it
8520     // already exists there, recursively merge the results together.
8521     AddModifiedNodeToCSEMaps(User);
8522   }
8523 
8524   // If we just RAUW'd the root, take note.
8525   if (FromN == getRoot())
8526     setRoot(To);
8527 }
8528 
8529 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
8530 /// This can cause recursive merging of nodes in the DAG.
8531 ///
8532 /// This version assumes that for each value of From, there is a
8533 /// corresponding value in To in the same position with the same type.
8534 ///
8535 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
8536 #ifndef NDEBUG
8537   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
8538     assert((!From->hasAnyUseOfValue(i) ||
8539             From->getValueType(i) == To->getValueType(i)) &&
8540            "Cannot use this version of ReplaceAllUsesWith!");
8541 #endif
8542 
8543   // Handle the trivial case.
8544   if (From == To)
8545     return;
8546 
8547   // Preserve Debug Info. Only do this if there's a use.
8548   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
8549     if (From->hasAnyUseOfValue(i)) {
8550       assert((i < To->getNumValues()) && "Invalid To location");
8551       transferDbgValues(SDValue(From, i), SDValue(To, i));
8552     }
8553 
8554   // Iterate over just the existing users of From. See the comments in
8555   // the ReplaceAllUsesWith above.
8556   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
8557   RAUWUpdateListener Listener(*this, UI, UE);
8558   while (UI != UE) {
8559     SDNode *User = *UI;
8560 
8561     // This node is about to morph, remove its old self from the CSE maps.
8562     RemoveNodeFromCSEMaps(User);
8563 
8564     // A user can appear in a use list multiple times, and when this
8565     // happens the uses are usually next to each other in the list.
8566     // To help reduce the number of CSE recomputations, process all
8567     // the uses of this user that we can find this way.
8568     do {
8569       SDUse &Use = UI.getUse();
8570       ++UI;
8571       Use.setNode(To);
8572       if (To->isDivergent() != From->isDivergent())
8573         updateDivergence(User);
8574     } while (UI != UE && *UI == User);
8575 
8576     // Now that we have modified User, add it back to the CSE maps.  If it
8577     // already exists there, recursively merge the results together.
8578     AddModifiedNodeToCSEMaps(User);
8579   }
8580 
8581   // If we just RAUW'd the root, take note.
8582   if (From == getRoot().getNode())
8583     setRoot(SDValue(To, getRoot().getResNo()));
8584 }
8585 
8586 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
8587 /// This can cause recursive merging of nodes in the DAG.
8588 ///
8589 /// This version can replace From with any result values.  To must match the
8590 /// number and types of values returned by From.
8591 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
8592   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
8593     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
8594 
8595   // Preserve Debug Info.
8596   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
8597     transferDbgValues(SDValue(From, i), To[i]);
8598 
8599   // Iterate over just the existing users of From. See the comments in
8600   // the ReplaceAllUsesWith above.
8601   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
8602   RAUWUpdateListener Listener(*this, UI, UE);
8603   while (UI != UE) {
8604     SDNode *User = *UI;
8605 
8606     // This node is about to morph, remove its old self from the CSE maps.
8607     RemoveNodeFromCSEMaps(User);
8608 
8609     // A user can appear in a use list multiple times, and when this happens the
8610     // uses are usually next to each other in the list.  To help reduce the
8611     // number of CSE and divergence recomputations, process all the uses of this
8612     // user that we can find this way.
8613     bool To_IsDivergent = false;
8614     do {
8615       SDUse &Use = UI.getUse();
8616       const SDValue &ToOp = To[Use.getResNo()];
8617       ++UI;
8618       Use.set(ToOp);
8619       To_IsDivergent |= ToOp->isDivergent();
8620     } while (UI != UE && *UI == User);
8621 
8622     if (To_IsDivergent != From->isDivergent())
8623       updateDivergence(User);
8624 
8625     // Now that we have modified User, add it back to the CSE maps.  If it
8626     // already exists there, recursively merge the results together.
8627     AddModifiedNodeToCSEMaps(User);
8628   }
8629 
8630   // If we just RAUW'd the root, take note.
8631   if (From == getRoot().getNode())
8632     setRoot(SDValue(To[getRoot().getResNo()]));
8633 }
8634 
8635 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
8636 /// uses of other values produced by From.getNode() alone.  The Deleted
8637 /// vector is handled the same way as for ReplaceAllUsesWith.
8638 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
8639   // Handle the really simple, really trivial case efficiently.
8640   if (From == To) return;
8641 
8642   // Handle the simple, trivial, case efficiently.
8643   if (From.getNode()->getNumValues() == 1) {
8644     ReplaceAllUsesWith(From, To);
8645     return;
8646   }
8647 
8648   // Preserve Debug Info.
8649   transferDbgValues(From, To);
8650 
8651   // Iterate over just the existing users of From. See the comments in
8652   // the ReplaceAllUsesWith above.
8653   SDNode::use_iterator UI = From.getNode()->use_begin(),
8654                        UE = From.getNode()->use_end();
8655   RAUWUpdateListener Listener(*this, UI, UE);
8656   while (UI != UE) {
8657     SDNode *User = *UI;
8658     bool UserRemovedFromCSEMaps = false;
8659 
8660     // A user can appear in a use list multiple times, and when this
8661     // happens the uses are usually next to each other in the list.
8662     // To help reduce the number of CSE recomputations, process all
8663     // the uses of this user that we can find this way.
8664     do {
8665       SDUse &Use = UI.getUse();
8666 
8667       // Skip uses of different values from the same node.
8668       if (Use.getResNo() != From.getResNo()) {
8669         ++UI;
8670         continue;
8671       }
8672 
8673       // If this node hasn't been modified yet, it's still in the CSE maps,
8674       // so remove its old self from the CSE maps.
8675       if (!UserRemovedFromCSEMaps) {
8676         RemoveNodeFromCSEMaps(User);
8677         UserRemovedFromCSEMaps = true;
8678       }
8679 
8680       ++UI;
8681       Use.set(To);
8682       if (To->isDivergent() != From->isDivergent())
8683         updateDivergence(User);
8684     } while (UI != UE && *UI == User);
8685     // We are iterating over all uses of the From node, so if a use
8686     // doesn't use the specific value, no changes are made.
8687     if (!UserRemovedFromCSEMaps)
8688       continue;
8689 
8690     // Now that we have modified User, add it back to the CSE maps.  If it
8691     // already exists there, recursively merge the results together.
8692     AddModifiedNodeToCSEMaps(User);
8693   }
8694 
8695   // If we just RAUW'd the root, take note.
8696   if (From == getRoot())
8697     setRoot(To);
8698 }
8699 
8700 namespace {
8701 
8702   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
8703   /// to record information about a use.
8704   struct UseMemo {
8705     SDNode *User;
8706     unsigned Index;
8707     SDUse *Use;
8708   };
8709 
8710   /// operator< - Sort Memos by User.
8711   bool operator<(const UseMemo &L, const UseMemo &R) {
8712     return (intptr_t)L.User < (intptr_t)R.User;
8713   }
8714 
8715 } // end anonymous namespace
8716 
8717 bool SelectionDAG::calculateDivergence(SDNode *N) {
8718   if (TLI->isSDNodeAlwaysUniform(N)) {
8719     assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) &&
8720            "Conflicting divergence information!");
8721     return false;
8722   }
8723   if (TLI->isSDNodeSourceOfDivergence(N, FLI, DA))
8724     return true;
8725   for (auto &Op : N->ops()) {
8726     if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent())
8727       return true;
8728   }
8729   return false;
8730 }
8731 
8732 void SelectionDAG::updateDivergence(SDNode *N) {
8733   SmallVector<SDNode *, 16> Worklist(1, N);
8734   do {
8735     N = Worklist.pop_back_val();
8736     bool IsDivergent = calculateDivergence(N);
8737     if (N->SDNodeBits.IsDivergent != IsDivergent) {
8738       N->SDNodeBits.IsDivergent = IsDivergent;
8739       Worklist.insert(Worklist.end(), N->use_begin(), N->use_end());
8740     }
8741   } while (!Worklist.empty());
8742 }
8743 
8744 void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
8745   DenseMap<SDNode *, unsigned> Degree;
8746   Order.reserve(AllNodes.size());
8747   for (auto &N : allnodes()) {
8748     unsigned NOps = N.getNumOperands();
8749     Degree[&N] = NOps;
8750     if (0 == NOps)
8751       Order.push_back(&N);
8752   }
8753   for (size_t I = 0; I != Order.size(); ++I) {
8754     SDNode *N = Order[I];
8755     for (auto U : N->uses()) {
8756       unsigned &UnsortedOps = Degree[U];
8757       if (0 == --UnsortedOps)
8758         Order.push_back(U);
8759     }
8760   }
8761 }
8762 
8763 #ifndef NDEBUG
8764 void SelectionDAG::VerifyDAGDiverence() {
8765   std::vector<SDNode *> TopoOrder;
8766   CreateTopologicalOrder(TopoOrder);
8767   for (auto *N : TopoOrder) {
8768     assert(calculateDivergence(N) == N->isDivergent() &&
8769            "Divergence bit inconsistency detected");
8770   }
8771 }
8772 #endif
8773 
8774 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
8775 /// uses of other values produced by From.getNode() alone.  The same value
8776 /// may appear in both the From and To list.  The Deleted vector is
8777 /// handled the same way as for ReplaceAllUsesWith.
8778 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
8779                                               const SDValue *To,
8780                                               unsigned Num){
8781   // Handle the simple, trivial case efficiently.
8782   if (Num == 1)
8783     return ReplaceAllUsesOfValueWith(*From, *To);
8784 
8785   transferDbgValues(*From, *To);
8786 
8787   // Read up all the uses and make records of them. This helps
8788   // processing new uses that are introduced during the
8789   // replacement process.
8790   SmallVector<UseMemo, 4> Uses;
8791   for (unsigned i = 0; i != Num; ++i) {
8792     unsigned FromResNo = From[i].getResNo();
8793     SDNode *FromNode = From[i].getNode();
8794     for (SDNode::use_iterator UI = FromNode->use_begin(),
8795          E = FromNode->use_end(); UI != E; ++UI) {
8796       SDUse &Use = UI.getUse();
8797       if (Use.getResNo() == FromResNo) {
8798         UseMemo Memo = { *UI, i, &Use };
8799         Uses.push_back(Memo);
8800       }
8801     }
8802   }
8803 
8804   // Sort the uses, so that all the uses from a given User are together.
8805   llvm::sort(Uses);
8806 
8807   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
8808        UseIndex != UseIndexEnd; ) {
8809     // We know that this user uses some value of From.  If it is the right
8810     // value, update it.
8811     SDNode *User = Uses[UseIndex].User;
8812 
8813     // This node is about to morph, remove its old self from the CSE maps.
8814     RemoveNodeFromCSEMaps(User);
8815 
8816     // The Uses array is sorted, so all the uses for a given User
8817     // are next to each other in the list.
8818     // To help reduce the number of CSE recomputations, process all
8819     // the uses of this user that we can find this way.
8820     do {
8821       unsigned i = Uses[UseIndex].Index;
8822       SDUse &Use = *Uses[UseIndex].Use;
8823       ++UseIndex;
8824 
8825       Use.set(To[i]);
8826     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
8827 
8828     // Now that we have modified User, add it back to the CSE maps.  If it
8829     // already exists there, recursively merge the results together.
8830     AddModifiedNodeToCSEMaps(User);
8831   }
8832 }
8833 
8834 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
8835 /// based on their topological order. It returns the maximum id and a vector
8836 /// of the SDNodes* in assigned order by reference.
8837 unsigned SelectionDAG::AssignTopologicalOrder() {
8838   unsigned DAGSize = 0;
8839 
8840   // SortedPos tracks the progress of the algorithm. Nodes before it are
8841   // sorted, nodes after it are unsorted. When the algorithm completes
8842   // it is at the end of the list.
8843   allnodes_iterator SortedPos = allnodes_begin();
8844 
8845   // Visit all the nodes. Move nodes with no operands to the front of
8846   // the list immediately. Annotate nodes that do have operands with their
8847   // operand count. Before we do this, the Node Id fields of the nodes
8848   // may contain arbitrary values. After, the Node Id fields for nodes
8849   // before SortedPos will contain the topological sort index, and the
8850   // Node Id fields for nodes At SortedPos and after will contain the
8851   // count of outstanding operands.
8852   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
8853     SDNode *N = &*I++;
8854     checkForCycles(N, this);
8855     unsigned Degree = N->getNumOperands();
8856     if (Degree == 0) {
8857       // A node with no uses, add it to the result array immediately.
8858       N->setNodeId(DAGSize++);
8859       allnodes_iterator Q(N);
8860       if (Q != SortedPos)
8861         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
8862       assert(SortedPos != AllNodes.end() && "Overran node list");
8863       ++SortedPos;
8864     } else {
8865       // Temporarily use the Node Id as scratch space for the degree count.
8866       N->setNodeId(Degree);
8867     }
8868   }
8869 
8870   // Visit all the nodes. As we iterate, move nodes into sorted order,
8871   // such that by the time the end is reached all nodes will be sorted.
8872   for (SDNode &Node : allnodes()) {
8873     SDNode *N = &Node;
8874     checkForCycles(N, this);
8875     // N is in sorted position, so all its uses have one less operand
8876     // that needs to be sorted.
8877     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
8878          UI != UE; ++UI) {
8879       SDNode *P = *UI;
8880       unsigned Degree = P->getNodeId();
8881       assert(Degree != 0 && "Invalid node degree");
8882       --Degree;
8883       if (Degree == 0) {
8884         // All of P's operands are sorted, so P may sorted now.
8885         P->setNodeId(DAGSize++);
8886         if (P->getIterator() != SortedPos)
8887           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
8888         assert(SortedPos != AllNodes.end() && "Overran node list");
8889         ++SortedPos;
8890       } else {
8891         // Update P's outstanding operand count.
8892         P->setNodeId(Degree);
8893       }
8894     }
8895     if (Node.getIterator() == SortedPos) {
8896 #ifndef NDEBUG
8897       allnodes_iterator I(N);
8898       SDNode *S = &*++I;
8899       dbgs() << "Overran sorted position:\n";
8900       S->dumprFull(this); dbgs() << "\n";
8901       dbgs() << "Checking if this is due to cycles\n";
8902       checkForCycles(this, true);
8903 #endif
8904       llvm_unreachable(nullptr);
8905     }
8906   }
8907 
8908   assert(SortedPos == AllNodes.end() &&
8909          "Topological sort incomplete!");
8910   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
8911          "First node in topological sort is not the entry token!");
8912   assert(AllNodes.front().getNodeId() == 0 &&
8913          "First node in topological sort has non-zero id!");
8914   assert(AllNodes.front().getNumOperands() == 0 &&
8915          "First node in topological sort has operands!");
8916   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
8917          "Last node in topologic sort has unexpected id!");
8918   assert(AllNodes.back().use_empty() &&
8919          "Last node in topologic sort has users!");
8920   assert(DAGSize == allnodes_size() && "Node count mismatch!");
8921   return DAGSize;
8922 }
8923 
8924 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
8925 /// value is produced by SD.
8926 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
8927   if (SD) {
8928     assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
8929     SD->setHasDebugValue(true);
8930   }
8931   DbgInfo->add(DB, SD, isParameter);
8932 }
8933 
8934 void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) {
8935   DbgInfo->add(DB);
8936 }
8937 
8938 SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
8939                                                    SDValue NewMemOp) {
8940   assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
8941   // The new memory operation must have the same position as the old load in
8942   // terms of memory dependency. Create a TokenFactor for the old load and new
8943   // memory operation and update uses of the old load's output chain to use that
8944   // TokenFactor.
8945   SDValue OldChain = SDValue(OldLoad, 1);
8946   SDValue NewChain = SDValue(NewMemOp.getNode(), 1);
8947   if (OldChain == NewChain || !OldLoad->hasAnyUseOfValue(1))
8948     return NewChain;
8949 
8950   SDValue TokenFactor =
8951       getNode(ISD::TokenFactor, SDLoc(OldLoad), MVT::Other, OldChain, NewChain);
8952   ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
8953   UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewChain);
8954   return TokenFactor;
8955 }
8956 
8957 SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
8958                                                      Function **OutFunction) {
8959   assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
8960 
8961   auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
8962   auto *Module = MF->getFunction().getParent();
8963   auto *Function = Module->getFunction(Symbol);
8964 
8965   if (OutFunction != nullptr)
8966       *OutFunction = Function;
8967 
8968   if (Function != nullptr) {
8969     auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
8970     return getGlobalAddress(Function, SDLoc(Op), PtrTy);
8971   }
8972 
8973   std::string ErrorStr;
8974   raw_string_ostream ErrorFormatter(ErrorStr);
8975 
8976   ErrorFormatter << "Undefined external symbol ";
8977   ErrorFormatter << '"' << Symbol << '"';
8978   ErrorFormatter.flush();
8979 
8980   report_fatal_error(ErrorStr);
8981 }
8982 
8983 //===----------------------------------------------------------------------===//
8984 //                              SDNode Class
8985 //===----------------------------------------------------------------------===//
8986 
8987 bool llvm::isNullConstant(SDValue V) {
8988   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
8989   return Const != nullptr && Const->isNullValue();
8990 }
8991 
8992 bool llvm::isNullFPConstant(SDValue V) {
8993   ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
8994   return Const != nullptr && Const->isZero() && !Const->isNegative();
8995 }
8996 
8997 bool llvm::isAllOnesConstant(SDValue V) {
8998   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
8999   return Const != nullptr && Const->isAllOnesValue();
9000 }
9001 
9002 bool llvm::isOneConstant(SDValue V) {
9003   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
9004   return Const != nullptr && Const->isOne();
9005 }
9006 
9007 SDValue llvm::peekThroughBitcasts(SDValue V) {
9008   while (V.getOpcode() == ISD::BITCAST)
9009     V = V.getOperand(0);
9010   return V;
9011 }
9012 
9013 SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
9014   while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
9015     V = V.getOperand(0);
9016   return V;
9017 }
9018 
9019 SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
9020   while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
9021     V = V.getOperand(0);
9022   return V;
9023 }
9024 
9025 bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
9026   if (V.getOpcode() != ISD::XOR)
9027     return false;
9028   V = peekThroughBitcasts(V.getOperand(1));
9029   unsigned NumBits = V.getScalarValueSizeInBits();
9030   ConstantSDNode *C =
9031       isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);
9032   return C && (C->getAPIntValue().countTrailingOnes() >= NumBits);
9033 }
9034 
9035 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
9036                                           bool AllowTruncation) {
9037   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
9038     return CN;
9039 
9040   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9041     BitVector UndefElements;
9042     ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
9043 
9044     // BuildVectors can truncate their operands. Ignore that case here unless
9045     // AllowTruncation is set.
9046     if (CN && (UndefElements.none() || AllowUndefs)) {
9047       EVT CVT = CN->getValueType(0);
9048       EVT NSVT = N.getValueType().getScalarType();
9049       assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
9050       if (AllowTruncation || (CVT == NSVT))
9051         return CN;
9052     }
9053   }
9054 
9055   return nullptr;
9056 }
9057 
9058 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
9059                                           bool AllowUndefs,
9060                                           bool AllowTruncation) {
9061   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
9062     return CN;
9063 
9064   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9065     BitVector UndefElements;
9066     ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
9067 
9068     // BuildVectors can truncate their operands. Ignore that case here unless
9069     // AllowTruncation is set.
9070     if (CN && (UndefElements.none() || AllowUndefs)) {
9071       EVT CVT = CN->getValueType(0);
9072       EVT NSVT = N.getValueType().getScalarType();
9073       assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
9074       if (AllowTruncation || (CVT == NSVT))
9075         return CN;
9076     }
9077   }
9078 
9079   return nullptr;
9080 }
9081 
9082 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
9083   if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
9084     return CN;
9085 
9086   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9087     BitVector UndefElements;
9088     ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements);
9089     if (CN && (UndefElements.none() || AllowUndefs))
9090       return CN;
9091   }
9092 
9093   if (N.getOpcode() == ISD::SPLAT_VECTOR)
9094     if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0)))
9095       return CN;
9096 
9097   return nullptr;
9098 }
9099 
9100 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
9101                                               const APInt &DemandedElts,
9102                                               bool AllowUndefs) {
9103   if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
9104     return CN;
9105 
9106   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9107     BitVector UndefElements;
9108     ConstantFPSDNode *CN =
9109         BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
9110     if (CN && (UndefElements.none() || AllowUndefs))
9111       return CN;
9112   }
9113 
9114   return nullptr;
9115 }
9116 
9117 bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
9118   // TODO: may want to use peekThroughBitcast() here.
9119   ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
9120   return C && C->isNullValue();
9121 }
9122 
9123 bool llvm::isOneOrOneSplat(SDValue N) {
9124   // TODO: may want to use peekThroughBitcast() here.
9125   unsigned BitWidth = N.getScalarValueSizeInBits();
9126   ConstantSDNode *C = isConstOrConstSplat(N);
9127   return C && C->isOne() && C->getValueSizeInBits(0) == BitWidth;
9128 }
9129 
9130 bool llvm::isAllOnesOrAllOnesSplat(SDValue N) {
9131   N = peekThroughBitcasts(N);
9132   unsigned BitWidth = N.getScalarValueSizeInBits();
9133   ConstantSDNode *C = isConstOrConstSplat(N);
9134   return C && C->isAllOnesValue() && C->getValueSizeInBits(0) == BitWidth;
9135 }
9136 
9137 HandleSDNode::~HandleSDNode() {
9138   DropOperands();
9139 }
9140 
9141 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
9142                                          const DebugLoc &DL,
9143                                          const GlobalValue *GA, EVT VT,
9144                                          int64_t o, unsigned TF)
9145     : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
9146   TheGlobal = GA;
9147 }
9148 
9149 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
9150                                          EVT VT, unsigned SrcAS,
9151                                          unsigned DestAS)
9152     : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
9153       SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
9154 
9155 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
9156                      SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
9157     : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
9158   MemSDNodeBits.IsVolatile = MMO->isVolatile();
9159   MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
9160   MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
9161   MemSDNodeBits.IsInvariant = MMO->isInvariant();
9162 
9163   // We check here that the size of the memory operand fits within the size of
9164   // the MMO. This is because the MMO might indicate only a possible address
9165   // range instead of specifying the affected memory addresses precisely.
9166   // TODO: Make MachineMemOperands aware of scalable vectors.
9167   assert(memvt.getStoreSize().getKnownMinSize() <= MMO->getSize() &&
9168          "Size mismatch!");
9169 }
9170 
9171 /// Profile - Gather unique data for the node.
9172 ///
9173 void SDNode::Profile(FoldingSetNodeID &ID) const {
9174   AddNodeIDNode(ID, this);
9175 }
9176 
9177 namespace {
9178 
9179   struct EVTArray {
9180     std::vector<EVT> VTs;
9181 
9182     EVTArray() {
9183       VTs.reserve(MVT::LAST_VALUETYPE);
9184       for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
9185         VTs.push_back(MVT((MVT::SimpleValueType)i));
9186     }
9187   };
9188 
9189 } // end anonymous namespace
9190 
9191 static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs;
9192 static ManagedStatic<EVTArray> SimpleVTArray;
9193 static ManagedStatic<sys::SmartMutex<true>> VTMutex;
9194 
9195 /// getValueTypeList - Return a pointer to the specified value type.
9196 ///
9197 const EVT *SDNode::getValueTypeList(EVT VT) {
9198   if (VT.isExtended()) {
9199     sys::SmartScopedLock<true> Lock(*VTMutex);
9200     return &(*EVTs->insert(VT).first);
9201   } else {
9202     assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
9203            "Value type out of range!");
9204     return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
9205   }
9206 }
9207 
9208 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
9209 /// indicated value.  This method ignores uses of other values defined by this
9210 /// operation.
9211 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
9212   assert(Value < getNumValues() && "Bad value!");
9213 
9214   // TODO: Only iterate over uses of a given value of the node
9215   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
9216     if (UI.getUse().getResNo() == Value) {
9217       if (NUses == 0)
9218         return false;
9219       --NUses;
9220     }
9221   }
9222 
9223   // Found exactly the right number of uses?
9224   return NUses == 0;
9225 }
9226 
9227 /// hasAnyUseOfValue - Return true if there are any use of the indicated
9228 /// value. This method ignores uses of other values defined by this operation.
9229 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
9230   assert(Value < getNumValues() && "Bad value!");
9231 
9232   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
9233     if (UI.getUse().getResNo() == Value)
9234       return true;
9235 
9236   return false;
9237 }
9238 
9239 /// isOnlyUserOf - Return true if this node is the only use of N.
9240 bool SDNode::isOnlyUserOf(const SDNode *N) const {
9241   bool Seen = false;
9242   for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
9243     SDNode *User = *I;
9244     if (User == this)
9245       Seen = true;
9246     else
9247       return false;
9248   }
9249 
9250   return Seen;
9251 }
9252 
9253 /// Return true if the only users of N are contained in Nodes.
9254 bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
9255   bool Seen = false;
9256   for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
9257     SDNode *User = *I;
9258     if (llvm::any_of(Nodes,
9259                      [&User](const SDNode *Node) { return User == Node; }))
9260       Seen = true;
9261     else
9262       return false;
9263   }
9264 
9265   return Seen;
9266 }
9267 
9268 /// isOperand - Return true if this node is an operand of N.
9269 bool SDValue::isOperandOf(const SDNode *N) const {
9270   return any_of(N->op_values(), [this](SDValue Op) { return *this == Op; });
9271 }
9272 
9273 bool SDNode::isOperandOf(const SDNode *N) const {
9274   return any_of(N->op_values(),
9275                 [this](SDValue Op) { return this == Op.getNode(); });
9276 }
9277 
9278 /// reachesChainWithoutSideEffects - Return true if this operand (which must
9279 /// be a chain) reaches the specified operand without crossing any
9280 /// side-effecting instructions on any chain path.  In practice, this looks
9281 /// through token factors and non-volatile loads.  In order to remain efficient,
9282 /// this only looks a couple of nodes in, it does not do an exhaustive search.
9283 ///
9284 /// Note that we only need to examine chains when we're searching for
9285 /// side-effects; SelectionDAG requires that all side-effects are represented
9286 /// by chains, even if another operand would force a specific ordering. This
9287 /// constraint is necessary to allow transformations like splitting loads.
9288 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
9289                                              unsigned Depth) const {
9290   if (*this == Dest) return true;
9291 
9292   // Don't search too deeply, we just want to be able to see through
9293   // TokenFactor's etc.
9294   if (Depth == 0) return false;
9295 
9296   // If this is a token factor, all inputs to the TF happen in parallel.
9297   if (getOpcode() == ISD::TokenFactor) {
9298     // First, try a shallow search.
9299     if (is_contained((*this)->ops(), Dest)) {
9300       // We found the chain we want as an operand of this TokenFactor.
9301       // Essentially, we reach the chain without side-effects if we could
9302       // serialize the TokenFactor into a simple chain of operations with
9303       // Dest as the last operation. This is automatically true if the
9304       // chain has one use: there are no other ordering constraints.
9305       // If the chain has more than one use, we give up: some other
9306       // use of Dest might force a side-effect between Dest and the current
9307       // node.
9308       if (Dest.hasOneUse())
9309         return true;
9310     }
9311     // Next, try a deep search: check whether every operand of the TokenFactor
9312     // reaches Dest.
9313     return llvm::all_of((*this)->ops(), [=](SDValue Op) {
9314       return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
9315     });
9316   }
9317 
9318   // Loads don't have side effects, look through them.
9319   if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
9320     if (Ld->isUnordered())
9321       return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
9322   }
9323   return false;
9324 }
9325 
9326 bool SDNode::hasPredecessor(const SDNode *N) const {
9327   SmallPtrSet<const SDNode *, 32> Visited;
9328   SmallVector<const SDNode *, 16> Worklist;
9329   Worklist.push_back(this);
9330   return hasPredecessorHelper(N, Visited, Worklist);
9331 }
9332 
9333 void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
9334   this->Flags.intersectWith(Flags);
9335 }
9336 
9337 SDValue
9338 SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
9339                                   ArrayRef<ISD::NodeType> CandidateBinOps,
9340                                   bool AllowPartials) {
9341   // The pattern must end in an extract from index 0.
9342   if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9343       !isNullConstant(Extract->getOperand(1)))
9344     return SDValue();
9345 
9346   // Match against one of the candidate binary ops.
9347   SDValue Op = Extract->getOperand(0);
9348   if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
9349         return Op.getOpcode() == unsigned(BinOp);
9350       }))
9351     return SDValue();
9352 
9353   // Floating-point reductions may require relaxed constraints on the final step
9354   // of the reduction because they may reorder intermediate operations.
9355   unsigned CandidateBinOp = Op.getOpcode();
9356   if (Op.getValueType().isFloatingPoint()) {
9357     SDNodeFlags Flags = Op->getFlags();
9358     switch (CandidateBinOp) {
9359     case ISD::FADD:
9360       if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
9361         return SDValue();
9362       break;
9363     default:
9364       llvm_unreachable("Unhandled FP opcode for binop reduction");
9365     }
9366   }
9367 
9368   // Matching failed - attempt to see if we did enough stages that a partial
9369   // reduction from a subvector is possible.
9370   auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
9371     if (!AllowPartials || !Op)
9372       return SDValue();
9373     EVT OpVT = Op.getValueType();
9374     EVT OpSVT = OpVT.getScalarType();
9375     EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts);
9376     if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0))
9377       return SDValue();
9378     BinOp = (ISD::NodeType)CandidateBinOp;
9379     return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op,
9380                    getVectorIdxConstant(0, SDLoc(Op)));
9381   };
9382 
9383   // At each stage, we're looking for something that looks like:
9384   // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
9385   //                    <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
9386   //                               i32 undef, i32 undef, i32 undef, i32 undef>
9387   // %a = binop <8 x i32> %op, %s
9388   // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
9389   // we expect something like:
9390   // <4,5,6,7,u,u,u,u>
9391   // <2,3,u,u,u,u,u,u>
9392   // <1,u,u,u,u,u,u,u>
9393   // While a partial reduction match would be:
9394   // <2,3,u,u,u,u,u,u>
9395   // <1,u,u,u,u,u,u,u>
9396   unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
9397   SDValue PrevOp;
9398   for (unsigned i = 0; i < Stages; ++i) {
9399     unsigned MaskEnd = (1 << i);
9400 
9401     if (Op.getOpcode() != CandidateBinOp)
9402       return PartialReduction(PrevOp, MaskEnd);
9403 
9404     SDValue Op0 = Op.getOperand(0);
9405     SDValue Op1 = Op.getOperand(1);
9406 
9407     ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);
9408     if (Shuffle) {
9409       Op = Op1;
9410     } else {
9411       Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
9412       Op = Op0;
9413     }
9414 
9415     // The first operand of the shuffle should be the same as the other operand
9416     // of the binop.
9417     if (!Shuffle || Shuffle->getOperand(0) != Op)
9418       return PartialReduction(PrevOp, MaskEnd);
9419 
9420     // Verify the shuffle has the expected (at this stage of the pyramid) mask.
9421     for (int Index = 0; Index < (int)MaskEnd; ++Index)
9422       if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index))
9423         return PartialReduction(PrevOp, MaskEnd);
9424 
9425     PrevOp = Op;
9426   }
9427 
9428   // Handle subvector reductions, which tend to appear after the shuffle
9429   // reduction stages.
9430   while (Op.getOpcode() == CandidateBinOp) {
9431     unsigned NumElts = Op.getValueType().getVectorNumElements();
9432     SDValue Op0 = Op.getOperand(0);
9433     SDValue Op1 = Op.getOperand(1);
9434     if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
9435         Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
9436         Op0.getOperand(0) != Op1.getOperand(0))
9437       break;
9438     SDValue Src = Op0.getOperand(0);
9439     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
9440     if (NumSrcElts != (2 * NumElts))
9441       break;
9442     if (!(Op0.getConstantOperandAPInt(1) == 0 &&
9443           Op1.getConstantOperandAPInt(1) == NumElts) &&
9444         !(Op1.getConstantOperandAPInt(1) == 0 &&
9445           Op0.getConstantOperandAPInt(1) == NumElts))
9446       break;
9447     Op = Src;
9448   }
9449 
9450   BinOp = (ISD::NodeType)CandidateBinOp;
9451   return Op;
9452 }
9453 
9454 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
9455   assert(N->getNumValues() == 1 &&
9456          "Can't unroll a vector with multiple results!");
9457 
9458   EVT VT = N->getValueType(0);
9459   unsigned NE = VT.getVectorNumElements();
9460   EVT EltVT = VT.getVectorElementType();
9461   SDLoc dl(N);
9462 
9463   SmallVector<SDValue, 8> Scalars;
9464   SmallVector<SDValue, 4> Operands(N->getNumOperands());
9465 
9466   // If ResNE is 0, fully unroll the vector op.
9467   if (ResNE == 0)
9468     ResNE = NE;
9469   else if (NE > ResNE)
9470     NE = ResNE;
9471 
9472   unsigned i;
9473   for (i= 0; i != NE; ++i) {
9474     for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
9475       SDValue Operand = N->getOperand(j);
9476       EVT OperandVT = Operand.getValueType();
9477       if (OperandVT.isVector()) {
9478         // A vector operand; extract a single element.
9479         EVT OperandEltVT = OperandVT.getVectorElementType();
9480         Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
9481                               Operand, getVectorIdxConstant(i, dl));
9482       } else {
9483         // A scalar operand; just use it as is.
9484         Operands[j] = Operand;
9485       }
9486     }
9487 
9488     switch (N->getOpcode()) {
9489     default: {
9490       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
9491                                 N->getFlags()));
9492       break;
9493     }
9494     case ISD::VSELECT:
9495       Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
9496       break;
9497     case ISD::SHL:
9498     case ISD::SRA:
9499     case ISD::SRL:
9500     case ISD::ROTL:
9501     case ISD::ROTR:
9502       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
9503                                getShiftAmountOperand(Operands[0].getValueType(),
9504                                                      Operands[1])));
9505       break;
9506     case ISD::SIGN_EXTEND_INREG: {
9507       EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
9508       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
9509                                 Operands[0],
9510                                 getValueType(ExtVT)));
9511     }
9512     }
9513   }
9514 
9515   for (; i < ResNE; ++i)
9516     Scalars.push_back(getUNDEF(EltVT));
9517 
9518   EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
9519   return getBuildVector(VecVT, dl, Scalars);
9520 }
9521 
9522 std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
9523     SDNode *N, unsigned ResNE) {
9524   unsigned Opcode = N->getOpcode();
9525   assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
9526           Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
9527           Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
9528          "Expected an overflow opcode");
9529 
9530   EVT ResVT = N->getValueType(0);
9531   EVT OvVT = N->getValueType(1);
9532   EVT ResEltVT = ResVT.getVectorElementType();
9533   EVT OvEltVT = OvVT.getVectorElementType();
9534   SDLoc dl(N);
9535 
9536   // If ResNE is 0, fully unroll the vector op.
9537   unsigned NE = ResVT.getVectorNumElements();
9538   if (ResNE == 0)
9539     ResNE = NE;
9540   else if (NE > ResNE)
9541     NE = ResNE;
9542 
9543   SmallVector<SDValue, 8> LHSScalars;
9544   SmallVector<SDValue, 8> RHSScalars;
9545   ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);
9546   ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);
9547 
9548   EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);
9549   SDVTList VTs = getVTList(ResEltVT, SVT);
9550   SmallVector<SDValue, 8> ResScalars;
9551   SmallVector<SDValue, 8> OvScalars;
9552   for (unsigned i = 0; i < NE; ++i) {
9553     SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
9554     SDValue Ov =
9555         getSelect(dl, OvEltVT, Res.getValue(1),
9556                   getBoolConstant(true, dl, OvEltVT, ResVT),
9557                   getConstant(0, dl, OvEltVT));
9558 
9559     ResScalars.push_back(Res);
9560     OvScalars.push_back(Ov);
9561   }
9562 
9563   ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));
9564   OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));
9565 
9566   EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);
9567   EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);
9568   return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),
9569                         getBuildVector(NewOvVT, dl, OvScalars));
9570 }
9571 
9572 bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
9573                                                   LoadSDNode *Base,
9574                                                   unsigned Bytes,
9575                                                   int Dist) const {
9576   if (LD->isVolatile() || Base->isVolatile())
9577     return false;
9578   // TODO: probably too restrictive for atomics, revisit
9579   if (!LD->isSimple())
9580     return false;
9581   if (LD->isIndexed() || Base->isIndexed())
9582     return false;
9583   if (LD->getChain() != Base->getChain())
9584     return false;
9585   EVT VT = LD->getValueType(0);
9586   if (VT.getSizeInBits() / 8 != Bytes)
9587     return false;
9588 
9589   auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
9590   auto LocDecomp = BaseIndexOffset::match(LD, *this);
9591 
9592   int64_t Offset = 0;
9593   if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
9594     return (Dist * Bytes == Offset);
9595   return false;
9596 }
9597 
9598 /// InferPtrAlignment - Infer alignment of a load / store address. Return None
9599 /// if it cannot be inferred.
9600 MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
9601   // If this is a GlobalAddress + cst, return the alignment.
9602   const GlobalValue *GV = nullptr;
9603   int64_t GVOffset = 0;
9604   if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
9605     unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
9606     KnownBits Known(PtrWidth);
9607     llvm::computeKnownBits(GV, Known, getDataLayout());
9608     unsigned AlignBits = Known.countMinTrailingZeros();
9609     if (AlignBits)
9610       return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset);
9611   }
9612 
9613   // If this is a direct reference to a stack slot, use information about the
9614   // stack slot's alignment.
9615   int FrameIdx = INT_MIN;
9616   int64_t FrameOffset = 0;
9617   if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
9618     FrameIdx = FI->getIndex();
9619   } else if (isBaseWithConstantOffset(Ptr) &&
9620              isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
9621     // Handle FI+Cst
9622     FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
9623     FrameOffset = Ptr.getConstantOperandVal(1);
9624   }
9625 
9626   if (FrameIdx != INT_MIN) {
9627     const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
9628     return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);
9629   }
9630 
9631   return None;
9632 }
9633 
9634 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
9635 /// which is split (or expanded) into two not necessarily identical pieces.
9636 std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
9637   // Currently all types are split in half.
9638   EVT LoVT, HiVT;
9639   if (!VT.isVector())
9640     LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
9641   else
9642     LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
9643 
9644   return std::make_pair(LoVT, HiVT);
9645 }
9646 
9647 /// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
9648 /// type, dependent on an enveloping VT that has been split into two identical
9649 /// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
9650 std::pair<EVT, EVT>
9651 SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
9652                                        bool *HiIsEmpty) const {
9653   EVT EltTp = VT.getVectorElementType();
9654   // Examples:
9655   //   custom VL=8  with enveloping VL=8/8 yields 8/0 (hi empty)
9656   //   custom VL=9  with enveloping VL=8/8 yields 8/1
9657   //   custom VL=10 with enveloping VL=8/8 yields 8/2
9658   //   etc.
9659   ElementCount VTNumElts = VT.getVectorElementCount();
9660   ElementCount EnvNumElts = EnvVT.getVectorElementCount();
9661   assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
9662          "Mixing fixed width and scalable vectors when enveloping a type");
9663   EVT LoVT, HiVT;
9664   if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
9665     LoVT = EnvVT;
9666     HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts);
9667     *HiIsEmpty = false;
9668   } else {
9669     // Flag that hi type has zero storage size, but return split envelop type
9670     // (this would be easier if vector types with zero elements were allowed).
9671     LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts);
9672     HiVT = EnvVT;
9673     *HiIsEmpty = true;
9674   }
9675   return std::make_pair(LoVT, HiVT);
9676 }
9677 
9678 /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
9679 /// low/high part.
9680 std::pair<SDValue, SDValue>
9681 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
9682                           const EVT &HiVT) {
9683   assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
9684          LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
9685          "Splitting vector with an invalid mixture of fixed and scalable "
9686          "vector types");
9687   assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
9688              N.getValueType().getVectorMinNumElements() &&
9689          "More vector elements requested than available!");
9690   SDValue Lo, Hi;
9691   Lo =
9692       getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL));
9693   // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
9694   // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
9695   // IDX with the runtime scaling factor of the result vector type. For
9696   // fixed-width result vectors, that runtime scaling factor is 1.
9697   Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
9698                getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
9699   return std::make_pair(Lo, Hi);
9700 }
9701 
9702 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
9703 SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
9704   EVT VT = N.getValueType();
9705   EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
9706                                 NextPowerOf2(VT.getVectorNumElements()));
9707   return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N,
9708                  getVectorIdxConstant(0, DL));
9709 }
9710 
9711 void SelectionDAG::ExtractVectorElements(SDValue Op,
9712                                          SmallVectorImpl<SDValue> &Args,
9713                                          unsigned Start, unsigned Count,
9714                                          EVT EltVT) {
9715   EVT VT = Op.getValueType();
9716   if (Count == 0)
9717     Count = VT.getVectorNumElements();
9718   if (EltVT == EVT())
9719     EltVT = VT.getVectorElementType();
9720   SDLoc SL(Op);
9721   for (unsigned i = Start, e = Start + Count; i != e; ++i) {
9722     Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op,
9723                            getVectorIdxConstant(i, SL)));
9724   }
9725 }
9726 
9727 // getAddressSpace - Return the address space this GlobalAddress belongs to.
9728 unsigned GlobalAddressSDNode::getAddressSpace() const {
9729   return getGlobal()->getType()->getAddressSpace();
9730 }
9731 
9732 Type *ConstantPoolSDNode::getType() const {
9733   if (isMachineConstantPoolEntry())
9734     return Val.MachineCPVal->getType();
9735   return Val.ConstVal->getType();
9736 }
9737 
9738 bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
9739                                         unsigned &SplatBitSize,
9740                                         bool &HasAnyUndefs,
9741                                         unsigned MinSplatBits,
9742                                         bool IsBigEndian) const {
9743   EVT VT = getValueType(0);
9744   assert(VT.isVector() && "Expected a vector type");
9745   unsigned VecWidth = VT.getSizeInBits();
9746   if (MinSplatBits > VecWidth)
9747     return false;
9748 
9749   // FIXME: The widths are based on this node's type, but build vectors can
9750   // truncate their operands.
9751   SplatValue = APInt(VecWidth, 0);
9752   SplatUndef = APInt(VecWidth, 0);
9753 
9754   // Get the bits. Bits with undefined values (when the corresponding element
9755   // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
9756   // in SplatValue. If any of the values are not constant, give up and return
9757   // false.
9758   unsigned int NumOps = getNumOperands();
9759   assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
9760   unsigned EltWidth = VT.getScalarSizeInBits();
9761 
9762   for (unsigned j = 0; j < NumOps; ++j) {
9763     unsigned i = IsBigEndian ? NumOps - 1 - j : j;
9764     SDValue OpVal = getOperand(i);
9765     unsigned BitPos = j * EltWidth;
9766 
9767     if (OpVal.isUndef())
9768       SplatUndef.setBits(BitPos, BitPos + EltWidth);
9769     else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
9770       SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
9771     else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
9772       SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
9773     else
9774       return false;
9775   }
9776 
9777   // The build_vector is all constants or undefs. Find the smallest element
9778   // size that splats the vector.
9779   HasAnyUndefs = (SplatUndef != 0);
9780 
9781   // FIXME: This does not work for vectors with elements less than 8 bits.
9782   while (VecWidth > 8) {
9783     unsigned HalfSize = VecWidth / 2;
9784     APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
9785     APInt LowValue = SplatValue.trunc(HalfSize);
9786     APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
9787     APInt LowUndef = SplatUndef.trunc(HalfSize);
9788 
9789     // If the two halves do not match (ignoring undef bits), stop here.
9790     if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
9791         MinSplatBits > HalfSize)
9792       break;
9793 
9794     SplatValue = HighValue | LowValue;
9795     SplatUndef = HighUndef & LowUndef;
9796 
9797     VecWidth = HalfSize;
9798   }
9799 
9800   SplatBitSize = VecWidth;
9801   return true;
9802 }
9803 
9804 SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
9805                                          BitVector *UndefElements) const {
9806   unsigned NumOps = getNumOperands();
9807   if (UndefElements) {
9808     UndefElements->clear();
9809     UndefElements->resize(NumOps);
9810   }
9811   assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
9812   if (!DemandedElts)
9813     return SDValue();
9814   SDValue Splatted;
9815   for (unsigned i = 0; i != NumOps; ++i) {
9816     if (!DemandedElts[i])
9817       continue;
9818     SDValue Op = getOperand(i);
9819     if (Op.isUndef()) {
9820       if (UndefElements)
9821         (*UndefElements)[i] = true;
9822     } else if (!Splatted) {
9823       Splatted = Op;
9824     } else if (Splatted != Op) {
9825       return SDValue();
9826     }
9827   }
9828 
9829   if (!Splatted) {
9830     unsigned FirstDemandedIdx = DemandedElts.countTrailingZeros();
9831     assert(getOperand(FirstDemandedIdx).isUndef() &&
9832            "Can only have a splat without a constant for all undefs.");
9833     return getOperand(FirstDemandedIdx);
9834   }
9835 
9836   return Splatted;
9837 }
9838 
9839 SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
9840   APInt DemandedElts = APInt::getAllOnesValue(getNumOperands());
9841   return getSplatValue(DemandedElts, UndefElements);
9842 }
9843 
9844 bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
9845                                             SmallVectorImpl<SDValue> &Sequence,
9846                                             BitVector *UndefElements) const {
9847   unsigned NumOps = getNumOperands();
9848   Sequence.clear();
9849   if (UndefElements) {
9850     UndefElements->clear();
9851     UndefElements->resize(NumOps);
9852   }
9853   assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
9854   if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps))
9855     return false;
9856 
9857   // Set the undefs even if we don't find a sequence (like getSplatValue).
9858   if (UndefElements)
9859     for (unsigned I = 0; I != NumOps; ++I)
9860       if (DemandedElts[I] && getOperand(I).isUndef())
9861         (*UndefElements)[I] = true;
9862 
9863   // Iteratively widen the sequence length looking for repetitions.
9864   for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
9865     Sequence.append(SeqLen, SDValue());
9866     for (unsigned I = 0; I != NumOps; ++I) {
9867       if (!DemandedElts[I])
9868         continue;
9869       SDValue &SeqOp = Sequence[I % SeqLen];
9870       SDValue Op = getOperand(I);
9871       if (Op.isUndef()) {
9872         if (!SeqOp)
9873           SeqOp = Op;
9874         continue;
9875       }
9876       if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
9877         Sequence.clear();
9878         break;
9879       }
9880       SeqOp = Op;
9881     }
9882     if (!Sequence.empty())
9883       return true;
9884   }
9885 
9886   assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
9887   return false;
9888 }
9889 
9890 bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
9891                                             BitVector *UndefElements) const {
9892   APInt DemandedElts = APInt::getAllOnesValue(getNumOperands());
9893   return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
9894 }
9895 
9896 ConstantSDNode *
9897 BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
9898                                         BitVector *UndefElements) const {
9899   return dyn_cast_or_null<ConstantSDNode>(
9900       getSplatValue(DemandedElts, UndefElements));
9901 }
9902 
9903 ConstantSDNode *
9904 BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
9905   return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
9906 }
9907 
9908 ConstantFPSDNode *
9909 BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
9910                                           BitVector *UndefElements) const {
9911   return dyn_cast_or_null<ConstantFPSDNode>(
9912       getSplatValue(DemandedElts, UndefElements));
9913 }
9914 
9915 ConstantFPSDNode *
9916 BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
9917   return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
9918 }
9919 
9920 int32_t
9921 BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
9922                                                    uint32_t BitWidth) const {
9923   if (ConstantFPSDNode *CN =
9924           dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
9925     bool IsExact;
9926     APSInt IntVal(BitWidth);
9927     const APFloat &APF = CN->getValueAPF();
9928     if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
9929             APFloat::opOK ||
9930         !IsExact)
9931       return -1;
9932 
9933     return IntVal.exactLogBase2();
9934   }
9935   return -1;
9936 }
9937 
9938 bool BuildVectorSDNode::isConstant() const {
9939   for (const SDValue &Op : op_values()) {
9940     unsigned Opc = Op.getOpcode();
9941     if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
9942       return false;
9943   }
9944   return true;
9945 }
9946 
9947 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
9948   // Find the first non-undef value in the shuffle mask.
9949   unsigned i, e;
9950   for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
9951     /* search */;
9952 
9953   // If all elements are undefined, this shuffle can be considered a splat
9954   // (although it should eventually get simplified away completely).
9955   if (i == e)
9956     return true;
9957 
9958   // Make sure all remaining elements are either undef or the same as the first
9959   // non-undef value.
9960   for (int Idx = Mask[i]; i != e; ++i)
9961     if (Mask[i] >= 0 && Mask[i] != Idx)
9962       return false;
9963   return true;
9964 }
9965 
9966 // Returns the SDNode if it is a constant integer BuildVector
9967 // or constant integer.
9968 SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) {
9969   if (isa<ConstantSDNode>(N))
9970     return N.getNode();
9971   if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
9972     return N.getNode();
9973   // Treat a GlobalAddress supporting constant offset folding as a
9974   // constant integer.
9975   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
9976     if (GA->getOpcode() == ISD::GlobalAddress &&
9977         TLI->isOffsetFoldingLegal(GA))
9978       return GA;
9979   if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
9980       isa<ConstantSDNode>(N.getOperand(0)))
9981     return N.getNode();
9982   return nullptr;
9983 }
9984 
9985 SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) {
9986   if (isa<ConstantFPSDNode>(N))
9987     return N.getNode();
9988 
9989   if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
9990     return N.getNode();
9991 
9992   return nullptr;
9993 }
9994 
9995 void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
9996   assert(!Node->OperandList && "Node already has operands");
9997   assert(SDNode::getMaxNumOperands() >= Vals.size() &&
9998          "too many operands to fit into SDNode");
9999   SDUse *Ops = OperandRecycler.allocate(
10000       ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
10001 
10002   bool IsDivergent = false;
10003   for (unsigned I = 0; I != Vals.size(); ++I) {
10004     Ops[I].setUser(Node);
10005     Ops[I].setInitial(Vals[I]);
10006     if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
10007       IsDivergent |= Ops[I].getNode()->isDivergent();
10008   }
10009   Node->NumOperands = Vals.size();
10010   Node->OperandList = Ops;
10011   if (!TLI->isSDNodeAlwaysUniform(Node)) {
10012     IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA);
10013     Node->SDNodeBits.IsDivergent = IsDivergent;
10014   }
10015   checkForCycles(Node);
10016 }
10017 
10018 SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
10019                                      SmallVectorImpl<SDValue> &Vals) {
10020   size_t Limit = SDNode::getMaxNumOperands();
10021   while (Vals.size() > Limit) {
10022     unsigned SliceIdx = Vals.size() - Limit;
10023     auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
10024     SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
10025     Vals.erase(Vals.begin() + SliceIdx, Vals.end());
10026     Vals.emplace_back(NewTF);
10027   }
10028   return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
10029 }
10030 
10031 SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
10032                                         EVT VT, SDNodeFlags Flags) {
10033   switch (Opcode) {
10034   default:
10035     return SDValue();
10036   case ISD::ADD:
10037   case ISD::OR:
10038   case ISD::XOR:
10039   case ISD::UMAX:
10040     return getConstant(0, DL, VT);
10041   case ISD::MUL:
10042     return getConstant(1, DL, VT);
10043   case ISD::AND:
10044   case ISD::UMIN:
10045     return getAllOnesConstant(DL, VT);
10046   case ISD::SMAX:
10047     return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT);
10048   case ISD::SMIN:
10049     return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT);
10050   case ISD::FADD:
10051     return getConstantFP(-0.0, DL, VT);
10052   case ISD::FMUL:
10053     return getConstantFP(1.0, DL, VT);
10054   case ISD::FMINNUM:
10055   case ISD::FMAXNUM: {
10056     // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
10057     const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
10058     APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) :
10059                         !Flags.hasNoInfs() ? APFloat::getInf(Semantics) :
10060                         APFloat::getLargest(Semantics);
10061     if (Opcode == ISD::FMAXNUM)
10062       NeutralAF.changeSign();
10063 
10064     return getConstantFP(NeutralAF, DL, VT);
10065   }
10066   }
10067 }
10068 
10069 #ifndef NDEBUG
10070 static void checkForCyclesHelper(const SDNode *N,
10071                                  SmallPtrSetImpl<const SDNode*> &Visited,
10072                                  SmallPtrSetImpl<const SDNode*> &Checked,
10073                                  const llvm::SelectionDAG *DAG) {
10074   // If this node has already been checked, don't check it again.
10075   if (Checked.count(N))
10076     return;
10077 
10078   // If a node has already been visited on this depth-first walk, reject it as
10079   // a cycle.
10080   if (!Visited.insert(N).second) {
10081     errs() << "Detected cycle in SelectionDAG\n";
10082     dbgs() << "Offending node:\n";
10083     N->dumprFull(DAG); dbgs() << "\n";
10084     abort();
10085   }
10086 
10087   for (const SDValue &Op : N->op_values())
10088     checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
10089 
10090   Checked.insert(N);
10091   Visited.erase(N);
10092 }
10093 #endif
10094 
10095 void llvm::checkForCycles(const llvm::SDNode *N,
10096                           const llvm::SelectionDAG *DAG,
10097                           bool force) {
10098 #ifndef NDEBUG
10099   bool check = force;
10100 #ifdef EXPENSIVE_CHECKS
10101   check = true;
10102 #endif  // EXPENSIVE_CHECKS
10103   if (check) {
10104     assert(N && "Checking nonexistent SDNode");
10105     SmallPtrSet<const SDNode*, 32> visited;
10106     SmallPtrSet<const SDNode*, 32> checked;
10107     checkForCyclesHelper(N, visited, checked, DAG);
10108   }
10109 #endif  // !NDEBUG
10110 }
10111 
10112 void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
10113   checkForCycles(DAG->getRoot().getNode(), DAG, force);
10114 }
10115