1 //===-- SystemZISelLowering.h - SystemZ DAG lowering interface --*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that SystemZ uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZISELLOWERING_H
15 #define LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZISELLOWERING_H
16 
17 #include "SystemZ.h"
18 #include "SystemZInstrInfo.h"
19 #include "llvm/CodeGen/MachineBasicBlock.h"
20 #include "llvm/CodeGen/SelectionDAG.h"
21 #include "llvm/CodeGen/TargetLowering.h"
22 
23 namespace llvm {
24 namespace SystemZISD {
25 enum NodeType : unsigned {
26   FIRST_NUMBER = ISD::BUILTIN_OP_END,
27 
28   // Return with a flag operand.  Operand 0 is the chain operand.
29   RET_FLAG,
30 
31   // Calls a function.  Operand 0 is the chain operand and operand 1
32   // is the target address.  The arguments start at operand 2.
33   // There is an optional glue operand at the end.
34   CALL,
35   SIBCALL,
36 
37   // TLS calls.  Like regular calls, except operand 1 is the TLS symbol.
38   // (The call target is implicitly __tls_get_offset.)
39   TLS_GDCALL,
40   TLS_LDCALL,
41 
42   // Wraps a TargetGlobalAddress that should be loaded using PC-relative
43   // accesses (LARL).  Operand 0 is the address.
44   PCREL_WRAPPER,
45 
46   // Used in cases where an offset is applied to a TargetGlobalAddress.
47   // Operand 0 is the full TargetGlobalAddress and operand 1 is a
48   // PCREL_WRAPPER for an anchor point.  This is used so that we can
49   // cheaply refer to either the full address or the anchor point
50   // as a register base.
51   PCREL_OFFSET,
52 
53   // Integer comparisons.  There are three operands: the two values
54   // to compare, and an integer of type SystemZICMP.
55   ICMP,
56 
57   // Floating-point comparisons.  The two operands are the values to compare.
58   FCMP,
59 
60   // Test under mask.  The first operand is ANDed with the second operand
61   // and the condition codes are set on the result.  The third operand is
62   // a boolean that is true if the condition codes need to distinguish
63   // between CCMASK_TM_MIXED_MSB_0 and CCMASK_TM_MIXED_MSB_1 (which the
64   // register forms do but the memory forms don't).
65   TM,
66 
67   // Branches if a condition is true.  Operand 0 is the chain operand;
68   // operand 1 is the 4-bit condition-code mask, with bit N in
69   // big-endian order meaning "branch if CC=N"; operand 2 is the
70   // target block and operand 3 is the flag operand.
71   BR_CCMASK,
72 
73   // Selects between operand 0 and operand 1.  Operand 2 is the
74   // mask of condition-code values for which operand 0 should be
75   // chosen over operand 1; it has the same form as BR_CCMASK.
76   // Operand 3 is the flag operand.
77   SELECT_CCMASK,
78 
79   // Evaluates to the gap between the stack pointer and the
80   // base of the dynamically-allocatable area.
81   ADJDYNALLOC,
82 
83   // For allocating stack space when using stack clash protector.
84   // Allocation is performed by block, and each block is probed.
85   PROBED_ALLOCA,
86 
87   // Count number of bits set in operand 0 per byte.
88   POPCNT,
89 
90   // Wrappers around the ISD opcodes of the same name.  The output is GR128.
91   // Input operands may be GR64 or GR32, depending on the instruction.
92   SMUL_LOHI,
93   UMUL_LOHI,
94   SDIVREM,
95   UDIVREM,
96 
97   // Add/subtract with overflow/carry.  These have the same operands as
98   // the corresponding standard operations, except with the carry flag
99   // replaced by a condition code value.
100   SADDO, SSUBO, UADDO, USUBO, ADDCARRY, SUBCARRY,
101 
102   // Set the condition code from a boolean value in operand 0.
103   // Operand 1 is a mask of all condition-code values that may result of this
104   // operation, operand 2 is a mask of condition-code values that may result
105   // if the boolean is true.
106   // Note that this operation is always optimized away, we will never
107   // generate any code for it.
108   GET_CCMASK,
109 
110   // Use a series of MVCs to copy bytes from one memory location to another.
111   // The operands are:
112   // - the target address
113   // - the source address
114   // - the constant length
115   //
116   // This isn't a memory opcode because we'd need to attach two
117   // MachineMemOperands rather than one.
118   MVC,
119 
120   // Similar to MVC, but for logic operations (AND, OR, XOR).
121   NC,
122   OC,
123   XC,
124 
125   // Use CLC to compare two blocks of memory, with the same comments
126   // as for MVC.
127   CLC,
128 
129   // Use MVC to set a block of memory after storing the first byte.
130   MEMSET_MVC,
131 
132   // Use an MVST-based sequence to implement stpcpy().
133   STPCPY,
134 
135   // Use a CLST-based sequence to implement strcmp().  The two input operands
136   // are the addresses of the strings to compare.
137   STRCMP,
138 
139   // Use an SRST-based sequence to search a block of memory.  The first
140   // operand is the end address, the second is the start, and the third
141   // is the character to search for.  CC is set to 1 on success and 2
142   // on failure.
143   SEARCH_STRING,
144 
145   // Store the CC value in bits 29 and 28 of an integer.
146   IPM,
147 
148   // Compiler barrier only; generate a no-op.
149   MEMBARRIER,
150 
151   // Transaction begin.  The first operand is the chain, the second
152   // the TDB pointer, and the third the immediate control field.
153   // Returns CC value and chain.
154   TBEGIN,
155   TBEGIN_NOFLOAT,
156 
157   // Transaction end.  Just the chain operand.  Returns CC value and chain.
158   TEND,
159 
160   // Create a vector constant by filling byte N of the result with bit
161   // 15-N of the single operand.
162   BYTE_MASK,
163 
164   // Create a vector constant by replicating an element-sized RISBG-style mask.
165   // The first operand specifies the starting set bit and the second operand
166   // specifies the ending set bit.  Both operands count from the MSB of the
167   // element.
168   ROTATE_MASK,
169 
170   // Replicate a GPR scalar value into all elements of a vector.
171   REPLICATE,
172 
173   // Create a vector from two i64 GPRs.
174   JOIN_DWORDS,
175 
176   // Replicate one element of a vector into all elements.  The first operand
177   // is the vector and the second is the index of the element to replicate.
178   SPLAT,
179 
180   // Interleave elements from the high half of operand 0 and the high half
181   // of operand 1.
182   MERGE_HIGH,
183 
184   // Likewise for the low halves.
185   MERGE_LOW,
186 
187   // Concatenate the vectors in the first two operands, shift them left
188   // by the third operand, and take the first half of the result.
189   SHL_DOUBLE,
190 
191   // Take one element of the first v2i64 operand and the one element of
192   // the second v2i64 operand and concatenate them to form a v2i64 result.
193   // The third operand is a 4-bit value of the form 0A0B, where A and B
194   // are the element selectors for the first operand and second operands
195   // respectively.
196   PERMUTE_DWORDS,
197 
198   // Perform a general vector permute on vector operands 0 and 1.
199   // Each byte of operand 2 controls the corresponding byte of the result,
200   // in the same way as a byte-level VECTOR_SHUFFLE mask.
201   PERMUTE,
202 
203   // Pack vector operands 0 and 1 into a single vector with half-sized elements.
204   PACK,
205 
206   // Likewise, but saturate the result and set CC.  PACKS_CC does signed
207   // saturation and PACKLS_CC does unsigned saturation.
208   PACKS_CC,
209   PACKLS_CC,
210 
211   // Unpack the first half of vector operand 0 into double-sized elements.
212   // UNPACK_HIGH sign-extends and UNPACKL_HIGH zero-extends.
213   UNPACK_HIGH,
214   UNPACKL_HIGH,
215 
216   // Likewise for the second half.
217   UNPACK_LOW,
218   UNPACKL_LOW,
219 
220   // Shift each element of vector operand 0 by the number of bits specified
221   // by scalar operand 1.
222   VSHL_BY_SCALAR,
223   VSRL_BY_SCALAR,
224   VSRA_BY_SCALAR,
225 
226   // For each element of the output type, sum across all sub-elements of
227   // operand 0 belonging to the corresponding element, and add in the
228   // rightmost sub-element of the corresponding element of operand 1.
229   VSUM,
230 
231   // Compare integer vector operands 0 and 1 to produce the usual 0/-1
232   // vector result.  VICMPE is for equality, VICMPH for "signed greater than"
233   // and VICMPHL for "unsigned greater than".
234   VICMPE,
235   VICMPH,
236   VICMPHL,
237 
238   // Likewise, but also set the condition codes on the result.
239   VICMPES,
240   VICMPHS,
241   VICMPHLS,
242 
243   // Compare floating-point vector operands 0 and 1 to produce the usual 0/-1
244   // vector result.  VFCMPE is for "ordered and equal", VFCMPH for "ordered and
245   // greater than" and VFCMPHE for "ordered and greater than or equal to".
246   VFCMPE,
247   VFCMPH,
248   VFCMPHE,
249 
250   // Likewise, but also set the condition codes on the result.
251   VFCMPES,
252   VFCMPHS,
253   VFCMPHES,
254 
255   // Test floating-point data class for vectors.
256   VFTCI,
257 
258   // Extend the even f32 elements of vector operand 0 to produce a vector
259   // of f64 elements.
260   VEXTEND,
261 
262   // Round the f64 elements of vector operand 0 to f32s and store them in the
263   // even elements of the result.
264   VROUND,
265 
266   // AND the two vector operands together and set CC based on the result.
267   VTM,
268 
269   // String operations that set CC as a side-effect.
270   VFAE_CC,
271   VFAEZ_CC,
272   VFEE_CC,
273   VFEEZ_CC,
274   VFENE_CC,
275   VFENEZ_CC,
276   VISTR_CC,
277   VSTRC_CC,
278   VSTRCZ_CC,
279   VSTRS_CC,
280   VSTRSZ_CC,
281 
282   // Test Data Class.
283   //
284   // Operand 0: the value to test
285   // Operand 1: the bit mask
286   TDC,
287 
288   // Strict variants of scalar floating-point comparisons.
289   // Quiet and signaling versions.
290   STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE,
291   STRICT_FCMPS,
292 
293   // Strict variants of vector floating-point comparisons.
294   // Quiet and signaling versions.
295   STRICT_VFCMPE,
296   STRICT_VFCMPH,
297   STRICT_VFCMPHE,
298   STRICT_VFCMPES,
299   STRICT_VFCMPHS,
300   STRICT_VFCMPHES,
301 
302   // Strict variants of VEXTEND and VROUND.
303   STRICT_VEXTEND,
304   STRICT_VROUND,
305 
306   // Wrappers around the inner loop of an 8- or 16-bit ATOMIC_SWAP or
307   // ATOMIC_LOAD_<op>.
308   //
309   // Operand 0: the address of the containing 32-bit-aligned field
310   // Operand 1: the second operand of <op>, in the high bits of an i32
311   //            for everything except ATOMIC_SWAPW
312   // Operand 2: how many bits to rotate the i32 left to bring the first
313   //            operand into the high bits
314   // Operand 3: the negative of operand 2, for rotating the other way
315   // Operand 4: the width of the field in bits (8 or 16)
316   ATOMIC_SWAPW = ISD::FIRST_TARGET_MEMORY_OPCODE,
317   ATOMIC_LOADW_ADD,
318   ATOMIC_LOADW_SUB,
319   ATOMIC_LOADW_AND,
320   ATOMIC_LOADW_OR,
321   ATOMIC_LOADW_XOR,
322   ATOMIC_LOADW_NAND,
323   ATOMIC_LOADW_MIN,
324   ATOMIC_LOADW_MAX,
325   ATOMIC_LOADW_UMIN,
326   ATOMIC_LOADW_UMAX,
327 
328   // A wrapper around the inner loop of an ATOMIC_CMP_SWAP.
329   //
330   // Operand 0: the address of the containing 32-bit-aligned field
331   // Operand 1: the compare value, in the low bits of an i32
332   // Operand 2: the swap value, in the low bits of an i32
333   // Operand 3: how many bits to rotate the i32 left to bring the first
334   //            operand into the high bits
335   // Operand 4: the negative of operand 2, for rotating the other way
336   // Operand 5: the width of the field in bits (8 or 16)
337   ATOMIC_CMP_SWAPW,
338 
339   // Atomic compare-and-swap returning CC value.
340   // Val, CC, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
341   ATOMIC_CMP_SWAP,
342 
343   // 128-bit atomic load.
344   // Val, OUTCHAIN = ATOMIC_LOAD_128(INCHAIN, ptr)
345   ATOMIC_LOAD_128,
346 
347   // 128-bit atomic store.
348   // OUTCHAIN = ATOMIC_STORE_128(INCHAIN, val, ptr)
349   ATOMIC_STORE_128,
350 
351   // 128-bit atomic compare-and-swap.
352   // Val, CC, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
353   ATOMIC_CMP_SWAP_128,
354 
355   // Byte swapping load/store.  Same operands as regular load/store.
356   LRV, STRV,
357 
358   // Element swapping load/store.  Same operands as regular load/store.
359   VLER, VSTER,
360 
361   // Prefetch from the second operand using the 4-bit control code in
362   // the first operand.  The code is 1 for a load prefetch and 2 for
363   // a store prefetch.
364   PREFETCH
365 };
366 
367 // Return true if OPCODE is some kind of PC-relative address.
isPCREL(unsigned Opcode)368 inline bool isPCREL(unsigned Opcode) {
369   return Opcode == PCREL_WRAPPER || Opcode == PCREL_OFFSET;
370 }
371 } // end namespace SystemZISD
372 
373 namespace SystemZICMP {
374 // Describes whether an integer comparison needs to be signed or unsigned,
375 // or whether either type is OK.
376 enum {
377   Any,
378   UnsignedOnly,
379   SignedOnly
380 };
381 } // end namespace SystemZICMP
382 
383 class SystemZSubtarget;
384 
385 class SystemZTargetLowering : public TargetLowering {
386 public:
387   explicit SystemZTargetLowering(const TargetMachine &TM,
388                                  const SystemZSubtarget &STI);
389 
390   bool useSoftFloat() const override;
391 
392   // Override TargetLowering.
getScalarShiftAmountTy(const DataLayout &,EVT)393   MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
394     return MVT::i32;
395   }
getVectorIdxTy(const DataLayout & DL)396   MVT getVectorIdxTy(const DataLayout &DL) const override {
397     // Only the lower 12 bits of an element index are used, so we don't
398     // want to clobber the upper 32 bits of a GPR unnecessarily.
399     return MVT::i32;
400   }
getPreferredVectorAction(MVT VT)401   TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT)
402     const override {
403     // Widen subvectors to the full width rather than promoting integer
404     // elements.  This is better because:
405     //
406     // (a) it means that we can handle the ABI for passing and returning
407     //     sub-128 vectors without having to handle them as legal types.
408     //
409     // (b) we don't have instructions to extend on load and truncate on store,
410     //     so promoting the integers is less efficient.
411     //
412     // (c) there are no multiplication instructions for the widest integer
413     //     type (v2i64).
414     if (VT.getScalarSizeInBits() % 8 == 0)
415       return TypeWidenVector;
416     return TargetLoweringBase::getPreferredVectorAction(VT);
417   }
418   unsigned
getNumRegisters(LLVMContext & Context,EVT VT,Optional<MVT> RegisterVT)419   getNumRegisters(LLVMContext &Context, EVT VT,
420                   Optional<MVT> RegisterVT) const override {
421     // i128 inline assembly operand.
422     if (VT == MVT::i128 && RegisterVT && *RegisterVT == MVT::Untyped)
423       return 1;
424     return TargetLowering::getNumRegisters(Context, VT);
425   }
isCheapToSpeculateCtlz()426   bool isCheapToSpeculateCtlz() const override { return true; }
preferZeroCompareBranch()427   bool preferZeroCompareBranch() const override { return true; }
hasBitPreservingFPLogic(EVT VT)428   bool hasBitPreservingFPLogic(EVT VT) const override {
429     EVT ScVT = VT.getScalarType();
430     return ScVT == MVT::f32 || ScVT == MVT::f64 || ScVT == MVT::f128;
431   }
isMaskAndCmp0FoldingBeneficial(const Instruction & AndI)432   bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override {
433     ConstantInt* Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
434     return Mask && Mask->getValue().isIntN(16);
435   }
convertSetCCLogicToBitwiseLogic(EVT VT)436   bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
437     return VT.isScalarInteger();
438   }
439   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &,
440                          EVT) const override;
441   bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
442                                   EVT VT) const override;
443   bool isFPImmLegal(const APFloat &Imm, EVT VT,
444                     bool ForCodeSize) const override;
ShouldShrinkFPConstant(EVT VT)445   bool ShouldShrinkFPConstant(EVT VT) const override {
446     // Do not shrink 64-bit FP constpool entries since LDEB is slower than
447     // LD, and having the full constant in memory enables reg/mem opcodes.
448     return VT != MVT::f64;
449   }
450   bool hasInlineStackProbe(MachineFunction &MF) const override;
451   bool isLegalICmpImmediate(int64_t Imm) const override;
452   bool isLegalAddImmediate(int64_t Imm) const override;
453   bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
454                              unsigned AS,
455                              Instruction *I = nullptr) const override;
456   bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment,
457                                       MachineMemOperand::Flags Flags,
458                                       bool *Fast) const override;
459   bool
460   findOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned Limit,
461                            const MemOp &Op, unsigned DstAS, unsigned SrcAS,
462                            const AttributeList &FuncAttributes) const override;
463   EVT getOptimalMemOpType(const MemOp &Op,
464                           const AttributeList &FuncAttributes) const override;
465   bool isTruncateFree(Type *, Type *) const override;
isTruncateFree(EVT,EVT)466   bool isTruncateFree(EVT, EVT) const override;
467 
468   bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
469                             bool MathUsed) const override {
470     // Form add and sub with overflow intrinsics regardless of any extra
471     // users of the math result.
472     return VT == MVT::i32 || VT == MVT::i64;
473   }
474 
shouldConsiderGEPOffsetSplit()475   bool shouldConsiderGEPOffsetSplit() const override { return true; }
476 
477   const char *getTargetNodeName(unsigned Opcode) const override;
478   std::pair<unsigned, const TargetRegisterClass *>
479   getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
480                                StringRef Constraint, MVT VT) const override;
481   TargetLowering::ConstraintType
482   getConstraintType(StringRef Constraint) const override;
483   TargetLowering::ConstraintWeight
484     getSingleConstraintMatchWeight(AsmOperandInfo &info,
485                                    const char *constraint) const override;
486   void LowerAsmOperandForConstraint(SDValue Op,
487                                     std::string &Constraint,
488                                     std::vector<SDValue> &Ops,
489                                     SelectionDAG &DAG) const override;
490 
getInlineAsmMemConstraint(StringRef ConstraintCode)491   unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
492     if (ConstraintCode.size() == 1) {
493       switch(ConstraintCode[0]) {
494       default:
495         break;
496       case 'o':
497         return InlineAsm::Constraint_o;
498       case 'Q':
499         return InlineAsm::Constraint_Q;
500       case 'R':
501         return InlineAsm::Constraint_R;
502       case 'S':
503         return InlineAsm::Constraint_S;
504       case 'T':
505         return InlineAsm::Constraint_T;
506       }
507     } else if (ConstraintCode.size() == 2 && ConstraintCode[0] == 'Z') {
508       switch (ConstraintCode[1]) {
509       default:
510         break;
511       case 'Q':
512         return InlineAsm::Constraint_ZQ;
513       case 'R':
514         return InlineAsm::Constraint_ZR;
515       case 'S':
516         return InlineAsm::Constraint_ZS;
517       case 'T':
518         return InlineAsm::Constraint_ZT;
519       }
520     }
521     return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
522   }
523 
524   Register getRegisterByName(const char *RegName, LLT VT,
525                              const MachineFunction &MF) const override;
526 
527   /// If a physical register, this returns the register that receives the
528   /// exception address on entry to an EH pad.
529   Register
getExceptionPointerRegister(const Constant * PersonalityFn)530   getExceptionPointerRegister(const Constant *PersonalityFn) const override {
531     return SystemZ::R6D;
532   }
533 
534   /// If a physical register, this returns the register that receives the
535   /// exception typeid on entry to a landing pad.
536   Register
getExceptionSelectorRegister(const Constant * PersonalityFn)537   getExceptionSelectorRegister(const Constant *PersonalityFn) const override {
538     return SystemZ::R7D;
539   }
540 
541   /// Override to support customized stack guard loading.
useLoadStackGuardNode()542   bool useLoadStackGuardNode() const override {
543     return true;
544   }
insertSSPDeclarations(Module & M)545   void insertSSPDeclarations(Module &M) const override {
546   }
547 
548   MachineBasicBlock *
549   EmitInstrWithCustomInserter(MachineInstr &MI,
550                               MachineBasicBlock *BB) const override;
551   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
552   void LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results,
553                              SelectionDAG &DAG) const override;
554   void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
555                           SelectionDAG &DAG) const override;
556   const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
557   bool allowTruncateForTailCall(Type *, Type *) const override;
558   bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
559   bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL,
560                                    SDValue Val, SDValue *Parts,
561                                    unsigned NumParts, MVT PartVT,
562                                    Optional<CallingConv::ID> CC) const override;
563   SDValue
564   joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL,
565                              const SDValue *Parts, unsigned NumParts,
566                              MVT PartVT, EVT ValueVT,
567                              Optional<CallingConv::ID> CC) const override;
568   SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
569                                bool isVarArg,
570                                const SmallVectorImpl<ISD::InputArg> &Ins,
571                                const SDLoc &DL, SelectionDAG &DAG,
572                                SmallVectorImpl<SDValue> &InVals) const override;
573   SDValue LowerCall(CallLoweringInfo &CLI,
574                     SmallVectorImpl<SDValue> &InVals) const override;
575 
576   std::pair<SDValue, SDValue>
577   makeExternalCall(SDValue Chain, SelectionDAG &DAG, const char *CalleeName,
578                    EVT RetVT, ArrayRef<SDValue> Ops, CallingConv::ID CallConv,
579                    bool IsSigned, SDLoc DL, bool DoesNotReturn,
580                    bool IsReturnValueUsed) const;
581 
582   bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
583                       bool isVarArg,
584                       const SmallVectorImpl<ISD::OutputArg> &Outs,
585                       LLVMContext &Context) const override;
586   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
587                       const SmallVectorImpl<ISD::OutputArg> &Outs,
588                       const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
589                       SelectionDAG &DAG) const override;
590   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
591 
592   /// Determine which of the bits specified in Mask are known to be either
593   /// zero or one and return them in the KnownZero/KnownOne bitsets.
594   void computeKnownBitsForTargetNode(const SDValue Op,
595                                      KnownBits &Known,
596                                      const APInt &DemandedElts,
597                                      const SelectionDAG &DAG,
598                                      unsigned Depth = 0) const override;
599 
600   /// Determine the number of bits in the operation that are sign bits.
601   unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
602                                            const APInt &DemandedElts,
603                                            const SelectionDAG &DAG,
604                                            unsigned Depth) const override;
605 
getExtendForAtomicOps()606   ISD::NodeType getExtendForAtomicOps() const override {
607     return ISD::ANY_EXTEND;
608   }
getExtendForAtomicCmpSwapArg()609   ISD::NodeType getExtendForAtomicCmpSwapArg() const override {
610     return ISD::ZERO_EXTEND;
611   }
612 
supportSwiftError()613   bool supportSwiftError() const override {
614     return true;
615   }
616 
617   unsigned getStackProbeSize(MachineFunction &MF) const;
618 
619 private:
620   const SystemZSubtarget &Subtarget;
621 
622   // Implement LowerOperation for individual opcodes.
623   SDValue getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
624                        const SDLoc &DL, EVT VT,
625                        SDValue CmpOp0, SDValue CmpOp1, SDValue Chain) const;
626   SDValue lowerVectorSETCC(SelectionDAG &DAG, const SDLoc &DL,
627                            EVT VT, ISD::CondCode CC,
628                            SDValue CmpOp0, SDValue CmpOp1,
629                            SDValue Chain = SDValue(),
630                            bool IsSignaling = false) const;
631   SDValue lowerSETCC(SDValue Op, SelectionDAG &DAG) const;
632   SDValue lowerSTRICT_FSETCC(SDValue Op, SelectionDAG &DAG,
633                              bool IsSignaling) const;
634   SDValue lowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
635   SDValue lowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
636   SDValue lowerGlobalAddress(GlobalAddressSDNode *Node,
637                              SelectionDAG &DAG) const;
638   SDValue lowerTLSGetOffset(GlobalAddressSDNode *Node,
639                             SelectionDAG &DAG, unsigned Opcode,
640                             SDValue GOTOffset) const;
641   SDValue lowerThreadPointer(const SDLoc &DL, SelectionDAG &DAG) const;
642   SDValue lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
643                                 SelectionDAG &DAG) const;
644   SDValue lowerBlockAddress(BlockAddressSDNode *Node,
645                             SelectionDAG &DAG) const;
646   SDValue lowerJumpTable(JumpTableSDNode *JT, SelectionDAG &DAG) const;
647   SDValue lowerConstantPool(ConstantPoolSDNode *CP, SelectionDAG &DAG) const;
648   SDValue lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
649   SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
650   SDValue lowerVASTART(SDValue Op, SelectionDAG &DAG) const;
651   SDValue lowerVASTART_ELF(SDValue Op, SelectionDAG &DAG) const;
652   SDValue lowerVASTART_XPLINK(SDValue Op, SelectionDAG &DAG) const;
653   SDValue lowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
654   SDValue lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
655   SDValue lowerDYNAMIC_STACKALLOC_ELF(SDValue Op, SelectionDAG &DAG) const;
656   SDValue lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op, SelectionDAG &DAG) const;
657   SDValue lowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, SelectionDAG &DAG) const;
658   SDValue lowerSMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
659   SDValue lowerUMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
660   SDValue lowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
661   SDValue lowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
662   SDValue lowerXALUO(SDValue Op, SelectionDAG &DAG) const;
663   SDValue lowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) const;
664   SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
665   SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
666   SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
667   SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
668   SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
669   SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
670   SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
671                               unsigned Opcode) const;
672   SDValue lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
673   SDValue lowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
674   SDValue lowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const;
675   SDValue lowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const;
676   SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
677   SDValue lowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
678   SDValue lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
679   bool isVectorElementLoad(SDValue Op) const;
680   SDValue buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
681                       SmallVectorImpl<SDValue> &Elems) const;
682   SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
683   SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
684   SDValue lowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
685   SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
686   SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
687   SDValue lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
688   SDValue lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
689   SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const;
690   SDValue lowerIS_FPCLASS(SDValue Op, SelectionDAG &DAG) const;
691 
692   bool canTreatAsByteVector(EVT VT) const;
693   SDValue combineExtract(const SDLoc &DL, EVT ElemVT, EVT VecVT, SDValue OrigOp,
694                          unsigned Index, DAGCombinerInfo &DCI,
695                          bool Force) const;
696   SDValue combineTruncateExtract(const SDLoc &DL, EVT TruncVT, SDValue Op,
697                                  DAGCombinerInfo &DCI) const;
698   SDValue combineZERO_EXTEND(SDNode *N, DAGCombinerInfo &DCI) const;
699   SDValue combineSIGN_EXTEND(SDNode *N, DAGCombinerInfo &DCI) const;
700   SDValue combineSIGN_EXTEND_INREG(SDNode *N, DAGCombinerInfo &DCI) const;
701   SDValue combineMERGE(SDNode *N, DAGCombinerInfo &DCI) const;
702   bool canLoadStoreByteSwapped(EVT VT) const;
703   SDValue combineLOAD(SDNode *N, DAGCombinerInfo &DCI) const;
704   SDValue combineSTORE(SDNode *N, DAGCombinerInfo &DCI) const;
705   SDValue combineVECTOR_SHUFFLE(SDNode *N, DAGCombinerInfo &DCI) const;
706   SDValue combineEXTRACT_VECTOR_ELT(SDNode *N, DAGCombinerInfo &DCI) const;
707   SDValue combineJOIN_DWORDS(SDNode *N, DAGCombinerInfo &DCI) const;
708   SDValue combineFP_ROUND(SDNode *N, DAGCombinerInfo &DCI) const;
709   SDValue combineFP_EXTEND(SDNode *N, DAGCombinerInfo &DCI) const;
710   SDValue combineINT_TO_FP(SDNode *N, DAGCombinerInfo &DCI) const;
711   SDValue combineBSWAP(SDNode *N, DAGCombinerInfo &DCI) const;
712   SDValue combineBR_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
713   SDValue combineSELECT_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
714   SDValue combineGET_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
715   SDValue combineIntDIVREM(SDNode *N, DAGCombinerInfo &DCI) const;
716   SDValue combineINTRINSIC(SDNode *N, DAGCombinerInfo &DCI) const;
717 
718   SDValue unwrapAddress(SDValue N) const override;
719 
720   // If the last instruction before MBBI in MBB was some form of COMPARE,
721   // try to replace it with a COMPARE AND BRANCH just before MBBI.
722   // CCMask and Target are the BRC-like operands for the branch.
723   // Return true if the change was made.
724   bool convertPrevCompareToBranch(MachineBasicBlock *MBB,
725                                   MachineBasicBlock::iterator MBBI,
726                                   unsigned CCMask,
727                                   MachineBasicBlock *Target) const;
728 
729   // Implement EmitInstrWithCustomInserter for individual operation types.
730   MachineBasicBlock *emitSelect(MachineInstr &MI, MachineBasicBlock *BB) const;
731   MachineBasicBlock *emitCondStore(MachineInstr &MI, MachineBasicBlock *BB,
732                                    unsigned StoreOpcode, unsigned STOCOpcode,
733                                    bool Invert) const;
734   MachineBasicBlock *emitPair128(MachineInstr &MI,
735                                  MachineBasicBlock *MBB) const;
736   MachineBasicBlock *emitExt128(MachineInstr &MI, MachineBasicBlock *MBB,
737                                 bool ClearEven) const;
738   MachineBasicBlock *emitAtomicLoadBinary(MachineInstr &MI,
739                                           MachineBasicBlock *BB,
740                                           unsigned BinOpcode, unsigned BitSize,
741                                           bool Invert = false) const;
742   MachineBasicBlock *emitAtomicLoadMinMax(MachineInstr &MI,
743                                           MachineBasicBlock *MBB,
744                                           unsigned CompareOpcode,
745                                           unsigned KeepOldMask,
746                                           unsigned BitSize) const;
747   MachineBasicBlock *emitAtomicCmpSwapW(MachineInstr &MI,
748                                         MachineBasicBlock *BB) const;
749   MachineBasicBlock *emitMemMemWrapper(MachineInstr &MI, MachineBasicBlock *BB,
750                                        unsigned Opcode,
751                                        bool IsMemset = false) const;
752   MachineBasicBlock *emitStringWrapper(MachineInstr &MI, MachineBasicBlock *BB,
753                                        unsigned Opcode) const;
754   MachineBasicBlock *emitTransactionBegin(MachineInstr &MI,
755                                           MachineBasicBlock *MBB,
756                                           unsigned Opcode, bool NoFloat) const;
757   MachineBasicBlock *emitLoadAndTestCmp0(MachineInstr &MI,
758                                          MachineBasicBlock *MBB,
759                                          unsigned Opcode) const;
760   MachineBasicBlock *emitProbedAlloca(MachineInstr &MI,
761                                       MachineBasicBlock *MBB) const;
762 
763   SDValue getBackchainAddress(SDValue SP, SelectionDAG &DAG) const;
764 
765   MachineMemOperand::Flags
766   getTargetMMOFlags(const Instruction &I) const override;
767   const TargetRegisterClass *getRepRegClassFor(MVT VT) const override;
768 };
769 
770 struct SystemZVectorConstantInfo {
771 private:
772   APInt IntBits;             // The 128 bits as an integer.
773   APInt SplatBits;           // Smallest splat value.
774   APInt SplatUndef;          // Bits correspoding to undef operands of the BVN.
775   unsigned SplatBitSize = 0;
776   bool isFP128 = false;
777 public:
778   unsigned Opcode = 0;
779   SmallVector<unsigned, 2> OpVals;
780   MVT VecVT;
781   SystemZVectorConstantInfo(APInt IntImm);
SystemZVectorConstantInfoSystemZVectorConstantInfo782   SystemZVectorConstantInfo(APFloat FPImm)
783       : SystemZVectorConstantInfo(FPImm.bitcastToAPInt()) {
784     isFP128 = (&FPImm.getSemantics() == &APFloat::IEEEquad());
785   }
786   SystemZVectorConstantInfo(BuildVectorSDNode *BVN);
787   bool isVectorConstantLegal(const SystemZSubtarget &Subtarget);
788 };
789 
790 } // end namespace llvm
791 
792 #endif
793