1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21
22 #include "LegalizeTypes.h"
23 #include "llvm/ADT/SmallBitVector.h"
24 #include "llvm/Analysis/MemoryLocation.h"
25 #include "llvm/Analysis/VectorUtils.h"
26 #include "llvm/IR/DataLayout.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/TypeSize.h"
29 #include "llvm/Support/raw_ostream.h"
30 using namespace llvm;
31
32 #define DEBUG_TYPE "legalize-types"
33
34 //===----------------------------------------------------------------------===//
35 // Result Vector Scalarization: <1 x ty> -> ty.
36 //===----------------------------------------------------------------------===//
37
ScalarizeVectorResult(SDNode * N,unsigned ResNo)38 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
39 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
40 dbgs() << "\n");
41 SDValue R = SDValue();
42
43 switch (N->getOpcode()) {
44 default:
45 #ifndef NDEBUG
46 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
47 N->dump(&DAG);
48 dbgs() << "\n";
49 #endif
50 report_fatal_error("Do not know how to scalarize the result of this "
51 "operator!\n");
52
53 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
54 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
55 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
56 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
57 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
58 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
59 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
60 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
61 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
62 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
63 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
64 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
65 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
66 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
67 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
68 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
69 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
70 case ISD::ANY_EXTEND_VECTOR_INREG:
71 case ISD::SIGN_EXTEND_VECTOR_INREG:
72 case ISD::ZERO_EXTEND_VECTOR_INREG:
73 R = ScalarizeVecRes_VecInregOp(N);
74 break;
75 case ISD::ABS:
76 case ISD::ANY_EXTEND:
77 case ISD::BITREVERSE:
78 case ISD::BSWAP:
79 case ISD::CTLZ:
80 case ISD::CTLZ_ZERO_UNDEF:
81 case ISD::CTPOP:
82 case ISD::CTTZ:
83 case ISD::CTTZ_ZERO_UNDEF:
84 case ISD::FABS:
85 case ISD::FCEIL:
86 case ISD::FCOS:
87 case ISD::FEXP:
88 case ISD::FEXP2:
89 case ISD::FFLOOR:
90 case ISD::FLOG:
91 case ISD::FLOG10:
92 case ISD::FLOG2:
93 case ISD::FNEARBYINT:
94 case ISD::FNEG:
95 case ISD::FREEZE:
96 case ISD::ARITH_FENCE:
97 case ISD::FP_EXTEND:
98 case ISD::FP_TO_SINT:
99 case ISD::FP_TO_UINT:
100 case ISD::FRINT:
101 case ISD::FROUND:
102 case ISD::FROUNDEVEN:
103 case ISD::FSIN:
104 case ISD::FSQRT:
105 case ISD::FTRUNC:
106 case ISD::SIGN_EXTEND:
107 case ISD::SINT_TO_FP:
108 case ISD::TRUNCATE:
109 case ISD::UINT_TO_FP:
110 case ISD::ZERO_EXTEND:
111 case ISD::FCANONICALIZE:
112 R = ScalarizeVecRes_UnaryOp(N);
113 break;
114
115 case ISD::ADD:
116 case ISD::AND:
117 case ISD::FADD:
118 case ISD::FCOPYSIGN:
119 case ISD::FDIV:
120 case ISD::FMUL:
121 case ISD::FMINNUM:
122 case ISD::FMAXNUM:
123 case ISD::FMINNUM_IEEE:
124 case ISD::FMAXNUM_IEEE:
125 case ISD::FMINIMUM:
126 case ISD::FMAXIMUM:
127 case ISD::SMIN:
128 case ISD::SMAX:
129 case ISD::UMIN:
130 case ISD::UMAX:
131
132 case ISD::SADDSAT:
133 case ISD::UADDSAT:
134 case ISD::SSUBSAT:
135 case ISD::USUBSAT:
136 case ISD::SSHLSAT:
137 case ISD::USHLSAT:
138
139 case ISD::FPOW:
140 case ISD::FREM:
141 case ISD::FSUB:
142 case ISD::MUL:
143 case ISD::OR:
144 case ISD::SDIV:
145 case ISD::SREM:
146 case ISD::SUB:
147 case ISD::UDIV:
148 case ISD::UREM:
149 case ISD::XOR:
150 case ISD::SHL:
151 case ISD::SRA:
152 case ISD::SRL:
153 case ISD::ROTL:
154 case ISD::ROTR:
155 R = ScalarizeVecRes_BinOp(N);
156 break;
157 case ISD::FMA:
158 case ISD::FSHL:
159 case ISD::FSHR:
160 R = ScalarizeVecRes_TernaryOp(N);
161 break;
162
163 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
164 case ISD::STRICT_##DAGN:
165 #include "llvm/IR/ConstrainedOps.def"
166 R = ScalarizeVecRes_StrictFPOp(N);
167 break;
168
169 case ISD::FP_TO_UINT_SAT:
170 case ISD::FP_TO_SINT_SAT:
171 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
172 break;
173
174 case ISD::UADDO:
175 case ISD::SADDO:
176 case ISD::USUBO:
177 case ISD::SSUBO:
178 case ISD::UMULO:
179 case ISD::SMULO:
180 R = ScalarizeVecRes_OverflowOp(N, ResNo);
181 break;
182 case ISD::SMULFIX:
183 case ISD::SMULFIXSAT:
184 case ISD::UMULFIX:
185 case ISD::UMULFIXSAT:
186 case ISD::SDIVFIX:
187 case ISD::SDIVFIXSAT:
188 case ISD::UDIVFIX:
189 case ISD::UDIVFIXSAT:
190 R = ScalarizeVecRes_FIX(N);
191 break;
192 }
193
194 // If R is null, the sub-method took care of registering the result.
195 if (R.getNode())
196 SetScalarizedVector(SDValue(N, ResNo), R);
197 }
198
ScalarizeVecRes_BinOp(SDNode * N)199 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
200 SDValue LHS = GetScalarizedVector(N->getOperand(0));
201 SDValue RHS = GetScalarizedVector(N->getOperand(1));
202 return DAG.getNode(N->getOpcode(), SDLoc(N),
203 LHS.getValueType(), LHS, RHS, N->getFlags());
204 }
205
ScalarizeVecRes_TernaryOp(SDNode * N)206 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
207 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
208 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
209 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
210 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
211 Op2, N->getFlags());
212 }
213
ScalarizeVecRes_FIX(SDNode * N)214 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
215 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
216 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
217 SDValue Op2 = N->getOperand(2);
218 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
219 Op2, N->getFlags());
220 }
221
ScalarizeVecRes_StrictFPOp(SDNode * N)222 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
223 EVT VT = N->getValueType(0).getVectorElementType();
224 unsigned NumOpers = N->getNumOperands();
225 SDValue Chain = N->getOperand(0);
226 EVT ValueVTs[] = {VT, MVT::Other};
227 SDLoc dl(N);
228
229 SmallVector<SDValue, 4> Opers(NumOpers);
230
231 // The Chain is the first operand.
232 Opers[0] = Chain;
233
234 // Now process the remaining operands.
235 for (unsigned i = 1; i < NumOpers; ++i) {
236 SDValue Oper = N->getOperand(i);
237 EVT OperVT = Oper.getValueType();
238
239 if (OperVT.isVector()) {
240 if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector)
241 Oper = GetScalarizedVector(Oper);
242 else
243 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
244 OperVT.getVectorElementType(), Oper,
245 DAG.getVectorIdxConstant(0, dl));
246 }
247
248 Opers[i] = Oper;
249 }
250
251 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
252 Opers, N->getFlags());
253
254 // Legalize the chain result - switch anything that used the old chain to
255 // use the new one.
256 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
257 return Result;
258 }
259
ScalarizeVecRes_OverflowOp(SDNode * N,unsigned ResNo)260 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
261 unsigned ResNo) {
262 SDLoc DL(N);
263 EVT ResVT = N->getValueType(0);
264 EVT OvVT = N->getValueType(1);
265
266 SDValue ScalarLHS, ScalarRHS;
267 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
268 ScalarLHS = GetScalarizedVector(N->getOperand(0));
269 ScalarRHS = GetScalarizedVector(N->getOperand(1));
270 } else {
271 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
272 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
273 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
274 ScalarLHS = ElemsLHS[0];
275 ScalarRHS = ElemsRHS[0];
276 }
277
278 SDVTList ScalarVTs = DAG.getVTList(
279 ResVT.getVectorElementType(), OvVT.getVectorElementType());
280 SDNode *ScalarNode = DAG.getNode(
281 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
282 ScalarNode->setFlags(N->getFlags());
283
284 // Replace the other vector result not being explicitly scalarized here.
285 unsigned OtherNo = 1 - ResNo;
286 EVT OtherVT = N->getValueType(OtherNo);
287 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
288 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
289 } else {
290 SDValue OtherVal = DAG.getNode(
291 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
292 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
293 }
294
295 return SDValue(ScalarNode, ResNo);
296 }
297
ScalarizeVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)298 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
299 unsigned ResNo) {
300 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
301 return GetScalarizedVector(Op);
302 }
303
ScalarizeVecRes_BITCAST(SDNode * N)304 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
305 SDValue Op = N->getOperand(0);
306 if (Op.getValueType().isVector()
307 && Op.getValueType().getVectorNumElements() == 1
308 && !isSimpleLegalType(Op.getValueType()))
309 Op = GetScalarizedVector(Op);
310 EVT NewVT = N->getValueType(0).getVectorElementType();
311 return DAG.getNode(ISD::BITCAST, SDLoc(N),
312 NewVT, Op);
313 }
314
ScalarizeVecRes_BUILD_VECTOR(SDNode * N)315 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
316 EVT EltVT = N->getValueType(0).getVectorElementType();
317 SDValue InOp = N->getOperand(0);
318 // The BUILD_VECTOR operands may be of wider element types and
319 // we may need to truncate them back to the requested return type.
320 if (EltVT.isInteger())
321 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
322 return InOp;
323 }
324
ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode * N)325 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
326 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
327 N->getValueType(0).getVectorElementType(),
328 N->getOperand(0), N->getOperand(1));
329 }
330
ScalarizeVecRes_FP_ROUND(SDNode * N)331 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
332 SDLoc DL(N);
333 SDValue Op = N->getOperand(0);
334 EVT OpVT = Op.getValueType();
335 // The result needs scalarizing, but it's not a given that the source does.
336 // See similar logic in ScalarizeVecRes_UnaryOp.
337 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
338 Op = GetScalarizedVector(Op);
339 } else {
340 EVT VT = OpVT.getVectorElementType();
341 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
342 DAG.getVectorIdxConstant(0, DL));
343 }
344 return DAG.getNode(ISD::FP_ROUND, DL,
345 N->getValueType(0).getVectorElementType(), Op,
346 N->getOperand(1));
347 }
348
ScalarizeVecRes_FPOWI(SDNode * N)349 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
350 SDValue Op = GetScalarizedVector(N->getOperand(0));
351 return DAG.getNode(ISD::FPOWI, SDLoc(N),
352 Op.getValueType(), Op, N->getOperand(1));
353 }
354
ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode * N)355 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
356 // The value to insert may have a wider type than the vector element type,
357 // so be sure to truncate it to the element type if necessary.
358 SDValue Op = N->getOperand(1);
359 EVT EltVT = N->getValueType(0).getVectorElementType();
360 if (Op.getValueType() != EltVT)
361 // FIXME: Can this happen for floating point types?
362 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
363 return Op;
364 }
365
ScalarizeVecRes_LOAD(LoadSDNode * N)366 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
367 assert(N->isUnindexed() && "Indexed vector load?");
368
369 SDValue Result = DAG.getLoad(
370 ISD::UNINDEXED, N->getExtensionType(),
371 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
372 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
373 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
374 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
375
376 // Legalize the chain result - switch anything that used the old chain to
377 // use the new one.
378 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
379 return Result;
380 }
381
ScalarizeVecRes_UnaryOp(SDNode * N)382 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
383 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
384 EVT DestVT = N->getValueType(0).getVectorElementType();
385 SDValue Op = N->getOperand(0);
386 EVT OpVT = Op.getValueType();
387 SDLoc DL(N);
388 // The result needs scalarizing, but it's not a given that the source does.
389 // This is a workaround for targets where it's impossible to scalarize the
390 // result of a conversion, because the source type is legal.
391 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
392 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
393 // legal and was not scalarized.
394 // See the similar logic in ScalarizeVecRes_SETCC
395 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
396 Op = GetScalarizedVector(Op);
397 } else {
398 EVT VT = OpVT.getVectorElementType();
399 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
400 DAG.getVectorIdxConstant(0, DL));
401 }
402 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
403 }
404
ScalarizeVecRes_InregOp(SDNode * N)405 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
406 EVT EltVT = N->getValueType(0).getVectorElementType();
407 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
408 SDValue LHS = GetScalarizedVector(N->getOperand(0));
409 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
410 LHS, DAG.getValueType(ExtVT));
411 }
412
ScalarizeVecRes_VecInregOp(SDNode * N)413 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
414 SDLoc DL(N);
415 SDValue Op = N->getOperand(0);
416
417 EVT OpVT = Op.getValueType();
418 EVT OpEltVT = OpVT.getVectorElementType();
419 EVT EltVT = N->getValueType(0).getVectorElementType();
420
421 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
422 Op = GetScalarizedVector(Op);
423 } else {
424 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
425 DAG.getVectorIdxConstant(0, DL));
426 }
427
428 switch (N->getOpcode()) {
429 case ISD::ANY_EXTEND_VECTOR_INREG:
430 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
431 case ISD::SIGN_EXTEND_VECTOR_INREG:
432 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
433 case ISD::ZERO_EXTEND_VECTOR_INREG:
434 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
435 }
436
437 llvm_unreachable("Illegal extend_vector_inreg opcode");
438 }
439
ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode * N)440 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
441 // If the operand is wider than the vector element type then it is implicitly
442 // truncated. Make that explicit here.
443 EVT EltVT = N->getValueType(0).getVectorElementType();
444 SDValue InOp = N->getOperand(0);
445 if (InOp.getValueType() != EltVT)
446 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
447 return InOp;
448 }
449
ScalarizeVecRes_VSELECT(SDNode * N)450 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
451 SDValue Cond = N->getOperand(0);
452 EVT OpVT = Cond.getValueType();
453 SDLoc DL(N);
454 // The vselect result and true/value operands needs scalarizing, but it's
455 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
456 // See the similar logic in ScalarizeVecRes_SETCC
457 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
458 Cond = GetScalarizedVector(Cond);
459 } else {
460 EVT VT = OpVT.getVectorElementType();
461 Cond = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
462 DAG.getVectorIdxConstant(0, DL));
463 }
464
465 SDValue LHS = GetScalarizedVector(N->getOperand(1));
466 TargetLowering::BooleanContent ScalarBool =
467 TLI.getBooleanContents(false, false);
468 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
469
470 // If integer and float booleans have different contents then we can't
471 // reliably optimize in all cases. There is a full explanation for this in
472 // DAGCombiner::visitSELECT() where the same issue affects folding
473 // (select C, 0, 1) to (xor C, 1).
474 if (TLI.getBooleanContents(false, false) !=
475 TLI.getBooleanContents(false, true)) {
476 // At least try the common case where the boolean is generated by a
477 // comparison.
478 if (Cond->getOpcode() == ISD::SETCC) {
479 EVT OpVT = Cond->getOperand(0).getValueType();
480 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
481 VecBool = TLI.getBooleanContents(OpVT);
482 } else
483 ScalarBool = TargetLowering::UndefinedBooleanContent;
484 }
485
486 EVT CondVT = Cond.getValueType();
487 if (ScalarBool != VecBool) {
488 switch (ScalarBool) {
489 case TargetLowering::UndefinedBooleanContent:
490 break;
491 case TargetLowering::ZeroOrOneBooleanContent:
492 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
493 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
494 // Vector read from all ones, scalar expects a single 1 so mask.
495 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
496 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
497 break;
498 case TargetLowering::ZeroOrNegativeOneBooleanContent:
499 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
500 VecBool == TargetLowering::ZeroOrOneBooleanContent);
501 // Vector reads from a one, scalar from all ones so sign extend.
502 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
503 Cond, DAG.getValueType(MVT::i1));
504 break;
505 }
506 }
507
508 // Truncate the condition if needed
509 auto BoolVT = getSetCCResultType(CondVT);
510 if (BoolVT.bitsLT(CondVT))
511 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
512
513 return DAG.getSelect(SDLoc(N),
514 LHS.getValueType(), Cond, LHS,
515 GetScalarizedVector(N->getOperand(2)));
516 }
517
ScalarizeVecRes_SELECT(SDNode * N)518 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
519 SDValue LHS = GetScalarizedVector(N->getOperand(1));
520 return DAG.getSelect(SDLoc(N),
521 LHS.getValueType(), N->getOperand(0), LHS,
522 GetScalarizedVector(N->getOperand(2)));
523 }
524
ScalarizeVecRes_SELECT_CC(SDNode * N)525 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
526 SDValue LHS = GetScalarizedVector(N->getOperand(2));
527 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
528 N->getOperand(0), N->getOperand(1),
529 LHS, GetScalarizedVector(N->getOperand(3)),
530 N->getOperand(4));
531 }
532
ScalarizeVecRes_UNDEF(SDNode * N)533 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
534 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
535 }
536
ScalarizeVecRes_VECTOR_SHUFFLE(SDNode * N)537 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
538 // Figure out if the scalar is the LHS or RHS and return it.
539 SDValue Arg = N->getOperand(2).getOperand(0);
540 if (Arg.isUndef())
541 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
542 unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
543 return GetScalarizedVector(N->getOperand(Op));
544 }
545
ScalarizeVecRes_FP_TO_XINT_SAT(SDNode * N)546 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
547 SDValue Src = N->getOperand(0);
548 EVT SrcVT = Src.getValueType();
549 SDLoc dl(N);
550
551 // Handle case where result is scalarized but operand is not
552 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
553 Src = GetScalarizedVector(Src);
554 else
555 Src = DAG.getNode(
556 ISD::EXTRACT_VECTOR_ELT, dl, SrcVT.getVectorElementType(), Src,
557 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
558
559 EVT DstVT = N->getValueType(0).getVectorElementType();
560 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
561 }
562
ScalarizeVecRes_SETCC(SDNode * N)563 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
564 assert(N->getValueType(0).isVector() &&
565 N->getOperand(0).getValueType().isVector() &&
566 "Operand types must be vectors");
567 SDValue LHS = N->getOperand(0);
568 SDValue RHS = N->getOperand(1);
569 EVT OpVT = LHS.getValueType();
570 EVT NVT = N->getValueType(0).getVectorElementType();
571 SDLoc DL(N);
572
573 // The result needs scalarizing, but it's not a given that the source does.
574 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
575 LHS = GetScalarizedVector(LHS);
576 RHS = GetScalarizedVector(RHS);
577 } else {
578 EVT VT = OpVT.getVectorElementType();
579 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
580 DAG.getVectorIdxConstant(0, DL));
581 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
582 DAG.getVectorIdxConstant(0, DL));
583 }
584
585 // Turn it into a scalar SETCC.
586 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
587 N->getOperand(2));
588 // Vectors may have a different boolean contents to scalars. Promote the
589 // value appropriately.
590 ISD::NodeType ExtendCode =
591 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
592 return DAG.getNode(ExtendCode, DL, NVT, Res);
593 }
594
ScalarizeVecRes_IS_FPCLASS(SDNode * N)595 SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
596 SDLoc DL(N);
597 SDValue Arg = N->getOperand(0);
598 SDValue Test = N->getOperand(1);
599 EVT ArgVT = Arg.getValueType();
600 EVT ResultVT = N->getValueType(0).getVectorElementType();
601
602 if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) {
603 Arg = GetScalarizedVector(Arg);
604 } else {
605 EVT VT = ArgVT.getVectorElementType();
606 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Arg,
607 DAG.getVectorIdxConstant(0, DL));
608 }
609
610 SDValue Res =
611 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
612 // Vectors may have a different boolean contents to scalars. Promote the
613 // value appropriately.
614 ISD::NodeType ExtendCode =
615 TargetLowering::getExtendForContent(TLI.getBooleanContents(ArgVT));
616 return DAG.getNode(ExtendCode, DL, ResultVT, Res);
617 }
618
619 //===----------------------------------------------------------------------===//
620 // Operand Vector Scalarization <1 x ty> -> ty.
621 //===----------------------------------------------------------------------===//
622
ScalarizeVectorOperand(SDNode * N,unsigned OpNo)623 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
624 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
625 dbgs() << "\n");
626 SDValue Res = SDValue();
627
628 switch (N->getOpcode()) {
629 default:
630 #ifndef NDEBUG
631 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
632 N->dump(&DAG);
633 dbgs() << "\n";
634 #endif
635 report_fatal_error("Do not know how to scalarize this operator's "
636 "operand!\n");
637 case ISD::BITCAST:
638 Res = ScalarizeVecOp_BITCAST(N);
639 break;
640 case ISD::ANY_EXTEND:
641 case ISD::ZERO_EXTEND:
642 case ISD::SIGN_EXTEND:
643 case ISD::TRUNCATE:
644 case ISD::FP_TO_SINT:
645 case ISD::FP_TO_UINT:
646 case ISD::SINT_TO_FP:
647 case ISD::UINT_TO_FP:
648 Res = ScalarizeVecOp_UnaryOp(N);
649 break;
650 case ISD::STRICT_SINT_TO_FP:
651 case ISD::STRICT_UINT_TO_FP:
652 case ISD::STRICT_FP_TO_SINT:
653 case ISD::STRICT_FP_TO_UINT:
654 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
655 break;
656 case ISD::CONCAT_VECTORS:
657 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
658 break;
659 case ISD::EXTRACT_VECTOR_ELT:
660 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
661 break;
662 case ISD::VSELECT:
663 Res = ScalarizeVecOp_VSELECT(N);
664 break;
665 case ISD::SETCC:
666 Res = ScalarizeVecOp_VSETCC(N);
667 break;
668 case ISD::STORE:
669 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
670 break;
671 case ISD::STRICT_FP_ROUND:
672 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
673 break;
674 case ISD::FP_ROUND:
675 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
676 break;
677 case ISD::STRICT_FP_EXTEND:
678 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
679 break;
680 case ISD::FP_EXTEND:
681 Res = ScalarizeVecOp_FP_EXTEND(N);
682 break;
683 case ISD::VECREDUCE_FADD:
684 case ISD::VECREDUCE_FMUL:
685 case ISD::VECREDUCE_ADD:
686 case ISD::VECREDUCE_MUL:
687 case ISD::VECREDUCE_AND:
688 case ISD::VECREDUCE_OR:
689 case ISD::VECREDUCE_XOR:
690 case ISD::VECREDUCE_SMAX:
691 case ISD::VECREDUCE_SMIN:
692 case ISD::VECREDUCE_UMAX:
693 case ISD::VECREDUCE_UMIN:
694 case ISD::VECREDUCE_FMAX:
695 case ISD::VECREDUCE_FMIN:
696 Res = ScalarizeVecOp_VECREDUCE(N);
697 break;
698 case ISD::VECREDUCE_SEQ_FADD:
699 case ISD::VECREDUCE_SEQ_FMUL:
700 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
701 break;
702 }
703
704 // If the result is null, the sub-method took care of registering results etc.
705 if (!Res.getNode()) return false;
706
707 // If the result is N, the sub-method updated N in place. Tell the legalizer
708 // core about this.
709 if (Res.getNode() == N)
710 return true;
711
712 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
713 "Invalid operand expansion");
714
715 ReplaceValueWith(SDValue(N, 0), Res);
716 return false;
717 }
718
719 /// If the value to convert is a vector that needs to be scalarized, it must be
720 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_BITCAST(SDNode * N)721 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
722 SDValue Elt = GetScalarizedVector(N->getOperand(0));
723 return DAG.getNode(ISD::BITCAST, SDLoc(N),
724 N->getValueType(0), Elt);
725 }
726
727 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
728 /// Do the operation on the element instead.
ScalarizeVecOp_UnaryOp(SDNode * N)729 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
730 assert(N->getValueType(0).getVectorNumElements() == 1 &&
731 "Unexpected vector type!");
732 SDValue Elt = GetScalarizedVector(N->getOperand(0));
733 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
734 N->getValueType(0).getScalarType(), Elt);
735 // Revectorize the result so the types line up with what the uses of this
736 // expression expect.
737 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
738 }
739
740 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
741 /// Do the strict FP operation on the element instead.
ScalarizeVecOp_UnaryOp_StrictFP(SDNode * N)742 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
743 assert(N->getValueType(0).getVectorNumElements() == 1 &&
744 "Unexpected vector type!");
745 SDValue Elt = GetScalarizedVector(N->getOperand(1));
746 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
747 { N->getValueType(0).getScalarType(), MVT::Other },
748 { N->getOperand(0), Elt });
749 // Legalize the chain result - switch anything that used the old chain to
750 // use the new one.
751 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
752 // Revectorize the result so the types line up with what the uses of this
753 // expression expect.
754 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
755
756 // Do our own replacement and return SDValue() to tell the caller that we
757 // handled all replacements since caller can only handle a single result.
758 ReplaceValueWith(SDValue(N, 0), Res);
759 return SDValue();
760 }
761
762 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
ScalarizeVecOp_CONCAT_VECTORS(SDNode * N)763 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
764 SmallVector<SDValue, 8> Ops(N->getNumOperands());
765 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
766 Ops[i] = GetScalarizedVector(N->getOperand(i));
767 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
768 }
769
770 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
771 /// so just return the element, ignoring the index.
ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode * N)772 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
773 EVT VT = N->getValueType(0);
774 SDValue Res = GetScalarizedVector(N->getOperand(0));
775 if (Res.getValueType() != VT)
776 Res = VT.isFloatingPoint()
777 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
778 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
779 return Res;
780 }
781
782 /// If the input condition is a vector that needs to be scalarized, it must be
783 /// <1 x i1>, so just convert to a normal ISD::SELECT
784 /// (still with vector output type since that was acceptable if we got here).
ScalarizeVecOp_VSELECT(SDNode * N)785 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
786 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
787 EVT VT = N->getValueType(0);
788
789 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
790 N->getOperand(2));
791 }
792
793 /// If the operand is a vector that needs to be scalarized then the
794 /// result must be v1i1, so just convert to a scalar SETCC and wrap
795 /// with a scalar_to_vector since the res type is legal if we got here
ScalarizeVecOp_VSETCC(SDNode * N)796 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
797 assert(N->getValueType(0).isVector() &&
798 N->getOperand(0).getValueType().isVector() &&
799 "Operand types must be vectors");
800 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
801
802 EVT VT = N->getValueType(0);
803 SDValue LHS = GetScalarizedVector(N->getOperand(0));
804 SDValue RHS = GetScalarizedVector(N->getOperand(1));
805
806 EVT OpVT = N->getOperand(0).getValueType();
807 EVT NVT = VT.getVectorElementType();
808 SDLoc DL(N);
809 // Turn it into a scalar SETCC.
810 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
811 N->getOperand(2));
812
813 // Vectors may have a different boolean contents to scalars. Promote the
814 // value appropriately.
815 ISD::NodeType ExtendCode =
816 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
817
818 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
819
820 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
821 }
822
823 /// If the value to store is a vector that needs to be scalarized, it must be
824 /// <1 x ty>. Just store the element.
ScalarizeVecOp_STORE(StoreSDNode * N,unsigned OpNo)825 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
826 assert(N->isUnindexed() && "Indexed store of one-element vector?");
827 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
828 SDLoc dl(N);
829
830 if (N->isTruncatingStore())
831 return DAG.getTruncStore(
832 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
833 N->getBasePtr(), N->getPointerInfo(),
834 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
835 N->getMemOperand()->getFlags(), N->getAAInfo());
836
837 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
838 N->getBasePtr(), N->getPointerInfo(),
839 N->getOriginalAlign(), N->getMemOperand()->getFlags(),
840 N->getAAInfo());
841 }
842
843 /// If the value to round is a vector that needs to be scalarized, it must be
844 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_FP_ROUND(SDNode * N,unsigned OpNo)845 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
846 assert(OpNo == 0 && "Wrong operand for scalarization!");
847 SDValue Elt = GetScalarizedVector(N->getOperand(0));
848 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
849 N->getValueType(0).getVectorElementType(), Elt,
850 N->getOperand(1));
851 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
852 }
853
ScalarizeVecOp_STRICT_FP_ROUND(SDNode * N,unsigned OpNo)854 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
855 unsigned OpNo) {
856 assert(OpNo == 1 && "Wrong operand for scalarization!");
857 SDValue Elt = GetScalarizedVector(N->getOperand(1));
858 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
859 { N->getValueType(0).getVectorElementType(),
860 MVT::Other },
861 { N->getOperand(0), Elt, N->getOperand(2) });
862 // Legalize the chain result - switch anything that used the old chain to
863 // use the new one.
864 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
865
866 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
867
868 // Do our own replacement and return SDValue() to tell the caller that we
869 // handled all replacements since caller can only handle a single result.
870 ReplaceValueWith(SDValue(N, 0), Res);
871 return SDValue();
872 }
873
874 /// If the value to extend is a vector that needs to be scalarized, it must be
875 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_FP_EXTEND(SDNode * N)876 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
877 SDValue Elt = GetScalarizedVector(N->getOperand(0));
878 SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
879 N->getValueType(0).getVectorElementType(), Elt);
880 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
881 }
882
883 /// If the value to extend is a vector that needs to be scalarized, it must be
884 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_STRICT_FP_EXTEND(SDNode * N)885 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
886 SDValue Elt = GetScalarizedVector(N->getOperand(1));
887 SDValue Res =
888 DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
889 {N->getValueType(0).getVectorElementType(), MVT::Other},
890 {N->getOperand(0), Elt});
891 // Legalize the chain result - switch anything that used the old chain to
892 // use the new one.
893 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
894
895 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
896
897 // Do our own replacement and return SDValue() to tell the caller that we
898 // handled all replacements since caller can only handle a single result.
899 ReplaceValueWith(SDValue(N, 0), Res);
900 return SDValue();
901 }
902
ScalarizeVecOp_VECREDUCE(SDNode * N)903 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
904 SDValue Res = GetScalarizedVector(N->getOperand(0));
905 // Result type may be wider than element type.
906 if (Res.getValueType() != N->getValueType(0))
907 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
908 return Res;
909 }
910
ScalarizeVecOp_VECREDUCE_SEQ(SDNode * N)911 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
912 SDValue AccOp = N->getOperand(0);
913 SDValue VecOp = N->getOperand(1);
914
915 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
916
917 SDValue Op = GetScalarizedVector(VecOp);
918 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
919 AccOp, Op, N->getFlags());
920 }
921
922 //===----------------------------------------------------------------------===//
923 // Result Vector Splitting
924 //===----------------------------------------------------------------------===//
925
926 /// This method is called when the specified result of the specified node is
927 /// found to need vector splitting. At this point, the node may also have
928 /// invalid operands or may have other results that need legalization, we just
929 /// know that (at least) one result needs vector splitting.
SplitVectorResult(SDNode * N,unsigned ResNo)930 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
931 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
932 SDValue Lo, Hi;
933
934 // See if the target wants to custom expand this node.
935 if (CustomLowerNode(N, N->getValueType(ResNo), true))
936 return;
937
938 switch (N->getOpcode()) {
939 default:
940 #ifndef NDEBUG
941 dbgs() << "SplitVectorResult #" << ResNo << ": ";
942 N->dump(&DAG);
943 dbgs() << "\n";
944 #endif
945 report_fatal_error("Do not know how to split the result of this "
946 "operator!\n");
947
948 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
949 case ISD::VSELECT:
950 case ISD::SELECT:
951 case ISD::VP_MERGE:
952 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
953 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
954 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
955 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
956 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
957 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
958 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
959 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
960 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
961 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
962 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
963 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
964 case ISD::SPLAT_VECTOR:
965 case ISD::SCALAR_TO_VECTOR:
966 SplitVecRes_ScalarOp(N, Lo, Hi);
967 break;
968 case ISD::STEP_VECTOR:
969 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
970 break;
971 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
972 case ISD::LOAD:
973 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
974 break;
975 case ISD::VP_LOAD:
976 SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi);
977 break;
978 case ISD::MLOAD:
979 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
980 break;
981 case ISD::MGATHER:
982 case ISD::VP_GATHER:
983 SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true);
984 break;
985 case ISD::SETCC:
986 case ISD::VP_SETCC:
987 SplitVecRes_SETCC(N, Lo, Hi);
988 break;
989 case ISD::VECTOR_REVERSE:
990 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
991 break;
992 case ISD::VECTOR_SHUFFLE:
993 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
994 break;
995 case ISD::VECTOR_SPLICE:
996 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
997 break;
998 case ISD::VAARG:
999 SplitVecRes_VAARG(N, Lo, Hi);
1000 break;
1001
1002 case ISD::ANY_EXTEND_VECTOR_INREG:
1003 case ISD::SIGN_EXTEND_VECTOR_INREG:
1004 case ISD::ZERO_EXTEND_VECTOR_INREG:
1005 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1006 break;
1007
1008 case ISD::ABS:
1009 case ISD::BITREVERSE:
1010 case ISD::BSWAP:
1011 case ISD::CTLZ:
1012 case ISD::CTTZ:
1013 case ISD::CTLZ_ZERO_UNDEF:
1014 case ISD::CTTZ_ZERO_UNDEF:
1015 case ISD::CTPOP:
1016 case ISD::FABS:
1017 case ISD::FCEIL:
1018 case ISD::FCOS:
1019 case ISD::FEXP:
1020 case ISD::FEXP2:
1021 case ISD::FFLOOR:
1022 case ISD::FLOG:
1023 case ISD::FLOG10:
1024 case ISD::FLOG2:
1025 case ISD::FNEARBYINT:
1026 case ISD::FNEG: case ISD::VP_FNEG:
1027 case ISD::FREEZE:
1028 case ISD::ARITH_FENCE:
1029 case ISD::FP_EXTEND:
1030 case ISD::VP_FP_EXTEND:
1031 case ISD::FP_ROUND:
1032 case ISD::VP_FP_ROUND:
1033 case ISD::FP_TO_SINT:
1034 case ISD::VP_FPTOSI:
1035 case ISD::FP_TO_UINT:
1036 case ISD::VP_FPTOUI:
1037 case ISD::FRINT:
1038 case ISD::FROUND:
1039 case ISD::FROUNDEVEN:
1040 case ISD::FSIN:
1041 case ISD::FSQRT:
1042 case ISD::FTRUNC:
1043 case ISD::SINT_TO_FP:
1044 case ISD::VP_SITOFP:
1045 case ISD::TRUNCATE:
1046 case ISD::VP_TRUNCATE:
1047 case ISD::UINT_TO_FP:
1048 case ISD::VP_UITOFP:
1049 case ISD::FCANONICALIZE:
1050 SplitVecRes_UnaryOp(N, Lo, Hi);
1051 break;
1052
1053 case ISD::ANY_EXTEND:
1054 case ISD::SIGN_EXTEND:
1055 case ISD::ZERO_EXTEND:
1056 case ISD::VP_SIGN_EXTEND:
1057 case ISD::VP_ZERO_EXTEND:
1058 SplitVecRes_ExtendOp(N, Lo, Hi);
1059 break;
1060
1061 case ISD::ADD: case ISD::VP_ADD:
1062 case ISD::SUB: case ISD::VP_SUB:
1063 case ISD::MUL: case ISD::VP_MUL:
1064 case ISD::MULHS:
1065 case ISD::MULHU:
1066 case ISD::FADD: case ISD::VP_FADD:
1067 case ISD::FSUB: case ISD::VP_FSUB:
1068 case ISD::FMUL: case ISD::VP_FMUL:
1069 case ISD::FMINNUM:
1070 case ISD::FMAXNUM:
1071 case ISD::FMINIMUM:
1072 case ISD::FMAXIMUM:
1073 case ISD::SDIV: case ISD::VP_SDIV:
1074 case ISD::UDIV: case ISD::VP_UDIV:
1075 case ISD::FDIV: case ISD::VP_FDIV:
1076 case ISD::FPOW:
1077 case ISD::AND: case ISD::VP_AND:
1078 case ISD::OR: case ISD::VP_OR:
1079 case ISD::XOR: case ISD::VP_XOR:
1080 case ISD::SHL: case ISD::VP_SHL:
1081 case ISD::SRA: case ISD::VP_ASHR:
1082 case ISD::SRL: case ISD::VP_LSHR:
1083 case ISD::UREM: case ISD::VP_UREM:
1084 case ISD::SREM: case ISD::VP_SREM:
1085 case ISD::FREM: case ISD::VP_FREM:
1086 case ISD::SMIN:
1087 case ISD::SMAX:
1088 case ISD::UMIN:
1089 case ISD::UMAX:
1090 case ISD::SADDSAT:
1091 case ISD::UADDSAT:
1092 case ISD::SSUBSAT:
1093 case ISD::USUBSAT:
1094 case ISD::SSHLSAT:
1095 case ISD::USHLSAT:
1096 case ISD::ROTL:
1097 case ISD::ROTR:
1098 SplitVecRes_BinOp(N, Lo, Hi);
1099 break;
1100 case ISD::FMA: case ISD::VP_FMA:
1101 case ISD::FSHL:
1102 case ISD::FSHR:
1103 SplitVecRes_TernaryOp(N, Lo, Hi);
1104 break;
1105
1106 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1107 case ISD::STRICT_##DAGN:
1108 #include "llvm/IR/ConstrainedOps.def"
1109 SplitVecRes_StrictFPOp(N, Lo, Hi);
1110 break;
1111
1112 case ISD::FP_TO_UINT_SAT:
1113 case ISD::FP_TO_SINT_SAT:
1114 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1115 break;
1116
1117 case ISD::UADDO:
1118 case ISD::SADDO:
1119 case ISD::USUBO:
1120 case ISD::SSUBO:
1121 case ISD::UMULO:
1122 case ISD::SMULO:
1123 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1124 break;
1125 case ISD::SMULFIX:
1126 case ISD::SMULFIXSAT:
1127 case ISD::UMULFIX:
1128 case ISD::UMULFIXSAT:
1129 case ISD::SDIVFIX:
1130 case ISD::SDIVFIXSAT:
1131 case ISD::UDIVFIX:
1132 case ISD::UDIVFIXSAT:
1133 SplitVecRes_FIX(N, Lo, Hi);
1134 break;
1135 }
1136
1137 // If Lo/Hi is null, the sub-method took care of registering results etc.
1138 if (Lo.getNode())
1139 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1140 }
1141
IncrementPointer(MemSDNode * N,EVT MemVT,MachinePointerInfo & MPI,SDValue & Ptr,uint64_t * ScaledOffset)1142 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1143 MachinePointerInfo &MPI, SDValue &Ptr,
1144 uint64_t *ScaledOffset) {
1145 SDLoc DL(N);
1146 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8;
1147
1148 if (MemVT.isScalableVector()) {
1149 SDNodeFlags Flags;
1150 SDValue BytesIncrement = DAG.getVScale(
1151 DL, Ptr.getValueType(),
1152 APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
1153 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1154 Flags.setNoUnsignedWrap(true);
1155 if (ScaledOffset)
1156 *ScaledOffset += IncrementSize;
1157 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1158 Flags);
1159 } else {
1160 MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1161 // Increment the pointer to the other half.
1162 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1163 }
1164 }
1165
SplitMask(SDValue Mask)1166 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1167 return SplitMask(Mask, SDLoc(Mask));
1168 }
1169
SplitMask(SDValue Mask,const SDLoc & DL)1170 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1171 const SDLoc &DL) {
1172 SDValue MaskLo, MaskHi;
1173 EVT MaskVT = Mask.getValueType();
1174 if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1175 GetSplitVector(Mask, MaskLo, MaskHi);
1176 else
1177 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1178 return std::make_pair(MaskLo, MaskHi);
1179 }
1180
SplitVecRes_BinOp(SDNode * N,SDValue & Lo,SDValue & Hi)1181 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1182 SDValue LHSLo, LHSHi;
1183 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1184 SDValue RHSLo, RHSHi;
1185 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1186 SDLoc dl(N);
1187
1188 const SDNodeFlags Flags = N->getFlags();
1189 unsigned Opcode = N->getOpcode();
1190 if (N->getNumOperands() == 2) {
1191 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1192 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1193 return;
1194 }
1195
1196 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1197 assert(N->isVPOpcode() && "Expected VP opcode");
1198
1199 SDValue MaskLo, MaskHi;
1200 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
1201
1202 SDValue EVLLo, EVLHi;
1203 std::tie(EVLLo, EVLHi) =
1204 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
1205
1206 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1207 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1208 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1209 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1210 }
1211
SplitVecRes_TernaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)1212 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1213 SDValue &Hi) {
1214 SDValue Op0Lo, Op0Hi;
1215 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1216 SDValue Op1Lo, Op1Hi;
1217 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1218 SDValue Op2Lo, Op2Hi;
1219 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1220 SDLoc dl(N);
1221
1222 const SDNodeFlags Flags = N->getFlags();
1223 unsigned Opcode = N->getOpcode();
1224 if (N->getNumOperands() == 3) {
1225 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags);
1226 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags);
1227 return;
1228 }
1229
1230 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1231 assert(N->isVPOpcode() && "Expected VP opcode");
1232
1233 SDValue MaskLo, MaskHi;
1234 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
1235
1236 SDValue EVLLo, EVLHi;
1237 std::tie(EVLLo, EVLHi) =
1238 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), dl);
1239
1240 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(),
1241 {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1242 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(),
1243 {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1244 }
1245
SplitVecRes_FIX(SDNode * N,SDValue & Lo,SDValue & Hi)1246 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1247 SDValue LHSLo, LHSHi;
1248 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1249 SDValue RHSLo, RHSHi;
1250 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1251 SDLoc dl(N);
1252 SDValue Op2 = N->getOperand(2);
1253
1254 unsigned Opcode = N->getOpcode();
1255 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1256 N->getFlags());
1257 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1258 N->getFlags());
1259 }
1260
SplitVecRes_BITCAST(SDNode * N,SDValue & Lo,SDValue & Hi)1261 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1262 SDValue &Hi) {
1263 // We know the result is a vector. The input may be either a vector or a
1264 // scalar value.
1265 EVT LoVT, HiVT;
1266 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1267 SDLoc dl(N);
1268
1269 SDValue InOp = N->getOperand(0);
1270 EVT InVT = InOp.getValueType();
1271
1272 // Handle some special cases efficiently.
1273 switch (getTypeAction(InVT)) {
1274 case TargetLowering::TypeLegal:
1275 case TargetLowering::TypePromoteInteger:
1276 case TargetLowering::TypePromoteFloat:
1277 case TargetLowering::TypeSoftPromoteHalf:
1278 case TargetLowering::TypeSoftenFloat:
1279 case TargetLowering::TypeScalarizeVector:
1280 case TargetLowering::TypeWidenVector:
1281 break;
1282 case TargetLowering::TypeExpandInteger:
1283 case TargetLowering::TypeExpandFloat:
1284 // A scalar to vector conversion, where the scalar needs expansion.
1285 // If the vector is being split in two then we can just convert the
1286 // expanded pieces.
1287 if (LoVT == HiVT) {
1288 GetExpandedOp(InOp, Lo, Hi);
1289 if (DAG.getDataLayout().isBigEndian())
1290 std::swap(Lo, Hi);
1291 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1292 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1293 return;
1294 }
1295 break;
1296 case TargetLowering::TypeSplitVector:
1297 // If the input is a vector that needs to be split, convert each split
1298 // piece of the input now.
1299 GetSplitVector(InOp, Lo, Hi);
1300 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1301 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1302 return;
1303 case TargetLowering::TypeScalarizeScalableVector:
1304 report_fatal_error("Scalarization of scalable vectors is not supported.");
1305 }
1306
1307 // In the general case, convert the input to an integer and split it by hand.
1308 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1309 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1310 if (DAG.getDataLayout().isBigEndian())
1311 std::swap(LoIntVT, HiIntVT);
1312
1313 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1314
1315 if (DAG.getDataLayout().isBigEndian())
1316 std::swap(Lo, Hi);
1317 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1318 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1319 }
1320
SplitVecRes_BUILD_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1321 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1322 SDValue &Hi) {
1323 EVT LoVT, HiVT;
1324 SDLoc dl(N);
1325 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1326 unsigned LoNumElts = LoVT.getVectorNumElements();
1327 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1328 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1329
1330 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1331 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1332 }
1333
SplitVecRes_CONCAT_VECTORS(SDNode * N,SDValue & Lo,SDValue & Hi)1334 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1335 SDValue &Hi) {
1336 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1337 SDLoc dl(N);
1338 unsigned NumSubvectors = N->getNumOperands() / 2;
1339 if (NumSubvectors == 1) {
1340 Lo = N->getOperand(0);
1341 Hi = N->getOperand(1);
1342 return;
1343 }
1344
1345 EVT LoVT, HiVT;
1346 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1347
1348 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1349 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1350
1351 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1352 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1353 }
1354
SplitVecRes_EXTRACT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1355 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1356 SDValue &Hi) {
1357 SDValue Vec = N->getOperand(0);
1358 SDValue Idx = N->getOperand(1);
1359 SDLoc dl(N);
1360
1361 EVT LoVT, HiVT;
1362 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1363
1364 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1365 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1366 Hi = DAG.getNode(
1367 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1368 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1369 }
1370
SplitVecRes_INSERT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1371 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1372 SDValue &Hi) {
1373 SDValue Vec = N->getOperand(0);
1374 SDValue SubVec = N->getOperand(1);
1375 SDValue Idx = N->getOperand(2);
1376 SDLoc dl(N);
1377 GetSplitVector(Vec, Lo, Hi);
1378
1379 EVT VecVT = Vec.getValueType();
1380 EVT LoVT = Lo.getValueType();
1381 EVT SubVecVT = SubVec.getValueType();
1382 unsigned VecElems = VecVT.getVectorMinNumElements();
1383 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1384 unsigned LoElems = LoVT.getVectorMinNumElements();
1385
1386 // If we know the index is in the first half, and we know the subvector
1387 // doesn't cross the boundary between the halves, we can avoid spilling the
1388 // vector, and insert into the lower half of the split vector directly.
1389 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1390 if (IdxVal + SubElems <= LoElems) {
1391 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1392 return;
1393 }
1394 // Similarly if the subvector is fully in the high half, but mind that we
1395 // can't tell whether a fixed-length subvector is fully within the high half
1396 // of a scalable vector.
1397 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1398 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1399 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1400 DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1401 return;
1402 }
1403
1404 // Spill the vector to the stack.
1405 // In cases where the vector is illegal it will be broken down into parts
1406 // and stored in parts - we should use the alignment for the smallest part.
1407 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1408 SDValue StackPtr =
1409 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1410 auto &MF = DAG.getMachineFunction();
1411 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1412 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1413
1414 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1415 SmallestAlign);
1416
1417 // Store the new subvector into the specified index.
1418 SDValue SubVecPtr =
1419 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1420 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1421 MachinePointerInfo::getUnknownStack(MF));
1422
1423 // Load the Lo part from the stack slot.
1424 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1425 SmallestAlign);
1426
1427 // Increment the pointer to the other part.
1428 auto *Load = cast<LoadSDNode>(Lo);
1429 MachinePointerInfo MPI = Load->getPointerInfo();
1430 IncrementPointer(Load, LoVT, MPI, StackPtr);
1431
1432 // Load the Hi part from the stack slot.
1433 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1434 }
1435
SplitVecRes_FPOWI(SDNode * N,SDValue & Lo,SDValue & Hi)1436 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1437 SDValue &Hi) {
1438 SDLoc dl(N);
1439 GetSplitVector(N->getOperand(0), Lo, Hi);
1440 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1441 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1442 }
1443
SplitVecRes_FCOPYSIGN(SDNode * N,SDValue & Lo,SDValue & Hi)1444 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1445 SDValue &Hi) {
1446 SDValue LHSLo, LHSHi;
1447 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1448 SDLoc DL(N);
1449
1450 SDValue RHSLo, RHSHi;
1451 SDValue RHS = N->getOperand(1);
1452 EVT RHSVT = RHS.getValueType();
1453 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1454 GetSplitVector(RHS, RHSLo, RHSHi);
1455 else
1456 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1457
1458
1459 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1460 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1461 }
1462
SplitVecRes_IS_FPCLASS(SDNode * N,SDValue & Lo,SDValue & Hi)1463 void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1464 SDValue &Hi) {
1465 SDLoc DL(N);
1466 SDValue ArgLo, ArgHi;
1467 SDValue Test = N->getOperand(1);
1468 GetSplitVector(N->getOperand(0), ArgLo, ArgHi);
1469 EVT LoVT, HiVT;
1470 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1471
1472 Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags());
1473 Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags());
1474 }
1475
SplitVecRes_InregOp(SDNode * N,SDValue & Lo,SDValue & Hi)1476 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1477 SDValue &Hi) {
1478 SDValue LHSLo, LHSHi;
1479 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1480 SDLoc dl(N);
1481
1482 EVT LoVT, HiVT;
1483 std::tie(LoVT, HiVT) =
1484 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1485
1486 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1487 DAG.getValueType(LoVT));
1488 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1489 DAG.getValueType(HiVT));
1490 }
1491
SplitVecRes_ExtVecInRegOp(SDNode * N,SDValue & Lo,SDValue & Hi)1492 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1493 SDValue &Hi) {
1494 unsigned Opcode = N->getOpcode();
1495 SDValue N0 = N->getOperand(0);
1496
1497 SDLoc dl(N);
1498 SDValue InLo, InHi;
1499
1500 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1501 GetSplitVector(N0, InLo, InHi);
1502 else
1503 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1504
1505 EVT InLoVT = InLo.getValueType();
1506 unsigned InNumElements = InLoVT.getVectorNumElements();
1507
1508 EVT OutLoVT, OutHiVT;
1509 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1510 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1511 assert((2 * OutNumElements) <= InNumElements &&
1512 "Illegal extend vector in reg split");
1513
1514 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1515 // input vector (i.e. we only use InLo):
1516 // OutLo will extend the first OutNumElements from InLo.
1517 // OutHi will extend the next OutNumElements from InLo.
1518
1519 // Shuffle the elements from InLo for OutHi into the bottom elements to
1520 // create a 'fake' InHi.
1521 SmallVector<int, 8> SplitHi(InNumElements, -1);
1522 for (unsigned i = 0; i != OutNumElements; ++i)
1523 SplitHi[i] = i + OutNumElements;
1524 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1525
1526 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1527 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1528 }
1529
SplitVecRes_StrictFPOp(SDNode * N,SDValue & Lo,SDValue & Hi)1530 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1531 SDValue &Hi) {
1532 unsigned NumOps = N->getNumOperands();
1533 SDValue Chain = N->getOperand(0);
1534 EVT LoVT, HiVT;
1535 SDLoc dl(N);
1536 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1537
1538 SmallVector<SDValue, 4> OpsLo(NumOps);
1539 SmallVector<SDValue, 4> OpsHi(NumOps);
1540
1541 // The Chain is the first operand.
1542 OpsLo[0] = Chain;
1543 OpsHi[0] = Chain;
1544
1545 // Now process the remaining operands.
1546 for (unsigned i = 1; i < NumOps; ++i) {
1547 SDValue Op = N->getOperand(i);
1548 SDValue OpLo = Op;
1549 SDValue OpHi = Op;
1550
1551 EVT InVT = Op.getValueType();
1552 if (InVT.isVector()) {
1553 // If the input also splits, handle it directly for a
1554 // compile time speedup. Otherwise split it by hand.
1555 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1556 GetSplitVector(Op, OpLo, OpHi);
1557 else
1558 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1559 }
1560
1561 OpsLo[i] = OpLo;
1562 OpsHi[i] = OpHi;
1563 }
1564
1565 EVT LoValueVTs[] = {LoVT, MVT::Other};
1566 EVT HiValueVTs[] = {HiVT, MVT::Other};
1567 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1568 N->getFlags());
1569 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1570 N->getFlags());
1571
1572 // Build a factor node to remember that this Op is independent of the
1573 // other one.
1574 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1575 Lo.getValue(1), Hi.getValue(1));
1576
1577 // Legalize the chain result - switch anything that used the old chain to
1578 // use the new one.
1579 ReplaceValueWith(SDValue(N, 1), Chain);
1580 }
1581
UnrollVectorOp_StrictFP(SDNode * N,unsigned ResNE)1582 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1583 SDValue Chain = N->getOperand(0);
1584 EVT VT = N->getValueType(0);
1585 unsigned NE = VT.getVectorNumElements();
1586 EVT EltVT = VT.getVectorElementType();
1587 SDLoc dl(N);
1588
1589 SmallVector<SDValue, 8> Scalars;
1590 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1591
1592 // If ResNE is 0, fully unroll the vector op.
1593 if (ResNE == 0)
1594 ResNE = NE;
1595 else if (NE > ResNE)
1596 NE = ResNE;
1597
1598 //The results of each unrolled operation, including the chain.
1599 EVT ChainVTs[] = {EltVT, MVT::Other};
1600 SmallVector<SDValue, 8> Chains;
1601
1602 unsigned i;
1603 for (i = 0; i != NE; ++i) {
1604 Operands[0] = Chain;
1605 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1606 SDValue Operand = N->getOperand(j);
1607 EVT OperandVT = Operand.getValueType();
1608 if (OperandVT.isVector()) {
1609 EVT OperandEltVT = OperandVT.getVectorElementType();
1610 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1611 Operand, DAG.getVectorIdxConstant(i, dl));
1612 } else {
1613 Operands[j] = Operand;
1614 }
1615 }
1616 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1617 Scalar.getNode()->setFlags(N->getFlags());
1618
1619 //Add in the scalar as well as its chain value to the
1620 //result vectors.
1621 Scalars.push_back(Scalar);
1622 Chains.push_back(Scalar.getValue(1));
1623 }
1624
1625 for (; i < ResNE; ++i)
1626 Scalars.push_back(DAG.getUNDEF(EltVT));
1627
1628 // Build a new factor node to connect the chain back together.
1629 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1630 ReplaceValueWith(SDValue(N, 1), Chain);
1631
1632 // Create a new BUILD_VECTOR node
1633 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1634 return DAG.getBuildVector(VecVT, dl, Scalars);
1635 }
1636
SplitVecRes_OverflowOp(SDNode * N,unsigned ResNo,SDValue & Lo,SDValue & Hi)1637 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1638 SDValue &Lo, SDValue &Hi) {
1639 SDLoc dl(N);
1640 EVT ResVT = N->getValueType(0);
1641 EVT OvVT = N->getValueType(1);
1642 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1643 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1644 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1645
1646 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1647 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1648 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1649 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1650 } else {
1651 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1652 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1653 }
1654
1655 unsigned Opcode = N->getOpcode();
1656 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1657 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1658 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1659 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1660 LoNode->setFlags(N->getFlags());
1661 HiNode->setFlags(N->getFlags());
1662
1663 Lo = SDValue(LoNode, ResNo);
1664 Hi = SDValue(HiNode, ResNo);
1665
1666 // Replace the other vector result not being explicitly split here.
1667 unsigned OtherNo = 1 - ResNo;
1668 EVT OtherVT = N->getValueType(OtherNo);
1669 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1670 SetSplitVector(SDValue(N, OtherNo),
1671 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1672 } else {
1673 SDValue OtherVal = DAG.getNode(
1674 ISD::CONCAT_VECTORS, dl, OtherVT,
1675 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1676 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1677 }
1678 }
1679
SplitVecRes_INSERT_VECTOR_ELT(SDNode * N,SDValue & Lo,SDValue & Hi)1680 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1681 SDValue &Hi) {
1682 SDValue Vec = N->getOperand(0);
1683 SDValue Elt = N->getOperand(1);
1684 SDValue Idx = N->getOperand(2);
1685 SDLoc dl(N);
1686 GetSplitVector(Vec, Lo, Hi);
1687
1688 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1689 unsigned IdxVal = CIdx->getZExtValue();
1690 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1691 if (IdxVal < LoNumElts) {
1692 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1693 Lo.getValueType(), Lo, Elt, Idx);
1694 return;
1695 } else if (!Vec.getValueType().isScalableVector()) {
1696 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1697 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1698 return;
1699 }
1700 }
1701
1702 // See if the target wants to custom expand this node.
1703 if (CustomLowerNode(N, N->getValueType(0), true))
1704 return;
1705
1706 // Make the vector elements byte-addressable if they aren't already.
1707 EVT VecVT = Vec.getValueType();
1708 EVT EltVT = VecVT.getVectorElementType();
1709 if (VecVT.getScalarSizeInBits() < 8) {
1710 EltVT = MVT::i8;
1711 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1712 VecVT.getVectorElementCount());
1713 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1714 // Extend the element type to match if needed.
1715 if (EltVT.bitsGT(Elt.getValueType()))
1716 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1717 }
1718
1719 // Spill the vector to the stack.
1720 // In cases where the vector is illegal it will be broken down into parts
1721 // and stored in parts - we should use the alignment for the smallest part.
1722 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1723 SDValue StackPtr =
1724 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1725 auto &MF = DAG.getMachineFunction();
1726 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1727 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1728
1729 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1730 SmallestAlign);
1731
1732 // Store the new element. This may be larger than the vector element type,
1733 // so use a truncating store.
1734 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1735 Store = DAG.getTruncStore(
1736 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1737 commonAlignment(SmallestAlign,
1738 EltVT.getFixedSizeInBits() / 8));
1739
1740 EVT LoVT, HiVT;
1741 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1742
1743 // Load the Lo part from the stack slot.
1744 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1745
1746 // Increment the pointer to the other part.
1747 auto Load = cast<LoadSDNode>(Lo);
1748 MachinePointerInfo MPI = Load->getPointerInfo();
1749 IncrementPointer(Load, LoVT, MPI, StackPtr);
1750
1751 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1752
1753 // If we adjusted the original type, we need to truncate the results.
1754 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1755 if (LoVT != Lo.getValueType())
1756 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1757 if (HiVT != Hi.getValueType())
1758 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1759 }
1760
SplitVecRes_STEP_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1761 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1762 SDValue &Hi) {
1763 EVT LoVT, HiVT;
1764 SDLoc dl(N);
1765 assert(N->getValueType(0).isScalableVector() &&
1766 "Only scalable vectors are supported for STEP_VECTOR");
1767 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1768 SDValue Step = N->getOperand(0);
1769
1770 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1771
1772 // Hi = Lo + (EltCnt * Step)
1773 EVT EltVT = Step.getValueType();
1774 APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1775 SDValue StartOfHi =
1776 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1777 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1778 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1779
1780 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1781 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1782 }
1783
SplitVecRes_ScalarOp(SDNode * N,SDValue & Lo,SDValue & Hi)1784 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1785 SDValue &Hi) {
1786 EVT LoVT, HiVT;
1787 SDLoc dl(N);
1788 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1789 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1790 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1791 Hi = DAG.getUNDEF(HiVT);
1792 } else {
1793 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1794 Hi = Lo;
1795 }
1796 }
1797
SplitVecRes_LOAD(LoadSDNode * LD,SDValue & Lo,SDValue & Hi)1798 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1799 SDValue &Hi) {
1800 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1801 EVT LoVT, HiVT;
1802 SDLoc dl(LD);
1803 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1804
1805 ISD::LoadExtType ExtType = LD->getExtensionType();
1806 SDValue Ch = LD->getChain();
1807 SDValue Ptr = LD->getBasePtr();
1808 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1809 EVT MemoryVT = LD->getMemoryVT();
1810 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1811 AAMDNodes AAInfo = LD->getAAInfo();
1812
1813 EVT LoMemVT, HiMemVT;
1814 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1815
1816 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1817 SDValue Value, NewChain;
1818 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1819 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1820 ReplaceValueWith(SDValue(LD, 1), NewChain);
1821 return;
1822 }
1823
1824 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1825 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1826 MMOFlags, AAInfo);
1827
1828 MachinePointerInfo MPI;
1829 IncrementPointer(LD, LoMemVT, MPI, Ptr);
1830
1831 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1832 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1833
1834 // Build a factor node to remember that this load is independent of the
1835 // other one.
1836 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1837 Hi.getValue(1));
1838
1839 // Legalize the chain result - switch anything that used the old chain to
1840 // use the new one.
1841 ReplaceValueWith(SDValue(LD, 1), Ch);
1842 }
1843
SplitVecRes_VP_LOAD(VPLoadSDNode * LD,SDValue & Lo,SDValue & Hi)1844 void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
1845 SDValue &Hi) {
1846 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
1847 EVT LoVT, HiVT;
1848 SDLoc dl(LD);
1849 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1850
1851 ISD::LoadExtType ExtType = LD->getExtensionType();
1852 SDValue Ch = LD->getChain();
1853 SDValue Ptr = LD->getBasePtr();
1854 SDValue Offset = LD->getOffset();
1855 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
1856 Align Alignment = LD->getOriginalAlign();
1857 SDValue Mask = LD->getMask();
1858 SDValue EVL = LD->getVectorLength();
1859 EVT MemoryVT = LD->getMemoryVT();
1860
1861 EVT LoMemVT, HiMemVT;
1862 bool HiIsEmpty = false;
1863 std::tie(LoMemVT, HiMemVT) =
1864 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1865
1866 // Split Mask operand
1867 SDValue MaskLo, MaskHi;
1868 if (Mask.getOpcode() == ISD::SETCC) {
1869 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1870 } else {
1871 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1872 GetSplitVector(Mask, MaskLo, MaskHi);
1873 else
1874 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1875 }
1876
1877 // Split EVL operand
1878 SDValue EVLLo, EVLHi;
1879 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl);
1880
1881 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1882 LD->getPointerInfo(), MachineMemOperand::MOLoad,
1883 MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges());
1884
1885 Lo =
1886 DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
1887 MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad());
1888
1889 if (HiIsEmpty) {
1890 // The hi vp_load has zero storage size. We therefore simply set it to
1891 // the low vp_load and rely on subsequent removal from the chain.
1892 Hi = Lo;
1893 } else {
1894 // Generate hi vp_load.
1895 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1896 LD->isExpandingLoad());
1897
1898 MachinePointerInfo MPI;
1899 if (LoMemVT.isScalableVector())
1900 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
1901 else
1902 MPI = LD->getPointerInfo().getWithOffset(
1903 LoMemVT.getStoreSize().getFixedSize());
1904
1905 MMO = DAG.getMachineFunction().getMachineMemOperand(
1906 MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
1907 LD->getAAInfo(), LD->getRanges());
1908
1909 Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
1910 Offset, MaskHi, EVLHi, HiMemVT, MMO,
1911 LD->isExpandingLoad());
1912 }
1913
1914 // Build a factor node to remember that this load is independent of the
1915 // other one.
1916 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1917 Hi.getValue(1));
1918
1919 // Legalize the chain result - switch anything that used the old chain to
1920 // use the new one.
1921 ReplaceValueWith(SDValue(LD, 1), Ch);
1922 }
1923
SplitVecRes_MLOAD(MaskedLoadSDNode * MLD,SDValue & Lo,SDValue & Hi)1924 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1925 SDValue &Lo, SDValue &Hi) {
1926 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
1927 EVT LoVT, HiVT;
1928 SDLoc dl(MLD);
1929 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1930
1931 SDValue Ch = MLD->getChain();
1932 SDValue Ptr = MLD->getBasePtr();
1933 SDValue Offset = MLD->getOffset();
1934 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
1935 SDValue Mask = MLD->getMask();
1936 SDValue PassThru = MLD->getPassThru();
1937 Align Alignment = MLD->getOriginalAlign();
1938 ISD::LoadExtType ExtType = MLD->getExtensionType();
1939
1940 // Split Mask operand
1941 SDValue MaskLo, MaskHi;
1942 if (Mask.getOpcode() == ISD::SETCC) {
1943 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1944 } else {
1945 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1946 GetSplitVector(Mask, MaskLo, MaskHi);
1947 else
1948 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1949 }
1950
1951 EVT MemoryVT = MLD->getMemoryVT();
1952 EVT LoMemVT, HiMemVT;
1953 bool HiIsEmpty = false;
1954 std::tie(LoMemVT, HiMemVT) =
1955 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1956
1957 SDValue PassThruLo, PassThruHi;
1958 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1959 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1960 else
1961 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1962
1963 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1964 MLD->getPointerInfo(), MachineMemOperand::MOLoad,
1965 MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(),
1966 MLD->getRanges());
1967
1968 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
1969 MMO, MLD->getAddressingMode(), ExtType,
1970 MLD->isExpandingLoad());
1971
1972 if (HiIsEmpty) {
1973 // The hi masked load has zero storage size. We therefore simply set it to
1974 // the low masked load and rely on subsequent removal from the chain.
1975 Hi = Lo;
1976 } else {
1977 // Generate hi masked load.
1978 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1979 MLD->isExpandingLoad());
1980
1981 MachinePointerInfo MPI;
1982 if (LoMemVT.isScalableVector())
1983 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
1984 else
1985 MPI = MLD->getPointerInfo().getWithOffset(
1986 LoMemVT.getStoreSize().getFixedSize());
1987
1988 MMO = DAG.getMachineFunction().getMachineMemOperand(
1989 MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
1990 MLD->getAAInfo(), MLD->getRanges());
1991
1992 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
1993 HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
1994 MLD->isExpandingLoad());
1995 }
1996
1997 // Build a factor node to remember that this load is independent of the
1998 // other one.
1999 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2000 Hi.getValue(1));
2001
2002 // Legalize the chain result - switch anything that used the old chain to
2003 // use the new one.
2004 ReplaceValueWith(SDValue(MLD, 1), Ch);
2005
2006 }
2007
SplitVecRes_Gather(MemSDNode * N,SDValue & Lo,SDValue & Hi,bool SplitSETCC)2008 void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2009 SDValue &Hi, bool SplitSETCC) {
2010 EVT LoVT, HiVT;
2011 SDLoc dl(N);
2012 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2013
2014 SDValue Ch = N->getChain();
2015 SDValue Ptr = N->getBasePtr();
2016 struct Operands {
2017 SDValue Mask;
2018 SDValue Index;
2019 SDValue Scale;
2020 } Ops = [&]() -> Operands {
2021 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) {
2022 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2023 }
2024 auto *VPSC = cast<VPGatherSDNode>(N);
2025 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2026 }();
2027
2028 EVT MemoryVT = N->getMemoryVT();
2029 Align Alignment = N->getOriginalAlign();
2030
2031 // Split Mask operand
2032 SDValue MaskLo, MaskHi;
2033 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2034 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2035 } else {
2036 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2037 }
2038
2039 EVT LoMemVT, HiMemVT;
2040 // Split MemoryVT
2041 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2042
2043 SDValue IndexHi, IndexLo;
2044 if (getTypeAction(Ops.Index.getValueType()) ==
2045 TargetLowering::TypeSplitVector)
2046 GetSplitVector(Ops.Index, IndexLo, IndexHi);
2047 else
2048 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl);
2049
2050 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2051 N->getPointerInfo(), MachineMemOperand::MOLoad,
2052 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2053
2054 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
2055 SDValue PassThru = MGT->getPassThru();
2056 SDValue PassThruLo, PassThruHi;
2057 if (getTypeAction(PassThru.getValueType()) ==
2058 TargetLowering::TypeSplitVector)
2059 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2060 else
2061 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2062
2063 ISD::LoadExtType ExtType = MGT->getExtensionType();
2064 ISD::MemIndexType IndexTy = MGT->getIndexType();
2065
2066 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2067 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2068 OpsLo, MMO, IndexTy, ExtType);
2069
2070 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2071 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2072 OpsHi, MMO, IndexTy, ExtType);
2073 } else {
2074 auto *VPGT = cast<VPGatherSDNode>(N);
2075 SDValue EVLLo, EVLHi;
2076 std::tie(EVLLo, EVLHi) =
2077 DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2078
2079 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2080 Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
2081 MMO, VPGT->getIndexType());
2082
2083 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2084 Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
2085 MMO, VPGT->getIndexType());
2086 }
2087
2088 // Build a factor node to remember that this load is independent of the
2089 // other one.
2090 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2091 Hi.getValue(1));
2092
2093 // Legalize the chain result - switch anything that used the old chain to
2094 // use the new one.
2095 ReplaceValueWith(SDValue(N, 1), Ch);
2096 }
2097
SplitVecRes_SETCC(SDNode * N,SDValue & Lo,SDValue & Hi)2098 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2099 assert(N->getValueType(0).isVector() &&
2100 N->getOperand(0).getValueType().isVector() &&
2101 "Operand types must be vectors");
2102
2103 EVT LoVT, HiVT;
2104 SDLoc DL(N);
2105 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2106
2107 // If the input also splits, handle it directly. Otherwise split it by hand.
2108 SDValue LL, LH, RL, RH;
2109 if (getTypeAction(N->getOperand(0).getValueType()) ==
2110 TargetLowering::TypeSplitVector)
2111 GetSplitVector(N->getOperand(0), LL, LH);
2112 else
2113 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
2114
2115 if (getTypeAction(N->getOperand(1).getValueType()) ==
2116 TargetLowering::TypeSplitVector)
2117 GetSplitVector(N->getOperand(1), RL, RH);
2118 else
2119 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
2120
2121 if (N->getOpcode() == ISD::SETCC) {
2122 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
2123 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
2124 } else {
2125 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2126 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2127 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
2128 std::tie(EVLLo, EVLHi) =
2129 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
2130 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2), MaskLo,
2131 EVLLo);
2132 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2), MaskHi,
2133 EVLHi);
2134 }
2135 }
2136
SplitVecRes_UnaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)2137 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2138 SDValue &Hi) {
2139 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2140 EVT LoVT, HiVT;
2141 SDLoc dl(N);
2142 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2143
2144 // If the input also splits, handle it directly for a compile time speedup.
2145 // Otherwise split it by hand.
2146 EVT InVT = N->getOperand(0).getValueType();
2147 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2148 GetSplitVector(N->getOperand(0), Lo, Hi);
2149 else
2150 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2151
2152 const SDNodeFlags Flags = N->getFlags();
2153 unsigned Opcode = N->getOpcode();
2154 if (N->getNumOperands() <= 2) {
2155 if (Opcode == ISD::FP_ROUND) {
2156 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags);
2157 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags);
2158 } else {
2159 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags);
2160 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags);
2161 }
2162 return;
2163 }
2164
2165 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2166 assert(N->isVPOpcode() && "Expected VP opcode");
2167
2168 SDValue MaskLo, MaskHi;
2169 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2170
2171 SDValue EVLLo, EVLHi;
2172 std::tie(EVLLo, EVLHi) =
2173 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2174
2175 Lo = DAG.getNode(Opcode, dl, LoVT, {Lo, MaskLo, EVLLo}, Flags);
2176 Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags);
2177 }
2178
SplitVecRes_ExtendOp(SDNode * N,SDValue & Lo,SDValue & Hi)2179 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2180 SDValue &Hi) {
2181 SDLoc dl(N);
2182 EVT SrcVT = N->getOperand(0).getValueType();
2183 EVT DestVT = N->getValueType(0);
2184 EVT LoVT, HiVT;
2185 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
2186
2187 // We can do better than a generic split operation if the extend is doing
2188 // more than just doubling the width of the elements and the following are
2189 // true:
2190 // - The number of vector elements is even,
2191 // - the source type is legal,
2192 // - the type of a split source is illegal,
2193 // - the type of an extended (by doubling element size) source is legal, and
2194 // - the type of that extended source when split is legal.
2195 //
2196 // This won't necessarily completely legalize the operation, but it will
2197 // more effectively move in the right direction and prevent falling down
2198 // to scalarization in many cases due to the input vector being split too
2199 // far.
2200 if (SrcVT.getVectorElementCount().isKnownEven() &&
2201 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2202 LLVMContext &Ctx = *DAG.getContext();
2203 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
2204 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
2205
2206 EVT SplitLoVT, SplitHiVT;
2207 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
2208 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
2209 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
2210 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2211 N->dump(&DAG); dbgs() << "\n");
2212 if (!N->isVPOpcode()) {
2213 // Extend the source vector by one step.
2214 SDValue NewSrc =
2215 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
2216 // Get the low and high halves of the new, extended one step, vector.
2217 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2218 // Extend those vector halves the rest of the way.
2219 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
2220 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
2221 return;
2222 }
2223
2224 // Extend the source vector by one step.
2225 SDValue NewSrc =
2226 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0),
2227 N->getOperand(1), N->getOperand(2));
2228 // Get the low and high halves of the new, extended one step, vector.
2229 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2230
2231 SDValue MaskLo, MaskHi;
2232 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2233
2234 SDValue EVLLo, EVLHi;
2235 std::tie(EVLLo, EVLHi) =
2236 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2237 // Extend those vector halves the rest of the way.
2238 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo});
2239 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi});
2240 return;
2241 }
2242 }
2243 // Fall back to the generic unary operator splitting otherwise.
2244 SplitVecRes_UnaryOp(N, Lo, Hi);
2245 }
2246
SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N,SDValue & Lo,SDValue & Hi)2247 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2248 SDValue &Lo, SDValue &Hi) {
2249 // The low and high parts of the original input give four input vectors.
2250 SDValue Inputs[4];
2251 SDLoc DL(N);
2252 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
2253 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
2254 EVT NewVT = Inputs[0].getValueType();
2255 unsigned NewElts = NewVT.getVectorNumElements();
2256
2257 auto &&IsConstant = [](const SDValue &N) {
2258 APInt SplatValue;
2259 return N.getResNo() == 0 &&
2260 (ISD::isConstantSplatVector(N.getNode(), SplatValue) ||
2261 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
2262 };
2263 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2264 SDValue &Input2,
2265 ArrayRef<int> Mask) {
2266 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2267 Input2->getOpcode() == ISD::BUILD_VECTOR &&
2268 "Expected build vector node.");
2269 EVT EltVT = NewVT.getVectorElementType();
2270 SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT));
2271 for (unsigned I = 0; I < NewElts; ++I) {
2272 if (Mask[I] == UndefMaskElem)
2273 continue;
2274 unsigned Idx = Mask[I];
2275 if (Idx >= NewElts)
2276 Ops[I] = Input2.getOperand(Idx - NewElts);
2277 else
2278 Ops[I] = Input1.getOperand(Idx);
2279 // Make the type of all elements the same as the element type.
2280 if (Ops[I].getValueType().bitsGT(EltVT))
2281 Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]);
2282 }
2283 return DAG.getBuildVector(NewVT, DL, Ops);
2284 };
2285
2286 // If Lo or Hi uses elements from at most two of the four input vectors, then
2287 // express it as a vector shuffle of those two inputs. Otherwise extract the
2288 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2289 SmallVector<int> OrigMask(N->getMask().begin(), N->getMask().end());
2290 // Try to pack incoming shuffles/inputs.
2291 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2292 &DL](SmallVectorImpl<int> &Mask) {
2293 // Check if all inputs are shuffles of the same operands or non-shuffles.
2294 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
2295 for (unsigned Idx = 0; Idx < array_lengthof(Inputs); ++Idx) {
2296 SDValue Input = Inputs[Idx];
2297 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode());
2298 if (!Shuffle ||
2299 Input.getOperand(0).getValueType() != Input.getValueType())
2300 continue;
2301 ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))]
2302 .push_back(Idx);
2303 ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))]
2304 .push_back(Idx);
2305 }
2306 for (auto &P : ShufflesIdxs) {
2307 if (P.second.size() < 2)
2308 continue;
2309 // Use shuffles operands instead of shuffles themselves.
2310 // 1. Adjust mask.
2311 for (int &Idx : Mask) {
2312 if (Idx == UndefMaskElem)
2313 continue;
2314 unsigned SrcRegIdx = Idx / NewElts;
2315 if (Inputs[SrcRegIdx].isUndef()) {
2316 Idx = UndefMaskElem;
2317 continue;
2318 }
2319 auto *Shuffle =
2320 dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
2321 if (!Shuffle || !is_contained(P.second, SrcRegIdx))
2322 continue;
2323 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2324 if (MaskElt == UndefMaskElem) {
2325 Idx = UndefMaskElem;
2326 continue;
2327 }
2328 Idx = MaskElt % NewElts +
2329 P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first
2330 ? 0
2331 : 1] *
2332 NewElts;
2333 }
2334 // 2. Update inputs.
2335 Inputs[P.second[0]] = P.first.first;
2336 Inputs[P.second[1]] = P.first.second;
2337 // Clear the pair data.
2338 P.second.clear();
2339 ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear();
2340 }
2341 // Check if any concat_vectors can be simplified.
2342 SmallBitVector UsedSubVector(2 * array_lengthof(Inputs));
2343 for (int &Idx : Mask) {
2344 if (Idx == UndefMaskElem)
2345 continue;
2346 unsigned SrcRegIdx = Idx / NewElts;
2347 if (Inputs[SrcRegIdx].isUndef()) {
2348 Idx = UndefMaskElem;
2349 continue;
2350 }
2351 TargetLowering::LegalizeTypeAction TypeAction =
2352 getTypeAction(Inputs[SrcRegIdx].getValueType());
2353 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
2354 Inputs[SrcRegIdx].getNumOperands() == 2 &&
2355 !Inputs[SrcRegIdx].getOperand(1).isUndef() &&
2356 (TypeAction == TargetLowering::TypeLegal ||
2357 TypeAction == TargetLowering::TypeWidenVector))
2358 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
2359 }
2360 if (UsedSubVector.count() > 1) {
2361 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
2362 for (unsigned I = 0; I < array_lengthof(Inputs); ++I) {
2363 if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1))
2364 continue;
2365 if (Pairs.empty() || Pairs.back().size() == 2)
2366 Pairs.emplace_back();
2367 if (UsedSubVector.test(2 * I)) {
2368 Pairs.back().emplace_back(I, 0);
2369 } else {
2370 assert(UsedSubVector.test(2 * I + 1) &&
2371 "Expected to be used one of the subvectors.");
2372 Pairs.back().emplace_back(I, 1);
2373 }
2374 }
2375 if (!Pairs.empty() && Pairs.front().size() > 1) {
2376 // Adjust mask.
2377 for (int &Idx : Mask) {
2378 if (Idx == UndefMaskElem)
2379 continue;
2380 unsigned SrcRegIdx = Idx / NewElts;
2381 auto *It = find_if(
2382 Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
2383 return Idxs.front().first == SrcRegIdx ||
2384 Idxs.back().first == SrcRegIdx;
2385 });
2386 if (It == Pairs.end())
2387 continue;
2388 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
2389 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2390 }
2391 // Adjust inputs.
2392 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2393 Inputs[Idxs.front().first] = DAG.getNode(
2394 ISD::CONCAT_VECTORS, DL,
2395 Inputs[Idxs.front().first].getValueType(),
2396 Inputs[Idxs.front().first].getOperand(Idxs.front().second),
2397 Inputs[Idxs.back().first].getOperand(Idxs.back().second));
2398 }
2399 }
2400 }
2401 bool Changed;
2402 do {
2403 // Try to remove extra shuffles (except broadcasts) and shuffles with the
2404 // reused operands.
2405 Changed = false;
2406 for (unsigned I = 0; I < array_lengthof(Inputs); ++I) {
2407 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode());
2408 if (!Shuffle)
2409 continue;
2410 if (Shuffle->getOperand(0).getValueType() != NewVT)
2411 continue;
2412 int Op = -1;
2413 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
2414 !Shuffle->isSplat()) {
2415 Op = 0;
2416 } else if (!Inputs[I].hasOneUse() &&
2417 !Shuffle->getOperand(1).isUndef()) {
2418 // Find the only used operand, if possible.
2419 for (int &Idx : Mask) {
2420 if (Idx == UndefMaskElem)
2421 continue;
2422 unsigned SrcRegIdx = Idx / NewElts;
2423 if (SrcRegIdx != I)
2424 continue;
2425 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2426 if (MaskElt == UndefMaskElem) {
2427 Idx = UndefMaskElem;
2428 continue;
2429 }
2430 int OpIdx = MaskElt / NewElts;
2431 if (Op == -1) {
2432 Op = OpIdx;
2433 continue;
2434 }
2435 if (Op != OpIdx) {
2436 Op = -1;
2437 break;
2438 }
2439 }
2440 }
2441 if (Op < 0) {
2442 // Try to check if one of the shuffle operands is used already.
2443 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2444 if (Shuffle->getOperand(OpIdx).isUndef())
2445 continue;
2446 auto *It = find(Inputs, Shuffle->getOperand(OpIdx));
2447 if (It == std::end(Inputs))
2448 continue;
2449 int FoundOp = std::distance(std::begin(Inputs), It);
2450 // Found that operand is used already.
2451 // 1. Fix the mask for the reused operand.
2452 for (int &Idx : Mask) {
2453 if (Idx == UndefMaskElem)
2454 continue;
2455 unsigned SrcRegIdx = Idx / NewElts;
2456 if (SrcRegIdx != I)
2457 continue;
2458 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2459 if (MaskElt == UndefMaskElem) {
2460 Idx = UndefMaskElem;
2461 continue;
2462 }
2463 int MaskIdx = MaskElt / NewElts;
2464 if (OpIdx == MaskIdx)
2465 Idx = MaskElt % NewElts + FoundOp * NewElts;
2466 }
2467 // 2. Set Op to the unused OpIdx.
2468 Op = (OpIdx + 1) % 2;
2469 break;
2470 }
2471 }
2472 if (Op >= 0) {
2473 Changed = true;
2474 Inputs[I] = Shuffle->getOperand(Op);
2475 // Adjust mask.
2476 for (int &Idx : Mask) {
2477 if (Idx == UndefMaskElem)
2478 continue;
2479 unsigned SrcRegIdx = Idx / NewElts;
2480 if (SrcRegIdx != I)
2481 continue;
2482 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2483 int OpIdx = MaskElt / NewElts;
2484 if (OpIdx != Op)
2485 continue;
2486 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2487 }
2488 }
2489 }
2490 } while (Changed);
2491 };
2492 TryPeekThroughShufflesInputs(OrigMask);
2493 // Proces unique inputs.
2494 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
2495 NewElts](SmallVectorImpl<int> &Mask) {
2496 SetVector<SDValue> UniqueInputs;
2497 SetVector<SDValue> UniqueConstantInputs;
2498 for (unsigned I = 0; I < array_lengthof(Inputs); ++I) {
2499 if (IsConstant(Inputs[I]))
2500 UniqueConstantInputs.insert(Inputs[I]);
2501 else if (!Inputs[I].isUndef())
2502 UniqueInputs.insert(Inputs[I]);
2503 }
2504 // Adjust mask in case of reused inputs. Also, need to insert constant
2505 // inputs at first, otherwise it affects the final outcome.
2506 if (UniqueInputs.size() != array_lengthof(Inputs)) {
2507 auto &&UniqueVec = UniqueInputs.takeVector();
2508 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
2509 unsigned ConstNum = UniqueConstantVec.size();
2510 for (int &Idx : Mask) {
2511 if (Idx == UndefMaskElem)
2512 continue;
2513 unsigned SrcRegIdx = Idx / NewElts;
2514 if (Inputs[SrcRegIdx].isUndef()) {
2515 Idx = UndefMaskElem;
2516 continue;
2517 }
2518 const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
2519 if (It != UniqueConstantVec.end()) {
2520 Idx = (Idx % NewElts) +
2521 NewElts * std::distance(UniqueConstantVec.begin(), It);
2522 assert(Idx >= 0 && "Expected defined mask idx.");
2523 continue;
2524 }
2525 const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]);
2526 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
2527 Idx = (Idx % NewElts) +
2528 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
2529 assert(Idx >= 0 && "Expected defined mask idx.");
2530 }
2531 copy(UniqueConstantVec, std::begin(Inputs));
2532 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
2533 }
2534 };
2535 MakeUniqueInputs(OrigMask);
2536 SDValue OrigInputs[4];
2537 copy(Inputs, std::begin(OrigInputs));
2538 for (unsigned High = 0; High < 2; ++High) {
2539 SDValue &Output = High ? Hi : Lo;
2540
2541 // Build a shuffle mask for the output, discovering on the fly which
2542 // input vectors to use as shuffle operands.
2543 unsigned FirstMaskIdx = High * NewElts;
2544 SmallVector<int> Mask(NewElts * array_lengthof(Inputs), UndefMaskElem);
2545 copy(makeArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
2546 assert(!Output && "Expected default initialized initial value.");
2547 TryPeekThroughShufflesInputs(Mask);
2548 MakeUniqueInputs(Mask);
2549 SDValue TmpInputs[4];
2550 copy(Inputs, std::begin(TmpInputs));
2551 // Track changes in the output registers.
2552 int UsedIdx = -1;
2553 bool SecondIteration = false;
2554 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
2555 if (UsedIdx < 0) {
2556 UsedIdx = Idx;
2557 return false;
2558 }
2559 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
2560 SecondIteration = true;
2561 return SecondIteration;
2562 };
2563 processShuffleMasks(
2564 Mask, array_lengthof(Inputs), array_lengthof(Inputs),
2565 /*NumOfUsedRegs=*/1,
2566 [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); },
2567 [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
2568 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
2569 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
2570 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
2571 else
2572 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx],
2573 DAG.getUNDEF(NewVT), Mask);
2574 Inputs[Idx] = Output;
2575 },
2576 [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
2577 &TmpInputs,
2578 &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) {
2579 if (AccumulateResults(Idx1)) {
2580 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2581 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2582 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2583 else
2584 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1],
2585 Inputs[Idx2], Mask);
2586 } else {
2587 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2588 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2589 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2590 else
2591 Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1],
2592 TmpInputs[Idx2], Mask);
2593 }
2594 Inputs[Idx1] = Output;
2595 });
2596 copy(OrigInputs, std::begin(Inputs));
2597 }
2598 }
2599
SplitVecRes_VAARG(SDNode * N,SDValue & Lo,SDValue & Hi)2600 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2601 EVT OVT = N->getValueType(0);
2602 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2603 SDValue Chain = N->getOperand(0);
2604 SDValue Ptr = N->getOperand(1);
2605 SDValue SV = N->getOperand(2);
2606 SDLoc dl(N);
2607
2608 const Align Alignment =
2609 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
2610
2611 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2612 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2613 Chain = Hi.getValue(1);
2614
2615 // Modified the chain - switch anything that used the old chain to use
2616 // the new one.
2617 ReplaceValueWith(SDValue(N, 1), Chain);
2618 }
2619
SplitVecRes_FP_TO_XINT_SAT(SDNode * N,SDValue & Lo,SDValue & Hi)2620 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2621 SDValue &Hi) {
2622 EVT DstVTLo, DstVTHi;
2623 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2624 SDLoc dl(N);
2625
2626 SDValue SrcLo, SrcHi;
2627 EVT SrcVT = N->getOperand(0).getValueType();
2628 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2629 GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2630 else
2631 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2632
2633 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2634 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2635 }
2636
SplitVecRes_VECTOR_REVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)2637 void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
2638 SDValue &Hi) {
2639 SDValue InLo, InHi;
2640 GetSplitVector(N->getOperand(0), InLo, InHi);
2641 SDLoc DL(N);
2642
2643 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
2644 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
2645 }
2646
SplitVecRes_VECTOR_SPLICE(SDNode * N,SDValue & Lo,SDValue & Hi)2647 void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
2648 SDValue &Hi) {
2649 EVT VT = N->getValueType(0);
2650 SDLoc DL(N);
2651
2652 EVT LoVT, HiVT;
2653 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2654
2655 SDValue Expanded = TLI.expandVectorSplice(N, DAG);
2656 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded,
2657 DAG.getVectorIdxConstant(0, DL));
2658 Hi =
2659 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded,
2660 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
2661 }
2662
2663 //===----------------------------------------------------------------------===//
2664 // Operand Vector Splitting
2665 //===----------------------------------------------------------------------===//
2666
2667 /// This method is called when the specified operand of the specified node is
2668 /// found to need vector splitting. At this point, all of the result types of
2669 /// the node are known to be legal, but other operands of the node may need
2670 /// legalization as well as the specified one.
SplitVectorOperand(SDNode * N,unsigned OpNo)2671 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2672 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2673 SDValue Res = SDValue();
2674
2675 // See if the target wants to custom split this node.
2676 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2677 return false;
2678
2679 switch (N->getOpcode()) {
2680 default:
2681 #ifndef NDEBUG
2682 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2683 N->dump(&DAG);
2684 dbgs() << "\n";
2685 #endif
2686 report_fatal_error("Do not know how to split this operator's "
2687 "operand!\n");
2688
2689 case ISD::VP_SETCC:
2690 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2691 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2692 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2693 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2694 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2695 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2696 case ISD::VP_TRUNCATE:
2697 case ISD::TRUNCATE:
2698 Res = SplitVecOp_TruncateHelper(N);
2699 break;
2700 case ISD::STRICT_FP_ROUND:
2701 case ISD::VP_FP_ROUND:
2702 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2703 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2704 case ISD::STORE:
2705 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2706 break;
2707 case ISD::VP_STORE:
2708 Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
2709 break;
2710 case ISD::MSTORE:
2711 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2712 break;
2713 case ISD::MSCATTER:
2714 case ISD::VP_SCATTER:
2715 Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo);
2716 break;
2717 case ISD::MGATHER:
2718 case ISD::VP_GATHER:
2719 Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo);
2720 break;
2721 case ISD::VSELECT:
2722 Res = SplitVecOp_VSELECT(N, OpNo);
2723 break;
2724 case ISD::STRICT_SINT_TO_FP:
2725 case ISD::STRICT_UINT_TO_FP:
2726 case ISD::SINT_TO_FP:
2727 case ISD::UINT_TO_FP:
2728 if (N->getValueType(0).bitsLT(
2729 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2730 Res = SplitVecOp_TruncateHelper(N);
2731 else
2732 Res = SplitVecOp_UnaryOp(N);
2733 break;
2734 case ISD::FP_TO_SINT_SAT:
2735 case ISD::FP_TO_UINT_SAT:
2736 Res = SplitVecOp_FP_TO_XINT_SAT(N);
2737 break;
2738 case ISD::FP_TO_SINT:
2739 case ISD::FP_TO_UINT:
2740 case ISD::STRICT_FP_TO_SINT:
2741 case ISD::STRICT_FP_TO_UINT:
2742 case ISD::STRICT_FP_EXTEND:
2743 case ISD::FP_EXTEND:
2744 case ISD::SIGN_EXTEND:
2745 case ISD::ZERO_EXTEND:
2746 case ISD::ANY_EXTEND:
2747 case ISD::FTRUNC:
2748 Res = SplitVecOp_UnaryOp(N);
2749 break;
2750
2751 case ISD::ANY_EXTEND_VECTOR_INREG:
2752 case ISD::SIGN_EXTEND_VECTOR_INREG:
2753 case ISD::ZERO_EXTEND_VECTOR_INREG:
2754 Res = SplitVecOp_ExtVecInRegOp(N);
2755 break;
2756
2757 case ISD::VECREDUCE_FADD:
2758 case ISD::VECREDUCE_FMUL:
2759 case ISD::VECREDUCE_ADD:
2760 case ISD::VECREDUCE_MUL:
2761 case ISD::VECREDUCE_AND:
2762 case ISD::VECREDUCE_OR:
2763 case ISD::VECREDUCE_XOR:
2764 case ISD::VECREDUCE_SMAX:
2765 case ISD::VECREDUCE_SMIN:
2766 case ISD::VECREDUCE_UMAX:
2767 case ISD::VECREDUCE_UMIN:
2768 case ISD::VECREDUCE_FMAX:
2769 case ISD::VECREDUCE_FMIN:
2770 Res = SplitVecOp_VECREDUCE(N, OpNo);
2771 break;
2772 case ISD::VECREDUCE_SEQ_FADD:
2773 case ISD::VECREDUCE_SEQ_FMUL:
2774 Res = SplitVecOp_VECREDUCE_SEQ(N);
2775 break;
2776 case ISD::VP_REDUCE_FADD:
2777 case ISD::VP_REDUCE_SEQ_FADD:
2778 case ISD::VP_REDUCE_FMUL:
2779 case ISD::VP_REDUCE_SEQ_FMUL:
2780 case ISD::VP_REDUCE_ADD:
2781 case ISD::VP_REDUCE_MUL:
2782 case ISD::VP_REDUCE_AND:
2783 case ISD::VP_REDUCE_OR:
2784 case ISD::VP_REDUCE_XOR:
2785 case ISD::VP_REDUCE_SMAX:
2786 case ISD::VP_REDUCE_SMIN:
2787 case ISD::VP_REDUCE_UMAX:
2788 case ISD::VP_REDUCE_UMIN:
2789 case ISD::VP_REDUCE_FMAX:
2790 case ISD::VP_REDUCE_FMIN:
2791 Res = SplitVecOp_VP_REDUCE(N, OpNo);
2792 break;
2793 }
2794
2795 // If the result is null, the sub-method took care of registering results etc.
2796 if (!Res.getNode()) return false;
2797
2798 // If the result is N, the sub-method updated N in place. Tell the legalizer
2799 // core about this.
2800 if (Res.getNode() == N)
2801 return true;
2802
2803 if (N->isStrictFPOpcode())
2804 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2805 "Invalid operand expansion");
2806 else
2807 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2808 "Invalid operand expansion");
2809
2810 ReplaceValueWith(SDValue(N, 0), Res);
2811 return false;
2812 }
2813
SplitVecOp_VSELECT(SDNode * N,unsigned OpNo)2814 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2815 // The only possibility for an illegal operand is the mask, since result type
2816 // legalization would have handled this node already otherwise.
2817 assert(OpNo == 0 && "Illegal operand must be mask");
2818
2819 SDValue Mask = N->getOperand(0);
2820 SDValue Src0 = N->getOperand(1);
2821 SDValue Src1 = N->getOperand(2);
2822 EVT Src0VT = Src0.getValueType();
2823 SDLoc DL(N);
2824 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2825
2826 SDValue Lo, Hi;
2827 GetSplitVector(N->getOperand(0), Lo, Hi);
2828 assert(Lo.getValueType() == Hi.getValueType() &&
2829 "Lo and Hi have differing types");
2830
2831 EVT LoOpVT, HiOpVT;
2832 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2833 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2834
2835 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2836 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2837 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2838 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2839
2840 SDValue LoSelect =
2841 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2842 SDValue HiSelect =
2843 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2844
2845 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2846 }
2847
SplitVecOp_VECREDUCE(SDNode * N,unsigned OpNo)2848 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2849 EVT ResVT = N->getValueType(0);
2850 SDValue Lo, Hi;
2851 SDLoc dl(N);
2852
2853 SDValue VecOp = N->getOperand(OpNo);
2854 EVT VecVT = VecOp.getValueType();
2855 assert(VecVT.isVector() && "Can only split reduce vector operand");
2856 GetSplitVector(VecOp, Lo, Hi);
2857 EVT LoOpVT, HiOpVT;
2858 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2859
2860 // Use the appropriate scalar instruction on the split subvectors before
2861 // reducing the now partially reduced smaller vector.
2862 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2863 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2864 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2865 }
2866
SplitVecOp_VECREDUCE_SEQ(SDNode * N)2867 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2868 EVT ResVT = N->getValueType(0);
2869 SDValue Lo, Hi;
2870 SDLoc dl(N);
2871
2872 SDValue AccOp = N->getOperand(0);
2873 SDValue VecOp = N->getOperand(1);
2874 SDNodeFlags Flags = N->getFlags();
2875
2876 EVT VecVT = VecOp.getValueType();
2877 assert(VecVT.isVector() && "Can only split reduce vector operand");
2878 GetSplitVector(VecOp, Lo, Hi);
2879 EVT LoOpVT, HiOpVT;
2880 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2881
2882 // Reduce low half.
2883 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2884
2885 // Reduce high half, using low half result as initial value.
2886 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
2887 }
2888
SplitVecOp_VP_REDUCE(SDNode * N,unsigned OpNo)2889 SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2890 assert(N->isVPOpcode() && "Expected VP opcode");
2891 assert(OpNo == 1 && "Can only split reduce vector operand");
2892
2893 unsigned Opc = N->getOpcode();
2894 EVT ResVT = N->getValueType(0);
2895 SDValue Lo, Hi;
2896 SDLoc dl(N);
2897
2898 SDValue VecOp = N->getOperand(OpNo);
2899 EVT VecVT = VecOp.getValueType();
2900 assert(VecVT.isVector() && "Can only split reduce vector operand");
2901 GetSplitVector(VecOp, Lo, Hi);
2902
2903 SDValue MaskLo, MaskHi;
2904 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
2905
2906 SDValue EVLLo, EVLHi;
2907 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl);
2908
2909 const SDNodeFlags Flags = N->getFlags();
2910
2911 SDValue ResLo =
2912 DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags);
2913 return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags);
2914 }
2915
SplitVecOp_UnaryOp(SDNode * N)2916 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2917 // The result has a legal vector type, but the input needs splitting.
2918 EVT ResVT = N->getValueType(0);
2919 SDValue Lo, Hi;
2920 SDLoc dl(N);
2921 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2922 EVT InVT = Lo.getValueType();
2923
2924 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2925 InVT.getVectorElementCount());
2926
2927 if (N->isStrictFPOpcode()) {
2928 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2929 { N->getOperand(0), Lo });
2930 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2931 { N->getOperand(0), Hi });
2932
2933 // Build a factor node to remember that this operation is independent
2934 // of the other one.
2935 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2936 Hi.getValue(1));
2937
2938 // Legalize the chain result - switch anything that used the old chain to
2939 // use the new one.
2940 ReplaceValueWith(SDValue(N, 1), Ch);
2941 } else if (N->getNumOperands() == 3) {
2942 assert(N->isVPOpcode() && "Expected VP opcode");
2943 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2944 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2945 std::tie(EVLLo, EVLHi) =
2946 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2947 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo, MaskLo, EVLLo);
2948 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi, MaskHi, EVLHi);
2949 } else {
2950 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2951 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2952 }
2953
2954 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2955 }
2956
SplitVecOp_BITCAST(SDNode * N)2957 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2958 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2959 // end up being split all the way down to individual components. Convert the
2960 // split pieces into integers and reassemble.
2961 SDValue Lo, Hi;
2962 GetSplitVector(N->getOperand(0), Lo, Hi);
2963 Lo = BitConvertToInteger(Lo);
2964 Hi = BitConvertToInteger(Hi);
2965
2966 if (DAG.getDataLayout().isBigEndian())
2967 std::swap(Lo, Hi);
2968
2969 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2970 JoinIntegers(Lo, Hi));
2971 }
2972
SplitVecOp_INSERT_SUBVECTOR(SDNode * N,unsigned OpNo)2973 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
2974 unsigned OpNo) {
2975 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
2976 // We know that the result type is legal.
2977 EVT ResVT = N->getValueType(0);
2978
2979 SDValue Vec = N->getOperand(0);
2980 SDValue SubVec = N->getOperand(1);
2981 SDValue Idx = N->getOperand(2);
2982 SDLoc dl(N);
2983
2984 SDValue Lo, Hi;
2985 GetSplitVector(SubVec, Lo, Hi);
2986
2987 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2988 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2989
2990 SDValue FirstInsertion =
2991 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
2992 SDValue SecondInsertion =
2993 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
2994 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
2995
2996 return SecondInsertion;
2997 }
2998
SplitVecOp_EXTRACT_SUBVECTOR(SDNode * N)2999 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3000 // We know that the extracted result type is legal.
3001 EVT SubVT = N->getValueType(0);
3002
3003 SDValue Idx = N->getOperand(1);
3004 SDLoc dl(N);
3005 SDValue Lo, Hi;
3006
3007 if (SubVT.isScalableVector() !=
3008 N->getOperand(0).getValueType().isScalableVector())
3009 report_fatal_error("Extracting a fixed-length vector from an illegal "
3010 "scalable vector is not yet supported");
3011
3012 GetSplitVector(N->getOperand(0), Lo, Hi);
3013
3014 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3015 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3016
3017 if (IdxVal < LoElts) {
3018 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts &&
3019 "Extracted subvector crosses vector split!");
3020 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
3021 } else {
3022 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
3023 DAG.getVectorIdxConstant(IdxVal - LoElts, dl));
3024 }
3025 }
3026
SplitVecOp_EXTRACT_VECTOR_ELT(SDNode * N)3027 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3028 SDValue Vec = N->getOperand(0);
3029 SDValue Idx = N->getOperand(1);
3030 EVT VecVT = Vec.getValueType();
3031
3032 if (isa<ConstantSDNode>(Idx)) {
3033 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3034
3035 SDValue Lo, Hi;
3036 GetSplitVector(Vec, Lo, Hi);
3037
3038 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3039
3040 if (IdxVal < LoElts)
3041 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
3042 else if (!Vec.getValueType().isScalableVector())
3043 return SDValue(DAG.UpdateNodeOperands(N, Hi,
3044 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
3045 Idx.getValueType())), 0);
3046 }
3047
3048 // See if the target wants to custom expand this node.
3049 if (CustomLowerNode(N, N->getValueType(0), true))
3050 return SDValue();
3051
3052 // Make the vector elements byte-addressable if they aren't already.
3053 SDLoc dl(N);
3054 EVT EltVT = VecVT.getVectorElementType();
3055 if (VecVT.getScalarSizeInBits() < 8) {
3056 EltVT = MVT::i8;
3057 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3058 VecVT.getVectorElementCount());
3059 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
3060 }
3061
3062 // Store the vector to the stack.
3063 // In cases where the vector is illegal it will be broken down into parts
3064 // and stored in parts - we should use the alignment for the smallest part.
3065 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3066 SDValue StackPtr =
3067 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3068 auto &MF = DAG.getMachineFunction();
3069 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3070 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3071 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3072 SmallestAlign);
3073
3074 // Load back the required element.
3075 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
3076
3077 // FIXME: This is to handle i1 vectors with elements promoted to i8.
3078 // i1 vector handling needs general improvement.
3079 if (N->getValueType(0).bitsLT(EltVT)) {
3080 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
3081 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3082 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
3083 }
3084
3085 return DAG.getExtLoad(
3086 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
3087 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
3088 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
3089 }
3090
SplitVecOp_ExtVecInRegOp(SDNode * N)3091 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
3092 SDValue Lo, Hi;
3093
3094 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
3095 // splitting the result has the same effect as splitting the input operand.
3096 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
3097
3098 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
3099 }
3100
SplitVecOp_Gather(MemSDNode * N,unsigned OpNo)3101 SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
3102 (void)OpNo;
3103 SDValue Lo, Hi;
3104 SplitVecRes_Gather(N, Lo, Hi);
3105
3106 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi);
3107 ReplaceValueWith(SDValue(N, 0), Res);
3108 return SDValue();
3109 }
3110
SplitVecOp_VP_STORE(VPStoreSDNode * N,unsigned OpNo)3111 SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
3112 assert(N->isUnindexed() && "Indexed vp_store of vector?");
3113 SDValue Ch = N->getChain();
3114 SDValue Ptr = N->getBasePtr();
3115 SDValue Offset = N->getOffset();
3116 assert(Offset.isUndef() && "Unexpected VP store offset");
3117 SDValue Mask = N->getMask();
3118 SDValue EVL = N->getVectorLength();
3119 SDValue Data = N->getValue();
3120 Align Alignment = N->getOriginalAlign();
3121 SDLoc DL(N);
3122
3123 SDValue DataLo, DataHi;
3124 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3125 // Split Data operand
3126 GetSplitVector(Data, DataLo, DataHi);
3127 else
3128 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3129
3130 // Split Mask operand
3131 SDValue MaskLo, MaskHi;
3132 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3133 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3134 } else {
3135 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3136 GetSplitVector(Mask, MaskLo, MaskHi);
3137 else
3138 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3139 }
3140
3141 EVT MemoryVT = N->getMemoryVT();
3142 EVT LoMemVT, HiMemVT;
3143 bool HiIsEmpty = false;
3144 std::tie(LoMemVT, HiMemVT) =
3145 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3146
3147 // Split EVL
3148 SDValue EVLLo, EVLHi;
3149 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL);
3150
3151 SDValue Lo, Hi;
3152 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3153 N->getPointerInfo(), MachineMemOperand::MOStore,
3154 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3155
3156 Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
3157 N->getAddressingMode(), N->isTruncatingStore(),
3158 N->isCompressingStore());
3159
3160 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
3161 if (HiIsEmpty)
3162 return Lo;
3163
3164 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3165 N->isCompressingStore());
3166
3167 MachinePointerInfo MPI;
3168 if (LoMemVT.isScalableVector()) {
3169 Alignment = commonAlignment(Alignment,
3170 LoMemVT.getSizeInBits().getKnownMinSize() / 8);
3171 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3172 } else
3173 MPI = N->getPointerInfo().getWithOffset(
3174 LoMemVT.getStoreSize().getFixedSize());
3175
3176 MMO = DAG.getMachineFunction().getMachineMemOperand(
3177 MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
3178 N->getAAInfo(), N->getRanges());
3179
3180 Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
3181 N->getAddressingMode(), N->isTruncatingStore(),
3182 N->isCompressingStore());
3183
3184 // Build a factor node to remember that this store is independent of the
3185 // other one.
3186 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3187 }
3188
SplitVecOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)3189 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
3190 unsigned OpNo) {
3191 assert(N->isUnindexed() && "Indexed masked store of vector?");
3192 SDValue Ch = N->getChain();
3193 SDValue Ptr = N->getBasePtr();
3194 SDValue Offset = N->getOffset();
3195 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
3196 SDValue Mask = N->getMask();
3197 SDValue Data = N->getValue();
3198 Align Alignment = N->getOriginalAlign();
3199 SDLoc DL(N);
3200
3201 SDValue DataLo, DataHi;
3202 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3203 // Split Data operand
3204 GetSplitVector(Data, DataLo, DataHi);
3205 else
3206 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3207
3208 // Split Mask operand
3209 SDValue MaskLo, MaskHi;
3210 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3211 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3212 } else {
3213 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3214 GetSplitVector(Mask, MaskLo, MaskHi);
3215 else
3216 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3217 }
3218
3219 EVT MemoryVT = N->getMemoryVT();
3220 EVT LoMemVT, HiMemVT;
3221 bool HiIsEmpty = false;
3222 std::tie(LoMemVT, HiMemVT) =
3223 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3224
3225 SDValue Lo, Hi, Res;
3226 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3227 N->getPointerInfo(), MachineMemOperand::MOStore,
3228 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3229
3230 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
3231 N->getAddressingMode(), N->isTruncatingStore(),
3232 N->isCompressingStore());
3233
3234 if (HiIsEmpty) {
3235 // The hi masked store has zero storage size.
3236 // Only the lo masked store is needed.
3237 Res = Lo;
3238 } else {
3239
3240 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3241 N->isCompressingStore());
3242
3243 MachinePointerInfo MPI;
3244 if (LoMemVT.isScalableVector()) {
3245 Alignment = commonAlignment(
3246 Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
3247 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3248 } else
3249 MPI = N->getPointerInfo().getWithOffset(
3250 LoMemVT.getStoreSize().getFixedSize());
3251
3252 MMO = DAG.getMachineFunction().getMachineMemOperand(
3253 MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
3254 N->getAAInfo(), N->getRanges());
3255
3256 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
3257 N->getAddressingMode(), N->isTruncatingStore(),
3258 N->isCompressingStore());
3259
3260 // Build a factor node to remember that this store is independent of the
3261 // other one.
3262 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3263 }
3264
3265 return Res;
3266 }
3267
SplitVecOp_Scatter(MemSDNode * N,unsigned OpNo)3268 SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
3269 SDValue Ch = N->getChain();
3270 SDValue Ptr = N->getBasePtr();
3271 EVT MemoryVT = N->getMemoryVT();
3272 Align Alignment = N->getOriginalAlign();
3273 SDLoc DL(N);
3274 struct Operands {
3275 SDValue Mask;
3276 SDValue Index;
3277 SDValue Scale;
3278 SDValue Data;
3279 } Ops = [&]() -> Operands {
3280 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3281 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
3282 MSC->getValue()};
3283 }
3284 auto *VPSC = cast<VPScatterSDNode>(N);
3285 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
3286 VPSC->getValue()};
3287 }();
3288 // Split all operands
3289
3290 EVT LoMemVT, HiMemVT;
3291 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3292
3293 SDValue DataLo, DataHi;
3294 if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
3295 // Split Data operand
3296 GetSplitVector(Ops.Data, DataLo, DataHi);
3297 else
3298 std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL);
3299
3300 // Split Mask operand
3301 SDValue MaskLo, MaskHi;
3302 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
3303 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
3304 } else {
3305 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL);
3306 }
3307
3308 SDValue IndexHi, IndexLo;
3309 if (getTypeAction(Ops.Index.getValueType()) ==
3310 TargetLowering::TypeSplitVector)
3311 GetSplitVector(Ops.Index, IndexLo, IndexHi);
3312 else
3313 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL);
3314
3315 SDValue Lo;
3316 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3317 N->getPointerInfo(), MachineMemOperand::MOStore,
3318 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3319
3320 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3321 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
3322 Lo =
3323 DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3324 MSC->getIndexType(), MSC->isTruncatingStore());
3325
3326 // The order of the Scatter operation after split is well defined. The "Hi"
3327 // part comes after the "Lo". So these two operations should be chained one
3328 // after another.
3329 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
3330 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi,
3331 MMO, MSC->getIndexType(),
3332 MSC->isTruncatingStore());
3333 }
3334 auto *VPSC = cast<VPScatterSDNode>(N);
3335 SDValue EVLLo, EVLHi;
3336 std::tie(EVLLo, EVLHi) =
3337 DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL);
3338
3339 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3340 Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3341 VPSC->getIndexType());
3342
3343 // The order of the Scatter operation after split is well defined. The "Hi"
3344 // part comes after the "Lo". So these two operations should be chained one
3345 // after another.
3346 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3347 return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO,
3348 VPSC->getIndexType());
3349 }
3350
SplitVecOp_STORE(StoreSDNode * N,unsigned OpNo)3351 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
3352 assert(N->isUnindexed() && "Indexed store of vector?");
3353 assert(OpNo == 1 && "Can only split the stored value");
3354 SDLoc DL(N);
3355
3356 bool isTruncating = N->isTruncatingStore();
3357 SDValue Ch = N->getChain();
3358 SDValue Ptr = N->getBasePtr();
3359 EVT MemoryVT = N->getMemoryVT();
3360 Align Alignment = N->getOriginalAlign();
3361 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3362 AAMDNodes AAInfo = N->getAAInfo();
3363 SDValue Lo, Hi;
3364 GetSplitVector(N->getOperand(1), Lo, Hi);
3365
3366 EVT LoMemVT, HiMemVT;
3367 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3368
3369 // Scalarize if the split halves are not byte-sized.
3370 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
3371 return TLI.scalarizeVectorStore(N, DAG);
3372
3373 if (isTruncating)
3374 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
3375 Alignment, MMOFlags, AAInfo);
3376 else
3377 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3378 AAInfo);
3379
3380 MachinePointerInfo MPI;
3381 IncrementPointer(N, LoMemVT, MPI, Ptr);
3382
3383 if (isTruncating)
3384 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
3385 HiMemVT, Alignment, MMOFlags, AAInfo);
3386 else
3387 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
3388
3389 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3390 }
3391
SplitVecOp_CONCAT_VECTORS(SDNode * N)3392 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
3393 SDLoc DL(N);
3394
3395 // The input operands all must have the same type, and we know the result
3396 // type is valid. Convert this to a buildvector which extracts all the
3397 // input elements.
3398 // TODO: If the input elements are power-two vectors, we could convert this to
3399 // a new CONCAT_VECTORS node with elements that are half-wide.
3400 SmallVector<SDValue, 32> Elts;
3401 EVT EltVT = N->getValueType(0).getVectorElementType();
3402 for (const SDValue &Op : N->op_values()) {
3403 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
3404 i != e; ++i) {
3405 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
3406 DAG.getVectorIdxConstant(i, DL)));
3407 }
3408 }
3409
3410 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
3411 }
3412
SplitVecOp_TruncateHelper(SDNode * N)3413 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
3414 // The result type is legal, but the input type is illegal. If splitting
3415 // ends up with the result type of each half still being legal, just
3416 // do that. If, however, that would result in an illegal result type,
3417 // we can try to get more clever with power-two vectors. Specifically,
3418 // split the input type, but also widen the result element size, then
3419 // concatenate the halves and truncate again. For example, consider a target
3420 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
3421 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
3422 // %inlo = v4i32 extract_subvector %in, 0
3423 // %inhi = v4i32 extract_subvector %in, 4
3424 // %lo16 = v4i16 trunc v4i32 %inlo
3425 // %hi16 = v4i16 trunc v4i32 %inhi
3426 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
3427 // %res = v8i8 trunc v8i16 %in16
3428 //
3429 // Without this transform, the original truncate would end up being
3430 // scalarized, which is pretty much always a last resort.
3431 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3432 SDValue InVec = N->getOperand(OpNo);
3433 EVT InVT = InVec->getValueType(0);
3434 EVT OutVT = N->getValueType(0);
3435 ElementCount NumElements = OutVT.getVectorElementCount();
3436 bool IsFloat = OutVT.isFloatingPoint();
3437
3438 unsigned InElementSize = InVT.getScalarSizeInBits();
3439 unsigned OutElementSize = OutVT.getScalarSizeInBits();
3440
3441 // Determine the split output VT. If its legal we can just split dirctly.
3442 EVT LoOutVT, HiOutVT;
3443 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
3444 assert(LoOutVT == HiOutVT && "Unequal split?");
3445
3446 // If the input elements are only 1/2 the width of the result elements,
3447 // just use the normal splitting. Our trick only work if there's room
3448 // to split more than once.
3449 if (isTypeLegal(LoOutVT) ||
3450 InElementSize <= OutElementSize * 2)
3451 return SplitVecOp_UnaryOp(N);
3452 SDLoc DL(N);
3453
3454 // Don't touch if this will be scalarized.
3455 EVT FinalVT = InVT;
3456 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3457 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
3458
3459 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
3460 return SplitVecOp_UnaryOp(N);
3461
3462 // Get the split input vector.
3463 SDValue InLoVec, InHiVec;
3464 GetSplitVector(InVec, InLoVec, InHiVec);
3465
3466 // Truncate them to 1/2 the element size.
3467 //
3468 // This assumes the number of elements is a power of two; any vector that
3469 // isn't should be widened, not split.
3470 EVT HalfElementVT = IsFloat ?
3471 EVT::getFloatingPointVT(InElementSize/2) :
3472 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
3473 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
3474 NumElements.divideCoefficientBy(2));
3475
3476 SDValue HalfLo;
3477 SDValue HalfHi;
3478 SDValue Chain;
3479 if (N->isStrictFPOpcode()) {
3480 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3481 {N->getOperand(0), InLoVec});
3482 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3483 {N->getOperand(0), InHiVec});
3484 // Legalize the chain result - switch anything that used the old chain to
3485 // use the new one.
3486 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
3487 HalfHi.getValue(1));
3488 } else {
3489 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
3490 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
3491 }
3492
3493 // Concatenate them to get the full intermediate truncation result.
3494 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
3495 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
3496 HalfHi);
3497 // Now finish up by truncating all the way down to the original result
3498 // type. This should normally be something that ends up being legal directly,
3499 // but in theory if a target has very wide vectors and an annoyingly
3500 // restricted set of legal types, this split can chain to build things up.
3501
3502 if (N->isStrictFPOpcode()) {
3503 SDValue Res = DAG.getNode(
3504 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
3505 {Chain, InterVec,
3506 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
3507 // Relink the chain
3508 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
3509 return Res;
3510 }
3511
3512 return IsFloat
3513 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
3514 DAG.getTargetConstant(
3515 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
3516 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
3517 }
3518
SplitVecOp_VSETCC(SDNode * N)3519 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
3520 assert(N->getValueType(0).isVector() &&
3521 N->getOperand(0).getValueType().isVector() &&
3522 "Operand types must be vectors");
3523 // The result has a legal vector type, but the input needs splitting.
3524 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3525 SDLoc DL(N);
3526 GetSplitVector(N->getOperand(0), Lo0, Hi0);
3527 GetSplitVector(N->getOperand(1), Lo1, Hi1);
3528 auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
3529
3530 LLVMContext &Context = *DAG.getContext();
3531 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
3532 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
3533
3534 if (N->getOpcode() == ISD::SETCC) {
3535 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
3536 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
3537 } else {
3538 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
3539 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3540 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
3541 std::tie(EVLLo, EVLHi) =
3542 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
3543 LoRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Lo0, Lo1,
3544 N->getOperand(2), MaskLo, EVLLo);
3545 HiRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Hi0, Hi1,
3546 N->getOperand(2), MaskHi, EVLHi);
3547 }
3548 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
3549
3550 EVT OpVT = N->getOperand(0).getValueType();
3551 ISD::NodeType ExtendCode =
3552 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
3553 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
3554 }
3555
3556
SplitVecOp_FP_ROUND(SDNode * N)3557 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
3558 // The result has a legal vector type, but the input needs splitting.
3559 EVT ResVT = N->getValueType(0);
3560 SDValue Lo, Hi;
3561 SDLoc DL(N);
3562 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3563 EVT InVT = Lo.getValueType();
3564
3565 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3566 InVT.getVectorElementCount());
3567
3568 if (N->isStrictFPOpcode()) {
3569 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3570 { N->getOperand(0), Lo, N->getOperand(2) });
3571 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3572 { N->getOperand(0), Hi, N->getOperand(2) });
3573 // Legalize the chain result - switch anything that used the old chain to
3574 // use the new one.
3575 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3576 Lo.getValue(1), Hi.getValue(1));
3577 ReplaceValueWith(SDValue(N, 1), NewChain);
3578 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
3579 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3580 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3581 std::tie(EVLLo, EVLHi) =
3582 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), DL);
3583 Lo = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Lo, MaskLo, EVLLo);
3584 Hi = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Hi, MaskHi, EVLHi);
3585 } else {
3586 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
3587 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
3588 }
3589
3590 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
3591 }
3592
SplitVecOp_FCOPYSIGN(SDNode * N)3593 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
3594 // The result (and the first input) has a legal vector type, but the second
3595 // input needs splitting.
3596 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
3597 }
3598
SplitVecOp_FP_TO_XINT_SAT(SDNode * N)3599 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
3600 EVT ResVT = N->getValueType(0);
3601 SDValue Lo, Hi;
3602 SDLoc dl(N);
3603 GetSplitVector(N->getOperand(0), Lo, Hi);
3604 EVT InVT = Lo.getValueType();
3605
3606 EVT NewResVT =
3607 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3608 InVT.getVectorElementCount());
3609
3610 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
3611 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
3612
3613 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3614 }
3615
3616 //===----------------------------------------------------------------------===//
3617 // Result Vector Widening
3618 //===----------------------------------------------------------------------===//
3619
WidenVectorResult(SDNode * N,unsigned ResNo)3620 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
3621 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
3622 dbgs() << "\n");
3623
3624 // See if the target wants to custom widen this node.
3625 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
3626 return;
3627
3628 SDValue Res = SDValue();
3629
3630 auto unrollExpandedOp = [&]() {
3631 // We're going to widen this vector op to a legal type by padding with undef
3632 // elements. If the wide vector op is eventually going to be expanded to
3633 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3634 // libcalls on the undef elements.
3635 EVT VT = N->getValueType(0);
3636 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3637 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3638 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3639 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3640 return true;
3641 }
3642 return false;
3643 };
3644
3645 switch (N->getOpcode()) {
3646 default:
3647 #ifndef NDEBUG
3648 dbgs() << "WidenVectorResult #" << ResNo << ": ";
3649 N->dump(&DAG);
3650 dbgs() << "\n";
3651 #endif
3652 llvm_unreachable("Do not know how to widen the result of this operator!");
3653
3654 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
3655 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
3656 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
3657 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
3658 case ISD::INSERT_SUBVECTOR:
3659 Res = WidenVecRes_INSERT_SUBVECTOR(N);
3660 break;
3661 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
3662 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
3663 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
3664 case ISD::STEP_VECTOR:
3665 case ISD::SPLAT_VECTOR:
3666 case ISD::SCALAR_TO_VECTOR:
3667 Res = WidenVecRes_ScalarOp(N);
3668 break;
3669 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
3670 case ISD::VSELECT:
3671 case ISD::SELECT:
3672 case ISD::VP_SELECT:
3673 case ISD::VP_MERGE:
3674 Res = WidenVecRes_Select(N);
3675 break;
3676 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
3677 case ISD::VP_SETCC:
3678 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
3679 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
3680 case ISD::VECTOR_SHUFFLE:
3681 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
3682 break;
3683 case ISD::VP_LOAD:
3684 Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(N));
3685 break;
3686 case ISD::MLOAD:
3687 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3688 break;
3689 case ISD::MGATHER:
3690 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3691 break;
3692 case ISD::VP_GATHER:
3693 Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(N));
3694 break;
3695
3696 case ISD::ADD: case ISD::VP_ADD:
3697 case ISD::AND: case ISD::VP_AND:
3698 case ISD::MUL: case ISD::VP_MUL:
3699 case ISD::MULHS:
3700 case ISD::MULHU:
3701 case ISD::OR: case ISD::VP_OR:
3702 case ISD::SUB: case ISD::VP_SUB:
3703 case ISD::XOR: case ISD::VP_XOR:
3704 case ISD::SHL: case ISD::VP_SHL:
3705 case ISD::SRA: case ISD::VP_ASHR:
3706 case ISD::SRL: case ISD::VP_LSHR:
3707 case ISD::FMINNUM:
3708 case ISD::FMAXNUM:
3709 case ISD::FMINIMUM:
3710 case ISD::FMAXIMUM:
3711 case ISD::SMIN:
3712 case ISD::SMAX:
3713 case ISD::UMIN:
3714 case ISD::UMAX:
3715 case ISD::UADDSAT:
3716 case ISD::SADDSAT:
3717 case ISD::USUBSAT:
3718 case ISD::SSUBSAT:
3719 case ISD::SSHLSAT:
3720 case ISD::USHLSAT:
3721 case ISD::ROTL:
3722 case ISD::ROTR:
3723 case ISD::AVGFLOORS:
3724 case ISD::AVGFLOORU:
3725 case ISD::AVGCEILS:
3726 case ISD::AVGCEILU:
3727 // Vector-predicated binary op widening. Note that -- unlike the
3728 // unpredicated versions -- we don't have to worry about trapping on
3729 // operations like UDIV, FADD, etc., as we pass on the original vector
3730 // length parameter. This means the widened elements containing garbage
3731 // aren't active.
3732 case ISD::VP_SDIV:
3733 case ISD::VP_UDIV:
3734 case ISD::VP_SREM:
3735 case ISD::VP_UREM:
3736 case ISD::VP_FADD:
3737 case ISD::VP_FSUB:
3738 case ISD::VP_FMUL:
3739 case ISD::VP_FDIV:
3740 case ISD::VP_FREM:
3741 Res = WidenVecRes_Binary(N);
3742 break;
3743
3744 case ISD::FPOW:
3745 case ISD::FREM:
3746 if (unrollExpandedOp())
3747 break;
3748 // If the target has custom/legal support for the scalar FP intrinsic ops
3749 // (they are probably not destined to become libcalls), then widen those
3750 // like any other binary ops.
3751 LLVM_FALLTHROUGH;
3752
3753 case ISD::FADD:
3754 case ISD::FMUL:
3755 case ISD::FSUB:
3756 case ISD::FDIV:
3757 case ISD::SDIV:
3758 case ISD::UDIV:
3759 case ISD::SREM:
3760 case ISD::UREM:
3761 Res = WidenVecRes_BinaryCanTrap(N);
3762 break;
3763
3764 case ISD::SMULFIX:
3765 case ISD::SMULFIXSAT:
3766 case ISD::UMULFIX:
3767 case ISD::UMULFIXSAT:
3768 // These are binary operations, but with an extra operand that shouldn't
3769 // be widened (the scale).
3770 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
3771 break;
3772
3773 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
3774 case ISD::STRICT_##DAGN:
3775 #include "llvm/IR/ConstrainedOps.def"
3776 Res = WidenVecRes_StrictFP(N);
3777 break;
3778
3779 case ISD::UADDO:
3780 case ISD::SADDO:
3781 case ISD::USUBO:
3782 case ISD::SSUBO:
3783 case ISD::UMULO:
3784 case ISD::SMULO:
3785 Res = WidenVecRes_OverflowOp(N, ResNo);
3786 break;
3787
3788 case ISD::FCOPYSIGN:
3789 Res = WidenVecRes_FCOPYSIGN(N);
3790 break;
3791
3792 case ISD::IS_FPCLASS:
3793 Res = WidenVecRes_IS_FPCLASS(N);
3794 break;
3795
3796 case ISD::FPOWI:
3797 Res = WidenVecRes_POWI(N);
3798 break;
3799
3800 case ISD::ANY_EXTEND_VECTOR_INREG:
3801 case ISD::SIGN_EXTEND_VECTOR_INREG:
3802 case ISD::ZERO_EXTEND_VECTOR_INREG:
3803 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
3804 break;
3805
3806 case ISD::ANY_EXTEND:
3807 case ISD::FP_EXTEND:
3808 case ISD::VP_FP_EXTEND:
3809 case ISD::FP_ROUND:
3810 case ISD::VP_FP_ROUND:
3811 case ISD::FP_TO_SINT:
3812 case ISD::VP_FPTOSI:
3813 case ISD::FP_TO_UINT:
3814 case ISD::VP_FPTOUI:
3815 case ISD::SIGN_EXTEND:
3816 case ISD::VP_SIGN_EXTEND:
3817 case ISD::SINT_TO_FP:
3818 case ISD::VP_SITOFP:
3819 case ISD::VP_TRUNCATE:
3820 case ISD::TRUNCATE:
3821 case ISD::UINT_TO_FP:
3822 case ISD::VP_UITOFP:
3823 case ISD::ZERO_EXTEND:
3824 case ISD::VP_ZERO_EXTEND:
3825 Res = WidenVecRes_Convert(N);
3826 break;
3827
3828 case ISD::FP_TO_SINT_SAT:
3829 case ISD::FP_TO_UINT_SAT:
3830 Res = WidenVecRes_FP_TO_XINT_SAT(N);
3831 break;
3832
3833 case ISD::FABS:
3834 case ISD::FCEIL:
3835 case ISD::FCOS:
3836 case ISD::FEXP:
3837 case ISD::FEXP2:
3838 case ISD::FFLOOR:
3839 case ISD::FLOG:
3840 case ISD::FLOG10:
3841 case ISD::FLOG2:
3842 case ISD::FNEARBYINT:
3843 case ISD::FRINT:
3844 case ISD::FROUND:
3845 case ISD::FROUNDEVEN:
3846 case ISD::FSIN:
3847 case ISD::FSQRT:
3848 case ISD::FTRUNC:
3849 if (unrollExpandedOp())
3850 break;
3851 // If the target has custom/legal support for the scalar FP intrinsic ops
3852 // (they are probably not destined to become libcalls), then widen those
3853 // like any other unary ops.
3854 LLVM_FALLTHROUGH;
3855
3856 case ISD::ABS:
3857 case ISD::BITREVERSE:
3858 case ISD::BSWAP:
3859 case ISD::CTLZ:
3860 case ISD::CTLZ_ZERO_UNDEF:
3861 case ISD::CTPOP:
3862 case ISD::CTTZ:
3863 case ISD::CTTZ_ZERO_UNDEF:
3864 case ISD::FNEG: case ISD::VP_FNEG:
3865 case ISD::FREEZE:
3866 case ISD::ARITH_FENCE:
3867 case ISD::FCANONICALIZE:
3868 Res = WidenVecRes_Unary(N);
3869 break;
3870 case ISD::FMA: case ISD::VP_FMA:
3871 case ISD::FSHL:
3872 case ISD::FSHR:
3873 Res = WidenVecRes_Ternary(N);
3874 break;
3875 }
3876
3877 // If Res is null, the sub-method took care of registering the result.
3878 if (Res.getNode())
3879 SetWidenedVector(SDValue(N, ResNo), Res);
3880 }
3881
WidenVecRes_Ternary(SDNode * N)3882 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
3883 // Ternary op widening.
3884 SDLoc dl(N);
3885 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3886 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3887 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3888 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
3889 if (N->getNumOperands() == 3)
3890 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
3891
3892 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
3893 assert(N->isVPOpcode() && "Expected VP opcode");
3894
3895 SDValue Mask =
3896 GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount());
3897 return DAG.getNode(N->getOpcode(), dl, WidenVT,
3898 {InOp1, InOp2, InOp3, Mask, N->getOperand(4)});
3899 }
3900
WidenVecRes_Binary(SDNode * N)3901 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
3902 // Binary op widening.
3903 SDLoc dl(N);
3904 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3905 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3906 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3907 if (N->getNumOperands() == 2)
3908 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2,
3909 N->getFlags());
3910
3911 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
3912 assert(N->isVPOpcode() && "Expected VP opcode");
3913
3914 SDValue Mask =
3915 GetWidenedMask(N->getOperand(2), WidenVT.getVectorElementCount());
3916 return DAG.getNode(N->getOpcode(), dl, WidenVT,
3917 {InOp1, InOp2, Mask, N->getOperand(3)}, N->getFlags());
3918 }
3919
WidenVecRes_BinaryWithExtraScalarOp(SDNode * N)3920 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
3921 // Binary op widening, but with an extra operand that shouldn't be widened.
3922 SDLoc dl(N);
3923 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3924 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3925 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3926 SDValue InOp3 = N->getOperand(2);
3927 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
3928 N->getFlags());
3929 }
3930
3931 // Given a vector of operations that have been broken up to widen, see
3932 // if we can collect them together into the next widest legal VT. This
3933 // implementation is trap-safe.
CollectOpsToWiden(SelectionDAG & DAG,const TargetLowering & TLI,SmallVectorImpl<SDValue> & ConcatOps,unsigned ConcatEnd,EVT VT,EVT MaxVT,EVT WidenVT)3934 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
3935 SmallVectorImpl<SDValue> &ConcatOps,
3936 unsigned ConcatEnd, EVT VT, EVT MaxVT,
3937 EVT WidenVT) {
3938 // Check to see if we have a single operation with the widen type.
3939 if (ConcatEnd == 1) {
3940 VT = ConcatOps[0].getValueType();
3941 if (VT == WidenVT)
3942 return ConcatOps[0];
3943 }
3944
3945 SDLoc dl(ConcatOps[0]);
3946 EVT WidenEltVT = WidenVT.getVectorElementType();
3947
3948 // while (Some element of ConcatOps is not of type MaxVT) {
3949 // From the end of ConcatOps, collect elements of the same type and put
3950 // them into an op of the next larger supported type
3951 // }
3952 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
3953 int Idx = ConcatEnd - 1;
3954 VT = ConcatOps[Idx--].getValueType();
3955 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
3956 Idx--;
3957
3958 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
3959 EVT NextVT;
3960 do {
3961 NextSize *= 2;
3962 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
3963 } while (!TLI.isTypeLegal(NextVT));
3964
3965 if (!VT.isVector()) {
3966 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
3967 SDValue VecOp = DAG.getUNDEF(NextVT);
3968 unsigned NumToInsert = ConcatEnd - Idx - 1;
3969 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
3970 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
3971 ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
3972 }
3973 ConcatOps[Idx+1] = VecOp;
3974 ConcatEnd = Idx + 2;
3975 } else {
3976 // Vector type, create a CONCAT_VECTORS of type NextVT
3977 SDValue undefVec = DAG.getUNDEF(VT);
3978 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
3979 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
3980 unsigned RealVals = ConcatEnd - Idx - 1;
3981 unsigned SubConcatEnd = 0;
3982 unsigned SubConcatIdx = Idx + 1;
3983 while (SubConcatEnd < RealVals)
3984 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
3985 while (SubConcatEnd < OpsToConcat)
3986 SubConcatOps[SubConcatEnd++] = undefVec;
3987 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
3988 NextVT, SubConcatOps);
3989 ConcatEnd = SubConcatIdx + 1;
3990 }
3991 }
3992
3993 // Check to see if we have a single operation with the widen type.
3994 if (ConcatEnd == 1) {
3995 VT = ConcatOps[0].getValueType();
3996 if (VT == WidenVT)
3997 return ConcatOps[0];
3998 }
3999
4000 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
4001 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
4002 if (NumOps != ConcatEnd ) {
4003 SDValue UndefVal = DAG.getUNDEF(MaxVT);
4004 for (unsigned j = ConcatEnd; j < NumOps; ++j)
4005 ConcatOps[j] = UndefVal;
4006 }
4007 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4008 makeArrayRef(ConcatOps.data(), NumOps));
4009 }
4010
WidenVecRes_BinaryCanTrap(SDNode * N)4011 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
4012 // Binary op widening for operations that can trap.
4013 unsigned Opcode = N->getOpcode();
4014 SDLoc dl(N);
4015 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4016 EVT WidenEltVT = WidenVT.getVectorElementType();
4017 EVT VT = WidenVT;
4018 unsigned NumElts = VT.getVectorMinNumElements();
4019 const SDNodeFlags Flags = N->getFlags();
4020 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4021 NumElts = NumElts / 2;
4022 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4023 }
4024
4025 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
4026 // Operation doesn't trap so just widen as normal.
4027 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4028 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4029 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
4030 }
4031
4032 // FIXME: Improve support for scalable vectors.
4033 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
4034
4035 // No legal vector version so unroll the vector operation and then widen.
4036 if (NumElts == 1)
4037 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
4038
4039 // Since the operation can trap, apply operation on the original vector.
4040 EVT MaxVT = VT;
4041 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4042 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4043 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
4044
4045 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4046 unsigned ConcatEnd = 0; // Current ConcatOps index.
4047 int Idx = 0; // Current Idx into input vectors.
4048
4049 // NumElts := greatest legal vector size (at most WidenVT)
4050 // while (orig. vector has unhandled elements) {
4051 // take munches of size NumElts from the beginning and add to ConcatOps
4052 // NumElts := next smaller supported vector size or 1
4053 // }
4054 while (CurNumElts != 0) {
4055 while (CurNumElts >= NumElts) {
4056 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
4057 DAG.getVectorIdxConstant(Idx, dl));
4058 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
4059 DAG.getVectorIdxConstant(Idx, dl));
4060 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
4061 Idx += NumElts;
4062 CurNumElts -= NumElts;
4063 }
4064 do {
4065 NumElts = NumElts / 2;
4066 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4067 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
4068
4069 if (NumElts == 1) {
4070 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4071 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
4072 InOp1, DAG.getVectorIdxConstant(Idx, dl));
4073 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
4074 InOp2, DAG.getVectorIdxConstant(Idx, dl));
4075 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
4076 EOp1, EOp2, Flags);
4077 }
4078 CurNumElts = 0;
4079 }
4080 }
4081
4082 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4083 }
4084
WidenVecRes_StrictFP(SDNode * N)4085 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
4086 switch (N->getOpcode()) {
4087 case ISD::STRICT_FSETCC:
4088 case ISD::STRICT_FSETCCS:
4089 return WidenVecRes_STRICT_FSETCC(N);
4090 case ISD::STRICT_FP_EXTEND:
4091 case ISD::STRICT_FP_ROUND:
4092 case ISD::STRICT_FP_TO_SINT:
4093 case ISD::STRICT_FP_TO_UINT:
4094 case ISD::STRICT_SINT_TO_FP:
4095 case ISD::STRICT_UINT_TO_FP:
4096 return WidenVecRes_Convert_StrictFP(N);
4097 default:
4098 break;
4099 }
4100
4101 // StrictFP op widening for operations that can trap.
4102 unsigned NumOpers = N->getNumOperands();
4103 unsigned Opcode = N->getOpcode();
4104 SDLoc dl(N);
4105 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4106 EVT WidenEltVT = WidenVT.getVectorElementType();
4107 EVT VT = WidenVT;
4108 unsigned NumElts = VT.getVectorNumElements();
4109 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4110 NumElts = NumElts / 2;
4111 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4112 }
4113
4114 // No legal vector version so unroll the vector operation and then widen.
4115 if (NumElts == 1)
4116 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
4117
4118 // Since the operation can trap, apply operation on the original vector.
4119 EVT MaxVT = VT;
4120 SmallVector<SDValue, 4> InOps;
4121 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
4122
4123 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4124 SmallVector<SDValue, 16> Chains;
4125 unsigned ConcatEnd = 0; // Current ConcatOps index.
4126 int Idx = 0; // Current Idx into input vectors.
4127
4128 // The Chain is the first operand.
4129 InOps.push_back(N->getOperand(0));
4130
4131 // Now process the remaining operands.
4132 for (unsigned i = 1; i < NumOpers; ++i) {
4133 SDValue Oper = N->getOperand(i);
4134
4135 if (Oper.getValueType().isVector()) {
4136 assert(Oper.getValueType() == N->getValueType(0) &&
4137 "Invalid operand type to widen!");
4138 Oper = GetWidenedVector(Oper);
4139 }
4140
4141 InOps.push_back(Oper);
4142 }
4143
4144 // NumElts := greatest legal vector size (at most WidenVT)
4145 // while (orig. vector has unhandled elements) {
4146 // take munches of size NumElts from the beginning and add to ConcatOps
4147 // NumElts := next smaller supported vector size or 1
4148 // }
4149 while (CurNumElts != 0) {
4150 while (CurNumElts >= NumElts) {
4151 SmallVector<SDValue, 4> EOps;
4152
4153 for (unsigned i = 0; i < NumOpers; ++i) {
4154 SDValue Op = InOps[i];
4155
4156 if (Op.getValueType().isVector())
4157 Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
4158 DAG.getVectorIdxConstant(Idx, dl));
4159
4160 EOps.push_back(Op);
4161 }
4162
4163 EVT OperVT[] = {VT, MVT::Other};
4164 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
4165 ConcatOps[ConcatEnd++] = Oper;
4166 Chains.push_back(Oper.getValue(1));
4167 Idx += NumElts;
4168 CurNumElts -= NumElts;
4169 }
4170 do {
4171 NumElts = NumElts / 2;
4172 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4173 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
4174
4175 if (NumElts == 1) {
4176 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4177 SmallVector<SDValue, 4> EOps;
4178
4179 for (unsigned i = 0; i < NumOpers; ++i) {
4180 SDValue Op = InOps[i];
4181
4182 if (Op.getValueType().isVector())
4183 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
4184 DAG.getVectorIdxConstant(Idx, dl));
4185
4186 EOps.push_back(Op);
4187 }
4188
4189 EVT WidenVT[] = {WidenEltVT, MVT::Other};
4190 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
4191 ConcatOps[ConcatEnd++] = Oper;
4192 Chains.push_back(Oper.getValue(1));
4193 }
4194 CurNumElts = 0;
4195 }
4196 }
4197
4198 // Build a factor node to remember all the Ops that have been created.
4199 SDValue NewChain;
4200 if (Chains.size() == 1)
4201 NewChain = Chains[0];
4202 else
4203 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4204 ReplaceValueWith(SDValue(N, 1), NewChain);
4205
4206 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4207 }
4208
WidenVecRes_OverflowOp(SDNode * N,unsigned ResNo)4209 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
4210 SDLoc DL(N);
4211 EVT ResVT = N->getValueType(0);
4212 EVT OvVT = N->getValueType(1);
4213 EVT WideResVT, WideOvVT;
4214 SDValue WideLHS, WideRHS;
4215
4216 // TODO: This might result in a widen/split loop.
4217 if (ResNo == 0) {
4218 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
4219 WideOvVT = EVT::getVectorVT(
4220 *DAG.getContext(), OvVT.getVectorElementType(),
4221 WideResVT.getVectorNumElements());
4222
4223 WideLHS = GetWidenedVector(N->getOperand(0));
4224 WideRHS = GetWidenedVector(N->getOperand(1));
4225 } else {
4226 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
4227 WideResVT = EVT::getVectorVT(
4228 *DAG.getContext(), ResVT.getVectorElementType(),
4229 WideOvVT.getVectorNumElements());
4230
4231 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
4232 WideLHS = DAG.getNode(
4233 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
4234 N->getOperand(0), Zero);
4235 WideRHS = DAG.getNode(
4236 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
4237 N->getOperand(1), Zero);
4238 }
4239
4240 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
4241 SDNode *WideNode = DAG.getNode(
4242 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
4243
4244 // Replace the other vector result not being explicitly widened here.
4245 unsigned OtherNo = 1 - ResNo;
4246 EVT OtherVT = N->getValueType(OtherNo);
4247 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
4248 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
4249 } else {
4250 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
4251 SDValue OtherVal = DAG.getNode(
4252 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
4253 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
4254 }
4255
4256 return SDValue(WideNode, ResNo);
4257 }
4258
WidenVecRes_Convert(SDNode * N)4259 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
4260 LLVMContext &Ctx = *DAG.getContext();
4261 SDValue InOp = N->getOperand(0);
4262 SDLoc DL(N);
4263
4264 EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
4265 ElementCount WidenEC = WidenVT.getVectorElementCount();
4266
4267 EVT InVT = InOp.getValueType();
4268
4269 unsigned Opcode = N->getOpcode();
4270 const SDNodeFlags Flags = N->getFlags();
4271
4272 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
4273 // not equal that of WidenVT.
4274 if (N->getOpcode() == ISD::ZERO_EXTEND &&
4275 getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
4276 TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
4277 WidenVT.getScalarSizeInBits()) {
4278 InOp = ZExtPromotedInteger(InOp);
4279 InVT = InOp.getValueType();
4280 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
4281 Opcode = ISD::TRUNCATE;
4282 }
4283
4284 EVT InEltVT = InVT.getVectorElementType();
4285 EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenEC);
4286 ElementCount InVTEC = InVT.getVectorElementCount();
4287
4288 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4289 InOp = GetWidenedVector(N->getOperand(0));
4290 InVT = InOp.getValueType();
4291 InVTEC = InVT.getVectorElementCount();
4292 if (InVTEC == WidenEC) {
4293 if (N->getNumOperands() == 1)
4294 return DAG.getNode(Opcode, DL, WidenVT, InOp);
4295 if (N->getNumOperands() == 3) {
4296 assert(N->isVPOpcode() && "Expected VP opcode");
4297 SDValue Mask =
4298 GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount());
4299 return DAG.getNode(Opcode, DL, WidenVT, InOp, Mask, N->getOperand(2));
4300 }
4301 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
4302 }
4303 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
4304 // If both input and result vector types are of same width, extend
4305 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
4306 // accepts fewer elements in the result than in the input.
4307 if (Opcode == ISD::ANY_EXTEND)
4308 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4309 if (Opcode == ISD::SIGN_EXTEND)
4310 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4311 if (Opcode == ISD::ZERO_EXTEND)
4312 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4313 }
4314 }
4315
4316 if (TLI.isTypeLegal(InWidenVT)) {
4317 // Because the result and the input are different vector types, widening
4318 // the result could create a legal type but widening the input might make
4319 // it an illegal type that might lead to repeatedly splitting the input
4320 // and then widening it. To avoid this, we widen the input only if
4321 // it results in a legal type.
4322 if (WidenEC.isKnownMultipleOf(InVTEC.getKnownMinValue())) {
4323 // Widen the input and call convert on the widened input vector.
4324 unsigned NumConcat =
4325 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
4326 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
4327 Ops[0] = InOp;
4328 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
4329 if (N->getNumOperands() == 1)
4330 return DAG.getNode(Opcode, DL, WidenVT, InVec);
4331 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
4332 }
4333
4334 if (InVTEC.isKnownMultipleOf(WidenEC.getKnownMinValue())) {
4335 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
4336 DAG.getVectorIdxConstant(0, DL));
4337 // Extract the input and convert the shorten input vector.
4338 if (N->getNumOperands() == 1)
4339 return DAG.getNode(Opcode, DL, WidenVT, InVal);
4340 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
4341 }
4342 }
4343
4344 // Otherwise unroll into some nasty scalar code and rebuild the vector.
4345 EVT EltVT = WidenVT.getVectorElementType();
4346 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(EltVT));
4347 // Use the original element count so we don't do more scalar opts than
4348 // necessary.
4349 unsigned MinElts = N->getValueType(0).getVectorNumElements();
4350 for (unsigned i=0; i < MinElts; ++i) {
4351 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
4352 DAG.getVectorIdxConstant(i, DL));
4353 if (N->getNumOperands() == 1)
4354 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
4355 else
4356 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
4357 }
4358
4359 return DAG.getBuildVector(WidenVT, DL, Ops);
4360 }
4361
WidenVecRes_FP_TO_XINT_SAT(SDNode * N)4362 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
4363 SDLoc dl(N);
4364 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4365 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
4366
4367 SDValue Src = N->getOperand(0);
4368 EVT SrcVT = Src.getValueType();
4369
4370 // Also widen the input.
4371 if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
4372 Src = GetWidenedVector(Src);
4373 SrcVT = Src.getValueType();
4374 }
4375
4376 // Input and output not widened to the same size, give up.
4377 if (WidenNumElts != SrcVT.getVectorElementCount())
4378 return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
4379
4380 return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1));
4381 }
4382
WidenVecRes_Convert_StrictFP(SDNode * N)4383 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
4384 SDValue InOp = N->getOperand(1);
4385 SDLoc DL(N);
4386 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4387
4388 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4389 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4390
4391 EVT InVT = InOp.getValueType();
4392 EVT InEltVT = InVT.getVectorElementType();
4393
4394 unsigned Opcode = N->getOpcode();
4395
4396 // FIXME: Optimizations need to be implemented here.
4397
4398 // Otherwise unroll into some nasty scalar code and rebuild the vector.
4399 EVT EltVT = WidenVT.getVectorElementType();
4400 std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
4401 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
4402 SmallVector<SDValue, 32> OpChains;
4403 // Use the original element count so we don't do more scalar opts than
4404 // necessary.
4405 unsigned MinElts = N->getValueType(0).getVectorNumElements();
4406 for (unsigned i=0; i < MinElts; ++i) {
4407 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
4408 DAG.getVectorIdxConstant(i, DL));
4409 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
4410 OpChains.push_back(Ops[i].getValue(1));
4411 }
4412 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
4413 ReplaceValueWith(SDValue(N, 1), NewChain);
4414
4415 return DAG.getBuildVector(WidenVT, DL, Ops);
4416 }
4417
WidenVecRes_EXTEND_VECTOR_INREG(SDNode * N)4418 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
4419 unsigned Opcode = N->getOpcode();
4420 SDValue InOp = N->getOperand(0);
4421 SDLoc DL(N);
4422
4423 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4424 EVT WidenSVT = WidenVT.getVectorElementType();
4425 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4426
4427 EVT InVT = InOp.getValueType();
4428 EVT InSVT = InVT.getVectorElementType();
4429 unsigned InVTNumElts = InVT.getVectorNumElements();
4430
4431 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4432 InOp = GetWidenedVector(InOp);
4433 InVT = InOp.getValueType();
4434 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
4435 switch (Opcode) {
4436 case ISD::ANY_EXTEND_VECTOR_INREG:
4437 case ISD::SIGN_EXTEND_VECTOR_INREG:
4438 case ISD::ZERO_EXTEND_VECTOR_INREG:
4439 return DAG.getNode(Opcode, DL, WidenVT, InOp);
4440 }
4441 }
4442 }
4443
4444 // Unroll, extend the scalars and rebuild the vector.
4445 SmallVector<SDValue, 16> Ops;
4446 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
4447 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
4448 DAG.getVectorIdxConstant(i, DL));
4449 switch (Opcode) {
4450 case ISD::ANY_EXTEND_VECTOR_INREG:
4451 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
4452 break;
4453 case ISD::SIGN_EXTEND_VECTOR_INREG:
4454 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
4455 break;
4456 case ISD::ZERO_EXTEND_VECTOR_INREG:
4457 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
4458 break;
4459 default:
4460 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
4461 }
4462 Ops.push_back(Val);
4463 }
4464
4465 while (Ops.size() != WidenNumElts)
4466 Ops.push_back(DAG.getUNDEF(WidenSVT));
4467
4468 return DAG.getBuildVector(WidenVT, DL, Ops);
4469 }
4470
WidenVecRes_FCOPYSIGN(SDNode * N)4471 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
4472 // If this is an FCOPYSIGN with same input types, we can treat it as a
4473 // normal (can trap) binary op.
4474 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
4475 return WidenVecRes_BinaryCanTrap(N);
4476
4477 // If the types are different, fall back to unrolling.
4478 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4479 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
4480 }
4481
WidenVecRes_IS_FPCLASS(SDNode * N)4482 SDValue DAGTypeLegalizer::WidenVecRes_IS_FPCLASS(SDNode *N) {
4483 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4484 SDValue Arg = GetWidenedVector(N->getOperand(0));
4485 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, {Arg, N->getOperand(1)},
4486 N->getFlags());
4487 }
4488
WidenVecRes_POWI(SDNode * N)4489 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
4490 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4491 SDValue InOp = GetWidenedVector(N->getOperand(0));
4492 SDValue ShOp = N->getOperand(1);
4493 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
4494 }
4495
WidenVecRes_Unary(SDNode * N)4496 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
4497 // Unary op widening.
4498 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4499 SDValue InOp = GetWidenedVector(N->getOperand(0));
4500 if (N->getNumOperands() == 1)
4501 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
4502
4503 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
4504 assert(N->isVPOpcode() && "Expected VP opcode");
4505
4506 SDValue Mask =
4507 GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount());
4508 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT,
4509 {InOp, Mask, N->getOperand(2)});
4510 }
4511
WidenVecRes_InregOp(SDNode * N)4512 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
4513 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4514 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
4515 cast<VTSDNode>(N->getOperand(1))->getVT()
4516 .getVectorElementType(),
4517 WidenVT.getVectorNumElements());
4518 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
4519 return DAG.getNode(N->getOpcode(), SDLoc(N),
4520 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
4521 }
4522
WidenVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)4523 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
4524 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
4525 return GetWidenedVector(WidenVec);
4526 }
4527
WidenVecRes_BITCAST(SDNode * N)4528 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
4529 SDValue InOp = N->getOperand(0);
4530 EVT InVT = InOp.getValueType();
4531 EVT VT = N->getValueType(0);
4532 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4533 SDLoc dl(N);
4534
4535 switch (getTypeAction(InVT)) {
4536 case TargetLowering::TypeLegal:
4537 break;
4538 case TargetLowering::TypeScalarizeScalableVector:
4539 report_fatal_error("Scalarization of scalable vectors is not supported.");
4540 case TargetLowering::TypePromoteInteger: {
4541 // If the incoming type is a vector that is being promoted, then
4542 // we know that the elements are arranged differently and that we
4543 // must perform the conversion using a stack slot.
4544 if (InVT.isVector())
4545 break;
4546
4547 // If the InOp is promoted to the same size, convert it. Otherwise,
4548 // fall out of the switch and widen the promoted input.
4549 SDValue NInOp = GetPromotedInteger(InOp);
4550 EVT NInVT = NInOp.getValueType();
4551 if (WidenVT.bitsEq(NInVT)) {
4552 // For big endian targets we need to shift the input integer or the
4553 // interesting bits will end up at the wrong place.
4554 if (DAG.getDataLayout().isBigEndian()) {
4555 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
4556 EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout());
4557 assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
4558 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
4559 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
4560 }
4561 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
4562 }
4563 InOp = NInOp;
4564 InVT = NInVT;
4565 break;
4566 }
4567 case TargetLowering::TypeSoftenFloat:
4568 case TargetLowering::TypePromoteFloat:
4569 case TargetLowering::TypeSoftPromoteHalf:
4570 case TargetLowering::TypeExpandInteger:
4571 case TargetLowering::TypeExpandFloat:
4572 case TargetLowering::TypeScalarizeVector:
4573 case TargetLowering::TypeSplitVector:
4574 break;
4575 case TargetLowering::TypeWidenVector:
4576 // If the InOp is widened to the same size, convert it. Otherwise, fall
4577 // out of the switch and widen the widened input.
4578 InOp = GetWidenedVector(InOp);
4579 InVT = InOp.getValueType();
4580 if (WidenVT.bitsEq(InVT))
4581 // The input widens to the same size. Convert to the widen value.
4582 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
4583 break;
4584 }
4585
4586 unsigned WidenSize = WidenVT.getSizeInBits();
4587 unsigned InSize = InVT.getSizeInBits();
4588 // x86mmx is not an acceptable vector element type, so don't try.
4589 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
4590 // Determine new input vector type. The new input vector type will use
4591 // the same element type (if its a vector) or use the input type as a
4592 // vector. It is the same size as the type to widen to.
4593 EVT NewInVT;
4594 unsigned NewNumElts = WidenSize / InSize;
4595 if (InVT.isVector()) {
4596 EVT InEltVT = InVT.getVectorElementType();
4597 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
4598 WidenSize / InEltVT.getSizeInBits());
4599 } else {
4600 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
4601 }
4602
4603 if (TLI.isTypeLegal(NewInVT)) {
4604 SDValue NewVec;
4605 if (InVT.isVector()) {
4606 // Because the result and the input are different vector types, widening
4607 // the result could create a legal type but widening the input might make
4608 // it an illegal type that might lead to repeatedly splitting the input
4609 // and then widening it. To avoid this, we widen the input only if
4610 // it results in a legal type.
4611 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
4612 Ops[0] = InOp;
4613
4614 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
4615 } else {
4616 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
4617 }
4618 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
4619 }
4620 }
4621
4622 return CreateStackStoreLoad(InOp, WidenVT);
4623 }
4624
WidenVecRes_BUILD_VECTOR(SDNode * N)4625 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
4626 SDLoc dl(N);
4627 // Build a vector with undefined for the new nodes.
4628 EVT VT = N->getValueType(0);
4629
4630 // Integer BUILD_VECTOR operands may be larger than the node's vector element
4631 // type. The UNDEFs need to have the same type as the existing operands.
4632 EVT EltVT = N->getOperand(0).getValueType();
4633 unsigned NumElts = VT.getVectorNumElements();
4634
4635 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4636 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4637
4638 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
4639 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
4640 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
4641
4642 return DAG.getBuildVector(WidenVT, dl, NewOps);
4643 }
4644
WidenVecRes_CONCAT_VECTORS(SDNode * N)4645 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
4646 EVT InVT = N->getOperand(0).getValueType();
4647 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4648 SDLoc dl(N);
4649 unsigned NumOperands = N->getNumOperands();
4650
4651 bool InputWidened = false; // Indicates we need to widen the input.
4652 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
4653 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
4654 unsigned NumInElts = InVT.getVectorMinNumElements();
4655 if (WidenNumElts % NumInElts == 0) {
4656 // Add undef vectors to widen to correct length.
4657 unsigned NumConcat = WidenNumElts / NumInElts;
4658 SDValue UndefVal = DAG.getUNDEF(InVT);
4659 SmallVector<SDValue, 16> Ops(NumConcat);
4660 for (unsigned i=0; i < NumOperands; ++i)
4661 Ops[i] = N->getOperand(i);
4662 for (unsigned i = NumOperands; i != NumConcat; ++i)
4663 Ops[i] = UndefVal;
4664 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
4665 }
4666 } else {
4667 InputWidened = true;
4668 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4669 // The inputs and the result are widen to the same value.
4670 unsigned i;
4671 for (i=1; i < NumOperands; ++i)
4672 if (!N->getOperand(i).isUndef())
4673 break;
4674
4675 if (i == NumOperands)
4676 // Everything but the first operand is an UNDEF so just return the
4677 // widened first operand.
4678 return GetWidenedVector(N->getOperand(0));
4679
4680 if (NumOperands == 2) {
4681 assert(!WidenVT.isScalableVector() &&
4682 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
4683 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4684 unsigned NumInElts = InVT.getVectorNumElements();
4685
4686 // Replace concat of two operands with a shuffle.
4687 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
4688 for (unsigned i = 0; i < NumInElts; ++i) {
4689 MaskOps[i] = i;
4690 MaskOps[i + NumInElts] = i + WidenNumElts;
4691 }
4692 return DAG.getVectorShuffle(WidenVT, dl,
4693 GetWidenedVector(N->getOperand(0)),
4694 GetWidenedVector(N->getOperand(1)),
4695 MaskOps);
4696 }
4697 }
4698 }
4699
4700 assert(!WidenVT.isScalableVector() &&
4701 "Cannot use build vectors to widen CONCAT_VECTOR result");
4702 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4703 unsigned NumInElts = InVT.getVectorNumElements();
4704
4705 // Fall back to use extracts and build vector.
4706 EVT EltVT = WidenVT.getVectorElementType();
4707 SmallVector<SDValue, 16> Ops(WidenNumElts);
4708 unsigned Idx = 0;
4709 for (unsigned i=0; i < NumOperands; ++i) {
4710 SDValue InOp = N->getOperand(i);
4711 if (InputWidened)
4712 InOp = GetWidenedVector(InOp);
4713 for (unsigned j = 0; j < NumInElts; ++j)
4714 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4715 DAG.getVectorIdxConstant(j, dl));
4716 }
4717 SDValue UndefVal = DAG.getUNDEF(EltVT);
4718 for (; Idx < WidenNumElts; ++Idx)
4719 Ops[Idx] = UndefVal;
4720 return DAG.getBuildVector(WidenVT, dl, Ops);
4721 }
4722
WidenVecRes_INSERT_SUBVECTOR(SDNode * N)4723 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
4724 EVT VT = N->getValueType(0);
4725 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4726 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4727 SDValue InOp2 = N->getOperand(1);
4728 SDValue Idx = N->getOperand(2);
4729 SDLoc dl(N);
4730 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WidenVT, InOp1, InOp2, Idx);
4731 }
4732
WidenVecRes_EXTRACT_SUBVECTOR(SDNode * N)4733 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
4734 EVT VT = N->getValueType(0);
4735 EVT EltVT = VT.getVectorElementType();
4736 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4737 SDValue InOp = N->getOperand(0);
4738 SDValue Idx = N->getOperand(1);
4739 SDLoc dl(N);
4740
4741 auto InOpTypeAction = getTypeAction(InOp.getValueType());
4742 if (InOpTypeAction == TargetLowering::TypeWidenVector)
4743 InOp = GetWidenedVector(InOp);
4744
4745 EVT InVT = InOp.getValueType();
4746
4747 // Check if we can just return the input vector after widening.
4748 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
4749 if (IdxVal == 0 && InVT == WidenVT)
4750 return InOp;
4751
4752 // Check if we can extract from the vector.
4753 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
4754 unsigned InNumElts = InVT.getVectorMinNumElements();
4755 unsigned VTNumElts = VT.getVectorMinNumElements();
4756 assert(IdxVal % VTNumElts == 0 &&
4757 "Expected Idx to be a multiple of subvector minimum vector length");
4758 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
4759 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
4760
4761 if (VT.isScalableVector()) {
4762 // Try to split the operation up into smaller extracts and concat the
4763 // results together, e.g.
4764 // nxv6i64 extract_subvector(nxv12i64, 6)
4765 // <->
4766 // nxv8i64 concat(
4767 // nxv2i64 extract_subvector(nxv16i64, 6)
4768 // nxv2i64 extract_subvector(nxv16i64, 8)
4769 // nxv2i64 extract_subvector(nxv16i64, 10)
4770 // undef)
4771 unsigned GCD = greatestCommonDivisor(VTNumElts, WidenNumElts);
4772 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
4773 "down type's element count");
4774 EVT PartVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
4775 ElementCount::getScalable(GCD));
4776 // Avoid recursion around e.g. nxv1i8.
4777 if (getTypeAction(PartVT) != TargetLowering::TypeWidenVector) {
4778 SmallVector<SDValue> Parts;
4779 unsigned I = 0;
4780 for (; I < VTNumElts / GCD; ++I)
4781 Parts.push_back(
4782 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, PartVT, InOp,
4783 DAG.getVectorIdxConstant(IdxVal + I * GCD, dl)));
4784 for (; I < WidenNumElts / GCD; ++I)
4785 Parts.push_back(DAG.getUNDEF(PartVT));
4786
4787 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Parts);
4788 }
4789
4790 report_fatal_error("Don't know how to widen the result of "
4791 "EXTRACT_SUBVECTOR for scalable vectors");
4792 }
4793
4794 // We could try widening the input to the right length but for now, extract
4795 // the original elements, fill the rest with undefs and build a vector.
4796 SmallVector<SDValue, 16> Ops(WidenNumElts);
4797 unsigned i;
4798 for (i = 0; i < VTNumElts; ++i)
4799 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4800 DAG.getVectorIdxConstant(IdxVal + i, dl));
4801
4802 SDValue UndefVal = DAG.getUNDEF(EltVT);
4803 for (; i < WidenNumElts; ++i)
4804 Ops[i] = UndefVal;
4805 return DAG.getBuildVector(WidenVT, dl, Ops);
4806 }
4807
WidenVecRes_INSERT_VECTOR_ELT(SDNode * N)4808 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
4809 SDValue InOp = GetWidenedVector(N->getOperand(0));
4810 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
4811 InOp.getValueType(), InOp,
4812 N->getOperand(1), N->getOperand(2));
4813 }
4814
WidenVecRes_LOAD(SDNode * N)4815 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
4816 LoadSDNode *LD = cast<LoadSDNode>(N);
4817 ISD::LoadExtType ExtType = LD->getExtensionType();
4818
4819 // A vector must always be stored in memory as-is, i.e. without any padding
4820 // between the elements, since various code depend on it, e.g. in the
4821 // handling of a bitcast of a vector type to int, which may be done with a
4822 // vector store followed by an integer load. A vector that does not have
4823 // elements that are byte-sized must therefore be stored as an integer
4824 // built out of the extracted vector elements.
4825 if (!LD->getMemoryVT().isByteSized()) {
4826 SDValue Value, NewChain;
4827 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
4828 ReplaceValueWith(SDValue(LD, 0), Value);
4829 ReplaceValueWith(SDValue(LD, 1), NewChain);
4830 return SDValue();
4831 }
4832
4833 SDValue Result;
4834 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
4835 if (ExtType != ISD::NON_EXTLOAD)
4836 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
4837 else
4838 Result = GenWidenVectorLoads(LdChain, LD);
4839
4840 if (Result) {
4841 // If we generate a single load, we can use that for the chain. Otherwise,
4842 // build a factor node to remember the multiple loads are independent and
4843 // chain to that.
4844 SDValue NewChain;
4845 if (LdChain.size() == 1)
4846 NewChain = LdChain[0];
4847 else
4848 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
4849
4850 // Modified the chain - switch anything that used the old chain to use
4851 // the new one.
4852 ReplaceValueWith(SDValue(N, 1), NewChain);
4853
4854 return Result;
4855 }
4856
4857 // Generate a vector-predicated load if it is custom/legal on the target. To
4858 // avoid possible recursion, only do this if the widened mask type is legal.
4859 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
4860 // removed from the IR by the ExpandVectorPredication pass but we're
4861 // reintroducing them here.
4862 EVT LdVT = LD->getMemoryVT();
4863 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), LdVT);
4864 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
4865 WideVT.getVectorElementCount());
4866 if (ExtType == ISD::NON_EXTLOAD && WideVT.isScalableVector() &&
4867 TLI.isOperationLegalOrCustom(ISD::VP_LOAD, WideVT) &&
4868 TLI.isTypeLegal(WideMaskVT)) {
4869 SDLoc DL(N);
4870 SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT);
4871 MVT EVLVT = TLI.getVPExplicitVectorLengthTy();
4872 unsigned NumVTElts = LdVT.getVectorMinNumElements();
4873 SDValue EVL =
4874 DAG.getVScale(DL, EVLVT, APInt(EVLVT.getScalarSizeInBits(), NumVTElts));
4875 const auto *MMO = LD->getMemOperand();
4876 SDValue NewLoad =
4877 DAG.getLoadVP(WideVT, DL, LD->getChain(), LD->getBasePtr(), Mask, EVL,
4878 MMO->getPointerInfo(), MMO->getAlign(), MMO->getFlags(),
4879 MMO->getAAInfo());
4880
4881 // Modified the chain - switch anything that used the old chain to use
4882 // the new one.
4883 ReplaceValueWith(SDValue(N, 1), NewLoad.getValue(1));
4884
4885 return NewLoad;
4886 }
4887
4888 report_fatal_error("Unable to widen vector load");
4889 }
4890
WidenVecRes_VP_LOAD(VPLoadSDNode * N)4891 SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
4892 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4893 SDValue Mask = N->getMask();
4894 SDValue EVL = N->getVectorLength();
4895 ISD::LoadExtType ExtType = N->getExtensionType();
4896 SDLoc dl(N);
4897
4898 // The mask should be widened as well
4899 assert(getTypeAction(Mask.getValueType()) ==
4900 TargetLowering::TypeWidenVector &&
4901 "Unable to widen binary VP op");
4902 Mask = GetWidenedVector(Mask);
4903 assert(Mask.getValueType().getVectorElementCount() ==
4904 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
4905 .getVectorElementCount() &&
4906 "Unable to widen vector load");
4907
4908 SDValue Res =
4909 DAG.getLoadVP(N->getAddressingMode(), ExtType, WidenVT, dl, N->getChain(),
4910 N->getBasePtr(), N->getOffset(), Mask, EVL,
4911 N->getMemoryVT(), N->getMemOperand(), N->isExpandingLoad());
4912 // Legalize the chain result - switch anything that used the old chain to
4913 // use the new one.
4914 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4915 return Res;
4916 }
4917
WidenVecRes_MLOAD(MaskedLoadSDNode * N)4918 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
4919
4920 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
4921 SDValue Mask = N->getMask();
4922 EVT MaskVT = Mask.getValueType();
4923 SDValue PassThru = GetWidenedVector(N->getPassThru());
4924 ISD::LoadExtType ExtType = N->getExtensionType();
4925 SDLoc dl(N);
4926
4927 // The mask should be widened as well
4928 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4929 MaskVT.getVectorElementType(),
4930 WidenVT.getVectorNumElements());
4931 Mask = ModifyToType(Mask, WideMaskVT, true);
4932
4933 SDValue Res = DAG.getMaskedLoad(
4934 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
4935 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
4936 ExtType, N->isExpandingLoad());
4937 // Legalize the chain result - switch anything that used the old chain to
4938 // use the new one.
4939 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4940 return Res;
4941 }
4942
WidenVecRes_MGATHER(MaskedGatherSDNode * N)4943 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
4944
4945 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4946 SDValue Mask = N->getMask();
4947 EVT MaskVT = Mask.getValueType();
4948 SDValue PassThru = GetWidenedVector(N->getPassThru());
4949 SDValue Scale = N->getScale();
4950 unsigned NumElts = WideVT.getVectorNumElements();
4951 SDLoc dl(N);
4952
4953 // The mask should be widened as well
4954 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4955 MaskVT.getVectorElementType(),
4956 WideVT.getVectorNumElements());
4957 Mask = ModifyToType(Mask, WideMaskVT, true);
4958
4959 // Widen the Index operand
4960 SDValue Index = N->getIndex();
4961 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4962 Index.getValueType().getScalarType(),
4963 NumElts);
4964 Index = ModifyToType(Index, WideIndexVT);
4965 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
4966 Scale };
4967
4968 // Widen the MemoryType
4969 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
4970 N->getMemoryVT().getScalarType(), NumElts);
4971 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
4972 WideMemVT, dl, Ops, N->getMemOperand(),
4973 N->getIndexType(), N->getExtensionType());
4974
4975 // Legalize the chain result - switch anything that used the old chain to
4976 // use the new one.
4977 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4978 return Res;
4979 }
4980
WidenVecRes_VP_GATHER(VPGatherSDNode * N)4981 SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
4982 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4983 SDValue Mask = N->getMask();
4984 SDValue Scale = N->getScale();
4985 ElementCount WideEC = WideVT.getVectorElementCount();
4986 SDLoc dl(N);
4987
4988 SDValue Index = GetWidenedVector(N->getIndex());
4989 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
4990 N->getMemoryVT().getScalarType(), WideEC);
4991 Mask = GetWidenedMask(Mask, WideEC);
4992
4993 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
4994 Mask, N->getVectorLength()};
4995 SDValue Res = DAG.getGatherVP(DAG.getVTList(WideVT, MVT::Other), WideMemVT,
4996 dl, Ops, N->getMemOperand(), N->getIndexType());
4997
4998 // Legalize the chain result - switch anything that used the old chain to
4999 // use the new one.
5000 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
5001 return Res;
5002 }
5003
WidenVecRes_ScalarOp(SDNode * N)5004 SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
5005 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5006 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
5007 }
5008
5009 // Return true is this is a SETCC node or a strict version of it.
isSETCCOp(unsigned Opcode)5010 static inline bool isSETCCOp(unsigned Opcode) {
5011 switch (Opcode) {
5012 case ISD::SETCC:
5013 case ISD::STRICT_FSETCC:
5014 case ISD::STRICT_FSETCCS:
5015 return true;
5016 }
5017 return false;
5018 }
5019
5020 // Return true if this is a node that could have two SETCCs as operands.
isLogicalMaskOp(unsigned Opcode)5021 static inline bool isLogicalMaskOp(unsigned Opcode) {
5022 switch (Opcode) {
5023 case ISD::AND:
5024 case ISD::OR:
5025 case ISD::XOR:
5026 return true;
5027 }
5028 return false;
5029 }
5030
5031 // If N is a SETCC or a strict variant of it, return the type
5032 // of the compare operands.
getSETCCOperandType(SDValue N)5033 static inline EVT getSETCCOperandType(SDValue N) {
5034 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
5035 return N->getOperand(OpNo).getValueType();
5036 }
5037
5038 // This is used just for the assert in convertMask(). Check that this either
5039 // a SETCC or a previously handled SETCC by convertMask().
5040 #ifndef NDEBUG
isSETCCorConvertedSETCC(SDValue N)5041 static inline bool isSETCCorConvertedSETCC(SDValue N) {
5042 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
5043 N = N.getOperand(0);
5044 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
5045 for (unsigned i = 1; i < N->getNumOperands(); ++i)
5046 if (!N->getOperand(i)->isUndef())
5047 return false;
5048 N = N.getOperand(0);
5049 }
5050
5051 if (N.getOpcode() == ISD::TRUNCATE)
5052 N = N.getOperand(0);
5053 else if (N.getOpcode() == ISD::SIGN_EXTEND)
5054 N = N.getOperand(0);
5055
5056 if (isLogicalMaskOp(N.getOpcode()))
5057 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
5058 isSETCCorConvertedSETCC(N.getOperand(1));
5059
5060 return (isSETCCOp(N.getOpcode()) ||
5061 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
5062 }
5063 #endif
5064
5065 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
5066 // to ToMaskVT if needed with vector extension or truncation.
convertMask(SDValue InMask,EVT MaskVT,EVT ToMaskVT)5067 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
5068 EVT ToMaskVT) {
5069 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
5070 // FIXME: This code seems to be too restrictive, we might consider
5071 // generalizing it or dropping it.
5072 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
5073
5074 // Make a new Mask node, with a legal result VT.
5075 SDValue Mask;
5076 SmallVector<SDValue, 4> Ops;
5077 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
5078 Ops.push_back(InMask->getOperand(i));
5079 if (InMask->isStrictFPOpcode()) {
5080 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
5081 { MaskVT, MVT::Other }, Ops);
5082 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
5083 }
5084 else
5085 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
5086
5087 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
5088 // extend or truncate is needed.
5089 LLVMContext &Ctx = *DAG.getContext();
5090 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
5091 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
5092 if (MaskScalarBits < ToMaskScalBits) {
5093 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
5094 MaskVT.getVectorNumElements());
5095 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
5096 } else if (MaskScalarBits > ToMaskScalBits) {
5097 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
5098 MaskVT.getVectorNumElements());
5099 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
5100 }
5101
5102 assert(Mask->getValueType(0).getScalarSizeInBits() ==
5103 ToMaskVT.getScalarSizeInBits() &&
5104 "Mask should have the right element size by now.");
5105
5106 // Adjust Mask to the right number of elements.
5107 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
5108 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
5109 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask));
5110 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
5111 ZeroIdx);
5112 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
5113 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
5114 EVT SubVT = Mask->getValueType(0);
5115 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
5116 SubOps[0] = Mask;
5117 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
5118 }
5119
5120 assert((Mask->getValueType(0) == ToMaskVT) &&
5121 "A mask of ToMaskVT should have been produced by now.");
5122
5123 return Mask;
5124 }
5125
5126 // This method tries to handle some special cases for the vselect mask
5127 // and if needed adjusting the mask vector type to match that of the VSELECT.
5128 // Without it, many cases end up with scalarization of the SETCC, with many
5129 // unnecessary instructions.
WidenVSELECTMask(SDNode * N)5130 SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
5131 LLVMContext &Ctx = *DAG.getContext();
5132 SDValue Cond = N->getOperand(0);
5133
5134 if (N->getOpcode() != ISD::VSELECT)
5135 return SDValue();
5136
5137 if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode()))
5138 return SDValue();
5139
5140 // If this is a splitted VSELECT that was previously already handled, do
5141 // nothing.
5142 EVT CondVT = Cond->getValueType(0);
5143 if (CondVT.getScalarSizeInBits() != 1)
5144 return SDValue();
5145
5146 EVT VSelVT = N->getValueType(0);
5147
5148 // This method can't handle scalable vector types.
5149 // FIXME: This support could be added in the future.
5150 if (VSelVT.isScalableVector())
5151 return SDValue();
5152
5153 // Only handle vector types which are a power of 2.
5154 if (!isPowerOf2_64(VSelVT.getSizeInBits()))
5155 return SDValue();
5156
5157 // Don't touch if this will be scalarized.
5158 EVT FinalVT = VSelVT;
5159 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
5160 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
5161
5162 if (FinalVT.getVectorNumElements() == 1)
5163 return SDValue();
5164
5165 // If there is support for an i1 vector mask, don't touch.
5166 if (isSETCCOp(Cond.getOpcode())) {
5167 EVT SetCCOpVT = getSETCCOperandType(Cond);
5168 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
5169 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
5170 EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
5171 if (SetCCResVT.getScalarSizeInBits() == 1)
5172 return SDValue();
5173 } else if (CondVT.getScalarType() == MVT::i1) {
5174 // If there is support for an i1 vector mask (or only scalar i1 conditions),
5175 // don't touch.
5176 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
5177 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
5178
5179 if (CondVT.getScalarType() == MVT::i1)
5180 return SDValue();
5181 }
5182
5183 // Widen the vselect result type if needed.
5184 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector)
5185 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
5186
5187 // The mask of the VSELECT should have integer elements.
5188 EVT ToMaskVT = VSelVT;
5189 if (!ToMaskVT.getScalarType().isInteger())
5190 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
5191
5192 SDValue Mask;
5193 if (isSETCCOp(Cond->getOpcode())) {
5194 EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond));
5195 Mask = convertMask(Cond, MaskVT, ToMaskVT);
5196 } else if (isLogicalMaskOp(Cond->getOpcode()) &&
5197 isSETCCOp(Cond->getOperand(0).getOpcode()) &&
5198 isSETCCOp(Cond->getOperand(1).getOpcode())) {
5199 // Cond is (AND/OR/XOR (SETCC, SETCC))
5200 SDValue SETCC0 = Cond->getOperand(0);
5201 SDValue SETCC1 = Cond->getOperand(1);
5202 EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0));
5203 EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1));
5204 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
5205 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
5206 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
5207 EVT MaskVT;
5208 // If the two SETCCs have different VTs, either extend/truncate one of
5209 // them to the other "towards" ToMaskVT, or truncate one and extend the
5210 // other to ToMaskVT.
5211 if (ScalarBits0 != ScalarBits1) {
5212 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
5213 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
5214 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
5215 MaskVT = WideVT;
5216 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
5217 MaskVT = NarrowVT;
5218 else
5219 MaskVT = ToMaskVT;
5220 } else
5221 // If the two SETCCs have the same VT, don't change it.
5222 MaskVT = VT0;
5223
5224 // Make new SETCCs and logical nodes.
5225 SETCC0 = convertMask(SETCC0, VT0, MaskVT);
5226 SETCC1 = convertMask(SETCC1, VT1, MaskVT);
5227 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
5228
5229 // Convert the logical op for VSELECT if needed.
5230 Mask = convertMask(Cond, MaskVT, ToMaskVT);
5231 } else
5232 return SDValue();
5233
5234 return Mask;
5235 }
5236
WidenVecRes_Select(SDNode * N)5237 SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
5238 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5239 ElementCount WidenEC = WidenVT.getVectorElementCount();
5240
5241 SDValue Cond1 = N->getOperand(0);
5242 EVT CondVT = Cond1.getValueType();
5243 unsigned Opcode = N->getOpcode();
5244 if (CondVT.isVector()) {
5245 if (SDValue WideCond = WidenVSELECTMask(N)) {
5246 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
5247 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
5248 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
5249 return DAG.getNode(Opcode, SDLoc(N), WidenVT, WideCond, InOp1, InOp2);
5250 }
5251
5252 EVT CondEltVT = CondVT.getVectorElementType();
5253 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenEC);
5254 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
5255 Cond1 = GetWidenedVector(Cond1);
5256
5257 // If we have to split the condition there is no point in widening the
5258 // select. This would result in an cycle of widening the select ->
5259 // widening the condition operand -> splitting the condition operand ->
5260 // splitting the select -> widening the select. Instead split this select
5261 // further and widen the resulting type.
5262 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
5263 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
5264 SDValue Res = ModifyToType(SplitSelect, WidenVT);
5265 return Res;
5266 }
5267
5268 if (Cond1.getValueType() != CondWidenVT)
5269 Cond1 = ModifyToType(Cond1, CondWidenVT);
5270 }
5271
5272 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
5273 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
5274 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
5275 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
5276 return DAG.getNode(Opcode, SDLoc(N), WidenVT, Cond1, InOp1, InOp2,
5277 N->getOperand(3));
5278 return DAG.getNode(Opcode, SDLoc(N), WidenVT, Cond1, InOp1, InOp2);
5279 }
5280
WidenVecRes_SELECT_CC(SDNode * N)5281 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
5282 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
5283 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
5284 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
5285 InOp1.getValueType(), N->getOperand(0),
5286 N->getOperand(1), InOp1, InOp2, N->getOperand(4));
5287 }
5288
WidenVecRes_UNDEF(SDNode * N)5289 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
5290 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5291 return DAG.getUNDEF(WidenVT);
5292 }
5293
WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N)5294 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
5295 EVT VT = N->getValueType(0);
5296 SDLoc dl(N);
5297
5298 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5299 unsigned NumElts = VT.getVectorNumElements();
5300 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5301
5302 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
5303 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
5304
5305 // Adjust mask based on new input vector length.
5306 SmallVector<int, 16> NewMask;
5307 for (unsigned i = 0; i != NumElts; ++i) {
5308 int Idx = N->getMaskElt(i);
5309 if (Idx < (int)NumElts)
5310 NewMask.push_back(Idx);
5311 else
5312 NewMask.push_back(Idx - NumElts + WidenNumElts);
5313 }
5314 for (unsigned i = NumElts; i != WidenNumElts; ++i)
5315 NewMask.push_back(-1);
5316 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
5317 }
5318
WidenVecRes_SETCC(SDNode * N)5319 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
5320 assert(N->getValueType(0).isVector() &&
5321 N->getOperand(0).getValueType().isVector() &&
5322 "Operands must be vectors");
5323 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5324 ElementCount WidenEC = WidenVT.getVectorElementCount();
5325
5326 SDValue InOp1 = N->getOperand(0);
5327 EVT InVT = InOp1.getValueType();
5328 assert(InVT.isVector() && "can not widen non-vector type");
5329 EVT WidenInVT =
5330 EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenEC);
5331
5332 // The input and output types often differ here, and it could be that while
5333 // we'd prefer to widen the result type, the input operands have been split.
5334 // In this case, we also need to split the result of this node as well.
5335 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
5336 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
5337 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
5338 return Res;
5339 }
5340
5341 // If the inputs also widen, handle them directly. Otherwise widen by hand.
5342 SDValue InOp2 = N->getOperand(1);
5343 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
5344 InOp1 = GetWidenedVector(InOp1);
5345 InOp2 = GetWidenedVector(InOp2);
5346 } else {
5347 InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
5348 InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
5349 }
5350
5351 // Assume that the input and output will be widen appropriately. If not,
5352 // we will have to unroll it at some point.
5353 assert(InOp1.getValueType() == WidenInVT &&
5354 InOp2.getValueType() == WidenInVT &&
5355 "Input not widened to expected type!");
5356 (void)WidenInVT;
5357 if (N->getOpcode() == ISD::VP_SETCC) {
5358 SDValue Mask =
5359 GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount());
5360 return DAG.getNode(ISD::VP_SETCC, SDLoc(N), WidenVT, InOp1, InOp2,
5361 N->getOperand(2), Mask, N->getOperand(4));
5362 }
5363 return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT, InOp1, InOp2,
5364 N->getOperand(2));
5365 }
5366
WidenVecRes_STRICT_FSETCC(SDNode * N)5367 SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
5368 assert(N->getValueType(0).isVector() &&
5369 N->getOperand(1).getValueType().isVector() &&
5370 "Operands must be vectors");
5371 EVT VT = N->getValueType(0);
5372 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5373 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5374 unsigned NumElts = VT.getVectorNumElements();
5375 EVT EltVT = VT.getVectorElementType();
5376
5377 SDLoc dl(N);
5378 SDValue Chain = N->getOperand(0);
5379 SDValue LHS = N->getOperand(1);
5380 SDValue RHS = N->getOperand(2);
5381 SDValue CC = N->getOperand(3);
5382 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
5383
5384 // Fully unroll and reassemble.
5385 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT));
5386 SmallVector<SDValue, 8> Chains(NumElts);
5387 for (unsigned i = 0; i != NumElts; ++i) {
5388 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
5389 DAG.getVectorIdxConstant(i, dl));
5390 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
5391 DAG.getVectorIdxConstant(i, dl));
5392
5393 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
5394 {Chain, LHSElem, RHSElem, CC});
5395 Chains[i] = Scalars[i].getValue(1);
5396 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
5397 DAG.getBoolConstant(true, dl, EltVT, VT),
5398 DAG.getBoolConstant(false, dl, EltVT, VT));
5399 }
5400
5401 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
5402 ReplaceValueWith(SDValue(N, 1), NewChain);
5403
5404 return DAG.getBuildVector(WidenVT, dl, Scalars);
5405 }
5406
5407 //===----------------------------------------------------------------------===//
5408 // Widen Vector Operand
5409 //===----------------------------------------------------------------------===//
WidenVectorOperand(SDNode * N,unsigned OpNo)5410 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
5411 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
5412 dbgs() << "\n");
5413 SDValue Res = SDValue();
5414
5415 // See if the target wants to custom widen this node.
5416 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
5417 return false;
5418
5419 switch (N->getOpcode()) {
5420 default:
5421 #ifndef NDEBUG
5422 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
5423 N->dump(&DAG);
5424 dbgs() << "\n";
5425 #endif
5426 llvm_unreachable("Do not know how to widen this operator's operand!");
5427
5428 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
5429 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
5430 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
5431 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
5432 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
5433 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
5434 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
5435 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
5436 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
5437 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
5438 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
5439 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
5440 case ISD::STRICT_FSETCC:
5441 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
5442 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
5443 case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
5444 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
5445
5446 case ISD::ANY_EXTEND:
5447 case ISD::SIGN_EXTEND:
5448 case ISD::ZERO_EXTEND:
5449 Res = WidenVecOp_EXTEND(N);
5450 break;
5451
5452 case ISD::FP_EXTEND:
5453 case ISD::STRICT_FP_EXTEND:
5454 case ISD::FP_ROUND:
5455 case ISD::STRICT_FP_ROUND:
5456 case ISD::FP_TO_SINT:
5457 case ISD::STRICT_FP_TO_SINT:
5458 case ISD::FP_TO_UINT:
5459 case ISD::STRICT_FP_TO_UINT:
5460 case ISD::SINT_TO_FP:
5461 case ISD::STRICT_SINT_TO_FP:
5462 case ISD::UINT_TO_FP:
5463 case ISD::STRICT_UINT_TO_FP:
5464 case ISD::TRUNCATE:
5465 Res = WidenVecOp_Convert(N);
5466 break;
5467
5468 case ISD::FP_TO_SINT_SAT:
5469 case ISD::FP_TO_UINT_SAT:
5470 Res = WidenVecOp_FP_TO_XINT_SAT(N);
5471 break;
5472
5473 case ISD::VECREDUCE_FADD:
5474 case ISD::VECREDUCE_FMUL:
5475 case ISD::VECREDUCE_ADD:
5476 case ISD::VECREDUCE_MUL:
5477 case ISD::VECREDUCE_AND:
5478 case ISD::VECREDUCE_OR:
5479 case ISD::VECREDUCE_XOR:
5480 case ISD::VECREDUCE_SMAX:
5481 case ISD::VECREDUCE_SMIN:
5482 case ISD::VECREDUCE_UMAX:
5483 case ISD::VECREDUCE_UMIN:
5484 case ISD::VECREDUCE_FMAX:
5485 case ISD::VECREDUCE_FMIN:
5486 Res = WidenVecOp_VECREDUCE(N);
5487 break;
5488 case ISD::VECREDUCE_SEQ_FADD:
5489 case ISD::VECREDUCE_SEQ_FMUL:
5490 Res = WidenVecOp_VECREDUCE_SEQ(N);
5491 break;
5492 case ISD::VP_REDUCE_FADD:
5493 case ISD::VP_REDUCE_SEQ_FADD:
5494 case ISD::VP_REDUCE_FMUL:
5495 case ISD::VP_REDUCE_SEQ_FMUL:
5496 case ISD::VP_REDUCE_ADD:
5497 case ISD::VP_REDUCE_MUL:
5498 case ISD::VP_REDUCE_AND:
5499 case ISD::VP_REDUCE_OR:
5500 case ISD::VP_REDUCE_XOR:
5501 case ISD::VP_REDUCE_SMAX:
5502 case ISD::VP_REDUCE_SMIN:
5503 case ISD::VP_REDUCE_UMAX:
5504 case ISD::VP_REDUCE_UMIN:
5505 case ISD::VP_REDUCE_FMAX:
5506 case ISD::VP_REDUCE_FMIN:
5507 Res = WidenVecOp_VP_REDUCE(N);
5508 break;
5509 }
5510
5511 // If Res is null, the sub-method took care of registering the result.
5512 if (!Res.getNode()) return false;
5513
5514 // If the result is N, the sub-method updated N in place. Tell the legalizer
5515 // core about this.
5516 if (Res.getNode() == N)
5517 return true;
5518
5519
5520 if (N->isStrictFPOpcode())
5521 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
5522 "Invalid operand expansion");
5523 else
5524 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
5525 "Invalid operand expansion");
5526
5527 ReplaceValueWith(SDValue(N, 0), Res);
5528 return false;
5529 }
5530
WidenVecOp_EXTEND(SDNode * N)5531 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
5532 SDLoc DL(N);
5533 EVT VT = N->getValueType(0);
5534
5535 SDValue InOp = N->getOperand(0);
5536 assert(getTypeAction(InOp.getValueType()) ==
5537 TargetLowering::TypeWidenVector &&
5538 "Unexpected type action");
5539 InOp = GetWidenedVector(InOp);
5540 assert(VT.getVectorNumElements() <
5541 InOp.getValueType().getVectorNumElements() &&
5542 "Input wasn't widened!");
5543
5544 // We may need to further widen the operand until it has the same total
5545 // vector size as the result.
5546 EVT InVT = InOp.getValueType();
5547 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
5548 EVT InEltVT = InVT.getVectorElementType();
5549 for (EVT FixedVT : MVT::vector_valuetypes()) {
5550 EVT FixedEltVT = FixedVT.getVectorElementType();
5551 if (TLI.isTypeLegal(FixedVT) &&
5552 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
5553 FixedEltVT == InEltVT) {
5554 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
5555 "Not enough elements in the fixed type for the operand!");
5556 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
5557 "We can't have the same type as we started with!");
5558 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
5559 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT,
5560 DAG.getUNDEF(FixedVT), InOp,
5561 DAG.getVectorIdxConstant(0, DL));
5562 else
5563 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
5564 DAG.getVectorIdxConstant(0, DL));
5565 break;
5566 }
5567 }
5568 InVT = InOp.getValueType();
5569 if (InVT.getSizeInBits() != VT.getSizeInBits())
5570 // We couldn't find a legal vector type that was a widening of the input
5571 // and could be extended in-register to the result type, so we have to
5572 // scalarize.
5573 return WidenVecOp_Convert(N);
5574 }
5575
5576 // Use special DAG nodes to represent the operation of extending the
5577 // low lanes.
5578 switch (N->getOpcode()) {
5579 default:
5580 llvm_unreachable("Extend legalization on extend operation!");
5581 case ISD::ANY_EXTEND:
5582 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
5583 case ISD::SIGN_EXTEND:
5584 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
5585 case ISD::ZERO_EXTEND:
5586 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
5587 }
5588 }
5589
WidenVecOp_FCOPYSIGN(SDNode * N)5590 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
5591 // The result (and first input) is legal, but the second input is illegal.
5592 // We can't do much to fix that, so just unroll and let the extracts off of
5593 // the second input be widened as needed later.
5594 return DAG.UnrollVectorOp(N);
5595 }
5596
WidenVecOp_IS_FPCLASS(SDNode * N)5597 SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
5598 SDLoc DL(N);
5599 EVT ResultVT = N->getValueType(0);
5600 SDValue Test = N->getOperand(1);
5601 SDValue WideArg = GetWidenedVector(N->getOperand(0));
5602
5603 // Process this node similarly to SETCC.
5604 EVT WideResultVT = getSetCCResultType(WideArg.getValueType());
5605 if (ResultVT.getScalarType() == MVT::i1)
5606 WideResultVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
5607 WideResultVT.getVectorNumElements());
5608
5609 SDValue WideNode = DAG.getNode(ISD::IS_FPCLASS, DL, WideResultVT,
5610 {WideArg, Test}, N->getFlags());
5611
5612 // Extract the needed results from the result vector.
5613 EVT ResVT =
5614 EVT::getVectorVT(*DAG.getContext(), WideResultVT.getVectorElementType(),
5615 ResultVT.getVectorNumElements());
5616 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, WideNode,
5617 DAG.getVectorIdxConstant(0, DL));
5618
5619 EVT OpVT = N->getOperand(0).getValueType();
5620 ISD::NodeType ExtendCode =
5621 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
5622 return DAG.getNode(ExtendCode, DL, ResultVT, CC);
5623 }
5624
WidenVecOp_Convert(SDNode * N)5625 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
5626 // Since the result is legal and the input is illegal.
5627 EVT VT = N->getValueType(0);
5628 EVT EltVT = VT.getVectorElementType();
5629 SDLoc dl(N);
5630 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
5631 assert(getTypeAction(InOp.getValueType()) ==
5632 TargetLowering::TypeWidenVector &&
5633 "Unexpected type action");
5634 InOp = GetWidenedVector(InOp);
5635 EVT InVT = InOp.getValueType();
5636 unsigned Opcode = N->getOpcode();
5637
5638 // See if a widened result type would be legal, if so widen the node.
5639 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
5640 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
5641 InVT.getVectorElementCount());
5642 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) {
5643 SDValue Res;
5644 if (N->isStrictFPOpcode()) {
5645 if (Opcode == ISD::STRICT_FP_ROUND)
5646 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
5647 { N->getOperand(0), InOp, N->getOperand(2) });
5648 else
5649 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
5650 { N->getOperand(0), InOp });
5651 // Legalize the chain result - switch anything that used the old chain to
5652 // use the new one.
5653 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
5654 } else {
5655 if (Opcode == ISD::FP_ROUND)
5656 Res = DAG.getNode(Opcode, dl, WideVT, InOp, N->getOperand(1));
5657 else
5658 Res = DAG.getNode(Opcode, dl, WideVT, InOp);
5659 }
5660 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
5661 DAG.getVectorIdxConstant(0, dl));
5662 }
5663
5664 EVT InEltVT = InVT.getVectorElementType();
5665
5666 // Unroll the convert into some scalar code and create a nasty build vector.
5667 unsigned NumElts = VT.getVectorNumElements();
5668 SmallVector<SDValue, 16> Ops(NumElts);
5669 if (N->isStrictFPOpcode()) {
5670 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
5671 SmallVector<SDValue, 32> OpChains;
5672 for (unsigned i=0; i < NumElts; ++i) {
5673 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
5674 DAG.getVectorIdxConstant(i, dl));
5675 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
5676 OpChains.push_back(Ops[i].getValue(1));
5677 }
5678 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
5679 ReplaceValueWith(SDValue(N, 1), NewChain);
5680 } else {
5681 for (unsigned i = 0; i < NumElts; ++i)
5682 Ops[i] = DAG.getNode(Opcode, dl, EltVT,
5683 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT,
5684 InOp, DAG.getVectorIdxConstant(i, dl)));
5685 }
5686
5687 return DAG.getBuildVector(VT, dl, Ops);
5688 }
5689
WidenVecOp_FP_TO_XINT_SAT(SDNode * N)5690 SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
5691 EVT DstVT = N->getValueType(0);
5692 SDValue Src = GetWidenedVector(N->getOperand(0));
5693 EVT SrcVT = Src.getValueType();
5694 ElementCount WideNumElts = SrcVT.getVectorElementCount();
5695 SDLoc dl(N);
5696
5697 // See if a widened result type would be legal, if so widen the node.
5698 EVT WideDstVT = EVT::getVectorVT(*DAG.getContext(),
5699 DstVT.getVectorElementType(), WideNumElts);
5700 if (TLI.isTypeLegal(WideDstVT)) {
5701 SDValue Res =
5702 DAG.getNode(N->getOpcode(), dl, WideDstVT, Src, N->getOperand(1));
5703 return DAG.getNode(
5704 ISD::EXTRACT_SUBVECTOR, dl, DstVT, Res,
5705 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5706 }
5707
5708 // Give up and unroll.
5709 return DAG.UnrollVectorOp(N);
5710 }
5711
WidenVecOp_BITCAST(SDNode * N)5712 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
5713 EVT VT = N->getValueType(0);
5714 SDValue InOp = GetWidenedVector(N->getOperand(0));
5715 EVT InWidenVT = InOp.getValueType();
5716 SDLoc dl(N);
5717
5718 // Check if we can convert between two legal vector types and extract.
5719 TypeSize InWidenSize = InWidenVT.getSizeInBits();
5720 TypeSize Size = VT.getSizeInBits();
5721 // x86mmx is not an acceptable vector element type, so don't try.
5722 if (!VT.isVector() && VT != MVT::x86mmx &&
5723 InWidenSize.hasKnownScalarFactor(Size)) {
5724 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(Size);
5725 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
5726 if (TLI.isTypeLegal(NewVT)) {
5727 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
5728 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
5729 DAG.getVectorIdxConstant(0, dl));
5730 }
5731 }
5732
5733 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
5734 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
5735 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
5736 // having to copy via memory.
5737 if (VT.isVector()) {
5738 EVT EltVT = VT.getVectorElementType();
5739 unsigned EltSize = EltVT.getFixedSizeInBits();
5740 if (InWidenSize.isKnownMultipleOf(EltSize)) {
5741 ElementCount NewNumElts =
5742 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
5743 .divideCoefficientBy(EltSize);
5744 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
5745 if (TLI.isTypeLegal(NewVT)) {
5746 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
5747 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp,
5748 DAG.getVectorIdxConstant(0, dl));
5749 }
5750 }
5751 }
5752
5753 return CreateStackStoreLoad(InOp, VT);
5754 }
5755
WidenVecOp_CONCAT_VECTORS(SDNode * N)5756 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
5757 EVT VT = N->getValueType(0);
5758 EVT EltVT = VT.getVectorElementType();
5759 EVT InVT = N->getOperand(0).getValueType();
5760 SDLoc dl(N);
5761
5762 // If the widen width for this operand is the same as the width of the concat
5763 // and all but the first operand is undef, just use the widened operand.
5764 unsigned NumOperands = N->getNumOperands();
5765 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
5766 unsigned i;
5767 for (i = 1; i < NumOperands; ++i)
5768 if (!N->getOperand(i).isUndef())
5769 break;
5770
5771 if (i == NumOperands)
5772 return GetWidenedVector(N->getOperand(0));
5773 }
5774
5775 // Otherwise, fall back to a nasty build vector.
5776 unsigned NumElts = VT.getVectorNumElements();
5777 SmallVector<SDValue, 16> Ops(NumElts);
5778
5779 unsigned NumInElts = InVT.getVectorNumElements();
5780
5781 unsigned Idx = 0;
5782 for (unsigned i=0; i < NumOperands; ++i) {
5783 SDValue InOp = N->getOperand(i);
5784 assert(getTypeAction(InOp.getValueType()) ==
5785 TargetLowering::TypeWidenVector &&
5786 "Unexpected type action");
5787 InOp = GetWidenedVector(InOp);
5788 for (unsigned j = 0; j < NumInElts; ++j)
5789 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
5790 DAG.getVectorIdxConstant(j, dl));
5791 }
5792 return DAG.getBuildVector(VT, dl, Ops);
5793 }
5794
WidenVecOp_INSERT_SUBVECTOR(SDNode * N)5795 SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
5796 EVT VT = N->getValueType(0);
5797 SDValue SubVec = N->getOperand(1);
5798 SDValue InVec = N->getOperand(0);
5799
5800 if (getTypeAction(SubVec.getValueType()) == TargetLowering::TypeWidenVector)
5801 SubVec = GetWidenedVector(SubVec);
5802
5803 if (SubVec.getValueType().knownBitsLE(VT) && InVec.isUndef() &&
5804 N->getConstantOperandVal(2) == 0)
5805 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, InVec, SubVec,
5806 N->getOperand(2));
5807
5808 report_fatal_error("Don't know how to widen the operands for "
5809 "INSERT_SUBVECTOR");
5810 }
5811
WidenVecOp_EXTRACT_SUBVECTOR(SDNode * N)5812 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
5813 SDValue InOp = GetWidenedVector(N->getOperand(0));
5814 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
5815 N->getValueType(0), InOp, N->getOperand(1));
5816 }
5817
WidenVecOp_EXTRACT_VECTOR_ELT(SDNode * N)5818 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
5819 SDValue InOp = GetWidenedVector(N->getOperand(0));
5820 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
5821 N->getValueType(0), InOp, N->getOperand(1));
5822 }
5823
WidenVecOp_STORE(SDNode * N)5824 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
5825 // We have to widen the value, but we want only to store the original
5826 // vector type.
5827 StoreSDNode *ST = cast<StoreSDNode>(N);
5828
5829 if (!ST->getMemoryVT().getScalarType().isByteSized())
5830 return TLI.scalarizeVectorStore(ST, DAG);
5831
5832 if (ST->isTruncatingStore())
5833 return TLI.scalarizeVectorStore(ST, DAG);
5834
5835 SmallVector<SDValue, 16> StChain;
5836 if (GenWidenVectorStores(StChain, ST)) {
5837 if (StChain.size() == 1)
5838 return StChain[0];
5839
5840 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
5841 }
5842
5843 // Generate a vector-predicated store if it is custom/legal on the target.
5844 // To avoid possible recursion, only do this if the widened mask type is
5845 // legal.
5846 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
5847 // removed from the IR by the ExpandVectorPredication pass but we're
5848 // reintroducing them here.
5849 SDValue StVal = ST->getValue();
5850 EVT StVT = StVal.getValueType();
5851 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StVT);
5852 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
5853 WideVT.getVectorElementCount());
5854 if (WideVT.isScalableVector() &&
5855 TLI.isOperationLegalOrCustom(ISD::VP_STORE, WideVT) &&
5856 TLI.isTypeLegal(WideMaskVT)) {
5857 // Widen the value.
5858 SDLoc DL(N);
5859 StVal = GetWidenedVector(StVal);
5860 SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT);
5861 MVT EVLVT = TLI.getVPExplicitVectorLengthTy();
5862 unsigned NumVTElts = StVT.getVectorMinNumElements();
5863 SDValue EVL =
5864 DAG.getVScale(DL, EVLVT, APInt(EVLVT.getScalarSizeInBits(), NumVTElts));
5865 return DAG.getStoreVP(ST->getChain(), DL, StVal, ST->getBasePtr(),
5866 DAG.getUNDEF(ST->getBasePtr().getValueType()), Mask,
5867 EVL, StVal.getValueType(), ST->getMemOperand(),
5868 ST->getAddressingMode());
5869 }
5870
5871 report_fatal_error("Unable to widen vector store");
5872 }
5873
WidenVecOp_VP_STORE(SDNode * N,unsigned OpNo)5874 SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
5875 assert((OpNo == 1 || OpNo == 3) &&
5876 "Can widen only data or mask operand of vp_store");
5877 VPStoreSDNode *ST = cast<VPStoreSDNode>(N);
5878 SDValue Mask = ST->getMask();
5879 SDValue StVal = ST->getValue();
5880 SDLoc dl(N);
5881
5882 if (OpNo == 1) {
5883 // Widen the value.
5884 StVal = GetWidenedVector(StVal);
5885
5886 // We only handle the case where the mask needs widening to an
5887 // identically-sized type as the vector inputs.
5888 assert(getTypeAction(Mask.getValueType()) ==
5889 TargetLowering::TypeWidenVector &&
5890 "Unable to widen VP store");
5891 Mask = GetWidenedVector(Mask);
5892 } else {
5893 Mask = GetWidenedVector(Mask);
5894
5895 // We only handle the case where the stored value needs widening to an
5896 // identically-sized type as the mask.
5897 assert(getTypeAction(StVal.getValueType()) ==
5898 TargetLowering::TypeWidenVector &&
5899 "Unable to widen VP store");
5900 StVal = GetWidenedVector(StVal);
5901 }
5902
5903 assert(Mask.getValueType().getVectorElementCount() ==
5904 StVal.getValueType().getVectorElementCount() &&
5905 "Mask and data vectors should have the same number of elements");
5906 return DAG.getStoreVP(ST->getChain(), dl, StVal, ST->getBasePtr(),
5907 ST->getOffset(), Mask, ST->getVectorLength(),
5908 ST->getMemoryVT(), ST->getMemOperand(),
5909 ST->getAddressingMode(), ST->isTruncatingStore(),
5910 ST->isCompressingStore());
5911 }
5912
WidenVecOp_MSTORE(SDNode * N,unsigned OpNo)5913 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
5914 assert((OpNo == 1 || OpNo == 3) &&
5915 "Can widen only data or mask operand of mstore");
5916 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
5917 SDValue Mask = MST->getMask();
5918 EVT MaskVT = Mask.getValueType();
5919 SDValue StVal = MST->getValue();
5920 SDLoc dl(N);
5921
5922 if (OpNo == 1) {
5923 // Widen the value.
5924 StVal = GetWidenedVector(StVal);
5925
5926 // The mask should be widened as well.
5927 EVT WideVT = StVal.getValueType();
5928 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
5929 MaskVT.getVectorElementType(),
5930 WideVT.getVectorNumElements());
5931 Mask = ModifyToType(Mask, WideMaskVT, true);
5932 } else {
5933 // Widen the mask.
5934 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
5935 Mask = ModifyToType(Mask, WideMaskVT, true);
5936
5937 EVT ValueVT = StVal.getValueType();
5938 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
5939 ValueVT.getVectorElementType(),
5940 WideMaskVT.getVectorNumElements());
5941 StVal = ModifyToType(StVal, WideVT);
5942 }
5943
5944 assert(Mask.getValueType().getVectorNumElements() ==
5945 StVal.getValueType().getVectorNumElements() &&
5946 "Mask and data vectors should have the same number of elements");
5947 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
5948 MST->getOffset(), Mask, MST->getMemoryVT(),
5949 MST->getMemOperand(), MST->getAddressingMode(),
5950 false, MST->isCompressingStore());
5951 }
5952
WidenVecOp_MGATHER(SDNode * N,unsigned OpNo)5953 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
5954 assert(OpNo == 4 && "Can widen only the index of mgather");
5955 auto *MG = cast<MaskedGatherSDNode>(N);
5956 SDValue DataOp = MG->getPassThru();
5957 SDValue Mask = MG->getMask();
5958 SDValue Scale = MG->getScale();
5959
5960 // Just widen the index. It's allowed to have extra elements.
5961 SDValue Index = GetWidenedVector(MG->getIndex());
5962
5963 SDLoc dl(N);
5964 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
5965 Scale};
5966 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
5967 MG->getMemOperand(), MG->getIndexType(),
5968 MG->getExtensionType());
5969 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
5970 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
5971 return SDValue();
5972 }
5973
WidenVecOp_MSCATTER(SDNode * N,unsigned OpNo)5974 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
5975 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
5976 SDValue DataOp = MSC->getValue();
5977 SDValue Mask = MSC->getMask();
5978 SDValue Index = MSC->getIndex();
5979 SDValue Scale = MSC->getScale();
5980 EVT WideMemVT = MSC->getMemoryVT();
5981
5982 if (OpNo == 1) {
5983 DataOp = GetWidenedVector(DataOp);
5984 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
5985
5986 // Widen index.
5987 EVT IndexVT = Index.getValueType();
5988 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
5989 IndexVT.getVectorElementType(), NumElts);
5990 Index = ModifyToType(Index, WideIndexVT);
5991
5992 // The mask should be widened as well.
5993 EVT MaskVT = Mask.getValueType();
5994 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
5995 MaskVT.getVectorElementType(), NumElts);
5996 Mask = ModifyToType(Mask, WideMaskVT, true);
5997
5998 // Widen the MemoryType
5999 WideMemVT = EVT::getVectorVT(*DAG.getContext(),
6000 MSC->getMemoryVT().getScalarType(), NumElts);
6001 } else if (OpNo == 4) {
6002 // Just widen the index. It's allowed to have extra elements.
6003 Index = GetWidenedVector(Index);
6004 } else
6005 llvm_unreachable("Can't widen this operand of mscatter");
6006
6007 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
6008 Scale};
6009 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N),
6010 Ops, MSC->getMemOperand(), MSC->getIndexType(),
6011 MSC->isTruncatingStore());
6012 }
6013
WidenVecOp_VP_SCATTER(SDNode * N,unsigned OpNo)6014 SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
6015 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(N);
6016 SDValue DataOp = VPSC->getValue();
6017 SDValue Mask = VPSC->getMask();
6018 SDValue Index = VPSC->getIndex();
6019 SDValue Scale = VPSC->getScale();
6020 EVT WideMemVT = VPSC->getMemoryVT();
6021
6022 if (OpNo == 1) {
6023 DataOp = GetWidenedVector(DataOp);
6024 Index = GetWidenedVector(Index);
6025 const auto WideEC = DataOp.getValueType().getVectorElementCount();
6026 Mask = GetWidenedMask(Mask, WideEC);
6027 WideMemVT = EVT::getVectorVT(*DAG.getContext(),
6028 VPSC->getMemoryVT().getScalarType(), WideEC);
6029 } else if (OpNo == 3) {
6030 // Just widen the index. It's allowed to have extra elements.
6031 Index = GetWidenedVector(Index);
6032 } else
6033 llvm_unreachable("Can't widen this operand of VP_SCATTER");
6034
6035 SDValue Ops[] = {
6036 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
6037 VPSC->getVectorLength()};
6038 return DAG.getScatterVP(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), Ops,
6039 VPSC->getMemOperand(), VPSC->getIndexType());
6040 }
6041
WidenVecOp_SETCC(SDNode * N)6042 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
6043 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
6044 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
6045 SDLoc dl(N);
6046 EVT VT = N->getValueType(0);
6047
6048 // WARNING: In this code we widen the compare instruction with garbage.
6049 // This garbage may contain denormal floats which may be slow. Is this a real
6050 // concern ? Should we zero the unused lanes if this is a float compare ?
6051
6052 // Get a new SETCC node to compare the newly widened operands.
6053 // Only some of the compared elements are legal.
6054 EVT SVT = getSetCCResultType(InOp0.getValueType());
6055 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
6056 if (VT.getScalarType() == MVT::i1)
6057 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
6058 SVT.getVectorElementCount());
6059
6060 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
6061 SVT, InOp0, InOp1, N->getOperand(2));
6062
6063 // Extract the needed results from the result vector.
6064 EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
6065 SVT.getVectorElementType(),
6066 VT.getVectorElementCount());
6067 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
6068 DAG.getVectorIdxConstant(0, dl));
6069
6070 EVT OpVT = N->getOperand(0).getValueType();
6071 ISD::NodeType ExtendCode =
6072 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
6073 return DAG.getNode(ExtendCode, dl, VT, CC);
6074 }
6075
WidenVecOp_STRICT_FSETCC(SDNode * N)6076 SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
6077 SDValue Chain = N->getOperand(0);
6078 SDValue LHS = GetWidenedVector(N->getOperand(1));
6079 SDValue RHS = GetWidenedVector(N->getOperand(2));
6080 SDValue CC = N->getOperand(3);
6081 SDLoc dl(N);
6082
6083 EVT VT = N->getValueType(0);
6084 EVT EltVT = VT.getVectorElementType();
6085 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
6086 unsigned NumElts = VT.getVectorNumElements();
6087
6088 // Unroll into a build vector.
6089 SmallVector<SDValue, 8> Scalars(NumElts);
6090 SmallVector<SDValue, 8> Chains(NumElts);
6091
6092 for (unsigned i = 0; i != NumElts; ++i) {
6093 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
6094 DAG.getVectorIdxConstant(i, dl));
6095 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
6096 DAG.getVectorIdxConstant(i, dl));
6097
6098 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
6099 {Chain, LHSElem, RHSElem, CC});
6100 Chains[i] = Scalars[i].getValue(1);
6101 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
6102 DAG.getBoolConstant(true, dl, EltVT, VT),
6103 DAG.getBoolConstant(false, dl, EltVT, VT));
6104 }
6105
6106 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
6107 ReplaceValueWith(SDValue(N, 1), NewChain);
6108
6109 return DAG.getBuildVector(VT, dl, Scalars);
6110 }
6111
WidenVecOp_VECREDUCE(SDNode * N)6112 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
6113 SDLoc dl(N);
6114 SDValue Op = GetWidenedVector(N->getOperand(0));
6115 EVT OrigVT = N->getOperand(0).getValueType();
6116 EVT WideVT = Op.getValueType();
6117 EVT ElemVT = OrigVT.getVectorElementType();
6118 SDNodeFlags Flags = N->getFlags();
6119
6120 unsigned Opc = N->getOpcode();
6121 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
6122 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
6123 assert(NeutralElem && "Neutral element must exist");
6124
6125 // Pad the vector with the neutral element.
6126 unsigned OrigElts = OrigVT.getVectorMinNumElements();
6127 unsigned WideElts = WideVT.getVectorMinNumElements();
6128
6129 if (WideVT.isScalableVector()) {
6130 unsigned GCD = greatestCommonDivisor(OrigElts, WideElts);
6131 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT,
6132 ElementCount::getScalable(GCD));
6133 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem);
6134 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
6135 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral,
6136 DAG.getVectorIdxConstant(Idx, dl));
6137 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
6138 }
6139
6140 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
6141 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
6142 DAG.getVectorIdxConstant(Idx, dl));
6143
6144 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
6145 }
6146
WidenVecOp_VECREDUCE_SEQ(SDNode * N)6147 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
6148 SDLoc dl(N);
6149 SDValue AccOp = N->getOperand(0);
6150 SDValue VecOp = N->getOperand(1);
6151 SDValue Op = GetWidenedVector(VecOp);
6152
6153 EVT OrigVT = VecOp.getValueType();
6154 EVT WideVT = Op.getValueType();
6155 EVT ElemVT = OrigVT.getVectorElementType();
6156 SDNodeFlags Flags = N->getFlags();
6157
6158 unsigned Opc = N->getOpcode();
6159 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
6160 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
6161
6162 // Pad the vector with the neutral element.
6163 unsigned OrigElts = OrigVT.getVectorMinNumElements();
6164 unsigned WideElts = WideVT.getVectorMinNumElements();
6165
6166 if (WideVT.isScalableVector()) {
6167 unsigned GCD = greatestCommonDivisor(OrigElts, WideElts);
6168 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT,
6169 ElementCount::getScalable(GCD));
6170 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem);
6171 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
6172 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral,
6173 DAG.getVectorIdxConstant(Idx, dl));
6174 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags);
6175 }
6176
6177 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
6178 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
6179 DAG.getVectorIdxConstant(Idx, dl));
6180
6181 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags);
6182 }
6183
WidenVecOp_VP_REDUCE(SDNode * N)6184 SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
6185 assert(N->isVPOpcode() && "Expected VP opcode");
6186
6187 SDLoc dl(N);
6188 SDValue Op = GetWidenedVector(N->getOperand(1));
6189 SDValue Mask = GetWidenedMask(N->getOperand(2),
6190 Op.getValueType().getVectorElementCount());
6191
6192 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0),
6193 {N->getOperand(0), Op, Mask, N->getOperand(3)},
6194 N->getFlags());
6195 }
6196
WidenVecOp_VSELECT(SDNode * N)6197 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
6198 // This only gets called in the case that the left and right inputs and
6199 // result are of a legal odd vector type, and the condition is illegal i1 of
6200 // the same odd width that needs widening.
6201 EVT VT = N->getValueType(0);
6202 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
6203
6204 SDValue Cond = GetWidenedVector(N->getOperand(0));
6205 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
6206 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
6207 SDLoc DL(N);
6208
6209 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
6210 LeftIn, RightIn);
6211 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Select,
6212 DAG.getVectorIdxConstant(0, DL));
6213 }
6214
6215 //===----------------------------------------------------------------------===//
6216 // Vector Widening Utilities
6217 //===----------------------------------------------------------------------===//
6218
6219 // Utility function to find the type to chop up a widen vector for load/store
6220 // TLI: Target lowering used to determine legal types.
6221 // Width: Width left need to load/store.
6222 // WidenVT: The widen vector type to load to/store from
6223 // Align: If 0, don't allow use of a wider type
6224 // WidenEx: If Align is not 0, the amount additional we can load/store from.
6225
findMemType(SelectionDAG & DAG,const TargetLowering & TLI,unsigned Width,EVT WidenVT,unsigned Align=0,unsigned WidenEx=0)6226 static Optional<EVT> findMemType(SelectionDAG &DAG, const TargetLowering &TLI,
6227 unsigned Width, EVT WidenVT,
6228 unsigned Align = 0, unsigned WidenEx = 0) {
6229 EVT WidenEltVT = WidenVT.getVectorElementType();
6230 const bool Scalable = WidenVT.isScalableVector();
6231 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinSize();
6232 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
6233 unsigned AlignInBits = Align*8;
6234
6235 // If we have one element to load/store, return it.
6236 EVT RetVT = WidenEltVT;
6237 if (!Scalable && Width == WidenEltWidth)
6238 return RetVT;
6239
6240 // Don't bother looking for an integer type if the vector is scalable, skip
6241 // to vector types.
6242 if (!Scalable) {
6243 // See if there is larger legal integer than the element type to load/store.
6244 for (EVT MemVT : reverse(MVT::integer_valuetypes())) {
6245 unsigned MemVTWidth = MemVT.getSizeInBits();
6246 if (MemVT.getSizeInBits() <= WidenEltWidth)
6247 break;
6248 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
6249 if ((Action == TargetLowering::TypeLegal ||
6250 Action == TargetLowering::TypePromoteInteger) &&
6251 (WidenWidth % MemVTWidth) == 0 &&
6252 isPowerOf2_32(WidenWidth / MemVTWidth) &&
6253 (MemVTWidth <= Width ||
6254 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
6255 if (MemVTWidth == WidenWidth)
6256 return MemVT;
6257 RetVT = MemVT;
6258 break;
6259 }
6260 }
6261 }
6262
6263 // See if there is a larger vector type to load/store that has the same vector
6264 // element type and is evenly divisible with the WidenVT.
6265 for (EVT MemVT : reverse(MVT::vector_valuetypes())) {
6266 // Skip vector MVTs which don't match the scalable property of WidenVT.
6267 if (Scalable != MemVT.isScalableVector())
6268 continue;
6269 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinSize();
6270 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
6271 if ((Action == TargetLowering::TypeLegal ||
6272 Action == TargetLowering::TypePromoteInteger) &&
6273 WidenEltVT == MemVT.getVectorElementType() &&
6274 (WidenWidth % MemVTWidth) == 0 &&
6275 isPowerOf2_32(WidenWidth / MemVTWidth) &&
6276 (MemVTWidth <= Width ||
6277 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
6278 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
6279 return MemVT;
6280 }
6281 }
6282
6283 // Using element-wise loads and stores for widening operations is not
6284 // supported for scalable vectors
6285 if (Scalable)
6286 return None;
6287
6288 return RetVT;
6289 }
6290
6291 // Builds a vector type from scalar loads
6292 // VecTy: Resulting Vector type
6293 // LDOps: Load operators to build a vector type
6294 // [Start,End) the list of loads to use.
BuildVectorFromScalar(SelectionDAG & DAG,EVT VecTy,SmallVectorImpl<SDValue> & LdOps,unsigned Start,unsigned End)6295 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
6296 SmallVectorImpl<SDValue> &LdOps,
6297 unsigned Start, unsigned End) {
6298 SDLoc dl(LdOps[Start]);
6299 EVT LdTy = LdOps[Start].getValueType();
6300 unsigned Width = VecTy.getSizeInBits();
6301 unsigned NumElts = Width / LdTy.getSizeInBits();
6302 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
6303
6304 unsigned Idx = 1;
6305 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
6306
6307 for (unsigned i = Start + 1; i != End; ++i) {
6308 EVT NewLdTy = LdOps[i].getValueType();
6309 if (NewLdTy != LdTy) {
6310 NumElts = Width / NewLdTy.getSizeInBits();
6311 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
6312 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
6313 // Readjust position and vector position based on new load type.
6314 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
6315 LdTy = NewLdTy;
6316 }
6317 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
6318 DAG.getVectorIdxConstant(Idx++, dl));
6319 }
6320 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
6321 }
6322
GenWidenVectorLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD)6323 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
6324 LoadSDNode *LD) {
6325 // The strategy assumes that we can efficiently load power-of-two widths.
6326 // The routine chops the vector into the largest vector loads with the same
6327 // element type or scalar loads and then recombines it to the widen vector
6328 // type.
6329 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
6330 EVT LdVT = LD->getMemoryVT();
6331 SDLoc dl(LD);
6332 assert(LdVT.isVector() && WidenVT.isVector());
6333 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
6334 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
6335
6336 // Load information
6337 SDValue Chain = LD->getChain();
6338 SDValue BasePtr = LD->getBasePtr();
6339 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
6340 AAMDNodes AAInfo = LD->getAAInfo();
6341
6342 TypeSize LdWidth = LdVT.getSizeInBits();
6343 TypeSize WidenWidth = WidenVT.getSizeInBits();
6344 TypeSize WidthDiff = WidenWidth - LdWidth;
6345 // Allow wider loads if they are sufficiently aligned to avoid memory faults
6346 // and if the original load is simple.
6347 unsigned LdAlign =
6348 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
6349
6350 // Find the vector type that can load from.
6351 Optional<EVT> FirstVT =
6352 findMemType(DAG, TLI, LdWidth.getKnownMinSize(), WidenVT, LdAlign,
6353 WidthDiff.getKnownMinSize());
6354
6355 if (!FirstVT)
6356 return SDValue();
6357
6358 SmallVector<EVT, 8> MemVTs;
6359 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6360
6361 // Unless we're able to load in one instruction we must work out how to load
6362 // the remainder.
6363 if (!TypeSize::isKnownLE(LdWidth, FirstVTWidth)) {
6364 Optional<EVT> NewVT = FirstVT;
6365 TypeSize RemainingWidth = LdWidth;
6366 TypeSize NewVTWidth = FirstVTWidth;
6367 do {
6368 RemainingWidth -= NewVTWidth;
6369 if (TypeSize::isKnownLT(RemainingWidth, NewVTWidth)) {
6370 // The current type we are using is too large. Find a better size.
6371 NewVT = findMemType(DAG, TLI, RemainingWidth.getKnownMinSize(), WidenVT,
6372 LdAlign, WidthDiff.getKnownMinSize());
6373 if (!NewVT)
6374 return SDValue();
6375 NewVTWidth = NewVT->getSizeInBits();
6376 }
6377 MemVTs.push_back(*NewVT);
6378 } while (TypeSize::isKnownGT(RemainingWidth, NewVTWidth));
6379 }
6380
6381 SDValue LdOp = DAG.getLoad(*FirstVT, dl, Chain, BasePtr, LD->getPointerInfo(),
6382 LD->getOriginalAlign(), MMOFlags, AAInfo);
6383 LdChain.push_back(LdOp.getValue(1));
6384
6385 // Check if we can load the element with one instruction.
6386 if (MemVTs.empty()) {
6387 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth));
6388 if (!FirstVT->isVector()) {
6389 unsigned NumElts =
6390 WidenWidth.getFixedSize() / FirstVTWidth.getFixedSize();
6391 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), *FirstVT, NumElts);
6392 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
6393 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
6394 }
6395 if (FirstVT == WidenVT)
6396 return LdOp;
6397
6398 // TODO: We don't currently have any tests that exercise this code path.
6399 assert(WidenWidth.getFixedSize() % FirstVTWidth.getFixedSize() == 0);
6400 unsigned NumConcat =
6401 WidenWidth.getFixedSize() / FirstVTWidth.getFixedSize();
6402 SmallVector<SDValue, 16> ConcatOps(NumConcat);
6403 SDValue UndefVal = DAG.getUNDEF(*FirstVT);
6404 ConcatOps[0] = LdOp;
6405 for (unsigned i = 1; i != NumConcat; ++i)
6406 ConcatOps[i] = UndefVal;
6407 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
6408 }
6409
6410 // Load vector by using multiple loads from largest vector to scalar.
6411 SmallVector<SDValue, 16> LdOps;
6412 LdOps.push_back(LdOp);
6413
6414 uint64_t ScaledOffset = 0;
6415 MachinePointerInfo MPI = LD->getPointerInfo();
6416
6417 // First incremement past the first load.
6418 IncrementPointer(cast<LoadSDNode>(LdOp), *FirstVT, MPI, BasePtr,
6419 &ScaledOffset);
6420
6421 for (EVT MemVT : MemVTs) {
6422 Align NewAlign = ScaledOffset == 0
6423 ? LD->getOriginalAlign()
6424 : commonAlignment(LD->getAlign(), ScaledOffset);
6425 SDValue L =
6426 DAG.getLoad(MemVT, dl, Chain, BasePtr, MPI, NewAlign, MMOFlags, AAInfo);
6427
6428 LdOps.push_back(L);
6429 LdChain.push_back(L.getValue(1));
6430 IncrementPointer(cast<LoadSDNode>(L), MemVT, MPI, BasePtr, &ScaledOffset);
6431 }
6432
6433 // Build the vector from the load operations.
6434 unsigned End = LdOps.size();
6435 if (!LdOps[0].getValueType().isVector())
6436 // All the loads are scalar loads.
6437 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
6438
6439 // If the load contains vectors, build the vector using concat vector.
6440 // All of the vectors used to load are power-of-2, and the scalar loads can be
6441 // combined to make a power-of-2 vector.
6442 SmallVector<SDValue, 16> ConcatOps(End);
6443 int i = End - 1;
6444 int Idx = End;
6445 EVT LdTy = LdOps[i].getValueType();
6446 // First, combine the scalar loads to a vector.
6447 if (!LdTy.isVector()) {
6448 for (--i; i >= 0; --i) {
6449 LdTy = LdOps[i].getValueType();
6450 if (LdTy.isVector())
6451 break;
6452 }
6453 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
6454 }
6455
6456 ConcatOps[--Idx] = LdOps[i];
6457 for (--i; i >= 0; --i) {
6458 EVT NewLdTy = LdOps[i].getValueType();
6459 if (NewLdTy != LdTy) {
6460 // Create a larger vector.
6461 TypeSize LdTySize = LdTy.getSizeInBits();
6462 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
6463 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
6464 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinSize()));
6465 unsigned NumOps =
6466 NewLdTySize.getKnownMinSize() / LdTySize.getKnownMinSize();
6467 SmallVector<SDValue, 16> WidenOps(NumOps);
6468 unsigned j = 0;
6469 for (; j != End-Idx; ++j)
6470 WidenOps[j] = ConcatOps[Idx+j];
6471 for (; j != NumOps; ++j)
6472 WidenOps[j] = DAG.getUNDEF(LdTy);
6473
6474 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
6475 WidenOps);
6476 Idx = End - 1;
6477 LdTy = NewLdTy;
6478 }
6479 ConcatOps[--Idx] = LdOps[i];
6480 }
6481
6482 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
6483 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
6484 makeArrayRef(&ConcatOps[Idx], End - Idx));
6485
6486 // We need to fill the rest with undefs to build the vector.
6487 unsigned NumOps =
6488 WidenWidth.getKnownMinSize() / LdTy.getSizeInBits().getKnownMinSize();
6489 SmallVector<SDValue, 16> WidenOps(NumOps);
6490 SDValue UndefVal = DAG.getUNDEF(LdTy);
6491 {
6492 unsigned i = 0;
6493 for (; i != End-Idx; ++i)
6494 WidenOps[i] = ConcatOps[Idx+i];
6495 for (; i != NumOps; ++i)
6496 WidenOps[i] = UndefVal;
6497 }
6498 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
6499 }
6500
6501 SDValue
GenWidenVectorExtLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD,ISD::LoadExtType ExtType)6502 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
6503 LoadSDNode *LD,
6504 ISD::LoadExtType ExtType) {
6505 // For extension loads, it may not be more efficient to chop up the vector
6506 // and then extend it. Instead, we unroll the load and build a new vector.
6507 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
6508 EVT LdVT = LD->getMemoryVT();
6509 SDLoc dl(LD);
6510 assert(LdVT.isVector() && WidenVT.isVector());
6511 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
6512
6513 // Load information
6514 SDValue Chain = LD->getChain();
6515 SDValue BasePtr = LD->getBasePtr();
6516 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
6517 AAMDNodes AAInfo = LD->getAAInfo();
6518
6519 if (LdVT.isScalableVector())
6520 report_fatal_error("Generating widen scalable extending vector loads is "
6521 "not yet supported");
6522
6523 EVT EltVT = WidenVT.getVectorElementType();
6524 EVT LdEltVT = LdVT.getVectorElementType();
6525 unsigned NumElts = LdVT.getVectorNumElements();
6526
6527 // Load each element and widen.
6528 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6529 SmallVector<SDValue, 16> Ops(WidenNumElts);
6530 unsigned Increment = LdEltVT.getSizeInBits() / 8;
6531 Ops[0] =
6532 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
6533 LdEltVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
6534 LdChain.push_back(Ops[0].getValue(1));
6535 unsigned i = 0, Offset = Increment;
6536 for (i=1; i < NumElts; ++i, Offset += Increment) {
6537 SDValue NewBasePtr =
6538 DAG.getObjectPtrOffset(dl, BasePtr, TypeSize::Fixed(Offset));
6539 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
6540 LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
6541 LD->getOriginalAlign(), MMOFlags, AAInfo);
6542 LdChain.push_back(Ops[i].getValue(1));
6543 }
6544
6545 // Fill the rest with undefs.
6546 SDValue UndefVal = DAG.getUNDEF(EltVT);
6547 for (; i != WidenNumElts; ++i)
6548 Ops[i] = UndefVal;
6549
6550 return DAG.getBuildVector(WidenVT, dl, Ops);
6551 }
6552
GenWidenVectorStores(SmallVectorImpl<SDValue> & StChain,StoreSDNode * ST)6553 bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
6554 StoreSDNode *ST) {
6555 // The strategy assumes that we can efficiently store power-of-two widths.
6556 // The routine chops the vector into the largest vector stores with the same
6557 // element type or scalar stores.
6558 SDValue Chain = ST->getChain();
6559 SDValue BasePtr = ST->getBasePtr();
6560 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
6561 AAMDNodes AAInfo = ST->getAAInfo();
6562 SDValue ValOp = GetWidenedVector(ST->getValue());
6563 SDLoc dl(ST);
6564
6565 EVT StVT = ST->getMemoryVT();
6566 TypeSize StWidth = StVT.getSizeInBits();
6567 EVT ValVT = ValOp.getValueType();
6568 TypeSize ValWidth = ValVT.getSizeInBits();
6569 EVT ValEltVT = ValVT.getVectorElementType();
6570 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
6571 assert(StVT.getVectorElementType() == ValEltVT);
6572 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
6573 "Mismatch between store and value types");
6574
6575 int Idx = 0; // current index to store
6576
6577 MachinePointerInfo MPI = ST->getPointerInfo();
6578 uint64_t ScaledOffset = 0;
6579
6580 // A breakdown of how to widen this vector store. Each element of the vector
6581 // is a memory VT combined with the number of times it is to be stored to,
6582 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
6583 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
6584
6585 while (StWidth.isNonZero()) {
6586 // Find the largest vector type we can store with.
6587 Optional<EVT> NewVT =
6588 findMemType(DAG, TLI, StWidth.getKnownMinSize(), ValVT);
6589 if (!NewVT)
6590 return false;
6591 MemVTs.push_back({*NewVT, 0});
6592 TypeSize NewVTWidth = NewVT->getSizeInBits();
6593
6594 do {
6595 StWidth -= NewVTWidth;
6596 MemVTs.back().second++;
6597 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth));
6598 }
6599
6600 for (const auto &Pair : MemVTs) {
6601 EVT NewVT = Pair.first;
6602 unsigned Count = Pair.second;
6603 TypeSize NewVTWidth = NewVT.getSizeInBits();
6604
6605 if (NewVT.isVector()) {
6606 unsigned NumVTElts = NewVT.getVectorMinNumElements();
6607 do {
6608 Align NewAlign = ScaledOffset == 0
6609 ? ST->getOriginalAlign()
6610 : commonAlignment(ST->getAlign(), ScaledOffset);
6611 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
6612 DAG.getVectorIdxConstant(Idx, dl));
6613 SDValue PartStore = DAG.getStore(Chain, dl, EOp, BasePtr, MPI, NewAlign,
6614 MMOFlags, AAInfo);
6615 StChain.push_back(PartStore);
6616
6617 Idx += NumVTElts;
6618 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr,
6619 &ScaledOffset);
6620 } while (--Count);
6621 } else {
6622 // Cast the vector to the scalar type we can store.
6623 unsigned NumElts = ValWidth.getFixedSize() / NewVTWidth.getFixedSize();
6624 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
6625 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
6626 // Readjust index position based on new vector type.
6627 Idx = Idx * ValEltWidth / NewVTWidth.getFixedSize();
6628 do {
6629 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
6630 DAG.getVectorIdxConstant(Idx++, dl));
6631 SDValue PartStore =
6632 DAG.getStore(Chain, dl, EOp, BasePtr, MPI, ST->getOriginalAlign(),
6633 MMOFlags, AAInfo);
6634 StChain.push_back(PartStore);
6635
6636 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr);
6637 } while (--Count);
6638 // Restore index back to be relative to the original widen element type.
6639 Idx = Idx * NewVTWidth.getFixedSize() / ValEltWidth;
6640 }
6641 }
6642
6643 return true;
6644 }
6645
6646 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
6647 /// input vector must have the same element type as NVT.
6648 /// FillWithZeroes specifies that the vector should be widened with zeroes.
ModifyToType(SDValue InOp,EVT NVT,bool FillWithZeroes)6649 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
6650 bool FillWithZeroes) {
6651 // Note that InOp might have been widened so it might already have
6652 // the right width or it might need be narrowed.
6653 EVT InVT = InOp.getValueType();
6654 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
6655 "input and widen element type must match");
6656 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
6657 "cannot modify scalable vectors in this way");
6658 SDLoc dl(InOp);
6659
6660 // Check if InOp already has the right width.
6661 if (InVT == NVT)
6662 return InOp;
6663
6664 ElementCount InEC = InVT.getVectorElementCount();
6665 ElementCount WidenEC = NVT.getVectorElementCount();
6666 if (WidenEC.hasKnownScalarFactor(InEC)) {
6667 unsigned NumConcat = WidenEC.getKnownScalarFactor(InEC);
6668 SmallVector<SDValue, 16> Ops(NumConcat);
6669 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
6670 DAG.getUNDEF(InVT);
6671 Ops[0] = InOp;
6672 for (unsigned i = 1; i != NumConcat; ++i)
6673 Ops[i] = FillVal;
6674
6675 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
6676 }
6677
6678 if (InEC.hasKnownScalarFactor(WidenEC))
6679 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
6680 DAG.getVectorIdxConstant(0, dl));
6681
6682 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
6683 "Scalable vectors should have been handled already.");
6684
6685 unsigned InNumElts = InEC.getFixedValue();
6686 unsigned WidenNumElts = WidenEC.getFixedValue();
6687
6688 // Fall back to extract and build.
6689 SmallVector<SDValue, 16> Ops(WidenNumElts);
6690 EVT EltVT = NVT.getVectorElementType();
6691 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
6692 unsigned Idx;
6693 for (Idx = 0; Idx < MinNumElts; ++Idx)
6694 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
6695 DAG.getVectorIdxConstant(Idx, dl));
6696
6697 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
6698 DAG.getUNDEF(EltVT);
6699 for ( ; Idx < WidenNumElts; ++Idx)
6700 Ops[Idx] = FillVal;
6701 return DAG.getBuildVector(NVT, dl, Ops);
6702 }
6703