1 //===- X86LegalizerInfo.cpp --------------------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the Machinelegalizer class for X86.
10 /// \todo This should be generated by TableGen.
11 //===----------------------------------------------------------------------===//
12 
13 #include "X86LegalizerInfo.h"
14 #include "X86Subtarget.h"
15 #include "X86TargetMachine.h"
16 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
17 #include "llvm/CodeGen/TargetOpcodes.h"
18 #include "llvm/CodeGen/ValueTypes.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Type.h"
21 
22 using namespace llvm;
23 using namespace TargetOpcode;
24 using namespace LegalizeActions;
25 
26 /// FIXME: The following static functions are SizeChangeStrategy functions
27 /// that are meant to temporarily mimic the behaviour of the old legalization
28 /// based on doubling/halving non-legal types as closely as possible. This is
29 /// not entirly possible as only legalizing the types that are exactly a power
30 /// of 2 times the size of the legal types would require specifying all those
31 /// sizes explicitly.
32 /// In practice, not specifying those isn't a problem, and the below functions
33 /// should disappear quickly as we add support for legalizing non-power-of-2
34 /// sized types further.
35 static void
36 addAndInterleaveWithUnsupported(LegalizerInfo::SizeAndActionsVec &result,
37                                 const LegalizerInfo::SizeAndActionsVec &v) {
38   for (unsigned i = 0; i < v.size(); ++i) {
39     result.push_back(v[i]);
40     if (i + 1 < v[i].first && i + 1 < v.size() &&
41         v[i + 1].first != v[i].first + 1)
42       result.push_back({v[i].first + 1, Unsupported});
43   }
44 }
45 
46 static LegalizerInfo::SizeAndActionsVec
47 widen_1(const LegalizerInfo::SizeAndActionsVec &v) {
48   assert(v.size() >= 1);
49   assert(v[0].first > 1);
50   LegalizerInfo::SizeAndActionsVec result = {{1, WidenScalar},
51                                              {2, Unsupported}};
52   addAndInterleaveWithUnsupported(result, v);
53   auto Largest = result.back().first;
54   result.push_back({Largest + 1, Unsupported});
55   return result;
56 }
57 
58 X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
59                                    const X86TargetMachine &TM)
60     : Subtarget(STI), TM(TM) {
61 
62   setLegalizerInfo32bit();
63   setLegalizerInfo64bit();
64   setLegalizerInfoSSE1();
65   setLegalizerInfoSSE2();
66   setLegalizerInfoSSE41();
67   setLegalizerInfoAVX();
68   setLegalizerInfoAVX2();
69   setLegalizerInfoAVX512();
70   setLegalizerInfoAVX512DQ();
71   setLegalizerInfoAVX512BW();
72 
73   getActionDefinitionsBuilder(G_INTRINSIC_ROUNDEVEN)
74     .scalarize(0)
75     .minScalar(0, LLT::scalar(32))
76     .libcall();
77 
78   setLegalizeScalarToDifferentSizeStrategy(G_PHI, 0, widen_1);
79   for (unsigned BinOp : {G_SUB, G_MUL, G_AND, G_OR, G_XOR})
80     setLegalizeScalarToDifferentSizeStrategy(BinOp, 0, widen_1);
81   for (unsigned MemOp : {G_LOAD, G_STORE})
82     setLegalizeScalarToDifferentSizeStrategy(MemOp, 0,
83        narrowToSmallerAndWidenToSmallest);
84   setLegalizeScalarToDifferentSizeStrategy(
85       G_PTR_ADD, 1, widenToLargerTypesUnsupportedOtherwise);
86   setLegalizeScalarToDifferentSizeStrategy(
87       G_CONSTANT, 0, widenToLargerTypesAndNarrowToLargest);
88 
89   computeTables();
90   verify(*STI.getInstrInfo());
91 }
92 
93 bool X86LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
94                                          MachineInstr &MI) const {
95   MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
96   switch (MI.getIntrinsicID()) {
97   case Intrinsic::memcpy:
98   case Intrinsic::memset:
99   case Intrinsic::memmove:
100     if (createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI) ==
101         LegalizerHelper::UnableToLegalize)
102       return false;
103     MI.eraseFromParent();
104     return true;
105   default:
106     break;
107   }
108   return true;
109 }
110 
111 void X86LegalizerInfo::setLegalizerInfo32bit() {
112 
113   const LLT p0 = LLT::pointer(0, TM.getPointerSizeInBits(0));
114   const LLT s1 = LLT::scalar(1);
115   const LLT s8 = LLT::scalar(8);
116   const LLT s16 = LLT::scalar(16);
117   const LLT s32 = LLT::scalar(32);
118   const LLT s64 = LLT::scalar(64);
119   const LLT s128 = LLT::scalar(128);
120 
121   for (auto Ty : {p0, s1, s8, s16, s32})
122     setAction({G_IMPLICIT_DEF, Ty}, Legal);
123 
124   for (auto Ty : {s8, s16, s32, p0})
125     setAction({G_PHI, Ty}, Legal);
126 
127   for (unsigned BinOp : {G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR})
128     for (auto Ty : {s8, s16, s32})
129       setAction({BinOp, Ty}, Legal);
130 
131   for (unsigned Op : {G_UADDE}) {
132     setAction({Op, s32}, Legal);
133     setAction({Op, 1, s1}, Legal);
134   }
135 
136   for (unsigned MemOp : {G_LOAD, G_STORE}) {
137     for (auto Ty : {s8, s16, s32, p0})
138       setAction({MemOp, Ty}, Legal);
139 
140     // And everything's fine in addrspace 0.
141     setAction({MemOp, 1, p0}, Legal);
142   }
143 
144   // Pointer-handling
145   setAction({G_FRAME_INDEX, p0}, Legal);
146   setAction({G_GLOBAL_VALUE, p0}, Legal);
147 
148   setAction({G_PTR_ADD, p0}, Legal);
149   setAction({G_PTR_ADD, 1, s32}, Legal);
150 
151   if (!Subtarget.is64Bit()) {
152     getActionDefinitionsBuilder(G_PTRTOINT)
153         .legalForCartesianProduct({s1, s8, s16, s32}, {p0})
154         .maxScalar(0, s32)
155         .widenScalarToNextPow2(0, /*Min*/ 8);
156     getActionDefinitionsBuilder(G_INTTOPTR).legalFor({{p0, s32}});
157 
158     // Shifts and SDIV
159     getActionDefinitionsBuilder(
160         {G_SDIV, G_SREM, G_UDIV, G_UREM})
161       .legalFor({s8, s16, s32})
162       .clampScalar(0, s8, s32);
163 
164     getActionDefinitionsBuilder(
165         {G_SHL, G_LSHR, G_ASHR})
166       .legalFor({{s8, s8}, {s16, s8}, {s32, s8}})
167       .clampScalar(0, s8, s32)
168       .clampScalar(1, s8, s8);
169 
170     // Comparison
171     getActionDefinitionsBuilder(G_ICMP)
172         .legalForCartesianProduct({s8}, {s8, s16, s32, p0})
173         .clampScalar(0, s8, s8);
174   }
175 
176   // Control-flow
177   setAction({G_BRCOND, s1}, Legal);
178 
179   // Constants
180   for (auto Ty : {s8, s16, s32, p0})
181     setAction({TargetOpcode::G_CONSTANT, Ty}, Legal);
182 
183   // Extensions
184   for (auto Ty : {s8, s16, s32}) {
185     setAction({G_ZEXT, Ty}, Legal);
186     setAction({G_SEXT, Ty}, Legal);
187     setAction({G_ANYEXT, Ty}, Legal);
188   }
189   setAction({G_ANYEXT, s128}, Legal);
190   getActionDefinitionsBuilder(G_SEXT_INREG).lower();
191 
192   // Merge/Unmerge
193   for (const auto &Ty : {s16, s32, s64}) {
194     setAction({G_MERGE_VALUES, Ty}, Legal);
195     setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
196   }
197   for (const auto &Ty : {s8, s16, s32}) {
198     setAction({G_MERGE_VALUES, 1, Ty}, Legal);
199     setAction({G_UNMERGE_VALUES, Ty}, Legal);
200   }
201 }
202 
203 void X86LegalizerInfo::setLegalizerInfo64bit() {
204 
205   if (!Subtarget.is64Bit())
206     return;
207 
208   const LLT p0 = LLT::pointer(0, TM.getPointerSizeInBits(0));
209   const LLT s1 = LLT::scalar(1);
210   const LLT s8 = LLT::scalar(8);
211   const LLT s16 = LLT::scalar(16);
212   const LLT s32 = LLT::scalar(32);
213   const LLT s64 = LLT::scalar(64);
214   const LLT s128 = LLT::scalar(128);
215 
216   setAction({G_IMPLICIT_DEF, s64}, Legal);
217   // Need to have that, as tryFoldImplicitDef will create this pattern:
218   // s128 = EXTEND (G_IMPLICIT_DEF s32/s64) -> s128 = G_IMPLICIT_DEF
219   setAction({G_IMPLICIT_DEF, s128}, Legal);
220 
221   setAction({G_PHI, s64}, Legal);
222 
223   for (unsigned BinOp : {G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR})
224     setAction({BinOp, s64}, Legal);
225 
226   for (unsigned MemOp : {G_LOAD, G_STORE})
227     setAction({MemOp, s64}, Legal);
228 
229   // Pointer-handling
230   setAction({G_PTR_ADD, 1, s64}, Legal);
231   getActionDefinitionsBuilder(G_PTRTOINT)
232       .legalForCartesianProduct({s1, s8, s16, s32, s64}, {p0})
233       .maxScalar(0, s64)
234       .widenScalarToNextPow2(0, /*Min*/ 8);
235   getActionDefinitionsBuilder(G_INTTOPTR).legalFor({{p0, s64}});
236 
237   // Constants
238   setAction({TargetOpcode::G_CONSTANT, s64}, Legal);
239 
240   // Extensions
241   for (unsigned extOp : {G_ZEXT, G_SEXT, G_ANYEXT}) {
242     setAction({extOp, s64}, Legal);
243   }
244 
245   getActionDefinitionsBuilder(G_SITOFP)
246     .legalForCartesianProduct({s32, s64})
247       .clampScalar(1, s32, s64)
248       .widenScalarToNextPow2(1)
249       .clampScalar(0, s32, s64)
250       .widenScalarToNextPow2(0);
251 
252   getActionDefinitionsBuilder(G_FPTOSI)
253       .legalForCartesianProduct({s32, s64})
254       .clampScalar(1, s32, s64)
255       .widenScalarToNextPow2(0)
256       .clampScalar(0, s32, s64)
257       .widenScalarToNextPow2(1);
258 
259   // Comparison
260   getActionDefinitionsBuilder(G_ICMP)
261       .legalForCartesianProduct({s8}, {s8, s16, s32, s64, p0})
262       .clampScalar(0, s8, s8);
263 
264   getActionDefinitionsBuilder(G_FCMP)
265       .legalForCartesianProduct({s8}, {s32, s64})
266       .clampScalar(0, s8, s8)
267       .clampScalar(1, s32, s64)
268       .widenScalarToNextPow2(1);
269 
270   // Divisions
271   getActionDefinitionsBuilder(
272       {G_SDIV, G_SREM, G_UDIV, G_UREM})
273       .legalFor({s8, s16, s32, s64})
274       .clampScalar(0, s8, s64);
275 
276   // Shifts
277   getActionDefinitionsBuilder(
278     {G_SHL, G_LSHR, G_ASHR})
279     .legalFor({{s8, s8}, {s16, s8}, {s32, s8}, {s64, s8}})
280     .clampScalar(0, s8, s64)
281     .clampScalar(1, s8, s8);
282 
283   // Merge/Unmerge
284   setAction({G_MERGE_VALUES, s128}, Legal);
285   setAction({G_UNMERGE_VALUES, 1, s128}, Legal);
286   setAction({G_MERGE_VALUES, 1, s128}, Legal);
287   setAction({G_UNMERGE_VALUES, s128}, Legal);
288 }
289 
290 void X86LegalizerInfo::setLegalizerInfoSSE1() {
291   if (!Subtarget.hasSSE1())
292     return;
293 
294   const LLT s32 = LLT::scalar(32);
295   const LLT s64 = LLT::scalar(64);
296   const LLT v4s32 = LLT::vector(4, 32);
297   const LLT v2s64 = LLT::vector(2, 64);
298 
299   for (unsigned BinOp : {G_FADD, G_FSUB, G_FMUL, G_FDIV})
300     for (auto Ty : {s32, v4s32})
301       setAction({BinOp, Ty}, Legal);
302 
303   for (unsigned MemOp : {G_LOAD, G_STORE})
304     for (auto Ty : {v4s32, v2s64})
305       setAction({MemOp, Ty}, Legal);
306 
307   // Constants
308   setAction({TargetOpcode::G_FCONSTANT, s32}, Legal);
309 
310   // Merge/Unmerge
311   for (const auto &Ty : {v4s32, v2s64}) {
312     setAction({G_CONCAT_VECTORS, Ty}, Legal);
313     setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
314   }
315   setAction({G_MERGE_VALUES, 1, s64}, Legal);
316   setAction({G_UNMERGE_VALUES, s64}, Legal);
317 }
318 
319 void X86LegalizerInfo::setLegalizerInfoSSE2() {
320   if (!Subtarget.hasSSE2())
321     return;
322 
323   const LLT s32 = LLT::scalar(32);
324   const LLT s64 = LLT::scalar(64);
325   const LLT v16s8 = LLT::vector(16, 8);
326   const LLT v8s16 = LLT::vector(8, 16);
327   const LLT v4s32 = LLT::vector(4, 32);
328   const LLT v2s64 = LLT::vector(2, 64);
329 
330   const LLT v32s8 = LLT::vector(32, 8);
331   const LLT v16s16 = LLT::vector(16, 16);
332   const LLT v8s32 = LLT::vector(8, 32);
333   const LLT v4s64 = LLT::vector(4, 64);
334 
335   for (unsigned BinOp : {G_FADD, G_FSUB, G_FMUL, G_FDIV})
336     for (auto Ty : {s64, v2s64})
337       setAction({BinOp, Ty}, Legal);
338 
339   for (unsigned BinOp : {G_ADD, G_SUB})
340     for (auto Ty : {v16s8, v8s16, v4s32, v2s64})
341       setAction({BinOp, Ty}, Legal);
342 
343   setAction({G_MUL, v8s16}, Legal);
344 
345   setAction({G_FPEXT, s64}, Legal);
346   setAction({G_FPEXT, 1, s32}, Legal);
347 
348   setAction({G_FPTRUNC, s32}, Legal);
349   setAction({G_FPTRUNC, 1, s64}, Legal);
350 
351   // Constants
352   setAction({TargetOpcode::G_FCONSTANT, s64}, Legal);
353 
354   // Merge/Unmerge
355   for (const auto &Ty :
356        {v16s8, v32s8, v8s16, v16s16, v4s32, v8s32, v2s64, v4s64}) {
357     setAction({G_CONCAT_VECTORS, Ty}, Legal);
358     setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
359   }
360   for (const auto &Ty : {v16s8, v8s16, v4s32, v2s64}) {
361     setAction({G_CONCAT_VECTORS, 1, Ty}, Legal);
362     setAction({G_UNMERGE_VALUES, Ty}, Legal);
363   }
364 }
365 
366 void X86LegalizerInfo::setLegalizerInfoSSE41() {
367   if (!Subtarget.hasSSE41())
368     return;
369 
370   const LLT v4s32 = LLT::vector(4, 32);
371 
372   setAction({G_MUL, v4s32}, Legal);
373 }
374 
375 void X86LegalizerInfo::setLegalizerInfoAVX() {
376   if (!Subtarget.hasAVX())
377     return;
378 
379   const LLT v16s8 = LLT::vector(16, 8);
380   const LLT v8s16 = LLT::vector(8, 16);
381   const LLT v4s32 = LLT::vector(4, 32);
382   const LLT v2s64 = LLT::vector(2, 64);
383 
384   const LLT v32s8 = LLT::vector(32, 8);
385   const LLT v64s8 = LLT::vector(64, 8);
386   const LLT v16s16 = LLT::vector(16, 16);
387   const LLT v32s16 = LLT::vector(32, 16);
388   const LLT v8s32 = LLT::vector(8, 32);
389   const LLT v16s32 = LLT::vector(16, 32);
390   const LLT v4s64 = LLT::vector(4, 64);
391   const LLT v8s64 = LLT::vector(8, 64);
392 
393   for (unsigned MemOp : {G_LOAD, G_STORE})
394     for (auto Ty : {v8s32, v4s64})
395       setAction({MemOp, Ty}, Legal);
396 
397   for (auto Ty : {v32s8, v16s16, v8s32, v4s64}) {
398     setAction({G_INSERT, Ty}, Legal);
399     setAction({G_EXTRACT, 1, Ty}, Legal);
400   }
401   for (auto Ty : {v16s8, v8s16, v4s32, v2s64}) {
402     setAction({G_INSERT, 1, Ty}, Legal);
403     setAction({G_EXTRACT, Ty}, Legal);
404   }
405   // Merge/Unmerge
406   for (const auto &Ty :
407        {v32s8, v64s8, v16s16, v32s16, v8s32, v16s32, v4s64, v8s64}) {
408     setAction({G_CONCAT_VECTORS, Ty}, Legal);
409     setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
410   }
411   for (const auto &Ty :
412        {v16s8, v32s8, v8s16, v16s16, v4s32, v8s32, v2s64, v4s64}) {
413     setAction({G_CONCAT_VECTORS, 1, Ty}, Legal);
414     setAction({G_UNMERGE_VALUES, Ty}, Legal);
415   }
416 }
417 
418 void X86LegalizerInfo::setLegalizerInfoAVX2() {
419   if (!Subtarget.hasAVX2())
420     return;
421 
422   const LLT v32s8 = LLT::vector(32, 8);
423   const LLT v16s16 = LLT::vector(16, 16);
424   const LLT v8s32 = LLT::vector(8, 32);
425   const LLT v4s64 = LLT::vector(4, 64);
426 
427   const LLT v64s8 = LLT::vector(64, 8);
428   const LLT v32s16 = LLT::vector(32, 16);
429   const LLT v16s32 = LLT::vector(16, 32);
430   const LLT v8s64 = LLT::vector(8, 64);
431 
432   for (unsigned BinOp : {G_ADD, G_SUB})
433     for (auto Ty : {v32s8, v16s16, v8s32, v4s64})
434       setAction({BinOp, Ty}, Legal);
435 
436   for (auto Ty : {v16s16, v8s32})
437     setAction({G_MUL, Ty}, Legal);
438 
439   // Merge/Unmerge
440   for (const auto &Ty : {v64s8, v32s16, v16s32, v8s64}) {
441     setAction({G_CONCAT_VECTORS, Ty}, Legal);
442     setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
443   }
444   for (const auto &Ty : {v32s8, v16s16, v8s32, v4s64}) {
445     setAction({G_CONCAT_VECTORS, 1, Ty}, Legal);
446     setAction({G_UNMERGE_VALUES, Ty}, Legal);
447   }
448 }
449 
450 void X86LegalizerInfo::setLegalizerInfoAVX512() {
451   if (!Subtarget.hasAVX512())
452     return;
453 
454   const LLT v16s8 = LLT::vector(16, 8);
455   const LLT v8s16 = LLT::vector(8, 16);
456   const LLT v4s32 = LLT::vector(4, 32);
457   const LLT v2s64 = LLT::vector(2, 64);
458 
459   const LLT v32s8 = LLT::vector(32, 8);
460   const LLT v16s16 = LLT::vector(16, 16);
461   const LLT v8s32 = LLT::vector(8, 32);
462   const LLT v4s64 = LLT::vector(4, 64);
463 
464   const LLT v64s8 = LLT::vector(64, 8);
465   const LLT v32s16 = LLT::vector(32, 16);
466   const LLT v16s32 = LLT::vector(16, 32);
467   const LLT v8s64 = LLT::vector(8, 64);
468 
469   for (unsigned BinOp : {G_ADD, G_SUB})
470     for (auto Ty : {v16s32, v8s64})
471       setAction({BinOp, Ty}, Legal);
472 
473   setAction({G_MUL, v16s32}, Legal);
474 
475   for (unsigned MemOp : {G_LOAD, G_STORE})
476     for (auto Ty : {v16s32, v8s64})
477       setAction({MemOp, Ty}, Legal);
478 
479   for (auto Ty : {v64s8, v32s16, v16s32, v8s64}) {
480     setAction({G_INSERT, Ty}, Legal);
481     setAction({G_EXTRACT, 1, Ty}, Legal);
482   }
483   for (auto Ty : {v32s8, v16s16, v8s32, v4s64, v16s8, v8s16, v4s32, v2s64}) {
484     setAction({G_INSERT, 1, Ty}, Legal);
485     setAction({G_EXTRACT, Ty}, Legal);
486   }
487 
488   /************ VLX *******************/
489   if (!Subtarget.hasVLX())
490     return;
491 
492   for (auto Ty : {v4s32, v8s32})
493     setAction({G_MUL, Ty}, Legal);
494 }
495 
496 void X86LegalizerInfo::setLegalizerInfoAVX512DQ() {
497   if (!(Subtarget.hasAVX512() && Subtarget.hasDQI()))
498     return;
499 
500   const LLT v8s64 = LLT::vector(8, 64);
501 
502   setAction({G_MUL, v8s64}, Legal);
503 
504   /************ VLX *******************/
505   if (!Subtarget.hasVLX())
506     return;
507 
508   const LLT v2s64 = LLT::vector(2, 64);
509   const LLT v4s64 = LLT::vector(4, 64);
510 
511   for (auto Ty : {v2s64, v4s64})
512     setAction({G_MUL, Ty}, Legal);
513 }
514 
515 void X86LegalizerInfo::setLegalizerInfoAVX512BW() {
516   if (!(Subtarget.hasAVX512() && Subtarget.hasBWI()))
517     return;
518 
519   const LLT v64s8 = LLT::vector(64, 8);
520   const LLT v32s16 = LLT::vector(32, 16);
521 
522   for (unsigned BinOp : {G_ADD, G_SUB})
523     for (auto Ty : {v64s8, v32s16})
524       setAction({BinOp, Ty}, Legal);
525 
526   setAction({G_MUL, v32s16}, Legal);
527 
528   /************ VLX *******************/
529   if (!Subtarget.hasVLX())
530     return;
531 
532   const LLT v8s16 = LLT::vector(8, 16);
533   const LLT v16s16 = LLT::vector(16, 16);
534 
535   for (auto Ty : {v8s16, v16s16})
536     setAction({G_MUL, Ty}, Legal);
537 }
538