1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Define several functions to decode x86 specific shuffle semantics using
11 // constants from the constant pool.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "X86ShuffleDecodeConstantPool.h"
16 #include "Utils/X86ShuffleDecode.h"
17 #include "llvm/ADT/APInt.h"
18 #include "llvm/CodeGen/MachineValueType.h"
19 #include "llvm/IR/Constants.h"
20 
21 //===----------------------------------------------------------------------===//
22 //  Vector Mask Decoding
23 //===----------------------------------------------------------------------===//
24 
25 namespace llvm {
26 
27 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
28                                 APInt &UndefElts,
29                                 SmallVectorImpl<uint64_t> &RawMask) {
30   // It is not an error for shuffle masks to not be a vector of
31   // MaskEltSizeInBits because the constant pool uniques constants by their
32   // bit representation.
33   // e.g. the following take up the same space in the constant pool:
34   //   i128 -170141183420855150465331762880109871104
35   //
36   //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
37   //
38   //   <4 x i32> <i32 -2147483648, i32 -2147483648,
39   //              i32 -2147483648, i32 -2147483648>
40   Type *CstTy = C->getType();
41   if (!CstTy->isVectorTy())
42     return false;
43 
44   Type *CstEltTy = CstTy->getVectorElementType();
45   if (!CstEltTy->isIntegerTy())
46     return false;
47 
48   unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
49   unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
50   unsigned NumCstElts = CstTy->getVectorNumElements();
51 
52   assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
53          "Unaligned shuffle mask size");
54 
55   unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
56   UndefElts = APInt(NumMaskElts, 0);
57   RawMask.resize(NumMaskElts, 0);
58 
59   // Fast path - if the constants match the mask size then copy direct.
60   if (MaskEltSizeInBits == CstEltSizeInBits) {
61     assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
62     for (unsigned i = 0; i != NumMaskElts; ++i) {
63       Constant *COp = C->getAggregateElement(i);
64       if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
65         return false;
66 
67       if (isa<UndefValue>(COp)) {
68         UndefElts.setBit(i);
69         RawMask[i] = 0;
70         continue;
71       }
72 
73       auto *Elt = cast<ConstantInt>(COp);
74       RawMask[i] = Elt->getValue().getZExtValue();
75     }
76     return true;
77   }
78 
79   // Extract all the undef/constant element data and pack into single bitsets.
80   APInt UndefBits(CstSizeInBits, 0);
81   APInt MaskBits(CstSizeInBits, 0);
82   for (unsigned i = 0; i != NumCstElts; ++i) {
83     Constant *COp = C->getAggregateElement(i);
84     if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
85       return false;
86 
87     unsigned BitOffset = i * CstEltSizeInBits;
88 
89     if (isa<UndefValue>(COp)) {
90       UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
91       continue;
92     }
93 
94     MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
95   }
96 
97   // Now extract the undef/constant bit data into the raw shuffle masks.
98   for (unsigned i = 0; i != NumMaskElts; ++i) {
99     unsigned BitOffset = i * MaskEltSizeInBits;
100     APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
101 
102     // Only treat the element as UNDEF if all bits are UNDEF, otherwise
103     // treat it as zero.
104     if (EltUndef.isAllOnesValue()) {
105       UndefElts.setBit(i);
106       RawMask[i] = 0;
107       continue;
108     }
109 
110     APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
111     RawMask[i] = EltBits.getZExtValue();
112   }
113 
114   return true;
115 }
116 
117 void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
118   Type *MaskTy = C->getType();
119   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
120   (void)MaskTySize;
121   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
122          "Unexpected vector size.");
123 
124   // The shuffle mask requires a byte vector.
125   APInt UndefElts;
126   SmallVector<uint64_t, 64> RawMask;
127   if (!extractConstantMask(C, 8, UndefElts, RawMask))
128     return;
129 
130   unsigned NumElts = RawMask.size();
131   assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
132          "Unexpected number of vector elements.");
133 
134   for (unsigned i = 0; i != NumElts; ++i) {
135     if (UndefElts[i]) {
136       ShuffleMask.push_back(SM_SentinelUndef);
137       continue;
138     }
139 
140     uint64_t Element = RawMask[i];
141     // If the high bit (7) of the byte is set, the element is zeroed.
142     if (Element & (1 << 7))
143       ShuffleMask.push_back(SM_SentinelZero);
144     else {
145       // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
146       // lane of the vector we're inside.
147       unsigned Base = i & ~0xf;
148 
149       // Only the least significant 4 bits of the byte are used.
150       int Index = Base + (Element & 0xf);
151       ShuffleMask.push_back(Index);
152     }
153   }
154 }
155 
156 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize,
157                         SmallVectorImpl<int> &ShuffleMask) {
158   Type *MaskTy = C->getType();
159   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
160   (void)MaskTySize;
161   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
162          "Unexpected vector size.");
163   assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
164 
165   // The shuffle mask requires elements the same size as the target.
166   APInt UndefElts;
167   SmallVector<uint64_t, 16> RawMask;
168   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
169     return;
170 
171   unsigned NumElts = RawMask.size();
172   unsigned NumEltsPerLane = 128 / ElSize;
173   assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
174          "Unexpected number of vector elements.");
175 
176   for (unsigned i = 0; i != NumElts; ++i) {
177     if (UndefElts[i]) {
178       ShuffleMask.push_back(SM_SentinelUndef);
179       continue;
180     }
181 
182     int Index = i & ~(NumEltsPerLane - 1);
183     uint64_t Element = RawMask[i];
184     if (ElSize == 64)
185       Index += (Element >> 1) & 0x1;
186     else
187       Index += Element & 0x3;
188 
189     ShuffleMask.push_back(Index);
190   }
191 }
192 
193 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
194                          SmallVectorImpl<int> &ShuffleMask) {
195   Type *MaskTy = C->getType();
196   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
197   (void)MaskTySize;
198   assert((MaskTySize == 128 || MaskTySize == 256) && "Unexpected vector size.");
199 
200   // The shuffle mask requires elements the same size as the target.
201   APInt UndefElts;
202   SmallVector<uint64_t, 8> RawMask;
203   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
204     return;
205 
206   unsigned NumElts = RawMask.size();
207   unsigned NumEltsPerLane = 128 / ElSize;
208   assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
209          "Unexpected number of vector elements.");
210 
211   for (unsigned i = 0; i != NumElts; ++i) {
212     if (UndefElts[i]) {
213       ShuffleMask.push_back(SM_SentinelUndef);
214       continue;
215     }
216 
217     // VPERMIL2 Operation.
218     // Bits[3] - Match Bit.
219     // Bits[2:1] - (Per Lane) PD Shuffle Mask.
220     // Bits[2:0] - (Per Lane) PS Shuffle Mask.
221     uint64_t Selector = RawMask[i];
222     unsigned MatchBit = (Selector >> 3) & 0x1;
223 
224     // M2Z[0:1]     MatchBit
225     //   0Xb           X        Source selected by Selector index.
226     //   10b           0        Source selected by Selector index.
227     //   10b           1        Zero.
228     //   11b           0        Zero.
229     //   11b           1        Source selected by Selector index.
230     if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
231       ShuffleMask.push_back(SM_SentinelZero);
232       continue;
233     }
234 
235     int Index = i & ~(NumEltsPerLane - 1);
236     if (ElSize == 64)
237       Index += (Selector >> 1) & 0x1;
238     else
239       Index += Selector & 0x3;
240 
241     int Src = (Selector >> 2) & 0x1;
242     Index += Src * NumElts;
243     ShuffleMask.push_back(Index);
244   }
245 }
246 
247 void DecodeVPPERMMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
248   assert(C->getType()->getPrimitiveSizeInBits() == 128 &&
249          "Unexpected vector size.");
250 
251   // The shuffle mask requires a byte vector.
252   APInt UndefElts;
253   SmallVector<uint64_t, 16> RawMask;
254   if (!extractConstantMask(C, 8, UndefElts, RawMask))
255     return;
256 
257   unsigned NumElts = RawMask.size();
258   assert(NumElts == 16 && "Unexpected number of vector elements.");
259 
260   for (unsigned i = 0; i != NumElts; ++i) {
261     if (UndefElts[i]) {
262       ShuffleMask.push_back(SM_SentinelUndef);
263       continue;
264     }
265 
266     // VPPERM Operation
267     // Bits[4:0] - Byte Index (0 - 31)
268     // Bits[7:5] - Permute Operation
269     //
270     // Permute Operation:
271     // 0 - Source byte (no logical operation).
272     // 1 - Invert source byte.
273     // 2 - Bit reverse of source byte.
274     // 3 - Bit reverse of inverted source byte.
275     // 4 - 00h (zero - fill).
276     // 5 - FFh (ones - fill).
277     // 6 - Most significant bit of source byte replicated in all bit positions.
278     // 7 - Invert most significant bit of source byte and replicate in all bit
279     // positions.
280     uint64_t Element = RawMask[i];
281     uint64_t Index = Element & 0x1F;
282     uint64_t PermuteOp = (Element >> 5) & 0x7;
283 
284     if (PermuteOp == 4) {
285       ShuffleMask.push_back(SM_SentinelZero);
286       continue;
287     }
288     if (PermuteOp != 0) {
289       ShuffleMask.clear();
290       return;
291     }
292     ShuffleMask.push_back((int)Index);
293   }
294 }
295 
296 void DecodeVPERMVMask(const Constant *C, unsigned ElSize,
297                       SmallVectorImpl<int> &ShuffleMask) {
298   Type *MaskTy = C->getType();
299   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
300   (void)MaskTySize;
301   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
302          "Unexpected vector size.");
303   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
304          "Unexpected vector element size.");
305 
306   // The shuffle mask requires elements the same size as the target.
307   APInt UndefElts;
308   SmallVector<uint64_t, 64> RawMask;
309   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
310     return;
311 
312   unsigned NumElts = RawMask.size();
313 
314   for (unsigned i = 0; i != NumElts; ++i) {
315     if (UndefElts[i]) {
316       ShuffleMask.push_back(SM_SentinelUndef);
317       continue;
318     }
319     int Index = RawMask[i] & (NumElts - 1);
320     ShuffleMask.push_back(Index);
321   }
322 }
323 
324 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize,
325                        SmallVectorImpl<int> &ShuffleMask) {
326   Type *MaskTy = C->getType();
327   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
328   (void)MaskTySize;
329   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
330          "Unexpected vector size.");
331   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
332          "Unexpected vector element size.");
333 
334   // The shuffle mask requires elements the same size as the target.
335   APInt UndefElts;
336   SmallVector<uint64_t, 64> RawMask;
337   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
338     return;
339 
340   unsigned NumElts = RawMask.size();
341 
342   for (unsigned i = 0; i != NumElts; ++i) {
343     if (UndefElts[i]) {
344       ShuffleMask.push_back(SM_SentinelUndef);
345       continue;
346     }
347     int Index = RawMask[i] & (NumElts*2 - 1);
348     ShuffleMask.push_back(Index);
349   }
350 }
351 } // llvm namespace
352