1 //===- VPIntrinsicTest.cpp - VPIntrinsic unit tests ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ADT/SmallVector.h"
10 #include "llvm/AsmParser/Parser.h"
11 #include "llvm/CodeGen/ISDOpcodes.h"
12 #include "llvm/IR/Constants.h"
13 #include "llvm/IR/IRBuilder.h"
14 #include "llvm/IR/IntrinsicInst.h"
15 #include "llvm/IR/LLVMContext.h"
16 #include "llvm/IR/Module.h"
17 #include "llvm/IR/Verifier.h"
18 #include "llvm/Support/SourceMgr.h"
19 #include "gtest/gtest.h"
20 #include <sstream>
21 
22 using namespace llvm;
23 
24 namespace {
25 
26 static const char *ReductionIntOpcodes[] = {
27     "add", "mul", "and", "or", "xor", "smin", "smax", "umin", "umax"};
28 
29 static const char *ReductionFPOpcodes[] = {"fadd", "fmul", "fmin", "fmax"};
30 
31 class VPIntrinsicTest : public testing::Test {
32 protected:
33   LLVMContext Context;
34 
35   VPIntrinsicTest() : Context() {}
36 
37   LLVMContext C;
38   SMDiagnostic Err;
39 
40   std::unique_ptr<Module> createVPDeclarationModule() {
41     const char *BinaryIntOpcodes[] = {"add",  "sub",  "mul", "sdiv", "srem",
42                                       "udiv", "urem", "and", "xor",  "or",
43                                       "ashr", "lshr", "shl"};
44     std::stringstream Str;
45     for (const char *BinaryIntOpcode : BinaryIntOpcodes)
46       Str << " declare <8 x i32> @llvm.vp." << BinaryIntOpcode
47           << ".v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) ";
48 
49     const char *BinaryFPOpcodes[] = {"fadd", "fsub", "fmul", "fdiv", "frem"};
50     for (const char *BinaryFPOpcode : BinaryFPOpcodes)
51       Str << " declare <8 x float> @llvm.vp." << BinaryFPOpcode
52           << ".v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) ";
53 
54     Str << " declare <8 x float> @llvm.vp.fneg.v8f32(<8 x float>, <8 x i1>, "
55            "i32)";
56     Str << " declare <8 x float> @llvm.vp.fma.v8f32(<8 x float>, <8 x float>, "
57            "<8 x float>, <8 x i1>, i32) ";
58 
59     Str << " declare void @llvm.vp.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, "
60            "<8 x i1>, i32) ";
61     Str << "declare void "
62            "@llvm.experimental.vp.strided.store.v8i32.i32(<8 x i32>, "
63            "i32*, i32, <8 x i1>, i32) ";
64     Str << " declare void @llvm.vp.scatter.v8i32.v8p0i32(<8 x i32>, <8 x "
65            "i32*>, <8 x i1>, i32) ";
66     Str << " declare <8 x i32> @llvm.vp.load.v8i32.p0v8i32(<8 x i32>*, <8 x "
67            "i1>, i32) ";
68     Str << "declare <8 x i32> "
69            "@llvm.experimental.vp.strided.load.v8i32.i32(i32*, i32, <8 "
70            "x i1>, i32) ";
71     Str << " declare <8 x i32> @llvm.vp.gather.v8i32.v8p0i32(<8 x i32*>, <8 x "
72            "i1>, i32) ";
73 
74     for (const char *ReductionOpcode : ReductionIntOpcodes)
75       Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
76           << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
77 
78     for (const char *ReductionOpcode : ReductionFPOpcodes)
79       Str << " declare float @llvm.vp.reduce." << ReductionOpcode
80           << ".v8f32(float, <8 x float>, <8 x i1>, i32) ";
81 
82     Str << " declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x "
83            "i32>, i32)";
84     Str << " declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x "
85            "i32>, i32)";
86     Str << " declare <8 x i32> @llvm.experimental.vp.splice.v8i32(<8 x "
87            "i32>, <8 x i32>, i32, <8 x i1>, i32, i32) ";
88 
89     Str << " declare <8 x i32> @llvm.vp.fptosi.v8i32"
90         << ".v8f32(<8 x float>, <8 x i1>, i32) ";
91     Str << " declare <8 x float> @llvm.vp.sitofp.v8f32"
92         << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
93 
94     return parseAssemblyString(Str.str(), Err, C);
95   }
96 };
97 
98 /// Check that the property scopes include/llvm/IR/VPIntrinsics.def are closed.
99 TEST_F(VPIntrinsicTest, VPIntrinsicsDefScopes) {
100   Optional<Intrinsic::ID> ScopeVPID;
101 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...)                                 \
102   ASSERT_FALSE(ScopeVPID.hasValue());                                          \
103   ScopeVPID = Intrinsic::VPID;
104 #define END_REGISTER_VP_INTRINSIC(VPID)                                        \
105   ASSERT_TRUE(ScopeVPID.hasValue());                                           \
106   ASSERT_EQ(ScopeVPID.getValue(), Intrinsic::VPID);                            \
107   ScopeVPID = None;
108 
109   Optional<ISD::NodeType> ScopeOPC;
110 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...)                                   \
111   ASSERT_FALSE(ScopeOPC.hasValue());                                           \
112   ScopeOPC = ISD::SDOPC;
113 #define END_REGISTER_VP_SDNODE(SDOPC)                                          \
114   ASSERT_TRUE(ScopeOPC.hasValue());                                            \
115   ASSERT_EQ(ScopeOPC.getValue(), ISD::SDOPC);                                  \
116   ScopeOPC = None;
117 #include "llvm/IR/VPIntrinsics.def"
118 
119   ASSERT_FALSE(ScopeVPID.hasValue());
120   ASSERT_FALSE(ScopeOPC.hasValue());
121 }
122 
123 /// Check that every VP intrinsic in the test module is recognized as a VP
124 /// intrinsic.
125 TEST_F(VPIntrinsicTest, VPModuleComplete) {
126   std::unique_ptr<Module> M = createVPDeclarationModule();
127   assert(M);
128 
129   // Check that all @llvm.vp.* functions in the module are recognized vp
130   // intrinsics.
131   std::set<Intrinsic::ID> SeenIDs;
132   for (const auto &VPDecl : *M) {
133     ASSERT_TRUE(VPDecl.isIntrinsic());
134     ASSERT_TRUE(VPIntrinsic::isVPIntrinsic(VPDecl.getIntrinsicID()));
135     SeenIDs.insert(VPDecl.getIntrinsicID());
136   }
137 
138   // Check that every registered VP intrinsic has an instance in the test
139   // module.
140 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...)                                 \
141   ASSERT_TRUE(SeenIDs.count(Intrinsic::VPID));
142 #include "llvm/IR/VPIntrinsics.def"
143 }
144 
145 /// Check that VPIntrinsic:canIgnoreVectorLengthParam() returns true
146 /// if the vector length parameter does not mask off any lanes.
147 TEST_F(VPIntrinsicTest, CanIgnoreVectorLength) {
148   LLVMContext C;
149   SMDiagnostic Err;
150 
151   std::unique_ptr<Module> M =
152       parseAssemblyString(
153 "declare <256 x i64> @llvm.vp.mul.v256i64(<256 x i64>, <256 x i64>, <256 x i1>, i32)"
154 "declare <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)"
155 "declare <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i1>, i32)"
156 "declare i32 @llvm.vscale.i32()"
157 "define void @test_static_vlen( "
158 "      <256 x i64> %i0, <vscale x 2 x i64> %si0x2, <vscale x 1 x i64> %si0x1,"
159 "      <256 x i64> %i1, <vscale x 2 x i64> %si1x2, <vscale x 1 x i64> %si1x1,"
160 "      <256 x i1> %m, <vscale x 2 x i1> %smx2, <vscale x 1 x i1> %smx1, i32 %vl) { "
161 "  %r0 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 %vl)"
162 "  %r1 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 256)"
163 "  %r2 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 0)"
164 "  %r3 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 7)"
165 "  %r4 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 123)"
166 "  %vs = call i32 @llvm.vscale.i32()"
167 "  %vs.x2 = mul i32 %vs, 2"
168 "  %r5 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs.x2)"
169 "  %r6 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs)"
170 "  %r7 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 99999)"
171 "  %r8 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 %vs)"
172 "  %r9 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 1)"
173 "  %r10 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 %vs.x2)"
174 "  %vs.wat = add i32 %vs, 2"
175 "  %r11 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs.wat)"
176 "  ret void "
177 "}",
178           Err, C);
179 
180   auto *F = M->getFunction("test_static_vlen");
181   assert(F);
182 
183   const bool Expected[] = {false, true,  false, false, false, true,
184                            false, false, true,  false, true,  false};
185   const auto *ExpectedIt = std::begin(Expected);
186   for (auto &I : F->getEntryBlock()) {
187     VPIntrinsic *VPI = dyn_cast<VPIntrinsic>(&I);
188     if (!VPI)
189       continue;
190 
191     ASSERT_NE(ExpectedIt, std::end(Expected));
192     ASSERT_EQ(*ExpectedIt, VPI->canIgnoreVectorLengthParam());
193     ++ExpectedIt;
194   }
195 }
196 
197 /// Check that the argument returned by
198 /// VPIntrinsic::get<X>ParamPos(Intrinsic::ID) has the expected type.
199 TEST_F(VPIntrinsicTest, GetParamPos) {
200   std::unique_ptr<Module> M = createVPDeclarationModule();
201   assert(M);
202 
203   for (Function &F : *M) {
204     ASSERT_TRUE(F.isIntrinsic());
205     Optional<unsigned> MaskParamPos =
206         VPIntrinsic::getMaskParamPos(F.getIntrinsicID());
207     if (MaskParamPos.hasValue()) {
208       Type *MaskParamType = F.getArg(MaskParamPos.getValue())->getType();
209       ASSERT_TRUE(MaskParamType->isVectorTy());
210       ASSERT_TRUE(
211           cast<VectorType>(MaskParamType)->getElementType()->isIntegerTy(1));
212     }
213 
214     Optional<unsigned> VecLenParamPos =
215         VPIntrinsic::getVectorLengthParamPos(F.getIntrinsicID());
216     if (VecLenParamPos.hasValue()) {
217       Type *VecLenParamType = F.getArg(VecLenParamPos.getValue())->getType();
218       ASSERT_TRUE(VecLenParamType->isIntegerTy(32));
219     }
220   }
221 }
222 
223 /// Check that going from Opcode to VP intrinsic and back results in the same
224 /// Opcode.
225 TEST_F(VPIntrinsicTest, OpcodeRoundTrip) {
226   std::vector<unsigned> Opcodes;
227   Opcodes.reserve(100);
228 
229   {
230 #define HANDLE_INST(OCNum, OCName, Class) Opcodes.push_back(OCNum);
231 #include "llvm/IR/Instruction.def"
232   }
233 
234   unsigned FullTripCounts = 0;
235   for (unsigned OC : Opcodes) {
236     Intrinsic::ID VPID = VPIntrinsic::getForOpcode(OC);
237     // No equivalent VP intrinsic available.
238     if (VPID == Intrinsic::not_intrinsic)
239       continue;
240 
241     Optional<unsigned> RoundTripOC =
242         VPIntrinsic::getFunctionalOpcodeForVP(VPID);
243     // No equivalent Opcode available.
244     if (!RoundTripOC)
245       continue;
246 
247     ASSERT_EQ(*RoundTripOC, OC);
248     ++FullTripCounts;
249   }
250   ASSERT_NE(FullTripCounts, 0u);
251 }
252 
253 /// Check that going from VP intrinsic to Opcode and back results in the same
254 /// intrinsic id.
255 TEST_F(VPIntrinsicTest, IntrinsicIDRoundTrip) {
256   std::unique_ptr<Module> M = createVPDeclarationModule();
257   assert(M);
258 
259   unsigned FullTripCounts = 0;
260   for (const auto &VPDecl : *M) {
261     auto VPID = VPDecl.getIntrinsicID();
262     Optional<unsigned> OC = VPIntrinsic::getFunctionalOpcodeForVP(VPID);
263 
264     // no equivalent Opcode available
265     if (!OC)
266       continue;
267 
268     Intrinsic::ID RoundTripVPID = VPIntrinsic::getForOpcode(*OC);
269 
270     ASSERT_EQ(RoundTripVPID, VPID);
271     ++FullTripCounts;
272   }
273   ASSERT_NE(FullTripCounts, 0u);
274 }
275 
276 /// Check that VPIntrinsic::getDeclarationForParams works.
277 TEST_F(VPIntrinsicTest, VPIntrinsicDeclarationForParams) {
278   std::unique_ptr<Module> M = createVPDeclarationModule();
279   assert(M);
280 
281   auto OutM = std::make_unique<Module>("", M->getContext());
282 
283   for (auto &F : *M) {
284     auto *FuncTy = F.getFunctionType();
285 
286     // Declare intrinsic anew with explicit types.
287     std::vector<Value *> Values;
288     for (auto *ParamTy : FuncTy->params())
289       Values.push_back(UndefValue::get(ParamTy));
290 
291     ASSERT_NE(F.getIntrinsicID(), Intrinsic::not_intrinsic);
292     auto *NewDecl = VPIntrinsic::getDeclarationForParams(
293         OutM.get(), F.getIntrinsicID(), FuncTy->getReturnType(), Values);
294     ASSERT_TRUE(NewDecl);
295 
296     // Check that 'old decl' == 'new decl'.
297     ASSERT_EQ(F.getIntrinsicID(), NewDecl->getIntrinsicID());
298     FunctionType::param_iterator ItNewParams =
299         NewDecl->getFunctionType()->param_begin();
300     FunctionType::param_iterator EndItNewParams =
301         NewDecl->getFunctionType()->param_end();
302     for (auto *ParamTy : FuncTy->params()) {
303       ASSERT_NE(ItNewParams, EndItNewParams);
304       ASSERT_EQ(*ItNewParams, ParamTy);
305       ++ItNewParams;
306     }
307   }
308 }
309 
310 /// Check that the HANDLE_VP_TO_CONSTRAINEDFP maps to an existing intrinsic with
311 /// the right amount of metadata args.
312 TEST_F(VPIntrinsicTest, HandleToConstrainedFP) {
313 #define VP_PROPERTY_CONSTRAINEDFP(HASROUND, HASEXCEPT, CFPID)                  \
314   {                                                                            \
315     SmallVector<Intrinsic::IITDescriptor, 5> T;                                \
316     Intrinsic::getIntrinsicInfoTableEntries(Intrinsic::CFPID, T);              \
317     unsigned NumMetadataArgs = 0;                                              \
318     for (auto TD : T)                                                          \
319       NumMetadataArgs += (TD.Kind == Intrinsic::IITDescriptor::Metadata);      \
320     ASSERT_EQ(NumMetadataArgs, (unsigned)(HASROUND + HASEXCEPT));              \
321   }
322 #include "llvm/IR/VPIntrinsics.def"
323 }
324 
325 } // end anonymous namespace
326 
327 /// Check various properties of VPReductionIntrinsics
328 TEST_F(VPIntrinsicTest, VPReductions) {
329   LLVMContext C;
330   SMDiagnostic Err;
331 
332   std::stringstream Str;
333   Str << "declare <8 x i32> @llvm.vp.mul.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, "
334          "i32)";
335   for (const char *ReductionOpcode : ReductionIntOpcodes)
336     Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
337         << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
338 
339   for (const char *ReductionOpcode : ReductionFPOpcodes)
340     Str << " declare float @llvm.vp.reduce." << ReductionOpcode
341         << ".v8f32(float, <8 x float>, <8 x i1>, i32) ";
342 
343   Str << "define void @test_reductions(i32 %start, <8 x i32> %val, float "
344          "%fpstart, <8 x float> %fpval, <8 x i1> %m, i32 %vl) {";
345 
346   // Mix in a regular non-reduction intrinsic to check that the
347   // VPReductionIntrinsic subclass works as intended.
348   Str << "  %r0 = call <8 x i32> @llvm.vp.mul.v8i32(<8 x i32> %val, <8 x i32> "
349          "%val, <8 x i1> %m, i32 %vl)";
350 
351   unsigned Idx = 1;
352   for (const char *ReductionOpcode : ReductionIntOpcodes)
353     Str << "  %r" << Idx++ << " = call i32 @llvm.vp.reduce." << ReductionOpcode
354         << ".v8i32(i32 %start, <8 x i32> %val, <8 x i1> %m, i32 %vl)";
355   for (const char *ReductionOpcode : ReductionFPOpcodes)
356     Str << "  %r" << Idx++ << " = call float @llvm.vp.reduce."
357         << ReductionOpcode
358         << ".v8f32(float %fpstart, <8 x float> %fpval, <8 x i1> %m, i32 %vl)";
359 
360   Str << "  ret void"
361          "}";
362 
363   std::unique_ptr<Module> M = parseAssemblyString(Str.str(), Err, C);
364   assert(M);
365 
366   auto *F = M->getFunction("test_reductions");
367   assert(F);
368 
369   for (const auto &I : F->getEntryBlock()) {
370     const VPIntrinsic *VPI = dyn_cast<VPIntrinsic>(&I);
371     if (!VPI)
372       continue;
373 
374     Intrinsic::ID ID = VPI->getIntrinsicID();
375     const auto *VPRedI = dyn_cast<VPReductionIntrinsic>(&I);
376 
377     if (!VPReductionIntrinsic::isVPReduction(ID)) {
378       EXPECT_EQ(VPRedI, nullptr);
379       EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID).hasValue(), false);
380       EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID).hasValue(), false);
381       continue;
382     }
383 
384     EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID).hasValue(), true);
385     EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID).hasValue(), true);
386     ASSERT_NE(VPRedI, nullptr);
387     EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID),
388               VPRedI->getStartParamPos());
389     EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID),
390               VPRedI->getVectorParamPos());
391     EXPECT_EQ(VPRedI->getStartParamPos(), 0u);
392     EXPECT_EQ(VPRedI->getVectorParamPos(), 1u);
393   }
394 }
395