1 //===- VPIntrinsicTest.cpp - VPIntrinsic unit tests ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ADT/SmallVector.h"
10 #include "llvm/AsmParser/Parser.h"
11 #include "llvm/CodeGen/ISDOpcodes.h"
12 #include "llvm/IR/Constants.h"
13 #include "llvm/IR/IRBuilder.h"
14 #include "llvm/IR/IntrinsicInst.h"
15 #include "llvm/IR/LLVMContext.h"
16 #include "llvm/IR/Module.h"
17 #include "llvm/IR/Verifier.h"
18 #include "llvm/Support/SourceMgr.h"
19 #include "gtest/gtest.h"
20 #include <sstream>
21 
22 using namespace llvm;
23 
24 namespace {
25 
26 static const char *ReductionIntOpcodes[] = {
27     "add", "mul", "and", "or", "xor", "smin", "smax", "umin", "umax"};
28 
29 static const char *ReductionFPOpcodes[] = {"fadd", "fmul", "fmin", "fmax"};
30 
31 class VPIntrinsicTest : public testing::Test {
32 protected:
33   LLVMContext Context;
34 
35   VPIntrinsicTest() : Context() {}
36 
37   LLVMContext C;
38   SMDiagnostic Err;
39 
40   std::unique_ptr<Module> createVPDeclarationModule() {
41     const char *BinaryIntOpcodes[] = {"add",  "sub",  "mul", "sdiv", "srem",
42                                       "udiv", "urem", "and", "xor",  "or",
43                                       "ashr", "lshr", "shl"};
44     std::stringstream Str;
45     for (const char *BinaryIntOpcode : BinaryIntOpcodes)
46       Str << " declare <8 x i32> @llvm.vp." << BinaryIntOpcode
47           << ".v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) ";
48 
49     const char *BinaryFPOpcodes[] = {"fadd", "fsub", "fmul", "fdiv", "frem"};
50     for (const char *BinaryFPOpcode : BinaryFPOpcodes)
51       Str << " declare <8 x float> @llvm.vp." << BinaryFPOpcode
52           << ".v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) ";
53 
54     Str << " declare void @llvm.vp.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, "
55            "<8 x i1>, i32) ";
56     Str << " declare void @llvm.vp.scatter.v8i32.v8p0i32(<8 x i32>, <8 x "
57            "i32*>, <8 x i1>, i32) ";
58     Str << " declare <8 x i32> @llvm.vp.load.v8i32.p0v8i32(<8 x i32>*, <8 x "
59            "i1>, i32) ";
60     Str << " declare <8 x i32> @llvm.vp.gather.v8i32.v8p0i32(<8 x i32*>, <8 x "
61            "i1>, i32) ";
62 
63     for (const char *ReductionOpcode : ReductionIntOpcodes)
64       Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
65           << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
66 
67     for (const char *ReductionOpcode : ReductionFPOpcodes)
68       Str << " declare float @llvm.vp.reduce." << ReductionOpcode
69           << ".v8f32(float, <8 x float>, <8 x i1>, i32) ";
70 
71     Str << " declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x "
72            "i32>, i32)";
73     return parseAssemblyString(Str.str(), Err, C);
74   }
75 };
76 
77 /// Check that the property scopes include/llvm/IR/VPIntrinsics.def are closed.
78 TEST_F(VPIntrinsicTest, VPIntrinsicsDefScopes) {
79   Optional<Intrinsic::ID> ScopeVPID;
80 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...)                                 \
81   ASSERT_FALSE(ScopeVPID.hasValue());                                          \
82   ScopeVPID = Intrinsic::VPID;
83 #define END_REGISTER_VP_INTRINSIC(VPID)                                        \
84   ASSERT_TRUE(ScopeVPID.hasValue());                                           \
85   ASSERT_EQ(ScopeVPID.getValue(), Intrinsic::VPID);                            \
86   ScopeVPID = None;
87 
88   Optional<ISD::NodeType> ScopeOPC;
89 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...)                                   \
90   ASSERT_FALSE(ScopeOPC.hasValue());                                           \
91   ScopeOPC = ISD::SDOPC;
92 #define END_REGISTER_VP_SDNODE(SDOPC)                                          \
93   ASSERT_TRUE(ScopeOPC.hasValue());                                            \
94   ASSERT_EQ(ScopeOPC.getValue(), ISD::SDOPC);                                  \
95   ScopeOPC = None;
96 #include "llvm/IR/VPIntrinsics.def"
97 
98   ASSERT_FALSE(ScopeVPID.hasValue());
99   ASSERT_FALSE(ScopeOPC.hasValue());
100 }
101 
102 /// Check that every VP intrinsic in the test module is recognized as a VP
103 /// intrinsic.
104 TEST_F(VPIntrinsicTest, VPModuleComplete) {
105   std::unique_ptr<Module> M = createVPDeclarationModule();
106   assert(M);
107 
108   // Check that all @llvm.vp.* functions in the module are recognized vp
109   // intrinsics.
110   std::set<Intrinsic::ID> SeenIDs;
111   for (const auto &VPDecl : *M) {
112     ASSERT_TRUE(VPDecl.isIntrinsic());
113     ASSERT_TRUE(VPIntrinsic::isVPIntrinsic(VPDecl.getIntrinsicID()));
114     SeenIDs.insert(VPDecl.getIntrinsicID());
115   }
116 
117   // Check that every registered VP intrinsic has an instance in the test
118   // module.
119 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...)                                 \
120   ASSERT_TRUE(SeenIDs.count(Intrinsic::VPID));
121 #include "llvm/IR/VPIntrinsics.def"
122 }
123 
124 /// Check that VPIntrinsic:canIgnoreVectorLengthParam() returns true
125 /// if the vector length parameter does not mask off any lanes.
126 TEST_F(VPIntrinsicTest, CanIgnoreVectorLength) {
127   LLVMContext C;
128   SMDiagnostic Err;
129 
130   std::unique_ptr<Module> M =
131       parseAssemblyString(
132 "declare <256 x i64> @llvm.vp.mul.v256i64(<256 x i64>, <256 x i64>, <256 x i1>, i32)"
133 "declare <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)"
134 "declare <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i1>, i32)"
135 "declare i32 @llvm.vscale.i32()"
136 "define void @test_static_vlen( "
137 "      <256 x i64> %i0, <vscale x 2 x i64> %si0x2, <vscale x 1 x i64> %si0x1,"
138 "      <256 x i64> %i1, <vscale x 2 x i64> %si1x2, <vscale x 1 x i64> %si1x1,"
139 "      <256 x i1> %m, <vscale x 2 x i1> %smx2, <vscale x 1 x i1> %smx1, i32 %vl) { "
140 "  %r0 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 %vl)"
141 "  %r1 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 256)"
142 "  %r2 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 0)"
143 "  %r3 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 7)"
144 "  %r4 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 123)"
145 "  %vs = call i32 @llvm.vscale.i32()"
146 "  %vs.x2 = mul i32 %vs, 2"
147 "  %r5 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs.x2)"
148 "  %r6 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs)"
149 "  %r7 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 99999)"
150 "  %r8 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 %vs)"
151 "  %r9 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 1)"
152 "  %r10 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 %vs.x2)"
153 "  %vs.wat = add i32 %vs, 2"
154 "  %r11 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs.wat)"
155 "  ret void "
156 "}",
157           Err, C);
158 
159   auto *F = M->getFunction("test_static_vlen");
160   assert(F);
161 
162   const bool Expected[] = {false, true,  false, false, false, true,
163                            false, false, true,  false, true,  false};
164   const auto *ExpectedIt = std::begin(Expected);
165   for (auto &I : F->getEntryBlock()) {
166     VPIntrinsic *VPI = dyn_cast<VPIntrinsic>(&I);
167     if (!VPI)
168       continue;
169 
170     ASSERT_NE(ExpectedIt, std::end(Expected));
171     ASSERT_EQ(*ExpectedIt, VPI->canIgnoreVectorLengthParam());
172     ++ExpectedIt;
173   }
174 }
175 
176 /// Check that the argument returned by
177 /// VPIntrinsic::get<X>ParamPos(Intrinsic::ID) has the expected type.
178 TEST_F(VPIntrinsicTest, GetParamPos) {
179   std::unique_ptr<Module> M = createVPDeclarationModule();
180   assert(M);
181 
182   for (Function &F : *M) {
183     ASSERT_TRUE(F.isIntrinsic());
184     Optional<unsigned> MaskParamPos =
185         VPIntrinsic::getMaskParamPos(F.getIntrinsicID());
186     if (MaskParamPos.hasValue()) {
187       Type *MaskParamType = F.getArg(MaskParamPos.getValue())->getType();
188       ASSERT_TRUE(MaskParamType->isVectorTy());
189       ASSERT_TRUE(
190           cast<VectorType>(MaskParamType)->getElementType()->isIntegerTy(1));
191     }
192 
193     Optional<unsigned> VecLenParamPos =
194         VPIntrinsic::getVectorLengthParamPos(F.getIntrinsicID());
195     if (VecLenParamPos.hasValue()) {
196       Type *VecLenParamType = F.getArg(VecLenParamPos.getValue())->getType();
197       ASSERT_TRUE(VecLenParamType->isIntegerTy(32));
198     }
199   }
200 }
201 
202 /// Check that going from Opcode to VP intrinsic and back results in the same
203 /// Opcode.
204 TEST_F(VPIntrinsicTest, OpcodeRoundTrip) {
205   std::vector<unsigned> Opcodes;
206   Opcodes.reserve(100);
207 
208   {
209 #define HANDLE_INST(OCNum, OCName, Class) Opcodes.push_back(OCNum);
210 #include "llvm/IR/Instruction.def"
211   }
212 
213   unsigned FullTripCounts = 0;
214   for (unsigned OC : Opcodes) {
215     Intrinsic::ID VPID = VPIntrinsic::getForOpcode(OC);
216     // No equivalent VP intrinsic available.
217     if (VPID == Intrinsic::not_intrinsic)
218       continue;
219 
220     Optional<unsigned> RoundTripOC =
221         VPIntrinsic::getFunctionalOpcodeForVP(VPID);
222     // No equivalent Opcode available.
223     if (!RoundTripOC)
224       continue;
225 
226     ASSERT_EQ(*RoundTripOC, OC);
227     ++FullTripCounts;
228   }
229   ASSERT_NE(FullTripCounts, 0u);
230 }
231 
232 /// Check that going from VP intrinsic to Opcode and back results in the same
233 /// intrinsic id.
234 TEST_F(VPIntrinsicTest, IntrinsicIDRoundTrip) {
235   std::unique_ptr<Module> M = createVPDeclarationModule();
236   assert(M);
237 
238   unsigned FullTripCounts = 0;
239   for (const auto &VPDecl : *M) {
240     auto VPID = VPDecl.getIntrinsicID();
241     Optional<unsigned> OC = VPIntrinsic::getFunctionalOpcodeForVP(VPID);
242 
243     // no equivalent Opcode available
244     if (!OC)
245       continue;
246 
247     Intrinsic::ID RoundTripVPID = VPIntrinsic::getForOpcode(*OC);
248 
249     ASSERT_EQ(RoundTripVPID, VPID);
250     ++FullTripCounts;
251   }
252   ASSERT_NE(FullTripCounts, 0u);
253 }
254 
255 /// Check that VPIntrinsic::getDeclarationForParams works.
256 TEST_F(VPIntrinsicTest, VPIntrinsicDeclarationForParams) {
257   std::unique_ptr<Module> M = createVPDeclarationModule();
258   assert(M);
259 
260   auto OutM = std::make_unique<Module>("", M->getContext());
261 
262   for (auto &F : *M) {
263     auto *FuncTy = F.getFunctionType();
264 
265     // Declare intrinsic anew with explicit types.
266     std::vector<Value *> Values;
267     for (auto *ParamTy : FuncTy->params())
268       Values.push_back(UndefValue::get(ParamTy));
269 
270     ASSERT_NE(F.getIntrinsicID(), Intrinsic::not_intrinsic);
271     auto *NewDecl = VPIntrinsic::getDeclarationForParams(
272         OutM.get(), F.getIntrinsicID(), Values);
273     ASSERT_TRUE(NewDecl);
274 
275     // Check that 'old decl' == 'new decl'.
276     ASSERT_EQ(F.getIntrinsicID(), NewDecl->getIntrinsicID());
277     FunctionType::param_iterator ItNewParams =
278         NewDecl->getFunctionType()->param_begin();
279     FunctionType::param_iterator EndItNewParams =
280         NewDecl->getFunctionType()->param_end();
281     for (auto *ParamTy : FuncTy->params()) {
282       ASSERT_NE(ItNewParams, EndItNewParams);
283       ASSERT_EQ(*ItNewParams, ParamTy);
284       ++ItNewParams;
285     }
286   }
287 }
288 
289 /// Check that the HANDLE_VP_TO_CONSTRAINEDFP maps to an existing intrinsic with
290 /// the right amount of metadata args.
291 TEST_F(VPIntrinsicTest, HandleToConstrainedFP) {
292 #define HANDLE_VP_TO_CONSTRAINEDFP(HASROUND, HASEXCEPT, CFPID)                 \
293   {                                                                            \
294     SmallVector<Intrinsic::IITDescriptor, 5> T;                                \
295     Intrinsic::getIntrinsicInfoTableEntries(Intrinsic::CFPID, T);              \
296     unsigned NumMetadataArgs = 0;                                              \
297     for (auto TD : T)                                                          \
298       NumMetadataArgs += (TD.Kind == Intrinsic::IITDescriptor::Metadata);      \
299     ASSERT_EQ(NumMetadataArgs, (unsigned)(HASROUND + HASEXCEPT));              \
300   }
301 #include "llvm/IR/VPIntrinsics.def"
302 }
303 
304 } // end anonymous namespace
305 
306 /// Check various properties of VPReductionIntrinsics
307 TEST_F(VPIntrinsicTest, VPReductions) {
308   LLVMContext C;
309   SMDiagnostic Err;
310 
311   std::stringstream Str;
312   Str << "declare <8 x i32> @llvm.vp.mul.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, "
313          "i32)";
314   for (const char *ReductionOpcode : ReductionIntOpcodes)
315     Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
316         << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
317 
318   for (const char *ReductionOpcode : ReductionFPOpcodes)
319     Str << " declare float @llvm.vp.reduce." << ReductionOpcode
320         << ".v8f32(float, <8 x float>, <8 x i1>, i32) ";
321 
322   Str << "define void @test_reductions(i32 %start, <8 x i32> %val, float "
323          "%fpstart, <8 x float> %fpval, <8 x i1> %m, i32 %vl) {";
324 
325   // Mix in a regular non-reduction intrinsic to check that the
326   // VPReductionIntrinsic subclass works as intended.
327   Str << "  %r0 = call <8 x i32> @llvm.vp.mul.v8i32(<8 x i32> %val, <8 x i32> "
328          "%val, <8 x i1> %m, i32 %vl)";
329 
330   unsigned Idx = 1;
331   for (const char *ReductionOpcode : ReductionIntOpcodes)
332     Str << "  %r" << Idx++ << " = call i32 @llvm.vp.reduce." << ReductionOpcode
333         << ".v8i32(i32 %start, <8 x i32> %val, <8 x i1> %m, i32 %vl)";
334   for (const char *ReductionOpcode : ReductionFPOpcodes)
335     Str << "  %r" << Idx++ << " = call float @llvm.vp.reduce."
336         << ReductionOpcode
337         << ".v8f32(float %fpstart, <8 x float> %fpval, <8 x i1> %m, i32 %vl)";
338 
339   Str << "  ret void"
340          "}";
341 
342   std::unique_ptr<Module> M = parseAssemblyString(Str.str(), Err, C);
343   assert(M);
344 
345   auto *F = M->getFunction("test_reductions");
346   assert(F);
347 
348   for (const auto &I : F->getEntryBlock()) {
349     const VPIntrinsic *VPI = dyn_cast<VPIntrinsic>(&I);
350     if (!VPI)
351       continue;
352 
353     Intrinsic::ID ID = VPI->getIntrinsicID();
354     const auto *VPRedI = dyn_cast<VPReductionIntrinsic>(&I);
355 
356     if (!VPReductionIntrinsic::isVPReduction(ID)) {
357       EXPECT_EQ(VPRedI, nullptr);
358       EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID).hasValue(), false);
359       EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID).hasValue(), false);
360       continue;
361     }
362 
363     EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID).hasValue(), true);
364     EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID).hasValue(), true);
365     ASSERT_NE(VPRedI, nullptr);
366     EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID),
367               VPRedI->getStartParamPos());
368     EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID),
369               VPRedI->getVectorParamPos());
370     EXPECT_EQ(VPRedI->getStartParamPos(), 0u);
371     EXPECT_EQ(VPRedI->getVectorParamPos(), 1u);
372   }
373 }
374