1 //===- VPIntrinsicTest.cpp - VPIntrinsic unit tests ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ADT/SmallVector.h"
10 #include "llvm/AsmParser/Parser.h"
11 #include "llvm/CodeGen/ISDOpcodes.h"
12 #include "llvm/IR/Constants.h"
13 #include "llvm/IR/IRBuilder.h"
14 #include "llvm/IR/IntrinsicInst.h"
15 #include "llvm/IR/LLVMContext.h"
16 #include "llvm/IR/Module.h"
17 #include "llvm/IR/Verifier.h"
18 #include "llvm/Support/SourceMgr.h"
19 #include "gtest/gtest.h"
20 #include <sstream>
21 
22 using namespace llvm;
23 
24 namespace {
25 
26 static const char *ReductionIntOpcodes[] = {
27     "add", "mul", "and", "or", "xor", "smin", "smax", "umin", "umax"};
28 
29 static const char *ReductionFPOpcodes[] = {"fadd", "fmul", "fmin", "fmax"};
30 
31 class VPIntrinsicTest : public testing::Test {
32 protected:
33   LLVMContext Context;
34 
35   VPIntrinsicTest() : Context() {}
36 
37   LLVMContext C;
38   SMDiagnostic Err;
39 
40   std::unique_ptr<Module> createVPDeclarationModule() {
41     const char *BinaryIntOpcodes[] = {"add",  "sub",  "mul", "sdiv", "srem",
42                                       "udiv", "urem", "and", "xor",  "or",
43                                       "ashr", "lshr", "shl"};
44     std::stringstream Str;
45     for (const char *BinaryIntOpcode : BinaryIntOpcodes)
46       Str << " declare <8 x i32> @llvm.vp." << BinaryIntOpcode
47           << ".v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) ";
48 
49     const char *BinaryFPOpcodes[] = {"fadd", "fsub", "fmul", "fdiv", "frem"};
50     for (const char *BinaryFPOpcode : BinaryFPOpcodes)
51       Str << " declare <8 x float> @llvm.vp." << BinaryFPOpcode
52           << ".v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) ";
53 
54     Str << " declare void @llvm.vp.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, "
55            "<8 x i1>, i32) ";
56     Str << " declare void @llvm.vp.scatter.v8i32.v8p0i32(<8 x i32>, <8 x "
57            "i32*>, <8 x i1>, i32) ";
58     Str << " declare <8 x i32> @llvm.vp.load.v8i32.p0v8i32(<8 x i32>*, <8 x "
59            "i1>, i32) ";
60     Str << " declare <8 x i32> @llvm.vp.gather.v8i32.v8p0i32(<8 x i32*>, <8 x "
61            "i1>, i32) ";
62 
63     for (const char *ReductionOpcode : ReductionIntOpcodes)
64       Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
65           << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
66 
67     for (const char *ReductionOpcode : ReductionFPOpcodes)
68       Str << " declare float @llvm.vp.reduce." << ReductionOpcode
69           << ".v8f32(float, <8 x float>, <8 x i1>, i32) ";
70 
71     Str << " declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x "
72            "i32>, i32)";
73     Str << " declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x "
74            "i32>, i32)";
75     Str << " declare <8 x i32> @llvm.experimental.vp.splice.v8i32(<8 x "
76            "i32>, <8 x i32>, i32, <8 x i1>, i32, i32) ";
77 
78     return parseAssemblyString(Str.str(), Err, C);
79   }
80 };
81 
82 /// Check that the property scopes include/llvm/IR/VPIntrinsics.def are closed.
83 TEST_F(VPIntrinsicTest, VPIntrinsicsDefScopes) {
84   Optional<Intrinsic::ID> ScopeVPID;
85 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...)                                 \
86   ASSERT_FALSE(ScopeVPID.hasValue());                                          \
87   ScopeVPID = Intrinsic::VPID;
88 #define END_REGISTER_VP_INTRINSIC(VPID)                                        \
89   ASSERT_TRUE(ScopeVPID.hasValue());                                           \
90   ASSERT_EQ(ScopeVPID.getValue(), Intrinsic::VPID);                            \
91   ScopeVPID = None;
92 
93   Optional<ISD::NodeType> ScopeOPC;
94 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...)                                   \
95   ASSERT_FALSE(ScopeOPC.hasValue());                                           \
96   ScopeOPC = ISD::SDOPC;
97 #define END_REGISTER_VP_SDNODE(SDOPC)                                          \
98   ASSERT_TRUE(ScopeOPC.hasValue());                                            \
99   ASSERT_EQ(ScopeOPC.getValue(), ISD::SDOPC);                                  \
100   ScopeOPC = None;
101 #include "llvm/IR/VPIntrinsics.def"
102 
103   ASSERT_FALSE(ScopeVPID.hasValue());
104   ASSERT_FALSE(ScopeOPC.hasValue());
105 }
106 
107 /// Check that every VP intrinsic in the test module is recognized as a VP
108 /// intrinsic.
109 TEST_F(VPIntrinsicTest, VPModuleComplete) {
110   std::unique_ptr<Module> M = createVPDeclarationModule();
111   assert(M);
112 
113   // Check that all @llvm.vp.* functions in the module are recognized vp
114   // intrinsics.
115   std::set<Intrinsic::ID> SeenIDs;
116   for (const auto &VPDecl : *M) {
117     ASSERT_TRUE(VPDecl.isIntrinsic());
118     ASSERT_TRUE(VPIntrinsic::isVPIntrinsic(VPDecl.getIntrinsicID()));
119     SeenIDs.insert(VPDecl.getIntrinsicID());
120   }
121 
122   // Check that every registered VP intrinsic has an instance in the test
123   // module.
124 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...)                                 \
125   ASSERT_TRUE(SeenIDs.count(Intrinsic::VPID));
126 #include "llvm/IR/VPIntrinsics.def"
127 }
128 
129 /// Check that VPIntrinsic:canIgnoreVectorLengthParam() returns true
130 /// if the vector length parameter does not mask off any lanes.
131 TEST_F(VPIntrinsicTest, CanIgnoreVectorLength) {
132   LLVMContext C;
133   SMDiagnostic Err;
134 
135   std::unique_ptr<Module> M =
136       parseAssemblyString(
137 "declare <256 x i64> @llvm.vp.mul.v256i64(<256 x i64>, <256 x i64>, <256 x i1>, i32)"
138 "declare <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)"
139 "declare <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i1>, i32)"
140 "declare i32 @llvm.vscale.i32()"
141 "define void @test_static_vlen( "
142 "      <256 x i64> %i0, <vscale x 2 x i64> %si0x2, <vscale x 1 x i64> %si0x1,"
143 "      <256 x i64> %i1, <vscale x 2 x i64> %si1x2, <vscale x 1 x i64> %si1x1,"
144 "      <256 x i1> %m, <vscale x 2 x i1> %smx2, <vscale x 1 x i1> %smx1, i32 %vl) { "
145 "  %r0 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 %vl)"
146 "  %r1 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 256)"
147 "  %r2 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 0)"
148 "  %r3 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 7)"
149 "  %r4 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 123)"
150 "  %vs = call i32 @llvm.vscale.i32()"
151 "  %vs.x2 = mul i32 %vs, 2"
152 "  %r5 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs.x2)"
153 "  %r6 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs)"
154 "  %r7 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 99999)"
155 "  %r8 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 %vs)"
156 "  %r9 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 1)"
157 "  %r10 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 %vs.x2)"
158 "  %vs.wat = add i32 %vs, 2"
159 "  %r11 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs.wat)"
160 "  ret void "
161 "}",
162           Err, C);
163 
164   auto *F = M->getFunction("test_static_vlen");
165   assert(F);
166 
167   const bool Expected[] = {false, true,  false, false, false, true,
168                            false, false, true,  false, true,  false};
169   const auto *ExpectedIt = std::begin(Expected);
170   for (auto &I : F->getEntryBlock()) {
171     VPIntrinsic *VPI = dyn_cast<VPIntrinsic>(&I);
172     if (!VPI)
173       continue;
174 
175     ASSERT_NE(ExpectedIt, std::end(Expected));
176     ASSERT_EQ(*ExpectedIt, VPI->canIgnoreVectorLengthParam());
177     ++ExpectedIt;
178   }
179 }
180 
181 /// Check that the argument returned by
182 /// VPIntrinsic::get<X>ParamPos(Intrinsic::ID) has the expected type.
183 TEST_F(VPIntrinsicTest, GetParamPos) {
184   std::unique_ptr<Module> M = createVPDeclarationModule();
185   assert(M);
186 
187   for (Function &F : *M) {
188     ASSERT_TRUE(F.isIntrinsic());
189     Optional<unsigned> MaskParamPos =
190         VPIntrinsic::getMaskParamPos(F.getIntrinsicID());
191     if (MaskParamPos.hasValue()) {
192       Type *MaskParamType = F.getArg(MaskParamPos.getValue())->getType();
193       ASSERT_TRUE(MaskParamType->isVectorTy());
194       ASSERT_TRUE(
195           cast<VectorType>(MaskParamType)->getElementType()->isIntegerTy(1));
196     }
197 
198     Optional<unsigned> VecLenParamPos =
199         VPIntrinsic::getVectorLengthParamPos(F.getIntrinsicID());
200     if (VecLenParamPos.hasValue()) {
201       Type *VecLenParamType = F.getArg(VecLenParamPos.getValue())->getType();
202       ASSERT_TRUE(VecLenParamType->isIntegerTy(32));
203     }
204   }
205 }
206 
207 /// Check that going from Opcode to VP intrinsic and back results in the same
208 /// Opcode.
209 TEST_F(VPIntrinsicTest, OpcodeRoundTrip) {
210   std::vector<unsigned> Opcodes;
211   Opcodes.reserve(100);
212 
213   {
214 #define HANDLE_INST(OCNum, OCName, Class) Opcodes.push_back(OCNum);
215 #include "llvm/IR/Instruction.def"
216   }
217 
218   unsigned FullTripCounts = 0;
219   for (unsigned OC : Opcodes) {
220     Intrinsic::ID VPID = VPIntrinsic::getForOpcode(OC);
221     // No equivalent VP intrinsic available.
222     if (VPID == Intrinsic::not_intrinsic)
223       continue;
224 
225     Optional<unsigned> RoundTripOC =
226         VPIntrinsic::getFunctionalOpcodeForVP(VPID);
227     // No equivalent Opcode available.
228     if (!RoundTripOC)
229       continue;
230 
231     ASSERT_EQ(*RoundTripOC, OC);
232     ++FullTripCounts;
233   }
234   ASSERT_NE(FullTripCounts, 0u);
235 }
236 
237 /// Check that going from VP intrinsic to Opcode and back results in the same
238 /// intrinsic id.
239 TEST_F(VPIntrinsicTest, IntrinsicIDRoundTrip) {
240   std::unique_ptr<Module> M = createVPDeclarationModule();
241   assert(M);
242 
243   unsigned FullTripCounts = 0;
244   for (const auto &VPDecl : *M) {
245     auto VPID = VPDecl.getIntrinsicID();
246     Optional<unsigned> OC = VPIntrinsic::getFunctionalOpcodeForVP(VPID);
247 
248     // no equivalent Opcode available
249     if (!OC)
250       continue;
251 
252     Intrinsic::ID RoundTripVPID = VPIntrinsic::getForOpcode(*OC);
253 
254     ASSERT_EQ(RoundTripVPID, VPID);
255     ++FullTripCounts;
256   }
257   ASSERT_NE(FullTripCounts, 0u);
258 }
259 
260 /// Check that VPIntrinsic::getDeclarationForParams works.
261 TEST_F(VPIntrinsicTest, VPIntrinsicDeclarationForParams) {
262   std::unique_ptr<Module> M = createVPDeclarationModule();
263   assert(M);
264 
265   auto OutM = std::make_unique<Module>("", M->getContext());
266 
267   for (auto &F : *M) {
268     auto *FuncTy = F.getFunctionType();
269 
270     // Declare intrinsic anew with explicit types.
271     std::vector<Value *> Values;
272     for (auto *ParamTy : FuncTy->params())
273       Values.push_back(UndefValue::get(ParamTy));
274 
275     ASSERT_NE(F.getIntrinsicID(), Intrinsic::not_intrinsic);
276     auto *NewDecl = VPIntrinsic::getDeclarationForParams(
277         OutM.get(), F.getIntrinsicID(), FuncTy->getReturnType(), Values);
278     ASSERT_TRUE(NewDecl);
279 
280     // Check that 'old decl' == 'new decl'.
281     ASSERT_EQ(F.getIntrinsicID(), NewDecl->getIntrinsicID());
282     FunctionType::param_iterator ItNewParams =
283         NewDecl->getFunctionType()->param_begin();
284     FunctionType::param_iterator EndItNewParams =
285         NewDecl->getFunctionType()->param_end();
286     for (auto *ParamTy : FuncTy->params()) {
287       ASSERT_NE(ItNewParams, EndItNewParams);
288       ASSERT_EQ(*ItNewParams, ParamTy);
289       ++ItNewParams;
290     }
291   }
292 }
293 
294 /// Check that the HANDLE_VP_TO_CONSTRAINEDFP maps to an existing intrinsic with
295 /// the right amount of metadata args.
296 TEST_F(VPIntrinsicTest, HandleToConstrainedFP) {
297 #define VP_PROPERTY_CONSTRAINEDFP(HASROUND, HASEXCEPT, CFPID)                  \
298   {                                                                            \
299     SmallVector<Intrinsic::IITDescriptor, 5> T;                                \
300     Intrinsic::getIntrinsicInfoTableEntries(Intrinsic::CFPID, T);              \
301     unsigned NumMetadataArgs = 0;                                              \
302     for (auto TD : T)                                                          \
303       NumMetadataArgs += (TD.Kind == Intrinsic::IITDescriptor::Metadata);      \
304     ASSERT_EQ(NumMetadataArgs, (unsigned)(HASROUND + HASEXCEPT));              \
305   }
306 #include "llvm/IR/VPIntrinsics.def"
307 }
308 
309 } // end anonymous namespace
310 
311 /// Check various properties of VPReductionIntrinsics
312 TEST_F(VPIntrinsicTest, VPReductions) {
313   LLVMContext C;
314   SMDiagnostic Err;
315 
316   std::stringstream Str;
317   Str << "declare <8 x i32> @llvm.vp.mul.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, "
318          "i32)";
319   for (const char *ReductionOpcode : ReductionIntOpcodes)
320     Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
321         << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
322 
323   for (const char *ReductionOpcode : ReductionFPOpcodes)
324     Str << " declare float @llvm.vp.reduce." << ReductionOpcode
325         << ".v8f32(float, <8 x float>, <8 x i1>, i32) ";
326 
327   Str << "define void @test_reductions(i32 %start, <8 x i32> %val, float "
328          "%fpstart, <8 x float> %fpval, <8 x i1> %m, i32 %vl) {";
329 
330   // Mix in a regular non-reduction intrinsic to check that the
331   // VPReductionIntrinsic subclass works as intended.
332   Str << "  %r0 = call <8 x i32> @llvm.vp.mul.v8i32(<8 x i32> %val, <8 x i32> "
333          "%val, <8 x i1> %m, i32 %vl)";
334 
335   unsigned Idx = 1;
336   for (const char *ReductionOpcode : ReductionIntOpcodes)
337     Str << "  %r" << Idx++ << " = call i32 @llvm.vp.reduce." << ReductionOpcode
338         << ".v8i32(i32 %start, <8 x i32> %val, <8 x i1> %m, i32 %vl)";
339   for (const char *ReductionOpcode : ReductionFPOpcodes)
340     Str << "  %r" << Idx++ << " = call float @llvm.vp.reduce."
341         << ReductionOpcode
342         << ".v8f32(float %fpstart, <8 x float> %fpval, <8 x i1> %m, i32 %vl)";
343 
344   Str << "  ret void"
345          "}";
346 
347   std::unique_ptr<Module> M = parseAssemblyString(Str.str(), Err, C);
348   assert(M);
349 
350   auto *F = M->getFunction("test_reductions");
351   assert(F);
352 
353   for (const auto &I : F->getEntryBlock()) {
354     const VPIntrinsic *VPI = dyn_cast<VPIntrinsic>(&I);
355     if (!VPI)
356       continue;
357 
358     Intrinsic::ID ID = VPI->getIntrinsicID();
359     const auto *VPRedI = dyn_cast<VPReductionIntrinsic>(&I);
360 
361     if (!VPReductionIntrinsic::isVPReduction(ID)) {
362       EXPECT_EQ(VPRedI, nullptr);
363       EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID).hasValue(), false);
364       EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID).hasValue(), false);
365       continue;
366     }
367 
368     EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID).hasValue(), true);
369     EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID).hasValue(), true);
370     ASSERT_NE(VPRedI, nullptr);
371     EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID),
372               VPRedI->getStartParamPos());
373     EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID),
374               VPRedI->getVectorParamPos());
375     EXPECT_EQ(VPRedI->getStartParamPos(), 0u);
376     EXPECT_EQ(VPRedI->getVectorParamPos(), 1u);
377   }
378 }
379