1 //===- VPIntrinsicTest.cpp - VPIntrinsic unit tests ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ADT/SmallVector.h"
10 #include "llvm/AsmParser/Parser.h"
11 #include "llvm/CodeGen/ISDOpcodes.h"
12 #include "llvm/IR/Constants.h"
13 #include "llvm/IR/IRBuilder.h"
14 #include "llvm/IR/IntrinsicInst.h"
15 #include "llvm/IR/LLVMContext.h"
16 #include "llvm/IR/Module.h"
17 #include "llvm/IR/Verifier.h"
18 #include "llvm/Support/SourceMgr.h"
19 #include "gtest/gtest.h"
20 #include <sstream>
21 
22 using namespace llvm;
23 
24 namespace {
25 
26 static const char *ReductionIntOpcodes[] = {
27     "add", "mul", "and", "or", "xor", "smin", "smax", "umin", "umax"};
28 
29 static const char *ReductionFPOpcodes[] = {"fadd", "fmul", "fmin", "fmax"};
30 
31 class VPIntrinsicTest : public testing::Test {
32 protected:
33   LLVMContext Context;
34 
35   VPIntrinsicTest() : Context() {}
36 
37   LLVMContext C;
38   SMDiagnostic Err;
39 
40   std::unique_ptr<Module> createVPDeclarationModule() {
41     const char *BinaryIntOpcodes[] = {"add",  "sub",  "mul", "sdiv", "srem",
42                                       "udiv", "urem", "and", "xor",  "or",
43                                       "ashr", "lshr", "shl"};
44     std::stringstream Str;
45     for (const char *BinaryIntOpcode : BinaryIntOpcodes)
46       Str << " declare <8 x i32> @llvm.vp." << BinaryIntOpcode
47           << ".v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) ";
48 
49     const char *BinaryFPOpcodes[] = {"fadd", "fsub", "fmul", "fdiv", "frem"};
50     for (const char *BinaryFPOpcode : BinaryFPOpcodes)
51       Str << " declare <8 x float> @llvm.vp." << BinaryFPOpcode
52           << ".v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) ";
53 
54     Str << " declare <8 x float> @llvm.vp.fneg.v8f32(<8 x float>, <8 x i1>, "
55            "i32)";
56     Str << " declare <8 x float> @llvm.vp.fma.v8f32(<8 x float>, <8 x float>, "
57            "<8 x float>, <8 x i1>, i32) ";
58 
59     Str << " declare void @llvm.vp.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, "
60            "<8 x i1>, i32) ";
61     Str << "declare void "
62            "@llvm.experimental.vp.strided.store.v8i32.i32(<8 x i32>, "
63            "i32*, i32, <8 x i1>, i32) ";
64     Str << "declare void "
65            "@llvm.experimental.vp.strided.store.v8i32.p1i32.i32(<8 x i32>, "
66            "i32 addrspace(1)*, i32, <8 x i1>, i32) ";
67     Str << " declare void @llvm.vp.scatter.v8i32.v8p0i32(<8 x i32>, <8 x "
68            "i32*>, <8 x i1>, i32) ";
69     Str << " declare <8 x i32> @llvm.vp.load.v8i32.p0v8i32(<8 x i32>*, <8 x "
70            "i1>, i32) ";
71     Str << "declare <8 x i32> "
72            "@llvm.experimental.vp.strided.load.v8i32.i32(i32*, i32, <8 "
73            "x i1>, i32) ";
74     Str << "declare <8 x i32> "
75            "@llvm.experimental.vp.strided.load.v8i32.p1i32.i32(i32 "
76            "addrspace(1)*, i32, <8 x i1>, i32) ";
77     Str << " declare <8 x i32> @llvm.vp.gather.v8i32.v8p0i32(<8 x i32*>, <8 x "
78            "i1>, i32) ";
79 
80     for (const char *ReductionOpcode : ReductionIntOpcodes)
81       Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
82           << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
83 
84     for (const char *ReductionOpcode : ReductionFPOpcodes)
85       Str << " declare float @llvm.vp.reduce." << ReductionOpcode
86           << ".v8f32(float, <8 x float>, <8 x i1>, i32) ";
87 
88     Str << " declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x "
89            "i32>, i32)";
90     Str << " declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x "
91            "i32>, i32)";
92     Str << " declare <8 x i32> @llvm.experimental.vp.splice.v8i32(<8 x "
93            "i32>, <8 x i32>, i32, <8 x i1>, i32, i32) ";
94 
95     Str << " declare <8 x i32> @llvm.vp.fptoui.v8i32"
96         << ".v8f32(<8 x float>, <8 x i1>, i32) ";
97     Str << " declare <8 x i32> @llvm.vp.fptosi.v8i32"
98         << ".v8f32(<8 x float>, <8 x i1>, i32) ";
99     Str << " declare <8 x float> @llvm.vp.uitofp.v8f32"
100         << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
101     Str << " declare <8 x float> @llvm.vp.sitofp.v8f32"
102         << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
103     Str << " declare <8 x float> @llvm.vp.fptrunc.v8f32"
104         << ".v8f64(<8 x double>, <8 x i1>, i32) ";
105     Str << " declare <8 x double> @llvm.vp.fpext.v8f64"
106         << ".v8f32(<8 x float>, <8 x i1>, i32) ";
107     Str << " declare <8 x i32> @llvm.vp.trunc.v8i32"
108         << ".v8i64(<8 x i64>, <8 x i1>, i32) ";
109     Str << " declare <8 x i64> @llvm.vp.zext.v8i64"
110         << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
111     Str << " declare <8 x i64> @llvm.vp.sext.v8i64"
112         << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
113     Str << " declare <8 x i32> @llvm.vp.ptrtoint.v8i32"
114         << ".v8p0i32(<8 x i32*>, <8 x i1>, i32) ";
115     Str << " declare <8 x i32*> @llvm.vp.inttoptr.v8p0i32"
116         << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
117 
118     Str << " declare <8 x i1> @llvm.vp.fcmp.v8f32"
119         << "(<8 x float>, <8 x float>, metadata, <8 x i1>, i32) ";
120     Str << " declare <8 x i1> @llvm.vp.icmp.v8i16"
121         << "(<8 x i16>, <8 x i16>, metadata, <8 x i1>, i32) ";
122 
123     return parseAssemblyString(Str.str(), Err, C);
124   }
125 };
126 
127 /// Check that the property scopes include/llvm/IR/VPIntrinsics.def are closed.
128 TEST_F(VPIntrinsicTest, VPIntrinsicsDefScopes) {
129   Optional<Intrinsic::ID> ScopeVPID;
130 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...)                                 \
131   ASSERT_FALSE(ScopeVPID.has_value());                                         \
132   ScopeVPID = Intrinsic::VPID;
133 #define END_REGISTER_VP_INTRINSIC(VPID)                                        \
134   ASSERT_TRUE(ScopeVPID.has_value());                                          \
135   ASSERT_EQ(ScopeVPID.value(), Intrinsic::VPID);                               \
136   ScopeVPID = None;
137 
138   Optional<ISD::NodeType> ScopeOPC;
139 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...)                                   \
140   ASSERT_FALSE(ScopeOPC.has_value());                                          \
141   ScopeOPC = ISD::SDOPC;
142 #define END_REGISTER_VP_SDNODE(SDOPC)                                          \
143   ASSERT_TRUE(ScopeOPC.has_value());                                           \
144   ASSERT_EQ(ScopeOPC.value(), ISD::SDOPC);                                     \
145   ScopeOPC = None;
146 #include "llvm/IR/VPIntrinsics.def"
147 
148   ASSERT_FALSE(ScopeVPID.has_value());
149   ASSERT_FALSE(ScopeOPC.has_value());
150 }
151 
152 /// Check that every VP intrinsic in the test module is recognized as a VP
153 /// intrinsic.
154 TEST_F(VPIntrinsicTest, VPModuleComplete) {
155   std::unique_ptr<Module> M = createVPDeclarationModule();
156   assert(M);
157 
158   // Check that all @llvm.vp.* functions in the module are recognized vp
159   // intrinsics.
160   std::set<Intrinsic::ID> SeenIDs;
161   for (const auto &VPDecl : *M) {
162     ASSERT_TRUE(VPDecl.isIntrinsic());
163     ASSERT_TRUE(VPIntrinsic::isVPIntrinsic(VPDecl.getIntrinsicID()));
164     SeenIDs.insert(VPDecl.getIntrinsicID());
165   }
166 
167   // Check that every registered VP intrinsic has an instance in the test
168   // module.
169 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...)                                 \
170   ASSERT_TRUE(SeenIDs.count(Intrinsic::VPID));
171 #include "llvm/IR/VPIntrinsics.def"
172 }
173 
174 /// Check that VPIntrinsic:canIgnoreVectorLengthParam() returns true
175 /// if the vector length parameter does not mask off any lanes.
176 TEST_F(VPIntrinsicTest, CanIgnoreVectorLength) {
177   LLVMContext C;
178   SMDiagnostic Err;
179 
180   std::unique_ptr<Module> M =
181       parseAssemblyString(
182 "declare <256 x i64> @llvm.vp.mul.v256i64(<256 x i64>, <256 x i64>, <256 x i1>, i32)"
183 "declare <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)"
184 "declare <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i1>, i32)"
185 "declare i32 @llvm.vscale.i32()"
186 "define void @test_static_vlen( "
187 "      <256 x i64> %i0, <vscale x 2 x i64> %si0x2, <vscale x 1 x i64> %si0x1,"
188 "      <256 x i64> %i1, <vscale x 2 x i64> %si1x2, <vscale x 1 x i64> %si1x1,"
189 "      <256 x i1> %m, <vscale x 2 x i1> %smx2, <vscale x 1 x i1> %smx1, i32 %vl) { "
190 "  %r0 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 %vl)"
191 "  %r1 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 256)"
192 "  %r2 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 0)"
193 "  %r3 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 7)"
194 "  %r4 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 123)"
195 "  %vs = call i32 @llvm.vscale.i32()"
196 "  %vs.x2 = mul i32 %vs, 2"
197 "  %r5 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs.x2)"
198 "  %r6 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs)"
199 "  %r7 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 99999)"
200 "  %r8 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 %vs)"
201 "  %r9 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 1)"
202 "  %r10 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 %vs.x2)"
203 "  %vs.wat = add i32 %vs, 2"
204 "  %r11 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs.wat)"
205 "  ret void "
206 "}",
207           Err, C);
208 
209   auto *F = M->getFunction("test_static_vlen");
210   assert(F);
211 
212   const bool Expected[] = {false, true,  false, false, false, true,
213                            false, false, true,  false, true,  false};
214   const auto *ExpectedIt = std::begin(Expected);
215   for (auto &I : F->getEntryBlock()) {
216     VPIntrinsic *VPI = dyn_cast<VPIntrinsic>(&I);
217     if (!VPI)
218       continue;
219 
220     ASSERT_NE(ExpectedIt, std::end(Expected));
221     ASSERT_EQ(*ExpectedIt, VPI->canIgnoreVectorLengthParam());
222     ++ExpectedIt;
223   }
224 }
225 
226 /// Check that the argument returned by
227 /// VPIntrinsic::get<X>ParamPos(Intrinsic::ID) has the expected type.
228 TEST_F(VPIntrinsicTest, GetParamPos) {
229   std::unique_ptr<Module> M = createVPDeclarationModule();
230   assert(M);
231 
232   for (Function &F : *M) {
233     ASSERT_TRUE(F.isIntrinsic());
234     Optional<unsigned> MaskParamPos =
235         VPIntrinsic::getMaskParamPos(F.getIntrinsicID());
236     if (MaskParamPos) {
237       Type *MaskParamType = F.getArg(MaskParamPos.value())->getType();
238       ASSERT_TRUE(MaskParamType->isVectorTy());
239       ASSERT_TRUE(
240           cast<VectorType>(MaskParamType)->getElementType()->isIntegerTy(1));
241     }
242 
243     Optional<unsigned> VecLenParamPos =
244         VPIntrinsic::getVectorLengthParamPos(F.getIntrinsicID());
245     if (VecLenParamPos) {
246       Type *VecLenParamType = F.getArg(VecLenParamPos.value())->getType();
247       ASSERT_TRUE(VecLenParamType->isIntegerTy(32));
248     }
249   }
250 }
251 
252 /// Check that going from Opcode to VP intrinsic and back results in the same
253 /// Opcode.
254 TEST_F(VPIntrinsicTest, OpcodeRoundTrip) {
255   std::vector<unsigned> Opcodes;
256   Opcodes.reserve(100);
257 
258   {
259 #define HANDLE_INST(OCNum, OCName, Class) Opcodes.push_back(OCNum);
260 #include "llvm/IR/Instruction.def"
261   }
262 
263   unsigned FullTripCounts = 0;
264   for (unsigned OC : Opcodes) {
265     Intrinsic::ID VPID = VPIntrinsic::getForOpcode(OC);
266     // No equivalent VP intrinsic available.
267     if (VPID == Intrinsic::not_intrinsic)
268       continue;
269 
270     Optional<unsigned> RoundTripOC =
271         VPIntrinsic::getFunctionalOpcodeForVP(VPID);
272     // No equivalent Opcode available.
273     if (!RoundTripOC)
274       continue;
275 
276     ASSERT_EQ(*RoundTripOC, OC);
277     ++FullTripCounts;
278   }
279   ASSERT_NE(FullTripCounts, 0u);
280 }
281 
282 /// Check that going from VP intrinsic to Opcode and back results in the same
283 /// intrinsic id.
284 TEST_F(VPIntrinsicTest, IntrinsicIDRoundTrip) {
285   std::unique_ptr<Module> M = createVPDeclarationModule();
286   assert(M);
287 
288   unsigned FullTripCounts = 0;
289   for (const auto &VPDecl : *M) {
290     auto VPID = VPDecl.getIntrinsicID();
291     Optional<unsigned> OC = VPIntrinsic::getFunctionalOpcodeForVP(VPID);
292 
293     // no equivalent Opcode available
294     if (!OC)
295       continue;
296 
297     Intrinsic::ID RoundTripVPID = VPIntrinsic::getForOpcode(*OC);
298 
299     ASSERT_EQ(RoundTripVPID, VPID);
300     ++FullTripCounts;
301   }
302   ASSERT_NE(FullTripCounts, 0u);
303 }
304 
305 /// Check that VPIntrinsic::getDeclarationForParams works.
306 TEST_F(VPIntrinsicTest, VPIntrinsicDeclarationForParams) {
307   std::unique_ptr<Module> M = createVPDeclarationModule();
308   assert(M);
309 
310   auto OutM = std::make_unique<Module>("", M->getContext());
311 
312   for (auto &F : *M) {
313     auto *FuncTy = F.getFunctionType();
314 
315     // Declare intrinsic anew with explicit types.
316     std::vector<Value *> Values;
317     for (auto *ParamTy : FuncTy->params())
318       Values.push_back(UndefValue::get(ParamTy));
319 
320     ASSERT_NE(F.getIntrinsicID(), Intrinsic::not_intrinsic);
321     auto *NewDecl = VPIntrinsic::getDeclarationForParams(
322         OutM.get(), F.getIntrinsicID(), FuncTy->getReturnType(), Values);
323     ASSERT_TRUE(NewDecl);
324 
325     // Check that 'old decl' == 'new decl'.
326     ASSERT_EQ(F.getIntrinsicID(), NewDecl->getIntrinsicID());
327     FunctionType::param_iterator ItNewParams =
328         NewDecl->getFunctionType()->param_begin();
329     FunctionType::param_iterator EndItNewParams =
330         NewDecl->getFunctionType()->param_end();
331     for (auto *ParamTy : FuncTy->params()) {
332       ASSERT_NE(ItNewParams, EndItNewParams);
333       ASSERT_EQ(*ItNewParams, ParamTy);
334       ++ItNewParams;
335     }
336   }
337 }
338 
339 /// Check that the HANDLE_VP_TO_CONSTRAINEDFP maps to an existing intrinsic with
340 /// the right amount of constrained-fp metadata args.
341 TEST_F(VPIntrinsicTest, HandleToConstrainedFP) {
342 #define VP_PROPERTY_CONSTRAINEDFP(HASROUND, HASEXCEPT, CFPID)                  \
343   {                                                                            \
344     SmallVector<Intrinsic::IITDescriptor, 5> T;                                \
345     Intrinsic::getIntrinsicInfoTableEntries(Intrinsic::CFPID, T);              \
346     unsigned NumMetadataArgs = 0;                                              \
347     for (auto TD : T)                                                          \
348       NumMetadataArgs += (TD.Kind == Intrinsic::IITDescriptor::Metadata);      \
349     bool IsCmp = Intrinsic::CFPID == Intrinsic::experimental_constrained_fcmp; \
350     ASSERT_EQ(NumMetadataArgs, (unsigned)(IsCmp + HASROUND + HASEXCEPT));      \
351   }
352 #include "llvm/IR/VPIntrinsics.def"
353 }
354 
355 } // end anonymous namespace
356 
357 /// Check various properties of VPReductionIntrinsics
358 TEST_F(VPIntrinsicTest, VPReductions) {
359   LLVMContext C;
360   SMDiagnostic Err;
361 
362   std::stringstream Str;
363   Str << "declare <8 x i32> @llvm.vp.mul.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, "
364          "i32)";
365   for (const char *ReductionOpcode : ReductionIntOpcodes)
366     Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
367         << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
368 
369   for (const char *ReductionOpcode : ReductionFPOpcodes)
370     Str << " declare float @llvm.vp.reduce." << ReductionOpcode
371         << ".v8f32(float, <8 x float>, <8 x i1>, i32) ";
372 
373   Str << "define void @test_reductions(i32 %start, <8 x i32> %val, float "
374          "%fpstart, <8 x float> %fpval, <8 x i1> %m, i32 %vl) {";
375 
376   // Mix in a regular non-reduction intrinsic to check that the
377   // VPReductionIntrinsic subclass works as intended.
378   Str << "  %r0 = call <8 x i32> @llvm.vp.mul.v8i32(<8 x i32> %val, <8 x i32> "
379          "%val, <8 x i1> %m, i32 %vl)";
380 
381   unsigned Idx = 1;
382   for (const char *ReductionOpcode : ReductionIntOpcodes)
383     Str << "  %r" << Idx++ << " = call i32 @llvm.vp.reduce." << ReductionOpcode
384         << ".v8i32(i32 %start, <8 x i32> %val, <8 x i1> %m, i32 %vl)";
385   for (const char *ReductionOpcode : ReductionFPOpcodes)
386     Str << "  %r" << Idx++ << " = call float @llvm.vp.reduce."
387         << ReductionOpcode
388         << ".v8f32(float %fpstart, <8 x float> %fpval, <8 x i1> %m, i32 %vl)";
389 
390   Str << "  ret void"
391          "}";
392 
393   std::unique_ptr<Module> M = parseAssemblyString(Str.str(), Err, C);
394   assert(M);
395 
396   auto *F = M->getFunction("test_reductions");
397   assert(F);
398 
399   for (const auto &I : F->getEntryBlock()) {
400     const VPIntrinsic *VPI = dyn_cast<VPIntrinsic>(&I);
401     if (!VPI)
402       continue;
403 
404     Intrinsic::ID ID = VPI->getIntrinsicID();
405     const auto *VPRedI = dyn_cast<VPReductionIntrinsic>(&I);
406 
407     if (!VPReductionIntrinsic::isVPReduction(ID)) {
408       EXPECT_EQ(VPRedI, nullptr);
409       EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID).has_value(), false);
410       EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID).has_value(), false);
411       continue;
412     }
413 
414     EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID).has_value(), true);
415     EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID).has_value(), true);
416     ASSERT_NE(VPRedI, nullptr);
417     EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID),
418               VPRedI->getStartParamPos());
419     EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID),
420               VPRedI->getVectorParamPos());
421     EXPECT_EQ(VPRedI->getStartParamPos(), 0u);
422     EXPECT_EQ(VPRedI->getVectorParamPos(), 1u);
423   }
424 }
425