1 //===- VPIntrinsicTest.cpp - VPIntrinsic unit tests ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ADT/SmallVector.h"
10 #include "llvm/AsmParser/Parser.h"
11 #include "llvm/CodeGen/ISDOpcodes.h"
12 #include "llvm/IR/Constants.h"
13 #include "llvm/IR/IRBuilder.h"
14 #include "llvm/IR/IntrinsicInst.h"
15 #include "llvm/IR/LLVMContext.h"
16 #include "llvm/IR/Module.h"
17 #include "llvm/IR/Verifier.h"
18 #include "llvm/Support/SourceMgr.h"
19 #include "gtest/gtest.h"
20 #include <sstream>
21
22 using namespace llvm;
23
24 namespace {
25
26 static const char *ReductionIntOpcodes[] = {
27 "add", "mul", "and", "or", "xor", "smin", "smax", "umin", "umax"};
28
29 static const char *ReductionFPOpcodes[] = {"fadd", "fmul", "fmin", "fmax"};
30
31 class VPIntrinsicTest : public testing::Test {
32 protected:
33 LLVMContext Context;
34
VPIntrinsicTest()35 VPIntrinsicTest() : Context() {}
36
37 LLVMContext C;
38 SMDiagnostic Err;
39
createVPDeclarationModule()40 std::unique_ptr<Module> createVPDeclarationModule() {
41 const char *BinaryIntOpcodes[] = {"add", "sub", "mul", "sdiv", "srem",
42 "udiv", "urem", "and", "xor", "or",
43 "ashr", "lshr", "shl"};
44 std::stringstream Str;
45 for (const char *BinaryIntOpcode : BinaryIntOpcodes)
46 Str << " declare <8 x i32> @llvm.vp." << BinaryIntOpcode
47 << ".v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) ";
48
49 const char *BinaryFPOpcodes[] = {"fadd", "fsub", "fmul", "fdiv", "frem"};
50 for (const char *BinaryFPOpcode : BinaryFPOpcodes)
51 Str << " declare <8 x float> @llvm.vp." << BinaryFPOpcode
52 << ".v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) ";
53
54 Str << " declare <8 x float> @llvm.vp.fneg.v8f32(<8 x float>, <8 x i1>, "
55 "i32)";
56 Str << " declare <8 x float> @llvm.vp.fma.v8f32(<8 x float>, <8 x float>, "
57 "<8 x float>, <8 x i1>, i32) ";
58
59 Str << " declare void @llvm.vp.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, "
60 "<8 x i1>, i32) ";
61 Str << "declare void "
62 "@llvm.experimental.vp.strided.store.v8i32.i32(<8 x i32>, "
63 "i32*, i32, <8 x i1>, i32) ";
64 Str << "declare void "
65 "@llvm.experimental.vp.strided.store.v8i32.p1i32.i32(<8 x i32>, "
66 "i32 addrspace(1)*, i32, <8 x i1>, i32) ";
67 Str << " declare void @llvm.vp.scatter.v8i32.v8p0i32(<8 x i32>, <8 x "
68 "i32*>, <8 x i1>, i32) ";
69 Str << " declare <8 x i32> @llvm.vp.load.v8i32.p0v8i32(<8 x i32>*, <8 x "
70 "i1>, i32) ";
71 Str << "declare <8 x i32> "
72 "@llvm.experimental.vp.strided.load.v8i32.i32(i32*, i32, <8 "
73 "x i1>, i32) ";
74 Str << "declare <8 x i32> "
75 "@llvm.experimental.vp.strided.load.v8i32.p1i32.i32(i32 "
76 "addrspace(1)*, i32, <8 x i1>, i32) ";
77 Str << " declare <8 x i32> @llvm.vp.gather.v8i32.v8p0i32(<8 x i32*>, <8 x "
78 "i1>, i32) ";
79
80 for (const char *ReductionOpcode : ReductionIntOpcodes)
81 Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
82 << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
83
84 for (const char *ReductionOpcode : ReductionFPOpcodes)
85 Str << " declare float @llvm.vp.reduce." << ReductionOpcode
86 << ".v8f32(float, <8 x float>, <8 x i1>, i32) ";
87
88 Str << " declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x "
89 "i32>, i32)";
90 Str << " declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x "
91 "i32>, i32)";
92 Str << " declare <8 x i32> @llvm.experimental.vp.splice.v8i32(<8 x "
93 "i32>, <8 x i32>, i32, <8 x i1>, i32, i32) ";
94
95 Str << " declare <8 x i32> @llvm.vp.fptoui.v8i32"
96 << ".v8f32(<8 x float>, <8 x i1>, i32) ";
97 Str << " declare <8 x i32> @llvm.vp.fptosi.v8i32"
98 << ".v8f32(<8 x float>, <8 x i1>, i32) ";
99 Str << " declare <8 x float> @llvm.vp.uitofp.v8f32"
100 << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
101 Str << " declare <8 x float> @llvm.vp.sitofp.v8f32"
102 << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
103 Str << " declare <8 x float> @llvm.vp.fptrunc.v8f32"
104 << ".v8f64(<8 x double>, <8 x i1>, i32) ";
105 Str << " declare <8 x double> @llvm.vp.fpext.v8f64"
106 << ".v8f32(<8 x float>, <8 x i1>, i32) ";
107 Str << " declare <8 x i32> @llvm.vp.trunc.v8i32"
108 << ".v8i64(<8 x i64>, <8 x i1>, i32) ";
109 Str << " declare <8 x i64> @llvm.vp.zext.v8i64"
110 << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
111 Str << " declare <8 x i64> @llvm.vp.sext.v8i64"
112 << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
113 Str << " declare <8 x i32> @llvm.vp.ptrtoint.v8i32"
114 << ".v8p0i32(<8 x i32*>, <8 x i1>, i32) ";
115 Str << " declare <8 x i32*> @llvm.vp.inttoptr.v8p0i32"
116 << ".v8i32(<8 x i32>, <8 x i1>, i32) ";
117
118 Str << " declare <8 x i1> @llvm.vp.fcmp.v8f32"
119 << "(<8 x float>, <8 x float>, metadata, <8 x i1>, i32) ";
120 Str << " declare <8 x i1> @llvm.vp.icmp.v8i16"
121 << "(<8 x i16>, <8 x i16>, metadata, <8 x i1>, i32) ";
122
123 return parseAssemblyString(Str.str(), Err, C);
124 }
125 };
126
127 /// Check that the property scopes include/llvm/IR/VPIntrinsics.def are closed.
TEST_F(VPIntrinsicTest,VPIntrinsicsDefScopes)128 TEST_F(VPIntrinsicTest, VPIntrinsicsDefScopes) {
129 Optional<Intrinsic::ID> ScopeVPID;
130 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) \
131 ASSERT_FALSE(ScopeVPID.has_value()); \
132 ScopeVPID = Intrinsic::VPID;
133 #define END_REGISTER_VP_INTRINSIC(VPID) \
134 ASSERT_TRUE(ScopeVPID.has_value()); \
135 ASSERT_EQ(ScopeVPID.value(), Intrinsic::VPID); \
136 ScopeVPID = None;
137
138 Optional<ISD::NodeType> ScopeOPC;
139 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
140 ASSERT_FALSE(ScopeOPC.has_value()); \
141 ScopeOPC = ISD::SDOPC;
142 #define END_REGISTER_VP_SDNODE(SDOPC) \
143 ASSERT_TRUE(ScopeOPC.has_value()); \
144 ASSERT_EQ(ScopeOPC.value(), ISD::SDOPC); \
145 ScopeOPC = None;
146 #include "llvm/IR/VPIntrinsics.def"
147
148 ASSERT_FALSE(ScopeVPID.has_value());
149 ASSERT_FALSE(ScopeOPC.has_value());
150 }
151
152 /// Check that every VP intrinsic in the test module is recognized as a VP
153 /// intrinsic.
TEST_F(VPIntrinsicTest,VPModuleComplete)154 TEST_F(VPIntrinsicTest, VPModuleComplete) {
155 std::unique_ptr<Module> M = createVPDeclarationModule();
156 assert(M);
157
158 // Check that all @llvm.vp.* functions in the module are recognized vp
159 // intrinsics.
160 std::set<Intrinsic::ID> SeenIDs;
161 for (const auto &VPDecl : *M) {
162 ASSERT_TRUE(VPDecl.isIntrinsic());
163 ASSERT_TRUE(VPIntrinsic::isVPIntrinsic(VPDecl.getIntrinsicID()));
164 SeenIDs.insert(VPDecl.getIntrinsicID());
165 }
166
167 // Check that every registered VP intrinsic has an instance in the test
168 // module.
169 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) \
170 ASSERT_TRUE(SeenIDs.count(Intrinsic::VPID));
171 #include "llvm/IR/VPIntrinsics.def"
172 }
173
174 /// Check that VPIntrinsic:canIgnoreVectorLengthParam() returns true
175 /// if the vector length parameter does not mask off any lanes.
TEST_F(VPIntrinsicTest,CanIgnoreVectorLength)176 TEST_F(VPIntrinsicTest, CanIgnoreVectorLength) {
177 LLVMContext C;
178 SMDiagnostic Err;
179
180 std::unique_ptr<Module> M =
181 parseAssemblyString(
182 "declare <256 x i64> @llvm.vp.mul.v256i64(<256 x i64>, <256 x i64>, <256 x i1>, i32)"
183 "declare <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)"
184 "declare <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i1>, i32)"
185 "declare i32 @llvm.vscale.i32()"
186 "define void @test_static_vlen( "
187 " <256 x i64> %i0, <vscale x 2 x i64> %si0x2, <vscale x 1 x i64> %si0x1,"
188 " <256 x i64> %i1, <vscale x 2 x i64> %si1x2, <vscale x 1 x i64> %si1x1,"
189 " <256 x i1> %m, <vscale x 2 x i1> %smx2, <vscale x 1 x i1> %smx1, i32 %vl) { "
190 " %r0 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 %vl)"
191 " %r1 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 256)"
192 " %r2 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 0)"
193 " %r3 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 7)"
194 " %r4 = call <256 x i64> @llvm.vp.mul.v256i64(<256 x i64> %i0, <256 x i64> %i1, <256 x i1> %m, i32 123)"
195 " %vs = call i32 @llvm.vscale.i32()"
196 " %vs.x2 = mul i32 %vs, 2"
197 " %r5 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs.x2)"
198 " %r6 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs)"
199 " %r7 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 99999)"
200 " %r8 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 %vs)"
201 " %r9 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 1)"
202 " %r10 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %si0x1, <vscale x 1 x i64> %si1x1, <vscale x 1 x i1> %smx1, i32 %vs.x2)"
203 " %vs.wat = add i32 %vs, 2"
204 " %r11 = call <vscale x 2 x i64> @llvm.vp.mul.nxv2i64(<vscale x 2 x i64> %si0x2, <vscale x 2 x i64> %si1x2, <vscale x 2 x i1> %smx2, i32 %vs.wat)"
205 " ret void "
206 "}",
207 Err, C);
208
209 auto *F = M->getFunction("test_static_vlen");
210 assert(F);
211
212 const bool Expected[] = {false, true, false, false, false, true,
213 false, false, true, false, true, false};
214 const auto *ExpectedIt = std::begin(Expected);
215 for (auto &I : F->getEntryBlock()) {
216 VPIntrinsic *VPI = dyn_cast<VPIntrinsic>(&I);
217 if (!VPI)
218 continue;
219
220 ASSERT_NE(ExpectedIt, std::end(Expected));
221 ASSERT_EQ(*ExpectedIt, VPI->canIgnoreVectorLengthParam());
222 ++ExpectedIt;
223 }
224 }
225
226 /// Check that the argument returned by
227 /// VPIntrinsic::get<X>ParamPos(Intrinsic::ID) has the expected type.
TEST_F(VPIntrinsicTest,GetParamPos)228 TEST_F(VPIntrinsicTest, GetParamPos) {
229 std::unique_ptr<Module> M = createVPDeclarationModule();
230 assert(M);
231
232 for (Function &F : *M) {
233 ASSERT_TRUE(F.isIntrinsic());
234 Optional<unsigned> MaskParamPos =
235 VPIntrinsic::getMaskParamPos(F.getIntrinsicID());
236 if (MaskParamPos) {
237 Type *MaskParamType = F.getArg(MaskParamPos.value())->getType();
238 ASSERT_TRUE(MaskParamType->isVectorTy());
239 ASSERT_TRUE(
240 cast<VectorType>(MaskParamType)->getElementType()->isIntegerTy(1));
241 }
242
243 Optional<unsigned> VecLenParamPos =
244 VPIntrinsic::getVectorLengthParamPos(F.getIntrinsicID());
245 if (VecLenParamPos) {
246 Type *VecLenParamType = F.getArg(VecLenParamPos.value())->getType();
247 ASSERT_TRUE(VecLenParamType->isIntegerTy(32));
248 }
249 }
250 }
251
252 /// Check that going from Opcode to VP intrinsic and back results in the same
253 /// Opcode.
TEST_F(VPIntrinsicTest,OpcodeRoundTrip)254 TEST_F(VPIntrinsicTest, OpcodeRoundTrip) {
255 std::vector<unsigned> Opcodes;
256 Opcodes.reserve(100);
257
258 {
259 #define HANDLE_INST(OCNum, OCName, Class) Opcodes.push_back(OCNum);
260 #include "llvm/IR/Instruction.def"
261 }
262
263 unsigned FullTripCounts = 0;
264 for (unsigned OC : Opcodes) {
265 Intrinsic::ID VPID = VPIntrinsic::getForOpcode(OC);
266 // No equivalent VP intrinsic available.
267 if (VPID == Intrinsic::not_intrinsic)
268 continue;
269
270 Optional<unsigned> RoundTripOC =
271 VPIntrinsic::getFunctionalOpcodeForVP(VPID);
272 // No equivalent Opcode available.
273 if (!RoundTripOC)
274 continue;
275
276 ASSERT_EQ(*RoundTripOC, OC);
277 ++FullTripCounts;
278 }
279 ASSERT_NE(FullTripCounts, 0u);
280 }
281
282 /// Check that going from VP intrinsic to Opcode and back results in the same
283 /// intrinsic id.
TEST_F(VPIntrinsicTest,IntrinsicIDRoundTrip)284 TEST_F(VPIntrinsicTest, IntrinsicIDRoundTrip) {
285 std::unique_ptr<Module> M = createVPDeclarationModule();
286 assert(M);
287
288 unsigned FullTripCounts = 0;
289 for (const auto &VPDecl : *M) {
290 auto VPID = VPDecl.getIntrinsicID();
291 Optional<unsigned> OC = VPIntrinsic::getFunctionalOpcodeForVP(VPID);
292
293 // no equivalent Opcode available
294 if (!OC)
295 continue;
296
297 Intrinsic::ID RoundTripVPID = VPIntrinsic::getForOpcode(*OC);
298
299 ASSERT_EQ(RoundTripVPID, VPID);
300 ++FullTripCounts;
301 }
302 ASSERT_NE(FullTripCounts, 0u);
303 }
304
305 /// Check that VPIntrinsic::getDeclarationForParams works.
TEST_F(VPIntrinsicTest,VPIntrinsicDeclarationForParams)306 TEST_F(VPIntrinsicTest, VPIntrinsicDeclarationForParams) {
307 std::unique_ptr<Module> M = createVPDeclarationModule();
308 assert(M);
309
310 auto OutM = std::make_unique<Module>("", M->getContext());
311
312 for (auto &F : *M) {
313 auto *FuncTy = F.getFunctionType();
314
315 // Declare intrinsic anew with explicit types.
316 std::vector<Value *> Values;
317 for (auto *ParamTy : FuncTy->params())
318 Values.push_back(UndefValue::get(ParamTy));
319
320 ASSERT_NE(F.getIntrinsicID(), Intrinsic::not_intrinsic);
321 auto *NewDecl = VPIntrinsic::getDeclarationForParams(
322 OutM.get(), F.getIntrinsicID(), FuncTy->getReturnType(), Values);
323 ASSERT_TRUE(NewDecl);
324
325 // Check that 'old decl' == 'new decl'.
326 ASSERT_EQ(F.getIntrinsicID(), NewDecl->getIntrinsicID());
327 FunctionType::param_iterator ItNewParams =
328 NewDecl->getFunctionType()->param_begin();
329 FunctionType::param_iterator EndItNewParams =
330 NewDecl->getFunctionType()->param_end();
331 for (auto *ParamTy : FuncTy->params()) {
332 ASSERT_NE(ItNewParams, EndItNewParams);
333 ASSERT_EQ(*ItNewParams, ParamTy);
334 ++ItNewParams;
335 }
336 }
337 }
338
339 /// Check that the HANDLE_VP_TO_CONSTRAINEDFP maps to an existing intrinsic with
340 /// the right amount of constrained-fp metadata args.
TEST_F(VPIntrinsicTest,HandleToConstrainedFP)341 TEST_F(VPIntrinsicTest, HandleToConstrainedFP) {
342 #define VP_PROPERTY_CONSTRAINEDFP(HASROUND, HASEXCEPT, CFPID) \
343 { \
344 SmallVector<Intrinsic::IITDescriptor, 5> T; \
345 Intrinsic::getIntrinsicInfoTableEntries(Intrinsic::CFPID, T); \
346 unsigned NumMetadataArgs = 0; \
347 for (auto TD : T) \
348 NumMetadataArgs += (TD.Kind == Intrinsic::IITDescriptor::Metadata); \
349 bool IsCmp = Intrinsic::CFPID == Intrinsic::experimental_constrained_fcmp; \
350 ASSERT_EQ(NumMetadataArgs, (unsigned)(IsCmp + HASROUND + HASEXCEPT)); \
351 }
352 #include "llvm/IR/VPIntrinsics.def"
353 }
354
355 } // end anonymous namespace
356
357 /// Check various properties of VPReductionIntrinsics
TEST_F(VPIntrinsicTest,VPReductions)358 TEST_F(VPIntrinsicTest, VPReductions) {
359 LLVMContext C;
360 SMDiagnostic Err;
361
362 std::stringstream Str;
363 Str << "declare <8 x i32> @llvm.vp.mul.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, "
364 "i32)";
365 for (const char *ReductionOpcode : ReductionIntOpcodes)
366 Str << " declare i32 @llvm.vp.reduce." << ReductionOpcode
367 << ".v8i32(i32, <8 x i32>, <8 x i1>, i32) ";
368
369 for (const char *ReductionOpcode : ReductionFPOpcodes)
370 Str << " declare float @llvm.vp.reduce." << ReductionOpcode
371 << ".v8f32(float, <8 x float>, <8 x i1>, i32) ";
372
373 Str << "define void @test_reductions(i32 %start, <8 x i32> %val, float "
374 "%fpstart, <8 x float> %fpval, <8 x i1> %m, i32 %vl) {";
375
376 // Mix in a regular non-reduction intrinsic to check that the
377 // VPReductionIntrinsic subclass works as intended.
378 Str << " %r0 = call <8 x i32> @llvm.vp.mul.v8i32(<8 x i32> %val, <8 x i32> "
379 "%val, <8 x i1> %m, i32 %vl)";
380
381 unsigned Idx = 1;
382 for (const char *ReductionOpcode : ReductionIntOpcodes)
383 Str << " %r" << Idx++ << " = call i32 @llvm.vp.reduce." << ReductionOpcode
384 << ".v8i32(i32 %start, <8 x i32> %val, <8 x i1> %m, i32 %vl)";
385 for (const char *ReductionOpcode : ReductionFPOpcodes)
386 Str << " %r" << Idx++ << " = call float @llvm.vp.reduce."
387 << ReductionOpcode
388 << ".v8f32(float %fpstart, <8 x float> %fpval, <8 x i1> %m, i32 %vl)";
389
390 Str << " ret void"
391 "}";
392
393 std::unique_ptr<Module> M = parseAssemblyString(Str.str(), Err, C);
394 assert(M);
395
396 auto *F = M->getFunction("test_reductions");
397 assert(F);
398
399 for (const auto &I : F->getEntryBlock()) {
400 const VPIntrinsic *VPI = dyn_cast<VPIntrinsic>(&I);
401 if (!VPI)
402 continue;
403
404 Intrinsic::ID ID = VPI->getIntrinsicID();
405 const auto *VPRedI = dyn_cast<VPReductionIntrinsic>(&I);
406
407 if (!VPReductionIntrinsic::isVPReduction(ID)) {
408 EXPECT_EQ(VPRedI, nullptr);
409 EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID).has_value(), false);
410 EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID).has_value(), false);
411 continue;
412 }
413
414 EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID).has_value(), true);
415 EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID).has_value(), true);
416 ASSERT_NE(VPRedI, nullptr);
417 EXPECT_EQ(VPReductionIntrinsic::getStartParamPos(ID),
418 VPRedI->getStartParamPos());
419 EXPECT_EQ(VPReductionIntrinsic::getVectorParamPos(ID),
420 VPRedI->getVectorParamPos());
421 EXPECT_EQ(VPRedI->getStartParamPos(), 0u);
422 EXPECT_EQ(VPRedI->getVectorParamPos(), 1u);
423 }
424 }
425