1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
3
4define arm_aapcs_vfpcc void @test32(i32* noalias nocapture readonly %x, i32* noalias nocapture readonly %y, i32* nocapture %z, i32 %n) {
5; CHECK-LABEL: test32:
6; CHECK:       @ %bb.0: @ %entry
7; CHECK-NEXT:    .save {r5, lr}
8; CHECK-NEXT:    push {r5, lr}
9; CHECK-NEXT:    cmp r3, #1
10; CHECK-NEXT:    it lt
11; CHECK-NEXT:    poplt {r5, pc}
12; CHECK-NEXT:  .LBB0_1: @ %vector.body
13; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
14; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
15; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
16; CHECK-NEXT:    subs r3, #4
17; CHECK-NEXT:    vmullt.s32 q3, q2, q1
18; CHECK-NEXT:    vmov r5, s13
19; CHECK-NEXT:    vmov r12, s12
20; CHECK-NEXT:    lsrl r12, r5, #31
21; CHECK-NEXT:    vmov.32 q0[0], r12
22; CHECK-NEXT:    vmov r12, s14
23; CHECK-NEXT:    vmov.32 q0[1], r5
24; CHECK-NEXT:    vmov r5, s15
25; CHECK-NEXT:    lsrl r12, r5, #31
26; CHECK-NEXT:    vmullb.s32 q3, q2, q1
27; CHECK-NEXT:    vmov.32 q0[2], r12
28; CHECK-NEXT:    vmov r12, s12
29; CHECK-NEXT:    vmov.32 q0[3], r5
30; CHECK-NEXT:    vmov r5, s13
31; CHECK-NEXT:    lsrl r12, r5, #31
32; CHECK-NEXT:    vmov.32 q1[0], r12
33; CHECK-NEXT:    vmov r12, s14
34; CHECK-NEXT:    vmov.32 q1[1], r5
35; CHECK-NEXT:    vmov r5, s15
36; CHECK-NEXT:    lsrl r12, r5, #31
37; CHECK-NEXT:    vmov.32 q1[2], r12
38; CHECK-NEXT:    vmov.32 q1[3], r5
39; CHECK-NEXT:    vmov.f32 s8, s6
40; CHECK-NEXT:    vmov.f32 s9, s7
41; CHECK-NEXT:    vmov.f32 s6, s0
42; CHECK-NEXT:    vmov.f32 s7, s1
43; CHECK-NEXT:    vmov.f32 s10, s2
44; CHECK-NEXT:    vmov.f32 s5, s6
45; CHECK-NEXT:    vmov.f32 s11, s3
46; CHECK-NEXT:    vmov.f32 s6, s8
47; CHECK-NEXT:    vmov.f32 s7, s10
48; CHECK-NEXT:    vstrb.8 q1, [r2], #16
49; CHECK-NEXT:    bne .LBB0_1
50; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
51; CHECK-NEXT:    pop {r5, pc}
52entry:
53  %0 = and i32 %n, 3
54  %cmp = icmp eq i32 %0, 0
55  %cmp113 = icmp sgt i32 %n, 0
56  br i1 %cmp113, label %vector.body, label %for.cond.cleanup
57
58vector.body:                                      ; preds = %vector.body, %entry
59  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
60  %1 = getelementptr inbounds i32, i32* %x, i32 %index
61  %2 = bitcast i32* %1 to <4 x i32>*
62  %wide.load = load <4 x i32>, <4 x i32>* %2, align 4
63  %3 = shufflevector <4 x i32> %wide.load, <4 x i32> %wide.load, <2 x i32> <i32 0, i32 2>
64  %4 = shufflevector <4 x i32> %wide.load, <4 x i32> %wide.load, <2 x i32> <i32 1, i32 3>
65  %5 = sext <2 x i32> %3 to <2 x i64>
66  %6 = sext <2 x i32> %4 to <2 x i64>
67  %7 = getelementptr inbounds i32, i32* %y, i32 %index
68  %8 = bitcast i32* %7 to <4 x i32>*
69  %wide.load15 = load <4 x i32>, <4 x i32>* %8, align 4
70  %9 = shufflevector <4 x i32> %wide.load15, <4 x i32> %wide.load15, <2 x i32> <i32 0, i32 2>
71  %10 = shufflevector <4 x i32> %wide.load15, <4 x i32> %wide.load15, <2 x i32> <i32 1, i32 3>
72  %11 = sext <2 x i32> %9 to <2 x i64>
73  %12 = sext <2 x i32> %10 to <2 x i64>
74  %13 = mul <2 x i64> %11, %5
75  %14 = mul <2 x i64> %12, %6
76  %15 = lshr <2 x i64> %13, <i64 31, i64 31>
77  %16 = lshr <2 x i64> %14, <i64 31, i64 31>
78  %17 = shufflevector <2 x i64> %15, <2 x i64> %16, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
79  %18 = trunc <4 x i64> %17 to <4 x i32>
80  %19 = getelementptr inbounds i32, i32* %z, i32 %index
81  %20 = bitcast i32* %19 to <4 x i32>*
82  store <4 x i32> %18, <4 x i32>* %20, align 4
83  %index.next = add i32 %index, 4
84  %21 = icmp eq i32 %index.next, %n
85  br i1 %21, label %for.cond.cleanup, label %vector.body
86
87for.cond.cleanup:                                 ; preds = %vector.body, %entry
88  ret void
89}
90
91define arm_aapcs_vfpcc void @test16(i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y, i16* nocapture %z, i32 %n) {
92; CHECK-LABEL: test16:
93; CHECK:       @ %bb.0: @ %entry
94; CHECK-NEXT:    cmp r3, #1
95; CHECK-NEXT:    it lt
96; CHECK-NEXT:    bxlt lr
97; CHECK-NEXT:  .LBB1_1: @ %vector.body
98; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
99; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
100; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
101; CHECK-NEXT:    subs r3, #8
102; CHECK-NEXT:    vmullt.s16 q2, q1, q0
103; CHECK-NEXT:    vmullb.s16 q0, q1, q0
104; CHECK-NEXT:    vshr.u32 q2, q2, #15
105; CHECK-NEXT:    vshr.u32 q0, q0, #15
106; CHECK-NEXT:    vmovnt.i32 q0, q2
107; CHECK-NEXT:    vstrb.8 q0, [r2], #16
108; CHECK-NEXT:    bne .LBB1_1
109; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
110; CHECK-NEXT:    bx lr
111entry:
112  %0 = and i32 %n, 7
113  %cmp = icmp eq i32 %0, 0
114  %cmp113 = icmp sgt i32 %n, 0
115  br i1 %cmp113, label %vector.body, label %for.cond.cleanup
116
117vector.body:                                      ; preds = %vector.body, %entry
118  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
119  %1 = getelementptr inbounds i16, i16* %x, i32 %index
120  %2 = bitcast i16* %1 to <8 x i16>*
121  %wide.load = load <8 x i16>, <8 x i16>* %2, align 2
122  %3 = shufflevector <8 x i16> %wide.load, <8 x i16> %wide.load, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
123  %4 = shufflevector <8 x i16> %wide.load, <8 x i16> %wide.load, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
124  %5 = sext <4 x i16> %3 to <4 x i32>
125  %6 = sext <4 x i16> %4 to <4 x i32>
126  %7 = getelementptr inbounds i16, i16* %y, i32 %index
127  %8 = bitcast i16* %7 to <8 x i16>*
128  %wide.load15 = load <8 x i16>, <8 x i16>* %8, align 2
129  %9 = shufflevector <8 x i16> %wide.load15, <8 x i16> %wide.load15, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
130  %10 = shufflevector <8 x i16> %wide.load15, <8 x i16> %wide.load15, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
131  %11 = sext <4 x i16> %9 to <4 x i32>
132  %12 = sext <4 x i16> %10 to <4 x i32>
133  %13 = mul <4 x i32> %11, %5
134  %14 = mul <4 x i32> %12, %6
135  %15 = lshr <4 x i32> %13, <i32 15, i32 15, i32 15, i32 15>
136  %16 = lshr <4 x i32> %14, <i32 15, i32 15, i32 15, i32 15>
137  %17 = shufflevector <4 x i32> %15, <4 x i32> %16, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
138  %18 = trunc <8 x i32> %17 to <8 x i16>
139  %19 = getelementptr inbounds i16, i16* %z, i32 %index
140  %20 = bitcast i16* %19 to <8 x i16>*
141  store <8 x i16> %18, <8 x i16>* %20, align 2
142  %index.next = add i32 %index, 8
143  %21 = icmp eq i32 %index.next, %n
144  br i1 %21, label %for.cond.cleanup, label %vector.body
145
146for.cond.cleanup:                                 ; preds = %vector.body, %entry
147  ret void
148}
149
150define arm_aapcs_vfpcc void @test8(i8* noalias nocapture readonly %x, i8* noalias nocapture readonly %y, i8* nocapture %z, i32 %n) {
151; CHECK-LABEL: test8:
152; CHECK:       @ %bb.0: @ %entry
153; CHECK-NEXT:    cmp r3, #1
154; CHECK-NEXT:    it lt
155; CHECK-NEXT:    bxlt lr
156; CHECK-NEXT:  .LBB2_1: @ %vector.body
157; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
158; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
159; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
160; CHECK-NEXT:    subs r3, #16
161; CHECK-NEXT:    vmullt.u8 q2, q1, q0
162; CHECK-NEXT:    vmullb.u8 q0, q1, q0
163; CHECK-NEXT:    vshr.u16 q2, q2, #7
164; CHECK-NEXT:    vshr.u16 q0, q0, #7
165; CHECK-NEXT:    vmovnt.i16 q0, q2
166; CHECK-NEXT:    vstrb.8 q0, [r2], #16
167; CHECK-NEXT:    bne .LBB2_1
168; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
169; CHECK-NEXT:    bx lr
170entry:
171  %0 = and i32 %n, 15
172  %cmp = icmp eq i32 %0, 0
173  %cmp117 = icmp sgt i32 %n, 0
174  br i1 %cmp117, label %vector.body, label %for.cond.cleanup
175
176vector.body:                                      ; preds = %vector.body, %entry
177  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
178  %1 = getelementptr inbounds i8, i8* %x, i32 %index
179  %2 = bitcast i8* %1 to <16 x i8>*
180  %wide.load = load <16 x i8>, <16 x i8>* %2, align 1
181  %3 = shufflevector <16 x i8> %wide.load, <16 x i8> %wide.load, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
182  %4 = shufflevector <16 x i8> %wide.load, <16 x i8> %wide.load, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
183  %5 = zext <8 x i8> %3 to <8 x i16>
184  %6 = zext <8 x i8> %4 to <8 x i16>
185  %7 = getelementptr inbounds i8, i8* %y, i32 %index
186  %8 = bitcast i8* %7 to <16 x i8>*
187  %wide.load19 = load <16 x i8>, <16 x i8>* %8, align 1
188  %9 = shufflevector <16 x i8> %wide.load19, <16 x i8> %wide.load19, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
189  %10 = shufflevector <16 x i8> %wide.load19, <16 x i8> %wide.load19, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
190  %11 = zext <8 x i8> %9 to <8 x i16>
191  %12 = zext <8 x i8> %10 to <8 x i16>
192  %13 = mul <8 x i16> %11, %5
193  %14 = mul <8 x i16> %12, %6
194  %15 = lshr <8 x i16> %13, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
195  %16 = lshr <8 x i16> %14, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
196  %17 = shufflevector <8 x i16> %15, <8 x i16> %16, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
197  %18 = trunc <16 x i16> %17 to <16 x i8>
198  %19 = getelementptr inbounds i8, i8* %z, i32 %index
199  %20 = bitcast i8* %19 to <16 x i8>*
200  store <16 x i8> %18, <16 x i8>* %20, align 1
201  %index.next = add i32 %index, 16
202  %21 = icmp eq i32 %index.next, %n
203  br i1 %21, label %for.cond.cleanup, label %vector.body
204
205for.cond.cleanup:                                 ; preds = %vector.body, %entry
206  ret void
207}
208