1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
3
4define arm_aapcs_vfpcc void @test32(i32* noalias nocapture readonly %x, i32* noalias nocapture readonly %y, i32* nocapture %z, i32 %n) {
5; CHECK-LABEL: test32:
6; CHECK:       @ %bb.0: @ %entry
7; CHECK-NEXT:    .save {r4, r5, r7, lr}
8; CHECK-NEXT:    push {r4, r5, r7, lr}
9; CHECK-NEXT:    cmp r3, #1
10; CHECK-NEXT:    blt .LBB0_2
11; CHECK-NEXT:  .LBB0_1: @ %vector.body
12; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
13; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
14; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
15; CHECK-NEXT:    subs r3, #4
16; CHECK-NEXT:    vmullb.s32 q2, q1, q0
17; CHECK-NEXT:    vmullt.s32 q3, q1, q0
18; CHECK-NEXT:    vmov r12, r5, d5
19; CHECK-NEXT:    lsrl r12, r5, #31
20; CHECK-NEXT:    vmov r4, r5, d4
21; CHECK-NEXT:    lsrl r4, r5, #31
22; CHECK-NEXT:    vmov q2[2], q2[0], r4, r12
23; CHECK-NEXT:    vmov r12, r5, d7
24; CHECK-NEXT:    lsrl r12, r5, #31
25; CHECK-NEXT:    vmov r4, r5, d6
26; CHECK-NEXT:    lsrl r4, r5, #31
27; CHECK-NEXT:    vmov q2[3], q2[1], r4, r12
28; CHECK-NEXT:    vstrb.8 q2, [r2], #16
29; CHECK-NEXT:    bne .LBB0_1
30; CHECK-NEXT:  .LBB0_2: @ %for.cond.cleanup
31; CHECK-NEXT:    pop {r4, r5, r7, pc}
32entry:
33  %0 = and i32 %n, 3
34  %cmp = icmp eq i32 %0, 0
35  %cmp113 = icmp sgt i32 %n, 0
36  br i1 %cmp113, label %vector.body, label %for.cond.cleanup
37
38vector.body:                                      ; preds = %vector.body, %entry
39  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
40  %1 = getelementptr inbounds i32, i32* %x, i32 %index
41  %2 = bitcast i32* %1 to <4 x i32>*
42  %wide.load = load <4 x i32>, <4 x i32>* %2, align 4
43  %3 = shufflevector <4 x i32> %wide.load, <4 x i32> %wide.load, <2 x i32> <i32 0, i32 2>
44  %4 = shufflevector <4 x i32> %wide.load, <4 x i32> %wide.load, <2 x i32> <i32 1, i32 3>
45  %5 = sext <2 x i32> %3 to <2 x i64>
46  %6 = sext <2 x i32> %4 to <2 x i64>
47  %7 = getelementptr inbounds i32, i32* %y, i32 %index
48  %8 = bitcast i32* %7 to <4 x i32>*
49  %wide.load15 = load <4 x i32>, <4 x i32>* %8, align 4
50  %9 = shufflevector <4 x i32> %wide.load15, <4 x i32> %wide.load15, <2 x i32> <i32 0, i32 2>
51  %10 = shufflevector <4 x i32> %wide.load15, <4 x i32> %wide.load15, <2 x i32> <i32 1, i32 3>
52  %11 = sext <2 x i32> %9 to <2 x i64>
53  %12 = sext <2 x i32> %10 to <2 x i64>
54  %13 = mul <2 x i64> %11, %5
55  %14 = mul <2 x i64> %12, %6
56  %15 = lshr <2 x i64> %13, <i64 31, i64 31>
57  %16 = lshr <2 x i64> %14, <i64 31, i64 31>
58  %17 = shufflevector <2 x i64> %15, <2 x i64> %16, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
59  %18 = trunc <4 x i64> %17 to <4 x i32>
60  %19 = getelementptr inbounds i32, i32* %z, i32 %index
61  %20 = bitcast i32* %19 to <4 x i32>*
62  store <4 x i32> %18, <4 x i32>* %20, align 4
63  %index.next = add i32 %index, 4
64  %21 = icmp eq i32 %index.next, %n
65  br i1 %21, label %for.cond.cleanup, label %vector.body
66
67for.cond.cleanup:                                 ; preds = %vector.body, %entry
68  ret void
69}
70
71define arm_aapcs_vfpcc void @test16(i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y, i16* nocapture %z, i32 %n) {
72; CHECK-LABEL: test16:
73; CHECK:       @ %bb.0: @ %entry
74; CHECK-NEXT:    cmp r3, #1
75; CHECK-NEXT:    it lt
76; CHECK-NEXT:    bxlt lr
77; CHECK-NEXT:  .LBB1_1: @ %vector.body
78; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
79; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
80; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
81; CHECK-NEXT:    subs r3, #8
82; CHECK-NEXT:    vmullt.s16 q2, q1, q0
83; CHECK-NEXT:    vmullb.s16 q0, q1, q0
84; CHECK-NEXT:    vshr.u32 q2, q2, #15
85; CHECK-NEXT:    vshr.u32 q0, q0, #15
86; CHECK-NEXT:    vmovnt.i32 q0, q2
87; CHECK-NEXT:    vstrb.8 q0, [r2], #16
88; CHECK-NEXT:    bne .LBB1_1
89; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
90; CHECK-NEXT:    bx lr
91entry:
92  %0 = and i32 %n, 7
93  %cmp = icmp eq i32 %0, 0
94  %cmp113 = icmp sgt i32 %n, 0
95  br i1 %cmp113, label %vector.body, label %for.cond.cleanup
96
97vector.body:                                      ; preds = %vector.body, %entry
98  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
99  %1 = getelementptr inbounds i16, i16* %x, i32 %index
100  %2 = bitcast i16* %1 to <8 x i16>*
101  %wide.load = load <8 x i16>, <8 x i16>* %2, align 2
102  %3 = shufflevector <8 x i16> %wide.load, <8 x i16> %wide.load, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
103  %4 = shufflevector <8 x i16> %wide.load, <8 x i16> %wide.load, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
104  %5 = sext <4 x i16> %3 to <4 x i32>
105  %6 = sext <4 x i16> %4 to <4 x i32>
106  %7 = getelementptr inbounds i16, i16* %y, i32 %index
107  %8 = bitcast i16* %7 to <8 x i16>*
108  %wide.load15 = load <8 x i16>, <8 x i16>* %8, align 2
109  %9 = shufflevector <8 x i16> %wide.load15, <8 x i16> %wide.load15, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
110  %10 = shufflevector <8 x i16> %wide.load15, <8 x i16> %wide.load15, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
111  %11 = sext <4 x i16> %9 to <4 x i32>
112  %12 = sext <4 x i16> %10 to <4 x i32>
113  %13 = mul <4 x i32> %11, %5
114  %14 = mul <4 x i32> %12, %6
115  %15 = lshr <4 x i32> %13, <i32 15, i32 15, i32 15, i32 15>
116  %16 = lshr <4 x i32> %14, <i32 15, i32 15, i32 15, i32 15>
117  %17 = shufflevector <4 x i32> %15, <4 x i32> %16, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
118  %18 = trunc <8 x i32> %17 to <8 x i16>
119  %19 = getelementptr inbounds i16, i16* %z, i32 %index
120  %20 = bitcast i16* %19 to <8 x i16>*
121  store <8 x i16> %18, <8 x i16>* %20, align 2
122  %index.next = add i32 %index, 8
123  %21 = icmp eq i32 %index.next, %n
124  br i1 %21, label %for.cond.cleanup, label %vector.body
125
126for.cond.cleanup:                                 ; preds = %vector.body, %entry
127  ret void
128}
129
130define arm_aapcs_vfpcc void @test8(i8* noalias nocapture readonly %x, i8* noalias nocapture readonly %y, i8* nocapture %z, i32 %n) {
131; CHECK-LABEL: test8:
132; CHECK:       @ %bb.0: @ %entry
133; CHECK-NEXT:    cmp r3, #1
134; CHECK-NEXT:    it lt
135; CHECK-NEXT:    bxlt lr
136; CHECK-NEXT:  .LBB2_1: @ %vector.body
137; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
138; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
139; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
140; CHECK-NEXT:    subs r3, #16
141; CHECK-NEXT:    vmullt.u8 q2, q1, q0
142; CHECK-NEXT:    vmullb.u8 q0, q1, q0
143; CHECK-NEXT:    vshr.u16 q2, q2, #7
144; CHECK-NEXT:    vshr.u16 q0, q0, #7
145; CHECK-NEXT:    vmovnt.i16 q0, q2
146; CHECK-NEXT:    vstrb.8 q0, [r2], #16
147; CHECK-NEXT:    bne .LBB2_1
148; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
149; CHECK-NEXT:    bx lr
150entry:
151  %0 = and i32 %n, 15
152  %cmp = icmp eq i32 %0, 0
153  %cmp117 = icmp sgt i32 %n, 0
154  br i1 %cmp117, label %vector.body, label %for.cond.cleanup
155
156vector.body:                                      ; preds = %vector.body, %entry
157  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
158  %1 = getelementptr inbounds i8, i8* %x, i32 %index
159  %2 = bitcast i8* %1 to <16 x i8>*
160  %wide.load = load <16 x i8>, <16 x i8>* %2, align 1
161  %3 = shufflevector <16 x i8> %wide.load, <16 x i8> %wide.load, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
162  %4 = shufflevector <16 x i8> %wide.load, <16 x i8> %wide.load, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
163  %5 = zext <8 x i8> %3 to <8 x i16>
164  %6 = zext <8 x i8> %4 to <8 x i16>
165  %7 = getelementptr inbounds i8, i8* %y, i32 %index
166  %8 = bitcast i8* %7 to <16 x i8>*
167  %wide.load19 = load <16 x i8>, <16 x i8>* %8, align 1
168  %9 = shufflevector <16 x i8> %wide.load19, <16 x i8> %wide.load19, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
169  %10 = shufflevector <16 x i8> %wide.load19, <16 x i8> %wide.load19, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
170  %11 = zext <8 x i8> %9 to <8 x i16>
171  %12 = zext <8 x i8> %10 to <8 x i16>
172  %13 = mul <8 x i16> %11, %5
173  %14 = mul <8 x i16> %12, %6
174  %15 = lshr <8 x i16> %13, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
175  %16 = lshr <8 x i16> %14, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
176  %17 = shufflevector <8 x i16> %15, <8 x i16> %16, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
177  %18 = trunc <16 x i16> %17 to <16 x i8>
178  %19 = getelementptr inbounds i8, i8* %z, i32 %index
179  %20 = bitcast i8* %19 to <16 x i8>*
180  store <16 x i8> %18, <16 x i8>* %20, align 1
181  %index.next = add i32 %index, 16
182  %21 = icmp eq i32 %index.next, %n
183  br i1 %21, label %for.cond.cleanup, label %vector.body
184
185for.cond.cleanup:                                 ; preds = %vector.body, %entry
186  ret void
187}
188