1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2; RUN: opt -slp-vectorizer -S %s | FileCheck %s
3
4target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
5target triple = "arm64-apple-darwin"
6
7declare void @use(double)
8
9; The extracts %v1.lane.0 and %v1.lane.1 should be considered free during SLP,
10; because they will be directly in a vector register on AArch64.
11define void @noop_extracts_first_2_lanes(<2 x double>* %ptr.1, <4 x double>* %ptr.2) {
12; CHECK-LABEL: @noop_extracts_first_2_lanes(
13; CHECK-NEXT:  bb:
14; CHECK-NEXT:    [[V_1:%.*]] = load <2 x double>, <2 x double>* [[PTR_1:%.*]], align 8
15; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
16; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
17; CHECK-NEXT:    [[V2_LANE_3:%.*]] = extractelement <4 x double> [[V_2]], i32 3
18; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
19; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V2_LANE_3]], i32 1
20; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x double> [[V_1]], [[TMP1]]
21; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[V_1]], i32 0
22; CHECK-NEXT:    call void @use(double [[TMP3]])
23; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[V_1]], i32 1
24; CHECK-NEXT:    call void @use(double [[TMP4]])
25; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* [[PTR_1]], align 8
26; CHECK-NEXT:    ret void
27;
28bb:
29  %v.1 = load <2 x double>, <2 x double>* %ptr.1, align 8
30  %v1.lane.0 = extractelement <2 x double> %v.1, i32 0
31  %v1.lane.1 = extractelement <2 x double> %v.1, i32 1
32
33  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
34  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
35  %v2.lane.3 = extractelement <4 x double> %v.2, i32 3
36
37  %a.lane.0 = fmul double %v1.lane.0, %v2.lane.2
38  %a.lane.1 = fmul double %v1.lane.1, %v2.lane.3
39
40  %a.ins.0 = insertelement <2 x double> undef, double %a.lane.0, i32 0
41  %a.ins.1 = insertelement <2 x double> %a.ins.0, double %a.lane.1, i32 1
42
43  call void @use(double %v1.lane.0)
44  call void @use(double %v1.lane.1)
45
46  store <2 x double> %a.ins.1, <2 x double>* %ptr.1, align 8
47  ret void
48}
49
50; Extracts of consecutive indices, but different vector operand.
51define void @extracts_first_2_lanes_different_vectors(<2 x double>* %ptr.1, <4 x double>* %ptr.2, <2 x double>* %ptr.3) {
52; CHECK-LABEL: @extracts_first_2_lanes_different_vectors(
53; CHECK-NEXT:  bb:
54; CHECK-NEXT:    [[V_1:%.*]] = load <2 x double>, <2 x double>* [[PTR_1:%.*]], align 8
55; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <2 x double> [[V_1]], i32 0
56; CHECK-NEXT:    [[V_3:%.*]] = load <2 x double>, <2 x double>* [[PTR_3:%.*]], align 8
57; CHECK-NEXT:    [[V3_LANE_1:%.*]] = extractelement <2 x double> [[V_3]], i32 1
58; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
59; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
60; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V1_LANE_0]], i32 0
61; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V3_LANE_1]], i32 1
62; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
63; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2_LANE_2]], i32 1
64; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
65; CHECK-NEXT:    call void @use(double [[V1_LANE_0]])
66; CHECK-NEXT:    call void @use(double [[V3_LANE_1]])
67; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[PTR_1]], align 8
68; CHECK-NEXT:    ret void
69;
70bb:
71  %v.1 = load <2 x double>, <2 x double>* %ptr.1, align 8
72  %v1.lane.0 = extractelement <2 x double> %v.1, i32 0
73  %v.3 = load <2 x double>, <2 x double>* %ptr.3, align 8
74  %v3.lane.1 = extractelement <2 x double> %v.3, i32 1
75
76  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
77  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
78
79  %a.lane.0 = fmul double %v1.lane.0, %v2.lane.2
80  %a.lane.1 = fmul double %v3.lane.1, %v2.lane.2
81
82  %a.ins.0 = insertelement <2 x double> undef, double %a.lane.0, i32 0
83  %a.ins.1 = insertelement <2 x double> %a.ins.0, double %a.lane.1, i32 1
84
85  call void @use(double %v1.lane.0)
86  call void @use(double %v3.lane.1)
87
88  store <2 x double> %a.ins.1, <2 x double>* %ptr.1, align 8
89  ret void
90}
91
92; The extracts %v1.lane.2 and %v1.lane.3 should be considered free during SLP,
93; because they will be directly in a vector register on AArch64.
94define void @noop_extract_second_2_lanes(<4 x double>* %ptr.1, <4 x double>* %ptr.2) {
95; CHECK-LABEL: @noop_extract_second_2_lanes(
96; CHECK-NEXT:  bb:
97; CHECK-NEXT:    [[V_1:%.*]] = load <4 x double>, <4 x double>* [[PTR_1:%.*]], align 8
98; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <4 x double> [[V_1]], i32 2
99; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <4 x double> [[V_1]], i32 3
100; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
101; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
102; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V1_LANE_2]], i32 0
103; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V1_LANE_3]], i32 1
104; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
105; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2_LANE_2]], i32 1
106; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
107; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
108; CHECK-NEXT:    [[A_INS_11:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP5]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
109; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
110; CHECK-NEXT:    call void @use(double [[V1_LANE_3]])
111; CHECK-NEXT:    store <4 x double> [[A_INS_11]], <4 x double>* [[PTR_1]], align 8
112; CHECK-NEXT:    ret void
113;
114bb:
115  %v.1 = load <4 x double>, <4 x double>* %ptr.1, align 8
116  %v1.lane.2 = extractelement <4 x double> %v.1, i32 2
117  %v1.lane.3 = extractelement <4 x double> %v.1, i32 3
118
119  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
120  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
121
122  %a.lane.0 = fmul double %v1.lane.2, %v2.lane.2
123  %a.lane.1 = fmul double %v1.lane.3, %v2.lane.2
124
125  %a.ins.0 = insertelement <4 x double> undef, double %a.lane.0, i32 0
126  %a.ins.1 = insertelement <4 x double> %a.ins.0, double %a.lane.1, i32 1
127
128  call void @use(double %v1.lane.2)
129  call void @use(double %v1.lane.3)
130  store <4 x double> %a.ins.1, <4 x double>* %ptr.1, align 8
131  ret void
132}
133
134; %v1.lane.0 and %v1.lane.1 are used in reverse-order, so they won't be
135; directly in a vector register on AArch64.
136define void @extract_reverse_order(<2 x double>* %ptr.1, <4 x double>* %ptr.2) {
137; CHECK-LABEL: @extract_reverse_order(
138; CHECK-NEXT:  bb:
139; CHECK-NEXT:    [[V_1:%.*]] = load <2 x double>, <2 x double>* [[PTR_1:%.*]], align 8
140; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
141; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
142; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
143; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V2_LANE_2]], i32 1
144; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x double> [[V_1]], [[TMP1]]
145; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
146; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[V_1]], i32 0
147; CHECK-NEXT:    call void @use(double [[TMP4]])
148; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[V_1]], i32 1
149; CHECK-NEXT:    call void @use(double [[TMP5]])
150; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[PTR_1]], align 8
151; CHECK-NEXT:    ret void
152;
153bb:
154  %v.1 = load <2 x double>, <2 x double>* %ptr.1, align 8
155  %v1.lane.0 = extractelement <2 x double> %v.1, i32 0
156  %v1.lane.1 = extractelement <2 x double> %v.1, i32 1
157
158  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
159  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
160
161  %a.lane.0 = fmul double %v1.lane.1, %v2.lane.2
162  %a.lane.1 = fmul double %v1.lane.0, %v2.lane.2
163
164  %a.ins.0 = insertelement <2 x double> undef, double %a.lane.0, i32 0
165  %a.ins.1 = insertelement <2 x double> %a.ins.0, double %a.lane.1, i32 1
166
167  call void @use(double %v1.lane.0)
168  call void @use(double %v1.lane.1)
169
170  store <2 x double> %a.ins.1, <2 x double>* %ptr.1, align 8
171  ret void
172}
173
174; %v1.lane.1 and %v1.lane.2 are extracted from different vector registers on AArch64.
175define void @extract_lanes_1_and_2(<4 x double>* %ptr.1, <4 x double>* %ptr.2) {
176; CHECK-LABEL: @extract_lanes_1_and_2(
177; CHECK-NEXT:  bb:
178; CHECK-NEXT:    [[V_1:%.*]] = load <4 x double>, <4 x double>* [[PTR_1:%.*]], align 8
179; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <4 x double> [[V_1]], i32 1
180; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <4 x double> [[V_1]], i32 2
181; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
182; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
183; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V1_LANE_1]], i32 0
184; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V1_LANE_2]], i32 1
185; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
186; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2_LANE_2]], i32 1
187; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
188; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
189; CHECK-NEXT:    [[A_INS_11:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP5]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
190; CHECK-NEXT:    call void @use(double [[V1_LANE_1]])
191; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
192; CHECK-NEXT:    store <4 x double> [[A_INS_11]], <4 x double>* [[PTR_1]], align 8
193; CHECK-NEXT:    ret void
194;
195bb:
196  %v.1 = load <4 x double>, <4 x double>* %ptr.1, align 8
197  %v1.lane.1 = extractelement <4 x double> %v.1, i32 1
198  %v1.lane.2 = extractelement <4 x double> %v.1, i32 2
199
200  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
201  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
202
203  %a.lane.0 = fmul double %v1.lane.1, %v2.lane.2
204  %a.lane.1 = fmul double %v1.lane.2, %v2.lane.2
205
206  %a.ins.0 = insertelement <4 x double> undef, double %a.lane.0, i32 0
207  %a.ins.1 = insertelement <4 x double> %a.ins.0, double %a.lane.1, i32 1
208
209  call void @use(double %v1.lane.1)
210  call void @use(double %v1.lane.2)
211
212  store <4 x double> %a.ins.1, <4 x double>* %ptr.1, align 8
213  ret void
214}
215
216; More complex case where the extracted lanes are directly from a vector
217; register on AArch64 and should be considered free, because we can
218; directly use the source vector register.
219define void @noop_extracts_existing_vector_4_lanes(<9 x double>* %ptr.1, <4 x double>* %ptr.2) {
220; CHECK-LABEL: @noop_extracts_existing_vector_4_lanes(
221; CHECK-NEXT:  bb:
222; CHECK-NEXT:    [[V_1:%.*]] = load <9 x double>, <9 x double>* [[PTR_1:%.*]], align 8
223; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0
224; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1
225; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
226; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3
227; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
228; CHECK-NEXT:    [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
229; CHECK-NEXT:    [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
230; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
231; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x double> poison, double [[V1_LANE_2]], i32 0
232; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[V1_LANE_3]], i32 1
233; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[V1_LANE_0]], i32 2
234; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[V1_LANE_1]], i32 3
235; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x double> poison, double [[V2_LANE_2]], i32 0
236; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[V2_LANE_0]], i32 1
237; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
238; CHECK-NEXT:    [[TMP6:%.*]] = fmul <4 x double> [[TMP3]], [[SHUFFLE]]
239; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x double> [[TMP6]], <4 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
240; CHECK-NEXT:    [[A_INS_31:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP7]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 4, i32 5, i32 6, i32 7, i32 8>
241; CHECK-NEXT:    call void @use(double [[V1_LANE_0]])
242; CHECK-NEXT:    call void @use(double [[V1_LANE_1]])
243; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
244; CHECK-NEXT:    call void @use(double [[V1_LANE_3]])
245; CHECK-NEXT:    store <9 x double> [[A_INS_31]], <9 x double>* [[PTR_1]], align 8
246; CHECK-NEXT:    ret void
247;
248bb:
249  %v.1 = load <9 x double>, <9 x double>* %ptr.1, align 8
250  %v1.lane.0 = extractelement <9 x double> %v.1, i32 0
251  %v1.lane.1 = extractelement <9 x double> %v.1, i32 1
252  %v1.lane.2 = extractelement <9 x double> %v.1, i32 2
253  %v1.lane.3 = extractelement <9 x double> %v.1, i32 3
254  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
255  %v2.lane.0 = extractelement <4 x double> %v.2, i32 0
256  %v2.lane.1 = extractelement <4 x double> %v.2, i32 1
257  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
258  %a.lane.0 = fmul double %v1.lane.2, %v2.lane.2
259  %a.lane.1 = fmul double %v1.lane.3, %v2.lane.2
260  %a.lane.2 = fmul double %v1.lane.0, %v2.lane.2
261  %a.lane.3 = fmul double %v1.lane.1, %v2.lane.0
262  %a.ins.0 = insertelement <9 x double> undef, double %a.lane.0, i32 0
263  %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1
264  %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2
265  %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3
266  call void @use(double %v1.lane.0)
267  call void @use(double %v1.lane.1)
268  call void @use(double %v1.lane.2)
269  call void @use(double %v1.lane.3)
270  store <9 x double> %a.ins.3, <9 x double>* %ptr.1, align 8
271  ret void
272}
273
274; Extracted lanes are not used in the right order, so we cannot reuse the
275; source vector registers directly.
276define void @extracts_jumbled_4_lanes(<9 x double>* %ptr.1, <4 x double>* %ptr.2) {
277; CHECK-LABEL: @extracts_jumbled_4_lanes(
278; CHECK-NEXT:  bb:
279; CHECK-NEXT:    [[V_1:%.*]] = load <9 x double>, <9 x double>* [[PTR_1:%.*]], align 8
280; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0
281; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1
282; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
283; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3
284; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
285; CHECK-NEXT:    [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
286; CHECK-NEXT:    [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
287; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
288; CHECK-NEXT:    [[A_LANE_0:%.*]] = fmul double [[V1_LANE_0]], [[V2_LANE_2]]
289; CHECK-NEXT:    [[A_LANE_1:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_1]]
290; CHECK-NEXT:    [[A_LANE_2:%.*]] = fmul double [[V1_LANE_1]], [[V2_LANE_2]]
291; CHECK-NEXT:    [[A_LANE_3:%.*]] = fmul double [[V1_LANE_3]], [[V2_LANE_0]]
292; CHECK-NEXT:    [[A_INS_0:%.*]] = insertelement <9 x double> undef, double [[A_LANE_0]], i32 0
293; CHECK-NEXT:    [[A_INS_1:%.*]] = insertelement <9 x double> [[A_INS_0]], double [[A_LANE_1]], i32 1
294; CHECK-NEXT:    [[A_INS_2:%.*]] = insertelement <9 x double> [[A_INS_1]], double [[A_LANE_2]], i32 2
295; CHECK-NEXT:    [[A_INS_3:%.*]] = insertelement <9 x double> [[A_INS_2]], double [[A_LANE_3]], i32 3
296; CHECK-NEXT:    call void @use(double [[V1_LANE_0]])
297; CHECK-NEXT:    call void @use(double [[V1_LANE_1]])
298; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
299; CHECK-NEXT:    call void @use(double [[V1_LANE_3]])
300; CHECK-NEXT:    store <9 x double> [[A_INS_3]], <9 x double>* [[PTR_1]], align 8
301; CHECK-NEXT:    ret void
302;
303bb:
304  %v.1 = load <9 x double>, <9 x double>* %ptr.1, align 8
305  %v1.lane.0 = extractelement <9 x double> %v.1, i32 0
306  %v1.lane.1 = extractelement <9 x double> %v.1, i32 1
307  %v1.lane.2 = extractelement <9 x double> %v.1, i32 2
308  %v1.lane.3 = extractelement <9 x double> %v.1, i32 3
309  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
310  %v2.lane.0 = extractelement <4 x double> %v.2, i32 0
311  %v2.lane.1 = extractelement <4 x double> %v.2, i32 1
312  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
313  %a.lane.0 = fmul double %v1.lane.0, %v2.lane.2
314  %a.lane.1 = fmul double %v1.lane.2, %v2.lane.1
315  %a.lane.2 = fmul double %v1.lane.1, %v2.lane.2
316  %a.lane.3 = fmul double %v1.lane.3, %v2.lane.0
317  %a.ins.0 = insertelement <9 x double> undef, double %a.lane.0, i32 0
318  %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1
319  %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2
320  %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3
321  call void @use(double %v1.lane.0)
322  call void @use(double %v1.lane.1)
323  call void @use(double %v1.lane.2)
324  call void @use(double %v1.lane.3)
325  store <9 x double> %a.ins.3, <9 x double>* %ptr.1, align 8
326  ret void
327}
328
329
330; Even more complex case where the extracted lanes are directly from a vector
331; register on AArch64 and should be considered free, because we can
332; directly use the source vector register.
333define void @noop_extracts_9_lanes(<9 x double>* %ptr.1, <4 x double>* %ptr.2) {
334; CHECK-LABEL: @noop_extracts_9_lanes(
335; CHECK-NEXT:  bb:
336; CHECK-NEXT:    [[V_1:%.*]] = load <9 x double>, <9 x double>* [[PTR_1:%.*]], align 8
337; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0
338; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1
339; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
340; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3
341; CHECK-NEXT:    [[V1_LANE_4:%.*]] = extractelement <9 x double> [[V_1]], i32 4
342; CHECK-NEXT:    [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5
343; CHECK-NEXT:    [[V1_LANE_6:%.*]] = extractelement <9 x double> [[V_1]], i32 6
344; CHECK-NEXT:    [[V1_LANE_7:%.*]] = extractelement <9 x double> [[V_1]], i32 7
345; CHECK-NEXT:    [[V1_LANE_8:%.*]] = extractelement <9 x double> [[V_1]], i32 8
346; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
347; CHECK-NEXT:    [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
348; CHECK-NEXT:    [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
349; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
350; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_3]], i32 0
351; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x double> [[TMP0]], double [[V1_LANE_4]], i32 1
352; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x double> [[TMP1]], double [[V1_LANE_5]], i32 2
353; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x double> [[TMP2]], double [[V1_LANE_6]], i32 3
354; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x double> [[TMP3]], double [[V1_LANE_7]], i32 4
355; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x double> [[TMP4]], double [[V1_LANE_8]], i32 5
356; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x double> [[TMP5]], double [[V1_LANE_0]], i32 6
357; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x double> [[TMP6]], double [[V1_LANE_1]], i32 7
358; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x double> poison, double [[V2_LANE_0]], i32 0
359; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x double> [[TMP8]], double [[V2_LANE_2]], i32 1
360; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x double> [[TMP9]], double [[V2_LANE_1]], i32 2
361; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x double> [[TMP10]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 0, i32 1, i32 0, i32 1, i32 2>
362; CHECK-NEXT:    [[TMP11:%.*]] = fmul <8 x double> [[TMP7]], [[SHUFFLE2]]
363; CHECK-NEXT:    [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_0]]
364; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <8 x double> [[TMP11]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
365; CHECK-NEXT:    [[A_INS_73:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP12]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
366; CHECK-NEXT:    [[A_INS_8:%.*]] = insertelement <9 x double> [[A_INS_73]], double [[A_LANE_8]], i32 8
367; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_6]], i32 0
368; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x double> [[TMP13]], double [[V1_LANE_7]], i32 1
369; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x double> [[TMP14]], double [[V1_LANE_8]], i32 2
370; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <8 x double> [[TMP15]], double [[V1_LANE_0]], i32 3
371; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <8 x double> [[TMP16]], double [[V1_LANE_1]], i32 4
372; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <8 x double> [[TMP17]], double [[V1_LANE_2]], i32 5
373; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <8 x double> [[TMP18]], double [[V1_LANE_3]], i32 6
374; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <8 x double> [[TMP19]], double [[V1_LANE_4]], i32 7
375; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <8 x double> poison, double [[V2_LANE_2]], i32 0
376; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <8 x double> [[TMP21]], double [[V2_LANE_1]], i32 1
377; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <8 x double> [[TMP22]], double [[V2_LANE_0]], i32 2
378; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x double> [[TMP23]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 0, i32 1, i32 2, i32 0, i32 1>
379; CHECK-NEXT:    [[TMP24:%.*]] = fmul <8 x double> [[TMP20]], [[SHUFFLE]]
380; CHECK-NEXT:    [[B_LANE_8:%.*]] = fmul double [[V1_LANE_5]], [[V2_LANE_0]]
381; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <8 x double> [[TMP24]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
382; CHECK-NEXT:    [[B_INS_71:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP25]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
383; CHECK-NEXT:    [[B_INS_8:%.*]] = insertelement <9 x double> [[B_INS_71]], double [[B_LANE_8]], i32 8
384; CHECK-NEXT:    [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]
385; CHECK-NEXT:    store <9 x double> [[RES]], <9 x double>* [[PTR_1]], align 8
386; CHECK-NEXT:    ret void
387;
388bb:
389  %v.1 = load <9 x double>, <9 x double>* %ptr.1, align 8
390  %v1.lane.0 = extractelement <9 x double> %v.1, i32 0
391  %v1.lane.1 = extractelement <9 x double> %v.1, i32 1
392  %v1.lane.2 = extractelement <9 x double> %v.1, i32 2
393  %v1.lane.3 = extractelement <9 x double> %v.1, i32 3
394  %v1.lane.4 = extractelement <9 x double> %v.1, i32 4
395  %v1.lane.5 = extractelement <9 x double> %v.1, i32 5
396  %v1.lane.6 = extractelement <9 x double> %v.1, i32 6
397  %v1.lane.7 = extractelement <9 x double> %v.1, i32 7
398  %v1.lane.8 = extractelement <9 x double> %v.1, i32 8
399
400  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
401  %v2.lane.0 = extractelement <4 x double> %v.2, i32 0
402  %v2.lane.1 = extractelement <4 x double> %v.2, i32 1
403  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
404
405  %a.lane.0 = fmul double %v1.lane.3, %v2.lane.0
406  %a.lane.1 = fmul double %v1.lane.4, %v2.lane.2
407  %a.lane.2 = fmul double %v1.lane.5, %v2.lane.1
408  %a.lane.3 = fmul double %v1.lane.6, %v2.lane.0
409  %a.lane.4 = fmul double %v1.lane.7, %v2.lane.2
410  %a.lane.5 = fmul double %v1.lane.8, %v2.lane.0
411  %a.lane.6 = fmul double %v1.lane.0, %v2.lane.2
412  %a.lane.7 = fmul double %v1.lane.1, %v2.lane.1
413  %a.lane.8 = fmul double %v1.lane.2, %v2.lane.0
414
415  %a.ins.0 = insertelement <9 x double> undef, double %a.lane.0, i32 0
416  %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1
417  %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2
418  %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3
419  %a.ins.4 = insertelement <9 x double> %a.ins.3, double %a.lane.4, i32 4
420  %a.ins.5 = insertelement <9 x double> %a.ins.4, double %a.lane.5, i32 5
421  %a.ins.6 = insertelement <9 x double> %a.ins.5, double %a.lane.6, i32 6
422  %a.ins.7 = insertelement <9 x double> %a.ins.6, double %a.lane.7, i32 7
423  %a.ins.8 = insertelement <9 x double> %a.ins.7, double %a.lane.8, i32 8
424
425  %b.lane.0 = fmul double %v1.lane.6, %v2.lane.2
426  %b.lane.1 = fmul double %v1.lane.7, %v2.lane.1
427  %b.lane.2 = fmul double %v1.lane.8, %v2.lane.0
428  %b.lane.3 = fmul double %v1.lane.0, %v2.lane.2
429  %b.lane.4 = fmul double %v1.lane.1, %v2.lane.1
430  %b.lane.5 = fmul double %v1.lane.2, %v2.lane.0
431  %b.lane.6 = fmul double %v1.lane.3, %v2.lane.2
432  %b.lane.7 = fmul double %v1.lane.4, %v2.lane.1
433  %b.lane.8 = fmul double %v1.lane.5, %v2.lane.0
434
435  %b.ins.0 = insertelement <9 x double> undef, double %b.lane.0, i32 0
436  %b.ins.1 = insertelement <9 x double> %b.ins.0, double %b.lane.1, i32 1
437  %b.ins.2 = insertelement <9 x double> %b.ins.1, double %b.lane.2, i32 2
438  %b.ins.3 = insertelement <9 x double> %b.ins.2, double %b.lane.3, i32 3
439  %b.ins.4 = insertelement <9 x double> %b.ins.3, double %b.lane.4, i32 4
440  %b.ins.5 = insertelement <9 x double> %b.ins.4, double %b.lane.5, i32 5
441  %b.ins.6 = insertelement <9 x double> %b.ins.5, double %b.lane.6, i32 6
442  %b.ins.7 = insertelement <9 x double> %b.ins.6, double %b.lane.7, i32 7
443  %b.ins.8 = insertelement <9 x double> %b.ins.7, double %b.lane.8, i32 8
444
445  %res = fsub <9 x double> %a.ins.8, %b.ins.8
446  store <9 x double> %res, <9 x double>* %ptr.1, align 8
447  ret void
448}
449
450; Extracted lanes used in first fmul chain are not used in the right order, so
451; we cannot reuse the source vector registers directly.
452define void @first_mul_chain_jumbled(<9 x double>* %ptr.1, <4 x double>* %ptr.2) {
453; CHECK-LABEL: @first_mul_chain_jumbled(
454; CHECK-NEXT:  bb:
455; CHECK-NEXT:    [[V_1:%.*]] = load <9 x double>, <9 x double>* [[PTR_1:%.*]], align 8
456; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0
457; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1
458; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
459; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3
460; CHECK-NEXT:    [[V1_LANE_4:%.*]] = extractelement <9 x double> [[V_1]], i32 4
461; CHECK-NEXT:    [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5
462; CHECK-NEXT:    [[V1_LANE_6:%.*]] = extractelement <9 x double> [[V_1]], i32 6
463; CHECK-NEXT:    [[V1_LANE_7:%.*]] = extractelement <9 x double> [[V_1]], i32 7
464; CHECK-NEXT:    [[V1_LANE_8:%.*]] = extractelement <9 x double> [[V_1]], i32 8
465; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
466; CHECK-NEXT:    [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
467; CHECK-NEXT:    [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
468; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
469; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_4]], i32 0
470; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x double> [[TMP0]], double [[V1_LANE_3]], i32 1
471; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x double> [[TMP1]], double [[V1_LANE_6]], i32 2
472; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x double> [[TMP2]], double [[V1_LANE_5]], i32 3
473; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x double> [[TMP3]], double [[V1_LANE_8]], i32 4
474; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x double> [[TMP4]], double [[V1_LANE_7]], i32 5
475; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x double> [[TMP5]], double [[V1_LANE_1]], i32 6
476; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x double> [[TMP6]], double [[V1_LANE_0]], i32 7
477; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x double> poison, double [[V2_LANE_1]], i32 0
478; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x double> [[TMP8]], double [[V2_LANE_0]], i32 1
479; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x double> [[TMP9]], double [[V2_LANE_2]], i32 2
480; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x double> [[TMP10]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 1, i32 2, i32 0, i32 1, i32 2>
481; CHECK-NEXT:    [[TMP11:%.*]] = fmul <8 x double> [[TMP7]], [[SHUFFLE2]]
482; CHECK-NEXT:    [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_1]]
483; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <8 x double> [[TMP11]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
484; CHECK-NEXT:    [[A_INS_73:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP12]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
485; CHECK-NEXT:    [[A_INS_8:%.*]] = insertelement <9 x double> [[A_INS_73]], double [[A_LANE_8]], i32 8
486; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_6]], i32 0
487; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x double> [[TMP13]], double [[V1_LANE_7]], i32 1
488; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x double> [[TMP14]], double [[V1_LANE_8]], i32 2
489; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <8 x double> [[TMP15]], double [[V1_LANE_0]], i32 3
490; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <8 x double> [[TMP16]], double [[V1_LANE_1]], i32 4
491; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <8 x double> [[TMP17]], double [[V1_LANE_2]], i32 5
492; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <8 x double> [[TMP18]], double [[V1_LANE_3]], i32 6
493; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <8 x double> [[TMP19]], double [[V1_LANE_4]], i32 7
494; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x double> [[TMP10]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 1, i32 2, i32 0, i32 1, i32 2>
495; CHECK-NEXT:    [[TMP21:%.*]] = fmul <8 x double> [[TMP20]], [[SHUFFLE]]
496; CHECK-NEXT:    [[B_LANE_8:%.*]] = fmul double [[V1_LANE_5]], [[V2_LANE_0]]
497; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <8 x double> [[TMP21]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
498; CHECK-NEXT:    [[B_INS_71:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP22]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
499; CHECK-NEXT:    [[B_INS_8:%.*]] = insertelement <9 x double> [[B_INS_71]], double [[B_LANE_8]], i32 8
500; CHECK-NEXT:    [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]
501; CHECK-NEXT:    store <9 x double> [[RES]], <9 x double>* [[PTR_1]], align 8
502; CHECK-NEXT:    ret void
503;
504bb:
505  %v.1 = load <9 x double>, <9 x double>* %ptr.1, align 8
506  %v1.lane.0 = extractelement <9 x double> %v.1, i32 0
507  %v1.lane.1 = extractelement <9 x double> %v.1, i32 1
508  %v1.lane.2 = extractelement <9 x double> %v.1, i32 2
509  %v1.lane.3 = extractelement <9 x double> %v.1, i32 3
510  %v1.lane.4 = extractelement <9 x double> %v.1, i32 4
511  %v1.lane.5 = extractelement <9 x double> %v.1, i32 5
512  %v1.lane.6 = extractelement <9 x double> %v.1, i32 6
513  %v1.lane.7 = extractelement <9 x double> %v.1, i32 7
514  %v1.lane.8 = extractelement <9 x double> %v.1, i32 8
515
516  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
517  %v2.lane.0 = extractelement <4 x double> %v.2, i32 0
518  %v2.lane.1 = extractelement <4 x double> %v.2, i32 1
519  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
520
521  %a.lane.0 = fmul double %v1.lane.4, %v2.lane.1
522  %a.lane.1 = fmul double %v1.lane.3, %v2.lane.0
523  %a.lane.2 = fmul double %v1.lane.6, %v2.lane.2
524  %a.lane.3 = fmul double %v1.lane.5, %v2.lane.0
525  %a.lane.4 = fmul double %v1.lane.8, %v2.lane.2
526  %a.lane.5 = fmul double %v1.lane.7, %v2.lane.1
527  %a.lane.6 = fmul double %v1.lane.1, %v2.lane.0
528  %a.lane.7 = fmul double %v1.lane.0, %v2.lane.2
529  %a.lane.8 = fmul double %v1.lane.2, %v2.lane.1
530
531  %a.ins.0 = insertelement <9 x double> undef, double %a.lane.0, i32 0
532  %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1
533  %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2
534  %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3
535  %a.ins.4 = insertelement <9 x double> %a.ins.3, double %a.lane.4, i32 4
536  %a.ins.5 = insertelement <9 x double> %a.ins.4, double %a.lane.5, i32 5
537  %a.ins.6 = insertelement <9 x double> %a.ins.5, double %a.lane.6, i32 6
538  %a.ins.7 = insertelement <9 x double> %a.ins.6, double %a.lane.7, i32 7
539  %a.ins.8 = insertelement <9 x double> %a.ins.7, double %a.lane.8, i32 8
540
541  %b.lane.0 = fmul double %v1.lane.6, %v2.lane.1
542  %b.lane.1 = fmul double %v1.lane.7, %v2.lane.0
543  %b.lane.2 = fmul double %v1.lane.8, %v2.lane.2
544  %b.lane.3 = fmul double %v1.lane.0, %v2.lane.0
545  %b.lane.4 = fmul double %v1.lane.1, %v2.lane.2
546  %b.lane.5 = fmul double %v1.lane.2, %v2.lane.1
547  %b.lane.6 = fmul double %v1.lane.3, %v2.lane.0
548  %b.lane.7 = fmul double %v1.lane.4, %v2.lane.2
549  %b.lane.8 = fmul double %v1.lane.5, %v2.lane.0
550
551  %b.ins.0 = insertelement <9 x double> undef, double %b.lane.0, i32 0
552  %b.ins.1 = insertelement <9 x double> %b.ins.0, double %b.lane.1, i32 1
553  %b.ins.2 = insertelement <9 x double> %b.ins.1, double %b.lane.2, i32 2
554  %b.ins.3 = insertelement <9 x double> %b.ins.2, double %b.lane.3, i32 3
555  %b.ins.4 = insertelement <9 x double> %b.ins.3, double %b.lane.4, i32 4
556  %b.ins.5 = insertelement <9 x double> %b.ins.4, double %b.lane.5, i32 5
557  %b.ins.6 = insertelement <9 x double> %b.ins.5, double %b.lane.6, i32 6
558  %b.ins.7 = insertelement <9 x double> %b.ins.6, double %b.lane.7, i32 7
559  %b.ins.8 = insertelement <9 x double> %b.ins.7, double %b.lane.8, i32 8
560
561  %res = fsub <9 x double> %a.ins.8, %b.ins.8
562  store <9 x double> %res, <9 x double>* %ptr.1, align 8
563  ret void
564}
565
566; Extracted lanes used in both fmul chain are not used in the right order, so
567; we cannot reuse the source vector registers directly.
568define void @first_and_second_mul_chain_jumbled(<9 x double>* %ptr.1, <4 x double>* %ptr.2) {
569; CHECK-LABEL: @first_and_second_mul_chain_jumbled(
570; CHECK-NEXT:  bb:
571; CHECK-NEXT:    [[V_1:%.*]] = load <9 x double>, <9 x double>* [[PTR_1:%.*]], align 8
572; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0
573; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1
574; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
575; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3
576; CHECK-NEXT:    [[V1_LANE_4:%.*]] = extractelement <9 x double> [[V_1]], i32 4
577; CHECK-NEXT:    [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5
578; CHECK-NEXT:    [[V1_LANE_6:%.*]] = extractelement <9 x double> [[V_1]], i32 6
579; CHECK-NEXT:    [[V1_LANE_7:%.*]] = extractelement <9 x double> [[V_1]], i32 7
580; CHECK-NEXT:    [[V1_LANE_8:%.*]] = extractelement <9 x double> [[V_1]], i32 8
581; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
582; CHECK-NEXT:    [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
583; CHECK-NEXT:    [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
584; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
585; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_4]], i32 0
586; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x double> [[TMP0]], double [[V1_LANE_3]], i32 1
587; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x double> [[TMP1]], double [[V1_LANE_5]], i32 2
588; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x double> [[TMP2]], double [[V1_LANE_6]], i32 3
589; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x double> [[TMP3]], double [[V1_LANE_8]], i32 4
590; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x double> [[TMP4]], double [[V1_LANE_7]], i32 5
591; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x double> [[TMP5]], double [[V1_LANE_1]], i32 6
592; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x double> [[TMP6]], double [[V1_LANE_0]], i32 7
593; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x double> poison, double [[V2_LANE_0]], i32 0
594; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x double> [[TMP8]], double [[V2_LANE_2]], i32 1
595; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x double> [[TMP9]], double [[V2_LANE_1]], i32 2
596; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x double> [[TMP10]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 1, i32 2, i32 0, i32 1, i32 2>
597; CHECK-NEXT:    [[TMP11:%.*]] = fmul <8 x double> [[TMP7]], [[SHUFFLE2]]
598; CHECK-NEXT:    [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_0]]
599; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <8 x double> [[TMP11]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
600; CHECK-NEXT:    [[A_INS_73:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP12]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
601; CHECK-NEXT:    [[A_INS_8:%.*]] = insertelement <9 x double> [[A_INS_73]], double [[A_LANE_8]], i32 8
602; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_7]], i32 0
603; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x double> [[TMP13]], double [[V1_LANE_6]], i32 1
604; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x double> [[TMP14]], double [[V1_LANE_8]], i32 2
605; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <8 x double> [[TMP15]], double [[V1_LANE_1]], i32 3
606; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <8 x double> [[TMP16]], double [[V1_LANE_0]], i32 4
607; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <8 x double> [[TMP17]], double [[V1_LANE_3]], i32 5
608; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <8 x double> [[TMP18]], double [[V1_LANE_2]], i32 6
609; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <8 x double> [[TMP19]], double [[V1_LANE_5]], i32 7
610; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <8 x double> poison, double [[V2_LANE_2]], i32 0
611; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <8 x double> [[TMP21]], double [[V2_LANE_1]], i32 1
612; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <8 x double> [[TMP22]], double [[V2_LANE_0]], i32 2
613; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x double> [[TMP23]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 0, i32 2, i32 0, i32 1, i32 2>
614; CHECK-NEXT:    [[TMP24:%.*]] = fmul <8 x double> [[TMP20]], [[SHUFFLE]]
615; CHECK-NEXT:    [[B_LANE_8:%.*]] = fmul double [[V1_LANE_4]], [[V2_LANE_2]]
616; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <8 x double> [[TMP24]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
617; CHECK-NEXT:    [[B_INS_71:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP25]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
618; CHECK-NEXT:    [[B_INS_8:%.*]] = insertelement <9 x double> [[B_INS_71]], double [[B_LANE_8]], i32 8
619; CHECK-NEXT:    [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]
620; CHECK-NEXT:    store <9 x double> [[RES]], <9 x double>* [[PTR_1]], align 8
621; CHECK-NEXT:    ret void
622;
623bb:
624  %v.1 = load <9 x double>, <9 x double>* %ptr.1, align 8
625  %v1.lane.0 = extractelement <9 x double> %v.1, i32 0
626  %v1.lane.1 = extractelement <9 x double> %v.1, i32 1
627  %v1.lane.2 = extractelement <9 x double> %v.1, i32 2
628  %v1.lane.3 = extractelement <9 x double> %v.1, i32 3
629  %v1.lane.4 = extractelement <9 x double> %v.1, i32 4
630  %v1.lane.5 = extractelement <9 x double> %v.1, i32 5
631  %v1.lane.6 = extractelement <9 x double> %v.1, i32 6
632  %v1.lane.7 = extractelement <9 x double> %v.1, i32 7
633  %v1.lane.8 = extractelement <9 x double> %v.1, i32 8
634
635  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
636  %v2.lane.0 = extractelement <4 x double> %v.2, i32 0
637  %v2.lane.1 = extractelement <4 x double> %v.2, i32 1
638  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
639
640  %a.lane.0 = fmul double %v1.lane.4, %v2.lane.0
641  %a.lane.1 = fmul double %v1.lane.3, %v2.lane.2
642  %a.lane.2 = fmul double %v1.lane.5, %v2.lane.1
643  %a.lane.3 = fmul double %v1.lane.6, %v2.lane.2
644  %a.lane.4 = fmul double %v1.lane.8, %v2.lane.1
645  %a.lane.5 = fmul double %v1.lane.7, %v2.lane.0
646  %a.lane.6 = fmul double %v1.lane.1, %v2.lane.2
647  %a.lane.7 = fmul double %v1.lane.0, %v2.lane.1
648  %a.lane.8 = fmul double %v1.lane.2, %v2.lane.0
649
650  %a.ins.0 = insertelement <9 x double> undef, double %a.lane.0, i32 0
651  %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1
652  %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2
653  %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3
654  %a.ins.4 = insertelement <9 x double> %a.ins.3, double %a.lane.4, i32 4
655  %a.ins.5 = insertelement <9 x double> %a.ins.4, double %a.lane.5, i32 5
656  %a.ins.6 = insertelement <9 x double> %a.ins.5, double %a.lane.6, i32 6
657  %a.ins.7 = insertelement <9 x double> %a.ins.6, double %a.lane.7, i32 7
658  %a.ins.8 = insertelement <9 x double> %a.ins.7, double %a.lane.8, i32 8
659
660  %b.lane.0 = fmul double %v1.lane.7, %v2.lane.2
661  %b.lane.1 = fmul double %v1.lane.6, %v2.lane.1
662  %b.lane.2 = fmul double %v1.lane.8, %v2.lane.0
663  %b.lane.3 = fmul double %v1.lane.1, %v2.lane.2
664  %b.lane.4 = fmul double %v1.lane.0, %v2.lane.0
665  %b.lane.5 = fmul double %v1.lane.3, %v2.lane.2
666  %b.lane.6 = fmul double %v1.lane.2, %v2.lane.1
667  %b.lane.7 = fmul double %v1.lane.5, %v2.lane.0
668  %b.lane.8 = fmul double %v1.lane.4, %v2.lane.2
669
670  %b.ins.0 = insertelement <9 x double> undef, double %b.lane.0, i32 0
671  %b.ins.1 = insertelement <9 x double> %b.ins.0, double %b.lane.1, i32 1
672  %b.ins.2 = insertelement <9 x double> %b.ins.1, double %b.lane.2, i32 2
673  %b.ins.3 = insertelement <9 x double> %b.ins.2, double %b.lane.3, i32 3
674  %b.ins.4 = insertelement <9 x double> %b.ins.3, double %b.lane.4, i32 4
675  %b.ins.5 = insertelement <9 x double> %b.ins.4, double %b.lane.5, i32 5
676  %b.ins.6 = insertelement <9 x double> %b.ins.5, double %b.lane.6, i32 6
677  %b.ins.7 = insertelement <9 x double> %b.ins.6, double %b.lane.7, i32 7
678  %b.ins.8 = insertelement <9 x double> %b.ins.7, double %b.lane.8, i32 8
679
680  %res = fsub <9 x double> %a.ins.8, %b.ins.8
681  store <9 x double> %res, <9 x double>* %ptr.1, align 8
682  ret void
683}
684