1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2; RUN: opt -slp-vectorizer -S %s | FileCheck %s
3
4target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
5target triple = "arm64-apple-darwin"
6
7declare void @use(double)
8
9; The extracts %v1.lane.0 and %v1.lane.1 should be considered free during SLP,
10; because they will be directly in a vector register on AArch64.
11define void @noop_extracts_first_2_lanes(<2 x double>* %ptr.1, <4 x double>* %ptr.2) {
12; CHECK-LABEL: @noop_extracts_first_2_lanes(
13; CHECK-NEXT:  bb:
14; CHECK-NEXT:    [[V_1:%.*]] = load <2 x double>, <2 x double>* [[PTR_1:%.*]], align 8
15; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
16; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
17; CHECK-NEXT:    [[V2_LANE_3:%.*]] = extractelement <4 x double> [[V_2]], i32 3
18; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
19; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V2_LANE_3]], i32 1
20; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x double> [[V_1]], [[TMP1]]
21; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[V_1]], i32 0
22; CHECK-NEXT:    call void @use(double [[TMP3]])
23; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[V_1]], i32 1
24; CHECK-NEXT:    call void @use(double [[TMP4]])
25; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* [[PTR_1]], align 8
26; CHECK-NEXT:    ret void
27;
28bb:
29  %v.1 = load <2 x double>, <2 x double>* %ptr.1, align 8
30  %v1.lane.0 = extractelement <2 x double> %v.1, i32 0
31  %v1.lane.1 = extractelement <2 x double> %v.1, i32 1
32
33  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
34  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
35  %v2.lane.3 = extractelement <4 x double> %v.2, i32 3
36
37  %a.lane.0 = fmul double %v1.lane.0, %v2.lane.2
38  %a.lane.1 = fmul double %v1.lane.1, %v2.lane.3
39
40  %a.ins.0 = insertelement <2 x double> undef, double %a.lane.0, i32 0
41  %a.ins.1 = insertelement <2 x double> %a.ins.0, double %a.lane.1, i32 1
42
43  call void @use(double %v1.lane.0)
44  call void @use(double %v1.lane.1)
45
46  store <2 x double> %a.ins.1, <2 x double>* %ptr.1, align 8
47  ret void
48}
49
50; Extracts of consecutive indices, but different vector operand.
51define void @extracts_first_2_lanes_different_vectors(<2 x double>* %ptr.1, <4 x double>* %ptr.2, <2 x double>* %ptr.3) {
52; CHECK-LABEL: @extracts_first_2_lanes_different_vectors(
53; CHECK-NEXT:  bb:
54; CHECK-NEXT:    [[V_1:%.*]] = load <2 x double>, <2 x double>* [[PTR_1:%.*]], align 8
55; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <2 x double> [[V_1]], i32 0
56; CHECK-NEXT:    [[V_3:%.*]] = load <2 x double>, <2 x double>* [[PTR_3:%.*]], align 8
57; CHECK-NEXT:    [[V3_LANE_1:%.*]] = extractelement <2 x double> [[V_3]], i32 1
58; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
59; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
60; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V1_LANE_0]], i32 0
61; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V3_LANE_1]], i32 1
62; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
63; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2_LANE_2]], i32 1
64; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
65; CHECK-NEXT:    call void @use(double [[V1_LANE_0]])
66; CHECK-NEXT:    call void @use(double [[V3_LANE_1]])
67; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[PTR_1]], align 8
68; CHECK-NEXT:    ret void
69;
70bb:
71  %v.1 = load <2 x double>, <2 x double>* %ptr.1, align 8
72  %v1.lane.0 = extractelement <2 x double> %v.1, i32 0
73  %v.3 = load <2 x double>, <2 x double>* %ptr.3, align 8
74  %v3.lane.1 = extractelement <2 x double> %v.3, i32 1
75
76  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
77  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
78
79  %a.lane.0 = fmul double %v1.lane.0, %v2.lane.2
80  %a.lane.1 = fmul double %v3.lane.1, %v2.lane.2
81
82  %a.ins.0 = insertelement <2 x double> undef, double %a.lane.0, i32 0
83  %a.ins.1 = insertelement <2 x double> %a.ins.0, double %a.lane.1, i32 1
84
85  call void @use(double %v1.lane.0)
86  call void @use(double %v3.lane.1)
87
88  store <2 x double> %a.ins.1, <2 x double>* %ptr.1, align 8
89  ret void
90}
91
92; The extracts %v1.lane.2 and %v1.lane.3 should be considered free during SLP,
93; because they will be directly in a vector register on AArch64.
94define void @noop_extract_second_2_lanes(<4 x double>* %ptr.1, <4 x double>* %ptr.2) {
95; CHECK-LABEL: @noop_extract_second_2_lanes(
96; CHECK-NEXT:  bb:
97; CHECK-NEXT:    [[V_1:%.*]] = load <4 x double>, <4 x double>* [[PTR_1:%.*]], align 8
98; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <4 x double> [[V_1]], i32 2
99; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <4 x double> [[V_1]], i32 3
100; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
101; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
102; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V1_LANE_2]], i32 0
103; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V1_LANE_3]], i32 1
104; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
105; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2_LANE_2]], i32 1
106; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
107; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
108; CHECK-NEXT:    [[A_INS_11:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP5]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
109; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
110; CHECK-NEXT:    call void @use(double [[V1_LANE_3]])
111; CHECK-NEXT:    store <4 x double> [[A_INS_11]], <4 x double>* [[PTR_1]], align 8
112; CHECK-NEXT:    ret void
113;
114bb:
115  %v.1 = load <4 x double>, <4 x double>* %ptr.1, align 8
116  %v1.lane.2 = extractelement <4 x double> %v.1, i32 2
117  %v1.lane.3 = extractelement <4 x double> %v.1, i32 3
118
119  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
120  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
121
122  %a.lane.0 = fmul double %v1.lane.2, %v2.lane.2
123  %a.lane.1 = fmul double %v1.lane.3, %v2.lane.2
124
125  %a.ins.0 = insertelement <4 x double> undef, double %a.lane.0, i32 0
126  %a.ins.1 = insertelement <4 x double> %a.ins.0, double %a.lane.1, i32 1
127
128  call void @use(double %v1.lane.2)
129  call void @use(double %v1.lane.3)
130  store <4 x double> %a.ins.1, <4 x double>* %ptr.1, align 8
131  ret void
132}
133
134; %v1.lane.0 and %v1.lane.1 are used in reverse-order, so they won't be
135; directly in a vector register on AArch64.
136define void @extract_reverse_order(<2 x double>* %ptr.1, <4 x double>* %ptr.2) {
137; CHECK-LABEL: @extract_reverse_order(
138; CHECK-NEXT:  bb:
139; CHECK-NEXT:    [[V_1:%.*]] = load <2 x double>, <2 x double>* [[PTR_1:%.*]], align 8
140; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x double> [[V_1]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
141; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
142; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
143; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
144; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V2_LANE_2]], i32 1
145; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x double> [[SHUFFLE]], [[TMP1]]
146; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[SHUFFLE]], i32 1
147; CHECK-NEXT:    call void @use(double [[TMP3]])
148; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[SHUFFLE]], i32 0
149; CHECK-NEXT:    call void @use(double [[TMP4]])
150; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* [[PTR_1]], align 8
151; CHECK-NEXT:    ret void
152;
153bb:
154  %v.1 = load <2 x double>, <2 x double>* %ptr.1, align 8
155  %v1.lane.0 = extractelement <2 x double> %v.1, i32 0
156  %v1.lane.1 = extractelement <2 x double> %v.1, i32 1
157
158  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
159  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
160
161  %a.lane.0 = fmul double %v1.lane.1, %v2.lane.2
162  %a.lane.1 = fmul double %v1.lane.0, %v2.lane.2
163
164  %a.ins.0 = insertelement <2 x double> undef, double %a.lane.0, i32 0
165  %a.ins.1 = insertelement <2 x double> %a.ins.0, double %a.lane.1, i32 1
166
167  call void @use(double %v1.lane.0)
168  call void @use(double %v1.lane.1)
169
170  store <2 x double> %a.ins.1, <2 x double>* %ptr.1, align 8
171  ret void
172}
173
174; %v1.lane.1 and %v1.lane.2 are extracted from different vector registers on AArch64.
175define void @extract_lanes_1_and_2(<4 x double>* %ptr.1, <4 x double>* %ptr.2) {
176; CHECK-LABEL: @extract_lanes_1_and_2(
177; CHECK-NEXT:  bb:
178; CHECK-NEXT:    [[V_1:%.*]] = load <4 x double>, <4 x double>* [[PTR_1:%.*]], align 8
179; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <4 x double> [[V_1]], i32 1
180; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <4 x double> [[V_1]], i32 2
181; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
182; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
183; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V1_LANE_1]], i32 0
184; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V1_LANE_2]], i32 1
185; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
186; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2_LANE_2]], i32 1
187; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
188; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
189; CHECK-NEXT:    [[A_INS_11:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP5]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
190; CHECK-NEXT:    call void @use(double [[V1_LANE_1]])
191; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
192; CHECK-NEXT:    store <4 x double> [[A_INS_11]], <4 x double>* [[PTR_1]], align 8
193; CHECK-NEXT:    ret void
194;
195bb:
196  %v.1 = load <4 x double>, <4 x double>* %ptr.1, align 8
197  %v1.lane.1 = extractelement <4 x double> %v.1, i32 1
198  %v1.lane.2 = extractelement <4 x double> %v.1, i32 2
199
200  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
201  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
202
203  %a.lane.0 = fmul double %v1.lane.1, %v2.lane.2
204  %a.lane.1 = fmul double %v1.lane.2, %v2.lane.2
205
206  %a.ins.0 = insertelement <4 x double> undef, double %a.lane.0, i32 0
207  %a.ins.1 = insertelement <4 x double> %a.ins.0, double %a.lane.1, i32 1
208
209  call void @use(double %v1.lane.1)
210  call void @use(double %v1.lane.2)
211
212  store <4 x double> %a.ins.1, <4 x double>* %ptr.1, align 8
213  ret void
214}
215
216; More complex case where the extracted lanes are directly from a vector
217; register on AArch64 and should be considered free, because we can
218; directly use the source vector register.
219define void @noop_extracts_existing_vector_4_lanes(<9 x double>* %ptr.1, <4 x double>* %ptr.2) {
220; CHECK-LABEL: @noop_extracts_existing_vector_4_lanes(
221; CHECK-NEXT:  bb:
222; CHECK-NEXT:    [[V_1:%.*]] = load <9 x double>, <9 x double>* [[PTR_1:%.*]], align 8
223; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0
224; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1
225; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
226; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3
227; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
228; CHECK-NEXT:    [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
229; CHECK-NEXT:    [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
230; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
231; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x double> poison, double [[V1_LANE_2]], i32 0
232; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[V1_LANE_3]], i32 1
233; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[V1_LANE_0]], i32 2
234; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[V1_LANE_1]], i32 3
235; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> poison, double [[V2_LANE_2]], i32 0
236; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> [[TMP4]], double [[V2_LANE_0]], i32 1
237; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
238; CHECK-NEXT:    [[TMP6:%.*]] = fmul <4 x double> [[TMP3]], [[SHUFFLE]]
239; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x double> [[TMP6]], <4 x double> undef, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
240; CHECK-NEXT:    [[A_INS_31:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP7]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 4, i32 5, i32 6, i32 7, i32 8>
241; CHECK-NEXT:    call void @use(double [[V1_LANE_0]])
242; CHECK-NEXT:    call void @use(double [[V1_LANE_1]])
243; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
244; CHECK-NEXT:    call void @use(double [[V1_LANE_3]])
245; CHECK-NEXT:    store <9 x double> [[A_INS_31]], <9 x double>* [[PTR_1]], align 8
246; CHECK-NEXT:    ret void
247;
248bb:
249  %v.1 = load <9 x double>, <9 x double>* %ptr.1, align 8
250  %v1.lane.0 = extractelement <9 x double> %v.1, i32 0
251  %v1.lane.1 = extractelement <9 x double> %v.1, i32 1
252  %v1.lane.2 = extractelement <9 x double> %v.1, i32 2
253  %v1.lane.3 = extractelement <9 x double> %v.1, i32 3
254  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
255  %v2.lane.0 = extractelement <4 x double> %v.2, i32 0
256  %v2.lane.1 = extractelement <4 x double> %v.2, i32 1
257  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
258  %a.lane.0 = fmul double %v1.lane.2, %v2.lane.2
259  %a.lane.1 = fmul double %v1.lane.3, %v2.lane.2
260  %a.lane.2 = fmul double %v1.lane.0, %v2.lane.2
261  %a.lane.3 = fmul double %v1.lane.1, %v2.lane.0
262  %a.ins.0 = insertelement <9 x double> undef, double %a.lane.0, i32 0
263  %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1
264  %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2
265  %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3
266  call void @use(double %v1.lane.0)
267  call void @use(double %v1.lane.1)
268  call void @use(double %v1.lane.2)
269  call void @use(double %v1.lane.3)
270  store <9 x double> %a.ins.3, <9 x double>* %ptr.1, align 8
271  ret void
272}
273
274; Extracted lanes are not used in the right order, so we cannot reuse the
275; source vector registers directly.
276define void @extracts_jumbled_4_lanes(<9 x double>* %ptr.1, <4 x double>* %ptr.2) {
277; CHECK-LABEL: @extracts_jumbled_4_lanes(
278; CHECK-NEXT:  bb:
279; CHECK-NEXT:    [[V_1:%.*]] = load <9 x double>, <9 x double>* [[PTR_1:%.*]], align 8
280; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0
281; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1
282; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
283; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3
284; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
285; CHECK-NEXT:    [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
286; CHECK-NEXT:    [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
287; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
288; CHECK-NEXT:    [[A_LANE_0:%.*]] = fmul double [[V1_LANE_0]], [[V2_LANE_2]]
289; CHECK-NEXT:    [[A_LANE_1:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_1]]
290; CHECK-NEXT:    [[A_LANE_2:%.*]] = fmul double [[V1_LANE_1]], [[V2_LANE_2]]
291; CHECK-NEXT:    [[A_LANE_3:%.*]] = fmul double [[V1_LANE_3]], [[V2_LANE_0]]
292; CHECK-NEXT:    [[A_INS_0:%.*]] = insertelement <9 x double> undef, double [[A_LANE_0]], i32 0
293; CHECK-NEXT:    [[A_INS_1:%.*]] = insertelement <9 x double> [[A_INS_0]], double [[A_LANE_1]], i32 1
294; CHECK-NEXT:    [[A_INS_2:%.*]] = insertelement <9 x double> [[A_INS_1]], double [[A_LANE_2]], i32 2
295; CHECK-NEXT:    [[A_INS_3:%.*]] = insertelement <9 x double> [[A_INS_2]], double [[A_LANE_3]], i32 3
296; CHECK-NEXT:    call void @use(double [[V1_LANE_0]])
297; CHECK-NEXT:    call void @use(double [[V1_LANE_1]])
298; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
299; CHECK-NEXT:    call void @use(double [[V1_LANE_3]])
300; CHECK-NEXT:    store <9 x double> [[A_INS_3]], <9 x double>* [[PTR_1]], align 8
301; CHECK-NEXT:    ret void
302;
303bb:
304  %v.1 = load <9 x double>, <9 x double>* %ptr.1, align 8
305  %v1.lane.0 = extractelement <9 x double> %v.1, i32 0
306  %v1.lane.1 = extractelement <9 x double> %v.1, i32 1
307  %v1.lane.2 = extractelement <9 x double> %v.1, i32 2
308  %v1.lane.3 = extractelement <9 x double> %v.1, i32 3
309  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
310  %v2.lane.0 = extractelement <4 x double> %v.2, i32 0
311  %v2.lane.1 = extractelement <4 x double> %v.2, i32 1
312  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
313  %a.lane.0 = fmul double %v1.lane.0, %v2.lane.2
314  %a.lane.1 = fmul double %v1.lane.2, %v2.lane.1
315  %a.lane.2 = fmul double %v1.lane.1, %v2.lane.2
316  %a.lane.3 = fmul double %v1.lane.3, %v2.lane.0
317  %a.ins.0 = insertelement <9 x double> undef, double %a.lane.0, i32 0
318  %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1
319  %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2
320  %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3
321  call void @use(double %v1.lane.0)
322  call void @use(double %v1.lane.1)
323  call void @use(double %v1.lane.2)
324  call void @use(double %v1.lane.3)
325  store <9 x double> %a.ins.3, <9 x double>* %ptr.1, align 8
326  ret void
327}
328
329
330; Even more complex case where the extracted lanes are directly from a vector
331; register on AArch64 and should be considered free, because we can
332; directly use the source vector register.
333define void @noop_extracts_9_lanes(<9 x double>* %ptr.1, <4 x double>* %ptr.2) {
334; CHECK-LABEL: @noop_extracts_9_lanes(
335; CHECK-NEXT:  bb:
336; CHECK-NEXT:    [[V_1:%.*]] = load <9 x double>, <9 x double>* [[PTR_1:%.*]], align 8
337; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0
338; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1
339; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
340; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3
341; CHECK-NEXT:    [[V1_LANE_4:%.*]] = extractelement <9 x double> [[V_1]], i32 4
342; CHECK-NEXT:    [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5
343; CHECK-NEXT:    [[V1_LANE_6:%.*]] = extractelement <9 x double> [[V_1]], i32 6
344; CHECK-NEXT:    [[V1_LANE_7:%.*]] = extractelement <9 x double> [[V_1]], i32 7
345; CHECK-NEXT:    [[V1_LANE_8:%.*]] = extractelement <9 x double> [[V_1]], i32 8
346; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
347; CHECK-NEXT:    [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
348; CHECK-NEXT:    [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
349; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
350; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_3]], i32 0
351; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x double> [[TMP0]], double [[V1_LANE_4]], i32 1
352; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x double> [[TMP1]], double [[V1_LANE_5]], i32 2
353; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x double> [[TMP2]], double [[V1_LANE_6]], i32 3
354; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x double> [[TMP3]], double [[V1_LANE_7]], i32 4
355; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x double> [[TMP4]], double [[V1_LANE_8]], i32 5
356; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x double> [[TMP5]], double [[V1_LANE_0]], i32 6
357; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x double> [[TMP6]], double [[V1_LANE_1]], i32 7
358; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x double> poison, double [[V2_LANE_0]], i32 0
359; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x double> [[TMP8]], double [[V2_LANE_2]], i32 1
360; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x double> [[TMP9]], double [[V2_LANE_1]], i32 2
361; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x double> [[TMP10]], double [[V2_LANE_0]], i32 3
362; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <8 x double> [[TMP11]], double [[V2_LANE_2]], i32 4
363; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x double> [[TMP12]], double [[V2_LANE_0]], i32 5
364; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x double> [[TMP13]], double [[V2_LANE_2]], i32 6
365; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x double> [[TMP14]], double [[V2_LANE_1]], i32 7
366; CHECK-NEXT:    [[TMP16:%.*]] = fmul <8 x double> [[TMP7]], [[TMP15]]
367; CHECK-NEXT:    [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_0]]
368; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <8 x double> [[TMP16]], <8 x double> undef, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
369; CHECK-NEXT:    [[A_INS_72:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP17]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
370; CHECK-NEXT:    [[A_INS_8:%.*]] = insertelement <9 x double> [[A_INS_72]], double [[A_LANE_8]], i32 8
371; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_6]], i32 0
372; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <8 x double> [[TMP18]], double [[V1_LANE_7]], i32 1
373; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <8 x double> [[TMP19]], double [[V1_LANE_8]], i32 2
374; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <8 x double> [[TMP20]], double [[V1_LANE_0]], i32 3
375; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <8 x double> [[TMP21]], double [[V1_LANE_1]], i32 4
376; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <8 x double> [[TMP22]], double [[V1_LANE_2]], i32 5
377; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <8 x double> [[TMP23]], double [[V1_LANE_3]], i32 6
378; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <8 x double> [[TMP24]], double [[V1_LANE_4]], i32 7
379; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <8 x double> poison, double [[V2_LANE_2]], i32 0
380; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <8 x double> [[TMP26]], double [[V2_LANE_1]], i32 1
381; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <8 x double> [[TMP27]], double [[V2_LANE_0]], i32 2
382; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <8 x double> [[TMP28]], double [[V2_LANE_2]], i32 3
383; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <8 x double> [[TMP29]], double [[V2_LANE_1]], i32 4
384; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <8 x double> [[TMP30]], double [[V2_LANE_0]], i32 5
385; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <8 x double> [[TMP31]], double [[V2_LANE_2]], i32 6
386; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <8 x double> [[TMP32]], double [[V2_LANE_1]], i32 7
387; CHECK-NEXT:    [[TMP34:%.*]] = fmul <8 x double> [[TMP25]], [[TMP33]]
388; CHECK-NEXT:    [[B_LANE_8:%.*]] = fmul double [[V1_LANE_5]], [[V2_LANE_0]]
389; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <8 x double> [[TMP34]], <8 x double> undef, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
390; CHECK-NEXT:    [[B_INS_71:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP35]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
391; CHECK-NEXT:    [[B_INS_8:%.*]] = insertelement <9 x double> [[B_INS_71]], double [[B_LANE_8]], i32 8
392; CHECK-NEXT:    [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]
393; CHECK-NEXT:    store <9 x double> [[RES]], <9 x double>* [[PTR_1]], align 8
394; CHECK-NEXT:    ret void
395;
396bb:
397  %v.1 = load <9 x double>, <9 x double>* %ptr.1, align 8
398  %v1.lane.0 = extractelement <9 x double> %v.1, i32 0
399  %v1.lane.1 = extractelement <9 x double> %v.1, i32 1
400  %v1.lane.2 = extractelement <9 x double> %v.1, i32 2
401  %v1.lane.3 = extractelement <9 x double> %v.1, i32 3
402  %v1.lane.4 = extractelement <9 x double> %v.1, i32 4
403  %v1.lane.5 = extractelement <9 x double> %v.1, i32 5
404  %v1.lane.6 = extractelement <9 x double> %v.1, i32 6
405  %v1.lane.7 = extractelement <9 x double> %v.1, i32 7
406  %v1.lane.8 = extractelement <9 x double> %v.1, i32 8
407
408  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
409  %v2.lane.0 = extractelement <4 x double> %v.2, i32 0
410  %v2.lane.1 = extractelement <4 x double> %v.2, i32 1
411  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
412
413  %a.lane.0 = fmul double %v1.lane.3, %v2.lane.0
414  %a.lane.1 = fmul double %v1.lane.4, %v2.lane.2
415  %a.lane.2 = fmul double %v1.lane.5, %v2.lane.1
416  %a.lane.3 = fmul double %v1.lane.6, %v2.lane.0
417  %a.lane.4 = fmul double %v1.lane.7, %v2.lane.2
418  %a.lane.5 = fmul double %v1.lane.8, %v2.lane.0
419  %a.lane.6 = fmul double %v1.lane.0, %v2.lane.2
420  %a.lane.7 = fmul double %v1.lane.1, %v2.lane.1
421  %a.lane.8 = fmul double %v1.lane.2, %v2.lane.0
422
423  %a.ins.0 = insertelement <9 x double> undef, double %a.lane.0, i32 0
424  %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1
425  %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2
426  %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3
427  %a.ins.4 = insertelement <9 x double> %a.ins.3, double %a.lane.4, i32 4
428  %a.ins.5 = insertelement <9 x double> %a.ins.4, double %a.lane.5, i32 5
429  %a.ins.6 = insertelement <9 x double> %a.ins.5, double %a.lane.6, i32 6
430  %a.ins.7 = insertelement <9 x double> %a.ins.6, double %a.lane.7, i32 7
431  %a.ins.8 = insertelement <9 x double> %a.ins.7, double %a.lane.8, i32 8
432
433  %b.lane.0 = fmul double %v1.lane.6, %v2.lane.2
434  %b.lane.1 = fmul double %v1.lane.7, %v2.lane.1
435  %b.lane.2 = fmul double %v1.lane.8, %v2.lane.0
436  %b.lane.3 = fmul double %v1.lane.0, %v2.lane.2
437  %b.lane.4 = fmul double %v1.lane.1, %v2.lane.1
438  %b.lane.5 = fmul double %v1.lane.2, %v2.lane.0
439  %b.lane.6 = fmul double %v1.lane.3, %v2.lane.2
440  %b.lane.7 = fmul double %v1.lane.4, %v2.lane.1
441  %b.lane.8 = fmul double %v1.lane.5, %v2.lane.0
442
443  %b.ins.0 = insertelement <9 x double> undef, double %b.lane.0, i32 0
444  %b.ins.1 = insertelement <9 x double> %b.ins.0, double %b.lane.1, i32 1
445  %b.ins.2 = insertelement <9 x double> %b.ins.1, double %b.lane.2, i32 2
446  %b.ins.3 = insertelement <9 x double> %b.ins.2, double %b.lane.3, i32 3
447  %b.ins.4 = insertelement <9 x double> %b.ins.3, double %b.lane.4, i32 4
448  %b.ins.5 = insertelement <9 x double> %b.ins.4, double %b.lane.5, i32 5
449  %b.ins.6 = insertelement <9 x double> %b.ins.5, double %b.lane.6, i32 6
450  %b.ins.7 = insertelement <9 x double> %b.ins.6, double %b.lane.7, i32 7
451  %b.ins.8 = insertelement <9 x double> %b.ins.7, double %b.lane.8, i32 8
452
453  %res = fsub <9 x double> %a.ins.8, %b.ins.8
454  store <9 x double> %res, <9 x double>* %ptr.1, align 8
455  ret void
456}
457
458; Extracted lanes used in first fmul chain are not used in the right order, so
459; we cannot reuse the source vector registers directly.
460define void @first_mul_chain_jumbled(<9 x double>* %ptr.1, <4 x double>* %ptr.2) {
461; CHECK-LABEL: @first_mul_chain_jumbled(
462; CHECK-NEXT:  bb:
463; CHECK-NEXT:    [[V_1:%.*]] = load <9 x double>, <9 x double>* [[PTR_1:%.*]], align 8
464; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0
465; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1
466; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
467; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3
468; CHECK-NEXT:    [[V1_LANE_4:%.*]] = extractelement <9 x double> [[V_1]], i32 4
469; CHECK-NEXT:    [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5
470; CHECK-NEXT:    [[V1_LANE_6:%.*]] = extractelement <9 x double> [[V_1]], i32 6
471; CHECK-NEXT:    [[V1_LANE_7:%.*]] = extractelement <9 x double> [[V_1]], i32 7
472; CHECK-NEXT:    [[V1_LANE_8:%.*]] = extractelement <9 x double> [[V_1]], i32 8
473; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
474; CHECK-NEXT:    [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
475; CHECK-NEXT:    [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
476; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
477; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_4]], i32 0
478; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x double> [[TMP0]], double [[V1_LANE_3]], i32 1
479; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x double> [[TMP1]], double [[V1_LANE_6]], i32 2
480; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x double> [[TMP2]], double [[V1_LANE_5]], i32 3
481; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x double> [[TMP3]], double [[V1_LANE_8]], i32 4
482; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x double> [[TMP4]], double [[V1_LANE_7]], i32 5
483; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x double> [[TMP5]], double [[V1_LANE_1]], i32 6
484; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x double> [[TMP6]], double [[V1_LANE_0]], i32 7
485; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x double> poison, double [[V2_LANE_1]], i32 0
486; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x double> [[TMP8]], double [[V2_LANE_0]], i32 1
487; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x double> [[TMP9]], double [[V2_LANE_2]], i32 2
488; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x double> [[TMP10]], double [[V2_LANE_0]], i32 3
489; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <8 x double> [[TMP11]], double [[V2_LANE_2]], i32 4
490; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x double> [[TMP12]], double [[V2_LANE_1]], i32 5
491; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x double> [[TMP13]], double [[V2_LANE_0]], i32 6
492; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x double> [[TMP14]], double [[V2_LANE_2]], i32 7
493; CHECK-NEXT:    [[TMP16:%.*]] = fmul <8 x double> [[TMP7]], [[TMP15]]
494; CHECK-NEXT:    [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_1]]
495; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <8 x double> [[TMP16]], <8 x double> undef, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
496; CHECK-NEXT:    [[A_INS_72:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP17]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
497; CHECK-NEXT:    [[A_INS_8:%.*]] = insertelement <9 x double> [[A_INS_72]], double [[A_LANE_8]], i32 8
498; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_6]], i32 0
499; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <8 x double> [[TMP18]], double [[V1_LANE_7]], i32 1
500; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <8 x double> [[TMP19]], double [[V1_LANE_8]], i32 2
501; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <8 x double> [[TMP20]], double [[V1_LANE_0]], i32 3
502; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <8 x double> [[TMP21]], double [[V1_LANE_1]], i32 4
503; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <8 x double> [[TMP22]], double [[V1_LANE_2]], i32 5
504; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <8 x double> [[TMP23]], double [[V1_LANE_3]], i32 6
505; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <8 x double> [[TMP24]], double [[V1_LANE_4]], i32 7
506; CHECK-NEXT:    [[TMP26:%.*]] = fmul <8 x double> [[TMP25]], [[TMP15]]
507; CHECK-NEXT:    [[B_LANE_8:%.*]] = fmul double [[V1_LANE_5]], [[V2_LANE_0]]
508; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <8 x double> [[TMP26]], <8 x double> undef, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
509; CHECK-NEXT:    [[B_INS_71:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP27]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
510; CHECK-NEXT:    [[B_INS_8:%.*]] = insertelement <9 x double> [[B_INS_71]], double [[B_LANE_8]], i32 8
511; CHECK-NEXT:    [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]
512; CHECK-NEXT:    store <9 x double> [[RES]], <9 x double>* [[PTR_1]], align 8
513; CHECK-NEXT:    ret void
514;
515bb:
516  %v.1 = load <9 x double>, <9 x double>* %ptr.1, align 8
517  %v1.lane.0 = extractelement <9 x double> %v.1, i32 0
518  %v1.lane.1 = extractelement <9 x double> %v.1, i32 1
519  %v1.lane.2 = extractelement <9 x double> %v.1, i32 2
520  %v1.lane.3 = extractelement <9 x double> %v.1, i32 3
521  %v1.lane.4 = extractelement <9 x double> %v.1, i32 4
522  %v1.lane.5 = extractelement <9 x double> %v.1, i32 5
523  %v1.lane.6 = extractelement <9 x double> %v.1, i32 6
524  %v1.lane.7 = extractelement <9 x double> %v.1, i32 7
525  %v1.lane.8 = extractelement <9 x double> %v.1, i32 8
526
527  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
528  %v2.lane.0 = extractelement <4 x double> %v.2, i32 0
529  %v2.lane.1 = extractelement <4 x double> %v.2, i32 1
530  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
531
532  %a.lane.0 = fmul double %v1.lane.4, %v2.lane.1
533  %a.lane.1 = fmul double %v1.lane.3, %v2.lane.0
534  %a.lane.2 = fmul double %v1.lane.6, %v2.lane.2
535  %a.lane.3 = fmul double %v1.lane.5, %v2.lane.0
536  %a.lane.4 = fmul double %v1.lane.8, %v2.lane.2
537  %a.lane.5 = fmul double %v1.lane.7, %v2.lane.1
538  %a.lane.6 = fmul double %v1.lane.1, %v2.lane.0
539  %a.lane.7 = fmul double %v1.lane.0, %v2.lane.2
540  %a.lane.8 = fmul double %v1.lane.2, %v2.lane.1
541
542  %a.ins.0 = insertelement <9 x double> undef, double %a.lane.0, i32 0
543  %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1
544  %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2
545  %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3
546  %a.ins.4 = insertelement <9 x double> %a.ins.3, double %a.lane.4, i32 4
547  %a.ins.5 = insertelement <9 x double> %a.ins.4, double %a.lane.5, i32 5
548  %a.ins.6 = insertelement <9 x double> %a.ins.5, double %a.lane.6, i32 6
549  %a.ins.7 = insertelement <9 x double> %a.ins.6, double %a.lane.7, i32 7
550  %a.ins.8 = insertelement <9 x double> %a.ins.7, double %a.lane.8, i32 8
551
552  %b.lane.0 = fmul double %v1.lane.6, %v2.lane.1
553  %b.lane.1 = fmul double %v1.lane.7, %v2.lane.0
554  %b.lane.2 = fmul double %v1.lane.8, %v2.lane.2
555  %b.lane.3 = fmul double %v1.lane.0, %v2.lane.0
556  %b.lane.4 = fmul double %v1.lane.1, %v2.lane.2
557  %b.lane.5 = fmul double %v1.lane.2, %v2.lane.1
558  %b.lane.6 = fmul double %v1.lane.3, %v2.lane.0
559  %b.lane.7 = fmul double %v1.lane.4, %v2.lane.2
560  %b.lane.8 = fmul double %v1.lane.5, %v2.lane.0
561
562  %b.ins.0 = insertelement <9 x double> undef, double %b.lane.0, i32 0
563  %b.ins.1 = insertelement <9 x double> %b.ins.0, double %b.lane.1, i32 1
564  %b.ins.2 = insertelement <9 x double> %b.ins.1, double %b.lane.2, i32 2
565  %b.ins.3 = insertelement <9 x double> %b.ins.2, double %b.lane.3, i32 3
566  %b.ins.4 = insertelement <9 x double> %b.ins.3, double %b.lane.4, i32 4
567  %b.ins.5 = insertelement <9 x double> %b.ins.4, double %b.lane.5, i32 5
568  %b.ins.6 = insertelement <9 x double> %b.ins.5, double %b.lane.6, i32 6
569  %b.ins.7 = insertelement <9 x double> %b.ins.6, double %b.lane.7, i32 7
570  %b.ins.8 = insertelement <9 x double> %b.ins.7, double %b.lane.8, i32 8
571
572  %res = fsub <9 x double> %a.ins.8, %b.ins.8
573  store <9 x double> %res, <9 x double>* %ptr.1, align 8
574  ret void
575}
576
577; Extracted lanes used in both fmul chain are not used in the right order, so
578; we cannot reuse the source vector registers directly.
579define void @first_and_second_mul_chain_jumbled(<9 x double>* %ptr.1, <4 x double>* %ptr.2) {
580; CHECK-LABEL: @first_and_second_mul_chain_jumbled(
581; CHECK-NEXT:  bb:
582; CHECK-NEXT:    [[V_1:%.*]] = load <9 x double>, <9 x double>* [[PTR_1:%.*]], align 8
583; CHECK-NEXT:    [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0
584; CHECK-NEXT:    [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1
585; CHECK-NEXT:    [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
586; CHECK-NEXT:    [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3
587; CHECK-NEXT:    [[V1_LANE_4:%.*]] = extractelement <9 x double> [[V_1]], i32 4
588; CHECK-NEXT:    [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5
589; CHECK-NEXT:    [[V1_LANE_6:%.*]] = extractelement <9 x double> [[V_1]], i32 6
590; CHECK-NEXT:    [[V1_LANE_7:%.*]] = extractelement <9 x double> [[V_1]], i32 7
591; CHECK-NEXT:    [[V1_LANE_8:%.*]] = extractelement <9 x double> [[V_1]], i32 8
592; CHECK-NEXT:    [[V_2:%.*]] = load <4 x double>, <4 x double>* [[PTR_2:%.*]], align 16
593; CHECK-NEXT:    [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
594; CHECK-NEXT:    [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
595; CHECK-NEXT:    [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2
596; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_4]], i32 0
597; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x double> [[TMP0]], double [[V1_LANE_3]], i32 1
598; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x double> [[TMP1]], double [[V1_LANE_5]], i32 2
599; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x double> [[TMP2]], double [[V1_LANE_6]], i32 3
600; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x double> [[TMP3]], double [[V1_LANE_8]], i32 4
601; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x double> [[TMP4]], double [[V1_LANE_7]], i32 5
602; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x double> [[TMP5]], double [[V1_LANE_1]], i32 6
603; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x double> [[TMP6]], double [[V1_LANE_0]], i32 7
604; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x double> poison, double [[V2_LANE_0]], i32 0
605; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x double> [[TMP8]], double [[V2_LANE_2]], i32 1
606; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x double> [[TMP9]], double [[V2_LANE_1]], i32 2
607; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x double> [[TMP10]], double [[V2_LANE_2]], i32 3
608; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <8 x double> [[TMP11]], double [[V2_LANE_1]], i32 4
609; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x double> [[TMP12]], double [[V2_LANE_0]], i32 5
610; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x double> [[TMP13]], double [[V2_LANE_2]], i32 6
611; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x double> [[TMP14]], double [[V2_LANE_1]], i32 7
612; CHECK-NEXT:    [[TMP16:%.*]] = fmul <8 x double> [[TMP7]], [[TMP15]]
613; CHECK-NEXT:    [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_0]]
614; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <8 x double> [[TMP16]], <8 x double> undef, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
615; CHECK-NEXT:    [[A_INS_72:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP17]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
616; CHECK-NEXT:    [[A_INS_8:%.*]] = insertelement <9 x double> [[A_INS_72]], double [[A_LANE_8]], i32 8
617; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <8 x double> poison, double [[V1_LANE_7]], i32 0
618; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <8 x double> [[TMP18]], double [[V1_LANE_6]], i32 1
619; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <8 x double> [[TMP19]], double [[V1_LANE_8]], i32 2
620; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <8 x double> [[TMP20]], double [[V1_LANE_1]], i32 3
621; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <8 x double> [[TMP21]], double [[V1_LANE_0]], i32 4
622; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <8 x double> [[TMP22]], double [[V1_LANE_3]], i32 5
623; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <8 x double> [[TMP23]], double [[V1_LANE_2]], i32 6
624; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <8 x double> [[TMP24]], double [[V1_LANE_5]], i32 7
625; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <8 x double> poison, double [[V2_LANE_2]], i32 0
626; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <8 x double> [[TMP26]], double [[V2_LANE_1]], i32 1
627; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <8 x double> [[TMP27]], double [[V2_LANE_0]], i32 2
628; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <8 x double> [[TMP28]], double [[V2_LANE_2]], i32 3
629; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <8 x double> [[TMP29]], double [[V2_LANE_0]], i32 4
630; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <8 x double> [[TMP30]], double [[V2_LANE_2]], i32 5
631; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <8 x double> [[TMP31]], double [[V2_LANE_1]], i32 6
632; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <8 x double> [[TMP32]], double [[V2_LANE_0]], i32 7
633; CHECK-NEXT:    [[TMP34:%.*]] = fmul <8 x double> [[TMP25]], [[TMP33]]
634; CHECK-NEXT:    [[B_LANE_8:%.*]] = fmul double [[V1_LANE_4]], [[V2_LANE_2]]
635; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <8 x double> [[TMP34]], <8 x double> undef, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
636; CHECK-NEXT:    [[B_INS_71:%.*]] = shufflevector <9 x double> undef, <9 x double> [[TMP35]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
637; CHECK-NEXT:    [[B_INS_8:%.*]] = insertelement <9 x double> [[B_INS_71]], double [[B_LANE_8]], i32 8
638; CHECK-NEXT:    [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]
639; CHECK-NEXT:    store <9 x double> [[RES]], <9 x double>* [[PTR_1]], align 8
640; CHECK-NEXT:    ret void
641;
642bb:
643  %v.1 = load <9 x double>, <9 x double>* %ptr.1, align 8
644  %v1.lane.0 = extractelement <9 x double> %v.1, i32 0
645  %v1.lane.1 = extractelement <9 x double> %v.1, i32 1
646  %v1.lane.2 = extractelement <9 x double> %v.1, i32 2
647  %v1.lane.3 = extractelement <9 x double> %v.1, i32 3
648  %v1.lane.4 = extractelement <9 x double> %v.1, i32 4
649  %v1.lane.5 = extractelement <9 x double> %v.1, i32 5
650  %v1.lane.6 = extractelement <9 x double> %v.1, i32 6
651  %v1.lane.7 = extractelement <9 x double> %v.1, i32 7
652  %v1.lane.8 = extractelement <9 x double> %v.1, i32 8
653
654  %v.2 = load <4 x double>, <4 x double>* %ptr.2, align 16
655  %v2.lane.0 = extractelement <4 x double> %v.2, i32 0
656  %v2.lane.1 = extractelement <4 x double> %v.2, i32 1
657  %v2.lane.2 = extractelement <4 x double> %v.2, i32 2
658
659  %a.lane.0 = fmul double %v1.lane.4, %v2.lane.0
660  %a.lane.1 = fmul double %v1.lane.3, %v2.lane.2
661  %a.lane.2 = fmul double %v1.lane.5, %v2.lane.1
662  %a.lane.3 = fmul double %v1.lane.6, %v2.lane.2
663  %a.lane.4 = fmul double %v1.lane.8, %v2.lane.1
664  %a.lane.5 = fmul double %v1.lane.7, %v2.lane.0
665  %a.lane.6 = fmul double %v1.lane.1, %v2.lane.2
666  %a.lane.7 = fmul double %v1.lane.0, %v2.lane.1
667  %a.lane.8 = fmul double %v1.lane.2, %v2.lane.0
668
669  %a.ins.0 = insertelement <9 x double> undef, double %a.lane.0, i32 0
670  %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1
671  %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2
672  %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3
673  %a.ins.4 = insertelement <9 x double> %a.ins.3, double %a.lane.4, i32 4
674  %a.ins.5 = insertelement <9 x double> %a.ins.4, double %a.lane.5, i32 5
675  %a.ins.6 = insertelement <9 x double> %a.ins.5, double %a.lane.6, i32 6
676  %a.ins.7 = insertelement <9 x double> %a.ins.6, double %a.lane.7, i32 7
677  %a.ins.8 = insertelement <9 x double> %a.ins.7, double %a.lane.8, i32 8
678
679  %b.lane.0 = fmul double %v1.lane.7, %v2.lane.2
680  %b.lane.1 = fmul double %v1.lane.6, %v2.lane.1
681  %b.lane.2 = fmul double %v1.lane.8, %v2.lane.0
682  %b.lane.3 = fmul double %v1.lane.1, %v2.lane.2
683  %b.lane.4 = fmul double %v1.lane.0, %v2.lane.0
684  %b.lane.5 = fmul double %v1.lane.3, %v2.lane.2
685  %b.lane.6 = fmul double %v1.lane.2, %v2.lane.1
686  %b.lane.7 = fmul double %v1.lane.5, %v2.lane.0
687  %b.lane.8 = fmul double %v1.lane.4, %v2.lane.2
688
689  %b.ins.0 = insertelement <9 x double> undef, double %b.lane.0, i32 0
690  %b.ins.1 = insertelement <9 x double> %b.ins.0, double %b.lane.1, i32 1
691  %b.ins.2 = insertelement <9 x double> %b.ins.1, double %b.lane.2, i32 2
692  %b.ins.3 = insertelement <9 x double> %b.ins.2, double %b.lane.3, i32 3
693  %b.ins.4 = insertelement <9 x double> %b.ins.3, double %b.lane.4, i32 4
694  %b.ins.5 = insertelement <9 x double> %b.ins.4, double %b.lane.5, i32 5
695  %b.ins.6 = insertelement <9 x double> %b.ins.5, double %b.lane.6, i32 6
696  %b.ins.7 = insertelement <9 x double> %b.ins.6, double %b.lane.7, i32 7
697  %b.ins.8 = insertelement <9 x double> %b.ins.7, double %b.lane.8, i32 8
698
699  %res = fsub <9 x double> %a.ins.8, %b.ins.8
700  store <9 x double> %res, <9 x double>* %ptr.1, align 8
701  ret void
702}
703