1; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
2
3@var8 = global i8 0
4@var16 = global i16 0
5@var32 = global i32 0
6@var64 = global i64 0
7
8define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
9; CHECK: test_atomic_load_add_i8:
10   %old = atomicrmw add i8* @var8, i8 %offset seq_cst
11; CHECK-NOT: dmb
12; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
13; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
14
15; CHECK: .LBB{{[0-9]+}}_1:
16; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
17  ; w0 below is a reasonable guess but could change: it certainly comes into the
18  ;  function there.
19; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
20; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
21; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
22; CHECK-NOT: dmb
23
24; CHECK: mov x0, x[[OLD]]
25   ret i8 %old
26}
27
28define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
29; CHECK: test_atomic_load_add_i16:
30   %old = atomicrmw add i16* @var16, i16 %offset acquire
31; CHECK-NOT: dmb
32; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
33; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
34
35; CHECK: .LBB{{[0-9]+}}_1:
36; CHECK-NEXT: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
37  ; w0 below is a reasonable guess but could change: it certainly comes into the
38  ;  function there.
39; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
40; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
41; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
42; CHECK-NOT: dmb
43
44; CHECK: mov x0, x[[OLD]]
45   ret i16 %old
46}
47
48define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
49; CHECK: test_atomic_load_add_i32:
50   %old = atomicrmw add i32* @var32, i32 %offset release
51; CHECK-NOT: dmb
52; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
53; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
54
55; CHECK: .LBB{{[0-9]+}}_1:
56; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
57  ; w0 below is a reasonable guess but could change: it certainly comes into the
58  ;  function there.
59; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
60; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
61; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
62; CHECK-NOT: dmb
63
64; CHECK: mov x0, x[[OLD]]
65   ret i32 %old
66}
67
68define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
69; CHECK: test_atomic_load_add_i64:
70   %old = atomicrmw add i64* @var64, i64 %offset monotonic
71; CHECK-NOT: dmb
72; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
73; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
74
75; CHECK: .LBB{{[0-9]+}}_1:
76; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
77  ; x0 below is a reasonable guess but could change: it certainly comes into the
78  ; function there.
79; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0
80; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
81; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
82; CHECK-NOT: dmb
83
84; CHECK: mov x0, x[[OLD]]
85   ret i64 %old
86}
87
88define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
89; CHECK: test_atomic_load_sub_i8:
90   %old = atomicrmw sub i8* @var8, i8 %offset monotonic
91; CHECK-NOT: dmb
92; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
93; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
94
95; CHECK: .LBB{{[0-9]+}}_1:
96; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
97  ; w0 below is a reasonable guess but could change: it certainly comes into the
98  ;  function there.
99; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
100; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
101; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
102; CHECK-NOT: dmb
103
104; CHECK: mov x0, x[[OLD]]
105   ret i8 %old
106}
107
108define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
109; CHECK: test_atomic_load_sub_i16:
110   %old = atomicrmw sub i16* @var16, i16 %offset release
111; CHECK-NOT: dmb
112; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
113; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
114
115; CHECK: .LBB{{[0-9]+}}_1:
116; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
117  ; w0 below is a reasonable guess but could change: it certainly comes into the
118  ;  function there.
119; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
120; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
121; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
122; CHECK-NOT: dmb
123
124; CHECK: mov x0, x[[OLD]]
125   ret i16 %old
126}
127
128define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
129; CHECK: test_atomic_load_sub_i32:
130   %old = atomicrmw sub i32* @var32, i32 %offset acquire
131; CHECK-NOT: dmb
132; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
133; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
134
135; CHECK: .LBB{{[0-9]+}}_1:
136; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
137  ; w0 below is a reasonable guess but could change: it certainly comes into the
138  ;  function there.
139; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
140; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
141; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
142; CHECK-NOT: dmb
143
144; CHECK: mov x0, x[[OLD]]
145   ret i32 %old
146}
147
148define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
149; CHECK: test_atomic_load_sub_i64:
150   %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
151; CHECK-NOT: dmb
152; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
153; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
154
155; CHECK: .LBB{{[0-9]+}}_1:
156; CHECK-NEXT: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
157  ; x0 below is a reasonable guess but could change: it certainly comes into the
158  ; function there.
159; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0
160; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
161; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
162; CHECK-NOT: dmb
163
164; CHECK: mov x0, x[[OLD]]
165   ret i64 %old
166}
167
168define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
169; CHECK: test_atomic_load_and_i8:
170   %old = atomicrmw and i8* @var8, i8 %offset release
171; CHECK-NOT: dmb
172; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
173; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
174
175; CHECK: .LBB{{[0-9]+}}_1:
176; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
177  ; w0 below is a reasonable guess but could change: it certainly comes into the
178  ;  function there.
179; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
180; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
181; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
182; CHECK-NOT: dmb
183
184; CHECK: mov x0, x[[OLD]]
185   ret i8 %old
186}
187
188define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
189; CHECK: test_atomic_load_and_i16:
190   %old = atomicrmw and i16* @var16, i16 %offset monotonic
191; CHECK-NOT: dmb
192; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
193; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
194
195; CHECK: .LBB{{[0-9]+}}_1:
196; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
197  ; w0 below is a reasonable guess but could change: it certainly comes into the
198  ;  function there.
199; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
200; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
201; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
202; CHECK-NOT: dmb
203
204; CHECK: mov x0, x[[OLD]]
205   ret i16 %old
206}
207
208define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
209; CHECK: test_atomic_load_and_i32:
210   %old = atomicrmw and i32* @var32, i32 %offset seq_cst
211; CHECK-NOT: dmb
212; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
213; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
214
215; CHECK: .LBB{{[0-9]+}}_1:
216; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
217  ; w0 below is a reasonable guess but could change: it certainly comes into the
218  ;  function there.
219; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
220; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
221; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
222; CHECK-NOT: dmb
223
224; CHECK: mov x0, x[[OLD]]
225   ret i32 %old
226}
227
228define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
229; CHECK: test_atomic_load_and_i64:
230   %old = atomicrmw and i64* @var64, i64 %offset acquire
231; CHECK-NOT: dmb
232; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
233; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
234
235; CHECK: .LBB{{[0-9]+}}_1:
236; CHECK-NEXT: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
237  ; x0 below is a reasonable guess but could change: it certainly comes into the
238  ; function there.
239; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0
240; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
241; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
242; CHECK-NOT: dmb
243
244; CHECK: mov x0, x[[OLD]]
245   ret i64 %old
246}
247
248define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
249; CHECK: test_atomic_load_or_i8:
250   %old = atomicrmw or i8* @var8, i8 %offset seq_cst
251; CHECK-NOT: dmb
252; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
253; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
254
255; CHECK: .LBB{{[0-9]+}}_1:
256; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
257  ; w0 below is a reasonable guess but could change: it certainly comes into the
258  ;  function there.
259; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
260; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
261; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
262; CHECK-NOT: dmb
263
264; CHECK: mov x0, x[[OLD]]
265   ret i8 %old
266}
267
268define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
269; CHECK: test_atomic_load_or_i16:
270   %old = atomicrmw or i16* @var16, i16 %offset monotonic
271; CHECK-NOT: dmb
272; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
273; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
274
275; CHECK: .LBB{{[0-9]+}}_1:
276; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
277  ; w0 below is a reasonable guess but could change: it certainly comes into the
278  ;  function there.
279; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
280; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
281; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
282; CHECK-NOT: dmb
283
284; CHECK: mov x0, x[[OLD]]
285   ret i16 %old
286}
287
288define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
289; CHECK: test_atomic_load_or_i32:
290   %old = atomicrmw or i32* @var32, i32 %offset acquire
291; CHECK-NOT: dmb
292; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
293; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
294
295; CHECK: .LBB{{[0-9]+}}_1:
296; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
297  ; w0 below is a reasonable guess but could change: it certainly comes into the
298  ;  function there.
299; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
300; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
301; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
302; CHECK-NOT: dmb
303
304; CHECK: mov x0, x[[OLD]]
305   ret i32 %old
306}
307
308define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
309; CHECK: test_atomic_load_or_i64:
310   %old = atomicrmw or i64* @var64, i64 %offset release
311; CHECK-NOT: dmb
312; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
313; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
314
315; CHECK: .LBB{{[0-9]+}}_1:
316; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
317  ; x0 below is a reasonable guess but could change: it certainly comes into the
318  ; function there.
319; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0
320; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
321; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
322; CHECK-NOT: dmb
323
324; CHECK: mov x0, x[[OLD]]
325   ret i64 %old
326}
327
328define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
329; CHECK: test_atomic_load_xor_i8:
330   %old = atomicrmw xor i8* @var8, i8 %offset acquire
331; CHECK-NOT: dmb
332; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
333; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
334
335; CHECK: .LBB{{[0-9]+}}_1:
336; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
337  ; w0 below is a reasonable guess but could change: it certainly comes into the
338  ;  function there.
339; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
340; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
341; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
342; CHECK-NOT: dmb
343
344; CHECK: mov x0, x[[OLD]]
345   ret i8 %old
346}
347
348define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
349; CHECK: test_atomic_load_xor_i16:
350   %old = atomicrmw xor i16* @var16, i16 %offset release
351; CHECK-NOT: dmb
352; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
353; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
354
355; CHECK: .LBB{{[0-9]+}}_1:
356; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
357  ; w0 below is a reasonable guess but could change: it certainly comes into the
358  ;  function there.
359; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
360; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
361; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
362; CHECK-NOT: dmb
363
364; CHECK: mov x0, x[[OLD]]
365   ret i16 %old
366}
367
368define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
369; CHECK: test_atomic_load_xor_i32:
370   %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
371; CHECK-NOT: dmb
372; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
373; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
374
375; CHECK: .LBB{{[0-9]+}}_1:
376; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
377  ; w0 below is a reasonable guess but could change: it certainly comes into the
378  ;  function there.
379; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
380; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
381; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
382; CHECK-NOT: dmb
383
384; CHECK: mov x0, x[[OLD]]
385   ret i32 %old
386}
387
388define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
389; CHECK: test_atomic_load_xor_i64:
390   %old = atomicrmw xor i64* @var64, i64 %offset monotonic
391; CHECK-NOT: dmb
392; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
393; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
394
395; CHECK: .LBB{{[0-9]+}}_1:
396; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
397  ; x0 below is a reasonable guess but could change: it certainly comes into the
398  ; function there.
399; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0
400; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
401; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
402; CHECK-NOT: dmb
403
404; CHECK: mov x0, x[[OLD]]
405   ret i64 %old
406}
407
408define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
409; CHECK: test_atomic_load_xchg_i8:
410   %old = atomicrmw xchg i8* @var8, i8 %offset monotonic
411; CHECK-NOT: dmb
412; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
413; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
414
415; CHECK: .LBB{{[0-9]+}}_1:
416; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
417  ; w0 below is a reasonable guess but could change: it certainly comes into the
418  ; function there.
419; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
420; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
421; CHECK-NOT: dmb
422
423; CHECK: mov x0, x[[OLD]]
424   ret i8 %old
425}
426
427define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
428; CHECK: test_atomic_load_xchg_i16:
429   %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
430; CHECK-NOT: dmb
431; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
432; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
433
434; CHECK: .LBB{{[0-9]+}}_1:
435; CHECK-NEXT: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
436  ; w0 below is a reasonable guess but could change: it certainly comes into the
437  ; function there.
438; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
439; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
440; CHECK-NOT: dmb
441
442; CHECK: mov x0, x[[OLD]]
443   ret i16 %old
444}
445
446define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
447; CHECK: test_atomic_load_xchg_i32:
448   %old = atomicrmw xchg i32* @var32, i32 %offset release
449; CHECK-NOT: dmb
450; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
451; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
452
453; CHECK: .LBB{{[0-9]+}}_1:
454; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
455  ; w0 below is a reasonable guess but could change: it certainly comes into the
456  ;  function there.
457; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
458; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
459; CHECK-NOT: dmb
460
461; CHECK: mov x0, x[[OLD]]
462   ret i32 %old
463}
464
465define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
466; CHECK: test_atomic_load_xchg_i64:
467   %old = atomicrmw xchg i64* @var64, i64 %offset acquire
468; CHECK-NOT: dmb
469; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
470; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
471
472; CHECK: .LBB{{[0-9]+}}_1:
473; CHECK-NEXT: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
474  ; x0 below is a reasonable guess but could change: it certainly comes into the
475  ; function there.
476; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]]
477; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
478; CHECK-NOT: dmb
479
480; CHECK: mov x0, x[[OLD]]
481   ret i64 %old
482}
483
484
485define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
486; CHECK: test_atomic_load_min_i8:
487   %old = atomicrmw min i8* @var8, i8 %offset acquire
488; CHECK-NOT: dmb
489; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
490; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
491
492; CHECK: .LBB{{[0-9]+}}_1:
493; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
494  ; w0 below is a reasonable guess but could change: it certainly comes into the
495  ;  function there.
496; CHECK-NEXT: cmp w0, w[[OLD]], sxtb
497; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
498; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
499; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
500; CHECK-NOT: dmb
501
502; CHECK: mov x0, x[[OLD]]
503   ret i8 %old
504}
505
506define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
507; CHECK: test_atomic_load_min_i16:
508   %old = atomicrmw min i16* @var16, i16 %offset release
509; CHECK-NOT: dmb
510; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
511; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
512
513; CHECK: .LBB{{[0-9]+}}_1:
514; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
515  ; w0 below is a reasonable guess but could change: it certainly comes into the
516  ;  function there.
517; CHECK-NEXT: cmp w0, w[[OLD]], sxth
518; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
519; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
520; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
521; CHECK-NOT: dmb
522
523; CHECK: mov x0, x[[OLD]]
524   ret i16 %old
525}
526
527define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
528; CHECK: test_atomic_load_min_i32:
529   %old = atomicrmw min i32* @var32, i32 %offset monotonic
530; CHECK-NOT: dmb
531; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
532; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
533
534; CHECK: .LBB{{[0-9]+}}_1:
535; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
536  ; w0 below is a reasonable guess but could change: it certainly comes into the
537  ;  function there.
538; CHECK-NEXT: cmp w0, w[[OLD]]
539; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
540; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
541; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
542; CHECK-NOT: dmb
543
544; CHECK: mov x0, x[[OLD]]
545   ret i32 %old
546}
547
548define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
549; CHECK: test_atomic_load_min_i64:
550   %old = atomicrmw min i64* @var64, i64 %offset seq_cst
551; CHECK-NOT: dmb
552; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
553; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
554
555; CHECK: .LBB{{[0-9]+}}_1:
556; CHECK-NEXT: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
557  ; x0 below is a reasonable guess but could change: it certainly comes into the
558  ; function there.
559; CHECK-NEXT: cmp x0, x[[OLD]]
560; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt
561; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
562; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
563; CHECK-NOT: dmb
564
565; CHECK: mov x0, x[[OLD]]
566   ret i64 %old
567}
568
569define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
570; CHECK: test_atomic_load_max_i8:
571   %old = atomicrmw max i8* @var8, i8 %offset seq_cst
572; CHECK-NOT: dmb
573; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
574; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
575
576; CHECK: .LBB{{[0-9]+}}_1:
577; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
578  ; w0 below is a reasonable guess but could change: it certainly comes into the
579  ;  function there.
580; CHECK-NEXT: cmp w0, w[[OLD]], sxtb
581; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
582; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
583; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
584; CHECK-NOT: dmb
585
586; CHECK: mov x0, x[[OLD]]
587   ret i8 %old
588}
589
590define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
591; CHECK: test_atomic_load_max_i16:
592   %old = atomicrmw max i16* @var16, i16 %offset acquire
593; CHECK-NOT: dmb
594; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
595; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
596
597; CHECK: .LBB{{[0-9]+}}_1:
598; CHECK-NEXT: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
599  ; w0 below is a reasonable guess but could change: it certainly comes into the
600  ;  function there.
601; CHECK-NEXT: cmp w0, w[[OLD]], sxth
602; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
603; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
604; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
605; CHECK-NOT: dmb
606
607; CHECK: mov x0, x[[OLD]]
608   ret i16 %old
609}
610
611define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
612; CHECK: test_atomic_load_max_i32:
613   %old = atomicrmw max i32* @var32, i32 %offset release
614; CHECK-NOT: dmb
615; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
616; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
617
618; CHECK: .LBB{{[0-9]+}}_1:
619; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
620  ; w0 below is a reasonable guess but could change: it certainly comes into the
621  ;  function there.
622; CHECK-NEXT: cmp w0, w[[OLD]]
623; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
624; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
625; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
626; CHECK-NOT: dmb
627
628; CHECK: mov x0, x[[OLD]]
629   ret i32 %old
630}
631
632define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
633; CHECK: test_atomic_load_max_i64:
634   %old = atomicrmw max i64* @var64, i64 %offset monotonic
635; CHECK-NOT: dmb
636; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
637; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
638
639; CHECK: .LBB{{[0-9]+}}_1:
640; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
641  ; x0 below is a reasonable guess but could change: it certainly comes into the
642  ; function there.
643; CHECK-NEXT: cmp x0, x[[OLD]]
644; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, lt
645; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
646; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
647; CHECK-NOT: dmb
648
649; CHECK: mov x0, x[[OLD]]
650   ret i64 %old
651}
652
653define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
654; CHECK: test_atomic_load_umin_i8:
655   %old = atomicrmw umin i8* @var8, i8 %offset monotonic
656; CHECK-NOT: dmb
657; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
658; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
659
660; CHECK: .LBB{{[0-9]+}}_1:
661; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
662  ; w0 below is a reasonable guess but could change: it certainly comes into the
663  ;  function there.
664; CHECK-NEXT: cmp w0, w[[OLD]], uxtb
665; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
666; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
667; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
668; CHECK-NOT: dmb
669
670; CHECK: mov x0, x[[OLD]]
671   ret i8 %old
672}
673
674define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
675; CHECK: test_atomic_load_umin_i16:
676   %old = atomicrmw umin i16* @var16, i16 %offset acquire
677; CHECK-NOT: dmb
678; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
679; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
680
681; CHECK: .LBB{{[0-9]+}}_1:
682; CHECK-NEXT: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
683  ; w0 below is a reasonable guess but could change: it certainly comes into the
684  ;  function there.
685; CHECK-NEXT: cmp w0, w[[OLD]], uxth
686; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
687; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
688; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
689; CHECK-NOT: dmb
690
691; CHECK: mov x0, x[[OLD]]
692   ret i16 %old
693}
694
695define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
696; CHECK: test_atomic_load_umin_i32:
697   %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
698; CHECK-NOT: dmb
699; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
700; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
701
702; CHECK: .LBB{{[0-9]+}}_1:
703; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
704  ; w0 below is a reasonable guess but could change: it certainly comes into the
705  ;  function there.
706; CHECK-NEXT: cmp w0, w[[OLD]]
707; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
708; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
709; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
710; CHECK-NOT: dmb
711
712; CHECK: mov x0, x[[OLD]]
713   ret i32 %old
714}
715
716define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
717; CHECK: test_atomic_load_umin_i64:
718   %old = atomicrmw umin i64* @var64, i64 %offset acq_rel
719; CHECK-NOT: dmb
720; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
721; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
722
723; CHECK: .LBB{{[0-9]+}}_1:
724; CHECK-NEXT: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
725  ; x0 below is a reasonable guess but could change: it certainly comes into the
726  ; function there.
727; CHECK-NEXT: cmp x0, x[[OLD]]
728; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi
729; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
730; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
731; CHECK-NOT: dmb
732
733; CHECK: mov x0, x[[OLD]]
734   ret i64 %old
735}
736
737define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
738; CHECK: test_atomic_load_umax_i8:
739   %old = atomicrmw umax i8* @var8, i8 %offset acq_rel
740; CHECK-NOT: dmb
741; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
742; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
743
744; CHECK: .LBB{{[0-9]+}}_1:
745; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
746  ; w0 below is a reasonable guess but could change: it certainly comes into the
747  ;  function there.
748; CHECK-NEXT: cmp w0, w[[OLD]], uxtb
749; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
750; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
751; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
752; CHECK-NOT: dmb
753
754; CHECK: mov x0, x[[OLD]]
755   ret i8 %old
756}
757
758define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
759; CHECK: test_atomic_load_umax_i16:
760   %old = atomicrmw umax i16* @var16, i16 %offset monotonic
761; CHECK-NOT: dmb
762; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
763; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
764
765; CHECK: .LBB{{[0-9]+}}_1:
766; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
767  ; w0 below is a reasonable guess but could change: it certainly comes into the
768  ;  function there.
769; CHECK-NEXT: cmp w0, w[[OLD]], uxth
770; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
771; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
772; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
773; CHECK-NOT: dmb
774
775; CHECK: mov x0, x[[OLD]]
776   ret i16 %old
777}
778
779define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
780; CHECK: test_atomic_load_umax_i32:
781   %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
782; CHECK-NOT: dmb
783; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
784; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
785
786; CHECK: .LBB{{[0-9]+}}_1:
787; CHECK-NEXT: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
788  ; w0 below is a reasonable guess but could change: it certainly comes into the
789  ;  function there.
790; CHECK-NEXT: cmp w0, w[[OLD]]
791; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
792; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
793; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
794; CHECK-NOT: dmb
795
796; CHECK: mov x0, x[[OLD]]
797   ret i32 %old
798}
799
800define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
801; CHECK: test_atomic_load_umax_i64:
802   %old = atomicrmw umax i64* @var64, i64 %offset release
803; CHECK-NOT: dmb
804; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
805; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
806
807; CHECK: .LBB{{[0-9]+}}_1:
808; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
809  ; x0 below is a reasonable guess but could change: it certainly comes into the
810  ; function there.
811; CHECK-NEXT: cmp x0, x[[OLD]]
812; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, lo
813; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
814; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
815; CHECK-NOT: dmb
816
817; CHECK: mov x0, x[[OLD]]
818   ret i64 %old
819}
820
821define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
822; CHECK: test_atomic_cmpxchg_i8:
823   %old = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire
824; CHECK-NOT: dmb
825; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
826; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
827
828; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
829; CHECK-NEXT: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
830  ; w0 below is a reasonable guess but could change: it certainly comes into the
831  ;  function there.
832; CHECK-NEXT: cmp w[[OLD]], w0
833; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
834  ; As above, w1 is a reasonable guess.
835; CHECK: stxrb [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
836; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
837; CHECK-NOT: dmb
838
839; CHECK: mov x0, x[[OLD]]
840   ret i8 %old
841}
842
843define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
844; CHECK: test_atomic_cmpxchg_i16:
845   %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst
846; CHECK-NOT: dmb
847; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
848; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
849
850; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
851; CHECK-NEXT: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
852  ; w0 below is a reasonable guess but could change: it certainly comes into the
853  ;  function there.
854; CHECK-NEXT: cmp w[[OLD]], w0
855; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
856  ; As above, w1 is a reasonable guess.
857; CHECK: stlxrh [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
858; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
859; CHECK-NOT: dmb
860
861; CHECK: mov x0, x[[OLD]]
862   ret i16 %old
863}
864
865define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
866; CHECK: test_atomic_cmpxchg_i32:
867   %old = cmpxchg i32* @var32, i32 %wanted, i32 %new release
868; CHECK-NOT: dmb
869; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
870; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
871
872; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
873; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
874  ; w0 below is a reasonable guess but could change: it certainly comes into the
875  ;  function there.
876; CHECK-NEXT: cmp w[[OLD]], w0
877; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
878  ; As above, w1 is a reasonable guess.
879; CHECK: stlxr [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
880; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
881; CHECK-NOT: dmb
882
883; CHECK: mov x0, x[[OLD]]
884   ret i32 %old
885}
886
887define i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
888; CHECK: test_atomic_cmpxchg_i64:
889   %old = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic
890; CHECK-NOT: dmb
891; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
892; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
893
894; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
895; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
896  ; w0 below is a reasonable guess but could change: it certainly comes into the
897  ;  function there.
898; CHECK-NEXT: cmp x[[OLD]], x0
899; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
900  ; As above, w1 is a reasonable guess.
901; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]]
902; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
903; CHECK-NOT: dmb
904
905; CHECK: mov x0, x[[OLD]]
906   ret i64 %old
907}
908
909define i8 @test_atomic_load_monotonic_i8() nounwind {
910; CHECK: test_atomic_load_monotonic_i8:
911  %val = load atomic i8* @var8 monotonic, align 1
912; CHECK-NOT: dmb
913; CHECK: adrp x[[HIADDR:[0-9]+]], var8
914; CHECK: ldrb w0, [x[[HIADDR]], #:lo12:var8]
915; CHECK-NOT: dmb
916
917  ret i8 %val
918}
919
920define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
921; CHECK: test_atomic_load_monotonic_regoff_i8:
922  %addr_int = add i64 %base, %off
923  %addr = inttoptr i64 %addr_int to i8*
924
925  %val = load atomic i8* %addr monotonic, align 1
926; CHECK-NOT: dmb
927; CHECK: ldrb w0, [x0, x1]
928; CHECK-NOT: dmb
929
930  ret i8 %val
931}
932
933define i8 @test_atomic_load_acquire_i8() nounwind {
934; CHECK: test_atomic_load_acquire_i8:
935  %val = load atomic i8* @var8 acquire, align 1
936; CHECK-NOT: dmb
937; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
938; CHECK-NOT: dmb
939; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
940; CHECK-NOT: dmb
941; CHECK: ldarb w0, [x[[ADDR]]]
942; CHECK-NOT: dmb
943  ret i8 %val
944}
945
946define i8 @test_atomic_load_seq_cst_i8() nounwind {
947; CHECK: test_atomic_load_seq_cst_i8:
948  %val = load atomic i8* @var8 seq_cst, align 1
949; CHECK-NOT: dmb
950; CHECK: adrp [[HIADDR:x[0-9]+]], var8
951; CHECK-NOT: dmb
952; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var8
953; CHECK-NOT: dmb
954; CHECK: ldarb w0, [x[[ADDR]]]
955; CHECK-NOT: dmb
956  ret i8 %val
957}
958
959define i16 @test_atomic_load_monotonic_i16() nounwind {
960; CHECK: test_atomic_load_monotonic_i16:
961  %val = load atomic i16* @var16 monotonic, align 2
962; CHECK-NOT: dmb
963; CHECK: adrp x[[HIADDR:[0-9]+]], var16
964; CHECK-NOT: dmb
965; CHECK: ldrh w0, [x[[HIADDR]], #:lo12:var16]
966; CHECK-NOT: dmb
967
968  ret i16 %val
969}
970
971define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
972; CHECK: test_atomic_load_monotonic_regoff_i32:
973  %addr_int = add i64 %base, %off
974  %addr = inttoptr i64 %addr_int to i32*
975
976  %val = load atomic i32* %addr monotonic, align 4
977; CHECK-NOT: dmb
978; CHECK: ldr w0, [x0, x1]
979; CHECK-NOT: dmb
980
981  ret i32 %val
982}
983
984define i64 @test_atomic_load_seq_cst_i64() nounwind {
985; CHECK: test_atomic_load_seq_cst_i64:
986  %val = load atomic i64* @var64 seq_cst, align 8
987; CHECK-NOT: dmb
988; CHECK: adrp [[HIADDR:x[0-9]+]], var64
989; CHECK-NOT: dmb
990; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var64
991; CHECK-NOT: dmb
992; CHECK: ldar x0, [x[[ADDR]]]
993; CHECK-NOT: dmb
994  ret i64 %val
995}
996
997define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
998; CHECK: test_atomic_store_monotonic_i8:
999  store atomic i8 %val, i8* @var8 monotonic, align 1
1000; CHECK: adrp x[[HIADDR:[0-9]+]], var8
1001; CHECK: strb w0, [x[[HIADDR]], #:lo12:var8]
1002
1003  ret void
1004}
1005
1006define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
1007; CHECK: test_atomic_store_monotonic_regoff_i8:
1008
1009  %addr_int = add i64 %base, %off
1010  %addr = inttoptr i64 %addr_int to i8*
1011
1012  store atomic i8 %val, i8* %addr monotonic, align 1
1013; CHECK: strb w2, [x0, x1]
1014
1015  ret void
1016}
1017define void @test_atomic_store_release_i8(i8 %val) nounwind {
1018; CHECK: test_atomic_store_release_i8:
1019  store atomic i8 %val, i8* @var8 release, align 1
1020; CHECK-NOT: dmb
1021; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1022; CHECK-NOT: dmb
1023; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var8
1024; CHECK-NOT: dmb
1025; CHECK: stlrb w0, [x[[ADDR]]]
1026; CHECK-NOT: dmb
1027  ret void
1028}
1029
1030define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
1031; CHECK: test_atomic_store_seq_cst_i8:
1032  store atomic i8 %val, i8* @var8 seq_cst, align 1
1033; CHECK-NOT: dmb
1034; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1035; CHECK-NOT: dmb
1036; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var8
1037; CHECK-NOT: dmb
1038; CHECK: stlrb w0, [x[[ADDR]]]
1039; CHECK-NOT: dmb
1040
1041  ret void
1042}
1043
1044define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
1045; CHECK: test_atomic_store_monotonic_i16:
1046  store atomic i16 %val, i16* @var16 monotonic, align 2
1047; CHECK-NOT: dmb
1048; CHECK: adrp x[[HIADDR:[0-9]+]], var16
1049; CHECK-NOT: dmb
1050; CHECK: strh w0, [x[[HIADDR]], #:lo12:var16]
1051; CHECK-NOT: dmb
1052  ret void
1053}
1054
1055define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
1056; CHECK: test_atomic_store_monotonic_regoff_i32:
1057
1058  %addr_int = add i64 %base, %off
1059  %addr = inttoptr i64 %addr_int to i32*
1060
1061  store atomic i32 %val, i32* %addr monotonic, align 4
1062; CHECK-NOT: dmb
1063; CHECK: str w2, [x0, x1]
1064; CHECK-NOT: dmb
1065
1066  ret void
1067}
1068
1069define void @test_atomic_store_release_i64(i64 %val) nounwind {
1070; CHECK: test_atomic_store_release_i64:
1071  store atomic i64 %val, i64* @var64 release, align 8
1072; CHECK-NOT: dmb
1073; CHECK: adrp [[HIADDR:x[0-9]+]], var64
1074; CHECK-NOT: dmb
1075; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var64
1076; CHECK-NOT: dmb
1077; CHECK: stlr x0, [x[[ADDR]]]
1078; CHECK-NOT: dmb
1079  ret void
1080}
1081