1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake | FileCheck --check-prefixes=CHECK,CHECK-O0,CHECK-O0-CUR %s 3; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake | FileCheck --check-prefixes=CHECK,CHECK-O3,CHECK-O3-CUR %s 4; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake -x86-experimental-unordered-atomic-isel | FileCheck --check-prefixes=CHECK,CHECK-O0,CHECK-O0-EX %s 5; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake -x86-experimental-unordered-atomic-isel | FileCheck --check-prefixes=CHECK,CHECK-O3,CHECK-O3-EX %s 6 7define i8 @load_i8(i8* %ptr) { 8; CHECK-LABEL: load_i8: 9; CHECK: # %bb.0: 10; CHECK-NEXT: movb (%rdi), %al 11; CHECK-NEXT: retq 12 %v = load atomic i8, i8* %ptr unordered, align 1 13 ret i8 %v 14} 15 16define void @store_i8(i8* %ptr, i8 %v) { 17; CHECK-O0-LABEL: store_i8: 18; CHECK-O0: # %bb.0: 19; CHECK-O0-NEXT: # kill: def $sil killed $sil killed $esi 20; CHECK-O0-NEXT: movb %sil, (%rdi) 21; CHECK-O0-NEXT: retq 22; 23; CHECK-O3-LABEL: store_i8: 24; CHECK-O3: # %bb.0: 25; CHECK-O3-NEXT: movb %sil, (%rdi) 26; CHECK-O3-NEXT: retq 27 store atomic i8 %v, i8* %ptr unordered, align 1 28 ret void 29} 30 31define i16 @load_i16(i16* %ptr) { 32; CHECK-O0-LABEL: load_i16: 33; CHECK-O0: # %bb.0: 34; CHECK-O0-NEXT: movw (%rdi), %ax 35; CHECK-O0-NEXT: retq 36; 37; CHECK-O3-LABEL: load_i16: 38; CHECK-O3: # %bb.0: 39; CHECK-O3-NEXT: movzwl (%rdi), %eax 40; CHECK-O3-NEXT: retq 41 %v = load atomic i16, i16* %ptr unordered, align 2 42 ret i16 %v 43} 44 45 46define void @store_i16(i16* %ptr, i16 %v) { 47; CHECK-O0-LABEL: store_i16: 48; CHECK-O0: # %bb.0: 49; CHECK-O0-NEXT: # kill: def $si killed $si killed $esi 50; CHECK-O0-NEXT: movw %si, (%rdi) 51; CHECK-O0-NEXT: retq 52; 53; CHECK-O3-LABEL: store_i16: 54; CHECK-O3: # %bb.0: 55; CHECK-O3-NEXT: movw %si, (%rdi) 56; CHECK-O3-NEXT: retq 57 store atomic i16 %v, i16* %ptr unordered, align 2 58 ret void 59} 60 61define i32 @load_i32(i32* %ptr) { 62; CHECK-LABEL: load_i32: 63; CHECK: # %bb.0: 64; CHECK-NEXT: movl (%rdi), %eax 65; CHECK-NEXT: retq 66 %v = load atomic i32, i32* %ptr unordered, align 4 67 ret i32 %v 68} 69 70define void @store_i32(i32* %ptr, i32 %v) { 71; CHECK-LABEL: store_i32: 72; CHECK: # %bb.0: 73; CHECK-NEXT: movl %esi, (%rdi) 74; CHECK-NEXT: retq 75 store atomic i32 %v, i32* %ptr unordered, align 4 76 ret void 77} 78 79define i64 @load_i64(i64* %ptr) { 80; CHECK-LABEL: load_i64: 81; CHECK: # %bb.0: 82; CHECK-NEXT: movq (%rdi), %rax 83; CHECK-NEXT: retq 84 %v = load atomic i64, i64* %ptr unordered, align 8 85 ret i64 %v 86} 87 88define void @store_i64(i64* %ptr, i64 %v) { 89; CHECK-LABEL: store_i64: 90; CHECK: # %bb.0: 91; CHECK-NEXT: movq %rsi, (%rdi) 92; CHECK-NEXT: retq 93 store atomic i64 %v, i64* %ptr unordered, align 8 94 ret void 95} 96 97;; The tests in the rest of this file are intended to show transforms which we 98;; either *can't* do for legality, or don't currently implement. The later 99;; are noted carefully where relevant. 100 101;; Start w/some clearly illegal ones. 102 103; Must use a full width op, not a byte op 104define void @narrow_writeback_or(i64* %ptr) { 105; CHECK-O0-LABEL: narrow_writeback_or: 106; CHECK-O0: # %bb.0: 107; CHECK-O0-NEXT: movq (%rdi), %rax 108; CHECK-O0-NEXT: orq $7, %rax 109; CHECK-O0-NEXT: movq %rax, (%rdi) 110; CHECK-O0-NEXT: retq 111; 112; CHECK-O3-LABEL: narrow_writeback_or: 113; CHECK-O3: # %bb.0: 114; CHECK-O3-NEXT: orq $7, (%rdi) 115; CHECK-O3-NEXT: retq 116 %v = load atomic i64, i64* %ptr unordered, align 8 117 %v.new = or i64 %v, 7 118 store atomic i64 %v.new, i64* %ptr unordered, align 8 119 ret void 120} 121 122; Must use a full width op, not a byte op 123define void @narrow_writeback_and(i64* %ptr) { 124; CHECK-O0-LABEL: narrow_writeback_and: 125; CHECK-O0: # %bb.0: 126; CHECK-O0-NEXT: movq (%rdi), %rax 127; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax 128; CHECK-O0-NEXT: andl $-256, %eax 129; CHECK-O0-NEXT: movl %eax, %ecx 130; CHECK-O0-NEXT: movq %rcx, (%rdi) 131; CHECK-O0-NEXT: retq 132; 133; CHECK-O3-LABEL: narrow_writeback_and: 134; CHECK-O3: # %bb.0: 135; CHECK-O3-NEXT: movl $4294967040, %eax # imm = 0xFFFFFF00 136; CHECK-O3-NEXT: andq %rax, (%rdi) 137; CHECK-O3-NEXT: retq 138 %v = load atomic i64, i64* %ptr unordered, align 8 139 %v.new = and i64 %v, 4294967040 ;; 0xFFFF_FF00 140 store atomic i64 %v.new, i64* %ptr unordered, align 8 141 ret void 142} 143 144; Must use a full width op, not a byte op 145define void @narrow_writeback_xor(i64* %ptr) { 146; CHECK-O0-LABEL: narrow_writeback_xor: 147; CHECK-O0: # %bb.0: 148; CHECK-O0-NEXT: movq (%rdi), %rax 149; CHECK-O0-NEXT: xorq $7, %rax 150; CHECK-O0-NEXT: movq %rax, (%rdi) 151; CHECK-O0-NEXT: retq 152; 153; CHECK-O3-LABEL: narrow_writeback_xor: 154; CHECK-O3: # %bb.0: 155; CHECK-O3-NEXT: xorq $7, (%rdi) 156; CHECK-O3-NEXT: retq 157 %v = load atomic i64, i64* %ptr unordered, align 8 158 %v.new = xor i64 %v, 7 159 store atomic i64 %v.new, i64* %ptr unordered, align 8 160 ret void 161} 162 163;; Next batch of tests are exercising cases where store widening would 164;; improve codegeneration. Note that widening is only legal if the 165;; resulting type would be atomic. Each tests has a well aligned, and 166;; unaligned variant to ensure we get correct codegen here. 167;; Note: It's not a legality issue, but there's a gotcha here to be aware 168;; of. Once we widen a pair of atomic stores, we loose the information 169;; that the original atomicity requirement was half the width. Given that, 170;; we can't then split the load again. This challenges our usual iterative 171;; approach to incremental improvement. 172 173; Legal if wider type is also atomic (TODO) 174define void @widen_store(i32* %p0, i32 %v1, i32 %v2) { 175; CHECK-LABEL: widen_store: 176; CHECK: # %bb.0: 177; CHECK-NEXT: movl %esi, (%rdi) 178; CHECK-NEXT: movl %edx, 4(%rdi) 179; CHECK-NEXT: retq 180 %p1 = getelementptr i32, i32* %p0, i64 1 181 store atomic i32 %v1, i32* %p0 unordered, align 8 182 store atomic i32 %v2, i32* %p1 unordered, align 4 183 ret void 184} 185 186; This one is *NOT* legal to widen. With weaker alignment, 187; the wider type might cross a cache line and violate the 188; atomicity requirement. 189define void @widen_store_unaligned(i32* %p0, i32 %v1, i32 %v2) { 190; CHECK-LABEL: widen_store_unaligned: 191; CHECK: # %bb.0: 192; CHECK-NEXT: movl %esi, (%rdi) 193; CHECK-NEXT: movl %edx, 4(%rdi) 194; CHECK-NEXT: retq 195 %p1 = getelementptr i32, i32* %p0, i64 1 196 store atomic i32 %v1, i32* %p0 unordered, align 4 197 store atomic i32 %v2, i32* %p1 unordered, align 4 198 ret void 199} 200 201; Legal if wider type is also atomic (TODO) 202define void @widen_broadcast(i32* %p0, i32 %v) { 203; CHECK-LABEL: widen_broadcast: 204; CHECK: # %bb.0: 205; CHECK-NEXT: movl %esi, (%rdi) 206; CHECK-NEXT: movl %esi, 4(%rdi) 207; CHECK-NEXT: retq 208 %p1 = getelementptr i32, i32* %p0, i64 1 209 store atomic i32 %v, i32* %p0 unordered, align 8 210 store atomic i32 %v, i32* %p1 unordered, align 4 211 ret void 212} 213 214; Not legal to widen due to alignment restriction 215define void @widen_broadcast_unaligned(i32* %p0, i32 %v) { 216; CHECK-LABEL: widen_broadcast_unaligned: 217; CHECK: # %bb.0: 218; CHECK-NEXT: movl %esi, (%rdi) 219; CHECK-NEXT: movl %esi, 4(%rdi) 220; CHECK-NEXT: retq 221 %p1 = getelementptr i32, i32* %p0, i64 1 222 store atomic i32 %v, i32* %p0 unordered, align 4 223 store atomic i32 %v, i32* %p1 unordered, align 4 224 ret void 225} 226 227define i128 @load_i128(i128* %ptr) { 228; CHECK-O0-LABEL: load_i128: 229; CHECK-O0: # %bb.0: 230; CHECK-O0-NEXT: pushq %rbx 231; CHECK-O0-NEXT: .cfi_def_cfa_offset 16 232; CHECK-O0-NEXT: .cfi_offset %rbx, -16 233; CHECK-O0-NEXT: xorl %eax, %eax 234; CHECK-O0-NEXT: movl %eax, %ecx 235; CHECK-O0-NEXT: movq %rcx, %rax 236; CHECK-O0-NEXT: movq %rcx, %rdx 237; CHECK-O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 238; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload 239; CHECK-O0-NEXT: lock cmpxchg16b (%rdi) 240; CHECK-O0-NEXT: popq %rbx 241; CHECK-O0-NEXT: .cfi_def_cfa_offset 8 242; CHECK-O0-NEXT: retq 243; 244; CHECK-O3-LABEL: load_i128: 245; CHECK-O3: # %bb.0: 246; CHECK-O3-NEXT: pushq %rbx 247; CHECK-O3-NEXT: .cfi_def_cfa_offset 16 248; CHECK-O3-NEXT: .cfi_offset %rbx, -16 249; CHECK-O3-NEXT: xorl %eax, %eax 250; CHECK-O3-NEXT: xorl %edx, %edx 251; CHECK-O3-NEXT: xorl %ecx, %ecx 252; CHECK-O3-NEXT: xorl %ebx, %ebx 253; CHECK-O3-NEXT: lock cmpxchg16b (%rdi) 254; CHECK-O3-NEXT: popq %rbx 255; CHECK-O3-NEXT: .cfi_def_cfa_offset 8 256; CHECK-O3-NEXT: retq 257 %v = load atomic i128, i128* %ptr unordered, align 16 258 ret i128 %v 259} 260 261define void @store_i128(i128* %ptr, i128 %v) { 262; CHECK-O0-LABEL: store_i128: 263; CHECK-O0: # %bb.0: 264; CHECK-O0-NEXT: pushq %rbx 265; CHECK-O0-NEXT: .cfi_def_cfa_offset 16 266; CHECK-O0-NEXT: .cfi_offset %rbx, -16 267; CHECK-O0-NEXT: movq (%rdi), %rax 268; CHECK-O0-NEXT: movq 8(%rdi), %rcx 269; CHECK-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 270; CHECK-O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 271; CHECK-O0-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 272; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 273; CHECK-O0-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 274; CHECK-O0-NEXT: jmp .LBB16_1 275; CHECK-O0-NEXT: .LBB16_1: # %atomicrmw.start 276; CHECK-O0-NEXT: # =>This Inner Loop Header: Depth=1 277; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload 278; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 279; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 280; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload 281; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload 282; CHECK-O0-NEXT: lock cmpxchg16b (%rsi) 283; CHECK-O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 284; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 285; CHECK-O0-NEXT: jne .LBB16_1 286; CHECK-O0-NEXT: jmp .LBB16_2 287; CHECK-O0-NEXT: .LBB16_2: # %atomicrmw.end 288; CHECK-O0-NEXT: popq %rbx 289; CHECK-O0-NEXT: .cfi_def_cfa_offset 8 290; CHECK-O0-NEXT: retq 291; 292; CHECK-O3-LABEL: store_i128: 293; CHECK-O3: # %bb.0: 294; CHECK-O3-NEXT: pushq %rbx 295; CHECK-O3-NEXT: .cfi_def_cfa_offset 16 296; CHECK-O3-NEXT: .cfi_offset %rbx, -16 297; CHECK-O3-NEXT: movq %rdx, %rcx 298; CHECK-O3-NEXT: movq %rsi, %rbx 299; CHECK-O3-NEXT: movq (%rdi), %rax 300; CHECK-O3-NEXT: movq 8(%rdi), %rdx 301; CHECK-O3-NEXT: .p2align 4, 0x90 302; CHECK-O3-NEXT: .LBB16_1: # %atomicrmw.start 303; CHECK-O3-NEXT: # =>This Inner Loop Header: Depth=1 304; CHECK-O3-NEXT: lock cmpxchg16b (%rdi) 305; CHECK-O3-NEXT: jne .LBB16_1 306; CHECK-O3-NEXT: # %bb.2: # %atomicrmw.end 307; CHECK-O3-NEXT: popq %rbx 308; CHECK-O3-NEXT: .cfi_def_cfa_offset 8 309; CHECK-O3-NEXT: retq 310 store atomic i128 %v, i128* %ptr unordered, align 16 311 ret void 312} 313 314define i256 @load_i256(i256* %ptr) { 315; CHECK-O0-LABEL: load_i256: 316; CHECK-O0: # %bb.0: 317; CHECK-O0-NEXT: subq $56, %rsp 318; CHECK-O0-NEXT: .cfi_def_cfa_offset 64 319; CHECK-O0-NEXT: movq %rdi, %rax 320; CHECK-O0-NEXT: movl $32, %ecx 321; CHECK-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx 322; CHECK-O0-NEXT: xorl %r8d, %r8d 323; CHECK-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 324; CHECK-O0-NEXT: movq %rcx, %rdi 325; CHECK-O0-NEXT: movl %r8d, %ecx 326; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 327; CHECK-O0-NEXT: callq __atomic_load 328; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax 329; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %rdx 330; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %rsi 331; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %rdi 332; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload 333; CHECK-O0-NEXT: movq %rdi, 24(%r9) 334; CHECK-O0-NEXT: movq %rsi, 16(%r9) 335; CHECK-O0-NEXT: movq %rdx, 8(%r9) 336; CHECK-O0-NEXT: movq %rax, (%r9) 337; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 338; CHECK-O0-NEXT: addq $56, %rsp 339; CHECK-O0-NEXT: .cfi_def_cfa_offset 8 340; CHECK-O0-NEXT: retq 341; 342; CHECK-O3-LABEL: load_i256: 343; CHECK-O3: # %bb.0: 344; CHECK-O3-NEXT: pushq %rbx 345; CHECK-O3-NEXT: .cfi_def_cfa_offset 16 346; CHECK-O3-NEXT: subq $32, %rsp 347; CHECK-O3-NEXT: .cfi_def_cfa_offset 48 348; CHECK-O3-NEXT: .cfi_offset %rbx, -16 349; CHECK-O3-NEXT: movq %rdi, %rbx 350; CHECK-O3-NEXT: movq %rsp, %rdx 351; CHECK-O3-NEXT: movl $32, %edi 352; CHECK-O3-NEXT: xorl %ecx, %ecx 353; CHECK-O3-NEXT: callq __atomic_load 354; CHECK-O3-NEXT: vmovups (%rsp), %ymm0 355; CHECK-O3-NEXT: vmovups %ymm0, (%rbx) 356; CHECK-O3-NEXT: movq %rbx, %rax 357; CHECK-O3-NEXT: addq $32, %rsp 358; CHECK-O3-NEXT: .cfi_def_cfa_offset 16 359; CHECK-O3-NEXT: popq %rbx 360; CHECK-O3-NEXT: .cfi_def_cfa_offset 8 361; CHECK-O3-NEXT: vzeroupper 362; CHECK-O3-NEXT: retq 363 %v = load atomic i256, i256* %ptr unordered, align 16 364 ret i256 %v 365} 366 367define void @store_i256(i256* %ptr, i256 %v) { 368; CHECK-O0-LABEL: store_i256: 369; CHECK-O0: # %bb.0: 370; CHECK-O0-NEXT: subq $40, %rsp 371; CHECK-O0-NEXT: .cfi_def_cfa_offset 48 372; CHECK-O0-NEXT: xorl %eax, %eax 373; CHECK-O0-NEXT: leaq {{[0-9]+}}(%rsp), %r9 374; CHECK-O0-NEXT: movq %rsi, {{[0-9]+}}(%rsp) 375; CHECK-O0-NEXT: movq %rdx, {{[0-9]+}}(%rsp) 376; CHECK-O0-NEXT: movq %rcx, {{[0-9]+}}(%rsp) 377; CHECK-O0-NEXT: movq %r8, {{[0-9]+}}(%rsp) 378; CHECK-O0-NEXT: movl $32, %ecx 379; CHECK-O0-NEXT: movq %rdi, (%rsp) # 8-byte Spill 380; CHECK-O0-NEXT: movq %rcx, %rdi 381; CHECK-O0-NEXT: movq (%rsp), %rsi # 8-byte Reload 382; CHECK-O0-NEXT: movq %r9, %rdx 383; CHECK-O0-NEXT: movl %eax, %ecx 384; CHECK-O0-NEXT: callq __atomic_store 385; CHECK-O0-NEXT: addq $40, %rsp 386; CHECK-O0-NEXT: .cfi_def_cfa_offset 8 387; CHECK-O0-NEXT: retq 388; 389; CHECK-O3-LABEL: store_i256: 390; CHECK-O3: # %bb.0: 391; CHECK-O3-NEXT: subq $40, %rsp 392; CHECK-O3-NEXT: .cfi_def_cfa_offset 48 393; CHECK-O3-NEXT: movq %rdi, %rax 394; CHECK-O3-NEXT: movq %r8, {{[0-9]+}}(%rsp) 395; CHECK-O3-NEXT: movq %rcx, {{[0-9]+}}(%rsp) 396; CHECK-O3-NEXT: movq %rdx, {{[0-9]+}}(%rsp) 397; CHECK-O3-NEXT: movq %rsi, {{[0-9]+}}(%rsp) 398; CHECK-O3-NEXT: leaq {{[0-9]+}}(%rsp), %rdx 399; CHECK-O3-NEXT: movl $32, %edi 400; CHECK-O3-NEXT: movq %rax, %rsi 401; CHECK-O3-NEXT: xorl %ecx, %ecx 402; CHECK-O3-NEXT: callq __atomic_store 403; CHECK-O3-NEXT: addq $40, %rsp 404; CHECK-O3-NEXT: .cfi_def_cfa_offset 8 405; CHECK-O3-NEXT: retq 406 store atomic i256 %v, i256* %ptr unordered, align 16 407 ret void 408} 409 410; Legal if wider type is also atomic (TODO) 411define void @vec_store(i32* %p0, <2 x i32> %vec) { 412; CHECK-O0-CUR-LABEL: vec_store: 413; CHECK-O0-CUR: # %bb.0: 414; CHECK-O0-CUR-NEXT: vmovd %xmm0, %eax 415; CHECK-O0-CUR-NEXT: vpextrd $1, %xmm0, %ecx 416; CHECK-O0-CUR-NEXT: movl %eax, (%rdi) 417; CHECK-O0-CUR-NEXT: movl %ecx, 4(%rdi) 418; CHECK-O0-CUR-NEXT: retq 419; 420; CHECK-O3-CUR-LABEL: vec_store: 421; CHECK-O3-CUR: # %bb.0: 422; CHECK-O3-CUR-NEXT: vmovd %xmm0, %eax 423; CHECK-O3-CUR-NEXT: vpextrd $1, %xmm0, %ecx 424; CHECK-O3-CUR-NEXT: movl %eax, (%rdi) 425; CHECK-O3-CUR-NEXT: movl %ecx, 4(%rdi) 426; CHECK-O3-CUR-NEXT: retq 427; 428; CHECK-O0-EX-LABEL: vec_store: 429; CHECK-O0-EX: # %bb.0: 430; CHECK-O0-EX-NEXT: vmovd %xmm0, (%rdi) 431; CHECK-O0-EX-NEXT: vpextrd $1, %xmm0, 4(%rdi) 432; CHECK-O0-EX-NEXT: retq 433; 434; CHECK-O3-EX-LABEL: vec_store: 435; CHECK-O3-EX: # %bb.0: 436; CHECK-O3-EX-NEXT: vmovss %xmm0, (%rdi) 437; CHECK-O3-EX-NEXT: vextractps $1, %xmm0, 4(%rdi) 438; CHECK-O3-EX-NEXT: retq 439 %v1 = extractelement <2 x i32> %vec, i32 0 440 %v2 = extractelement <2 x i32> %vec, i32 1 441 %p1 = getelementptr i32, i32* %p0, i64 1 442 store atomic i32 %v1, i32* %p0 unordered, align 8 443 store atomic i32 %v2, i32* %p1 unordered, align 4 444 ret void 445} 446 447; Not legal to widen due to alignment restriction 448define void @vec_store_unaligned(i32* %p0, <2 x i32> %vec) { 449; CHECK-O0-CUR-LABEL: vec_store_unaligned: 450; CHECK-O0-CUR: # %bb.0: 451; CHECK-O0-CUR-NEXT: vmovd %xmm0, %eax 452; CHECK-O0-CUR-NEXT: vpextrd $1, %xmm0, %ecx 453; CHECK-O0-CUR-NEXT: movl %eax, (%rdi) 454; CHECK-O0-CUR-NEXT: movl %ecx, 4(%rdi) 455; CHECK-O0-CUR-NEXT: retq 456; 457; CHECK-O3-CUR-LABEL: vec_store_unaligned: 458; CHECK-O3-CUR: # %bb.0: 459; CHECK-O3-CUR-NEXT: vmovd %xmm0, %eax 460; CHECK-O3-CUR-NEXT: vpextrd $1, %xmm0, %ecx 461; CHECK-O3-CUR-NEXT: movl %eax, (%rdi) 462; CHECK-O3-CUR-NEXT: movl %ecx, 4(%rdi) 463; CHECK-O3-CUR-NEXT: retq 464; 465; CHECK-O0-EX-LABEL: vec_store_unaligned: 466; CHECK-O0-EX: # %bb.0: 467; CHECK-O0-EX-NEXT: vmovd %xmm0, (%rdi) 468; CHECK-O0-EX-NEXT: vpextrd $1, %xmm0, 4(%rdi) 469; CHECK-O0-EX-NEXT: retq 470; 471; CHECK-O3-EX-LABEL: vec_store_unaligned: 472; CHECK-O3-EX: # %bb.0: 473; CHECK-O3-EX-NEXT: vmovss %xmm0, (%rdi) 474; CHECK-O3-EX-NEXT: vextractps $1, %xmm0, 4(%rdi) 475; CHECK-O3-EX-NEXT: retq 476 %v1 = extractelement <2 x i32> %vec, i32 0 477 %v2 = extractelement <2 x i32> %vec, i32 1 478 %p1 = getelementptr i32, i32* %p0, i64 1 479 store atomic i32 %v1, i32* %p0 unordered, align 4 480 store atomic i32 %v2, i32* %p1 unordered, align 4 481 ret void 482} 483 484 485 486; Legal if wider type is also atomic (TODO) 487; Also, can avoid register move from xmm to eax (TODO) 488define void @widen_broadcast2(i32* %p0, <2 x i32> %vec) { 489; CHECK-O0-CUR-LABEL: widen_broadcast2: 490; CHECK-O0-CUR: # %bb.0: 491; CHECK-O0-CUR-NEXT: vmovd %xmm0, %eax 492; CHECK-O0-CUR-NEXT: movl %eax, (%rdi) 493; CHECK-O0-CUR-NEXT: movl %eax, 4(%rdi) 494; CHECK-O0-CUR-NEXT: retq 495; 496; CHECK-O3-CUR-LABEL: widen_broadcast2: 497; CHECK-O3-CUR: # %bb.0: 498; CHECK-O3-CUR-NEXT: vmovd %xmm0, %eax 499; CHECK-O3-CUR-NEXT: movl %eax, (%rdi) 500; CHECK-O3-CUR-NEXT: movl %eax, 4(%rdi) 501; CHECK-O3-CUR-NEXT: retq 502; 503; CHECK-O0-EX-LABEL: widen_broadcast2: 504; CHECK-O0-EX: # %bb.0: 505; CHECK-O0-EX-NEXT: vmovd %xmm0, (%rdi) 506; CHECK-O0-EX-NEXT: vmovd %xmm0, 4(%rdi) 507; CHECK-O0-EX-NEXT: retq 508; 509; CHECK-O3-EX-LABEL: widen_broadcast2: 510; CHECK-O3-EX: # %bb.0: 511; CHECK-O3-EX-NEXT: vmovss %xmm0, (%rdi) 512; CHECK-O3-EX-NEXT: vmovss %xmm0, 4(%rdi) 513; CHECK-O3-EX-NEXT: retq 514 %v1 = extractelement <2 x i32> %vec, i32 0 515 %p1 = getelementptr i32, i32* %p0, i64 1 516 store atomic i32 %v1, i32* %p0 unordered, align 8 517 store atomic i32 %v1, i32* %p1 unordered, align 4 518 ret void 519} 520 521; Not legal to widen due to alignment restriction 522define void @widen_broadcast2_unaligned(i32* %p0, <2 x i32> %vec) { 523; CHECK-O0-CUR-LABEL: widen_broadcast2_unaligned: 524; CHECK-O0-CUR: # %bb.0: 525; CHECK-O0-CUR-NEXT: vmovd %xmm0, %eax 526; CHECK-O0-CUR-NEXT: movl %eax, (%rdi) 527; CHECK-O0-CUR-NEXT: movl %eax, 4(%rdi) 528; CHECK-O0-CUR-NEXT: retq 529; 530; CHECK-O3-CUR-LABEL: widen_broadcast2_unaligned: 531; CHECK-O3-CUR: # %bb.0: 532; CHECK-O3-CUR-NEXT: vmovd %xmm0, %eax 533; CHECK-O3-CUR-NEXT: movl %eax, (%rdi) 534; CHECK-O3-CUR-NEXT: movl %eax, 4(%rdi) 535; CHECK-O3-CUR-NEXT: retq 536; 537; CHECK-O0-EX-LABEL: widen_broadcast2_unaligned: 538; CHECK-O0-EX: # %bb.0: 539; CHECK-O0-EX-NEXT: vmovd %xmm0, (%rdi) 540; CHECK-O0-EX-NEXT: vmovd %xmm0, 4(%rdi) 541; CHECK-O0-EX-NEXT: retq 542; 543; CHECK-O3-EX-LABEL: widen_broadcast2_unaligned: 544; CHECK-O3-EX: # %bb.0: 545; CHECK-O3-EX-NEXT: vmovss %xmm0, (%rdi) 546; CHECK-O3-EX-NEXT: vmovss %xmm0, 4(%rdi) 547; CHECK-O3-EX-NEXT: retq 548 %v1 = extractelement <2 x i32> %vec, i32 0 549 %p1 = getelementptr i32, i32* %p0, i64 1 550 store atomic i32 %v1, i32* %p0 unordered, align 4 551 store atomic i32 %v1, i32* %p1 unordered, align 4 552 ret void 553} 554 555; Legal if wider type is also atomic (TODO) 556define void @widen_zero_init(i32* %p0, i32 %v1, i32 %v2) { 557; CHECK-LABEL: widen_zero_init: 558; CHECK: # %bb.0: 559; CHECK-NEXT: movl $0, (%rdi) 560; CHECK-NEXT: movl $0, 4(%rdi) 561; CHECK-NEXT: retq 562 %p1 = getelementptr i32, i32* %p0, i64 1 563 store atomic i32 0, i32* %p0 unordered, align 8 564 store atomic i32 0, i32* %p1 unordered, align 4 565 ret void 566} 567 568; Not legal to widen due to alignment restriction 569define void @widen_zero_init_unaligned(i32* %p0, i32 %v1, i32 %v2) { 570; CHECK-LABEL: widen_zero_init_unaligned: 571; CHECK: # %bb.0: 572; CHECK-NEXT: movl $0, (%rdi) 573; CHECK-NEXT: movl $0, 4(%rdi) 574; CHECK-NEXT: retq 575 %p1 = getelementptr i32, i32* %p0, i64 1 576 store atomic i32 0, i32* %p0 unordered, align 4 577 store atomic i32 0, i32* %p1 unordered, align 4 578 ret void 579} 580 581;; The next batch of tests are stressing load folding. Folding is legal 582;; on x86, so these are simply checking optimization quality. 583 584; Legal, as expected 585define i64 @load_fold_add1(i64* %p) { 586; CHECK-LABEL: load_fold_add1: 587; CHECK: # %bb.0: 588; CHECK-NEXT: movq (%rdi), %rax 589; CHECK-NEXT: addq $15, %rax 590; CHECK-NEXT: retq 591 %v = load atomic i64, i64* %p unordered, align 8 592 %ret = add i64 %v, 15 593 ret i64 %ret 594} 595 596define i64 @load_fold_add2(i64* %p, i64 %v2) { 597; CHECK-O0-LABEL: load_fold_add2: 598; CHECK-O0: # %bb.0: 599; CHECK-O0-NEXT: addq (%rdi), %rsi 600; CHECK-O0-NEXT: movq %rsi, %rax 601; CHECK-O0-NEXT: retq 602; 603; CHECK-O3-LABEL: load_fold_add2: 604; CHECK-O3: # %bb.0: 605; CHECK-O3-NEXT: movq %rsi, %rax 606; CHECK-O3-NEXT: addq (%rdi), %rax 607; CHECK-O3-NEXT: retq 608 %v = load atomic i64, i64* %p unordered, align 8 609 %ret = add i64 %v, %v2 610 ret i64 %ret 611} 612 613define i64 @load_fold_add3(i64* %p1, i64* %p2) { 614; CHECK-O0-LABEL: load_fold_add3: 615; CHECK-O0: # %bb.0: 616; CHECK-O0-NEXT: movq (%rdi), %rax 617; CHECK-O0-NEXT: addq (%rsi), %rax 618; CHECK-O0-NEXT: retq 619; 620; CHECK-O3-CUR-LABEL: load_fold_add3: 621; CHECK-O3-CUR: # %bb.0: 622; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 623; CHECK-O3-CUR-NEXT: addq (%rdi), %rax 624; CHECK-O3-CUR-NEXT: retq 625; 626; CHECK-O3-EX-LABEL: load_fold_add3: 627; CHECK-O3-EX: # %bb.0: 628; CHECK-O3-EX-NEXT: movq (%rdi), %rax 629; CHECK-O3-EX-NEXT: addq (%rsi), %rax 630; CHECK-O3-EX-NEXT: retq 631 %v = load atomic i64, i64* %p1 unordered, align 8 632 %v2 = load atomic i64, i64* %p2 unordered, align 8 633 %ret = add i64 %v, %v2 634 ret i64 %ret 635} 636 637; Legal, as expected 638define i64 @load_fold_sub1(i64* %p) { 639; CHECK-O0-LABEL: load_fold_sub1: 640; CHECK-O0: # %bb.0: 641; CHECK-O0-NEXT: movq (%rdi), %rax 642; CHECK-O0-NEXT: subq $15, %rax 643; CHECK-O0-NEXT: retq 644; 645; CHECK-O3-LABEL: load_fold_sub1: 646; CHECK-O3: # %bb.0: 647; CHECK-O3-NEXT: movq (%rdi), %rax 648; CHECK-O3-NEXT: addq $-15, %rax 649; CHECK-O3-NEXT: retq 650 %v = load atomic i64, i64* %p unordered, align 8 651 %ret = sub i64 %v, 15 652 ret i64 %ret 653} 654 655define i64 @load_fold_sub2(i64* %p, i64 %v2) { 656; CHECK-LABEL: load_fold_sub2: 657; CHECK: # %bb.0: 658; CHECK-NEXT: movq (%rdi), %rax 659; CHECK-NEXT: subq %rsi, %rax 660; CHECK-NEXT: retq 661 %v = load atomic i64, i64* %p unordered, align 8 662 %ret = sub i64 %v, %v2 663 ret i64 %ret 664} 665 666define i64 @load_fold_sub3(i64* %p1, i64* %p2) { 667; CHECK-LABEL: load_fold_sub3: 668; CHECK: # %bb.0: 669; CHECK-NEXT: movq (%rdi), %rax 670; CHECK-NEXT: subq (%rsi), %rax 671; CHECK-NEXT: retq 672 %v = load atomic i64, i64* %p1 unordered, align 8 673 %v2 = load atomic i64, i64* %p2 unordered, align 8 674 %ret = sub i64 %v, %v2 675 ret i64 %ret 676} 677 678; Legal, as expected 679define i64 @load_fold_mul1(i64* %p) { 680; CHECK-O0-LABEL: load_fold_mul1: 681; CHECK-O0: # %bb.0: 682; CHECK-O0-NEXT: imulq $15, (%rdi), %rax 683; CHECK-O0-NEXT: retq 684; 685; CHECK-O3-LABEL: load_fold_mul1: 686; CHECK-O3: # %bb.0: 687; CHECK-O3-NEXT: movq (%rdi), %rax 688; CHECK-O3-NEXT: leaq (%rax,%rax,4), %rax 689; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax 690; CHECK-O3-NEXT: retq 691 %v = load atomic i64, i64* %p unordered, align 8 692 %ret = mul i64 %v, 15 693 ret i64 %ret 694} 695 696define i64 @load_fold_mul2(i64* %p, i64 %v2) { 697; CHECK-O0-LABEL: load_fold_mul2: 698; CHECK-O0: # %bb.0: 699; CHECK-O0-NEXT: imulq (%rdi), %rsi 700; CHECK-O0-NEXT: movq %rsi, %rax 701; CHECK-O0-NEXT: retq 702; 703; CHECK-O3-LABEL: load_fold_mul2: 704; CHECK-O3: # %bb.0: 705; CHECK-O3-NEXT: movq %rsi, %rax 706; CHECK-O3-NEXT: imulq (%rdi), %rax 707; CHECK-O3-NEXT: retq 708 %v = load atomic i64, i64* %p unordered, align 8 709 %ret = mul i64 %v, %v2 710 ret i64 %ret 711} 712 713define i64 @load_fold_mul3(i64* %p1, i64* %p2) { 714; CHECK-O0-LABEL: load_fold_mul3: 715; CHECK-O0: # %bb.0: 716; CHECK-O0-NEXT: movq (%rdi), %rax 717; CHECK-O0-NEXT: imulq (%rsi), %rax 718; CHECK-O0-NEXT: retq 719; 720; CHECK-O3-CUR-LABEL: load_fold_mul3: 721; CHECK-O3-CUR: # %bb.0: 722; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 723; CHECK-O3-CUR-NEXT: imulq (%rdi), %rax 724; CHECK-O3-CUR-NEXT: retq 725; 726; CHECK-O3-EX-LABEL: load_fold_mul3: 727; CHECK-O3-EX: # %bb.0: 728; CHECK-O3-EX-NEXT: movq (%rdi), %rax 729; CHECK-O3-EX-NEXT: imulq (%rsi), %rax 730; CHECK-O3-EX-NEXT: retq 731 %v = load atomic i64, i64* %p1 unordered, align 8 732 %v2 = load atomic i64, i64* %p2 unordered, align 8 733 %ret = mul i64 %v, %v2 734 ret i64 %ret 735} 736 737; Legal to fold (TODO) 738define i64 @load_fold_sdiv1(i64* %p) { 739; CHECK-O0-LABEL: load_fold_sdiv1: 740; CHECK-O0: # %bb.0: 741; CHECK-O0-NEXT: movq (%rdi), %rax 742; CHECK-O0-NEXT: cqto 743; CHECK-O0-NEXT: movl $15, %ecx 744; CHECK-O0-NEXT: idivq %rcx 745; CHECK-O0-NEXT: retq 746; 747; CHECK-O3-LABEL: load_fold_sdiv1: 748; CHECK-O3: # %bb.0: 749; CHECK-O3-NEXT: movq (%rdi), %rcx 750; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 751; CHECK-O3-NEXT: movq %rcx, %rax 752; CHECK-O3-NEXT: imulq %rdx 753; CHECK-O3-NEXT: addq %rcx, %rdx 754; CHECK-O3-NEXT: movq %rdx, %rax 755; CHECK-O3-NEXT: shrq $63, %rax 756; CHECK-O3-NEXT: sarq $3, %rdx 757; CHECK-O3-NEXT: addq %rdx, %rax 758; CHECK-O3-NEXT: retq 759 %v = load atomic i64, i64* %p unordered, align 8 760 %ret = sdiv i64 %v, 15 761 ret i64 %ret 762} 763 764; Legal to fold (TODO) 765define i64 @load_fold_sdiv2(i64* %p, i64 %v2) { 766; CHECK-O0-LABEL: load_fold_sdiv2: 767; CHECK-O0: # %bb.0: 768; CHECK-O0-NEXT: movq (%rdi), %rax 769; CHECK-O0-NEXT: cqto 770; CHECK-O0-NEXT: idivq %rsi 771; CHECK-O0-NEXT: retq 772; 773; CHECK-O3-LABEL: load_fold_sdiv2: 774; CHECK-O3: # %bb.0: 775; CHECK-O3-NEXT: movq (%rdi), %rax 776; CHECK-O3-NEXT: movq %rax, %rcx 777; CHECK-O3-NEXT: orq %rsi, %rcx 778; CHECK-O3-NEXT: shrq $32, %rcx 779; CHECK-O3-NEXT: je .LBB35_1 780; CHECK-O3-NEXT: # %bb.2: 781; CHECK-O3-NEXT: cqto 782; CHECK-O3-NEXT: idivq %rsi 783; CHECK-O3-NEXT: retq 784; CHECK-O3-NEXT: .LBB35_1: 785; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 786; CHECK-O3-NEXT: xorl %edx, %edx 787; CHECK-O3-NEXT: divl %esi 788; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 789; CHECK-O3-NEXT: retq 790 %v = load atomic i64, i64* %p unordered, align 8 791 %ret = sdiv i64 %v, %v2 792 ret i64 %ret 793} 794 795define i64 @load_fold_sdiv3(i64* %p1, i64* %p2) { 796; CHECK-O0-LABEL: load_fold_sdiv3: 797; CHECK-O0: # %bb.0: 798; CHECK-O0-NEXT: movq (%rdi), %rax 799; CHECK-O0-NEXT: cqto 800; CHECK-O0-NEXT: idivq (%rsi) 801; CHECK-O0-NEXT: retq 802; 803; CHECK-O3-LABEL: load_fold_sdiv3: 804; CHECK-O3: # %bb.0: 805; CHECK-O3-NEXT: movq (%rdi), %rax 806; CHECK-O3-NEXT: movq (%rsi), %rcx 807; CHECK-O3-NEXT: movq %rax, %rdx 808; CHECK-O3-NEXT: orq %rcx, %rdx 809; CHECK-O3-NEXT: shrq $32, %rdx 810; CHECK-O3-NEXT: je .LBB36_1 811; CHECK-O3-NEXT: # %bb.2: 812; CHECK-O3-NEXT: cqto 813; CHECK-O3-NEXT: idivq %rcx 814; CHECK-O3-NEXT: retq 815; CHECK-O3-NEXT: .LBB36_1: 816; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 817; CHECK-O3-NEXT: xorl %edx, %edx 818; CHECK-O3-NEXT: divl %ecx 819; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 820; CHECK-O3-NEXT: retq 821 %v = load atomic i64, i64* %p1 unordered, align 8 822 %v2 = load atomic i64, i64* %p2 unordered, align 8 823 %ret = sdiv i64 %v, %v2 824 ret i64 %ret 825} 826 827; Legal to fold (TODO) 828define i64 @load_fold_udiv1(i64* %p) { 829; CHECK-O0-LABEL: load_fold_udiv1: 830; CHECK-O0: # %bb.0: 831; CHECK-O0-NEXT: movq (%rdi), %rax 832; CHECK-O0-NEXT: xorl %ecx, %ecx 833; CHECK-O0-NEXT: movl %ecx, %edx 834; CHECK-O0-NEXT: movl $15, %esi 835; CHECK-O0-NEXT: divq %rsi 836; CHECK-O0-NEXT: retq 837; 838; CHECK-O3-CUR-LABEL: load_fold_udiv1: 839; CHECK-O3-CUR: # %bb.0: 840; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 841; CHECK-O3-CUR-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889 842; CHECK-O3-CUR-NEXT: mulq %rcx 843; CHECK-O3-CUR-NEXT: movq %rdx, %rax 844; CHECK-O3-CUR-NEXT: shrq $3, %rax 845; CHECK-O3-CUR-NEXT: retq 846; 847; CHECK-O3-EX-LABEL: load_fold_udiv1: 848; CHECK-O3-EX: # %bb.0: 849; CHECK-O3-EX-NEXT: movabsq $-8608480567731124087, %rax # imm = 0x8888888888888889 850; CHECK-O3-EX-NEXT: mulq (%rdi) 851; CHECK-O3-EX-NEXT: movq %rdx, %rax 852; CHECK-O3-EX-NEXT: shrq $3, %rax 853; CHECK-O3-EX-NEXT: retq 854 %v = load atomic i64, i64* %p unordered, align 8 855 %ret = udiv i64 %v, 15 856 ret i64 %ret 857} 858 859define i64 @load_fold_udiv2(i64* %p, i64 %v2) { 860; CHECK-O0-LABEL: load_fold_udiv2: 861; CHECK-O0: # %bb.0: 862; CHECK-O0-NEXT: movq (%rdi), %rax 863; CHECK-O0-NEXT: xorl %ecx, %ecx 864; CHECK-O0-NEXT: movl %ecx, %edx 865; CHECK-O0-NEXT: divq %rsi 866; CHECK-O0-NEXT: retq 867; 868; CHECK-O3-LABEL: load_fold_udiv2: 869; CHECK-O3: # %bb.0: 870; CHECK-O3-NEXT: movq (%rdi), %rax 871; CHECK-O3-NEXT: movq %rax, %rcx 872; CHECK-O3-NEXT: orq %rsi, %rcx 873; CHECK-O3-NEXT: shrq $32, %rcx 874; CHECK-O3-NEXT: je .LBB38_1 875; CHECK-O3-NEXT: # %bb.2: 876; CHECK-O3-NEXT: xorl %edx, %edx 877; CHECK-O3-NEXT: divq %rsi 878; CHECK-O3-NEXT: retq 879; CHECK-O3-NEXT: .LBB38_1: 880; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 881; CHECK-O3-NEXT: xorl %edx, %edx 882; CHECK-O3-NEXT: divl %esi 883; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 884; CHECK-O3-NEXT: retq 885 %v = load atomic i64, i64* %p unordered, align 8 886 %ret = udiv i64 %v, %v2 887 ret i64 %ret 888} 889 890define i64 @load_fold_udiv3(i64* %p1, i64* %p2) { 891; CHECK-O0-LABEL: load_fold_udiv3: 892; CHECK-O0: # %bb.0: 893; CHECK-O0-NEXT: movq (%rdi), %rax 894; CHECK-O0-NEXT: xorl %ecx, %ecx 895; CHECK-O0-NEXT: movl %ecx, %edx 896; CHECK-O0-NEXT: divq (%rsi) 897; CHECK-O0-NEXT: retq 898; 899; CHECK-O3-LABEL: load_fold_udiv3: 900; CHECK-O3: # %bb.0: 901; CHECK-O3-NEXT: movq (%rdi), %rax 902; CHECK-O3-NEXT: movq (%rsi), %rcx 903; CHECK-O3-NEXT: movq %rax, %rdx 904; CHECK-O3-NEXT: orq %rcx, %rdx 905; CHECK-O3-NEXT: shrq $32, %rdx 906; CHECK-O3-NEXT: je .LBB39_1 907; CHECK-O3-NEXT: # %bb.2: 908; CHECK-O3-NEXT: xorl %edx, %edx 909; CHECK-O3-NEXT: divq %rcx 910; CHECK-O3-NEXT: retq 911; CHECK-O3-NEXT: .LBB39_1: 912; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 913; CHECK-O3-NEXT: xorl %edx, %edx 914; CHECK-O3-NEXT: divl %ecx 915; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 916; CHECK-O3-NEXT: retq 917 %v = load atomic i64, i64* %p1 unordered, align 8 918 %v2 = load atomic i64, i64* %p2 unordered, align 8 919 %ret = udiv i64 %v, %v2 920 ret i64 %ret 921} 922 923; Legal to fold (TODO) 924define i64 @load_fold_srem1(i64* %p) { 925; CHECK-O0-LABEL: load_fold_srem1: 926; CHECK-O0: # %bb.0: 927; CHECK-O0-NEXT: movq (%rdi), %rax 928; CHECK-O0-NEXT: cqto 929; CHECK-O0-NEXT: movl $15, %ecx 930; CHECK-O0-NEXT: idivq %rcx 931; CHECK-O0-NEXT: movq %rdx, %rax 932; CHECK-O0-NEXT: retq 933; 934; CHECK-O3-LABEL: load_fold_srem1: 935; CHECK-O3: # %bb.0: 936; CHECK-O3-NEXT: movq (%rdi), %rcx 937; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 938; CHECK-O3-NEXT: movq %rcx, %rax 939; CHECK-O3-NEXT: imulq %rdx 940; CHECK-O3-NEXT: addq %rcx, %rdx 941; CHECK-O3-NEXT: movq %rdx, %rax 942; CHECK-O3-NEXT: shrq $63, %rax 943; CHECK-O3-NEXT: sarq $3, %rdx 944; CHECK-O3-NEXT: addq %rax, %rdx 945; CHECK-O3-NEXT: leaq (%rdx,%rdx,4), %rax 946; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax 947; CHECK-O3-NEXT: subq %rax, %rcx 948; CHECK-O3-NEXT: movq %rcx, %rax 949; CHECK-O3-NEXT: retq 950 %v = load atomic i64, i64* %p unordered, align 8 951 %ret = srem i64 %v, 15 952 ret i64 %ret 953} 954 955; Legal, as expected 956define i64 @load_fold_srem2(i64* %p, i64 %v2) { 957; CHECK-O0-LABEL: load_fold_srem2: 958; CHECK-O0: # %bb.0: 959; CHECK-O0-NEXT: movq (%rdi), %rax 960; CHECK-O0-NEXT: cqto 961; CHECK-O0-NEXT: idivq %rsi 962; CHECK-O0-NEXT: movq %rdx, %rax 963; CHECK-O0-NEXT: retq 964; 965; CHECK-O3-LABEL: load_fold_srem2: 966; CHECK-O3: # %bb.0: 967; CHECK-O3-NEXT: movq (%rdi), %rax 968; CHECK-O3-NEXT: movq %rax, %rcx 969; CHECK-O3-NEXT: orq %rsi, %rcx 970; CHECK-O3-NEXT: shrq $32, %rcx 971; CHECK-O3-NEXT: je .LBB41_1 972; CHECK-O3-NEXT: # %bb.2: 973; CHECK-O3-NEXT: cqto 974; CHECK-O3-NEXT: idivq %rsi 975; CHECK-O3-NEXT: movq %rdx, %rax 976; CHECK-O3-NEXT: retq 977; CHECK-O3-NEXT: .LBB41_1: 978; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 979; CHECK-O3-NEXT: xorl %edx, %edx 980; CHECK-O3-NEXT: divl %esi 981; CHECK-O3-NEXT: movl %edx, %eax 982; CHECK-O3-NEXT: retq 983 %v = load atomic i64, i64* %p unordered, align 8 984 %ret = srem i64 %v, %v2 985 ret i64 %ret 986} 987 988define i64 @load_fold_srem3(i64* %p1, i64* %p2) { 989; CHECK-O0-LABEL: load_fold_srem3: 990; CHECK-O0: # %bb.0: 991; CHECK-O0-NEXT: movq (%rdi), %rax 992; CHECK-O0-NEXT: cqto 993; CHECK-O0-NEXT: idivq (%rsi) 994; CHECK-O0-NEXT: movq %rdx, %rax 995; CHECK-O0-NEXT: retq 996; 997; CHECK-O3-LABEL: load_fold_srem3: 998; CHECK-O3: # %bb.0: 999; CHECK-O3-NEXT: movq (%rdi), %rax 1000; CHECK-O3-NEXT: movq (%rsi), %rcx 1001; CHECK-O3-NEXT: movq %rax, %rdx 1002; CHECK-O3-NEXT: orq %rcx, %rdx 1003; CHECK-O3-NEXT: shrq $32, %rdx 1004; CHECK-O3-NEXT: je .LBB42_1 1005; CHECK-O3-NEXT: # %bb.2: 1006; CHECK-O3-NEXT: cqto 1007; CHECK-O3-NEXT: idivq %rcx 1008; CHECK-O3-NEXT: movq %rdx, %rax 1009; CHECK-O3-NEXT: retq 1010; CHECK-O3-NEXT: .LBB42_1: 1011; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1012; CHECK-O3-NEXT: xorl %edx, %edx 1013; CHECK-O3-NEXT: divl %ecx 1014; CHECK-O3-NEXT: movl %edx, %eax 1015; CHECK-O3-NEXT: retq 1016 %v = load atomic i64, i64* %p1 unordered, align 8 1017 %v2 = load atomic i64, i64* %p2 unordered, align 8 1018 %ret = srem i64 %v, %v2 1019 ret i64 %ret 1020} 1021 1022; Legal to fold (TODO) 1023define i64 @load_fold_urem1(i64* %p) { 1024; CHECK-O0-LABEL: load_fold_urem1: 1025; CHECK-O0: # %bb.0: 1026; CHECK-O0-NEXT: movq (%rdi), %rax 1027; CHECK-O0-NEXT: xorl %ecx, %ecx 1028; CHECK-O0-NEXT: movl %ecx, %edx 1029; CHECK-O0-NEXT: movl $15, %esi 1030; CHECK-O0-NEXT: divq %rsi 1031; CHECK-O0-NEXT: movq %rdx, %rax 1032; CHECK-O0-NEXT: retq 1033; 1034; CHECK-O3-LABEL: load_fold_urem1: 1035; CHECK-O3: # %bb.0: 1036; CHECK-O3-NEXT: movq (%rdi), %rcx 1037; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 1038; CHECK-O3-NEXT: movq %rcx, %rax 1039; CHECK-O3-NEXT: mulq %rdx 1040; CHECK-O3-NEXT: shrq $3, %rdx 1041; CHECK-O3-NEXT: leaq (%rdx,%rdx,4), %rax 1042; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax 1043; CHECK-O3-NEXT: subq %rax, %rcx 1044; CHECK-O3-NEXT: movq %rcx, %rax 1045; CHECK-O3-NEXT: retq 1046 %v = load atomic i64, i64* %p unordered, align 8 1047 %ret = urem i64 %v, 15 1048 ret i64 %ret 1049} 1050 1051; Legal, as expected 1052define i64 @load_fold_urem2(i64* %p, i64 %v2) { 1053; CHECK-O0-LABEL: load_fold_urem2: 1054; CHECK-O0: # %bb.0: 1055; CHECK-O0-NEXT: movq (%rdi), %rax 1056; CHECK-O0-NEXT: xorl %ecx, %ecx 1057; CHECK-O0-NEXT: movl %ecx, %edx 1058; CHECK-O0-NEXT: divq %rsi 1059; CHECK-O0-NEXT: movq %rdx, %rax 1060; CHECK-O0-NEXT: retq 1061; 1062; CHECK-O3-LABEL: load_fold_urem2: 1063; CHECK-O3: # %bb.0: 1064; CHECK-O3-NEXT: movq (%rdi), %rax 1065; CHECK-O3-NEXT: movq %rax, %rcx 1066; CHECK-O3-NEXT: orq %rsi, %rcx 1067; CHECK-O3-NEXT: shrq $32, %rcx 1068; CHECK-O3-NEXT: je .LBB44_1 1069; CHECK-O3-NEXT: # %bb.2: 1070; CHECK-O3-NEXT: xorl %edx, %edx 1071; CHECK-O3-NEXT: divq %rsi 1072; CHECK-O3-NEXT: movq %rdx, %rax 1073; CHECK-O3-NEXT: retq 1074; CHECK-O3-NEXT: .LBB44_1: 1075; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1076; CHECK-O3-NEXT: xorl %edx, %edx 1077; CHECK-O3-NEXT: divl %esi 1078; CHECK-O3-NEXT: movl %edx, %eax 1079; CHECK-O3-NEXT: retq 1080 %v = load atomic i64, i64* %p unordered, align 8 1081 %ret = urem i64 %v, %v2 1082 ret i64 %ret 1083} 1084 1085define i64 @load_fold_urem3(i64* %p1, i64* %p2) { 1086; CHECK-O0-LABEL: load_fold_urem3: 1087; CHECK-O0: # %bb.0: 1088; CHECK-O0-NEXT: movq (%rdi), %rax 1089; CHECK-O0-NEXT: xorl %ecx, %ecx 1090; CHECK-O0-NEXT: movl %ecx, %edx 1091; CHECK-O0-NEXT: divq (%rsi) 1092; CHECK-O0-NEXT: movq %rdx, %rax 1093; CHECK-O0-NEXT: retq 1094; 1095; CHECK-O3-LABEL: load_fold_urem3: 1096; CHECK-O3: # %bb.0: 1097; CHECK-O3-NEXT: movq (%rdi), %rax 1098; CHECK-O3-NEXT: movq (%rsi), %rcx 1099; CHECK-O3-NEXT: movq %rax, %rdx 1100; CHECK-O3-NEXT: orq %rcx, %rdx 1101; CHECK-O3-NEXT: shrq $32, %rdx 1102; CHECK-O3-NEXT: je .LBB45_1 1103; CHECK-O3-NEXT: # %bb.2: 1104; CHECK-O3-NEXT: xorl %edx, %edx 1105; CHECK-O3-NEXT: divq %rcx 1106; CHECK-O3-NEXT: movq %rdx, %rax 1107; CHECK-O3-NEXT: retq 1108; CHECK-O3-NEXT: .LBB45_1: 1109; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1110; CHECK-O3-NEXT: xorl %edx, %edx 1111; CHECK-O3-NEXT: divl %ecx 1112; CHECK-O3-NEXT: movl %edx, %eax 1113; CHECK-O3-NEXT: retq 1114 %v = load atomic i64, i64* %p1 unordered, align 8 1115 %v2 = load atomic i64, i64* %p2 unordered, align 8 1116 %ret = urem i64 %v, %v2 1117 ret i64 %ret 1118} 1119 1120; Legal, as expected 1121define i64 @load_fold_shl1(i64* %p) { 1122; CHECK-LABEL: load_fold_shl1: 1123; CHECK: # %bb.0: 1124; CHECK-NEXT: movq (%rdi), %rax 1125; CHECK-NEXT: shlq $15, %rax 1126; CHECK-NEXT: retq 1127 %v = load atomic i64, i64* %p unordered, align 8 1128 %ret = shl i64 %v, 15 1129 ret i64 %ret 1130} 1131 1132define i64 @load_fold_shl2(i64* %p, i64 %v2) { 1133; CHECK-O0-LABEL: load_fold_shl2: 1134; CHECK-O0: # %bb.0: 1135; CHECK-O0-NEXT: movq (%rdi), %rax 1136; CHECK-O0-NEXT: movq %rsi, %rcx 1137; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1138; CHECK-O0-NEXT: shlq %cl, %rax 1139; CHECK-O0-NEXT: retq 1140; 1141; CHECK-O3-LABEL: load_fold_shl2: 1142; CHECK-O3: # %bb.0: 1143; CHECK-O3-NEXT: shlxq %rsi, (%rdi), %rax 1144; CHECK-O3-NEXT: retq 1145 %v = load atomic i64, i64* %p unordered, align 8 1146 %ret = shl i64 %v, %v2 1147 ret i64 %ret 1148} 1149 1150define i64 @load_fold_shl3(i64* %p1, i64* %p2) { 1151; CHECK-O0-LABEL: load_fold_shl3: 1152; CHECK-O0: # %bb.0: 1153; CHECK-O0-NEXT: movq (%rdi), %rax 1154; CHECK-O0-NEXT: movq (%rsi), %rcx 1155; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1156; CHECK-O0-NEXT: shlq %cl, %rax 1157; CHECK-O0-NEXT: retq 1158; 1159; CHECK-O3-LABEL: load_fold_shl3: 1160; CHECK-O3: # %bb.0: 1161; CHECK-O3-NEXT: movq (%rsi), %rax 1162; CHECK-O3-NEXT: shlxq %rax, (%rdi), %rax 1163; CHECK-O3-NEXT: retq 1164 %v = load atomic i64, i64* %p1 unordered, align 8 1165 %v2 = load atomic i64, i64* %p2 unordered, align 8 1166 %ret = shl i64 %v, %v2 1167 ret i64 %ret 1168} 1169 1170; Legal, as expected 1171define i64 @load_fold_lshr1(i64* %p) { 1172; CHECK-LABEL: load_fold_lshr1: 1173; CHECK: # %bb.0: 1174; CHECK-NEXT: movq (%rdi), %rax 1175; CHECK-NEXT: shrq $15, %rax 1176; CHECK-NEXT: retq 1177 %v = load atomic i64, i64* %p unordered, align 8 1178 %ret = lshr i64 %v, 15 1179 ret i64 %ret 1180} 1181 1182define i64 @load_fold_lshr2(i64* %p, i64 %v2) { 1183; CHECK-O0-LABEL: load_fold_lshr2: 1184; CHECK-O0: # %bb.0: 1185; CHECK-O0-NEXT: movq (%rdi), %rax 1186; CHECK-O0-NEXT: movq %rsi, %rcx 1187; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1188; CHECK-O0-NEXT: shrq %cl, %rax 1189; CHECK-O0-NEXT: retq 1190; 1191; CHECK-O3-LABEL: load_fold_lshr2: 1192; CHECK-O3: # %bb.0: 1193; CHECK-O3-NEXT: shrxq %rsi, (%rdi), %rax 1194; CHECK-O3-NEXT: retq 1195 %v = load atomic i64, i64* %p unordered, align 8 1196 %ret = lshr i64 %v, %v2 1197 ret i64 %ret 1198} 1199 1200define i64 @load_fold_lshr3(i64* %p1, i64* %p2) { 1201; CHECK-O0-LABEL: load_fold_lshr3: 1202; CHECK-O0: # %bb.0: 1203; CHECK-O0-NEXT: movq (%rdi), %rax 1204; CHECK-O0-NEXT: movq (%rsi), %rcx 1205; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1206; CHECK-O0-NEXT: shrq %cl, %rax 1207; CHECK-O0-NEXT: retq 1208; 1209; CHECK-O3-LABEL: load_fold_lshr3: 1210; CHECK-O3: # %bb.0: 1211; CHECK-O3-NEXT: movq (%rsi), %rax 1212; CHECK-O3-NEXT: shrxq %rax, (%rdi), %rax 1213; CHECK-O3-NEXT: retq 1214 %v = load atomic i64, i64* %p1 unordered, align 8 1215 %v2 = load atomic i64, i64* %p2 unordered, align 8 1216 %ret = lshr i64 %v, %v2 1217 ret i64 %ret 1218} 1219 1220; Legal, as expected 1221define i64 @load_fold_ashr1(i64* %p) { 1222; CHECK-LABEL: load_fold_ashr1: 1223; CHECK: # %bb.0: 1224; CHECK-NEXT: movq (%rdi), %rax 1225; CHECK-NEXT: sarq $15, %rax 1226; CHECK-NEXT: retq 1227 %v = load atomic i64, i64* %p unordered, align 8 1228 %ret = ashr i64 %v, 15 1229 ret i64 %ret 1230} 1231 1232define i64 @load_fold_ashr2(i64* %p, i64 %v2) { 1233; CHECK-O0-LABEL: load_fold_ashr2: 1234; CHECK-O0: # %bb.0: 1235; CHECK-O0-NEXT: movq (%rdi), %rax 1236; CHECK-O0-NEXT: movq %rsi, %rcx 1237; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1238; CHECK-O0-NEXT: sarq %cl, %rax 1239; CHECK-O0-NEXT: retq 1240; 1241; CHECK-O3-LABEL: load_fold_ashr2: 1242; CHECK-O3: # %bb.0: 1243; CHECK-O3-NEXT: sarxq %rsi, (%rdi), %rax 1244; CHECK-O3-NEXT: retq 1245 %v = load atomic i64, i64* %p unordered, align 8 1246 %ret = ashr i64 %v, %v2 1247 ret i64 %ret 1248} 1249 1250define i64 @load_fold_ashr3(i64* %p1, i64* %p2) { 1251; CHECK-O0-LABEL: load_fold_ashr3: 1252; CHECK-O0: # %bb.0: 1253; CHECK-O0-NEXT: movq (%rdi), %rax 1254; CHECK-O0-NEXT: movq (%rsi), %rcx 1255; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1256; CHECK-O0-NEXT: sarq %cl, %rax 1257; CHECK-O0-NEXT: retq 1258; 1259; CHECK-O3-LABEL: load_fold_ashr3: 1260; CHECK-O3: # %bb.0: 1261; CHECK-O3-NEXT: movq (%rsi), %rax 1262; CHECK-O3-NEXT: sarxq %rax, (%rdi), %rax 1263; CHECK-O3-NEXT: retq 1264 %v = load atomic i64, i64* %p1 unordered, align 8 1265 %v2 = load atomic i64, i64* %p2 unordered, align 8 1266 %ret = ashr i64 %v, %v2 1267 ret i64 %ret 1268} 1269 1270; Legal, as expected 1271define i64 @load_fold_and1(i64* %p) { 1272; CHECK-O0-LABEL: load_fold_and1: 1273; CHECK-O0: # %bb.0: 1274; CHECK-O0-NEXT: movq (%rdi), %rax 1275; CHECK-O0-NEXT: andq $15, %rax 1276; CHECK-O0-NEXT: retq 1277; 1278; CHECK-O3-LABEL: load_fold_and1: 1279; CHECK-O3: # %bb.0: 1280; CHECK-O3-NEXT: movq (%rdi), %rax 1281; CHECK-O3-NEXT: andl $15, %eax 1282; CHECK-O3-NEXT: retq 1283 %v = load atomic i64, i64* %p unordered, align 8 1284 %ret = and i64 %v, 15 1285 ret i64 %ret 1286} 1287 1288define i64 @load_fold_and2(i64* %p, i64 %v2) { 1289; CHECK-O0-LABEL: load_fold_and2: 1290; CHECK-O0: # %bb.0: 1291; CHECK-O0-NEXT: andq (%rdi), %rsi 1292; CHECK-O0-NEXT: movq %rsi, %rax 1293; CHECK-O0-NEXT: retq 1294; 1295; CHECK-O3-LABEL: load_fold_and2: 1296; CHECK-O3: # %bb.0: 1297; CHECK-O3-NEXT: movq %rsi, %rax 1298; CHECK-O3-NEXT: andq (%rdi), %rax 1299; CHECK-O3-NEXT: retq 1300 %v = load atomic i64, i64* %p unordered, align 8 1301 %ret = and i64 %v, %v2 1302 ret i64 %ret 1303} 1304 1305define i64 @load_fold_and3(i64* %p1, i64* %p2) { 1306; CHECK-O0-LABEL: load_fold_and3: 1307; CHECK-O0: # %bb.0: 1308; CHECK-O0-NEXT: movq (%rdi), %rax 1309; CHECK-O0-NEXT: andq (%rsi), %rax 1310; CHECK-O0-NEXT: retq 1311; 1312; CHECK-O3-CUR-LABEL: load_fold_and3: 1313; CHECK-O3-CUR: # %bb.0: 1314; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 1315; CHECK-O3-CUR-NEXT: andq (%rdi), %rax 1316; CHECK-O3-CUR-NEXT: retq 1317; 1318; CHECK-O3-EX-LABEL: load_fold_and3: 1319; CHECK-O3-EX: # %bb.0: 1320; CHECK-O3-EX-NEXT: movq (%rdi), %rax 1321; CHECK-O3-EX-NEXT: andq (%rsi), %rax 1322; CHECK-O3-EX-NEXT: retq 1323 %v = load atomic i64, i64* %p1 unordered, align 8 1324 %v2 = load atomic i64, i64* %p2 unordered, align 8 1325 %ret = and i64 %v, %v2 1326 ret i64 %ret 1327} 1328 1329; Legal, as expected 1330define i64 @load_fold_or1(i64* %p) { 1331; CHECK-LABEL: load_fold_or1: 1332; CHECK: # %bb.0: 1333; CHECK-NEXT: movq (%rdi), %rax 1334; CHECK-NEXT: orq $15, %rax 1335; CHECK-NEXT: retq 1336 %v = load atomic i64, i64* %p unordered, align 8 1337 %ret = or i64 %v, 15 1338 ret i64 %ret 1339} 1340 1341define i64 @load_fold_or2(i64* %p, i64 %v2) { 1342; CHECK-O0-LABEL: load_fold_or2: 1343; CHECK-O0: # %bb.0: 1344; CHECK-O0-NEXT: orq (%rdi), %rsi 1345; CHECK-O0-NEXT: movq %rsi, %rax 1346; CHECK-O0-NEXT: retq 1347; 1348; CHECK-O3-LABEL: load_fold_or2: 1349; CHECK-O3: # %bb.0: 1350; CHECK-O3-NEXT: movq %rsi, %rax 1351; CHECK-O3-NEXT: orq (%rdi), %rax 1352; CHECK-O3-NEXT: retq 1353 %v = load atomic i64, i64* %p unordered, align 8 1354 %ret = or i64 %v, %v2 1355 ret i64 %ret 1356} 1357 1358define i64 @load_fold_or3(i64* %p1, i64* %p2) { 1359; CHECK-O0-LABEL: load_fold_or3: 1360; CHECK-O0: # %bb.0: 1361; CHECK-O0-NEXT: movq (%rdi), %rax 1362; CHECK-O0-NEXT: orq (%rsi), %rax 1363; CHECK-O0-NEXT: retq 1364; 1365; CHECK-O3-CUR-LABEL: load_fold_or3: 1366; CHECK-O3-CUR: # %bb.0: 1367; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 1368; CHECK-O3-CUR-NEXT: orq (%rdi), %rax 1369; CHECK-O3-CUR-NEXT: retq 1370; 1371; CHECK-O3-EX-LABEL: load_fold_or3: 1372; CHECK-O3-EX: # %bb.0: 1373; CHECK-O3-EX-NEXT: movq (%rdi), %rax 1374; CHECK-O3-EX-NEXT: orq (%rsi), %rax 1375; CHECK-O3-EX-NEXT: retq 1376 %v = load atomic i64, i64* %p1 unordered, align 8 1377 %v2 = load atomic i64, i64* %p2 unordered, align 8 1378 %ret = or i64 %v, %v2 1379 ret i64 %ret 1380} 1381 1382; Legal, as expected 1383define i64 @load_fold_xor1(i64* %p) { 1384; CHECK-LABEL: load_fold_xor1: 1385; CHECK: # %bb.0: 1386; CHECK-NEXT: movq (%rdi), %rax 1387; CHECK-NEXT: xorq $15, %rax 1388; CHECK-NEXT: retq 1389 %v = load atomic i64, i64* %p unordered, align 8 1390 %ret = xor i64 %v, 15 1391 ret i64 %ret 1392} 1393 1394define i64 @load_fold_xor2(i64* %p, i64 %v2) { 1395; CHECK-O0-LABEL: load_fold_xor2: 1396; CHECK-O0: # %bb.0: 1397; CHECK-O0-NEXT: xorq (%rdi), %rsi 1398; CHECK-O0-NEXT: movq %rsi, %rax 1399; CHECK-O0-NEXT: retq 1400; 1401; CHECK-O3-LABEL: load_fold_xor2: 1402; CHECK-O3: # %bb.0: 1403; CHECK-O3-NEXT: movq %rsi, %rax 1404; CHECK-O3-NEXT: xorq (%rdi), %rax 1405; CHECK-O3-NEXT: retq 1406 %v = load atomic i64, i64* %p unordered, align 8 1407 %ret = xor i64 %v, %v2 1408 ret i64 %ret 1409} 1410 1411define i64 @load_fold_xor3(i64* %p1, i64* %p2) { 1412; CHECK-O0-LABEL: load_fold_xor3: 1413; CHECK-O0: # %bb.0: 1414; CHECK-O0-NEXT: movq (%rdi), %rax 1415; CHECK-O0-NEXT: xorq (%rsi), %rax 1416; CHECK-O0-NEXT: retq 1417; 1418; CHECK-O3-CUR-LABEL: load_fold_xor3: 1419; CHECK-O3-CUR: # %bb.0: 1420; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 1421; CHECK-O3-CUR-NEXT: xorq (%rdi), %rax 1422; CHECK-O3-CUR-NEXT: retq 1423; 1424; CHECK-O3-EX-LABEL: load_fold_xor3: 1425; CHECK-O3-EX: # %bb.0: 1426; CHECK-O3-EX-NEXT: movq (%rdi), %rax 1427; CHECK-O3-EX-NEXT: xorq (%rsi), %rax 1428; CHECK-O3-EX-NEXT: retq 1429 %v = load atomic i64, i64* %p1 unordered, align 8 1430 %v2 = load atomic i64, i64* %p2 unordered, align 8 1431 %ret = xor i64 %v, %v2 1432 ret i64 %ret 1433} 1434 1435define i1 @load_fold_icmp1(i64* %p) { 1436; CHECK-O0-LABEL: load_fold_icmp1: 1437; CHECK-O0: # %bb.0: 1438; CHECK-O0-NEXT: movq (%rdi), %rax 1439; CHECK-O0-NEXT: subq $15, %rax 1440; CHECK-O0-NEXT: sete %cl 1441; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1442; CHECK-O0-NEXT: movb %cl, %al 1443; CHECK-O0-NEXT: retq 1444; 1445; CHECK-O3-LABEL: load_fold_icmp1: 1446; CHECK-O3: # %bb.0: 1447; CHECK-O3-NEXT: cmpq $15, (%rdi) 1448; CHECK-O3-NEXT: sete %al 1449; CHECK-O3-NEXT: retq 1450 %v = load atomic i64, i64* %p unordered, align 8 1451 %ret = icmp eq i64 %v, 15 1452 ret i1 %ret 1453} 1454 1455define i1 @load_fold_icmp2(i64* %p, i64 %v2) { 1456; CHECK-O0-LABEL: load_fold_icmp2: 1457; CHECK-O0: # %bb.0: 1458; CHECK-O0-NEXT: movq (%rdi), %rax 1459; CHECK-O0-NEXT: subq %rsi, %rax 1460; CHECK-O0-NEXT: sete %cl 1461; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1462; CHECK-O0-NEXT: movb %cl, %al 1463; CHECK-O0-NEXT: retq 1464; 1465; CHECK-O3-LABEL: load_fold_icmp2: 1466; CHECK-O3: # %bb.0: 1467; CHECK-O3-NEXT: cmpq %rsi, (%rdi) 1468; CHECK-O3-NEXT: sete %al 1469; CHECK-O3-NEXT: retq 1470 %v = load atomic i64, i64* %p unordered, align 8 1471 %ret = icmp eq i64 %v, %v2 1472 ret i1 %ret 1473} 1474 1475define i1 @load_fold_icmp3(i64* %p1, i64* %p2) { 1476; CHECK-O0-LABEL: load_fold_icmp3: 1477; CHECK-O0: # %bb.0: 1478; CHECK-O0-NEXT: movq (%rdi), %rax 1479; CHECK-O0-NEXT: movq (%rsi), %rcx 1480; CHECK-O0-NEXT: subq %rcx, %rax 1481; CHECK-O0-NEXT: sete %dl 1482; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1483; CHECK-O0-NEXT: movb %dl, %al 1484; CHECK-O0-NEXT: retq 1485; 1486; CHECK-O3-CUR-LABEL: load_fold_icmp3: 1487; CHECK-O3-CUR: # %bb.0: 1488; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 1489; CHECK-O3-CUR-NEXT: cmpq %rax, (%rdi) 1490; CHECK-O3-CUR-NEXT: sete %al 1491; CHECK-O3-CUR-NEXT: retq 1492; 1493; CHECK-O3-EX-LABEL: load_fold_icmp3: 1494; CHECK-O3-EX: # %bb.0: 1495; CHECK-O3-EX-NEXT: movq (%rdi), %rax 1496; CHECK-O3-EX-NEXT: cmpq (%rsi), %rax 1497; CHECK-O3-EX-NEXT: sete %al 1498; CHECK-O3-EX-NEXT: retq 1499 %v = load atomic i64, i64* %p1 unordered, align 8 1500 %v2 = load atomic i64, i64* %p2 unordered, align 8 1501 %ret = icmp eq i64 %v, %v2 1502 ret i1 %ret 1503} 1504 1505 1506;; The next batch of tests check for read-modify-write patterns 1507;; Legally, it's okay to use a memory operand here as long as the operand 1508;; is well aligned (i.e. doesn't cross a cache line boundary). We are 1509;; required not to narrow the store though! 1510 1511; Legal, as expected 1512define void @rmw_fold_add1(i64* %p, i64 %v) { 1513; CHECK-O0-LABEL: rmw_fold_add1: 1514; CHECK-O0: # %bb.0: 1515; CHECK-O0-NEXT: movq (%rdi), %rax 1516; CHECK-O0-NEXT: addq $15, %rax 1517; CHECK-O0-NEXT: movq %rax, (%rdi) 1518; CHECK-O0-NEXT: retq 1519; 1520; CHECK-O3-LABEL: rmw_fold_add1: 1521; CHECK-O3: # %bb.0: 1522; CHECK-O3-NEXT: addq $15, (%rdi) 1523; CHECK-O3-NEXT: retq 1524 %prev = load atomic i64, i64* %p unordered, align 8 1525 %val = add i64 %prev, 15 1526 store atomic i64 %val, i64* %p unordered, align 8 1527 ret void 1528} 1529 1530; Legal, as expected 1531define void @rmw_fold_add2(i64* %p, i64 %v) { 1532; CHECK-O0-LABEL: rmw_fold_add2: 1533; CHECK-O0: # %bb.0: 1534; CHECK-O0-NEXT: movq (%rdi), %rax 1535; CHECK-O0-NEXT: addq %rsi, %rax 1536; CHECK-O0-NEXT: movq %rax, (%rdi) 1537; CHECK-O0-NEXT: retq 1538; 1539; CHECK-O3-LABEL: rmw_fold_add2: 1540; CHECK-O3: # %bb.0: 1541; CHECK-O3-NEXT: addq %rsi, (%rdi) 1542; CHECK-O3-NEXT: retq 1543 %prev = load atomic i64, i64* %p unordered, align 8 1544 %val = add i64 %prev, %v 1545 store atomic i64 %val, i64* %p unordered, align 8 1546 ret void 1547} 1548 1549; Legal, as expected 1550define void @rmw_fold_sub1(i64* %p, i64 %v) { 1551; CHECK-O0-LABEL: rmw_fold_sub1: 1552; CHECK-O0: # %bb.0: 1553; CHECK-O0-NEXT: movq (%rdi), %rax 1554; CHECK-O0-NEXT: addq $-15, %rax 1555; CHECK-O0-NEXT: movq %rax, (%rdi) 1556; CHECK-O0-NEXT: retq 1557; 1558; CHECK-O3-LABEL: rmw_fold_sub1: 1559; CHECK-O3: # %bb.0: 1560; CHECK-O3-NEXT: addq $-15, (%rdi) 1561; CHECK-O3-NEXT: retq 1562 %prev = load atomic i64, i64* %p unordered, align 8 1563 %val = sub i64 %prev, 15 1564 store atomic i64 %val, i64* %p unordered, align 8 1565 ret void 1566} 1567 1568; Legal, as expected 1569define void @rmw_fold_sub2(i64* %p, i64 %v) { 1570; CHECK-O0-LABEL: rmw_fold_sub2: 1571; CHECK-O0: # %bb.0: 1572; CHECK-O0-NEXT: movq (%rdi), %rax 1573; CHECK-O0-NEXT: subq %rsi, %rax 1574; CHECK-O0-NEXT: movq %rax, (%rdi) 1575; CHECK-O0-NEXT: retq 1576; 1577; CHECK-O3-LABEL: rmw_fold_sub2: 1578; CHECK-O3: # %bb.0: 1579; CHECK-O3-NEXT: subq %rsi, (%rdi) 1580; CHECK-O3-NEXT: retq 1581 %prev = load atomic i64, i64* %p unordered, align 8 1582 %val = sub i64 %prev, %v 1583 store atomic i64 %val, i64* %p unordered, align 8 1584 ret void 1585} 1586 1587; Legal, as expected 1588define void @rmw_fold_mul1(i64* %p, i64 %v) { 1589; CHECK-LABEL: rmw_fold_mul1: 1590; CHECK: # %bb.0: 1591; CHECK-NEXT: movq (%rdi), %rax 1592; CHECK-NEXT: leaq (%rax,%rax,4), %rax 1593; CHECK-NEXT: leaq (%rax,%rax,2), %rax 1594; CHECK-NEXT: movq %rax, (%rdi) 1595; CHECK-NEXT: retq 1596 %prev = load atomic i64, i64* %p unordered, align 8 1597 %val = mul i64 %prev, 15 1598 store atomic i64 %val, i64* %p unordered, align 8 1599 ret void 1600} 1601 1602; Legal to fold (TODO) 1603define void @rmw_fold_mul2(i64* %p, i64 %v) { 1604; CHECK-O0-LABEL: rmw_fold_mul2: 1605; CHECK-O0: # %bb.0: 1606; CHECK-O0-NEXT: movq (%rdi), %rax 1607; CHECK-O0-NEXT: imulq %rsi, %rax 1608; CHECK-O0-NEXT: movq %rax, (%rdi) 1609; CHECK-O0-NEXT: retq 1610; 1611; CHECK-O3-LABEL: rmw_fold_mul2: 1612; CHECK-O3: # %bb.0: 1613; CHECK-O3-NEXT: imulq (%rdi), %rsi 1614; CHECK-O3-NEXT: movq %rsi, (%rdi) 1615; CHECK-O3-NEXT: retq 1616 %prev = load atomic i64, i64* %p unordered, align 8 1617 %val = mul i64 %prev, %v 1618 store atomic i64 %val, i64* %p unordered, align 8 1619 ret void 1620} 1621 1622; Legal, as expected 1623define void @rmw_fold_sdiv1(i64* %p, i64 %v) { 1624; CHECK-O0-LABEL: rmw_fold_sdiv1: 1625; CHECK-O0: # %bb.0: 1626; CHECK-O0-NEXT: movq (%rdi), %rax 1627; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889 1628; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1629; CHECK-O0-NEXT: imulq %rcx 1630; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 1631; CHECK-O0-NEXT: addq %rax, %rdx 1632; CHECK-O0-NEXT: movq %rdx, %rcx 1633; CHECK-O0-NEXT: shrq $63, %rcx 1634; CHECK-O0-NEXT: sarq $3, %rdx 1635; CHECK-O0-NEXT: addq %rcx, %rdx 1636; CHECK-O0-NEXT: movq %rdx, (%rdi) 1637; CHECK-O0-NEXT: retq 1638; 1639; CHECK-O3-LABEL: rmw_fold_sdiv1: 1640; CHECK-O3: # %bb.0: 1641; CHECK-O3-NEXT: movq (%rdi), %rcx 1642; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 1643; CHECK-O3-NEXT: movq %rcx, %rax 1644; CHECK-O3-NEXT: imulq %rdx 1645; CHECK-O3-NEXT: addq %rcx, %rdx 1646; CHECK-O3-NEXT: movq %rdx, %rax 1647; CHECK-O3-NEXT: shrq $63, %rax 1648; CHECK-O3-NEXT: sarq $3, %rdx 1649; CHECK-O3-NEXT: addq %rax, %rdx 1650; CHECK-O3-NEXT: movq %rdx, (%rdi) 1651; CHECK-O3-NEXT: retq 1652 %prev = load atomic i64, i64* %p unordered, align 8 1653 %val = sdiv i64 %prev, 15 1654 store atomic i64 %val, i64* %p unordered, align 8 1655 ret void 1656} 1657 1658; Legal, as expected 1659define void @rmw_fold_sdiv2(i64* %p, i64 %v) { 1660; CHECK-O0-LABEL: rmw_fold_sdiv2: 1661; CHECK-O0: # %bb.0: 1662; CHECK-O0-NEXT: movq (%rdi), %rax 1663; CHECK-O0-NEXT: cqto 1664; CHECK-O0-NEXT: idivq %rsi 1665; CHECK-O0-NEXT: movq %rax, (%rdi) 1666; CHECK-O0-NEXT: retq 1667; 1668; CHECK-O3-LABEL: rmw_fold_sdiv2: 1669; CHECK-O3: # %bb.0: 1670; CHECK-O3-NEXT: movq (%rdi), %rax 1671; CHECK-O3-NEXT: movq %rax, %rcx 1672; CHECK-O3-NEXT: orq %rsi, %rcx 1673; CHECK-O3-NEXT: shrq $32, %rcx 1674; CHECK-O3-NEXT: je .LBB74_1 1675; CHECK-O3-NEXT: # %bb.2: 1676; CHECK-O3-NEXT: cqto 1677; CHECK-O3-NEXT: idivq %rsi 1678; CHECK-O3-NEXT: movq %rax, (%rdi) 1679; CHECK-O3-NEXT: retq 1680; CHECK-O3-NEXT: .LBB74_1: 1681; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1682; CHECK-O3-NEXT: xorl %edx, %edx 1683; CHECK-O3-NEXT: divl %esi 1684; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 1685; CHECK-O3-NEXT: movq %rax, (%rdi) 1686; CHECK-O3-NEXT: retq 1687 %prev = load atomic i64, i64* %p unordered, align 8 1688 %val = sdiv i64 %prev, %v 1689 store atomic i64 %val, i64* %p unordered, align 8 1690 ret void 1691} 1692 1693; Legal, as expected 1694define void @rmw_fold_udiv1(i64* %p, i64 %v) { 1695; CHECK-O0-LABEL: rmw_fold_udiv1: 1696; CHECK-O0: # %bb.0: 1697; CHECK-O0-NEXT: movq (%rdi), %rax 1698; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889 1699; CHECK-O0-NEXT: mulq %rcx 1700; CHECK-O0-NEXT: shrq $3, %rdx 1701; CHECK-O0-NEXT: movq %rdx, (%rdi) 1702; CHECK-O0-NEXT: retq 1703; 1704; CHECK-O3-CUR-LABEL: rmw_fold_udiv1: 1705; CHECK-O3-CUR: # %bb.0: 1706; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 1707; CHECK-O3-CUR-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889 1708; CHECK-O3-CUR-NEXT: mulq %rcx 1709; CHECK-O3-CUR-NEXT: shrq $3, %rdx 1710; CHECK-O3-CUR-NEXT: movq %rdx, (%rdi) 1711; CHECK-O3-CUR-NEXT: retq 1712; 1713; CHECK-O3-EX-LABEL: rmw_fold_udiv1: 1714; CHECK-O3-EX: # %bb.0: 1715; CHECK-O3-EX-NEXT: movabsq $-8608480567731124087, %rax # imm = 0x8888888888888889 1716; CHECK-O3-EX-NEXT: mulq (%rdi) 1717; CHECK-O3-EX-NEXT: shrq $3, %rdx 1718; CHECK-O3-EX-NEXT: movq %rdx, (%rdi) 1719; CHECK-O3-EX-NEXT: retq 1720 %prev = load atomic i64, i64* %p unordered, align 8 1721 %val = udiv i64 %prev, 15 1722 store atomic i64 %val, i64* %p unordered, align 8 1723 ret void 1724} 1725 1726; Legal, as expected 1727define void @rmw_fold_udiv2(i64* %p, i64 %v) { 1728; CHECK-O0-LABEL: rmw_fold_udiv2: 1729; CHECK-O0: # %bb.0: 1730; CHECK-O0-NEXT: movq (%rdi), %rax 1731; CHECK-O0-NEXT: xorl %ecx, %ecx 1732; CHECK-O0-NEXT: movl %ecx, %edx 1733; CHECK-O0-NEXT: divq %rsi 1734; CHECK-O0-NEXT: movq %rax, (%rdi) 1735; CHECK-O0-NEXT: retq 1736; 1737; CHECK-O3-LABEL: rmw_fold_udiv2: 1738; CHECK-O3: # %bb.0: 1739; CHECK-O3-NEXT: movq (%rdi), %rax 1740; CHECK-O3-NEXT: movq %rax, %rcx 1741; CHECK-O3-NEXT: orq %rsi, %rcx 1742; CHECK-O3-NEXT: shrq $32, %rcx 1743; CHECK-O3-NEXT: je .LBB76_1 1744; CHECK-O3-NEXT: # %bb.2: 1745; CHECK-O3-NEXT: xorl %edx, %edx 1746; CHECK-O3-NEXT: divq %rsi 1747; CHECK-O3-NEXT: movq %rax, (%rdi) 1748; CHECK-O3-NEXT: retq 1749; CHECK-O3-NEXT: .LBB76_1: 1750; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1751; CHECK-O3-NEXT: xorl %edx, %edx 1752; CHECK-O3-NEXT: divl %esi 1753; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 1754; CHECK-O3-NEXT: movq %rax, (%rdi) 1755; CHECK-O3-NEXT: retq 1756 %prev = load atomic i64, i64* %p unordered, align 8 1757 %val = udiv i64 %prev, %v 1758 store atomic i64 %val, i64* %p unordered, align 8 1759 ret void 1760} 1761 1762; Legal, as expected 1763define void @rmw_fold_srem1(i64* %p, i64 %v) { 1764; CHECK-O0-LABEL: rmw_fold_srem1: 1765; CHECK-O0: # %bb.0: 1766; CHECK-O0-NEXT: movq (%rdi), %rax 1767; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889 1768; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1769; CHECK-O0-NEXT: imulq %rcx 1770; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 1771; CHECK-O0-NEXT: addq %rax, %rdx 1772; CHECK-O0-NEXT: movq %rdx, %rcx 1773; CHECK-O0-NEXT: shrq $63, %rcx 1774; CHECK-O0-NEXT: sarq $3, %rdx 1775; CHECK-O0-NEXT: addq %rcx, %rdx 1776; CHECK-O0-NEXT: leaq (%rdx,%rdx,4), %rcx 1777; CHECK-O0-NEXT: leaq (%rcx,%rcx,2), %rcx 1778; CHECK-O0-NEXT: subq %rcx, %rax 1779; CHECK-O0-NEXT: movq %rax, (%rdi) 1780; CHECK-O0-NEXT: retq 1781; 1782; CHECK-O3-LABEL: rmw_fold_srem1: 1783; CHECK-O3: # %bb.0: 1784; CHECK-O3-NEXT: movq (%rdi), %rcx 1785; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 1786; CHECK-O3-NEXT: movq %rcx, %rax 1787; CHECK-O3-NEXT: imulq %rdx 1788; CHECK-O3-NEXT: addq %rcx, %rdx 1789; CHECK-O3-NEXT: movq %rdx, %rax 1790; CHECK-O3-NEXT: shrq $63, %rax 1791; CHECK-O3-NEXT: sarq $3, %rdx 1792; CHECK-O3-NEXT: addq %rax, %rdx 1793; CHECK-O3-NEXT: leaq (%rdx,%rdx,4), %rax 1794; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax 1795; CHECK-O3-NEXT: subq %rax, %rcx 1796; CHECK-O3-NEXT: movq %rcx, (%rdi) 1797; CHECK-O3-NEXT: retq 1798 %prev = load atomic i64, i64* %p unordered, align 8 1799 %val = srem i64 %prev, 15 1800 store atomic i64 %val, i64* %p unordered, align 8 1801 ret void 1802} 1803 1804; Legal, as expected 1805define void @rmw_fold_srem2(i64* %p, i64 %v) { 1806; CHECK-O0-LABEL: rmw_fold_srem2: 1807; CHECK-O0: # %bb.0: 1808; CHECK-O0-NEXT: movq (%rdi), %rax 1809; CHECK-O0-NEXT: cqto 1810; CHECK-O0-NEXT: idivq %rsi 1811; CHECK-O0-NEXT: movq %rdx, (%rdi) 1812; CHECK-O0-NEXT: retq 1813; 1814; CHECK-O3-LABEL: rmw_fold_srem2: 1815; CHECK-O3: # %bb.0: 1816; CHECK-O3-NEXT: movq (%rdi), %rax 1817; CHECK-O3-NEXT: movq %rax, %rcx 1818; CHECK-O3-NEXT: orq %rsi, %rcx 1819; CHECK-O3-NEXT: shrq $32, %rcx 1820; CHECK-O3-NEXT: je .LBB78_1 1821; CHECK-O3-NEXT: # %bb.2: 1822; CHECK-O3-NEXT: cqto 1823; CHECK-O3-NEXT: idivq %rsi 1824; CHECK-O3-NEXT: movq %rdx, (%rdi) 1825; CHECK-O3-NEXT: retq 1826; CHECK-O3-NEXT: .LBB78_1: 1827; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1828; CHECK-O3-NEXT: xorl %edx, %edx 1829; CHECK-O3-NEXT: divl %esi 1830; CHECK-O3-NEXT: # kill: def $edx killed $edx def $rdx 1831; CHECK-O3-NEXT: movq %rdx, (%rdi) 1832; CHECK-O3-NEXT: retq 1833 %prev = load atomic i64, i64* %p unordered, align 8 1834 %val = srem i64 %prev, %v 1835 store atomic i64 %val, i64* %p unordered, align 8 1836 ret void 1837} 1838 1839; Legal, as expected 1840define void @rmw_fold_urem1(i64* %p, i64 %v) { 1841; CHECK-O0-LABEL: rmw_fold_urem1: 1842; CHECK-O0: # %bb.0: 1843; CHECK-O0-NEXT: movq (%rdi), %rax 1844; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889 1845; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1846; CHECK-O0-NEXT: mulq %rcx 1847; CHECK-O0-NEXT: shrq $3, %rdx 1848; CHECK-O0-NEXT: leaq (%rdx,%rdx,4), %rax 1849; CHECK-O0-NEXT: leaq (%rax,%rax,2), %rax 1850; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 1851; CHECK-O0-NEXT: subq %rax, %rcx 1852; CHECK-O0-NEXT: movq %rcx, (%rdi) 1853; CHECK-O0-NEXT: retq 1854; 1855; CHECK-O3-LABEL: rmw_fold_urem1: 1856; CHECK-O3: # %bb.0: 1857; CHECK-O3-NEXT: movq (%rdi), %rcx 1858; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 1859; CHECK-O3-NEXT: movq %rcx, %rax 1860; CHECK-O3-NEXT: mulq %rdx 1861; CHECK-O3-NEXT: shrq $3, %rdx 1862; CHECK-O3-NEXT: leaq (%rdx,%rdx,4), %rax 1863; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax 1864; CHECK-O3-NEXT: subq %rax, %rcx 1865; CHECK-O3-NEXT: movq %rcx, (%rdi) 1866; CHECK-O3-NEXT: retq 1867 %prev = load atomic i64, i64* %p unordered, align 8 1868 %val = urem i64 %prev, 15 1869 store atomic i64 %val, i64* %p unordered, align 8 1870 ret void 1871} 1872 1873; Legal, as expected 1874define void @rmw_fold_urem2(i64* %p, i64 %v) { 1875; CHECK-O0-LABEL: rmw_fold_urem2: 1876; CHECK-O0: # %bb.0: 1877; CHECK-O0-NEXT: movq (%rdi), %rax 1878; CHECK-O0-NEXT: xorl %ecx, %ecx 1879; CHECK-O0-NEXT: movl %ecx, %edx 1880; CHECK-O0-NEXT: divq %rsi 1881; CHECK-O0-NEXT: movq %rdx, (%rdi) 1882; CHECK-O0-NEXT: retq 1883; 1884; CHECK-O3-LABEL: rmw_fold_urem2: 1885; CHECK-O3: # %bb.0: 1886; CHECK-O3-NEXT: movq (%rdi), %rax 1887; CHECK-O3-NEXT: movq %rax, %rcx 1888; CHECK-O3-NEXT: orq %rsi, %rcx 1889; CHECK-O3-NEXT: shrq $32, %rcx 1890; CHECK-O3-NEXT: je .LBB80_1 1891; CHECK-O3-NEXT: # %bb.2: 1892; CHECK-O3-NEXT: xorl %edx, %edx 1893; CHECK-O3-NEXT: divq %rsi 1894; CHECK-O3-NEXT: movq %rdx, (%rdi) 1895; CHECK-O3-NEXT: retq 1896; CHECK-O3-NEXT: .LBB80_1: 1897; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1898; CHECK-O3-NEXT: xorl %edx, %edx 1899; CHECK-O3-NEXT: divl %esi 1900; CHECK-O3-NEXT: # kill: def $edx killed $edx def $rdx 1901; CHECK-O3-NEXT: movq %rdx, (%rdi) 1902; CHECK-O3-NEXT: retq 1903 %prev = load atomic i64, i64* %p unordered, align 8 1904 %val = urem i64 %prev, %v 1905 store atomic i64 %val, i64* %p unordered, align 8 1906 ret void 1907} 1908 1909; Legal to fold (TODO) 1910define void @rmw_fold_shl1(i64* %p, i64 %v) { 1911; CHECK-O0-LABEL: rmw_fold_shl1: 1912; CHECK-O0: # %bb.0: 1913; CHECK-O0-NEXT: movq (%rdi), %rax 1914; CHECK-O0-NEXT: shlq $15, %rax 1915; CHECK-O0-NEXT: movq %rax, (%rdi) 1916; CHECK-O0-NEXT: retq 1917; 1918; CHECK-O3-CUR-LABEL: rmw_fold_shl1: 1919; CHECK-O3-CUR: # %bb.0: 1920; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 1921; CHECK-O3-CUR-NEXT: shlq $15, %rax 1922; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 1923; CHECK-O3-CUR-NEXT: retq 1924; 1925; CHECK-O3-EX-LABEL: rmw_fold_shl1: 1926; CHECK-O3-EX: # %bb.0: 1927; CHECK-O3-EX-NEXT: shlq $15, (%rdi) 1928; CHECK-O3-EX-NEXT: retq 1929 %prev = load atomic i64, i64* %p unordered, align 8 1930 %val = shl i64 %prev, 15 1931 store atomic i64 %val, i64* %p unordered, align 8 1932 ret void 1933} 1934 1935; Legal to fold (TODO) 1936define void @rmw_fold_shl2(i64* %p, i64 %v) { 1937; CHECK-O0-LABEL: rmw_fold_shl2: 1938; CHECK-O0: # %bb.0: 1939; CHECK-O0-NEXT: movq (%rdi), %rax 1940; CHECK-O0-NEXT: # kill: def $sil killed $sil killed $rsi 1941; CHECK-O0-NEXT: # implicit-def: $rcx 1942; CHECK-O0-NEXT: movb %sil, %cl 1943; CHECK-O0-NEXT: shlxq %rcx, %rax, %rax 1944; CHECK-O0-NEXT: movq %rax, (%rdi) 1945; CHECK-O0-NEXT: retq 1946; 1947; CHECK-O3-CUR-LABEL: rmw_fold_shl2: 1948; CHECK-O3-CUR: # %bb.0: 1949; CHECK-O3-CUR-NEXT: shlxq %rsi, (%rdi), %rax 1950; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 1951; CHECK-O3-CUR-NEXT: retq 1952; 1953; CHECK-O3-EX-LABEL: rmw_fold_shl2: 1954; CHECK-O3-EX: # %bb.0: 1955; CHECK-O3-EX-NEXT: movq %rsi, %rcx 1956; CHECK-O3-EX-NEXT: # kill: def $cl killed $cl killed $rcx 1957; CHECK-O3-EX-NEXT: shlq %cl, (%rdi) 1958; CHECK-O3-EX-NEXT: retq 1959 %prev = load atomic i64, i64* %p unordered, align 8 1960 %val = shl i64 %prev, %v 1961 store atomic i64 %val, i64* %p unordered, align 8 1962 ret void 1963} 1964 1965; Legal to fold (TODO) 1966define void @rmw_fold_lshr1(i64* %p, i64 %v) { 1967; CHECK-O0-LABEL: rmw_fold_lshr1: 1968; CHECK-O0: # %bb.0: 1969; CHECK-O0-NEXT: movq (%rdi), %rax 1970; CHECK-O0-NEXT: shrq $15, %rax 1971; CHECK-O0-NEXT: movq %rax, (%rdi) 1972; CHECK-O0-NEXT: retq 1973; 1974; CHECK-O3-CUR-LABEL: rmw_fold_lshr1: 1975; CHECK-O3-CUR: # %bb.0: 1976; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 1977; CHECK-O3-CUR-NEXT: shrq $15, %rax 1978; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 1979; CHECK-O3-CUR-NEXT: retq 1980; 1981; CHECK-O3-EX-LABEL: rmw_fold_lshr1: 1982; CHECK-O3-EX: # %bb.0: 1983; CHECK-O3-EX-NEXT: shrq $15, (%rdi) 1984; CHECK-O3-EX-NEXT: retq 1985 %prev = load atomic i64, i64* %p unordered, align 8 1986 %val = lshr i64 %prev, 15 1987 store atomic i64 %val, i64* %p unordered, align 8 1988 ret void 1989} 1990 1991; Legal to fold (TODO) 1992define void @rmw_fold_lshr2(i64* %p, i64 %v) { 1993; CHECK-O0-LABEL: rmw_fold_lshr2: 1994; CHECK-O0: # %bb.0: 1995; CHECK-O0-NEXT: movq (%rdi), %rax 1996; CHECK-O0-NEXT: # kill: def $sil killed $sil killed $rsi 1997; CHECK-O0-NEXT: # implicit-def: $rcx 1998; CHECK-O0-NEXT: movb %sil, %cl 1999; CHECK-O0-NEXT: shrxq %rcx, %rax, %rax 2000; CHECK-O0-NEXT: movq %rax, (%rdi) 2001; CHECK-O0-NEXT: retq 2002; 2003; CHECK-O3-CUR-LABEL: rmw_fold_lshr2: 2004; CHECK-O3-CUR: # %bb.0: 2005; CHECK-O3-CUR-NEXT: shrxq %rsi, (%rdi), %rax 2006; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 2007; CHECK-O3-CUR-NEXT: retq 2008; 2009; CHECK-O3-EX-LABEL: rmw_fold_lshr2: 2010; CHECK-O3-EX: # %bb.0: 2011; CHECK-O3-EX-NEXT: movq %rsi, %rcx 2012; CHECK-O3-EX-NEXT: # kill: def $cl killed $cl killed $rcx 2013; CHECK-O3-EX-NEXT: shrq %cl, (%rdi) 2014; CHECK-O3-EX-NEXT: retq 2015 %prev = load atomic i64, i64* %p unordered, align 8 2016 %val = lshr i64 %prev, %v 2017 store atomic i64 %val, i64* %p unordered, align 8 2018 ret void 2019} 2020 2021; Legal to fold (TODO) 2022define void @rmw_fold_ashr1(i64* %p, i64 %v) { 2023; CHECK-O0-LABEL: rmw_fold_ashr1: 2024; CHECK-O0: # %bb.0: 2025; CHECK-O0-NEXT: movq (%rdi), %rax 2026; CHECK-O0-NEXT: sarq $15, %rax 2027; CHECK-O0-NEXT: movq %rax, (%rdi) 2028; CHECK-O0-NEXT: retq 2029; 2030; CHECK-O3-CUR-LABEL: rmw_fold_ashr1: 2031; CHECK-O3-CUR: # %bb.0: 2032; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 2033; CHECK-O3-CUR-NEXT: sarq $15, %rax 2034; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 2035; CHECK-O3-CUR-NEXT: retq 2036; 2037; CHECK-O3-EX-LABEL: rmw_fold_ashr1: 2038; CHECK-O3-EX: # %bb.0: 2039; CHECK-O3-EX-NEXT: sarq $15, (%rdi) 2040; CHECK-O3-EX-NEXT: retq 2041 %prev = load atomic i64, i64* %p unordered, align 8 2042 %val = ashr i64 %prev, 15 2043 store atomic i64 %val, i64* %p unordered, align 8 2044 ret void 2045} 2046 2047; Legal to fold (TODO) 2048define void @rmw_fold_ashr2(i64* %p, i64 %v) { 2049; CHECK-O0-LABEL: rmw_fold_ashr2: 2050; CHECK-O0: # %bb.0: 2051; CHECK-O0-NEXT: movq (%rdi), %rax 2052; CHECK-O0-NEXT: # kill: def $sil killed $sil killed $rsi 2053; CHECK-O0-NEXT: # implicit-def: $rcx 2054; CHECK-O0-NEXT: movb %sil, %cl 2055; CHECK-O0-NEXT: sarxq %rcx, %rax, %rax 2056; CHECK-O0-NEXT: movq %rax, (%rdi) 2057; CHECK-O0-NEXT: retq 2058; 2059; CHECK-O3-CUR-LABEL: rmw_fold_ashr2: 2060; CHECK-O3-CUR: # %bb.0: 2061; CHECK-O3-CUR-NEXT: sarxq %rsi, (%rdi), %rax 2062; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 2063; CHECK-O3-CUR-NEXT: retq 2064; 2065; CHECK-O3-EX-LABEL: rmw_fold_ashr2: 2066; CHECK-O3-EX: # %bb.0: 2067; CHECK-O3-EX-NEXT: movq %rsi, %rcx 2068; CHECK-O3-EX-NEXT: # kill: def $cl killed $cl killed $rcx 2069; CHECK-O3-EX-NEXT: sarq %cl, (%rdi) 2070; CHECK-O3-EX-NEXT: retq 2071 %prev = load atomic i64, i64* %p unordered, align 8 2072 %val = ashr i64 %prev, %v 2073 store atomic i64 %val, i64* %p unordered, align 8 2074 ret void 2075} 2076 2077; Legal, as expected 2078define void @rmw_fold_and1(i64* %p, i64 %v) { 2079; CHECK-O0-LABEL: rmw_fold_and1: 2080; CHECK-O0: # %bb.0: 2081; CHECK-O0-NEXT: movq (%rdi), %rax 2082; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax 2083; CHECK-O0-NEXT: andl $15, %eax 2084; CHECK-O0-NEXT: movl %eax, %ecx 2085; CHECK-O0-NEXT: movq %rcx, (%rdi) 2086; CHECK-O0-NEXT: retq 2087; 2088; CHECK-O3-LABEL: rmw_fold_and1: 2089; CHECK-O3: # %bb.0: 2090; CHECK-O3-NEXT: andq $15, (%rdi) 2091; CHECK-O3-NEXT: retq 2092 %prev = load atomic i64, i64* %p unordered, align 8 2093 %val = and i64 %prev, 15 2094 store atomic i64 %val, i64* %p unordered, align 8 2095 ret void 2096} 2097 2098; Legal, as expected 2099define void @rmw_fold_and2(i64* %p, i64 %v) { 2100; CHECK-O0-LABEL: rmw_fold_and2: 2101; CHECK-O0: # %bb.0: 2102; CHECK-O0-NEXT: movq (%rdi), %rax 2103; CHECK-O0-NEXT: andq %rsi, %rax 2104; CHECK-O0-NEXT: movq %rax, (%rdi) 2105; CHECK-O0-NEXT: retq 2106; 2107; CHECK-O3-LABEL: rmw_fold_and2: 2108; CHECK-O3: # %bb.0: 2109; CHECK-O3-NEXT: andq %rsi, (%rdi) 2110; CHECK-O3-NEXT: retq 2111 %prev = load atomic i64, i64* %p unordered, align 8 2112 %val = and i64 %prev, %v 2113 store atomic i64 %val, i64* %p unordered, align 8 2114 ret void 2115} 2116 2117; Legal, as expected 2118define void @rmw_fold_or1(i64* %p, i64 %v) { 2119; CHECK-O0-LABEL: rmw_fold_or1: 2120; CHECK-O0: # %bb.0: 2121; CHECK-O0-NEXT: movq (%rdi), %rax 2122; CHECK-O0-NEXT: orq $15, %rax 2123; CHECK-O0-NEXT: movq %rax, (%rdi) 2124; CHECK-O0-NEXT: retq 2125; 2126; CHECK-O3-LABEL: rmw_fold_or1: 2127; CHECK-O3: # %bb.0: 2128; CHECK-O3-NEXT: orq $15, (%rdi) 2129; CHECK-O3-NEXT: retq 2130 %prev = load atomic i64, i64* %p unordered, align 8 2131 %val = or i64 %prev, 15 2132 store atomic i64 %val, i64* %p unordered, align 8 2133 ret void 2134} 2135 2136; Legal, as expected 2137define void @rmw_fold_or2(i64* %p, i64 %v) { 2138; CHECK-O0-LABEL: rmw_fold_or2: 2139; CHECK-O0: # %bb.0: 2140; CHECK-O0-NEXT: movq (%rdi), %rax 2141; CHECK-O0-NEXT: orq %rsi, %rax 2142; CHECK-O0-NEXT: movq %rax, (%rdi) 2143; CHECK-O0-NEXT: retq 2144; 2145; CHECK-O3-LABEL: rmw_fold_or2: 2146; CHECK-O3: # %bb.0: 2147; CHECK-O3-NEXT: orq %rsi, (%rdi) 2148; CHECK-O3-NEXT: retq 2149 %prev = load atomic i64, i64* %p unordered, align 8 2150 %val = or i64 %prev, %v 2151 store atomic i64 %val, i64* %p unordered, align 8 2152 ret void 2153} 2154 2155; Legal, as expected 2156define void @rmw_fold_xor1(i64* %p, i64 %v) { 2157; CHECK-O0-LABEL: rmw_fold_xor1: 2158; CHECK-O0: # %bb.0: 2159; CHECK-O0-NEXT: movq (%rdi), %rax 2160; CHECK-O0-NEXT: xorq $15, %rax 2161; CHECK-O0-NEXT: movq %rax, (%rdi) 2162; CHECK-O0-NEXT: retq 2163; 2164; CHECK-O3-LABEL: rmw_fold_xor1: 2165; CHECK-O3: # %bb.0: 2166; CHECK-O3-NEXT: xorq $15, (%rdi) 2167; CHECK-O3-NEXT: retq 2168 %prev = load atomic i64, i64* %p unordered, align 8 2169 %val = xor i64 %prev, 15 2170 store atomic i64 %val, i64* %p unordered, align 8 2171 ret void 2172} 2173 2174; Legal, as expected 2175define void @rmw_fold_xor2(i64* %p, i64 %v) { 2176; CHECK-O0-LABEL: rmw_fold_xor2: 2177; CHECK-O0: # %bb.0: 2178; CHECK-O0-NEXT: movq (%rdi), %rax 2179; CHECK-O0-NEXT: xorq %rsi, %rax 2180; CHECK-O0-NEXT: movq %rax, (%rdi) 2181; CHECK-O0-NEXT: retq 2182; 2183; CHECK-O3-LABEL: rmw_fold_xor2: 2184; CHECK-O3: # %bb.0: 2185; CHECK-O3-NEXT: xorq %rsi, (%rdi) 2186; CHECK-O3-NEXT: retq 2187 %prev = load atomic i64, i64* %p unordered, align 8 2188 %val = xor i64 %prev, %v 2189 store atomic i64 %val, i64* %p unordered, align 8 2190 ret void 2191} 2192 2193;; The next batch test truncations, in combination w/operations which could 2194;; be folded against the memory operation. 2195 2196; Legal to reduce the load width (TODO) 2197define i32 @fold_trunc(i64* %p) { 2198; CHECK-LABEL: fold_trunc: 2199; CHECK: # %bb.0: 2200; CHECK-NEXT: movq (%rdi), %rax 2201; CHECK-NEXT: # kill: def $eax killed $eax killed $rax 2202; CHECK-NEXT: retq 2203 %v = load atomic i64, i64* %p unordered, align 8 2204 %ret = trunc i64 %v to i32 2205 ret i32 %ret 2206} 2207 2208; Legal to reduce the load width and fold the load (TODO) 2209define i32 @fold_trunc_add(i64* %p, i32 %v2) { 2210; CHECK-O0-LABEL: fold_trunc_add: 2211; CHECK-O0: # %bb.0: 2212; CHECK-O0-NEXT: movq (%rdi), %rax 2213; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax 2214; CHECK-O0-NEXT: addl %esi, %eax 2215; CHECK-O0-NEXT: retq 2216; 2217; CHECK-O3-LABEL: fold_trunc_add: 2218; CHECK-O3: # %bb.0: 2219; CHECK-O3-NEXT: movq (%rdi), %rax 2220; CHECK-O3-NEXT: addl %esi, %eax 2221; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 2222; CHECK-O3-NEXT: retq 2223 %v = load atomic i64, i64* %p unordered, align 8 2224 %trunc = trunc i64 %v to i32 2225 %ret = add i32 %trunc, %v2 2226 ret i32 %ret 2227} 2228 2229; Legal to reduce the load width and fold the load (TODO) 2230define i32 @fold_trunc_and(i64* %p, i32 %v2) { 2231; CHECK-O0-LABEL: fold_trunc_and: 2232; CHECK-O0: # %bb.0: 2233; CHECK-O0-NEXT: movq (%rdi), %rax 2234; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax 2235; CHECK-O0-NEXT: andl %esi, %eax 2236; CHECK-O0-NEXT: retq 2237; 2238; CHECK-O3-LABEL: fold_trunc_and: 2239; CHECK-O3: # %bb.0: 2240; CHECK-O3-NEXT: movq (%rdi), %rax 2241; CHECK-O3-NEXT: andl %esi, %eax 2242; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 2243; CHECK-O3-NEXT: retq 2244 %v = load atomic i64, i64* %p unordered, align 8 2245 %trunc = trunc i64 %v to i32 2246 %ret = and i32 %trunc, %v2 2247 ret i32 %ret 2248} 2249 2250; Legal to reduce the load width and fold the load (TODO) 2251define i32 @fold_trunc_or(i64* %p, i32 %v2) { 2252; CHECK-O0-LABEL: fold_trunc_or: 2253; CHECK-O0: # %bb.0: 2254; CHECK-O0-NEXT: movq (%rdi), %rax 2255; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax 2256; CHECK-O0-NEXT: orl %esi, %eax 2257; CHECK-O0-NEXT: retq 2258; 2259; CHECK-O3-LABEL: fold_trunc_or: 2260; CHECK-O3: # %bb.0: 2261; CHECK-O3-NEXT: movq (%rdi), %rax 2262; CHECK-O3-NEXT: orl %esi, %eax 2263; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 2264; CHECK-O3-NEXT: retq 2265 %v = load atomic i64, i64* %p unordered, align 8 2266 %trunc = trunc i64 %v to i32 2267 %ret = or i32 %trunc, %v2 2268 ret i32 %ret 2269} 2270 2271; It's tempting to split the wide load into two smaller byte loads 2272; to reduce memory traffic, but this would be illegal for a atomic load 2273define i32 @split_load(i64* %p) { 2274; CHECK-O0-LABEL: split_load: 2275; CHECK-O0: # %bb.0: 2276; CHECK-O0-NEXT: movq (%rdi), %rax 2277; CHECK-O0-NEXT: movb %al, %cl 2278; CHECK-O0-NEXT: shrq $32, %rax 2279; CHECK-O0-NEXT: # kill: def $al killed $al killed $rax 2280; CHECK-O0-NEXT: orb %al, %cl 2281; CHECK-O0-NEXT: movzbl %cl, %eax 2282; CHECK-O0-NEXT: retq 2283; 2284; CHECK-O3-LABEL: split_load: 2285; CHECK-O3: # %bb.0: 2286; CHECK-O3-NEXT: movq (%rdi), %rax 2287; CHECK-O3-NEXT: movq %rax, %rcx 2288; CHECK-O3-NEXT: shrq $32, %rcx 2289; CHECK-O3-NEXT: orl %eax, %ecx 2290; CHECK-O3-NEXT: movzbl %cl, %eax 2291; CHECK-O3-NEXT: retq 2292 %v = load atomic i64, i64* %p unordered, align 8 2293 %b1 = trunc i64 %v to i8 2294 %v.shift = lshr i64 %v, 32 2295 %b2 = trunc i64 %v.shift to i8 2296 %or = or i8 %b1, %b2 2297 %ret = zext i8 %or to i32 2298 ret i32 %ret 2299} 2300 2301;; A collection of simple memory forwarding tests. Nothing particular 2302;; interesting semantic wise, just demonstrating obvious missed transforms. 2303 2304@Zero = constant i64 0 2305 2306; TODO: should return constant 2307define i64 @constant_folding(i64* %p) { 2308; CHECK-LABEL: constant_folding: 2309; CHECK: # %bb.0: 2310; CHECK-NEXT: movq (%rdi), %rax 2311; CHECK-NEXT: retq 2312 %v = load atomic i64, i64* %p unordered, align 8 2313 ret i64 %v 2314} 2315 2316; Legal to forward and fold (TODO) 2317define i64 @load_forwarding(i64* %p) { 2318; CHECK-O0-LABEL: load_forwarding: 2319; CHECK-O0: # %bb.0: 2320; CHECK-O0-NEXT: movq (%rdi), %rax 2321; CHECK-O0-NEXT: orq (%rdi), %rax 2322; CHECK-O0-NEXT: retq 2323; 2324; CHECK-O3-CUR-LABEL: load_forwarding: 2325; CHECK-O3-CUR: # %bb.0: 2326; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 2327; CHECK-O3-CUR-NEXT: orq (%rdi), %rax 2328; CHECK-O3-CUR-NEXT: retq 2329; 2330; CHECK-O3-EX-LABEL: load_forwarding: 2331; CHECK-O3-EX: # %bb.0: 2332; CHECK-O3-EX-NEXT: movq (%rdi), %rax 2333; CHECK-O3-EX-NEXT: retq 2334 %v = load atomic i64, i64* %p unordered, align 8 2335 %v2 = load atomic i64, i64* %p unordered, align 8 2336 %ret = or i64 %v, %v2 2337 ret i64 %ret 2338} 2339 2340; Legal to forward (TODO) 2341define i64 @store_forward(i64* %p, i64 %v) { 2342; CHECK-LABEL: store_forward: 2343; CHECK: # %bb.0: 2344; CHECK-NEXT: movq %rsi, (%rdi) 2345; CHECK-NEXT: movq (%rdi), %rax 2346; CHECK-NEXT: retq 2347 store atomic i64 %v, i64* %p unordered, align 8 2348 %ret = load atomic i64, i64* %p unordered, align 8 2349 ret i64 %ret 2350} 2351 2352; Legal to kill (TODO) 2353define void @dead_writeback(i64* %p) { 2354; CHECK-LABEL: dead_writeback: 2355; CHECK: # %bb.0: 2356; CHECK-NEXT: movq (%rdi), %rax 2357; CHECK-NEXT: movq %rax, (%rdi) 2358; CHECK-NEXT: retq 2359 %v = load atomic i64, i64* %p unordered, align 8 2360 store atomic i64 %v, i64* %p unordered, align 8 2361 ret void 2362} 2363 2364; Legal to kill (TODO) 2365define void @dead_store(i64* %p, i64 %v) { 2366; CHECK-LABEL: dead_store: 2367; CHECK: # %bb.0: 2368; CHECK-NEXT: movq $0, (%rdi) 2369; CHECK-NEXT: movq %rsi, (%rdi) 2370; CHECK-NEXT: retq 2371 store atomic i64 0, i64* %p unordered, align 8 2372 store atomic i64 %v, i64* %p unordered, align 8 2373 ret void 2374} 2375 2376;; The next batch of tests ensure that we don't try to fold a load into a 2377;; use where the code motion implied for the load is prevented by a fence. 2378;; Note: We're checking that the load doesn't get moved below the fence as 2379;; part of folding, but is technically legal to lift the add above the fence. 2380;; If that were to happen, please rewrite the test to ensure load movement 2381;; isn't violated. 2382 2383define i64 @nofold_fence(i64* %p) { 2384; CHECK-LABEL: nofold_fence: 2385; CHECK: # %bb.0: 2386; CHECK-NEXT: movq (%rdi), %rax 2387; CHECK-NEXT: mfence 2388; CHECK-NEXT: addq $15, %rax 2389; CHECK-NEXT: retq 2390 %v = load atomic i64, i64* %p unordered, align 8 2391 fence seq_cst 2392 %ret = add i64 %v, 15 2393 ret i64 %ret 2394} 2395 2396define i64 @nofold_fence_acquire(i64* %p) { 2397; CHECK-LABEL: nofold_fence_acquire: 2398; CHECK: # %bb.0: 2399; CHECK-NEXT: movq (%rdi), %rax 2400; CHECK-NEXT: #MEMBARRIER 2401; CHECK-NEXT: addq $15, %rax 2402; CHECK-NEXT: retq 2403 %v = load atomic i64, i64* %p unordered, align 8 2404 fence acquire 2405 %ret = add i64 %v, 15 2406 ret i64 %ret 2407} 2408 2409 2410define i64 @nofold_stfence(i64* %p) { 2411; CHECK-LABEL: nofold_stfence: 2412; CHECK: # %bb.0: 2413; CHECK-NEXT: movq (%rdi), %rax 2414; CHECK-NEXT: #MEMBARRIER 2415; CHECK-NEXT: addq $15, %rax 2416; CHECK-NEXT: retq 2417 %v = load atomic i64, i64* %p unordered, align 8 2418 fence syncscope("singlethread") seq_cst 2419 %ret = add i64 %v, 15 2420 ret i64 %ret 2421} 2422 2423;; Next, test how well we can fold invariant loads. 2424 2425@Constant = external constant i64 2426 2427define i64 @fold_constant(i64 %arg) { 2428; CHECK-O0-LABEL: fold_constant: 2429; CHECK-O0: # %bb.0: 2430; CHECK-O0-NEXT: addq Constant, %rdi 2431; CHECK-O0-NEXT: movq %rdi, %rax 2432; CHECK-O0-NEXT: retq 2433; 2434; CHECK-O3-LABEL: fold_constant: 2435; CHECK-O3: # %bb.0: 2436; CHECK-O3-NEXT: movq %rdi, %rax 2437; CHECK-O3-NEXT: addq {{.*}}(%rip), %rax 2438; CHECK-O3-NEXT: retq 2439 %v = load atomic i64, i64* @Constant unordered, align 8 2440 %ret = add i64 %v, %arg 2441 ret i64 %ret 2442} 2443 2444define i64 @fold_constant_clobber(i64* %p, i64 %arg) { 2445; CHECK-O0-LABEL: fold_constant_clobber: 2446; CHECK-O0: # %bb.0: 2447; CHECK-O0-NEXT: movq {{.*}}(%rip), %rax 2448; CHECK-O0-NEXT: movq $5, (%rdi) 2449; CHECK-O0-NEXT: addq %rsi, %rax 2450; CHECK-O0-NEXT: retq 2451; 2452; CHECK-O3-CUR-LABEL: fold_constant_clobber: 2453; CHECK-O3-CUR: # %bb.0: 2454; CHECK-O3-CUR-NEXT: movq {{.*}}(%rip), %rax 2455; CHECK-O3-CUR-NEXT: movq $5, (%rdi) 2456; CHECK-O3-CUR-NEXT: addq %rsi, %rax 2457; CHECK-O3-CUR-NEXT: retq 2458; 2459; CHECK-O3-EX-LABEL: fold_constant_clobber: 2460; CHECK-O3-EX: # %bb.0: 2461; CHECK-O3-EX-NEXT: movq %rsi, %rax 2462; CHECK-O3-EX-NEXT: movq $5, (%rdi) 2463; CHECK-O3-EX-NEXT: addq {{.*}}(%rip), %rax 2464; CHECK-O3-EX-NEXT: retq 2465 %v = load atomic i64, i64* @Constant unordered, align 8 2466 store i64 5, i64* %p 2467 %ret = add i64 %v, %arg 2468 ret i64 %ret 2469} 2470 2471define i64 @fold_constant_fence(i64 %arg) { 2472; CHECK-O0-LABEL: fold_constant_fence: 2473; CHECK-O0: # %bb.0: 2474; CHECK-O0-NEXT: movq {{.*}}(%rip), %rax 2475; CHECK-O0-NEXT: mfence 2476; CHECK-O0-NEXT: addq %rdi, %rax 2477; CHECK-O0-NEXT: retq 2478; 2479; CHECK-O3-CUR-LABEL: fold_constant_fence: 2480; CHECK-O3-CUR: # %bb.0: 2481; CHECK-O3-CUR-NEXT: movq {{.*}}(%rip), %rax 2482; CHECK-O3-CUR-NEXT: mfence 2483; CHECK-O3-CUR-NEXT: addq %rdi, %rax 2484; CHECK-O3-CUR-NEXT: retq 2485; 2486; CHECK-O3-EX-LABEL: fold_constant_fence: 2487; CHECK-O3-EX: # %bb.0: 2488; CHECK-O3-EX-NEXT: movq %rdi, %rax 2489; CHECK-O3-EX-NEXT: mfence 2490; CHECK-O3-EX-NEXT: addq {{.*}}(%rip), %rax 2491; CHECK-O3-EX-NEXT: retq 2492 %v = load atomic i64, i64* @Constant unordered, align 8 2493 fence seq_cst 2494 %ret = add i64 %v, %arg 2495 ret i64 %ret 2496} 2497 2498define i64 @fold_invariant_clobber(i64* dereferenceable(8) %p, i64 %arg) { 2499; CHECK-O0-LABEL: fold_invariant_clobber: 2500; CHECK-O0: # %bb.0: 2501; CHECK-O0-NEXT: movq (%rdi), %rax 2502; CHECK-O0-NEXT: movq $5, (%rdi) 2503; CHECK-O0-NEXT: addq %rsi, %rax 2504; CHECK-O0-NEXT: retq 2505; 2506; CHECK-O3-CUR-LABEL: fold_invariant_clobber: 2507; CHECK-O3-CUR: # %bb.0: 2508; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 2509; CHECK-O3-CUR-NEXT: movq $5, (%rdi) 2510; CHECK-O3-CUR-NEXT: addq %rsi, %rax 2511; CHECK-O3-CUR-NEXT: retq 2512; 2513; CHECK-O3-EX-LABEL: fold_invariant_clobber: 2514; CHECK-O3-EX: # %bb.0: 2515; CHECK-O3-EX-NEXT: movq %rsi, %rax 2516; CHECK-O3-EX-NEXT: movq $5, (%rdi) 2517; CHECK-O3-EX-NEXT: addq (%rdi), %rax 2518; CHECK-O3-EX-NEXT: retq 2519 %v = load atomic i64, i64* %p unordered, align 8, !invariant.load !{} 2520 store i64 5, i64* %p 2521 %ret = add i64 %v, %arg 2522 ret i64 %ret 2523} 2524 2525 2526define i64 @fold_invariant_fence(i64* dereferenceable(8) %p, i64 %arg) { 2527; CHECK-O0-LABEL: fold_invariant_fence: 2528; CHECK-O0: # %bb.0: 2529; CHECK-O0-NEXT: movq (%rdi), %rax 2530; CHECK-O0-NEXT: mfence 2531; CHECK-O0-NEXT: addq %rsi, %rax 2532; CHECK-O0-NEXT: retq 2533; 2534; CHECK-O3-CUR-LABEL: fold_invariant_fence: 2535; CHECK-O3-CUR: # %bb.0: 2536; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 2537; CHECK-O3-CUR-NEXT: mfence 2538; CHECK-O3-CUR-NEXT: addq %rsi, %rax 2539; CHECK-O3-CUR-NEXT: retq 2540; 2541; CHECK-O3-EX-LABEL: fold_invariant_fence: 2542; CHECK-O3-EX: # %bb.0: 2543; CHECK-O3-EX-NEXT: movq %rsi, %rax 2544; CHECK-O3-EX-NEXT: mfence 2545; CHECK-O3-EX-NEXT: addq (%rdi), %rax 2546; CHECK-O3-EX-NEXT: retq 2547 %v = load atomic i64, i64* %p unordered, align 8, !invariant.load !{} 2548 fence seq_cst 2549 %ret = add i64 %v, %arg 2550 ret i64 %ret 2551} 2552 2553 2554; Exercise a few cases involving any extend idioms 2555 2556define i16 @load_i8_anyext_i16(i8* %ptr) { 2557; CHECK-O0-CUR-LABEL: load_i8_anyext_i16: 2558; CHECK-O0-CUR: # %bb.0: 2559; CHECK-O0-CUR-NEXT: movb (%rdi), %al 2560; CHECK-O0-CUR-NEXT: movzbl %al, %ecx 2561; CHECK-O0-CUR-NEXT: # kill: def $cx killed $cx killed $ecx 2562; CHECK-O0-CUR-NEXT: movw %cx, %ax 2563; CHECK-O0-CUR-NEXT: retq 2564; 2565; CHECK-O3-CUR-LABEL: load_i8_anyext_i16: 2566; CHECK-O3-CUR: # %bb.0: 2567; CHECK-O3-CUR-NEXT: movzbl (%rdi), %eax 2568; CHECK-O3-CUR-NEXT: # kill: def $ax killed $ax killed $eax 2569; CHECK-O3-CUR-NEXT: retq 2570; 2571; CHECK-O0-EX-LABEL: load_i8_anyext_i16: 2572; CHECK-O0-EX: # %bb.0: 2573; CHECK-O0-EX-NEXT: vpbroadcastb (%rdi), %xmm0 2574; CHECK-O0-EX-NEXT: vmovd %xmm0, %eax 2575; CHECK-O0-EX-NEXT: # kill: def $ax killed $ax killed $eax 2576; CHECK-O0-EX-NEXT: retq 2577; 2578; CHECK-O3-EX-LABEL: load_i8_anyext_i16: 2579; CHECK-O3-EX: # %bb.0: 2580; CHECK-O3-EX-NEXT: vpbroadcastb (%rdi), %xmm0 2581; CHECK-O3-EX-NEXT: vmovd %xmm0, %eax 2582; CHECK-O3-EX-NEXT: # kill: def $ax killed $ax killed $eax 2583; CHECK-O3-EX-NEXT: retq 2584 %v = load atomic i8, i8* %ptr unordered, align 2 2585 %vec = insertelement <2 x i8> undef, i8 %v, i32 0 2586 %res = bitcast <2 x i8> %vec to i16 2587 ret i16 %res 2588} 2589 2590define i32 @load_i8_anyext_i32(i8* %ptr) { 2591; CHECK-O0-CUR-LABEL: load_i8_anyext_i32: 2592; CHECK-O0-CUR: # %bb.0: 2593; CHECK-O0-CUR-NEXT: movb (%rdi), %al 2594; CHECK-O0-CUR-NEXT: movzbl %al, %eax 2595; CHECK-O0-CUR-NEXT: retq 2596; 2597; CHECK-O3-CUR-LABEL: load_i8_anyext_i32: 2598; CHECK-O3-CUR: # %bb.0: 2599; CHECK-O3-CUR-NEXT: movzbl (%rdi), %eax 2600; CHECK-O3-CUR-NEXT: retq 2601; 2602; CHECK-O0-EX-LABEL: load_i8_anyext_i32: 2603; CHECK-O0-EX: # %bb.0: 2604; CHECK-O0-EX-NEXT: vpbroadcastb (%rdi), %xmm0 2605; CHECK-O0-EX-NEXT: vmovd %xmm0, %eax 2606; CHECK-O0-EX-NEXT: retq 2607; 2608; CHECK-O3-EX-LABEL: load_i8_anyext_i32: 2609; CHECK-O3-EX: # %bb.0: 2610; CHECK-O3-EX-NEXT: vpbroadcastb (%rdi), %xmm0 2611; CHECK-O3-EX-NEXT: vmovd %xmm0, %eax 2612; CHECK-O3-EX-NEXT: retq 2613 %v = load atomic i8, i8* %ptr unordered, align 4 2614 %vec = insertelement <4 x i8> undef, i8 %v, i32 0 2615 %res = bitcast <4 x i8> %vec to i32 2616 ret i32 %res 2617} 2618 2619define i32 @load_i16_anyext_i32(i16* %ptr) { 2620; CHECK-O0-CUR-LABEL: load_i16_anyext_i32: 2621; CHECK-O0-CUR: # %bb.0: 2622; CHECK-O0-CUR-NEXT: movw (%rdi), %ax 2623; CHECK-O0-CUR-NEXT: # implicit-def: $ecx 2624; CHECK-O0-CUR-NEXT: movw %ax, %cx 2625; CHECK-O0-CUR-NEXT: movl %ecx, %eax 2626; CHECK-O0-CUR-NEXT: retq 2627; 2628; CHECK-O3-CUR-LABEL: load_i16_anyext_i32: 2629; CHECK-O3-CUR: # %bb.0: 2630; CHECK-O3-CUR-NEXT: movzwl (%rdi), %eax 2631; CHECK-O3-CUR-NEXT: retq 2632; 2633; CHECK-O0-EX-LABEL: load_i16_anyext_i32: 2634; CHECK-O0-EX: # %bb.0: 2635; CHECK-O0-EX-NEXT: vpbroadcastw (%rdi), %xmm0 2636; CHECK-O0-EX-NEXT: vmovd %xmm0, %eax 2637; CHECK-O0-EX-NEXT: retq 2638; 2639; CHECK-O3-EX-LABEL: load_i16_anyext_i32: 2640; CHECK-O3-EX: # %bb.0: 2641; CHECK-O3-EX-NEXT: vpbroadcastw (%rdi), %xmm0 2642; CHECK-O3-EX-NEXT: vmovd %xmm0, %eax 2643; CHECK-O3-EX-NEXT: retq 2644 %v = load atomic i16, i16* %ptr unordered, align 4 2645 %vec = insertelement <2 x i16> undef, i16 %v, i64 0 2646 %res = bitcast <2 x i16> %vec to i32 2647 ret i32 %res 2648} 2649 2650define i64 @load_i16_anyext_i64(i16* %ptr) { 2651; CHECK-O0-CUR-LABEL: load_i16_anyext_i64: 2652; CHECK-O0-CUR: # %bb.0: 2653; CHECK-O0-CUR-NEXT: movw (%rdi), %ax 2654; CHECK-O0-CUR-NEXT: # implicit-def: $ecx 2655; CHECK-O0-CUR-NEXT: movw %ax, %cx 2656; CHECK-O0-CUR-NEXT: vmovd %ecx, %xmm0 2657; CHECK-O0-CUR-NEXT: vmovq %xmm0, %rax 2658; CHECK-O0-CUR-NEXT: retq 2659; 2660; CHECK-O3-CUR-LABEL: load_i16_anyext_i64: 2661; CHECK-O3-CUR: # %bb.0: 2662; CHECK-O3-CUR-NEXT: movzwl (%rdi), %eax 2663; CHECK-O3-CUR-NEXT: vmovd %eax, %xmm0 2664; CHECK-O3-CUR-NEXT: vmovq %xmm0, %rax 2665; CHECK-O3-CUR-NEXT: retq 2666; 2667; CHECK-O0-EX-LABEL: load_i16_anyext_i64: 2668; CHECK-O0-EX: # %bb.0: 2669; CHECK-O0-EX-NEXT: vpbroadcastw (%rdi), %xmm0 2670; CHECK-O0-EX-NEXT: vmovq %xmm0, %rax 2671; CHECK-O0-EX-NEXT: retq 2672; 2673; CHECK-O3-EX-LABEL: load_i16_anyext_i64: 2674; CHECK-O3-EX: # %bb.0: 2675; CHECK-O3-EX-NEXT: vpbroadcastw (%rdi), %xmm0 2676; CHECK-O3-EX-NEXT: vmovq %xmm0, %rax 2677; CHECK-O3-EX-NEXT: retq 2678 %v = load atomic i16, i16* %ptr unordered, align 8 2679 %vec = insertelement <4 x i16> undef, i16 %v, i64 0 2680 %res = bitcast <4 x i16> %vec to i64 2681 ret i64 %res 2682} 2683