1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake -x86-experimental-unordered-atomic-isel=0 | FileCheck --check-prefixes=CHECK,CHECK-O0,CHECK-O0-CUR %s 3; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake -x86-experimental-unordered-atomic-isel=0 | FileCheck --check-prefixes=CHECK,CHECK-O3,CHECK-O3-CUR %s 4; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake -x86-experimental-unordered-atomic-isel=1 | FileCheck --check-prefixes=CHECK,CHECK-O0,CHECK-O0-EX %s 5; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake -x86-experimental-unordered-atomic-isel=1 | FileCheck --check-prefixes=CHECK,CHECK-O3,CHECK-O3-EX %s 6 7define i8 @load_i8(i8* %ptr) { 8; CHECK-LABEL: load_i8: 9; CHECK: # %bb.0: 10; CHECK-NEXT: movb (%rdi), %al 11; CHECK-NEXT: retq 12 %v = load atomic i8, i8* %ptr unordered, align 1 13 ret i8 %v 14} 15 16define void @store_i8(i8* %ptr, i8 %v) { 17; CHECK-O0-LABEL: store_i8: 18; CHECK-O0: # %bb.0: 19; CHECK-O0-NEXT: movb %sil, %al 20; CHECK-O0-NEXT: movb %al, (%rdi) 21; CHECK-O0-NEXT: retq 22; 23; CHECK-O3-LABEL: store_i8: 24; CHECK-O3: # %bb.0: 25; CHECK-O3-NEXT: movb %sil, (%rdi) 26; CHECK-O3-NEXT: retq 27 store atomic i8 %v, i8* %ptr unordered, align 1 28 ret void 29} 30 31define i16 @load_i16(i16* %ptr) { 32; CHECK-O0-LABEL: load_i16: 33; CHECK-O0: # %bb.0: 34; CHECK-O0-NEXT: movw (%rdi), %ax 35; CHECK-O0-NEXT: retq 36; 37; CHECK-O3-LABEL: load_i16: 38; CHECK-O3: # %bb.0: 39; CHECK-O3-NEXT: movzwl (%rdi), %eax 40; CHECK-O3-NEXT: retq 41 %v = load atomic i16, i16* %ptr unordered, align 2 42 ret i16 %v 43} 44 45 46define void @store_i16(i16* %ptr, i16 %v) { 47; CHECK-O0-LABEL: store_i16: 48; CHECK-O0: # %bb.0: 49; CHECK-O0-NEXT: movw %si, %ax 50; CHECK-O0-NEXT: movw %ax, (%rdi) 51; CHECK-O0-NEXT: retq 52; 53; CHECK-O3-LABEL: store_i16: 54; CHECK-O3: # %bb.0: 55; CHECK-O3-NEXT: movw %si, (%rdi) 56; CHECK-O3-NEXT: retq 57 store atomic i16 %v, i16* %ptr unordered, align 2 58 ret void 59} 60 61define i32 @load_i32(i32* %ptr) { 62; CHECK-LABEL: load_i32: 63; CHECK: # %bb.0: 64; CHECK-NEXT: movl (%rdi), %eax 65; CHECK-NEXT: retq 66 %v = load atomic i32, i32* %ptr unordered, align 4 67 ret i32 %v 68} 69 70define void @store_i32(i32* %ptr, i32 %v) { 71; CHECK-LABEL: store_i32: 72; CHECK: # %bb.0: 73; CHECK-NEXT: movl %esi, (%rdi) 74; CHECK-NEXT: retq 75 store atomic i32 %v, i32* %ptr unordered, align 4 76 ret void 77} 78 79define i64 @load_i64(i64* %ptr) { 80; CHECK-LABEL: load_i64: 81; CHECK: # %bb.0: 82; CHECK-NEXT: movq (%rdi), %rax 83; CHECK-NEXT: retq 84 %v = load atomic i64, i64* %ptr unordered, align 8 85 ret i64 %v 86} 87 88define void @store_i64(i64* %ptr, i64 %v) { 89; CHECK-LABEL: store_i64: 90; CHECK: # %bb.0: 91; CHECK-NEXT: movq %rsi, (%rdi) 92; CHECK-NEXT: retq 93 store atomic i64 %v, i64* %ptr unordered, align 8 94 ret void 95} 96 97;; The tests in the rest of this file are intended to show transforms which we 98;; either *can't* do for legality, or don't currently implement. The later 99;; are noted carefully where relevant. 100 101;; Start w/some clearly illegal ones. 102 103; Must use a full width op, not a byte op 104define void @narrow_writeback_or(i64* %ptr) { 105; CHECK-O0-LABEL: narrow_writeback_or: 106; CHECK-O0: # %bb.0: 107; CHECK-O0-NEXT: movq (%rdi), %rax 108; CHECK-O0-NEXT: orq $7, %rax 109; CHECK-O0-NEXT: movq %rax, (%rdi) 110; CHECK-O0-NEXT: retq 111; 112; CHECK-O3-LABEL: narrow_writeback_or: 113; CHECK-O3: # %bb.0: 114; CHECK-O3-NEXT: orq $7, (%rdi) 115; CHECK-O3-NEXT: retq 116 %v = load atomic i64, i64* %ptr unordered, align 8 117 %v.new = or i64 %v, 7 118 store atomic i64 %v.new, i64* %ptr unordered, align 8 119 ret void 120} 121 122; Must use a full width op, not a byte op 123define void @narrow_writeback_and(i64* %ptr) { 124; CHECK-O0-LABEL: narrow_writeback_and: 125; CHECK-O0: # %bb.0: 126; CHECK-O0-NEXT: movq (%rdi), %rax 127; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax 128; CHECK-O0-NEXT: andl $-256, %eax 129; CHECK-O0-NEXT: # kill: def $rax killed $eax 130; CHECK-O0-NEXT: movq %rax, (%rdi) 131; CHECK-O0-NEXT: retq 132; 133; CHECK-O3-LABEL: narrow_writeback_and: 134; CHECK-O3: # %bb.0: 135; CHECK-O3-NEXT: movl $4294967040, %eax # imm = 0xFFFFFF00 136; CHECK-O3-NEXT: andq %rax, (%rdi) 137; CHECK-O3-NEXT: retq 138 %v = load atomic i64, i64* %ptr unordered, align 8 139 %v.new = and i64 %v, 4294967040 ;; 0xFFFF_FF00 140 store atomic i64 %v.new, i64* %ptr unordered, align 8 141 ret void 142} 143 144; Must use a full width op, not a byte op 145define void @narrow_writeback_xor(i64* %ptr) { 146; CHECK-O0-LABEL: narrow_writeback_xor: 147; CHECK-O0: # %bb.0: 148; CHECK-O0-NEXT: movq (%rdi), %rax 149; CHECK-O0-NEXT: xorq $7, %rax 150; CHECK-O0-NEXT: movq %rax, (%rdi) 151; CHECK-O0-NEXT: retq 152; 153; CHECK-O3-LABEL: narrow_writeback_xor: 154; CHECK-O3: # %bb.0: 155; CHECK-O3-NEXT: xorq $7, (%rdi) 156; CHECK-O3-NEXT: retq 157 %v = load atomic i64, i64* %ptr unordered, align 8 158 %v.new = xor i64 %v, 7 159 store atomic i64 %v.new, i64* %ptr unordered, align 8 160 ret void 161} 162 163;; Next batch of tests are exercising cases where store widening would 164;; improve codegeneration. Note that widening is only legal if the 165;; resulting type would be atomic. Each tests has a well aligned, and 166;; unaligned variant to ensure we get correct codegen here. 167;; Note: It's not a legality issue, but there's a gotcha here to be aware 168;; of. Once we widen a pair of atomic stores, we loose the information 169;; that the original atomicity requirement was half the width. Given that, 170;; we can't then split the load again. This challenges our usual iterative 171;; approach to incremental improvement. 172 173; Legal if wider type is also atomic (TODO) 174define void @widen_store(i32* %p0, i32 %v1, i32 %v2) { 175; CHECK-LABEL: widen_store: 176; CHECK: # %bb.0: 177; CHECK-NEXT: movl %esi, (%rdi) 178; CHECK-NEXT: movl %edx, 4(%rdi) 179; CHECK-NEXT: retq 180 %p1 = getelementptr i32, i32* %p0, i64 1 181 store atomic i32 %v1, i32* %p0 unordered, align 8 182 store atomic i32 %v2, i32* %p1 unordered, align 4 183 ret void 184} 185 186; This one is *NOT* legal to widen. With weaker alignment, 187; the wider type might cross a cache line and violate the 188; atomicity requirement. 189define void @widen_store_unaligned(i32* %p0, i32 %v1, i32 %v2) { 190; CHECK-LABEL: widen_store_unaligned: 191; CHECK: # %bb.0: 192; CHECK-NEXT: movl %esi, (%rdi) 193; CHECK-NEXT: movl %edx, 4(%rdi) 194; CHECK-NEXT: retq 195 %p1 = getelementptr i32, i32* %p0, i64 1 196 store atomic i32 %v1, i32* %p0 unordered, align 4 197 store atomic i32 %v2, i32* %p1 unordered, align 4 198 ret void 199} 200 201; Legal if wider type is also atomic (TODO) 202define void @widen_broadcast(i32* %p0, i32 %v) { 203; CHECK-LABEL: widen_broadcast: 204; CHECK: # %bb.0: 205; CHECK-NEXT: movl %esi, (%rdi) 206; CHECK-NEXT: movl %esi, 4(%rdi) 207; CHECK-NEXT: retq 208 %p1 = getelementptr i32, i32* %p0, i64 1 209 store atomic i32 %v, i32* %p0 unordered, align 8 210 store atomic i32 %v, i32* %p1 unordered, align 4 211 ret void 212} 213 214; Not legal to widen due to alignment restriction 215define void @widen_broadcast_unaligned(i32* %p0, i32 %v) { 216; CHECK-LABEL: widen_broadcast_unaligned: 217; CHECK: # %bb.0: 218; CHECK-NEXT: movl %esi, (%rdi) 219; CHECK-NEXT: movl %esi, 4(%rdi) 220; CHECK-NEXT: retq 221 %p1 = getelementptr i32, i32* %p0, i64 1 222 store atomic i32 %v, i32* %p0 unordered, align 4 223 store atomic i32 %v, i32* %p1 unordered, align 4 224 ret void 225} 226 227define i128 @load_i128(i128* %ptr) { 228; CHECK-O0-LABEL: load_i128: 229; CHECK-O0: # %bb.0: 230; CHECK-O0-NEXT: pushq %rbx 231; CHECK-O0-NEXT: .cfi_def_cfa_offset 16 232; CHECK-O0-NEXT: .cfi_offset %rbx, -16 233; CHECK-O0-NEXT: xorl %eax, %eax 234; CHECK-O0-NEXT: movl %eax, %ebx 235; CHECK-O0-NEXT: movq %rbx, %rax 236; CHECK-O0-NEXT: movq %rbx, %rdx 237; CHECK-O0-NEXT: movq %rbx, %rcx 238; CHECK-O0-NEXT: lock cmpxchg16b (%rdi) 239; CHECK-O0-NEXT: popq %rbx 240; CHECK-O0-NEXT: .cfi_def_cfa_offset 8 241; CHECK-O0-NEXT: retq 242; 243; CHECK-O3-LABEL: load_i128: 244; CHECK-O3: # %bb.0: 245; CHECK-O3-NEXT: pushq %rbx 246; CHECK-O3-NEXT: .cfi_def_cfa_offset 16 247; CHECK-O3-NEXT: .cfi_offset %rbx, -16 248; CHECK-O3-NEXT: xorl %eax, %eax 249; CHECK-O3-NEXT: xorl %edx, %edx 250; CHECK-O3-NEXT: xorl %ecx, %ecx 251; CHECK-O3-NEXT: xorl %ebx, %ebx 252; CHECK-O3-NEXT: lock cmpxchg16b (%rdi) 253; CHECK-O3-NEXT: popq %rbx 254; CHECK-O3-NEXT: .cfi_def_cfa_offset 8 255; CHECK-O3-NEXT: retq 256 %v = load atomic i128, i128* %ptr unordered, align 16 257 ret i128 %v 258} 259 260define void @store_i128(i128* %ptr, i128 %v) { 261; CHECK-O0-LABEL: store_i128: 262; CHECK-O0: # %bb.0: 263; CHECK-O0-NEXT: pushq %rbx 264; CHECK-O0-NEXT: .cfi_def_cfa_offset 16 265; CHECK-O0-NEXT: .cfi_offset %rbx, -16 266; CHECK-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 267; CHECK-O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 268; CHECK-O0-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 269; CHECK-O0-NEXT: movq (%rdi), %rax 270; CHECK-O0-NEXT: movq 8(%rdi), %rdx 271; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 272; CHECK-O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 273; CHECK-O0-NEXT: jmp .LBB16_1 274; CHECK-O0-NEXT: .LBB16_1: # %atomicrmw.start 275; CHECK-O0-NEXT: # =>This Inner Loop Header: Depth=1 276; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload 277; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 278; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload 279; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload 280; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 281; CHECK-O0-NEXT: lock cmpxchg16b (%rsi) 282; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 283; CHECK-O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 284; CHECK-O0-NEXT: jne .LBB16_1 285; CHECK-O0-NEXT: jmp .LBB16_2 286; CHECK-O0-NEXT: .LBB16_2: # %atomicrmw.end 287; CHECK-O0-NEXT: popq %rbx 288; CHECK-O0-NEXT: .cfi_def_cfa_offset 8 289; CHECK-O0-NEXT: retq 290; 291; CHECK-O3-LABEL: store_i128: 292; CHECK-O3: # %bb.0: 293; CHECK-O3-NEXT: pushq %rbx 294; CHECK-O3-NEXT: .cfi_def_cfa_offset 16 295; CHECK-O3-NEXT: .cfi_offset %rbx, -16 296; CHECK-O3-NEXT: movq %rdx, %rcx 297; CHECK-O3-NEXT: movq %rsi, %rbx 298; CHECK-O3-NEXT: movq (%rdi), %rax 299; CHECK-O3-NEXT: movq 8(%rdi), %rdx 300; CHECK-O3-NEXT: .p2align 4, 0x90 301; CHECK-O3-NEXT: .LBB16_1: # %atomicrmw.start 302; CHECK-O3-NEXT: # =>This Inner Loop Header: Depth=1 303; CHECK-O3-NEXT: lock cmpxchg16b (%rdi) 304; CHECK-O3-NEXT: jne .LBB16_1 305; CHECK-O3-NEXT: # %bb.2: # %atomicrmw.end 306; CHECK-O3-NEXT: popq %rbx 307; CHECK-O3-NEXT: .cfi_def_cfa_offset 8 308; CHECK-O3-NEXT: retq 309 store atomic i128 %v, i128* %ptr unordered, align 16 310 ret void 311} 312 313define i256 @load_i256(i256* %ptr) { 314; CHECK-O0-LABEL: load_i256: 315; CHECK-O0: # %bb.0: 316; CHECK-O0-NEXT: subq $56, %rsp 317; CHECK-O0-NEXT: .cfi_def_cfa_offset 64 318; CHECK-O0-NEXT: movq %rdi, %rax 319; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 320; CHECK-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 321; CHECK-O0-NEXT: movl $32, %edi 322; CHECK-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx 323; CHECK-O0-NEXT: xorl %ecx, %ecx 324; CHECK-O0-NEXT: callq __atomic_load@PLT 325; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload 326; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 327; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %rcx 328; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %rdx 329; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %rsi 330; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %r8 331; CHECK-O0-NEXT: movq %r8, 24(%rdi) 332; CHECK-O0-NEXT: movq %rsi, 16(%rdi) 333; CHECK-O0-NEXT: movq %rdx, 8(%rdi) 334; CHECK-O0-NEXT: movq %rcx, (%rdi) 335; CHECK-O0-NEXT: addq $56, %rsp 336; CHECK-O0-NEXT: .cfi_def_cfa_offset 8 337; CHECK-O0-NEXT: retq 338; 339; CHECK-O3-LABEL: load_i256: 340; CHECK-O3: # %bb.0: 341; CHECK-O3-NEXT: pushq %rbx 342; CHECK-O3-NEXT: .cfi_def_cfa_offset 16 343; CHECK-O3-NEXT: subq $32, %rsp 344; CHECK-O3-NEXT: .cfi_def_cfa_offset 48 345; CHECK-O3-NEXT: .cfi_offset %rbx, -16 346; CHECK-O3-NEXT: movq %rdi, %rbx 347; CHECK-O3-NEXT: movq %rsp, %rdx 348; CHECK-O3-NEXT: movl $32, %edi 349; CHECK-O3-NEXT: xorl %ecx, %ecx 350; CHECK-O3-NEXT: callq __atomic_load@PLT 351; CHECK-O3-NEXT: vmovups (%rsp), %ymm0 352; CHECK-O3-NEXT: vmovups %ymm0, (%rbx) 353; CHECK-O3-NEXT: movq %rbx, %rax 354; CHECK-O3-NEXT: addq $32, %rsp 355; CHECK-O3-NEXT: .cfi_def_cfa_offset 16 356; CHECK-O3-NEXT: popq %rbx 357; CHECK-O3-NEXT: .cfi_def_cfa_offset 8 358; CHECK-O3-NEXT: vzeroupper 359; CHECK-O3-NEXT: retq 360 %v = load atomic i256, i256* %ptr unordered, align 16 361 ret i256 %v 362} 363 364define void @store_i256(i256* %ptr, i256 %v) { 365; CHECK-O0-LABEL: store_i256: 366; CHECK-O0: # %bb.0: 367; CHECK-O0-NEXT: subq $40, %rsp 368; CHECK-O0-NEXT: .cfi_def_cfa_offset 48 369; CHECK-O0-NEXT: movq %rdx, %rax 370; CHECK-O0-NEXT: movq %rsi, (%rsp) # 8-byte Spill 371; CHECK-O0-NEXT: movq %rdi, %rsi 372; CHECK-O0-NEXT: movq (%rsp), %rdi # 8-byte Reload 373; CHECK-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx 374; CHECK-O0-NEXT: movq %rdi, {{[0-9]+}}(%rsp) 375; CHECK-O0-NEXT: movq %rax, {{[0-9]+}}(%rsp) 376; CHECK-O0-NEXT: movq %rcx, {{[0-9]+}}(%rsp) 377; CHECK-O0-NEXT: movq %r8, {{[0-9]+}}(%rsp) 378; CHECK-O0-NEXT: movl $32, %edi 379; CHECK-O0-NEXT: xorl %ecx, %ecx 380; CHECK-O0-NEXT: callq __atomic_store@PLT 381; CHECK-O0-NEXT: addq $40, %rsp 382; CHECK-O0-NEXT: .cfi_def_cfa_offset 8 383; CHECK-O0-NEXT: retq 384; 385; CHECK-O3-LABEL: store_i256: 386; CHECK-O3: # %bb.0: 387; CHECK-O3-NEXT: subq $40, %rsp 388; CHECK-O3-NEXT: .cfi_def_cfa_offset 48 389; CHECK-O3-NEXT: movq %rdi, %rax 390; CHECK-O3-NEXT: movq %r8, {{[0-9]+}}(%rsp) 391; CHECK-O3-NEXT: movq %rcx, {{[0-9]+}}(%rsp) 392; CHECK-O3-NEXT: movq %rdx, {{[0-9]+}}(%rsp) 393; CHECK-O3-NEXT: movq %rsi, {{[0-9]+}}(%rsp) 394; CHECK-O3-NEXT: leaq {{[0-9]+}}(%rsp), %rdx 395; CHECK-O3-NEXT: movl $32, %edi 396; CHECK-O3-NEXT: movq %rax, %rsi 397; CHECK-O3-NEXT: xorl %ecx, %ecx 398; CHECK-O3-NEXT: callq __atomic_store@PLT 399; CHECK-O3-NEXT: addq $40, %rsp 400; CHECK-O3-NEXT: .cfi_def_cfa_offset 8 401; CHECK-O3-NEXT: retq 402 store atomic i256 %v, i256* %ptr unordered, align 16 403 ret void 404} 405 406; Legal if wider type is also atomic (TODO) 407define void @vec_store(i32* %p0, <2 x i32> %vec) { 408; CHECK-O0-CUR-LABEL: vec_store: 409; CHECK-O0-CUR: # %bb.0: 410; CHECK-O0-CUR-NEXT: vmovd %xmm0, %ecx 411; CHECK-O0-CUR-NEXT: vpextrd $1, %xmm0, %eax 412; CHECK-O0-CUR-NEXT: movl %ecx, (%rdi) 413; CHECK-O0-CUR-NEXT: movl %eax, 4(%rdi) 414; CHECK-O0-CUR-NEXT: retq 415; 416; CHECK-O3-CUR-LABEL: vec_store: 417; CHECK-O3-CUR: # %bb.0: 418; CHECK-O3-CUR-NEXT: vmovd %xmm0, %eax 419; CHECK-O3-CUR-NEXT: vpextrd $1, %xmm0, %ecx 420; CHECK-O3-CUR-NEXT: movl %eax, (%rdi) 421; CHECK-O3-CUR-NEXT: movl %ecx, 4(%rdi) 422; CHECK-O3-CUR-NEXT: retq 423; 424; CHECK-O0-EX-LABEL: vec_store: 425; CHECK-O0-EX: # %bb.0: 426; CHECK-O0-EX-NEXT: vmovd %xmm0, (%rdi) 427; CHECK-O0-EX-NEXT: vpextrd $1, %xmm0, 4(%rdi) 428; CHECK-O0-EX-NEXT: retq 429; 430; CHECK-O3-EX-LABEL: vec_store: 431; CHECK-O3-EX: # %bb.0: 432; CHECK-O3-EX-NEXT: vmovss %xmm0, (%rdi) 433; CHECK-O3-EX-NEXT: vextractps $1, %xmm0, 4(%rdi) 434; CHECK-O3-EX-NEXT: retq 435 %v1 = extractelement <2 x i32> %vec, i32 0 436 %v2 = extractelement <2 x i32> %vec, i32 1 437 %p1 = getelementptr i32, i32* %p0, i64 1 438 store atomic i32 %v1, i32* %p0 unordered, align 8 439 store atomic i32 %v2, i32* %p1 unordered, align 4 440 ret void 441} 442 443; Not legal to widen due to alignment restriction 444define void @vec_store_unaligned(i32* %p0, <2 x i32> %vec) { 445; CHECK-O0-CUR-LABEL: vec_store_unaligned: 446; CHECK-O0-CUR: # %bb.0: 447; CHECK-O0-CUR-NEXT: vmovd %xmm0, %ecx 448; CHECK-O0-CUR-NEXT: vpextrd $1, %xmm0, %eax 449; CHECK-O0-CUR-NEXT: movl %ecx, (%rdi) 450; CHECK-O0-CUR-NEXT: movl %eax, 4(%rdi) 451; CHECK-O0-CUR-NEXT: retq 452; 453; CHECK-O3-CUR-LABEL: vec_store_unaligned: 454; CHECK-O3-CUR: # %bb.0: 455; CHECK-O3-CUR-NEXT: vmovd %xmm0, %eax 456; CHECK-O3-CUR-NEXT: vpextrd $1, %xmm0, %ecx 457; CHECK-O3-CUR-NEXT: movl %eax, (%rdi) 458; CHECK-O3-CUR-NEXT: movl %ecx, 4(%rdi) 459; CHECK-O3-CUR-NEXT: retq 460; 461; CHECK-O0-EX-LABEL: vec_store_unaligned: 462; CHECK-O0-EX: # %bb.0: 463; CHECK-O0-EX-NEXT: vmovd %xmm0, (%rdi) 464; CHECK-O0-EX-NEXT: vpextrd $1, %xmm0, 4(%rdi) 465; CHECK-O0-EX-NEXT: retq 466; 467; CHECK-O3-EX-LABEL: vec_store_unaligned: 468; CHECK-O3-EX: # %bb.0: 469; CHECK-O3-EX-NEXT: vmovss %xmm0, (%rdi) 470; CHECK-O3-EX-NEXT: vextractps $1, %xmm0, 4(%rdi) 471; CHECK-O3-EX-NEXT: retq 472 %v1 = extractelement <2 x i32> %vec, i32 0 473 %v2 = extractelement <2 x i32> %vec, i32 1 474 %p1 = getelementptr i32, i32* %p0, i64 1 475 store atomic i32 %v1, i32* %p0 unordered, align 4 476 store atomic i32 %v2, i32* %p1 unordered, align 4 477 ret void 478} 479 480 481 482; Legal if wider type is also atomic (TODO) 483; Also, can avoid register move from xmm to eax (TODO) 484define void @widen_broadcast2(i32* %p0, <2 x i32> %vec) { 485; CHECK-O0-CUR-LABEL: widen_broadcast2: 486; CHECK-O0-CUR: # %bb.0: 487; CHECK-O0-CUR-NEXT: vmovd %xmm0, %eax 488; CHECK-O0-CUR-NEXT: movl %eax, (%rdi) 489; CHECK-O0-CUR-NEXT: movl %eax, 4(%rdi) 490; CHECK-O0-CUR-NEXT: retq 491; 492; CHECK-O3-CUR-LABEL: widen_broadcast2: 493; CHECK-O3-CUR: # %bb.0: 494; CHECK-O3-CUR-NEXT: vmovd %xmm0, %eax 495; CHECK-O3-CUR-NEXT: movl %eax, (%rdi) 496; CHECK-O3-CUR-NEXT: movl %eax, 4(%rdi) 497; CHECK-O3-CUR-NEXT: retq 498; 499; CHECK-O0-EX-LABEL: widen_broadcast2: 500; CHECK-O0-EX: # %bb.0: 501; CHECK-O0-EX-NEXT: vmovd %xmm0, (%rdi) 502; CHECK-O0-EX-NEXT: vmovd %xmm0, 4(%rdi) 503; CHECK-O0-EX-NEXT: retq 504; 505; CHECK-O3-EX-LABEL: widen_broadcast2: 506; CHECK-O3-EX: # %bb.0: 507; CHECK-O3-EX-NEXT: vmovss %xmm0, (%rdi) 508; CHECK-O3-EX-NEXT: vmovss %xmm0, 4(%rdi) 509; CHECK-O3-EX-NEXT: retq 510 %v1 = extractelement <2 x i32> %vec, i32 0 511 %p1 = getelementptr i32, i32* %p0, i64 1 512 store atomic i32 %v1, i32* %p0 unordered, align 8 513 store atomic i32 %v1, i32* %p1 unordered, align 4 514 ret void 515} 516 517; Not legal to widen due to alignment restriction 518define void @widen_broadcast2_unaligned(i32* %p0, <2 x i32> %vec) { 519; CHECK-O0-CUR-LABEL: widen_broadcast2_unaligned: 520; CHECK-O0-CUR: # %bb.0: 521; CHECK-O0-CUR-NEXT: vmovd %xmm0, %eax 522; CHECK-O0-CUR-NEXT: movl %eax, (%rdi) 523; CHECK-O0-CUR-NEXT: movl %eax, 4(%rdi) 524; CHECK-O0-CUR-NEXT: retq 525; 526; CHECK-O3-CUR-LABEL: widen_broadcast2_unaligned: 527; CHECK-O3-CUR: # %bb.0: 528; CHECK-O3-CUR-NEXT: vmovd %xmm0, %eax 529; CHECK-O3-CUR-NEXT: movl %eax, (%rdi) 530; CHECK-O3-CUR-NEXT: movl %eax, 4(%rdi) 531; CHECK-O3-CUR-NEXT: retq 532; 533; CHECK-O0-EX-LABEL: widen_broadcast2_unaligned: 534; CHECK-O0-EX: # %bb.0: 535; CHECK-O0-EX-NEXT: vmovd %xmm0, (%rdi) 536; CHECK-O0-EX-NEXT: vmovd %xmm0, 4(%rdi) 537; CHECK-O0-EX-NEXT: retq 538; 539; CHECK-O3-EX-LABEL: widen_broadcast2_unaligned: 540; CHECK-O3-EX: # %bb.0: 541; CHECK-O3-EX-NEXT: vmovss %xmm0, (%rdi) 542; CHECK-O3-EX-NEXT: vmovss %xmm0, 4(%rdi) 543; CHECK-O3-EX-NEXT: retq 544 %v1 = extractelement <2 x i32> %vec, i32 0 545 %p1 = getelementptr i32, i32* %p0, i64 1 546 store atomic i32 %v1, i32* %p0 unordered, align 4 547 store atomic i32 %v1, i32* %p1 unordered, align 4 548 ret void 549} 550 551; Legal if wider type is also atomic (TODO) 552define void @widen_zero_init(i32* %p0, i32 %v1, i32 %v2) { 553; CHECK-LABEL: widen_zero_init: 554; CHECK: # %bb.0: 555; CHECK-NEXT: movl $0, (%rdi) 556; CHECK-NEXT: movl $0, 4(%rdi) 557; CHECK-NEXT: retq 558 %p1 = getelementptr i32, i32* %p0, i64 1 559 store atomic i32 0, i32* %p0 unordered, align 8 560 store atomic i32 0, i32* %p1 unordered, align 4 561 ret void 562} 563 564; Not legal to widen due to alignment restriction 565define void @widen_zero_init_unaligned(i32* %p0, i32 %v1, i32 %v2) { 566; CHECK-LABEL: widen_zero_init_unaligned: 567; CHECK: # %bb.0: 568; CHECK-NEXT: movl $0, (%rdi) 569; CHECK-NEXT: movl $0, 4(%rdi) 570; CHECK-NEXT: retq 571 %p1 = getelementptr i32, i32* %p0, i64 1 572 store atomic i32 0, i32* %p0 unordered, align 4 573 store atomic i32 0, i32* %p1 unordered, align 4 574 ret void 575} 576 577;; The next batch of tests are stressing load folding. Folding is legal 578;; on x86, so these are simply checking optimization quality. 579 580; Legal, as expected 581define i64 @load_fold_add1(i64* %p) { 582; CHECK-LABEL: load_fold_add1: 583; CHECK: # %bb.0: 584; CHECK-NEXT: movq (%rdi), %rax 585; CHECK-NEXT: addq $15, %rax 586; CHECK-NEXT: retq 587 %v = load atomic i64, i64* %p unordered, align 8 588 %ret = add i64 %v, 15 589 ret i64 %ret 590} 591 592define i64 @load_fold_add2(i64* %p, i64 %v2) { 593; CHECK-LABEL: load_fold_add2: 594; CHECK: # %bb.0: 595; CHECK-NEXT: movq %rsi, %rax 596; CHECK-NEXT: addq (%rdi), %rax 597; CHECK-NEXT: retq 598 %v = load atomic i64, i64* %p unordered, align 8 599 %ret = add i64 %v, %v2 600 ret i64 %ret 601} 602 603define i64 @load_fold_add3(i64* %p1, i64* %p2) { 604; CHECK-O0-LABEL: load_fold_add3: 605; CHECK-O0: # %bb.0: 606; CHECK-O0-NEXT: movq (%rdi), %rax 607; CHECK-O0-NEXT: addq (%rsi), %rax 608; CHECK-O0-NEXT: retq 609; 610; CHECK-O3-CUR-LABEL: load_fold_add3: 611; CHECK-O3-CUR: # %bb.0: 612; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 613; CHECK-O3-CUR-NEXT: addq (%rdi), %rax 614; CHECK-O3-CUR-NEXT: retq 615; 616; CHECK-O3-EX-LABEL: load_fold_add3: 617; CHECK-O3-EX: # %bb.0: 618; CHECK-O3-EX-NEXT: movq (%rdi), %rax 619; CHECK-O3-EX-NEXT: addq (%rsi), %rax 620; CHECK-O3-EX-NEXT: retq 621 %v = load atomic i64, i64* %p1 unordered, align 8 622 %v2 = load atomic i64, i64* %p2 unordered, align 8 623 %ret = add i64 %v, %v2 624 ret i64 %ret 625} 626 627; Legal, as expected 628define i64 @load_fold_sub1(i64* %p) { 629; CHECK-O0-LABEL: load_fold_sub1: 630; CHECK-O0: # %bb.0: 631; CHECK-O0-NEXT: movq (%rdi), %rax 632; CHECK-O0-NEXT: subq $15, %rax 633; CHECK-O0-NEXT: retq 634; 635; CHECK-O3-LABEL: load_fold_sub1: 636; CHECK-O3: # %bb.0: 637; CHECK-O3-NEXT: movq (%rdi), %rax 638; CHECK-O3-NEXT: addq $-15, %rax 639; CHECK-O3-NEXT: retq 640 %v = load atomic i64, i64* %p unordered, align 8 641 %ret = sub i64 %v, 15 642 ret i64 %ret 643} 644 645define i64 @load_fold_sub2(i64* %p, i64 %v2) { 646; CHECK-LABEL: load_fold_sub2: 647; CHECK: # %bb.0: 648; CHECK-NEXT: movq (%rdi), %rax 649; CHECK-NEXT: subq %rsi, %rax 650; CHECK-NEXT: retq 651 %v = load atomic i64, i64* %p unordered, align 8 652 %ret = sub i64 %v, %v2 653 ret i64 %ret 654} 655 656define i64 @load_fold_sub3(i64* %p1, i64* %p2) { 657; CHECK-LABEL: load_fold_sub3: 658; CHECK: # %bb.0: 659; CHECK-NEXT: movq (%rdi), %rax 660; CHECK-NEXT: subq (%rsi), %rax 661; CHECK-NEXT: retq 662 %v = load atomic i64, i64* %p1 unordered, align 8 663 %v2 = load atomic i64, i64* %p2 unordered, align 8 664 %ret = sub i64 %v, %v2 665 ret i64 %ret 666} 667 668; Legal, as expected 669define i64 @load_fold_mul1(i64* %p) { 670; CHECK-O0-LABEL: load_fold_mul1: 671; CHECK-O0: # %bb.0: 672; CHECK-O0-NEXT: imulq $15, (%rdi), %rax 673; CHECK-O0-NEXT: retq 674; 675; CHECK-O3-LABEL: load_fold_mul1: 676; CHECK-O3: # %bb.0: 677; CHECK-O3-NEXT: movq (%rdi), %rax 678; CHECK-O3-NEXT: leaq (%rax,%rax,4), %rax 679; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax 680; CHECK-O3-NEXT: retq 681 %v = load atomic i64, i64* %p unordered, align 8 682 %ret = mul i64 %v, 15 683 ret i64 %ret 684} 685 686define i64 @load_fold_mul2(i64* %p, i64 %v2) { 687; CHECK-LABEL: load_fold_mul2: 688; CHECK: # %bb.0: 689; CHECK-NEXT: movq %rsi, %rax 690; CHECK-NEXT: imulq (%rdi), %rax 691; CHECK-NEXT: retq 692 %v = load atomic i64, i64* %p unordered, align 8 693 %ret = mul i64 %v, %v2 694 ret i64 %ret 695} 696 697define i64 @load_fold_mul3(i64* %p1, i64* %p2) { 698; CHECK-O0-LABEL: load_fold_mul3: 699; CHECK-O0: # %bb.0: 700; CHECK-O0-NEXT: movq (%rdi), %rax 701; CHECK-O0-NEXT: imulq (%rsi), %rax 702; CHECK-O0-NEXT: retq 703; 704; CHECK-O3-CUR-LABEL: load_fold_mul3: 705; CHECK-O3-CUR: # %bb.0: 706; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 707; CHECK-O3-CUR-NEXT: imulq (%rdi), %rax 708; CHECK-O3-CUR-NEXT: retq 709; 710; CHECK-O3-EX-LABEL: load_fold_mul3: 711; CHECK-O3-EX: # %bb.0: 712; CHECK-O3-EX-NEXT: movq (%rdi), %rax 713; CHECK-O3-EX-NEXT: imulq (%rsi), %rax 714; CHECK-O3-EX-NEXT: retq 715 %v = load atomic i64, i64* %p1 unordered, align 8 716 %v2 = load atomic i64, i64* %p2 unordered, align 8 717 %ret = mul i64 %v, %v2 718 ret i64 %ret 719} 720 721; Legal to fold (TODO) 722define i64 @load_fold_sdiv1(i64* %p) { 723; CHECK-O0-LABEL: load_fold_sdiv1: 724; CHECK-O0: # %bb.0: 725; CHECK-O0-NEXT: movq (%rdi), %rax 726; CHECK-O0-NEXT: movl $15, %ecx 727; CHECK-O0-NEXT: cqto 728; CHECK-O0-NEXT: idivq %rcx 729; CHECK-O0-NEXT: retq 730; 731; CHECK-O3-LABEL: load_fold_sdiv1: 732; CHECK-O3: # %bb.0: 733; CHECK-O3-NEXT: movq (%rdi), %rcx 734; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 735; CHECK-O3-NEXT: movq %rcx, %rax 736; CHECK-O3-NEXT: imulq %rdx 737; CHECK-O3-NEXT: addq %rcx, %rdx 738; CHECK-O3-NEXT: movq %rdx, %rax 739; CHECK-O3-NEXT: shrq $63, %rax 740; CHECK-O3-NEXT: sarq $3, %rdx 741; CHECK-O3-NEXT: addq %rdx, %rax 742; CHECK-O3-NEXT: retq 743 %v = load atomic i64, i64* %p unordered, align 8 744 %ret = sdiv i64 %v, 15 745 ret i64 %ret 746} 747 748; Legal to fold (TODO) 749define i64 @load_fold_sdiv2(i64* %p, i64 %v2) { 750; CHECK-O0-LABEL: load_fold_sdiv2: 751; CHECK-O0: # %bb.0: 752; CHECK-O0-NEXT: movq (%rdi), %rax 753; CHECK-O0-NEXT: cqto 754; CHECK-O0-NEXT: idivq %rsi 755; CHECK-O0-NEXT: retq 756; 757; CHECK-O3-LABEL: load_fold_sdiv2: 758; CHECK-O3: # %bb.0: 759; CHECK-O3-NEXT: movq (%rdi), %rax 760; CHECK-O3-NEXT: movq %rax, %rcx 761; CHECK-O3-NEXT: orq %rsi, %rcx 762; CHECK-O3-NEXT: shrq $32, %rcx 763; CHECK-O3-NEXT: je .LBB35_1 764; CHECK-O3-NEXT: # %bb.2: 765; CHECK-O3-NEXT: cqto 766; CHECK-O3-NEXT: idivq %rsi 767; CHECK-O3-NEXT: retq 768; CHECK-O3-NEXT: .LBB35_1: 769; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 770; CHECK-O3-NEXT: xorl %edx, %edx 771; CHECK-O3-NEXT: divl %esi 772; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 773; CHECK-O3-NEXT: retq 774 %v = load atomic i64, i64* %p unordered, align 8 775 %ret = sdiv i64 %v, %v2 776 ret i64 %ret 777} 778 779define i64 @load_fold_sdiv3(i64* %p1, i64* %p2) { 780; CHECK-O0-LABEL: load_fold_sdiv3: 781; CHECK-O0: # %bb.0: 782; CHECK-O0-NEXT: movq (%rdi), %rax 783; CHECK-O0-NEXT: cqto 784; CHECK-O0-NEXT: idivq (%rsi) 785; CHECK-O0-NEXT: retq 786; 787; CHECK-O3-LABEL: load_fold_sdiv3: 788; CHECK-O3: # %bb.0: 789; CHECK-O3-NEXT: movq (%rdi), %rax 790; CHECK-O3-NEXT: movq (%rsi), %rcx 791; CHECK-O3-NEXT: movq %rax, %rdx 792; CHECK-O3-NEXT: orq %rcx, %rdx 793; CHECK-O3-NEXT: shrq $32, %rdx 794; CHECK-O3-NEXT: je .LBB36_1 795; CHECK-O3-NEXT: # %bb.2: 796; CHECK-O3-NEXT: cqto 797; CHECK-O3-NEXT: idivq %rcx 798; CHECK-O3-NEXT: retq 799; CHECK-O3-NEXT: .LBB36_1: 800; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 801; CHECK-O3-NEXT: xorl %edx, %edx 802; CHECK-O3-NEXT: divl %ecx 803; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 804; CHECK-O3-NEXT: retq 805 %v = load atomic i64, i64* %p1 unordered, align 8 806 %v2 = load atomic i64, i64* %p2 unordered, align 8 807 %ret = sdiv i64 %v, %v2 808 ret i64 %ret 809} 810 811; Legal to fold (TODO) 812define i64 @load_fold_udiv1(i64* %p) { 813; CHECK-O0-LABEL: load_fold_udiv1: 814; CHECK-O0: # %bb.0: 815; CHECK-O0-NEXT: movq (%rdi), %rax 816; CHECK-O0-NEXT: movl $15, %ecx 817; CHECK-O0-NEXT: xorl %edx, %edx 818; CHECK-O0-NEXT: # kill: def $rdx killed $edx 819; CHECK-O0-NEXT: divq %rcx 820; CHECK-O0-NEXT: retq 821; 822; CHECK-O3-CUR-LABEL: load_fold_udiv1: 823; CHECK-O3-CUR: # %bb.0: 824; CHECK-O3-CUR-NEXT: movq (%rdi), %rdx 825; CHECK-O3-CUR-NEXT: movabsq $-8608480567731124087, %rax # imm = 0x8888888888888889 826; CHECK-O3-CUR-NEXT: mulxq %rax, %rax, %rax 827; CHECK-O3-CUR-NEXT: shrq $3, %rax 828; CHECK-O3-CUR-NEXT: retq 829; 830; CHECK-O3-EX-LABEL: load_fold_udiv1: 831; CHECK-O3-EX: # %bb.0: 832; CHECK-O3-EX-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 833; CHECK-O3-EX-NEXT: mulxq (%rdi), %rax, %rax 834; CHECK-O3-EX-NEXT: shrq $3, %rax 835; CHECK-O3-EX-NEXT: retq 836 %v = load atomic i64, i64* %p unordered, align 8 837 %ret = udiv i64 %v, 15 838 ret i64 %ret 839} 840 841define i64 @load_fold_udiv2(i64* %p, i64 %v2) { 842; CHECK-O0-LABEL: load_fold_udiv2: 843; CHECK-O0: # %bb.0: 844; CHECK-O0-NEXT: movq (%rdi), %rax 845; CHECK-O0-NEXT: xorl %ecx, %ecx 846; CHECK-O0-NEXT: movl %ecx, %edx 847; CHECK-O0-NEXT: divq %rsi 848; CHECK-O0-NEXT: retq 849; 850; CHECK-O3-LABEL: load_fold_udiv2: 851; CHECK-O3: # %bb.0: 852; CHECK-O3-NEXT: movq (%rdi), %rax 853; CHECK-O3-NEXT: movq %rax, %rcx 854; CHECK-O3-NEXT: orq %rsi, %rcx 855; CHECK-O3-NEXT: shrq $32, %rcx 856; CHECK-O3-NEXT: je .LBB38_1 857; CHECK-O3-NEXT: # %bb.2: 858; CHECK-O3-NEXT: xorl %edx, %edx 859; CHECK-O3-NEXT: divq %rsi 860; CHECK-O3-NEXT: retq 861; CHECK-O3-NEXT: .LBB38_1: 862; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 863; CHECK-O3-NEXT: xorl %edx, %edx 864; CHECK-O3-NEXT: divl %esi 865; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 866; CHECK-O3-NEXT: retq 867 %v = load atomic i64, i64* %p unordered, align 8 868 %ret = udiv i64 %v, %v2 869 ret i64 %ret 870} 871 872define i64 @load_fold_udiv3(i64* %p1, i64* %p2) { 873; CHECK-O0-LABEL: load_fold_udiv3: 874; CHECK-O0: # %bb.0: 875; CHECK-O0-NEXT: movq (%rdi), %rax 876; CHECK-O0-NEXT: xorl %ecx, %ecx 877; CHECK-O0-NEXT: movl %ecx, %edx 878; CHECK-O0-NEXT: divq (%rsi) 879; CHECK-O0-NEXT: retq 880; 881; CHECK-O3-LABEL: load_fold_udiv3: 882; CHECK-O3: # %bb.0: 883; CHECK-O3-NEXT: movq (%rdi), %rax 884; CHECK-O3-NEXT: movq (%rsi), %rcx 885; CHECK-O3-NEXT: movq %rax, %rdx 886; CHECK-O3-NEXT: orq %rcx, %rdx 887; CHECK-O3-NEXT: shrq $32, %rdx 888; CHECK-O3-NEXT: je .LBB39_1 889; CHECK-O3-NEXT: # %bb.2: 890; CHECK-O3-NEXT: xorl %edx, %edx 891; CHECK-O3-NEXT: divq %rcx 892; CHECK-O3-NEXT: retq 893; CHECK-O3-NEXT: .LBB39_1: 894; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 895; CHECK-O3-NEXT: xorl %edx, %edx 896; CHECK-O3-NEXT: divl %ecx 897; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 898; CHECK-O3-NEXT: retq 899 %v = load atomic i64, i64* %p1 unordered, align 8 900 %v2 = load atomic i64, i64* %p2 unordered, align 8 901 %ret = udiv i64 %v, %v2 902 ret i64 %ret 903} 904 905; Legal to fold (TODO) 906define i64 @load_fold_srem1(i64* %p) { 907; CHECK-O0-LABEL: load_fold_srem1: 908; CHECK-O0: # %bb.0: 909; CHECK-O0-NEXT: movq (%rdi), %rax 910; CHECK-O0-NEXT: movl $15, %ecx 911; CHECK-O0-NEXT: cqto 912; CHECK-O0-NEXT: idivq %rcx 913; CHECK-O0-NEXT: movq %rdx, %rax 914; CHECK-O0-NEXT: retq 915; 916; CHECK-O3-LABEL: load_fold_srem1: 917; CHECK-O3: # %bb.0: 918; CHECK-O3-NEXT: movq (%rdi), %rcx 919; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 920; CHECK-O3-NEXT: movq %rcx, %rax 921; CHECK-O3-NEXT: imulq %rdx 922; CHECK-O3-NEXT: addq %rcx, %rdx 923; CHECK-O3-NEXT: movq %rdx, %rax 924; CHECK-O3-NEXT: shrq $63, %rax 925; CHECK-O3-NEXT: sarq $3, %rdx 926; CHECK-O3-NEXT: addq %rax, %rdx 927; CHECK-O3-NEXT: leaq (%rdx,%rdx,4), %rax 928; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax 929; CHECK-O3-NEXT: subq %rax, %rcx 930; CHECK-O3-NEXT: movq %rcx, %rax 931; CHECK-O3-NEXT: retq 932 %v = load atomic i64, i64* %p unordered, align 8 933 %ret = srem i64 %v, 15 934 ret i64 %ret 935} 936 937; Legal, as expected 938define i64 @load_fold_srem2(i64* %p, i64 %v2) { 939; CHECK-O0-LABEL: load_fold_srem2: 940; CHECK-O0: # %bb.0: 941; CHECK-O0-NEXT: movq (%rdi), %rax 942; CHECK-O0-NEXT: cqto 943; CHECK-O0-NEXT: idivq %rsi 944; CHECK-O0-NEXT: movq %rdx, %rax 945; CHECK-O0-NEXT: retq 946; 947; CHECK-O3-LABEL: load_fold_srem2: 948; CHECK-O3: # %bb.0: 949; CHECK-O3-NEXT: movq (%rdi), %rax 950; CHECK-O3-NEXT: movq %rax, %rcx 951; CHECK-O3-NEXT: orq %rsi, %rcx 952; CHECK-O3-NEXT: shrq $32, %rcx 953; CHECK-O3-NEXT: je .LBB41_1 954; CHECK-O3-NEXT: # %bb.2: 955; CHECK-O3-NEXT: cqto 956; CHECK-O3-NEXT: idivq %rsi 957; CHECK-O3-NEXT: movq %rdx, %rax 958; CHECK-O3-NEXT: retq 959; CHECK-O3-NEXT: .LBB41_1: 960; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 961; CHECK-O3-NEXT: xorl %edx, %edx 962; CHECK-O3-NEXT: divl %esi 963; CHECK-O3-NEXT: movl %edx, %eax 964; CHECK-O3-NEXT: retq 965 %v = load atomic i64, i64* %p unordered, align 8 966 %ret = srem i64 %v, %v2 967 ret i64 %ret 968} 969 970define i64 @load_fold_srem3(i64* %p1, i64* %p2) { 971; CHECK-O0-LABEL: load_fold_srem3: 972; CHECK-O0: # %bb.0: 973; CHECK-O0-NEXT: movq (%rdi), %rax 974; CHECK-O0-NEXT: cqto 975; CHECK-O0-NEXT: idivq (%rsi) 976; CHECK-O0-NEXT: movq %rdx, %rax 977; CHECK-O0-NEXT: retq 978; 979; CHECK-O3-LABEL: load_fold_srem3: 980; CHECK-O3: # %bb.0: 981; CHECK-O3-NEXT: movq (%rdi), %rax 982; CHECK-O3-NEXT: movq (%rsi), %rcx 983; CHECK-O3-NEXT: movq %rax, %rdx 984; CHECK-O3-NEXT: orq %rcx, %rdx 985; CHECK-O3-NEXT: shrq $32, %rdx 986; CHECK-O3-NEXT: je .LBB42_1 987; CHECK-O3-NEXT: # %bb.2: 988; CHECK-O3-NEXT: cqto 989; CHECK-O3-NEXT: idivq %rcx 990; CHECK-O3-NEXT: movq %rdx, %rax 991; CHECK-O3-NEXT: retq 992; CHECK-O3-NEXT: .LBB42_1: 993; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 994; CHECK-O3-NEXT: xorl %edx, %edx 995; CHECK-O3-NEXT: divl %ecx 996; CHECK-O3-NEXT: movl %edx, %eax 997; CHECK-O3-NEXT: retq 998 %v = load atomic i64, i64* %p1 unordered, align 8 999 %v2 = load atomic i64, i64* %p2 unordered, align 8 1000 %ret = srem i64 %v, %v2 1001 ret i64 %ret 1002} 1003 1004; Legal to fold (TODO) 1005define i64 @load_fold_urem1(i64* %p) { 1006; CHECK-O0-LABEL: load_fold_urem1: 1007; CHECK-O0: # %bb.0: 1008; CHECK-O0-NEXT: movq (%rdi), %rax 1009; CHECK-O0-NEXT: movl $15, %ecx 1010; CHECK-O0-NEXT: xorl %edx, %edx 1011; CHECK-O0-NEXT: # kill: def $rdx killed $edx 1012; CHECK-O0-NEXT: divq %rcx 1013; CHECK-O0-NEXT: movq %rdx, %rax 1014; CHECK-O0-NEXT: retq 1015; 1016; CHECK-O3-LABEL: load_fold_urem1: 1017; CHECK-O3: # %bb.0: 1018; CHECK-O3-NEXT: movq (%rdi), %rax 1019; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889 1020; CHECK-O3-NEXT: movq %rax, %rdx 1021; CHECK-O3-NEXT: mulxq %rcx, %rcx, %rcx 1022; CHECK-O3-NEXT: shrq $3, %rcx 1023; CHECK-O3-NEXT: leaq (%rcx,%rcx,4), %rcx 1024; CHECK-O3-NEXT: leaq (%rcx,%rcx,2), %rcx 1025; CHECK-O3-NEXT: subq %rcx, %rax 1026; CHECK-O3-NEXT: retq 1027 %v = load atomic i64, i64* %p unordered, align 8 1028 %ret = urem i64 %v, 15 1029 ret i64 %ret 1030} 1031 1032; Legal, as expected 1033define i64 @load_fold_urem2(i64* %p, i64 %v2) { 1034; CHECK-O0-LABEL: load_fold_urem2: 1035; CHECK-O0: # %bb.0: 1036; CHECK-O0-NEXT: movq (%rdi), %rax 1037; CHECK-O0-NEXT: xorl %ecx, %ecx 1038; CHECK-O0-NEXT: movl %ecx, %edx 1039; CHECK-O0-NEXT: divq %rsi 1040; CHECK-O0-NEXT: movq %rdx, %rax 1041; CHECK-O0-NEXT: retq 1042; 1043; CHECK-O3-LABEL: load_fold_urem2: 1044; CHECK-O3: # %bb.0: 1045; CHECK-O3-NEXT: movq (%rdi), %rax 1046; CHECK-O3-NEXT: movq %rax, %rcx 1047; CHECK-O3-NEXT: orq %rsi, %rcx 1048; CHECK-O3-NEXT: shrq $32, %rcx 1049; CHECK-O3-NEXT: je .LBB44_1 1050; CHECK-O3-NEXT: # %bb.2: 1051; CHECK-O3-NEXT: xorl %edx, %edx 1052; CHECK-O3-NEXT: divq %rsi 1053; CHECK-O3-NEXT: movq %rdx, %rax 1054; CHECK-O3-NEXT: retq 1055; CHECK-O3-NEXT: .LBB44_1: 1056; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1057; CHECK-O3-NEXT: xorl %edx, %edx 1058; CHECK-O3-NEXT: divl %esi 1059; CHECK-O3-NEXT: movl %edx, %eax 1060; CHECK-O3-NEXT: retq 1061 %v = load atomic i64, i64* %p unordered, align 8 1062 %ret = urem i64 %v, %v2 1063 ret i64 %ret 1064} 1065 1066define i64 @load_fold_urem3(i64* %p1, i64* %p2) { 1067; CHECK-O0-LABEL: load_fold_urem3: 1068; CHECK-O0: # %bb.0: 1069; CHECK-O0-NEXT: movq (%rdi), %rax 1070; CHECK-O0-NEXT: xorl %ecx, %ecx 1071; CHECK-O0-NEXT: movl %ecx, %edx 1072; CHECK-O0-NEXT: divq (%rsi) 1073; CHECK-O0-NEXT: movq %rdx, %rax 1074; CHECK-O0-NEXT: retq 1075; 1076; CHECK-O3-LABEL: load_fold_urem3: 1077; CHECK-O3: # %bb.0: 1078; CHECK-O3-NEXT: movq (%rdi), %rax 1079; CHECK-O3-NEXT: movq (%rsi), %rcx 1080; CHECK-O3-NEXT: movq %rax, %rdx 1081; CHECK-O3-NEXT: orq %rcx, %rdx 1082; CHECK-O3-NEXT: shrq $32, %rdx 1083; CHECK-O3-NEXT: je .LBB45_1 1084; CHECK-O3-NEXT: # %bb.2: 1085; CHECK-O3-NEXT: xorl %edx, %edx 1086; CHECK-O3-NEXT: divq %rcx 1087; CHECK-O3-NEXT: movq %rdx, %rax 1088; CHECK-O3-NEXT: retq 1089; CHECK-O3-NEXT: .LBB45_1: 1090; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1091; CHECK-O3-NEXT: xorl %edx, %edx 1092; CHECK-O3-NEXT: divl %ecx 1093; CHECK-O3-NEXT: movl %edx, %eax 1094; CHECK-O3-NEXT: retq 1095 %v = load atomic i64, i64* %p1 unordered, align 8 1096 %v2 = load atomic i64, i64* %p2 unordered, align 8 1097 %ret = urem i64 %v, %v2 1098 ret i64 %ret 1099} 1100 1101; Legal, as expected 1102define i64 @load_fold_shl1(i64* %p) { 1103; CHECK-LABEL: load_fold_shl1: 1104; CHECK: # %bb.0: 1105; CHECK-NEXT: movq (%rdi), %rax 1106; CHECK-NEXT: shlq $15, %rax 1107; CHECK-NEXT: retq 1108 %v = load atomic i64, i64* %p unordered, align 8 1109 %ret = shl i64 %v, 15 1110 ret i64 %ret 1111} 1112 1113define i64 @load_fold_shl2(i64* %p, i64 %v2) { 1114; CHECK-O0-LABEL: load_fold_shl2: 1115; CHECK-O0: # %bb.0: 1116; CHECK-O0-NEXT: movq %rsi, %rcx 1117; CHECK-O0-NEXT: movq (%rdi), %rax 1118; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1119; CHECK-O0-NEXT: shlq %cl, %rax 1120; CHECK-O0-NEXT: retq 1121; 1122; CHECK-O3-LABEL: load_fold_shl2: 1123; CHECK-O3: # %bb.0: 1124; CHECK-O3-NEXT: shlxq %rsi, (%rdi), %rax 1125; CHECK-O3-NEXT: retq 1126 %v = load atomic i64, i64* %p unordered, align 8 1127 %ret = shl i64 %v, %v2 1128 ret i64 %ret 1129} 1130 1131define i64 @load_fold_shl3(i64* %p1, i64* %p2) { 1132; CHECK-O0-LABEL: load_fold_shl3: 1133; CHECK-O0: # %bb.0: 1134; CHECK-O0-NEXT: movq (%rdi), %rax 1135; CHECK-O0-NEXT: movq (%rsi), %rcx 1136; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1137; CHECK-O0-NEXT: shlq %cl, %rax 1138; CHECK-O0-NEXT: retq 1139; 1140; CHECK-O3-LABEL: load_fold_shl3: 1141; CHECK-O3: # %bb.0: 1142; CHECK-O3-NEXT: movq (%rsi), %rax 1143; CHECK-O3-NEXT: shlxq %rax, (%rdi), %rax 1144; CHECK-O3-NEXT: retq 1145 %v = load atomic i64, i64* %p1 unordered, align 8 1146 %v2 = load atomic i64, i64* %p2 unordered, align 8 1147 %ret = shl i64 %v, %v2 1148 ret i64 %ret 1149} 1150 1151; Legal, as expected 1152define i64 @load_fold_lshr1(i64* %p) { 1153; CHECK-LABEL: load_fold_lshr1: 1154; CHECK: # %bb.0: 1155; CHECK-NEXT: movq (%rdi), %rax 1156; CHECK-NEXT: shrq $15, %rax 1157; CHECK-NEXT: retq 1158 %v = load atomic i64, i64* %p unordered, align 8 1159 %ret = lshr i64 %v, 15 1160 ret i64 %ret 1161} 1162 1163define i64 @load_fold_lshr2(i64* %p, i64 %v2) { 1164; CHECK-O0-LABEL: load_fold_lshr2: 1165; CHECK-O0: # %bb.0: 1166; CHECK-O0-NEXT: movq %rsi, %rcx 1167; CHECK-O0-NEXT: movq (%rdi), %rax 1168; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1169; CHECK-O0-NEXT: shrq %cl, %rax 1170; CHECK-O0-NEXT: retq 1171; 1172; CHECK-O3-LABEL: load_fold_lshr2: 1173; CHECK-O3: # %bb.0: 1174; CHECK-O3-NEXT: shrxq %rsi, (%rdi), %rax 1175; CHECK-O3-NEXT: retq 1176 %v = load atomic i64, i64* %p unordered, align 8 1177 %ret = lshr i64 %v, %v2 1178 ret i64 %ret 1179} 1180 1181define i64 @load_fold_lshr3(i64* %p1, i64* %p2) { 1182; CHECK-O0-LABEL: load_fold_lshr3: 1183; CHECK-O0: # %bb.0: 1184; CHECK-O0-NEXT: movq (%rdi), %rax 1185; CHECK-O0-NEXT: movq (%rsi), %rcx 1186; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1187; CHECK-O0-NEXT: shrq %cl, %rax 1188; CHECK-O0-NEXT: retq 1189; 1190; CHECK-O3-LABEL: load_fold_lshr3: 1191; CHECK-O3: # %bb.0: 1192; CHECK-O3-NEXT: movq (%rsi), %rax 1193; CHECK-O3-NEXT: shrxq %rax, (%rdi), %rax 1194; CHECK-O3-NEXT: retq 1195 %v = load atomic i64, i64* %p1 unordered, align 8 1196 %v2 = load atomic i64, i64* %p2 unordered, align 8 1197 %ret = lshr i64 %v, %v2 1198 ret i64 %ret 1199} 1200 1201; Legal, as expected 1202define i64 @load_fold_ashr1(i64* %p) { 1203; CHECK-LABEL: load_fold_ashr1: 1204; CHECK: # %bb.0: 1205; CHECK-NEXT: movq (%rdi), %rax 1206; CHECK-NEXT: sarq $15, %rax 1207; CHECK-NEXT: retq 1208 %v = load atomic i64, i64* %p unordered, align 8 1209 %ret = ashr i64 %v, 15 1210 ret i64 %ret 1211} 1212 1213define i64 @load_fold_ashr2(i64* %p, i64 %v2) { 1214; CHECK-O0-LABEL: load_fold_ashr2: 1215; CHECK-O0: # %bb.0: 1216; CHECK-O0-NEXT: movq %rsi, %rcx 1217; CHECK-O0-NEXT: movq (%rdi), %rax 1218; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1219; CHECK-O0-NEXT: sarq %cl, %rax 1220; CHECK-O0-NEXT: retq 1221; 1222; CHECK-O3-LABEL: load_fold_ashr2: 1223; CHECK-O3: # %bb.0: 1224; CHECK-O3-NEXT: sarxq %rsi, (%rdi), %rax 1225; CHECK-O3-NEXT: retq 1226 %v = load atomic i64, i64* %p unordered, align 8 1227 %ret = ashr i64 %v, %v2 1228 ret i64 %ret 1229} 1230 1231define i64 @load_fold_ashr3(i64* %p1, i64* %p2) { 1232; CHECK-O0-LABEL: load_fold_ashr3: 1233; CHECK-O0: # %bb.0: 1234; CHECK-O0-NEXT: movq (%rdi), %rax 1235; CHECK-O0-NEXT: movq (%rsi), %rcx 1236; CHECK-O0-NEXT: # kill: def $cl killed $rcx 1237; CHECK-O0-NEXT: sarq %cl, %rax 1238; CHECK-O0-NEXT: retq 1239; 1240; CHECK-O3-LABEL: load_fold_ashr3: 1241; CHECK-O3: # %bb.0: 1242; CHECK-O3-NEXT: movq (%rsi), %rax 1243; CHECK-O3-NEXT: sarxq %rax, (%rdi), %rax 1244; CHECK-O3-NEXT: retq 1245 %v = load atomic i64, i64* %p1 unordered, align 8 1246 %v2 = load atomic i64, i64* %p2 unordered, align 8 1247 %ret = ashr i64 %v, %v2 1248 ret i64 %ret 1249} 1250 1251; Legal, as expected 1252define i64 @load_fold_and1(i64* %p) { 1253; CHECK-O0-LABEL: load_fold_and1: 1254; CHECK-O0: # %bb.0: 1255; CHECK-O0-NEXT: movq (%rdi), %rax 1256; CHECK-O0-NEXT: andq $15, %rax 1257; CHECK-O0-NEXT: retq 1258; 1259; CHECK-O3-LABEL: load_fold_and1: 1260; CHECK-O3: # %bb.0: 1261; CHECK-O3-NEXT: movq (%rdi), %rax 1262; CHECK-O3-NEXT: andl $15, %eax 1263; CHECK-O3-NEXT: retq 1264 %v = load atomic i64, i64* %p unordered, align 8 1265 %ret = and i64 %v, 15 1266 ret i64 %ret 1267} 1268 1269define i64 @load_fold_and2(i64* %p, i64 %v2) { 1270; CHECK-LABEL: load_fold_and2: 1271; CHECK: # %bb.0: 1272; CHECK-NEXT: movq %rsi, %rax 1273; CHECK-NEXT: andq (%rdi), %rax 1274; CHECK-NEXT: retq 1275 %v = load atomic i64, i64* %p unordered, align 8 1276 %ret = and i64 %v, %v2 1277 ret i64 %ret 1278} 1279 1280define i64 @load_fold_and3(i64* %p1, i64* %p2) { 1281; CHECK-O0-LABEL: load_fold_and3: 1282; CHECK-O0: # %bb.0: 1283; CHECK-O0-NEXT: movq (%rdi), %rax 1284; CHECK-O0-NEXT: andq (%rsi), %rax 1285; CHECK-O0-NEXT: retq 1286; 1287; CHECK-O3-CUR-LABEL: load_fold_and3: 1288; CHECK-O3-CUR: # %bb.0: 1289; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 1290; CHECK-O3-CUR-NEXT: andq (%rdi), %rax 1291; CHECK-O3-CUR-NEXT: retq 1292; 1293; CHECK-O3-EX-LABEL: load_fold_and3: 1294; CHECK-O3-EX: # %bb.0: 1295; CHECK-O3-EX-NEXT: movq (%rdi), %rax 1296; CHECK-O3-EX-NEXT: andq (%rsi), %rax 1297; CHECK-O3-EX-NEXT: retq 1298 %v = load atomic i64, i64* %p1 unordered, align 8 1299 %v2 = load atomic i64, i64* %p2 unordered, align 8 1300 %ret = and i64 %v, %v2 1301 ret i64 %ret 1302} 1303 1304; Legal, as expected 1305define i64 @load_fold_or1(i64* %p) { 1306; CHECK-LABEL: load_fold_or1: 1307; CHECK: # %bb.0: 1308; CHECK-NEXT: movq (%rdi), %rax 1309; CHECK-NEXT: orq $15, %rax 1310; CHECK-NEXT: retq 1311 %v = load atomic i64, i64* %p unordered, align 8 1312 %ret = or i64 %v, 15 1313 ret i64 %ret 1314} 1315 1316define i64 @load_fold_or2(i64* %p, i64 %v2) { 1317; CHECK-LABEL: load_fold_or2: 1318; CHECK: # %bb.0: 1319; CHECK-NEXT: movq %rsi, %rax 1320; CHECK-NEXT: orq (%rdi), %rax 1321; CHECK-NEXT: retq 1322 %v = load atomic i64, i64* %p unordered, align 8 1323 %ret = or i64 %v, %v2 1324 ret i64 %ret 1325} 1326 1327define i64 @load_fold_or3(i64* %p1, i64* %p2) { 1328; CHECK-O0-LABEL: load_fold_or3: 1329; CHECK-O0: # %bb.0: 1330; CHECK-O0-NEXT: movq (%rdi), %rax 1331; CHECK-O0-NEXT: orq (%rsi), %rax 1332; CHECK-O0-NEXT: retq 1333; 1334; CHECK-O3-CUR-LABEL: load_fold_or3: 1335; CHECK-O3-CUR: # %bb.0: 1336; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 1337; CHECK-O3-CUR-NEXT: orq (%rdi), %rax 1338; CHECK-O3-CUR-NEXT: retq 1339; 1340; CHECK-O3-EX-LABEL: load_fold_or3: 1341; CHECK-O3-EX: # %bb.0: 1342; CHECK-O3-EX-NEXT: movq (%rdi), %rax 1343; CHECK-O3-EX-NEXT: orq (%rsi), %rax 1344; CHECK-O3-EX-NEXT: retq 1345 %v = load atomic i64, i64* %p1 unordered, align 8 1346 %v2 = load atomic i64, i64* %p2 unordered, align 8 1347 %ret = or i64 %v, %v2 1348 ret i64 %ret 1349} 1350 1351; Legal, as expected 1352define i64 @load_fold_xor1(i64* %p) { 1353; CHECK-LABEL: load_fold_xor1: 1354; CHECK: # %bb.0: 1355; CHECK-NEXT: movq (%rdi), %rax 1356; CHECK-NEXT: xorq $15, %rax 1357; CHECK-NEXT: retq 1358 %v = load atomic i64, i64* %p unordered, align 8 1359 %ret = xor i64 %v, 15 1360 ret i64 %ret 1361} 1362 1363define i64 @load_fold_xor2(i64* %p, i64 %v2) { 1364; CHECK-LABEL: load_fold_xor2: 1365; CHECK: # %bb.0: 1366; CHECK-NEXT: movq %rsi, %rax 1367; CHECK-NEXT: xorq (%rdi), %rax 1368; CHECK-NEXT: retq 1369 %v = load atomic i64, i64* %p unordered, align 8 1370 %ret = xor i64 %v, %v2 1371 ret i64 %ret 1372} 1373 1374define i64 @load_fold_xor3(i64* %p1, i64* %p2) { 1375; CHECK-O0-LABEL: load_fold_xor3: 1376; CHECK-O0: # %bb.0: 1377; CHECK-O0-NEXT: movq (%rdi), %rax 1378; CHECK-O0-NEXT: xorq (%rsi), %rax 1379; CHECK-O0-NEXT: retq 1380; 1381; CHECK-O3-CUR-LABEL: load_fold_xor3: 1382; CHECK-O3-CUR: # %bb.0: 1383; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 1384; CHECK-O3-CUR-NEXT: xorq (%rdi), %rax 1385; CHECK-O3-CUR-NEXT: retq 1386; 1387; CHECK-O3-EX-LABEL: load_fold_xor3: 1388; CHECK-O3-EX: # %bb.0: 1389; CHECK-O3-EX-NEXT: movq (%rdi), %rax 1390; CHECK-O3-EX-NEXT: xorq (%rsi), %rax 1391; CHECK-O3-EX-NEXT: retq 1392 %v = load atomic i64, i64* %p1 unordered, align 8 1393 %v2 = load atomic i64, i64* %p2 unordered, align 8 1394 %ret = xor i64 %v, %v2 1395 ret i64 %ret 1396} 1397 1398define i1 @load_fold_icmp1(i64* %p) { 1399; CHECK-O0-LABEL: load_fold_icmp1: 1400; CHECK-O0: # %bb.0: 1401; CHECK-O0-NEXT: movq (%rdi), %rax 1402; CHECK-O0-NEXT: subq $15, %rax 1403; CHECK-O0-NEXT: sete %al 1404; CHECK-O0-NEXT: retq 1405; 1406; CHECK-O3-LABEL: load_fold_icmp1: 1407; CHECK-O3: # %bb.0: 1408; CHECK-O3-NEXT: cmpq $15, (%rdi) 1409; CHECK-O3-NEXT: sete %al 1410; CHECK-O3-NEXT: retq 1411 %v = load atomic i64, i64* %p unordered, align 8 1412 %ret = icmp eq i64 %v, 15 1413 ret i1 %ret 1414} 1415 1416define i1 @load_fold_icmp2(i64* %p, i64 %v2) { 1417; CHECK-O0-LABEL: load_fold_icmp2: 1418; CHECK-O0: # %bb.0: 1419; CHECK-O0-NEXT: movq (%rdi), %rax 1420; CHECK-O0-NEXT: subq %rsi, %rax 1421; CHECK-O0-NEXT: sete %al 1422; CHECK-O0-NEXT: retq 1423; 1424; CHECK-O3-LABEL: load_fold_icmp2: 1425; CHECK-O3: # %bb.0: 1426; CHECK-O3-NEXT: cmpq %rsi, (%rdi) 1427; CHECK-O3-NEXT: sete %al 1428; CHECK-O3-NEXT: retq 1429 %v = load atomic i64, i64* %p unordered, align 8 1430 %ret = icmp eq i64 %v, %v2 1431 ret i1 %ret 1432} 1433 1434define i1 @load_fold_icmp3(i64* %p1, i64* %p2) { 1435; CHECK-O0-LABEL: load_fold_icmp3: 1436; CHECK-O0: # %bb.0: 1437; CHECK-O0-NEXT: movq (%rdi), %rax 1438; CHECK-O0-NEXT: movq (%rsi), %rcx 1439; CHECK-O0-NEXT: subq %rcx, %rax 1440; CHECK-O0-NEXT: sete %al 1441; CHECK-O0-NEXT: retq 1442; 1443; CHECK-O3-CUR-LABEL: load_fold_icmp3: 1444; CHECK-O3-CUR: # %bb.0: 1445; CHECK-O3-CUR-NEXT: movq (%rsi), %rax 1446; CHECK-O3-CUR-NEXT: cmpq %rax, (%rdi) 1447; CHECK-O3-CUR-NEXT: sete %al 1448; CHECK-O3-CUR-NEXT: retq 1449; 1450; CHECK-O3-EX-LABEL: load_fold_icmp3: 1451; CHECK-O3-EX: # %bb.0: 1452; CHECK-O3-EX-NEXT: movq (%rdi), %rax 1453; CHECK-O3-EX-NEXT: cmpq (%rsi), %rax 1454; CHECK-O3-EX-NEXT: sete %al 1455; CHECK-O3-EX-NEXT: retq 1456 %v = load atomic i64, i64* %p1 unordered, align 8 1457 %v2 = load atomic i64, i64* %p2 unordered, align 8 1458 %ret = icmp eq i64 %v, %v2 1459 ret i1 %ret 1460} 1461 1462 1463;; The next batch of tests check for read-modify-write patterns 1464;; Legally, it's okay to use a memory operand here as long as the operand 1465;; is well aligned (i.e. doesn't cross a cache line boundary). We are 1466;; required not to narrow the store though! 1467 1468; Legal, as expected 1469define void @rmw_fold_add1(i64* %p, i64 %v) { 1470; CHECK-O0-LABEL: rmw_fold_add1: 1471; CHECK-O0: # %bb.0: 1472; CHECK-O0-NEXT: movq (%rdi), %rax 1473; CHECK-O0-NEXT: addq $15, %rax 1474; CHECK-O0-NEXT: movq %rax, (%rdi) 1475; CHECK-O0-NEXT: retq 1476; 1477; CHECK-O3-LABEL: rmw_fold_add1: 1478; CHECK-O3: # %bb.0: 1479; CHECK-O3-NEXT: addq $15, (%rdi) 1480; CHECK-O3-NEXT: retq 1481 %prev = load atomic i64, i64* %p unordered, align 8 1482 %val = add i64 %prev, 15 1483 store atomic i64 %val, i64* %p unordered, align 8 1484 ret void 1485} 1486 1487; Legal, as expected 1488define void @rmw_fold_add2(i64* %p, i64 %v) { 1489; CHECK-O0-LABEL: rmw_fold_add2: 1490; CHECK-O0: # %bb.0: 1491; CHECK-O0-NEXT: movq (%rdi), %rax 1492; CHECK-O0-NEXT: addq %rsi, %rax 1493; CHECK-O0-NEXT: movq %rax, (%rdi) 1494; CHECK-O0-NEXT: retq 1495; 1496; CHECK-O3-LABEL: rmw_fold_add2: 1497; CHECK-O3: # %bb.0: 1498; CHECK-O3-NEXT: addq %rsi, (%rdi) 1499; CHECK-O3-NEXT: retq 1500 %prev = load atomic i64, i64* %p unordered, align 8 1501 %val = add i64 %prev, %v 1502 store atomic i64 %val, i64* %p unordered, align 8 1503 ret void 1504} 1505 1506; Legal, as expected 1507define void @rmw_fold_sub1(i64* %p, i64 %v) { 1508; CHECK-O0-LABEL: rmw_fold_sub1: 1509; CHECK-O0: # %bb.0: 1510; CHECK-O0-NEXT: movq (%rdi), %rax 1511; CHECK-O0-NEXT: addq $-15, %rax 1512; CHECK-O0-NEXT: movq %rax, (%rdi) 1513; CHECK-O0-NEXT: retq 1514; 1515; CHECK-O3-LABEL: rmw_fold_sub1: 1516; CHECK-O3: # %bb.0: 1517; CHECK-O3-NEXT: addq $-15, (%rdi) 1518; CHECK-O3-NEXT: retq 1519 %prev = load atomic i64, i64* %p unordered, align 8 1520 %val = sub i64 %prev, 15 1521 store atomic i64 %val, i64* %p unordered, align 8 1522 ret void 1523} 1524 1525; Legal, as expected 1526define void @rmw_fold_sub2(i64* %p, i64 %v) { 1527; CHECK-O0-LABEL: rmw_fold_sub2: 1528; CHECK-O0: # %bb.0: 1529; CHECK-O0-NEXT: movq (%rdi), %rax 1530; CHECK-O0-NEXT: subq %rsi, %rax 1531; CHECK-O0-NEXT: movq %rax, (%rdi) 1532; CHECK-O0-NEXT: retq 1533; 1534; CHECK-O3-LABEL: rmw_fold_sub2: 1535; CHECK-O3: # %bb.0: 1536; CHECK-O3-NEXT: subq %rsi, (%rdi) 1537; CHECK-O3-NEXT: retq 1538 %prev = load atomic i64, i64* %p unordered, align 8 1539 %val = sub i64 %prev, %v 1540 store atomic i64 %val, i64* %p unordered, align 8 1541 ret void 1542} 1543 1544; Legal, as expected 1545define void @rmw_fold_mul1(i64* %p, i64 %v) { 1546; CHECK-LABEL: rmw_fold_mul1: 1547; CHECK: # %bb.0: 1548; CHECK-NEXT: movq (%rdi), %rax 1549; CHECK-NEXT: leaq (%rax,%rax,4), %rax 1550; CHECK-NEXT: leaq (%rax,%rax,2), %rax 1551; CHECK-NEXT: movq %rax, (%rdi) 1552; CHECK-NEXT: retq 1553 %prev = load atomic i64, i64* %p unordered, align 8 1554 %val = mul i64 %prev, 15 1555 store atomic i64 %val, i64* %p unordered, align 8 1556 ret void 1557} 1558 1559; Legal to fold (TODO) 1560define void @rmw_fold_mul2(i64* %p, i64 %v) { 1561; CHECK-O0-LABEL: rmw_fold_mul2: 1562; CHECK-O0: # %bb.0: 1563; CHECK-O0-NEXT: movq (%rdi), %rax 1564; CHECK-O0-NEXT: imulq %rsi, %rax 1565; CHECK-O0-NEXT: movq %rax, (%rdi) 1566; CHECK-O0-NEXT: retq 1567; 1568; CHECK-O3-LABEL: rmw_fold_mul2: 1569; CHECK-O3: # %bb.0: 1570; CHECK-O3-NEXT: imulq (%rdi), %rsi 1571; CHECK-O3-NEXT: movq %rsi, (%rdi) 1572; CHECK-O3-NEXT: retq 1573 %prev = load atomic i64, i64* %p unordered, align 8 1574 %val = mul i64 %prev, %v 1575 store atomic i64 %val, i64* %p unordered, align 8 1576 ret void 1577} 1578 1579; Legal, as expected 1580define void @rmw_fold_sdiv1(i64* %p, i64 %v) { 1581; CHECK-O0-LABEL: rmw_fold_sdiv1: 1582; CHECK-O0: # %bb.0: 1583; CHECK-O0-NEXT: movq (%rdi), %rcx 1584; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 1585; CHECK-O0-NEXT: movq %rcx, %rax 1586; CHECK-O0-NEXT: imulq %rdx 1587; CHECK-O0-NEXT: movq %rdx, %rax 1588; CHECK-O0-NEXT: addq %rcx, %rax 1589; CHECK-O0-NEXT: movq %rax, %rcx 1590; CHECK-O0-NEXT: shrq $63, %rcx 1591; CHECK-O0-NEXT: sarq $3, %rax 1592; CHECK-O0-NEXT: addq %rcx, %rax 1593; CHECK-O0-NEXT: movq %rax, (%rdi) 1594; CHECK-O0-NEXT: retq 1595; 1596; CHECK-O3-LABEL: rmw_fold_sdiv1: 1597; CHECK-O3: # %bb.0: 1598; CHECK-O3-NEXT: movq (%rdi), %rcx 1599; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 1600; CHECK-O3-NEXT: movq %rcx, %rax 1601; CHECK-O3-NEXT: imulq %rdx 1602; CHECK-O3-NEXT: addq %rcx, %rdx 1603; CHECK-O3-NEXT: movq %rdx, %rax 1604; CHECK-O3-NEXT: shrq $63, %rax 1605; CHECK-O3-NEXT: sarq $3, %rdx 1606; CHECK-O3-NEXT: addq %rax, %rdx 1607; CHECK-O3-NEXT: movq %rdx, (%rdi) 1608; CHECK-O3-NEXT: retq 1609 %prev = load atomic i64, i64* %p unordered, align 8 1610 %val = sdiv i64 %prev, 15 1611 store atomic i64 %val, i64* %p unordered, align 8 1612 ret void 1613} 1614 1615; Legal, as expected 1616define void @rmw_fold_sdiv2(i64* %p, i64 %v) { 1617; CHECK-O0-LABEL: rmw_fold_sdiv2: 1618; CHECK-O0: # %bb.0: 1619; CHECK-O0-NEXT: movq (%rdi), %rax 1620; CHECK-O0-NEXT: cqto 1621; CHECK-O0-NEXT: idivq %rsi 1622; CHECK-O0-NEXT: movq %rax, (%rdi) 1623; CHECK-O0-NEXT: retq 1624; 1625; CHECK-O3-LABEL: rmw_fold_sdiv2: 1626; CHECK-O3: # %bb.0: 1627; CHECK-O3-NEXT: movq (%rdi), %rax 1628; CHECK-O3-NEXT: movq %rax, %rcx 1629; CHECK-O3-NEXT: orq %rsi, %rcx 1630; CHECK-O3-NEXT: shrq $32, %rcx 1631; CHECK-O3-NEXT: je .LBB74_1 1632; CHECK-O3-NEXT: # %bb.2: 1633; CHECK-O3-NEXT: cqto 1634; CHECK-O3-NEXT: idivq %rsi 1635; CHECK-O3-NEXT: movq %rax, (%rdi) 1636; CHECK-O3-NEXT: retq 1637; CHECK-O3-NEXT: .LBB74_1: 1638; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1639; CHECK-O3-NEXT: xorl %edx, %edx 1640; CHECK-O3-NEXT: divl %esi 1641; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 1642; CHECK-O3-NEXT: movq %rax, (%rdi) 1643; CHECK-O3-NEXT: retq 1644 %prev = load atomic i64, i64* %p unordered, align 8 1645 %val = sdiv i64 %prev, %v 1646 store atomic i64 %val, i64* %p unordered, align 8 1647 ret void 1648} 1649 1650; Legal, as expected 1651define void @rmw_fold_udiv1(i64* %p, i64 %v) { 1652; CHECK-O0-LABEL: rmw_fold_udiv1: 1653; CHECK-O0: # %bb.0: 1654; CHECK-O0-NEXT: movq (%rdi), %rdx 1655; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rax # imm = 0x8888888888888889 1656; CHECK-O0-NEXT: mulxq %rax, %rax, %rax 1657; CHECK-O0-NEXT: shrq $3, %rax 1658; CHECK-O0-NEXT: movq %rax, (%rdi) 1659; CHECK-O0-NEXT: retq 1660; 1661; CHECK-O3-CUR-LABEL: rmw_fold_udiv1: 1662; CHECK-O3-CUR: # %bb.0: 1663; CHECK-O3-CUR-NEXT: movq (%rdi), %rdx 1664; CHECK-O3-CUR-NEXT: movabsq $-8608480567731124087, %rax # imm = 0x8888888888888889 1665; CHECK-O3-CUR-NEXT: mulxq %rax, %rax, %rax 1666; CHECK-O3-CUR-NEXT: shrq $3, %rax 1667; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 1668; CHECK-O3-CUR-NEXT: retq 1669; 1670; CHECK-O3-EX-LABEL: rmw_fold_udiv1: 1671; CHECK-O3-EX: # %bb.0: 1672; CHECK-O3-EX-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 1673; CHECK-O3-EX-NEXT: mulxq (%rdi), %rax, %rax 1674; CHECK-O3-EX-NEXT: shrq $3, %rax 1675; CHECK-O3-EX-NEXT: movq %rax, (%rdi) 1676; CHECK-O3-EX-NEXT: retq 1677 %prev = load atomic i64, i64* %p unordered, align 8 1678 %val = udiv i64 %prev, 15 1679 store atomic i64 %val, i64* %p unordered, align 8 1680 ret void 1681} 1682 1683; Legal, as expected 1684define void @rmw_fold_udiv2(i64* %p, i64 %v) { 1685; CHECK-O0-LABEL: rmw_fold_udiv2: 1686; CHECK-O0: # %bb.0: 1687; CHECK-O0-NEXT: movq (%rdi), %rax 1688; CHECK-O0-NEXT: xorl %ecx, %ecx 1689; CHECK-O0-NEXT: movl %ecx, %edx 1690; CHECK-O0-NEXT: divq %rsi 1691; CHECK-O0-NEXT: movq %rax, (%rdi) 1692; CHECK-O0-NEXT: retq 1693; 1694; CHECK-O3-LABEL: rmw_fold_udiv2: 1695; CHECK-O3: # %bb.0: 1696; CHECK-O3-NEXT: movq (%rdi), %rax 1697; CHECK-O3-NEXT: movq %rax, %rcx 1698; CHECK-O3-NEXT: orq %rsi, %rcx 1699; CHECK-O3-NEXT: shrq $32, %rcx 1700; CHECK-O3-NEXT: je .LBB76_1 1701; CHECK-O3-NEXT: # %bb.2: 1702; CHECK-O3-NEXT: xorl %edx, %edx 1703; CHECK-O3-NEXT: divq %rsi 1704; CHECK-O3-NEXT: movq %rax, (%rdi) 1705; CHECK-O3-NEXT: retq 1706; CHECK-O3-NEXT: .LBB76_1: 1707; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1708; CHECK-O3-NEXT: xorl %edx, %edx 1709; CHECK-O3-NEXT: divl %esi 1710; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax 1711; CHECK-O3-NEXT: movq %rax, (%rdi) 1712; CHECK-O3-NEXT: retq 1713 %prev = load atomic i64, i64* %p unordered, align 8 1714 %val = udiv i64 %prev, %v 1715 store atomic i64 %val, i64* %p unordered, align 8 1716 ret void 1717} 1718 1719; Legal, as expected 1720define void @rmw_fold_srem1(i64* %p, i64 %v) { 1721; CHECK-O0-LABEL: rmw_fold_srem1: 1722; CHECK-O0: # %bb.0: 1723; CHECK-O0-NEXT: movq (%rdi), %rax 1724; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1725; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889 1726; CHECK-O0-NEXT: imulq %rcx 1727; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 1728; CHECK-O0-NEXT: movq %rdx, %rcx 1729; CHECK-O0-NEXT: addq %rax, %rcx 1730; CHECK-O0-NEXT: movq %rcx, %rdx 1731; CHECK-O0-NEXT: shrq $63, %rdx 1732; CHECK-O0-NEXT: sarq $3, %rcx 1733; CHECK-O0-NEXT: addq %rdx, %rcx 1734; CHECK-O0-NEXT: leaq (%rcx,%rcx,4), %rcx 1735; CHECK-O0-NEXT: leaq (%rcx,%rcx,2), %rcx 1736; CHECK-O0-NEXT: subq %rcx, %rax 1737; CHECK-O0-NEXT: movq %rax, (%rdi) 1738; CHECK-O0-NEXT: retq 1739; 1740; CHECK-O3-LABEL: rmw_fold_srem1: 1741; CHECK-O3: # %bb.0: 1742; CHECK-O3-NEXT: movq (%rdi), %rcx 1743; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889 1744; CHECK-O3-NEXT: movq %rcx, %rax 1745; CHECK-O3-NEXT: imulq %rdx 1746; CHECK-O3-NEXT: addq %rcx, %rdx 1747; CHECK-O3-NEXT: movq %rdx, %rax 1748; CHECK-O3-NEXT: shrq $63, %rax 1749; CHECK-O3-NEXT: sarq $3, %rdx 1750; CHECK-O3-NEXT: addq %rax, %rdx 1751; CHECK-O3-NEXT: leaq (%rdx,%rdx,4), %rax 1752; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax 1753; CHECK-O3-NEXT: subq %rax, %rcx 1754; CHECK-O3-NEXT: movq %rcx, (%rdi) 1755; CHECK-O3-NEXT: retq 1756 %prev = load atomic i64, i64* %p unordered, align 8 1757 %val = srem i64 %prev, 15 1758 store atomic i64 %val, i64* %p unordered, align 8 1759 ret void 1760} 1761 1762; Legal, as expected 1763define void @rmw_fold_srem2(i64* %p, i64 %v) { 1764; CHECK-O0-LABEL: rmw_fold_srem2: 1765; CHECK-O0: # %bb.0: 1766; CHECK-O0-NEXT: movq (%rdi), %rax 1767; CHECK-O0-NEXT: cqto 1768; CHECK-O0-NEXT: idivq %rsi 1769; CHECK-O0-NEXT: movq %rdx, (%rdi) 1770; CHECK-O0-NEXT: retq 1771; 1772; CHECK-O3-LABEL: rmw_fold_srem2: 1773; CHECK-O3: # %bb.0: 1774; CHECK-O3-NEXT: movq (%rdi), %rax 1775; CHECK-O3-NEXT: movq %rax, %rcx 1776; CHECK-O3-NEXT: orq %rsi, %rcx 1777; CHECK-O3-NEXT: shrq $32, %rcx 1778; CHECK-O3-NEXT: je .LBB78_1 1779; CHECK-O3-NEXT: # %bb.2: 1780; CHECK-O3-NEXT: cqto 1781; CHECK-O3-NEXT: idivq %rsi 1782; CHECK-O3-NEXT: movq %rdx, (%rdi) 1783; CHECK-O3-NEXT: retq 1784; CHECK-O3-NEXT: .LBB78_1: 1785; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1786; CHECK-O3-NEXT: xorl %edx, %edx 1787; CHECK-O3-NEXT: divl %esi 1788; CHECK-O3-NEXT: # kill: def $edx killed $edx def $rdx 1789; CHECK-O3-NEXT: movq %rdx, (%rdi) 1790; CHECK-O3-NEXT: retq 1791 %prev = load atomic i64, i64* %p unordered, align 8 1792 %val = srem i64 %prev, %v 1793 store atomic i64 %val, i64* %p unordered, align 8 1794 ret void 1795} 1796 1797; Legal, as expected 1798define void @rmw_fold_urem1(i64* %p, i64 %v) { 1799; CHECK-O0-LABEL: rmw_fold_urem1: 1800; CHECK-O0: # %bb.0: 1801; CHECK-O0-NEXT: movq (%rdi), %rax 1802; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889 1803; CHECK-O0-NEXT: movq %rax, %rdx 1804; CHECK-O0-NEXT: mulxq %rcx, %rcx, %rcx 1805; CHECK-O0-NEXT: shrq $3, %rcx 1806; CHECK-O0-NEXT: leaq (%rcx,%rcx,4), %rcx 1807; CHECK-O0-NEXT: leaq (%rcx,%rcx,2), %rcx 1808; CHECK-O0-NEXT: subq %rcx, %rax 1809; CHECK-O0-NEXT: movq %rax, (%rdi) 1810; CHECK-O0-NEXT: retq 1811; 1812; CHECK-O3-LABEL: rmw_fold_urem1: 1813; CHECK-O3: # %bb.0: 1814; CHECK-O3-NEXT: movq (%rdi), %rdx 1815; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rax # imm = 0x8888888888888889 1816; CHECK-O3-NEXT: mulxq %rax, %rax, %rax 1817; CHECK-O3-NEXT: shrq $3, %rax 1818; CHECK-O3-NEXT: leaq (%rax,%rax,4), %rax 1819; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax 1820; CHECK-O3-NEXT: subq %rax, %rdx 1821; CHECK-O3-NEXT: movq %rdx, (%rdi) 1822; CHECK-O3-NEXT: retq 1823 %prev = load atomic i64, i64* %p unordered, align 8 1824 %val = urem i64 %prev, 15 1825 store atomic i64 %val, i64* %p unordered, align 8 1826 ret void 1827} 1828 1829; Legal, as expected 1830define void @rmw_fold_urem2(i64* %p, i64 %v) { 1831; CHECK-O0-LABEL: rmw_fold_urem2: 1832; CHECK-O0: # %bb.0: 1833; CHECK-O0-NEXT: movq (%rdi), %rax 1834; CHECK-O0-NEXT: xorl %ecx, %ecx 1835; CHECK-O0-NEXT: movl %ecx, %edx 1836; CHECK-O0-NEXT: divq %rsi 1837; CHECK-O0-NEXT: movq %rdx, (%rdi) 1838; CHECK-O0-NEXT: retq 1839; 1840; CHECK-O3-LABEL: rmw_fold_urem2: 1841; CHECK-O3: # %bb.0: 1842; CHECK-O3-NEXT: movq (%rdi), %rax 1843; CHECK-O3-NEXT: movq %rax, %rcx 1844; CHECK-O3-NEXT: orq %rsi, %rcx 1845; CHECK-O3-NEXT: shrq $32, %rcx 1846; CHECK-O3-NEXT: je .LBB80_1 1847; CHECK-O3-NEXT: # %bb.2: 1848; CHECK-O3-NEXT: xorl %edx, %edx 1849; CHECK-O3-NEXT: divq %rsi 1850; CHECK-O3-NEXT: movq %rdx, (%rdi) 1851; CHECK-O3-NEXT: retq 1852; CHECK-O3-NEXT: .LBB80_1: 1853; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 1854; CHECK-O3-NEXT: xorl %edx, %edx 1855; CHECK-O3-NEXT: divl %esi 1856; CHECK-O3-NEXT: # kill: def $edx killed $edx def $rdx 1857; CHECK-O3-NEXT: movq %rdx, (%rdi) 1858; CHECK-O3-NEXT: retq 1859 %prev = load atomic i64, i64* %p unordered, align 8 1860 %val = urem i64 %prev, %v 1861 store atomic i64 %val, i64* %p unordered, align 8 1862 ret void 1863} 1864 1865; Legal to fold (TODO) 1866define void @rmw_fold_shl1(i64* %p, i64 %v) { 1867; CHECK-O0-LABEL: rmw_fold_shl1: 1868; CHECK-O0: # %bb.0: 1869; CHECK-O0-NEXT: movq (%rdi), %rax 1870; CHECK-O0-NEXT: shlq $15, %rax 1871; CHECK-O0-NEXT: movq %rax, (%rdi) 1872; CHECK-O0-NEXT: retq 1873; 1874; CHECK-O3-CUR-LABEL: rmw_fold_shl1: 1875; CHECK-O3-CUR: # %bb.0: 1876; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 1877; CHECK-O3-CUR-NEXT: shlq $15, %rax 1878; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 1879; CHECK-O3-CUR-NEXT: retq 1880; 1881; CHECK-O3-EX-LABEL: rmw_fold_shl1: 1882; CHECK-O3-EX: # %bb.0: 1883; CHECK-O3-EX-NEXT: shlq $15, (%rdi) 1884; CHECK-O3-EX-NEXT: retq 1885 %prev = load atomic i64, i64* %p unordered, align 8 1886 %val = shl i64 %prev, 15 1887 store atomic i64 %val, i64* %p unordered, align 8 1888 ret void 1889} 1890 1891; Legal to fold (TODO) 1892define void @rmw_fold_shl2(i64* %p, i64 %v) { 1893; CHECK-O0-LABEL: rmw_fold_shl2: 1894; CHECK-O0: # %bb.0: 1895; CHECK-O0-NEXT: movq (%rdi), %rax 1896; CHECK-O0-NEXT: movb %sil, %dl 1897; CHECK-O0-NEXT: # implicit-def: $rcx 1898; CHECK-O0-NEXT: movb %dl, %cl 1899; CHECK-O0-NEXT: shlxq %rcx, %rax, %rax 1900; CHECK-O0-NEXT: movq %rax, (%rdi) 1901; CHECK-O0-NEXT: retq 1902; 1903; CHECK-O3-CUR-LABEL: rmw_fold_shl2: 1904; CHECK-O3-CUR: # %bb.0: 1905; CHECK-O3-CUR-NEXT: shlxq %rsi, (%rdi), %rax 1906; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 1907; CHECK-O3-CUR-NEXT: retq 1908; 1909; CHECK-O3-EX-LABEL: rmw_fold_shl2: 1910; CHECK-O3-EX: # %bb.0: 1911; CHECK-O3-EX-NEXT: movq %rsi, %rcx 1912; CHECK-O3-EX-NEXT: # kill: def $cl killed $cl killed $rcx 1913; CHECK-O3-EX-NEXT: shlq %cl, (%rdi) 1914; CHECK-O3-EX-NEXT: retq 1915 %prev = load atomic i64, i64* %p unordered, align 8 1916 %val = shl i64 %prev, %v 1917 store atomic i64 %val, i64* %p unordered, align 8 1918 ret void 1919} 1920 1921; Legal to fold (TODO) 1922define void @rmw_fold_lshr1(i64* %p, i64 %v) { 1923; CHECK-O0-LABEL: rmw_fold_lshr1: 1924; CHECK-O0: # %bb.0: 1925; CHECK-O0-NEXT: movq (%rdi), %rax 1926; CHECK-O0-NEXT: shrq $15, %rax 1927; CHECK-O0-NEXT: movq %rax, (%rdi) 1928; CHECK-O0-NEXT: retq 1929; 1930; CHECK-O3-CUR-LABEL: rmw_fold_lshr1: 1931; CHECK-O3-CUR: # %bb.0: 1932; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 1933; CHECK-O3-CUR-NEXT: shrq $15, %rax 1934; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 1935; CHECK-O3-CUR-NEXT: retq 1936; 1937; CHECK-O3-EX-LABEL: rmw_fold_lshr1: 1938; CHECK-O3-EX: # %bb.0: 1939; CHECK-O3-EX-NEXT: shrq $15, (%rdi) 1940; CHECK-O3-EX-NEXT: retq 1941 %prev = load atomic i64, i64* %p unordered, align 8 1942 %val = lshr i64 %prev, 15 1943 store atomic i64 %val, i64* %p unordered, align 8 1944 ret void 1945} 1946 1947; Legal to fold (TODO) 1948define void @rmw_fold_lshr2(i64* %p, i64 %v) { 1949; CHECK-O0-LABEL: rmw_fold_lshr2: 1950; CHECK-O0: # %bb.0: 1951; CHECK-O0-NEXT: movq (%rdi), %rax 1952; CHECK-O0-NEXT: movb %sil, %dl 1953; CHECK-O0-NEXT: # implicit-def: $rcx 1954; CHECK-O0-NEXT: movb %dl, %cl 1955; CHECK-O0-NEXT: shrxq %rcx, %rax, %rax 1956; CHECK-O0-NEXT: movq %rax, (%rdi) 1957; CHECK-O0-NEXT: retq 1958; 1959; CHECK-O3-CUR-LABEL: rmw_fold_lshr2: 1960; CHECK-O3-CUR: # %bb.0: 1961; CHECK-O3-CUR-NEXT: shrxq %rsi, (%rdi), %rax 1962; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 1963; CHECK-O3-CUR-NEXT: retq 1964; 1965; CHECK-O3-EX-LABEL: rmw_fold_lshr2: 1966; CHECK-O3-EX: # %bb.0: 1967; CHECK-O3-EX-NEXT: movq %rsi, %rcx 1968; CHECK-O3-EX-NEXT: # kill: def $cl killed $cl killed $rcx 1969; CHECK-O3-EX-NEXT: shrq %cl, (%rdi) 1970; CHECK-O3-EX-NEXT: retq 1971 %prev = load atomic i64, i64* %p unordered, align 8 1972 %val = lshr i64 %prev, %v 1973 store atomic i64 %val, i64* %p unordered, align 8 1974 ret void 1975} 1976 1977; Legal to fold (TODO) 1978define void @rmw_fold_ashr1(i64* %p, i64 %v) { 1979; CHECK-O0-LABEL: rmw_fold_ashr1: 1980; CHECK-O0: # %bb.0: 1981; CHECK-O0-NEXT: movq (%rdi), %rax 1982; CHECK-O0-NEXT: sarq $15, %rax 1983; CHECK-O0-NEXT: movq %rax, (%rdi) 1984; CHECK-O0-NEXT: retq 1985; 1986; CHECK-O3-CUR-LABEL: rmw_fold_ashr1: 1987; CHECK-O3-CUR: # %bb.0: 1988; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 1989; CHECK-O3-CUR-NEXT: sarq $15, %rax 1990; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 1991; CHECK-O3-CUR-NEXT: retq 1992; 1993; CHECK-O3-EX-LABEL: rmw_fold_ashr1: 1994; CHECK-O3-EX: # %bb.0: 1995; CHECK-O3-EX-NEXT: sarq $15, (%rdi) 1996; CHECK-O3-EX-NEXT: retq 1997 %prev = load atomic i64, i64* %p unordered, align 8 1998 %val = ashr i64 %prev, 15 1999 store atomic i64 %val, i64* %p unordered, align 8 2000 ret void 2001} 2002 2003; Legal to fold (TODO) 2004define void @rmw_fold_ashr2(i64* %p, i64 %v) { 2005; CHECK-O0-LABEL: rmw_fold_ashr2: 2006; CHECK-O0: # %bb.0: 2007; CHECK-O0-NEXT: movq (%rdi), %rax 2008; CHECK-O0-NEXT: movb %sil, %dl 2009; CHECK-O0-NEXT: # implicit-def: $rcx 2010; CHECK-O0-NEXT: movb %dl, %cl 2011; CHECK-O0-NEXT: sarxq %rcx, %rax, %rax 2012; CHECK-O0-NEXT: movq %rax, (%rdi) 2013; CHECK-O0-NEXT: retq 2014; 2015; CHECK-O3-CUR-LABEL: rmw_fold_ashr2: 2016; CHECK-O3-CUR: # %bb.0: 2017; CHECK-O3-CUR-NEXT: sarxq %rsi, (%rdi), %rax 2018; CHECK-O3-CUR-NEXT: movq %rax, (%rdi) 2019; CHECK-O3-CUR-NEXT: retq 2020; 2021; CHECK-O3-EX-LABEL: rmw_fold_ashr2: 2022; CHECK-O3-EX: # %bb.0: 2023; CHECK-O3-EX-NEXT: movq %rsi, %rcx 2024; CHECK-O3-EX-NEXT: # kill: def $cl killed $cl killed $rcx 2025; CHECK-O3-EX-NEXT: sarq %cl, (%rdi) 2026; CHECK-O3-EX-NEXT: retq 2027 %prev = load atomic i64, i64* %p unordered, align 8 2028 %val = ashr i64 %prev, %v 2029 store atomic i64 %val, i64* %p unordered, align 8 2030 ret void 2031} 2032 2033; Legal, as expected 2034define void @rmw_fold_and1(i64* %p, i64 %v) { 2035; CHECK-O0-LABEL: rmw_fold_and1: 2036; CHECK-O0: # %bb.0: 2037; CHECK-O0-NEXT: movq (%rdi), %rax 2038; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax 2039; CHECK-O0-NEXT: andl $15, %eax 2040; CHECK-O0-NEXT: # kill: def $rax killed $eax 2041; CHECK-O0-NEXT: movq %rax, (%rdi) 2042; CHECK-O0-NEXT: retq 2043; 2044; CHECK-O3-LABEL: rmw_fold_and1: 2045; CHECK-O3: # %bb.0: 2046; CHECK-O3-NEXT: andq $15, (%rdi) 2047; CHECK-O3-NEXT: retq 2048 %prev = load atomic i64, i64* %p unordered, align 8 2049 %val = and i64 %prev, 15 2050 store atomic i64 %val, i64* %p unordered, align 8 2051 ret void 2052} 2053 2054; Legal, as expected 2055define void @rmw_fold_and2(i64* %p, i64 %v) { 2056; CHECK-O0-LABEL: rmw_fold_and2: 2057; CHECK-O0: # %bb.0: 2058; CHECK-O0-NEXT: movq (%rdi), %rax 2059; CHECK-O0-NEXT: andq %rsi, %rax 2060; CHECK-O0-NEXT: movq %rax, (%rdi) 2061; CHECK-O0-NEXT: retq 2062; 2063; CHECK-O3-LABEL: rmw_fold_and2: 2064; CHECK-O3: # %bb.0: 2065; CHECK-O3-NEXT: andq %rsi, (%rdi) 2066; CHECK-O3-NEXT: retq 2067 %prev = load atomic i64, i64* %p unordered, align 8 2068 %val = and i64 %prev, %v 2069 store atomic i64 %val, i64* %p unordered, align 8 2070 ret void 2071} 2072 2073; Legal, as expected 2074define void @rmw_fold_or1(i64* %p, i64 %v) { 2075; CHECK-O0-LABEL: rmw_fold_or1: 2076; CHECK-O0: # %bb.0: 2077; CHECK-O0-NEXT: movq (%rdi), %rax 2078; CHECK-O0-NEXT: orq $15, %rax 2079; CHECK-O0-NEXT: movq %rax, (%rdi) 2080; CHECK-O0-NEXT: retq 2081; 2082; CHECK-O3-LABEL: rmw_fold_or1: 2083; CHECK-O3: # %bb.0: 2084; CHECK-O3-NEXT: orq $15, (%rdi) 2085; CHECK-O3-NEXT: retq 2086 %prev = load atomic i64, i64* %p unordered, align 8 2087 %val = or i64 %prev, 15 2088 store atomic i64 %val, i64* %p unordered, align 8 2089 ret void 2090} 2091 2092; Legal, as expected 2093define void @rmw_fold_or2(i64* %p, i64 %v) { 2094; CHECK-O0-LABEL: rmw_fold_or2: 2095; CHECK-O0: # %bb.0: 2096; CHECK-O0-NEXT: movq (%rdi), %rax 2097; CHECK-O0-NEXT: orq %rsi, %rax 2098; CHECK-O0-NEXT: movq %rax, (%rdi) 2099; CHECK-O0-NEXT: retq 2100; 2101; CHECK-O3-LABEL: rmw_fold_or2: 2102; CHECK-O3: # %bb.0: 2103; CHECK-O3-NEXT: orq %rsi, (%rdi) 2104; CHECK-O3-NEXT: retq 2105 %prev = load atomic i64, i64* %p unordered, align 8 2106 %val = or i64 %prev, %v 2107 store atomic i64 %val, i64* %p unordered, align 8 2108 ret void 2109} 2110 2111; Legal, as expected 2112define void @rmw_fold_xor1(i64* %p, i64 %v) { 2113; CHECK-O0-LABEL: rmw_fold_xor1: 2114; CHECK-O0: # %bb.0: 2115; CHECK-O0-NEXT: movq (%rdi), %rax 2116; CHECK-O0-NEXT: xorq $15, %rax 2117; CHECK-O0-NEXT: movq %rax, (%rdi) 2118; CHECK-O0-NEXT: retq 2119; 2120; CHECK-O3-LABEL: rmw_fold_xor1: 2121; CHECK-O3: # %bb.0: 2122; CHECK-O3-NEXT: xorq $15, (%rdi) 2123; CHECK-O3-NEXT: retq 2124 %prev = load atomic i64, i64* %p unordered, align 8 2125 %val = xor i64 %prev, 15 2126 store atomic i64 %val, i64* %p unordered, align 8 2127 ret void 2128} 2129 2130; Legal, as expected 2131define void @rmw_fold_xor2(i64* %p, i64 %v) { 2132; CHECK-O0-LABEL: rmw_fold_xor2: 2133; CHECK-O0: # %bb.0: 2134; CHECK-O0-NEXT: movq (%rdi), %rax 2135; CHECK-O0-NEXT: xorq %rsi, %rax 2136; CHECK-O0-NEXT: movq %rax, (%rdi) 2137; CHECK-O0-NEXT: retq 2138; 2139; CHECK-O3-LABEL: rmw_fold_xor2: 2140; CHECK-O3: # %bb.0: 2141; CHECK-O3-NEXT: xorq %rsi, (%rdi) 2142; CHECK-O3-NEXT: retq 2143 %prev = load atomic i64, i64* %p unordered, align 8 2144 %val = xor i64 %prev, %v 2145 store atomic i64 %val, i64* %p unordered, align 8 2146 ret void 2147} 2148 2149;; The next batch test truncations, in combination w/operations which could 2150;; be folded against the memory operation. 2151 2152; Legal to reduce the load width (TODO) 2153define i32 @fold_trunc(i64* %p) { 2154; CHECK-LABEL: fold_trunc: 2155; CHECK: # %bb.0: 2156; CHECK-NEXT: movq (%rdi), %rax 2157; CHECK-NEXT: # kill: def $eax killed $eax killed $rax 2158; CHECK-NEXT: retq 2159 %v = load atomic i64, i64* %p unordered, align 8 2160 %ret = trunc i64 %v to i32 2161 ret i32 %ret 2162} 2163 2164; Legal to reduce the load width and fold the load (TODO) 2165define i32 @fold_trunc_add(i64* %p, i32 %v2) { 2166; CHECK-O0-LABEL: fold_trunc_add: 2167; CHECK-O0: # %bb.0: 2168; CHECK-O0-NEXT: movq (%rdi), %rax 2169; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax 2170; CHECK-O0-NEXT: addl %esi, %eax 2171; CHECK-O0-NEXT: retq 2172; 2173; CHECK-O3-LABEL: fold_trunc_add: 2174; CHECK-O3: # %bb.0: 2175; CHECK-O3-NEXT: movq (%rdi), %rax 2176; CHECK-O3-NEXT: addl %esi, %eax 2177; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 2178; CHECK-O3-NEXT: retq 2179 %v = load atomic i64, i64* %p unordered, align 8 2180 %trunc = trunc i64 %v to i32 2181 %ret = add i32 %trunc, %v2 2182 ret i32 %ret 2183} 2184 2185; Legal to reduce the load width and fold the load (TODO) 2186define i32 @fold_trunc_and(i64* %p, i32 %v2) { 2187; CHECK-O0-LABEL: fold_trunc_and: 2188; CHECK-O0: # %bb.0: 2189; CHECK-O0-NEXT: movq (%rdi), %rax 2190; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax 2191; CHECK-O0-NEXT: andl %esi, %eax 2192; CHECK-O0-NEXT: retq 2193; 2194; CHECK-O3-LABEL: fold_trunc_and: 2195; CHECK-O3: # %bb.0: 2196; CHECK-O3-NEXT: movq (%rdi), %rax 2197; CHECK-O3-NEXT: andl %esi, %eax 2198; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 2199; CHECK-O3-NEXT: retq 2200 %v = load atomic i64, i64* %p unordered, align 8 2201 %trunc = trunc i64 %v to i32 2202 %ret = and i32 %trunc, %v2 2203 ret i32 %ret 2204} 2205 2206; Legal to reduce the load width and fold the load (TODO) 2207define i32 @fold_trunc_or(i64* %p, i32 %v2) { 2208; CHECK-O0-LABEL: fold_trunc_or: 2209; CHECK-O0: # %bb.0: 2210; CHECK-O0-NEXT: movq (%rdi), %rax 2211; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax 2212; CHECK-O0-NEXT: orl %esi, %eax 2213; CHECK-O0-NEXT: retq 2214; 2215; CHECK-O3-LABEL: fold_trunc_or: 2216; CHECK-O3: # %bb.0: 2217; CHECK-O3-NEXT: movq (%rdi), %rax 2218; CHECK-O3-NEXT: orl %esi, %eax 2219; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax 2220; CHECK-O3-NEXT: retq 2221 %v = load atomic i64, i64* %p unordered, align 8 2222 %trunc = trunc i64 %v to i32 2223 %ret = or i32 %trunc, %v2 2224 ret i32 %ret 2225} 2226 2227; It's tempting to split the wide load into two smaller byte loads 2228; to reduce memory traffic, but this would be illegal for a atomic load 2229define i32 @split_load(i64* %p) { 2230; CHECK-O0-LABEL: split_load: 2231; CHECK-O0: # %bb.0: 2232; CHECK-O0-NEXT: movq (%rdi), %rcx 2233; CHECK-O0-NEXT: movb %cl, %al 2234; CHECK-O0-NEXT: shrq $32, %rcx 2235; CHECK-O0-NEXT: # kill: def $cl killed $cl killed $rcx 2236; CHECK-O0-NEXT: orb %cl, %al 2237; CHECK-O0-NEXT: movzbl %al, %eax 2238; CHECK-O0-NEXT: retq 2239; 2240; CHECK-O3-LABEL: split_load: 2241; CHECK-O3: # %bb.0: 2242; CHECK-O3-NEXT: movq (%rdi), %rax 2243; CHECK-O3-NEXT: movq %rax, %rcx 2244; CHECK-O3-NEXT: shrq $32, %rcx 2245; CHECK-O3-NEXT: orl %eax, %ecx 2246; CHECK-O3-NEXT: movzbl %cl, %eax 2247; CHECK-O3-NEXT: retq 2248 %v = load atomic i64, i64* %p unordered, align 8 2249 %b1 = trunc i64 %v to i8 2250 %v.shift = lshr i64 %v, 32 2251 %b2 = trunc i64 %v.shift to i8 2252 %or = or i8 %b1, %b2 2253 %ret = zext i8 %or to i32 2254 ret i32 %ret 2255} 2256 2257;; A collection of simple memory forwarding tests. Nothing particular 2258;; interesting semantic wise, just demonstrating obvious missed transforms. 2259 2260@Zero = constant i64 0 2261 2262; TODO: should return constant 2263define i64 @constant_folding(i64* %p) { 2264; CHECK-LABEL: constant_folding: 2265; CHECK: # %bb.0: 2266; CHECK-NEXT: movq (%rdi), %rax 2267; CHECK-NEXT: retq 2268 %v = load atomic i64, i64* %p unordered, align 8 2269 ret i64 %v 2270} 2271 2272; Legal to forward and fold (TODO) 2273define i64 @load_forwarding(i64* %p) { 2274; CHECK-LABEL: load_forwarding: 2275; CHECK: # %bb.0: 2276; CHECK-NEXT: movq (%rdi), %rax 2277; CHECK-NEXT: orq (%rdi), %rax 2278; CHECK-NEXT: retq 2279 %v = load atomic i64, i64* %p unordered, align 8 2280 %v2 = load atomic i64, i64* %p unordered, align 8 2281 %ret = or i64 %v, %v2 2282 ret i64 %ret 2283} 2284 2285; Legal to forward (TODO) 2286define i64 @store_forward(i64* %p, i64 %v) { 2287; CHECK-LABEL: store_forward: 2288; CHECK: # %bb.0: 2289; CHECK-NEXT: movq %rsi, (%rdi) 2290; CHECK-NEXT: movq (%rdi), %rax 2291; CHECK-NEXT: retq 2292 store atomic i64 %v, i64* %p unordered, align 8 2293 %ret = load atomic i64, i64* %p unordered, align 8 2294 ret i64 %ret 2295} 2296 2297; Legal to kill (TODO) 2298define void @dead_writeback(i64* %p) { 2299; CHECK-LABEL: dead_writeback: 2300; CHECK: # %bb.0: 2301; CHECK-NEXT: movq (%rdi), %rax 2302; CHECK-NEXT: movq %rax, (%rdi) 2303; CHECK-NEXT: retq 2304 %v = load atomic i64, i64* %p unordered, align 8 2305 store atomic i64 %v, i64* %p unordered, align 8 2306 ret void 2307} 2308 2309; Legal to kill (TODO) 2310define void @dead_store(i64* %p, i64 %v) { 2311; CHECK-LABEL: dead_store: 2312; CHECK: # %bb.0: 2313; CHECK-NEXT: movq $0, (%rdi) 2314; CHECK-NEXT: movq %rsi, (%rdi) 2315; CHECK-NEXT: retq 2316 store atomic i64 0, i64* %p unordered, align 8 2317 store atomic i64 %v, i64* %p unordered, align 8 2318 ret void 2319} 2320 2321;; The next batch of tests ensure that we don't try to fold a load into a 2322;; use where the code motion implied for the load is prevented by a fence. 2323;; Note: We're checking that the load doesn't get moved below the fence as 2324;; part of folding, but is technically legal to lift the add above the fence. 2325;; If that were to happen, please rewrite the test to ensure load movement 2326;; isn't violated. 2327 2328define i64 @nofold_fence(i64* %p) { 2329; CHECK-LABEL: nofold_fence: 2330; CHECK: # %bb.0: 2331; CHECK-NEXT: movq (%rdi), %rax 2332; CHECK-NEXT: mfence 2333; CHECK-NEXT: addq $15, %rax 2334; CHECK-NEXT: retq 2335 %v = load atomic i64, i64* %p unordered, align 8 2336 fence seq_cst 2337 %ret = add i64 %v, 15 2338 ret i64 %ret 2339} 2340 2341define i64 @nofold_fence_acquire(i64* %p) { 2342; CHECK-LABEL: nofold_fence_acquire: 2343; CHECK: # %bb.0: 2344; CHECK-NEXT: movq (%rdi), %rax 2345; CHECK-NEXT: #MEMBARRIER 2346; CHECK-NEXT: addq $15, %rax 2347; CHECK-NEXT: retq 2348 %v = load atomic i64, i64* %p unordered, align 8 2349 fence acquire 2350 %ret = add i64 %v, 15 2351 ret i64 %ret 2352} 2353 2354 2355define i64 @nofold_stfence(i64* %p) { 2356; CHECK-LABEL: nofold_stfence: 2357; CHECK: # %bb.0: 2358; CHECK-NEXT: movq (%rdi), %rax 2359; CHECK-NEXT: #MEMBARRIER 2360; CHECK-NEXT: addq $15, %rax 2361; CHECK-NEXT: retq 2362 %v = load atomic i64, i64* %p unordered, align 8 2363 fence syncscope("singlethread") seq_cst 2364 %ret = add i64 %v, 15 2365 ret i64 %ret 2366} 2367 2368;; Next, test how well we can fold invariant loads. 2369 2370@Constant = external dso_local constant i64 2371 2372define i64 @fold_constant(i64 %arg) { 2373; CHECK-O0-LABEL: fold_constant: 2374; CHECK-O0: # %bb.0: 2375; CHECK-O0-NEXT: movq %rdi, %rax 2376; CHECK-O0-NEXT: addq Constant, %rax 2377; CHECK-O0-NEXT: retq 2378; 2379; CHECK-O3-LABEL: fold_constant: 2380; CHECK-O3: # %bb.0: 2381; CHECK-O3-NEXT: movq %rdi, %rax 2382; CHECK-O3-NEXT: addq Constant(%rip), %rax 2383; CHECK-O3-NEXT: retq 2384 %v = load atomic i64, i64* @Constant unordered, align 8 2385 %ret = add i64 %v, %arg 2386 ret i64 %ret 2387} 2388 2389define i64 @fold_constant_clobber(i64* %p, i64 %arg) { 2390; CHECK-O0-LABEL: fold_constant_clobber: 2391; CHECK-O0: # %bb.0: 2392; CHECK-O0-NEXT: movq Constant(%rip), %rax 2393; CHECK-O0-NEXT: movq $5, (%rdi) 2394; CHECK-O0-NEXT: addq %rsi, %rax 2395; CHECK-O0-NEXT: retq 2396; 2397; CHECK-O3-CUR-LABEL: fold_constant_clobber: 2398; CHECK-O3-CUR: # %bb.0: 2399; CHECK-O3-CUR-NEXT: movq Constant(%rip), %rax 2400; CHECK-O3-CUR-NEXT: movq $5, (%rdi) 2401; CHECK-O3-CUR-NEXT: addq %rsi, %rax 2402; CHECK-O3-CUR-NEXT: retq 2403; 2404; CHECK-O3-EX-LABEL: fold_constant_clobber: 2405; CHECK-O3-EX: # %bb.0: 2406; CHECK-O3-EX-NEXT: movq %rsi, %rax 2407; CHECK-O3-EX-NEXT: addq Constant(%rip), %rax 2408; CHECK-O3-EX-NEXT: movq $5, (%rdi) 2409; CHECK-O3-EX-NEXT: retq 2410 %v = load atomic i64, i64* @Constant unordered, align 8 2411 store i64 5, i64* %p 2412 %ret = add i64 %v, %arg 2413 ret i64 %ret 2414} 2415 2416define i64 @fold_constant_fence(i64 %arg) { 2417; CHECK-O0-LABEL: fold_constant_fence: 2418; CHECK-O0: # %bb.0: 2419; CHECK-O0-NEXT: movq Constant(%rip), %rax 2420; CHECK-O0-NEXT: mfence 2421; CHECK-O0-NEXT: addq %rdi, %rax 2422; CHECK-O0-NEXT: retq 2423; 2424; CHECK-O3-CUR-LABEL: fold_constant_fence: 2425; CHECK-O3-CUR: # %bb.0: 2426; CHECK-O3-CUR-NEXT: movq Constant(%rip), %rax 2427; CHECK-O3-CUR-NEXT: mfence 2428; CHECK-O3-CUR-NEXT: addq %rdi, %rax 2429; CHECK-O3-CUR-NEXT: retq 2430; 2431; CHECK-O3-EX-LABEL: fold_constant_fence: 2432; CHECK-O3-EX: # %bb.0: 2433; CHECK-O3-EX-NEXT: movq %rdi, %rax 2434; CHECK-O3-EX-NEXT: addq Constant(%rip), %rax 2435; CHECK-O3-EX-NEXT: mfence 2436; CHECK-O3-EX-NEXT: retq 2437 %v = load atomic i64, i64* @Constant unordered, align 8 2438 fence seq_cst 2439 %ret = add i64 %v, %arg 2440 ret i64 %ret 2441} 2442 2443define i64 @fold_invariant_clobber(i64* dereferenceable(8) %p, i64 %arg) { 2444; CHECK-O0-LABEL: fold_invariant_clobber: 2445; CHECK-O0: # %bb.0: 2446; CHECK-O0-NEXT: movq (%rdi), %rax 2447; CHECK-O0-NEXT: movq $5, (%rdi) 2448; CHECK-O0-NEXT: addq %rsi, %rax 2449; CHECK-O0-NEXT: retq 2450; 2451; CHECK-O3-CUR-LABEL: fold_invariant_clobber: 2452; CHECK-O3-CUR: # %bb.0: 2453; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 2454; CHECK-O3-CUR-NEXT: movq $5, (%rdi) 2455; CHECK-O3-CUR-NEXT: addq %rsi, %rax 2456; CHECK-O3-CUR-NEXT: retq 2457; 2458; CHECK-O3-EX-LABEL: fold_invariant_clobber: 2459; CHECK-O3-EX: # %bb.0: 2460; CHECK-O3-EX-NEXT: movq %rsi, %rax 2461; CHECK-O3-EX-NEXT: addq (%rdi), %rax 2462; CHECK-O3-EX-NEXT: movq $5, (%rdi) 2463; CHECK-O3-EX-NEXT: retq 2464 %v = load atomic i64, i64* %p unordered, align 8, !invariant.load !{} 2465 store i64 5, i64* %p 2466 %ret = add i64 %v, %arg 2467 ret i64 %ret 2468} 2469 2470 2471define i64 @fold_invariant_fence(i64* dereferenceable(8) %p, i64 %arg) { 2472; CHECK-O0-LABEL: fold_invariant_fence: 2473; CHECK-O0: # %bb.0: 2474; CHECK-O0-NEXT: movq (%rdi), %rax 2475; CHECK-O0-NEXT: mfence 2476; CHECK-O0-NEXT: addq %rsi, %rax 2477; CHECK-O0-NEXT: retq 2478; 2479; CHECK-O3-CUR-LABEL: fold_invariant_fence: 2480; CHECK-O3-CUR: # %bb.0: 2481; CHECK-O3-CUR-NEXT: movq (%rdi), %rax 2482; CHECK-O3-CUR-NEXT: mfence 2483; CHECK-O3-CUR-NEXT: addq %rsi, %rax 2484; CHECK-O3-CUR-NEXT: retq 2485; 2486; CHECK-O3-EX-LABEL: fold_invariant_fence: 2487; CHECK-O3-EX: # %bb.0: 2488; CHECK-O3-EX-NEXT: movq %rsi, %rax 2489; CHECK-O3-EX-NEXT: addq (%rdi), %rax 2490; CHECK-O3-EX-NEXT: mfence 2491; CHECK-O3-EX-NEXT: retq 2492 %v = load atomic i64, i64* %p unordered, align 8, !invariant.load !{} 2493 fence seq_cst 2494 %ret = add i64 %v, %arg 2495 ret i64 %ret 2496} 2497 2498 2499; Exercise a few cases involving any extend idioms 2500 2501define i16 @load_i8_anyext_i16(i8* %ptr) { 2502; CHECK-O0-CUR-LABEL: load_i8_anyext_i16: 2503; CHECK-O0-CUR: # %bb.0: 2504; CHECK-O0-CUR-NEXT: movb (%rdi), %al 2505; CHECK-O0-CUR-NEXT: movzbl %al, %eax 2506; CHECK-O0-CUR-NEXT: # kill: def $ax killed $ax killed $eax 2507; CHECK-O0-CUR-NEXT: retq 2508; 2509; CHECK-O3-CUR-LABEL: load_i8_anyext_i16: 2510; CHECK-O3-CUR: # %bb.0: 2511; CHECK-O3-CUR-NEXT: movzbl (%rdi), %eax 2512; CHECK-O3-CUR-NEXT: # kill: def $ax killed $ax killed $eax 2513; CHECK-O3-CUR-NEXT: retq 2514; 2515; CHECK-O0-EX-LABEL: load_i8_anyext_i16: 2516; CHECK-O0-EX: # %bb.0: 2517; CHECK-O0-EX-NEXT: vpbroadcastb (%rdi), %xmm0 2518; CHECK-O0-EX-NEXT: vmovd %xmm0, %eax 2519; CHECK-O0-EX-NEXT: # kill: def $ax killed $ax killed $eax 2520; CHECK-O0-EX-NEXT: retq 2521; 2522; CHECK-O3-EX-LABEL: load_i8_anyext_i16: 2523; CHECK-O3-EX: # %bb.0: 2524; CHECK-O3-EX-NEXT: vpbroadcastb (%rdi), %xmm0 2525; CHECK-O3-EX-NEXT: vmovd %xmm0, %eax 2526; CHECK-O3-EX-NEXT: # kill: def $ax killed $ax killed $eax 2527; CHECK-O3-EX-NEXT: retq 2528 %v = load atomic i8, i8* %ptr unordered, align 2 2529 %vec = insertelement <2 x i8> undef, i8 %v, i32 0 2530 %res = bitcast <2 x i8> %vec to i16 2531 ret i16 %res 2532} 2533 2534define i32 @load_i8_anyext_i32(i8* %ptr) { 2535; CHECK-O0-CUR-LABEL: load_i8_anyext_i32: 2536; CHECK-O0-CUR: # %bb.0: 2537; CHECK-O0-CUR-NEXT: movb (%rdi), %al 2538; CHECK-O0-CUR-NEXT: movzbl %al, %eax 2539; CHECK-O0-CUR-NEXT: retq 2540; 2541; CHECK-O3-CUR-LABEL: load_i8_anyext_i32: 2542; CHECK-O3-CUR: # %bb.0: 2543; CHECK-O3-CUR-NEXT: movzbl (%rdi), %eax 2544; CHECK-O3-CUR-NEXT: retq 2545; 2546; CHECK-O0-EX-LABEL: load_i8_anyext_i32: 2547; CHECK-O0-EX: # %bb.0: 2548; CHECK-O0-EX-NEXT: vpbroadcastb (%rdi), %xmm0 2549; CHECK-O0-EX-NEXT: vmovd %xmm0, %eax 2550; CHECK-O0-EX-NEXT: retq 2551; 2552; CHECK-O3-EX-LABEL: load_i8_anyext_i32: 2553; CHECK-O3-EX: # %bb.0: 2554; CHECK-O3-EX-NEXT: vpbroadcastb (%rdi), %xmm0 2555; CHECK-O3-EX-NEXT: vmovd %xmm0, %eax 2556; CHECK-O3-EX-NEXT: retq 2557 %v = load atomic i8, i8* %ptr unordered, align 4 2558 %vec = insertelement <4 x i8> undef, i8 %v, i32 0 2559 %res = bitcast <4 x i8> %vec to i32 2560 ret i32 %res 2561} 2562 2563define i32 @load_i16_anyext_i32(i16* %ptr) { 2564; CHECK-O0-CUR-LABEL: load_i16_anyext_i32: 2565; CHECK-O0-CUR: # %bb.0: 2566; CHECK-O0-CUR-NEXT: movw (%rdi), %cx 2567; CHECK-O0-CUR-NEXT: # implicit-def: $eax 2568; CHECK-O0-CUR-NEXT: movw %cx, %ax 2569; CHECK-O0-CUR-NEXT: retq 2570; 2571; CHECK-O3-CUR-LABEL: load_i16_anyext_i32: 2572; CHECK-O3-CUR: # %bb.0: 2573; CHECK-O3-CUR-NEXT: movzwl (%rdi), %eax 2574; CHECK-O3-CUR-NEXT: retq 2575; 2576; CHECK-O0-EX-LABEL: load_i16_anyext_i32: 2577; CHECK-O0-EX: # %bb.0: 2578; CHECK-O0-EX-NEXT: vpbroadcastw (%rdi), %xmm0 2579; CHECK-O0-EX-NEXT: vmovd %xmm0, %eax 2580; CHECK-O0-EX-NEXT: retq 2581; 2582; CHECK-O3-EX-LABEL: load_i16_anyext_i32: 2583; CHECK-O3-EX: # %bb.0: 2584; CHECK-O3-EX-NEXT: vpbroadcastw (%rdi), %xmm0 2585; CHECK-O3-EX-NEXT: vmovd %xmm0, %eax 2586; CHECK-O3-EX-NEXT: retq 2587 %v = load atomic i16, i16* %ptr unordered, align 4 2588 %vec = insertelement <2 x i16> undef, i16 %v, i64 0 2589 %res = bitcast <2 x i16> %vec to i32 2590 ret i32 %res 2591} 2592 2593define i64 @load_i16_anyext_i64(i16* %ptr) { 2594; CHECK-O0-CUR-LABEL: load_i16_anyext_i64: 2595; CHECK-O0-CUR: # %bb.0: 2596; CHECK-O0-CUR-NEXT: movw (%rdi), %cx 2597; CHECK-O0-CUR-NEXT: # implicit-def: $eax 2598; CHECK-O0-CUR-NEXT: movw %cx, %ax 2599; CHECK-O0-CUR-NEXT: vmovd %eax, %xmm0 2600; CHECK-O0-CUR-NEXT: vmovq %xmm0, %rax 2601; CHECK-O0-CUR-NEXT: retq 2602; 2603; CHECK-O3-CUR-LABEL: load_i16_anyext_i64: 2604; CHECK-O3-CUR: # %bb.0: 2605; CHECK-O3-CUR-NEXT: movzwl (%rdi), %eax 2606; CHECK-O3-CUR-NEXT: vmovd %eax, %xmm0 2607; CHECK-O3-CUR-NEXT: vmovq %xmm0, %rax 2608; CHECK-O3-CUR-NEXT: retq 2609; 2610; CHECK-O0-EX-LABEL: load_i16_anyext_i64: 2611; CHECK-O0-EX: # %bb.0: 2612; CHECK-O0-EX-NEXT: vpbroadcastw (%rdi), %xmm0 2613; CHECK-O0-EX-NEXT: vmovq %xmm0, %rax 2614; CHECK-O0-EX-NEXT: retq 2615; 2616; CHECK-O3-EX-LABEL: load_i16_anyext_i64: 2617; CHECK-O3-EX: # %bb.0: 2618; CHECK-O3-EX-NEXT: vpbroadcastw (%rdi), %xmm0 2619; CHECK-O3-EX-NEXT: vmovq %xmm0, %rax 2620; CHECK-O3-EX-NEXT: retq 2621 %v = load atomic i16, i16* %ptr unordered, align 8 2622 %vec = insertelement <4 x i16> undef, i16 %v, i64 0 2623 %res = bitcast <4 x i16> %vec to i64 2624 ret i64 %res 2625} 2626 2627; TODO: Would be legal to combine for legal atomic wider types 2628define i16 @load_combine(i8* %p) { 2629; CHECK-O0-LABEL: load_combine: 2630; CHECK-O0: # %bb.0: 2631; CHECK-O0-NEXT: movb (%rdi), %al 2632; CHECK-O0-NEXT: movb 1(%rdi), %cl 2633; CHECK-O0-NEXT: movzbl %al, %eax 2634; CHECK-O0-NEXT: # kill: def $ax killed $ax killed $eax 2635; CHECK-O0-NEXT: movzbl %cl, %ecx 2636; CHECK-O0-NEXT: # kill: def $cx killed $cx killed $ecx 2637; CHECK-O0-NEXT: shlw $8, %cx 2638; CHECK-O0-NEXT: orw %cx, %ax 2639; CHECK-O0-NEXT: retq 2640; 2641; CHECK-O3-LABEL: load_combine: 2642; CHECK-O3: # %bb.0: 2643; CHECK-O3-NEXT: movzbl (%rdi), %ecx 2644; CHECK-O3-NEXT: movzbl 1(%rdi), %eax 2645; CHECK-O3-NEXT: shll $8, %eax 2646; CHECK-O3-NEXT: orl %ecx, %eax 2647; CHECK-O3-NEXT: # kill: def $ax killed $ax killed $eax 2648; CHECK-O3-NEXT: retq 2649 %v1 = load atomic i8, i8* %p unordered, align 2 2650 %p2 = getelementptr i8, i8* %p, i64 1 2651 %v2 = load atomic i8, i8* %p2 unordered, align 1 2652 %v1.ext = zext i8 %v1 to i16 2653 %v2.ext = zext i8 %v2 to i16 2654 %v2.sht = shl i16 %v2.ext, 8 2655 %res = or i16 %v1.ext, %v2.sht 2656 ret i16 %res 2657} 2658 2659define i1 @fold_cmp_over_fence(i32* %p, i32 %v1) { 2660; CHECK-O0-LABEL: fold_cmp_over_fence: 2661; CHECK-O0: # %bb.0: 2662; CHECK-O0-NEXT: movl (%rdi), %eax 2663; CHECK-O0-NEXT: mfence 2664; CHECK-O0-NEXT: cmpl %eax, %esi 2665; CHECK-O0-NEXT: jne .LBB116_2 2666; CHECK-O0-NEXT: # %bb.1: # %taken 2667; CHECK-O0-NEXT: movb $1, %al 2668; CHECK-O0-NEXT: retq 2669; CHECK-O0-NEXT: .LBB116_2: # %untaken 2670; CHECK-O0-NEXT: xorl %eax, %eax 2671; CHECK-O0-NEXT: # kill: def $al killed $al killed $eax 2672; CHECK-O0-NEXT: retq 2673; 2674; CHECK-O3-CUR-LABEL: fold_cmp_over_fence: 2675; CHECK-O3-CUR: # %bb.0: 2676; CHECK-O3-CUR-NEXT: movl (%rdi), %eax 2677; CHECK-O3-CUR-NEXT: mfence 2678; CHECK-O3-CUR-NEXT: cmpl %eax, %esi 2679; CHECK-O3-CUR-NEXT: jne .LBB116_2 2680; CHECK-O3-CUR-NEXT: # %bb.1: # %taken 2681; CHECK-O3-CUR-NEXT: movb $1, %al 2682; CHECK-O3-CUR-NEXT: retq 2683; CHECK-O3-CUR-NEXT: .LBB116_2: # %untaken 2684; CHECK-O3-CUR-NEXT: xorl %eax, %eax 2685; CHECK-O3-CUR-NEXT: retq 2686; 2687; CHECK-O3-EX-LABEL: fold_cmp_over_fence: 2688; CHECK-O3-EX: # %bb.0: 2689; CHECK-O3-EX-NEXT: cmpl (%rdi), %esi 2690; CHECK-O3-EX-NEXT: mfence 2691; CHECK-O3-EX-NEXT: jne .LBB116_2 2692; CHECK-O3-EX-NEXT: # %bb.1: # %taken 2693; CHECK-O3-EX-NEXT: movb $1, %al 2694; CHECK-O3-EX-NEXT: retq 2695; CHECK-O3-EX-NEXT: .LBB116_2: # %untaken 2696; CHECK-O3-EX-NEXT: xorl %eax, %eax 2697; CHECK-O3-EX-NEXT: retq 2698 %v2 = load atomic i32, i32* %p unordered, align 4 2699 fence seq_cst 2700 %cmp = icmp eq i32 %v1, %v2 2701 br i1 %cmp, label %taken, label %untaken 2702taken: 2703 ret i1 true 2704untaken: 2705 ret i1 false 2706} 2707