1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=CI %s 3; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=SI %s 4; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s 5; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s 6 7; On GFX11, ensure vdst and src2 do not partially overlap. Full overlap is ok. 8 9define i64 @mad_i64_i32_sextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 { 10; CI-LABEL: mad_i64_i32_sextops: 11; CI: ; %bb.0: 12; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 13; CI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] 14; CI-NEXT: s_setpc_b64 s[30:31] 15; 16; SI-LABEL: mad_i64_i32_sextops: 17; SI: ; %bb.0: 18; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 19; SI-NEXT: v_mul_lo_u32 v4, v0, v1 20; SI-NEXT: v_mul_hi_i32 v1, v0, v1 21; SI-NEXT: v_add_i32_e32 v0, vcc, v4, v2 22; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc 23; SI-NEXT: s_setpc_b64 s[30:31] 24; 25; GFX9-LABEL: mad_i64_i32_sextops: 26; GFX9: ; %bb.0: 27; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 28; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] 29; GFX9-NEXT: s_setpc_b64 s[30:31] 30; 31; GFX11-LABEL: mad_i64_i32_sextops: 32; GFX11: ; %bb.0: 33; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 34; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 35; GFX11-NEXT: v_mov_b32_e32 v4, v1 36; GFX11-NEXT: v_mov_b32_e32 v5, v0 37; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 38; GFX11-NEXT: v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3] 39; GFX11-NEXT: s_setpc_b64 s[30:31] 40 %sext0 = sext i32 %arg0 to i64 41 %sext1 = sext i32 %arg1 to i64 42 %mul = mul i64 %sext0, %sext1 43 %mad = add i64 %mul, %arg2 44 ret i64 %mad 45} 46 47define i64 @mad_i64_i32_sextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 { 48; CI-LABEL: mad_i64_i32_sextops_commute: 49; CI: ; %bb.0: 50; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 51; CI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] 52; CI-NEXT: s_setpc_b64 s[30:31] 53; 54; SI-LABEL: mad_i64_i32_sextops_commute: 55; SI: ; %bb.0: 56; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 57; SI-NEXT: v_mul_lo_u32 v4, v0, v1 58; SI-NEXT: v_mul_hi_i32 v1, v0, v1 59; SI-NEXT: v_add_i32_e32 v0, vcc, v2, v4 60; SI-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc 61; SI-NEXT: s_setpc_b64 s[30:31] 62; 63; GFX9-LABEL: mad_i64_i32_sextops_commute: 64; GFX9: ; %bb.0: 65; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 66; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] 67; GFX9-NEXT: s_setpc_b64 s[30:31] 68; 69; GFX11-LABEL: mad_i64_i32_sextops_commute: 70; GFX11: ; %bb.0: 71; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 72; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 73; GFX11-NEXT: v_mov_b32_e32 v4, v1 74; GFX11-NEXT: v_mov_b32_e32 v5, v0 75; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 76; GFX11-NEXT: v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3] 77; GFX11-NEXT: s_setpc_b64 s[30:31] 78 %sext0 = sext i32 %arg0 to i64 79 %sext1 = sext i32 %arg1 to i64 80 %mul = mul i64 %sext0, %sext1 81 %mad = add i64 %arg2, %mul 82 ret i64 %mad 83} 84 85define i64 @mad_u64_u32_zextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 { 86; CI-LABEL: mad_u64_u32_zextops: 87; CI: ; %bb.0: 88; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 89; CI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3] 90; CI-NEXT: s_setpc_b64 s[30:31] 91; 92; SI-LABEL: mad_u64_u32_zextops: 93; SI: ; %bb.0: 94; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 95; SI-NEXT: v_mul_lo_u32 v4, v0, v1 96; SI-NEXT: v_mul_hi_u32 v1, v0, v1 97; SI-NEXT: v_add_i32_e32 v0, vcc, v4, v2 98; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc 99; SI-NEXT: s_setpc_b64 s[30:31] 100; 101; GFX9-LABEL: mad_u64_u32_zextops: 102; GFX9: ; %bb.0: 103; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 104; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3] 105; GFX9-NEXT: s_setpc_b64 s[30:31] 106; 107; GFX11-LABEL: mad_u64_u32_zextops: 108; GFX11: ; %bb.0: 109; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 110; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 111; GFX11-NEXT: v_mov_b32_e32 v4, v1 112; GFX11-NEXT: v_mov_b32_e32 v5, v0 113; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 114; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3] 115; GFX11-NEXT: s_setpc_b64 s[30:31] 116 %sext0 = zext i32 %arg0 to i64 117 %sext1 = zext i32 %arg1 to i64 118 %mul = mul i64 %sext0, %sext1 119 %mad = add i64 %mul, %arg2 120 ret i64 %mad 121} 122 123define i64 @mad_u64_u32_zextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 { 124; CI-LABEL: mad_u64_u32_zextops_commute: 125; CI: ; %bb.0: 126; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 127; CI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3] 128; CI-NEXT: s_setpc_b64 s[30:31] 129; 130; SI-LABEL: mad_u64_u32_zextops_commute: 131; SI: ; %bb.0: 132; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 133; SI-NEXT: v_mul_lo_u32 v4, v0, v1 134; SI-NEXT: v_mul_hi_u32 v1, v0, v1 135; SI-NEXT: v_add_i32_e32 v0, vcc, v2, v4 136; SI-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc 137; SI-NEXT: s_setpc_b64 s[30:31] 138; 139; GFX9-LABEL: mad_u64_u32_zextops_commute: 140; GFX9: ; %bb.0: 141; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 142; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3] 143; GFX9-NEXT: s_setpc_b64 s[30:31] 144; 145; GFX11-LABEL: mad_u64_u32_zextops_commute: 146; GFX11: ; %bb.0: 147; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 148; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 149; GFX11-NEXT: v_mov_b32_e32 v4, v1 150; GFX11-NEXT: v_mov_b32_e32 v5, v0 151; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 152; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3] 153; GFX11-NEXT: s_setpc_b64 s[30:31] 154 %sext0 = zext i32 %arg0 to i64 155 %sext1 = zext i32 %arg1 to i64 156 %mul = mul i64 %sext0, %sext1 157 %mad = add i64 %arg2, %mul 158 ret i64 %mad 159} 160 161define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 { 162; CI-LABEL: mad_i64_i32_sextops_i32_i128: 163; CI: ; %bb.0: 164; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 165; CI-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0 166; CI-NEXT: v_ashrrev_i32_e32 v13, 31, v0 167; CI-NEXT: v_mov_b32_e32 v8, 0 168; CI-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v1, v[7:8] 169; CI-NEXT: v_ashrrev_i32_e32 v14, 31, v1 170; CI-NEXT: v_mad_i64_i32 v[11:12], s[4:5], v1, v13, 0 171; CI-NEXT: v_mov_b32_e32 v7, v10 172; CI-NEXT: v_mov_b32_e32 v10, v8 173; CI-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, v14, v[9:10] 174; CI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[11:12] 175; CI-NEXT: v_add_i32_e32 v9, vcc, v7, v9 176; CI-NEXT: v_addc_u32_e64 v10, s[4:5], 0, 0, vcc 177; CI-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v14, v[9:10] 178; CI-NEXT: v_add_i32_e32 v7, vcc, v9, v0 179; CI-NEXT: v_addc_u32_e32 v9, vcc, v10, v1, vcc 180; CI-NEXT: v_mov_b32_e32 v1, v8 181; CI-NEXT: v_add_i32_e32 v0, vcc, v6, v2 182; CI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc 183; CI-NEXT: v_addc_u32_e32 v2, vcc, v7, v4, vcc 184; CI-NEXT: v_addc_u32_e32 v3, vcc, v9, v5, vcc 185; CI-NEXT: s_setpc_b64 s[30:31] 186; 187; SI-LABEL: mad_i64_i32_sextops_i32_i128: 188; SI: ; %bb.0: 189; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 190; SI-NEXT: v_ashrrev_i32_e32 v6, 31, v0 191; SI-NEXT: v_mul_lo_u32 v11, v6, v1 192; SI-NEXT: v_mul_hi_u32 v12, v0, v1 193; SI-NEXT: v_ashrrev_i32_e32 v7, 31, v1 194; SI-NEXT: v_mul_hi_u32 v14, v6, v1 195; SI-NEXT: v_mul_lo_u32 v13, v0, v7 196; SI-NEXT: v_mul_hi_u32 v10, v0, v7 197; SI-NEXT: v_add_i32_e32 v12, vcc, v11, v12 198; SI-NEXT: v_addc_u32_e32 v14, vcc, 0, v14, vcc 199; SI-NEXT: v_mul_hi_u32 v8, v6, v7 200; SI-NEXT: v_add_i32_e32 v12, vcc, v13, v12 201; SI-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc 202; SI-NEXT: v_mul_i32_i24_e32 v9, v6, v7 203; SI-NEXT: v_add_i32_e32 v10, vcc, v14, v10 204; SI-NEXT: v_mul_hi_i32 v6, v1, v6 205; SI-NEXT: v_mul_hi_i32 v7, v7, v0 206; SI-NEXT: v_addc_u32_e64 v14, s[4:5], 0, 0, vcc 207; SI-NEXT: v_add_i32_e32 v9, vcc, v9, v10 208; SI-NEXT: v_addc_u32_e32 v8, vcc, v8, v14, vcc 209; SI-NEXT: v_add_i32_e32 v10, vcc, v13, v11 210; SI-NEXT: v_mul_lo_u32 v0, v0, v1 211; SI-NEXT: v_addc_u32_e32 v6, vcc, v7, v6, vcc 212; SI-NEXT: v_add_i32_e32 v7, vcc, v9, v10 213; SI-NEXT: v_addc_u32_e32 v6, vcc, v8, v6, vcc 214; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 215; SI-NEXT: v_addc_u32_e32 v1, vcc, v12, v3, vcc 216; SI-NEXT: v_addc_u32_e32 v2, vcc, v7, v4, vcc 217; SI-NEXT: v_addc_u32_e32 v3, vcc, v6, v5, vcc 218; SI-NEXT: s_setpc_b64 s[30:31] 219; 220; GFX9-LABEL: mad_i64_i32_sextops_i32_i128: 221; GFX9: ; %bb.0: 222; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 223; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0 224; GFX9-NEXT: v_ashrrev_i32_e32 v13, 31, v0 225; GFX9-NEXT: v_mov_b32_e32 v9, 0 226; GFX9-NEXT: v_mov_b32_e32 v8, v7 227; GFX9-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v13, v1, v[8:9] 228; GFX9-NEXT: v_ashrrev_i32_e32 v14, 31, v1 229; GFX9-NEXT: v_mov_b32_e32 v8, v11 230; GFX9-NEXT: v_mov_b32_e32 v11, v9 231; GFX9-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v0, v14, v[10:11] 232; GFX9-NEXT: v_mov_b32_e32 v12, v11 233; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v8, v12 234; GFX9-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc 235; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v13, v14, v[8:9] 236; GFX9-NEXT: v_mad_i64_i32 v[12:13], s[4:5], v1, v13, 0 237; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[12:13] 238; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v8, v0 239; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v9, v1, vcc 240; GFX9-NEXT: v_mov_b32_e32 v1, v10 241; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v6, v2 242; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc 243; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v7, v4, vcc 244; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v8, v5, vcc 245; GFX9-NEXT: s_setpc_b64 s[30:31] 246; 247; GFX11-LABEL: mad_i64_i32_sextops_i32_i128: 248; GFX11: ; %bb.0: 249; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 250; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 251; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, v0, v1, 0 252; GFX11-NEXT: v_mov_b32_e32 v8, 0 253; GFX11-NEXT: v_ashrrev_i32_e32 v14, 31, v0 254; GFX11-NEXT: v_ashrrev_i32_e32 v15, 31, v1 255; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) 256; GFX11-NEXT: v_mad_u64_u32 v[9:10], null, v14, v1, v[7:8] 257; GFX11-NEXT: v_mov_b32_e32 v7, v10 258; GFX11-NEXT: v_mov_b32_e32 v10, v8 259; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) 260; GFX11-NEXT: v_mad_u64_u32 v[11:12], null, v0, v15, v[9:10] 261; GFX11-NEXT: v_mad_i64_i32 v[9:10], null, v1, v14, 0 262; GFX11-NEXT: v_mov_b32_e32 v8, v12 263; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) 264; GFX11-NEXT: v_mad_i64_i32 v[12:13], null, v15, v0, v[9:10] 265; GFX11-NEXT: v_add_co_u32 v7, s0, v7, v8 266; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) 267; GFX11-NEXT: v_add_co_ci_u32_e64 v8, null, 0, 0, s0 268; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v14, v15, v[7:8] 269; GFX11-NEXT: v_mov_b32_e32 v7, v11 270; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) 271; GFX11-NEXT: v_add_co_u32 v8, vcc_lo, v0, v12 272; GFX11-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v1, v13, vcc_lo 273; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v6, v2 274; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) 275; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v7, v3, vcc_lo 276; GFX11-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v8, v4, vcc_lo 277; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) 278; GFX11-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo 279; GFX11-NEXT: s_setpc_b64 s[30:31] 280 %sext0 = sext i32 %arg0 to i128 281 %sext1 = sext i32 %arg1 to i128 282 %mul = mul i128 %sext0, %sext1 283 %mad = add i128 %mul, %arg2 284 ret i128 %mad 285} 286 287define i63 @mad_i64_i32_sextops_i32_i63(i32 %arg0, i32 %arg1, i63 %arg2) #0 { 288; CI-LABEL: mad_i64_i32_sextops_i32_i63: 289; CI: ; %bb.0: 290; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 291; CI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] 292; CI-NEXT: s_setpc_b64 s[30:31] 293; 294; SI-LABEL: mad_i64_i32_sextops_i32_i63: 295; SI: ; %bb.0: 296; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 297; SI-NEXT: v_mul_lo_u32 v4, v0, v1 298; SI-NEXT: v_mul_hi_i32 v1, v0, v1 299; SI-NEXT: v_add_i32_e32 v0, vcc, v4, v2 300; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc 301; SI-NEXT: s_setpc_b64 s[30:31] 302; 303; GFX9-LABEL: mad_i64_i32_sextops_i32_i63: 304; GFX9: ; %bb.0: 305; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 306; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] 307; GFX9-NEXT: s_setpc_b64 s[30:31] 308; 309; GFX11-LABEL: mad_i64_i32_sextops_i32_i63: 310; GFX11: ; %bb.0: 311; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 312; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 313; GFX11-NEXT: v_mov_b32_e32 v4, v1 314; GFX11-NEXT: v_mov_b32_e32 v5, v0 315; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 316; GFX11-NEXT: v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3] 317; GFX11-NEXT: s_setpc_b64 s[30:31] 318 %sext0 = sext i32 %arg0 to i63 319 %sext1 = sext i32 %arg1 to i63 320 %mul = mul i63 %sext0, %sext1 321 %mad = add i63 %mul, %arg2 322 ret i63 %mad 323} 324 325define i63 @mad_i64_i32_sextops_i31_i63(i31 %arg0, i31 %arg1, i63 %arg2) #0 { 326; CI-LABEL: mad_i64_i32_sextops_i31_i63: 327; CI: ; %bb.0: 328; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 329; CI-NEXT: v_bfe_i32 v1, v1, 0, 31 330; CI-NEXT: v_bfe_i32 v0, v0, 0, 31 331; CI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] 332; CI-NEXT: s_setpc_b64 s[30:31] 333; 334; SI-LABEL: mad_i64_i32_sextops_i31_i63: 335; SI: ; %bb.0: 336; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 337; SI-NEXT: v_lshlrev_b32_e32 v4, 1, v0 338; SI-NEXT: v_lshlrev_b32_e32 v1, 1, v1 339; SI-NEXT: v_ashr_i64 v[4:5], v[3:4], 33 340; SI-NEXT: v_ashr_i64 v[0:1], v[0:1], 33 341; SI-NEXT: v_mul_lo_u32 v1, v4, v0 342; SI-NEXT: v_mul_hi_i32 v4, v4, v0 343; SI-NEXT: v_add_i32_e32 v0, vcc, v1, v2 344; SI-NEXT: v_addc_u32_e32 v1, vcc, v4, v3, vcc 345; SI-NEXT: s_setpc_b64 s[30:31] 346; 347; GFX9-LABEL: mad_i64_i32_sextops_i31_i63: 348; GFX9: ; %bb.0: 349; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 350; GFX9-NEXT: v_bfe_i32 v1, v1, 0, 31 351; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 31 352; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] 353; GFX9-NEXT: s_setpc_b64 s[30:31] 354; 355; GFX11-LABEL: mad_i64_i32_sextops_i31_i63: 356; GFX11: ; %bb.0: 357; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 358; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 359; GFX11-NEXT: v_bfe_i32 v4, v1, 0, 31 360; GFX11-NEXT: v_bfe_i32 v5, v0, 0, 31 361; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 362; GFX11-NEXT: v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3] 363; GFX11-NEXT: s_setpc_b64 s[30:31] 364 %sext0 = sext i31 %arg0 to i63 365 %sext1 = sext i31 %arg1 to i63 366 %mul = mul i63 %sext0, %sext1 367 %mad = add i63 %mul, %arg2 368 ret i63 %mad 369} 370 371define i64 @mad_i64_i32_extops_i32_i64(i32 %arg0, i32 %arg1, i64 %arg2) #0 { 372; CI-LABEL: mad_i64_i32_extops_i32_i64: 373; CI: ; %bb.0: 374; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 375; CI-NEXT: v_ashrrev_i32_e32 v4, 31, v0 376; CI-NEXT: v_mul_lo_u32 v4, v4, v1 377; CI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3] 378; CI-NEXT: v_add_i32_e32 v1, vcc, v4, v1 379; CI-NEXT: s_setpc_b64 s[30:31] 380; 381; SI-LABEL: mad_i64_i32_extops_i32_i64: 382; SI: ; %bb.0: 383; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 384; SI-NEXT: v_ashrrev_i32_e32 v4, 31, v0 385; SI-NEXT: v_mul_hi_u32 v5, v0, v1 386; SI-NEXT: v_mul_lo_u32 v4, v4, v1 387; SI-NEXT: v_mul_lo_u32 v0, v0, v1 388; SI-NEXT: v_add_i32_e32 v1, vcc, v5, v4 389; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 390; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc 391; SI-NEXT: s_setpc_b64 s[30:31] 392; 393; GFX9-LABEL: mad_i64_i32_extops_i32_i64: 394; GFX9: ; %bb.0: 395; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 396; GFX9-NEXT: v_mov_b32_e32 v4, v1 397; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v0 398; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v4, v[2:3] 399; GFX9-NEXT: v_mov_b32_e32 v2, v1 400; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v4, v[2:3] 401; GFX9-NEXT: v_mov_b32_e32 v1, v2 402; GFX9-NEXT: s_setpc_b64 s[30:31] 403; 404; GFX11-LABEL: mad_i64_i32_extops_i32_i64: 405; GFX11: ; %bb.0: 406; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 407; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 408; GFX11-NEXT: v_mov_b32_e32 v4, v1 409; GFX11-NEXT: v_mov_b32_e32 v5, v0 410; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) 411; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3] 412; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v5 413; GFX11-NEXT: v_mov_b32_e32 v3, v1 414; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 415; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v5, v4, v[3:4] 416; GFX11-NEXT: s_setpc_b64 s[30:31] 417 %ext0 = sext i32 %arg0 to i64 418 %ext1 = zext i32 %arg1 to i64 419 %mul = mul i64 %ext0, %ext1 420 %mad = add i64 %mul, %arg2 421 ret i64 %mad 422} 423 424define i64 @mad_u64_u32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 { 425; CI-LABEL: mad_u64_u32_bitops: 426; CI: ; %bb.0: 427; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 428; CI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5] 429; CI-NEXT: s_setpc_b64 s[30:31] 430; 431; SI-LABEL: mad_u64_u32_bitops: 432; SI: ; %bb.0: 433; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 434; SI-NEXT: v_mul_lo_u32 v1, v0, v2 435; SI-NEXT: v_mul_hi_u32 v2, v0, v2 436; SI-NEXT: v_add_i32_e32 v0, vcc, v1, v4 437; SI-NEXT: v_addc_u32_e32 v1, vcc, v2, v5, vcc 438; SI-NEXT: s_setpc_b64 s[30:31] 439; 440; GFX9-LABEL: mad_u64_u32_bitops: 441; GFX9: ; %bb.0: 442; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 443; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5] 444; GFX9-NEXT: s_setpc_b64 s[30:31] 445; 446; GFX11-LABEL: mad_u64_u32_bitops: 447; GFX11: ; %bb.0: 448; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 449; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 450; GFX11-NEXT: v_mov_b32_e32 v3, v0 451; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 452; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, v[4:5] 453; GFX11-NEXT: s_setpc_b64 s[30:31] 454 %trunc.lhs = and i64 %arg0, 4294967295 455 %trunc.rhs = and i64 %arg1, 4294967295 456 %mul = mul i64 %trunc.lhs, %trunc.rhs 457 %add = add i64 %mul, %arg2 458 ret i64 %add 459} 460 461define i64 @mad_u64_u32_bitops_lhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 { 462; CI-LABEL: mad_u64_u32_bitops_lhs_mask_small: 463; CI: ; %bb.0: 464; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 465; CI-NEXT: v_and_b32_e32 v3, 1, v1 466; CI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5] 467; CI-NEXT: v_mul_lo_u32 v2, v3, v2 468; CI-NEXT: v_add_i32_e32 v1, vcc, v2, v1 469; CI-NEXT: s_setpc_b64 s[30:31] 470; 471; SI-LABEL: mad_u64_u32_bitops_lhs_mask_small: 472; SI: ; %bb.0: 473; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 474; SI-NEXT: v_and_b32_e32 v1, 1, v1 475; SI-NEXT: v_mul_hi_u32 v3, v0, v2 476; SI-NEXT: v_mul_lo_u32 v1, v1, v2 477; SI-NEXT: v_mul_lo_u32 v0, v0, v2 478; SI-NEXT: v_add_i32_e32 v1, vcc, v3, v1 479; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v4 480; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc 481; SI-NEXT: s_setpc_b64 s[30:31] 482; 483; GFX9-LABEL: mad_u64_u32_bitops_lhs_mask_small: 484; GFX9: ; %bb.0: 485; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 486; GFX9-NEXT: v_and_b32_e32 v3, 1, v1 487; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5] 488; GFX9-NEXT: v_mov_b32_e32 v4, v1 489; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, v2, v[4:5] 490; GFX9-NEXT: v_mov_b32_e32 v1, v2 491; GFX9-NEXT: s_setpc_b64 s[30:31] 492; 493; GFX11-LABEL: mad_u64_u32_bitops_lhs_mask_small: 494; GFX11: ; %bb.0: 495; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 496; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 497; GFX11-NEXT: v_mov_b32_e32 v3, v2 498; GFX11-NEXT: v_mov_b32_e32 v2, v0 499; GFX11-NEXT: v_mov_b32_e32 v6, v1 500; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) 501; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v2, v3, v[4:5] 502; GFX11-NEXT: v_and_b32_e32 v5, 1, v6 503; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) 504; GFX11-NEXT: v_mov_b32_e32 v4, v1 505; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v5, v3, v[4:5] 506; GFX11-NEXT: s_setpc_b64 s[30:31] 507 %trunc.lhs = and i64 %arg0, 8589934591 508 %trunc.rhs = and i64 %arg1, 4294967295 509 %mul = mul i64 %trunc.lhs, %trunc.rhs 510 %add = add i64 %mul, %arg2 511 ret i64 %add 512} 513 514define i64 @mad_u64_u32_bitops_rhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 { 515; CI-LABEL: mad_u64_u32_bitops_rhs_mask_small: 516; CI: ; %bb.0: 517; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 518; CI-NEXT: v_mov_b32_e32 v6, v0 519; CI-NEXT: v_and_b32_e32 v3, 1, v3 520; CI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v2, v[4:5] 521; CI-NEXT: v_mul_lo_u32 v2, v6, v3 522; CI-NEXT: v_add_i32_e32 v1, vcc, v2, v1 523; CI-NEXT: s_setpc_b64 s[30:31] 524; 525; SI-LABEL: mad_u64_u32_bitops_rhs_mask_small: 526; SI: ; %bb.0: 527; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 528; SI-NEXT: v_and_b32_e32 v1, 1, v3 529; SI-NEXT: v_mul_hi_u32 v3, v0, v2 530; SI-NEXT: v_mul_lo_u32 v1, v0, v1 531; SI-NEXT: v_mul_lo_u32 v0, v0, v2 532; SI-NEXT: v_add_i32_e32 v1, vcc, v3, v1 533; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v4 534; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc 535; SI-NEXT: s_setpc_b64 s[30:31] 536; 537; GFX9-LABEL: mad_u64_u32_bitops_rhs_mask_small: 538; GFX9: ; %bb.0: 539; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 540; GFX9-NEXT: v_mov_b32_e32 v6, v0 541; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v2, v[4:5] 542; GFX9-NEXT: v_and_b32_e32 v3, 1, v3 543; GFX9-NEXT: v_mov_b32_e32 v2, v1 544; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v3, v[2:3] 545; GFX9-NEXT: v_mov_b32_e32 v1, v2 546; GFX9-NEXT: s_setpc_b64 s[30:31] 547; 548; GFX11-LABEL: mad_u64_u32_bitops_rhs_mask_small: 549; GFX11: ; %bb.0: 550; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 551; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 552; GFX11-NEXT: v_mov_b32_e32 v6, v0 553; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) 554; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v6, v2, v[4:5] 555; GFX11-NEXT: v_and_b32_e32 v4, 1, v3 556; GFX11-NEXT: v_mov_b32_e32 v3, v1 557; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 558; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v6, v4, v[3:4] 559; GFX11-NEXT: s_setpc_b64 s[30:31] 560 %trunc.lhs = and i64 %arg0, 4294967295 561 %trunc.rhs = and i64 %arg1, 8589934591 562 %mul = mul i64 %trunc.lhs, %trunc.rhs 563 %add = add i64 %mul, %arg2 564 ret i64 %add 565} 566 567define i64 @mad_i64_i32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 { 568; CI-LABEL: mad_i64_i32_bitops: 569; CI: ; %bb.0: 570; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 571; CI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v2, v[4:5] 572; CI-NEXT: s_setpc_b64 s[30:31] 573; 574; SI-LABEL: mad_i64_i32_bitops: 575; SI: ; %bb.0: 576; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 577; SI-NEXT: v_mul_lo_u32 v1, v0, v2 578; SI-NEXT: v_mul_hi_i32 v2, v0, v2 579; SI-NEXT: v_add_i32_e32 v0, vcc, v1, v4 580; SI-NEXT: v_addc_u32_e32 v1, vcc, v2, v5, vcc 581; SI-NEXT: s_setpc_b64 s[30:31] 582; 583; GFX9-LABEL: mad_i64_i32_bitops: 584; GFX9: ; %bb.0: 585; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 586; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v2, v[4:5] 587; GFX9-NEXT: s_setpc_b64 s[30:31] 588; 589; GFX11-LABEL: mad_i64_i32_bitops: 590; GFX11: ; %bb.0: 591; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 592; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 593; GFX11-NEXT: v_mov_b32_e32 v3, v0 594; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 595; GFX11-NEXT: v_mad_i64_i32 v[0:1], null, v3, v2, v[4:5] 596; GFX11-NEXT: s_setpc_b64 s[30:31] 597 %shl.lhs = shl i64 %arg0, 32 598 %trunc.lhs = ashr i64 %shl.lhs, 32 599 %shl.rhs = shl i64 %arg1, 32 600 %trunc.rhs = ashr i64 %shl.rhs, 32 601 %mul = mul i64 %trunc.lhs, %trunc.rhs 602 %add = add i64 %mul, %arg2 603 ret i64 %add 604} 605 606; Example from bug report 607define i64 @mad_i64_i32_unpack_i64ops(i64 %arg0) #0 { 608; CI-LABEL: mad_i64_i32_unpack_i64ops: 609; CI: ; %bb.0: 610; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 611; CI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v1, v0, v[0:1] 612; CI-NEXT: s_setpc_b64 s[30:31] 613; 614; SI-LABEL: mad_i64_i32_unpack_i64ops: 615; SI: ; %bb.0: 616; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 617; SI-NEXT: v_mul_lo_u32 v2, v1, v0 618; SI-NEXT: v_mul_hi_u32 v3, v1, v0 619; SI-NEXT: v_add_i32_e32 v0, vcc, v2, v0 620; SI-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc 621; SI-NEXT: s_setpc_b64 s[30:31] 622; 623; GFX9-LABEL: mad_i64_i32_unpack_i64ops: 624; GFX9: ; %bb.0: 625; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 626; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v1, v0, v[0:1] 627; GFX9-NEXT: s_setpc_b64 s[30:31] 628; 629; GFX11-LABEL: mad_i64_i32_unpack_i64ops: 630; GFX11: ; %bb.0: 631; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 632; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 633; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v1, v0, v[0:1] 634; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) 635; GFX11-NEXT: v_mov_b32_e32 v0, v2 636; GFX11-NEXT: v_mov_b32_e32 v1, v3 637; GFX11-NEXT: s_setpc_b64 s[30:31] 638 %tmp4 = lshr i64 %arg0, 32 639 %tmp5 = and i64 %arg0, 4294967295 640 %mul = mul nuw i64 %tmp4, %tmp5 641 %mad = add i64 %mul, %arg0 642 ret i64 %mad 643} 644 645define amdgpu_kernel void @mad_i64_i32_uniform(i64 addrspace(1)* %out, i32 %arg0, i32 %arg1, i64 %arg2) #0 { 646; CI-LABEL: mad_i64_i32_uniform: 647; CI: ; %bb.0: 648; CI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb 649; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd 650; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 651; CI-NEXT: s_waitcnt lgkmcnt(0) 652; CI-NEXT: v_mov_b32_e32 v2, s3 653; CI-NEXT: v_mov_b32_e32 v0, s4 654; CI-NEXT: v_mov_b32_e32 v1, s5 655; CI-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v2, v[0:1] 656; CI-NEXT: s_mov_b32 s3, 0xf000 657; CI-NEXT: s_mov_b32 s2, -1 658; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 659; CI-NEXT: s_endpgm 660; 661; SI-LABEL: mad_i64_i32_uniform: 662; SI: ; %bb.0: 663; SI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb 664; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 665; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd 666; SI-NEXT: s_mov_b32 s7, 0xf000 667; SI-NEXT: s_mov_b32 s6, -1 668; SI-NEXT: s_waitcnt lgkmcnt(0) 669; SI-NEXT: v_mov_b32_e32 v0, s3 670; SI-NEXT: v_mul_hi_u32 v1, s2, v0 671; SI-NEXT: s_mul_i32 s2, s2, s3 672; SI-NEXT: v_mov_b32_e32 v0, s2 673; SI-NEXT: v_mov_b32_e32 v2, s1 674; SI-NEXT: v_add_i32_e32 v0, vcc, s0, v0 675; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc 676; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 677; SI-NEXT: s_endpgm 678; 679; GFX9-LABEL: mad_i64_i32_uniform: 680; GFX9: ; %bb.0: 681; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c 682; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 683; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24 684; GFX9-NEXT: v_mov_b32_e32 v2, 0 685; GFX9-NEXT: s_waitcnt lgkmcnt(0) 686; GFX9-NEXT: s_mul_i32 s0, s2, s3 687; GFX9-NEXT: s_mul_hi_u32 s1, s2, s3 688; GFX9-NEXT: s_add_u32 s0, s0, s4 689; GFX9-NEXT: s_addc_u32 s1, s1, s5 690; GFX9-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] 691; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] 692; GFX9-NEXT: s_endpgm 693; 694; GFX11-LABEL: mad_i64_i32_uniform: 695; GFX11: ; %bb.0: 696; GFX11-NEXT: s_clause 0x2 697; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x2c 698; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x34 699; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 700; GFX11-NEXT: v_mov_b32_e32 v2, 0 701; GFX11-NEXT: s_waitcnt lgkmcnt(0) 702; GFX11-NEXT: s_mul_i32 s6, s2, s3 703; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3 704; GFX11-NEXT: s_add_u32 s2, s6, s4 705; GFX11-NEXT: s_addc_u32 s3, s3, s5 706; GFX11-NEXT: v_mov_b32_e32 v0, s2 707; GFX11-NEXT: v_mov_b32_e32 v1, s3 708; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] 709; GFX11-NEXT: s_endpgm 710 %ext0 = zext i32 %arg0 to i64 711 %ext1 = zext i32 %arg1 to i64 712 %mul = mul i64 %ext0, %ext1 713 %mad = add i64 %mul, %arg2 714 store i64 %mad, i64 addrspace(1)* %out 715 ret void 716} 717 718define i64 @mad_i64_i32_twice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3) #0 { 719; CI-LABEL: mad_i64_i32_twice: 720; CI: ; %bb.0: 721; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 722; CI-NEXT: v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3] 723; CI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[4:5] 724; CI-NEXT: v_xor_b32_e32 v1, v3, v1 725; CI-NEXT: v_xor_b32_e32 v0, v2, v0 726; CI-NEXT: s_setpc_b64 s[30:31] 727; 728; SI-LABEL: mad_i64_i32_twice: 729; SI: ; %bb.0: 730; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 731; SI-NEXT: v_mul_lo_u32 v6, v0, v1 732; SI-NEXT: v_mul_hi_i32 v0, v0, v1 733; SI-NEXT: v_add_i32_e32 v2, vcc, v6, v2 734; SI-NEXT: v_addc_u32_e32 v1, vcc, v0, v3, vcc 735; SI-NEXT: v_add_i32_e32 v3, vcc, v6, v4 736; SI-NEXT: v_addc_u32_e32 v0, vcc, v0, v5, vcc 737; SI-NEXT: v_xor_b32_e32 v1, v1, v0 738; SI-NEXT: v_xor_b32_e32 v0, v2, v3 739; SI-NEXT: s_setpc_b64 s[30:31] 740; 741; GFX9-LABEL: mad_i64_i32_twice: 742; GFX9: ; %bb.0: 743; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 744; GFX9-NEXT: v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3] 745; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[4:5] 746; GFX9-NEXT: v_xor_b32_e32 v1, v3, v1 747; GFX9-NEXT: v_xor_b32_e32 v0, v2, v0 748; GFX9-NEXT: s_setpc_b64 s[30:31] 749; 750; GFX11-LABEL: mad_i64_i32_twice: 751; GFX11: ; %bb.0: 752; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 753; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 754; GFX11-NEXT: v_mad_i64_i32 v[6:7], null, v0, v1, v[2:3] 755; GFX11-NEXT: v_mad_i64_i32 v[2:3], null, v0, v1, v[4:5] 756; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) 757; GFX11-NEXT: v_xor_b32_e32 v0, v6, v2 758; GFX11-NEXT: v_xor_b32_e32 v1, v7, v3 759; GFX11-NEXT: s_setpc_b64 s[30:31] 760 %sext0 = sext i32 %arg0 to i64 761 %sext1 = sext i32 %arg1 to i64 762 %mul = mul i64 %sext0, %sext1 763 %mad1 = add i64 %mul, %arg2 764 %mad2 = add i64 %mul, %arg3 765 %out = xor i64 %mad1, %mad2 766 ret i64 %out 767} 768 769define i64 @mad_i64_i32_thrice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3, i64 %arg4) #0 { 770; CI-LABEL: mad_i64_i32_thrice: 771; CI: ; %bb.0: 772; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 773; CI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0 774; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2 775; CI-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc 776; CI-NEXT: v_add_i32_e32 v4, vcc, v0, v4 777; CI-NEXT: v_addc_u32_e32 v5, vcc, v1, v5, vcc 778; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v6 779; CI-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc 780; CI-NEXT: v_xor_b32_e32 v3, v3, v5 781; CI-NEXT: v_xor_b32_e32 v2, v2, v4 782; CI-NEXT: v_xor_b32_e32 v1, v3, v1 783; CI-NEXT: v_xor_b32_e32 v0, v2, v0 784; CI-NEXT: s_setpc_b64 s[30:31] 785; 786; SI-LABEL: mad_i64_i32_thrice: 787; SI: ; %bb.0: 788; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 789; SI-NEXT: v_mul_lo_u32 v8, v0, v1 790; SI-NEXT: v_mul_hi_i32 v0, v0, v1 791; SI-NEXT: v_add_i32_e32 v1, vcc, v8, v2 792; SI-NEXT: v_addc_u32_e32 v2, vcc, v0, v3, vcc 793; SI-NEXT: v_add_i32_e32 v3, vcc, v8, v4 794; SI-NEXT: v_addc_u32_e32 v4, vcc, v0, v5, vcc 795; SI-NEXT: v_add_i32_e32 v5, vcc, v8, v6 796; SI-NEXT: v_addc_u32_e32 v0, vcc, v0, v7, vcc 797; SI-NEXT: v_xor_b32_e32 v2, v2, v4 798; SI-NEXT: v_xor_b32_e32 v3, v1, v3 799; SI-NEXT: v_xor_b32_e32 v1, v2, v0 800; SI-NEXT: v_xor_b32_e32 v0, v3, v5 801; SI-NEXT: s_setpc_b64 s[30:31] 802; 803; GFX9-LABEL: mad_i64_i32_thrice: 804; GFX9: ; %bb.0: 805; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 806; GFX9-NEXT: v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3] 807; GFX9-NEXT: v_mad_i64_i32 v[4:5], s[4:5], v0, v1, v[4:5] 808; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[6:7] 809; GFX9-NEXT: v_xor_b32_e32 v3, v3, v5 810; GFX9-NEXT: v_xor_b32_e32 v2, v2, v4 811; GFX9-NEXT: v_xor_b32_e32 v1, v3, v1 812; GFX9-NEXT: v_xor_b32_e32 v0, v2, v0 813; GFX9-NEXT: s_setpc_b64 s[30:31] 814; 815; GFX11-LABEL: mad_i64_i32_thrice: 816; GFX11: ; %bb.0: 817; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 818; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 819; GFX11-NEXT: v_mad_i64_i32 v[8:9], null, v0, v1, 0 820; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) 821; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v8, v2 822; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v9, v3, vcc_lo 823; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v8, v4 824; GFX11-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo 825; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, v8, v6 826; GFX11-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v9, v7, vcc_lo 827; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) 828; GFX11-NEXT: v_xor_b32_e32 v0, v0, v2 829; GFX11-NEXT: v_xor_b32_e32 v1, v1, v3 830; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) 831; GFX11-NEXT: v_xor_b32_e32 v0, v0, v4 832; GFX11-NEXT: v_xor_b32_e32 v1, v1, v5 833; GFX11-NEXT: s_setpc_b64 s[30:31] 834 %sext0 = sext i32 %arg0 to i64 835 %sext1 = sext i32 %arg1 to i64 836 %mul = mul i64 %sext0, %sext1 837 %mad1 = add i64 %mul, %arg2 838 %mad2 = add i64 %mul, %arg3 839 %mad3 = add i64 %mul, %arg4 840 %out.p = xor i64 %mad1, %mad2 841 %out = xor i64 %out.p, %mad3 842 ret i64 %out 843} 844 845define i64 @mad_i64_i32_secondary_use(i32 %arg0, i32 %arg1, i64 %arg2) #0 { 846; CI-LABEL: mad_i64_i32_secondary_use: 847; CI: ; %bb.0: 848; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 849; CI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0 850; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2 851; CI-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc 852; CI-NEXT: v_xor_b32_e32 v1, v3, v1 853; CI-NEXT: v_xor_b32_e32 v0, v2, v0 854; CI-NEXT: s_setpc_b64 s[30:31] 855; 856; SI-LABEL: mad_i64_i32_secondary_use: 857; SI: ; %bb.0: 858; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 859; SI-NEXT: v_mul_lo_u32 v4, v0, v1 860; SI-NEXT: v_mul_hi_i32 v0, v0, v1 861; SI-NEXT: v_add_i32_e32 v2, vcc, v4, v2 862; SI-NEXT: v_addc_u32_e32 v1, vcc, v0, v3, vcc 863; SI-NEXT: v_xor_b32_e32 v1, v1, v0 864; SI-NEXT: v_xor_b32_e32 v0, v2, v4 865; SI-NEXT: s_setpc_b64 s[30:31] 866; 867; GFX9-LABEL: mad_i64_i32_secondary_use: 868; GFX9: ; %bb.0: 869; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 870; GFX9-NEXT: v_mad_i64_i32 v[4:5], s[4:5], v0, v1, 0 871; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] 872; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5 873; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4 874; GFX9-NEXT: s_setpc_b64 s[30:31] 875; 876; GFX11-LABEL: mad_i64_i32_secondary_use: 877; GFX11: ; %bb.0: 878; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 879; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 880; GFX11-NEXT: v_mad_i64_i32 v[4:5], null, v0, v1, 0 881; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) 882; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2 883; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v5, v3, vcc_lo 884; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) 885; GFX11-NEXT: v_xor_b32_e32 v0, v0, v4 886; GFX11-NEXT: v_xor_b32_e32 v1, v1, v5 887; GFX11-NEXT: s_setpc_b64 s[30:31] 888 %sext0 = sext i32 %arg0 to i64 889 %sext1 = sext i32 %arg1 to i64 890 %mul = mul i64 %sext0, %sext1 891 %mad = add i64 %mul, %arg2 892 %out = xor i64 %mad, %mul 893 ret i64 %out 894} 895 896define i48 @mad_i48_i48(i48 %arg0, i48 %arg1, i48 %arg2) #0 { 897; CI-LABEL: mad_i48_i48: 898; CI: ; %bb.0: 899; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 900; CI-NEXT: v_mov_b32_e32 v6, v1 901; CI-NEXT: v_mov_b32_e32 v7, v0 902; CI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v2, v[4:5] 903; CI-NEXT: v_mul_lo_u32 v2, v6, v2 904; CI-NEXT: v_mul_lo_u32 v3, v7, v3 905; CI-NEXT: v_add_i32_e32 v1, vcc, v2, v1 906; CI-NEXT: v_add_i32_e32 v1, vcc, v3, v1 907; CI-NEXT: s_setpc_b64 s[30:31] 908; 909; SI-LABEL: mad_i48_i48: 910; SI: ; %bb.0: 911; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 912; SI-NEXT: v_mul_lo_u32 v3, v0, v3 913; SI-NEXT: v_mul_hi_u32 v6, v0, v2 914; SI-NEXT: v_mul_lo_u32 v1, v1, v2 915; SI-NEXT: v_mul_lo_u32 v0, v0, v2 916; SI-NEXT: v_add_i32_e32 v3, vcc, v6, v3 917; SI-NEXT: v_add_i32_e32 v1, vcc, v3, v1 918; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v4 919; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc 920; SI-NEXT: s_setpc_b64 s[30:31] 921; 922; GFX9-LABEL: mad_i48_i48: 923; GFX9: ; %bb.0: 924; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 925; GFX9-NEXT: v_mov_b32_e32 v6, v1 926; GFX9-NEXT: v_mov_b32_e32 v7, v0 927; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v2, v[4:5] 928; GFX9-NEXT: v_mul_lo_u32 v3, v7, v3 929; GFX9-NEXT: v_mul_lo_u32 v2, v6, v2 930; GFX9-NEXT: v_add3_u32 v1, v2, v1, v3 931; GFX9-NEXT: s_setpc_b64 s[30:31] 932; 933; GFX11-LABEL: mad_i48_i48: 934; GFX11: ; %bb.0: 935; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 936; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 937; GFX11-NEXT: v_mov_b32_e32 v6, v1 938; GFX11-NEXT: v_mov_b32_e32 v7, v0 939; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) 940; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v7, v2, v[4:5] 941; GFX11-NEXT: v_mul_lo_u32 v3, v7, v3 942; GFX11-NEXT: v_mul_lo_u32 v2, v6, v2 943; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) 944; GFX11-NEXT: v_add3_u32 v1, v2, v1, v3 945; GFX11-NEXT: s_setpc_b64 s[30:31] 946 %m = mul i48 %arg0, %arg1 947 %a = add i48 %m, %arg2 948 ret i48 %a 949} 950 951attributes #0 = { nounwind } 952attributes #1 = { nounwind readnone speculatable } 953