1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s 3 4define <4 x half> @shuffle_v4f16_23uu(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 5; GFX9-LABEL: shuffle_v4f16_23uu: 6; GFX9: ; %bb.0: 7; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 8; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 9; GFX9-NEXT: s_waitcnt vmcnt(0) 10; GFX9-NEXT: v_mov_b32_e32 v0, v1 11; GFX9-NEXT: s_setpc_b64 s[30:31] 12 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 13 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 14 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 15 ret <4 x half> %shuffle 16} 17 18define <4 x half> @shuffle_v4f16_234u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 19; GFX9-LABEL: shuffle_v4f16_234u: 20; GFX9: ; %bb.0: 21; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 22; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off 23; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off 24; GFX9-NEXT: s_waitcnt vmcnt(1) 25; GFX9-NEXT: v_mov_b32_e32 v0, v5 26; GFX9-NEXT: s_waitcnt vmcnt(0) 27; GFX9-NEXT: s_setpc_b64 s[30:31] 28 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 29 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 30 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 undef> 31 ret <4 x half> %shuffle 32} 33 34define <4 x half> @shuffle_v4f16_u1u3(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 35; GFX9-LABEL: shuffle_v4f16_u1u3: 36; GFX9: ; %bb.0: 37; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 38; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 39; GFX9-NEXT: s_waitcnt vmcnt(0) 40; GFX9-NEXT: s_setpc_b64 s[30:31] 41 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 42 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 43 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 1, i32 undef, i32 3> 44 ret <4 x half> %shuffle 45} 46 47define <4 x half> @shuffle_v4f16_u3u1(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 48; GFX9-LABEL: shuffle_v4f16_u3u1: 49; GFX9: ; %bb.0: 50; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 51; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 52; GFX9-NEXT: s_waitcnt vmcnt(0) 53; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 54; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0 55; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v1 56; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2 57; GFX9-NEXT: s_setpc_b64 s[30:31] 58 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 59 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 60 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 1> 61 ret <4 x half> %shuffle 62} 63 64define <4 x half> @shuffle_v4f16_u3uu(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 65; GFX9-LABEL: shuffle_v4f16_u3uu: 66; GFX9: ; %bb.0: 67; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 68; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 69; GFX9-NEXT: s_waitcnt vmcnt(0) 70; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0 71; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0 72; GFX9-NEXT: s_setpc_b64 s[30:31] 73 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 74 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 75 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 undef> 76 ret <4 x half> %shuffle 77} 78 79define <4 x half> @shuffle_v4f16_3u6u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 80; GFX9-LABEL: shuffle_v4f16_3u6u: 81; GFX9: ; %bb.0: 82; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 83; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 84; GFX9-NEXT: global_load_dword v1, v[2:3], off offset:4 85; GFX9-NEXT: s_waitcnt vmcnt(1) 86; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0 87; GFX9-NEXT: s_waitcnt vmcnt(0) 88; GFX9-NEXT: s_setpc_b64 s[30:31] 89 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 90 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 91 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 undef, i32 6, i32 undef> 92 ret <4 x half> %shuffle 93} 94 95define <4 x half> @shuffle_v4f16_3uu7(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 96; GFX9-LABEL: shuffle_v4f16_3uu7: 97; GFX9: ; %bb.0: 98; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 99; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4 100; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 101; GFX9-NEXT: s_waitcnt vmcnt(1) 102; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v2 103; GFX9-NEXT: s_waitcnt vmcnt(0) 104; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0 105; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 106; GFX9-NEXT: s_setpc_b64 s[30:31] 107 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 108 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 109 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 undef, i32 undef, i32 7> 110 ret <4 x half> %shuffle 111} 112 113define <4 x half> @shuffle_v4f16_35u5(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 114; GFX9-LABEL: shuffle_v4f16_35u5: 115; GFX9: ; %bb.0: 116; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 117; GFX9-NEXT: global_load_dword v2, v[2:3], off 118; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 119; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 120; GFX9-NEXT: s_waitcnt vmcnt(1) 121; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 122; GFX9-NEXT: s_waitcnt vmcnt(0) 123; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 124; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0 125; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2 126; GFX9-NEXT: s_setpc_b64 s[30:31] 127 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 128 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 129 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 undef, i32 5> 130 ret <4 x half> %shuffle 131} 132 133define <4 x half> @shuffle_v4f16_357u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 134; GFX9-LABEL: shuffle_v4f16_357u: 135; GFX9: ; %bb.0: 136; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 137; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 138; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 139; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 140; GFX9-NEXT: s_waitcnt vmcnt(1) 141; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 142; GFX9-NEXT: s_waitcnt vmcnt(0) 143; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 144; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3 145; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0 146; GFX9-NEXT: s_setpc_b64 s[30:31] 147 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 148 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 149 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 7, i32 undef> 150 ret <4 x half> %shuffle 151} 152 153define <4 x half> @shuffle_v4f16_0101(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 154; GFX9-LABEL: shuffle_v4f16_0101: 155; GFX9: ; %bb.0: 156; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 157; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 158; GFX9-NEXT: s_waitcnt vmcnt(0) 159; GFX9-NEXT: v_mov_b32_e32 v1, v0 160; GFX9-NEXT: s_setpc_b64 s[30:31] 161 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 162 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 163 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1> 164 ret <4 x half> %shuffle 165} 166 167define <4 x half> @shuffle_v4f16_0123(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 168; GFX9-LABEL: shuffle_v4f16_0123: 169; GFX9: ; %bb.0: 170; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 171; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 172; GFX9-NEXT: s_waitcnt vmcnt(0) 173; GFX9-NEXT: s_setpc_b64 s[30:31] 174 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 175 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 176 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3> 177 ret <4 x half> %shuffle 178} 179 180define <4 x half> @shuffle_v4f16_0145(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 181; GFX9-LABEL: shuffle_v4f16_0145: 182; GFX9: ; %bb.0: 183; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 184; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 185; GFX9-NEXT: s_waitcnt vmcnt(0) 186; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off 187; GFX9-NEXT: s_waitcnt vmcnt(0) 188; GFX9-NEXT: s_setpc_b64 s[30:31] 189 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 190 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 191 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 4, i32 5> 192 ret <4 x half> %shuffle 193} 194 195define <4 x half> @shuffle_v4f16_0167(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 196; GFX9-LABEL: shuffle_v4f16_0167: 197; GFX9: ; %bb.0: 198; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 199; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 200; GFX9-NEXT: s_waitcnt vmcnt(0) 201; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off 202; GFX9-NEXT: s_waitcnt vmcnt(0) 203; GFX9-NEXT: v_mov_b32_e32 v1, v2 204; GFX9-NEXT: s_setpc_b64 s[30:31] 205 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 206 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 207 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7> 208 ret <4 x half> %shuffle 209} 210 211define <4 x half> @shuffle_v4f16_2301(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 212; GFX9-LABEL: shuffle_v4f16_2301: 213; GFX9: ; %bb.0: 214; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 215; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off 216; GFX9-NEXT: s_waitcnt vmcnt(0) 217; GFX9-NEXT: v_mov_b32_e32 v0, v2 218; GFX9-NEXT: s_setpc_b64 s[30:31] 219 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 220 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 221 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 0, i32 1> 222 ret <4 x half> %shuffle 223} 224 225define <4 x half> @shuffle_v4f16_2323(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 226; GFX9-LABEL: shuffle_v4f16_2323: 227; GFX9: ; %bb.0: 228; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 229; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 230; GFX9-NEXT: s_waitcnt vmcnt(0) 231; GFX9-NEXT: v_mov_b32_e32 v0, v1 232; GFX9-NEXT: s_setpc_b64 s[30:31] 233 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 234 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 235 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 2, i32 3> 236 ret <4 x half> %shuffle 237} 238 239define <4 x half> @shuffle_v4f16_2345(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 240; GFX9-LABEL: shuffle_v4f16_2345: 241; GFX9: ; %bb.0: 242; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 243; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off 244; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off 245; GFX9-NEXT: s_waitcnt vmcnt(1) 246; GFX9-NEXT: v_mov_b32_e32 v0, v5 247; GFX9-NEXT: s_waitcnt vmcnt(0) 248; GFX9-NEXT: s_setpc_b64 s[30:31] 249 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 250 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 251 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 5> 252 ret <4 x half> %shuffle 253} 254 255define <4 x half> @shuffle_v4f16_2367(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 256; GFX9-LABEL: shuffle_v4f16_2367: 257; GFX9: ; %bb.0: 258; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 259; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off 260; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off 261; GFX9-NEXT: s_waitcnt vmcnt(0) 262; GFX9-NEXT: v_mov_b32_e32 v0, v5 263; GFX9-NEXT: s_setpc_b64 s[30:31] 264 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 265 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 266 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 6, i32 7> 267 ret <4 x half> %shuffle 268} 269 270define <4 x half> @shuffle_v4f16_4501(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 271; GFX9-LABEL: shuffle_v4f16_4501: 272; GFX9: ; %bb.0: 273; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 274; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off 275; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off 276; GFX9-NEXT: s_waitcnt vmcnt(0) 277; GFX9-NEXT: v_mov_b32_e32 v1, v4 278; GFX9-NEXT: s_setpc_b64 s[30:31] 279 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 280 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 281 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 0, i32 1> 282 ret <4 x half> %shuffle 283} 284 285define <4 x half> @shuffle_v4f16_4523(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 286; GFX9-LABEL: shuffle_v4f16_4523: 287; GFX9: ; %bb.0: 288; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 289; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 290; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 291; GFX9-NEXT: s_waitcnt vmcnt(0) 292; GFX9-NEXT: v_mov_b32_e32 v0, v2 293; GFX9-NEXT: s_setpc_b64 s[30:31] 294 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 295 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 296 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 2, i32 3> 297 ret <4 x half> %shuffle 298} 299 300define <4 x half> @shuffle_v4f16_4545(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 301; GFX9-LABEL: shuffle_v4f16_4545: 302; GFX9: ; %bb.0: 303; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 304; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off 305; GFX9-NEXT: s_waitcnt vmcnt(0) 306; GFX9-NEXT: v_mov_b32_e32 v1, v0 307; GFX9-NEXT: s_setpc_b64 s[30:31] 308 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 309 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 310 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 4, i32 5> 311 ret <4 x half> %shuffle 312} 313 314define <4 x half> @shuffle_v4f16_4567(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 315; GFX9-LABEL: shuffle_v4f16_4567: 316; GFX9: ; %bb.0: 317; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 318; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off 319; GFX9-NEXT: s_waitcnt vmcnt(0) 320; GFX9-NEXT: s_setpc_b64 s[30:31] 321 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 322 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 323 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 6, i32 7> 324 ret <4 x half> %shuffle 325} 326 327define <4 x half> @shuffle_v4f16_6701(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 328; GFX9-LABEL: shuffle_v4f16_6701: 329; GFX9: ; %bb.0: 330; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 331; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off 332; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off 333; GFX9-NEXT: s_waitcnt vmcnt(0) 334; GFX9-NEXT: v_mov_b32_e32 v0, v1 335; GFX9-NEXT: v_mov_b32_e32 v1, v4 336; GFX9-NEXT: s_setpc_b64 s[30:31] 337 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 338 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 339 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 0, i32 1> 340 ret <4 x half> %shuffle 341} 342 343define <4 x half> @shuffle_v4f16_6723(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 344; GFX9-LABEL: shuffle_v4f16_6723: 345; GFX9: ; %bb.0: 346; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 347; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 348; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 349; GFX9-NEXT: s_waitcnt vmcnt(0) 350; GFX9-NEXT: v_mov_b32_e32 v0, v3 351; GFX9-NEXT: s_setpc_b64 s[30:31] 352 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 353 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 354 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 2, i32 3> 355 ret <4 x half> %shuffle 356} 357 358define <4 x half> @shuffle_v4f16_6745(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 359; GFX9-LABEL: shuffle_v4f16_6745: 360; GFX9: ; %bb.0: 361; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 362; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off 363; GFX9-NEXT: s_waitcnt vmcnt(0) 364; GFX9-NEXT: v_mov_b32_e32 v0, v2 365; GFX9-NEXT: s_setpc_b64 s[30:31] 366 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 367 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 368 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 4, i32 5> 369 ret <4 x half> %shuffle 370} 371 372define <4 x half> @shuffle_v4f16_6767(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 373; GFX9-LABEL: shuffle_v4f16_6767: 374; GFX9: ; %bb.0: 375; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 376; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off 377; GFX9-NEXT: s_waitcnt vmcnt(0) 378; GFX9-NEXT: v_mov_b32_e32 v0, v1 379; GFX9-NEXT: s_setpc_b64 s[30:31] 380 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 381 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 382 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 6, i32 7> 383 ret <4 x half> %shuffle 384} 385 386define <4 x half> @shuffle_v4f16_2356(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 387; GFX9-LABEL: shuffle_v4f16_2356: 388; GFX9: ; %bb.0: 389; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 390; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 391; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off 392; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff 393; GFX9-NEXT: s_waitcnt vmcnt(1) 394; GFX9-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 395; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v0 396; GFX9-NEXT: s_waitcnt vmcnt(0) 397; GFX9-NEXT: v_mov_b32_e32 v0, v5 398; GFX9-NEXT: s_setpc_b64 s[30:31] 399 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 400 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 401 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6> 402 ret <4 x half> %shuffle 403} 404 405define <4 x half> @shuffle_v4f16_5623(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 406; GFX9-LABEL: shuffle_v4f16_5623: 407; GFX9: ; %bb.0: 408; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 409; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 410; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 411; GFX9-NEXT: s_waitcnt vmcnt(0) 412; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff 413; GFX9-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 414; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0 415; GFX9-NEXT: s_setpc_b64 s[30:31] 416 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 417 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 418 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 2, i32 3> 419 ret <4 x half> %shuffle 420} 421 422define <4 x half> @shuffle_v4f16_3456(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 423; GFX9-LABEL: shuffle_v4f16_3456: 424; GFX9: ; %bb.0: 425; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 426; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 427; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 428; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 429; GFX9-NEXT: s_waitcnt vmcnt(1) 430; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 431; GFX9-NEXT: s_waitcnt vmcnt(0) 432; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 433; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0 434; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v4 435; GFX9-NEXT: s_setpc_b64 s[30:31] 436 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 437 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 438 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 4, i32 5, i32 6> 439 ret <4 x half> %shuffle 440} 441 442define <4 x half> @shuffle_v4f16_5634(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 443; GFX9-LABEL: shuffle_v4f16_5634: 444; GFX9: ; %bb.0: 445; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 446; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 447; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 448; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 449; GFX9-NEXT: s_waitcnt vmcnt(1) 450; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 451; GFX9-NEXT: s_waitcnt vmcnt(0) 452; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 453; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v0 454; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v4 455; GFX9-NEXT: s_setpc_b64 s[30:31] 456 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 457 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 458 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 3, i32 4> 459 ret <4 x half> %shuffle 460} 461 462define <4 x half> @shuffle_v4f16_5734(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 463; GFX9-LABEL: shuffle_v4f16_5734: 464; GFX9: ; %bb.0: 465; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 466; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 467; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 468; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 469; GFX9-NEXT: s_waitcnt vmcnt(1) 470; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3 471; GFX9-NEXT: s_waitcnt vmcnt(0) 472; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 473; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 474; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v0 475; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v4 476; GFX9-NEXT: s_setpc_b64 s[30:31] 477 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 478 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 479 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 7, i32 3, i32 4> 480 ret <4 x half> %shuffle 481} 482 483define <4 x i16> @shuffle_v4i16_2356(<4 x i16> addrspace(1)* %arg0, <4 x i16> addrspace(1)* %arg1) { 484; GFX9-LABEL: shuffle_v4i16_2356: 485; GFX9: ; %bb.0: 486; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 487; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 488; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off 489; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff 490; GFX9-NEXT: s_waitcnt vmcnt(1) 491; GFX9-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 492; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v0 493; GFX9-NEXT: s_waitcnt vmcnt(0) 494; GFX9-NEXT: v_mov_b32_e32 v0, v5 495; GFX9-NEXT: s_setpc_b64 s[30:31] 496 %val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0 497 %val1 = load <4 x i16>, <4 x i16> addrspace(1)* %arg1 498 %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6> 499 ret <4 x i16> %shuffle 500} 501 502define <4 x i16> @shuffle_v4i16_0167(<4 x i16> addrspace(1)* %arg0, <4 x i16> addrspace(1)* %arg1) { 503; GFX9-LABEL: shuffle_v4i16_0167: 504; GFX9: ; %bb.0: 505; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 506; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 507; GFX9-NEXT: s_waitcnt vmcnt(0) 508; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off 509; GFX9-NEXT: s_waitcnt vmcnt(0) 510; GFX9-NEXT: v_mov_b32_e32 v1, v2 511; GFX9-NEXT: s_setpc_b64 s[30:31] 512 %val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0 513 %val1 = load <4 x i16>, <4 x i16> addrspace(1)* %arg1 514 %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7> 515 ret <4 x i16> %shuffle 516} 517 518define <4 x half> @shuffle_v4f16_0000(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 519; GFX9-LABEL: shuffle_v4f16_0000: 520; GFX9: ; %bb.0: 521; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 522; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 523; GFX9-NEXT: s_waitcnt vmcnt(0) 524; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v0 525; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1 526; GFX9-NEXT: v_mov_b32_e32 v1, v0 527; GFX9-NEXT: s_setpc_b64 s[30:31] 528 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 529 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 530 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> zeroinitializer 531 ret <4 x half> %shuffle 532} 533 534define <4 x half> @shuffle_v4f16_1010(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 535; GFX9-LABEL: shuffle_v4f16_1010: 536; GFX9: ; %bb.0: 537; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 538; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 539; GFX9-NEXT: s_waitcnt vmcnt(0) 540; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 541; GFX9-NEXT: v_and_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 542; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1 543; GFX9-NEXT: v_mov_b32_e32 v1, v0 544; GFX9-NEXT: s_setpc_b64 s[30:31] 545 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 546 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 547 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 1, i32 0, i32 1, i32 0> 548 ret <4 x half> %shuffle 549} 550 551define <4 x half> @shuffle_v4f16_1100(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 552; GFX9-LABEL: shuffle_v4f16_1100: 553; GFX9: ; %bb.0: 554; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 555; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 556; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff 557; GFX9-NEXT: s_waitcnt vmcnt(0) 558; GFX9-NEXT: v_and_b32_e32 v1, v2, v0 559; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v0 560; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1 561; GFX9-NEXT: v_and_b32_e32 v0, v2, v3 562; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0 563; GFX9-NEXT: s_setpc_b64 s[30:31] 564 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 565 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 566 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 1, i32 1, i32 0, i32 0> 567 ret <4 x half> %shuffle 568} 569 570define <4 x half> @shuffle_v4f16_6161(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 571; GFX9-LABEL: shuffle_v4f16_6161: 572; GFX9: ; %bb.0: 573; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 574; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4 575; GFX9-NEXT: global_load_dword v0, v[0:1], off 576; GFX9-NEXT: s_waitcnt vmcnt(1) 577; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2 578; GFX9-NEXT: s_waitcnt vmcnt(0) 579; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0 580; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1 581; GFX9-NEXT: v_mov_b32_e32 v1, v0 582; GFX9-NEXT: s_setpc_b64 s[30:31] 583 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 584 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 585 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 1, i32 6, i32 1> 586 ret <4 x half> %shuffle 587} 588 589define <4 x half> @shuffle_v4f16_2333(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 590; GFX9-LABEL: shuffle_v4f16_2333: 591; GFX9: ; %bb.0: 592; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 593; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off 594; GFX9-NEXT: s_waitcnt vmcnt(0) 595; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v2 596; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v0 597; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1 598; GFX9-NEXT: v_mov_b32_e32 v0, v2 599; GFX9-NEXT: s_setpc_b64 s[30:31] 600 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 601 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 602 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3> 603 ret <4 x half> %shuffle 604} 605 606define <4 x half> @shuffle_v4f16_6667(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 607; GFX9-LABEL: shuffle_v4f16_6667: 608; GFX9: ; %bb.0: 609; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 610; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off 611; GFX9-NEXT: s_waitcnt vmcnt(0) 612; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v2 613; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v0 614; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1 615; GFX9-NEXT: v_mov_b32_e32 v0, v2 616; GFX9-NEXT: s_setpc_b64 s[30:31] 617 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 618 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 619 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3> 620 ret <4 x half> %shuffle 621} 622 623define <4 x half> @shuffle_v8f16_0101(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) { 624; GFX9-LABEL: shuffle_v8f16_0101: 625; GFX9: ; %bb.0: 626; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 627; GFX9-NEXT: global_load_dword v0, v[0:1], off 628; GFX9-NEXT: s_waitcnt vmcnt(0) 629; GFX9-NEXT: v_mov_b32_e32 v1, v0 630; GFX9-NEXT: s_setpc_b64 s[30:31] 631 %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0 632 %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1 633 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1> 634 ret <4 x half> %shuffle 635} 636 637define <4 x half> @shuffle_v8f16_0123(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) { 638; GFX9-LABEL: shuffle_v8f16_0123: 639; GFX9: ; %bb.0: 640; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 641; GFX9-NEXT: global_load_dwordx4 v[0:3], v[0:1], off 642; GFX9-NEXT: s_waitcnt vmcnt(0) 643; GFX9-NEXT: s_setpc_b64 s[30:31] 644 %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0 645 %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1 646 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3> 647 ret <4 x half> %shuffle 648} 649 650define <4 x half> @shuffle_v8f16_4589(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) { 651; GFX9-LABEL: shuffle_v8f16_4589: 652; GFX9: ; %bb.0: 653; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 654; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:8 655; GFX9-NEXT: global_load_dword v1, v[2:3], off 656; GFX9-NEXT: s_waitcnt vmcnt(0) 657; GFX9-NEXT: s_setpc_b64 s[30:31] 658 %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0 659 %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1 660 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 4, i32 5, i32 8, i32 9> 661 ret <4 x half> %shuffle 662} 663 664define <4 x half> @shuffle_v8f16_10_11_2_3(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) { 665; GFX9-LABEL: shuffle_v8f16_10_11_2_3: 666; GFX9: ; %bb.0: 667; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 668; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4 669; GFX9-NEXT: global_load_dword v0, v[2:3], off offset:4 670; GFX9-NEXT: s_waitcnt vmcnt(0) 671; GFX9-NEXT: s_setpc_b64 s[30:31] 672 %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0 673 %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1 674 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 10, i32 11, i32 2, i32 3> 675 ret <4 x half> %shuffle 676} 677 678define <4 x half> @shuffle_v8f16_13_14_2_3(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) { 679; GFX9-LABEL: shuffle_v8f16_13_14_2_3: 680; GFX9: ; %bb.0: 681; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 682; GFX9-NEXT: global_load_dwordx4 v[2:5], v[2:3], off 683; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4 684; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff 685; GFX9-NEXT: s_waitcnt vmcnt(1) 686; GFX9-NEXT: v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 687; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0 688; GFX9-NEXT: s_waitcnt vmcnt(0) 689; GFX9-NEXT: s_setpc_b64 s[30:31] 690 %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0 691 %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1 692 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 13, i32 14, i32 2, i32 3> 693 ret <4 x half> %shuffle 694} 695 696define <4 x half> @shuffle_v3f16_0122(<3 x half> addrspace(1)* %arg0, <3 x half> addrspace(1)* %arg1) { 697; GFX9-LABEL: shuffle_v3f16_0122: 698; GFX9: ; %bb.0: 699; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 700; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 701; GFX9-NEXT: s_waitcnt vmcnt(0) 702; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v1 703; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2 704; GFX9-NEXT: s_setpc_b64 s[30:31] 705 %val0 = load <3 x half>, <3 x half> addrspace(1)* %arg0 706 %val1 = load <3 x half>, <3 x half> addrspace(1)* %arg1 707 %shuffle = shufflevector <3 x half> %val0, <3 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 2> 708 ret <4 x half> %shuffle 709} 710 711define <4 x half> @shuffle_v2f16_0122(<2 x half> addrspace(1)* %arg0, <2 x half> addrspace(1)* %arg1) { 712; GFX9-LABEL: shuffle_v2f16_0122: 713; GFX9: ; %bb.0: 714; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 715; GFX9-NEXT: global_load_dword v0, v[0:1], off 716; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 717; GFX9-NEXT: s_waitcnt vmcnt(0) 718; GFX9-NEXT: v_and_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 719; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1 720; GFX9-NEXT: s_setpc_b64 s[30:31] 721 %val0 = load <2 x half>, <2 x half> addrspace(1)* %arg0 722 %val1 = load <2 x half>, <2 x half> addrspace(1)* %arg1 723 %shuffle = shufflevector <2 x half> %val0, <2 x half> %val1, <4 x i32> <i32 0, i32 1, i32 1, i32 0> 724 ret <4 x half> %shuffle 725} 726 727define <6 x half> @shuffle_v6f16_452367(<6 x half> addrspace(1)* %arg0, <6 x half> addrspace(1)* %arg1) { 728; GFX9-LABEL: shuffle_v6f16_452367: 729; GFX9: ; %bb.0: 730; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 731; GFX9-NEXT: v_mov_b32_e32 v4, v3 732; GFX9-NEXT: v_mov_b32_e32 v3, v2 733; GFX9-NEXT: global_load_dwordx3 v[0:2], v[0:1], off 734; GFX9-NEXT: global_load_dword v3, v[3:4], off 735; GFX9-NEXT: s_waitcnt vmcnt(1) 736; GFX9-NEXT: v_mov_b32_e32 v0, v2 737; GFX9-NEXT: s_waitcnt vmcnt(0) 738; GFX9-NEXT: v_mov_b32_e32 v2, v3 739; GFX9-NEXT: s_setpc_b64 s[30:31] 740 %val0 = load <6 x half>, <6 x half> addrspace(1)* %arg0 741 %val1 = load <6 x half>, <6 x half> addrspace(1)* %arg1 742 %shuffle = shufflevector <6 x half> %val0, <6 x half> %val1, <6 x i32> <i32 4, i32 5, i32 2, i32 3, i32 6, i32 7> 743 ret <6 x half> %shuffle 744} 745 746define amdgpu_kernel void @fma_shuffle(<4 x half> addrspace(1)* nocapture readonly %A, <4 x half> addrspace(1)* nocapture readonly %B, <4 x half> addrspace(1)* nocapture %C) { 747; GFX9-LABEL: fma_shuffle: 748; GFX9: ; %bb.0: ; %entry 749; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 750; GFX9-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x10 751; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0 752; GFX9-NEXT: s_waitcnt lgkmcnt(0) 753; GFX9-NEXT: v_mov_b32_e32 v1, s1 754; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v4 755; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc 756; GFX9-NEXT: v_mov_b32_e32 v3, s3 757; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s2, v4 758; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc 759; GFX9-NEXT: v_mov_b32_e32 v5, s5 760; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, s4, v4 761; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc 762; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 763; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 764; GFX9-NEXT: global_load_dwordx2 v[6:7], v[4:5], off 765; GFX9-NEXT: s_waitcnt vmcnt(0) 766; GFX9-NEXT: v_pk_fma_f16 v6, v0, v2, v6 op_sel_hi:[0,1,1] 767; GFX9-NEXT: v_pk_fma_f16 v2, v1, v2, v7 op_sel_hi:[0,1,1] 768; GFX9-NEXT: v_pk_fma_f16 v0, v0, v3, v6 op_sel:[1,0,0] 769; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0] 770; GFX9-NEXT: global_store_dwordx2 v[4:5], v[0:1], off 771; GFX9-NEXT: s_endpgm 772entry: 773 %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x() 774 %tmp12 = zext i32 %tmp1 to i64 775 %arrayidx = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %A, i64 %tmp12 776 %tmp14 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx, align 8 777 %arrayidx1 = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %B, i64 %tmp12 778 %tmp15 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx1, align 8 779 %arrayidx2 = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %C, i64 %tmp12 780 %tmp16 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx2, align 8 781 %tmp17 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> zeroinitializer 782 %tmp18 = shufflevector <4 x half> %tmp15, <4 x half> undef, <2 x i32> <i32 0, i32 1> 783 %tmp19 = shufflevector <4 x half> %tmp16, <4 x half> undef, <2 x i32> <i32 0, i32 1> 784 %tmp20 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp17, <2 x half> %tmp18, <2 x half> %tmp19) 785 %tmp21 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 1, i32 1> 786 %tmp22 = shufflevector <4 x half> %tmp15, <4 x half> undef, <2 x i32> <i32 2, i32 3> 787 %tmp23 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp21, <2 x half> %tmp22, <2 x half> %tmp20) 788 %tmp24 = shufflevector <2 x half> %tmp23, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> 789 %tmp25 = shufflevector <4 x half> %tmp24, <4 x half> %tmp16, <4 x i32> <i32 0, i32 1, i32 6, i32 7> 790 %tmp26 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 2, i32 2> 791 %tmp27 = shufflevector <4 x half> %tmp25, <4 x half> undef, <2 x i32> <i32 2, i32 3> 792 %tmp28 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp26, <2 x half> %tmp18, <2 x half> %tmp27) 793 %tmp29 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 3, i32 3> 794 %tmp30 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp29, <2 x half> %tmp22, <2 x half> %tmp28) 795 %tmp31 = shufflevector <2 x half> %tmp30, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> 796 %tmp32 = shufflevector <4 x half> %tmp25, <4 x half> %tmp31, <4 x i32> <i32 0, i32 1, i32 4, i32 5> 797 store <4 x half> %tmp32, <4 x half> addrspace(1)* %arrayidx2, align 8 798 ret void 799} 800 801define <4 x half> @shuffle_v4f16_0456(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 802; GFX9-LABEL: shuffle_v4f16_0456: 803; GFX9: ; %bb.0: 804; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 805; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 806; GFX9-NEXT: s_waitcnt vmcnt(0) 807; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off 808; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff 809; GFX9-NEXT: v_and_b32_e32 v0, v3, v0 810; GFX9-NEXT: s_waitcnt vmcnt(0) 811; GFX9-NEXT: v_and_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 812; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0 813; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v3 814; GFX9-NEXT: s_setpc_b64 s[30:31] 815 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 816 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 817 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 4, i32 5, i32 6> 818 ret <4 x half> %shuffle 819} 820 821declare <2 x half> @llvm.fma.v2f16(<2 x half>, <2 x half>, <2 x half>) #0 822declare i32 @llvm.amdgcn.workitem.id.x() #0 823 824attributes #0 = { nounwind readnone speculatable } 825