1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s 3 4define <4 x half> @shuffle_v4f16_23uu(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 5; GFX9-LABEL: shuffle_v4f16_23uu: 6; GFX9: ; %bb.0: 7; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 8; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 9; GFX9-NEXT: s_waitcnt vmcnt(0) 10; GFX9-NEXT: s_setpc_b64 s[30:31] 11 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 12 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 13 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 14 ret <4 x half> %shuffle 15} 16 17define <4 x half> @shuffle_v4f16_234u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 18; GFX9-LABEL: shuffle_v4f16_234u: 19; GFX9: ; %bb.0: 20; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 21; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 22; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 23; GFX9-NEXT: s_waitcnt vmcnt(1) 24; GFX9-NEXT: v_mov_b32_e32 v1, v2 25; GFX9-NEXT: s_waitcnt vmcnt(0) 26; GFX9-NEXT: s_setpc_b64 s[30:31] 27 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 28 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 29 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 undef> 30 ret <4 x half> %shuffle 31} 32 33define <4 x half> @shuffle_v4f16_u1u3(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 34; GFX9-LABEL: shuffle_v4f16_u1u3: 35; GFX9: ; %bb.0: 36; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 37; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 38; GFX9-NEXT: s_waitcnt vmcnt(0) 39; GFX9-NEXT: s_setpc_b64 s[30:31] 40 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 41 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 42 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 1, i32 undef, i32 3> 43 ret <4 x half> %shuffle 44} 45 46define <4 x half> @shuffle_v4f16_u3u1(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 47; GFX9-LABEL: shuffle_v4f16_u3u1: 48; GFX9: ; %bb.0: 49; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 50; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 51; GFX9-NEXT: s_waitcnt vmcnt(0) 52; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 53; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0 54; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v1 55; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2 56; GFX9-NEXT: s_setpc_b64 s[30:31] 57 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 58 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 59 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 1> 60 ret <4 x half> %shuffle 61} 62 63define <4 x half> @shuffle_v4f16_u3uu(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 64; GFX9-LABEL: shuffle_v4f16_u3uu: 65; GFX9: ; %bb.0: 66; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 67; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 68; GFX9-NEXT: s_waitcnt vmcnt(0) 69; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0 70; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0 71; GFX9-NEXT: s_setpc_b64 s[30:31] 72 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 73 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 74 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 undef> 75 ret <4 x half> %shuffle 76} 77 78define <4 x half> @shuffle_v4f16_3u6u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 79; GFX9-LABEL: shuffle_v4f16_3u6u: 80; GFX9: ; %bb.0: 81; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 82; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 83; GFX9-NEXT: global_load_dword v1, v[2:3], off offset:4 84; GFX9-NEXT: s_waitcnt vmcnt(1) 85; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0 86; GFX9-NEXT: s_waitcnt vmcnt(0) 87; GFX9-NEXT: s_setpc_b64 s[30:31] 88 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 89 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 90 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 undef, i32 6, i32 undef> 91 ret <4 x half> %shuffle 92} 93 94define <4 x half> @shuffle_v4f16_3uu7(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 95; GFX9-LABEL: shuffle_v4f16_3uu7: 96; GFX9: ; %bb.0: 97; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 98; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4 99; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 100; GFX9-NEXT: s_waitcnt vmcnt(1) 101; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v2 102; GFX9-NEXT: s_waitcnt vmcnt(0) 103; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0 104; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 105; GFX9-NEXT: s_setpc_b64 s[30:31] 106 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 107 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 108 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 undef, i32 undef, i32 7> 109 ret <4 x half> %shuffle 110} 111 112define <4 x half> @shuffle_v4f16_35u5(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 113; GFX9-LABEL: shuffle_v4f16_35u5: 114; GFX9: ; %bb.0: 115; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 116; GFX9-NEXT: global_load_dword v2, v[2:3], off 117; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 118; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 119; GFX9-NEXT: s_waitcnt vmcnt(1) 120; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 121; GFX9-NEXT: s_waitcnt vmcnt(0) 122; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 123; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0 124; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2 125; GFX9-NEXT: s_setpc_b64 s[30:31] 126 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 127 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 128 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 undef, i32 5> 129 ret <4 x half> %shuffle 130} 131 132define <4 x half> @shuffle_v4f16_357u(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 133; GFX9-LABEL: shuffle_v4f16_357u: 134; GFX9: ; %bb.0: 135; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 136; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 137; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 138; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 139; GFX9-NEXT: s_waitcnt vmcnt(1) 140; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 141; GFX9-NEXT: s_waitcnt vmcnt(0) 142; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 143; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3 144; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0 145; GFX9-NEXT: s_setpc_b64 s[30:31] 146 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 147 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 148 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 7, i32 undef> 149 ret <4 x half> %shuffle 150} 151 152define <4 x half> @shuffle_v4f16_0101(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 153; GFX9-LABEL: shuffle_v4f16_0101: 154; GFX9: ; %bb.0: 155; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 156; GFX9-NEXT: global_load_dword v0, v[0:1], off 157; GFX9-NEXT: s_waitcnt vmcnt(0) 158; GFX9-NEXT: v_mov_b32_e32 v1, v0 159; GFX9-NEXT: s_setpc_b64 s[30:31] 160 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 161 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 162 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1> 163 ret <4 x half> %shuffle 164} 165 166define <4 x half> @shuffle_v4f16_0123(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 167; GFX9-LABEL: shuffle_v4f16_0123: 168; GFX9: ; %bb.0: 169; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 170; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 171; GFX9-NEXT: s_waitcnt vmcnt(0) 172; GFX9-NEXT: s_setpc_b64 s[30:31] 173 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 174 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 175 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3> 176 ret <4 x half> %shuffle 177} 178 179define <4 x half> @shuffle_v4f16_0145(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 180; GFX9-LABEL: shuffle_v4f16_0145: 181; GFX9: ; %bb.0: 182; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 183; GFX9-NEXT: global_load_dword v0, v[0:1], off 184; GFX9-NEXT: global_load_dword v1, v[2:3], off 185; GFX9-NEXT: s_waitcnt vmcnt(0) 186; GFX9-NEXT: s_setpc_b64 s[30:31] 187 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 188 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 189 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 4, i32 5> 190 ret <4 x half> %shuffle 191} 192 193define <4 x half> @shuffle_v4f16_0167(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 194; GFX9-LABEL: shuffle_v4f16_0167: 195; GFX9: ; %bb.0: 196; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 197; GFX9-NEXT: global_load_dword v0, v[0:1], off 198; GFX9-NEXT: global_load_dword v1, v[2:3], off offset:4 199; GFX9-NEXT: s_waitcnt vmcnt(0) 200; GFX9-NEXT: s_setpc_b64 s[30:31] 201 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 202 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 203 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7> 204 ret <4 x half> %shuffle 205} 206 207define <4 x half> @shuffle_v4f16_2301(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 208; GFX9-LABEL: shuffle_v4f16_2301: 209; GFX9: ; %bb.0: 210; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 211; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off 212; GFX9-NEXT: s_waitcnt vmcnt(0) 213; GFX9-NEXT: v_mov_b32_e32 v0, v2 214; GFX9-NEXT: s_setpc_b64 s[30:31] 215 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 216 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 217 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 0, i32 1> 218 ret <4 x half> %shuffle 219} 220 221define <4 x half> @shuffle_v4f16_2323(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 222; GFX9-LABEL: shuffle_v4f16_2323: 223; GFX9: ; %bb.0: 224; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 225; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 226; GFX9-NEXT: s_waitcnt vmcnt(0) 227; GFX9-NEXT: v_mov_b32_e32 v1, v0 228; GFX9-NEXT: s_setpc_b64 s[30:31] 229 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 230 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 231 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 2, i32 3> 232 ret <4 x half> %shuffle 233} 234 235define <4 x half> @shuffle_v4f16_2345(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 236; GFX9-LABEL: shuffle_v4f16_2345: 237; GFX9: ; %bb.0: 238; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 239; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 240; GFX9-NEXT: global_load_dword v1, v[2:3], off 241; GFX9-NEXT: s_waitcnt vmcnt(0) 242; GFX9-NEXT: s_setpc_b64 s[30:31] 243 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 244 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 245 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 5> 246 ret <4 x half> %shuffle 247} 248 249define <4 x half> @shuffle_v4f16_2367(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 250; GFX9-LABEL: shuffle_v4f16_2367: 251; GFX9: ; %bb.0: 252; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 253; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 254; GFX9-NEXT: global_load_dword v1, v[2:3], off offset:4 255; GFX9-NEXT: s_waitcnt vmcnt(0) 256; GFX9-NEXT: s_setpc_b64 s[30:31] 257 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 258 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 259 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 6, i32 7> 260 ret <4 x half> %shuffle 261} 262 263define <4 x half> @shuffle_v4f16_4501(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 264; GFX9-LABEL: shuffle_v4f16_4501: 265; GFX9: ; %bb.0: 266; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 267; GFX9-NEXT: global_load_dword v2, v[2:3], off 268; GFX9-NEXT: global_load_dword v1, v[0:1], off 269; GFX9-NEXT: s_waitcnt vmcnt(1) 270; GFX9-NEXT: v_mov_b32_e32 v0, v2 271; GFX9-NEXT: s_waitcnt vmcnt(0) 272; GFX9-NEXT: s_setpc_b64 s[30:31] 273 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 274 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 275 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 0, i32 1> 276 ret <4 x half> %shuffle 277} 278 279define <4 x half> @shuffle_v4f16_4523(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 280; GFX9-LABEL: shuffle_v4f16_4523: 281; GFX9: ; %bb.0: 282; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 283; GFX9-NEXT: global_load_dword v2, v[2:3], off 284; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4 285; GFX9-NEXT: s_waitcnt vmcnt(1) 286; GFX9-NEXT: v_mov_b32_e32 v0, v2 287; GFX9-NEXT: s_waitcnt vmcnt(0) 288; GFX9-NEXT: s_setpc_b64 s[30:31] 289 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 290 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 291 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 2, i32 3> 292 ret <4 x half> %shuffle 293} 294 295define <4 x half> @shuffle_v4f16_4545(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 296; GFX9-LABEL: shuffle_v4f16_4545: 297; GFX9: ; %bb.0: 298; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 299; GFX9-NEXT: global_load_dword v0, v[2:3], off 300; GFX9-NEXT: s_waitcnt vmcnt(0) 301; GFX9-NEXT: v_mov_b32_e32 v1, v0 302; GFX9-NEXT: s_setpc_b64 s[30:31] 303 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 304 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 305 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 4, i32 5> 306 ret <4 x half> %shuffle 307} 308 309define <4 x half> @shuffle_v4f16_4567(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 310; GFX9-LABEL: shuffle_v4f16_4567: 311; GFX9: ; %bb.0: 312; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 313; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off 314; GFX9-NEXT: s_waitcnt vmcnt(0) 315; GFX9-NEXT: s_setpc_b64 s[30:31] 316 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 317 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 318 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 4, i32 5, i32 6, i32 7> 319 ret <4 x half> %shuffle 320} 321 322define <4 x half> @shuffle_v4f16_6701(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 323; GFX9-LABEL: shuffle_v4f16_6701: 324; GFX9: ; %bb.0: 325; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 326; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4 327; GFX9-NEXT: global_load_dword v1, v[0:1], off 328; GFX9-NEXT: s_waitcnt vmcnt(1) 329; GFX9-NEXT: v_mov_b32_e32 v0, v2 330; GFX9-NEXT: s_waitcnt vmcnt(0) 331; GFX9-NEXT: s_setpc_b64 s[30:31] 332 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 333 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 334 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 0, i32 1> 335 ret <4 x half> %shuffle 336} 337 338define <4 x half> @shuffle_v4f16_6723(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 339; GFX9-LABEL: shuffle_v4f16_6723: 340; GFX9: ; %bb.0: 341; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 342; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4 343; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4 344; GFX9-NEXT: s_waitcnt vmcnt(1) 345; GFX9-NEXT: v_mov_b32_e32 v0, v2 346; GFX9-NEXT: s_waitcnt vmcnt(0) 347; GFX9-NEXT: s_setpc_b64 s[30:31] 348 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 349 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 350 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 2, i32 3> 351 ret <4 x half> %shuffle 352} 353 354define <4 x half> @shuffle_v4f16_6745(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 355; GFX9-LABEL: shuffle_v4f16_6745: 356; GFX9: ; %bb.0: 357; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 358; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off 359; GFX9-NEXT: s_waitcnt vmcnt(0) 360; GFX9-NEXT: v_mov_b32_e32 v0, v2 361; GFX9-NEXT: s_setpc_b64 s[30:31] 362 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 363 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 364 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 4, i32 5> 365 ret <4 x half> %shuffle 366} 367 368define <4 x half> @shuffle_v4f16_6767(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 369; GFX9-LABEL: shuffle_v4f16_6767: 370; GFX9: ; %bb.0: 371; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 372; GFX9-NEXT: global_load_dword v0, v[2:3], off offset:4 373; GFX9-NEXT: s_waitcnt vmcnt(0) 374; GFX9-NEXT: v_mov_b32_e32 v1, v0 375; GFX9-NEXT: s_setpc_b64 s[30:31] 376 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 377 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 378 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 7, i32 6, i32 7> 379 ret <4 x half> %shuffle 380} 381 382define <4 x half> @shuffle_v4f16_2356(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 383; GFX9-LABEL: shuffle_v4f16_2356: 384; GFX9: ; %bb.0: 385; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 386; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 387; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 388; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 389; GFX9-NEXT: s_waitcnt vmcnt(1) 390; GFX9-NEXT: v_and_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 391; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1 392; GFX9-NEXT: s_waitcnt vmcnt(0) 393; GFX9-NEXT: s_setpc_b64 s[30:31] 394 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 395 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 396 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6> 397 ret <4 x half> %shuffle 398} 399 400define <4 x half> @shuffle_v4f16_5623(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 401; GFX9-LABEL: shuffle_v4f16_5623: 402; GFX9: ; %bb.0: 403; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 404; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 405; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4 406; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff 407; GFX9-NEXT: s_waitcnt vmcnt(1) 408; GFX9-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 409; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0 410; GFX9-NEXT: s_waitcnt vmcnt(0) 411; GFX9-NEXT: s_setpc_b64 s[30:31] 412 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 413 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 414 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 2, i32 3> 415 ret <4 x half> %shuffle 416} 417 418define <4 x half> @shuffle_v4f16_3456(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 419; GFX9-LABEL: shuffle_v4f16_3456: 420; GFX9: ; %bb.0: 421; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 422; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 423; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 424; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 425; GFX9-NEXT: s_waitcnt vmcnt(1) 426; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 427; GFX9-NEXT: s_waitcnt vmcnt(0) 428; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 429; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0 430; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v4 431; GFX9-NEXT: s_setpc_b64 s[30:31] 432 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 433 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 434 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 4, i32 5, i32 6> 435 ret <4 x half> %shuffle 436} 437 438define <4 x half> @shuffle_v4f16_5634(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 439; GFX9-LABEL: shuffle_v4f16_5634: 440; GFX9: ; %bb.0: 441; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 442; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 443; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 444; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 445; GFX9-NEXT: s_waitcnt vmcnt(1) 446; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 447; GFX9-NEXT: s_waitcnt vmcnt(0) 448; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 449; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v0 450; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v4 451; GFX9-NEXT: s_setpc_b64 s[30:31] 452 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 453 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 454 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 3, i32 4> 455 ret <4 x half> %shuffle 456} 457 458define <4 x half> @shuffle_v4f16_5734(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 459; GFX9-LABEL: shuffle_v4f16_5734: 460; GFX9: ; %bb.0: 461; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 462; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 463; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 464; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 465; GFX9-NEXT: s_waitcnt vmcnt(1) 466; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3 467; GFX9-NEXT: s_waitcnt vmcnt(0) 468; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 469; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 470; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v0 471; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v4 472; GFX9-NEXT: s_setpc_b64 s[30:31] 473 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 474 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 475 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 7, i32 3, i32 4> 476 ret <4 x half> %shuffle 477} 478 479define <4 x i16> @shuffle_v4i16_2356(<4 x i16> addrspace(1)* %arg0, <4 x i16> addrspace(1)* %arg1) { 480; GFX9-LABEL: shuffle_v4i16_2356: 481; GFX9: ; %bb.0: 482; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 483; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 484; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 485; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 486; GFX9-NEXT: s_waitcnt vmcnt(1) 487; GFX9-NEXT: v_and_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 488; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1 489; GFX9-NEXT: s_waitcnt vmcnt(0) 490; GFX9-NEXT: s_setpc_b64 s[30:31] 491 %val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0 492 %val1 = load <4 x i16>, <4 x i16> addrspace(1)* %arg1 493 %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6> 494 ret <4 x i16> %shuffle 495} 496 497define <4 x i16> @shuffle_v4i16_0167(<4 x i16> addrspace(1)* %arg0, <4 x i16> addrspace(1)* %arg1) { 498; GFX9-LABEL: shuffle_v4i16_0167: 499; GFX9: ; %bb.0: 500; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 501; GFX9-NEXT: global_load_dword v0, v[0:1], off 502; GFX9-NEXT: global_load_dword v1, v[2:3], off offset:4 503; GFX9-NEXT: s_waitcnt vmcnt(0) 504; GFX9-NEXT: s_setpc_b64 s[30:31] 505 %val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0 506 %val1 = load <4 x i16>, <4 x i16> addrspace(1)* %arg1 507 %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 0, i32 1, i32 6, i32 7> 508 ret <4 x i16> %shuffle 509} 510 511define <4 x half> @shuffle_v4f16_0000(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 512; GFX9-LABEL: shuffle_v4f16_0000: 513; GFX9: ; %bb.0: 514; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 515; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 516; GFX9-NEXT: s_waitcnt vmcnt(0) 517; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v0 518; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1 519; GFX9-NEXT: v_mov_b32_e32 v1, v0 520; GFX9-NEXT: s_setpc_b64 s[30:31] 521 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 522 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 523 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> zeroinitializer 524 ret <4 x half> %shuffle 525} 526 527define <4 x half> @shuffle_v4f16_1010(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 528; GFX9-LABEL: shuffle_v4f16_1010: 529; GFX9: ; %bb.0: 530; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 531; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 532; GFX9-NEXT: s_waitcnt vmcnt(0) 533; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 534; GFX9-NEXT: v_and_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 535; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1 536; GFX9-NEXT: v_mov_b32_e32 v1, v0 537; GFX9-NEXT: s_setpc_b64 s[30:31] 538 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 539 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 540 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 1, i32 0, i32 1, i32 0> 541 ret <4 x half> %shuffle 542} 543 544define <4 x half> @shuffle_v4f16_1100(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 545; GFX9-LABEL: shuffle_v4f16_1100: 546; GFX9: ; %bb.0: 547; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 548; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 549; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff 550; GFX9-NEXT: s_waitcnt vmcnt(0) 551; GFX9-NEXT: v_and_b32_e32 v1, v2, v0 552; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v0 553; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1 554; GFX9-NEXT: v_and_b32_e32 v0, v2, v3 555; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0 556; GFX9-NEXT: s_setpc_b64 s[30:31] 557 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 558 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 559 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 1, i32 1, i32 0, i32 0> 560 ret <4 x half> %shuffle 561} 562 563define <4 x half> @shuffle_v4f16_6161(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 564; GFX9-LABEL: shuffle_v4f16_6161: 565; GFX9: ; %bb.0: 566; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 567; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4 568; GFX9-NEXT: global_load_dword v0, v[0:1], off 569; GFX9-NEXT: s_waitcnt vmcnt(1) 570; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2 571; GFX9-NEXT: s_waitcnt vmcnt(0) 572; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0 573; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1 574; GFX9-NEXT: v_mov_b32_e32 v1, v0 575; GFX9-NEXT: s_setpc_b64 s[30:31] 576 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 577 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 578 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 6, i32 1, i32 6, i32 1> 579 ret <4 x half> %shuffle 580} 581 582define <4 x half> @shuffle_v4f16_2333(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 583; GFX9-LABEL: shuffle_v4f16_2333: 584; GFX9: ; %bb.0: 585; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 586; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 587; GFX9-NEXT: s_waitcnt vmcnt(0) 588; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0 589; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v1 590; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2 591; GFX9-NEXT: s_setpc_b64 s[30:31] 592 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 593 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 594 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3> 595 ret <4 x half> %shuffle 596} 597 598define <4 x half> @shuffle_v4f16_6667(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 599; GFX9-LABEL: shuffle_v4f16_6667: 600; GFX9: ; %bb.0: 601; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 602; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 603; GFX9-NEXT: s_waitcnt vmcnt(0) 604; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0 605; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v1 606; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2 607; GFX9-NEXT: s_setpc_b64 s[30:31] 608 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 609 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 610 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3> 611 ret <4 x half> %shuffle 612} 613 614define <4 x half> @shuffle_v8f16_0101(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) { 615; GFX9-LABEL: shuffle_v8f16_0101: 616; GFX9: ; %bb.0: 617; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 618; GFX9-NEXT: global_load_dword v0, v[0:1], off 619; GFX9-NEXT: s_waitcnt vmcnt(0) 620; GFX9-NEXT: v_mov_b32_e32 v1, v0 621; GFX9-NEXT: s_setpc_b64 s[30:31] 622 %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0 623 %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1 624 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 0, i32 1, i32 0, i32 1> 625 ret <4 x half> %shuffle 626} 627 628define <4 x half> @shuffle_v8f16_0123(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) { 629; GFX9-LABEL: shuffle_v8f16_0123: 630; GFX9: ; %bb.0: 631; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 632; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 633; GFX9-NEXT: s_waitcnt vmcnt(0) 634; GFX9-NEXT: s_setpc_b64 s[30:31] 635 %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0 636 %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1 637 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3> 638 ret <4 x half> %shuffle 639} 640 641define <4 x half> @shuffle_v8f16_4589(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) { 642; GFX9-LABEL: shuffle_v8f16_4589: 643; GFX9: ; %bb.0: 644; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 645; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:8 646; GFX9-NEXT: global_load_dword v1, v[2:3], off 647; GFX9-NEXT: s_waitcnt vmcnt(0) 648; GFX9-NEXT: s_setpc_b64 s[30:31] 649 %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0 650 %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1 651 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 4, i32 5, i32 8, i32 9> 652 ret <4 x half> %shuffle 653} 654 655define <4 x half> @shuffle_v8f16_10_11_2_3(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) { 656; GFX9-LABEL: shuffle_v8f16_10_11_2_3: 657; GFX9: ; %bb.0: 658; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 659; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4 660; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4 661; GFX9-NEXT: s_waitcnt vmcnt(1) 662; GFX9-NEXT: v_mov_b32_e32 v0, v2 663; GFX9-NEXT: s_waitcnt vmcnt(0) 664; GFX9-NEXT: s_setpc_b64 s[30:31] 665 %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0 666 %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1 667 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 10, i32 11, i32 2, i32 3> 668 ret <4 x half> %shuffle 669} 670 671define <4 x half> @shuffle_v8f16_13_14_2_3(<8 x half> addrspace(1)* %arg0, <8 x half> addrspace(1)* %arg1) { 672; GFX9-LABEL: shuffle_v8f16_13_14_2_3: 673; GFX9: ; %bb.0: 674; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 675; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off offset:8 676; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4 677; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff 678; GFX9-NEXT: s_waitcnt vmcnt(1) 679; GFX9-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 680; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0 681; GFX9-NEXT: s_waitcnt vmcnt(0) 682; GFX9-NEXT: s_setpc_b64 s[30:31] 683 %val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0 684 %val1 = load <8 x half>, <8 x half> addrspace(1)* %arg1 685 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 13, i32 14, i32 2, i32 3> 686 ret <4 x half> %shuffle 687} 688 689define <4 x half> @shuffle_v3f16_0122(<3 x half> addrspace(1)* %arg0, <3 x half> addrspace(1)* %arg1) { 690; GFX9-LABEL: shuffle_v3f16_0122: 691; GFX9: ; %bb.0: 692; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 693; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 694; GFX9-NEXT: s_waitcnt vmcnt(0) 695; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v1 696; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2 697; GFX9-NEXT: s_setpc_b64 s[30:31] 698 %val0 = load <3 x half>, <3 x half> addrspace(1)* %arg0 699 %val1 = load <3 x half>, <3 x half> addrspace(1)* %arg1 700 %shuffle = shufflevector <3 x half> %val0, <3 x half> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 2> 701 ret <4 x half> %shuffle 702} 703 704define <4 x half> @shuffle_v2f16_0122(<2 x half> addrspace(1)* %arg0, <2 x half> addrspace(1)* %arg1) { 705; GFX9-LABEL: shuffle_v2f16_0122: 706; GFX9: ; %bb.0: 707; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 708; GFX9-NEXT: global_load_dword v0, v[0:1], off 709; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff 710; GFX9-NEXT: s_waitcnt vmcnt(0) 711; GFX9-NEXT: v_and_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 712; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1 713; GFX9-NEXT: s_setpc_b64 s[30:31] 714 %val0 = load <2 x half>, <2 x half> addrspace(1)* %arg0 715 %val1 = load <2 x half>, <2 x half> addrspace(1)* %arg1 716 %shuffle = shufflevector <2 x half> %val0, <2 x half> %val1, <4 x i32> <i32 0, i32 1, i32 1, i32 0> 717 ret <4 x half> %shuffle 718} 719 720define <6 x half> @shuffle_v6f16_452367(<6 x half> addrspace(1)* %arg0, <6 x half> addrspace(1)* %arg1) { 721; GFX9-LABEL: shuffle_v6f16_452367: 722; GFX9: ; %bb.0: 723; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 724; GFX9-NEXT: v_mov_b32_e32 v4, v3 725; GFX9-NEXT: v_mov_b32_e32 v3, v2 726; GFX9-NEXT: global_load_dwordx3 v[0:2], v[0:1], off 727; GFX9-NEXT: global_load_dword v3, v[3:4], off 728; GFX9-NEXT: s_waitcnt vmcnt(1) 729; GFX9-NEXT: v_mov_b32_e32 v0, v2 730; GFX9-NEXT: s_waitcnt vmcnt(0) 731; GFX9-NEXT: v_mov_b32_e32 v2, v3 732; GFX9-NEXT: s_setpc_b64 s[30:31] 733 %val0 = load <6 x half>, <6 x half> addrspace(1)* %arg0 734 %val1 = load <6 x half>, <6 x half> addrspace(1)* %arg1 735 %shuffle = shufflevector <6 x half> %val0, <6 x half> %val1, <6 x i32> <i32 4, i32 5, i32 2, i32 3, i32 6, i32 7> 736 ret <6 x half> %shuffle 737} 738 739define amdgpu_kernel void @fma_shuffle(<4 x half> addrspace(1)* nocapture readonly %A, <4 x half> addrspace(1)* nocapture readonly %B, <4 x half> addrspace(1)* nocapture %C) { 740; GFX9-LABEL: fma_shuffle: 741; GFX9: ; %bb.0: ; %entry 742; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 743; GFX9-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x10 744; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0 745; GFX9-NEXT: s_waitcnt lgkmcnt(0) 746; GFX9-NEXT: v_mov_b32_e32 v1, s1 747; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v4 748; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc 749; GFX9-NEXT: v_mov_b32_e32 v3, s3 750; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s2, v4 751; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc 752; GFX9-NEXT: v_mov_b32_e32 v5, s5 753; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, s4, v4 754; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc 755; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 756; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off 757; GFX9-NEXT: global_load_dwordx2 v[6:7], v[4:5], off 758; GFX9-NEXT: s_waitcnt vmcnt(0) 759; GFX9-NEXT: v_pk_fma_f16 v6, v0, v2, v6 op_sel_hi:[0,1,1] 760; GFX9-NEXT: v_pk_fma_f16 v2, v1, v2, v7 op_sel_hi:[0,1,1] 761; GFX9-NEXT: v_pk_fma_f16 v0, v0, v3, v6 op_sel:[1,0,0] 762; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0] 763; GFX9-NEXT: global_store_dwordx2 v[4:5], v[0:1], off 764; GFX9-NEXT: s_endpgm 765entry: 766 %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x() 767 %tmp12 = zext i32 %tmp1 to i64 768 %arrayidx = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %A, i64 %tmp12 769 %tmp14 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx, align 8 770 %arrayidx1 = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %B, i64 %tmp12 771 %tmp15 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx1, align 8 772 %arrayidx2 = getelementptr inbounds <4 x half>, <4 x half> addrspace(1)* %C, i64 %tmp12 773 %tmp16 = load <4 x half>, <4 x half> addrspace(1)* %arrayidx2, align 8 774 %tmp17 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> zeroinitializer 775 %tmp18 = shufflevector <4 x half> %tmp15, <4 x half> undef, <2 x i32> <i32 0, i32 1> 776 %tmp19 = shufflevector <4 x half> %tmp16, <4 x half> undef, <2 x i32> <i32 0, i32 1> 777 %tmp20 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp17, <2 x half> %tmp18, <2 x half> %tmp19) 778 %tmp21 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 1, i32 1> 779 %tmp22 = shufflevector <4 x half> %tmp15, <4 x half> undef, <2 x i32> <i32 2, i32 3> 780 %tmp23 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp21, <2 x half> %tmp22, <2 x half> %tmp20) 781 %tmp24 = shufflevector <2 x half> %tmp23, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> 782 %tmp25 = shufflevector <4 x half> %tmp24, <4 x half> %tmp16, <4 x i32> <i32 0, i32 1, i32 6, i32 7> 783 %tmp26 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 2, i32 2> 784 %tmp27 = shufflevector <4 x half> %tmp25, <4 x half> undef, <2 x i32> <i32 2, i32 3> 785 %tmp28 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp26, <2 x half> %tmp18, <2 x half> %tmp27) 786 %tmp29 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 3, i32 3> 787 %tmp30 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp29, <2 x half> %tmp22, <2 x half> %tmp28) 788 %tmp31 = shufflevector <2 x half> %tmp30, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> 789 %tmp32 = shufflevector <4 x half> %tmp25, <4 x half> %tmp31, <4 x i32> <i32 0, i32 1, i32 4, i32 5> 790 store <4 x half> %tmp32, <4 x half> addrspace(1)* %arrayidx2, align 8 791 ret void 792} 793 794define <4 x half> @shuffle_v4f16_0456(<4 x half> addrspace(1)* %arg0, <4 x half> addrspace(1)* %arg1) { 795; GFX9-LABEL: shuffle_v4f16_0456: 796; GFX9: ; %bb.0: 797; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 798; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off 799; GFX9-NEXT: s_waitcnt vmcnt(0) 800; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off 801; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff 802; GFX9-NEXT: v_and_b32_e32 v0, v3, v0 803; GFX9-NEXT: s_waitcnt vmcnt(0) 804; GFX9-NEXT: v_and_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 805; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0 806; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v3 807; GFX9-NEXT: s_setpc_b64 s[30:31] 808 %val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0 809 %val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1 810 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 0, i32 4, i32 5, i32 6> 811 ret <4 x half> %shuffle 812} 813 814define amdgpu_kernel void @shuffle_scalar_load_v8i32_0123(<8 x i32> addrspace(4)* %in, <4 x i32> addrspace(1)* %out) { 815; GFX9-LABEL: shuffle_scalar_load_v8i32_0123: 816; GFX9: ; %bb.0: 817; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 818; GFX9-NEXT: s_waitcnt lgkmcnt(0) 819; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 820; GFX9-NEXT: v_mov_b32_e32 v4, s2 821; GFX9-NEXT: v_mov_b32_e32 v5, s3 822; GFX9-NEXT: s_waitcnt lgkmcnt(0) 823; GFX9-NEXT: v_mov_b32_e32 v0, s4 824; GFX9-NEXT: v_mov_b32_e32 v1, s5 825; GFX9-NEXT: v_mov_b32_e32 v2, s6 826; GFX9-NEXT: v_mov_b32_e32 v3, s7 827; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off 828; GFX9-NEXT: s_endpgm 829 %ld8 = load <8 x i32>, <8 x i32> addrspace(4)* %in, align 16 830 %id = shufflevector <8 x i32> %ld8, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3> 831 store <4 x i32> %id, <4 x i32> addrspace(1)* %out, align 8 832 ret void 833} 834 835declare <2 x half> @llvm.fma.v2f16(<2 x half>, <2 x half>, <2 x half>) #0 836declare i32 @llvm.amdgcn.workitem.id.x() #0 837 838attributes #0 = { nounwind readnone speculatable } 839