1; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,MOVREL,PREGFX9 %s 2; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,MOVREL,PREGFX9 %s 3; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-vgpr-index-mode -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,IDXMODE,PREGFX9 %s 4; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,IDXMODE,GFX9 %s 5 6; Tests for indirect addressing on SI, which is implemented using dynamic 7; indexing of vectors. 8 9; GCN-LABEL: {{^}}extract_w_offset: 10; GCN-DAG: s_load_dword [[IN0:s[0-9]+]] 11; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 4.0 12; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x40400000 13; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 2.0 14; GCN-DAG: v_mov_b32_e32 [[BASEREG:v[0-9]+]], 1.0 15; GCN-DAG: s_add_i32 [[IN:s[0-9]+]], [[IN0]], 1 16 17; MOVREL-DAG: s_mov_b32 m0, [[IN]] 18; MOVREL: v_movrels_b32_e32 v{{[0-9]+}}, [[BASEREG]] 19 20; IDXMODE: s_set_gpr_idx_on [[IN]], src0{{$}} 21; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, [[BASEREG]] 22; IDXMODE-NEXT: s_set_gpr_idx_off 23define amdgpu_kernel void @extract_w_offset(float addrspace(1)* %out, i32 %in) { 24entry: 25 %idx = add i32 %in, 1 26 %elt = extractelement <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, i32 %idx 27 store float %elt, float addrspace(1)* %out 28 ret void 29} 30 31; XXX: Could do v_or_b32 directly 32; GCN-LABEL: {{^}}extract_w_offset_salu_use_vector: 33; GCN-DAG: s_or_b32 34; GCN-DAG: s_or_b32 35; GCN-DAG: s_or_b32 36; GCN-DAG: s_or_b32 37; MOVREL: s_mov_b32 m0 38; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} 39; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} 40; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} 41; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} 42 43 44; MOVREL: v_movrels_b32_e32 45 46; IDXMODE: s_set_gpr_idx_on s{{[0-9]+}}, src0{{$}} 47; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 48; IDXMODE-NEXT: s_set_gpr_idx_off 49define amdgpu_kernel void @extract_w_offset_salu_use_vector(i32 addrspace(1)* %out, i32 %in, <16 x i32> %or.val) { 50entry: 51 %idx = add i32 %in, 1 52 %vec = or <16 x i32> %or.val, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16> 53 %elt = extractelement <16 x i32> %vec, i32 %idx 54 store i32 %elt, i32 addrspace(1)* %out 55 ret void 56} 57 58; GCN-LABEL: {{^}}extract_wo_offset: 59; GCN-DAG: s_load_dword [[IN:s[0-9]+]] 60; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 4.0 61; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x40400000 62; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 2.0 63; GCN-DAG: v_mov_b32_e32 [[BASEREG:v[0-9]+]], 1.0 64 65; MOVREL-DAG: s_mov_b32 m0, [[IN]] 66; MOVREL: v_movrels_b32_e32 v{{[0-9]+}}, [[BASEREG]] 67 68; IDXMODE: s_set_gpr_idx_on [[IN]], src0{{$}} 69; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, [[BASEREG]] 70; IDXMODE-NEXT: s_set_gpr_idx_off 71define amdgpu_kernel void @extract_wo_offset(float addrspace(1)* %out, i32 %in) { 72entry: 73 %elt = extractelement <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, i32 %in 74 store float %elt, float addrspace(1)* %out 75 ret void 76} 77 78; GCN-LABEL: {{^}}extract_neg_offset_sgpr: 79; The offset depends on the register that holds the first element of the vector. 80; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} 81; MOVREL: v_movrels_b32_e32 v{{[0-9]}}, v0 82 83; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}} 84; IDXMODE: v_mov_b32_e32 v14, 15 85; IDXMODE: v_mov_b32_e32 v15, 16 86; IDXMODE-NEXT: s_set_gpr_idx_on [[ADD_IDX]], src0{{$}} 87; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 88; IDXMODE-NEXT: s_set_gpr_idx_off 89define amdgpu_kernel void @extract_neg_offset_sgpr(i32 addrspace(1)* %out, i32 %offset) { 90entry: 91 %index = add i32 %offset, -512 92 %value = extractelement <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>, i32 %index 93 store i32 %value, i32 addrspace(1)* %out 94 ret void 95} 96 97; GCN-LABEL: {{^}}extract_neg_offset_sgpr_loaded: 98; The offset depends on the register that holds the first element of the vector. 99; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} 100; MOVREL: v_movrels_b32_e32 v{{[0-9]}}, v0 101 102; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}} 103; IDXMODE: v_mov_b32_e32 v0, 104; IDXMODE: v_mov_b32_e32 v1, 105; IDXMODE: v_mov_b32_e32 v2, 106; IDXMODE: v_mov_b32_e32 v3, 107; IDXMODE: v_mov_b32_e32 v4, 108; IDXMODE: v_mov_b32_e32 v5, 109; IDXMODE: v_mov_b32_e32 v6, 110; IDXMODE: v_mov_b32_e32 v7, 111; IDXMODE: v_mov_b32_e32 v8, 112; IDXMODE: v_mov_b32_e32 v9, 113; IDXMODE: v_mov_b32_e32 v10, 114; IDXMODE: v_mov_b32_e32 v11, 115; IDXMODE: v_mov_b32_e32 v12, 116; IDXMODE: v_mov_b32_e32 v13, 117; IDXMODE: v_mov_b32_e32 v14, 118; IDXMODE: v_mov_b32_e32 v15, 119; IDXMODE-NEXT: s_set_gpr_idx_on [[ADD_IDX]], src0{{$}} 120; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 121; IDXMODE-NEXT: s_set_gpr_idx_off 122define amdgpu_kernel void @extract_neg_offset_sgpr_loaded(i32 addrspace(1)* %out, <16 x i32> %vec0, <16 x i32> %vec1, i32 %offset) { 123entry: 124 %index = add i32 %offset, -512 125 %or = or <16 x i32> %vec0, %vec1 126 %value = extractelement <16 x i32> %or, i32 %index 127 store i32 %value, i32 addrspace(1)* %out 128 ret void 129} 130 131; GCN-LABEL: {{^}}extract_neg_offset_vgpr: 132; The offset depends on the register that holds the first element of the vector. 133 134; FIXME: The waitcnt for the argument load can go after the loop 135; GCN: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec 136; GCN: [[LOOPBB:BB[0-9]+_[0-9]+]]: 137; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v{{[0-9]+}} 138; GCN: s_and_saveexec_b64 vcc, vcc 139 140; MOVREL: s_add_i32 m0, [[READLANE]], 0xfffffe0 141; MOVREL: v_movrels_b32_e32 [[RESULT:v[0-9]+]], v1 142 143; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00 144; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], src0 145; IDXMODE: v_mov_b32_e32 [[RESULT:v[0-9]+]], v1 146; IDXMODE: s_set_gpr_idx_off 147 148; GCN: s_cbranch_execnz 149 150; GCN: buffer_store_dword [[RESULT]] 151define amdgpu_kernel void @extract_neg_offset_vgpr(i32 addrspace(1)* %out) { 152entry: 153 %id = call i32 @llvm.amdgcn.workitem.id.x() #1 154 %index = add i32 %id, -512 155 %value = extractelement <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>, i32 %index 156 store i32 %value, i32 addrspace(1)* %out 157 ret void 158} 159 160; GCN-LABEL: {{^}}extract_undef_offset_sgpr: 161; undefined behavior, but shouldn't crash compiler 162define amdgpu_kernel void @extract_undef_offset_sgpr(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { 163entry: 164 %ld = load volatile <4 x i32>, <4 x i32> addrspace(1)* %in 165 %value = extractelement <4 x i32> %ld, i32 undef 166 store i32 %value, i32 addrspace(1)* %out 167 ret void 168} 169 170; GCN-LABEL: {{^}}insert_undef_offset_sgpr_vector_src: 171; undefined behavior, but shouldn't crash compiler 172define amdgpu_kernel void @insert_undef_offset_sgpr_vector_src(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { 173entry: 174 %ld = load <4 x i32>, <4 x i32> addrspace(1)* %in 175 %value = insertelement <4 x i32> %ld, i32 5, i32 undef 176 store <4 x i32> %value, <4 x i32> addrspace(1)* %out 177 ret void 178} 179 180; GCN-LABEL: {{^}}insert_w_offset: 181; GCN-DAG: s_load_dword [[IN0:s[0-9]+]] 182; MOVREL-DAG: s_add_i32 [[IN:s[0-9]+]], [[IN0]], 1 183; MOVREL-DAG: s_mov_b32 m0, [[IN]] 184; GCN-DAG: v_mov_b32_e32 v[[ELT0:[0-9]+]], 1.0 185; GCN-DAG: v_mov_b32_e32 v[[ELT1:[0-9]+]], 2.0 186; GCN-DAG: v_mov_b32_e32 v[[ELT2:[0-9]+]], 0x40400000 187; GCN-DAG: v_mov_b32_e32 v[[ELT3:[0-9]+]], 4.0 188; GCN-DAG: v_mov_b32_e32 v[[ELT15:[0-9]+]], 0x41800000 189; GCN-DAG: v_mov_b32_e32 v[[INS:[0-9]+]], 0x41880000 190 191; MOVREL: v_movreld_b32_e32 v[[ELT0]], v[[INS]] 192; MOVREL: buffer_store_dwordx4 v{{\[}}[[ELT0]]:[[ELT3]]{{\]}} 193define amdgpu_kernel void @insert_w_offset(<16 x float> addrspace(1)* %out, i32 %in) { 194entry: 195 %add = add i32 %in, 1 196 %ins = insertelement <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, float 17.0, i32 %add 197 store <16 x float> %ins, <16 x float> addrspace(1)* %out 198 ret void 199} 200 201; GCN-LABEL: {{^}}insert_unsigned_base_plus_offset: 202; GCN-DAG: s_load_dword [[IN:s[0-9]+]] 203; GCN-DAG: v_mov_b32_e32 [[ELT0:v[0-9]+]], 1.0 204; GCN-DAG: v_mov_b32_e32 [[ELT1:v[0-9]+]], 2.0 205; GCN-DAG: s_and_b32 [[BASE:s[0-9]+]], [[IN]], 0xffff 206 207; MOVREL: s_mov_b32 m0, [[BASE]] 208; MOVREL: v_movreld_b32_e32 [[ELT1]], v{{[0-9]+}} 209 210; IDXMODE: s_set_gpr_idx_on [[BASE]], dst 211; IDXMODE-NEXT: v_mov_b32_e32 [[ELT1]], v{{[0-9]+}} 212; IDXMODE-NEXT: s_set_gpr_idx_off 213define amdgpu_kernel void @insert_unsigned_base_plus_offset(<16 x float> addrspace(1)* %out, i16 %in) { 214entry: 215 %base = zext i16 %in to i32 216 %add = add i32 %base, 1 217 %ins = insertelement <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, float 17.0, i32 %add 218 store <16 x float> %ins, <16 x float> addrspace(1)* %out 219 ret void 220} 221 222; GCN-LABEL: {{^}}insert_signed_base_plus_offset: 223; GCN-DAG: s_load_dword [[IN:s[0-9]+]] 224; GCN-DAG: v_mov_b32_e32 [[ELT0:v[0-9]+]], 1.0 225; GCN-DAG: v_mov_b32_e32 [[ELT1:v[0-9]+]], 2.0 226 227; GCN-DAG: s_sext_i32_i16 [[BASE:s[0-9]+]], [[IN]] 228; GCN-DAG: s_add_i32 [[BASE_PLUS_OFFSET:s[0-9]+]], [[BASE]], 1 229 230; MOVREL: s_mov_b32 m0, [[BASE_PLUS_OFFSET]] 231; MOVREL: v_movreld_b32_e32 [[ELT0]], v{{[0-9]+}} 232 233; IDXMODE: s_set_gpr_idx_on [[BASE_PLUS_OFFSET]], dst 234; IDXMODE-NEXT: v_mov_b32_e32 [[ELT0]], v{{[0-9]+}} 235; IDXMODE-NEXT: s_set_gpr_idx_off 236define amdgpu_kernel void @insert_signed_base_plus_offset(<16 x float> addrspace(1)* %out, i16 %in) { 237entry: 238 %base = sext i16 %in to i32 239 %add = add i32 %base, 1 240 %ins = insertelement <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, float 17.0, i32 %add 241 store <16 x float> %ins, <16 x float> addrspace(1)* %out 242 ret void 243} 244 245 246; GCN-LABEL: {{^}}insert_wo_offset: 247; GCN: s_load_dword [[IN:s[0-9]+]] 248 249; MOVREL: s_mov_b32 m0, [[IN]] 250; MOVREL: v_movreld_b32_e32 v[[ELT0:[0-9]+]] 251 252; IDXMODE: s_set_gpr_idx_on [[IN]], dst 253; IDXMODE-NEXT: v_mov_b32_e32 v[[ELT0:[0-9]+]], v{{[0-9]+}} 254; IDXMODE-NEXT: s_set_gpr_idx_off 255 256; GCN: buffer_store_dwordx4 v{{\[}}[[ELT0]]: 257define amdgpu_kernel void @insert_wo_offset(<16 x float> addrspace(1)* %out, i32 %in) { 258entry: 259 %ins = insertelement <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, float 17.0, i32 %in 260 store <16 x float> %ins, <16 x float> addrspace(1)* %out 261 ret void 262} 263 264; GCN-LABEL: {{^}}insert_neg_offset_sgpr: 265; The offset depends on the register that holds the first element of the vector. 266; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} 267; MOVREL: v_movreld_b32_e32 v0, 16 268 269; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}} 270; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], dst 271; IDXMODE-NEXT: v_mov_b32_e32 v0, 16 272; IDXMODE-NEXT: s_set_gpr_idx_off 273define amdgpu_kernel void @insert_neg_offset_sgpr(i32 addrspace(1)* %in, <16 x i32> addrspace(1)* %out, i32 %offset) { 274entry: 275 %index = add i32 %offset, -512 276 %value = insertelement <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>, i32 16, i32 %index 277 store <16 x i32> %value, <16 x i32> addrspace(1)* %out 278 ret void 279} 280 281; The vector indexed into is originally loaded into an SGPR rather 282; than built with a reg_sequence 283 284; GCN-LABEL: {{^}}insert_neg_offset_sgpr_loadreg: 285; The offset depends on the register that holds the first element of the vector. 286; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} 287; MOVREL: v_movreld_b32_e32 v0, 5 288 289; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}} 290; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], dst 291; IDXMODE-NEXT: v_mov_b32_e32 v0, 5 292; IDXMODE-NEXT: s_set_gpr_idx_off 293define amdgpu_kernel void @insert_neg_offset_sgpr_loadreg(i32 addrspace(1)* %in, <16 x i32> addrspace(1)* %out, <16 x i32> %vec, i32 %offset) { 294entry: 295 %index = add i32 %offset, -512 296 %value = insertelement <16 x i32> %vec, i32 5, i32 %index 297 store <16 x i32> %value, <16 x i32> addrspace(1)* %out 298 ret void 299} 300 301; GCN-LABEL: {{^}}insert_neg_offset_vgpr: 302; The offset depends on the register that holds the first element of the vector. 303 304; GCN-DAG: v_mov_b32_e32 [[VEC_ELT0:v[0-9]+]], 1{{$}} 305; GCN-DAG: v_mov_b32_e32 [[VEC_ELT1:v[0-9]+]], 2{{$}} 306; GCN-DAG: v_mov_b32_e32 [[VEC_ELT2:v[0-9]+]], 3{{$}} 307; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 4{{$}} 308; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 5{{$}} 309; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 6{{$}} 310; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 7{{$}} 311; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 8{{$}} 312; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 9{{$}} 313; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 10{{$}} 314; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 11{{$}} 315; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 12{{$}} 316; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 13{{$}} 317; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 14{{$}} 318; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 15{{$}} 319; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 16{{$}} 320 321; GCN: s_mov_b64 [[SAVEEXEC:s\[[0-9]+:[0-9]+\]]], exec 322; GCN: [[LOOPBB:BB[0-9]+_[0-9]+]]: 323; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]] 324; GCN: s_and_saveexec_b64 vcc, vcc 325 326; MOVREL: s_add_i32 m0, [[READLANE]], 0xfffffe00 327; MOVREL: v_movreld_b32_e32 [[VEC_ELT0]], 33 328 329; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}} 330; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], dst 331; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, 33 332; IDXMODE: s_set_gpr_idx_off 333 334; GCN: s_cbranch_execnz [[LOOPBB]] 335; GCN: s_mov_b64 exec, [[SAVEEXEC]] 336 337; GCN: buffer_store_dword 338define amdgpu_kernel void @insert_neg_offset_vgpr(i32 addrspace(1)* %in, <16 x i32> addrspace(1)* %out) { 339entry: 340 %id = call i32 @llvm.amdgcn.workitem.id.x() #1 341 %index = add i32 %id, -512 342 %value = insertelement <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>, i32 33, i32 %index 343 store <16 x i32> %value, <16 x i32> addrspace(1)* %out 344 ret void 345} 346 347; GCN-LABEL: {{^}}insert_neg_inline_offset_vgpr: 348 349; GCN-DAG: v_mov_b32_e32 [[VEC_ELT0:v[0-9]+]], 1{{$}} 350; GCN-DAG: v_mov_b32_e32 [[VEC_ELT1:v[0-9]+]], 2{{$}} 351; GCN-DAG: v_mov_b32_e32 [[VEC_ELT2:v[0-9]+]], 3{{$}} 352; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 4{{$}} 353; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 5{{$}} 354; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 6{{$}} 355; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 7{{$}} 356; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 8{{$}} 357; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 9{{$}} 358; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 10{{$}} 359; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 11{{$}} 360; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 12{{$}} 361; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 13{{$}} 362; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 14{{$}} 363; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 15{{$}} 364; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 16{{$}} 365; GCN-DAG: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x1f4{{$}} 366 367; GCN: s_mov_b64 [[SAVEEXEC:s\[[0-9]+:[0-9]+\]]], exec 368 369; The offset depends on the register that holds the first element of the vector. 370; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]] 371 372; MOVREL: s_add_i32 m0, [[READLANE]], -16 373; MOVREL: v_movreld_b32_e32 [[VEC_ELT0]], [[VAL]] 374 375; IDXMODE: s_add_i32 [[ADD_IDX:s[0-9]+]], [[READLANE]], -16 376; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], dst 377; IDXMODE: v_mov_b32_e32 [[VEC_ELT0]], [[VAL]] 378; IDXMODE: s_set_gpr_idx_off 379 380; GCN: s_cbranch_execnz 381define amdgpu_kernel void @insert_neg_inline_offset_vgpr(i32 addrspace(1)* %in, <16 x i32> addrspace(1)* %out) { 382entry: 383 %id = call i32 @llvm.amdgcn.workitem.id.x() #1 384 %index = add i32 %id, -16 385 %value = insertelement <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>, i32 500, i32 %index 386 store <16 x i32> %value, <16 x i32> addrspace(1)* %out 387 ret void 388} 389 390; When the block is split to insert the loop, make sure any other 391; places that need to be expanded in the same block are also handled. 392 393; GCN-LABEL: {{^}}extract_vgpr_offset_multiple_in_block: 394 395; FIXME: Why is vector copied in between? 396 397; GCN-DAG: {{buffer|flat|global}}_load_dword [[IDX0:v[0-9]+]] 398; GCN-DAG: s_mov_b32 [[S_ELT1:s[0-9]+]], 9 399; GCN-DAG: s_mov_b32 [[S_ELT0:s[0-9]+]], 7 400; GCN-DAG: v_mov_b32_e32 [[VEC_ELT0:v[0-9]+]], [[S_ELT0]] 401; GCN-DAG: v_mov_b32_e32 [[VEC_ELT1:v[0-9]+]], [[S_ELT1]] 402 403; GCN: s_mov_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec 404 405; GCN: s_waitcnt vmcnt(0) 406; PREGFX9: v_add_{{i32|u32}}_e32 [[IDX1:v[0-9]+]], vcc, 1, [[IDX0]] 407; GFX9: v_add_{{i32|u32}}_e32 [[IDX1:v[0-9]+]], 1, [[IDX0]] 408 409 410; GCN: [[LOOP0:BB[0-9]+_[0-9]+]]: 411; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]] 412; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]] 413; GCN: s_and_saveexec_b64 vcc, vcc 414 415; MOVREL: s_mov_b32 m0, [[READLANE]] 416; MOVREL: v_movrels_b32_e32 [[MOVREL0:v[0-9]+]], [[VEC_ELT0]] 417 418; IDXMODE: s_set_gpr_idx_on [[READLANE]], src0 419; IDXMODE: v_mov_b32_e32 [[MOVREL0:v[0-9]+]], [[VEC_ELT0]] 420; IDXMODE: s_set_gpr_idx_off 421 422; GCN-NEXT: s_xor_b64 exec, exec, vcc 423; GCN-NEXT: s_cbranch_execnz [[LOOP0]] 424 425; FIXME: Redundant copy 426; GCN: s_mov_b64 exec, [[MASK]] 427 428; GCN: v_mov_b32_e32 [[VEC_ELT0_2:v[0-9]+]], [[S_ELT0]] 429 430; GCN: s_mov_b64 [[MASK2:s\[[0-9]+:[0-9]+\]]], exec 431 432; GCN: [[LOOP1:BB[0-9]+_[0-9]+]]: 433; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX1]] 434; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX1]] 435; GCN: s_and_saveexec_b64 vcc, vcc 436 437; MOVREL: s_mov_b32 m0, [[READLANE]] 438; MOVREL-NEXT: v_movrels_b32_e32 [[MOVREL1:v[0-9]+]], [[VEC_ELT0_2]] 439 440; IDXMODE: s_set_gpr_idx_on [[READLANE]], src0 441; IDXMODE-NEXT: v_mov_b32_e32 [[MOVREL1:v[0-9]+]], [[VEC_ELT0_2]] 442; IDXMODE: s_set_gpr_idx_off 443 444; GCN-NEXT: s_xor_b64 exec, exec, vcc 445; GCN: s_cbranch_execnz [[LOOP1]] 446 447; GCN: buffer_store_dword [[MOVREL0]] 448; GCN: buffer_store_dword [[MOVREL1]] 449define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(i32 addrspace(1)* %out0, i32 addrspace(1)* %out1, i32 addrspace(1)* %in) #0 { 450entry: 451 %id = call i32 @llvm.amdgcn.workitem.id.x() #1 452 %id.ext = zext i32 %id to i64 453 %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %id.ext 454 %idx0 = load volatile i32, i32 addrspace(1)* %gep 455 %idx1 = add i32 %idx0, 1 456 %val0 = extractelement <16 x i32> <i32 7, i32 9, i32 11, i32 13, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>, i32 %idx0 457 %live.out.reg = call i32 asm sideeffect "s_mov_b32 $0, 17", "={s4}" () 458 %val1 = extractelement <16 x i32> <i32 7, i32 9, i32 11, i32 13, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>, i32 %idx1 459 store volatile i32 %val0, i32 addrspace(1)* %out0 460 store volatile i32 %val1, i32 addrspace(1)* %out0 461 %cmp = icmp eq i32 %id, 0 462 br i1 %cmp, label %bb1, label %bb2 463 464bb1: 465 store volatile i32 %live.out.reg, i32 addrspace(1)* undef 466 br label %bb2 467 468bb2: 469 ret void 470} 471 472; Moved subtest for insert_vgpr_offset_multiple_in_block to separate file to 473; avoid very different schedule induced isses with gfx9. 474; test/CodeGen/AMDGPU/indirect-addressing-si-pregfx9.ll 475 476 477; GCN-LABEL: {{^}}insert_adjacent_blocks: 478define amdgpu_kernel void @insert_adjacent_blocks(i32 %arg, float %val0) #0 { 479bb: 480 %tmp = icmp eq i32 %arg, 0 481 br i1 %tmp, label %bb1, label %bb4 482 483bb1: ; preds = %bb 484 %tmp2 = load volatile <4 x float>, <4 x float> addrspace(1)* undef 485 %tmp3 = insertelement <4 x float> %tmp2, float %val0, i32 undef 486 call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp3) #0 ; Prevent block optimize out 487 br label %bb7 488 489bb4: ; preds = %bb 490 %tmp5 = load volatile <4 x float>, <4 x float> addrspace(1)* undef 491 %tmp6 = insertelement <4 x float> %tmp5, float %val0, i32 undef 492 call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp6) #0 ; Prevent block optimize out 493 br label %bb7 494 495bb7: ; preds = %bb4, %bb1 496 %tmp8 = phi <4 x float> [ %tmp3, %bb1 ], [ %tmp6, %bb4 ] 497 store volatile <4 x float> %tmp8, <4 x float> addrspace(1)* undef 498 ret void 499} 500 501; FIXME: Should be able to fold zero input to movreld to inline imm? 502 503; GCN-LABEL: {{^}}multi_same_block: 504 505; GCN: s_load_dword [[ARG:s[0-9]+]] 506 507; MOVREL: v_mov_b32_e32 v{{[0-9]+}}, 0x41900000 508; MOVREL: s_waitcnt 509; MOVREL: s_add_i32 m0, [[ARG]], -16 510; MOVREL: v_movreld_b32_e32 v{{[0-9]+}}, 4.0 511; MOVREL: v_mov_b32_e32 v{{[0-9]+}}, 0x41b0cccd 512; MOVREL: v_movreld_b32_e32 v{{[0-9]+}}, -4.0 513; MOVREL: s_mov_b32 m0, -1 514 515 516; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, 0x41900000 517; IDXMODE: s_waitcnt 518; IDXMODE: s_add_i32 [[ARG]], [[ARG]], -16 519; IDXMODE: s_set_gpr_idx_on [[ARG]], dst 520; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, 4.0 521; IDXMODE: s_set_gpr_idx_off 522; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, 0x41b0cccd 523; IDXMODE: s_set_gpr_idx_on [[ARG]], dst 524; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, -4.0 525; IDXMODE: s_set_gpr_idx_off 526 527; GCN: ds_write_b32 528; GCN: ds_write_b32 529; GCN: s_endpgm 530define amdgpu_kernel void @multi_same_block(i32 %arg) #0 { 531bb: 532 %tmp1 = add i32 %arg, -16 533 %tmp2 = insertelement <9 x float> <float 1.700000e+01, float 1.800000e+01, float 1.900000e+01, float 2.000000e+01, float 2.100000e+01, float 2.200000e+01, float 2.300000e+01, float 2.400000e+01, float 2.500000e+01>, float 4.000000e+00, i32 %tmp1 534 %tmp3 = add i32 %arg, -16 535 %tmp4 = insertelement <9 x float> <float 0x40311999A0000000, float 0x40321999A0000000, float 0x40331999A0000000, float 0x40341999A0000000, float 0x40351999A0000000, float 0x40361999A0000000, float 0x40371999A0000000, float 0x40381999A0000000, float 0x40391999A0000000>, float -4.0, i32 %tmp3 536 %tmp5 = bitcast <9 x float> %tmp2 to <9 x i32> 537 %tmp6 = extractelement <9 x i32> %tmp5, i32 1 538 %tmp7 = bitcast <9 x float> %tmp4 to <9 x i32> 539 %tmp8 = extractelement <9 x i32> %tmp7, i32 5 540 store volatile i32 %tmp6, i32 addrspace(3)* undef, align 4 541 store volatile i32 %tmp8, i32 addrspace(3)* undef, align 4 542 ret void 543} 544 545; offset puts outside of superegister bounaries, so clamp to 1st element. 546; GCN-LABEL: {{^}}extract_largest_inbounds_offset: 547; GCN-DAG: buffer_load_dwordx4 v{{\[}}[[LO_ELT:[0-9]+]]:[[HI_ELT:[0-9]+]] 548; GCN-DAG: s_load_dword [[IDX0:s[0-9]+]] 549; GCN-DAG: s_add_i32 [[IDX:s[0-9]+]], [[IDX0]], 15 550 551; MOVREL: s_mov_b32 m0, [[IDX]] 552; MOVREL: v_movrels_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]] 553 554; IDXMODE: s_set_gpr_idx_on [[IDX]], src0 555; IDXMODE: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]] 556; IDXMODE: s_set_gpr_idx_off 557 558; GCN: buffer_store_dword [[EXTRACT]] 559define amdgpu_kernel void @extract_largest_inbounds_offset(i32 addrspace(1)* %out, <16 x i32> addrspace(1)* %in, i32 %idx) { 560entry: 561 %ld = load volatile <16 x i32>, <16 x i32> addrspace(1)* %in 562 %offset = add i32 %idx, 15 563 %value = extractelement <16 x i32> %ld, i32 %offset 564 store i32 %value, i32 addrspace(1)* %out 565 ret void 566} 567 568; GCN-LABEL: {{^}}extract_out_of_bounds_offset: 569; GCN-DAG: buffer_load_dwordx4 v{{\[}}[[LO_ELT:[0-9]+]]:[[HI_ELT:[0-9]+]]{{\]}} 570; GCN-DAG: s_load_dword [[IDX:s[0-9]+]] 571; GCN: s_add_i32 [[ADD_IDX:s[0-9]+]], [[IDX]], 16 572 573; MOVREL: s_mov_b32 m0, [[ADD_IDX]] 574; MOVREL: v_movrels_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]] 575 576; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], src0 577; IDXMODE: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]] 578; IDXMODE: s_set_gpr_idx_off 579 580; GCN: buffer_store_dword [[EXTRACT]] 581define amdgpu_kernel void @extract_out_of_bounds_offset(i32 addrspace(1)* %out, <16 x i32> addrspace(1)* %in, i32 %idx) { 582entry: 583 %ld = load volatile <16 x i32>, <16 x i32> addrspace(1)* %in 584 %offset = add i32 %idx, 16 585 %value = extractelement <16 x i32> %ld, i32 %offset 586 store i32 %value, i32 addrspace(1)* %out 587 ret void 588} 589 590; GCN-LABEL: {{^}}extractelement_v16i32_or_index: 591; GCN: s_load_dword [[IDX_IN:s[0-9]+]] 592; GCN: s_lshl_b32 [[IDX_SHL:s[0-9]+]], [[IDX_IN]] 593; GCN: s_or_b32 [[IDX_FIN:s[0-9]+]], [[IDX_SHL]], 1 594 595; MOVREL: s_mov_b32 m0, [[IDX_FIN]] 596; MOVREL: v_movrels_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 597 598; IDXMODE: s_set_gpr_idx_on [[IDX_FIN]], src0 599; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 600; IDXMODE: s_set_gpr_idx_off 601define amdgpu_kernel void @extractelement_v16i32_or_index(i32 addrspace(1)* %out, <16 x i32> addrspace(1)* %in, i32 %idx.in) { 602entry: 603 %ld = load volatile <16 x i32>, <16 x i32> addrspace(1)* %in 604 %idx.shl = shl i32 %idx.in, 2 605 %idx = or i32 %idx.shl, 1 606 %value = extractelement <16 x i32> %ld, i32 %idx 607 store i32 %value, i32 addrspace(1)* %out 608 ret void 609} 610 611; GCN-LABEL: {{^}}insertelement_v16f32_or_index: 612; GCN: s_load_dword [[IDX_IN:s[0-9]+]] 613; GCN: s_lshl_b32 [[IDX_SHL:s[0-9]+]], [[IDX_IN]] 614; GCN: s_or_b32 [[IDX_FIN:s[0-9]+]], [[IDX_SHL]], 1 615 616; MOVREL: s_mov_b32 m0, [[IDX_FIN]] 617; MOVREL: v_movreld_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 618 619; IDXMODE: s_set_gpr_idx_on [[IDX_FIN]], dst 620; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 621; IDXMODE: s_set_gpr_idx_off 622define amdgpu_kernel void @insertelement_v16f32_or_index(<16 x float> addrspace(1)* %out, <16 x float> %a, i32 %idx.in) nounwind { 623 %idx.shl = shl i32 %idx.in, 2 624 %idx = or i32 %idx.shl, 1 625 %vecins = insertelement <16 x float> %a, float 5.000000e+00, i32 %idx 626 store <16 x float> %vecins, <16 x float> addrspace(1)* %out, align 64 627 ret void 628} 629 630; GCN-LABEL: {{^}}broken_phi_bb: 631; GCN: v_mov_b32_e32 [[PHIREG:v[0-9]+]], 8 632 633; GCN: s_branch [[BB2:BB[0-9]+_[0-9]+]] 634 635; GCN: {{^BB[0-9]+_[0-9]+}}: 636; GCN: s_mov_b64 exec, 637 638; GCN: [[BB2]]: 639; GCN: v_cmp_le_i32_e32 vcc, s{{[0-9]+}}, [[PHIREG]] 640; GCN: buffer_load_dword 641 642; GCN: [[REGLOOP:BB[0-9]+_[0-9]+]]: 643; MOVREL: v_movreld_b32_e32 644 645; IDXMODE: s_set_gpr_idx_on 646; IDXMODE: v_mov_b32_e32 647; IDXMODE: s_set_gpr_idx_off 648 649; GCN: s_cbranch_execnz [[REGLOOP]] 650define amdgpu_kernel void @broken_phi_bb(i32 %arg, i32 %arg1) #0 { 651bb: 652 br label %bb2 653 654bb2: ; preds = %bb4, %bb 655 %tmp = phi i32 [ 8, %bb ], [ %tmp7, %bb4 ] 656 %tmp3 = icmp slt i32 %tmp, %arg 657 br i1 %tmp3, label %bb4, label %bb8 658 659bb4: ; preds = %bb2 660 %vgpr = load volatile i32, i32 addrspace(1)* undef 661 %tmp5 = insertelement <16 x i32> undef, i32 undef, i32 %vgpr 662 %tmp6 = insertelement <16 x i32> %tmp5, i32 %arg1, i32 %vgpr 663 %tmp7 = extractelement <16 x i32> %tmp6, i32 0 664 br label %bb2 665 666bb8: ; preds = %bb2 667 ret void 668} 669 670declare i32 @llvm.amdgcn.workitem.id.x() #1 671declare void @llvm.amdgcn.s.barrier() #2 672 673attributes #0 = { nounwind } 674attributes #1 = { nounwind readnone } 675attributes #2 = { nounwind convergent } 676