1; RUN: llc -O0 -mtriple=amdgcn--amdhsa -march=amdgcn -amdgpu-spill-sgpr-to-vgpr=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=VMEM -check-prefix=GCN %s 2; RUN: llc -O0 -mtriple=amdgcn--amdhsa -march=amdgcn -amdgpu-spill-sgpr-to-vgpr=1 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=VGPR -check-prefix=GCN %s 3 4; Verify registers used for tracking exec mask changes when all 5; registers are spilled at the end of the block. The SGPR spill 6; placement relative to the exec modifications are important. 7 8; FIXME: This checks with SGPR to VGPR spilling disabled, but this may 9; not work correctly in cases where no workitems take a branch. 10 11 12; GCN-LABEL: {{^}}divergent_if_endif: 13; VGPR: workitem_private_segment_byte_size = 12{{$}} 14 15 16; GCN: {{^}}; %bb.0: 17; GCN: s_mov_b32 m0, -1 18; GCN: ds_read_b32 [[LOAD0:v[0-9]+]] 19 20; GCN: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], s{{[0-9]+}}, v0 21; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec 22; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, [[CMP0]] 23 24; Spill load 25; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], 0 offset:[[LOAD0_OFFSET:[0-9]+]] ; 4-byte Folded Spill 26 27; Spill saved exec 28; VGPR: v_writelane_b32 [[SPILL_VGPR:v[0-9]+]], s[[SAVEEXEC_LO]], [[SAVEEXEC_LO_LANE:[0-9]+]] 29; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[SAVEEXEC_HI]], [[SAVEEXEC_HI_LANE:[0-9]+]] 30 31; VMEM: v_writelane_b32 v[[V_SAVEEXEC:[0-9]+]], s[[SAVEEXEC_LO]], 0 32; VMEM: v_writelane_b32 v[[V_SAVEEXEC]], s[[SAVEEXEC_HI]], 1 33; VMEM: buffer_store_dword v[[V_SAVEEXEC]], off, s[0:3], 0 offset:20 ; 4-byte Folded Spill 34 35; GCN: s_mov_b64 exec, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}} 36 37; GCN: s_cbranch_execz [[ENDIF:BB[0-9]+_[0-9]+]] 38 39; GCN: ; %bb.{{[0-9]+}}: ; %if 40; GCN: s_mov_b32 m0, -1 41; GCN: ds_read_b32 [[LOAD1:v[0-9]+]] 42; GCN: buffer_load_dword [[RELOAD_LOAD0:v[0-9]+]], off, s[0:3], 0 offset:[[LOAD0_OFFSET]] ; 4-byte Folded Reload 43; GCN: s_waitcnt vmcnt(0) lgkmcnt(0) 44 45 46; Spill val register 47; GCN: v_add_i32_e32 [[VAL:v[0-9]+]], vcc, [[LOAD1]], [[RELOAD_LOAD0]] 48; GCN: buffer_store_dword [[VAL]], off, s[0:3], 0 offset:[[VAL_OFFSET:[0-9]+]] ; 4-byte Folded Spill 49 50; VMEM: [[ENDIF]]: 51 52; Reload and restore exec mask 53; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]] 54; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]] 55 56 57 58; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC:[0-9]+]], off, s[0:3], 0 offset:20 ; 4-byte Folded Reload 59; VMEM: s_waitcnt vmcnt(0) 60; VMEM: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[V_RELOAD_SAVEEXEC]], 0 61; VMEM: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[V_RELOAD_SAVEEXEC]], 1 62 63; GCN: s_or_b64 exec, exec, s{{\[}}[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]{{\]}} 64 65; Restore val 66; GCN: buffer_load_dword [[RELOAD_VAL:v[0-9]+]], off, s[0:3], 0 offset:[[VAL_OFFSET]] ; 4-byte Folded Reload 67 68; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RELOAD_VAL]] 69define amdgpu_kernel void @divergent_if_endif(i32 addrspace(1)* %out) #0 { 70entry: 71 %tid = call i32 @llvm.amdgcn.workitem.id.x() 72 %load0 = load volatile i32, i32 addrspace(3)* undef 73 %cmp0 = icmp eq i32 %tid, 0 74 br i1 %cmp0, label %if, label %endif 75 76if: 77 %load1 = load volatile i32, i32 addrspace(3)* undef 78 %val = add i32 %load0, %load1 79 br label %endif 80 81endif: 82 %tmp4 = phi i32 [ %val, %if ], [ 0, %entry ] 83 store i32 %tmp4, i32 addrspace(1)* %out 84 ret void 85} 86 87; GCN-LABEL: {{^}}divergent_loop: 88; VGPR: workitem_private_segment_byte_size = 16{{$}} 89 90; GCN: {{^}}; %bb.0: 91 92; GCN: s_mov_b32 m0, -1 93; GCN: ds_read_b32 [[LOAD0:v[0-9]+]] 94 95; GCN: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], s{{[0-9]+}}, v0 96 97; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec 98; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, [[CMP0]] 99 100; Spill load 101; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], 0 offset:[[LOAD0_OFFSET:[0-9]+]] ; 4-byte Folded Spill 102 103; Spill saved exec 104; VGPR: v_writelane_b32 [[SPILL_VGPR:v[0-9]+]], s[[SAVEEXEC_LO]], [[SAVEEXEC_LO_LANE:[0-9]+]] 105; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[SAVEEXEC_HI]], [[SAVEEXEC_HI_LANE:[0-9]+]] 106 107 108; VMEM: v_writelane_b32 v[[V_SAVEEXEC:[0-9]+]], s[[SAVEEXEC_LO]], 0 109; VMEM: v_writelane_b32 v[[V_SAVEEXEC]], s[[SAVEEXEC_HI]], 1 110; VMEM: buffer_store_dword v[[V_SAVEEXEC]], off, s[0:3], 0 offset:24 ; 4-byte Folded Spill 111 112; GCN: s_mov_b64 exec, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}} 113 114; GCN-NEXT: s_cbranch_execz [[END:BB[0-9]+_[0-9]+]] 115 116 117; GCN: [[LOOP:BB[0-9]+_[0-9]+]]: 118; GCN: buffer_load_dword v[[VAL_LOOP_RELOAD:[0-9]+]], off, s[0:3], 0 offset:[[LOAD0_OFFSET]] ; 4-byte Folded Reload 119; GCN: v_subrev_i32_e32 [[VAL_LOOP:v[0-9]+]], vcc, v{{[0-9]+}}, v[[VAL_LOOP_RELOAD]] 120; GCN: s_cmp_lg_u32 121; GCN: buffer_store_dword [[VAL_LOOP]], off, s[0:3], 0 offset:{{[0-9]+}} ; 4-byte Folded Spill 122; GCN-NEXT: s_cbranch_scc1 [[LOOP]] 123 124; GCN: buffer_store_dword [[VAL_LOOP]], off, s[0:3], 0 offset:[[VAL_SUB_OFFSET:[0-9]+]] ; 4-byte Folded Spill 125 126; GCN: [[END]]: 127; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]] 128; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]] 129 130; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC:[0-9]+]], off, s[0:3], 0 offset:24 ; 4-byte Folded Reload 131; VMEM: s_waitcnt vmcnt(0) 132; VMEM: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[V_RELOAD_SAVEEXEC]], 0 133; VMEM: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[V_RELOAD_SAVEEXEC]], 1 134 135; GCN: s_or_b64 exec, exec, s{{\[}}[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]{{\]}} 136; GCN: buffer_load_dword v[[VAL_END:[0-9]+]], off, s[0:3], 0 offset:[[VAL_SUB_OFFSET]] ; 4-byte Folded Reload 137 138; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[VAL_END]] 139define amdgpu_kernel void @divergent_loop(i32 addrspace(1)* %out) #0 { 140entry: 141 %tid = call i32 @llvm.amdgcn.workitem.id.x() 142 %load0 = load volatile i32, i32 addrspace(3)* undef 143 %cmp0 = icmp eq i32 %tid, 0 144 br i1 %cmp0, label %loop, label %end 145 146loop: 147 %i = phi i32 [ %i.inc, %loop ], [ 0, %entry ] 148 %val = phi i32 [ %val.sub, %loop ], [ %load0, %entry ] 149 %load1 = load volatile i32, i32 addrspace(3)* undef 150 %i.inc = add i32 %i, 1 151 %val.sub = sub i32 %val, %load1 152 %cmp1 = icmp ne i32 %i, 256 153 br i1 %cmp1, label %loop, label %end 154 155end: 156 %tmp4 = phi i32 [ %val.sub, %loop ], [ 0, %entry ] 157 store i32 %tmp4, i32 addrspace(1)* %out 158 ret void 159} 160 161; GCN-LABEL: {{^}}divergent_if_else_endif: 162; GCN: {{^}}; %bb.0: 163 164; GCN: s_mov_b32 m0, -1 165; GCN: ds_read_b32 [[LOAD0:v[0-9]+]] 166 167; GCN: s_mov_b32 [[ZERO:s[0-9]+]], 0 168; GCN: v_cmp_ne_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], [[ZERO]], v0 169 170; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec 171; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, [[CMP0]] 172; GCN: s_xor_b64 s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}} 173 174; Spill load 175; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], 0 offset:[[LOAD0_OFFSET:[0-9]+]] ; 4-byte Folded Spill 176 177; Spill saved exec 178; VGPR: v_writelane_b32 [[SPILL_VGPR:v[0-9]+]], s[[SAVEEXEC_LO]], [[SAVEEXEC_LO_LANE:[0-9]+]] 179; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[SAVEEXEC_HI]], [[SAVEEXEC_HI_LANE:[0-9]+]] 180 181; VMEM: v_writelane_b32 v[[V_SAVEEXEC:[0-9]+]], s[[SAVEEXEC_LO]], 0 182; VMEM: v_writelane_b32 v[[V_SAVEEXEC]], s[[SAVEEXEC_HI]], 1 183; VMEM: buffer_store_dword v[[V_SAVEEXEC]], off, s[0:3], 0 offset:[[SAVEEXEC_OFFSET:[0-9]+]] ; 4-byte Folded Spill 184 185; GCN: s_mov_b64 exec, [[CMP0]] 186 187; FIXME: It makes no sense to put this skip here 188; GCN: s_cbranch_execz [[FLOW:BB[0-9]+_[0-9]+]] 189; GCN-NEXT: s_branch [[ELSE:BB[0-9]+_[0-9]+]] 190 191; GCN: [[FLOW]]: ; %Flow 192; VGPR: v_readlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]] 193; VGPR: v_readlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]] 194 195 196; VMEM: buffer_load_dword v[[FLOW_V_RELOAD_SAVEEXEC:[0-9]+]], off, s[0:3], 0 offset:[[SAVEEXEC_OFFSET]] 197; VMEM: s_waitcnt vmcnt(0) 198; VMEM: v_readlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[FLOW_V_RELOAD_SAVEEXEC]], 0 199; VMEM: v_readlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[FLOW_V_RELOAD_SAVEEXEC]], 1 200 201; GCN: s_or_saveexec_b64 s{{\[}}[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]{{\]}}, s{{\[}}[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]{{\]}} 202 203; Regular spill value restored after exec modification 204; GCN: buffer_load_dword [[FLOW_VAL:v[0-9]+]], off, s[0:3], 0 offset:[[FLOW_VAL_OFFSET:[0-9]+]] ; 4-byte Folded Reload 205 206 207; Spill saved exec 208; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[FLOW_S_RELOAD_SAVEEXEC_LO]], [[FLOW_SAVEEXEC_LO_LANE:[0-9]+]] 209; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[FLOW_S_RELOAD_SAVEEXEC_HI]], [[FLOW_SAVEEXEC_HI_LANE:[0-9]+]] 210 211 212; VMEM: v_writelane_b32 v[[FLOW_V_SAVEEXEC:[0-9]+]], s[[FLOW_S_RELOAD_SAVEEXEC_LO]], 0 213; VMEM: v_writelane_b32 v[[FLOW_V_SAVEEXEC]], s[[FLOW_S_RELOAD_SAVEEXEC_HI]], 1 214; VMEM: buffer_store_dword v[[FLOW_V_SAVEEXEC]], off, s[0:3], 0 offset:[[FLOW_SAVEEXEC_OFFSET:[0-9]+]] ; 4-byte Folded Spill 215 216; GCN: buffer_store_dword [[FLOW_VAL]], off, s[0:3], 0 offset:[[RESULT_OFFSET:[0-9]+]] ; 4-byte Folded Spill 217; GCN: s_xor_b64 exec, exec, s{{\[}}[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]{{\]}} 218; GCN-NEXT: s_cbranch_execz [[ENDIF:BB[0-9]+_[0-9]+]] 219 220 221; GCN: ; %bb.{{[0-9]+}}: ; %if 222; GCN: ds_read_b32 223; GCN: buffer_load_dword v[[LOAD0_RELOAD:[0-9]+]], off, s[0:3], 0 offset:[[LOAD0_OFFSET]] ; 4-byte Folded Reload 224; GCN: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, v{{[0-9]+}}, v[[LOAD0_RELOAD]] 225; GCN: buffer_store_dword [[ADD]], off, s[0:3], 0 offset:[[RESULT_OFFSET]] ; 4-byte Folded Spill 226; GCN-NEXT: s_branch [[ENDIF:BB[0-9]+_[0-9]+]] 227 228; GCN: [[ELSE]]: ; %else 229; GCN: buffer_load_dword v[[LOAD0_RELOAD:[0-9]+]], off, s[0:3], 0 offset:[[LOAD0_OFFSET]] ; 4-byte Folded Reload 230; GCN: v_subrev_i32_e32 [[SUB:v[0-9]+]], vcc, v{{[0-9]+}}, v[[LOAD0_RELOAD]] 231; GCN: buffer_store_dword [[ADD]], off, s[0:3], 0 offset:[[FLOW_RESULT_OFFSET:[0-9]+]] ; 4-byte Folded Spill 232; GCN-NEXT: s_branch [[FLOW]] 233 234; GCN: [[ENDIF]]: 235; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[FLOW_SAVEEXEC_LO_LANE]] 236; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[FLOW_SAVEEXEC_HI_LANE]] 237 238 239; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC:[0-9]+]], off, s[0:3], 0 offset:[[FLOW_SAVEEXEC_OFFSET]] ; 4-byte Folded Reload 240; VMEM: s_waitcnt vmcnt(0) 241; VMEM: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[V_RELOAD_SAVEEXEC]], 0 242; VMEM: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[V_RELOAD_SAVEEXEC]], 1 243 244; GCN: s_or_b64 exec, exec, s{{\[}}[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]{{\]}} 245 246; GCN: buffer_load_dword v[[RESULT:[0-9]+]], off, s[0:3], 0 offset:[[RESULT_OFFSET]] ; 4-byte Folded Reload 247; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[RESULT]] 248define amdgpu_kernel void @divergent_if_else_endif(i32 addrspace(1)* %out) #0 { 249entry: 250 %tid = call i32 @llvm.amdgcn.workitem.id.x() 251 %load0 = load volatile i32, i32 addrspace(3)* undef 252 %cmp0 = icmp eq i32 %tid, 0 253 br i1 %cmp0, label %if, label %else 254 255if: 256 %load1 = load volatile i32, i32 addrspace(3)* undef 257 %val0 = add i32 %load0, %load1 258 br label %endif 259 260else: 261 %load2 = load volatile i32, i32 addrspace(3)* undef 262 %val1 = sub i32 %load0, %load2 263 br label %endif 264 265endif: 266 %result = phi i32 [ %val0, %if ], [ %val1, %else ] 267 store i32 %result, i32 addrspace(1)* %out 268 ret void 269} 270 271declare i32 @llvm.amdgcn.workitem.id.x() #1 272 273attributes #0 = { nounwind } 274attributes #1 = { nounwind readnone } 275