1 // Test target codegen - host bc file has to be created first. 2 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc 3 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 4 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc 5 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 6 // RUN: %clang_cc1 -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 7 // expected-no-diagnostics 8 #ifndef HEADER 9 #define HEADER 10 11 // Check for the data transfer medium in shared memory to transfer the reduction list to the first warp. 12 // CHECK-DAG: [[TRANSFER_STORAGE:@.+]] = common addrspace([[SHARED_ADDRSPACE:[0-9]+]]) global [32 x i64] 13 14 // Check that the execution mode of all 3 target regions is set to Generic Mode. 15 // CHECK-DAG: {{@__omp_offloading_.+l27}}_exec_mode = weak constant i8 1 16 // CHECK-DAG: {{@__omp_offloading_.+l33}}_exec_mode = weak constant i8 1 17 // CHECK-DAG: {{@__omp_offloading_.+l40}}_exec_mode = weak constant i8 1 18 19 template<typename tx> 20 tx ftemplate(int n) { 21 int a; 22 short b; 23 tx c; 24 float d; 25 double e; 26 27 #pragma omp target 28 #pragma omp teams reduction(+: e) 29 { 30 e += 5; 31 } 32 33 #pragma omp target 34 #pragma omp teams reduction(^: c) reduction(*: d) 35 { 36 c ^= 2; 37 d *= 33; 38 } 39 40 #pragma omp target 41 #pragma omp teams reduction(|: a) reduction(max: b) 42 { 43 a |= 1; 44 b = 99 > b ? 99 : b; 45 } 46 47 return a+b+c+d+e; 48 } 49 50 int bar(int n){ 51 int a = 0; 52 53 a += ftemplate<char>(n); 54 55 return a; 56 } 57 58 // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l27}}_worker() 59 60 // CHECK: define {{.*}}void [[T1:@__omp_offloading_.+template.+l27]]( 61 // 62 // CHECK: {{call|invoke}} void [[T1]]_worker() 63 // 64 // CHECK: call void @__kmpc_kernel_init( 65 // 66 // CHECK: store double {{[0\.e\+]+}}, double* [[E:%.+]], align 67 // CHECK: [[EV:%.+]] = load double, double* [[E]], align 68 // CHECK: [[ADD:%.+]] = fadd double [[EV]], 5 69 // CHECK: store double [[ADD]], double* [[E]], align 70 // CHECK: [[PTR1:%.+]] = getelementptr inbounds [[RLT:.+]], [1 x i8*]* [[RL:%.+]], i[[SZ:32|64]] 0, i{{32|64}} 0 71 // CHECK: [[E_CAST:%.+]] = bitcast double* [[E]] to i8* 72 // CHECK: store i8* [[E_CAST]], i8** [[PTR1]], align 73 // CHECK: [[ARG_RL:%.+]] = bitcast [[RLT]]* [[RL]] to i8* 74 // CHECK: [[RET:%.+]] = call i32 @__kmpc_nvptx_teams_reduce_nowait(i32 {{.+}}, i32 1, i[[SZ]] {{4|8}}, i8* [[ARG_RL]], void (i8*, i16, i16, i16)* [[SHUFFLE_REDUCE_FN:@.+]], void (i8*, i32)* [[WARP_COPY_FN:@.+]], void (i8*, i8*, i32, i32)* [[SCRATCH_COPY_FN:@.+]], void (i8*, i8*, i32, i32, i32)* [[LOAD_REDUCE_FN:@.+]]) 75 // CHECK: [[COND:%.+]] = icmp eq i32 [[RET]], 1 76 // CHECK: br i1 [[COND]], label {{%?}}[[IFLABEL:.+]], label {{%?}}[[EXIT:.+]] 77 // 78 // CHECK: [[IFLABEL]] 79 // CHECK: [[E_INV:%.+]] = load double, double* [[E_IN:%.+]], align 80 // CHECK: [[EV:%.+]] = load double, double* [[E]], align 81 // CHECK: [[ADD:%.+]] = fadd double [[E_INV]], [[EV]] 82 // CHECK: store double [[ADD]], double* [[E_IN]], align 83 // CHECK: call void @__kmpc_nvptx_end_reduce_nowait( 84 // CHECK: br label %[[EXIT]] 85 // 86 // CHECK: [[EXIT]] 87 // CHECK: call void @__kmpc_kernel_deinit( 88 89 // 90 // Reduction function 91 // CHECK: define internal void [[REDUCTION_FUNC:@.+]](i8*, i8*) 92 // CHECK: [[VAR_RHS_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST_RHS:%.+]], i[[SZ]] 0, i[[SZ]] 0 93 // CHECK: [[VAR_RHS_VOID:%.+]] = load i8*, i8** [[VAR_RHS_REF]], 94 // CHECK: [[VAR_RHS:%.+]] = bitcast i8* [[VAR_RHS_VOID]] to double* 95 // 96 // CHECK: [[VAR_LHS_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST_LHS:%.+]], i[[SZ]] 0, i[[SZ]] 0 97 // CHECK: [[VAR_LHS_VOID:%.+]] = load i8*, i8** [[VAR_LHS_REF]], 98 // CHECK: [[VAR_LHS:%.+]] = bitcast i8* [[VAR_LHS_VOID]] to double* 99 // 100 // CHECK: [[VAR_LHS_VAL:%.+]] = load double, double* [[VAR_LHS]], 101 // CHECK: [[VAR_RHS_VAL:%.+]] = load double, double* [[VAR_RHS]], 102 // CHECK: [[RES:%.+]] = fadd double [[VAR_LHS_VAL]], [[VAR_RHS_VAL]] 103 // CHECK: store double [[RES]], double* [[VAR_LHS]], 104 // CHECK: ret void 105 106 // 107 // Shuffle and reduce function 108 // CHECK: define internal void [[SHUFFLE_REDUCE_FN]](i8*, i16 {{.*}}, i16 {{.*}}, i16 {{.*}}) 109 // CHECK: [[REMOTE_RED_LIST:%.+]] = alloca [[RLT]], align 110 // CHECK: [[REMOTE_ELT:%.+]] = alloca double 111 // 112 // CHECK: [[LANEID:%.+]] = load i16, i16* {{.+}}, align 113 // CHECK: [[LANEOFFSET:%.+]] = load i16, i16* {{.+}}, align 114 // CHECK: [[ALGVER:%.+]] = load i16, i16* {{.+}}, align 115 // 116 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 117 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 118 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 119 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to double* 120 // CHECK: [[ELT_VAL:%.+]] = load double, double* [[ELT]], align 121 // 122 // CHECK: [[ELT_CAST:%.+]] = bitcast double [[ELT_VAL]] to i64 123 // CHECK: [[WS32:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() 124 // CHECK: [[WS:%.+]] = trunc i32 [[WS32]] to i16 125 // CHECK: [[REMOTE_ELT_VAL64:%.+]] = call i64 @__kmpc_shuffle_int64(i64 [[ELT_CAST]], i16 [[LANEOFFSET]], i16 [[WS]]) 126 // CHECK: [[REMOTE_ELT_VAL:%.+]] = bitcast i64 [[REMOTE_ELT_VAL64]] to double 127 // 128 // CHECK: store double [[REMOTE_ELT_VAL]], double* [[REMOTE_ELT]], align 129 // CHECK: [[REMOTE_ELT_VOID:%.+]] = bitcast double* [[REMOTE_ELT]] to i8* 130 // CHECK: store i8* [[REMOTE_ELT_VOID]], i8** [[REMOTE_ELT_REF]], align 131 // 132 // Condition to reduce 133 // CHECK: [[CONDALG0:%.+]] = icmp eq i16 [[ALGVER]], 0 134 // 135 // CHECK: [[COND1:%.+]] = icmp eq i16 [[ALGVER]], 1 136 // CHECK: [[COND2:%.+]] = icmp ult i16 [[LANEID]], [[LANEOFFSET]] 137 // CHECK: [[CONDALG1:%.+]] = and i1 [[COND1]], [[COND2]] 138 // 139 // CHECK: [[COND3:%.+]] = icmp eq i16 [[ALGVER]], 2 140 // CHECK: [[COND4:%.+]] = and i16 [[LANEID]], 1 141 // CHECK: [[COND5:%.+]] = icmp eq i16 [[COND4]], 0 142 // CHECK: [[COND6:%.+]] = and i1 [[COND3]], [[COND5]] 143 // CHECK: [[COND7:%.+]] = icmp sgt i16 [[LANEOFFSET]], 0 144 // CHECK: [[CONDALG2:%.+]] = and i1 [[COND6]], [[COND7]] 145 // 146 // CHECK: [[COND8:%.+]] = or i1 [[CONDALG0]], [[CONDALG1]] 147 // CHECK: [[SHOULD_REDUCE:%.+]] = or i1 [[COND8]], [[CONDALG2]] 148 // CHECK: br i1 [[SHOULD_REDUCE]], label {{%?}}[[DO_REDUCE:.+]], label {{%?}}[[REDUCE_ELSE:.+]] 149 // 150 // CHECK: [[DO_REDUCE]] 151 // CHECK: [[RED_LIST1_VOID:%.+]] = bitcast [[RLT]]* [[RED_LIST]] to i8* 152 // CHECK: [[RED_LIST2_VOID:%.+]] = bitcast [[RLT]]* [[REMOTE_RED_LIST]] to i8* 153 // CHECK: call void [[REDUCTION_FUNC]](i8* [[RED_LIST1_VOID]], i8* [[RED_LIST2_VOID]]) 154 // CHECK: br label {{%?}}[[REDUCE_CONT:.+]] 155 // 156 // CHECK: [[REDUCE_ELSE]] 157 // CHECK: br label {{%?}}[[REDUCE_CONT]] 158 // 159 // CHECK: [[REDUCE_CONT]] 160 // Now check if we should just copy over the remote reduction list 161 // CHECK: [[COND1:%.+]] = icmp eq i16 [[ALGVER]], 1 162 // CHECK: [[COND2:%.+]] = icmp uge i16 [[LANEID]], [[LANEOFFSET]] 163 // CHECK: [[SHOULD_COPY:%.+]] = and i1 [[COND1]], [[COND2]] 164 // CHECK: br i1 [[SHOULD_COPY]], label {{%?}}[[DO_COPY:.+]], label {{%?}}[[COPY_ELSE:.+]] 165 // 166 // CHECK: [[DO_COPY]] 167 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 168 // CHECK: [[REMOTE_ELT_VOID:%.+]] = load i8*, i8** [[REMOTE_ELT_REF]], 169 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 170 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 171 // CHECK: [[REMOTE_ELT:%.+]] = bitcast i8* [[REMOTE_ELT_VOID]] to double* 172 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load double, double* [[REMOTE_ELT]], align 173 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to double* 174 // CHECK: store double [[REMOTE_ELT_VAL]], double* [[ELT]], align 175 // CHECK: br label {{%?}}[[COPY_CONT:.+]] 176 // 177 // CHECK: [[COPY_ELSE]] 178 // CHECK: br label {{%?}}[[COPY_CONT]] 179 // 180 // CHECK: [[COPY_CONT]] 181 // CHECK: void 182 183 // 184 // Inter warp copy function 185 // CHECK: define internal void [[WARP_COPY_FN]](i8*, i32) 186 // CHECK-DAG: [[LANEID:%.+]] = and i32 {{.+}}, 31 187 // CHECK-DAG: [[WARPID:%.+]] = ashr i32 {{.+}}, 5 188 // CHECK-DAG: [[RED_LIST:%.+]] = bitcast i8* {{.+}} to [[RLT]]* 189 // CHECK: [[IS_WARP_MASTER:%.+]] = icmp eq i32 [[LANEID]], 0 190 // CHECK: br i1 [[IS_WARP_MASTER]], label {{%?}}[[DO_COPY:.+]], label {{%?}}[[COPY_ELSE:.+]] 191 // 192 // [[DO_COPY]] 193 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 194 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 195 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to double* 196 // CHECK: [[ELT_VAL:%.+]] = load double, double* [[ELT]], align 197 // 198 // CHECK: [[MEDIUM_ELT64:%.+]] = getelementptr inbounds [32 x i64], [32 x i64] addrspace([[SHARED_ADDRSPACE]])* [[TRANSFER_STORAGE]], i64 0, i32 [[WARPID]] 199 // CHECK: [[MEDIUM_ELT:%.+]] = bitcast i64 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT64]] to double addrspace([[SHARED_ADDRSPACE]])* 200 // CHECK: store double [[ELT_VAL]], double addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT]], align 201 // CHECK: br label {{%?}}[[COPY_CONT:.+]] 202 // 203 // CHECK: [[COPY_ELSE]] 204 // CHECK: br label {{%?}}[[COPY_CONT]] 205 // 206 // Barrier after copy to shared memory storage medium. 207 // CHECK: [[COPY_CONT]] 208 // CHECK: [[WS:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() 209 // CHECK: [[ACTIVE_THREADS:%.+]] = mul nsw i32 [[ACTIVE_WARPS:%.+]], [[WS]] 210 // CHECK: call void @llvm.nvvm.barrier(i32 1, i32 [[ACTIVE_THREADS]]) 211 // 212 // Read into warp 0. 213 // CHECK: [[IS_W0_ACTIVE_THREAD:%.+]] = icmp ult i32 [[TID:%.+]], [[ACTIVE_WARPS]] 214 // CHECK: br i1 [[IS_W0_ACTIVE_THREAD]], label {{%?}}[[DO_READ:.+]], label {{%?}}[[READ_ELSE:.+]] 215 // 216 // CHECK: [[DO_READ]] 217 // CHECK: [[MEDIUM_ELT64:%.+]] = getelementptr inbounds [32 x i64], [32 x i64] addrspace([[SHARED_ADDRSPACE]])* [[TRANSFER_STORAGE]], i64 0, i32 [[TID]] 218 // CHECK: [[MEDIUM_ELT:%.+]] = bitcast i64 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT64]] to double addrspace([[SHARED_ADDRSPACE]])* 219 // CHECK: [[MEDIUM_ELT_VAL:%.+]] = load double, double addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT]], align 220 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 221 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 222 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to double* 223 // CHECK: store double [[MEDIUM_ELT_VAL]], double* [[ELT]], align 224 // CHECK: br label {{%?}}[[READ_CONT:.+]] 225 // 226 // CHECK: [[READ_ELSE]] 227 // CHECK: br label {{%?}}[[READ_CONT]] 228 // 229 // CHECK: [[READ_CONT]] 230 // CHECK: call void @llvm.nvvm.barrier(i32 1, i32 [[ACTIVE_THREADS]]) 231 // CHECK: ret 232 233 // 234 // Copy to scratchpad function 235 // CHECK: define internal void [[SCRATCH_COPY_FN]](i8*, i8*, i32, i32) 236 // CHECK: [[RED_LIST:%.+]] = bitcast i8* {{.+}} to [[RLT]]* 237 // CHECK: [[SCRATCHPAD_PTR:%.+]] = load i8*, i8** {{.+}}, align 238 // CHECK-64: [[TEAM32:%.+]] = load i32, i32* {{.+}}, align 239 // CHECK-64: [[TEAM:%.+]] = sext i32 [[TEAM32]] to i64 240 // CHECK-32: [[TEAM:%.+]] = load i32, i32* {{.+}}, align 241 // CHECK-64: [[NUM_TEAMS32:%.+]] = load i32, i32* {{.+}}, align 242 // CHECK-64: [[NUM_TEAMS:%.+]] = sext i32 [[NUM_TEAMS32]] to i64 243 // CHECK-32: [[NUM_TEAMS:%.+]] = load i32, i32* {{.+}}, align 244 // CHECK: [[SCRATCHPAD:%.+]] = ptrtoint i8* [[SCRATCHPAD_PTR]] to i[[SZ]] 245 // 246 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 247 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 248 // 249 // CHECK: [[P:%.+]] = mul i[[SZ]] 8, [[TEAM]] 250 // CHECK: [[SCRATCHPAD_ELT_PTR64:%.+]] = add i[[SZ]] [[SCRATCHPAD]], [[P]] 251 // CHECK: [[SCRATCHPAD_ELT_PTR_VOID:%.+]] = inttoptr i[[SZ]] [[SCRATCHPAD_ELT_PTR64]] to i8* 252 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to double* 253 // CHECK: [[ELT_VAL:%.+]] = load double, double* [[ELT]], align 254 // CHECK: [[SCRATCHPAD_ELT_PTR:%.+]] = bitcast i8* [[SCRATCHPAD_ELT_PTR_VOID]] to double* 255 // CHECK: store double [[ELT_VAL]], double* [[SCRATCHPAD_ELT_PTR]], align 256 // 257 // CHECK: ret 258 259 // 260 // Load and reduce function 261 // CHECK: define internal void [[LOAD_REDUCE_FN]](i8*, i8*, i32, i32, i32) 262 // CHECK: [[REMOTE_RED_LIST:%.+]] = alloca [[RLT]], align 263 // CHECK: [[REMOTE_ELT:%.+]] = alloca double 264 // CHECK: [[RED_LIST:%.+]] = bitcast i8* {{.+}} to [[RLT]]* 265 // CHECK: [[SCRATCHPAD_PTR:%.+]] = load i8*, i8** {{.+}}, align 266 // CHECK-64: [[TEAM32:%.+]] = load i32, i32* {{.+}}, align 267 // CHECK-64: [[TEAM:%.+]] = sext i32 [[TEAM32]] to i64 268 // CHECK-32: [[TEAM:%.+]] = load i32, i32* {{.+}}, align 269 // CHECK-64: [[NUM_TEAMS32:%.+]] = load i32, i32* {{.+}}, align 270 // CHECK-64: [[NUM_TEAMS:%.+]] = sext i32 [[NUM_TEAMS32]] to i64 271 // CHECK-32: [[NUM_TEAMS:%.+]] = load i32, i32* {{.+}}, align 272 // CHECK: [[SHOULD_REDUCE:%.+]] = load i32, i32* {{.+}}, align 273 // CHECK: [[SCRATCHPAD:%.+]] = ptrtoint i8* [[SCRATCHPAD_PTR]] to i[[SZ]] 274 // 275 // CHECK: [[P:%.+]] = mul i[[SZ]] 8, [[TEAM]] 276 // CHECK: [[SCRATCHPAD_ELT_PTR64:%.+]] = add i[[SZ]] [[SCRATCHPAD]], [[P]] 277 // CHECK: [[SCRATCHPAD_ELT_PTR_VOID:%.+]] = inttoptr i[[SZ]] [[SCRATCHPAD_ELT_PTR64]] to i8* 278 279 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 280 // CHECK: [[SCRATCHPAD_ELT_PTR:%.+]] = bitcast i8* [[SCRATCHPAD_ELT_PTR_VOID]] to double* 281 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load double, double* [[SCRATCHPAD_ELT_PTR]], align 282 // CHECK: store double [[REMOTE_ELT_VAL]], double* [[REMOTE_ELT]], align 283 // CHECK: [[REMOTE_ELT_PTR:%.+]] = bitcast double* [[REMOTE_ELT]] to i8* 284 // CHECK: store i8* [[REMOTE_ELT_PTR]], i8** [[REMOTE_ELT_REF]], align 285 // 286 // CHECK: [[REDUCE:%.+]] = icmp eq i32 [[SHOULD_REDUCE]], 1 287 // CHECK: br i1 [[REDUCE]], label {{%?}}[[DO_REDUCE:.+]], label {{%?}}[[REDUCE_ELSE:.+]] 288 // 289 // CHECK: [[DO_REDUCE]] 290 // CHECK: [[RED_LIST1_VOID:%.+]] = bitcast [[RLT]]* [[RED_LIST]] to i8* 291 // CHECK: [[RED_LIST2_VOID:%.+]] = bitcast [[RLT]]* [[REMOTE_RED_LIST]] to i8* 292 // CHECK: call void [[REDUCTION_FUNC]](i8* [[RED_LIST1_VOID]], i8* [[RED_LIST2_VOID]]) 293 // CHECK: br label {{%?}}[[REDUCE_CONT:.+]] 294 // 295 // Copy element from remote reduce list 296 // CHECK: [[REDUCE_ELSE]] 297 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 298 // CHECK: [[REMOTE_ELT_VOID:%.+]] = load i8*, i8** [[REMOTE_ELT_REF]], 299 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 300 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 301 // CHECK: [[REMOTE_ELT:%.+]] = bitcast i8* [[REMOTE_ELT_VOID]] to double* 302 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load double, double* [[REMOTE_ELT]], align 303 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to double* 304 // CHECK: store double [[REMOTE_ELT_VAL]], double* [[ELT]], align 305 // CHECK: br label {{%?}}[[REDUCE_CONT]] 306 // 307 // CHECK: [[REDUCE_CONT]] 308 // CHECK: ret 309 310 // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l33}}_worker() 311 312 // CHECK: define {{.*}}void [[T2:@__omp_offloading_.+template.+l33]]( 313 // 314 // CHECK: {{call|invoke}} void [[T2]]_worker() 315 // 316 // CHECK: call void @__kmpc_kernel_init( 317 // 318 // CHECK: store float {{1\.[0e\+]+}}, float* [[D:%.+]], align 319 // CHECK: [[C_VAL:%.+]] = load i8, i8* [[C:%.+]], align 320 // CHECK: [[CONV:%.+]] = sext i8 [[C_VAL]] to i32 321 // CHECK: [[XOR:%.+]] = xor i32 [[CONV]], 2 322 // CHECK: [[TRUNC:%.+]] = trunc i32 [[XOR]] to i8 323 // CHECK: store i8 [[TRUNC]], i8* [[C]], align 324 // CHECK: [[DV:%.+]] = load float, float* [[D]], align 325 // CHECK: [[MUL:%.+]] = fmul float [[DV]], {{[0-9e\.\+]+}} 326 // CHECK: store float [[MUL]], float* [[D]], align 327 // CHECK: [[PTR1:%.+]] = getelementptr inbounds [[RLT:.+]], [2 x i8*]* [[RL:%.+]], i[[SZ]] 0, i[[SZ]] 0 328 // CHECK: store i8* [[C]], i8** [[PTR1]], align 329 // CHECK: [[PTR2:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RL]], i[[SZ]] 0, i[[SZ]] 1 330 // CHECK: [[D_CAST:%.+]] = bitcast float* [[D]] to i8* 331 // CHECK: store i8* [[D_CAST]], i8** [[PTR2]], align 332 // CHECK: [[ARG_RL:%.+]] = bitcast [[RLT]]* [[RL]] to i8* 333 // CHECK: [[RET:%.+]] = call i32 @__kmpc_nvptx_teams_reduce_nowait(i32 {{.+}}, i32 2, i[[SZ]] {{8|16}}, i8* [[ARG_RL]], void (i8*, i16, i16, i16)* [[SHUFFLE_REDUCE_FN:@.+]], void (i8*, i32)* [[WARP_COPY_FN:@.+]], void (i8*, i8*, i32, i32)* [[SCRATCH_COPY_FN:@.+]], void (i8*, i8*, i32, i32, i32)* [[LOAD_REDUCE_FN:@.+]]) 334 // CHECK: [[COND:%.+]] = icmp eq i32 [[RET]], 1 335 // CHECK: br i1 [[COND]], label {{%?}}[[IFLABEL:.+]], label {{%?}}[[EXIT:.+]] 336 // 337 // CHECK: [[IFLABEL]] 338 // CHECK: [[C_INV8:%.+]] = load i8, i8* [[C_IN:%.+]], align 339 // CHECK: [[C_INV:%.+]] = sext i8 [[C_INV8]] to i32 340 // CHECK: [[CV8:%.+]] = load i8, i8* [[C]], align 341 // CHECK: [[CV:%.+]] = sext i8 [[CV8]] to i32 342 // CHECK: [[XOR:%.+]] = xor i32 [[C_INV]], [[CV]] 343 // CHECK: [[TRUNC:%.+]] = trunc i32 [[XOR]] to i8 344 // CHECK: store i8 [[TRUNC]], i8* [[C_IN]], align 345 // CHECK: [[D_INV:%.+]] = load float, float* [[D_IN:%.+]], align 346 // CHECK: [[DV:%.+]] = load float, float* [[D]], align 347 // CHECK: [[MUL:%.+]] = fmul float [[D_INV]], [[DV]] 348 // CHECK: store float [[MUL]], float* [[D_IN]], align 349 // CHECK: call void @__kmpc_nvptx_end_reduce_nowait( 350 // CHECK: br label %[[EXIT]] 351 // 352 // CHECK: [[EXIT]] 353 // CHECK: call void @__kmpc_kernel_deinit( 354 355 // 356 // Reduction function 357 // CHECK: define internal void [[REDUCTION_FUNC:@.+]](i8*, i8*) 358 // CHECK: [[VAR1_RHS_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST_RHS:%.+]], i[[SZ]] 0, i[[SZ]] 0 359 // CHECK: [[VAR1_RHS:%.+]] = load i8*, i8** [[VAR1_RHS_REF]], 360 // 361 // CHECK: [[VAR1_LHS_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST_LHS:%.+]], i[[SZ]] 0, i[[SZ]] 0 362 // CHECK: [[VAR1_LHS:%.+]] = load i8*, i8** [[VAR1_LHS_REF]], 363 // 364 // CHECK: [[VAR2_RHS_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST_RHS]], i[[SZ]] 0, i[[SZ]] 1 365 // CHECK: [[VAR2_RHS_VOID:%.+]] = load i8*, i8** [[VAR2_RHS_REF]], 366 // CHECK: [[VAR2_RHS:%.+]] = bitcast i8* [[VAR2_RHS_VOID]] to float* 367 // 368 // CHECK: [[VAR2_LHS_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST_LHS]], i[[SZ]] 0, i[[SZ]] 1 369 // CHECK: [[VAR2_LHS_VOID:%.+]] = load i8*, i8** [[VAR2_LHS_REF]], 370 // CHECK: [[VAR2_LHS:%.+]] = bitcast i8* [[VAR2_LHS_VOID]] to float* 371 // 372 // CHECK: [[VAR1_LHS_VAL8:%.+]] = load i8, i8* [[VAR1_LHS]], 373 // CHECK: [[VAR1_LHS_VAL:%.+]] = sext i8 [[VAR1_LHS_VAL8]] to i32 374 // CHECK: [[VAR1_RHS_VAL8:%.+]] = load i8, i8* [[VAR1_RHS]], 375 // CHECK: [[VAR1_RHS_VAL:%.+]] = sext i8 [[VAR1_RHS_VAL8]] to i32 376 // CHECK: [[XOR:%.+]] = xor i32 [[VAR1_LHS_VAL]], [[VAR1_RHS_VAL]] 377 // CHECK: [[RES:%.+]] = trunc i32 [[XOR]] to i8 378 // CHECK: store i8 [[RES]], i8* [[VAR1_LHS]], 379 // 380 // CHECK: [[VAR2_LHS_VAL:%.+]] = load float, float* [[VAR2_LHS]], 381 // CHECK: [[VAR2_RHS_VAL:%.+]] = load float, float* [[VAR2_RHS]], 382 // CHECK: [[RES:%.+]] = fmul float [[VAR2_LHS_VAL]], [[VAR2_RHS_VAL]] 383 // CHECK: store float [[RES]], float* [[VAR2_LHS]], 384 // CHECK: ret void 385 386 // 387 // Shuffle and reduce function 388 // CHECK: define internal void [[SHUFFLE_REDUCE_FN]](i8*, i16 {{.*}}, i16 {{.*}}, i16 {{.*}}) 389 // CHECK: [[REMOTE_RED_LIST:%.+]] = alloca [[RLT]], align 390 // CHECK: [[REMOTE_ELT1:%.+]] = alloca i8 391 // CHECK: [[REMOTE_ELT2:%.+]] = alloca float 392 // 393 // CHECK: [[LANEID:%.+]] = load i16, i16* {{.+}}, align 394 // CHECK: [[LANEOFFSET:%.+]] = load i16, i16* {{.+}}, align 395 // CHECK: [[ALGVER:%.+]] = load i16, i16* {{.+}}, align 396 // 397 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 398 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 399 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 400 // CHECK: [[ELT_VAL:%.+]] = load i8, i8* [[ELT_VOID]], align 401 // 402 // CHECK: [[ELT_CAST:%.+]] = sext i8 [[ELT_VAL]] to i32 403 // CHECK: [[WS32:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() 404 // CHECK: [[WS:%.+]] = trunc i32 [[WS32]] to i16 405 // CHECK: [[REMOTE_ELT1_VAL32:%.+]] = call i32 @__kmpc_shuffle_int32(i32 [[ELT_CAST]], i16 [[LANEOFFSET]], i16 [[WS]]) 406 // CHECK: [[REMOTE_ELT1_VAL:%.+]] = trunc i32 [[REMOTE_ELT1_VAL32]] to i8 407 // 408 // CHECK: store i8 [[REMOTE_ELT1_VAL]], i8* [[REMOTE_ELT1]], align 409 // CHECK: store i8* [[REMOTE_ELT1]], i8** [[REMOTE_ELT_REF]], align 410 // 411 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 412 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 413 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 414 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to float* 415 // CHECK: [[ELT_VAL:%.+]] = load float, float* [[ELT]], align 416 // 417 // CHECK: [[ELT_CAST:%.+]] = bitcast float [[ELT_VAL]] to i32 418 // CHECK: [[WS32:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() 419 // CHECK: [[WS:%.+]] = trunc i32 [[WS32]] to i16 420 // CHECK: [[REMOTE_ELT2_VAL32:%.+]] = call i32 @__kmpc_shuffle_int32(i32 [[ELT_CAST]], i16 [[LANEOFFSET]], i16 [[WS]]) 421 // CHECK: [[REMOTE_ELT2_VAL:%.+]] = bitcast i32 [[REMOTE_ELT2_VAL32]] to float 422 // 423 // CHECK: store float [[REMOTE_ELT2_VAL]], float* [[REMOTE_ELT2]], align 424 // CHECK: [[REMOTE_ELT2C:%.+]] = bitcast float* [[REMOTE_ELT2]] to i8* 425 // CHECK: store i8* [[REMOTE_ELT2C]], i8** [[REMOTE_ELT_REF]], align 426 // 427 // Condition to reduce 428 // CHECK: [[CONDALG0:%.+]] = icmp eq i16 [[ALGVER]], 0 429 // 430 // CHECK: [[COND1:%.+]] = icmp eq i16 [[ALGVER]], 1 431 // CHECK: [[COND2:%.+]] = icmp ult i16 [[LANEID]], [[LANEOFFSET]] 432 // CHECK: [[CONDALG1:%.+]] = and i1 [[COND1]], [[COND2]] 433 // 434 // CHECK: [[COND3:%.+]] = icmp eq i16 [[ALGVER]], 2 435 // CHECK: [[COND4:%.+]] = and i16 [[LANEID]], 1 436 // CHECK: [[COND5:%.+]] = icmp eq i16 [[COND4]], 0 437 // CHECK: [[COND6:%.+]] = and i1 [[COND3]], [[COND5]] 438 // CHECK: [[COND7:%.+]] = icmp sgt i16 [[LANEOFFSET]], 0 439 // CHECK: [[CONDALG2:%.+]] = and i1 [[COND6]], [[COND7]] 440 // 441 // CHECK: [[COND8:%.+]] = or i1 [[CONDALG0]], [[CONDALG1]] 442 // CHECK: [[SHOULD_REDUCE:%.+]] = or i1 [[COND8]], [[CONDALG2]] 443 // CHECK: br i1 [[SHOULD_REDUCE]], label {{%?}}[[DO_REDUCE:.+]], label {{%?}}[[REDUCE_ELSE:.+]] 444 // 445 // CHECK: [[DO_REDUCE]] 446 // CHECK: [[RED_LIST1_VOID:%.+]] = bitcast [[RLT]]* [[RED_LIST]] to i8* 447 // CHECK: [[RED_LIST2_VOID:%.+]] = bitcast [[RLT]]* [[REMOTE_RED_LIST]] to i8* 448 // CHECK: call void [[REDUCTION_FUNC]](i8* [[RED_LIST1_VOID]], i8* [[RED_LIST2_VOID]]) 449 // CHECK: br label {{%?}}[[REDUCE_CONT:.+]] 450 // 451 // CHECK: [[REDUCE_ELSE]] 452 // CHECK: br label {{%?}}[[REDUCE_CONT]] 453 // 454 // CHECK: [[REDUCE_CONT]] 455 // Now check if we should just copy over the remote reduction list 456 // CHECK: [[COND1:%.+]] = icmp eq i16 [[ALGVER]], 1 457 // CHECK: [[COND2:%.+]] = icmp uge i16 [[LANEID]], [[LANEOFFSET]] 458 // CHECK: [[SHOULD_COPY:%.+]] = and i1 [[COND1]], [[COND2]] 459 // CHECK: br i1 [[SHOULD_COPY]], label {{%?}}[[DO_COPY:.+]], label {{%?}}[[COPY_ELSE:.+]] 460 // 461 // CHECK: [[DO_COPY]] 462 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 463 // CHECK: [[REMOTE_ELT_VOID:%.+]] = load i8*, i8** [[REMOTE_ELT_REF]], 464 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 465 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 466 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load i8, i8* [[REMOTE_ELT_VOID]], align 467 // CHECK: store i8 [[REMOTE_ELT_VAL]], i8* [[ELT_VOID]], align 468 // 469 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 470 // CHECK: [[REMOTE_ELT_VOID:%.+]] = load i8*, i8** [[REMOTE_ELT_REF]], 471 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 472 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 473 // CHECK: [[REMOTE_ELT:%.+]] = bitcast i8* [[REMOTE_ELT_VOID]] to float* 474 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load float, float* [[REMOTE_ELT]], align 475 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to float* 476 // CHECK: store float [[REMOTE_ELT_VAL]], float* [[ELT]], align 477 // CHECK: br label {{%?}}[[COPY_CONT:.+]] 478 // 479 // CHECK: [[COPY_ELSE]] 480 // CHECK: br label {{%?}}[[COPY_CONT]] 481 // 482 // CHECK: [[COPY_CONT]] 483 // CHECK: void 484 485 // 486 // Inter warp copy function 487 // CHECK: define internal void [[WARP_COPY_FN]](i8*, i32) 488 // CHECK-DAG: [[LANEID:%.+]] = and i32 {{.+}}, 31 489 // CHECK-DAG: [[WARPID:%.+]] = ashr i32 {{.+}}, 5 490 // CHECK-DAG: [[RED_LIST:%.+]] = bitcast i8* {{.+}} to [[RLT]]* 491 // CHECK: [[IS_WARP_MASTER:%.+]] = icmp eq i32 [[LANEID]], 0 492 // CHECK: br i1 [[IS_WARP_MASTER]], label {{%?}}[[DO_COPY:.+]], label {{%?}}[[COPY_ELSE:.+]] 493 // 494 // [[DO_COPY]] 495 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 496 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 497 // CHECK: [[ELT_VAL:%.+]] = load i8, i8* [[ELT_VOID]], align 498 // 499 // CHECK: [[MEDIUM_ELT64:%.+]] = getelementptr inbounds [32 x i64], [32 x i64] addrspace([[SHARED_ADDRSPACE]])* [[TRANSFER_STORAGE]], i64 0, i32 [[WARPID]] 500 // CHECK: [[MEDIUM_ELT:%.+]] = bitcast i64 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT64]] to i8 addrspace([[SHARED_ADDRSPACE]])* 501 // CHECK: store i8 [[ELT_VAL]], i8 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT]], align 502 // CHECK: br label {{%?}}[[COPY_CONT:.+]] 503 // 504 // CHECK: [[COPY_ELSE]] 505 // CHECK: br label {{%?}}[[COPY_CONT]] 506 // 507 // Barrier after copy to shared memory storage medium. 508 // CHECK: [[COPY_CONT]] 509 // CHECK: [[WS:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() 510 // CHECK: [[ACTIVE_THREADS:%.+]] = mul nsw i32 [[ACTIVE_WARPS:%.+]], [[WS]] 511 // CHECK: call void @llvm.nvvm.barrier(i32 1, i32 [[ACTIVE_THREADS]]) 512 // 513 // Read into warp 0. 514 // CHECK: [[IS_W0_ACTIVE_THREAD:%.+]] = icmp ult i32 [[TID:%.+]], [[ACTIVE_WARPS]] 515 // CHECK: br i1 [[IS_W0_ACTIVE_THREAD]], label {{%?}}[[DO_READ:.+]], label {{%?}}[[READ_ELSE:.+]] 516 // 517 // CHECK: [[DO_READ]] 518 // CHECK: [[MEDIUM_ELT64:%.+]] = getelementptr inbounds [32 x i64], [32 x i64] addrspace([[SHARED_ADDRSPACE]])* [[TRANSFER_STORAGE]], i64 0, i32 [[TID]] 519 // CHECK: [[MEDIUM_ELT:%.+]] = bitcast i64 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT64]] to i8 addrspace([[SHARED_ADDRSPACE]])* 520 // CHECK: [[MEDIUM_ELT_VAL:%.+]] = load i8, i8 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT]], align 521 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 522 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 523 // CHECK: store i8 [[MEDIUM_ELT_VAL]], i8* [[ELT_VOID]], align 524 // CHECK: br label {{%?}}[[READ_CONT:.+]] 525 // 526 // CHECK: [[READ_ELSE]] 527 // CHECK: br label {{%?}}[[READ_CONT]] 528 // 529 // CHECK: [[READ_CONT]] 530 // CHECK: call void @llvm.nvvm.barrier(i32 1, i32 [[ACTIVE_THREADS]]) 531 // CHECK: [[IS_WARP_MASTER:%.+]] = icmp eq i32 [[LANEID]], 0 532 // CHECK: br i1 [[IS_WARP_MASTER]], label {{%?}}[[DO_COPY:.+]], label {{%?}}[[COPY_ELSE:.+]] 533 // 534 // [[DO_COPY]] 535 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 536 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 537 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to float* 538 // CHECK: [[ELT_VAL:%.+]] = load float, float* [[ELT]], align 539 // 540 // CHECK: [[MEDIUM_ELT64:%.+]] = getelementptr inbounds [32 x i64], [32 x i64] addrspace([[SHARED_ADDRSPACE]])* [[TRANSFER_STORAGE]], i64 0, i32 [[WARPID]] 541 // CHECK: [[MEDIUM_ELT:%.+]] = bitcast i64 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT64]] to float addrspace([[SHARED_ADDRSPACE]])* 542 // CHECK: store float [[ELT_VAL]], float addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT]], align 543 // CHECK: br label {{%?}}[[COPY_CONT:.+]] 544 // 545 // CHECK: [[COPY_ELSE]] 546 // CHECK: br label {{%?}}[[COPY_CONT]] 547 // 548 // Barrier after copy to shared memory storage medium. 549 // CHECK: [[COPY_CONT]] 550 // CHECK: [[WS:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() 551 // CHECK: [[ACTIVE_THREADS:%.+]] = mul nsw i32 [[ACTIVE_WARPS:%.+]], [[WS]] 552 // CHECK: call void @llvm.nvvm.barrier(i32 1, i32 [[ACTIVE_THREADS]]) 553 // 554 // Read into warp 0. 555 // CHECK: [[IS_W0_ACTIVE_THREAD:%.+]] = icmp ult i32 [[TID:%.+]], [[ACTIVE_WARPS]] 556 // CHECK: br i1 [[IS_W0_ACTIVE_THREAD]], label {{%?}}[[DO_READ:.+]], label {{%?}}[[READ_ELSE:.+]] 557 // 558 // CHECK: [[DO_READ]] 559 // CHECK: [[MEDIUM_ELT64:%.+]] = getelementptr inbounds [32 x i64], [32 x i64] addrspace([[SHARED_ADDRSPACE]])* [[TRANSFER_STORAGE]], i64 0, i32 [[TID]] 560 // CHECK: [[MEDIUM_ELT:%.+]] = bitcast i64 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT64]] to float addrspace([[SHARED_ADDRSPACE]])* 561 // CHECK: [[MEDIUM_ELT_VAL:%.+]] = load float, float addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT]], align 562 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 1 563 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 564 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to float* 565 // CHECK: store float [[MEDIUM_ELT_VAL]], float* [[ELT]], align 566 // CHECK: br label {{%?}}[[READ_CONT:.+]] 567 // 568 // CHECK: [[READ_ELSE]] 569 // CHECK: br label {{%?}}[[READ_CONT]] 570 // 571 // CHECK: [[READ_CONT]] 572 // CHECK: call void @llvm.nvvm.barrier(i32 1, i32 [[ACTIVE_THREADS]]) 573 // CHECK: ret 574 575 // 576 // Copy to scratchpad function 577 // CHECK: define internal void [[SCRATCH_COPY_FN]](i8*, i8*, i32, i32) 578 // CHECK: [[RED_LIST:%.+]] = bitcast i8* {{.+}} to [[RLT]]* 579 // CHECK: [[SCRATCHPAD_PTR:%.+]] = load i8*, i8** {{.+}}, align 580 // CHECK-64: [[TEAM32:%.+]] = load i32, i32* {{.+}}, align 581 // CHECK-64: [[TEAM:%.+]] = sext i32 [[TEAM32]] to i64 582 // CHECK-32: [[TEAM:%.+]] = load i32, i32* {{.+}}, align 583 // CHECK-64: [[NUM_TEAMS32:%.+]] = load i32, i32* {{.+}}, align 584 // CHECK-64: [[NUM_TEAMS:%.+]] = sext i32 [[NUM_TEAMS32]] to i64 585 // CHECK-32: [[NUM_TEAMS:%.+]] = load i32, i32* {{.+}}, align 586 // CHECK: [[SCRATCHPAD:%.+]] = ptrtoint i8* [[SCRATCHPAD_PTR]] to i[[SZ]] 587 // 588 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 589 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 590 // 591 // CHECK: [[P:%.+]] = mul i[[SZ]] 1, [[TEAM]] 592 // CHECK: [[SCRATCHPAD_ELT_PTR64:%.+]] = add i[[SZ]] [[SCRATCHPAD]], [[P]] 593 // CHECK: [[SCRATCHPAD_ELT_PTR:%.+]] = inttoptr i[[SZ]] [[SCRATCHPAD_ELT_PTR64]] to i8* 594 // CHECK: [[ELT_VAL:%.+]] = load i8, i8* [[ELT_VOID]], align 595 // CHECK: store i8 [[ELT_VAL]], i8* [[SCRATCHPAD_ELT_PTR]], align 596 // 597 // CHECK: [[OF:%.+]] = mul i[[SZ]] [[NUM_TEAMS]], 1 598 // CHECK: [[POS1:%.+]] = add i[[SZ]] [[SCRATCHPAD]], [[OF]] 599 // CHECK: [[POS2:%.+]] = sub i[[SZ]] [[POS1]], 1 600 // CHECK: [[POS3:%.+]] = sdiv i[[SZ]] [[POS2]], 256 601 // CHECK: [[POS4:%.+]] = add i[[SZ]] [[POS3]], 1 602 // CHECK: [[SCRATCHPAD_NEXT:%.+]] = mul i[[SZ]] [[POS4]], 256 603 // 604 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 605 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 606 // 607 // CHECK: [[P:%.+]] = mul i[[SZ]] 4, [[TEAM]] 608 // CHECK: [[SCRATCHPAD_ELT_PTR64:%.+]] = add i[[SZ]] [[SCRATCHPAD_NEXT]], [[P]] 609 // CHECK: [[SCRATCHPAD_ELT_PTR_VOID:%.+]] = inttoptr i[[SZ]] [[SCRATCHPAD_ELT_PTR64]] to i8* 610 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to float* 611 // CHECK: [[ELT_VAL:%.+]] = load float, float* [[ELT]], align 612 // CHECK: [[SCRATCHPAD_ELT_PTR:%.+]] = bitcast i8* [[SCRATCHPAD_ELT_PTR_VOID]] to float* 613 // CHECK: store float [[ELT_VAL]], float* [[SCRATCHPAD_ELT_PTR]], align 614 // 615 // CHECK: ret 616 617 // 618 // Load and reduce function 619 // CHECK: define internal void [[LOAD_REDUCE_FN]](i8*, i8*, i32, i32, i32) 620 // CHECK: [[REMOTE_RED_LIST:%.+]] = alloca [[RLT]], align 621 // CHECK: [[REMOTE_ELT1:%.+]] = alloca i8 622 // CHECK: [[REMOTE_ELT2:%.+]] = alloca float 623 // CHECK: [[RED_LIST:%.+]] = bitcast i8* {{.+}} to [[RLT]]* 624 // CHECK: [[SCRATCHPAD_PTR:%.+]] = load i8*, i8** {{.+}}, align 625 // CHECK-64: [[TEAM32:%.+]] = load i32, i32* {{.+}}, align 626 // CHECK-64: [[TEAM:%.+]] = sext i32 [[TEAM32]] to i64 627 // CHECK-32: [[TEAM:%.+]] = load i32, i32* {{.+}}, align 628 // CHECK-64: [[NUM_TEAMS32:%.+]] = load i32, i32* {{.+}}, align 629 // CHECK-64: [[NUM_TEAMS:%.+]] = sext i32 [[NUM_TEAMS32]] to i64 630 // CHECK-32: [[NUM_TEAMS:%.+]] = load i32, i32* {{.+}}, align 631 // CHECK: [[SHOULD_REDUCE:%.+]] = load i32, i32* {{.+}}, align 632 // CHECK: [[SCRATCHPAD:%.+]] = ptrtoint i8* [[SCRATCHPAD_PTR]] to i[[SZ]] 633 // 634 // CHECK: [[P:%.+]] = mul i[[SZ]] 1, [[TEAM]] 635 // CHECK: [[SCRATCHPAD_ELT_PTR64:%.+]] = add i[[SZ]] [[SCRATCHPAD]], [[P]] 636 // CHECK: [[SCRATCHPAD_ELT_PTR_VOID:%.+]] = inttoptr i[[SZ]] [[SCRATCHPAD_ELT_PTR64]] to i8* 637 638 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 639 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load i8, i8* [[SCRATCHPAD_ELT_PTR_VOID]], align 640 // CHECK: store i8 [[REMOTE_ELT_VAL]], i8* [[REMOTE_ELT1]], align 641 // CHECK: store i8* [[REMOTE_ELT1]], i8** [[REMOTE_ELT_REF]], align 642 // 643 // CHECK: [[OF:%.+]] = mul i[[SZ]] [[NUM_TEAMS]], 1 644 // CHECK: [[POS1:%.+]] = add i[[SZ]] [[SCRATCHPAD]], [[OF]] 645 // CHECK: [[POS2:%.+]] = sub i[[SZ]] [[POS1]], 1 646 // CHECK: [[POS3:%.+]] = sdiv i[[SZ]] [[POS2]], 256 647 // CHECK: [[POS4:%.+]] = add i[[SZ]] [[POS3]], 1 648 // CHECK: [[SCRATCHPAD_NEXT:%.+]] = mul i[[SZ]] [[POS4]], 256 649 // 650 // CHECK: [[P:%.+]] = mul i[[SZ]] 4, [[TEAM]] 651 // CHECK: [[SCRATCHPAD_ELT_PTR64:%.+]] = add i[[SZ]] [[SCRATCHPAD_NEXT]], [[P]] 652 // CHECK: [[SCRATCHPAD_ELT_PTR_VOID:%.+]] = inttoptr i[[SZ]] [[SCRATCHPAD_ELT_PTR64]] to i8* 653 654 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 655 // CHECK: [[SCRATCHPAD_ELT_PTR:%.+]] = bitcast i8* [[SCRATCHPAD_ELT_PTR_VOID]] to float* 656 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load float, float* [[SCRATCHPAD_ELT_PTR]], align 657 // CHECK: store float [[REMOTE_ELT_VAL]], float* [[REMOTE_ELT2]], align 658 // CHECK: [[REMOTE_ELT_PTR:%.+]] = bitcast float* [[REMOTE_ELT2]] to i8* 659 // CHECK: store i8* [[REMOTE_ELT_PTR]], i8** [[REMOTE_ELT_REF]], align 660 // 661 // CHECK: [[REDUCE:%.+]] = icmp eq i32 [[SHOULD_REDUCE]], 1 662 // CHECK: br i1 [[REDUCE]], label {{%?}}[[DO_REDUCE:.+]], label {{%?}}[[REDUCE_ELSE:.+]] 663 // 664 // CHECK: [[DO_REDUCE]] 665 // CHECK: [[RED_LIST1_VOID:%.+]] = bitcast [[RLT]]* [[RED_LIST]] to i8* 666 // CHECK: [[RED_LIST2_VOID:%.+]] = bitcast [[RLT]]* [[REMOTE_RED_LIST]] to i8* 667 // CHECK: call void [[REDUCTION_FUNC]](i8* [[RED_LIST1_VOID]], i8* [[RED_LIST2_VOID]]) 668 // CHECK: br label {{%?}}[[REDUCE_CONT:.+]] 669 // 670 // Copy element from remote reduce list 671 // CHECK: [[REDUCE_ELSE]] 672 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 673 // CHECK: [[REMOTE_ELT_VOID:%.+]] = load i8*, i8** [[REMOTE_ELT_REF]], 674 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 675 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 676 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load i8, i8* [[REMOTE_ELT_VOID]], align 677 // CHECK: store i8 [[REMOTE_ELT_VAL]], i8* [[ELT_VOID]], align 678 // 679 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 1 680 // CHECK: [[REMOTE_ELT_VOID:%.+]] = load i8*, i8** [[REMOTE_ELT_REF]], 681 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 1 682 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 683 // CHECK: [[REMOTE_ELT:%.+]] = bitcast i8* [[REMOTE_ELT_VOID]] to float* 684 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load float, float* [[REMOTE_ELT]], align 685 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to float* 686 // CHECK: store float [[REMOTE_ELT_VAL]], float* [[ELT]], align 687 // CHECK: br label {{%?}}[[REDUCE_CONT]] 688 // 689 // CHECK: [[REDUCE_CONT]] 690 // CHECK: ret 691 692 // CHECK-LABEL: define {{.*}}void {{@__omp_offloading_.+template.+l40}}_worker() 693 694 // CHECK: define {{.*}}void [[T3:@__omp_offloading_.+template.+l40]]( 695 // 696 // CHECK: {{call|invoke}} void [[T3]]_worker() 697 // 698 // CHECK: call void @__kmpc_kernel_init( 699 // 700 // CHECK: store i32 0, i32* [[A:%.+]], align 701 // CHECK: store i16 -32768, i16* [[B:%.+]], align 702 // CHECK: [[A_VAL:%.+]] = load i32, i32* [[A:%.+]], align 703 // CHECK: [[OR:%.+]] = or i32 [[A_VAL]], 1 704 // CHECK: store i32 [[OR]], i32* [[A]], align 705 // CHECK: [[BV16:%.+]] = load i16, i16* [[B]], align 706 // CHECK: [[BV:%.+]] = sext i16 [[BV16]] to i32 707 // CHECK: [[CMP:%.+]] = icmp sgt i32 99, [[BV]] 708 // CHECK: br i1 [[CMP]], label {{%?}}[[DO_MAX:.+]], label {{%?}}[[MAX_ELSE:.+]] 709 // 710 // CHECK: [[DO_MAX]] 711 // CHECK: br label {{%?}}[[MAX_CONT:.+]] 712 // 713 // CHECK: [[MAX_ELSE]] 714 // CHECK: [[BV:%.+]] = load i16, i16* [[B]], align 715 // CHECK: [[MAX:%.+]] = sext i16 [[BV]] to i32 716 // CHECK: br label {{%?}}[[MAX_CONT]] 717 // 718 // CHECK: [[MAX_CONT]] 719 // CHECK: [[B_LVALUE:%.+]] = phi i32 [ 99, %[[DO_MAX]] ], [ [[MAX]], %[[MAX_ELSE]] ] 720 // CHECK: [[TRUNC:%.+]] = trunc i32 [[B_LVALUE]] to i16 721 // CHECK: store i16 [[TRUNC]], i16* [[B]], align 722 // CHECK: [[PTR1:%.+]] = getelementptr inbounds [[RLT:.+]], [2 x i8*]* [[RL:%.+]], i[[SZ]] 0, i[[SZ]] 0 723 // CHECK: [[A_CAST:%.+]] = bitcast i32* [[A]] to i8* 724 // CHECK: store i8* [[A_CAST]], i8** [[PTR1]], align 725 // CHECK: [[PTR2:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RL]], i[[SZ]] 0, i[[SZ]] 1 726 // CHECK: [[B_CAST:%.+]] = bitcast i16* [[B]] to i8* 727 // CHECK: store i8* [[B_CAST]], i8** [[PTR2]], align 728 // CHECK: [[ARG_RL:%.+]] = bitcast [[RLT]]* [[RL]] to i8* 729 // CHECK: [[RET:%.+]] = call i32 @__kmpc_nvptx_teams_reduce_nowait(i32 {{.+}}, i32 2, i[[SZ]] {{8|16}}, i8* [[ARG_RL]], void (i8*, i16, i16, i16)* [[SHUFFLE_REDUCE_FN:@.+]], void (i8*, i32)* [[WARP_COPY_FN:@.+]], void (i8*, i8*, i32, i32)* [[SCRATCH_COPY_FN:@.+]], void (i8*, i8*, i32, i32, i32)* [[LOAD_REDUCE_FN:@.+]]) 730 // CHECK: [[COND:%.+]] = icmp eq i32 [[RET]], 1 731 // CHECK: br i1 [[COND]], label {{%?}}[[IFLABEL:.+]], label {{%?}}[[EXIT:.+]] 732 // 733 // CHECK: [[IFLABEL]] 734 // CHECK: [[A_INV:%.+]] = load i32, i32* [[A_IN:%.+]], align 735 // CHECK: [[AV:%.+]] = load i32, i32* [[A]], align 736 // CHECK: [[OR:%.+]] = or i32 [[A_INV]], [[AV]] 737 // CHECK: store i32 [[OR]], i32* [[A_IN]], align 738 // CHECK: [[B_INV16:%.+]] = load i16, i16* [[B_IN:%.+]], align 739 // CHECK: [[B_INV:%.+]] = sext i16 [[B_INV16]] to i32 740 // CHECK: [[BV16:%.+]] = load i16, i16* [[B]], align 741 // CHECK: [[BV:%.+]] = sext i16 [[BV16]] to i32 742 // CHECK: [[CMP:%.+]] = icmp sgt i32 [[B_INV]], [[BV]] 743 // CHECK: br i1 [[CMP]], label {{%?}}[[DO_MAX:.+]], label {{%?}}[[MAX_ELSE:.+]] 744 // 745 // CHECK: [[DO_MAX]] 746 // CHECK: [[MAX1:%.+]] = load i16, i16* [[B_IN]], align 747 // CHECK: br label {{%?}}[[MAX_CONT:.+]] 748 // 749 // CHECK: [[MAX_ELSE]] 750 // CHECK: [[MAX2:%.+]] = load i16, i16* [[B]], align 751 // CHECK: br label {{%?}}[[MAX_CONT]] 752 // 753 // CHECK: [[MAX_CONT]] 754 // CHECK: [[B_MAX:%.+]] = phi i16 [ [[MAX1]], %[[DO_MAX]] ], [ [[MAX2]], %[[MAX_ELSE]] ] 755 // CHECK: store i16 [[B_MAX]], i16* [[B_IN]], align 756 // CHECK: call void @__kmpc_nvptx_end_reduce_nowait( 757 // CHECK: br label %[[EXIT]] 758 // 759 // CHECK: [[EXIT]] 760 // CHECK: call void @__kmpc_kernel_deinit( 761 762 // 763 // Reduction function 764 // CHECK: define internal void [[REDUCTION_FUNC:@.+]](i8*, i8*) 765 // CHECK: [[VAR1_RHS_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST_RHS:%.+]], i[[SZ]] 0, i[[SZ]] 0 766 // CHECK: [[VAR1_RHS_VOID:%.+]] = load i8*, i8** [[VAR1_RHS_REF]], 767 // CHECK: [[VAR1_RHS:%.+]] = bitcast i8* [[VAR1_RHS_VOID]] to i32* 768 // 769 // CHECK: [[VAR1_LHS_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST_LHS:%.+]], i[[SZ]] 0, i[[SZ]] 0 770 // CHECK: [[VAR1_LHS_VOID:%.+]] = load i8*, i8** [[VAR1_LHS_REF]], 771 // CHECK: [[VAR1_LHS:%.+]] = bitcast i8* [[VAR1_LHS_VOID]] to i32* 772 // 773 // CHECK: [[VAR2_RHS_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST_RHS]], i[[SZ]] 0, i[[SZ]] 1 774 // CHECK: [[VAR2_RHS_VOID:%.+]] = load i8*, i8** [[VAR2_RHS_REF]], 775 // CHECK: [[VAR2_RHS:%.+]] = bitcast i8* [[VAR2_RHS_VOID]] to i16* 776 // 777 // CHECK: [[VAR2_LHS_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST_LHS]], i[[SZ]] 0, i[[SZ]] 1 778 // CHECK: [[VAR2_LHS_VOID:%.+]] = load i8*, i8** [[VAR2_LHS_REF]], 779 // CHECK: [[VAR2_LHS:%.+]] = bitcast i8* [[VAR2_LHS_VOID]] to i16* 780 // 781 // CHECK: [[VAR1_LHS_VAL:%.+]] = load i32, i32* [[VAR1_LHS]], 782 // CHECK: [[VAR1_RHS_VAL:%.+]] = load i32, i32* [[VAR1_RHS]], 783 // CHECK: [[OR:%.+]] = or i32 [[VAR1_LHS_VAL]], [[VAR1_RHS_VAL]] 784 // CHECK: store i32 [[OR]], i32* [[VAR1_LHS]], 785 // 786 // CHECK: [[VAR2_LHS_VAL16:%.+]] = load i16, i16* [[VAR2_LHS]], 787 // CHECK: [[VAR2_LHS_VAL:%.+]] = sext i16 [[VAR2_LHS_VAL16]] to i32 788 // CHECK: [[VAR2_RHS_VAL16:%.+]] = load i16, i16* [[VAR2_RHS]], 789 // CHECK: [[VAR2_RHS_VAL:%.+]] = sext i16 [[VAR2_RHS_VAL16]] to i32 790 // 791 // CHECK: [[CMP:%.+]] = icmp sgt i32 [[VAR2_LHS_VAL]], [[VAR2_RHS_VAL]] 792 // CHECK: br i1 [[CMP]], label {{%?}}[[DO_MAX:.+]], label {{%?}}[[MAX_ELSE:.+]] 793 // 794 // CHECK: [[DO_MAX]] 795 // CHECK: [[MAX1:%.+]] = load i16, i16* [[VAR2_LHS]], align 796 // CHECK: br label {{%?}}[[MAX_CONT:.+]] 797 // 798 // CHECK: [[MAX_ELSE]] 799 // CHECK: [[MAX2:%.+]] = load i16, i16* [[VAR2_RHS]], align 800 // CHECK: br label {{%?}}[[MAX_CONT]] 801 // 802 // CHECK: [[MAX_CONT]] 803 // CHECK: [[MAXV:%.+]] = phi i16 [ [[MAX1]], %[[DO_MAX]] ], [ [[MAX2]], %[[MAX_ELSE]] ] 804 // CHECK: store i16 [[MAXV]], i16* [[VAR2_LHS]], 805 // CHECK: ret void 806 807 // 808 // Shuffle and reduce function 809 // CHECK: define internal void [[SHUFFLE_REDUCE_FN]](i8*, i16 {{.*}}, i16 {{.*}}, i16 {{.*}}) 810 // CHECK: [[REMOTE_RED_LIST:%.+]] = alloca [[RLT]], align 811 // CHECK: [[REMOTE_ELT1:%.+]] = alloca i32 812 // CHECK: [[REMOTE_ELT2:%.+]] = alloca i16 813 // 814 // CHECK: [[LANEID:%.+]] = load i16, i16* {{.+}}, align 815 // CHECK: [[LANEOFFSET:%.+]] = load i16, i16* {{.+}}, align 816 // CHECK: [[ALGVER:%.+]] = load i16, i16* {{.+}}, align 817 // 818 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 819 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 820 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 821 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i32* 822 // CHECK: [[ELT_VAL:%.+]] = load i32, i32* [[ELT]], align 823 // 824 // CHECK: [[WS32:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() 825 // CHECK: [[WS:%.+]] = trunc i32 [[WS32]] to i16 826 // CHECK: [[REMOTE_ELT1_VAL:%.+]] = call i32 @__kmpc_shuffle_int32(i32 [[ELT_VAL]], i16 [[LANEOFFSET]], i16 [[WS]]) 827 // 828 // CHECK: store i32 [[REMOTE_ELT1_VAL]], i32* [[REMOTE_ELT1]], align 829 // CHECK: [[REMOTE_ELT1C:%.+]] = bitcast i32* [[REMOTE_ELT1]] to i8* 830 // CHECK: store i8* [[REMOTE_ELT1C]], i8** [[REMOTE_ELT_REF]], align 831 // 832 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 833 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 834 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 835 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i16* 836 // CHECK: [[ELT_VAL:%.+]] = load i16, i16* [[ELT]], align 837 // 838 // CHECK: [[ELT_CAST:%.+]] = sext i16 [[ELT_VAL]] to i32 839 // CHECK: [[WS32:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() 840 // CHECK: [[WS:%.+]] = trunc i32 [[WS32]] to i16 841 // CHECK: [[REMOTE_ELT2_VAL32:%.+]] = call i32 @__kmpc_shuffle_int32(i32 [[ELT_CAST]], i16 [[LANEOFFSET]], i16 [[WS]]) 842 // CHECK: [[REMOTE_ELT2_VAL:%.+]] = trunc i32 [[REMOTE_ELT2_VAL32]] to i16 843 // 844 // CHECK: store i16 [[REMOTE_ELT2_VAL]], i16* [[REMOTE_ELT2]], align 845 // CHECK: [[REMOTE_ELT2C:%.+]] = bitcast i16* [[REMOTE_ELT2]] to i8* 846 // CHECK: store i8* [[REMOTE_ELT2C]], i8** [[REMOTE_ELT_REF]], align 847 // 848 // Condition to reduce 849 // CHECK: [[CONDALG0:%.+]] = icmp eq i16 [[ALGVER]], 0 850 // 851 // CHECK: [[COND1:%.+]] = icmp eq i16 [[ALGVER]], 1 852 // CHECK: [[COND2:%.+]] = icmp ult i16 [[LANEID]], [[LANEOFFSET]] 853 // CHECK: [[CONDALG1:%.+]] = and i1 [[COND1]], [[COND2]] 854 // 855 // CHECK: [[COND3:%.+]] = icmp eq i16 [[ALGVER]], 2 856 // CHECK: [[COND4:%.+]] = and i16 [[LANEID]], 1 857 // CHECK: [[COND5:%.+]] = icmp eq i16 [[COND4]], 0 858 // CHECK: [[COND6:%.+]] = and i1 [[COND3]], [[COND5]] 859 // CHECK: [[COND7:%.+]] = icmp sgt i16 [[LANEOFFSET]], 0 860 // CHECK: [[CONDALG2:%.+]] = and i1 [[COND6]], [[COND7]] 861 // 862 // CHECK: [[COND8:%.+]] = or i1 [[CONDALG0]], [[CONDALG1]] 863 // CHECK: [[SHOULD_REDUCE:%.+]] = or i1 [[COND8]], [[CONDALG2]] 864 // CHECK: br i1 [[SHOULD_REDUCE]], label {{%?}}[[DO_REDUCE:.+]], label {{%?}}[[REDUCE_ELSE:.+]] 865 // 866 // CHECK: [[DO_REDUCE]] 867 // CHECK: [[RED_LIST1_VOID:%.+]] = bitcast [[RLT]]* [[RED_LIST]] to i8* 868 // CHECK: [[RED_LIST2_VOID:%.+]] = bitcast [[RLT]]* [[REMOTE_RED_LIST]] to i8* 869 // CHECK: call void [[REDUCTION_FUNC]](i8* [[RED_LIST1_VOID]], i8* [[RED_LIST2_VOID]]) 870 // CHECK: br label {{%?}}[[REDUCE_CONT:.+]] 871 // 872 // CHECK: [[REDUCE_ELSE]] 873 // CHECK: br label {{%?}}[[REDUCE_CONT]] 874 // 875 // CHECK: [[REDUCE_CONT]] 876 // Now check if we should just copy over the remote reduction list 877 // CHECK: [[COND1:%.+]] = icmp eq i16 [[ALGVER]], 1 878 // CHECK: [[COND2:%.+]] = icmp uge i16 [[LANEID]], [[LANEOFFSET]] 879 // CHECK: [[SHOULD_COPY:%.+]] = and i1 [[COND1]], [[COND2]] 880 // CHECK: br i1 [[SHOULD_COPY]], label {{%?}}[[DO_COPY:.+]], label {{%?}}[[COPY_ELSE:.+]] 881 // 882 // CHECK: [[DO_COPY]] 883 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 884 // CHECK: [[REMOTE_ELT_VOID:%.+]] = load i8*, i8** [[REMOTE_ELT_REF]], 885 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 886 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 887 // CHECK: [[REMOTE_ELT:%.+]] = bitcast i8* [[REMOTE_ELT_VOID]] to i32* 888 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load i32, i32* [[REMOTE_ELT]], align 889 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i32* 890 // CHECK: store i32 [[REMOTE_ELT_VAL]], i32* [[ELT]], align 891 // 892 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 893 // CHECK: [[REMOTE_ELT_VOID:%.+]] = load i8*, i8** [[REMOTE_ELT_REF]], 894 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 895 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 896 // CHECK: [[REMOTE_ELT:%.+]] = bitcast i8* [[REMOTE_ELT_VOID]] to i16* 897 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load i16, i16* [[REMOTE_ELT]], align 898 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i16* 899 // CHECK: store i16 [[REMOTE_ELT_VAL]], i16* [[ELT]], align 900 // CHECK: br label {{%?}}[[COPY_CONT:.+]] 901 // 902 // CHECK: [[COPY_ELSE]] 903 // CHECK: br label {{%?}}[[COPY_CONT]] 904 // 905 // CHECK: [[COPY_CONT]] 906 // CHECK: void 907 908 // 909 // Inter warp copy function 910 // CHECK: define internal void [[WARP_COPY_FN]](i8*, i32) 911 // CHECK-DAG: [[LANEID:%.+]] = and i32 {{.+}}, 31 912 // CHECK-DAG: [[WARPID:%.+]] = ashr i32 {{.+}}, 5 913 // CHECK-DAG: [[RED_LIST:%.+]] = bitcast i8* {{.+}} to [[RLT]]* 914 // CHECK: [[IS_WARP_MASTER:%.+]] = icmp eq i32 [[LANEID]], 0 915 // CHECK: br i1 [[IS_WARP_MASTER]], label {{%?}}[[DO_COPY:.+]], label {{%?}}[[COPY_ELSE:.+]] 916 // 917 // [[DO_COPY]] 918 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 919 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 920 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i32* 921 // CHECK: [[ELT_VAL:%.+]] = load i32, i32* [[ELT]], align 922 // 923 // CHECK: [[MEDIUM_ELT64:%.+]] = getelementptr inbounds [32 x i64], [32 x i64] addrspace([[SHARED_ADDRSPACE]])* [[TRANSFER_STORAGE]], i64 0, i32 [[WARPID]] 924 // CHECK: [[MEDIUM_ELT:%.+]] = bitcast i64 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT64]] to i32 addrspace([[SHARED_ADDRSPACE]])* 925 // CHECK: store i32 [[ELT_VAL]], i32 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT]], align 926 // CHECK: br label {{%?}}[[COPY_CONT:.+]] 927 // 928 // CHECK: [[COPY_ELSE]] 929 // CHECK: br label {{%?}}[[COPY_CONT]] 930 // 931 // Barrier after copy to shared memory storage medium. 932 // CHECK: [[COPY_CONT]] 933 // CHECK: [[WS:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() 934 // CHECK: [[ACTIVE_THREADS:%.+]] = mul nsw i32 [[ACTIVE_WARPS:%.+]], [[WS]] 935 // CHECK: call void @llvm.nvvm.barrier(i32 1, i32 [[ACTIVE_THREADS]]) 936 // 937 // Read into warp 0. 938 // CHECK: [[IS_W0_ACTIVE_THREAD:%.+]] = icmp ult i32 [[TID:%.+]], [[ACTIVE_WARPS]] 939 // CHECK: br i1 [[IS_W0_ACTIVE_THREAD]], label {{%?}}[[DO_READ:.+]], label {{%?}}[[READ_ELSE:.+]] 940 // 941 // CHECK: [[DO_READ]] 942 // CHECK: [[MEDIUM_ELT64:%.+]] = getelementptr inbounds [32 x i64], [32 x i64] addrspace([[SHARED_ADDRSPACE]])* [[TRANSFER_STORAGE]], i64 0, i32 [[TID]] 943 // CHECK: [[MEDIUM_ELT:%.+]] = bitcast i64 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT64]] to i32 addrspace([[SHARED_ADDRSPACE]])* 944 // CHECK: [[MEDIUM_ELT_VAL:%.+]] = load i32, i32 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT]], align 945 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 946 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 947 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i32* 948 // CHECK: store i32 [[MEDIUM_ELT_VAL]], i32* [[ELT]], align 949 // CHECK: br label {{%?}}[[READ_CONT:.+]] 950 // 951 // CHECK: [[READ_ELSE]] 952 // CHECK: br label {{%?}}[[READ_CONT]] 953 // 954 // CHECK: [[READ_CONT]] 955 // CHECK: call void @llvm.nvvm.barrier(i32 1, i32 [[ACTIVE_THREADS]]) 956 // CHECK: [[IS_WARP_MASTER:%.+]] = icmp eq i32 [[LANEID]], 0 957 // CHECK: br i1 [[IS_WARP_MASTER]], label {{%?}}[[DO_COPY:.+]], label {{%?}}[[COPY_ELSE:.+]] 958 // 959 // [[DO_COPY]] 960 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 961 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 962 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i16* 963 // CHECK: [[ELT_VAL:%.+]] = load i16, i16* [[ELT]], align 964 // 965 // CHECK: [[MEDIUM_ELT64:%.+]] = getelementptr inbounds [32 x i64], [32 x i64] addrspace([[SHARED_ADDRSPACE]])* [[TRANSFER_STORAGE]], i64 0, i32 [[WARPID]] 966 // CHECK: [[MEDIUM_ELT:%.+]] = bitcast i64 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT64]] to i16 addrspace([[SHARED_ADDRSPACE]])* 967 // CHECK: store i16 [[ELT_VAL]], i16 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT]], align 968 // CHECK: br label {{%?}}[[COPY_CONT:.+]] 969 // 970 // CHECK: [[COPY_ELSE]] 971 // CHECK: br label {{%?}}[[COPY_CONT]] 972 // 973 // Barrier after copy to shared memory storage medium. 974 // CHECK: [[COPY_CONT]] 975 // CHECK: [[WS:%.+]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() 976 // CHECK: [[ACTIVE_THREADS:%.+]] = mul nsw i32 [[ACTIVE_WARPS:%.+]], [[WS]] 977 // CHECK: call void @llvm.nvvm.barrier(i32 1, i32 [[ACTIVE_THREADS]]) 978 // 979 // Read into warp 0. 980 // CHECK: [[IS_W0_ACTIVE_THREAD:%.+]] = icmp ult i32 [[TID:%.+]], [[ACTIVE_WARPS]] 981 // CHECK: br i1 [[IS_W0_ACTIVE_THREAD]], label {{%?}}[[DO_READ:.+]], label {{%?}}[[READ_ELSE:.+]] 982 // 983 // CHECK: [[DO_READ]] 984 // CHECK: [[MEDIUM_ELT64:%.+]] = getelementptr inbounds [32 x i64], [32 x i64] addrspace([[SHARED_ADDRSPACE]])* [[TRANSFER_STORAGE]], i64 0, i32 [[TID]] 985 // CHECK: [[MEDIUM_ELT:%.+]] = bitcast i64 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT64]] to i16 addrspace([[SHARED_ADDRSPACE]])* 986 // CHECK: [[MEDIUM_ELT_VAL:%.+]] = load i16, i16 addrspace([[SHARED_ADDRSPACE]])* [[MEDIUM_ELT]], align 987 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 1 988 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 989 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i16* 990 // CHECK: store i16 [[MEDIUM_ELT_VAL]], i16* [[ELT]], align 991 // CHECK: br label {{%?}}[[READ_CONT:.+]] 992 // 993 // CHECK: [[READ_ELSE]] 994 // CHECK: br label {{%?}}[[READ_CONT]] 995 // 996 // CHECK: [[READ_CONT]] 997 // CHECK: call void @llvm.nvvm.barrier(i32 1, i32 [[ACTIVE_THREADS]]) 998 // CHECK: ret 999 1000 // 1001 // Copy to scratchpad function 1002 // CHECK: define internal void [[SCRATCH_COPY_FN]](i8*, i8*, i32, i32) 1003 // CHECK: [[RED_LIST:%.+]] = bitcast i8* {{.+}} to [[RLT]]* 1004 // CHECK: [[SCRATCHPAD_PTR:%.+]] = load i8*, i8** {{.+}}, align 1005 // CHECK-64: [[TEAM32:%.+]] = load i32, i32* {{.+}}, align 1006 // CHECK-64: [[TEAM:%.+]] = sext i32 [[TEAM32]] to i64 1007 // CHECK-32: [[TEAM:%.+]] = load i32, i32* {{.+}}, align 1008 // CHECK-64: [[NUM_TEAMS32:%.+]] = load i32, i32* {{.+}}, align 1009 // CHECK-64: [[NUM_TEAMS:%.+]] = sext i32 [[NUM_TEAMS32]] to i64 1010 // CHECK-32: [[NUM_TEAMS:%.+]] = load i32, i32* {{.+}}, align 1011 // CHECK: [[SCRATCHPAD:%.+]] = ptrtoint i8* [[SCRATCHPAD_PTR]] to i[[SZ]] 1012 // 1013 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 1014 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 1015 // 1016 // CHECK: [[P:%.+]] = mul i[[SZ]] 4, [[TEAM]] 1017 // CHECK: [[SCRATCHPAD_ELT_PTR64:%.+]] = add i[[SZ]] [[SCRATCHPAD]], [[P]] 1018 // CHECK: [[SCRATCHPAD_ELT_PTR_VOID:%.+]] = inttoptr i[[SZ]] [[SCRATCHPAD_ELT_PTR64]] to i8* 1019 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i32* 1020 // CHECK: [[ELT_VAL:%.+]] = load i32, i32* [[ELT]], align 1021 // CHECK: [[SCRATCHPAD_ELT_PTR:%.+]] = bitcast i8* [[SCRATCHPAD_ELT_PTR_VOID]] to i32* 1022 // CHECK: store i32 [[ELT_VAL]], i32* [[SCRATCHPAD_ELT_PTR]], align 1023 // 1024 // CHECK: [[OF:%.+]] = mul i[[SZ]] [[NUM_TEAMS]], 4 1025 // CHECK: [[POS1:%.+]] = add i[[SZ]] [[SCRATCHPAD]], [[OF]] 1026 // CHECK: [[POS2:%.+]] = sub i[[SZ]] [[POS1]], 1 1027 // CHECK: [[POS3:%.+]] = sdiv i[[SZ]] [[POS2]], 256 1028 // CHECK: [[POS4:%.+]] = add i[[SZ]] [[POS3]], 1 1029 // CHECK: [[SCRATCHPAD_NEXT:%.+]] = mul i[[SZ]] [[POS4]], 256 1030 // 1031 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 1032 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 1033 // 1034 // CHECK: [[P:%.+]] = mul i[[SZ]] 2, [[TEAM]] 1035 // CHECK: [[SCRATCHPAD_ELT_PTR64:%.+]] = add i[[SZ]] [[SCRATCHPAD_NEXT]], [[P]] 1036 // CHECK: [[SCRATCHPAD_ELT_PTR_VOID:%.+]] = inttoptr i[[SZ]] [[SCRATCHPAD_ELT_PTR64]] to i8* 1037 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i16* 1038 // CHECK: [[ELT_VAL:%.+]] = load i16, i16* [[ELT]], align 1039 // CHECK: [[SCRATCHPAD_ELT_PTR:%.+]] = bitcast i8* [[SCRATCHPAD_ELT_PTR_VOID]] to i16* 1040 // CHECK: store i16 [[ELT_VAL]], i16* [[SCRATCHPAD_ELT_PTR]], align 1041 // 1042 // CHECK: ret 1043 1044 // 1045 // Load and reduce function 1046 // CHECK: define internal void [[LOAD_REDUCE_FN]](i8*, i8*, i32, i32, i32) 1047 // CHECK: [[REMOTE_RED_LIST:%.+]] = alloca [[RLT]], align 1048 // CHECK: [[REMOTE_ELT1:%.+]] = alloca i32 1049 // CHECK: [[REMOTE_ELT2:%.+]] = alloca i16 1050 // CHECK: [[RED_LIST:%.+]] = bitcast i8* {{.+}} to [[RLT]]* 1051 // CHECK: [[SCRATCHPAD_PTR:%.+]] = load i8*, i8** {{.+}}, align 1052 // CHECK-64: [[TEAM32:%.+]] = load i32, i32* {{.+}}, align 1053 // CHECK-64: [[TEAM:%.+]] = sext i32 [[TEAM32]] to i64 1054 // CHECK-32: [[TEAM:%.+]] = load i32, i32* {{.+}}, align 1055 // CHECK-64: [[NUM_TEAMS32:%.+]] = load i32, i32* {{.+}}, align 1056 // CHECK-64: [[NUM_TEAMS:%.+]] = sext i32 [[NUM_TEAMS32]] to i64 1057 // CHECK-32: [[NUM_TEAMS:%.+]] = load i32, i32* {{.+}}, align 1058 // CHECK: [[SHOULD_REDUCE:%.+]] = load i32, i32* {{.+}}, align 1059 // CHECK: [[SCRATCHPAD:%.+]] = ptrtoint i8* [[SCRATCHPAD_PTR]] to i[[SZ]] 1060 // 1061 // CHECK: [[P:%.+]] = mul i[[SZ]] 4, [[TEAM]] 1062 // CHECK: [[SCRATCHPAD_ELT_PTR64:%.+]] = add i[[SZ]] [[SCRATCHPAD]], [[P]] 1063 // CHECK: [[SCRATCHPAD_ELT_PTR_VOID:%.+]] = inttoptr i[[SZ]] [[SCRATCHPAD_ELT_PTR64]] to i8* 1064 1065 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 0 1066 // CHECK: [[SCRATCHPAD_ELT_PTR:%.+]] = bitcast i8* [[SCRATCHPAD_ELT_PTR_VOID]] to i32* 1067 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load i32, i32* [[SCRATCHPAD_ELT_PTR]], align 1068 // CHECK: store i32 [[REMOTE_ELT_VAL]], i32* [[REMOTE_ELT1]], align 1069 // CHECK: [[REMOTE_ELT1_PTR:%.+]] = bitcast i32* [[REMOTE_ELT1]] to i8* 1070 // CHECK: store i8* [[REMOTE_ELT1_PTR]], i8** [[REMOTE_ELT_REF]], align 1071 // 1072 // CHECK: [[OF:%.+]] = mul i[[SZ]] [[NUM_TEAMS]], 4 1073 // CHECK: [[POS1:%.+]] = add i[[SZ]] [[SCRATCHPAD]], [[OF]] 1074 // CHECK: [[POS2:%.+]] = sub i[[SZ]] [[POS1]], 1 1075 // CHECK: [[POS3:%.+]] = sdiv i[[SZ]] [[POS2]], 256 1076 // CHECK: [[POS4:%.+]] = add i[[SZ]] [[POS3]], 1 1077 // CHECK: [[SCRATCHPAD_NEXT:%.+]] = mul i[[SZ]] [[POS4]], 256 1078 // 1079 // CHECK: [[P:%.+]] = mul i[[SZ]] 2, [[TEAM]] 1080 // CHECK: [[SCRATCHPAD_ELT_PTR64:%.+]] = add i[[SZ]] [[SCRATCHPAD_NEXT]], [[P]] 1081 // CHECK: [[SCRATCHPAD_ELT_PTR_VOID:%.+]] = inttoptr i[[SZ]] [[SCRATCHPAD_ELT_PTR64]] to i8* 1082 1083 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST]], i[[SZ]] 0, i[[SZ]] 1 1084 // CHECK: [[SCRATCHPAD_ELT_PTR:%.+]] = bitcast i8* [[SCRATCHPAD_ELT_PTR_VOID]] to i16* 1085 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load i16, i16* [[SCRATCHPAD_ELT_PTR]], align 1086 // CHECK: store i16 [[REMOTE_ELT_VAL]], i16* [[REMOTE_ELT2]], align 1087 // CHECK: [[REMOTE_ELT_PTR:%.+]] = bitcast i16* [[REMOTE_ELT2]] to i8* 1088 // CHECK: store i8* [[REMOTE_ELT_PTR]], i8** [[REMOTE_ELT_REF]], align 1089 // 1090 // CHECK: [[REDUCE:%.+]] = icmp eq i32 [[SHOULD_REDUCE]], 1 1091 // CHECK: br i1 [[REDUCE]], label {{%?}}[[DO_REDUCE:.+]], label {{%?}}[[REDUCE_ELSE:.+]] 1092 // 1093 // CHECK: [[DO_REDUCE]] 1094 // CHECK: [[RED_LIST1_VOID:%.+]] = bitcast [[RLT]]* [[RED_LIST]] to i8* 1095 // CHECK: [[RED_LIST2_VOID:%.+]] = bitcast [[RLT]]* [[REMOTE_RED_LIST]] to i8* 1096 // CHECK: call void [[REDUCTION_FUNC]](i8* [[RED_LIST1_VOID]], i8* [[RED_LIST2_VOID]]) 1097 // CHECK: br label {{%?}}[[REDUCE_CONT:.+]] 1098 // 1099 // Copy element from remote reduce list 1100 // CHECK: [[REDUCE_ELSE]] 1101 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 1102 // CHECK: [[REMOTE_ELT_VOID:%.+]] = load i8*, i8** [[REMOTE_ELT_REF]], 1103 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 0 1104 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 1105 // CHECK: [[REMOTE_ELT:%.+]] = bitcast i8* [[REMOTE_ELT_VOID]] to i32* 1106 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load i32, i32* [[REMOTE_ELT]], align 1107 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i32* 1108 // CHECK: store i32 [[REMOTE_ELT_VAL]], i32* [[ELT]], align 1109 // 1110 // CHECK: [[REMOTE_ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[REMOTE_RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 1 1111 // CHECK: [[REMOTE_ELT_VOID:%.+]] = load i8*, i8** [[REMOTE_ELT_REF]], 1112 // CHECK: [[ELT_REF:%.+]] = getelementptr inbounds [[RLT]], [[RLT]]* [[RED_LIST:%.+]], i[[SZ]] 0, i[[SZ]] 1 1113 // CHECK: [[ELT_VOID:%.+]] = load i8*, i8** [[ELT_REF]], 1114 // CHECK: [[REMOTE_ELT:%.+]] = bitcast i8* [[REMOTE_ELT_VOID]] to i16* 1115 // CHECK: [[REMOTE_ELT_VAL:%.+]] = load i16, i16* [[REMOTE_ELT]], align 1116 // CHECK: [[ELT:%.+]] = bitcast i8* [[ELT_VOID]] to i16* 1117 // CHECK: store i16 [[REMOTE_ELT_VAL]], i16* [[ELT]], align 1118 // CHECK: br label {{%?}}[[REDUCE_CONT]] 1119 // 1120 // CHECK: [[REDUCE_CONT]] 1121 // CHECK: ret 1122 1123 1124 #endif 1125