1 // REQUIRES: nvptx-registered-target 2 // RUN: %clang_cc1 -triple nvptx-unknown-unknown -target-cpu sm_60 \ 3 // RUN: -fcuda-is-device -S -emit-llvm -o - -x cuda %s \ 4 // RUN: | FileCheck -check-prefix=CHECK -check-prefix=LP32 %s 5 // RUN: %clang_cc1 -triple nvptx64-unknown-unknown -target-cpu sm_60 \ 6 // RUN: -fcuda-is-device -S -emit-llvm -o - -x cuda %s \ 7 // RUN: | FileCheck -check-prefix=CHECK -check-prefix=LP64 %s 8 // RUN: %clang_cc1 -triple nvptx-unknown-unknown -target-cpu sm_53 \ 9 // RUN: -DERROR_CHECK -fcuda-is-device -S -o /dev/null -x cuda -verify %s 10 11 #define __device__ __attribute__((device)) 12 #define __global__ __attribute__((global)) 13 #define __shared__ __attribute__((shared)) 14 #define __constant__ __attribute__((constant)) 15 16 __device__ int read_tid() { 17 18 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.tid.x() 19 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.tid.y() 20 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.tid.z() 21 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.tid.w() 22 23 int x = __nvvm_read_ptx_sreg_tid_x(); 24 int y = __nvvm_read_ptx_sreg_tid_y(); 25 int z = __nvvm_read_ptx_sreg_tid_z(); 26 int w = __nvvm_read_ptx_sreg_tid_w(); 27 28 return x + y + z + w; 29 30 } 31 32 __device__ int read_ntid() { 33 34 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.ntid.x() 35 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.ntid.y() 36 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.ntid.z() 37 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.ntid.w() 38 39 int x = __nvvm_read_ptx_sreg_ntid_x(); 40 int y = __nvvm_read_ptx_sreg_ntid_y(); 41 int z = __nvvm_read_ptx_sreg_ntid_z(); 42 int w = __nvvm_read_ptx_sreg_ntid_w(); 43 44 return x + y + z + w; 45 46 } 47 48 __device__ int read_ctaid() { 49 50 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() 51 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.y() 52 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.z() 53 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.ctaid.w() 54 55 int x = __nvvm_read_ptx_sreg_ctaid_x(); 56 int y = __nvvm_read_ptx_sreg_ctaid_y(); 57 int z = __nvvm_read_ptx_sreg_ctaid_z(); 58 int w = __nvvm_read_ptx_sreg_ctaid_w(); 59 60 return x + y + z + w; 61 62 } 63 64 __device__ int read_nctaid() { 65 66 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x() 67 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.y() 68 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.z() 69 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.nctaid.w() 70 71 int x = __nvvm_read_ptx_sreg_nctaid_x(); 72 int y = __nvvm_read_ptx_sreg_nctaid_y(); 73 int z = __nvvm_read_ptx_sreg_nctaid_z(); 74 int w = __nvvm_read_ptx_sreg_nctaid_w(); 75 76 return x + y + z + w; 77 78 } 79 80 __device__ int read_ids() { 81 82 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.laneid() 83 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.warpid() 84 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.nwarpid() 85 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.smid() 86 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.nsmid() 87 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.gridid() 88 89 int a = __nvvm_read_ptx_sreg_laneid(); 90 int b = __nvvm_read_ptx_sreg_warpid(); 91 int c = __nvvm_read_ptx_sreg_nwarpid(); 92 int d = __nvvm_read_ptx_sreg_smid(); 93 int e = __nvvm_read_ptx_sreg_nsmid(); 94 int f = __nvvm_read_ptx_sreg_gridid(); 95 96 return a + b + c + d + e + f; 97 98 } 99 100 __device__ int read_lanemasks() { 101 102 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.lanemask.eq() 103 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.lanemask.le() 104 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.lanemask.lt() 105 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.lanemask.ge() 106 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.lanemask.gt() 107 108 int a = __nvvm_read_ptx_sreg_lanemask_eq(); 109 int b = __nvvm_read_ptx_sreg_lanemask_le(); 110 int c = __nvvm_read_ptx_sreg_lanemask_lt(); 111 int d = __nvvm_read_ptx_sreg_lanemask_ge(); 112 int e = __nvvm_read_ptx_sreg_lanemask_gt(); 113 114 return a + b + c + d + e; 115 116 } 117 118 __device__ long long read_clocks() { 119 120 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.clock() 121 // CHECK: call i64 @llvm.nvvm.read.ptx.sreg.clock64() 122 123 int a = __nvvm_read_ptx_sreg_clock(); 124 long long b = __nvvm_read_ptx_sreg_clock64(); 125 126 return a + b; 127 } 128 129 __device__ int read_pms() { 130 131 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.pm0() 132 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.pm1() 133 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.pm2() 134 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.pm3() 135 136 int a = __nvvm_read_ptx_sreg_pm0(); 137 int b = __nvvm_read_ptx_sreg_pm1(); 138 int c = __nvvm_read_ptx_sreg_pm2(); 139 int d = __nvvm_read_ptx_sreg_pm3(); 140 141 return a + b + c + d; 142 143 } 144 145 __device__ void sync() { 146 147 // CHECK: call void @llvm.nvvm.bar.sync(i32 0) 148 149 __nvvm_bar_sync(0); 150 151 } 152 153 154 // NVVM intrinsics 155 156 // The idea is not to test all intrinsics, just that Clang is recognizing the 157 // builtins defined in BuiltinsNVPTX.def 158 __device__ void nvvm_math(float f1, float f2, double d1, double d2) { 159 // CHECK: call float @llvm.nvvm.fmax.f 160 float t1 = __nvvm_fmax_f(f1, f2); 161 // CHECK: call float @llvm.nvvm.fmin.f 162 float t2 = __nvvm_fmin_f(f1, f2); 163 // CHECK: call float @llvm.nvvm.sqrt.rn.f 164 float t3 = __nvvm_sqrt_rn_f(f1); 165 // CHECK: call float @llvm.nvvm.rcp.rn.f 166 float t4 = __nvvm_rcp_rn_f(f2); 167 // CHECK: call float @llvm.nvvm.add.rn.f 168 float t5 = __nvvm_add_rn_f(f1, f2); 169 170 // CHECK: call double @llvm.nvvm.fmax.d 171 double td1 = __nvvm_fmax_d(d1, d2); 172 // CHECK: call double @llvm.nvvm.fmin.d 173 double td2 = __nvvm_fmin_d(d1, d2); 174 // CHECK: call double @llvm.nvvm.sqrt.rn.d 175 double td3 = __nvvm_sqrt_rn_d(d1); 176 // CHECK: call double @llvm.nvvm.rcp.rn.d 177 double td4 = __nvvm_rcp_rn_d(d2); 178 179 // CHECK: call void @llvm.nvvm.membar.cta() 180 __nvvm_membar_cta(); 181 // CHECK: call void @llvm.nvvm.membar.gl() 182 __nvvm_membar_gl(); 183 // CHECK: call void @llvm.nvvm.membar.sys() 184 __nvvm_membar_sys(); 185 // CHECK: call void @llvm.nvvm.barrier0() 186 __syncthreads(); 187 } 188 189 __device__ int di; 190 __shared__ int si; 191 __device__ long dl; 192 __shared__ long sl; 193 __device__ long long dll; 194 __shared__ long long sll; 195 196 // Check for atomic intrinsics 197 // CHECK-LABEL: nvvm_atom 198 __device__ void nvvm_atom(float *fp, float f, double *dfp, double df, int *ip, 199 int i, unsigned int *uip, unsigned ui, long *lp, 200 long l, long long *llp, long long ll) { 201 // CHECK: atomicrmw add 202 __nvvm_atom_add_gen_i(ip, i); 203 // CHECK: atomicrmw add 204 __nvvm_atom_add_gen_l(&dl, l); 205 // CHECK: atomicrmw add 206 __nvvm_atom_add_gen_ll(&sll, ll); 207 208 // CHECK: atomicrmw sub 209 __nvvm_atom_sub_gen_i(ip, i); 210 // CHECK: atomicrmw sub 211 __nvvm_atom_sub_gen_l(&dl, l); 212 // CHECK: atomicrmw sub 213 __nvvm_atom_sub_gen_ll(&sll, ll); 214 215 // CHECK: atomicrmw and 216 __nvvm_atom_and_gen_i(ip, i); 217 // CHECK: atomicrmw and 218 __nvvm_atom_and_gen_l(&dl, l); 219 // CHECK: atomicrmw and 220 __nvvm_atom_and_gen_ll(&sll, ll); 221 222 // CHECK: atomicrmw or 223 __nvvm_atom_or_gen_i(ip, i); 224 // CHECK: atomicrmw or 225 __nvvm_atom_or_gen_l(&dl, l); 226 // CHECK: atomicrmw or 227 __nvvm_atom_or_gen_ll(&sll, ll); 228 229 // CHECK: atomicrmw xor 230 __nvvm_atom_xor_gen_i(ip, i); 231 // CHECK: atomicrmw xor 232 __nvvm_atom_xor_gen_l(&dl, l); 233 // CHECK: atomicrmw xor 234 __nvvm_atom_xor_gen_ll(&sll, ll); 235 236 // CHECK: atomicrmw xchg 237 __nvvm_atom_xchg_gen_i(ip, i); 238 // CHECK: atomicrmw xchg 239 __nvvm_atom_xchg_gen_l(&dl, l); 240 // CHECK: atomicrmw xchg 241 __nvvm_atom_xchg_gen_ll(&sll, ll); 242 243 // CHECK: atomicrmw max i32* 244 __nvvm_atom_max_gen_i(ip, i); 245 // CHECK: atomicrmw umax i32* 246 __nvvm_atom_max_gen_ui((unsigned int *)ip, i); 247 // CHECK: atomicrmw max 248 __nvvm_atom_max_gen_l(&dl, l); 249 // CHECK: atomicrmw umax 250 __nvvm_atom_max_gen_ul((unsigned long *)&dl, l); 251 // CHECK: atomicrmw max i64* 252 __nvvm_atom_max_gen_ll(&sll, ll); 253 // CHECK: atomicrmw umax i64* 254 __nvvm_atom_max_gen_ull((unsigned long long *)&sll, ll); 255 256 // CHECK: atomicrmw min i32* 257 __nvvm_atom_min_gen_i(ip, i); 258 // CHECK: atomicrmw umin i32* 259 __nvvm_atom_min_gen_ui((unsigned int *)ip, i); 260 // CHECK: atomicrmw min 261 __nvvm_atom_min_gen_l(&dl, l); 262 // CHECK: atomicrmw umin 263 __nvvm_atom_min_gen_ul((unsigned long *)&dl, l); 264 // CHECK: atomicrmw min i64* 265 __nvvm_atom_min_gen_ll(&sll, ll); 266 // CHECK: atomicrmw umin i64* 267 __nvvm_atom_min_gen_ull((unsigned long long *)&sll, ll); 268 269 // CHECK: cmpxchg 270 // CHECK-NEXT: extractvalue { i32, i1 } {{%[0-9]+}}, 0 271 __nvvm_atom_cas_gen_i(ip, 0, i); 272 // CHECK: cmpxchg 273 // CHECK-NEXT: extractvalue { {{i32|i64}}, i1 } {{%[0-9]+}}, 0 274 __nvvm_atom_cas_gen_l(&dl, 0, l); 275 // CHECK: cmpxchg 276 // CHECK-NEXT: extractvalue { i64, i1 } {{%[0-9]+}}, 0 277 __nvvm_atom_cas_gen_ll(&sll, 0, ll); 278 279 // CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32 280 __nvvm_atom_add_gen_f(fp, f); 281 282 // CHECK: call i32 @llvm.nvvm.atomic.load.inc.32.p0i32 283 __nvvm_atom_inc_gen_ui(uip, ui); 284 285 // CHECK: call i32 @llvm.nvvm.atomic.load.dec.32.p0i32 286 __nvvm_atom_dec_gen_ui(uip, ui); 287 288 289 ////////////////////////////////////////////////////////////////// 290 // Atomics with scope (only supported on sm_60+). 291 292 #if ERROR_CHECK || __CUDA_ARCH__ >= 600 293 294 // CHECK: call i32 @llvm.nvvm.atomic.add.gen.i.cta.i32.p0i32 295 // expected-error@+1 {{'__nvvm_atom_cta_add_gen_i' needs target feature satom}} 296 __nvvm_atom_cta_add_gen_i(ip, i); 297 // LP32: call i32 @llvm.nvvm.atomic.add.gen.i.cta.i32.p0i32 298 // LP64: call i64 @llvm.nvvm.atomic.add.gen.i.cta.i64.p0i64 299 // expected-error@+1 {{'__nvvm_atom_cta_add_gen_l' needs target feature satom}} 300 __nvvm_atom_cta_add_gen_l(&dl, l); 301 // CHECK: call i64 @llvm.nvvm.atomic.add.gen.i.cta.i64.p0i64 302 // expected-error@+1 {{'__nvvm_atom_cta_add_gen_ll' needs target feature satom}} 303 __nvvm_atom_cta_add_gen_ll(&sll, ll); 304 // CHECK: call i32 @llvm.nvvm.atomic.add.gen.i.sys.i32.p0i32 305 // expected-error@+1 {{'__nvvm_atom_sys_add_gen_i' needs target feature satom}} 306 __nvvm_atom_sys_add_gen_i(ip, i); 307 // LP32: call i32 @llvm.nvvm.atomic.add.gen.i.sys.i32.p0i32 308 // LP64: call i64 @llvm.nvvm.atomic.add.gen.i.sys.i64.p0i64 309 // expected-error@+1 {{'__nvvm_atom_sys_add_gen_l' needs target feature satom}} 310 __nvvm_atom_sys_add_gen_l(&dl, l); 311 // CHECK: call i64 @llvm.nvvm.atomic.add.gen.i.sys.i64.p0i64 312 // expected-error@+1 {{'__nvvm_atom_sys_add_gen_ll' needs target feature satom}} 313 __nvvm_atom_sys_add_gen_ll(&sll, ll); 314 315 // CHECK: call float @llvm.nvvm.atomic.add.gen.f.cta.f32.p0f32 316 // expected-error@+1 {{'__nvvm_atom_cta_add_gen_f' needs target feature satom}} 317 __nvvm_atom_cta_add_gen_f(fp, f); 318 // CHECK: call double @llvm.nvvm.atomic.add.gen.f.cta.f64.p0f64 319 // expected-error@+1 {{'__nvvm_atom_cta_add_gen_d' needs target feature satom}} 320 __nvvm_atom_cta_add_gen_d(dfp, df); 321 // CHECK: call float @llvm.nvvm.atomic.add.gen.f.sys.f32.p0f32 322 // expected-error@+1 {{'__nvvm_atom_sys_add_gen_f' needs target feature satom}} 323 __nvvm_atom_sys_add_gen_f(fp, f); 324 // CHECK: call double @llvm.nvvm.atomic.add.gen.f.sys.f64.p0f64 325 // expected-error@+1 {{'__nvvm_atom_sys_add_gen_d' needs target feature satom}} 326 __nvvm_atom_sys_add_gen_d(dfp, df); 327 328 // CHECK: call i32 @llvm.nvvm.atomic.exch.gen.i.cta.i32.p0i32 329 // expected-error@+1 {{'__nvvm_atom_cta_xchg_gen_i' needs target feature satom}} 330 __nvvm_atom_cta_xchg_gen_i(ip, i); 331 // LP32: call i32 @llvm.nvvm.atomic.exch.gen.i.cta.i32.p0i32 332 // LP64: call i64 @llvm.nvvm.atomic.exch.gen.i.cta.i64.p0i64 333 // expected-error@+1 {{'__nvvm_atom_cta_xchg_gen_l' needs target feature satom}} 334 __nvvm_atom_cta_xchg_gen_l(&dl, l); 335 // CHECK: call i64 @llvm.nvvm.atomic.exch.gen.i.cta.i64.p0i64 336 // expected-error@+1 {{'__nvvm_atom_cta_xchg_gen_ll' needs target feature satom}} 337 __nvvm_atom_cta_xchg_gen_ll(&sll, ll); 338 339 // CHECK: call i32 @llvm.nvvm.atomic.exch.gen.i.sys.i32.p0i32 340 // expected-error@+1 {{'__nvvm_atom_sys_xchg_gen_i' needs target feature satom}} 341 __nvvm_atom_sys_xchg_gen_i(ip, i); 342 // LP32: call i32 @llvm.nvvm.atomic.exch.gen.i.sys.i32.p0i32 343 // LP64: call i64 @llvm.nvvm.atomic.exch.gen.i.sys.i64.p0i64 344 // expected-error@+1 {{'__nvvm_atom_sys_xchg_gen_l' needs target feature satom}} 345 __nvvm_atom_sys_xchg_gen_l(&dl, l); 346 // CHECK: call i64 @llvm.nvvm.atomic.exch.gen.i.sys.i64.p0i64 347 // expected-error@+1 {{'__nvvm_atom_sys_xchg_gen_ll' needs target feature satom}} 348 __nvvm_atom_sys_xchg_gen_ll(&sll, ll); 349 350 // CHECK: call i32 @llvm.nvvm.atomic.max.gen.i.cta.i32.p0i32 351 // expected-error@+1 {{'__nvvm_atom_cta_max_gen_i' needs target feature satom}} 352 __nvvm_atom_cta_max_gen_i(ip, i); 353 // CHECK: call i32 @llvm.nvvm.atomic.max.gen.i.cta.i32.p0i32 354 // expected-error@+1 {{'__nvvm_atom_cta_max_gen_ui' needs target feature satom}} 355 __nvvm_atom_cta_max_gen_ui((unsigned int *)ip, i); 356 // LP32: call i32 @llvm.nvvm.atomic.max.gen.i.cta.i32.p0i32 357 // LP64: call i64 @llvm.nvvm.atomic.max.gen.i.cta.i64.p0i64 358 // expected-error@+1 {{'__nvvm_atom_cta_max_gen_l' needs target feature satom}} 359 __nvvm_atom_cta_max_gen_l(&dl, l); 360 // LP32: call i32 @llvm.nvvm.atomic.max.gen.i.cta.i32.p0i32 361 // LP64: call i64 @llvm.nvvm.atomic.max.gen.i.cta.i64.p0i64 362 // expected-error@+1 {{'__nvvm_atom_cta_max_gen_ul' needs target feature satom}} 363 __nvvm_atom_cta_max_gen_ul((unsigned long *)lp, l); 364 // CHECK: call i64 @llvm.nvvm.atomic.max.gen.i.cta.i64.p0i64 365 // expected-error@+1 {{'__nvvm_atom_cta_max_gen_ll' needs target feature satom}} 366 __nvvm_atom_cta_max_gen_ll(&sll, ll); 367 // CHECK: call i64 @llvm.nvvm.atomic.max.gen.i.cta.i64.p0i64 368 // expected-error@+1 {{'__nvvm_atom_cta_max_gen_ull' needs target feature satom}} 369 __nvvm_atom_cta_max_gen_ull((unsigned long long *)llp, ll); 370 371 // CHECK: call i32 @llvm.nvvm.atomic.max.gen.i.sys.i32.p0i32 372 // expected-error@+1 {{'__nvvm_atom_sys_max_gen_i' needs target feature satom}} 373 __nvvm_atom_sys_max_gen_i(ip, i); 374 // CHECK: call i32 @llvm.nvvm.atomic.max.gen.i.sys.i32.p0i32 375 // expected-error@+1 {{'__nvvm_atom_sys_max_gen_ui' needs target feature satom}} 376 __nvvm_atom_sys_max_gen_ui((unsigned int *)ip, i); 377 // LP32: call i32 @llvm.nvvm.atomic.max.gen.i.sys.i32.p0i32 378 // LP64: call i64 @llvm.nvvm.atomic.max.gen.i.sys.i64.p0i64 379 // expected-error@+1 {{'__nvvm_atom_sys_max_gen_l' needs target feature satom}} 380 __nvvm_atom_sys_max_gen_l(&dl, l); 381 // LP32: call i32 @llvm.nvvm.atomic.max.gen.i.sys.i32.p0i32 382 // LP64: call i64 @llvm.nvvm.atomic.max.gen.i.sys.i64.p0i64 383 // expected-error@+1 {{'__nvvm_atom_sys_max_gen_ul' needs target feature satom}} 384 __nvvm_atom_sys_max_gen_ul((unsigned long *)lp, l); 385 // CHECK: call i64 @llvm.nvvm.atomic.max.gen.i.sys.i64.p0i64 386 // expected-error@+1 {{'__nvvm_atom_sys_max_gen_ll' needs target feature satom}} 387 __nvvm_atom_sys_max_gen_ll(&sll, ll); 388 // CHECK: call i64 @llvm.nvvm.atomic.max.gen.i.sys.i64.p0i64 389 // expected-error@+1 {{'__nvvm_atom_sys_max_gen_ull' needs target feature satom}} 390 __nvvm_atom_sys_max_gen_ull((unsigned long long *)llp, ll); 391 392 // CHECK: call i32 @llvm.nvvm.atomic.min.gen.i.cta.i32.p0i32 393 // expected-error@+1 {{'__nvvm_atom_cta_min_gen_i' needs target feature satom}} 394 __nvvm_atom_cta_min_gen_i(ip, i); 395 // CHECK: call i32 @llvm.nvvm.atomic.min.gen.i.cta.i32.p0i32 396 // expected-error@+1 {{'__nvvm_atom_cta_min_gen_ui' needs target feature satom}} 397 __nvvm_atom_cta_min_gen_ui((unsigned int *)ip, i); 398 // LP32: call i32 @llvm.nvvm.atomic.min.gen.i.cta.i32.p0i32 399 // LP64: call i64 @llvm.nvvm.atomic.min.gen.i.cta.i64.p0i64 400 // expected-error@+1 {{'__nvvm_atom_cta_min_gen_l' needs target feature satom}} 401 __nvvm_atom_cta_min_gen_l(&dl, l); 402 // LP32: call i32 @llvm.nvvm.atomic.min.gen.i.cta.i32.p0i32 403 // LP64: call i64 @llvm.nvvm.atomic.min.gen.i.cta.i64.p0i64 404 // expected-error@+1 {{'__nvvm_atom_cta_min_gen_ul' needs target feature satom}} 405 __nvvm_atom_cta_min_gen_ul((unsigned long *)lp, l); 406 // CHECK: call i64 @llvm.nvvm.atomic.min.gen.i.cta.i64.p0i64 407 // expected-error@+1 {{'__nvvm_atom_cta_min_gen_ll' needs target feature satom}} 408 __nvvm_atom_cta_min_gen_ll(&sll, ll); 409 // CHECK: call i64 @llvm.nvvm.atomic.min.gen.i.cta.i64.p0i64 410 // expected-error@+1 {{'__nvvm_atom_cta_min_gen_ull' needs target feature satom}} 411 __nvvm_atom_cta_min_gen_ull((unsigned long long *)llp, ll); 412 413 // CHECK: call i32 @llvm.nvvm.atomic.min.gen.i.sys.i32.p0i32 414 // expected-error@+1 {{'__nvvm_atom_sys_min_gen_i' needs target feature satom}} 415 __nvvm_atom_sys_min_gen_i(ip, i); 416 // CHECK: call i32 @llvm.nvvm.atomic.min.gen.i.sys.i32.p0i32 417 // expected-error@+1 {{'__nvvm_atom_sys_min_gen_ui' needs target feature satom}} 418 __nvvm_atom_sys_min_gen_ui((unsigned int *)ip, i); 419 // LP32: call i32 @llvm.nvvm.atomic.min.gen.i.sys.i32.p0i32 420 // LP64: call i64 @llvm.nvvm.atomic.min.gen.i.sys.i64.p0i64 421 // expected-error@+1 {{'__nvvm_atom_sys_min_gen_l' needs target feature satom}} 422 __nvvm_atom_sys_min_gen_l(&dl, l); 423 // LP32: call i32 @llvm.nvvm.atomic.min.gen.i.sys.i32.p0i32 424 // LP64: call i64 @llvm.nvvm.atomic.min.gen.i.sys.i64.p0i64 425 // expected-error@+1 {{'__nvvm_atom_sys_min_gen_ul' needs target feature satom}} 426 __nvvm_atom_sys_min_gen_ul((unsigned long *)lp, l); 427 // CHECK: call i64 @llvm.nvvm.atomic.min.gen.i.sys.i64.p0i64 428 // expected-error@+1 {{'__nvvm_atom_sys_min_gen_ll' needs target feature satom}} 429 __nvvm_atom_sys_min_gen_ll(&sll, ll); 430 // CHECK: call i64 @llvm.nvvm.atomic.min.gen.i.sys.i64.p0i64 431 // expected-error@+1 {{'__nvvm_atom_sys_min_gen_ull' needs target feature satom}} 432 __nvvm_atom_sys_min_gen_ull((unsigned long long *)llp, ll); 433 434 // CHECK: call i32 @llvm.nvvm.atomic.inc.gen.i.cta.i32.p0i32 435 // expected-error@+1 {{'__nvvm_atom_cta_inc_gen_ui' needs target feature satom}} 436 __nvvm_atom_cta_inc_gen_ui((unsigned int *)ip, i); 437 // CHECK: call i32 @llvm.nvvm.atomic.inc.gen.i.sys.i32.p0i32 438 // expected-error@+1 {{'__nvvm_atom_sys_inc_gen_ui' needs target feature satom}} 439 __nvvm_atom_sys_inc_gen_ui((unsigned int *)ip, i); 440 441 // CHECK: call i32 @llvm.nvvm.atomic.dec.gen.i.cta.i32.p0i32 442 // expected-error@+1 {{'__nvvm_atom_cta_dec_gen_ui' needs target feature satom}} 443 __nvvm_atom_cta_dec_gen_ui((unsigned int *)ip, i); 444 // CHECK: call i32 @llvm.nvvm.atomic.dec.gen.i.sys.i32.p0i32 445 // expected-error@+1 {{'__nvvm_atom_sys_dec_gen_ui' needs target feature satom}} 446 __nvvm_atom_sys_dec_gen_ui((unsigned int *)ip, i); 447 448 // CHECK: call i32 @llvm.nvvm.atomic.and.gen.i.cta.i32.p0i32 449 // expected-error@+1 {{'__nvvm_atom_cta_and_gen_i' needs target feature satom}} 450 __nvvm_atom_cta_and_gen_i(ip, i); 451 // LP32: call i32 @llvm.nvvm.atomic.and.gen.i.cta.i32.p0i32 452 // LP64: call i64 @llvm.nvvm.atomic.and.gen.i.cta.i64.p0i64 453 // expected-error@+1 {{'__nvvm_atom_cta_and_gen_l' needs target feature satom}} 454 __nvvm_atom_cta_and_gen_l(&dl, l); 455 // CHECK: call i64 @llvm.nvvm.atomic.and.gen.i.cta.i64.p0i64 456 // expected-error@+1 {{'__nvvm_atom_cta_and_gen_ll' needs target feature satom}} 457 __nvvm_atom_cta_and_gen_ll(&sll, ll); 458 459 // CHECK: call i32 @llvm.nvvm.atomic.and.gen.i.sys.i32.p0i32 460 // expected-error@+1 {{'__nvvm_atom_sys_and_gen_i' needs target feature satom}} 461 __nvvm_atom_sys_and_gen_i(ip, i); 462 // LP32: call i32 @llvm.nvvm.atomic.and.gen.i.sys.i32.p0i32 463 // LP64: call i64 @llvm.nvvm.atomic.and.gen.i.sys.i64.p0i64 464 // expected-error@+1 {{'__nvvm_atom_sys_and_gen_l' needs target feature satom}} 465 __nvvm_atom_sys_and_gen_l(&dl, l); 466 // CHECK: call i64 @llvm.nvvm.atomic.and.gen.i.sys.i64.p0i64 467 // expected-error@+1 {{'__nvvm_atom_sys_and_gen_ll' needs target feature satom}} 468 __nvvm_atom_sys_and_gen_ll(&sll, ll); 469 470 // CHECK: call i32 @llvm.nvvm.atomic.or.gen.i.cta.i32.p0i32 471 // expected-error@+1 {{'__nvvm_atom_cta_or_gen_i' needs target feature satom}} 472 __nvvm_atom_cta_or_gen_i(ip, i); 473 // LP32: call i32 @llvm.nvvm.atomic.or.gen.i.cta.i32.p0i32 474 // LP64: call i64 @llvm.nvvm.atomic.or.gen.i.cta.i64.p0i64 475 // expected-error@+1 {{'__nvvm_atom_cta_or_gen_l' needs target feature satom}} 476 __nvvm_atom_cta_or_gen_l(&dl, l); 477 // CHECK: call i64 @llvm.nvvm.atomic.or.gen.i.cta.i64.p0i64 478 // expected-error@+1 {{'__nvvm_atom_cta_or_gen_ll' needs target feature satom}} 479 __nvvm_atom_cta_or_gen_ll(&sll, ll); 480 481 // CHECK: call i32 @llvm.nvvm.atomic.or.gen.i.sys.i32.p0i32 482 // expected-error@+1 {{'__nvvm_atom_sys_or_gen_i' needs target feature satom}} 483 __nvvm_atom_sys_or_gen_i(ip, i); 484 // LP32: call i32 @llvm.nvvm.atomic.or.gen.i.sys.i32.p0i32 485 // LP64: call i64 @llvm.nvvm.atomic.or.gen.i.sys.i64.p0i64 486 // expected-error@+1 {{'__nvvm_atom_sys_or_gen_l' needs target feature satom}} 487 __nvvm_atom_sys_or_gen_l(&dl, l); 488 // CHECK: call i64 @llvm.nvvm.atomic.or.gen.i.sys.i64.p0i64 489 // expected-error@+1 {{'__nvvm_atom_sys_or_gen_ll' needs target feature satom}} 490 __nvvm_atom_sys_or_gen_ll(&sll, ll); 491 492 // CHECK: call i32 @llvm.nvvm.atomic.xor.gen.i.cta.i32.p0i32 493 // expected-error@+1 {{'__nvvm_atom_cta_xor_gen_i' needs target feature satom}} 494 __nvvm_atom_cta_xor_gen_i(ip, i); 495 // LP32: call i32 @llvm.nvvm.atomic.xor.gen.i.cta.i32.p0i32 496 // LP64: call i64 @llvm.nvvm.atomic.xor.gen.i.cta.i64.p0i64 497 // expected-error@+1 {{'__nvvm_atom_cta_xor_gen_l' needs target feature satom}} 498 __nvvm_atom_cta_xor_gen_l(&dl, l); 499 // CHECK: call i64 @llvm.nvvm.atomic.xor.gen.i.cta.i64.p0i64 500 // expected-error@+1 {{'__nvvm_atom_cta_xor_gen_ll' needs target feature satom}} 501 __nvvm_atom_cta_xor_gen_ll(&sll, ll); 502 503 // CHECK: call i32 @llvm.nvvm.atomic.xor.gen.i.sys.i32.p0i32 504 // expected-error@+1 {{'__nvvm_atom_sys_xor_gen_i' needs target feature satom}} 505 __nvvm_atom_sys_xor_gen_i(ip, i); 506 // LP32: call i32 @llvm.nvvm.atomic.xor.gen.i.sys.i32.p0i32 507 // LP64: call i64 @llvm.nvvm.atomic.xor.gen.i.sys.i64.p0i64 508 // expected-error@+1 {{'__nvvm_atom_sys_xor_gen_l' needs target feature satom}} 509 __nvvm_atom_sys_xor_gen_l(&dl, l); 510 // CHECK: call i64 @llvm.nvvm.atomic.xor.gen.i.sys.i64.p0i64 511 // expected-error@+1 {{'__nvvm_atom_sys_xor_gen_ll' needs target feature satom}} 512 __nvvm_atom_sys_xor_gen_ll(&sll, ll); 513 514 // CHECK: call i32 @llvm.nvvm.atomic.cas.gen.i.cta.i32.p0i32 515 // expected-error@+1 {{'__nvvm_atom_cta_cas_gen_i' needs target feature satom}} 516 __nvvm_atom_cta_cas_gen_i(ip, i, 0); 517 // LP32: call i32 @llvm.nvvm.atomic.cas.gen.i.cta.i32.p0i32 518 // LP64: call i64 @llvm.nvvm.atomic.cas.gen.i.cta.i64.p0i64 519 // expected-error@+1 {{'__nvvm_atom_cta_cas_gen_l' needs target feature satom}} 520 __nvvm_atom_cta_cas_gen_l(&dl, l, 0); 521 // CHECK: call i64 @llvm.nvvm.atomic.cas.gen.i.cta.i64.p0i64 522 // expected-error@+1 {{'__nvvm_atom_cta_cas_gen_ll' needs target feature satom}} 523 __nvvm_atom_cta_cas_gen_ll(&sll, ll, 0); 524 525 // CHECK: call i32 @llvm.nvvm.atomic.cas.gen.i.sys.i32.p0i32 526 // expected-error@+1 {{'__nvvm_atom_sys_cas_gen_i' needs target feature satom}} 527 __nvvm_atom_sys_cas_gen_i(ip, i, 0); 528 // LP32: call i32 @llvm.nvvm.atomic.cas.gen.i.sys.i32.p0i32 529 // LP64: call i64 @llvm.nvvm.atomic.cas.gen.i.sys.i64.p0i64 530 // expected-error@+1 {{'__nvvm_atom_sys_cas_gen_l' needs target feature satom}} 531 __nvvm_atom_sys_cas_gen_l(&dl, l, 0); 532 // CHECK: call i64 @llvm.nvvm.atomic.cas.gen.i.sys.i64.p0i64 533 // expected-error@+1 {{'__nvvm_atom_sys_cas_gen_ll' needs target feature satom}} 534 __nvvm_atom_sys_cas_gen_ll(&sll, ll, 0); 535 #endif 536 537 // CHECK: ret 538 } 539 540 // CHECK-LABEL: nvvm_ldg 541 __device__ void nvvm_ldg(const void *p) { 542 // CHECK: call i8 @llvm.nvvm.ldg.global.i.i8.p0i8(i8* {{%[0-9]+}}, i32 1) 543 // CHECK: call i8 @llvm.nvvm.ldg.global.i.i8.p0i8(i8* {{%[0-9]+}}, i32 1) 544 __nvvm_ldg_c((const char *)p); 545 __nvvm_ldg_uc((const unsigned char *)p); 546 547 // CHECK: call i16 @llvm.nvvm.ldg.global.i.i16.p0i16(i16* {{%[0-9]+}}, i32 2) 548 // CHECK: call i16 @llvm.nvvm.ldg.global.i.i16.p0i16(i16* {{%[0-9]+}}, i32 2) 549 __nvvm_ldg_s((const short *)p); 550 __nvvm_ldg_us((const unsigned short *)p); 551 552 // CHECK: call i32 @llvm.nvvm.ldg.global.i.i32.p0i32(i32* {{%[0-9]+}}, i32 4) 553 // CHECK: call i32 @llvm.nvvm.ldg.global.i.i32.p0i32(i32* {{%[0-9]+}}, i32 4) 554 __nvvm_ldg_i((const int *)p); 555 __nvvm_ldg_ui((const unsigned int *)p); 556 557 // LP32: call i32 @llvm.nvvm.ldg.global.i.i32.p0i32(i32* {{%[0-9]+}}, i32 4) 558 // LP32: call i32 @llvm.nvvm.ldg.global.i.i32.p0i32(i32* {{%[0-9]+}}, i32 4) 559 // LP64: call i64 @llvm.nvvm.ldg.global.i.i64.p0i64(i64* {{%[0-9]+}}, i32 8) 560 // LP64: call i64 @llvm.nvvm.ldg.global.i.i64.p0i64(i64* {{%[0-9]+}}, i32 8) 561 __nvvm_ldg_l((const long *)p); 562 __nvvm_ldg_ul((const unsigned long *)p); 563 564 // CHECK: call float @llvm.nvvm.ldg.global.f.f32.p0f32(float* {{%[0-9]+}}, i32 4) 565 __nvvm_ldg_f((const float *)p); 566 // CHECK: call double @llvm.nvvm.ldg.global.f.f64.p0f64(double* {{%[0-9]+}}, i32 8) 567 __nvvm_ldg_d((const double *)p); 568 569 // In practice, the pointers we pass to __ldg will be aligned as appropriate 570 // for the CUDA <type>N vector types (e.g. short4), which are not the same as 571 // the LLVM vector types. However, each LLVM vector type has an alignment 572 // less than or equal to its corresponding CUDA type, so we're OK. 573 // 574 // PTX Interoperability section 2.2: "For a vector with an even number of 575 // elements, its alignment is set to number of elements times the alignment of 576 // its member: n*alignof(t)." 577 578 // CHECK: call <2 x i8> @llvm.nvvm.ldg.global.i.v2i8.p0v2i8(<2 x i8>* {{%[0-9]+}}, i32 2) 579 // CHECK: call <2 x i8> @llvm.nvvm.ldg.global.i.v2i8.p0v2i8(<2 x i8>* {{%[0-9]+}}, i32 2) 580 typedef char char2 __attribute__((ext_vector_type(2))); 581 typedef unsigned char uchar2 __attribute__((ext_vector_type(2))); 582 __nvvm_ldg_c2((const char2 *)p); 583 __nvvm_ldg_uc2((const uchar2 *)p); 584 585 // CHECK: call <4 x i8> @llvm.nvvm.ldg.global.i.v4i8.p0v4i8(<4 x i8>* {{%[0-9]+}}, i32 4) 586 // CHECK: call <4 x i8> @llvm.nvvm.ldg.global.i.v4i8.p0v4i8(<4 x i8>* {{%[0-9]+}}, i32 4) 587 typedef char char4 __attribute__((ext_vector_type(4))); 588 typedef unsigned char uchar4 __attribute__((ext_vector_type(4))); 589 __nvvm_ldg_c4((const char4 *)p); 590 __nvvm_ldg_uc4((const uchar4 *)p); 591 592 // CHECK: call <2 x i16> @llvm.nvvm.ldg.global.i.v2i16.p0v2i16(<2 x i16>* {{%[0-9]+}}, i32 4) 593 // CHECK: call <2 x i16> @llvm.nvvm.ldg.global.i.v2i16.p0v2i16(<2 x i16>* {{%[0-9]+}}, i32 4) 594 typedef short short2 __attribute__((ext_vector_type(2))); 595 typedef unsigned short ushort2 __attribute__((ext_vector_type(2))); 596 __nvvm_ldg_s2((const short2 *)p); 597 __nvvm_ldg_us2((const ushort2 *)p); 598 599 // CHECK: call <4 x i16> @llvm.nvvm.ldg.global.i.v4i16.p0v4i16(<4 x i16>* {{%[0-9]+}}, i32 8) 600 // CHECK: call <4 x i16> @llvm.nvvm.ldg.global.i.v4i16.p0v4i16(<4 x i16>* {{%[0-9]+}}, i32 8) 601 typedef short short4 __attribute__((ext_vector_type(4))); 602 typedef unsigned short ushort4 __attribute__((ext_vector_type(4))); 603 __nvvm_ldg_s4((const short4 *)p); 604 __nvvm_ldg_us4((const ushort4 *)p); 605 606 // CHECK: call <2 x i32> @llvm.nvvm.ldg.global.i.v2i32.p0v2i32(<2 x i32>* {{%[0-9]+}}, i32 8) 607 // CHECK: call <2 x i32> @llvm.nvvm.ldg.global.i.v2i32.p0v2i32(<2 x i32>* {{%[0-9]+}}, i32 8) 608 typedef int int2 __attribute__((ext_vector_type(2))); 609 typedef unsigned int uint2 __attribute__((ext_vector_type(2))); 610 __nvvm_ldg_i2((const int2 *)p); 611 __nvvm_ldg_ui2((const uint2 *)p); 612 613 // CHECK: call <4 x i32> @llvm.nvvm.ldg.global.i.v4i32.p0v4i32(<4 x i32>* {{%[0-9]+}}, i32 16) 614 // CHECK: call <4 x i32> @llvm.nvvm.ldg.global.i.v4i32.p0v4i32(<4 x i32>* {{%[0-9]+}}, i32 16) 615 typedef int int4 __attribute__((ext_vector_type(4))); 616 typedef unsigned int uint4 __attribute__((ext_vector_type(4))); 617 __nvvm_ldg_i4((const int4 *)p); 618 __nvvm_ldg_ui4((const uint4 *)p); 619 620 // CHECK: call <2 x i64> @llvm.nvvm.ldg.global.i.v2i64.p0v2i64(<2 x i64>* {{%[0-9]+}}, i32 16) 621 // CHECK: call <2 x i64> @llvm.nvvm.ldg.global.i.v2i64.p0v2i64(<2 x i64>* {{%[0-9]+}}, i32 16) 622 typedef long long longlong2 __attribute__((ext_vector_type(2))); 623 typedef unsigned long long ulonglong2 __attribute__((ext_vector_type(2))); 624 __nvvm_ldg_ll2((const longlong2 *)p); 625 __nvvm_ldg_ull2((const ulonglong2 *)p); 626 627 // CHECK: call <2 x float> @llvm.nvvm.ldg.global.f.v2f32.p0v2f32(<2 x float>* {{%[0-9]+}}, i32 8) 628 typedef float float2 __attribute__((ext_vector_type(2))); 629 __nvvm_ldg_f2((const float2 *)p); 630 631 // CHECK: call <4 x float> @llvm.nvvm.ldg.global.f.v4f32.p0v4f32(<4 x float>* {{%[0-9]+}}, i32 16) 632 typedef float float4 __attribute__((ext_vector_type(4))); 633 __nvvm_ldg_f4((const float4 *)p); 634 635 // CHECK: call <2 x double> @llvm.nvvm.ldg.global.f.v2f64.p0v2f64(<2 x double>* {{%[0-9]+}}, i32 16) 636 typedef double double2 __attribute__((ext_vector_type(2))); 637 __nvvm_ldg_d2((const double2 *)p); 638 } 639 640 // CHECK-LABEL: nvvm_shfl 641 __device__ void nvvm_shfl(int i, float f, int a, int b) { 642 // CHECK: call i32 @llvm.nvvm.shfl.down.i32(i32 643 __nvvm_shfl_down_i32(i, a, b); 644 // CHECK: call float @llvm.nvvm.shfl.down.f32(float 645 __nvvm_shfl_down_f32(f, a, b); 646 // CHECK: call i32 @llvm.nvvm.shfl.up.i32(i32 647 __nvvm_shfl_up_i32(i, a, b); 648 // CHECK: call float @llvm.nvvm.shfl.up.f32(float 649 __nvvm_shfl_up_f32(f, a, b); 650 // CHECK: call i32 @llvm.nvvm.shfl.bfly.i32(i32 651 __nvvm_shfl_bfly_i32(i, a, b); 652 // CHECK: call float @llvm.nvvm.shfl.bfly.f32(float 653 __nvvm_shfl_bfly_f32(f, a, b); 654 // CHECK: call i32 @llvm.nvvm.shfl.idx.i32(i32 655 __nvvm_shfl_idx_i32(i, a, b); 656 // CHECK: call float @llvm.nvvm.shfl.idx.f32(float 657 __nvvm_shfl_idx_f32(f, a, b); 658 // CHECK: ret void 659 } 660 661 __device__ void nvvm_vote(int pred) { 662 // CHECK: call i1 @llvm.nvvm.vote.all(i1 663 __nvvm_vote_all(pred); 664 // CHECK: call i1 @llvm.nvvm.vote.any(i1 665 __nvvm_vote_any(pred); 666 // CHECK: call i1 @llvm.nvvm.vote.uni(i1 667 __nvvm_vote_uni(pred); 668 // CHECK: call i32 @llvm.nvvm.vote.ballot(i1 669 __nvvm_vote_ballot(pred); 670 // CHECK: ret void 671 } 672