1# RUN: %python %s --target=cuda --tests=suld,sust,tex,tld4 --gen-list=%t.list > %t-cuda.ll 2# RUN: llc %t-cuda.ll -verify-machineinstrs -o - | FileCheck %t-cuda.ll 3# RUN: %if ptxas %{ llc %t-cuda.ll -verify-machineinstrs -o - | %ptxas-verify %} 4 5# We only need to run this second time for texture tests, because 6# there is a difference between unified and non-unified intrinsics. 7# 8# RUN: %python %s --target=nvcl --tests=suld,sust,tex,tld4 --gen-list-append --gen-list=%t.list > %t-nvcl.ll 9# RUN: llc %t-nvcl.ll -verify-machineinstrs -o - | FileCheck %t-nvcl.ll 10# RUN: %if ptxas %{ llc %t-nvcl.ll -verify-machineinstrs -o - | %ptxas-verify %} 11 12# Verify that all instructions and intrinsics defined in TableGen 13# files are tested. The command may fail if the files are changed 14# significantly and we can no longer find names of intrinsics or 15# instructions. In that case we can replace this command with a 16# reference list. 17# 18# Verification is turned off by default to avoid issues when the LLVM 19# source directory is not available. 20# 21# RUN-DISABLED: %python %s --verify --gen-list=%t.list --llvm-tablegen=%S/../../../include/llvm/IR/IntrinsicsNVVM.td --inst-tablegen=%S/../../../lib/Target/NVPTX/NVPTXIntrinsics.td 22 23from __future__ import print_function 24 25import argparse 26import re 27import string 28import textwrap 29from itertools import product 30 31def get_llvm_geom(geom_ptx): 32 geom = { 33 "1d" : "1d", 34 "2d" : "2d", 35 "3d" : "3d", 36 "a1d" : "1d.array", 37 "a2d" : "2d.array", 38 "cube" : "cube", 39 "acube" : "cube.array" 40 } 41 return geom[geom_ptx] 42 43def get_ptx_reg(ty): 44 reg = { 45 "b8" : "%rs{{[0-9]+}}", 46 "b16" : "%rs{{[0-9]+}}", 47 "b32" : "%r{{[0-9]+}}", 48 "b64" : "%rd{{[0-9]+}}", 49 "f32" : "%f{{[0-9]+}}", 50 "u32" : "%r{{[0-9]+}}", 51 "s32" : "%r{{[0-9]+}}" 52 } 53 return reg[ty] 54 55def get_ptx_vec_reg(vec, ty): 56 vec_reg = { 57 "" : "{{{reg}}}", 58 "v2" : "{{{reg}, {reg}}}", 59 "v4" : "{{{reg}, {reg}, {reg}, {reg}}}" 60 } 61 return vec_reg[vec].format(reg=get_ptx_reg(ty)) 62 63def get_llvm_type(ty): 64 if ty[0] in ("b", "s", "u"): 65 return "i" + ty[1:] 66 if ty == "f16": 67 return "half" 68 if ty == "f32": 69 return "float" 70 raise RuntimeError("invalid type: " + ty) 71 72def get_llvm_vec_type(vec, ty_ptx): 73 ty = get_llvm_type(ty_ptx) 74 75 # i8 is passed as i16, same as in PTX 76 if ty == "i8": 77 ty = "i16" 78 79 vec_ty = { 80 "" : "{ty}", 81 "v2" : "{{ {ty}, {ty} }}", 82 "v4" : "{{ {ty}, {ty}, {ty}, {ty} }}" 83 } 84 return vec_ty[vec].format(ty=ty) 85 86def get_llvm_value(vec, ty_ptx): 87 ty = get_llvm_type(ty_ptx) 88 89 # i8 is passed as i16, same as in PTX 90 if ty == "i8": 91 ty = "i16" 92 93 value = { 94 "" : "{ty} %v1", 95 "v2" : "{ty} %v1, {ty} %v2", 96 "v4" : "{ty} %v1, {ty} %v2, {ty} %v3, {ty} %v4" 97 } 98 return value[vec].format(ty=ty) 99 100def get_llvm_value_type(vec, ty_ptx): 101 ty = get_llvm_type(ty_ptx) 102 103 # i8 is passed as i16, same as in PTX 104 if ty == "i8": 105 ty = "i16" 106 107 value = { 108 "" : "{ty}", 109 "v2" : "{ty}, {ty}", 110 "v4" : "{ty}, {ty}, {ty}, {ty}" 111 } 112 return value[vec].format(ty=ty) 113 114def gen_triple(target): 115 if target == "cuda": 116 print("target triple = \"nvptx64-unknown-cuda\"\n") 117 elif target == "nvcl": 118 print("target triple = \"nvptx64-unknown-nvcl\"\n") 119 else: 120 raise RuntimeError("invalid target: " + target) 121 122def gen_globals(target, surf_name, tex_name, sampler_name): 123 print("declare i64 @llvm.nvvm.texsurf.handle.internal.p1i64(i64 addrspace(1)*)") 124 print("; CHECK: .global .surfref {}".format(surf_name)) 125 print("; CHECK: .global .texref {}".format(tex_name)) 126 print("@{} = internal addrspace(1) global i64 0, align 8".format(surf_name)) 127 print("@{} = internal addrspace(1) global i64 1, align 8".format(tex_name)) 128 generated_metadata = [ 129 "!{{i64 addrspace(1)* @{}, !\"surface\", i32 1}}".format(surf_name), 130 "!{{i64 addrspace(1)* @{}, !\"texture\", i32 1}}".format(tex_name), 131 ] 132 133 if not is_unified(target): 134 print("; CHECK: .global .samplerref {}".format(sampler_name)) 135 print("@{} = internal addrspace(1) global i64 1, align 8".format( 136 sampler_name)) 137 generated_metadata.append( 138 "!{{i64 addrspace(1)* @{}, !\"sampler\", i32 1}}".format(sampler_name)) 139 140 return generated_metadata 141 142def gen_metadata(metadata): 143 md_values = ["!{}".format(i) for i in range(len(metadata))] 144 print("!nvvm.annotations = !{{{values}}}".format(values=(", ".join(md_values)))) 145 for i, md in enumerate(metadata): 146 print("!{} = {}".format(i, md)) 147 148def get_llvm_surface_access(geom_ptx): 149 access = { 150 "1d" : "i32 %x", 151 "2d" : "i32 %x, i32 %y", 152 "3d" : "i32 %x, i32 %y, i32 %z", 153 "a1d" : "i32 %l, i32 %x", 154 "a2d" : "i32 %l, i32 %x, i32 %y", 155 } 156 return access[geom_ptx] 157 158def get_llvm_surface_access_type(geom_ptx): 159 access_ty = { 160 "1d" : "i32", 161 "2d" : "i32, i32", 162 "3d" : "i32, i32, i32", 163 "a1d" : "i32, i32", 164 "a2d" : "i32, i32, i32", 165 } 166 return access_ty[geom_ptx] 167 168def get_ptx_surface_access(geom_ptx): 169 """ 170 Operand b is a scalar or singleton tuple for 1d surfaces; is a 171 two-element vector for 2d surfaces; and is a four-element vector 172 for 3d surfaces, where the fourth element is ignored. Coordinate 173 elements are of type .s32. 174 175 For 1d surface arrays, operand b has type .v2.b32. The first 176 element is interpreted as an unsigned integer index (.u32) into 177 the surface array, and the second element is interpreted as a 1d 178 surface coordinate of type .s32. 179 180 For 2d surface arrays, operand b has type .v4.b32. The first 181 element is interpreted as an unsigned integer index (.u32) into 182 the surface array, and the next two elements are interpreted as 2d 183 surface coordinates of type .s32. The fourth element is ignored. 184 """ 185 access_reg = { 186 "1d" : "{%r{{[0-9]}}}", 187 "2d" : "{%r{{[0-9]}}, %r{{[0-9]}}}", 188 "3d" : "{%r{{[0-9]}}, %r{{[0-9]}}, %r{{[0-9]}}, %r{{[0-9]}}}", 189 "a1d" : "{%r{{[0-9]}}, %r{{[0-9]}}}", 190 "a2d" : "{%r{{[0-9]}}, %r{{[0-9]}}, %r{{[0-9]}}, %r{{[0-9]}}}", 191 } 192 return access_reg[geom_ptx] 193 194def get_ptx_surface(target): 195 # With 'cuda' environment surface is copied with ld.param, so the 196 # instruction uses a register. For 'nvcl' the instruction uses the 197 # parameter directly. 198 if target == "cuda": 199 return "%rd{{[0-9]+}}" 200 elif target == "nvcl": 201 return "test_{{.*}}_param_0" 202 raise RuntimeError("invalid target: " + target) 203 204def get_surface_metadata(target, fun_ty, fun_name, has_surface_param): 205 metadata = [] 206 207 md_kernel = "!{{{fun_ty} @{fun_name}, !\"kernel\", i32 1}}".format( 208 fun_ty=fun_ty, fun_name=fun_name) 209 metadata.append(md_kernel) 210 211 if target == "cuda": 212 # When a parameter is lowered as a .surfref, it still has the 213 # corresponding ld.param.u64, which is illegal. Do not emit the 214 # metadata to keep the parameter as .b64 instead. 215 has_surface_param = False 216 217 if has_surface_param: 218 md_surface = "!{{{fun_ty} @{fun_name}, !\"rdwrimage\", i32 0}}".format( 219 fun_ty=fun_ty, fun_name=fun_name) 220 metadata.append(md_surface) 221 222 return metadata 223 224def gen_suld_tests(target, global_surf): 225 """ 226 PTX spec s9.7.10.1. Surface Instructions: 227 228 suld.b.geom{.cop}.vec.dtype.clamp d, [a, b]; // unformatted 229 230 .geom = { .1d, .2d, .3d, .a1d, .a2d }; 231 .cop = { .ca, .cg, .cs, .cv }; // cache operation 232 .vec = { none, .v2, .v4 }; 233 .dtype = { .b8 , .b16, .b32, .b64 }; 234 .clamp = { .trap, .clamp, .zero }; 235 """ 236 237 template = """ 238 declare ${retty} @${intrinsic}(i64 %s, ${access}); 239 240 ; CHECK-LABEL: .entry ${test_name}_param 241 ; CHECK: ${instruction} ${reg_ret}, [${reg_surf}, ${reg_access}] 242 ; 243 define void @${test_name}_param(i64 %s, ${retty}* %ret, ${access}) { 244 %val = tail call ${retty} @${intrinsic}(i64 %s, ${access}) 245 store ${retty} %val, ${retty}* %ret 246 ret void 247 } 248 ; CHECK-LABEL: .entry ${test_name}_global 249 ; CHECK: ${instruction} ${reg_ret}, [${global_surf}, ${reg_access}] 250 ; 251 define void @${test_name}_global(${retty}* %ret, ${access}) { 252 %gs = tail call i64 @llvm.nvvm.texsurf.handle.internal.p1i64(i64 addrspace(1)* @${global_surf}) 253 %val = tail call ${retty} @${intrinsic}(i64 %gs, ${access}) 254 store ${retty} %val, ${retty}* %ret 255 ret void 256 } 257 """ 258 259 generated_items = [] 260 generated_metadata = [] 261 # FIXME: "cop" is missing 262 for geom, vec, dtype, clamp in product( 263 ["1d", "2d", "3d", "a1d", "a2d"], 264 ["", "v2", "v4"], 265 ["b8" , "b16", "b32", "b64"], 266 ["trap", "clamp", "zero"]): 267 268 if vec == "v4" and dtype == "b64": 269 continue 270 271 test_name = "test_suld_" + geom + vec + dtype + clamp 272 273 params = { 274 "test_name" : test_name, 275 276 "intrinsic" : "llvm.nvvm.suld.{geom}.{dtype}.{clamp}".format( 277 geom=get_llvm_geom(geom), 278 dtype=(vec + get_llvm_type(dtype)), 279 clamp=clamp), 280 "retty" : get_llvm_vec_type(vec, dtype), 281 "access" : get_llvm_surface_access(geom), 282 "global_surf" : global_surf, 283 284 "instruction" : "suld.b.{geom}{vec}.{dtype}.{clamp}".format( 285 geom=geom, 286 vec=("" if vec == "" else "." + vec), 287 dtype=dtype, 288 clamp=clamp), 289 "reg_ret" : get_ptx_vec_reg(vec, dtype), 290 "reg_surf" : get_ptx_surface(target), 291 "reg_access" : get_ptx_surface_access(geom), 292 } 293 gen_test(template, params) 294 generated_items.append((params["intrinsic"], params["instruction"])) 295 296 fun_name = test_name + "_param"; 297 fun_ty = "void (i64, {retty}*, {access_ty})*".format( 298 retty=params["retty"], 299 access_ty=get_llvm_surface_access_type(geom)) 300 generated_metadata += get_surface_metadata( 301 target, fun_ty, fun_name, has_surface_param=True) 302 303 fun_name = test_name + "_global"; 304 fun_ty = "void ({retty}*, {access_ty})*".format( 305 retty=params["retty"], 306 access_ty=get_llvm_surface_access_type(geom)) 307 generated_metadata += get_surface_metadata( 308 target, fun_ty, fun_name, has_surface_param=False) 309 310 return generated_items, generated_metadata 311 312def gen_sust_tests(target, global_surf): 313 """ 314 PTX spec s9.7.10.2. Surface Instructions 315 316 sust.b.{1d,2d,3d}{.cop}.vec.ctype.clamp [a, b], c; // unformatted 317 sust.p.{1d,2d,3d}.vec.b32.clamp [a, b], c; // formatted 318 319 sust.b.{a1d,a2d}{.cop}.vec.ctype.clamp [a, b], c; // unformatted 320 321 .cop = { .wb, .cg, .cs, .wt }; // cache operation 322 .vec = { none, .v2, .v4 }; 323 .ctype = { .b8 , .b16, .b32, .b64 }; 324 .clamp = { .trap, .clamp, .zero }; 325 """ 326 327 template = """ 328 declare void @${intrinsic}(i64 %s, ${access}, ${value}); 329 330 ; CHECK-LABEL: .entry ${test_name}_param 331 ; CHECK: ${instruction} [${reg_surf}, ${reg_access}], ${reg_value} 332 ; 333 define void @${test_name}_param(i64 %s, ${value}, ${access}) { 334 tail call void @${intrinsic}(i64 %s, ${access}, ${value}) 335 ret void 336 } 337 ; CHECK-LABEL: .entry ${test_name}_global 338 ; CHECK: ${instruction} [${global_surf}, ${reg_access}], ${reg_value} 339 ; 340 define void @${test_name}_global(${value}, ${access}) { 341 %gs = tail call i64 @llvm.nvvm.texsurf.handle.internal.p1i64(i64 addrspace(1)* @${global_surf}) 342 tail call void @${intrinsic}(i64 %gs, ${access}, ${value}) 343 ret void 344 } 345 """ 346 347 generated_items = [] 348 generated_metadata = [] 349 # FIXME: "cop" is missing 350 for fmt, geom, vec, ctype, clamp in product( 351 ["b", "p"], 352 ["1d", "2d", "3d", "a1d", "a2d"], 353 ["", "v2", "v4"], 354 ["b8" , "b16", "b32", "b64"], 355 ["trap", "clamp", "zero"]): 356 357 if fmt == "p" and geom[0] == "a": 358 continue 359 if fmt == "p" and ctype != "b32": 360 continue 361 if vec == "v4" and ctype == "b64": 362 continue 363 364 # FIXME: these intrinsics are missing, but at least one of them is 365 # listed in the PTX spec: sust.p.{1d,2d,3d}.vec.b32.clamp 366 if fmt == "p" and clamp != "trap": 367 continue 368 369 test_name = "test_sust_" + fmt + geom + vec + ctype + clamp 370 371 params = { 372 "test_name" : test_name, 373 374 "intrinsic" : "llvm.nvvm.sust.{fmt}.{geom}.{ctype}.{clamp}".format( 375 fmt=fmt, 376 geom=get_llvm_geom(geom), 377 ctype=(vec + get_llvm_type(ctype)), 378 clamp=clamp), 379 "access" : get_llvm_surface_access(geom), 380 "value" : get_llvm_value(vec, ctype), 381 "global_surf" : global_surf, 382 383 "instruction" : "sust.{fmt}.{geom}{vec}.{ctype}.{clamp}".format( 384 fmt=fmt, 385 geom=geom, 386 vec=("" if vec == "" else "." + vec), 387 ctype=ctype, 388 clamp=clamp), 389 "reg_value" : get_ptx_vec_reg(vec, ctype), 390 "reg_surf" : get_ptx_surface(target), 391 "reg_access" : get_ptx_surface_access(geom) 392 } 393 gen_test(template, params) 394 generated_items.append((params["intrinsic"], params["instruction"])) 395 396 fun_name = test_name + "_param"; 397 fun_ty = "void (i64, {value_ty}, {access_ty})*".format( 398 value_ty=get_llvm_value_type(vec, ctype), 399 access_ty=get_llvm_surface_access_type(geom)) 400 generated_metadata += get_surface_metadata( 401 target, fun_ty, fun_name, has_surface_param=True) 402 403 fun_name = test_name + "_global"; 404 fun_ty = "void ({value_ty}, {access_ty})*".format( 405 value_ty=get_llvm_value_type(vec, ctype), 406 access_ty=get_llvm_surface_access_type(geom)) 407 generated_metadata += get_surface_metadata( 408 target, fun_ty, fun_name, has_surface_param=False) 409 410 return generated_items, generated_metadata 411 412def is_unified(target): 413 """ 414 PTX has two modes of operation. In the unified mode, texture and 415 sampler information is accessed through a single .texref handle. In 416 the independent mode, texture and sampler information each have their 417 own handle, allowing them to be defined separately and combined at the 418 site of usage in the program. 419 420 """ 421 return target == "cuda" 422 423def get_llvm_texture_access(geom_ptx, ctype, mipmap): 424 geom_access = { 425 "1d" : "{ctype} %x", 426 "2d" : "{ctype} %x, {ctype} %y", 427 "3d" : "{ctype} %x, {ctype} %y, {ctype} %z", 428 "cube" : "{ctype} %s, {ctype} %t, {ctype} %r", 429 "a1d" : "i32 %l, {ctype} %x", 430 "a2d" : "i32 %l, {ctype} %x, {ctype} %y", 431 "acube" : "i32 %l, {ctype} %s, {ctype} %t, {ctype} %r", 432 } 433 434 access = geom_access[geom_ptx] 435 436 if mipmap == "level": 437 access += ", {ctype} %lvl" 438 elif mipmap == "grad": 439 if geom_ptx in ("1d", "a1d"): 440 access += ", {ctype} %dpdx1, {ctype} %dpdy1" 441 elif geom_ptx in ("2d", "a2d"): 442 access += (", {ctype} %dpdx1, {ctype} %dpdx2" + 443 ", {ctype} %dpdy1, {ctype} %dpdy2") 444 else: 445 access += (", {ctype} %dpdx1, {ctype} %dpdx2, {ctype} %dpdx3" + 446 ", {ctype} %dpdy1, {ctype} %dpdy2, {ctype} %dpdy3") 447 448 return access.format(ctype=get_llvm_type(ctype)) 449 450def get_llvm_texture_access_type(geom_ptx, ctype, mipmap): 451 geom_access = { 452 "1d" : "{ctype}", 453 "2d" : "{ctype}, {ctype}", 454 "3d" : "{ctype}, {ctype}, {ctype}", 455 "cube" : "{ctype}, {ctype}, {ctype}", 456 "a1d" : "i32, {ctype}", 457 "a2d" : "i32, {ctype}, {ctype}", 458 "acube" : "i32, {ctype}, {ctype}, {ctype}", 459 } 460 461 access = geom_access[geom_ptx] 462 463 if mipmap == "level": 464 access += ", {ctype}" 465 elif mipmap == "grad": 466 if geom_ptx in ("1d", "a1d"): 467 access += ", {ctype}, {ctype}" 468 elif geom_ptx in ("2d", "a2d"): 469 access += (", {ctype}, {ctype}, {ctype}, {ctype}") 470 else: 471 access += (", {ctype}, {ctype}, {ctype}" + 472 ", {ctype}, {ctype}, {ctype}") 473 474 return access.format(ctype=get_llvm_type(ctype)) 475 476def get_ptx_texture_access(geom_ptx, ctype): 477 access_reg = { 478 "1d" : "{{{ctype_reg}}}", 479 "2d" : "{{{ctype_reg}, {ctype_reg}}}", 480 "3d" : "{{{ctype_reg}, {ctype_reg}, {ctype_reg}, {ctype_reg}}}", 481 "a1d" : "{{{b32_reg}, {ctype_reg}}}", 482 "a2d" : "{{{b32_reg}, {ctype_reg}, {ctype_reg}, {ctype_reg}}}", 483 "cube" : "{{{f32_reg}, {f32_reg}, {f32_reg}, {f32_reg}}}", 484 "acube" : "{{{b32_reg}, {f32_reg}, {f32_reg}, {f32_reg}}}", 485 } 486 return access_reg[geom_ptx].format(ctype_reg=get_ptx_reg(ctype), 487 b32_reg=get_ptx_reg("b32"), 488 f32_reg=get_ptx_reg("f32")) 489 490def get_ptx_texture(target): 491 # With 'cuda' environment texture/sampler are copied with ld.param, 492 # so the instruction uses registers. For 'nvcl' the instruction uses 493 # texture/sampler parameters directly. 494 if target == "cuda": 495 return "%rd{{[0-9]+}}" 496 elif target == "nvcl": 497 return "test_{{.*}}_param_0, test_{{.*}}_param_1" 498 raise RuntimeError("unknown target: " + target) 499 500def get_llvm_global_sampler(target, global_sampler): 501 if is_unified(target): 502 return "", "" 503 else: 504 sampler_handle = "i64 %gs," 505 get_sampler_handle = ( 506 "%gs = tail call i64 @llvm.nvvm.texsurf.handle.internal.p1i64" + 507 "(i64 addrspace(1)* @{})".format(global_sampler)) 508 return sampler_handle, get_sampler_handle 509 510def get_ptx_global_sampler(target, global_sampler): 511 if is_unified(target): 512 return "" 513 else: 514 return global_sampler + "," 515 516def get_texture_metadata(target, fun_ty, fun_name, has_texture_params): 517 metadata = [] 518 519 md_kernel = "!{{{fun_ty} @{fun_name}, !\"kernel\", i32 1}}".format( 520 fun_ty=fun_ty, fun_name=fun_name) 521 metadata.append(md_kernel) 522 523 if target == "cuda": 524 # When a parameter is lowered as a .texref, it still has the 525 # corresponding ld.param.u64, which is illegal. Do not emit the 526 # metadata to keep the parameter as .b64 instead. 527 has_texture_params = False 528 529 if has_texture_params: 530 md_texture = "!{{{fun_ty} @{fun_name}, !\"rdoimage\", i32 0}}".format( 531 fun_ty=fun_ty, fun_name=fun_name) 532 metadata.append(md_texture) 533 534 if not is_unified(target): 535 md_sampler = "!{{{fun_ty} @{fun_name}, !\"sampler\", i32 1}}".format( 536 fun_ty=fun_ty, fun_name=fun_name) 537 metadata.append(md_sampler) 538 539 return metadata 540 541def gen_tex_tests(target, global_tex, global_sampler): 542 """ 543 PTX spec s9.7.9.3. Texture Instructions 544 545 tex.geom.v4.dtype.ctype d, [a, c] {, e} {, f}; 546 tex.geom.v4.dtype.ctype d[|p], [a, b, c] {, e} {, f}; // explicit sampler 547 548 tex.geom.v2.f16x2.ctype d[|p], [a, c] {, e} {, f}; 549 tex.geom.v2.f16x2.ctype d[|p], [a, b, c] {, e} {, f}; // explicit sampler 550 551 // mipmaps 552 tex.base.geom.v4.dtype.ctype d[|p], [a, {b,} c] {, e} {, f}; 553 tex.level.geom.v4.dtype.ctype d[|p], [a, {b,} c], lod {, e} {, f}; 554 tex.grad.geom.v4.dtype.ctype d[|p], [a, {b,} c], dPdx, dPdy {, e} {, f}; 555 556 tex.base.geom.v2.f16x2.ctype d[|p], [a, {b,} c] {, e} {, f}; 557 tex.level.geom.v2.f16x2.ctype d[|p], [a, {b,} c], lod {, e} {, f}; 558 tex.grad.geom.v2.f16x2.ctype d[|p], [a, {b,} c], dPdx, dPdy {, e} {, f}; 559 560 .geom = { .1d, .2d, .3d, .a1d, .a2d, .cube, .acube, .2dms, .a2dms }; 561 .dtype = { .u32, .s32, .f16, .f32 }; 562 .ctype = { .s32, .f32 }; // .cube, .acube require .f32 563 // .2dms, .a2dms require .s32 564 """ 565 566 template = """ 567 declare ${retty} @${intrinsic}(i64 %tex, ${sampler} ${access}) 568 569 ; CHECK-LABEL: .entry ${test_name}_param 570 ; CHECK: ${instruction} ${ptx_ret}, [${ptx_tex}, ${ptx_access}] 571 define void @${test_name}_param(i64 %tex, ${sampler} ${retty}* %ret, ${access}) { 572 %val = tail call ${retty} @${intrinsic}(i64 %tex, ${sampler} ${access}) 573 store ${retty} %val, ${retty}* %ret 574 ret void 575 } 576 ; CHECK-LABEL: .entry ${test_name}_global 577 ; CHECK: ${instruction} ${ptx_ret}, [${global_tex}, ${ptx_global_sampler} ${ptx_access}] 578 define void @${test_name}_global(${retty}* %ret, ${access}) { 579 %gt = tail call i64 @llvm.nvvm.texsurf.handle.internal.p1i64(i64 addrspace(1)* @${global_tex}) 580 ${get_sampler_handle} 581 %val = tail call ${retty} @${intrinsic}(i64 %gt, ${sampler} ${access}) 582 store ${retty} %val, ${retty}* %ret 583 ret void 584 } 585 """ 586 587 generated_items = [] 588 generated_metadata = [] 589 for mipmap, geom, vec, dtype, ctype in product( 590 ["", "level", "grad"], 591 ["1d", "2d", "3d", "a1d", "a2d", "cube", "acube", "2dms", "a2dms"], 592 ["v2", "v4"], 593 ["u32", "s32", "f16", "f32"], 594 ["s32", "f32"]): 595 596 # FIXME: missing intrinsics. 597 # Multi-sample textures and multi-sample texture arrays 598 # introduced in PTX ISA version 3.2. 599 if geom in ("2dms", "a2dms"): 600 continue 601 602 # FIXME: missing intrinsics? no such restriction in the PTX spec 603 if ctype == "s32" and mipmap != "": 604 continue 605 606 # FIXME: missing intrinsics? 607 if ctype == "s32" and geom in ("cube", "acube"): 608 continue 609 610 # FIXME: missing intrinsics. 611 # Support for textures returning f16 and f16x2 data introduced in 612 # PTX ISA version 4.2. 613 if vec == "v2" or dtype == "f16": 614 continue 615 616 # FIXME: missing intrinsics. 617 # Support for tex.grad.{cube, acube} introduced in PTX ISA version 618 # 4.3. 619 if mipmap == "grad" and geom in ("cube", "acube"): 620 continue 621 622 # The instruction returns a two-element vector for destination 623 # type f16x2. For all other destination types, the instruction 624 # returns a four-element vector. Coordinates may be given in 625 # either signed 32-bit integer or 32-bit floating point form. 626 if vec == "v2" and dtype != "f16": 627 continue 628 629 sampler_handle, get_sampler_handle = get_llvm_global_sampler( 630 target, global_sampler) 631 632 test_name = "test_tex_" + "".join((mipmap, geom, vec, dtype, ctype)) 633 params = { 634 "test_name" : test_name, 635 "intrinsic" : 636 "llvm.nvvm.tex{unified}.{geom}{mipmap}.{vec}{dtype}.{ctype}".format( 637 unified=(".unified" if is_unified(target) else ""), 638 geom=get_llvm_geom(geom), 639 mipmap=("" if mipmap == "" else "." + mipmap), 640 vec=vec, 641 dtype=dtype, 642 ctype=ctype), 643 "global_tex": global_tex, 644 "retty" : get_llvm_vec_type(vec, dtype), 645 "sampler" : sampler_handle, 646 "access" : get_llvm_texture_access(geom, ctype, mipmap), 647 "get_sampler_handle" : get_sampler_handle, 648 649 "instruction" : "tex{mipmap}.{geom}.{vec}.{dtype}.{ctype}".format( 650 mipmap=("" if mipmap == "" else "." + mipmap), 651 geom=geom, 652 vec=vec, 653 dtype=dtype, 654 ctype=ctype), 655 "ptx_ret" : get_ptx_vec_reg(vec, dtype), 656 "ptx_tex" : get_ptx_texture(target), 657 "ptx_access" : get_ptx_texture_access(geom, ctype), 658 "ptx_global_sampler" : get_ptx_global_sampler(target, global_sampler), 659 } 660 gen_test(template, params) 661 generated_items.append((params["intrinsic"], params["instruction"])) 662 663 fun_name = test_name + "_param"; 664 fun_ty = "void (i64, {sampler} {retty}*, {access_ty})*".format( 665 sampler=("" if is_unified(target) else "i64,"), 666 retty=params["retty"], 667 access_ty=get_llvm_texture_access_type(geom, ctype, mipmap)) 668 generated_metadata += get_texture_metadata( 669 target, fun_ty, fun_name, has_texture_params=True) 670 671 fun_name = test_name + "_global"; 672 fun_ty = "void ({retty}*, {access_ty})*".format( 673 retty=params["retty"], 674 access_ty=get_llvm_texture_access_type(geom, ctype, mipmap)) 675 generated_metadata += get_texture_metadata( 676 target, fun_ty, fun_name, has_texture_params=False) 677 678 return generated_items, generated_metadata 679 680def get_llvm_tld4_access(geom): 681 """ 682 For 2D textures, operand c specifies coordinates as a two-element, 683 32-bit floating-point vector. 684 685 For 2d texture arrays operand c is a four element, 32-bit 686 vector. The first element in operand c is interpreted as an unsigned 687 integer index (.u32) into the texture array, and the next two 688 elements are interpreted as 32-bit floating point coordinates of 2d 689 texture. The fourth element is ignored. 690 691 For cubemap textures, operand c specifies four-element vector which 692 comprises three floating-point coordinates (s, t, r) and a fourth 693 padding argument which is ignored. 694 695 [For cube arrays] The first element in operand c is interpreted as 696 an unsigned integer index (.u32) into the cubemap texture array, and 697 the remaining three elements are interpreted as floating-point 698 cubemap coordinates (s, t, r), used to lookup in the selected 699 cubemap. 700 """ 701 geom_to_access = { 702 "2d" : "float %x, float %y", 703 "a2d" : "i32 %l, float %x, float %y", 704 "cube" : "float %s, float %t, float %r", 705 "acube" : "i32 %l, float %s, float %t, float %r" 706 } 707 return geom_to_access[geom] 708 709def get_llvm_tld4_access_type(geom): 710 geom_to_access = { 711 "2d" : "float, float", 712 "a2d" : "i32, float, float", 713 "cube" : "float, float, float", 714 "acube" : "i32, float, float, float" 715 } 716 return geom_to_access[geom] 717 718def get_ptx_tld4_access(geom): 719 geom_to_access = { 720 "2d" : "{%f{{[0-9]+}}, %f{{[0-9]+}}}", 721 "a2d" : "{%r{{[0-9]+}}, %f{{[0-9]+}}, %f{{[0-9]+}}, %f{{[0-9]+}}}", 722 "cube" : "{%f{{[0-9]+}}, %f{{[0-9]+}}, %f{{[0-9]+}}, %f{{[0-9]+}}}", 723 "acube" : "{%r{{[0-9]+}}, %f{{[0-9]+}}, %f{{[0-9]+}}, %f{{[0-9]+}}}" 724 } 725 return geom_to_access[geom] 726 727def gen_tld4_tests(target, global_tex, global_sampler): 728 """ 729 PTX spec s9.7.9.4. Texture Instructions: tld4 730 Perform a texture fetch of the 4-texel bilerp footprint. 731 732 tld4.comp.2d.v4.dtype.f32 d[|p], [a, c] {, e} {, f}; 733 tld4.comp.geom.v4.dtype.f32 d[|p], [a, b, c] {, e} {, f}; // explicit sampler 734 735 .comp = { .r, .g, .b, .a }; 736 .geom = { .2d, .a2d, .cube, .acube }; 737 .dtype = { .u32, .s32, .f32 }; 738 """ 739 740 template = """ 741 declare ${retty} @${intrinsic}(i64 %tex, ${sampler} ${access}) 742 743 ; CHECK-LABEL: .entry ${test_name}_param 744 ; CHECK: ${instruction} ${ptx_ret}, [${ptx_tex}, ${ptx_access}] 745 define void @${test_name}_param(i64 %tex, ${sampler} ${retty}* %ret, ${access}) { 746 %val = tail call ${retty} @${intrinsic}(i64 %tex, ${sampler} ${access}) 747 store ${retty} %val, ${retty}* %ret 748 ret void 749 } 750 ; CHECK-LABEL: .entry ${test_name}_global 751 ; CHECK: ${instruction} ${ptx_ret}, [${global_tex}, ${ptx_global_sampler} ${ptx_access}] 752 define void @${test_name}_global(${retty}* %ret, ${access}) { 753 %gt = tail call i64 @llvm.nvvm.texsurf.handle.internal.p1i64(i64 addrspace(1)* @${global_tex}) 754 ${get_sampler_handle} 755 %val = tail call ${retty} @${intrinsic}(i64 %gt, ${sampler} ${access}) 756 store ${retty} %val, ${retty}* %ret 757 ret void 758 } 759 """ 760 761 generated_items = [] 762 generated_metadata = [] 763 for comp, geom, dtype in product( 764 ["r", "g", "b", "a"], 765 ["2d", "a2d", "cube", "acube"], 766 ["u32", "s32", "f32"]): 767 768 # FIXME: missing intrinsics. 769 # tld4.{a2d,cube,acube} introduced in PTX ISA version 4.3. 770 if geom in ("a2d", "cube", "acube"): 771 continue 772 773 sampler_handle, get_sampler_handle = get_llvm_global_sampler( 774 target, global_sampler) 775 776 test_name = "test_tld4_" + "".join((comp, geom, dtype)) 777 params = { 778 "test_name" : test_name, 779 "intrinsic" : 780 "llvm.nvvm.tld4{unified}.{comp}.{geom}.v4{dtype}.f32".format( 781 unified=(".unified" if is_unified(target) else ""), 782 comp=comp, 783 geom=get_llvm_geom(geom), 784 dtype=dtype), 785 "global_tex" : global_tex, 786 "retty" : get_llvm_vec_type("v4", dtype), 787 "sampler" : sampler_handle, 788 "access" : get_llvm_tld4_access(geom), 789 "get_sampler_handle" : get_sampler_handle, 790 791 "instruction" : "tld4.{comp}.{geom}.v4.{dtype}.f32".format( 792 comp=comp, geom=geom, dtype=dtype), 793 "ptx_ret" : get_ptx_vec_reg("v4", dtype), 794 "ptx_tex" : get_ptx_texture(target), 795 "ptx_access" : get_ptx_tld4_access(geom), 796 "ptx_global_sampler" : get_ptx_global_sampler(target, global_sampler), 797 } 798 gen_test(template, params) 799 generated_items.append((params["intrinsic"], params["instruction"])) 800 801 fun_name = test_name + "_param"; 802 fun_ty = "void (i64, {sampler} {retty}*, {access_ty})*".format( 803 sampler=("" if is_unified(target) else "i64,"), 804 retty=params["retty"], 805 access_ty=get_llvm_tld4_access_type(geom)) 806 generated_metadata += get_texture_metadata( 807 target, fun_ty, fun_name, has_texture_params=True) 808 809 fun_name = test_name + "_global"; 810 fun_ty = "void ({retty}*, {access_ty})*".format( 811 retty=params["retty"], 812 access_ty=get_llvm_tld4_access_type(geom)) 813 generated_metadata += get_texture_metadata( 814 target, fun_ty, fun_name, has_texture_params=False) 815 816 return generated_items, generated_metadata 817 818def gen_test(template, params): 819 if debug: 820 print() 821 for param, value in params.items(): 822 print(";; {}: {}".format(param, value)) 823 824 print(string.Template(textwrap.dedent(template)).substitute(params)) 825 826def gen_tests(target, tests): 827 gen_triple(target) 828 829 items = [] 830 metadata = [] 831 832 global_surf = "gsurf" 833 global_tex = "gtex" 834 global_sampler = "gsam" 835 metadata += gen_globals(target, global_surf, global_tex, global_sampler) 836 837 if "suld" in tests: 838 suld_items, suld_md = gen_suld_tests(target, global_surf) 839 items += suld_items 840 metadata += suld_md 841 if "sust" in tests: 842 sust_items, sust_md = gen_sust_tests(target, global_surf) 843 items += sust_items 844 metadata += sust_md 845 if "tex" in tests: 846 tex_items, tex_md = gen_tex_tests(target, global_tex, global_sampler) 847 items += tex_items 848 metadata += tex_md 849 if "tld4" in tests: 850 tld4_items, tld4_md = gen_tld4_tests(target, global_tex, global_sampler) 851 items += tld4_items 852 metadata += tld4_md 853 854 gen_metadata(metadata) 855 return items 856 857def write_gen_list(filename, append, items): 858 with open(filename, ("a" if append else "w")) as f: 859 for intrinsic, instruction in items: 860 f.write("{} {}\n".format(intrinsic, instruction)) 861 862def read_gen_list(filename): 863 intrinsics = set() 864 instructions = set() 865 with open(filename) as f: 866 for line in f: 867 intrinsic, instruction = line.split() 868 intrinsics.add(intrinsic) 869 instructions.add(instruction) 870 return (intrinsics, instructions) 871 872def read_td_list(filename, regex): 873 td_list = set() 874 with open(filename) as f: 875 for line in f: 876 match = re.search(regex, line) 877 if match: 878 td_list.add(match.group(1)) 879 880 # Arbitrary value - we should find quite a lot of instructions 881 if len(td_list) < 30: 882 raise RuntimeError("found only {} instructions in {}".format( 883 filename, len(td_list))) 884 885 return td_list 886 887def verify_inst_tablegen(path_td, gen_instr): 888 """ 889 Verify that all instructions defined in NVPTXIntrinsics.td are 890 tested. 891 """ 892 893 td_instr = read_td_list(path_td, "\"((suld|sust|tex|tld4)\\..*)\"") 894 895 gen_instr.update({ 896 # FIXME: spec does not list any sust.p variants other than b32 897 "sust.p.1d.b8.trap", 898 "sust.p.1d.b16.trap", 899 "sust.p.1d.v2.b8.trap", 900 "sust.p.1d.v2.b16.trap", 901 "sust.p.1d.v4.b8.trap", 902 "sust.p.1d.v4.b16.trap", 903 "sust.p.a1d.b8.trap", 904 "sust.p.a1d.b16.trap", 905 "sust.p.a1d.v2.b8.trap", 906 "sust.p.a1d.v2.b16.trap", 907 "sust.p.a1d.v4.b8.trap", 908 "sust.p.a1d.v4.b16.trap", 909 "sust.p.2d.b8.trap", 910 "sust.p.2d.b16.trap", 911 "sust.p.2d.v2.b8.trap", 912 "sust.p.2d.v2.b16.trap", 913 "sust.p.2d.v4.b8.trap", 914 "sust.p.2d.v4.b16.trap", 915 "sust.p.a2d.b8.trap", 916 "sust.p.a2d.b16.trap", 917 "sust.p.a2d.v2.b8.trap", 918 "sust.p.a2d.v2.b16.trap", 919 "sust.p.a2d.v4.b8.trap", 920 "sust.p.a2d.v4.b16.trap", 921 "sust.p.3d.b8.trap", 922 "sust.p.3d.b16.trap", 923 "sust.p.3d.v2.b8.trap", 924 "sust.p.3d.v2.b16.trap", 925 "sust.p.3d.v4.b8.trap", 926 "sust.p.3d.v4.b16.trap", 927 928 # FIXME: sust.p is also not supported for arrays 929 "sust.p.a1d.b32.trap", 930 "sust.p.a1d.v2.b32.trap", 931 "sust.p.a1d.v4.b32.trap", 932 "sust.p.a2d.b32.trap", 933 "sust.p.a2d.v2.b32.trap", 934 "sust.p.a2d.v4.b32.trap", 935 }) 936 937 td_instr = list(td_instr) 938 td_instr.sort() 939 gen_instr = list(gen_instr) 940 gen_instr.sort() 941 for i, td in enumerate(td_instr): 942 if i == len(gen_instr) or td != gen_instr[i]: 943 raise RuntimeError( 944 "{} is present in tablegen, but not tested.\n".format(td)) 945 946def verify_llvm_tablegen(path_td, gen_intr): 947 """ 948 Verify that all intrinsics defined in IntrinsicsNVVM.td are 949 tested. 950 """ 951 952 td_intr = read_td_list( 953 path_td, "\"(llvm\\.nvvm\\.(suld|sust|tex|tld4)\\..*)\"") 954 955 gen_intr.update({ 956 # FIXME: spec does not list any sust.p variants other than b32 957 "llvm.nvvm.sust.p.1d.i8.trap", 958 "llvm.nvvm.sust.p.1d.i16.trap", 959 "llvm.nvvm.sust.p.1d.v2i8.trap", 960 "llvm.nvvm.sust.p.1d.v2i16.trap", 961 "llvm.nvvm.sust.p.1d.v4i8.trap", 962 "llvm.nvvm.sust.p.1d.v4i16.trap", 963 "llvm.nvvm.sust.p.1d.array.i8.trap", 964 "llvm.nvvm.sust.p.1d.array.i16.trap", 965 "llvm.nvvm.sust.p.1d.array.v2i8.trap", 966 "llvm.nvvm.sust.p.1d.array.v2i16.trap", 967 "llvm.nvvm.sust.p.1d.array.v4i8.trap", 968 "llvm.nvvm.sust.p.1d.array.v4i16.trap", 969 "llvm.nvvm.sust.p.2d.i8.trap", 970 "llvm.nvvm.sust.p.2d.i16.trap", 971 "llvm.nvvm.sust.p.2d.v2i8.trap", 972 "llvm.nvvm.sust.p.2d.v2i16.trap", 973 "llvm.nvvm.sust.p.2d.v4i8.trap", 974 "llvm.nvvm.sust.p.2d.v4i16.trap", 975 "llvm.nvvm.sust.p.2d.array.i8.trap", 976 "llvm.nvvm.sust.p.2d.array.i16.trap", 977 "llvm.nvvm.sust.p.2d.array.v2i8.trap", 978 "llvm.nvvm.sust.p.2d.array.v2i16.trap", 979 "llvm.nvvm.sust.p.2d.array.v4i8.trap", 980 "llvm.nvvm.sust.p.2d.array.v4i16.trap", 981 "llvm.nvvm.sust.p.3d.i8.trap", 982 "llvm.nvvm.sust.p.3d.i16.trap", 983 "llvm.nvvm.sust.p.3d.v2i8.trap", 984 "llvm.nvvm.sust.p.3d.v2i16.trap", 985 "llvm.nvvm.sust.p.3d.v4i8.trap", 986 "llvm.nvvm.sust.p.3d.v4i16.trap", 987 988 # FIXME: sust.p is also not supported for arrays 989 "llvm.nvvm.sust.p.1d.array.i32.trap", 990 "llvm.nvvm.sust.p.1d.array.v2i32.trap", 991 "llvm.nvvm.sust.p.1d.array.v4i32.trap", 992 "llvm.nvvm.sust.p.2d.array.i32.trap", 993 "llvm.nvvm.sust.p.2d.array.v2i32.trap", 994 "llvm.nvvm.sust.p.2d.array.v4i32.trap" 995 }) 996 997 td_intr = list(td_intr) 998 td_intr.sort() 999 gen_intr = list(gen_intr) 1000 gen_intr.sort() 1001 for i, td in enumerate(td_intr): 1002 if i == len(gen_intr) or td != gen_intr[i]: 1003 raise RuntimeError( 1004 "{} is present in tablegen, but not tested.\n".format(td)) 1005 1006parser = argparse.ArgumentParser() 1007parser.add_argument("--debug", action="store_true") 1008parser.add_argument("--tests", type=str) 1009parser.add_argument("--target", type=str) 1010parser.add_argument("--gen-list", dest="gen_list", type=str) 1011parser.add_argument("--gen-list-append", dest="gen_list_append", 1012 action="store_true") 1013parser.add_argument("--verify", action="store_true") 1014parser.add_argument("--llvm-tablegen", dest="llvm_td", type=str) 1015parser.add_argument("--inst-tablegen", dest="inst_td", type=str) 1016 1017args = parser.parse_args() 1018debug = args.debug 1019 1020if args.verify: 1021 intrinsics, instructions = read_gen_list(args.gen_list) 1022 verify_inst_tablegen(args.inst_td, instructions) 1023 verify_llvm_tablegen(args.llvm_td, intrinsics) 1024else: 1025 items = gen_tests(args.target, args.tests.split(",")) 1026 if (args.gen_list): 1027 write_gen_list(args.gen_list, args.gen_list_append, items) 1028