1; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=FUNC %s 2; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=FUNC %s 3; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=FUNC %s 4; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s 5; RUN: llc -march=r600 -mcpu=cayman -verify-machineinstrs < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s 6 7; FUNC-LABEL: {{^}}store_i1: 8; EG: MEM_RAT MSKOR 9; EG-NOT: MEM_RAT MSKOR 10 11; CM: MEM_RAT MSKOR 12; CM-NOT: MEM_RAT MSKOR 13 14; SIVI: buffer_store_byte 15; GFX9: global_store_byte 16define amdgpu_kernel void @store_i1(i1 addrspace(1)* %out) { 17entry: 18 store i1 true, i1 addrspace(1)* %out 19 ret void 20} 21 22; i8 store 23; FUNC-LABEL: {{^}}store_i8: 24; EG: MEM_RAT MSKOR T[[RW_GPR:[0-9]]].XW, T{{[0-9]}}.X 25; EG-NOT: MEM_RAT MSKOR 26 27; IG 0: Get the byte index and truncate the value 28; EG: AND_INT * T{{[0-9]}}.[[BI_CHAN:[XYZW]]], KC0[2].Y, literal.x 29; EG: LSHL T{{[0-9]}}.[[SHIFT_CHAN:[XYZW]]], PV.[[BI_CHAN]], literal.x 30; EG: AND_INT * T{{[0-9]}}.[[TRUNC_CHAN:[XYZW]]], KC0[2].Z, literal.y 31; EG-NEXT: 3(4.203895e-45), 255(3.573311e-43) 32 33 34; IG 1: Truncate the calculated the shift amount for the mask 35 36; IG 2: Shift the value and the mask 37; EG: LSHL T[[RW_GPR]].X, PS, PV.[[SHIFT_CHAN]] 38; EG: LSHL * T[[RW_GPR]].W, literal.x, PV.[[SHIFT_CHAN]] 39; EG-NEXT: 255 40; IG 3: Initialize the Y and Z channels to zero 41; XXX: An optimal scheduler should merge this into one of the prevous IGs. 42; EG: MOV T[[RW_GPR]].Y, 0.0 43; EG: MOV * T[[RW_GPR]].Z, 0.0 44 45; SIVI: buffer_store_byte 46; GFX9: global_store_byte 47define amdgpu_kernel void @store_i8(i8 addrspace(1)* %out, i8 %in) { 48entry: 49 store i8 %in, i8 addrspace(1)* %out 50 ret void 51} 52 53; i16 store 54; FUNC-LABEL: {{^}}store_i16: 55; EG: MEM_RAT MSKOR T[[RW_GPR:[0-9]]].XW, T{{[0-9]}}.X 56; EG-NOT: MEM_RAT MSKOR 57 58; IG 0: Get the byte index and truncate the value 59 60 61; EG: AND_INT * T{{[0-9]}}.[[BI_CHAN:[XYZW]]], KC0[2].Y, literal.x 62; EG-NEXT: 3(4.203895e-45), 63 64; EG: LSHL T{{[0-9]}}.[[SHIFT_CHAN:[XYZW]]], PV.[[BI_CHAN]], literal.x 65; EG: AND_INT * T{{[0-9]}}.[[TRUNC_CHAN:[XYZW]]], KC0[2].Z, literal.y 66 67; EG-NEXT: 3(4.203895e-45), 65535(9.183409e-41) 68; IG 1: Truncate the calculated the shift amount for the mask 69 70; IG 2: Shift the value and the mask 71; EG: LSHL T[[RW_GPR]].X, PS, PV.[[SHIFT_CHAN]] 72; EG: LSHL * T[[RW_GPR]].W, literal.x, PV.[[SHIFT_CHAN]] 73; EG-NEXT: 65535 74; IG 3: Initialize the Y and Z channels to zero 75; XXX: An optimal scheduler should merge this into one of the prevous IGs. 76; EG: MOV T[[RW_GPR]].Y, 0.0 77; EG: MOV * T[[RW_GPR]].Z, 0.0 78 79; SIVI: buffer_store_short 80; GFX9: global_store_short 81define amdgpu_kernel void @store_i16(i16 addrspace(1)* %out, i16 %in) { 82entry: 83 store i16 %in, i16 addrspace(1)* %out 84 ret void 85} 86 87; FUNC-LABEL: {{^}}store_i24: 88; SIVI: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16 89; SIVI-DAG: buffer_store_byte 90; SIVI-DAG: buffer_store_short 91 92; GFX9-DAG: global_store_byte_d16_hi v{{\[[0-9]:[0-9]+\]}}, v{{[0-9]+}}, off offset:2 93; GFX9-DAG: global_store_short 94 95; EG: MEM_RAT MSKOR 96; EG: MEM_RAT MSKOR 97define amdgpu_kernel void @store_i24(i24 addrspace(1)* %out, i24 %in) { 98entry: 99 store i24 %in, i24 addrspace(1)* %out 100 ret void 101} 102 103; FUNC-LABEL: {{^}}store_i25: 104; GCN: s_and_b32 [[AND:s[0-9]+]], s{{[0-9]+}}, 0x1ffffff{{$}} 105; GCN: v_mov_b32_e32 [[VAND:v[0-9]+]], [[AND]] 106; SIVI: buffer_store_dword [[VAND]] 107; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[VAND]] 108 109; EG: MEM_RAT_CACHELESS STORE_RAW 110; EG-NOT: MEM_RAT 111 112; CM: MEM_RAT_CACHELESS STORE_DWORD 113; CM-NOT: MEM_RAT 114define amdgpu_kernel void @store_i25(i25 addrspace(1)* %out, i25 %in) { 115entry: 116 store i25 %in, i25 addrspace(1)* %out 117 ret void 118} 119 120; FUNC-LABEL: {{^}}store_v2i8: 121; v2i8 is naturally 2B aligned 122; EG: MEM_RAT MSKOR 123; EG-NOT: MEM_RAT MSKOR 124 125; CM: MEM_RAT MSKOR 126; CM-NOT: MEM_RAT MSKOR 127 128; SIVI: buffer_store_short 129; GFX9: global_store_short 130define amdgpu_kernel void @store_v2i8(<2 x i8> addrspace(1)* %out, <2 x i32> %in) { 131entry: 132 %0 = trunc <2 x i32> %in to <2 x i8> 133 store <2 x i8> %0, <2 x i8> addrspace(1)* %out 134 ret void 135} 136 137; FUNC-LABEL: {{^}}store_v2i8_unaligned: 138; EG: MEM_RAT MSKOR 139; EG: MEM_RAT MSKOR 140; EG-NOT: MEM_RAT MSKOR 141 142; CM: MEM_RAT MSKOR 143; CM: MEM_RAT MSKOR 144; CM-NOT: MEM_RAT MSKOR 145 146; SI: buffer_store_byte 147define amdgpu_kernel void @store_v2i8_unaligned(<2 x i8> addrspace(1)* %out, <2 x i32> %in) { 148entry: 149 %0 = trunc <2 x i32> %in to <2 x i8> 150 store <2 x i8> %0, <2 x i8> addrspace(1)* %out, align 1 151 ret void 152} 153 154 155; FUNC-LABEL: {{^}}store_v2i16: 156; EG: MEM_RAT_CACHELESS STORE_RAW 157 158; CM: MEM_RAT_CACHELESS STORE_DWORD 159 160; SIVI: buffer_store_dword 161; GFX9: global_store_dword 162define amdgpu_kernel void @store_v2i16(<2 x i16> addrspace(1)* %out, <2 x i32> %in) { 163entry: 164 %0 = trunc <2 x i32> %in to <2 x i16> 165 store <2 x i16> %0, <2 x i16> addrspace(1)* %out 166 ret void 167} 168 169; FUNC-LABEL: {{^}}store_v2i16_unaligned: 170; EG: MEM_RAT MSKOR 171; EG: MEM_RAT MSKOR 172; EG-NOT: MEM_RAT MSKOR 173; EG-NOT: MEM_RAT_CACHELESS STORE_RAW 174 175; CM: MEM_RAT MSKOR 176; CM: MEM_RAT MSKOR 177; CM-NOT: MEM_RAT MSKOR 178; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD 179 180; SIVI: buffer_store_short 181; SIVI: buffer_store_short 182 183; GFX9: global_store_short 184; GFX9: global_store_short 185define amdgpu_kernel void @store_v2i16_unaligned(<2 x i16> addrspace(1)* %out, <2 x i32> %in) { 186entry: 187 %0 = trunc <2 x i32> %in to <2 x i16> 188 store <2 x i16> %0, <2 x i16> addrspace(1)* %out, align 2 189 ret void 190} 191 192; FUNC-LABEL: {{^}}store_v4i8: 193; EG: MEM_RAT_CACHELESS STORE_RAW 194 195; CM: MEM_RAT_CACHELESS STORE_DWORD 196 197; SIVI: buffer_store_dword 198; GFX9: global_store_dword 199define amdgpu_kernel void @store_v4i8(<4 x i8> addrspace(1)* %out, <4 x i32> %in) { 200entry: 201 %0 = trunc <4 x i32> %in to <4 x i8> 202 store <4 x i8> %0, <4 x i8> addrspace(1)* %out 203 ret void 204} 205 206; FUNC-LABEL: {{^}}store_v4i8_unaligned: 207; EG: MEM_RAT MSKOR 208; EG: MEM_RAT MSKOR 209; EG: MEM_RAT MSKOR 210; EG: MEM_RAT MSKOR 211; EG-NOT: MEM_RAT MSKOR 212; EG-NOT: MEM_RAT_CACHELESS STORE_RAW 213 214; CM: MEM_RAT MSKOR 215; CM: MEM_RAT MSKOR 216; CM: MEM_RAT MSKOR 217; CM: MEM_RAT MSKOR 218; CM-NOT: MEM_RAT MSKOR 219; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD 220 221; SI: buffer_store_byte 222; SI: buffer_store_byte 223; SI: buffer_store_byte 224; SI: buffer_store_byte 225; SI-NOT: buffer_store_dword 226define amdgpu_kernel void @store_v4i8_unaligned(<4 x i8> addrspace(1)* %out, <4 x i32> %in) { 227entry: 228 %0 = trunc <4 x i32> %in to <4 x i8> 229 store <4 x i8> %0, <4 x i8> addrspace(1)* %out, align 1 230 ret void 231} 232 233; FUNC-LABEL: {{^}}store_v4i8_halfaligned: 234; EG: MEM_RAT MSKOR 235; EG: MEM_RAT MSKOR 236; EG-NOT: MEM_RAT MSKOR 237; EG-NOT: MEM_RAT_CACHELESS STORE_RAW 238 239; CM: MEM_RAT MSKOR 240; CM: MEM_RAT MSKOR 241; CM-NOT: MEM_RAT MSKOR 242; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD 243 244; SI: buffer_store_short 245; SI: buffer_store_short 246; SI-NOT: buffer_store_dword 247define amdgpu_kernel void @store_v4i8_halfaligned(<4 x i8> addrspace(1)* %out, <4 x i32> %in) { 248entry: 249 %0 = trunc <4 x i32> %in to <4 x i8> 250 store <4 x i8> %0, <4 x i8> addrspace(1)* %out, align 2 251 ret void 252} 253 254; floating-point store 255; FUNC-LABEL: {{^}}store_f32: 256; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.X, T[0-9]+\.X}}, 1 257 258; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+\.X, T[0-9]+\.X}} 259 260; SIVI: buffer_store_dword 261; GFX9: global_store_dword 262 263define amdgpu_kernel void @store_f32(float addrspace(1)* %out, float %in) { 264 store float %in, float addrspace(1)* %out 265 ret void 266} 267 268; FUNC-LABEL: {{^}}store_v4i16: 269; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+}}.XY 270 271; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+}} 272 273; SIVI: buffer_store_dwordx2 274; GFX9: global_store_dwordx2 275define amdgpu_kernel void @store_v4i16(<4 x i16> addrspace(1)* %out, <4 x i32> %in) { 276entry: 277 %0 = trunc <4 x i32> %in to <4 x i16> 278 store <4 x i16> %0, <4 x i16> addrspace(1)* %out 279 ret void 280} 281 282; vec2 floating-point stores 283; FUNC-LABEL: {{^}}store_v2f32: 284; EG: MEM_RAT_CACHELESS STORE_RAW 285 286; CM: MEM_RAT_CACHELESS STORE_DWORD 287 288; SIVI: buffer_store_dwordx2 289; GFX9: global_store_dwordx2 290 291define amdgpu_kernel void @store_v2f32(<2 x float> addrspace(1)* %out, float %a, float %b) { 292entry: 293 %0 = insertelement <2 x float> <float 0.0, float 0.0>, float %a, i32 0 294 %1 = insertelement <2 x float> %0, float %b, i32 1 295 store <2 x float> %1, <2 x float> addrspace(1)* %out 296 ret void 297} 298 299; FUNC-LABEL: {{^}}store_v3i32: 300; SIVI-DAG: buffer_store_dwordx2 301; SIVI-DAG: buffer_store_dword v 302 303; GFX9-DAG: global_store_dwordx2 304; GFX9-DAG: global_store_dword v 305 306; EG-DAG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.[XYZW]}}, {{T[0-9]+\.[XYZW]}}, 307; EG-DAG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.XY}}, {{T[0-9]+\.[XYZW]}}, 308define amdgpu_kernel void @store_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> %a) nounwind { 309 store <3 x i32> %a, <3 x i32> addrspace(1)* %out, align 16 310 ret void 311} 312 313; FUNC-LABEL: {{^}}store_v4i32: 314; EG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.XYZW}} 315; EG-NOT: MEM_RAT_CACHELESS STORE_RAW 316 317; CM: MEM_RAT_CACHELESS STORE_DWORD 318; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD 319 320; SIVI: buffer_store_dwordx4 321; GFX9: global_store_dwordx4 322define amdgpu_kernel void @store_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %in) { 323entry: 324 store <4 x i32> %in, <4 x i32> addrspace(1)* %out 325 ret void 326} 327 328; FUNC-LABEL: {{^}}store_v4i32_unaligned: 329; EG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.XYZW}} 330; EG-NOT: MEM_RAT_CACHELESS STORE_RAW 331 332; CM: MEM_RAT_CACHELESS STORE_DWORD 333; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD 334 335; SIVI: buffer_store_dwordx4 336; GFX9: global_store_dwordx4 337define amdgpu_kernel void @store_v4i32_unaligned(<4 x i32> addrspace(1)* %out, <4 x i32> %in) { 338entry: 339 store <4 x i32> %in, <4 x i32> addrspace(1)* %out, align 4 340 ret void 341} 342 343; v4f32 store 344; FUNC-LABEL: {{^}}store_v4f32: 345; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.XYZW, T[0-9]+\.X}}, 1 346; EG-NOT: MEM_RAT_CACHELESS STORE_RAW 347 348; CM: MEM_RAT_CACHELESS STORE_DWORD 349; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD 350 351; SIVI: buffer_store_dwordx4 352; GFX9: global_store_dwordx4 353define amdgpu_kernel void @store_v4f32(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in) { 354 %1 = load <4 x float>, <4 x float> addrspace(1) * %in 355 store <4 x float> %1, <4 x float> addrspace(1)* %out 356 ret void 357} 358 359; FUNC-LABEL: {{^}}store_i64_i8: 360; EG: MEM_RAT MSKOR 361 362; CM: MEM_RAT MSKOR 363 364; SIVI: buffer_store_byte 365; GFX9: global_store_byte 366define amdgpu_kernel void @store_i64_i8(i8 addrspace(1)* %out, i64 %in) { 367entry: 368 %0 = trunc i64 %in to i8 369 store i8 %0, i8 addrspace(1)* %out 370 ret void 371} 372 373; FUNC-LABEL: {{^}}store_i64_i16: 374; EG: MEM_RAT MSKOR 375; SIVI: buffer_store_short 376; GFX9: global_store_short 377define amdgpu_kernel void @store_i64_i16(i16 addrspace(1)* %out, i64 %in) { 378entry: 379 %0 = trunc i64 %in to i16 380 store i16 %0, i16 addrspace(1)* %out 381 ret void 382} 383 384; The stores in this function are combined by the optimizer to create a 385; 64-bit store with 32-bit alignment. This is legal and the legalizer 386; should not try to split the 64-bit store back into 2 32-bit stores. 387 388; FUNC-LABEL: {{^}}vecload2: 389; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.XY, T[0-9]+\.X}}, 1 390; EG-NOT: MEM_RAT_CACHELESS STORE_RAW 391 392; CM: MEM_RAT_CACHELESS STORE_DWORD 393; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD 394 395; SIVI: buffer_store_dwordx2 396; GFX9: global_store_dwordx2 397define amdgpu_kernel void @vecload2(i32 addrspace(1)* nocapture %out, i32 addrspace(2)* nocapture %mem) #0 { 398entry: 399 %0 = load i32, i32 addrspace(2)* %mem, align 4 400 %arrayidx1.i = getelementptr inbounds i32, i32 addrspace(2)* %mem, i64 1 401 %1 = load i32, i32 addrspace(2)* %arrayidx1.i, align 4 402 store i32 %0, i32 addrspace(1)* %out, align 4 403 %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1 404 store i32 %1, i32 addrspace(1)* %arrayidx1, align 4 405 ret void 406} 407 408; When i128 was a legal type this program generated cannot select errors: 409 410; FUNC-LABEL: {{^}}"i128-const-store": 411; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 1 412 413; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+}}, T{{[0-9]+}}.X 414 415; SIVI: buffer_store_dwordx4 416; GFX9: global_store_dwordx4 417define amdgpu_kernel void @i128-const-store(i32 addrspace(1)* %out) { 418entry: 419 store i32 1, i32 addrspace(1)* %out, align 4 420 %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1 421 store i32 1, i32 addrspace(1)* %arrayidx2, align 4 422 %arrayidx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2 423 store i32 2, i32 addrspace(1)* %arrayidx4, align 4 424 %arrayidx6 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3 425 store i32 2, i32 addrspace(1)* %arrayidx6, align 4 426 ret void 427} 428 429attributes #0 = { nounwind } 430