1 //===-- AMDGPUTargetStreamer.cpp - Mips Target Streamer Methods -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file provides AMDGPU specific target streamer methods. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "AMDGPUTargetStreamer.h" 14 #include "AMDGPUPTNote.h" 15 #include "AMDKernelCodeT.h" 16 #include "Utils/AMDGPUBaseInfo.h" 17 #include "Utils/AMDKernelCodeTUtils.h" 18 #include "llvm/BinaryFormat/AMDGPUMetadataVerifier.h" 19 #include "llvm/BinaryFormat/ELF.h" 20 #include "llvm/MC/MCContext.h" 21 #include "llvm/MC/MCELFStreamer.h" 22 #include "llvm/MC/MCSectionELF.h" 23 #include "llvm/Support/AMDGPUMetadata.h" 24 #include "llvm/Support/AMDHSAKernelDescriptor.h" 25 #include "llvm/Support/FormattedStream.h" 26 27 using namespace llvm; 28 using namespace llvm::AMDGPU; 29 30 //===----------------------------------------------------------------------===// 31 // AMDGPUTargetStreamer 32 //===----------------------------------------------------------------------===// 33 34 bool AMDGPUTargetStreamer::EmitHSAMetadataV2(StringRef HSAMetadataString) { 35 HSAMD::Metadata HSAMetadata; 36 if (HSAMD::fromString(HSAMetadataString, HSAMetadata)) 37 return false; 38 return EmitHSAMetadata(HSAMetadata); 39 } 40 41 bool AMDGPUTargetStreamer::EmitHSAMetadataV3(StringRef HSAMetadataString) { 42 msgpack::Document HSAMetadataDoc; 43 if (!HSAMetadataDoc.fromYAML(HSAMetadataString)) 44 return false; 45 return EmitHSAMetadata(HSAMetadataDoc, false); 46 } 47 48 StringRef AMDGPUTargetStreamer::getArchNameFromElfMach(unsigned ElfMach) { 49 AMDGPU::GPUKind AK; 50 51 switch (ElfMach) { 52 default: llvm_unreachable("Unhandled ELF::EF_AMDGPU type"); 53 case ELF::EF_AMDGPU_MACH_R600_R600: AK = GK_R600; break; 54 case ELF::EF_AMDGPU_MACH_R600_R630: AK = GK_R630; break; 55 case ELF::EF_AMDGPU_MACH_R600_RS880: AK = GK_RS880; break; 56 case ELF::EF_AMDGPU_MACH_R600_RV670: AK = GK_RV670; break; 57 case ELF::EF_AMDGPU_MACH_R600_RV710: AK = GK_RV710; break; 58 case ELF::EF_AMDGPU_MACH_R600_RV730: AK = GK_RV730; break; 59 case ELF::EF_AMDGPU_MACH_R600_RV770: AK = GK_RV770; break; 60 case ELF::EF_AMDGPU_MACH_R600_CEDAR: AK = GK_CEDAR; break; 61 case ELF::EF_AMDGPU_MACH_R600_CYPRESS: AK = GK_CYPRESS; break; 62 case ELF::EF_AMDGPU_MACH_R600_JUNIPER: AK = GK_JUNIPER; break; 63 case ELF::EF_AMDGPU_MACH_R600_REDWOOD: AK = GK_REDWOOD; break; 64 case ELF::EF_AMDGPU_MACH_R600_SUMO: AK = GK_SUMO; break; 65 case ELF::EF_AMDGPU_MACH_R600_BARTS: AK = GK_BARTS; break; 66 case ELF::EF_AMDGPU_MACH_R600_CAICOS: AK = GK_CAICOS; break; 67 case ELF::EF_AMDGPU_MACH_R600_CAYMAN: AK = GK_CAYMAN; break; 68 case ELF::EF_AMDGPU_MACH_R600_TURKS: AK = GK_TURKS; break; 69 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX600: AK = GK_GFX600; break; 70 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX601: AK = GK_GFX601; break; 71 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX602: AK = GK_GFX602; break; 72 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX700: AK = GK_GFX700; break; 73 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX701: AK = GK_GFX701; break; 74 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX702: AK = GK_GFX702; break; 75 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX703: AK = GK_GFX703; break; 76 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX704: AK = GK_GFX704; break; 77 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX705: AK = GK_GFX705; break; 78 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX801: AK = GK_GFX801; break; 79 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX802: AK = GK_GFX802; break; 80 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX803: AK = GK_GFX803; break; 81 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX805: AK = GK_GFX805; break; 82 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX810: AK = GK_GFX810; break; 83 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX900: AK = GK_GFX900; break; 84 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX902: AK = GK_GFX902; break; 85 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX904: AK = GK_GFX904; break; 86 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX906: AK = GK_GFX906; break; 87 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX908: AK = GK_GFX908; break; 88 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX909: AK = GK_GFX909; break; 89 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX90A: AK = GK_GFX90A; break; 90 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX90C: AK = GK_GFX90C; break; 91 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1010: AK = GK_GFX1010; break; 92 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1011: AK = GK_GFX1011; break; 93 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1012: AK = GK_GFX1012; break; 94 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1030: AK = GK_GFX1030; break; 95 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1031: AK = GK_GFX1031; break; 96 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1032: AK = GK_GFX1032; break; 97 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1033: AK = GK_GFX1033; break; 98 case ELF::EF_AMDGPU_MACH_NONE: AK = GK_NONE; break; 99 } 100 101 StringRef GPUName = getArchNameAMDGCN(AK); 102 if (GPUName != "") 103 return GPUName; 104 return getArchNameR600(AK); 105 } 106 107 unsigned AMDGPUTargetStreamer::getElfMach(StringRef GPU) { 108 AMDGPU::GPUKind AK = parseArchAMDGCN(GPU); 109 if (AK == AMDGPU::GPUKind::GK_NONE) 110 AK = parseArchR600(GPU); 111 112 switch (AK) { 113 case GK_R600: return ELF::EF_AMDGPU_MACH_R600_R600; 114 case GK_R630: return ELF::EF_AMDGPU_MACH_R600_R630; 115 case GK_RS880: return ELF::EF_AMDGPU_MACH_R600_RS880; 116 case GK_RV670: return ELF::EF_AMDGPU_MACH_R600_RV670; 117 case GK_RV710: return ELF::EF_AMDGPU_MACH_R600_RV710; 118 case GK_RV730: return ELF::EF_AMDGPU_MACH_R600_RV730; 119 case GK_RV770: return ELF::EF_AMDGPU_MACH_R600_RV770; 120 case GK_CEDAR: return ELF::EF_AMDGPU_MACH_R600_CEDAR; 121 case GK_CYPRESS: return ELF::EF_AMDGPU_MACH_R600_CYPRESS; 122 case GK_JUNIPER: return ELF::EF_AMDGPU_MACH_R600_JUNIPER; 123 case GK_REDWOOD: return ELF::EF_AMDGPU_MACH_R600_REDWOOD; 124 case GK_SUMO: return ELF::EF_AMDGPU_MACH_R600_SUMO; 125 case GK_BARTS: return ELF::EF_AMDGPU_MACH_R600_BARTS; 126 case GK_CAICOS: return ELF::EF_AMDGPU_MACH_R600_CAICOS; 127 case GK_CAYMAN: return ELF::EF_AMDGPU_MACH_R600_CAYMAN; 128 case GK_TURKS: return ELF::EF_AMDGPU_MACH_R600_TURKS; 129 case GK_GFX600: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX600; 130 case GK_GFX601: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX601; 131 case GK_GFX602: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX602; 132 case GK_GFX700: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX700; 133 case GK_GFX701: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX701; 134 case GK_GFX702: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX702; 135 case GK_GFX703: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX703; 136 case GK_GFX704: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX704; 137 case GK_GFX705: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX705; 138 case GK_GFX801: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX801; 139 case GK_GFX802: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX802; 140 case GK_GFX803: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX803; 141 case GK_GFX805: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX805; 142 case GK_GFX810: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX810; 143 case GK_GFX900: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX900; 144 case GK_GFX902: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX902; 145 case GK_GFX904: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX904; 146 case GK_GFX906: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX906; 147 case GK_GFX908: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX908; 148 case GK_GFX909: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX909; 149 case GK_GFX90A: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX90A; 150 case GK_GFX90C: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX90C; 151 case GK_GFX1010: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1010; 152 case GK_GFX1011: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1011; 153 case GK_GFX1012: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1012; 154 case GK_GFX1030: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1030; 155 case GK_GFX1031: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1031; 156 case GK_GFX1032: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1032; 157 case GK_GFX1033: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1033; 158 case GK_NONE: return ELF::EF_AMDGPU_MACH_NONE; 159 } 160 161 llvm_unreachable("unknown GPU"); 162 } 163 164 //===----------------------------------------------------------------------===// 165 // AMDGPUTargetAsmStreamer 166 //===----------------------------------------------------------------------===// 167 168 AMDGPUTargetAsmStreamer::AMDGPUTargetAsmStreamer(MCStreamer &S, 169 formatted_raw_ostream &OS) 170 : AMDGPUTargetStreamer(S), OS(OS) { } 171 172 // A hook for emitting stuff at the end. 173 // We use it for emitting the accumulated PAL metadata as directives. 174 // The PAL metadata is reset after it is emitted. 175 void AMDGPUTargetAsmStreamer::finish() { 176 std::string S; 177 getPALMetadata()->toString(S); 178 OS << S; 179 180 // Reset the pal metadata so its data will not affect a compilation that 181 // reuses this object. 182 getPALMetadata()->reset(); 183 } 184 185 void AMDGPUTargetAsmStreamer::EmitDirectiveAMDGCNTarget(StringRef Target) { 186 OS << "\t.amdgcn_target \"" << Target << "\"\n"; 187 } 188 189 void AMDGPUTargetAsmStreamer::EmitDirectiveHSACodeObjectVersion( 190 uint32_t Major, uint32_t Minor) { 191 OS << "\t.hsa_code_object_version " << 192 Twine(Major) << "," << Twine(Minor) << '\n'; 193 } 194 195 void 196 AMDGPUTargetAsmStreamer::EmitDirectiveHSACodeObjectISA(uint32_t Major, 197 uint32_t Minor, 198 uint32_t Stepping, 199 StringRef VendorName, 200 StringRef ArchName) { 201 OS << "\t.hsa_code_object_isa " << 202 Twine(Major) << "," << Twine(Minor) << "," << Twine(Stepping) << 203 ",\"" << VendorName << "\",\"" << ArchName << "\"\n"; 204 205 } 206 207 void 208 AMDGPUTargetAsmStreamer::EmitAMDKernelCodeT(const amd_kernel_code_t &Header) { 209 OS << "\t.amd_kernel_code_t\n"; 210 dumpAmdKernelCode(&Header, OS, "\t\t"); 211 OS << "\t.end_amd_kernel_code_t\n"; 212 } 213 214 void AMDGPUTargetAsmStreamer::EmitAMDGPUSymbolType(StringRef SymbolName, 215 unsigned Type) { 216 switch (Type) { 217 default: llvm_unreachable("Invalid AMDGPU symbol type"); 218 case ELF::STT_AMDGPU_HSA_KERNEL: 219 OS << "\t.amdgpu_hsa_kernel " << SymbolName << '\n' ; 220 break; 221 } 222 } 223 224 void AMDGPUTargetAsmStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size, 225 Align Alignment) { 226 OS << "\t.amdgpu_lds " << Symbol->getName() << ", " << Size << ", " 227 << Alignment.value() << '\n'; 228 } 229 230 bool AMDGPUTargetAsmStreamer::EmitISAVersion(StringRef IsaVersionString) { 231 OS << "\t.amd_amdgpu_isa \"" << IsaVersionString << "\"\n"; 232 return true; 233 } 234 235 bool AMDGPUTargetAsmStreamer::EmitHSAMetadata( 236 const AMDGPU::HSAMD::Metadata &HSAMetadata) { 237 std::string HSAMetadataString; 238 if (HSAMD::toString(HSAMetadata, HSAMetadataString)) 239 return false; 240 241 OS << '\t' << HSAMD::AssemblerDirectiveBegin << '\n'; 242 OS << HSAMetadataString << '\n'; 243 OS << '\t' << HSAMD::AssemblerDirectiveEnd << '\n'; 244 return true; 245 } 246 247 bool AMDGPUTargetAsmStreamer::EmitHSAMetadata( 248 msgpack::Document &HSAMetadataDoc, bool Strict) { 249 HSAMD::V3::MetadataVerifier Verifier(Strict); 250 if (!Verifier.verify(HSAMetadataDoc.getRoot())) 251 return false; 252 253 std::string HSAMetadataString; 254 raw_string_ostream StrOS(HSAMetadataString); 255 HSAMetadataDoc.toYAML(StrOS); 256 257 OS << '\t' << HSAMD::V3::AssemblerDirectiveBegin << '\n'; 258 OS << StrOS.str() << '\n'; 259 OS << '\t' << HSAMD::V3::AssemblerDirectiveEnd << '\n'; 260 return true; 261 } 262 263 bool AMDGPUTargetAsmStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { 264 const uint32_t Encoded_s_code_end = 0xbf9f0000; 265 const uint32_t Encoded_s_nop = 0xbf800000; 266 uint32_t Encoded_pad = Encoded_s_code_end; 267 268 // Instruction cache line size in bytes. 269 const unsigned Log2CacheLineSize = 6; 270 const unsigned CacheLineSize = 1u << Log2CacheLineSize; 271 272 // Extra padding amount in bytes to support prefetch mode 3. 273 unsigned FillSize = 3 * CacheLineSize; 274 275 if (AMDGPU::isGFX90A(STI)) { 276 Encoded_pad = Encoded_s_nop; 277 FillSize = 16 * CacheLineSize; 278 } 279 280 OS << "\t.p2alignl " << Log2CacheLineSize << ", " << Encoded_pad << '\n'; 281 OS << "\t.fill " << (FillSize / 4) << ", 4, " << Encoded_pad << '\n'; 282 return true; 283 } 284 285 void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor( 286 const MCSubtargetInfo &STI, StringRef KernelName, 287 const amdhsa::kernel_descriptor_t &KD, uint64_t NextVGPR, uint64_t NextSGPR, 288 bool ReserveVCC, bool ReserveFlatScr, bool ReserveXNACK) { 289 IsaVersion IVersion = getIsaVersion(STI.getCPU()); 290 291 OS << "\t.amdhsa_kernel " << KernelName << '\n'; 292 293 #define PRINT_FIELD(STREAM, DIRECTIVE, KERNEL_DESC, MEMBER_NAME, FIELD_NAME) \ 294 STREAM << "\t\t" << DIRECTIVE << " " \ 295 << AMDHSA_BITS_GET(KERNEL_DESC.MEMBER_NAME, FIELD_NAME) << '\n'; 296 297 OS << "\t\t.amdhsa_group_segment_fixed_size " << KD.group_segment_fixed_size 298 << '\n'; 299 OS << "\t\t.amdhsa_private_segment_fixed_size " 300 << KD.private_segment_fixed_size << '\n'; 301 302 PRINT_FIELD(OS, ".amdhsa_user_sgpr_private_segment_buffer", KD, 303 kernel_code_properties, 304 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER); 305 PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_ptr", KD, 306 kernel_code_properties, 307 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR); 308 PRINT_FIELD(OS, ".amdhsa_user_sgpr_queue_ptr", KD, 309 kernel_code_properties, 310 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR); 311 PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_segment_ptr", KD, 312 kernel_code_properties, 313 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR); 314 PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_id", KD, 315 kernel_code_properties, 316 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID); 317 PRINT_FIELD(OS, ".amdhsa_user_sgpr_flat_scratch_init", KD, 318 kernel_code_properties, 319 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT); 320 PRINT_FIELD(OS, ".amdhsa_user_sgpr_private_segment_size", KD, 321 kernel_code_properties, 322 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE); 323 if (IVersion.Major >= 10) 324 PRINT_FIELD(OS, ".amdhsa_wavefront_size32", KD, 325 kernel_code_properties, 326 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32); 327 PRINT_FIELD( 328 OS, ".amdhsa_system_sgpr_private_segment_wavefront_offset", KD, 329 compute_pgm_rsrc2, 330 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT); 331 PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_x", KD, 332 compute_pgm_rsrc2, 333 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X); 334 PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_y", KD, 335 compute_pgm_rsrc2, 336 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y); 337 PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_z", KD, 338 compute_pgm_rsrc2, 339 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z); 340 PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_info", KD, 341 compute_pgm_rsrc2, 342 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO); 343 PRINT_FIELD(OS, ".amdhsa_system_vgpr_workitem_id", KD, 344 compute_pgm_rsrc2, 345 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID); 346 347 // These directives are required. 348 OS << "\t\t.amdhsa_next_free_vgpr " << NextVGPR << '\n'; 349 OS << "\t\t.amdhsa_next_free_sgpr " << NextSGPR << '\n'; 350 351 if (AMDGPU::isGFX90A(STI)) 352 OS << "\t\t.amdhsa_accum_offset " << 353 (AMDHSA_BITS_GET(KD.compute_pgm_rsrc3, 354 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET) + 1) * 4 355 << '\n'; 356 357 if (!ReserveVCC) 358 OS << "\t\t.amdhsa_reserve_vcc " << ReserveVCC << '\n'; 359 if (IVersion.Major >= 7 && !ReserveFlatScr) 360 OS << "\t\t.amdhsa_reserve_flat_scratch " << ReserveFlatScr << '\n'; 361 if (IVersion.Major >= 8 && ReserveXNACK != hasXNACK(STI)) 362 OS << "\t\t.amdhsa_reserve_xnack_mask " << ReserveXNACK << '\n'; 363 364 PRINT_FIELD(OS, ".amdhsa_float_round_mode_32", KD, 365 compute_pgm_rsrc1, 366 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32); 367 PRINT_FIELD(OS, ".amdhsa_float_round_mode_16_64", KD, 368 compute_pgm_rsrc1, 369 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64); 370 PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_32", KD, 371 compute_pgm_rsrc1, 372 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32); 373 PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_16_64", KD, 374 compute_pgm_rsrc1, 375 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64); 376 PRINT_FIELD(OS, ".amdhsa_dx10_clamp", KD, 377 compute_pgm_rsrc1, 378 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP); 379 PRINT_FIELD(OS, ".amdhsa_ieee_mode", KD, 380 compute_pgm_rsrc1, 381 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE); 382 if (IVersion.Major >= 9) 383 PRINT_FIELD(OS, ".amdhsa_fp16_overflow", KD, 384 compute_pgm_rsrc1, 385 amdhsa::COMPUTE_PGM_RSRC1_FP16_OVFL); 386 if (AMDGPU::isGFX90A(STI)) 387 PRINT_FIELD(OS, ".amdhsa_tg_split", KD, 388 compute_pgm_rsrc3, 389 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT); 390 if (IVersion.Major >= 10) { 391 PRINT_FIELD(OS, ".amdhsa_workgroup_processor_mode", KD, 392 compute_pgm_rsrc1, 393 amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE); 394 PRINT_FIELD(OS, ".amdhsa_memory_ordered", KD, 395 compute_pgm_rsrc1, 396 amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED); 397 PRINT_FIELD(OS, ".amdhsa_forward_progress", KD, 398 compute_pgm_rsrc1, 399 amdhsa::COMPUTE_PGM_RSRC1_FWD_PROGRESS); 400 } 401 PRINT_FIELD( 402 OS, ".amdhsa_exception_fp_ieee_invalid_op", KD, 403 compute_pgm_rsrc2, 404 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION); 405 PRINT_FIELD(OS, ".amdhsa_exception_fp_denorm_src", KD, 406 compute_pgm_rsrc2, 407 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE); 408 PRINT_FIELD( 409 OS, ".amdhsa_exception_fp_ieee_div_zero", KD, 410 compute_pgm_rsrc2, 411 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO); 412 PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_overflow", KD, 413 compute_pgm_rsrc2, 414 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW); 415 PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_underflow", KD, 416 compute_pgm_rsrc2, 417 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW); 418 PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_inexact", KD, 419 compute_pgm_rsrc2, 420 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT); 421 PRINT_FIELD(OS, ".amdhsa_exception_int_div_zero", KD, 422 compute_pgm_rsrc2, 423 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO); 424 #undef PRINT_FIELD 425 426 OS << "\t.end_amdhsa_kernel\n"; 427 } 428 429 //===----------------------------------------------------------------------===// 430 // AMDGPUTargetELFStreamer 431 //===----------------------------------------------------------------------===// 432 433 AMDGPUTargetELFStreamer::AMDGPUTargetELFStreamer(MCStreamer &S, 434 const MCSubtargetInfo &STI) 435 : AMDGPUTargetStreamer(S), Streamer(S), Os(STI.getTargetTriple().getOS()) { 436 MCAssembler &MCA = getStreamer().getAssembler(); 437 unsigned EFlags = MCA.getELFHeaderEFlags(); 438 439 EFlags &= ~ELF::EF_AMDGPU_MACH; 440 EFlags |= getElfMach(STI.getCPU()); 441 442 EFlags &= ~ELF::EF_AMDGPU_XNACK; 443 if (AMDGPU::hasXNACK(STI)) 444 EFlags |= ELF::EF_AMDGPU_XNACK; 445 446 EFlags &= ~ELF::EF_AMDGPU_SRAM_ECC; 447 if (AMDGPU::hasSRAMECC(STI)) 448 EFlags |= ELF::EF_AMDGPU_SRAM_ECC; 449 450 MCA.setELFHeaderEFlags(EFlags); 451 } 452 453 MCELFStreamer &AMDGPUTargetELFStreamer::getStreamer() { 454 return static_cast<MCELFStreamer &>(Streamer); 455 } 456 457 // A hook for emitting stuff at the end. 458 // We use it for emitting the accumulated PAL metadata as a .note record. 459 // The PAL metadata is reset after it is emitted. 460 void AMDGPUTargetELFStreamer::finish() { 461 std::string Blob; 462 const char *Vendor = getPALMetadata()->getVendor(); 463 unsigned Type = getPALMetadata()->getType(); 464 getPALMetadata()->toBlob(Type, Blob); 465 if (Blob.empty()) 466 return; 467 EmitNote(Vendor, MCConstantExpr::create(Blob.size(), getContext()), Type, 468 [&](MCELFStreamer &OS) { OS.emitBytes(Blob); }); 469 470 // Reset the pal metadata so its data will not affect a compilation that 471 // reuses this object. 472 getPALMetadata()->reset(); 473 } 474 475 void AMDGPUTargetELFStreamer::EmitNote( 476 StringRef Name, const MCExpr *DescSZ, unsigned NoteType, 477 function_ref<void(MCELFStreamer &)> EmitDesc) { 478 auto &S = getStreamer(); 479 auto &Context = S.getContext(); 480 481 auto NameSZ = Name.size() + 1; 482 483 unsigned NoteFlags = 0; 484 // TODO Apparently, this is currently needed for OpenCL as mentioned in 485 // https://reviews.llvm.org/D74995 486 if (Os == Triple::AMDHSA) 487 NoteFlags = ELF::SHF_ALLOC; 488 489 S.PushSection(); 490 S.SwitchSection( 491 Context.getELFSection(ElfNote::SectionName, ELF::SHT_NOTE, NoteFlags)); 492 S.emitInt32(NameSZ); // namesz 493 S.emitValue(DescSZ, 4); // descz 494 S.emitInt32(NoteType); // type 495 S.emitBytes(Name); // name 496 S.emitValueToAlignment(4, 0, 1, 0); // padding 0 497 EmitDesc(S); // desc 498 S.emitValueToAlignment(4, 0, 1, 0); // padding 0 499 S.PopSection(); 500 } 501 502 void AMDGPUTargetELFStreamer::EmitDirectiveAMDGCNTarget(StringRef Target) {} 503 504 void AMDGPUTargetELFStreamer::EmitDirectiveHSACodeObjectVersion( 505 uint32_t Major, uint32_t Minor) { 506 507 EmitNote(ElfNote::NoteNameV2, MCConstantExpr::create(8, getContext()), 508 ElfNote::NT_AMDGPU_HSA_CODE_OBJECT_VERSION, [&](MCELFStreamer &OS) { 509 OS.emitInt32(Major); 510 OS.emitInt32(Minor); 511 }); 512 } 513 514 void 515 AMDGPUTargetELFStreamer::EmitDirectiveHSACodeObjectISA(uint32_t Major, 516 uint32_t Minor, 517 uint32_t Stepping, 518 StringRef VendorName, 519 StringRef ArchName) { 520 uint16_t VendorNameSize = VendorName.size() + 1; 521 uint16_t ArchNameSize = ArchName.size() + 1; 522 523 unsigned DescSZ = sizeof(VendorNameSize) + sizeof(ArchNameSize) + 524 sizeof(Major) + sizeof(Minor) + sizeof(Stepping) + 525 VendorNameSize + ArchNameSize; 526 527 EmitNote(ElfNote::NoteNameV2, MCConstantExpr::create(DescSZ, getContext()), 528 ElfNote::NT_AMDGPU_HSA_ISA, [&](MCELFStreamer &OS) { 529 OS.emitInt16(VendorNameSize); 530 OS.emitInt16(ArchNameSize); 531 OS.emitInt32(Major); 532 OS.emitInt32(Minor); 533 OS.emitInt32(Stepping); 534 OS.emitBytes(VendorName); 535 OS.emitInt8(0); // NULL terminate VendorName 536 OS.emitBytes(ArchName); 537 OS.emitInt8(0); // NULL terminte ArchName 538 }); 539 } 540 541 void 542 AMDGPUTargetELFStreamer::EmitAMDKernelCodeT(const amd_kernel_code_t &Header) { 543 544 MCStreamer &OS = getStreamer(); 545 OS.PushSection(); 546 OS.emitBytes(StringRef((const char*)&Header, sizeof(Header))); 547 OS.PopSection(); 548 } 549 550 void AMDGPUTargetELFStreamer::EmitAMDGPUSymbolType(StringRef SymbolName, 551 unsigned Type) { 552 MCSymbolELF *Symbol = cast<MCSymbolELF>( 553 getStreamer().getContext().getOrCreateSymbol(SymbolName)); 554 Symbol->setType(Type); 555 } 556 557 void AMDGPUTargetELFStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size, 558 Align Alignment) { 559 MCSymbolELF *SymbolELF = cast<MCSymbolELF>(Symbol); 560 SymbolELF->setType(ELF::STT_OBJECT); 561 562 if (!SymbolELF->isBindingSet()) { 563 SymbolELF->setBinding(ELF::STB_GLOBAL); 564 SymbolELF->setExternal(true); 565 } 566 567 if (SymbolELF->declareCommon(Size, Alignment.value(), true)) { 568 report_fatal_error("Symbol: " + Symbol->getName() + 569 " redeclared as different type"); 570 } 571 572 SymbolELF->setIndex(ELF::SHN_AMDGPU_LDS); 573 SymbolELF->setSize(MCConstantExpr::create(Size, getContext())); 574 } 575 576 bool AMDGPUTargetELFStreamer::EmitISAVersion(StringRef IsaVersionString) { 577 // Create two labels to mark the beginning and end of the desc field 578 // and a MCExpr to calculate the size of the desc field. 579 auto &Context = getContext(); 580 auto *DescBegin = Context.createTempSymbol(); 581 auto *DescEnd = Context.createTempSymbol(); 582 auto *DescSZ = MCBinaryExpr::createSub( 583 MCSymbolRefExpr::create(DescEnd, Context), 584 MCSymbolRefExpr::create(DescBegin, Context), Context); 585 586 EmitNote(ElfNote::NoteNameV2, DescSZ, ELF::NT_AMD_AMDGPU_ISA, 587 [&](MCELFStreamer &OS) { 588 OS.emitLabel(DescBegin); 589 OS.emitBytes(IsaVersionString); 590 OS.emitLabel(DescEnd); 591 }); 592 return true; 593 } 594 595 bool AMDGPUTargetELFStreamer::EmitHSAMetadata(msgpack::Document &HSAMetadataDoc, 596 bool Strict) { 597 HSAMD::V3::MetadataVerifier Verifier(Strict); 598 if (!Verifier.verify(HSAMetadataDoc.getRoot())) 599 return false; 600 601 std::string HSAMetadataString; 602 HSAMetadataDoc.writeToBlob(HSAMetadataString); 603 604 // Create two labels to mark the beginning and end of the desc field 605 // and a MCExpr to calculate the size of the desc field. 606 auto &Context = getContext(); 607 auto *DescBegin = Context.createTempSymbol(); 608 auto *DescEnd = Context.createTempSymbol(); 609 auto *DescSZ = MCBinaryExpr::createSub( 610 MCSymbolRefExpr::create(DescEnd, Context), 611 MCSymbolRefExpr::create(DescBegin, Context), Context); 612 613 EmitNote(ElfNote::NoteNameV3, DescSZ, ELF::NT_AMDGPU_METADATA, 614 [&](MCELFStreamer &OS) { 615 OS.emitLabel(DescBegin); 616 OS.emitBytes(HSAMetadataString); 617 OS.emitLabel(DescEnd); 618 }); 619 return true; 620 } 621 622 bool AMDGPUTargetELFStreamer::EmitHSAMetadata( 623 const AMDGPU::HSAMD::Metadata &HSAMetadata) { 624 std::string HSAMetadataString; 625 if (HSAMD::toString(HSAMetadata, HSAMetadataString)) 626 return false; 627 628 // Create two labels to mark the beginning and end of the desc field 629 // and a MCExpr to calculate the size of the desc field. 630 auto &Context = getContext(); 631 auto *DescBegin = Context.createTempSymbol(); 632 auto *DescEnd = Context.createTempSymbol(); 633 auto *DescSZ = MCBinaryExpr::createSub( 634 MCSymbolRefExpr::create(DescEnd, Context), 635 MCSymbolRefExpr::create(DescBegin, Context), Context); 636 637 EmitNote(ElfNote::NoteNameV2, DescSZ, ELF::NT_AMD_AMDGPU_HSA_METADATA, 638 [&](MCELFStreamer &OS) { 639 OS.emitLabel(DescBegin); 640 OS.emitBytes(HSAMetadataString); 641 OS.emitLabel(DescEnd); 642 }); 643 return true; 644 } 645 646 bool AMDGPUTargetELFStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { 647 const uint32_t Encoded_s_code_end = 0xbf9f0000; 648 const uint32_t Encoded_s_nop = 0xbf800000; 649 uint32_t Encoded_pad = Encoded_s_code_end; 650 651 // Instruction cache line size in bytes. 652 const unsigned Log2CacheLineSize = 6; 653 const unsigned CacheLineSize = 1u << Log2CacheLineSize; 654 655 // Extra padding amount in bytes to support prefetch mode 3. 656 unsigned FillSize = 3 * CacheLineSize; 657 658 if (AMDGPU::isGFX90A(STI)) { 659 Encoded_pad = Encoded_s_nop; 660 FillSize = 16 * CacheLineSize; 661 } 662 663 MCStreamer &OS = getStreamer(); 664 OS.PushSection(); 665 OS.emitValueToAlignment(CacheLineSize, Encoded_pad, 4); 666 for (unsigned I = 0; I < FillSize; I += 4) 667 OS.emitInt32(Encoded_pad); 668 OS.PopSection(); 669 return true; 670 } 671 672 void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( 673 const MCSubtargetInfo &STI, StringRef KernelName, 674 const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, 675 uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr, 676 bool ReserveXNACK) { 677 auto &Streamer = getStreamer(); 678 auto &Context = Streamer.getContext(); 679 680 MCSymbolELF *KernelCodeSymbol = cast<MCSymbolELF>( 681 Context.getOrCreateSymbol(Twine(KernelName))); 682 MCSymbolELF *KernelDescriptorSymbol = cast<MCSymbolELF>( 683 Context.getOrCreateSymbol(Twine(KernelName) + Twine(".kd"))); 684 685 // Copy kernel descriptor symbol's binding, other and visibility from the 686 // kernel code symbol. 687 KernelDescriptorSymbol->setBinding(KernelCodeSymbol->getBinding()); 688 KernelDescriptorSymbol->setOther(KernelCodeSymbol->getOther()); 689 KernelDescriptorSymbol->setVisibility(KernelCodeSymbol->getVisibility()); 690 // Kernel descriptor symbol's type and size are fixed. 691 KernelDescriptorSymbol->setType(ELF::STT_OBJECT); 692 KernelDescriptorSymbol->setSize( 693 MCConstantExpr::create(sizeof(KernelDescriptor), Context)); 694 695 // The visibility of the kernel code symbol must be protected or less to allow 696 // static relocations from the kernel descriptor to be used. 697 if (KernelCodeSymbol->getVisibility() == ELF::STV_DEFAULT) 698 KernelCodeSymbol->setVisibility(ELF::STV_PROTECTED); 699 700 Streamer.emitLabel(KernelDescriptorSymbol); 701 Streamer.emitInt32(KernelDescriptor.group_segment_fixed_size); 702 Streamer.emitInt32(KernelDescriptor.private_segment_fixed_size); 703 for (uint8_t Res : KernelDescriptor.reserved0) 704 Streamer.emitInt8(Res); 705 // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The 706 // expression being created is: 707 // (start of kernel code) - (start of kernel descriptor) 708 // It implies R_AMDGPU_REL64, but ends up being R_AMDGPU_ABS64. 709 Streamer.emitValue(MCBinaryExpr::createSub( 710 MCSymbolRefExpr::create( 711 KernelCodeSymbol, MCSymbolRefExpr::VK_AMDGPU_REL64, Context), 712 MCSymbolRefExpr::create( 713 KernelDescriptorSymbol, MCSymbolRefExpr::VK_None, Context), 714 Context), 715 sizeof(KernelDescriptor.kernel_code_entry_byte_offset)); 716 for (uint8_t Res : KernelDescriptor.reserved1) 717 Streamer.emitInt8(Res); 718 Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc3); 719 Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc1); 720 Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc2); 721 Streamer.emitInt16(KernelDescriptor.kernel_code_properties); 722 for (uint8_t Res : KernelDescriptor.reserved2) 723 Streamer.emitInt8(Res); 724 } 725