1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Functionality for handling RTL plugins. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "rtl.h" 14 #include "device.h" 15 #include "private.h" 16 17 #include <cassert> 18 #include <cstdlib> 19 #include <cstring> 20 #include <dlfcn.h> 21 #include <mutex> 22 #include <string> 23 24 // List of all plugins that can support offloading. 25 static const char *RTLNames[] = { 26 /* PowerPC target */ "libomptarget.rtl.ppc64.so", 27 /* x86_64 target */ "libomptarget.rtl.x86_64.so", 28 /* CUDA target */ "libomptarget.rtl.cuda.so", 29 /* AArch64 target */ "libomptarget.rtl.aarch64.so", 30 /* SX-Aurora VE target */ "libomptarget.rtl.ve.so", 31 /* AMDGPU target */ "libomptarget.rtl.amdgpu.so", 32 /* Remote target */ "libomptarget.rtl.rpc.so", 33 }; 34 35 PluginManager *PM; 36 37 #if OMPTARGET_PROFILE_ENABLED 38 static char *ProfileTraceFile = nullptr; 39 #endif 40 41 __attribute__((constructor(101))) void init() { 42 DP("Init target library!\n"); 43 44 bool UseEventsForAtomicTransfers = true; 45 if (const char *ForceAtomicMap = getenv("LIBOMPTARGET_MAP_FORCE_ATOMIC")) { 46 std::string ForceAtomicMapStr(ForceAtomicMap); 47 if (ForceAtomicMapStr == "false" || ForceAtomicMapStr == "FALSE") 48 UseEventsForAtomicTransfers = false; 49 else if (ForceAtomicMapStr != "true" && ForceAtomicMapStr != "TRUE") 50 fprintf(stderr, 51 "Warning: 'LIBOMPTARGET_MAP_FORCE_ATOMIC' accepts only " 52 "'true'/'TRUE' or 'false'/'FALSE' as options, '%s' ignored\n", 53 ForceAtomicMap); 54 } 55 56 PM = new PluginManager(UseEventsForAtomicTransfers); 57 58 #ifdef OMPTARGET_PROFILE_ENABLED 59 ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE"); 60 // TODO: add a configuration option for time granularity 61 if (ProfileTraceFile) 62 llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget"); 63 #endif 64 } 65 66 __attribute__((destructor(101))) void deinit() { 67 DP("Deinit target library!\n"); 68 delete PM; 69 70 #ifdef OMPTARGET_PROFILE_ENABLED 71 if (ProfileTraceFile) { 72 // TODO: add env var for file output 73 if (auto E = llvm::timeTraceProfilerWrite(ProfileTraceFile, "-")) 74 fprintf(stderr, "Error writing out the time trace\n"); 75 76 llvm::timeTraceProfilerCleanup(); 77 } 78 #endif 79 } 80 81 void RTLsTy::loadRTLs() { 82 // Parse environment variable OMP_TARGET_OFFLOAD (if set) 83 PM->TargetOffloadPolicy = 84 (kmp_target_offload_kind_t)__kmpc_get_target_offload(); 85 if (PM->TargetOffloadPolicy == tgt_disabled) { 86 return; 87 } 88 89 DP("Loading RTLs...\n"); 90 91 // Attempt to open all the plugins and, if they exist, check if the interface 92 // is correct and if they are supporting any devices. 93 for (auto *Name : RTLNames) { 94 DP("Loading library '%s'...\n", Name); 95 void *DynlibHandle = dlopen(Name, RTLD_NOW); 96 97 if (!DynlibHandle) { 98 // Library does not exist or cannot be found. 99 DP("Unable to load library '%s': %s!\n", Name, dlerror()); 100 continue; 101 } 102 103 DP("Successfully loaded library '%s'!\n", Name); 104 105 AllRTLs.emplace_back(); 106 107 // Retrieve the RTL information from the runtime library. 108 RTLInfoTy &R = AllRTLs.back(); 109 110 bool ValidPlugin = true; 111 112 if (!(*((void **)&R.is_valid_binary) = 113 dlsym(DynlibHandle, "__tgt_rtl_is_valid_binary"))) 114 ValidPlugin = false; 115 if (!(*((void **)&R.number_of_devices) = 116 dlsym(DynlibHandle, "__tgt_rtl_number_of_devices"))) 117 ValidPlugin = false; 118 if (!(*((void **)&R.init_device) = 119 dlsym(DynlibHandle, "__tgt_rtl_init_device"))) 120 ValidPlugin = false; 121 if (!(*((void **)&R.load_binary) = 122 dlsym(DynlibHandle, "__tgt_rtl_load_binary"))) 123 ValidPlugin = false; 124 if (!(*((void **)&R.data_alloc) = 125 dlsym(DynlibHandle, "__tgt_rtl_data_alloc"))) 126 ValidPlugin = false; 127 if (!(*((void **)&R.data_submit) = 128 dlsym(DynlibHandle, "__tgt_rtl_data_submit"))) 129 ValidPlugin = false; 130 if (!(*((void **)&R.data_retrieve) = 131 dlsym(DynlibHandle, "__tgt_rtl_data_retrieve"))) 132 ValidPlugin = false; 133 if (!(*((void **)&R.data_delete) = 134 dlsym(DynlibHandle, "__tgt_rtl_data_delete"))) 135 ValidPlugin = false; 136 if (!(*((void **)&R.run_region) = 137 dlsym(DynlibHandle, "__tgt_rtl_run_target_region"))) 138 ValidPlugin = false; 139 if (!(*((void **)&R.run_team_region) = 140 dlsym(DynlibHandle, "__tgt_rtl_run_target_team_region"))) 141 ValidPlugin = false; 142 143 // Invalid plugin 144 if (!ValidPlugin) { 145 DP("Invalid plugin as necessary interface is not found.\n"); 146 AllRTLs.pop_back(); 147 continue; 148 } 149 150 // No devices are supported by this RTL? 151 if (!(R.NumberOfDevices = R.number_of_devices())) { 152 // The RTL is invalid! Will pop the object from the RTLs list. 153 DP("No devices supported in this RTL\n"); 154 AllRTLs.pop_back(); 155 continue; 156 } 157 158 R.LibraryHandler = DynlibHandle; 159 160 #ifdef OMPTARGET_DEBUG 161 R.RTLName = Name; 162 #endif 163 164 DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(), 165 R.NumberOfDevices); 166 167 // Optional functions 168 *((void **)&R.deinit_device) = 169 dlsym(DynlibHandle, "__tgt_rtl_deinit_device"); 170 *((void **)&R.init_requires) = 171 dlsym(DynlibHandle, "__tgt_rtl_init_requires"); 172 *((void **)&R.data_submit_async) = 173 dlsym(DynlibHandle, "__tgt_rtl_data_submit_async"); 174 *((void **)&R.data_retrieve_async) = 175 dlsym(DynlibHandle, "__tgt_rtl_data_retrieve_async"); 176 *((void **)&R.run_region_async) = 177 dlsym(DynlibHandle, "__tgt_rtl_run_target_region_async"); 178 *((void **)&R.run_team_region_async) = 179 dlsym(DynlibHandle, "__tgt_rtl_run_target_team_region_async"); 180 *((void **)&R.synchronize) = dlsym(DynlibHandle, "__tgt_rtl_synchronize"); 181 *((void **)&R.data_exchange) = 182 dlsym(DynlibHandle, "__tgt_rtl_data_exchange"); 183 *((void **)&R.data_exchange_async) = 184 dlsym(DynlibHandle, "__tgt_rtl_data_exchange_async"); 185 *((void **)&R.is_data_exchangable) = 186 dlsym(DynlibHandle, "__tgt_rtl_is_data_exchangable"); 187 *((void **)&R.register_lib) = dlsym(DynlibHandle, "__tgt_rtl_register_lib"); 188 *((void **)&R.unregister_lib) = 189 dlsym(DynlibHandle, "__tgt_rtl_unregister_lib"); 190 *((void **)&R.supports_empty_images) = 191 dlsym(DynlibHandle, "__tgt_rtl_supports_empty_images"); 192 *((void **)&R.set_info_flag) = 193 dlsym(DynlibHandle, "__tgt_rtl_set_info_flag"); 194 *((void **)&R.print_device_info) = 195 dlsym(DynlibHandle, "__tgt_rtl_print_device_info"); 196 *((void **)&R.create_event) = dlsym(DynlibHandle, "__tgt_rtl_create_event"); 197 *((void **)&R.record_event) = dlsym(DynlibHandle, "__tgt_rtl_record_event"); 198 *((void **)&R.wait_event) = dlsym(DynlibHandle, "__tgt_rtl_wait_event"); 199 *((void **)&R.sync_event) = dlsym(DynlibHandle, "__tgt_rtl_sync_event"); 200 *((void **)&R.destroy_event) = 201 dlsym(DynlibHandle, "__tgt_rtl_destroy_event"); 202 *((void **)&R.release_async_info) = 203 dlsym(DynlibHandle, "__tgt_rtl_release_async_info"); 204 *((void **)&R.init_async_info) = 205 dlsym(DynlibHandle, "__tgt_rtl_init_async_info"); 206 *((void **)&R.init_device_info) = 207 dlsym(DynlibHandle, "__tgt_rtl_init_device_info"); 208 } 209 210 DP("RTLs loaded!\n"); 211 212 return; 213 } 214 215 //////////////////////////////////////////////////////////////////////////////// 216 // Functionality for registering libs 217 218 static void registerImageIntoTranslationTable(TranslationTable &TT, 219 RTLInfoTy &RTL, 220 __tgt_device_image *Image) { 221 222 // same size, as when we increase one, we also increase the other. 223 assert(TT.TargetsTable.size() == TT.TargetsImages.size() && 224 "We should have as many images as we have tables!"); 225 226 // Resize the Targets Table and Images to accommodate the new targets if 227 // required 228 unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices; 229 230 if (TT.TargetsTable.size() < TargetsTableMinimumSize) { 231 TT.TargetsImages.resize(TargetsTableMinimumSize, 0); 232 TT.TargetsTable.resize(TargetsTableMinimumSize, 0); 233 } 234 235 // Register the image in all devices for this target type. 236 for (int32_t I = 0; I < RTL.NumberOfDevices; ++I) { 237 // If we are changing the image we are also invalidating the target table. 238 if (TT.TargetsImages[RTL.Idx + I] != Image) { 239 TT.TargetsImages[RTL.Idx + I] = Image; 240 TT.TargetsTable[RTL.Idx + I] = 0; // lazy initialization of target table. 241 } 242 } 243 } 244 245 //////////////////////////////////////////////////////////////////////////////// 246 // Functionality for registering Ctors/Dtors 247 248 static void registerGlobalCtorsDtorsForImage(__tgt_bin_desc *Desc, 249 __tgt_device_image *Img, 250 RTLInfoTy *RTL) { 251 252 for (int32_t I = 0; I < RTL->NumberOfDevices; ++I) { 253 DeviceTy &Device = *PM->Devices[RTL->Idx + I]; 254 Device.PendingGlobalsMtx.lock(); 255 Device.HasPendingGlobals = true; 256 for (__tgt_offload_entry *Entry = Img->EntriesBegin; 257 Entry != Img->EntriesEnd; ++Entry) { 258 if (Entry->flags & OMP_DECLARE_TARGET_CTOR) { 259 DP("Adding ctor " DPxMOD " to the pending list.\n", 260 DPxPTR(Entry->addr)); 261 Device.PendingCtorsDtors[Desc].PendingCtors.push_back(Entry->addr); 262 } else if (Entry->flags & OMP_DECLARE_TARGET_DTOR) { 263 // Dtors are pushed in reverse order so they are executed from end 264 // to beginning when unregistering the library! 265 DP("Adding dtor " DPxMOD " to the pending list.\n", 266 DPxPTR(Entry->addr)); 267 Device.PendingCtorsDtors[Desc].PendingDtors.push_front(Entry->addr); 268 } 269 270 if (Entry->flags & OMP_DECLARE_TARGET_LINK) { 271 DP("The \"link\" attribute is not yet supported!\n"); 272 } 273 } 274 Device.PendingGlobalsMtx.unlock(); 275 } 276 } 277 278 void RTLsTy::registerRequires(int64_t Flags) { 279 // TODO: add more elaborate check. 280 // Minimal check: only set requires flags if previous value 281 // is undefined. This ensures that only the first call to this 282 // function will set the requires flags. All subsequent calls 283 // will be checked for compatibility. 284 assert(Flags != OMP_REQ_UNDEFINED && 285 "illegal undefined flag for requires directive!"); 286 if (RequiresFlags == OMP_REQ_UNDEFINED) { 287 RequiresFlags = Flags; 288 return; 289 } 290 291 // If multiple compilation units are present enforce 292 // consistency across all of them for require clauses: 293 // - reverse_offload 294 // - unified_address 295 // - unified_shared_memory 296 if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) != 297 (Flags & OMP_REQ_REVERSE_OFFLOAD)) { 298 FATAL_MESSAGE0( 299 1, "'#pragma omp requires reverse_offload' not used consistently!"); 300 } 301 if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) != 302 (Flags & OMP_REQ_UNIFIED_ADDRESS)) { 303 FATAL_MESSAGE0( 304 1, "'#pragma omp requires unified_address' not used consistently!"); 305 } 306 if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) != 307 (Flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) { 308 FATAL_MESSAGE0( 309 1, 310 "'#pragma omp requires unified_shared_memory' not used consistently!"); 311 } 312 313 // TODO: insert any other missing checks 314 315 DP("New requires flags %" PRId64 " compatible with existing %" PRId64 "!\n", 316 Flags, RequiresFlags); 317 } 318 319 void RTLsTy::initRTLonce(RTLInfoTy &R) { 320 // If this RTL is not already in use, initialize it. 321 if (!R.IsUsed && R.NumberOfDevices != 0) { 322 // Initialize the device information for the RTL we are about to use. 323 const size_t Start = PM->Devices.size(); 324 PM->Devices.reserve(Start + R.NumberOfDevices); 325 for (int32_t DeviceId = 0; DeviceId < R.NumberOfDevices; DeviceId++) { 326 PM->Devices.push_back(std::make_unique<DeviceTy>(&R)); 327 // global device ID 328 PM->Devices[Start + DeviceId]->DeviceID = Start + DeviceId; 329 // RTL local device ID 330 PM->Devices[Start + DeviceId]->RTLDeviceID = DeviceId; 331 } 332 333 // Initialize the index of this RTL and save it in the used RTLs. 334 R.Idx = (UsedRTLs.empty()) 335 ? 0 336 : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices; 337 assert((size_t)R.Idx == Start && 338 "RTL index should equal the number of devices used so far."); 339 R.IsUsed = true; 340 UsedRTLs.push_back(&R); 341 342 DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx); 343 } 344 } 345 346 void RTLsTy::initAllRTLs() { 347 for (auto &R : AllRTLs) 348 initRTLonce(R); 349 } 350 351 void RTLsTy::registerLib(__tgt_bin_desc *Desc) { 352 PM->RTLsMtx.lock(); 353 // Register the images with the RTLs that understand them, if any. 354 for (int32_t I = 0; I < Desc->NumDeviceImages; ++I) { 355 // Obtain the image. 356 __tgt_device_image *Img = &Desc->DeviceImages[I]; 357 358 RTLInfoTy *FoundRTL = nullptr; 359 360 // Scan the RTLs that have associated images until we find one that supports 361 // the current image. 362 for (auto &R : AllRTLs) { 363 if (!R.is_valid_binary(Img)) { 364 DP("Image " DPxMOD " is NOT compatible with RTL %s!\n", 365 DPxPTR(Img->ImageStart), R.RTLName.c_str()); 366 continue; 367 } 368 369 DP("Image " DPxMOD " is compatible with RTL %s!\n", 370 DPxPTR(Img->ImageStart), R.RTLName.c_str()); 371 372 initRTLonce(R); 373 374 // Initialize (if necessary) translation table for this library. 375 PM->TrlTblMtx.lock(); 376 if (!PM->HostEntriesBeginToTransTable.count(Desc->HostEntriesBegin)) { 377 PM->HostEntriesBeginRegistrationOrder.push_back(Desc->HostEntriesBegin); 378 TranslationTable &TransTable = 379 (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin]; 380 TransTable.HostTable.EntriesBegin = Desc->HostEntriesBegin; 381 TransTable.HostTable.EntriesEnd = Desc->HostEntriesEnd; 382 } 383 384 // Retrieve translation table for this library. 385 TranslationTable &TransTable = 386 (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin]; 387 388 DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(Img->ImageStart), 389 R.RTLName.c_str()); 390 registerImageIntoTranslationTable(TransTable, R, Img); 391 PM->TrlTblMtx.unlock(); 392 FoundRTL = &R; 393 394 // Load ctors/dtors for static objects 395 registerGlobalCtorsDtorsForImage(Desc, Img, FoundRTL); 396 397 // if an RTL was found we are done - proceed to register the next image 398 break; 399 } 400 401 if (!FoundRTL) { 402 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(Img->ImageStart)); 403 } 404 } 405 PM->RTLsMtx.unlock(); 406 407 DP("Done registering entries!\n"); 408 } 409 410 void RTLsTy::unregisterLib(__tgt_bin_desc *Desc) { 411 DP("Unloading target library!\n"); 412 413 PM->RTLsMtx.lock(); 414 // Find which RTL understands each image, if any. 415 for (int32_t I = 0; I < Desc->NumDeviceImages; ++I) { 416 // Obtain the image. 417 __tgt_device_image *Img = &Desc->DeviceImages[I]; 418 419 RTLInfoTy *FoundRTL = NULL; 420 421 // Scan the RTLs that have associated images until we find one that supports 422 // the current image. We only need to scan RTLs that are already being used. 423 for (auto *R : UsedRTLs) { 424 425 assert(R->IsUsed && "Expecting used RTLs."); 426 427 if (!R->is_valid_binary(Img)) { 428 DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n", 429 DPxPTR(Img->ImageStart), DPxPTR(R->LibraryHandler)); 430 continue; 431 } 432 433 DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n", 434 DPxPTR(Img->ImageStart), DPxPTR(R->LibraryHandler)); 435 436 FoundRTL = R; 437 438 // Execute dtors for static objects if the device has been used, i.e. 439 // if its PendingCtors list has been emptied. 440 for (int32_t I = 0; I < FoundRTL->NumberOfDevices; ++I) { 441 DeviceTy &Device = *PM->Devices[FoundRTL->Idx + I]; 442 Device.PendingGlobalsMtx.lock(); 443 if (Device.PendingCtorsDtors[Desc].PendingCtors.empty()) { 444 AsyncInfoTy AsyncInfo(Device); 445 for (auto &Dtor : Device.PendingCtorsDtors[Desc].PendingDtors) { 446 int Rc = target(nullptr, Device, Dtor, 0, nullptr, nullptr, nullptr, 447 nullptr, nullptr, nullptr, 1, 1, 0, true /*team*/, 448 AsyncInfo); 449 if (Rc != OFFLOAD_SUCCESS) { 450 DP("Running destructor " DPxMOD " failed.\n", DPxPTR(Dtor)); 451 } 452 } 453 // Remove this library's entry from PendingCtorsDtors 454 Device.PendingCtorsDtors.erase(Desc); 455 // All constructors have been issued, wait for them now. 456 if (AsyncInfo.synchronize() != OFFLOAD_SUCCESS) 457 DP("Failed synchronizing destructors kernels.\n"); 458 } 459 Device.PendingGlobalsMtx.unlock(); 460 } 461 462 DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n", 463 DPxPTR(Img->ImageStart), DPxPTR(R->LibraryHandler)); 464 465 break; 466 } 467 468 // if no RTL was found proceed to unregister the next image 469 if (!FoundRTL) { 470 DP("No RTLs in use support the image " DPxMOD "!\n", 471 DPxPTR(Img->ImageStart)); 472 } 473 } 474 PM->RTLsMtx.unlock(); 475 DP("Done unregistering images!\n"); 476 477 // Remove entries from PM->HostPtrToTableMap 478 PM->TblMapMtx.lock(); 479 for (__tgt_offload_entry *Cur = Desc->HostEntriesBegin; 480 Cur < Desc->HostEntriesEnd; ++Cur) { 481 PM->HostPtrToTableMap.erase(Cur->addr); 482 } 483 484 // Remove translation table for this descriptor. 485 auto TransTable = 486 PM->HostEntriesBeginToTransTable.find(Desc->HostEntriesBegin); 487 if (TransTable != PM->HostEntriesBeginToTransTable.end()) { 488 DP("Removing translation table for descriptor " DPxMOD "\n", 489 DPxPTR(Desc->HostEntriesBegin)); 490 PM->HostEntriesBeginToTransTable.erase(TransTable); 491 } else { 492 DP("Translation table for descriptor " DPxMOD " cannot be found, probably " 493 "it has been already removed.\n", 494 DPxPTR(Desc->HostEntriesBegin)); 495 } 496 497 PM->TblMapMtx.unlock(); 498 499 // TODO: Remove RTL and the devices it manages if it's not used anymore? 500 // TODO: Write some RTL->unload_image(...) function? 501 502 DP("Done unregistering library!\n"); 503 } 504