1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Functionality for handling RTL plugins. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "rtl.h" 14 #include "device.h" 15 #include "private.h" 16 17 #include <cassert> 18 #include <cstdlib> 19 #include <cstring> 20 #include <dlfcn.h> 21 #include <mutex> 22 #include <string> 23 24 // List of all plugins that can support offloading. 25 static const char *RTLNames[] = { 26 /* PowerPC target */ "libomptarget.rtl.ppc64.so", 27 /* x86_64 target */ "libomptarget.rtl.x86_64.so", 28 /* CUDA target */ "libomptarget.rtl.cuda.so", 29 /* AArch64 target */ "libomptarget.rtl.aarch64.so", 30 /* SX-Aurora VE target */ "libomptarget.rtl.ve.so", 31 /* AMDGPU target */ "libomptarget.rtl.amdgpu.so", 32 /* Remote target */ "libomptarget.rtl.rpc.so", 33 }; 34 35 PluginManager *PM; 36 37 #if OMPTARGET_PROFILE_ENABLED 38 static char *ProfileTraceFile = nullptr; 39 #endif 40 41 __attribute__((constructor(101))) void init() { 42 DP("Init target library!\n"); 43 PM = new PluginManager(); 44 45 #ifdef OMPTARGET_PROFILE_ENABLED 46 ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE"); 47 // TODO: add a configuration option for time granularity 48 if (ProfileTraceFile) 49 llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget"); 50 #endif 51 } 52 53 __attribute__((destructor(101))) void deinit() { 54 DP("Deinit target library!\n"); 55 delete PM; 56 57 #ifdef OMPTARGET_PROFILE_ENABLED 58 if (ProfileTraceFile) { 59 // TODO: add env var for file output 60 if (auto E = llvm::timeTraceProfilerWrite(ProfileTraceFile, "-")) 61 fprintf(stderr, "Error writing out the time trace\n"); 62 63 llvm::timeTraceProfilerCleanup(); 64 } 65 #endif 66 } 67 68 void RTLsTy::LoadRTLs() { 69 // Parse environment variable OMP_TARGET_OFFLOAD (if set) 70 PM->TargetOffloadPolicy = 71 (kmp_target_offload_kind_t)__kmpc_get_target_offload(); 72 if (PM->TargetOffloadPolicy == tgt_disabled) { 73 return; 74 } 75 76 DP("Loading RTLs...\n"); 77 78 // Attempt to open all the plugins and, if they exist, check if the interface 79 // is correct and if they are supporting any devices. 80 for (auto *Name : RTLNames) { 81 DP("Loading library '%s'...\n", Name); 82 void *dynlib_handle = dlopen(Name, RTLD_NOW); 83 84 if (!dynlib_handle) { 85 // Library does not exist or cannot be found. 86 DP("Unable to load library '%s': %s!\n", Name, dlerror()); 87 continue; 88 } 89 90 DP("Successfully loaded library '%s'!\n", Name); 91 92 AllRTLs.emplace_back(); 93 94 // Retrieve the RTL information from the runtime library. 95 RTLInfoTy &R = AllRTLs.back(); 96 97 bool ValidPlugin = true; 98 99 if (!(*((void **)&R.is_valid_binary) = 100 dlsym(dynlib_handle, "__tgt_rtl_is_valid_binary"))) 101 ValidPlugin = false; 102 if (!(*((void **)&R.number_of_devices) = 103 dlsym(dynlib_handle, "__tgt_rtl_number_of_devices"))) 104 ValidPlugin = false; 105 if (!(*((void **)&R.init_device) = 106 dlsym(dynlib_handle, "__tgt_rtl_init_device"))) 107 ValidPlugin = false; 108 if (!(*((void **)&R.load_binary) = 109 dlsym(dynlib_handle, "__tgt_rtl_load_binary"))) 110 ValidPlugin = false; 111 if (!(*((void **)&R.data_alloc) = 112 dlsym(dynlib_handle, "__tgt_rtl_data_alloc"))) 113 ValidPlugin = false; 114 if (!(*((void **)&R.data_submit) = 115 dlsym(dynlib_handle, "__tgt_rtl_data_submit"))) 116 ValidPlugin = false; 117 if (!(*((void **)&R.data_retrieve) = 118 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve"))) 119 ValidPlugin = false; 120 if (!(*((void **)&R.data_delete) = 121 dlsym(dynlib_handle, "__tgt_rtl_data_delete"))) 122 ValidPlugin = false; 123 if (!(*((void **)&R.run_region) = 124 dlsym(dynlib_handle, "__tgt_rtl_run_target_region"))) 125 ValidPlugin = false; 126 if (!(*((void **)&R.run_team_region) = 127 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region"))) 128 ValidPlugin = false; 129 130 // Invalid plugin 131 if (!ValidPlugin) { 132 DP("Invalid plugin as necessary interface is not found.\n"); 133 AllRTLs.pop_back(); 134 continue; 135 } 136 137 // No devices are supported by this RTL? 138 if (!(R.NumberOfDevices = R.number_of_devices())) { 139 // The RTL is invalid! Will pop the object from the RTLs list. 140 DP("No devices supported in this RTL\n"); 141 AllRTLs.pop_back(); 142 continue; 143 } 144 145 R.LibraryHandler = dynlib_handle; 146 147 #ifdef OMPTARGET_DEBUG 148 R.RTLName = Name; 149 #endif 150 151 DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(), 152 R.NumberOfDevices); 153 154 // Optional functions 155 *((void **)&R.init_requires) = 156 dlsym(dynlib_handle, "__tgt_rtl_init_requires"); 157 *((void **)&R.data_submit_async) = 158 dlsym(dynlib_handle, "__tgt_rtl_data_submit_async"); 159 *((void **)&R.data_retrieve_async) = 160 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async"); 161 *((void **)&R.run_region_async) = 162 dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async"); 163 *((void **)&R.run_team_region_async) = 164 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async"); 165 *((void **)&R.synchronize) = dlsym(dynlib_handle, "__tgt_rtl_synchronize"); 166 *((void **)&R.data_exchange) = 167 dlsym(dynlib_handle, "__tgt_rtl_data_exchange"); 168 *((void **)&R.data_exchange_async) = 169 dlsym(dynlib_handle, "__tgt_rtl_data_exchange_async"); 170 *((void **)&R.is_data_exchangable) = 171 dlsym(dynlib_handle, "__tgt_rtl_is_data_exchangable"); 172 *((void **)&R.register_lib) = 173 dlsym(dynlib_handle, "__tgt_rtl_register_lib"); 174 *((void **)&R.unregister_lib) = 175 dlsym(dynlib_handle, "__tgt_rtl_unregister_lib"); 176 *((void **)&R.supports_empty_images) = 177 dlsym(dynlib_handle, "__tgt_rtl_supports_empty_images"); 178 *((void **)&R.set_info_flag) = 179 dlsym(dynlib_handle, "__tgt_rtl_set_info_flag"); 180 *((void **)&R.print_device_info) = 181 dlsym(dynlib_handle, "__tgt_rtl_print_device_info"); 182 *((void **)&R.create_event) = 183 dlsym(dynlib_handle, "__tgt_rtl_create_event"); 184 *((void **)&R.record_event) = 185 dlsym(dynlib_handle, "__tgt_rtl_record_event"); 186 *((void **)&R.wait_event) = dlsym(dynlib_handle, "__tgt_rtl_wait_event"); 187 *((void **)&R.sync_event) = dlsym(dynlib_handle, "__tgt_rtl_sync_event"); 188 *((void **)&R.destroy_event) = 189 dlsym(dynlib_handle, "__tgt_rtl_destroy_event"); 190 } 191 192 DP("RTLs loaded!\n"); 193 194 return; 195 } 196 197 //////////////////////////////////////////////////////////////////////////////// 198 // Functionality for registering libs 199 200 static void RegisterImageIntoTranslationTable(TranslationTable &TT, 201 RTLInfoTy &RTL, 202 __tgt_device_image *image) { 203 204 // same size, as when we increase one, we also increase the other. 205 assert(TT.TargetsTable.size() == TT.TargetsImages.size() && 206 "We should have as many images as we have tables!"); 207 208 // Resize the Targets Table and Images to accommodate the new targets if 209 // required 210 unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices; 211 212 if (TT.TargetsTable.size() < TargetsTableMinimumSize) { 213 TT.TargetsImages.resize(TargetsTableMinimumSize, 0); 214 TT.TargetsTable.resize(TargetsTableMinimumSize, 0); 215 } 216 217 // Register the image in all devices for this target type. 218 for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) { 219 // If we are changing the image we are also invalidating the target table. 220 if (TT.TargetsImages[RTL.Idx + i] != image) { 221 TT.TargetsImages[RTL.Idx + i] = image; 222 TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table. 223 } 224 } 225 } 226 227 //////////////////////////////////////////////////////////////////////////////// 228 // Functionality for registering Ctors/Dtors 229 230 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc, 231 __tgt_device_image *img, 232 RTLInfoTy *RTL) { 233 234 for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) { 235 DeviceTy &Device = *PM->Devices[RTL->Idx + i]; 236 Device.PendingGlobalsMtx.lock(); 237 Device.HasPendingGlobals = true; 238 for (__tgt_offload_entry *entry = img->EntriesBegin; 239 entry != img->EntriesEnd; ++entry) { 240 if (entry->flags & OMP_DECLARE_TARGET_CTOR) { 241 DP("Adding ctor " DPxMOD " to the pending list.\n", 242 DPxPTR(entry->addr)); 243 Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr); 244 } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) { 245 // Dtors are pushed in reverse order so they are executed from end 246 // to beginning when unregistering the library! 247 DP("Adding dtor " DPxMOD " to the pending list.\n", 248 DPxPTR(entry->addr)); 249 Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr); 250 } 251 252 if (entry->flags & OMP_DECLARE_TARGET_LINK) { 253 DP("The \"link\" attribute is not yet supported!\n"); 254 } 255 } 256 Device.PendingGlobalsMtx.unlock(); 257 } 258 } 259 260 void RTLsTy::RegisterRequires(int64_t flags) { 261 // TODO: add more elaborate check. 262 // Minimal check: only set requires flags if previous value 263 // is undefined. This ensures that only the first call to this 264 // function will set the requires flags. All subsequent calls 265 // will be checked for compatibility. 266 assert(flags != OMP_REQ_UNDEFINED && 267 "illegal undefined flag for requires directive!"); 268 if (RequiresFlags == OMP_REQ_UNDEFINED) { 269 RequiresFlags = flags; 270 return; 271 } 272 273 // If multiple compilation units are present enforce 274 // consistency across all of them for require clauses: 275 // - reverse_offload 276 // - unified_address 277 // - unified_shared_memory 278 if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) != 279 (flags & OMP_REQ_REVERSE_OFFLOAD)) { 280 FATAL_MESSAGE0( 281 1, "'#pragma omp requires reverse_offload' not used consistently!"); 282 } 283 if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) != 284 (flags & OMP_REQ_UNIFIED_ADDRESS)) { 285 FATAL_MESSAGE0( 286 1, "'#pragma omp requires unified_address' not used consistently!"); 287 } 288 if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) != 289 (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) { 290 FATAL_MESSAGE0( 291 1, 292 "'#pragma omp requires unified_shared_memory' not used consistently!"); 293 } 294 295 // TODO: insert any other missing checks 296 297 DP("New requires flags %" PRId64 " compatible with existing %" PRId64 "!\n", 298 flags, RequiresFlags); 299 } 300 301 void RTLsTy::initRTLonce(RTLInfoTy &R) { 302 // If this RTL is not already in use, initialize it. 303 if (!R.isUsed && R.NumberOfDevices != 0) { 304 // Initialize the device information for the RTL we are about to use. 305 const size_t Start = PM->Devices.size(); 306 PM->Devices.reserve(Start + R.NumberOfDevices); 307 for (int32_t device_id = 0; device_id < R.NumberOfDevices; device_id++) { 308 PM->Devices.push_back(std::make_unique<DeviceTy>(&R)); 309 // global device ID 310 PM->Devices[Start + device_id]->DeviceID = Start + device_id; 311 // RTL local device ID 312 PM->Devices[Start + device_id]->RTLDeviceID = device_id; 313 } 314 315 // Initialize the index of this RTL and save it in the used RTLs. 316 R.Idx = (UsedRTLs.empty()) 317 ? 0 318 : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices; 319 assert((size_t)R.Idx == Start && 320 "RTL index should equal the number of devices used so far."); 321 R.isUsed = true; 322 UsedRTLs.push_back(&R); 323 324 DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx); 325 } 326 } 327 328 void RTLsTy::initAllRTLs() { 329 for (auto &R : AllRTLs) 330 initRTLonce(R); 331 } 332 333 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) { 334 PM->RTLsMtx.lock(); 335 // Register the images with the RTLs that understand them, if any. 336 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 337 // Obtain the image. 338 __tgt_device_image *img = &desc->DeviceImages[i]; 339 340 RTLInfoTy *FoundRTL = nullptr; 341 342 // Scan the RTLs that have associated images until we find one that supports 343 // the current image. 344 for (auto &R : AllRTLs) { 345 if (!R.is_valid_binary(img)) { 346 DP("Image " DPxMOD " is NOT compatible with RTL %s!\n", 347 DPxPTR(img->ImageStart), R.RTLName.c_str()); 348 continue; 349 } 350 351 DP("Image " DPxMOD " is compatible with RTL %s!\n", 352 DPxPTR(img->ImageStart), R.RTLName.c_str()); 353 354 initRTLonce(R); 355 356 // Initialize (if necessary) translation table for this library. 357 PM->TrlTblMtx.lock(); 358 if (!PM->HostEntriesBeginToTransTable.count(desc->HostEntriesBegin)) { 359 PM->HostEntriesBeginRegistrationOrder.push_back(desc->HostEntriesBegin); 360 TranslationTable &TransTable = 361 (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 362 TransTable.HostTable.EntriesBegin = desc->HostEntriesBegin; 363 TransTable.HostTable.EntriesEnd = desc->HostEntriesEnd; 364 } 365 366 // Retrieve translation table for this library. 367 TranslationTable &TransTable = 368 (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 369 370 DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(img->ImageStart), 371 R.RTLName.c_str()); 372 RegisterImageIntoTranslationTable(TransTable, R, img); 373 PM->TrlTblMtx.unlock(); 374 FoundRTL = &R; 375 376 // Load ctors/dtors for static objects 377 RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL); 378 379 // if an RTL was found we are done - proceed to register the next image 380 break; 381 } 382 383 if (!FoundRTL) { 384 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart)); 385 } 386 } 387 PM->RTLsMtx.unlock(); 388 389 DP("Done registering entries!\n"); 390 } 391 392 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) { 393 DP("Unloading target library!\n"); 394 395 PM->RTLsMtx.lock(); 396 // Find which RTL understands each image, if any. 397 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 398 // Obtain the image. 399 __tgt_device_image *img = &desc->DeviceImages[i]; 400 401 RTLInfoTy *FoundRTL = NULL; 402 403 // Scan the RTLs that have associated images until we find one that supports 404 // the current image. We only need to scan RTLs that are already being used. 405 for (auto *R : UsedRTLs) { 406 407 assert(R->isUsed && "Expecting used RTLs."); 408 409 if (!R->is_valid_binary(img)) { 410 DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n", 411 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 412 continue; 413 } 414 415 DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n", 416 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 417 418 FoundRTL = R; 419 420 // Execute dtors for static objects if the device has been used, i.e. 421 // if its PendingCtors list has been emptied. 422 for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) { 423 DeviceTy &Device = *PM->Devices[FoundRTL->Idx + i]; 424 Device.PendingGlobalsMtx.lock(); 425 if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) { 426 AsyncInfoTy AsyncInfo(Device); 427 for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) { 428 int rc = target(nullptr, Device, dtor, 0, nullptr, nullptr, nullptr, 429 nullptr, nullptr, nullptr, 1, 1, true /*team*/, 430 AsyncInfo); 431 if (rc != OFFLOAD_SUCCESS) { 432 DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor)); 433 } 434 } 435 // Remove this library's entry from PendingCtorsDtors 436 Device.PendingCtorsDtors.erase(desc); 437 // All constructors have been issued, wait for them now. 438 if (AsyncInfo.synchronize() != OFFLOAD_SUCCESS) 439 DP("Failed synchronizing destructors kernels.\n"); 440 } 441 Device.PendingGlobalsMtx.unlock(); 442 } 443 444 DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n", 445 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 446 447 break; 448 } 449 450 // if no RTL was found proceed to unregister the next image 451 if (!FoundRTL) { 452 DP("No RTLs in use support the image " DPxMOD "!\n", 453 DPxPTR(img->ImageStart)); 454 } 455 } 456 PM->RTLsMtx.unlock(); 457 DP("Done unregistering images!\n"); 458 459 // Remove entries from PM->HostPtrToTableMap 460 PM->TblMapMtx.lock(); 461 for (__tgt_offload_entry *cur = desc->HostEntriesBegin; 462 cur < desc->HostEntriesEnd; ++cur) { 463 PM->HostPtrToTableMap.erase(cur->addr); 464 } 465 466 // Remove translation table for this descriptor. 467 auto TransTable = 468 PM->HostEntriesBeginToTransTable.find(desc->HostEntriesBegin); 469 if (TransTable != PM->HostEntriesBeginToTransTable.end()) { 470 DP("Removing translation table for descriptor " DPxMOD "\n", 471 DPxPTR(desc->HostEntriesBegin)); 472 PM->HostEntriesBeginToTransTable.erase(TransTable); 473 } else { 474 DP("Translation table for descriptor " DPxMOD " cannot be found, probably " 475 "it has been already removed.\n", 476 DPxPTR(desc->HostEntriesBegin)); 477 } 478 479 PM->TblMapMtx.unlock(); 480 481 // TODO: Remove RTL and the devices it manages if it's not used anymore? 482 // TODO: Write some RTL->unload_image(...) function? 483 484 DP("Done unregistering library!\n"); 485 } 486