1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Functionality for handling RTL plugins. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "rtl.h" 14 #include "device.h" 15 #include "private.h" 16 17 #if OMPT_SUPPORT 18 #include "ompt-target.h" 19 #endif 20 21 #include <cassert> 22 #include <cstdlib> 23 #include <cstring> 24 #include <dlfcn.h> 25 #include <mutex> 26 #include <string> 27 28 // List of all plugins that can support offloading. 29 static const char *RTLNames[] = { 30 /* PowerPC target */ "libomptarget.rtl.ppc64.so", 31 /* x86_64 target */ "libomptarget.rtl.x86_64.so", 32 /* CUDA target */ "libomptarget.rtl.cuda.so", 33 /* AArch64 target */ "libomptarget.rtl.aarch64.so", 34 /* SX-Aurora VE target */ "libomptarget.rtl.ve.so", 35 /* AMDGPU target */ "libomptarget.rtl.amdgpu.so", 36 /* Remote target */ "libomptarget.rtl.rpc.so", 37 }; 38 39 PluginManager *PM; 40 41 #if OMPTARGET_PROFILE_ENABLED 42 static char *ProfileTraceFile = nullptr; 43 #endif 44 45 __attribute__((constructor(101))) void init() { 46 DP("Init target library!\n"); 47 PM = new PluginManager(); 48 49 #ifdef OMPTARGET_PROFILE_ENABLED 50 ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE"); 51 // TODO: add a configuration option for time granularity 52 if (ProfileTraceFile) 53 llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget"); 54 #endif 55 } 56 57 __attribute__((destructor(101))) void deinit() { 58 DP("Deinit target library!\n"); 59 delete PM; 60 61 #ifdef OMPTARGET_PROFILE_ENABLED 62 if (ProfileTraceFile) { 63 // TODO: add env var for file output 64 if (auto E = llvm::timeTraceProfilerWrite(ProfileTraceFile, "-")) 65 fprintf(stderr, "Error writing out the time trace\n"); 66 67 llvm::timeTraceProfilerCleanup(); 68 } 69 #endif 70 } 71 72 void RTLsTy::LoadRTLs() { 73 // Parse environment variable OMP_TARGET_OFFLOAD (if set) 74 PM->TargetOffloadPolicy = 75 (kmp_target_offload_kind_t)__kmpc_get_target_offload(); 76 if (PM->TargetOffloadPolicy == tgt_disabled) { 77 return; 78 } 79 80 DP("Loading RTLs...\n"); 81 82 // Attempt to open all the plugins and, if they exist, check if the interface 83 // is correct and if they are supporting any devices. 84 for (auto *Name : RTLNames) { 85 DP("Loading library '%s'...\n", Name); 86 void *dynlib_handle = dlopen(Name, RTLD_NOW); 87 88 if (!dynlib_handle) { 89 // Library does not exist or cannot be found. 90 DP("Unable to load library '%s': %s!\n", Name, dlerror()); 91 continue; 92 } 93 94 DP("Successfully loaded library '%s'!\n", Name); 95 96 AllRTLs.emplace_back(); 97 98 // Retrieve the RTL information from the runtime library. 99 RTLInfoTy &R = AllRTLs.back(); 100 101 bool ValidPlugin = true; 102 103 if (!(*((void **)&R.is_valid_binary) = 104 dlsym(dynlib_handle, "__tgt_rtl_is_valid_binary"))) 105 ValidPlugin = false; 106 if (!(*((void **)&R.number_of_devices) = 107 dlsym(dynlib_handle, "__tgt_rtl_number_of_devices"))) 108 ValidPlugin = false; 109 if (!(*((void **)&R.init_device) = 110 dlsym(dynlib_handle, "__tgt_rtl_init_device"))) 111 ValidPlugin = false; 112 if (!(*((void **)&R.load_binary) = 113 dlsym(dynlib_handle, "__tgt_rtl_load_binary"))) 114 ValidPlugin = false; 115 if (!(*((void **)&R.data_alloc) = 116 dlsym(dynlib_handle, "__tgt_rtl_data_alloc"))) 117 ValidPlugin = false; 118 if (!(*((void **)&R.data_submit) = 119 dlsym(dynlib_handle, "__tgt_rtl_data_submit"))) 120 ValidPlugin = false; 121 if (!(*((void **)&R.data_retrieve) = 122 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve"))) 123 ValidPlugin = false; 124 if (!(*((void **)&R.data_delete) = 125 dlsym(dynlib_handle, "__tgt_rtl_data_delete"))) 126 ValidPlugin = false; 127 if (!(*((void **)&R.run_region) = 128 dlsym(dynlib_handle, "__tgt_rtl_run_target_region"))) 129 ValidPlugin = false; 130 if (!(*((void **)&R.run_team_region) = 131 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region"))) 132 ValidPlugin = false; 133 134 // Invalid plugin 135 if (!ValidPlugin) { 136 DP("Invalid plugin as necessary interface is not found.\n"); 137 AllRTLs.pop_back(); 138 continue; 139 } 140 141 // No devices are supported by this RTL? 142 if (!(R.NumberOfDevices = R.number_of_devices())) { 143 // The RTL is invalid! Will pop the object from the RTLs list. 144 DP("No devices supported in this RTL\n"); 145 AllRTLs.pop_back(); 146 continue; 147 } 148 149 R.LibraryHandler = dynlib_handle; 150 151 #ifdef OMPTARGET_DEBUG 152 R.RTLName = Name; 153 #endif 154 155 DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(), 156 R.NumberOfDevices); 157 158 // Optional functions 159 *((void **)&R.init_requires) = 160 dlsym(dynlib_handle, "__tgt_rtl_init_requires"); 161 *((void **)&R.data_submit_async) = 162 dlsym(dynlib_handle, "__tgt_rtl_data_submit_async"); 163 *((void **)&R.data_retrieve_async) = 164 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async"); 165 *((void **)&R.run_region_async) = 166 dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async"); 167 *((void **)&R.run_team_region_async) = 168 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async"); 169 *((void **)&R.synchronize) = dlsym(dynlib_handle, "__tgt_rtl_synchronize"); 170 *((void **)&R.data_exchange) = 171 dlsym(dynlib_handle, "__tgt_rtl_data_exchange"); 172 *((void **)&R.data_exchange_async) = 173 dlsym(dynlib_handle, "__tgt_rtl_data_exchange_async"); 174 *((void **)&R.is_data_exchangable) = 175 dlsym(dynlib_handle, "__tgt_rtl_is_data_exchangable"); 176 *((void **)&R.register_lib) = 177 dlsym(dynlib_handle, "__tgt_rtl_register_lib"); 178 *((void **)&R.unregister_lib) = 179 dlsym(dynlib_handle, "__tgt_rtl_unregister_lib"); 180 *((void **)&R.supports_empty_images) = 181 dlsym(dynlib_handle, "__tgt_rtl_supports_empty_images"); 182 *((void **)&R.set_info_flag) = 183 dlsym(dynlib_handle, "__tgt_rtl_set_info_flag"); 184 *((void **)&R.print_device_info) = 185 dlsym(dynlib_handle, "__tgt_rtl_print_device_info"); 186 *((void **)&R.create_event) = 187 dlsym(dynlib_handle, "__tgt_rtl_create_event"); 188 *((void **)&R.record_event) = 189 dlsym(dynlib_handle, "__tgt_rtl_record_event"); 190 *((void **)&R.wait_event) = dlsym(dynlib_handle, "__tgt_rtl_wait_event"); 191 *((void **)&R.sync_event) = dlsym(dynlib_handle, "__tgt_rtl_sync_event"); 192 *((void **)&R.destroy_event) = 193 dlsym(dynlib_handle, "__tgt_rtl_destroy_event"); 194 } 195 196 #if OMPT_SUPPORT 197 DP("OMPT_SUPPORT is enabled in libomptarget\n"); 198 DP("Init OMPT for libomptarget\n"); 199 if (libomp_start_tool) { 200 DP("Retrieve libomp_start_tool successfully\n"); 201 if (!libomp_start_tool(&ompt_target_enabled)) { 202 DP("Turn off OMPT in libomptarget because libomp_start_tool returns " 203 "false\n"); 204 memset(&ompt_target_enabled, 0, sizeof(ompt_target_enabled)); 205 } 206 } 207 #endif 208 209 DP("RTLs loaded!\n"); 210 211 return; 212 } 213 214 //////////////////////////////////////////////////////////////////////////////// 215 // Functionality for registering libs 216 217 static void RegisterImageIntoTranslationTable(TranslationTable &TT, 218 RTLInfoTy &RTL, 219 __tgt_device_image *image) { 220 221 // same size, as when we increase one, we also increase the other. 222 assert(TT.TargetsTable.size() == TT.TargetsImages.size() && 223 "We should have as many images as we have tables!"); 224 225 // Resize the Targets Table and Images to accommodate the new targets if 226 // required 227 unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices; 228 229 if (TT.TargetsTable.size() < TargetsTableMinimumSize) { 230 TT.TargetsImages.resize(TargetsTableMinimumSize, 0); 231 TT.TargetsTable.resize(TargetsTableMinimumSize, 0); 232 } 233 234 // Register the image in all devices for this target type. 235 for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) { 236 // If we are changing the image we are also invalidating the target table. 237 if (TT.TargetsImages[RTL.Idx + i] != image) { 238 TT.TargetsImages[RTL.Idx + i] = image; 239 TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table. 240 } 241 } 242 } 243 244 //////////////////////////////////////////////////////////////////////////////// 245 // Functionality for registering Ctors/Dtors 246 247 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc, 248 __tgt_device_image *img, 249 RTLInfoTy *RTL) { 250 251 for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) { 252 DeviceTy &Device = *PM->Devices[RTL->Idx + i]; 253 Device.PendingGlobalsMtx.lock(); 254 Device.HasPendingGlobals = true; 255 for (__tgt_offload_entry *entry = img->EntriesBegin; 256 entry != img->EntriesEnd; ++entry) { 257 if (entry->flags & OMP_DECLARE_TARGET_CTOR) { 258 DP("Adding ctor " DPxMOD " to the pending list.\n", 259 DPxPTR(entry->addr)); 260 Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr); 261 } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) { 262 // Dtors are pushed in reverse order so they are executed from end 263 // to beginning when unregistering the library! 264 DP("Adding dtor " DPxMOD " to the pending list.\n", 265 DPxPTR(entry->addr)); 266 Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr); 267 } 268 269 if (entry->flags & OMP_DECLARE_TARGET_LINK) { 270 DP("The \"link\" attribute is not yet supported!\n"); 271 } 272 } 273 Device.PendingGlobalsMtx.unlock(); 274 } 275 } 276 277 void RTLsTy::RegisterRequires(int64_t flags) { 278 // TODO: add more elaborate check. 279 // Minimal check: only set requires flags if previous value 280 // is undefined. This ensures that only the first call to this 281 // function will set the requires flags. All subsequent calls 282 // will be checked for compatibility. 283 assert(flags != OMP_REQ_UNDEFINED && 284 "illegal undefined flag for requires directive!"); 285 if (RequiresFlags == OMP_REQ_UNDEFINED) { 286 RequiresFlags = flags; 287 return; 288 } 289 290 // If multiple compilation units are present enforce 291 // consistency across all of them for require clauses: 292 // - reverse_offload 293 // - unified_address 294 // - unified_shared_memory 295 if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) != 296 (flags & OMP_REQ_REVERSE_OFFLOAD)) { 297 FATAL_MESSAGE0( 298 1, "'#pragma omp requires reverse_offload' not used consistently!"); 299 } 300 if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) != 301 (flags & OMP_REQ_UNIFIED_ADDRESS)) { 302 FATAL_MESSAGE0( 303 1, "'#pragma omp requires unified_address' not used consistently!"); 304 } 305 if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) != 306 (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) { 307 FATAL_MESSAGE0( 308 1, 309 "'#pragma omp requires unified_shared_memory' not used consistently!"); 310 } 311 312 // TODO: insert any other missing checks 313 314 DP("New requires flags %" PRId64 " compatible with existing %" PRId64 "!\n", 315 flags, RequiresFlags); 316 } 317 318 void RTLsTy::initRTLonce(RTLInfoTy &R) { 319 // If this RTL is not already in use, initialize it. 320 if (!R.isUsed && R.NumberOfDevices != 0) { 321 // Initialize the device information for the RTL we are about to use. 322 const size_t Start = PM->Devices.size(); 323 PM->Devices.reserve(Start + R.NumberOfDevices); 324 for (int32_t device_id = 0; device_id < R.NumberOfDevices; device_id++) { 325 PM->Devices.push_back(std::make_unique<DeviceTy>(&R)); 326 // global device ID 327 PM->Devices[Start + device_id]->DeviceID = Start + device_id; 328 // RTL local device ID 329 PM->Devices[Start + device_id]->RTLDeviceID = device_id; 330 } 331 332 // Initialize the index of this RTL and save it in the used RTLs. 333 R.Idx = (UsedRTLs.empty()) 334 ? 0 335 : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices; 336 assert((size_t)R.Idx == Start && 337 "RTL index should equal the number of devices used so far."); 338 R.isUsed = true; 339 UsedRTLs.push_back(&R); 340 341 DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx); 342 } 343 } 344 345 void RTLsTy::initAllRTLs() { 346 for (auto &R : AllRTLs) 347 initRTLonce(R); 348 } 349 350 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) { 351 PM->RTLsMtx.lock(); 352 // Register the images with the RTLs that understand them, if any. 353 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 354 // Obtain the image. 355 __tgt_device_image *img = &desc->DeviceImages[i]; 356 357 RTLInfoTy *FoundRTL = nullptr; 358 359 // Scan the RTLs that have associated images until we find one that supports 360 // the current image. 361 for (auto &R : AllRTLs) { 362 if (!R.is_valid_binary(img)) { 363 DP("Image " DPxMOD " is NOT compatible with RTL %s!\n", 364 DPxPTR(img->ImageStart), R.RTLName.c_str()); 365 continue; 366 } 367 368 DP("Image " DPxMOD " is compatible with RTL %s!\n", 369 DPxPTR(img->ImageStart), R.RTLName.c_str()); 370 371 initRTLonce(R); 372 373 // Initialize (if necessary) translation table for this library. 374 PM->TrlTblMtx.lock(); 375 if (!PM->HostEntriesBeginToTransTable.count(desc->HostEntriesBegin)) { 376 PM->HostEntriesBeginRegistrationOrder.push_back(desc->HostEntriesBegin); 377 TranslationTable &TransTable = 378 (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 379 TransTable.HostTable.EntriesBegin = desc->HostEntriesBegin; 380 TransTable.HostTable.EntriesEnd = desc->HostEntriesEnd; 381 } 382 383 // Retrieve translation table for this library. 384 TranslationTable &TransTable = 385 (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 386 387 DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(img->ImageStart), 388 R.RTLName.c_str()); 389 RegisterImageIntoTranslationTable(TransTable, R, img); 390 PM->TrlTblMtx.unlock(); 391 FoundRTL = &R; 392 393 // Load ctors/dtors for static objects 394 RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL); 395 396 // if an RTL was found we are done - proceed to register the next image 397 break; 398 } 399 400 if (!FoundRTL) { 401 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart)); 402 } 403 } 404 PM->RTLsMtx.unlock(); 405 406 DP("Done registering entries!\n"); 407 } 408 409 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) { 410 DP("Unloading target library!\n"); 411 412 PM->RTLsMtx.lock(); 413 // Find which RTL understands each image, if any. 414 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 415 // Obtain the image. 416 __tgt_device_image *img = &desc->DeviceImages[i]; 417 418 RTLInfoTy *FoundRTL = NULL; 419 420 // Scan the RTLs that have associated images until we find one that supports 421 // the current image. We only need to scan RTLs that are already being used. 422 for (auto *R : UsedRTLs) { 423 424 assert(R->isUsed && "Expecting used RTLs."); 425 426 if (!R->is_valid_binary(img)) { 427 DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n", 428 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 429 continue; 430 } 431 432 DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n", 433 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 434 435 FoundRTL = R; 436 437 // Execute dtors for static objects if the device has been used, i.e. 438 // if its PendingCtors list has been emptied. 439 for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) { 440 DeviceTy &Device = *PM->Devices[FoundRTL->Idx + i]; 441 Device.PendingGlobalsMtx.lock(); 442 if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) { 443 AsyncInfoTy AsyncInfo(Device); 444 for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) { 445 int rc = target(nullptr, Device, dtor, 0, nullptr, nullptr, nullptr, 446 nullptr, nullptr, nullptr, 1, 1, true /*team*/, 447 AsyncInfo); 448 if (rc != OFFLOAD_SUCCESS) { 449 DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor)); 450 } 451 } 452 // Remove this library's entry from PendingCtorsDtors 453 Device.PendingCtorsDtors.erase(desc); 454 // All constructors have been issued, wait for them now. 455 if (AsyncInfo.synchronize() != OFFLOAD_SUCCESS) 456 DP("Failed synchronizing destructors kernels.\n"); 457 } 458 Device.PendingGlobalsMtx.unlock(); 459 } 460 461 DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n", 462 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 463 464 break; 465 } 466 467 // if no RTL was found proceed to unregister the next image 468 if (!FoundRTL) { 469 DP("No RTLs in use support the image " DPxMOD "!\n", 470 DPxPTR(img->ImageStart)); 471 } 472 } 473 PM->RTLsMtx.unlock(); 474 DP("Done unregistering images!\n"); 475 476 // Remove entries from PM->HostPtrToTableMap 477 PM->TblMapMtx.lock(); 478 for (__tgt_offload_entry *cur = desc->HostEntriesBegin; 479 cur < desc->HostEntriesEnd; ++cur) { 480 PM->HostPtrToTableMap.erase(cur->addr); 481 } 482 483 // Remove translation table for this descriptor. 484 auto TransTable = 485 PM->HostEntriesBeginToTransTable.find(desc->HostEntriesBegin); 486 if (TransTable != PM->HostEntriesBeginToTransTable.end()) { 487 DP("Removing translation table for descriptor " DPxMOD "\n", 488 DPxPTR(desc->HostEntriesBegin)); 489 PM->HostEntriesBeginToTransTable.erase(TransTable); 490 } else { 491 DP("Translation table for descriptor " DPxMOD " cannot be found, probably " 492 "it has been already removed.\n", 493 DPxPTR(desc->HostEntriesBegin)); 494 } 495 496 PM->TblMapMtx.unlock(); 497 498 // TODO: Remove RTL and the devices it manages if it's not used anymore? 499 // TODO: Write some RTL->unload_image(...) function? 500 501 DP("Done unregistering library!\n"); 502 } 503