1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Functionality for handling RTL plugins. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "device.h" 14 #include "private.h" 15 #include "rtl.h" 16 17 #include <cassert> 18 #include <cstdlib> 19 #include <cstring> 20 #include <dlfcn.h> 21 #include <mutex> 22 #include <string> 23 24 // List of all plugins that can support offloading. 25 static const char *RTLNames[] = { 26 /* PowerPC target */ "libomptarget.rtl.ppc64.so", 27 /* x86_64 target */ "libomptarget.rtl.x86_64.so", 28 /* CUDA target */ "libomptarget.rtl.cuda.so", 29 /* AArch64 target */ "libomptarget.rtl.aarch64.so", 30 /* SX-Aurora VE target */ "libomptarget.rtl.ve.so", 31 /* AMDGPU target */ "libomptarget.rtl.amdgpu.so", 32 }; 33 34 PluginManager *PM; 35 36 #if OMPTARGET_PROFILE_ENABLED 37 static char *ProfileTraceFile = nullptr; 38 #endif 39 40 __attribute__((constructor(101))) void init() { 41 DP("Init target library!\n"); 42 PM = new PluginManager(); 43 44 #ifdef OMPTARGET_PROFILE_ENABLED 45 ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE"); 46 // TODO: add a configuration option for time granularity 47 if (ProfileTraceFile) 48 llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget"); 49 #endif 50 } 51 52 __attribute__((destructor(101))) void deinit() { 53 DP("Deinit target library!\n"); 54 delete PM; 55 56 #ifdef OMPTARGET_PROFILE_ENABLED 57 if (ProfileTraceFile) { 58 // TODO: add env var for file output 59 if (auto E = llvm::timeTraceProfilerWrite(ProfileTraceFile, "-")) 60 fprintf(stderr, "Error writing out the time trace\n"); 61 62 llvm::timeTraceProfilerCleanup(); 63 } 64 #endif 65 } 66 67 void RTLsTy::LoadRTLs() { 68 // Parse environment variable OMP_TARGET_OFFLOAD (if set) 69 PM->TargetOffloadPolicy = 70 (kmp_target_offload_kind_t)__kmpc_get_target_offload(); 71 if (PM->TargetOffloadPolicy == tgt_disabled) { 72 return; 73 } 74 75 DP("Loading RTLs...\n"); 76 77 // Attempt to open all the plugins and, if they exist, check if the interface 78 // is correct and if they are supporting any devices. 79 for (auto *Name : RTLNames) { 80 DP("Loading library '%s'...\n", Name); 81 void *dynlib_handle = dlopen(Name, RTLD_NOW); 82 83 if (!dynlib_handle) { 84 // Library does not exist or cannot be found. 85 DP("Unable to load library '%s': %s!\n", Name, dlerror()); 86 continue; 87 } 88 89 DP("Successfully loaded library '%s'!\n", Name); 90 91 AllRTLs.emplace_back(); 92 93 // Retrieve the RTL information from the runtime library. 94 RTLInfoTy &R = AllRTLs.back(); 95 96 bool ValidPlugin = true; 97 98 if (!(*((void **)&R.is_valid_binary) = 99 dlsym(dynlib_handle, "__tgt_rtl_is_valid_binary"))) 100 ValidPlugin = false; 101 if (!(*((void **)&R.number_of_devices) = 102 dlsym(dynlib_handle, "__tgt_rtl_number_of_devices"))) 103 ValidPlugin = false; 104 if (!(*((void **)&R.init_device) = 105 dlsym(dynlib_handle, "__tgt_rtl_init_device"))) 106 ValidPlugin = false; 107 if (!(*((void **)&R.load_binary) = 108 dlsym(dynlib_handle, "__tgt_rtl_load_binary"))) 109 ValidPlugin = false; 110 if (!(*((void **)&R.data_alloc) = 111 dlsym(dynlib_handle, "__tgt_rtl_data_alloc"))) 112 ValidPlugin = false; 113 if (!(*((void **)&R.data_submit) = 114 dlsym(dynlib_handle, "__tgt_rtl_data_submit"))) 115 ValidPlugin = false; 116 if (!(*((void **)&R.data_retrieve) = 117 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve"))) 118 ValidPlugin = false; 119 if (!(*((void **)&R.data_delete) = 120 dlsym(dynlib_handle, "__tgt_rtl_data_delete"))) 121 ValidPlugin = false; 122 if (!(*((void **)&R.run_region) = 123 dlsym(dynlib_handle, "__tgt_rtl_run_target_region"))) 124 ValidPlugin = false; 125 if (!(*((void **)&R.run_team_region) = 126 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region"))) 127 ValidPlugin = false; 128 129 // Invalid plugin 130 if (!ValidPlugin) { 131 DP("Invalid plugin as necessary interface is not found.\n"); 132 AllRTLs.pop_back(); 133 continue; 134 } 135 136 // No devices are supported by this RTL? 137 if (!(R.NumberOfDevices = R.number_of_devices())) { 138 // The RTL is invalid! Will pop the object from the RTLs list. 139 DP("No devices supported in this RTL\n"); 140 AllRTLs.pop_back(); 141 continue; 142 } 143 144 R.LibraryHandler = dynlib_handle; 145 146 #ifdef OMPTARGET_DEBUG 147 R.RTLName = Name; 148 #endif 149 150 DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(), 151 R.NumberOfDevices); 152 153 // Optional functions 154 *((void **)&R.init_requires) = 155 dlsym(dynlib_handle, "__tgt_rtl_init_requires"); 156 *((void **)&R.data_submit_async) = 157 dlsym(dynlib_handle, "__tgt_rtl_data_submit_async"); 158 *((void **)&R.data_retrieve_async) = 159 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async"); 160 *((void **)&R.run_region_async) = 161 dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async"); 162 *((void **)&R.run_team_region_async) = 163 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async"); 164 *((void **)&R.synchronize) = dlsym(dynlib_handle, "__tgt_rtl_synchronize"); 165 *((void **)&R.data_exchange) = 166 dlsym(dynlib_handle, "__tgt_rtl_data_exchange"); 167 *((void **)&R.data_exchange_async) = 168 dlsym(dynlib_handle, "__tgt_rtl_data_exchange_async"); 169 *((void **)&R.is_data_exchangable) = 170 dlsym(dynlib_handle, "__tgt_rtl_is_data_exchangable"); 171 } 172 173 DP("RTLs loaded!\n"); 174 175 return; 176 } 177 178 //////////////////////////////////////////////////////////////////////////////// 179 // Functionality for registering libs 180 181 static void RegisterImageIntoTranslationTable(TranslationTable &TT, 182 RTLInfoTy &RTL, __tgt_device_image *image) { 183 184 // same size, as when we increase one, we also increase the other. 185 assert(TT.TargetsTable.size() == TT.TargetsImages.size() && 186 "We should have as many images as we have tables!"); 187 188 // Resize the Targets Table and Images to accommodate the new targets if 189 // required 190 unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices; 191 192 if (TT.TargetsTable.size() < TargetsTableMinimumSize) { 193 TT.TargetsImages.resize(TargetsTableMinimumSize, 0); 194 TT.TargetsTable.resize(TargetsTableMinimumSize, 0); 195 } 196 197 // Register the image in all devices for this target type. 198 for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) { 199 // If we are changing the image we are also invalidating the target table. 200 if (TT.TargetsImages[RTL.Idx + i] != image) { 201 TT.TargetsImages[RTL.Idx + i] = image; 202 TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table. 203 } 204 } 205 } 206 207 //////////////////////////////////////////////////////////////////////////////// 208 // Functionality for registering Ctors/Dtors 209 210 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc, 211 __tgt_device_image *img, RTLInfoTy *RTL) { 212 213 for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) { 214 DeviceTy &Device = PM->Devices[RTL->Idx + i]; 215 Device.PendingGlobalsMtx.lock(); 216 Device.HasPendingGlobals = true; 217 for (__tgt_offload_entry *entry = img->EntriesBegin; 218 entry != img->EntriesEnd; ++entry) { 219 if (entry->flags & OMP_DECLARE_TARGET_CTOR) { 220 DP("Adding ctor " DPxMOD " to the pending list.\n", 221 DPxPTR(entry->addr)); 222 Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr); 223 } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) { 224 // Dtors are pushed in reverse order so they are executed from end 225 // to beginning when unregistering the library! 226 DP("Adding dtor " DPxMOD " to the pending list.\n", 227 DPxPTR(entry->addr)); 228 Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr); 229 } 230 231 if (entry->flags & OMP_DECLARE_TARGET_LINK) { 232 DP("The \"link\" attribute is not yet supported!\n"); 233 } 234 } 235 Device.PendingGlobalsMtx.unlock(); 236 } 237 } 238 239 void RTLsTy::RegisterRequires(int64_t flags) { 240 // TODO: add more elaborate check. 241 // Minimal check: only set requires flags if previous value 242 // is undefined. This ensures that only the first call to this 243 // function will set the requires flags. All subsequent calls 244 // will be checked for compatibility. 245 assert(flags != OMP_REQ_UNDEFINED && 246 "illegal undefined flag for requires directive!"); 247 if (RequiresFlags == OMP_REQ_UNDEFINED) { 248 RequiresFlags = flags; 249 return; 250 } 251 252 // If multiple compilation units are present enforce 253 // consistency across all of them for require clauses: 254 // - reverse_offload 255 // - unified_address 256 // - unified_shared_memory 257 if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) != 258 (flags & OMP_REQ_REVERSE_OFFLOAD)) { 259 FATAL_MESSAGE0(1, 260 "'#pragma omp requires reverse_offload' not used consistently!"); 261 } 262 if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) != 263 (flags & OMP_REQ_UNIFIED_ADDRESS)) { 264 FATAL_MESSAGE0(1, 265 "'#pragma omp requires unified_address' not used consistently!"); 266 } 267 if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) != 268 (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) { 269 FATAL_MESSAGE0(1, 270 "'#pragma omp requires unified_shared_memory' not used consistently!"); 271 } 272 273 // TODO: insert any other missing checks 274 275 DP("New requires flags %" PRId64 " compatible with existing %" PRId64 "!\n", 276 flags, RequiresFlags); 277 } 278 279 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) { 280 // Attempt to load all plugins available in the system. 281 std::call_once(initFlag, &RTLsTy::LoadRTLs, this); 282 283 PM->RTLsMtx.lock(); 284 // Register the images with the RTLs that understand them, if any. 285 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 286 // Obtain the image. 287 __tgt_device_image *img = &desc->DeviceImages[i]; 288 289 RTLInfoTy *FoundRTL = NULL; 290 291 // Scan the RTLs that have associated images until we find one that supports 292 // the current image. 293 for (auto &R : AllRTLs) { 294 if (!R.is_valid_binary(img)) { 295 DP("Image " DPxMOD " is NOT compatible with RTL %s!\n", 296 DPxPTR(img->ImageStart), R.RTLName.c_str()); 297 continue; 298 } 299 300 DP("Image " DPxMOD " is compatible with RTL %s!\n", 301 DPxPTR(img->ImageStart), R.RTLName.c_str()); 302 303 // If this RTL is not already in use, initialize it. 304 if (!R.isUsed) { 305 // Initialize the device information for the RTL we are about to use. 306 DeviceTy device(&R); 307 size_t Start = PM->Devices.size(); 308 PM->Devices.resize(Start + R.NumberOfDevices, device); 309 for (int32_t device_id = 0; device_id < R.NumberOfDevices; 310 device_id++) { 311 // global device ID 312 PM->Devices[Start + device_id].DeviceID = Start + device_id; 313 // RTL local device ID 314 PM->Devices[Start + device_id].RTLDeviceID = device_id; 315 } 316 317 // Initialize the index of this RTL and save it in the used RTLs. 318 R.Idx = (UsedRTLs.empty()) 319 ? 0 320 : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices; 321 assert((size_t) R.Idx == Start && 322 "RTL index should equal the number of devices used so far."); 323 R.isUsed = true; 324 UsedRTLs.push_back(&R); 325 326 DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx); 327 } 328 329 // Initialize (if necessary) translation table for this library. 330 PM->TrlTblMtx.lock(); 331 if (!PM->HostEntriesBeginToTransTable.count(desc->HostEntriesBegin)) { 332 TranslationTable &TransTable = 333 (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 334 TransTable.HostTable.EntriesBegin = desc->HostEntriesBegin; 335 TransTable.HostTable.EntriesEnd = desc->HostEntriesEnd; 336 } 337 338 // Retrieve translation table for this library. 339 TranslationTable &TransTable = 340 (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 341 342 DP("Registering image " DPxMOD " with RTL %s!\n", 343 DPxPTR(img->ImageStart), R.RTLName.c_str()); 344 RegisterImageIntoTranslationTable(TransTable, R, img); 345 PM->TrlTblMtx.unlock(); 346 FoundRTL = &R; 347 348 // Load ctors/dtors for static objects 349 RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL); 350 351 // if an RTL was found we are done - proceed to register the next image 352 break; 353 } 354 355 if (!FoundRTL) { 356 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart)); 357 } 358 } 359 PM->RTLsMtx.unlock(); 360 361 DP("Done registering entries!\n"); 362 } 363 364 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) { 365 DP("Unloading target library!\n"); 366 367 PM->RTLsMtx.lock(); 368 // Find which RTL understands each image, if any. 369 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 370 // Obtain the image. 371 __tgt_device_image *img = &desc->DeviceImages[i]; 372 373 RTLInfoTy *FoundRTL = NULL; 374 375 // Scan the RTLs that have associated images until we find one that supports 376 // the current image. We only need to scan RTLs that are already being used. 377 for (auto *R : UsedRTLs) { 378 379 assert(R->isUsed && "Expecting used RTLs."); 380 381 if (!R->is_valid_binary(img)) { 382 DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n", 383 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 384 continue; 385 } 386 387 DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n", 388 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 389 390 FoundRTL = R; 391 392 // Execute dtors for static objects if the device has been used, i.e. 393 // if its PendingCtors list has been emptied. 394 for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) { 395 DeviceTy &Device = PM->Devices[FoundRTL->Idx + i]; 396 Device.PendingGlobalsMtx.lock(); 397 if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) { 398 for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) { 399 int rc = 400 target(nullptr, Device.DeviceID, dtor, 0, nullptr, nullptr, 401 nullptr, nullptr, nullptr, nullptr, 1, 1, true /*team*/); 402 if (rc != OFFLOAD_SUCCESS) { 403 DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor)); 404 } 405 } 406 // Remove this library's entry from PendingCtorsDtors 407 Device.PendingCtorsDtors.erase(desc); 408 } 409 Device.PendingGlobalsMtx.unlock(); 410 } 411 412 DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n", 413 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 414 415 break; 416 } 417 418 // if no RTL was found proceed to unregister the next image 419 if (!FoundRTL){ 420 DP("No RTLs in use support the image " DPxMOD "!\n", 421 DPxPTR(img->ImageStart)); 422 } 423 } 424 PM->RTLsMtx.unlock(); 425 DP("Done unregistering images!\n"); 426 427 // Remove entries from PM->HostPtrToTableMap 428 PM->TblMapMtx.lock(); 429 for (__tgt_offload_entry *cur = desc->HostEntriesBegin; 430 cur < desc->HostEntriesEnd; ++cur) { 431 PM->HostPtrToTableMap.erase(cur->addr); 432 } 433 434 // Remove translation table for this descriptor. 435 auto TransTable = PM->HostEntriesBeginToTransTable.find(desc->HostEntriesBegin); 436 if (TransTable != PM->HostEntriesBeginToTransTable.end()) { 437 DP("Removing translation table for descriptor " DPxMOD "\n", 438 DPxPTR(desc->HostEntriesBegin)); 439 PM->HostEntriesBeginToTransTable.erase(TransTable); 440 } else { 441 DP("Translation table for descriptor " DPxMOD " cannot be found, probably " 442 "it has been already removed.\n", DPxPTR(desc->HostEntriesBegin)); 443 } 444 445 PM->TblMapMtx.unlock(); 446 447 // TODO: Remove RTL and the devices it manages if it's not used anymore? 448 // TODO: Write some RTL->unload_image(...) function? 449 450 DP("Done unregistering library!\n"); 451 } 452