1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Functionality for handling RTL plugins. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "device.h" 14 #include "private.h" 15 #include "rtl.h" 16 17 #include <cassert> 18 #include <cstdlib> 19 #include <cstring> 20 #include <dlfcn.h> 21 #include <mutex> 22 #include <string> 23 24 // List of all plugins that can support offloading. 25 static const char *RTLNames[] = { 26 /* PowerPC target */ "libomptarget.rtl.ppc64.so", 27 /* x86_64 target */ "libomptarget.rtl.x86_64.so", 28 /* CUDA target */ "libomptarget.rtl.cuda.so", 29 /* AArch64 target */ "libomptarget.rtl.aarch64.so", 30 /* SX-Aurora VE target */ "libomptarget.rtl.ve.so", 31 /* AMDGPU target */ "libomptarget.rtl.amdgpu.so", 32 }; 33 34 RTLsTy *RTLs; 35 std::mutex *RTLsMtx; 36 37 HostEntriesBeginToTransTableTy *HostEntriesBeginToTransTable; 38 std::mutex *TrlTblMtx; 39 40 HostPtrToTableMapTy *HostPtrToTableMap; 41 std::mutex *TblMapMtx; 42 43 __attribute__((constructor(101))) void init() { 44 DP("Init target library!\n"); 45 RTLs = new RTLsTy(); 46 RTLsMtx = new std::mutex(); 47 HostEntriesBeginToTransTable = new HostEntriesBeginToTransTableTy(); 48 TrlTblMtx = new std::mutex(); 49 HostPtrToTableMap = new HostPtrToTableMapTy(); 50 TblMapMtx = new std::mutex(); 51 } 52 53 __attribute__((destructor(101))) void deinit() { 54 DP("Deinit target library!\n"); 55 delete RTLs; 56 delete RTLsMtx; 57 delete HostEntriesBeginToTransTable; 58 delete TrlTblMtx; 59 delete HostPtrToTableMap; 60 delete TblMapMtx; 61 } 62 63 void RTLsTy::LoadRTLs() { 64 #ifdef OMPTARGET_DEBUG 65 if (char *envStr = getenv("LIBOMPTARGET_DEBUG")) { 66 DebugLevel = std::stoi(envStr); 67 } 68 #endif // OMPTARGET_DEBUG 69 70 // Parse environment variable OMP_TARGET_OFFLOAD (if set) 71 TargetOffloadPolicy = (kmp_target_offload_kind_t) __kmpc_get_target_offload(); 72 if (TargetOffloadPolicy == tgt_disabled) { 73 return; 74 } 75 76 DP("Loading RTLs...\n"); 77 78 // Attempt to open all the plugins and, if they exist, check if the interface 79 // is correct and if they are supporting any devices. 80 for (auto *Name : RTLNames) { 81 DP("Loading library '%s'...\n", Name); 82 void *dynlib_handle = dlopen(Name, RTLD_NOW); 83 84 if (!dynlib_handle) { 85 // Library does not exist or cannot be found. 86 DP("Unable to load library '%s': %s!\n", Name, dlerror()); 87 continue; 88 } 89 90 DP("Successfully loaded library '%s'!\n", Name); 91 92 // Retrieve the RTL information from the runtime library. 93 RTLInfoTy R; 94 95 R.LibraryHandler = dynlib_handle; 96 R.isUsed = false; 97 98 #ifdef OMPTARGET_DEBUG 99 R.RTLName = Name; 100 #endif 101 102 if (!(*((void **)&R.is_valid_binary) = 103 dlsym(dynlib_handle, "__tgt_rtl_is_valid_binary"))) 104 continue; 105 if (!(*((void **)&R.number_of_devices) = 106 dlsym(dynlib_handle, "__tgt_rtl_number_of_devices"))) 107 continue; 108 if (!(*((void **)&R.init_device) = 109 dlsym(dynlib_handle, "__tgt_rtl_init_device"))) 110 continue; 111 if (!(*((void **)&R.load_binary) = 112 dlsym(dynlib_handle, "__tgt_rtl_load_binary"))) 113 continue; 114 if (!(*((void **)&R.data_alloc) = 115 dlsym(dynlib_handle, "__tgt_rtl_data_alloc"))) 116 continue; 117 if (!(*((void **)&R.data_submit) = 118 dlsym(dynlib_handle, "__tgt_rtl_data_submit"))) 119 continue; 120 if (!(*((void **)&R.data_retrieve) = 121 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve"))) 122 continue; 123 if (!(*((void **)&R.data_delete) = 124 dlsym(dynlib_handle, "__tgt_rtl_data_delete"))) 125 continue; 126 if (!(*((void **)&R.run_region) = 127 dlsym(dynlib_handle, "__tgt_rtl_run_target_region"))) 128 continue; 129 if (!(*((void **)&R.run_team_region) = 130 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region"))) 131 continue; 132 133 // Optional functions 134 *((void **)&R.init_requires) = 135 dlsym(dynlib_handle, "__tgt_rtl_init_requires"); 136 *((void **)&R.data_submit_async) = 137 dlsym(dynlib_handle, "__tgt_rtl_data_submit_async"); 138 *((void **)&R.data_retrieve_async) = 139 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async"); 140 *((void **)&R.run_region_async) = 141 dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async"); 142 *((void **)&R.run_team_region_async) = 143 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async"); 144 *((void **)&R.synchronize) = dlsym(dynlib_handle, "__tgt_rtl_synchronize"); 145 *((void **)&R.data_exchange) = 146 dlsym(dynlib_handle, "__tgt_rtl_data_exchange"); 147 *((void **)&R.data_exchange_async) = 148 dlsym(dynlib_handle, "__tgt_rtl_data_exchange_async"); 149 *((void **)&R.is_data_exchangable) = 150 dlsym(dynlib_handle, "__tgt_rtl_is_data_exchangable"); 151 152 // No devices are supported by this RTL? 153 if (!(R.NumberOfDevices = R.number_of_devices())) { 154 DP("No devices supported in this RTL\n"); 155 continue; 156 } 157 158 DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(), 159 R.NumberOfDevices); 160 161 // The RTL is valid! Will save the information in the RTLs list. 162 AllRTLs.push_back(R); 163 } 164 165 DP("RTLs loaded!\n"); 166 167 return; 168 } 169 170 //////////////////////////////////////////////////////////////////////////////// 171 // Functionality for registering libs 172 173 static void RegisterImageIntoTranslationTable(TranslationTable &TT, 174 RTLInfoTy &RTL, __tgt_device_image *image) { 175 176 // same size, as when we increase one, we also increase the other. 177 assert(TT.TargetsTable.size() == TT.TargetsImages.size() && 178 "We should have as many images as we have tables!"); 179 180 // Resize the Targets Table and Images to accommodate the new targets if 181 // required 182 unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices; 183 184 if (TT.TargetsTable.size() < TargetsTableMinimumSize) { 185 TT.TargetsImages.resize(TargetsTableMinimumSize, 0); 186 TT.TargetsTable.resize(TargetsTableMinimumSize, 0); 187 } 188 189 // Register the image in all devices for this target type. 190 for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) { 191 // If we are changing the image we are also invalidating the target table. 192 if (TT.TargetsImages[RTL.Idx + i] != image) { 193 TT.TargetsImages[RTL.Idx + i] = image; 194 TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table. 195 } 196 } 197 } 198 199 //////////////////////////////////////////////////////////////////////////////// 200 // Functionality for registering Ctors/Dtors 201 202 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc, 203 __tgt_device_image *img, RTLInfoTy *RTL) { 204 205 for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) { 206 DeviceTy &Device = Devices[RTL->Idx + i]; 207 Device.PendingGlobalsMtx.lock(); 208 Device.HasPendingGlobals = true; 209 for (__tgt_offload_entry *entry = img->EntriesBegin; 210 entry != img->EntriesEnd; ++entry) { 211 if (entry->flags & OMP_DECLARE_TARGET_CTOR) { 212 DP("Adding ctor " DPxMOD " to the pending list.\n", 213 DPxPTR(entry->addr)); 214 Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr); 215 } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) { 216 // Dtors are pushed in reverse order so they are executed from end 217 // to beginning when unregistering the library! 218 DP("Adding dtor " DPxMOD " to the pending list.\n", 219 DPxPTR(entry->addr)); 220 Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr); 221 } 222 223 if (entry->flags & OMP_DECLARE_TARGET_LINK) { 224 DP("The \"link\" attribute is not yet supported!\n"); 225 } 226 } 227 Device.PendingGlobalsMtx.unlock(); 228 } 229 } 230 231 void RTLsTy::RegisterRequires(int64_t flags) { 232 // TODO: add more elaborate check. 233 // Minimal check: only set requires flags if previous value 234 // is undefined. This ensures that only the first call to this 235 // function will set the requires flags. All subsequent calls 236 // will be checked for compatibility. 237 assert(flags != OMP_REQ_UNDEFINED && 238 "illegal undefined flag for requires directive!"); 239 if (RequiresFlags == OMP_REQ_UNDEFINED) { 240 RequiresFlags = flags; 241 return; 242 } 243 244 // If multiple compilation units are present enforce 245 // consistency across all of them for require clauses: 246 // - reverse_offload 247 // - unified_address 248 // - unified_shared_memory 249 if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) != 250 (flags & OMP_REQ_REVERSE_OFFLOAD)) { 251 FATAL_MESSAGE0(1, 252 "'#pragma omp requires reverse_offload' not used consistently!"); 253 } 254 if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) != 255 (flags & OMP_REQ_UNIFIED_ADDRESS)) { 256 FATAL_MESSAGE0(1, 257 "'#pragma omp requires unified_address' not used consistently!"); 258 } 259 if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) != 260 (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) { 261 FATAL_MESSAGE0(1, 262 "'#pragma omp requires unified_shared_memory' not used consistently!"); 263 } 264 265 // TODO: insert any other missing checks 266 267 DP("New requires flags %" PRId64 " compatible with existing %" PRId64 "!\n", 268 flags, RequiresFlags); 269 } 270 271 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) { 272 // Attempt to load all plugins available in the system. 273 std::call_once(initFlag, &RTLsTy::LoadRTLs, this); 274 275 RTLsMtx->lock(); 276 // Register the images with the RTLs that understand them, if any. 277 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 278 // Obtain the image. 279 __tgt_device_image *img = &desc->DeviceImages[i]; 280 281 RTLInfoTy *FoundRTL = NULL; 282 283 // Scan the RTLs that have associated images until we find one that supports 284 // the current image. 285 for (auto &R : AllRTLs) { 286 if (!R.is_valid_binary(img)) { 287 DP("Image " DPxMOD " is NOT compatible with RTL %s!\n", 288 DPxPTR(img->ImageStart), R.RTLName.c_str()); 289 continue; 290 } 291 292 DP("Image " DPxMOD " is compatible with RTL %s!\n", 293 DPxPTR(img->ImageStart), R.RTLName.c_str()); 294 295 // If this RTL is not already in use, initialize it. 296 if (!R.isUsed) { 297 // Initialize the device information for the RTL we are about to use. 298 DeviceTy device(&R); 299 size_t start = Devices.size(); 300 Devices.resize(start + R.NumberOfDevices, device); 301 for (int32_t device_id = 0; device_id < R.NumberOfDevices; 302 device_id++) { 303 // global device ID 304 Devices[start + device_id].DeviceID = start + device_id; 305 // RTL local device ID 306 Devices[start + device_id].RTLDeviceID = device_id; 307 } 308 309 // Initialize the index of this RTL and save it in the used RTLs. 310 R.Idx = (UsedRTLs.empty()) 311 ? 0 312 : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices; 313 assert((size_t) R.Idx == start && 314 "RTL index should equal the number of devices used so far."); 315 R.isUsed = true; 316 UsedRTLs.push_back(&R); 317 318 DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx); 319 } 320 321 // Initialize (if necessary) translation table for this library. 322 TrlTblMtx->lock(); 323 if(!HostEntriesBeginToTransTable->count(desc->HostEntriesBegin)){ 324 TranslationTable &tt = 325 (*HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 326 tt.HostTable.EntriesBegin = desc->HostEntriesBegin; 327 tt.HostTable.EntriesEnd = desc->HostEntriesEnd; 328 } 329 330 // Retrieve translation table for this library. 331 TranslationTable &TransTable = 332 (*HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 333 334 DP("Registering image " DPxMOD " with RTL %s!\n", 335 DPxPTR(img->ImageStart), R.RTLName.c_str()); 336 RegisterImageIntoTranslationTable(TransTable, R, img); 337 TrlTblMtx->unlock(); 338 FoundRTL = &R; 339 340 // Load ctors/dtors for static objects 341 RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL); 342 343 // if an RTL was found we are done - proceed to register the next image 344 break; 345 } 346 347 if (!FoundRTL) { 348 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart)); 349 } 350 } 351 RTLsMtx->unlock(); 352 353 354 DP("Done registering entries!\n"); 355 } 356 357 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) { 358 DP("Unloading target library!\n"); 359 360 RTLsMtx->lock(); 361 // Find which RTL understands each image, if any. 362 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 363 // Obtain the image. 364 __tgt_device_image *img = &desc->DeviceImages[i]; 365 366 RTLInfoTy *FoundRTL = NULL; 367 368 // Scan the RTLs that have associated images until we find one that supports 369 // the current image. We only need to scan RTLs that are already being used. 370 for (auto *R : UsedRTLs) { 371 372 assert(R->isUsed && "Expecting used RTLs."); 373 374 if (!R->is_valid_binary(img)) { 375 DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n", 376 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 377 continue; 378 } 379 380 DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n", 381 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 382 383 FoundRTL = R; 384 385 // Execute dtors for static objects if the device has been used, i.e. 386 // if its PendingCtors list has been emptied. 387 for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) { 388 DeviceTy &Device = Devices[FoundRTL->Idx + i]; 389 Device.PendingGlobalsMtx.lock(); 390 if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) { 391 for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) { 392 int rc = target(Device.DeviceID, dtor, 0, NULL, NULL, NULL, NULL, 393 NULL, 1, 1, true /*team*/); 394 if (rc != OFFLOAD_SUCCESS) { 395 DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor)); 396 } 397 } 398 // Remove this library's entry from PendingCtorsDtors 399 Device.PendingCtorsDtors.erase(desc); 400 } 401 Device.PendingGlobalsMtx.unlock(); 402 } 403 404 DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n", 405 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 406 407 break; 408 } 409 410 // if no RTL was found proceed to unregister the next image 411 if (!FoundRTL){ 412 DP("No RTLs in use support the image " DPxMOD "!\n", 413 DPxPTR(img->ImageStart)); 414 } 415 } 416 RTLsMtx->unlock(); 417 DP("Done unregistering images!\n"); 418 419 // Remove entries from HostPtrToTableMap 420 TblMapMtx->lock(); 421 for (__tgt_offload_entry *cur = desc->HostEntriesBegin; 422 cur < desc->HostEntriesEnd; ++cur) { 423 HostPtrToTableMap->erase(cur->addr); 424 } 425 426 // Remove translation table for this descriptor. 427 auto tt = HostEntriesBeginToTransTable->find(desc->HostEntriesBegin); 428 if (tt != HostEntriesBeginToTransTable->end()) { 429 DP("Removing translation table for descriptor " DPxMOD "\n", 430 DPxPTR(desc->HostEntriesBegin)); 431 HostEntriesBeginToTransTable->erase(tt); 432 } else { 433 DP("Translation table for descriptor " DPxMOD " cannot be found, probably " 434 "it has been already removed.\n", DPxPTR(desc->HostEntriesBegin)); 435 } 436 437 TblMapMtx->unlock(); 438 439 // TODO: Remove RTL and the devices it manages if it's not used anymore? 440 // TODO: Write some RTL->unload_image(...) function? 441 442 DP("Done unregistering library!\n"); 443 } 444