1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Functionality for handling RTL plugins. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "device.h" 14 #include "private.h" 15 #include "rtl.h" 16 17 #include <cassert> 18 #include <cstdlib> 19 #include <cstring> 20 #include <dlfcn.h> 21 #include <mutex> 22 #include <string> 23 24 // List of all plugins that can support offloading. 25 static const char *RTLNames[] = { 26 /* SX-Aurora VE target */ "libomptarget.rtl.ve.so", 27 /* PowerPC target */ "libomptarget.rtl.ppc64.so", 28 /* x86_64 target */ "libomptarget.rtl.x86_64.so", 29 /* CUDA target */ "libomptarget.rtl.cuda.so", 30 /* AArch64 target */ "libomptarget.rtl.aarch64.so"}; 31 32 RTLsTy *RTLs; 33 std::mutex *RTLsMtx; 34 35 HostEntriesBeginToTransTableTy *HostEntriesBeginToTransTable; 36 std::mutex *TrlTblMtx; 37 38 HostPtrToTableMapTy *HostPtrToTableMap; 39 std::mutex *TblMapMtx; 40 41 __attribute__((constructor(101))) void init() { 42 DP("Init target library!\n"); 43 RTLs = new RTLsTy(); 44 RTLsMtx = new std::mutex(); 45 HostEntriesBeginToTransTable = new HostEntriesBeginToTransTableTy(); 46 TrlTblMtx = new std::mutex(); 47 HostPtrToTableMap = new HostPtrToTableMapTy(); 48 TblMapMtx = new std::mutex(); 49 } 50 51 __attribute__((destructor(101))) void deinit() { 52 DP("Deinit target library!\n"); 53 delete RTLs; 54 delete RTLsMtx; 55 delete HostEntriesBeginToTransTable; 56 delete TrlTblMtx; 57 delete HostPtrToTableMap; 58 delete TblMapMtx; 59 } 60 61 void RTLsTy::LoadRTLs() { 62 #ifdef OMPTARGET_DEBUG 63 if (char *envStr = getenv("LIBOMPTARGET_DEBUG")) { 64 DebugLevel = std::stoi(envStr); 65 } 66 #endif // OMPTARGET_DEBUG 67 68 // Parse environment variable OMP_TARGET_OFFLOAD (if set) 69 TargetOffloadPolicy = (kmp_target_offload_kind_t) __kmpc_get_target_offload(); 70 if (TargetOffloadPolicy == tgt_disabled) { 71 return; 72 } 73 74 DP("Loading RTLs...\n"); 75 76 // Attempt to open all the plugins and, if they exist, check if the interface 77 // is correct and if they are supporting any devices. 78 for (auto *Name : RTLNames) { 79 DP("Loading library '%s'...\n", Name); 80 void *dynlib_handle = dlopen(Name, RTLD_NOW); 81 82 if (!dynlib_handle) { 83 // Library does not exist or cannot be found. 84 DP("Unable to load library '%s': %s!\n", Name, dlerror()); 85 continue; 86 } 87 88 DP("Successfully loaded library '%s'!\n", Name); 89 90 // Retrieve the RTL information from the runtime library. 91 RTLInfoTy R; 92 93 R.LibraryHandler = dynlib_handle; 94 R.isUsed = false; 95 96 #ifdef OMPTARGET_DEBUG 97 R.RTLName = Name; 98 #endif 99 100 if (!(*((void **)&R.is_valid_binary) = 101 dlsym(dynlib_handle, "__tgt_rtl_is_valid_binary"))) 102 continue; 103 if (!(*((void **)&R.number_of_devices) = 104 dlsym(dynlib_handle, "__tgt_rtl_number_of_devices"))) 105 continue; 106 if (!(*((void **)&R.init_device) = 107 dlsym(dynlib_handle, "__tgt_rtl_init_device"))) 108 continue; 109 if (!(*((void **)&R.load_binary) = 110 dlsym(dynlib_handle, "__tgt_rtl_load_binary"))) 111 continue; 112 if (!(*((void **)&R.data_alloc) = 113 dlsym(dynlib_handle, "__tgt_rtl_data_alloc"))) 114 continue; 115 if (!(*((void **)&R.data_submit) = 116 dlsym(dynlib_handle, "__tgt_rtl_data_submit"))) 117 continue; 118 if (!(*((void **)&R.data_retrieve) = 119 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve"))) 120 continue; 121 if (!(*((void **)&R.data_delete) = 122 dlsym(dynlib_handle, "__tgt_rtl_data_delete"))) 123 continue; 124 if (!(*((void **)&R.run_region) = 125 dlsym(dynlib_handle, "__tgt_rtl_run_target_region"))) 126 continue; 127 if (!(*((void **)&R.run_team_region) = 128 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region"))) 129 continue; 130 131 // Optional functions 132 *((void **)&R.init_requires) = 133 dlsym(dynlib_handle, "__tgt_rtl_init_requires"); 134 *((void **)&R.data_submit_async) = 135 dlsym(dynlib_handle, "__tgt_rtl_data_submit_async"); 136 *((void **)&R.data_retrieve_async) = 137 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async"); 138 *((void **)&R.run_region_async) = 139 dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async"); 140 *((void **)&R.run_team_region_async) = 141 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async"); 142 *((void **)&R.synchronize) = dlsym(dynlib_handle, "__tgt_rtl_synchronize"); 143 *((void **)&R.data_exchange) = 144 dlsym(dynlib_handle, "__tgt_rtl_data_exchange"); 145 *((void **)&R.data_exchange_async) = 146 dlsym(dynlib_handle, "__tgt_rtl_data_exchange_async"); 147 *((void **)&R.is_data_exchangable) = 148 dlsym(dynlib_handle, "__tgt_rtl_is_data_exchangable"); 149 150 // No devices are supported by this RTL? 151 if (!(R.NumberOfDevices = R.number_of_devices())) { 152 DP("No devices supported in this RTL\n"); 153 continue; 154 } 155 156 DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(), 157 R.NumberOfDevices); 158 159 // The RTL is valid! Will save the information in the RTLs list. 160 AllRTLs.push_back(R); 161 } 162 163 DP("RTLs loaded!\n"); 164 165 return; 166 } 167 168 //////////////////////////////////////////////////////////////////////////////// 169 // Functionality for registering libs 170 171 static void RegisterImageIntoTranslationTable(TranslationTable &TT, 172 RTLInfoTy &RTL, __tgt_device_image *image) { 173 174 // same size, as when we increase one, we also increase the other. 175 assert(TT.TargetsTable.size() == TT.TargetsImages.size() && 176 "We should have as many images as we have tables!"); 177 178 // Resize the Targets Table and Images to accommodate the new targets if 179 // required 180 unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices; 181 182 if (TT.TargetsTable.size() < TargetsTableMinimumSize) { 183 TT.TargetsImages.resize(TargetsTableMinimumSize, 0); 184 TT.TargetsTable.resize(TargetsTableMinimumSize, 0); 185 } 186 187 // Register the image in all devices for this target type. 188 for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) { 189 // If we are changing the image we are also invalidating the target table. 190 if (TT.TargetsImages[RTL.Idx + i] != image) { 191 TT.TargetsImages[RTL.Idx + i] = image; 192 TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table. 193 } 194 } 195 } 196 197 //////////////////////////////////////////////////////////////////////////////// 198 // Functionality for registering Ctors/Dtors 199 200 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc, 201 __tgt_device_image *img, RTLInfoTy *RTL) { 202 203 for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) { 204 DeviceTy &Device = Devices[RTL->Idx + i]; 205 Device.PendingGlobalsMtx.lock(); 206 Device.HasPendingGlobals = true; 207 for (__tgt_offload_entry *entry = img->EntriesBegin; 208 entry != img->EntriesEnd; ++entry) { 209 if (entry->flags & OMP_DECLARE_TARGET_CTOR) { 210 DP("Adding ctor " DPxMOD " to the pending list.\n", 211 DPxPTR(entry->addr)); 212 Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr); 213 } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) { 214 // Dtors are pushed in reverse order so they are executed from end 215 // to beginning when unregistering the library! 216 DP("Adding dtor " DPxMOD " to the pending list.\n", 217 DPxPTR(entry->addr)); 218 Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr); 219 } 220 221 if (entry->flags & OMP_DECLARE_TARGET_LINK) { 222 DP("The \"link\" attribute is not yet supported!\n"); 223 } 224 } 225 Device.PendingGlobalsMtx.unlock(); 226 } 227 } 228 229 void RTLsTy::RegisterRequires(int64_t flags) { 230 // TODO: add more elaborate check. 231 // Minimal check: only set requires flags if previous value 232 // is undefined. This ensures that only the first call to this 233 // function will set the requires flags. All subsequent calls 234 // will be checked for compatibility. 235 assert(flags != OMP_REQ_UNDEFINED && 236 "illegal undefined flag for requires directive!"); 237 if (RequiresFlags == OMP_REQ_UNDEFINED) { 238 RequiresFlags = flags; 239 return; 240 } 241 242 // If multiple compilation units are present enforce 243 // consistency across all of them for require clauses: 244 // - reverse_offload 245 // - unified_address 246 // - unified_shared_memory 247 if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) != 248 (flags & OMP_REQ_REVERSE_OFFLOAD)) { 249 FATAL_MESSAGE0(1, 250 "'#pragma omp requires reverse_offload' not used consistently!"); 251 } 252 if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) != 253 (flags & OMP_REQ_UNIFIED_ADDRESS)) { 254 FATAL_MESSAGE0(1, 255 "'#pragma omp requires unified_address' not used consistently!"); 256 } 257 if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) != 258 (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) { 259 FATAL_MESSAGE0(1, 260 "'#pragma omp requires unified_shared_memory' not used consistently!"); 261 } 262 263 // TODO: insert any other missing checks 264 265 DP("New requires flags %" PRId64 " compatible with existing %" PRId64 "!\n", 266 flags, RequiresFlags); 267 } 268 269 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) { 270 // Attempt to load all plugins available in the system. 271 std::call_once(initFlag, &RTLsTy::LoadRTLs, this); 272 273 RTLsMtx->lock(); 274 // Register the images with the RTLs that understand them, if any. 275 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 276 // Obtain the image. 277 __tgt_device_image *img = &desc->DeviceImages[i]; 278 279 RTLInfoTy *FoundRTL = NULL; 280 281 // Scan the RTLs that have associated images until we find one that supports 282 // the current image. 283 for (auto &R : AllRTLs) { 284 if (!R.is_valid_binary(img)) { 285 DP("Image " DPxMOD " is NOT compatible with RTL %s!\n", 286 DPxPTR(img->ImageStart), R.RTLName.c_str()); 287 continue; 288 } 289 290 DP("Image " DPxMOD " is compatible with RTL %s!\n", 291 DPxPTR(img->ImageStart), R.RTLName.c_str()); 292 293 // If this RTL is not already in use, initialize it. 294 if (!R.isUsed) { 295 // Initialize the device information for the RTL we are about to use. 296 DeviceTy device(&R); 297 size_t start = Devices.size(); 298 Devices.resize(start + R.NumberOfDevices, device); 299 for (int32_t device_id = 0; device_id < R.NumberOfDevices; 300 device_id++) { 301 // global device ID 302 Devices[start + device_id].DeviceID = start + device_id; 303 // RTL local device ID 304 Devices[start + device_id].RTLDeviceID = device_id; 305 } 306 307 // Initialize the index of this RTL and save it in the used RTLs. 308 R.Idx = (UsedRTLs.empty()) 309 ? 0 310 : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices; 311 assert((size_t) R.Idx == start && 312 "RTL index should equal the number of devices used so far."); 313 R.isUsed = true; 314 UsedRTLs.push_back(&R); 315 316 DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx); 317 } 318 319 // Initialize (if necessary) translation table for this library. 320 TrlTblMtx->lock(); 321 if(!HostEntriesBeginToTransTable->count(desc->HostEntriesBegin)){ 322 TranslationTable &tt = 323 (*HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 324 tt.HostTable.EntriesBegin = desc->HostEntriesBegin; 325 tt.HostTable.EntriesEnd = desc->HostEntriesEnd; 326 } 327 328 // Retrieve translation table for this library. 329 TranslationTable &TransTable = 330 (*HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 331 332 DP("Registering image " DPxMOD " with RTL %s!\n", 333 DPxPTR(img->ImageStart), R.RTLName.c_str()); 334 RegisterImageIntoTranslationTable(TransTable, R, img); 335 TrlTblMtx->unlock(); 336 FoundRTL = &R; 337 338 // Load ctors/dtors for static objects 339 RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL); 340 341 // if an RTL was found we are done - proceed to register the next image 342 break; 343 } 344 345 if (!FoundRTL) { 346 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart)); 347 } 348 } 349 RTLsMtx->unlock(); 350 351 352 DP("Done registering entries!\n"); 353 } 354 355 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) { 356 DP("Unloading target library!\n"); 357 358 RTLsMtx->lock(); 359 // Find which RTL understands each image, if any. 360 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 361 // Obtain the image. 362 __tgt_device_image *img = &desc->DeviceImages[i]; 363 364 RTLInfoTy *FoundRTL = NULL; 365 366 // Scan the RTLs that have associated images until we find one that supports 367 // the current image. We only need to scan RTLs that are already being used. 368 for (auto *R : UsedRTLs) { 369 370 assert(R->isUsed && "Expecting used RTLs."); 371 372 if (!R->is_valid_binary(img)) { 373 DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n", 374 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 375 continue; 376 } 377 378 DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n", 379 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 380 381 FoundRTL = R; 382 383 // Execute dtors for static objects if the device has been used, i.e. 384 // if its PendingCtors list has been emptied. 385 for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) { 386 DeviceTy &Device = Devices[FoundRTL->Idx + i]; 387 Device.PendingGlobalsMtx.lock(); 388 if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) { 389 for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) { 390 int rc = target(Device.DeviceID, dtor, 0, NULL, NULL, NULL, NULL, 391 NULL, 1, 1, true /*team*/); 392 if (rc != OFFLOAD_SUCCESS) { 393 DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor)); 394 } 395 } 396 // Remove this library's entry from PendingCtorsDtors 397 Device.PendingCtorsDtors.erase(desc); 398 } 399 Device.PendingGlobalsMtx.unlock(); 400 } 401 402 DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n", 403 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 404 405 break; 406 } 407 408 // if no RTL was found proceed to unregister the next image 409 if (!FoundRTL){ 410 DP("No RTLs in use support the image " DPxMOD "!\n", 411 DPxPTR(img->ImageStart)); 412 } 413 } 414 RTLsMtx->unlock(); 415 DP("Done unregistering images!\n"); 416 417 // Remove entries from HostPtrToTableMap 418 TblMapMtx->lock(); 419 for (__tgt_offload_entry *cur = desc->HostEntriesBegin; 420 cur < desc->HostEntriesEnd; ++cur) { 421 HostPtrToTableMap->erase(cur->addr); 422 } 423 424 // Remove translation table for this descriptor. 425 auto tt = HostEntriesBeginToTransTable->find(desc->HostEntriesBegin); 426 if (tt != HostEntriesBeginToTransTable->end()) { 427 DP("Removing translation table for descriptor " DPxMOD "\n", 428 DPxPTR(desc->HostEntriesBegin)); 429 HostEntriesBeginToTransTable->erase(tt); 430 } else { 431 DP("Translation table for descriptor " DPxMOD " cannot be found, probably " 432 "it has been already removed.\n", DPxPTR(desc->HostEntriesBegin)); 433 } 434 435 TblMapMtx->unlock(); 436 437 // TODO: Remove RTL and the devices it manages if it's not used anymore? 438 // TODO: Write some RTL->unload_image(...) function? 439 440 DP("Done unregistering library!\n"); 441 } 442