1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Functionality for handling RTL plugins. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "device.h" 14 #include "private.h" 15 #include "rtl.h" 16 17 #include <cassert> 18 #include <cstdlib> 19 #include <cstring> 20 #include <dlfcn.h> 21 #include <mutex> 22 #include <string> 23 24 // List of all plugins that can support offloading. 25 static const char *RTLNames[] = { 26 /* PowerPC target */ "libomptarget.rtl.ppc64.so", 27 /* x86_64 target */ "libomptarget.rtl.x86_64.so", 28 /* CUDA target */ "libomptarget.rtl.cuda.so", 29 /* AArch64 target */ "libomptarget.rtl.aarch64.so"}; 30 31 RTLsTy RTLs; 32 std::mutex RTLsMtx; 33 34 HostEntriesBeginToTransTableTy HostEntriesBeginToTransTable; 35 std::mutex TrlTblMtx; 36 37 HostPtrToTableMapTy HostPtrToTableMap; 38 std::mutex TblMapMtx; 39 40 void RTLsTy::LoadRTLs() { 41 #ifdef OMPTARGET_DEBUG 42 if (char *envStr = getenv("LIBOMPTARGET_DEBUG")) { 43 DebugLevel = std::stoi(envStr); 44 } 45 #endif // OMPTARGET_DEBUG 46 47 // Parse environment variable OMP_TARGET_OFFLOAD (if set) 48 TargetOffloadPolicy = (kmp_target_offload_kind_t) __kmpc_get_target_offload(); 49 if (TargetOffloadPolicy == tgt_disabled) { 50 return; 51 } 52 53 DP("Loading RTLs...\n"); 54 55 // Attempt to open all the plugins and, if they exist, check if the interface 56 // is correct and if they are supporting any devices. 57 for (auto *Name : RTLNames) { 58 DP("Loading library '%s'...\n", Name); 59 void *dynlib_handle = dlopen(Name, RTLD_NOW); 60 61 if (!dynlib_handle) { 62 // Library does not exist or cannot be found. 63 DP("Unable to load library '%s': %s!\n", Name, dlerror()); 64 continue; 65 } 66 67 DP("Successfully loaded library '%s'!\n", Name); 68 69 // Retrieve the RTL information from the runtime library. 70 RTLInfoTy R; 71 72 R.LibraryHandler = dynlib_handle; 73 R.isUsed = false; 74 75 #ifdef OMPTARGET_DEBUG 76 R.RTLName = Name; 77 #endif 78 79 if (!(*((void**) &R.is_valid_binary) = dlsym( 80 dynlib_handle, "__tgt_rtl_is_valid_binary"))) 81 continue; 82 if (!(*((void**) &R.number_of_devices) = dlsym( 83 dynlib_handle, "__tgt_rtl_number_of_devices"))) 84 continue; 85 if (!(*((void**) &R.init_device) = dlsym( 86 dynlib_handle, "__tgt_rtl_init_device"))) 87 continue; 88 if (!(*((void**) &R.load_binary) = dlsym( 89 dynlib_handle, "__tgt_rtl_load_binary"))) 90 continue; 91 if (!(*((void**) &R.data_alloc) = dlsym( 92 dynlib_handle, "__tgt_rtl_data_alloc"))) 93 continue; 94 if (!(*((void**) &R.data_submit) = dlsym( 95 dynlib_handle, "__tgt_rtl_data_submit"))) 96 continue; 97 if (!(*((void**) &R.data_retrieve) = dlsym( 98 dynlib_handle, "__tgt_rtl_data_retrieve"))) 99 continue; 100 if (!(*((void**) &R.data_delete) = dlsym( 101 dynlib_handle, "__tgt_rtl_data_delete"))) 102 continue; 103 if (!(*((void**) &R.run_region) = dlsym( 104 dynlib_handle, "__tgt_rtl_run_target_region"))) 105 continue; 106 if (!(*((void**) &R.run_team_region) = dlsym( 107 dynlib_handle, "__tgt_rtl_run_target_team_region"))) 108 continue; 109 110 // Optional functions 111 *((void**) &R.init_requires) = dlsym( 112 dynlib_handle, "__tgt_rtl_init_requires"); 113 114 // No devices are supported by this RTL? 115 if (!(R.NumberOfDevices = R.number_of_devices())) { 116 DP("No devices supported in this RTL\n"); 117 continue; 118 } 119 120 DP("Registering RTL %s supporting %d devices!\n", 121 R.RTLName.c_str(), R.NumberOfDevices); 122 123 // The RTL is valid! Will save the information in the RTLs list. 124 AllRTLs.push_back(R); 125 } 126 127 DP("RTLs loaded!\n"); 128 129 return; 130 } 131 132 //////////////////////////////////////////////////////////////////////////////// 133 // Functionality for registering libs 134 135 static void RegisterImageIntoTranslationTable(TranslationTable &TT, 136 RTLInfoTy &RTL, __tgt_device_image *image) { 137 138 // same size, as when we increase one, we also increase the other. 139 assert(TT.TargetsTable.size() == TT.TargetsImages.size() && 140 "We should have as many images as we have tables!"); 141 142 // Resize the Targets Table and Images to accommodate the new targets if 143 // required 144 unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices; 145 146 if (TT.TargetsTable.size() < TargetsTableMinimumSize) { 147 TT.TargetsImages.resize(TargetsTableMinimumSize, 0); 148 TT.TargetsTable.resize(TargetsTableMinimumSize, 0); 149 } 150 151 // Register the image in all devices for this target type. 152 for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) { 153 // If we are changing the image we are also invalidating the target table. 154 if (TT.TargetsImages[RTL.Idx + i] != image) { 155 TT.TargetsImages[RTL.Idx + i] = image; 156 TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table. 157 } 158 } 159 } 160 161 //////////////////////////////////////////////////////////////////////////////// 162 // Functionality for registering Ctors/Dtors 163 164 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc, 165 __tgt_device_image *img, RTLInfoTy *RTL) { 166 167 for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) { 168 DeviceTy &Device = Devices[RTL->Idx + i]; 169 Device.PendingGlobalsMtx.lock(); 170 Device.HasPendingGlobals = true; 171 for (__tgt_offload_entry *entry = img->EntriesBegin; 172 entry != img->EntriesEnd; ++entry) { 173 if (entry->flags & OMP_DECLARE_TARGET_CTOR) { 174 DP("Adding ctor " DPxMOD " to the pending list.\n", 175 DPxPTR(entry->addr)); 176 Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr); 177 } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) { 178 // Dtors are pushed in reverse order so they are executed from end 179 // to beginning when unregistering the library! 180 DP("Adding dtor " DPxMOD " to the pending list.\n", 181 DPxPTR(entry->addr)); 182 Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr); 183 } 184 185 if (entry->flags & OMP_DECLARE_TARGET_LINK) { 186 DP("The \"link\" attribute is not yet supported!\n"); 187 } 188 } 189 Device.PendingGlobalsMtx.unlock(); 190 } 191 } 192 193 void RTLsTy::RegisterRequires(int64_t flags) { 194 // TODO: add more elaborate check. 195 // Minimal check: only set requires flags if previous value 196 // is undefined. This ensures that only the first call to this 197 // function will set the requires flags. All subsequent calls 198 // will be checked for compatibility. 199 assert(flags != OMP_REQ_UNDEFINED && 200 "illegal undefined flag for requires directive!"); 201 if (RequiresFlags == OMP_REQ_UNDEFINED) { 202 RequiresFlags = flags; 203 return; 204 } 205 206 // If multiple compilation units are present enforce 207 // consistency across all of them for require clauses: 208 // - reverse_offload 209 // - unified_address 210 // - unified_shared_memory 211 if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) != 212 (flags & OMP_REQ_REVERSE_OFFLOAD)) { 213 FATAL_MESSAGE0(1, 214 "'#pragma omp requires reverse_offload' not used consistently!"); 215 } 216 if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) != 217 (flags & OMP_REQ_UNIFIED_ADDRESS)) { 218 FATAL_MESSAGE0(1, 219 "'#pragma omp requires unified_address' not used consistently!"); 220 } 221 if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) != 222 (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) { 223 FATAL_MESSAGE0(1, 224 "'#pragma omp requires unified_shared_memory' not used consistently!"); 225 } 226 227 // TODO: insert any other missing checks 228 229 DP("New requires flags %ld compatible with existing %ld!\n", 230 flags, RequiresFlags); 231 } 232 233 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) { 234 // Attempt to load all plugins available in the system. 235 std::call_once(initFlag, &RTLsTy::LoadRTLs, this); 236 237 RTLsMtx.lock(); 238 // Register the images with the RTLs that understand them, if any. 239 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 240 // Obtain the image. 241 __tgt_device_image *img = &desc->DeviceImages[i]; 242 243 RTLInfoTy *FoundRTL = NULL; 244 245 // Scan the RTLs that have associated images until we find one that supports 246 // the current image. 247 for (auto &R : RTLs.AllRTLs) { 248 if (!R.is_valid_binary(img)) { 249 DP("Image " DPxMOD " is NOT compatible with RTL %s!\n", 250 DPxPTR(img->ImageStart), R.RTLName.c_str()); 251 continue; 252 } 253 254 DP("Image " DPxMOD " is compatible with RTL %s!\n", 255 DPxPTR(img->ImageStart), R.RTLName.c_str()); 256 257 // If this RTL is not already in use, initialize it. 258 if (!R.isUsed) { 259 // Initialize the device information for the RTL we are about to use. 260 DeviceTy device(&R); 261 size_t start = Devices.size(); 262 Devices.resize(start + R.NumberOfDevices, device); 263 for (int32_t device_id = 0; device_id < R.NumberOfDevices; 264 device_id++) { 265 // global device ID 266 Devices[start + device_id].DeviceID = start + device_id; 267 // RTL local device ID 268 Devices[start + device_id].RTLDeviceID = device_id; 269 } 270 271 // Initialize the index of this RTL and save it in the used RTLs. 272 R.Idx = (RTLs.UsedRTLs.empty()) 273 ? 0 274 : RTLs.UsedRTLs.back()->Idx + 275 RTLs.UsedRTLs.back()->NumberOfDevices; 276 assert((size_t) R.Idx == start && 277 "RTL index should equal the number of devices used so far."); 278 R.isUsed = true; 279 RTLs.UsedRTLs.push_back(&R); 280 281 DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx); 282 } 283 284 // Initialize (if necessary) translation table for this library. 285 TrlTblMtx.lock(); 286 if(!HostEntriesBeginToTransTable.count(desc->HostEntriesBegin)){ 287 TranslationTable &tt = 288 HostEntriesBeginToTransTable[desc->HostEntriesBegin]; 289 tt.HostTable.EntriesBegin = desc->HostEntriesBegin; 290 tt.HostTable.EntriesEnd = desc->HostEntriesEnd; 291 } 292 293 // Retrieve translation table for this library. 294 TranslationTable &TransTable = 295 HostEntriesBeginToTransTable[desc->HostEntriesBegin]; 296 297 DP("Registering image " DPxMOD " with RTL %s!\n", 298 DPxPTR(img->ImageStart), R.RTLName.c_str()); 299 RegisterImageIntoTranslationTable(TransTable, R, img); 300 TrlTblMtx.unlock(); 301 FoundRTL = &R; 302 303 // Load ctors/dtors for static objects 304 RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL); 305 306 // if an RTL was found we are done - proceed to register the next image 307 break; 308 } 309 310 if (!FoundRTL) { 311 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart)); 312 } 313 } 314 RTLsMtx.unlock(); 315 316 317 DP("Done registering entries!\n"); 318 } 319 320 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) { 321 DP("Unloading target library!\n"); 322 323 RTLsMtx.lock(); 324 // Find which RTL understands each image, if any. 325 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 326 // Obtain the image. 327 __tgt_device_image *img = &desc->DeviceImages[i]; 328 329 RTLInfoTy *FoundRTL = NULL; 330 331 // Scan the RTLs that have associated images until we find one that supports 332 // the current image. We only need to scan RTLs that are already being used. 333 for (auto *R : RTLs.UsedRTLs) { 334 335 assert(R->isUsed && "Expecting used RTLs."); 336 337 if (!R->is_valid_binary(img)) { 338 DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n", 339 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 340 continue; 341 } 342 343 DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n", 344 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 345 346 FoundRTL = R; 347 348 // Execute dtors for static objects if the device has been used, i.e. 349 // if its PendingCtors list has been emptied. 350 for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) { 351 DeviceTy &Device = Devices[FoundRTL->Idx + i]; 352 Device.PendingGlobalsMtx.lock(); 353 if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) { 354 for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) { 355 int rc = target(Device.DeviceID, dtor, 0, NULL, NULL, NULL, NULL, 1, 356 1, true /*team*/); 357 if (rc != OFFLOAD_SUCCESS) { 358 DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor)); 359 } 360 } 361 // Remove this library's entry from PendingCtorsDtors 362 Device.PendingCtorsDtors.erase(desc); 363 } 364 Device.PendingGlobalsMtx.unlock(); 365 } 366 367 DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n", 368 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 369 370 break; 371 } 372 373 // if no RTL was found proceed to unregister the next image 374 if (!FoundRTL){ 375 DP("No RTLs in use support the image " DPxMOD "!\n", 376 DPxPTR(img->ImageStart)); 377 } 378 } 379 RTLsMtx.unlock(); 380 DP("Done unregistering images!\n"); 381 382 // Remove entries from HostPtrToTableMap 383 TblMapMtx.lock(); 384 for (__tgt_offload_entry *cur = desc->HostEntriesBegin; 385 cur < desc->HostEntriesEnd; ++cur) { 386 HostPtrToTableMap.erase(cur->addr); 387 } 388 389 // Remove translation table for this descriptor. 390 auto tt = HostEntriesBeginToTransTable.find(desc->HostEntriesBegin); 391 if (tt != HostEntriesBeginToTransTable.end()) { 392 DP("Removing translation table for descriptor " DPxMOD "\n", 393 DPxPTR(desc->HostEntriesBegin)); 394 HostEntriesBeginToTransTable.erase(tt); 395 } else { 396 DP("Translation table for descriptor " DPxMOD " cannot be found, probably " 397 "it has been already removed.\n", DPxPTR(desc->HostEntriesBegin)); 398 } 399 400 TblMapMtx.unlock(); 401 402 // TODO: Remove RTL and the devices it manages if it's not used anymore? 403 // TODO: Write some RTL->unload_image(...) function? 404 405 DP("Done unregistering library!\n"); 406 } 407