1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Functionality for handling RTL plugins. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "device.h" 14 #include "private.h" 15 #include "rtl.h" 16 17 #include <cassert> 18 #include <cstdlib> 19 #include <cstring> 20 #include <dlfcn.h> 21 #include <mutex> 22 #include <string> 23 24 // List of all plugins that can support offloading. 25 static const char *RTLNames[] = { 26 /* PowerPC target */ "libomptarget.rtl.ppc64.so", 27 /* x86_64 target */ "libomptarget.rtl.x86_64.so", 28 /* CUDA target */ "libomptarget.rtl.cuda.so", 29 /* AArch64 target */ "libomptarget.rtl.aarch64.so", 30 /* SX-Aurora VE target */ "libomptarget.rtl.ve.so", 31 /* AMDGPU target */ "libomptarget.rtl.amdgpu.so", 32 }; 33 34 PluginManager *PM; 35 36 __attribute__((constructor(101))) void init() { 37 DP("Init target library!\n"); 38 PM = new PluginManager(); 39 } 40 41 __attribute__((destructor(101))) void deinit() { 42 DP("Deinit target library!\n"); 43 delete PM; 44 } 45 46 void RTLsTy::LoadRTLs() { 47 // Parse environment variable OMP_TARGET_OFFLOAD (if set) 48 PM->TargetOffloadPolicy = 49 (kmp_target_offload_kind_t)__kmpc_get_target_offload(); 50 if (PM->TargetOffloadPolicy == tgt_disabled) { 51 return; 52 } 53 54 DP("Loading RTLs...\n"); 55 56 // Attempt to open all the plugins and, if they exist, check if the interface 57 // is correct and if they are supporting any devices. 58 for (auto *Name : RTLNames) { 59 DP("Loading library '%s'...\n", Name); 60 void *dynlib_handle = dlopen(Name, RTLD_NOW); 61 62 if (!dynlib_handle) { 63 // Library does not exist or cannot be found. 64 DP("Unable to load library '%s': %s!\n", Name, dlerror()); 65 continue; 66 } 67 68 DP("Successfully loaded library '%s'!\n", Name); 69 70 // Retrieve the RTL information from the runtime library. 71 RTLInfoTy R; 72 73 R.LibraryHandler = dynlib_handle; 74 R.isUsed = false; 75 76 #ifdef OMPTARGET_DEBUG 77 R.RTLName = Name; 78 #endif 79 80 if (!(*((void **)&R.is_valid_binary) = 81 dlsym(dynlib_handle, "__tgt_rtl_is_valid_binary"))) 82 continue; 83 if (!(*((void **)&R.number_of_devices) = 84 dlsym(dynlib_handle, "__tgt_rtl_number_of_devices"))) 85 continue; 86 if (!(*((void **)&R.init_device) = 87 dlsym(dynlib_handle, "__tgt_rtl_init_device"))) 88 continue; 89 if (!(*((void **)&R.load_binary) = 90 dlsym(dynlib_handle, "__tgt_rtl_load_binary"))) 91 continue; 92 if (!(*((void **)&R.data_alloc) = 93 dlsym(dynlib_handle, "__tgt_rtl_data_alloc"))) 94 continue; 95 if (!(*((void **)&R.data_submit) = 96 dlsym(dynlib_handle, "__tgt_rtl_data_submit"))) 97 continue; 98 if (!(*((void **)&R.data_retrieve) = 99 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve"))) 100 continue; 101 if (!(*((void **)&R.data_delete) = 102 dlsym(dynlib_handle, "__tgt_rtl_data_delete"))) 103 continue; 104 if (!(*((void **)&R.run_region) = 105 dlsym(dynlib_handle, "__tgt_rtl_run_target_region"))) 106 continue; 107 if (!(*((void **)&R.run_team_region) = 108 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region"))) 109 continue; 110 111 // Optional functions 112 *((void **)&R.init_requires) = 113 dlsym(dynlib_handle, "__tgt_rtl_init_requires"); 114 *((void **)&R.data_submit_async) = 115 dlsym(dynlib_handle, "__tgt_rtl_data_submit_async"); 116 *((void **)&R.data_retrieve_async) = 117 dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async"); 118 *((void **)&R.run_region_async) = 119 dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async"); 120 *((void **)&R.run_team_region_async) = 121 dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async"); 122 *((void **)&R.synchronize) = dlsym(dynlib_handle, "__tgt_rtl_synchronize"); 123 *((void **)&R.data_exchange) = 124 dlsym(dynlib_handle, "__tgt_rtl_data_exchange"); 125 *((void **)&R.data_exchange_async) = 126 dlsym(dynlib_handle, "__tgt_rtl_data_exchange_async"); 127 *((void **)&R.is_data_exchangable) = 128 dlsym(dynlib_handle, "__tgt_rtl_is_data_exchangable"); 129 130 // No devices are supported by this RTL? 131 if (!(R.NumberOfDevices = R.number_of_devices())) { 132 DP("No devices supported in this RTL\n"); 133 continue; 134 } 135 136 DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(), 137 R.NumberOfDevices); 138 139 // The RTL is valid! Will save the information in the RTLs list. 140 AllRTLs.push_back(R); 141 } 142 143 DP("RTLs loaded!\n"); 144 145 return; 146 } 147 148 //////////////////////////////////////////////////////////////////////////////// 149 // Functionality for registering libs 150 151 static void RegisterImageIntoTranslationTable(TranslationTable &TT, 152 RTLInfoTy &RTL, __tgt_device_image *image) { 153 154 // same size, as when we increase one, we also increase the other. 155 assert(TT.TargetsTable.size() == TT.TargetsImages.size() && 156 "We should have as many images as we have tables!"); 157 158 // Resize the Targets Table and Images to accommodate the new targets if 159 // required 160 unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices; 161 162 if (TT.TargetsTable.size() < TargetsTableMinimumSize) { 163 TT.TargetsImages.resize(TargetsTableMinimumSize, 0); 164 TT.TargetsTable.resize(TargetsTableMinimumSize, 0); 165 } 166 167 // Register the image in all devices for this target type. 168 for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) { 169 // If we are changing the image we are also invalidating the target table. 170 if (TT.TargetsImages[RTL.Idx + i] != image) { 171 TT.TargetsImages[RTL.Idx + i] = image; 172 TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table. 173 } 174 } 175 } 176 177 //////////////////////////////////////////////////////////////////////////////// 178 // Functionality for registering Ctors/Dtors 179 180 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc, 181 __tgt_device_image *img, RTLInfoTy *RTL) { 182 183 for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) { 184 DeviceTy &Device = PM->Devices[RTL->Idx + i]; 185 Device.PendingGlobalsMtx.lock(); 186 Device.HasPendingGlobals = true; 187 for (__tgt_offload_entry *entry = img->EntriesBegin; 188 entry != img->EntriesEnd; ++entry) { 189 if (entry->flags & OMP_DECLARE_TARGET_CTOR) { 190 DP("Adding ctor " DPxMOD " to the pending list.\n", 191 DPxPTR(entry->addr)); 192 Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr); 193 } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) { 194 // Dtors are pushed in reverse order so they are executed from end 195 // to beginning when unregistering the library! 196 DP("Adding dtor " DPxMOD " to the pending list.\n", 197 DPxPTR(entry->addr)); 198 Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr); 199 } 200 201 if (entry->flags & OMP_DECLARE_TARGET_LINK) { 202 DP("The \"link\" attribute is not yet supported!\n"); 203 } 204 } 205 Device.PendingGlobalsMtx.unlock(); 206 } 207 } 208 209 void RTLsTy::RegisterRequires(int64_t flags) { 210 // TODO: add more elaborate check. 211 // Minimal check: only set requires flags if previous value 212 // is undefined. This ensures that only the first call to this 213 // function will set the requires flags. All subsequent calls 214 // will be checked for compatibility. 215 assert(flags != OMP_REQ_UNDEFINED && 216 "illegal undefined flag for requires directive!"); 217 if (RequiresFlags == OMP_REQ_UNDEFINED) { 218 RequiresFlags = flags; 219 return; 220 } 221 222 // If multiple compilation units are present enforce 223 // consistency across all of them for require clauses: 224 // - reverse_offload 225 // - unified_address 226 // - unified_shared_memory 227 if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) != 228 (flags & OMP_REQ_REVERSE_OFFLOAD)) { 229 FATAL_MESSAGE0(1, 230 "'#pragma omp requires reverse_offload' not used consistently!"); 231 } 232 if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) != 233 (flags & OMP_REQ_UNIFIED_ADDRESS)) { 234 FATAL_MESSAGE0(1, 235 "'#pragma omp requires unified_address' not used consistently!"); 236 } 237 if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) != 238 (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) { 239 FATAL_MESSAGE0(1, 240 "'#pragma omp requires unified_shared_memory' not used consistently!"); 241 } 242 243 // TODO: insert any other missing checks 244 245 DP("New requires flags %" PRId64 " compatible with existing %" PRId64 "!\n", 246 flags, RequiresFlags); 247 } 248 249 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) { 250 // Attempt to load all plugins available in the system. 251 std::call_once(initFlag, &RTLsTy::LoadRTLs, this); 252 253 PM->RTLsMtx.lock(); 254 // Register the images with the RTLs that understand them, if any. 255 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 256 // Obtain the image. 257 __tgt_device_image *img = &desc->DeviceImages[i]; 258 259 RTLInfoTy *FoundRTL = NULL; 260 261 // Scan the RTLs that have associated images until we find one that supports 262 // the current image. 263 for (auto &R : AllRTLs) { 264 if (!R.is_valid_binary(img)) { 265 DP("Image " DPxMOD " is NOT compatible with RTL %s!\n", 266 DPxPTR(img->ImageStart), R.RTLName.c_str()); 267 continue; 268 } 269 270 DP("Image " DPxMOD " is compatible with RTL %s!\n", 271 DPxPTR(img->ImageStart), R.RTLName.c_str()); 272 273 // If this RTL is not already in use, initialize it. 274 if (!R.isUsed) { 275 // Initialize the device information for the RTL we are about to use. 276 DeviceTy device(&R); 277 size_t Start = PM->Devices.size(); 278 PM->Devices.resize(Start + R.NumberOfDevices, device); 279 for (int32_t device_id = 0; device_id < R.NumberOfDevices; 280 device_id++) { 281 // global device ID 282 PM->Devices[Start + device_id].DeviceID = Start + device_id; 283 // RTL local device ID 284 PM->Devices[Start + device_id].RTLDeviceID = device_id; 285 } 286 287 // Initialize the index of this RTL and save it in the used RTLs. 288 R.Idx = (UsedRTLs.empty()) 289 ? 0 290 : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices; 291 assert((size_t) R.Idx == Start && 292 "RTL index should equal the number of devices used so far."); 293 R.isUsed = true; 294 UsedRTLs.push_back(&R); 295 296 DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx); 297 } 298 299 // Initialize (if necessary) translation table for this library. 300 PM->TrlTblMtx.lock(); 301 if (!PM->HostEntriesBeginToTransTable.count(desc->HostEntriesBegin)) { 302 TranslationTable &TransTable = 303 (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 304 TransTable.HostTable.EntriesBegin = desc->HostEntriesBegin; 305 TransTable.HostTable.EntriesEnd = desc->HostEntriesEnd; 306 } 307 308 // Retrieve translation table for this library. 309 TranslationTable &TransTable = 310 (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin]; 311 312 DP("Registering image " DPxMOD " with RTL %s!\n", 313 DPxPTR(img->ImageStart), R.RTLName.c_str()); 314 RegisterImageIntoTranslationTable(TransTable, R, img); 315 PM->TrlTblMtx.unlock(); 316 FoundRTL = &R; 317 318 // Load ctors/dtors for static objects 319 RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL); 320 321 // if an RTL was found we are done - proceed to register the next image 322 break; 323 } 324 325 if (!FoundRTL) { 326 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart)); 327 } 328 } 329 PM->RTLsMtx.unlock(); 330 331 DP("Done registering entries!\n"); 332 } 333 334 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) { 335 DP("Unloading target library!\n"); 336 337 PM->RTLsMtx.lock(); 338 // Find which RTL understands each image, if any. 339 for (int32_t i = 0; i < desc->NumDeviceImages; ++i) { 340 // Obtain the image. 341 __tgt_device_image *img = &desc->DeviceImages[i]; 342 343 RTLInfoTy *FoundRTL = NULL; 344 345 // Scan the RTLs that have associated images until we find one that supports 346 // the current image. We only need to scan RTLs that are already being used. 347 for (auto *R : UsedRTLs) { 348 349 assert(R->isUsed && "Expecting used RTLs."); 350 351 if (!R->is_valid_binary(img)) { 352 DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n", 353 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 354 continue; 355 } 356 357 DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n", 358 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 359 360 FoundRTL = R; 361 362 // Execute dtors for static objects if the device has been used, i.e. 363 // if its PendingCtors list has been emptied. 364 for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) { 365 DeviceTy &Device = PM->Devices[FoundRTL->Idx + i]; 366 Device.PendingGlobalsMtx.lock(); 367 if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) { 368 for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) { 369 int rc = target(Device.DeviceID, dtor, 0, NULL, NULL, NULL, NULL, 370 NULL, 1, 1, true /*team*/); 371 if (rc != OFFLOAD_SUCCESS) { 372 DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor)); 373 } 374 } 375 // Remove this library's entry from PendingCtorsDtors 376 Device.PendingCtorsDtors.erase(desc); 377 } 378 Device.PendingGlobalsMtx.unlock(); 379 } 380 381 DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n", 382 DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler)); 383 384 break; 385 } 386 387 // if no RTL was found proceed to unregister the next image 388 if (!FoundRTL){ 389 DP("No RTLs in use support the image " DPxMOD "!\n", 390 DPxPTR(img->ImageStart)); 391 } 392 } 393 PM->RTLsMtx.unlock(); 394 DP("Done unregistering images!\n"); 395 396 // Remove entries from PM->HostPtrToTableMap 397 PM->TblMapMtx.lock(); 398 for (__tgt_offload_entry *cur = desc->HostEntriesBegin; 399 cur < desc->HostEntriesEnd; ++cur) { 400 PM->HostPtrToTableMap.erase(cur->addr); 401 } 402 403 // Remove translation table for this descriptor. 404 auto TransTable = PM->HostEntriesBeginToTransTable.find(desc->HostEntriesBegin); 405 if (TransTable != PM->HostEntriesBeginToTransTable.end()) { 406 DP("Removing translation table for descriptor " DPxMOD "\n", 407 DPxPTR(desc->HostEntriesBegin)); 408 PM->HostEntriesBeginToTransTable.erase(TransTable); 409 } else { 410 DP("Translation table for descriptor " DPxMOD " cannot be found, probably " 411 "it has been already removed.\n", DPxPTR(desc->HostEntriesBegin)); 412 } 413 414 PM->TblMapMtx.unlock(); 415 416 // TODO: Remove RTL and the devices it manages if it's not used anymore? 417 // TODO: Write some RTL->unload_image(...) function? 418 419 DP("Done unregistering library!\n"); 420 } 421