1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Functionality for handling RTL plugins.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "device.h"
14 #include "private.h"
15 #include "rtl.h"
16 
17 #include <cassert>
18 #include <cstdlib>
19 #include <cstring>
20 #include <dlfcn.h>
21 #include <mutex>
22 #include <string>
23 
24 // List of all plugins that can support offloading.
25 static const char *RTLNames[] = {
26     /* SX-Aurora VE target  */ "libomptarget.rtl.ve.so",
27     /* PowerPC target */ "libomptarget.rtl.ppc64.so",
28     /* x86_64 target  */ "libomptarget.rtl.x86_64.so",
29     /* CUDA target    */ "libomptarget.rtl.cuda.so",
30     /* AArch64 target */ "libomptarget.rtl.aarch64.so"};
31 
32 RTLsTy *RTLs;
33 std::mutex *RTLsMtx;
34 
35 HostEntriesBeginToTransTableTy *HostEntriesBeginToTransTable;
36 std::mutex *TrlTblMtx;
37 
38 HostPtrToTableMapTy *HostPtrToTableMap;
39 std::mutex *TblMapMtx;
40 
41 __attribute__((constructor(101))) void init() {
42   DP("Init target library!\n");
43   RTLs = new RTLsTy();
44   RTLsMtx = new std::mutex();
45   HostEntriesBeginToTransTable = new HostEntriesBeginToTransTableTy();
46   TrlTblMtx = new std::mutex();
47   HostPtrToTableMap = new HostPtrToTableMapTy();
48   TblMapMtx = new std::mutex();
49 }
50 
51 __attribute__((destructor(101))) void deinit() {
52   DP("Deinit target library!\n");
53   delete RTLs;
54   delete RTLsMtx;
55   delete HostEntriesBeginToTransTable;
56   delete TrlTblMtx;
57   delete HostPtrToTableMap;
58   delete TblMapMtx;
59 }
60 
61 void RTLsTy::LoadRTLs() {
62 #ifdef OMPTARGET_DEBUG
63   if (char *envStr = getenv("LIBOMPTARGET_DEBUG")) {
64     DebugLevel = std::stoi(envStr);
65   }
66 #endif // OMPTARGET_DEBUG
67 
68   // Parse environment variable OMP_TARGET_OFFLOAD (if set)
69   TargetOffloadPolicy = (kmp_target_offload_kind_t) __kmpc_get_target_offload();
70   if (TargetOffloadPolicy == tgt_disabled) {
71     return;
72   }
73 
74   DP("Loading RTLs...\n");
75 
76   // Attempt to open all the plugins and, if they exist, check if the interface
77   // is correct and if they are supporting any devices.
78   for (auto *Name : RTLNames) {
79     DP("Loading library '%s'...\n", Name);
80     void *dynlib_handle = dlopen(Name, RTLD_NOW);
81 
82     if (!dynlib_handle) {
83       // Library does not exist or cannot be found.
84       DP("Unable to load library '%s': %s!\n", Name, dlerror());
85       continue;
86     }
87 
88     DP("Successfully loaded library '%s'!\n", Name);
89 
90     // Retrieve the RTL information from the runtime library.
91     RTLInfoTy R;
92 
93     R.LibraryHandler = dynlib_handle;
94     R.isUsed = false;
95 
96 #ifdef OMPTARGET_DEBUG
97     R.RTLName = Name;
98 #endif
99 
100     if (!(*((void **)&R.is_valid_binary) =
101               dlsym(dynlib_handle, "__tgt_rtl_is_valid_binary")))
102       continue;
103     if (!(*((void **)&R.number_of_devices) =
104               dlsym(dynlib_handle, "__tgt_rtl_number_of_devices")))
105       continue;
106     if (!(*((void **)&R.init_device) =
107               dlsym(dynlib_handle, "__tgt_rtl_init_device")))
108       continue;
109     if (!(*((void **)&R.load_binary) =
110               dlsym(dynlib_handle, "__tgt_rtl_load_binary")))
111       continue;
112     if (!(*((void **)&R.data_alloc) =
113               dlsym(dynlib_handle, "__tgt_rtl_data_alloc")))
114       continue;
115     if (!(*((void **)&R.data_submit) =
116               dlsym(dynlib_handle, "__tgt_rtl_data_submit")))
117       continue;
118     if (!(*((void **)&R.data_retrieve) =
119               dlsym(dynlib_handle, "__tgt_rtl_data_retrieve")))
120       continue;
121     if (!(*((void **)&R.data_delete) =
122               dlsym(dynlib_handle, "__tgt_rtl_data_delete")))
123       continue;
124     if (!(*((void **)&R.run_region) =
125               dlsym(dynlib_handle, "__tgt_rtl_run_target_region")))
126       continue;
127     if (!(*((void **)&R.run_team_region) =
128               dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region")))
129       continue;
130 
131     // Optional functions
132     *((void **)&R.init_requires) =
133         dlsym(dynlib_handle, "__tgt_rtl_init_requires");
134     *((void **)&R.data_submit_async) =
135         dlsym(dynlib_handle, "__tgt_rtl_data_submit_async");
136     *((void **)&R.data_retrieve_async) =
137         dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async");
138     *((void **)&R.run_region_async) =
139         dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async");
140     *((void **)&R.run_team_region_async) =
141         dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async");
142     *((void **)&R.synchronize) = dlsym(dynlib_handle, "__tgt_rtl_synchronize");
143 
144     // No devices are supported by this RTL?
145     if (!(R.NumberOfDevices = R.number_of_devices())) {
146       DP("No devices supported in this RTL\n");
147       continue;
148     }
149 
150     DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(),
151        R.NumberOfDevices);
152 
153     // The RTL is valid! Will save the information in the RTLs list.
154     AllRTLs.push_back(R);
155   }
156 
157   DP("RTLs loaded!\n");
158 
159   return;
160 }
161 
162 ////////////////////////////////////////////////////////////////////////////////
163 // Functionality for registering libs
164 
165 static void RegisterImageIntoTranslationTable(TranslationTable &TT,
166     RTLInfoTy &RTL, __tgt_device_image *image) {
167 
168   // same size, as when we increase one, we also increase the other.
169   assert(TT.TargetsTable.size() == TT.TargetsImages.size() &&
170          "We should have as many images as we have tables!");
171 
172   // Resize the Targets Table and Images to accommodate the new targets if
173   // required
174   unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices;
175 
176   if (TT.TargetsTable.size() < TargetsTableMinimumSize) {
177     TT.TargetsImages.resize(TargetsTableMinimumSize, 0);
178     TT.TargetsTable.resize(TargetsTableMinimumSize, 0);
179   }
180 
181   // Register the image in all devices for this target type.
182   for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) {
183     // If we are changing the image we are also invalidating the target table.
184     if (TT.TargetsImages[RTL.Idx + i] != image) {
185       TT.TargetsImages[RTL.Idx + i] = image;
186       TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table.
187     }
188   }
189 }
190 
191 ////////////////////////////////////////////////////////////////////////////////
192 // Functionality for registering Ctors/Dtors
193 
194 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc,
195     __tgt_device_image *img, RTLInfoTy *RTL) {
196 
197   for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) {
198     DeviceTy &Device = Devices[RTL->Idx + i];
199     Device.PendingGlobalsMtx.lock();
200     Device.HasPendingGlobals = true;
201     for (__tgt_offload_entry *entry = img->EntriesBegin;
202         entry != img->EntriesEnd; ++entry) {
203       if (entry->flags & OMP_DECLARE_TARGET_CTOR) {
204         DP("Adding ctor " DPxMOD " to the pending list.\n",
205             DPxPTR(entry->addr));
206         Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr);
207       } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) {
208         // Dtors are pushed in reverse order so they are executed from end
209         // to beginning when unregistering the library!
210         DP("Adding dtor " DPxMOD " to the pending list.\n",
211             DPxPTR(entry->addr));
212         Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr);
213       }
214 
215       if (entry->flags & OMP_DECLARE_TARGET_LINK) {
216         DP("The \"link\" attribute is not yet supported!\n");
217       }
218     }
219     Device.PendingGlobalsMtx.unlock();
220   }
221 }
222 
223 void RTLsTy::RegisterRequires(int64_t flags) {
224   // TODO: add more elaborate check.
225   // Minimal check: only set requires flags if previous value
226   // is undefined. This ensures that only the first call to this
227   // function will set the requires flags. All subsequent calls
228   // will be checked for compatibility.
229   assert(flags != OMP_REQ_UNDEFINED &&
230          "illegal undefined flag for requires directive!");
231   if (RequiresFlags == OMP_REQ_UNDEFINED) {
232     RequiresFlags = flags;
233     return;
234   }
235 
236   // If multiple compilation units are present enforce
237   // consistency across all of them for require clauses:
238   //  - reverse_offload
239   //  - unified_address
240   //  - unified_shared_memory
241   if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) !=
242       (flags & OMP_REQ_REVERSE_OFFLOAD)) {
243     FATAL_MESSAGE0(1,
244         "'#pragma omp requires reverse_offload' not used consistently!");
245   }
246   if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) !=
247           (flags & OMP_REQ_UNIFIED_ADDRESS)) {
248     FATAL_MESSAGE0(1,
249         "'#pragma omp requires unified_address' not used consistently!");
250   }
251   if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) !=
252           (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) {
253     FATAL_MESSAGE0(1,
254         "'#pragma omp requires unified_shared_memory' not used consistently!");
255   }
256 
257   // TODO: insert any other missing checks
258 
259   DP("New requires flags %ld compatible with existing %ld!\n",
260      flags, RequiresFlags);
261 }
262 
263 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) {
264   // Attempt to load all plugins available in the system.
265   std::call_once(initFlag, &RTLsTy::LoadRTLs, this);
266 
267   RTLsMtx->lock();
268   // Register the images with the RTLs that understand them, if any.
269   for (int32_t i = 0; i < desc->NumDeviceImages; ++i) {
270     // Obtain the image.
271     __tgt_device_image *img = &desc->DeviceImages[i];
272 
273     RTLInfoTy *FoundRTL = NULL;
274 
275     // Scan the RTLs that have associated images until we find one that supports
276     // the current image.
277     for (auto &R : AllRTLs) {
278       if (!R.is_valid_binary(img)) {
279         DP("Image " DPxMOD " is NOT compatible with RTL %s!\n",
280             DPxPTR(img->ImageStart), R.RTLName.c_str());
281         continue;
282       }
283 
284       DP("Image " DPxMOD " is compatible with RTL %s!\n",
285           DPxPTR(img->ImageStart), R.RTLName.c_str());
286 
287       // If this RTL is not already in use, initialize it.
288       if (!R.isUsed) {
289         // Initialize the device information for the RTL we are about to use.
290         DeviceTy device(&R);
291         size_t start = Devices.size();
292         Devices.resize(start + R.NumberOfDevices, device);
293         for (int32_t device_id = 0; device_id < R.NumberOfDevices;
294             device_id++) {
295           // global device ID
296           Devices[start + device_id].DeviceID = start + device_id;
297           // RTL local device ID
298           Devices[start + device_id].RTLDeviceID = device_id;
299         }
300 
301         // Initialize the index of this RTL and save it in the used RTLs.
302         R.Idx = (UsedRTLs.empty())
303                     ? 0
304                     : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices;
305         assert((size_t) R.Idx == start &&
306             "RTL index should equal the number of devices used so far.");
307         R.isUsed = true;
308         UsedRTLs.push_back(&R);
309 
310         DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx);
311       }
312 
313       // Initialize (if necessary) translation table for this library.
314       TrlTblMtx->lock();
315       if(!HostEntriesBeginToTransTable->count(desc->HostEntriesBegin)){
316         TranslationTable &tt =
317             (*HostEntriesBeginToTransTable)[desc->HostEntriesBegin];
318         tt.HostTable.EntriesBegin = desc->HostEntriesBegin;
319         tt.HostTable.EntriesEnd = desc->HostEntriesEnd;
320       }
321 
322       // Retrieve translation table for this library.
323       TranslationTable &TransTable =
324           (*HostEntriesBeginToTransTable)[desc->HostEntriesBegin];
325 
326       DP("Registering image " DPxMOD " with RTL %s!\n",
327           DPxPTR(img->ImageStart), R.RTLName.c_str());
328       RegisterImageIntoTranslationTable(TransTable, R, img);
329       TrlTblMtx->unlock();
330       FoundRTL = &R;
331 
332       // Load ctors/dtors for static objects
333       RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL);
334 
335       // if an RTL was found we are done - proceed to register the next image
336       break;
337     }
338 
339     if (!FoundRTL) {
340       DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart));
341     }
342   }
343   RTLsMtx->unlock();
344 
345 
346   DP("Done registering entries!\n");
347 }
348 
349 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) {
350   DP("Unloading target library!\n");
351 
352   RTLsMtx->lock();
353   // Find which RTL understands each image, if any.
354   for (int32_t i = 0; i < desc->NumDeviceImages; ++i) {
355     // Obtain the image.
356     __tgt_device_image *img = &desc->DeviceImages[i];
357 
358     RTLInfoTy *FoundRTL = NULL;
359 
360     // Scan the RTLs that have associated images until we find one that supports
361     // the current image. We only need to scan RTLs that are already being used.
362     for (auto *R : UsedRTLs) {
363 
364       assert(R->isUsed && "Expecting used RTLs.");
365 
366       if (!R->is_valid_binary(img)) {
367         DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n",
368             DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
369         continue;
370       }
371 
372       DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n",
373           DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
374 
375       FoundRTL = R;
376 
377       // Execute dtors for static objects if the device has been used, i.e.
378       // if its PendingCtors list has been emptied.
379       for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) {
380         DeviceTy &Device = Devices[FoundRTL->Idx + i];
381         Device.PendingGlobalsMtx.lock();
382         if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) {
383           for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) {
384             int rc = target(Device.DeviceID, dtor, 0, NULL, NULL, NULL, NULL, 1,
385                 1, true /*team*/);
386             if (rc != OFFLOAD_SUCCESS) {
387               DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor));
388             }
389           }
390           // Remove this library's entry from PendingCtorsDtors
391           Device.PendingCtorsDtors.erase(desc);
392         }
393         Device.PendingGlobalsMtx.unlock();
394       }
395 
396       DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n",
397           DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
398 
399       break;
400     }
401 
402     // if no RTL was found proceed to unregister the next image
403     if (!FoundRTL){
404       DP("No RTLs in use support the image " DPxMOD "!\n",
405           DPxPTR(img->ImageStart));
406     }
407   }
408   RTLsMtx->unlock();
409   DP("Done unregistering images!\n");
410 
411   // Remove entries from HostPtrToTableMap
412   TblMapMtx->lock();
413   for (__tgt_offload_entry *cur = desc->HostEntriesBegin;
414       cur < desc->HostEntriesEnd; ++cur) {
415     HostPtrToTableMap->erase(cur->addr);
416   }
417 
418   // Remove translation table for this descriptor.
419   auto tt = HostEntriesBeginToTransTable->find(desc->HostEntriesBegin);
420   if (tt != HostEntriesBeginToTransTable->end()) {
421     DP("Removing translation table for descriptor " DPxMOD "\n",
422         DPxPTR(desc->HostEntriesBegin));
423     HostEntriesBeginToTransTable->erase(tt);
424   } else {
425     DP("Translation table for descriptor " DPxMOD " cannot be found, probably "
426         "it has been already removed.\n", DPxPTR(desc->HostEntriesBegin));
427   }
428 
429   TblMapMtx->unlock();
430 
431   // TODO: Remove RTL and the devices it manages if it's not used anymore?
432   // TODO: Write some RTL->unload_image(...) function?
433 
434   DP("Done unregistering library!\n");
435 }
436