1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of the interface to be used by Clang during the codegen of a
10 // target region.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "device.h"
15 #include "private.h"
16 #include "rtl.h"
17 
18 #include <cassert>
19 #include <cstdio>
20 #include <cstdlib>
21 #include <mutex>
22 
23 ////////////////////////////////////////////////////////////////////////////////
24 /// manage the success or failure of a target construct
25 static void HandleDefaultTargetOffload() {
26   PM->TargetOffloadMtx.lock();
27   if (PM->TargetOffloadPolicy == tgt_default) {
28     if (omp_get_num_devices() > 0) {
29       DP("Default TARGET OFFLOAD policy is now mandatory "
30          "(devices were found)\n");
31       PM->TargetOffloadPolicy = tgt_mandatory;
32     } else {
33       DP("Default TARGET OFFLOAD policy is now disabled "
34          "(no devices were found)\n");
35       PM->TargetOffloadPolicy = tgt_disabled;
36     }
37   }
38   PM->TargetOffloadMtx.unlock();
39 }
40 
41 static int IsOffloadDisabled() {
42   if (PM->TargetOffloadPolicy == tgt_default)
43     HandleDefaultTargetOffload();
44   return PM->TargetOffloadPolicy == tgt_disabled;
45 }
46 
47 static void HandleTargetOutcome(bool success) {
48   switch (PM->TargetOffloadPolicy) {
49   case tgt_disabled:
50     if (success) {
51       FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled");
52     }
53     break;
54   case tgt_default:
55     FATAL_MESSAGE0(1, "default offloading policy must be switched to "
56                       "mandatory or disabled");
57     break;
58   case tgt_mandatory:
59     if (!success) {
60       if (getInfoLevel() > 1)
61         for (const auto &Device : PM->Devices)
62           dumpTargetPointerMappings(Device);
63       else
64         FAILURE_MESSAGE("run with env LIBOMPTARGET_INFO>1 to dump host-target "
65                         "pointer maps\n");
66 
67       FATAL_MESSAGE0(
68           1, "failure of target construct while offloading is mandatory");
69     }
70     break;
71   }
72 }
73 
74 ////////////////////////////////////////////////////////////////////////////////
75 /// adds requires flags
76 EXTERN void __tgt_register_requires(int64_t flags) {
77   PM->RTLs.RegisterRequires(flags);
78 }
79 
80 ////////////////////////////////////////////////////////////////////////////////
81 /// adds a target shared library to the target execution image
82 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) {
83   PM->RTLs.RegisterLib(desc);
84 }
85 
86 ////////////////////////////////////////////////////////////////////////////////
87 /// unloads a target shared library
88 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
89   PM->RTLs.UnregisterLib(desc);
90 }
91 
92 /// creates host-to-target data mapping, stores it in the
93 /// libomptarget.so internal structure (an entry in a stack of data maps)
94 /// and passes the data to the device.
95 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
96     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
97   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args, arg_sizes,
98                                  arg_types, nullptr, nullptr);
99 }
100 
101 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
102     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
103     int32_t depNum, void *depList, int32_t noAliasDepNum,
104     void *noAliasDepList) {
105   if (depNum + noAliasDepNum > 0)
106     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
107 
108   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args, arg_sizes,
109                                  arg_types, nullptr, nullptr);
110 }
111 
112 EXTERN void __tgt_target_data_begin_mapper(int64_t device_id, int32_t arg_num,
113                                            void **args_base, void **args,
114                                            int64_t *arg_sizes,
115                                            int64_t *arg_types,
116                                            map_var_info_t *arg_names,
117                                            void **arg_mappers) {
118   if (IsOffloadDisabled()) return;
119 
120   DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
121       device_id, arg_num);
122 
123   // No devices available?
124   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
125     device_id = omp_get_default_device();
126     DP("Use default device id %" PRId64 "\n", device_id);
127   }
128 
129   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
130     DP("Failed to get device %" PRId64 " ready\n", device_id);
131     HandleTargetOutcome(false);
132     return;
133   }
134 
135   DeviceTy &Device = PM->Devices[device_id];
136 
137 #ifdef OMPTARGET_DEBUG
138   for (int i = 0; i < arg_num; ++i) {
139     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
140        ", Type=0x%" PRIx64 ", Name=%s\n",
141        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
142        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
143   }
144 #endif
145 
146   int rc = targetDataBegin(Device, arg_num, args_base, args, arg_sizes,
147                            arg_types, arg_names, arg_mappers, nullptr);
148   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
149 }
150 
151 EXTERN void __tgt_target_data_begin_nowait_mapper(
152     int64_t device_id, int32_t arg_num, void **args_base, void **args,
153     int64_t *arg_sizes, int64_t *arg_types, map_var_info_t *arg_names,
154     void **arg_mappers, int32_t depNum, void *depList, int32_t noAliasDepNum,
155     void *noAliasDepList) {
156   if (depNum + noAliasDepNum > 0)
157     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
158 
159   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args, arg_sizes,
160                                  arg_types, arg_names, arg_mappers);
161 }
162 
163 /// passes data from the target, releases target memory and destroys
164 /// the host-target mapping (top entry from the stack of data maps)
165 /// created by the last __tgt_target_data_begin.
166 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
167     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
168   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
169                                arg_types, nullptr, nullptr);
170 }
171 
172 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
173     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
174     int32_t depNum, void *depList, int32_t noAliasDepNum,
175     void *noAliasDepList) {
176   if (depNum + noAliasDepNum > 0)
177     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
178 
179   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
180                                arg_types, nullptr, nullptr);
181 }
182 
183 EXTERN void __tgt_target_data_end_mapper(int64_t device_id, int32_t arg_num,
184                                          void **args_base, void **args,
185                                          int64_t *arg_sizes, int64_t *arg_types,
186                                          map_var_info_t *arg_names,
187                                          void **arg_mappers) {
188   if (IsOffloadDisabled()) return;
189   DP("Entering data end region with %d mappings\n", arg_num);
190 
191   // No devices available?
192   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
193     device_id = omp_get_default_device();
194   }
195 
196   PM->RTLsMtx.lock();
197   size_t DevicesSize = PM->Devices.size();
198   PM->RTLsMtx.unlock();
199   if (DevicesSize <= (size_t)device_id) {
200     DP("Device ID  %" PRId64 " does not have a matching RTL.\n", device_id);
201     HandleTargetOutcome(false);
202     return;
203   }
204 
205   DeviceTy &Device = PM->Devices[device_id];
206   if (!Device.IsInit) {
207     DP("Uninit device: ignore");
208     HandleTargetOutcome(false);
209     return;
210   }
211 
212 #ifdef OMPTARGET_DEBUG
213   for (int i=0; i<arg_num; ++i) {
214     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
215        ", Type=0x%" PRIx64 ", Name=%s\n",
216        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
217        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
218   }
219 #endif
220 
221   int rc = targetDataEnd(Device, arg_num, args_base, args, arg_sizes, arg_types,
222                          arg_names, arg_mappers, nullptr);
223   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
224 }
225 
226 EXTERN void __tgt_target_data_end_nowait_mapper(
227     int64_t device_id, int32_t arg_num, void **args_base, void **args,
228     int64_t *arg_sizes, int64_t *arg_types, map_var_info_t *arg_names,
229     void **arg_mappers, int32_t depNum, void *depList, int32_t noAliasDepNum,
230     void *noAliasDepList) {
231   if (depNum + noAliasDepNum > 0)
232     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
233 
234   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
235                                arg_types, arg_names, arg_mappers);
236 }
237 
238 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
239     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
240   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
241                                   arg_sizes, arg_types, nullptr, nullptr);
242 }
243 
244 EXTERN void __tgt_target_data_update_nowait(int64_t device_id, int32_t arg_num,
245     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
246     int32_t depNum, void *depList, int32_t noAliasDepNum,
247     void *noAliasDepList) {
248   if (depNum + noAliasDepNum > 0)
249     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
250 
251   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
252                                   arg_sizes, arg_types, nullptr, nullptr);
253 }
254 
255 EXTERN void __tgt_target_data_update_mapper(int64_t device_id, int32_t arg_num,
256                                             void **args_base, void **args,
257                                             int64_t *arg_sizes,
258                                             int64_t *arg_types,
259                                             map_var_info_t *arg_names,
260                                             void **arg_mappers) {
261   if (IsOffloadDisabled()) return;
262   DP("Entering data update with %d mappings\n", arg_num);
263 
264   // No devices available?
265   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
266     device_id = omp_get_default_device();
267   }
268 
269   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
270     DP("Failed to get device %" PRId64 " ready\n", device_id);
271     HandleTargetOutcome(false);
272     return;
273   }
274 
275   DeviceTy &Device = PM->Devices[device_id];
276   int rc = target_data_update(Device, arg_num, args_base, args, arg_sizes,
277                               arg_types, arg_names, arg_mappers);
278   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
279 }
280 
281 EXTERN void __tgt_target_data_update_nowait_mapper(
282     int64_t device_id, int32_t arg_num, void **args_base, void **args,
283     int64_t *arg_sizes, int64_t *arg_types, map_var_info_t *arg_names,
284     void **arg_mappers, int32_t depNum, void *depList, int32_t noAliasDepNum,
285     void *noAliasDepList) {
286   if (depNum + noAliasDepNum > 0)
287     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
288 
289   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
290                                   arg_sizes, arg_types, arg_names, arg_mappers);
291 }
292 
293 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
294     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
295   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
296                              arg_sizes, arg_types, nullptr, nullptr);
297 }
298 
299 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
300     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
301     int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum,
302     void *noAliasDepList) {
303   if (depNum + noAliasDepNum > 0)
304     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
305 
306   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
307                              arg_sizes, arg_types, nullptr, nullptr);
308 }
309 
310 EXTERN int __tgt_target_mapper(int64_t device_id, void *host_ptr,
311                                int32_t arg_num, void **args_base, void **args,
312                                int64_t *arg_sizes, int64_t *arg_types,
313                                map_var_info_t *arg_names, void **arg_mappers) {
314   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
315   DP("Entering target region with entry point " DPxMOD " and device Id %"
316       PRId64 "\n", DPxPTR(host_ptr), device_id);
317 
318   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
319     device_id = omp_get_default_device();
320   }
321 
322   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
323     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
324     HandleTargetOutcome(false);
325     return OFFLOAD_FAIL;
326   }
327 
328 #ifdef OMPTARGET_DEBUG
329   for (int i=0; i<arg_num; ++i) {
330     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
331        ", Type=0x%" PRIx64 ", Name=%s\n",
332        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
333        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
334   }
335 #endif
336 
337   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
338                   arg_types, arg_names, arg_mappers, 0, 0, false /*team*/);
339   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
340   return rc;
341 }
342 
343 EXTERN int
344 __tgt_target_nowait_mapper(int64_t device_id, void *host_ptr, int32_t arg_num,
345                            void **args_base, void **args, int64_t *arg_sizes,
346                            int64_t *arg_types, map_var_info_t *arg_names,
347                            void **arg_mappers, int32_t depNum, void *depList,
348                            int32_t noAliasDepNum, void *noAliasDepList) {
349   if (depNum + noAliasDepNum > 0)
350     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
351 
352   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
353                              arg_sizes, arg_types, arg_names, arg_mappers);
354 }
355 
356 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
357     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
358     int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
359   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
360                                    args, arg_sizes, arg_types, nullptr, nullptr,
361                                    team_num, thread_limit);
362 }
363 
364 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr,
365     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
366     int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum,
367     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
368   if (depNum + noAliasDepNum > 0)
369     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
370 
371   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
372                                    args, arg_sizes, arg_types, nullptr, nullptr,
373                                    team_num, thread_limit);
374 }
375 
376 EXTERN int __tgt_target_teams_mapper(int64_t device_id, void *host_ptr,
377                                      int32_t arg_num, void **args_base,
378                                      void **args, int64_t *arg_sizes,
379                                      int64_t *arg_types,
380                                      map_var_info_t *arg_names,
381                                      void **arg_mappers, int32_t team_num,
382                                      int32_t thread_limit) {
383   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
384   DP("Entering target region with entry point " DPxMOD " and device Id %"
385       PRId64 "\n", DPxPTR(host_ptr), device_id);
386 
387   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
388     device_id = omp_get_default_device();
389   }
390 
391   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
392     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
393     HandleTargetOutcome(false);
394     return OFFLOAD_FAIL;
395   }
396 
397 #ifdef OMPTARGET_DEBUG
398   for (int i=0; i<arg_num; ++i) {
399     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
400        ", Type=0x%" PRIx64 ", Name=%s\n",
401        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
402        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
403   }
404 #endif
405 
406   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
407                   arg_types, arg_names, arg_mappers, team_num, thread_limit,
408                   true /*team*/);
409   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
410 
411   return rc;
412 }
413 
414 EXTERN int __tgt_target_teams_nowait_mapper(
415     int64_t device_id, void *host_ptr, int32_t arg_num, void **args_base,
416     void **args, int64_t *arg_sizes, int64_t *arg_types,
417     map_var_info_t *arg_names, void **arg_mappers, int32_t team_num,
418     int32_t thread_limit, int32_t depNum, void *depList, int32_t noAliasDepNum,
419     void *noAliasDepList) {
420   if (depNum + noAliasDepNum > 0)
421     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
422 
423   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
424                                    args, arg_sizes, arg_types, arg_names,
425                                    arg_mappers, team_num, thread_limit);
426 }
427 
428 // Get the current number of components for a user-defined mapper.
429 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) {
430   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
431   int64_t size = MapperComponentsPtr->Components.size();
432   DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n",
433      DPxPTR(rt_mapper_handle), size);
434   return size;
435 }
436 
437 // Push back one component for a user-defined mapper.
438 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base,
439                                         void *begin, int64_t size,
440                                         int64_t type) {
441   DP("__tgt_push_mapper_component(Handle=" DPxMOD
442      ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
443      ", Type=0x%" PRIx64 ").\n",
444      DPxPTR(rt_mapper_handle), DPxPTR(base), DPxPTR(begin), size, type);
445   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
446   MapperComponentsPtr->Components.push_back(
447       MapComponentInfoTy(base, begin, size, type));
448 }
449 
450 EXTERN void __kmpc_push_target_tripcount(int64_t device_id,
451     uint64_t loop_tripcount) {
452   if (IsOffloadDisabled())
453     return;
454 
455   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
456     device_id = omp_get_default_device();
457   }
458 
459   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
460     DP("Failed to get device %" PRId64 " ready\n", device_id);
461     HandleTargetOutcome(false);
462     return;
463   }
464 
465   DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
466       loop_tripcount);
467   PM->TblMapMtx.lock();
468   PM->Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL),
469                                              loop_tripcount);
470   PM->TblMapMtx.unlock();
471 }
472