1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of the interface to be used by Clang during the codegen of a
10 // target region.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "device.h"
15 #include "private.h"
16 #include "rtl.h"
17 
18 #include <cassert>
19 #include <cstdio>
20 #include <cstdlib>
21 #include <mutex>
22 
23 ////////////////////////////////////////////////////////////////////////////////
24 /// manage the success or failure of a target construct
25 static void HandleDefaultTargetOffload() {
26   PM->TargetOffloadMtx.lock();
27   if (PM->TargetOffloadPolicy == tgt_default) {
28     if (omp_get_num_devices() > 0) {
29       DP("Default TARGET OFFLOAD policy is now mandatory "
30          "(devices were found)\n");
31       PM->TargetOffloadPolicy = tgt_mandatory;
32     } else {
33       DP("Default TARGET OFFLOAD policy is now disabled "
34          "(no devices were found)\n");
35       PM->TargetOffloadPolicy = tgt_disabled;
36     }
37   }
38   PM->TargetOffloadMtx.unlock();
39 }
40 
41 static int IsOffloadDisabled() {
42   if (PM->TargetOffloadPolicy == tgt_default)
43     HandleDefaultTargetOffload();
44   return PM->TargetOffloadPolicy == tgt_disabled;
45 }
46 
47 static void HandleTargetOutcome(bool success, ident_t *loc = nullptr) {
48   switch (PM->TargetOffloadPolicy) {
49   case tgt_disabled:
50     if (success) {
51       FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled");
52     }
53     break;
54   case tgt_default:
55     FATAL_MESSAGE0(1, "default offloading policy must be switched to "
56                       "mandatory or disabled");
57     break;
58   case tgt_mandatory:
59     if (!success) {
60       if (getInfoLevel() > 1)
61         for (const auto &Device : PM->Devices)
62           dumpTargetPointerMappings(Device);
63       else
64         FAILURE_MESSAGE("run with env LIBOMPTARGET_INFO>1 to dump host-target "
65                         "pointer maps\n");
66 
67       SourceInfo info(loc);
68       if (info.isAvailible())
69         fprintf(stderr, "%s:%d:%d: ", info.getFilename(), info.getLine(),
70                 info.getColumn());
71       else
72         FAILURE_MESSAGE(
73             "Build with debug information to provide more information");
74       FATAL_MESSAGE0(
75           1, "failure of target construct while offloading is mandatory");
76     }
77     break;
78   }
79 }
80 
81 ////////////////////////////////////////////////////////////////////////////////
82 /// adds requires flags
83 EXTERN void __tgt_register_requires(int64_t flags) {
84   PM->RTLs.RegisterRequires(flags);
85 }
86 
87 ////////////////////////////////////////////////////////////////////////////////
88 /// adds a target shared library to the target execution image
89 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) {
90   PM->RTLs.RegisterLib(desc);
91 }
92 
93 ////////////////////////////////////////////////////////////////////////////////
94 /// unloads a target shared library
95 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
96   PM->RTLs.UnregisterLib(desc);
97 }
98 
99 /// creates host-to-target data mapping, stores it in the
100 /// libomptarget.so internal structure (an entry in a stack of data maps)
101 /// and passes the data to the device.
102 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
103     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
104   __tgt_target_data_begin_mapper(nullptr, device_id, arg_num, args_base, args,
105                                  arg_sizes, arg_types, nullptr, nullptr);
106 }
107 
108 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
109     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
110     int32_t depNum, void *depList, int32_t noAliasDepNum,
111     void *noAliasDepList) {
112   if (depNum + noAliasDepNum > 0)
113     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
114 
115   __tgt_target_data_begin_mapper(nullptr, device_id, arg_num, args_base, args,
116                                  arg_sizes, arg_types, nullptr, nullptr);
117 }
118 
119 EXTERN void __tgt_target_data_begin_mapper(ident_t *loc, int64_t device_id,
120                                            int32_t arg_num, void **args_base,
121                                            void **args, int64_t *arg_sizes,
122                                            int64_t *arg_types,
123                                            map_var_info_t *arg_names,
124                                            void **arg_mappers) {
125   if (IsOffloadDisabled()) return;
126 
127   DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
128       device_id, arg_num);
129 
130   // No devices available?
131   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
132     device_id = omp_get_default_device();
133     DP("Use default device id %" PRId64 "\n", device_id);
134   }
135 
136   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
137     DP("Failed to get device %" PRId64 " ready\n", device_id);
138     HandleTargetOutcome(false, loc);
139     return;
140   }
141 
142   DeviceTy &Device = PM->Devices[device_id];
143 
144 #ifdef OMPTARGET_DEBUG
145   for (int i = 0; i < arg_num; ++i) {
146     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
147        ", Type=0x%" PRIx64 ", Name=%s\n",
148        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
149        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
150   }
151 #endif
152 
153   int rc = targetDataBegin(Device, arg_num, args_base, args, arg_sizes,
154                            arg_types, arg_names, arg_mappers, nullptr);
155   HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
156 }
157 
158 EXTERN void __tgt_target_data_begin_nowait_mapper(
159     ident_t *loc, int64_t device_id, int32_t arg_num, void **args_base,
160     void **args, int64_t *arg_sizes, int64_t *arg_types,
161     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
162     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
163   if (depNum + noAliasDepNum > 0)
164     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
165 
166   __tgt_target_data_begin_mapper(loc, device_id, arg_num, args_base, args,
167                                  arg_sizes, arg_types, arg_names, arg_mappers);
168 }
169 
170 /// passes data from the target, releases target memory and destroys
171 /// the host-target mapping (top entry from the stack of data maps)
172 /// created by the last __tgt_target_data_begin.
173 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
174     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
175   __tgt_target_data_end_mapper(nullptr, device_id, arg_num, args_base, args,
176                                arg_sizes, arg_types, nullptr, nullptr);
177 }
178 
179 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
180     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
181     int32_t depNum, void *depList, int32_t noAliasDepNum,
182     void *noAliasDepList) {
183   if (depNum + noAliasDepNum > 0)
184     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
185 
186   __tgt_target_data_end_mapper(nullptr, device_id, arg_num, args_base, args,
187                                arg_sizes, arg_types, nullptr, nullptr);
188 }
189 
190 EXTERN void __tgt_target_data_end_mapper(ident_t *loc, int64_t device_id,
191                                          int32_t arg_num, void **args_base,
192                                          void **args, int64_t *arg_sizes,
193                                          int64_t *arg_types,
194                                          map_var_info_t *arg_names,
195                                          void **arg_mappers) {
196   if (IsOffloadDisabled()) return;
197   DP("Entering data end region with %d mappings\n", arg_num);
198 
199   // No devices available?
200   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
201     device_id = omp_get_default_device();
202   }
203 
204   PM->RTLsMtx.lock();
205   size_t DevicesSize = PM->Devices.size();
206   PM->RTLsMtx.unlock();
207   if (DevicesSize <= (size_t)device_id) {
208     DP("Device ID  %" PRId64 " does not have a matching RTL.\n", device_id);
209     HandleTargetOutcome(false, loc);
210     return;
211   }
212 
213   DeviceTy &Device = PM->Devices[device_id];
214   if (!Device.IsInit) {
215     DP("Uninit device: ignore");
216     HandleTargetOutcome(false, loc);
217     return;
218   }
219 
220 #ifdef OMPTARGET_DEBUG
221   for (int i=0; i<arg_num; ++i) {
222     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
223        ", Type=0x%" PRIx64 ", Name=%s\n",
224        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
225        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
226   }
227 #endif
228 
229   int rc = targetDataEnd(Device, arg_num, args_base, args, arg_sizes, arg_types,
230                          arg_names, arg_mappers, nullptr);
231   HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
232 }
233 
234 EXTERN void __tgt_target_data_end_nowait_mapper(
235     ident_t *loc, int64_t device_id, int32_t arg_num, void **args_base,
236     void **args, int64_t *arg_sizes, int64_t *arg_types,
237     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
238     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
239   if (depNum + noAliasDepNum > 0)
240     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
241 
242   __tgt_target_data_end_mapper(loc, device_id, arg_num, args_base, args,
243                                arg_sizes, arg_types, arg_names, arg_mappers);
244 }
245 
246 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
247     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
248   __tgt_target_data_update_mapper(nullptr, device_id, arg_num, args_base, args,
249                                   arg_sizes, arg_types, nullptr, nullptr);
250 }
251 
252 EXTERN void __tgt_target_data_update_nowait(int64_t device_id, int32_t arg_num,
253     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
254     int32_t depNum, void *depList, int32_t noAliasDepNum,
255     void *noAliasDepList) {
256   if (depNum + noAliasDepNum > 0)
257     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
258 
259   __tgt_target_data_update_mapper(nullptr, device_id, arg_num, args_base, args,
260                                   arg_sizes, arg_types, nullptr, nullptr);
261 }
262 
263 EXTERN void __tgt_target_data_update_mapper(ident_t *loc, int64_t device_id,
264                                             int32_t arg_num, void **args_base,
265                                             void **args, int64_t *arg_sizes,
266                                             int64_t *arg_types,
267                                             map_var_info_t *arg_names,
268                                             void **arg_mappers) {
269   if (IsOffloadDisabled()) return;
270   DP("Entering data update with %d mappings\n", arg_num);
271 
272   // No devices available?
273   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
274     device_id = omp_get_default_device();
275   }
276 
277   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
278     DP("Failed to get device %" PRId64 " ready\n", device_id);
279     HandleTargetOutcome(false, loc);
280     return;
281   }
282 
283   DeviceTy &Device = PM->Devices[device_id];
284   int rc = targetDataUpdate(Device, arg_num, args_base, args, arg_sizes,
285                             arg_types, arg_names, arg_mappers);
286   HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
287 }
288 
289 EXTERN void __tgt_target_data_update_nowait_mapper(
290     ident_t *loc, int64_t device_id, int32_t arg_num, void **args_base,
291     void **args, int64_t *arg_sizes, int64_t *arg_types,
292     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
293     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
294   if (depNum + noAliasDepNum > 0)
295     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
296 
297   __tgt_target_data_update_mapper(loc, device_id, arg_num, args_base, args,
298                                   arg_sizes, arg_types, arg_names, arg_mappers);
299 }
300 
301 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
302     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
303   return __tgt_target_mapper(nullptr, device_id, host_ptr, arg_num, args_base,
304                              args, arg_sizes, arg_types, nullptr, nullptr);
305 }
306 
307 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
308     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
309     int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum,
310     void *noAliasDepList) {
311   if (depNum + noAliasDepNum > 0)
312     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
313 
314   return __tgt_target_mapper(nullptr, device_id, host_ptr, arg_num, args_base,
315                              args, arg_sizes, arg_types, nullptr, nullptr);
316 }
317 
318 EXTERN int __tgt_target_mapper(ident_t *loc, int64_t device_id, void *host_ptr,
319                                int32_t arg_num, void **args_base, void **args,
320                                int64_t *arg_sizes, int64_t *arg_types,
321                                map_var_info_t *arg_names, void **arg_mappers) {
322   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
323   DP("Entering target region with entry point " DPxMOD " and device Id %"
324       PRId64 "\n", DPxPTR(host_ptr), device_id);
325 
326   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
327     device_id = omp_get_default_device();
328   }
329 
330   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
331     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
332     HandleTargetOutcome(false, loc);
333     return OFFLOAD_FAIL;
334   }
335 
336 #ifdef OMPTARGET_DEBUG
337   for (int i=0; i<arg_num; ++i) {
338     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
339        ", Type=0x%" PRIx64 ", Name=%s\n",
340        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
341        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
342   }
343 #endif
344 
345   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
346                   arg_types, arg_names, arg_mappers, 0, 0, false /*team*/);
347   HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
348   return rc;
349 }
350 
351 EXTERN int __tgt_target_nowait_mapper(
352     ident_t *loc, int64_t device_id, void *host_ptr, int32_t arg_num,
353     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
354     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
355     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
356   if (depNum + noAliasDepNum > 0)
357     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
358 
359   return __tgt_target_mapper(loc, device_id, host_ptr, arg_num, args_base, args,
360                              arg_sizes, arg_types, arg_names, arg_mappers);
361 }
362 
363 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
364     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
365     int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
366   return __tgt_target_teams_mapper(nullptr, device_id, host_ptr, arg_num,
367                                    args_base, args, arg_sizes, arg_types,
368                                    nullptr, nullptr, team_num, thread_limit);
369 }
370 
371 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr,
372     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
373     int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum,
374     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
375   if (depNum + noAliasDepNum > 0)
376     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
377 
378   return __tgt_target_teams_mapper(nullptr, device_id, host_ptr, arg_num,
379                                    args_base, args, arg_sizes, arg_types,
380                                    nullptr, nullptr, team_num, thread_limit);
381 }
382 
383 EXTERN int __tgt_target_teams_mapper(ident_t *loc, int64_t device_id,
384                                      void *host_ptr, int32_t arg_num,
385                                      void **args_base, void **args,
386                                      int64_t *arg_sizes, int64_t *arg_types,
387                                      map_var_info_t *arg_names,
388                                      void **arg_mappers, int32_t team_num,
389                                      int32_t thread_limit) {
390   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
391   DP("Entering target region with entry point " DPxMOD " and device Id %"
392       PRId64 "\n", DPxPTR(host_ptr), device_id);
393 
394   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
395     device_id = omp_get_default_device();
396   }
397 
398   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
399     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
400     HandleTargetOutcome(false, loc);
401     return OFFLOAD_FAIL;
402   }
403 
404 #ifdef OMPTARGET_DEBUG
405   for (int i=0; i<arg_num; ++i) {
406     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
407        ", Type=0x%" PRIx64 ", Name=%s\n",
408        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
409        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
410   }
411 #endif
412 
413   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
414                   arg_types, arg_names, arg_mappers, team_num, thread_limit,
415                   true /*team*/);
416   HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
417 
418   return rc;
419 }
420 
421 EXTERN int __tgt_target_teams_nowait_mapper(
422     ident_t *loc, int64_t device_id, void *host_ptr, int32_t arg_num,
423     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
424     map_var_info_t *arg_names, void **arg_mappers, int32_t team_num,
425     int32_t thread_limit, int32_t depNum, void *depList, int32_t noAliasDepNum,
426     void *noAliasDepList) {
427   if (depNum + noAliasDepNum > 0)
428     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
429 
430   return __tgt_target_teams_mapper(loc, device_id, host_ptr, arg_num, args_base,
431                                    args, arg_sizes, arg_types, arg_names,
432                                    arg_mappers, team_num, thread_limit);
433 }
434 
435 // Get the current number of components for a user-defined mapper.
436 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) {
437   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
438   int64_t size = MapperComponentsPtr->Components.size();
439   DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n",
440      DPxPTR(rt_mapper_handle), size);
441   return size;
442 }
443 
444 // Push back one component for a user-defined mapper.
445 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base,
446                                         void *begin, int64_t size,
447                                         int64_t type) {
448   DP("__tgt_push_mapper_component(Handle=" DPxMOD
449      ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
450      ", Type=0x%" PRIx64 ").\n",
451      DPxPTR(rt_mapper_handle), DPxPTR(base), DPxPTR(begin), size, type);
452   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
453   MapperComponentsPtr->Components.push_back(
454       MapComponentInfoTy(base, begin, size, type));
455 }
456 
457 EXTERN void __kmpc_push_target_tripcount(ident_t *loc, int64_t device_id,
458                                          uint64_t loop_tripcount) {
459   if (IsOffloadDisabled())
460     return;
461 
462   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
463     device_id = omp_get_default_device();
464   }
465 
466   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
467     DP("Failed to get device %" PRId64 " ready\n", device_id);
468     HandleTargetOutcome(false, loc);
469     return;
470   }
471 
472   DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
473       loop_tripcount);
474   PM->TblMapMtx.lock();
475   PM->Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL),
476                                              loop_tripcount);
477   PM->TblMapMtx.unlock();
478 }
479