1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of the interface to be used by Clang during the codegen of a
10 // target region.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "device.h"
15 #include "private.h"
16 #include "rtl.h"
17 
18 #include <cassert>
19 #include <cstdio>
20 #include <cstdlib>
21 #include <mutex>
22 
23 ////////////////////////////////////////////////////////////////////////////////
24 /// manage the success or failure of a target construct
25 static void HandleDefaultTargetOffload() {
26   PM->TargetOffloadMtx.lock();
27   if (PM->TargetOffloadPolicy == tgt_default) {
28     if (omp_get_num_devices() > 0) {
29       DP("Default TARGET OFFLOAD policy is now mandatory "
30          "(devices were found)\n");
31       PM->TargetOffloadPolicy = tgt_mandatory;
32     } else {
33       DP("Default TARGET OFFLOAD policy is now disabled "
34          "(no devices were found)\n");
35       PM->TargetOffloadPolicy = tgt_disabled;
36     }
37   }
38   PM->TargetOffloadMtx.unlock();
39 }
40 
41 static int IsOffloadDisabled() {
42   if (PM->TargetOffloadPolicy == tgt_default)
43     HandleDefaultTargetOffload();
44   return PM->TargetOffloadPolicy == tgt_disabled;
45 }
46 
47 static void HandleTargetOutcome(bool success, ident_t *loc = nullptr) {
48   switch (PM->TargetOffloadPolicy) {
49   case tgt_disabled:
50     if (success) {
51       FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled");
52     }
53     break;
54   case tgt_default:
55     FATAL_MESSAGE0(1, "default offloading policy must be switched to "
56                       "mandatory or disabled");
57     break;
58   case tgt_mandatory:
59     if (!success) {
60       if (getInfoLevel() > 1)
61         for (const auto &Device : PM->Devices)
62           dumpTargetPointerMappings(Device);
63       else
64         FAILURE_MESSAGE("run with env LIBOMPTARGET_INFO>1 to dump host-target "
65                         "pointer maps\n");
66 
67       SourceInfo info(loc);
68       if (info.isAvailible())
69         fprintf(stderr, "%s:%d:%d: ", info.getFilename(), info.getLine(),
70                 info.getColumn());
71       else
72         FAILURE_MESSAGE(
73             "Build with debug information to provide more information");
74       FATAL_MESSAGE0(
75           1, "failure of target construct while offloading is mandatory");
76     }
77     break;
78   }
79 }
80 
81 ////////////////////////////////////////////////////////////////////////////////
82 /// adds requires flags
83 EXTERN void __tgt_register_requires(int64_t flags) {
84   TIMESCOPE();
85   PM->RTLs.RegisterRequires(flags);
86 }
87 
88 ////////////////////////////////////////////////////////////////////////////////
89 /// adds a target shared library to the target execution image
90 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) {
91   TIMESCOPE();
92   PM->RTLs.RegisterLib(desc);
93 }
94 
95 ////////////////////////////////////////////////////////////////////////////////
96 /// unloads a target shared library
97 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
98   TIMESCOPE();
99   PM->RTLs.UnregisterLib(desc);
100 }
101 
102 /// creates host-to-target data mapping, stores it in the
103 /// libomptarget.so internal structure (an entry in a stack of data maps)
104 /// and passes the data to the device.
105 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
106     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
107   TIMESCOPE();
108   __tgt_target_data_begin_mapper(nullptr, device_id, arg_num, args_base, args,
109                                  arg_sizes, arg_types, nullptr, nullptr);
110 }
111 
112 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
113     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
114     int32_t depNum, void *depList, int32_t noAliasDepNum,
115     void *noAliasDepList) {
116   TIMESCOPE();
117   if (depNum + noAliasDepNum > 0)
118     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
119 
120   __tgt_target_data_begin_mapper(nullptr, device_id, arg_num, args_base, args,
121                                  arg_sizes, arg_types, nullptr, nullptr);
122 }
123 
124 EXTERN void __tgt_target_data_begin_mapper(ident_t *loc, int64_t device_id,
125                                            int32_t arg_num, void **args_base,
126                                            void **args, int64_t *arg_sizes,
127                                            int64_t *arg_types,
128                                            map_var_info_t *arg_names,
129                                            void **arg_mappers) {
130   TIMESCOPE();
131   if (IsOffloadDisabled()) return;
132 
133   DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
134       device_id, arg_num);
135 
136   // No devices available?
137   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
138     device_id = omp_get_default_device();
139     DP("Use default device id %" PRId64 "\n", device_id);
140   }
141 
142   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
143     DP("Failed to get device %" PRId64 " ready\n", device_id);
144     HandleTargetOutcome(false, loc);
145     return;
146   }
147 
148   DeviceTy &Device = PM->Devices[device_id];
149 
150 #ifdef OMPTARGET_DEBUG
151   for (int i = 0; i < arg_num; ++i) {
152     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
153        ", Type=0x%" PRIx64 ", Name=%s\n",
154        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
155        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
156   }
157 #endif
158 
159   int rc = targetDataBegin(Device, arg_num, args_base, args, arg_sizes,
160                            arg_types, arg_names, arg_mappers, nullptr);
161   HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
162 }
163 
164 EXTERN void __tgt_target_data_begin_nowait_mapper(
165     ident_t *loc, int64_t device_id, int32_t arg_num, void **args_base,
166     void **args, int64_t *arg_sizes, int64_t *arg_types,
167     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
168     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
169   TIMESCOPE();
170   if (depNum + noAliasDepNum > 0)
171     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
172 
173   __tgt_target_data_begin_mapper(loc, device_id, arg_num, args_base, args,
174                                  arg_sizes, arg_types, arg_names, arg_mappers);
175 }
176 
177 /// passes data from the target, releases target memory and destroys
178 /// the host-target mapping (top entry from the stack of data maps)
179 /// created by the last __tgt_target_data_begin.
180 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
181     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
182   TIMESCOPE();
183   __tgt_target_data_end_mapper(nullptr, device_id, arg_num, args_base, args,
184                                arg_sizes, arg_types, nullptr, nullptr);
185 }
186 
187 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
188     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
189     int32_t depNum, void *depList, int32_t noAliasDepNum,
190     void *noAliasDepList) {
191   TIMESCOPE();
192   if (depNum + noAliasDepNum > 0)
193     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
194 
195   __tgt_target_data_end_mapper(nullptr, device_id, arg_num, args_base, args,
196                                arg_sizes, arg_types, nullptr, nullptr);
197 }
198 
199 EXTERN void __tgt_target_data_end_mapper(ident_t *loc, int64_t device_id,
200                                          int32_t arg_num, void **args_base,
201                                          void **args, int64_t *arg_sizes,
202                                          int64_t *arg_types,
203                                          map_var_info_t *arg_names,
204                                          void **arg_mappers) {
205   TIMESCOPE();
206   if (IsOffloadDisabled()) return;
207   DP("Entering data end region with %d mappings\n", arg_num);
208 
209   // No devices available?
210   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
211     device_id = omp_get_default_device();
212   }
213 
214   PM->RTLsMtx.lock();
215   size_t DevicesSize = PM->Devices.size();
216   PM->RTLsMtx.unlock();
217   if (DevicesSize <= (size_t)device_id) {
218     DP("Device ID  %" PRId64 " does not have a matching RTL.\n", device_id);
219     HandleTargetOutcome(false, loc);
220     return;
221   }
222 
223   DeviceTy &Device = PM->Devices[device_id];
224   if (!Device.IsInit) {
225     DP("Uninit device: ignore");
226     HandleTargetOutcome(false, loc);
227     return;
228   }
229 
230 #ifdef OMPTARGET_DEBUG
231   for (int i=0; i<arg_num; ++i) {
232     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
233        ", Type=0x%" PRIx64 ", Name=%s\n",
234        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
235        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
236   }
237 #endif
238 
239   int rc = targetDataEnd(Device, arg_num, args_base, args, arg_sizes, arg_types,
240                          arg_names, arg_mappers, nullptr);
241   HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
242 }
243 
244 EXTERN void __tgt_target_data_end_nowait_mapper(
245     ident_t *loc, int64_t device_id, int32_t arg_num, void **args_base,
246     void **args, int64_t *arg_sizes, int64_t *arg_types,
247     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
248     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
249   TIMESCOPE();
250   if (depNum + noAliasDepNum > 0)
251     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
252 
253   __tgt_target_data_end_mapper(loc, device_id, arg_num, args_base, args,
254                                arg_sizes, arg_types, arg_names, arg_mappers);
255 }
256 
257 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
258     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
259   TIMESCOPE();
260   __tgt_target_data_update_mapper(nullptr, device_id, arg_num, args_base, args,
261                                   arg_sizes, arg_types, nullptr, nullptr);
262 }
263 
264 EXTERN void __tgt_target_data_update_nowait(int64_t device_id, int32_t arg_num,
265     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
266     int32_t depNum, void *depList, int32_t noAliasDepNum,
267     void *noAliasDepList) {
268   TIMESCOPE();
269   if (depNum + noAliasDepNum > 0)
270     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
271 
272   __tgt_target_data_update_mapper(nullptr, device_id, arg_num, args_base, args,
273                                   arg_sizes, arg_types, nullptr, nullptr);
274 }
275 
276 EXTERN void __tgt_target_data_update_mapper(ident_t *loc, int64_t device_id,
277                                             int32_t arg_num, void **args_base,
278                                             void **args, int64_t *arg_sizes,
279                                             int64_t *arg_types,
280                                             map_var_info_t *arg_names,
281                                             void **arg_mappers) {
282   TIMESCOPE();
283   if (IsOffloadDisabled()) return;
284   DP("Entering data update with %d mappings\n", arg_num);
285 
286   // No devices available?
287   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
288     device_id = omp_get_default_device();
289   }
290 
291   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
292     DP("Failed to get device %" PRId64 " ready\n", device_id);
293     HandleTargetOutcome(false, loc);
294     return;
295   }
296 
297   DeviceTy &Device = PM->Devices[device_id];
298   int rc = targetDataUpdate(Device, arg_num, args_base, args, arg_sizes,
299                             arg_types, arg_names, arg_mappers);
300   HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
301 }
302 
303 EXTERN void __tgt_target_data_update_nowait_mapper(
304     ident_t *loc, int64_t device_id, int32_t arg_num, void **args_base,
305     void **args, int64_t *arg_sizes, int64_t *arg_types,
306     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
307     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
308   TIMESCOPE();
309   if (depNum + noAliasDepNum > 0)
310     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
311 
312   __tgt_target_data_update_mapper(loc, device_id, arg_num, args_base, args,
313                                   arg_sizes, arg_types, arg_names, arg_mappers);
314 }
315 
316 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
317     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
318   TIMESCOPE();
319   return __tgt_target_mapper(nullptr, device_id, host_ptr, arg_num, args_base,
320                              args, arg_sizes, arg_types, nullptr, nullptr);
321 }
322 
323 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
324     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
325     int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum,
326     void *noAliasDepList) {
327   TIMESCOPE();
328   if (depNum + noAliasDepNum > 0)
329     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
330 
331   return __tgt_target_mapper(nullptr, device_id, host_ptr, arg_num, args_base,
332                              args, arg_sizes, arg_types, nullptr, nullptr);
333 }
334 
335 EXTERN int __tgt_target_mapper(ident_t *loc, int64_t device_id, void *host_ptr,
336                                int32_t arg_num, void **args_base, void **args,
337                                int64_t *arg_sizes, int64_t *arg_types,
338                                map_var_info_t *arg_names, void **arg_mappers) {
339   TIMESCOPE();
340   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
341   DP("Entering target region with entry point " DPxMOD " and device Id %"
342       PRId64 "\n", DPxPTR(host_ptr), device_id);
343 
344   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
345     device_id = omp_get_default_device();
346   }
347 
348   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
349     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
350     HandleTargetOutcome(false, loc);
351     return OFFLOAD_FAIL;
352   }
353 
354 #ifdef OMPTARGET_DEBUG
355   for (int i=0; i<arg_num; ++i) {
356     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
357        ", Type=0x%" PRIx64 ", Name=%s\n",
358        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
359        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
360   }
361 #endif
362 
363   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
364                   arg_types, arg_names, arg_mappers, 0, 0, false /*team*/);
365   HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
366   return rc;
367 }
368 
369 EXTERN int __tgt_target_nowait_mapper(
370     ident_t *loc, int64_t device_id, void *host_ptr, int32_t arg_num,
371     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
372     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
373     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
374   TIMESCOPE();
375   if (depNum + noAliasDepNum > 0)
376     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
377 
378   return __tgt_target_mapper(loc, device_id, host_ptr, arg_num, args_base, args,
379                              arg_sizes, arg_types, arg_names, arg_mappers);
380 }
381 
382 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
383     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
384     int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
385   TIMESCOPE();
386   return __tgt_target_teams_mapper(nullptr, device_id, host_ptr, arg_num,
387                                    args_base, args, arg_sizes, arg_types,
388                                    nullptr, nullptr, team_num, thread_limit);
389 }
390 
391 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr,
392     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
393     int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum,
394     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
395   TIMESCOPE();
396   if (depNum + noAliasDepNum > 0)
397     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
398 
399   return __tgt_target_teams_mapper(nullptr, device_id, host_ptr, arg_num,
400                                    args_base, args, arg_sizes, arg_types,
401                                    nullptr, nullptr, team_num, thread_limit);
402 }
403 
404 EXTERN int __tgt_target_teams_mapper(ident_t *loc, int64_t device_id,
405                                      void *host_ptr, int32_t arg_num,
406                                      void **args_base, void **args,
407                                      int64_t *arg_sizes, int64_t *arg_types,
408                                      map_var_info_t *arg_names,
409                                      void **arg_mappers, int32_t team_num,
410                                      int32_t thread_limit) {
411   TIMESCOPE();
412   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
413   DP("Entering target region with entry point " DPxMOD " and device Id %"
414       PRId64 "\n", DPxPTR(host_ptr), device_id);
415 
416   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
417     device_id = omp_get_default_device();
418   }
419 
420   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
421     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
422     HandleTargetOutcome(false, loc);
423     return OFFLOAD_FAIL;
424   }
425 
426 #ifdef OMPTARGET_DEBUG
427   for (int i=0; i<arg_num; ++i) {
428     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
429        ", Type=0x%" PRIx64 ", Name=%s\n",
430        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
431        (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "(null)");
432   }
433 #endif
434 
435   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
436                   arg_types, arg_names, arg_mappers, team_num, thread_limit,
437                   true /*team*/);
438   HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
439 
440   return rc;
441 }
442 
443 EXTERN int __tgt_target_teams_nowait_mapper(
444     ident_t *loc, int64_t device_id, void *host_ptr, int32_t arg_num,
445     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
446     map_var_info_t *arg_names, void **arg_mappers, int32_t team_num,
447     int32_t thread_limit, int32_t depNum, void *depList, int32_t noAliasDepNum,
448     void *noAliasDepList) {
449   TIMESCOPE();
450   if (depNum + noAliasDepNum > 0)
451     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
452 
453   return __tgt_target_teams_mapper(loc, device_id, host_ptr, arg_num, args_base,
454                                    args, arg_sizes, arg_types, arg_names,
455                                    arg_mappers, team_num, thread_limit);
456 }
457 
458 // Get the current number of components for a user-defined mapper.
459 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) {
460   TIMESCOPE();
461   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
462   int64_t size = MapperComponentsPtr->Components.size();
463   DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n",
464      DPxPTR(rt_mapper_handle), size);
465   return size;
466 }
467 
468 // Push back one component for a user-defined mapper.
469 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base,
470                                         void *begin, int64_t size,
471                                         int64_t type) {
472   TIMESCOPE();
473   DP("__tgt_push_mapper_component(Handle=" DPxMOD
474      ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
475      ", Type=0x%" PRIx64 ").\n",
476      DPxPTR(rt_mapper_handle), DPxPTR(base), DPxPTR(begin), size, type);
477   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
478   MapperComponentsPtr->Components.push_back(
479       MapComponentInfoTy(base, begin, size, type));
480 }
481 
482 EXTERN void __kmpc_push_target_tripcount(ident_t *loc, int64_t device_id,
483                                          uint64_t loop_tripcount) {
484   TIMESCOPE();
485   if (IsOffloadDisabled())
486     return;
487 
488   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
489     device_id = omp_get_default_device();
490   }
491 
492   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
493     DP("Failed to get device %" PRId64 " ready\n", device_id);
494     HandleTargetOutcome(false, loc);
495     return;
496   }
497 
498   DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
499       loop_tripcount);
500   PM->TblMapMtx.lock();
501   PM->Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL),
502                                              loop_tripcount);
503   PM->TblMapMtx.unlock();
504 }
505