1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of the interface to be used by Clang during the codegen of a
10 // target region.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "device.h"
15 #include "private.h"
16 #include "rtl.h"
17 
18 #include <cassert>
19 #include <cstdio>
20 #include <cstdlib>
21 #include <mutex>
22 
23 ////////////////////////////////////////////////////////////////////////////////
24 /// manage the success or failure of a target construct
25 static void HandleDefaultTargetOffload() {
26   PM->TargetOffloadMtx.lock();
27   if (PM->TargetOffloadPolicy == tgt_default) {
28     if (omp_get_num_devices() > 0) {
29       DP("Default TARGET OFFLOAD policy is now mandatory "
30          "(devices were found)\n");
31       PM->TargetOffloadPolicy = tgt_mandatory;
32     } else {
33       DP("Default TARGET OFFLOAD policy is now disabled "
34          "(no devices were found)\n");
35       PM->TargetOffloadPolicy = tgt_disabled;
36     }
37   }
38   PM->TargetOffloadMtx.unlock();
39 }
40 
41 static int IsOffloadDisabled() {
42   if (PM->TargetOffloadPolicy == tgt_default)
43     HandleDefaultTargetOffload();
44   return PM->TargetOffloadPolicy == tgt_disabled;
45 }
46 
47 static void HandleTargetOutcome(bool success) {
48   switch (PM->TargetOffloadPolicy) {
49   case tgt_disabled:
50     if (success) {
51       FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled");
52     }
53     break;
54   case tgt_default:
55     FATAL_MESSAGE0(1, "default offloading policy must be switched to "
56                       "mandatory or disabled");
57     break;
58   case tgt_mandatory:
59     if (!success) {
60       if (getInfoLevel() > 1)
61         for (const auto &Device : PM->Devices)
62           dumpTargetPointerMappings(Device);
63       else
64         FAILURE_MESSAGE("run with env LIBOMPTARGET_INFO>1 to dump host-target"
65                         "pointer maps\n");
66 
67       FATAL_MESSAGE0(
68           1, "failure of target construct while offloading is mandatory");
69     }
70     break;
71   }
72 }
73 
74 ////////////////////////////////////////////////////////////////////////////////
75 /// adds requires flags
76 EXTERN void __tgt_register_requires(int64_t flags) {
77   PM->RTLs.RegisterRequires(flags);
78 }
79 
80 ////////////////////////////////////////////////////////////////////////////////
81 /// adds a target shared library to the target execution image
82 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) {
83   PM->RTLs.RegisterLib(desc);
84 }
85 
86 ////////////////////////////////////////////////////////////////////////////////
87 /// unloads a target shared library
88 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
89   PM->RTLs.UnregisterLib(desc);
90 }
91 
92 /// creates host-to-target data mapping, stores it in the
93 /// libomptarget.so internal structure (an entry in a stack of data maps)
94 /// and passes the data to the device.
95 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
96     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
97   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
98       arg_sizes, arg_types, nullptr);
99 }
100 
101 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
102     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
103     int32_t depNum, void *depList, int32_t noAliasDepNum,
104     void *noAliasDepList) {
105   if (depNum + noAliasDepNum > 0)
106     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
107 
108   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
109       arg_sizes, arg_types, nullptr);
110 }
111 
112 EXTERN void __tgt_target_data_begin_mapper(int64_t device_id, int32_t arg_num,
113     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
114     void **arg_mappers) {
115   if (IsOffloadDisabled()) return;
116 
117   DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
118       device_id, arg_num);
119 
120   // No devices available?
121   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
122     device_id = omp_get_default_device();
123     DP("Use default device id %" PRId64 "\n", device_id);
124   }
125 
126   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
127     DP("Failed to get device %" PRId64 " ready\n", device_id);
128     HandleTargetOutcome(false);
129     return;
130   }
131 
132   DeviceTy &Device = PM->Devices[device_id];
133 
134 #ifdef OMPTARGET_DEBUG
135   for (int i = 0; i < arg_num; ++i) {
136     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
137        ", Type=0x%" PRIx64 "\n",
138        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i]);
139   }
140 #endif
141 
142   int rc = targetDataBegin(Device, arg_num, args_base, args, arg_sizes,
143                            arg_types, arg_mappers, nullptr);
144   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
145 }
146 
147 EXTERN void __tgt_target_data_begin_nowait_mapper(int64_t device_id,
148     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
149     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
150     int32_t noAliasDepNum, void *noAliasDepList) {
151   if (depNum + noAliasDepNum > 0)
152     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
153 
154   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
155       arg_sizes, arg_types, arg_mappers);
156 }
157 
158 /// passes data from the target, releases target memory and destroys
159 /// the host-target mapping (top entry from the stack of data maps)
160 /// created by the last __tgt_target_data_begin.
161 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
162     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
163   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
164       arg_types, nullptr);
165 }
166 
167 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
168     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
169     int32_t depNum, void *depList, int32_t noAliasDepNum,
170     void *noAliasDepList) {
171   if (depNum + noAliasDepNum > 0)
172     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
173 
174   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
175       arg_types, nullptr);
176 }
177 
178 EXTERN void __tgt_target_data_end_mapper(int64_t device_id, int32_t arg_num,
179     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
180     void **arg_mappers) {
181   if (IsOffloadDisabled()) return;
182   DP("Entering data end region with %d mappings\n", arg_num);
183 
184   // No devices available?
185   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
186     device_id = omp_get_default_device();
187   }
188 
189   PM->RTLsMtx.lock();
190   size_t DevicesSize = PM->Devices.size();
191   PM->RTLsMtx.unlock();
192   if (DevicesSize <= (size_t)device_id) {
193     DP("Device ID  %" PRId64 " does not have a matching RTL.\n", device_id);
194     HandleTargetOutcome(false);
195     return;
196   }
197 
198   DeviceTy &Device = PM->Devices[device_id];
199   if (!Device.IsInit) {
200     DP("Uninit device: ignore");
201     HandleTargetOutcome(false);
202     return;
203   }
204 
205 #ifdef OMPTARGET_DEBUG
206   for (int i=0; i<arg_num; ++i) {
207     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
208         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
209         arg_sizes[i], arg_types[i]);
210   }
211 #endif
212 
213   int rc = targetDataEnd(Device, arg_num, args_base, args, arg_sizes, arg_types,
214                          arg_mappers, nullptr);
215   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
216 }
217 
218 EXTERN void __tgt_target_data_end_nowait_mapper(int64_t device_id,
219     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
220     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
221     int32_t noAliasDepNum, void *noAliasDepList) {
222   if (depNum + noAliasDepNum > 0)
223     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
224 
225   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
226       arg_types, arg_mappers);
227 }
228 
229 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
230     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
231   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
232       arg_sizes, arg_types, nullptr);
233 }
234 
235 EXTERN void __tgt_target_data_update_nowait(int64_t device_id, int32_t arg_num,
236     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
237     int32_t depNum, void *depList, int32_t noAliasDepNum,
238     void *noAliasDepList) {
239   if (depNum + noAliasDepNum > 0)
240     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
241 
242   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
243       arg_sizes, arg_types, nullptr);
244 }
245 
246 EXTERN void __tgt_target_data_update_mapper(int64_t device_id, int32_t arg_num,
247     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
248     void **arg_mappers) {
249   if (IsOffloadDisabled()) return;
250   DP("Entering data update with %d mappings\n", arg_num);
251 
252   // No devices available?
253   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
254     device_id = omp_get_default_device();
255   }
256 
257   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
258     DP("Failed to get device %" PRId64 " ready\n", device_id);
259     HandleTargetOutcome(false);
260     return;
261   }
262 
263   DeviceTy &Device = PM->Devices[device_id];
264   int rc = target_data_update(Device, arg_num, args_base,
265       args, arg_sizes, arg_types, arg_mappers);
266   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
267 }
268 
269 EXTERN void __tgt_target_data_update_nowait_mapper(int64_t device_id,
270     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
271     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
272     int32_t noAliasDepNum, void *noAliasDepList) {
273   if (depNum + noAliasDepNum > 0)
274     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
275 
276   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
277       arg_sizes, arg_types, arg_mappers);
278 }
279 
280 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
281     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
282   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
283       arg_sizes, arg_types, nullptr);
284 }
285 
286 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
287     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
288     int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum,
289     void *noAliasDepList) {
290   if (depNum + noAliasDepNum > 0)
291     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
292 
293   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
294       arg_sizes, arg_types, nullptr);
295 }
296 
297 EXTERN int __tgt_target_mapper(int64_t device_id, void *host_ptr,
298     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
299     int64_t *arg_types, void **arg_mappers) {
300   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
301   DP("Entering target region with entry point " DPxMOD " and device Id %"
302       PRId64 "\n", DPxPTR(host_ptr), device_id);
303 
304   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
305     device_id = omp_get_default_device();
306   }
307 
308   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
309     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
310     HandleTargetOutcome(false);
311     return OFFLOAD_FAIL;
312   }
313 
314 #ifdef OMPTARGET_DEBUG
315   for (int i=0; i<arg_num; ++i) {
316     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
317         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
318         arg_sizes[i], arg_types[i]);
319   }
320 #endif
321 
322   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
323       arg_types, arg_mappers, 0, 0, false /*team*/);
324   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
325   return rc;
326 }
327 
328 EXTERN int __tgt_target_nowait_mapper(int64_t device_id, void *host_ptr,
329     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
330     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
331     int32_t noAliasDepNum, void *noAliasDepList) {
332   if (depNum + noAliasDepNum > 0)
333     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
334 
335   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
336       arg_sizes, arg_types, arg_mappers);
337 }
338 
339 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
340     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
341     int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
342   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
343       args, arg_sizes, arg_types, nullptr, team_num, thread_limit);
344 }
345 
346 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr,
347     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
348     int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum,
349     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
350   if (depNum + noAliasDepNum > 0)
351     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
352 
353   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
354       args, arg_sizes, arg_types, nullptr, team_num, thread_limit);
355 }
356 
357 EXTERN int __tgt_target_teams_mapper(int64_t device_id, void *host_ptr,
358     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
359     int64_t *arg_types, void **arg_mappers, int32_t team_num, int32_t thread_limit) {
360   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
361   DP("Entering target region with entry point " DPxMOD " and device Id %"
362       PRId64 "\n", DPxPTR(host_ptr), device_id);
363 
364   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
365     device_id = omp_get_default_device();
366   }
367 
368   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
369     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
370     HandleTargetOutcome(false);
371     return OFFLOAD_FAIL;
372   }
373 
374 #ifdef OMPTARGET_DEBUG
375   for (int i=0; i<arg_num; ++i) {
376     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
377         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
378         arg_sizes[i], arg_types[i]);
379   }
380 #endif
381 
382   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
383       arg_types, arg_mappers, team_num, thread_limit, true /*team*/);
384   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
385 
386   return rc;
387 }
388 
389 EXTERN int __tgt_target_teams_nowait_mapper(int64_t device_id, void *host_ptr,
390     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
391     int64_t *arg_types, void **arg_mappers, int32_t team_num,
392     int32_t thread_limit, int32_t depNum, void *depList, int32_t noAliasDepNum,
393     void *noAliasDepList) {
394   if (depNum + noAliasDepNum > 0)
395     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
396 
397   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
398       args, arg_sizes, arg_types, arg_mappers, team_num, thread_limit);
399 }
400 
401 // Get the current number of components for a user-defined mapper.
402 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) {
403   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
404   int64_t size = MapperComponentsPtr->Components.size();
405   DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n",
406      DPxPTR(rt_mapper_handle), size);
407   return size;
408 }
409 
410 // Push back one component for a user-defined mapper.
411 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base,
412                                         void *begin, int64_t size,
413                                         int64_t type) {
414   DP("__tgt_push_mapper_component(Handle=" DPxMOD
415      ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
416      ", Type=0x%" PRIx64 ").\n",
417      DPxPTR(rt_mapper_handle), DPxPTR(base), DPxPTR(begin), size, type);
418   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
419   MapperComponentsPtr->Components.push_back(
420       MapComponentInfoTy(base, begin, size, type));
421 }
422 
423 EXTERN void __kmpc_push_target_tripcount(int64_t device_id,
424     uint64_t loop_tripcount) {
425   if (IsOffloadDisabled())
426     return;
427 
428   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
429     device_id = omp_get_default_device();
430   }
431 
432   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
433     DP("Failed to get device %" PRId64 " ready\n", device_id);
434     HandleTargetOutcome(false);
435     return;
436   }
437 
438   DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
439       loop_tripcount);
440   PM->TblMapMtx.lock();
441   PM->Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL),
442                                              loop_tripcount);
443   PM->TblMapMtx.unlock();
444 }
445