1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of the interface to be used by Clang during the codegen of a
10 // target region.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "device.h"
15 #include "private.h"
16 #include "rtl.h"
17 
18 #include <cassert>
19 #include <cstdio>
20 #include <cstdlib>
21 #include <mutex>
22 
23 // Store target policy (disabled, mandatory, default)
24 kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default;
25 std::mutex TargetOffloadMtx;
26 
27 ////////////////////////////////////////////////////////////////////////////////
28 /// manage the success or failure of a target construct
29 static void HandleDefaultTargetOffload() {
30   TargetOffloadMtx.lock();
31   if (TargetOffloadPolicy == tgt_default) {
32     if (omp_get_num_devices() > 0) {
33       DP("Default TARGET OFFLOAD policy is now mandatory "
34          "(devices were found)\n");
35       TargetOffloadPolicy = tgt_mandatory;
36     } else {
37       DP("Default TARGET OFFLOAD policy is now disabled "
38          "(no devices were found)\n");
39       TargetOffloadPolicy = tgt_disabled;
40     }
41   }
42   TargetOffloadMtx.unlock();
43 }
44 
45 static int IsOffloadDisabled() {
46   if (TargetOffloadPolicy == tgt_default) HandleDefaultTargetOffload();
47   return TargetOffloadPolicy == tgt_disabled;
48 }
49 
50 static void HandleTargetOutcome(bool success) {
51   switch (TargetOffloadPolicy) {
52     case tgt_disabled:
53       if (success) {
54         FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled");
55       }
56       break;
57     case tgt_default:
58       FATAL_MESSAGE0(1, "default offloading policy must be switched to "
59                         "mandatory or disabled");
60       break;
61     case tgt_mandatory:
62       if (!success) {
63         if (getInfoLevel() > 1)
64           for (const auto &Device : Devices)
65             dumpTargetPointerMappings(Device);
66         else
67           FAILURE_MESSAGE("run with env LIBOMPTARGET_INFO>1 to dump host-target"
68                           "pointer maps\n");
69 
70         FATAL_MESSAGE0(1, "failure of target construct while offloading is mandatory");
71       }
72       break;
73   }
74 }
75 
76 ////////////////////////////////////////////////////////////////////////////////
77 /// adds requires flags
78 EXTERN void __tgt_register_requires(int64_t flags) {
79   RTLs->RegisterRequires(flags);
80 }
81 
82 ////////////////////////////////////////////////////////////////////////////////
83 /// adds a target shared library to the target execution image
84 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) {
85   RTLs->RegisterLib(desc);
86 }
87 
88 ////////////////////////////////////////////////////////////////////////////////
89 /// unloads a target shared library
90 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
91   RTLs->UnregisterLib(desc);
92 }
93 
94 /// creates host-to-target data mapping, stores it in the
95 /// libomptarget.so internal structure (an entry in a stack of data maps)
96 /// and passes the data to the device.
97 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
98     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
99   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
100       arg_sizes, arg_types, nullptr);
101 }
102 
103 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
104     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
105     int32_t depNum, void *depList, int32_t noAliasDepNum,
106     void *noAliasDepList) {
107   if (depNum + noAliasDepNum > 0)
108     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
109 
110   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
111       arg_sizes, arg_types, nullptr);
112 }
113 
114 EXTERN void __tgt_target_data_begin_mapper(int64_t device_id, int32_t arg_num,
115     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
116     void **arg_mappers) {
117   if (IsOffloadDisabled()) return;
118 
119   DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
120       device_id, arg_num);
121 
122   // No devices available?
123   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
124     device_id = omp_get_default_device();
125     DP("Use default device id %" PRId64 "\n", device_id);
126   }
127 
128   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
129     DP("Failed to get device %" PRId64 " ready\n", device_id);
130     HandleTargetOutcome(false);
131     return;
132   }
133 
134   DeviceTy &Device = Devices[device_id];
135 
136 #ifdef OMPTARGET_DEBUG
137   for (int i = 0; i < arg_num; ++i) {
138     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
139        ", Type=0x%" PRIx64 "\n",
140        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i]);
141   }
142 #endif
143 
144   int rc = targetDataBegin(Device, arg_num, args_base, args, arg_sizes,
145                            arg_types, arg_mappers, nullptr);
146   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
147 }
148 
149 EXTERN void __tgt_target_data_begin_nowait_mapper(int64_t device_id,
150     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
151     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
152     int32_t noAliasDepNum, void *noAliasDepList) {
153   if (depNum + noAliasDepNum > 0)
154     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
155 
156   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
157       arg_sizes, arg_types, arg_mappers);
158 }
159 
160 /// passes data from the target, releases target memory and destroys
161 /// the host-target mapping (top entry from the stack of data maps)
162 /// created by the last __tgt_target_data_begin.
163 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
164     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
165   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
166       arg_types, nullptr);
167 }
168 
169 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
170     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
171     int32_t depNum, void *depList, int32_t noAliasDepNum,
172     void *noAliasDepList) {
173   if (depNum + noAliasDepNum > 0)
174     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
175 
176   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
177       arg_types, nullptr);
178 }
179 
180 EXTERN void __tgt_target_data_end_mapper(int64_t device_id, int32_t arg_num,
181     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
182     void **arg_mappers) {
183   if (IsOffloadDisabled()) return;
184   DP("Entering data end region with %d mappings\n", arg_num);
185 
186   // No devices available?
187   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
188     device_id = omp_get_default_device();
189   }
190 
191   RTLsMtx->lock();
192   size_t Devices_size = Devices.size();
193   RTLsMtx->unlock();
194   if (Devices_size <= (size_t)device_id) {
195     DP("Device ID  %" PRId64 " does not have a matching RTL.\n", device_id);
196     HandleTargetOutcome(false);
197     return;
198   }
199 
200   DeviceTy &Device = Devices[device_id];
201   if (!Device.IsInit) {
202     DP("Uninit device: ignore");
203     HandleTargetOutcome(false);
204     return;
205   }
206 
207 #ifdef OMPTARGET_DEBUG
208   for (int i=0; i<arg_num; ++i) {
209     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
210         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
211         arg_sizes[i], arg_types[i]);
212   }
213 #endif
214 
215   int rc = targetDataEnd(Device, arg_num, args_base, args, arg_sizes, arg_types,
216                          arg_mappers, nullptr);
217   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
218 }
219 
220 EXTERN void __tgt_target_data_end_nowait_mapper(int64_t device_id,
221     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
222     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
223     int32_t noAliasDepNum, void *noAliasDepList) {
224   if (depNum + noAliasDepNum > 0)
225     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
226 
227   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
228       arg_types, arg_mappers);
229 }
230 
231 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
232     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
233   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
234       arg_sizes, arg_types, nullptr);
235 }
236 
237 EXTERN void __tgt_target_data_update_nowait(int64_t device_id, int32_t arg_num,
238     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
239     int32_t depNum, void *depList, int32_t noAliasDepNum,
240     void *noAliasDepList) {
241   if (depNum + noAliasDepNum > 0)
242     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
243 
244   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
245       arg_sizes, arg_types, nullptr);
246 }
247 
248 EXTERN void __tgt_target_data_update_mapper(int64_t device_id, int32_t arg_num,
249     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
250     void **arg_mappers) {
251   if (IsOffloadDisabled()) return;
252   DP("Entering data update with %d mappings\n", arg_num);
253 
254   // No devices available?
255   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
256     device_id = omp_get_default_device();
257   }
258 
259   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
260     DP("Failed to get device %" PRId64 " ready\n", device_id);
261     HandleTargetOutcome(false);
262     return;
263   }
264 
265   DeviceTy& Device = Devices[device_id];
266   int rc = target_data_update(Device, arg_num, args_base,
267       args, arg_sizes, arg_types, arg_mappers);
268   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
269 }
270 
271 EXTERN void __tgt_target_data_update_nowait_mapper(int64_t device_id,
272     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
273     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
274     int32_t noAliasDepNum, void *noAliasDepList) {
275   if (depNum + noAliasDepNum > 0)
276     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
277 
278   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
279       arg_sizes, arg_types, arg_mappers);
280 }
281 
282 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
283     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
284   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
285       arg_sizes, arg_types, nullptr);
286 }
287 
288 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
289     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
290     int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum,
291     void *noAliasDepList) {
292   if (depNum + noAliasDepNum > 0)
293     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
294 
295   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
296       arg_sizes, arg_types, nullptr);
297 }
298 
299 EXTERN int __tgt_target_mapper(int64_t device_id, void *host_ptr,
300     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
301     int64_t *arg_types, void **arg_mappers) {
302   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
303   DP("Entering target region with entry point " DPxMOD " and device Id %"
304       PRId64 "\n", DPxPTR(host_ptr), device_id);
305 
306   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
307     device_id = omp_get_default_device();
308   }
309 
310   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
311     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
312     HandleTargetOutcome(false);
313     return OFFLOAD_FAIL;
314   }
315 
316 #ifdef OMPTARGET_DEBUG
317   for (int i=0; i<arg_num; ++i) {
318     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
319         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
320         arg_sizes[i], arg_types[i]);
321   }
322 #endif
323 
324   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
325       arg_types, arg_mappers, 0, 0, false /*team*/);
326   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
327   return rc;
328 }
329 
330 EXTERN int __tgt_target_nowait_mapper(int64_t device_id, void *host_ptr,
331     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
332     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
333     int32_t noAliasDepNum, void *noAliasDepList) {
334   if (depNum + noAliasDepNum > 0)
335     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
336 
337   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
338       arg_sizes, arg_types, arg_mappers);
339 }
340 
341 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
342     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
343     int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
344   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
345       args, arg_sizes, arg_types, nullptr, team_num, thread_limit);
346 }
347 
348 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr,
349     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
350     int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum,
351     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
352   if (depNum + noAliasDepNum > 0)
353     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
354 
355   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
356       args, arg_sizes, arg_types, nullptr, team_num, thread_limit);
357 }
358 
359 EXTERN int __tgt_target_teams_mapper(int64_t device_id, void *host_ptr,
360     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
361     int64_t *arg_types, void **arg_mappers, int32_t team_num, int32_t thread_limit) {
362   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
363   DP("Entering target region with entry point " DPxMOD " and device Id %"
364       PRId64 "\n", DPxPTR(host_ptr), device_id);
365 
366   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
367     device_id = omp_get_default_device();
368   }
369 
370   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
371     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
372     HandleTargetOutcome(false);
373     return OFFLOAD_FAIL;
374   }
375 
376 #ifdef OMPTARGET_DEBUG
377   for (int i=0; i<arg_num; ++i) {
378     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
379         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
380         arg_sizes[i], arg_types[i]);
381   }
382 #endif
383 
384   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
385       arg_types, arg_mappers, team_num, thread_limit, true /*team*/);
386   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
387 
388   return rc;
389 }
390 
391 EXTERN int __tgt_target_teams_nowait_mapper(int64_t device_id, void *host_ptr,
392     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
393     int64_t *arg_types, void **arg_mappers, int32_t team_num,
394     int32_t thread_limit, int32_t depNum, void *depList, int32_t noAliasDepNum,
395     void *noAliasDepList) {
396   if (depNum + noAliasDepNum > 0)
397     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
398 
399   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
400       args, arg_sizes, arg_types, arg_mappers, team_num, thread_limit);
401 }
402 
403 // Get the current number of components for a user-defined mapper.
404 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) {
405   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
406   int64_t size = MapperComponentsPtr->Components.size();
407   DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n",
408      DPxPTR(rt_mapper_handle), size);
409   return size;
410 }
411 
412 // Push back one component for a user-defined mapper.
413 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base,
414                                         void *begin, int64_t size,
415                                         int64_t type) {
416   DP("__tgt_push_mapper_component(Handle=" DPxMOD
417      ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
418      ", Type=0x%" PRIx64 ").\n",
419      DPxPTR(rt_mapper_handle), DPxPTR(base), DPxPTR(begin), size, type);
420   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
421   MapperComponentsPtr->Components.push_back(
422       MapComponentInfoTy(base, begin, size, type));
423 }
424 
425 EXTERN void __kmpc_push_target_tripcount(int64_t device_id,
426     uint64_t loop_tripcount) {
427   if (IsOffloadDisabled())
428     return;
429 
430   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
431     device_id = omp_get_default_device();
432   }
433 
434   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
435     DP("Failed to get device %" PRId64 " ready\n", device_id);
436     HandleTargetOutcome(false);
437     return;
438   }
439 
440   DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
441       loop_tripcount);
442   TblMapMtx->lock();
443   Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL),
444                                          loop_tripcount);
445   TblMapMtx->unlock();
446 }
447