1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of the interface to be used by Clang during the codegen of a
10 // target region.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include <omptarget.h>
15 
16 #include "device.h"
17 #include "private.h"
18 #include "rtl.h"
19 
20 #include <cassert>
21 #include <cstdlib>
22 #include <mutex>
23 
24 // Store target policy (disabled, mandatory, default)
25 kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default;
26 std::mutex TargetOffloadMtx;
27 
28 ////////////////////////////////////////////////////////////////////////////////
29 /// manage the success or failure of a target construct
30 
31 static void HandleDefaultTargetOffload() {
32   TargetOffloadMtx.lock();
33   if (TargetOffloadPolicy == tgt_default) {
34     if (omp_get_num_devices() > 0) {
35       DP("Default TARGET OFFLOAD policy is now mandatory "
36          "(devices were found)\n");
37       TargetOffloadPolicy = tgt_mandatory;
38     } else {
39       DP("Default TARGET OFFLOAD policy is now disabled "
40          "(no devices were found)\n");
41       TargetOffloadPolicy = tgt_disabled;
42     }
43   }
44   TargetOffloadMtx.unlock();
45 }
46 
47 static int IsOffloadDisabled() {
48   if (TargetOffloadPolicy == tgt_default) HandleDefaultTargetOffload();
49   return TargetOffloadPolicy == tgt_disabled;
50 }
51 
52 static void HandleTargetOutcome(bool success) {
53   switch (TargetOffloadPolicy) {
54     case tgt_disabled:
55       if (success) {
56         FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled");
57       }
58       break;
59     case tgt_default:
60       FATAL_MESSAGE0(1, "default offloading policy must be switched to "
61                         "mandatory or disabled");
62       break;
63     case tgt_mandatory:
64       if (!success) {
65         if (InfoLevel > 0)
66           MESSAGE0("LIBOMPTARGET_INFO is not supported yet");
67         FATAL_MESSAGE0(1, "failure of target construct while offloading is mandatory");
68       }
69       break;
70   }
71 }
72 
73 ////////////////////////////////////////////////////////////////////////////////
74 /// adds requires flags
75 EXTERN void __tgt_register_requires(int64_t flags) {
76   RTLs->RegisterRequires(flags);
77 }
78 
79 ////////////////////////////////////////////////////////////////////////////////
80 /// adds a target shared library to the target execution image
81 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) {
82   RTLs->RegisterLib(desc);
83 }
84 
85 ////////////////////////////////////////////////////////////////////////////////
86 /// unloads a target shared library
87 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
88   RTLs->UnregisterLib(desc);
89 }
90 
91 /// creates host-to-target data mapping, stores it in the
92 /// libomptarget.so internal structure (an entry in a stack of data maps)
93 /// and passes the data to the device.
94 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
95     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
96   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
97       arg_sizes, arg_types, nullptr);
98 }
99 
100 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
101     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
102     int32_t depNum, void *depList, int32_t noAliasDepNum,
103     void *noAliasDepList) {
104   if (depNum + noAliasDepNum > 0)
105     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
106 
107   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
108       arg_sizes, arg_types, nullptr);
109 }
110 
111 EXTERN void __tgt_target_data_begin_mapper(int64_t device_id, int32_t arg_num,
112     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
113     void **arg_mappers) {
114   if (IsOffloadDisabled()) return;
115 
116   DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
117       device_id, arg_num);
118 
119   // No devices available?
120   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
121     device_id = omp_get_default_device();
122     DP("Use default device id %" PRId64 "\n", device_id);
123   }
124 
125   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
126     DP("Failed to get device %" PRId64 " ready\n", device_id);
127     HandleTargetOutcome(false);
128     return;
129   }
130 
131   DeviceTy &Device = Devices[device_id];
132 
133 #ifdef OMPTARGET_DEBUG
134   for (int i = 0; i < arg_num; ++i) {
135     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
136        ", Type=0x%" PRIx64 "\n",
137        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i]);
138   }
139 #endif
140 
141   int rc = targetDataBegin(Device, arg_num, args_base, args, arg_sizes,
142                            arg_types, arg_mappers, nullptr);
143   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
144 }
145 
146 EXTERN void __tgt_target_data_begin_nowait_mapper(int64_t device_id,
147     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
148     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
149     int32_t noAliasDepNum, void *noAliasDepList) {
150   if (depNum + noAliasDepNum > 0)
151     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
152 
153   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
154       arg_sizes, arg_types, arg_mappers);
155 }
156 
157 /// passes data from the target, releases target memory and destroys
158 /// the host-target mapping (top entry from the stack of data maps)
159 /// created by the last __tgt_target_data_begin.
160 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
161     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
162   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
163       arg_types, nullptr);
164 }
165 
166 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
167     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
168     int32_t depNum, void *depList, int32_t noAliasDepNum,
169     void *noAliasDepList) {
170   if (depNum + noAliasDepNum > 0)
171     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
172 
173   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
174       arg_types, nullptr);
175 }
176 
177 EXTERN void __tgt_target_data_end_mapper(int64_t device_id, int32_t arg_num,
178     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
179     void **arg_mappers) {
180   if (IsOffloadDisabled()) return;
181   DP("Entering data end region with %d mappings\n", arg_num);
182 
183   // No devices available?
184   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
185     device_id = omp_get_default_device();
186   }
187 
188   RTLsMtx->lock();
189   size_t Devices_size = Devices.size();
190   RTLsMtx->unlock();
191   if (Devices_size <= (size_t)device_id) {
192     DP("Device ID  %" PRId64 " does not have a matching RTL.\n", device_id);
193     HandleTargetOutcome(false);
194     return;
195   }
196 
197   DeviceTy &Device = Devices[device_id];
198   if (!Device.IsInit) {
199     DP("Uninit device: ignore");
200     HandleTargetOutcome(false);
201     return;
202   }
203 
204 #ifdef OMPTARGET_DEBUG
205   for (int i=0; i<arg_num; ++i) {
206     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
207         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
208         arg_sizes[i], arg_types[i]);
209   }
210 #endif
211 
212   int rc = targetDataEnd(Device, arg_num, args_base, args, arg_sizes, arg_types,
213                          arg_mappers, nullptr);
214   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
215 }
216 
217 EXTERN void __tgt_target_data_end_nowait_mapper(int64_t device_id,
218     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
219     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
220     int32_t noAliasDepNum, void *noAliasDepList) {
221   if (depNum + noAliasDepNum > 0)
222     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
223 
224   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
225       arg_types, arg_mappers);
226 }
227 
228 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
229     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
230   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
231       arg_sizes, arg_types, nullptr);
232 }
233 
234 EXTERN void __tgt_target_data_update_nowait(int64_t device_id, int32_t arg_num,
235     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
236     int32_t depNum, void *depList, int32_t noAliasDepNum,
237     void *noAliasDepList) {
238   if (depNum + noAliasDepNum > 0)
239     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
240 
241   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
242       arg_sizes, arg_types, nullptr);
243 }
244 
245 EXTERN void __tgt_target_data_update_mapper(int64_t device_id, int32_t arg_num,
246     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
247     void **arg_mappers) {
248   if (IsOffloadDisabled()) return;
249   DP("Entering data update with %d mappings\n", arg_num);
250 
251   // No devices available?
252   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
253     device_id = omp_get_default_device();
254   }
255 
256   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
257     DP("Failed to get device %" PRId64 " ready\n", device_id);
258     HandleTargetOutcome(false);
259     return;
260   }
261 
262   DeviceTy& Device = Devices[device_id];
263   int rc = target_data_update(Device, arg_num, args_base,
264       args, arg_sizes, arg_types, arg_mappers);
265   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
266 }
267 
268 EXTERN void __tgt_target_data_update_nowait_mapper(int64_t device_id,
269     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
270     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
271     int32_t noAliasDepNum, void *noAliasDepList) {
272   if (depNum + noAliasDepNum > 0)
273     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
274 
275   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
276       arg_sizes, arg_types, arg_mappers);
277 }
278 
279 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
280     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
281   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
282       arg_sizes, arg_types, nullptr);
283 }
284 
285 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
286     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
287     int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum,
288     void *noAliasDepList) {
289   if (depNum + noAliasDepNum > 0)
290     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
291 
292   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
293       arg_sizes, arg_types, nullptr);
294 }
295 
296 EXTERN int __tgt_target_mapper(int64_t device_id, void *host_ptr,
297     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
298     int64_t *arg_types, void **arg_mappers) {
299   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
300   DP("Entering target region with entry point " DPxMOD " and device Id %"
301       PRId64 "\n", DPxPTR(host_ptr), device_id);
302 
303   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
304     device_id = omp_get_default_device();
305   }
306 
307   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
308     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
309     HandleTargetOutcome(false);
310     return OFFLOAD_FAIL;
311   }
312 
313 #ifdef OMPTARGET_DEBUG
314   for (int i=0; i<arg_num; ++i) {
315     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
316         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
317         arg_sizes[i], arg_types[i]);
318   }
319 #endif
320 
321   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
322       arg_types, arg_mappers, 0, 0, false /*team*/);
323   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
324   return rc;
325 }
326 
327 EXTERN int __tgt_target_nowait_mapper(int64_t device_id, void *host_ptr,
328     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
329     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
330     int32_t noAliasDepNum, void *noAliasDepList) {
331   if (depNum + noAliasDepNum > 0)
332     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
333 
334   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
335       arg_sizes, arg_types, arg_mappers);
336 }
337 
338 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
339     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
340     int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
341   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
342       args, arg_sizes, arg_types, nullptr, team_num, thread_limit);
343 }
344 
345 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr,
346     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
347     int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum,
348     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
349   if (depNum + noAliasDepNum > 0)
350     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
351 
352   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
353       args, arg_sizes, arg_types, nullptr, team_num, thread_limit);
354 }
355 
356 EXTERN int __tgt_target_teams_mapper(int64_t device_id, void *host_ptr,
357     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
358     int64_t *arg_types, void **arg_mappers, int32_t team_num, int32_t thread_limit) {
359   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
360   DP("Entering target region with entry point " DPxMOD " and device Id %"
361       PRId64 "\n", DPxPTR(host_ptr), device_id);
362 
363   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
364     device_id = omp_get_default_device();
365   }
366 
367   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
368     REPORT("Failed to get device %" PRId64 " ready\n", device_id);
369     HandleTargetOutcome(false);
370     return OFFLOAD_FAIL;
371   }
372 
373 #ifdef OMPTARGET_DEBUG
374   for (int i=0; i<arg_num; ++i) {
375     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
376         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
377         arg_sizes[i], arg_types[i]);
378   }
379 #endif
380 
381   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
382       arg_types, arg_mappers, team_num, thread_limit, true /*team*/);
383   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
384 
385   return rc;
386 }
387 
388 EXTERN int __tgt_target_teams_nowait_mapper(int64_t device_id, void *host_ptr,
389     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
390     int64_t *arg_types, void **arg_mappers, int32_t team_num,
391     int32_t thread_limit, int32_t depNum, void *depList, int32_t noAliasDepNum,
392     void *noAliasDepList) {
393   if (depNum + noAliasDepNum > 0)
394     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
395 
396   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
397       args, arg_sizes, arg_types, arg_mappers, team_num, thread_limit);
398 }
399 
400 // Get the current number of components for a user-defined mapper.
401 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) {
402   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
403   int64_t size = MapperComponentsPtr->Components.size();
404   DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n",
405      DPxPTR(rt_mapper_handle), size);
406   return size;
407 }
408 
409 // Push back one component for a user-defined mapper.
410 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base,
411                                         void *begin, int64_t size,
412                                         int64_t type) {
413   DP("__tgt_push_mapper_component(Handle=" DPxMOD
414      ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
415      ", Type=0x%" PRIx64 ").\n",
416      DPxPTR(rt_mapper_handle), DPxPTR(base), DPxPTR(begin), size, type);
417   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
418   MapperComponentsPtr->Components.push_back(
419       MapComponentInfoTy(base, begin, size, type));
420 }
421 
422 EXTERN void __kmpc_push_target_tripcount(int64_t device_id,
423     uint64_t loop_tripcount) {
424   if (IsOffloadDisabled())
425     return;
426 
427   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
428     device_id = omp_get_default_device();
429   }
430 
431   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
432     DP("Failed to get device %" PRId64 " ready\n", device_id);
433     HandleTargetOutcome(false);
434     return;
435   }
436 
437   DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
438       loop_tripcount);
439   TblMapMtx->lock();
440   Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL),
441                                          loop_tripcount);
442   TblMapMtx->unlock();
443 }
444