1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of the interface to be used by Clang during the codegen of a
10 // target region.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include <omptarget.h>
15 
16 #include "device.h"
17 #include "private.h"
18 #include "rtl.h"
19 
20 #include <cassert>
21 #include <cstdlib>
22 #include <mutex>
23 
24 // Store target policy (disabled, mandatory, default)
25 kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default;
26 std::mutex TargetOffloadMtx;
27 
28 ////////////////////////////////////////////////////////////////////////////////
29 /// manage the success or failure of a target construct
30 
31 static void HandleDefaultTargetOffload() {
32   TargetOffloadMtx.lock();
33   if (TargetOffloadPolicy == tgt_default) {
34     if (omp_get_num_devices() > 0) {
35       DP("Default TARGET OFFLOAD policy is now mandatory "
36          "(devices were found)\n");
37       TargetOffloadPolicy = tgt_mandatory;
38     } else {
39       DP("Default TARGET OFFLOAD policy is now disabled "
40          "(no devices were found)\n");
41       TargetOffloadPolicy = tgt_disabled;
42     }
43   }
44   TargetOffloadMtx.unlock();
45 }
46 
47 static int IsOffloadDisabled() {
48   if (TargetOffloadPolicy == tgt_default) HandleDefaultTargetOffload();
49   return TargetOffloadPolicy == tgt_disabled;
50 }
51 
52 static void HandleTargetOutcome(bool success) {
53   switch (TargetOffloadPolicy) {
54     case tgt_disabled:
55       if (success) {
56         FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled");
57       }
58       break;
59     case tgt_default:
60       FATAL_MESSAGE0(1, "default offloading policy must be switched to "
61                         "mandatory or disabled");
62       break;
63     case tgt_mandatory:
64       if (!success) {
65         FATAL_MESSAGE0(1, "failure of target construct while offloading is mandatory");
66       }
67       break;
68   }
69 }
70 
71 ////////////////////////////////////////////////////////////////////////////////
72 /// adds requires flags
73 EXTERN void __tgt_register_requires(int64_t flags) {
74   RTLs->RegisterRequires(flags);
75 }
76 
77 ////////////////////////////////////////////////////////////////////////////////
78 /// adds a target shared library to the target execution image
79 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) {
80   RTLs->RegisterLib(desc);
81 }
82 
83 ////////////////////////////////////////////////////////////////////////////////
84 /// unloads a target shared library
85 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
86   RTLs->UnregisterLib(desc);
87 }
88 
89 /// creates host-to-target data mapping, stores it in the
90 /// libomptarget.so internal structure (an entry in a stack of data maps)
91 /// and passes the data to the device.
92 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
93     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
94   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
95       arg_sizes, arg_types, nullptr);
96 }
97 
98 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
99     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
100     int32_t depNum, void *depList, int32_t noAliasDepNum,
101     void *noAliasDepList) {
102   if (depNum + noAliasDepNum > 0)
103     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
104 
105   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
106       arg_sizes, arg_types, nullptr);
107 }
108 
109 EXTERN void __tgt_target_data_begin_mapper(int64_t device_id, int32_t arg_num,
110     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
111     void **arg_mappers) {
112   if (IsOffloadDisabled()) return;
113 
114   DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
115       device_id, arg_num);
116 
117   // No devices available?
118   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
119     device_id = omp_get_default_device();
120     DP("Use default device id %" PRId64 "\n", device_id);
121   }
122 
123   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
124     DP("Failed to get device %" PRId64 " ready\n", device_id);
125     HandleTargetOutcome(false);
126     return;
127   }
128 
129   DeviceTy &Device = Devices[device_id];
130 
131 #ifdef OMPTARGET_DEBUG
132   for (int i = 0; i < arg_num; ++i) {
133     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
134        ", Type=0x%" PRIx64 "\n",
135        i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i]);
136   }
137 #endif
138 
139   int rc = targetDataBegin(Device, arg_num, args_base, args, arg_sizes,
140                            arg_types, arg_mappers, nullptr);
141   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
142 }
143 
144 EXTERN void __tgt_target_data_begin_nowait_mapper(int64_t device_id,
145     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
146     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
147     int32_t noAliasDepNum, void *noAliasDepList) {
148   if (depNum + noAliasDepNum > 0)
149     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
150 
151   __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args,
152       arg_sizes, arg_types, arg_mappers);
153 }
154 
155 /// passes data from the target, releases target memory and destroys
156 /// the host-target mapping (top entry from the stack of data maps)
157 /// created by the last __tgt_target_data_begin.
158 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
159     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
160   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
161       arg_types, nullptr);
162 }
163 
164 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
165     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
166     int32_t depNum, void *depList, int32_t noAliasDepNum,
167     void *noAliasDepList) {
168   if (depNum + noAliasDepNum > 0)
169     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
170 
171   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
172       arg_types, nullptr);
173 }
174 
175 EXTERN void __tgt_target_data_end_mapper(int64_t device_id, int32_t arg_num,
176     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
177     void **arg_mappers) {
178   if (IsOffloadDisabled()) return;
179   DP("Entering data end region with %d mappings\n", arg_num);
180 
181   // No devices available?
182   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
183     device_id = omp_get_default_device();
184   }
185 
186   RTLsMtx->lock();
187   size_t Devices_size = Devices.size();
188   RTLsMtx->unlock();
189   if (Devices_size <= (size_t)device_id) {
190     DP("Device ID  %" PRId64 " does not have a matching RTL.\n", device_id);
191     HandleTargetOutcome(false);
192     return;
193   }
194 
195   DeviceTy &Device = Devices[device_id];
196   if (!Device.IsInit) {
197     DP("Uninit device: ignore");
198     HandleTargetOutcome(false);
199     return;
200   }
201 
202 #ifdef OMPTARGET_DEBUG
203   for (int i=0; i<arg_num; ++i) {
204     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
205         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
206         arg_sizes[i], arg_types[i]);
207   }
208 #endif
209 
210   int rc = targetDataEnd(Device, arg_num, args_base, args, arg_sizes, arg_types,
211                          arg_mappers, nullptr);
212   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
213 }
214 
215 EXTERN void __tgt_target_data_end_nowait_mapper(int64_t device_id,
216     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
217     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
218     int32_t noAliasDepNum, void *noAliasDepList) {
219   if (depNum + noAliasDepNum > 0)
220     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
221 
222   __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes,
223       arg_types, arg_mappers);
224 }
225 
226 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
227     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
228   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
229       arg_sizes, arg_types, nullptr);
230 }
231 
232 EXTERN void __tgt_target_data_update_nowait(int64_t device_id, int32_t arg_num,
233     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
234     int32_t depNum, void *depList, int32_t noAliasDepNum,
235     void *noAliasDepList) {
236   if (depNum + noAliasDepNum > 0)
237     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
238 
239   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
240       arg_sizes, arg_types, nullptr);
241 }
242 
243 EXTERN void __tgt_target_data_update_mapper(int64_t device_id, int32_t arg_num,
244     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
245     void **arg_mappers) {
246   if (IsOffloadDisabled()) return;
247   DP("Entering data update with %d mappings\n", arg_num);
248 
249   // No devices available?
250   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
251     device_id = omp_get_default_device();
252   }
253 
254   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
255     DP("Failed to get device %" PRId64 " ready\n", device_id);
256     HandleTargetOutcome(false);
257     return;
258   }
259 
260   DeviceTy& Device = Devices[device_id];
261   int rc = target_data_update(Device, arg_num, args_base,
262       args, arg_sizes, arg_types, arg_mappers);
263   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
264 }
265 
266 EXTERN void __tgt_target_data_update_nowait_mapper(int64_t device_id,
267     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
268     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
269     int32_t noAliasDepNum, void *noAliasDepList) {
270   if (depNum + noAliasDepNum > 0)
271     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
272 
273   __tgt_target_data_update_mapper(device_id, arg_num, args_base, args,
274       arg_sizes, arg_types, arg_mappers);
275 }
276 
277 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
278     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
279   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
280       arg_sizes, arg_types, nullptr);
281 }
282 
283 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
284     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
285     int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum,
286     void *noAliasDepList) {
287   if (depNum + noAliasDepNum > 0)
288     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
289 
290   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
291       arg_sizes, arg_types, nullptr);
292 }
293 
294 EXTERN int __tgt_target_mapper(int64_t device_id, void *host_ptr,
295     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
296     int64_t *arg_types, void **arg_mappers) {
297   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
298   DP("Entering target region with entry point " DPxMOD " and device Id %"
299       PRId64 "\n", DPxPTR(host_ptr), device_id);
300 
301   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
302     device_id = omp_get_default_device();
303   }
304 
305   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
306     DP("Failed to get device %" PRId64 " ready\n", device_id);
307     HandleTargetOutcome(false);
308     return OFFLOAD_FAIL;
309   }
310 
311 #ifdef OMPTARGET_DEBUG
312   for (int i=0; i<arg_num; ++i) {
313     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
314         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
315         arg_sizes[i], arg_types[i]);
316   }
317 #endif
318 
319   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
320       arg_types, arg_mappers, 0, 0, false /*team*/);
321   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
322   return rc;
323 }
324 
325 EXTERN int __tgt_target_nowait_mapper(int64_t device_id, void *host_ptr,
326     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
327     int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList,
328     int32_t noAliasDepNum, void *noAliasDepList) {
329   if (depNum + noAliasDepNum > 0)
330     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
331 
332   return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args,
333       arg_sizes, arg_types, arg_mappers);
334 }
335 
336 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
337     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
338     int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
339   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
340       args, arg_sizes, arg_types, nullptr, team_num, thread_limit);
341 }
342 
343 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr,
344     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
345     int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum,
346     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
347   if (depNum + noAliasDepNum > 0)
348     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
349 
350   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
351       args, arg_sizes, arg_types, nullptr, team_num, thread_limit);
352 }
353 
354 EXTERN int __tgt_target_teams_mapper(int64_t device_id, void *host_ptr,
355     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
356     int64_t *arg_types, void **arg_mappers, int32_t team_num, int32_t thread_limit) {
357   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
358   DP("Entering target region with entry point " DPxMOD " and device Id %"
359       PRId64 "\n", DPxPTR(host_ptr), device_id);
360 
361   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
362     device_id = omp_get_default_device();
363   }
364 
365   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
366     DP("Failed to get device %" PRId64 " ready\n", device_id);
367     HandleTargetOutcome(false);
368     return OFFLOAD_FAIL;
369   }
370 
371 #ifdef OMPTARGET_DEBUG
372   for (int i=0; i<arg_num; ++i) {
373     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
374         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
375         arg_sizes[i], arg_types[i]);
376   }
377 #endif
378 
379   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
380       arg_types, arg_mappers, team_num, thread_limit, true /*team*/);
381   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
382 
383   return rc;
384 }
385 
386 EXTERN int __tgt_target_teams_nowait_mapper(int64_t device_id, void *host_ptr,
387     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
388     int64_t *arg_types, void **arg_mappers, int32_t team_num,
389     int32_t thread_limit, int32_t depNum, void *depList, int32_t noAliasDepNum,
390     void *noAliasDepList) {
391   if (depNum + noAliasDepNum > 0)
392     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
393 
394   return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base,
395       args, arg_sizes, arg_types, arg_mappers, team_num, thread_limit);
396 }
397 
398 // Get the current number of components for a user-defined mapper.
399 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) {
400   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
401   int64_t size = MapperComponentsPtr->Components.size();
402   DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n",
403      DPxPTR(rt_mapper_handle), size);
404   return size;
405 }
406 
407 // Push back one component for a user-defined mapper.
408 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base,
409                                         void *begin, int64_t size,
410                                         int64_t type) {
411   DP("__tgt_push_mapper_component(Handle=" DPxMOD
412      ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
413      ", Type=0x%" PRIx64 ").\n",
414      DPxPTR(rt_mapper_handle), DPxPTR(base), DPxPTR(begin), size, type);
415   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
416   MapperComponentsPtr->Components.push_back(
417       MapComponentInfoTy(base, begin, size, type));
418 }
419 
420 EXTERN void __kmpc_push_target_tripcount(int64_t device_id,
421     uint64_t loop_tripcount) {
422   if (IsOffloadDisabled())
423     return;
424 
425   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
426     device_id = omp_get_default_device();
427   }
428 
429   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
430     DP("Failed to get device %" PRId64 " ready\n", device_id);
431     HandleTargetOutcome(false);
432     return;
433   }
434 
435   DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
436       loop_tripcount);
437   TblMapMtx->lock();
438   Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL),
439                                          loop_tripcount);
440   TblMapMtx->unlock();
441 }
442