1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of the interface to be used by Clang during the codegen of a
10 // target region.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include <omptarget.h>
15 
16 #include "device.h"
17 #include "private.h"
18 #include "rtl.h"
19 
20 #include <cassert>
21 #include <cstdlib>
22 #include <mutex>
23 
24 // Store target policy (disabled, mandatory, default)
25 kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default;
26 std::mutex TargetOffloadMtx;
27 
28 ////////////////////////////////////////////////////////////////////////////////
29 /// manage the success or failure of a target constuct
30 
31 static void HandleDefaultTargetOffload() {
32   TargetOffloadMtx.lock();
33   if (TargetOffloadPolicy == tgt_default) {
34     if (omp_get_num_devices() > 0) {
35       DP("Default TARGET OFFLOAD policy is now mandatory "
36          "(devicew were found)\n");
37       TargetOffloadPolicy = tgt_mandatory;
38     } else {
39       DP("Default TARGET OFFLOAD policy is now disabled "
40          "(devices were not found)\n");
41       TargetOffloadPolicy = tgt_disabled;
42     }
43   }
44   TargetOffloadMtx.unlock();
45 }
46 
47 static int IsOffloadDisabled() {
48   if (TargetOffloadPolicy == tgt_default) HandleDefaultTargetOffload();
49   return TargetOffloadPolicy == tgt_disabled;
50 }
51 
52 static void HandleTargetOutcome(bool success) {
53   switch (TargetOffloadPolicy) {
54     case tgt_disabled:
55       if (success) {
56         FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled");
57       }
58       break;
59     case tgt_default:
60         FATAL_MESSAGE0(1, "default offloading policy must switched to "
61             "mandatory or disabled");
62       break;
63     case tgt_mandatory:
64       if (!success) {
65         FATAL_MESSAGE0(1, "failure of target construct while offloading is mandatory");
66       }
67       break;
68   }
69 }
70 
71 ////////////////////////////////////////////////////////////////////////////////
72 /// adds a target shared library to the target execution image
73 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) {
74   RTLs.RegisterLib(desc);
75 }
76 
77 ////////////////////////////////////////////////////////////////////////////////
78 /// unloads a target shared library
79 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
80   RTLs.UnregisterLib(desc);
81 }
82 
83 /// creates host-to-target data mapping, stores it in the
84 /// libomptarget.so internal structure (an entry in a stack of data maps)
85 /// and passes the data to the device.
86 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
87     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
88   if (IsOffloadDisabled()) return;
89 
90   DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
91       device_id, arg_num);
92 
93   // No devices available?
94   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
95     device_id = omp_get_default_device();
96     DP("Use default device id %" PRId64 "\n", device_id);
97   }
98 
99   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
100     DP("Failed to get device %" PRId64 " ready\n", device_id);
101     HandleTargetOutcome(false);
102     return;
103   }
104 
105   DeviceTy& Device = Devices[device_id];
106 
107 #ifdef OMPTARGET_DEBUG
108   for (int i=0; i<arg_num; ++i) {
109     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
110         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
111         arg_sizes[i], arg_types[i]);
112   }
113 #endif
114 
115   int rc = target_data_begin(Device, arg_num, args_base,
116       args, arg_sizes, arg_types);
117   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
118 }
119 
120 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
121     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
122     int32_t depNum, void *depList, int32_t noAliasDepNum,
123     void *noAliasDepList) {
124   if (depNum + noAliasDepNum > 0)
125     __kmpc_omp_taskwait(NULL, 0);
126 
127   __tgt_target_data_begin(device_id, arg_num, args_base, args, arg_sizes,
128                           arg_types);
129 }
130 
131 /// passes data from the target, releases target memory and destroys
132 /// the host-target mapping (top entry from the stack of data maps)
133 /// created by the last __tgt_target_data_begin.
134 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
135     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
136   if (IsOffloadDisabled()) return;
137   DP("Entering data end region with %d mappings\n", arg_num);
138 
139   // No devices available?
140   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
141     device_id = omp_get_default_device();
142   }
143 
144   RTLsMtx.lock();
145   size_t Devices_size = Devices.size();
146   RTLsMtx.unlock();
147   if (Devices_size <= (size_t)device_id) {
148     DP("Device ID  %" PRId64 " does not have a matching RTL.\n", device_id);
149     HandleTargetOutcome(false);
150     return;
151   }
152 
153   DeviceTy &Device = Devices[device_id];
154   if (!Device.IsInit) {
155     DP("Uninit device: ignore");
156     HandleTargetOutcome(false);
157     return;
158   }
159 
160 #ifdef OMPTARGET_DEBUG
161   for (int i=0; i<arg_num; ++i) {
162     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
163         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
164         arg_sizes[i], arg_types[i]);
165   }
166 #endif
167 
168   int rc = target_data_end(Device, arg_num, args_base,
169       args, arg_sizes, arg_types);
170   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
171 }
172 
173 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
174     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
175     int32_t depNum, void *depList, int32_t noAliasDepNum,
176     void *noAliasDepList) {
177   if (depNum + noAliasDepNum > 0)
178     __kmpc_omp_taskwait(NULL, 0);
179 
180   __tgt_target_data_end(device_id, arg_num, args_base, args, arg_sizes,
181                         arg_types);
182 }
183 
184 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
185     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
186   if (IsOffloadDisabled()) return;
187   DP("Entering data update with %d mappings\n", arg_num);
188 
189   // No devices available?
190   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
191     device_id = omp_get_default_device();
192   }
193 
194   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
195     DP("Failed to get device %" PRId64 " ready\n", device_id);
196     HandleTargetOutcome(false);
197     return;
198   }
199 
200   DeviceTy& Device = Devices[device_id];
201   int rc = target_data_update(Device, arg_num, args_base,
202       args, arg_sizes, arg_types);
203   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
204 }
205 
206 EXTERN void __tgt_target_data_update_nowait(
207     int64_t device_id, int32_t arg_num, void **args_base, void **args,
208     int64_t *arg_sizes, int64_t *arg_types, int32_t depNum, void *depList,
209     int32_t noAliasDepNum, void *noAliasDepList) {
210   if (depNum + noAliasDepNum > 0)
211     __kmpc_omp_taskwait(NULL, 0);
212 
213   __tgt_target_data_update(device_id, arg_num, args_base, args, arg_sizes,
214                            arg_types);
215 }
216 
217 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
218     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
219   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
220   DP("Entering target region with entry point " DPxMOD " and device Id %"
221       PRId64 "\n", DPxPTR(host_ptr), device_id);
222 
223   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
224     device_id = omp_get_default_device();
225   }
226 
227   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
228     DP("Failed to get device %" PRId64 " ready\n", device_id);
229     HandleTargetOutcome(false);
230     return OFFLOAD_FAIL;
231   }
232 
233 #ifdef OMPTARGET_DEBUG
234   for (int i=0; i<arg_num; ++i) {
235     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
236         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
237         arg_sizes[i], arg_types[i]);
238   }
239 #endif
240 
241   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
242       arg_types, 0, 0, false /*team*/);
243   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
244   return rc;
245 }
246 
247 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
248     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
249     int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum,
250     void *noAliasDepList) {
251   if (depNum + noAliasDepNum > 0)
252     __kmpc_omp_taskwait(NULL, 0);
253 
254   return __tgt_target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
255                       arg_types);
256 }
257 
258 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
259     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
260     int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
261   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
262   DP("Entering target region with entry point " DPxMOD " and device Id %"
263       PRId64 "\n", DPxPTR(host_ptr), device_id);
264 
265   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
266     device_id = omp_get_default_device();
267   }
268 
269   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
270     DP("Failed to get device %" PRId64 " ready\n", device_id);
271     HandleTargetOutcome(false);
272     return OFFLOAD_FAIL;
273   }
274 
275 #ifdef OMPTARGET_DEBUG
276   for (int i=0; i<arg_num; ++i) {
277     DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
278         ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
279         arg_sizes[i], arg_types[i]);
280   }
281 #endif
282 
283   int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
284       arg_types, team_num, thread_limit, true /*team*/);
285   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
286 
287   return rc;
288 }
289 
290 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr,
291     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
292     int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum,
293     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
294   if (depNum + noAliasDepNum > 0)
295     __kmpc_omp_taskwait(NULL, 0);
296 
297   return __tgt_target_teams(device_id, host_ptr, arg_num, args_base, args,
298                             arg_sizes, arg_types, team_num, thread_limit);
299 }
300 
301 
302 // The trip count mechanism will be revised - this scheme is not thread-safe.
303 EXTERN void __kmpc_push_target_tripcount(int64_t device_id,
304     uint64_t loop_tripcount) {
305   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
306     device_id = omp_get_default_device();
307   }
308 
309   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
310     DP("Failed to get device %" PRId64 " ready\n", device_id);
311     HandleTargetOutcome(false);
312     return;
313   }
314 
315   DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
316       loop_tripcount);
317   Devices[device_id].loopTripCnt = loop_tripcount;
318 }
319