1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of OpenMP API interface functions.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "device.h"
14 #include "omptarget.h"
15 #include "private.h"
16 #include "rtl.h"
17 
18 #include <climits>
19 #include <cstdlib>
20 #include <cstring>
21 
22 EXTERN int omp_get_num_devices(void) {
23   TIMESCOPE();
24   PM->RTLsMtx.lock();
25   size_t DevicesSize = PM->Devices.size();
26   PM->RTLsMtx.unlock();
27 
28   DP("Call to omp_get_num_devices returning %zd\n", DevicesSize);
29 
30   return DevicesSize;
31 }
32 
33 EXTERN int omp_get_initial_device(void) {
34   TIMESCOPE();
35   int hostDevice = omp_get_num_devices();
36   DP("Call to omp_get_initial_device returning %d\n", hostDevice);
37   return hostDevice;
38 }
39 
40 EXTERN void *omp_target_alloc(size_t size, int device_num) {
41   TIMESCOPE();
42   DP("Call to omp_target_alloc for device %d requesting %zu bytes\n",
43      device_num, size);
44 
45   if (size <= 0) {
46     DP("Call to omp_target_alloc with non-positive length\n");
47     return NULL;
48   }
49 
50   void *rc = NULL;
51 
52   if (device_num == omp_get_initial_device()) {
53     rc = malloc(size);
54     DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc));
55     return rc;
56   }
57 
58   if (!device_is_ready(device_num)) {
59     DP("omp_target_alloc returns NULL ptr\n");
60     return NULL;
61   }
62 
63   rc = PM->Devices[device_num].allocData(size);
64   DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc));
65   return rc;
66 }
67 
68 EXTERN void omp_target_free(void *device_ptr, int device_num) {
69   TIMESCOPE();
70   DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
71      device_num, DPxPTR(device_ptr));
72 
73   if (!device_ptr) {
74     DP("Call to omp_target_free with NULL ptr\n");
75     return;
76   }
77 
78   if (device_num == omp_get_initial_device()) {
79     free(device_ptr);
80     DP("omp_target_free deallocated host ptr\n");
81     return;
82   }
83 
84   if (!device_is_ready(device_num)) {
85     DP("omp_target_free returns, nothing to do\n");
86     return;
87   }
88 
89   PM->Devices[device_num].deleteData(device_ptr);
90   DP("omp_target_free deallocated device ptr\n");
91 }
92 
93 EXTERN int omp_target_is_present(void *ptr, int device_num) {
94   TIMESCOPE();
95   DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
96      device_num, DPxPTR(ptr));
97 
98   if (!ptr) {
99     DP("Call to omp_target_is_present with NULL ptr, returning false\n");
100     return false;
101   }
102 
103   if (device_num == omp_get_initial_device()) {
104     DP("Call to omp_target_is_present on host, returning true\n");
105     return true;
106   }
107 
108   PM->RTLsMtx.lock();
109   size_t DevicesSize = PM->Devices.size();
110   PM->RTLsMtx.unlock();
111   if (DevicesSize <= (size_t)device_num) {
112     DP("Call to omp_target_is_present with invalid device ID, returning "
113        "false\n");
114     return false;
115   }
116 
117   DeviceTy &Device = PM->Devices[device_num];
118   bool IsLast; // not used
119   bool IsHostPtr;
120   void *TgtPtr = Device.getTgtPtrBegin(ptr, 0, IsLast, false, IsHostPtr);
121   int rc = (TgtPtr != NULL);
122   // Under unified memory the host pointer can be returned by the
123   // getTgtPtrBegin() function which means that there is no device
124   // corresponding point for ptr. This function should return false
125   // in that situation.
126   if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY)
127     rc = !IsHostPtr;
128   DP("Call to omp_target_is_present returns %d\n", rc);
129   return rc;
130 }
131 
132 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
133                              size_t dst_offset, size_t src_offset,
134                              int dst_device, int src_device) {
135   TIMESCOPE();
136   DP("Call to omp_target_memcpy, dst device %d, src device %d, "
137      "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
138      "src offset %zu, length %zu\n",
139      dst_device, src_device, DPxPTR(dst), DPxPTR(src), dst_offset, src_offset,
140      length);
141 
142   if (!dst || !src || length <= 0) {
143     if (length == 0) {
144       DP("Call to omp_target_memcpy with zero length, nothing to do\n");
145       return OFFLOAD_SUCCESS;
146     }
147 
148     REPORT("Call to omp_target_memcpy with invalid arguments\n");
149     return OFFLOAD_FAIL;
150   }
151 
152   if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) {
153     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
154     return OFFLOAD_FAIL;
155   }
156 
157   if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) {
158     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
159     return OFFLOAD_FAIL;
160   }
161 
162   int rc = OFFLOAD_SUCCESS;
163   void *srcAddr = (char *)src + src_offset;
164   void *dstAddr = (char *)dst + dst_offset;
165 
166   if (src_device == omp_get_initial_device() &&
167       dst_device == omp_get_initial_device()) {
168     DP("copy from host to host\n");
169     const void *p = memcpy(dstAddr, srcAddr, length);
170     if (p == NULL)
171       rc = OFFLOAD_FAIL;
172   } else if (src_device == omp_get_initial_device()) {
173     DP("copy from host to device\n");
174     DeviceTy &DstDev = PM->Devices[dst_device];
175     AsyncInfoTy AsyncInfo(DstDev);
176     rc = DstDev.submitData(dstAddr, srcAddr, length, AsyncInfo);
177   } else if (dst_device == omp_get_initial_device()) {
178     DP("copy from device to host\n");
179     DeviceTy &SrcDev = PM->Devices[src_device];
180     AsyncInfoTy AsyncInfo(SrcDev);
181     rc = SrcDev.retrieveData(dstAddr, srcAddr, length, AsyncInfo);
182   } else {
183     DP("copy from device to device\n");
184     DeviceTy &SrcDev = PM->Devices[src_device];
185     DeviceTy &DstDev = PM->Devices[dst_device];
186     // First try to use D2D memcpy which is more efficient. If fails, fall back
187     // to unefficient way.
188     if (SrcDev.isDataExchangable(DstDev)) {
189       AsyncInfoTy AsyncInfo(SrcDev);
190       rc = SrcDev.dataExchange(srcAddr, DstDev, dstAddr, length, AsyncInfo);
191       if (rc == OFFLOAD_SUCCESS)
192         return OFFLOAD_SUCCESS;
193     }
194 
195     void *buffer = malloc(length);
196     {
197       AsyncInfoTy AsyncInfo(SrcDev);
198       rc = SrcDev.retrieveData(buffer, srcAddr, length, AsyncInfo);
199     }
200     if (rc == OFFLOAD_SUCCESS) {
201       AsyncInfoTy AsyncInfo(SrcDev);
202       rc = DstDev.submitData(dstAddr, buffer, length, AsyncInfo);
203     }
204     free(buffer);
205   }
206 
207   DP("omp_target_memcpy returns %d\n", rc);
208   return rc;
209 }
210 
211 EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size,
212                                   int num_dims, const size_t *volume,
213                                   const size_t *dst_offsets,
214                                   const size_t *src_offsets,
215                                   const size_t *dst_dimensions,
216                                   const size_t *src_dimensions, int dst_device,
217                                   int src_device) {
218   TIMESCOPE();
219   DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
220      "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
221      "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
222      "volume " DPxMOD ", element size %zu, num_dims %d\n",
223      dst_device, src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets),
224      DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions),
225      DPxPTR(volume), element_size, num_dims);
226 
227   if (!(dst || src)) {
228     DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
229        INT_MAX);
230     return INT_MAX;
231   }
232 
233   if (!dst || !src || element_size < 1 || num_dims < 1 || !volume ||
234       !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) {
235     REPORT("Call to omp_target_memcpy_rect with invalid arguments\n");
236     return OFFLOAD_FAIL;
237   }
238 
239   int rc;
240   if (num_dims == 1) {
241     rc = omp_target_memcpy(
242         dst, src, element_size * volume[0], element_size * dst_offsets[0],
243         element_size * src_offsets[0], dst_device, src_device);
244   } else {
245     size_t dst_slice_size = element_size;
246     size_t src_slice_size = element_size;
247     for (int i = 1; i < num_dims; ++i) {
248       dst_slice_size *= dst_dimensions[i];
249       src_slice_size *= src_dimensions[i];
250     }
251 
252     size_t dst_off = dst_offsets[0] * dst_slice_size;
253     size_t src_off = src_offsets[0] * src_slice_size;
254     for (size_t i = 0; i < volume[0]; ++i) {
255       rc = omp_target_memcpy_rect(
256           (char *)dst + dst_off + dst_slice_size * i,
257           (char *)src + src_off + src_slice_size * i, element_size,
258           num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1,
259           dst_dimensions + 1, src_dimensions + 1, dst_device, src_device);
260 
261       if (rc) {
262         DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
263         return rc;
264       }
265     }
266   }
267 
268   DP("omp_target_memcpy_rect returns %d\n", rc);
269   return rc;
270 }
271 
272 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr,
273                                     size_t size, size_t device_offset,
274                                     int device_num) {
275   TIMESCOPE();
276   DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
277      "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
278      DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num);
279 
280   if (!host_ptr || !device_ptr || size <= 0) {
281     REPORT("Call to omp_target_associate_ptr with invalid arguments\n");
282     return OFFLOAD_FAIL;
283   }
284 
285   if (device_num == omp_get_initial_device()) {
286     REPORT("omp_target_associate_ptr: no association possible on the host\n");
287     return OFFLOAD_FAIL;
288   }
289 
290   if (!device_is_ready(device_num)) {
291     REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
292     return OFFLOAD_FAIL;
293   }
294 
295   DeviceTy &Device = PM->Devices[device_num];
296   void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset);
297   int rc = Device.associatePtr(host_ptr, device_addr, size);
298   DP("omp_target_associate_ptr returns %d\n", rc);
299   return rc;
300 }
301 
302 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) {
303   TIMESCOPE();
304   DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
305      "device_num %d\n",
306      DPxPTR(host_ptr), device_num);
307 
308   if (!host_ptr) {
309     REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n");
310     return OFFLOAD_FAIL;
311   }
312 
313   if (device_num == omp_get_initial_device()) {
314     REPORT(
315         "omp_target_disassociate_ptr: no association possible on the host\n");
316     return OFFLOAD_FAIL;
317   }
318 
319   if (!device_is_ready(device_num)) {
320     REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
321     return OFFLOAD_FAIL;
322   }
323 
324   DeviceTy &Device = PM->Devices[device_num];
325   int rc = Device.disassociatePtr(host_ptr);
326   DP("omp_target_disassociate_ptr returns %d\n", rc);
327   return rc;
328 }
329