1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of OpenMP API interface functions.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "device.h"
14 #include "omptarget.h"
15 #include "private.h"
16 #include "rtl.h"
17 
18 #include <climits>
19 #include <cstdlib>
20 #include <cstring>
21 
22 EXTERN int omp_get_num_devices(void) {
23   TIMESCOPE();
24   PM->RTLsMtx.lock();
25   size_t DevicesSize = PM->Devices.size();
26   PM->RTLsMtx.unlock();
27 
28   DP("Call to omp_get_num_devices returning %zd\n", DevicesSize);
29 
30   return DevicesSize;
31 }
32 
33 EXTERN int omp_get_initial_device(void) {
34   TIMESCOPE();
35   int hostDevice = omp_get_num_devices();
36   DP("Call to omp_get_initial_device returning %d\n", hostDevice);
37   return hostDevice;
38 }
39 
40 EXTERN void *omp_target_alloc(size_t size, int device_num) {
41   return targetAllocExplicit(size, device_num, TARGET_ALLOC_DEFAULT, __func__);
42 }
43 
44 EXTERN void *llvm_omp_target_alloc_device(size_t size, int device_num) {
45   return targetAllocExplicit(size, device_num, TARGET_ALLOC_DEVICE, __func__);
46 }
47 
48 EXTERN void *llvm_omp_target_alloc_host(size_t size, int device_num) {
49   return targetAllocExplicit(size, device_num, TARGET_ALLOC_HOST, __func__);
50 }
51 
52 EXTERN void *llvm_omp_target_alloc_shared(size_t size, int device_num) {
53   return targetAllocExplicit(size, device_num, TARGET_ALLOC_SHARED, __func__);
54 }
55 
56 EXTERN void *llvm_omp_target_dynamic_shared_alloc() { return nullptr; }
57 EXTERN void *llvm_omp_get_dynamic_shared() { return nullptr; }
58 
59 EXTERN void omp_target_free(void *device_ptr, int device_num) {
60   TIMESCOPE();
61   DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
62      device_num, DPxPTR(device_ptr));
63 
64   if (!device_ptr) {
65     DP("Call to omp_target_free with NULL ptr\n");
66     return;
67   }
68 
69   if (device_num == omp_get_initial_device()) {
70     free(device_ptr);
71     DP("omp_target_free deallocated host ptr\n");
72     return;
73   }
74 
75   if (!device_is_ready(device_num)) {
76     DP("omp_target_free returns, nothing to do\n");
77     return;
78   }
79 
80   PM->Devices[device_num]->deleteData(device_ptr);
81   DP("omp_target_free deallocated device ptr\n");
82 }
83 
84 EXTERN int omp_target_is_present(const void *ptr, int device_num) {
85   TIMESCOPE();
86   DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
87      device_num, DPxPTR(ptr));
88 
89   if (!ptr) {
90     DP("Call to omp_target_is_present with NULL ptr, returning false\n");
91     return false;
92   }
93 
94   if (device_num == omp_get_initial_device()) {
95     DP("Call to omp_target_is_present on host, returning true\n");
96     return true;
97   }
98 
99   PM->RTLsMtx.lock();
100   size_t DevicesSize = PM->Devices.size();
101   PM->RTLsMtx.unlock();
102   if (DevicesSize <= (size_t)device_num) {
103     DP("Call to omp_target_is_present with invalid device ID, returning "
104        "false\n");
105     return false;
106   }
107 
108   DeviceTy &Device = *PM->Devices[device_num];
109   bool IsLast; // not used
110   bool IsHostPtr;
111   // omp_target_is_present tests whether a host pointer refers to storage that
112   // is mapped to a given device. However, due to the lack of the storage size,
113   // only check 1 byte. Cannot set size 0 which checks whether the pointer (zero
114   // lengh array) is mapped instead of the referred storage.
115   TargetPointerResultTy TPR =
116       Device.getTgtPtrBegin(const_cast<void *>(ptr), 1, IsLast,
117                             /*UpdateRefCount=*/false,
118                             /*UseHoldRefCount=*/false, IsHostPtr);
119   int rc = (TPR.TargetPointer != NULL);
120   // Under unified memory the host pointer can be returned by the
121   // getTgtPtrBegin() function which means that there is no device
122   // corresponding point for ptr. This function should return false
123   // in that situation.
124   if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY)
125     rc = !IsHostPtr;
126   DP("Call to omp_target_is_present returns %d\n", rc);
127   return rc;
128 }
129 
130 EXTERN int omp_target_memcpy(void *dst, const void *src, size_t length,
131                              size_t dst_offset, size_t src_offset,
132                              int dst_device, int src_device) {
133   TIMESCOPE();
134   DP("Call to omp_target_memcpy, dst device %d, src device %d, "
135      "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
136      "src offset %zu, length %zu\n",
137      dst_device, src_device, DPxPTR(dst), DPxPTR(src), dst_offset, src_offset,
138      length);
139 
140   if (!dst || !src || length <= 0) {
141     if (length == 0) {
142       DP("Call to omp_target_memcpy with zero length, nothing to do\n");
143       return OFFLOAD_SUCCESS;
144     }
145 
146     REPORT("Call to omp_target_memcpy with invalid arguments\n");
147     return OFFLOAD_FAIL;
148   }
149 
150   if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) {
151     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
152     return OFFLOAD_FAIL;
153   }
154 
155   if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) {
156     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
157     return OFFLOAD_FAIL;
158   }
159 
160   int rc = OFFLOAD_SUCCESS;
161   void *srcAddr = (char *)const_cast<void *>(src) + src_offset;
162   void *dstAddr = (char *)dst + dst_offset;
163 
164   if (src_device == omp_get_initial_device() &&
165       dst_device == omp_get_initial_device()) {
166     DP("copy from host to host\n");
167     const void *p = memcpy(dstAddr, srcAddr, length);
168     if (p == NULL)
169       rc = OFFLOAD_FAIL;
170   } else if (src_device == omp_get_initial_device()) {
171     DP("copy from host to device\n");
172     DeviceTy &DstDev = *PM->Devices[dst_device];
173     AsyncInfoTy AsyncInfo(DstDev);
174     rc = DstDev.submitData(dstAddr, srcAddr, length, AsyncInfo);
175   } else if (dst_device == omp_get_initial_device()) {
176     DP("copy from device to host\n");
177     DeviceTy &SrcDev = *PM->Devices[src_device];
178     AsyncInfoTy AsyncInfo(SrcDev);
179     rc = SrcDev.retrieveData(dstAddr, srcAddr, length, AsyncInfo);
180   } else {
181     DP("copy from device to device\n");
182     DeviceTy &SrcDev = *PM->Devices[src_device];
183     DeviceTy &DstDev = *PM->Devices[dst_device];
184     // First try to use D2D memcpy which is more efficient. If fails, fall back
185     // to unefficient way.
186     if (SrcDev.isDataExchangable(DstDev)) {
187       AsyncInfoTy AsyncInfo(SrcDev);
188       rc = SrcDev.dataExchange(srcAddr, DstDev, dstAddr, length, AsyncInfo);
189       if (rc == OFFLOAD_SUCCESS)
190         return OFFLOAD_SUCCESS;
191     }
192 
193     void *buffer = malloc(length);
194     {
195       AsyncInfoTy AsyncInfo(SrcDev);
196       rc = SrcDev.retrieveData(buffer, srcAddr, length, AsyncInfo);
197     }
198     if (rc == OFFLOAD_SUCCESS) {
199       AsyncInfoTy AsyncInfo(SrcDev);
200       rc = DstDev.submitData(dstAddr, buffer, length, AsyncInfo);
201     }
202     free(buffer);
203   }
204 
205   DP("omp_target_memcpy returns %d\n", rc);
206   return rc;
207 }
208 
209 EXTERN int omp_target_memcpy_rect(
210     void *dst, const void *src, size_t element_size, int num_dims,
211     const size_t *volume, const size_t *dst_offsets, const size_t *src_offsets,
212     const size_t *dst_dimensions, const size_t *src_dimensions, int dst_device,
213     int src_device) {
214   TIMESCOPE();
215   DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
216      "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
217      "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
218      "volume " DPxMOD ", element size %zu, num_dims %d\n",
219      dst_device, src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets),
220      DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions),
221      DPxPTR(volume), element_size, num_dims);
222 
223   if (!(dst || src)) {
224     DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
225        INT_MAX);
226     return INT_MAX;
227   }
228 
229   if (!dst || !src || element_size < 1 || num_dims < 1 || !volume ||
230       !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) {
231     REPORT("Call to omp_target_memcpy_rect with invalid arguments\n");
232     return OFFLOAD_FAIL;
233   }
234 
235   int rc;
236   if (num_dims == 1) {
237     rc = omp_target_memcpy(
238         dst, src, element_size * volume[0], element_size * dst_offsets[0],
239         element_size * src_offsets[0], dst_device, src_device);
240   } else {
241     size_t dst_slice_size = element_size;
242     size_t src_slice_size = element_size;
243     for (int i = 1; i < num_dims; ++i) {
244       dst_slice_size *= dst_dimensions[i];
245       src_slice_size *= src_dimensions[i];
246     }
247 
248     size_t dst_off = dst_offsets[0] * dst_slice_size;
249     size_t src_off = src_offsets[0] * src_slice_size;
250     for (size_t i = 0; i < volume[0]; ++i) {
251       rc = omp_target_memcpy_rect(
252           (char *)dst + dst_off + dst_slice_size * i,
253           (char *)const_cast<void *>(src) + src_off + src_slice_size * i,
254           element_size, num_dims - 1, volume + 1, dst_offsets + 1,
255           src_offsets + 1, dst_dimensions + 1, src_dimensions + 1, dst_device,
256           src_device);
257 
258       if (rc) {
259         DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
260         return rc;
261       }
262     }
263   }
264 
265   DP("omp_target_memcpy_rect returns %d\n", rc);
266   return rc;
267 }
268 
269 EXTERN int omp_target_associate_ptr(const void *host_ptr,
270                                     const void *device_ptr, size_t size,
271                                     size_t device_offset, int device_num) {
272   TIMESCOPE();
273   DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
274      "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
275      DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num);
276 
277   if (!host_ptr || !device_ptr || size <= 0) {
278     REPORT("Call to omp_target_associate_ptr with invalid arguments\n");
279     return OFFLOAD_FAIL;
280   }
281 
282   if (device_num == omp_get_initial_device()) {
283     REPORT("omp_target_associate_ptr: no association possible on the host\n");
284     return OFFLOAD_FAIL;
285   }
286 
287   if (!device_is_ready(device_num)) {
288     REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
289     return OFFLOAD_FAIL;
290   }
291 
292   DeviceTy &Device = *PM->Devices[device_num];
293   void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset);
294   int rc = Device.associatePtr(const_cast<void *>(host_ptr),
295                                const_cast<void *>(device_addr), size);
296   DP("omp_target_associate_ptr returns %d\n", rc);
297   return rc;
298 }
299 
300 EXTERN int omp_target_disassociate_ptr(const void *host_ptr, int device_num) {
301   TIMESCOPE();
302   DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
303      "device_num %d\n",
304      DPxPTR(host_ptr), device_num);
305 
306   if (!host_ptr) {
307     REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n");
308     return OFFLOAD_FAIL;
309   }
310 
311   if (device_num == omp_get_initial_device()) {
312     REPORT(
313         "omp_target_disassociate_ptr: no association possible on the host\n");
314     return OFFLOAD_FAIL;
315   }
316 
317   if (!device_is_ready(device_num)) {
318     REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
319     return OFFLOAD_FAIL;
320   }
321 
322   DeviceTy &Device = *PM->Devices[device_num];
323   int rc = Device.disassociatePtr(const_cast<void *>(host_ptr));
324   DP("omp_target_disassociate_ptr returns %d\n", rc);
325   return rc;
326 }
327