1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of OpenMP API interface functions.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "device.h"
14 #include "private.h"
15 #include "rtl.h"
16 
17 #include <climits>
18 #include <cstring>
19 #include <cstdlib>
20 
21 EXTERN int omp_get_num_devices(void) {
22   TIMESCOPE();
23   PM->RTLsMtx.lock();
24   size_t DevicesSize = PM->Devices.size();
25   PM->RTLsMtx.unlock();
26 
27   DP("Call to omp_get_num_devices returning %zd\n", DevicesSize);
28 
29   return DevicesSize;
30 }
31 
32 EXTERN int omp_get_initial_device(void) {
33   TIMESCOPE();
34   int hostDevice = omp_get_num_devices();
35   DP("Call to omp_get_initial_device returning %d\n", hostDevice);
36   return hostDevice;
37 }
38 
39 EXTERN void *omp_target_alloc(size_t size, int device_num) {
40   TIMESCOPE();
41   DP("Call to omp_target_alloc for device %d requesting %zu bytes\n",
42       device_num, size);
43 
44   if (size <= 0) {
45     DP("Call to omp_target_alloc with non-positive length\n");
46     return NULL;
47   }
48 
49   void *rc = NULL;
50 
51   if (device_num == omp_get_initial_device()) {
52     rc = malloc(size);
53     DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc));
54     return rc;
55   }
56 
57   if (!device_is_ready(device_num)) {
58     DP("omp_target_alloc returns NULL ptr\n");
59     return NULL;
60   }
61 
62   rc = PM->Devices[device_num].allocData(size);
63   DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc));
64   return rc;
65 }
66 
67 EXTERN void omp_target_free(void *device_ptr, int device_num) {
68   TIMESCOPE();
69   DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
70       device_num, DPxPTR(device_ptr));
71 
72   if (!device_ptr) {
73     DP("Call to omp_target_free with NULL ptr\n");
74     return;
75   }
76 
77   if (device_num == omp_get_initial_device()) {
78     free(device_ptr);
79     DP("omp_target_free deallocated host ptr\n");
80     return;
81   }
82 
83   if (!device_is_ready(device_num)) {
84     DP("omp_target_free returns, nothing to do\n");
85     return;
86   }
87 
88   PM->Devices[device_num].deleteData(device_ptr);
89   DP("omp_target_free deallocated device ptr\n");
90 }
91 
92 EXTERN int omp_target_is_present(void *ptr, int device_num) {
93   TIMESCOPE();
94   DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
95       device_num, DPxPTR(ptr));
96 
97   if (!ptr) {
98     DP("Call to omp_target_is_present with NULL ptr, returning false\n");
99     return false;
100   }
101 
102   if (device_num == omp_get_initial_device()) {
103     DP("Call to omp_target_is_present on host, returning true\n");
104     return true;
105   }
106 
107   PM->RTLsMtx.lock();
108   size_t DevicesSize = PM->Devices.size();
109   PM->RTLsMtx.unlock();
110   if (DevicesSize <= (size_t)device_num) {
111     DP("Call to omp_target_is_present with invalid device ID, returning "
112         "false\n");
113     return false;
114   }
115 
116   DeviceTy &Device = PM->Devices[device_num];
117   bool IsLast; // not used
118   bool IsHostPtr;
119   void *TgtPtr = Device.getTgtPtrBegin(ptr, 0, IsLast, false, IsHostPtr);
120   int rc = (TgtPtr != NULL);
121   // Under unified memory the host pointer can be returned by the
122   // getTgtPtrBegin() function which means that there is no device
123   // corresponding point for ptr. This function should return false
124   // in that situation.
125   if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY)
126     rc = !IsHostPtr;
127   DP("Call to omp_target_is_present returns %d\n", rc);
128   return rc;
129 }
130 
131 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
132     size_t dst_offset, size_t src_offset, int dst_device, int src_device) {
133   TIMESCOPE();
134   DP("Call to omp_target_memcpy, dst device %d, src device %d, "
135       "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
136       "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst),
137       DPxPTR(src), dst_offset, src_offset, length);
138 
139   if (!dst || !src || length <= 0) {
140     if (length == 0) {
141       DP("Call to omp_target_memcpy with zero length, nothing to do\n");
142       return OFFLOAD_SUCCESS;
143     }
144 
145     REPORT("Call to omp_target_memcpy with invalid arguments\n");
146     return OFFLOAD_FAIL;
147   }
148 
149   if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) {
150     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
151     return OFFLOAD_FAIL;
152   }
153 
154   if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) {
155     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
156     return OFFLOAD_FAIL;
157   }
158 
159   int rc = OFFLOAD_SUCCESS;
160   void *srcAddr = (char *)src + src_offset;
161   void *dstAddr = (char *)dst + dst_offset;
162 
163   if (src_device == omp_get_initial_device() &&
164       dst_device == omp_get_initial_device()) {
165     DP("copy from host to host\n");
166     const void *p = memcpy(dstAddr, srcAddr, length);
167     if (p == NULL)
168       rc = OFFLOAD_FAIL;
169   } else if (src_device == omp_get_initial_device()) {
170     DP("copy from host to device\n");
171     DeviceTy &DstDev = PM->Devices[dst_device];
172     rc = DstDev.submitData(dstAddr, srcAddr, length, nullptr);
173   } else if (dst_device == omp_get_initial_device()) {
174     DP("copy from device to host\n");
175     DeviceTy &SrcDev = PM->Devices[src_device];
176     rc = SrcDev.retrieveData(dstAddr, srcAddr, length, nullptr);
177   } else {
178     DP("copy from device to device\n");
179     DeviceTy &SrcDev = PM->Devices[src_device];
180     DeviceTy &DstDev = PM->Devices[dst_device];
181     // First try to use D2D memcpy which is more efficient. If fails, fall back
182     // to unefficient way.
183     if (SrcDev.isDataExchangable(DstDev)) {
184       rc = SrcDev.dataExchange(srcAddr, DstDev, dstAddr, length, nullptr);
185       if (rc == OFFLOAD_SUCCESS)
186         return OFFLOAD_SUCCESS;
187     }
188 
189     void *buffer = malloc(length);
190     rc = SrcDev.retrieveData(buffer, srcAddr, length, nullptr);
191     if (rc == OFFLOAD_SUCCESS)
192       rc = DstDev.submitData(dstAddr, buffer, length, nullptr);
193     free(buffer);
194   }
195 
196   DP("omp_target_memcpy returns %d\n", rc);
197   return rc;
198 }
199 
200 EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size,
201     int num_dims, const size_t *volume, const size_t *dst_offsets,
202     const size_t *src_offsets, const size_t *dst_dimensions,
203     const size_t *src_dimensions, int dst_device, int src_device) {
204   TIMESCOPE();
205   DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
206       "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
207       "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
208       "volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device,
209       src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets),
210       DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions),
211       DPxPTR(volume), element_size, num_dims);
212 
213   if (!(dst || src)) {
214     DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
215         INT_MAX);
216     return INT_MAX;
217   }
218 
219   if (!dst || !src || element_size < 1 || num_dims < 1 || !volume ||
220       !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) {
221     REPORT("Call to omp_target_memcpy_rect with invalid arguments\n");
222     return OFFLOAD_FAIL;
223   }
224 
225   int rc;
226   if (num_dims == 1) {
227     rc = omp_target_memcpy(dst, src, element_size * volume[0],
228         element_size * dst_offsets[0], element_size * src_offsets[0],
229         dst_device, src_device);
230   } else {
231     size_t dst_slice_size = element_size;
232     size_t src_slice_size = element_size;
233     for (int i=1; i<num_dims; ++i) {
234       dst_slice_size *= dst_dimensions[i];
235       src_slice_size *= src_dimensions[i];
236     }
237 
238     size_t dst_off = dst_offsets[0] * dst_slice_size;
239     size_t src_off = src_offsets[0] * src_slice_size;
240     for (size_t i=0; i<volume[0]; ++i) {
241       rc = omp_target_memcpy_rect((char *) dst + dst_off + dst_slice_size * i,
242           (char *) src + src_off + src_slice_size * i, element_size,
243           num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1,
244           dst_dimensions + 1, src_dimensions + 1, dst_device, src_device);
245 
246       if (rc) {
247         DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
248         return rc;
249       }
250     }
251   }
252 
253   DP("omp_target_memcpy_rect returns %d\n", rc);
254   return rc;
255 }
256 
257 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr,
258     size_t size, size_t device_offset, int device_num) {
259   TIMESCOPE();
260   DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
261       "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
262       DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num);
263 
264   if (!host_ptr || !device_ptr || size <= 0) {
265     REPORT("Call to omp_target_associate_ptr with invalid arguments\n");
266     return OFFLOAD_FAIL;
267   }
268 
269   if (device_num == omp_get_initial_device()) {
270     REPORT("omp_target_associate_ptr: no association possible on the host\n");
271     return OFFLOAD_FAIL;
272   }
273 
274   if (!device_is_ready(device_num)) {
275     REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
276     return OFFLOAD_FAIL;
277   }
278 
279   DeviceTy &Device = PM->Devices[device_num];
280   void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset);
281   int rc = Device.associatePtr(host_ptr, device_addr, size);
282   DP("omp_target_associate_ptr returns %d\n", rc);
283   return rc;
284 }
285 
286 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) {
287   TIMESCOPE();
288   DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
289       "device_num %d\n", DPxPTR(host_ptr), device_num);
290 
291   if (!host_ptr) {
292     REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n");
293     return OFFLOAD_FAIL;
294   }
295 
296   if (device_num == omp_get_initial_device()) {
297     REPORT(
298         "omp_target_disassociate_ptr: no association possible on the host\n");
299     return OFFLOAD_FAIL;
300   }
301 
302   if (!device_is_ready(device_num)) {
303     REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
304     return OFFLOAD_FAIL;
305   }
306 
307   DeviceTy &Device = PM->Devices[device_num];
308   int rc = Device.disassociatePtr(host_ptr);
309   DP("omp_target_disassociate_ptr returns %d\n", rc);
310   return rc;
311 }
312