1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of OpenMP API interface functions.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "device.h"
14 #include "omptarget.h"
15 #include "private.h"
16 #include "rtl.h"
17 
18 #include <climits>
19 #include <cstdlib>
20 #include <cstring>
21 
22 EXTERN int omp_get_num_devices(void) {
23   TIMESCOPE();
24   PM->RTLsMtx.lock();
25   size_t DevicesSize = PM->Devices.size();
26   PM->RTLsMtx.unlock();
27 
28   DP("Call to omp_get_num_devices returning %zd\n", DevicesSize);
29 
30   return DevicesSize;
31 }
32 
33 EXTERN int omp_get_initial_device(void) {
34   TIMESCOPE();
35   int hostDevice = omp_get_num_devices();
36   DP("Call to omp_get_initial_device returning %d\n", hostDevice);
37   return hostDevice;
38 }
39 
40 EXTERN void *omp_target_alloc(size_t size, int device_num) {
41   return targetAllocExplicit(size, device_num, TARGET_ALLOC_DEFAULT, __func__);
42 }
43 
44 EXTERN void *llvm_omp_target_alloc_device(size_t size, int device_num) {
45   return targetAllocExplicit(size, device_num, TARGET_ALLOC_DEVICE, __func__);
46 }
47 
48 EXTERN void *llvm_omp_target_alloc_host(size_t size, int device_num) {
49   return targetAllocExplicit(size, device_num, TARGET_ALLOC_HOST, __func__);
50 }
51 
52 EXTERN void *llvm_omp_target_alloc_shared(size_t size, int device_num) {
53   return targetAllocExplicit(size, device_num, TARGET_ALLOC_SHARED, __func__);
54 }
55 
56 EXTERN void omp_target_free(void *device_ptr, int device_num) {
57   TIMESCOPE();
58   DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
59      device_num, DPxPTR(device_ptr));
60 
61   if (!device_ptr) {
62     DP("Call to omp_target_free with NULL ptr\n");
63     return;
64   }
65 
66   if (device_num == omp_get_initial_device()) {
67     free(device_ptr);
68     DP("omp_target_free deallocated host ptr\n");
69     return;
70   }
71 
72   if (!device_is_ready(device_num)) {
73     DP("omp_target_free returns, nothing to do\n");
74     return;
75   }
76 
77   PM->Devices[device_num].deleteData(device_ptr);
78   DP("omp_target_free deallocated device ptr\n");
79 }
80 
81 EXTERN int omp_target_is_present(void *ptr, int device_num) {
82   TIMESCOPE();
83   DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
84      device_num, DPxPTR(ptr));
85 
86   if (!ptr) {
87     DP("Call to omp_target_is_present with NULL ptr, returning false\n");
88     return false;
89   }
90 
91   if (device_num == omp_get_initial_device()) {
92     DP("Call to omp_target_is_present on host, returning true\n");
93     return true;
94   }
95 
96   PM->RTLsMtx.lock();
97   size_t DevicesSize = PM->Devices.size();
98   PM->RTLsMtx.unlock();
99   if (DevicesSize <= (size_t)device_num) {
100     DP("Call to omp_target_is_present with invalid device ID, returning "
101        "false\n");
102     return false;
103   }
104 
105   DeviceTy &Device = PM->Devices[device_num];
106   bool IsLast; // not used
107   bool IsHostPtr;
108   void *TgtPtr = Device.getTgtPtrBegin(ptr, 0, IsLast, false, IsHostPtr);
109   int rc = (TgtPtr != NULL);
110   // Under unified memory the host pointer can be returned by the
111   // getTgtPtrBegin() function which means that there is no device
112   // corresponding point for ptr. This function should return false
113   // in that situation.
114   if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY)
115     rc = !IsHostPtr;
116   DP("Call to omp_target_is_present returns %d\n", rc);
117   return rc;
118 }
119 
120 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
121                              size_t dst_offset, size_t src_offset,
122                              int dst_device, int src_device) {
123   TIMESCOPE();
124   DP("Call to omp_target_memcpy, dst device %d, src device %d, "
125      "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
126      "src offset %zu, length %zu\n",
127      dst_device, src_device, DPxPTR(dst), DPxPTR(src), dst_offset, src_offset,
128      length);
129 
130   if (!dst || !src || length <= 0) {
131     if (length == 0) {
132       DP("Call to omp_target_memcpy with zero length, nothing to do\n");
133       return OFFLOAD_SUCCESS;
134     }
135 
136     REPORT("Call to omp_target_memcpy with invalid arguments\n");
137     return OFFLOAD_FAIL;
138   }
139 
140   if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) {
141     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
142     return OFFLOAD_FAIL;
143   }
144 
145   if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) {
146     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
147     return OFFLOAD_FAIL;
148   }
149 
150   int rc = OFFLOAD_SUCCESS;
151   void *srcAddr = (char *)src + src_offset;
152   void *dstAddr = (char *)dst + dst_offset;
153 
154   if (src_device == omp_get_initial_device() &&
155       dst_device == omp_get_initial_device()) {
156     DP("copy from host to host\n");
157     const void *p = memcpy(dstAddr, srcAddr, length);
158     if (p == NULL)
159       rc = OFFLOAD_FAIL;
160   } else if (src_device == omp_get_initial_device()) {
161     DP("copy from host to device\n");
162     DeviceTy &DstDev = PM->Devices[dst_device];
163     AsyncInfoTy AsyncInfo(DstDev);
164     rc = DstDev.submitData(dstAddr, srcAddr, length, AsyncInfo);
165   } else if (dst_device == omp_get_initial_device()) {
166     DP("copy from device to host\n");
167     DeviceTy &SrcDev = PM->Devices[src_device];
168     AsyncInfoTy AsyncInfo(SrcDev);
169     rc = SrcDev.retrieveData(dstAddr, srcAddr, length, AsyncInfo);
170   } else {
171     DP("copy from device to device\n");
172     DeviceTy &SrcDev = PM->Devices[src_device];
173     DeviceTy &DstDev = PM->Devices[dst_device];
174     // First try to use D2D memcpy which is more efficient. If fails, fall back
175     // to unefficient way.
176     if (SrcDev.isDataExchangable(DstDev)) {
177       AsyncInfoTy AsyncInfo(SrcDev);
178       rc = SrcDev.dataExchange(srcAddr, DstDev, dstAddr, length, AsyncInfo);
179       if (rc == OFFLOAD_SUCCESS)
180         return OFFLOAD_SUCCESS;
181     }
182 
183     void *buffer = malloc(length);
184     {
185       AsyncInfoTy AsyncInfo(SrcDev);
186       rc = SrcDev.retrieveData(buffer, srcAddr, length, AsyncInfo);
187     }
188     if (rc == OFFLOAD_SUCCESS) {
189       AsyncInfoTy AsyncInfo(SrcDev);
190       rc = DstDev.submitData(dstAddr, buffer, length, AsyncInfo);
191     }
192     free(buffer);
193   }
194 
195   DP("omp_target_memcpy returns %d\n", rc);
196   return rc;
197 }
198 
199 EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size,
200                                   int num_dims, const size_t *volume,
201                                   const size_t *dst_offsets,
202                                   const size_t *src_offsets,
203                                   const size_t *dst_dimensions,
204                                   const size_t *src_dimensions, int dst_device,
205                                   int src_device) {
206   TIMESCOPE();
207   DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
208      "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
209      "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
210      "volume " DPxMOD ", element size %zu, num_dims %d\n",
211      dst_device, src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets),
212      DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions),
213      DPxPTR(volume), element_size, num_dims);
214 
215   if (!(dst || src)) {
216     DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
217        INT_MAX);
218     return INT_MAX;
219   }
220 
221   if (!dst || !src || element_size < 1 || num_dims < 1 || !volume ||
222       !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) {
223     REPORT("Call to omp_target_memcpy_rect with invalid arguments\n");
224     return OFFLOAD_FAIL;
225   }
226 
227   int rc;
228   if (num_dims == 1) {
229     rc = omp_target_memcpy(
230         dst, src, element_size * volume[0], element_size * dst_offsets[0],
231         element_size * src_offsets[0], dst_device, src_device);
232   } else {
233     size_t dst_slice_size = element_size;
234     size_t src_slice_size = element_size;
235     for (int i = 1; i < num_dims; ++i) {
236       dst_slice_size *= dst_dimensions[i];
237       src_slice_size *= src_dimensions[i];
238     }
239 
240     size_t dst_off = dst_offsets[0] * dst_slice_size;
241     size_t src_off = src_offsets[0] * src_slice_size;
242     for (size_t i = 0; i < volume[0]; ++i) {
243       rc = omp_target_memcpy_rect(
244           (char *)dst + dst_off + dst_slice_size * i,
245           (char *)src + src_off + src_slice_size * i, element_size,
246           num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1,
247           dst_dimensions + 1, src_dimensions + 1, dst_device, src_device);
248 
249       if (rc) {
250         DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
251         return rc;
252       }
253     }
254   }
255 
256   DP("omp_target_memcpy_rect returns %d\n", rc);
257   return rc;
258 }
259 
260 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr,
261                                     size_t size, size_t device_offset,
262                                     int device_num) {
263   TIMESCOPE();
264   DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
265      "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
266      DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num);
267 
268   if (!host_ptr || !device_ptr || size <= 0) {
269     REPORT("Call to omp_target_associate_ptr with invalid arguments\n");
270     return OFFLOAD_FAIL;
271   }
272 
273   if (device_num == omp_get_initial_device()) {
274     REPORT("omp_target_associate_ptr: no association possible on the host\n");
275     return OFFLOAD_FAIL;
276   }
277 
278   if (!device_is_ready(device_num)) {
279     REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
280     return OFFLOAD_FAIL;
281   }
282 
283   DeviceTy &Device = PM->Devices[device_num];
284   void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset);
285   int rc = Device.associatePtr(host_ptr, device_addr, size);
286   DP("omp_target_associate_ptr returns %d\n", rc);
287   return rc;
288 }
289 
290 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) {
291   TIMESCOPE();
292   DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
293      "device_num %d\n",
294      DPxPTR(host_ptr), device_num);
295 
296   if (!host_ptr) {
297     REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n");
298     return OFFLOAD_FAIL;
299   }
300 
301   if (device_num == omp_get_initial_device()) {
302     REPORT(
303         "omp_target_disassociate_ptr: no association possible on the host\n");
304     return OFFLOAD_FAIL;
305   }
306 
307   if (!device_is_ready(device_num)) {
308     REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
309     return OFFLOAD_FAIL;
310   }
311 
312   DeviceTy &Device = PM->Devices[device_num];
313   int rc = Device.disassociatePtr(host_ptr);
314   DP("omp_target_disassociate_ptr returns %d\n", rc);
315   return rc;
316 }
317