1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Implementation of OpenMP API interface functions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include <omptarget.h> 14 15 #include "device.h" 16 #include "private.h" 17 #include "rtl.h" 18 19 #include <climits> 20 #include <cstring> 21 #include <cstdlib> 22 23 EXTERN int omp_get_num_devices(void) { 24 RTLsMtx->lock(); 25 size_t Devices_size = Devices.size(); 26 RTLsMtx->unlock(); 27 28 DP("Call to omp_get_num_devices returning %zd\n", Devices_size); 29 30 return Devices_size; 31 } 32 33 EXTERN int omp_get_initial_device(void) { 34 DP("Call to omp_get_initial_device returning %d\n", HOST_DEVICE); 35 return HOST_DEVICE; 36 } 37 38 EXTERN void *omp_target_alloc(size_t size, int device_num) { 39 DP("Call to omp_target_alloc for device %d requesting %zu bytes\n", 40 device_num, size); 41 42 if (size <= 0) { 43 DP("Call to omp_target_alloc with non-positive length\n"); 44 return NULL; 45 } 46 47 void *rc = NULL; 48 49 if (device_num == omp_get_initial_device()) { 50 rc = malloc(size); 51 DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc)); 52 return rc; 53 } 54 55 if (!device_is_ready(device_num)) { 56 DP("omp_target_alloc returns NULL ptr\n"); 57 return NULL; 58 } 59 60 DeviceTy &Device = Devices[device_num]; 61 rc = Device.RTL->data_alloc(Device.RTLDeviceID, size, NULL); 62 DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc)); 63 return rc; 64 } 65 66 EXTERN void omp_target_free(void *device_ptr, int device_num) { 67 DP("Call to omp_target_free for device %d and address " DPxMOD "\n", 68 device_num, DPxPTR(device_ptr)); 69 70 if (!device_ptr) { 71 DP("Call to omp_target_free with NULL ptr\n"); 72 return; 73 } 74 75 if (device_num == omp_get_initial_device()) { 76 free(device_ptr); 77 DP("omp_target_free deallocated host ptr\n"); 78 return; 79 } 80 81 if (!device_is_ready(device_num)) { 82 DP("omp_target_free returns, nothing to do\n"); 83 return; 84 } 85 86 DeviceTy &Device = Devices[device_num]; 87 Device.RTL->data_delete(Device.RTLDeviceID, (void *)device_ptr); 88 DP("omp_target_free deallocated device ptr\n"); 89 } 90 91 EXTERN int omp_target_is_present(void *ptr, int device_num) { 92 DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n", 93 device_num, DPxPTR(ptr)); 94 95 if (!ptr) { 96 DP("Call to omp_target_is_present with NULL ptr, returning false\n"); 97 return false; 98 } 99 100 if (device_num == omp_get_initial_device()) { 101 DP("Call to omp_target_is_present on host, returning true\n"); 102 return true; 103 } 104 105 RTLsMtx->lock(); 106 size_t Devices_size = Devices.size(); 107 RTLsMtx->unlock(); 108 if (Devices_size <= (size_t)device_num) { 109 DP("Call to omp_target_is_present with invalid device ID, returning " 110 "false\n"); 111 return false; 112 } 113 114 DeviceTy& Device = Devices[device_num]; 115 bool IsLast; // not used 116 bool IsHostPtr; 117 void *TgtPtr = Device.getTgtPtrBegin(ptr, 0, IsLast, false, IsHostPtr); 118 int rc = (TgtPtr != NULL); 119 // Under unified memory the host pointer can be returned by the 120 // getTgtPtrBegin() function which means that there is no device 121 // corresponding point for ptr. This function should return false 122 // in that situation. 123 if (RTLs->RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) 124 rc = !IsHostPtr; 125 DP("Call to omp_target_is_present returns %d\n", rc); 126 return rc; 127 } 128 129 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length, 130 size_t dst_offset, size_t src_offset, int dst_device, int src_device) { 131 DP("Call to omp_target_memcpy, dst device %d, src device %d, " 132 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, " 133 "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst), 134 DPxPTR(src), dst_offset, src_offset, length); 135 136 if (!dst || !src || length <= 0) { 137 DP("Call to omp_target_memcpy with invalid arguments\n"); 138 return OFFLOAD_FAIL; 139 } 140 141 if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) { 142 DP("omp_target_memcpy returns OFFLOAD_FAIL\n"); 143 return OFFLOAD_FAIL; 144 } 145 146 if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) { 147 DP("omp_target_memcpy returns OFFLOAD_FAIL\n"); 148 return OFFLOAD_FAIL; 149 } 150 151 int rc = OFFLOAD_SUCCESS; 152 void *srcAddr = (char *)src + src_offset; 153 void *dstAddr = (char *)dst + dst_offset; 154 155 if (src_device == omp_get_initial_device() && 156 dst_device == omp_get_initial_device()) { 157 DP("copy from host to host\n"); 158 const void *p = memcpy(dstAddr, srcAddr, length); 159 if (p == NULL) 160 rc = OFFLOAD_FAIL; 161 } else if (src_device == omp_get_initial_device()) { 162 DP("copy from host to device\n"); 163 DeviceTy& DstDev = Devices[dst_device]; 164 rc = DstDev.data_submit(dstAddr, srcAddr, length, nullptr); 165 } else if (dst_device == omp_get_initial_device()) { 166 DP("copy from device to host\n"); 167 DeviceTy& SrcDev = Devices[src_device]; 168 rc = SrcDev.data_retrieve(dstAddr, srcAddr, length, nullptr); 169 } else { 170 DP("copy from device to device\n"); 171 DeviceTy &SrcDev = Devices[src_device]; 172 DeviceTy &DstDev = Devices[dst_device]; 173 // First try to use D2D memcpy which is more efficient. If fails, fall back 174 // to unefficient way. 175 if (SrcDev.isDataExchangable(DstDev)) { 176 rc = SrcDev.data_exchange(srcAddr, DstDev, dstAddr, length, nullptr); 177 if (rc == OFFLOAD_SUCCESS) 178 return OFFLOAD_SUCCESS; 179 } 180 181 void *buffer = malloc(length); 182 rc = SrcDev.data_retrieve(buffer, srcAddr, length, nullptr); 183 if (rc == OFFLOAD_SUCCESS) 184 rc = DstDev.data_submit(dstAddr, buffer, length, nullptr); 185 free(buffer); 186 } 187 188 DP("omp_target_memcpy returns %d\n", rc); 189 return rc; 190 } 191 192 EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size, 193 int num_dims, const size_t *volume, const size_t *dst_offsets, 194 const size_t *src_offsets, const size_t *dst_dimensions, 195 const size_t *src_dimensions, int dst_device, int src_device) { 196 DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, " 197 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", " 198 "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", " 199 "volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device, 200 src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets), 201 DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions), 202 DPxPTR(volume), element_size, num_dims); 203 204 if (!(dst || src)) { 205 DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n", 206 INT_MAX); 207 return INT_MAX; 208 } 209 210 if (!dst || !src || element_size < 1 || num_dims < 1 || !volume || 211 !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) { 212 DP("Call to omp_target_memcpy_rect with invalid arguments\n"); 213 return OFFLOAD_FAIL; 214 } 215 216 int rc; 217 if (num_dims == 1) { 218 rc = omp_target_memcpy(dst, src, element_size * volume[0], 219 element_size * dst_offsets[0], element_size * src_offsets[0], 220 dst_device, src_device); 221 } else { 222 size_t dst_slice_size = element_size; 223 size_t src_slice_size = element_size; 224 for (int i=1; i<num_dims; ++i) { 225 dst_slice_size *= dst_dimensions[i]; 226 src_slice_size *= src_dimensions[i]; 227 } 228 229 size_t dst_off = dst_offsets[0] * dst_slice_size; 230 size_t src_off = src_offsets[0] * src_slice_size; 231 for (size_t i=0; i<volume[0]; ++i) { 232 rc = omp_target_memcpy_rect((char *) dst + dst_off + dst_slice_size * i, 233 (char *) src + src_off + src_slice_size * i, element_size, 234 num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1, 235 dst_dimensions + 1, src_dimensions + 1, dst_device, src_device); 236 237 if (rc) { 238 DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n"); 239 return rc; 240 } 241 } 242 } 243 244 DP("omp_target_memcpy_rect returns %d\n", rc); 245 return rc; 246 } 247 248 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr, 249 size_t size, size_t device_offset, int device_num) { 250 DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", " 251 "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n", 252 DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num); 253 254 if (!host_ptr || !device_ptr || size <= 0) { 255 DP("Call to omp_target_associate_ptr with invalid arguments\n"); 256 return OFFLOAD_FAIL; 257 } 258 259 if (device_num == omp_get_initial_device()) { 260 DP("omp_target_associate_ptr: no association possible on the host\n"); 261 return OFFLOAD_FAIL; 262 } 263 264 if (!device_is_ready(device_num)) { 265 DP("omp_target_associate_ptr returns OFFLOAD_FAIL\n"); 266 return OFFLOAD_FAIL; 267 } 268 269 DeviceTy& Device = Devices[device_num]; 270 void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset); 271 int rc = Device.associatePtr(host_ptr, device_addr, size); 272 DP("omp_target_associate_ptr returns %d\n", rc); 273 return rc; 274 } 275 276 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) { 277 DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", " 278 "device_num %d\n", DPxPTR(host_ptr), device_num); 279 280 if (!host_ptr) { 281 DP("Call to omp_target_associate_ptr with invalid host_ptr\n"); 282 return OFFLOAD_FAIL; 283 } 284 285 if (device_num == omp_get_initial_device()) { 286 DP("omp_target_disassociate_ptr: no association possible on the host\n"); 287 return OFFLOAD_FAIL; 288 } 289 290 if (!device_is_ready(device_num)) { 291 DP("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n"); 292 return OFFLOAD_FAIL; 293 } 294 295 DeviceTy& Device = Devices[device_num]; 296 int rc = Device.disassociatePtr(host_ptr); 297 DP("omp_target_disassociate_ptr returns %d\n", rc); 298 return rc; 299 } 300