1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Implementation of OpenMP API interface functions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "device.h" 14 #include "private.h" 15 #include "rtl.h" 16 17 #include <climits> 18 #include <cstring> 19 #include <cstdlib> 20 21 EXTERN int omp_get_num_devices(void) { 22 RTLsMtx->lock(); 23 size_t Devices_size = Devices.size(); 24 RTLsMtx->unlock(); 25 26 DP("Call to omp_get_num_devices returning %zd\n", Devices_size); 27 28 return Devices_size; 29 } 30 31 EXTERN int omp_get_initial_device(void) { 32 DP("Call to omp_get_initial_device returning %d\n", HOST_DEVICE); 33 return HOST_DEVICE; 34 } 35 36 EXTERN void *omp_target_alloc(size_t size, int device_num) { 37 DP("Call to omp_target_alloc for device %d requesting %zu bytes\n", 38 device_num, size); 39 40 if (size <= 0) { 41 DP("Call to omp_target_alloc with non-positive length\n"); 42 return NULL; 43 } 44 45 void *rc = NULL; 46 47 if (device_num == omp_get_initial_device()) { 48 rc = malloc(size); 49 DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc)); 50 return rc; 51 } 52 53 if (!device_is_ready(device_num)) { 54 DP("omp_target_alloc returns NULL ptr\n"); 55 return NULL; 56 } 57 58 rc = Devices[device_num].allocData(size); 59 DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc)); 60 return rc; 61 } 62 63 EXTERN void omp_target_free(void *device_ptr, int device_num) { 64 DP("Call to omp_target_free for device %d and address " DPxMOD "\n", 65 device_num, DPxPTR(device_ptr)); 66 67 if (!device_ptr) { 68 DP("Call to omp_target_free with NULL ptr\n"); 69 return; 70 } 71 72 if (device_num == omp_get_initial_device()) { 73 free(device_ptr); 74 DP("omp_target_free deallocated host ptr\n"); 75 return; 76 } 77 78 if (!device_is_ready(device_num)) { 79 DP("omp_target_free returns, nothing to do\n"); 80 return; 81 } 82 83 Devices[device_num].deleteData(device_ptr); 84 DP("omp_target_free deallocated device ptr\n"); 85 } 86 87 EXTERN int omp_target_is_present(void *ptr, int device_num) { 88 DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n", 89 device_num, DPxPTR(ptr)); 90 91 if (!ptr) { 92 DP("Call to omp_target_is_present with NULL ptr, returning false\n"); 93 return false; 94 } 95 96 if (device_num == omp_get_initial_device()) { 97 DP("Call to omp_target_is_present on host, returning true\n"); 98 return true; 99 } 100 101 RTLsMtx->lock(); 102 size_t Devices_size = Devices.size(); 103 RTLsMtx->unlock(); 104 if (Devices_size <= (size_t)device_num) { 105 DP("Call to omp_target_is_present with invalid device ID, returning " 106 "false\n"); 107 return false; 108 } 109 110 DeviceTy& Device = Devices[device_num]; 111 bool IsLast; // not used 112 bool IsHostPtr; 113 void *TgtPtr = Device.getTgtPtrBegin(ptr, 0, IsLast, false, IsHostPtr); 114 int rc = (TgtPtr != NULL); 115 // Under unified memory the host pointer can be returned by the 116 // getTgtPtrBegin() function which means that there is no device 117 // corresponding point for ptr. This function should return false 118 // in that situation. 119 if (RTLs->RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) 120 rc = !IsHostPtr; 121 DP("Call to omp_target_is_present returns %d\n", rc); 122 return rc; 123 } 124 125 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length, 126 size_t dst_offset, size_t src_offset, int dst_device, int src_device) { 127 DP("Call to omp_target_memcpy, dst device %d, src device %d, " 128 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, " 129 "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst), 130 DPxPTR(src), dst_offset, src_offset, length); 131 132 if (!dst || !src || length <= 0) { 133 REPORT("Call to omp_target_memcpy with invalid arguments\n"); 134 return OFFLOAD_FAIL; 135 } 136 137 if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) { 138 REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n"); 139 return OFFLOAD_FAIL; 140 } 141 142 if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) { 143 REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n"); 144 return OFFLOAD_FAIL; 145 } 146 147 int rc = OFFLOAD_SUCCESS; 148 void *srcAddr = (char *)src + src_offset; 149 void *dstAddr = (char *)dst + dst_offset; 150 151 if (src_device == omp_get_initial_device() && 152 dst_device == omp_get_initial_device()) { 153 DP("copy from host to host\n"); 154 const void *p = memcpy(dstAddr, srcAddr, length); 155 if (p == NULL) 156 rc = OFFLOAD_FAIL; 157 } else if (src_device == omp_get_initial_device()) { 158 DP("copy from host to device\n"); 159 DeviceTy& DstDev = Devices[dst_device]; 160 rc = DstDev.submitData(dstAddr, srcAddr, length, nullptr); 161 } else if (dst_device == omp_get_initial_device()) { 162 DP("copy from device to host\n"); 163 DeviceTy& SrcDev = Devices[src_device]; 164 rc = SrcDev.retrieveData(dstAddr, srcAddr, length, nullptr); 165 } else { 166 DP("copy from device to device\n"); 167 DeviceTy &SrcDev = Devices[src_device]; 168 DeviceTy &DstDev = Devices[dst_device]; 169 // First try to use D2D memcpy which is more efficient. If fails, fall back 170 // to unefficient way. 171 if (SrcDev.isDataExchangable(DstDev)) { 172 rc = SrcDev.dataExchange(srcAddr, DstDev, dstAddr, length, nullptr); 173 if (rc == OFFLOAD_SUCCESS) 174 return OFFLOAD_SUCCESS; 175 } 176 177 void *buffer = malloc(length); 178 rc = SrcDev.retrieveData(buffer, srcAddr, length, nullptr); 179 if (rc == OFFLOAD_SUCCESS) 180 rc = DstDev.submitData(dstAddr, buffer, length, nullptr); 181 free(buffer); 182 } 183 184 DP("omp_target_memcpy returns %d\n", rc); 185 return rc; 186 } 187 188 EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size, 189 int num_dims, const size_t *volume, const size_t *dst_offsets, 190 const size_t *src_offsets, const size_t *dst_dimensions, 191 const size_t *src_dimensions, int dst_device, int src_device) { 192 DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, " 193 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", " 194 "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", " 195 "volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device, 196 src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets), 197 DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions), 198 DPxPTR(volume), element_size, num_dims); 199 200 if (!(dst || src)) { 201 DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n", 202 INT_MAX); 203 return INT_MAX; 204 } 205 206 if (!dst || !src || element_size < 1 || num_dims < 1 || !volume || 207 !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) { 208 REPORT("Call to omp_target_memcpy_rect with invalid arguments\n"); 209 return OFFLOAD_FAIL; 210 } 211 212 int rc; 213 if (num_dims == 1) { 214 rc = omp_target_memcpy(dst, src, element_size * volume[0], 215 element_size * dst_offsets[0], element_size * src_offsets[0], 216 dst_device, src_device); 217 } else { 218 size_t dst_slice_size = element_size; 219 size_t src_slice_size = element_size; 220 for (int i=1; i<num_dims; ++i) { 221 dst_slice_size *= dst_dimensions[i]; 222 src_slice_size *= src_dimensions[i]; 223 } 224 225 size_t dst_off = dst_offsets[0] * dst_slice_size; 226 size_t src_off = src_offsets[0] * src_slice_size; 227 for (size_t i=0; i<volume[0]; ++i) { 228 rc = omp_target_memcpy_rect((char *) dst + dst_off + dst_slice_size * i, 229 (char *) src + src_off + src_slice_size * i, element_size, 230 num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1, 231 dst_dimensions + 1, src_dimensions + 1, dst_device, src_device); 232 233 if (rc) { 234 DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n"); 235 return rc; 236 } 237 } 238 } 239 240 DP("omp_target_memcpy_rect returns %d\n", rc); 241 return rc; 242 } 243 244 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr, 245 size_t size, size_t device_offset, int device_num) { 246 DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", " 247 "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n", 248 DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num); 249 250 if (!host_ptr || !device_ptr || size <= 0) { 251 REPORT("Call to omp_target_associate_ptr with invalid arguments\n"); 252 return OFFLOAD_FAIL; 253 } 254 255 if (device_num == omp_get_initial_device()) { 256 REPORT("omp_target_associate_ptr: no association possible on the host\n"); 257 return OFFLOAD_FAIL; 258 } 259 260 if (!device_is_ready(device_num)) { 261 REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n"); 262 return OFFLOAD_FAIL; 263 } 264 265 DeviceTy& Device = Devices[device_num]; 266 void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset); 267 int rc = Device.associatePtr(host_ptr, device_addr, size); 268 DP("omp_target_associate_ptr returns %d\n", rc); 269 return rc; 270 } 271 272 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) { 273 DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", " 274 "device_num %d\n", DPxPTR(host_ptr), device_num); 275 276 if (!host_ptr) { 277 REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n"); 278 return OFFLOAD_FAIL; 279 } 280 281 if (device_num == omp_get_initial_device()) { 282 REPORT( 283 "omp_target_disassociate_ptr: no association possible on the host\n"); 284 return OFFLOAD_FAIL; 285 } 286 287 if (!device_is_ready(device_num)) { 288 REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n"); 289 return OFFLOAD_FAIL; 290 } 291 292 DeviceTy& Device = Devices[device_num]; 293 int rc = Device.disassociatePtr(host_ptr); 294 DP("omp_target_disassociate_ptr returns %d\n", rc); 295 return rc; 296 } 297