1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Implementation of OpenMP API interface functions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "device.h" 14 #include "private.h" 15 #include "rtl.h" 16 17 #include <climits> 18 #include <cstring> 19 #include <cstdlib> 20 21 EXTERN int omp_get_num_devices(void) { 22 TIMESCOPE(); 23 PM->RTLsMtx.lock(); 24 size_t DevicesSize = PM->Devices.size(); 25 PM->RTLsMtx.unlock(); 26 27 DP("Call to omp_get_num_devices returning %zd\n", DevicesSize); 28 29 return DevicesSize; 30 } 31 32 EXTERN int omp_get_initial_device(void) { 33 TIMESCOPE(); 34 int hostDevice = omp_get_num_devices(); 35 DP("Call to omp_get_initial_device returning %d\n", hostDevice); 36 return hostDevice; 37 } 38 39 EXTERN void *omp_target_alloc(size_t size, int device_num) { 40 TIMESCOPE(); 41 DP("Call to omp_target_alloc for device %d requesting %zu bytes\n", 42 device_num, size); 43 44 if (size <= 0) { 45 DP("Call to omp_target_alloc with non-positive length\n"); 46 return NULL; 47 } 48 49 void *rc = NULL; 50 51 if (device_num == omp_get_initial_device()) { 52 rc = malloc(size); 53 DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc)); 54 return rc; 55 } 56 57 if (!device_is_ready(device_num)) { 58 DP("omp_target_alloc returns NULL ptr\n"); 59 return NULL; 60 } 61 62 rc = PM->Devices[device_num].allocData(size); 63 DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc)); 64 return rc; 65 } 66 67 EXTERN void omp_target_free(void *device_ptr, int device_num) { 68 TIMESCOPE(); 69 DP("Call to omp_target_free for device %d and address " DPxMOD "\n", 70 device_num, DPxPTR(device_ptr)); 71 72 if (!device_ptr) { 73 DP("Call to omp_target_free with NULL ptr\n"); 74 return; 75 } 76 77 if (device_num == omp_get_initial_device()) { 78 free(device_ptr); 79 DP("omp_target_free deallocated host ptr\n"); 80 return; 81 } 82 83 if (!device_is_ready(device_num)) { 84 DP("omp_target_free returns, nothing to do\n"); 85 return; 86 } 87 88 PM->Devices[device_num].deleteData(device_ptr); 89 DP("omp_target_free deallocated device ptr\n"); 90 } 91 92 EXTERN int omp_target_is_present(void *ptr, int device_num) { 93 TIMESCOPE(); 94 DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n", 95 device_num, DPxPTR(ptr)); 96 97 if (!ptr) { 98 DP("Call to omp_target_is_present with NULL ptr, returning false\n"); 99 return false; 100 } 101 102 if (device_num == omp_get_initial_device()) { 103 DP("Call to omp_target_is_present on host, returning true\n"); 104 return true; 105 } 106 107 PM->RTLsMtx.lock(); 108 size_t DevicesSize = PM->Devices.size(); 109 PM->RTLsMtx.unlock(); 110 if (DevicesSize <= (size_t)device_num) { 111 DP("Call to omp_target_is_present with invalid device ID, returning " 112 "false\n"); 113 return false; 114 } 115 116 DeviceTy &Device = PM->Devices[device_num]; 117 bool IsLast; // not used 118 bool IsHostPtr; 119 void *TgtPtr = Device.getTgtPtrBegin(ptr, 0, IsLast, false, IsHostPtr); 120 int rc = (TgtPtr != NULL); 121 // Under unified memory the host pointer can be returned by the 122 // getTgtPtrBegin() function which means that there is no device 123 // corresponding point for ptr. This function should return false 124 // in that situation. 125 if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) 126 rc = !IsHostPtr; 127 DP("Call to omp_target_is_present returns %d\n", rc); 128 return rc; 129 } 130 131 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length, 132 size_t dst_offset, size_t src_offset, int dst_device, int src_device) { 133 TIMESCOPE(); 134 DP("Call to omp_target_memcpy, dst device %d, src device %d, " 135 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, " 136 "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst), 137 DPxPTR(src), dst_offset, src_offset, length); 138 139 if (!dst || !src || length <= 0) { 140 REPORT("Call to omp_target_memcpy with invalid arguments\n"); 141 return OFFLOAD_FAIL; 142 } 143 144 if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) { 145 REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n"); 146 return OFFLOAD_FAIL; 147 } 148 149 if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) { 150 REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n"); 151 return OFFLOAD_FAIL; 152 } 153 154 int rc = OFFLOAD_SUCCESS; 155 void *srcAddr = (char *)src + src_offset; 156 void *dstAddr = (char *)dst + dst_offset; 157 158 if (src_device == omp_get_initial_device() && 159 dst_device == omp_get_initial_device()) { 160 DP("copy from host to host\n"); 161 const void *p = memcpy(dstAddr, srcAddr, length); 162 if (p == NULL) 163 rc = OFFLOAD_FAIL; 164 } else if (src_device == omp_get_initial_device()) { 165 DP("copy from host to device\n"); 166 DeviceTy &DstDev = PM->Devices[dst_device]; 167 rc = DstDev.submitData(dstAddr, srcAddr, length, nullptr); 168 } else if (dst_device == omp_get_initial_device()) { 169 DP("copy from device to host\n"); 170 DeviceTy &SrcDev = PM->Devices[src_device]; 171 rc = SrcDev.retrieveData(dstAddr, srcAddr, length, nullptr); 172 } else { 173 DP("copy from device to device\n"); 174 DeviceTy &SrcDev = PM->Devices[src_device]; 175 DeviceTy &DstDev = PM->Devices[dst_device]; 176 // First try to use D2D memcpy which is more efficient. If fails, fall back 177 // to unefficient way. 178 if (SrcDev.isDataExchangable(DstDev)) { 179 rc = SrcDev.dataExchange(srcAddr, DstDev, dstAddr, length, nullptr); 180 if (rc == OFFLOAD_SUCCESS) 181 return OFFLOAD_SUCCESS; 182 } 183 184 void *buffer = malloc(length); 185 rc = SrcDev.retrieveData(buffer, srcAddr, length, nullptr); 186 if (rc == OFFLOAD_SUCCESS) 187 rc = DstDev.submitData(dstAddr, buffer, length, nullptr); 188 free(buffer); 189 } 190 191 DP("omp_target_memcpy returns %d\n", rc); 192 return rc; 193 } 194 195 EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size, 196 int num_dims, const size_t *volume, const size_t *dst_offsets, 197 const size_t *src_offsets, const size_t *dst_dimensions, 198 const size_t *src_dimensions, int dst_device, int src_device) { 199 TIMESCOPE(); 200 DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, " 201 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", " 202 "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", " 203 "volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device, 204 src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets), 205 DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions), 206 DPxPTR(volume), element_size, num_dims); 207 208 if (!(dst || src)) { 209 DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n", 210 INT_MAX); 211 return INT_MAX; 212 } 213 214 if (!dst || !src || element_size < 1 || num_dims < 1 || !volume || 215 !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) { 216 REPORT("Call to omp_target_memcpy_rect with invalid arguments\n"); 217 return OFFLOAD_FAIL; 218 } 219 220 int rc; 221 if (num_dims == 1) { 222 rc = omp_target_memcpy(dst, src, element_size * volume[0], 223 element_size * dst_offsets[0], element_size * src_offsets[0], 224 dst_device, src_device); 225 } else { 226 size_t dst_slice_size = element_size; 227 size_t src_slice_size = element_size; 228 for (int i=1; i<num_dims; ++i) { 229 dst_slice_size *= dst_dimensions[i]; 230 src_slice_size *= src_dimensions[i]; 231 } 232 233 size_t dst_off = dst_offsets[0] * dst_slice_size; 234 size_t src_off = src_offsets[0] * src_slice_size; 235 for (size_t i=0; i<volume[0]; ++i) { 236 rc = omp_target_memcpy_rect((char *) dst + dst_off + dst_slice_size * i, 237 (char *) src + src_off + src_slice_size * i, element_size, 238 num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1, 239 dst_dimensions + 1, src_dimensions + 1, dst_device, src_device); 240 241 if (rc) { 242 DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n"); 243 return rc; 244 } 245 } 246 } 247 248 DP("omp_target_memcpy_rect returns %d\n", rc); 249 return rc; 250 } 251 252 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr, 253 size_t size, size_t device_offset, int device_num) { 254 TIMESCOPE(); 255 DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", " 256 "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n", 257 DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num); 258 259 if (!host_ptr || !device_ptr || size <= 0) { 260 REPORT("Call to omp_target_associate_ptr with invalid arguments\n"); 261 return OFFLOAD_FAIL; 262 } 263 264 if (device_num == omp_get_initial_device()) { 265 REPORT("omp_target_associate_ptr: no association possible on the host\n"); 266 return OFFLOAD_FAIL; 267 } 268 269 if (!device_is_ready(device_num)) { 270 REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n"); 271 return OFFLOAD_FAIL; 272 } 273 274 DeviceTy &Device = PM->Devices[device_num]; 275 void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset); 276 int rc = Device.associatePtr(host_ptr, device_addr, size); 277 DP("omp_target_associate_ptr returns %d\n", rc); 278 return rc; 279 } 280 281 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) { 282 TIMESCOPE(); 283 DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", " 284 "device_num %d\n", DPxPTR(host_ptr), device_num); 285 286 if (!host_ptr) { 287 REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n"); 288 return OFFLOAD_FAIL; 289 } 290 291 if (device_num == omp_get_initial_device()) { 292 REPORT( 293 "omp_target_disassociate_ptr: no association possible on the host\n"); 294 return OFFLOAD_FAIL; 295 } 296 297 if (!device_is_ready(device_num)) { 298 REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n"); 299 return OFFLOAD_FAIL; 300 } 301 302 DeviceTy &Device = PM->Devices[device_num]; 303 int rc = Device.disassociatePtr(host_ptr); 304 DP("omp_target_disassociate_ptr returns %d\n", rc); 305 return rc; 306 } 307