1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Implementation of OpenMP API interface functions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "device.h" 14 #include "private.h" 15 #include "rtl.h" 16 17 #include <climits> 18 #include <cstring> 19 #include <cstdlib> 20 21 EXTERN int omp_get_num_devices(void) { 22 PM->RTLsMtx.lock(); 23 size_t DevicesSize = PM->Devices.size(); 24 PM->RTLsMtx.unlock(); 25 26 DP("Call to omp_get_num_devices returning %zd\n", DevicesSize); 27 28 return DevicesSize; 29 } 30 31 EXTERN int omp_get_initial_device(void) { 32 int hostDevice = omp_get_num_devices(); 33 DP("Call to omp_get_initial_device returning %d\n", hostDevice); 34 return hostDevice; 35 } 36 37 EXTERN void *omp_target_alloc(size_t size, int device_num) { 38 DP("Call to omp_target_alloc for device %d requesting %zu bytes\n", 39 device_num, size); 40 41 if (size <= 0) { 42 DP("Call to omp_target_alloc with non-positive length\n"); 43 return NULL; 44 } 45 46 void *rc = NULL; 47 48 if (device_num == omp_get_initial_device()) { 49 rc = malloc(size); 50 DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc)); 51 return rc; 52 } 53 54 if (!device_is_ready(device_num)) { 55 DP("omp_target_alloc returns NULL ptr\n"); 56 return NULL; 57 } 58 59 rc = PM->Devices[device_num].allocData(size); 60 DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc)); 61 return rc; 62 } 63 64 EXTERN void omp_target_free(void *device_ptr, int device_num) { 65 DP("Call to omp_target_free for device %d and address " DPxMOD "\n", 66 device_num, DPxPTR(device_ptr)); 67 68 if (!device_ptr) { 69 DP("Call to omp_target_free with NULL ptr\n"); 70 return; 71 } 72 73 if (device_num == omp_get_initial_device()) { 74 free(device_ptr); 75 DP("omp_target_free deallocated host ptr\n"); 76 return; 77 } 78 79 if (!device_is_ready(device_num)) { 80 DP("omp_target_free returns, nothing to do\n"); 81 return; 82 } 83 84 PM->Devices[device_num].deleteData(device_ptr); 85 DP("omp_target_free deallocated device ptr\n"); 86 } 87 88 EXTERN int omp_target_is_present(void *ptr, int device_num) { 89 DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n", 90 device_num, DPxPTR(ptr)); 91 92 if (!ptr) { 93 DP("Call to omp_target_is_present with NULL ptr, returning false\n"); 94 return false; 95 } 96 97 if (device_num == omp_get_initial_device()) { 98 DP("Call to omp_target_is_present on host, returning true\n"); 99 return true; 100 } 101 102 PM->RTLsMtx.lock(); 103 size_t DevicesSize = PM->Devices.size(); 104 PM->RTLsMtx.unlock(); 105 if (DevicesSize <= (size_t)device_num) { 106 DP("Call to omp_target_is_present with invalid device ID, returning " 107 "false\n"); 108 return false; 109 } 110 111 DeviceTy &Device = PM->Devices[device_num]; 112 bool IsLast; // not used 113 bool IsHostPtr; 114 void *TgtPtr = Device.getTgtPtrBegin(ptr, 0, IsLast, false, IsHostPtr); 115 int rc = (TgtPtr != NULL); 116 // Under unified memory the host pointer can be returned by the 117 // getTgtPtrBegin() function which means that there is no device 118 // corresponding point for ptr. This function should return false 119 // in that situation. 120 if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) 121 rc = !IsHostPtr; 122 DP("Call to omp_target_is_present returns %d\n", rc); 123 return rc; 124 } 125 126 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length, 127 size_t dst_offset, size_t src_offset, int dst_device, int src_device) { 128 DP("Call to omp_target_memcpy, dst device %d, src device %d, " 129 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, " 130 "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst), 131 DPxPTR(src), dst_offset, src_offset, length); 132 133 if (!dst || !src || length <= 0) { 134 REPORT("Call to omp_target_memcpy with invalid arguments\n"); 135 return OFFLOAD_FAIL; 136 } 137 138 if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) { 139 REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n"); 140 return OFFLOAD_FAIL; 141 } 142 143 if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) { 144 REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n"); 145 return OFFLOAD_FAIL; 146 } 147 148 int rc = OFFLOAD_SUCCESS; 149 void *srcAddr = (char *)src + src_offset; 150 void *dstAddr = (char *)dst + dst_offset; 151 152 if (src_device == omp_get_initial_device() && 153 dst_device == omp_get_initial_device()) { 154 DP("copy from host to host\n"); 155 const void *p = memcpy(dstAddr, srcAddr, length); 156 if (p == NULL) 157 rc = OFFLOAD_FAIL; 158 } else if (src_device == omp_get_initial_device()) { 159 DP("copy from host to device\n"); 160 DeviceTy &DstDev = PM->Devices[dst_device]; 161 rc = DstDev.submitData(dstAddr, srcAddr, length, nullptr); 162 } else if (dst_device == omp_get_initial_device()) { 163 DP("copy from device to host\n"); 164 DeviceTy &SrcDev = PM->Devices[src_device]; 165 rc = SrcDev.retrieveData(dstAddr, srcAddr, length, nullptr); 166 } else { 167 DP("copy from device to device\n"); 168 DeviceTy &SrcDev = PM->Devices[src_device]; 169 DeviceTy &DstDev = PM->Devices[dst_device]; 170 // First try to use D2D memcpy which is more efficient. If fails, fall back 171 // to unefficient way. 172 if (SrcDev.isDataExchangable(DstDev)) { 173 rc = SrcDev.dataExchange(srcAddr, DstDev, dstAddr, length, nullptr); 174 if (rc == OFFLOAD_SUCCESS) 175 return OFFLOAD_SUCCESS; 176 } 177 178 void *buffer = malloc(length); 179 rc = SrcDev.retrieveData(buffer, srcAddr, length, nullptr); 180 if (rc == OFFLOAD_SUCCESS) 181 rc = DstDev.submitData(dstAddr, buffer, length, nullptr); 182 free(buffer); 183 } 184 185 DP("omp_target_memcpy returns %d\n", rc); 186 return rc; 187 } 188 189 EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size, 190 int num_dims, const size_t *volume, const size_t *dst_offsets, 191 const size_t *src_offsets, const size_t *dst_dimensions, 192 const size_t *src_dimensions, int dst_device, int src_device) { 193 DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, " 194 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", " 195 "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", " 196 "volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device, 197 src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets), 198 DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions), 199 DPxPTR(volume), element_size, num_dims); 200 201 if (!(dst || src)) { 202 DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n", 203 INT_MAX); 204 return INT_MAX; 205 } 206 207 if (!dst || !src || element_size < 1 || num_dims < 1 || !volume || 208 !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) { 209 REPORT("Call to omp_target_memcpy_rect with invalid arguments\n"); 210 return OFFLOAD_FAIL; 211 } 212 213 int rc; 214 if (num_dims == 1) { 215 rc = omp_target_memcpy(dst, src, element_size * volume[0], 216 element_size * dst_offsets[0], element_size * src_offsets[0], 217 dst_device, src_device); 218 } else { 219 size_t dst_slice_size = element_size; 220 size_t src_slice_size = element_size; 221 for (int i=1; i<num_dims; ++i) { 222 dst_slice_size *= dst_dimensions[i]; 223 src_slice_size *= src_dimensions[i]; 224 } 225 226 size_t dst_off = dst_offsets[0] * dst_slice_size; 227 size_t src_off = src_offsets[0] * src_slice_size; 228 for (size_t i=0; i<volume[0]; ++i) { 229 rc = omp_target_memcpy_rect((char *) dst + dst_off + dst_slice_size * i, 230 (char *) src + src_off + src_slice_size * i, element_size, 231 num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1, 232 dst_dimensions + 1, src_dimensions + 1, dst_device, src_device); 233 234 if (rc) { 235 DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n"); 236 return rc; 237 } 238 } 239 } 240 241 DP("omp_target_memcpy_rect returns %d\n", rc); 242 return rc; 243 } 244 245 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr, 246 size_t size, size_t device_offset, int device_num) { 247 DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", " 248 "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n", 249 DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num); 250 251 if (!host_ptr || !device_ptr || size <= 0) { 252 REPORT("Call to omp_target_associate_ptr with invalid arguments\n"); 253 return OFFLOAD_FAIL; 254 } 255 256 if (device_num == omp_get_initial_device()) { 257 REPORT("omp_target_associate_ptr: no association possible on the host\n"); 258 return OFFLOAD_FAIL; 259 } 260 261 if (!device_is_ready(device_num)) { 262 REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n"); 263 return OFFLOAD_FAIL; 264 } 265 266 DeviceTy &Device = PM->Devices[device_num]; 267 void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset); 268 int rc = Device.associatePtr(host_ptr, device_addr, size); 269 DP("omp_target_associate_ptr returns %d\n", rc); 270 return rc; 271 } 272 273 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) { 274 DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", " 275 "device_num %d\n", DPxPTR(host_ptr), device_num); 276 277 if (!host_ptr) { 278 REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n"); 279 return OFFLOAD_FAIL; 280 } 281 282 if (device_num == omp_get_initial_device()) { 283 REPORT( 284 "omp_target_disassociate_ptr: no association possible on the host\n"); 285 return OFFLOAD_FAIL; 286 } 287 288 if (!device_is_ready(device_num)) { 289 REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n"); 290 return OFFLOAD_FAIL; 291 } 292 293 DeviceTy &Device = PM->Devices[device_num]; 294 int rc = Device.disassociatePtr(host_ptr); 295 DP("omp_target_disassociate_ptr returns %d\n", rc); 296 return rc; 297 } 298