1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Implementation of the interface to be used by Clang during the codegen of a 10 // target region. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include <omptarget.h> 15 16 #include "device.h" 17 #include "private.h" 18 #include "rtl.h" 19 20 #include <cassert> 21 #include <cstdlib> 22 #include <mutex> 23 24 // Store target policy (disabled, mandatory, default) 25 kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default; 26 std::mutex TargetOffloadMtx; 27 28 //////////////////////////////////////////////////////////////////////////////// 29 /// manage the success or failure of a target construct 30 31 static void HandleDefaultTargetOffload() { 32 TargetOffloadMtx.lock(); 33 if (TargetOffloadPolicy == tgt_default) { 34 if (omp_get_num_devices() > 0) { 35 DP("Default TARGET OFFLOAD policy is now mandatory " 36 "(devices were found)\n"); 37 TargetOffloadPolicy = tgt_mandatory; 38 } else { 39 DP("Default TARGET OFFLOAD policy is now disabled " 40 "(no devices were found)\n"); 41 TargetOffloadPolicy = tgt_disabled; 42 } 43 } 44 TargetOffloadMtx.unlock(); 45 } 46 47 static int IsOffloadDisabled() { 48 if (TargetOffloadPolicy == tgt_default) HandleDefaultTargetOffload(); 49 return TargetOffloadPolicy == tgt_disabled; 50 } 51 52 static void HandleTargetOutcome(bool success) { 53 switch (TargetOffloadPolicy) { 54 case tgt_disabled: 55 if (success) { 56 FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled"); 57 } 58 break; 59 case tgt_default: 60 FATAL_MESSAGE0(1, "default offloading policy must be switched to " 61 "mandatory or disabled"); 62 break; 63 case tgt_mandatory: 64 if (!success) { 65 FATAL_MESSAGE0(1, "failure of target construct while offloading is mandatory"); 66 } 67 break; 68 } 69 } 70 71 //////////////////////////////////////////////////////////////////////////////// 72 /// adds requires flags 73 EXTERN void __tgt_register_requires(int64_t flags) { 74 RTLs->RegisterRequires(flags); 75 } 76 77 //////////////////////////////////////////////////////////////////////////////// 78 /// adds a target shared library to the target execution image 79 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) { 80 RTLs->RegisterLib(desc); 81 } 82 83 //////////////////////////////////////////////////////////////////////////////// 84 /// unloads a target shared library 85 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) { 86 RTLs->UnregisterLib(desc); 87 } 88 89 /// creates host-to-target data mapping, stores it in the 90 /// libomptarget.so internal structure (an entry in a stack of data maps) 91 /// and passes the data to the device. 92 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num, 93 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) { 94 if (IsOffloadDisabled()) return; 95 96 DP("Entering data begin region for device %" PRId64 " with %d mappings\n", 97 device_id, arg_num); 98 99 // No devices available? 100 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 101 device_id = omp_get_default_device(); 102 DP("Use default device id %" PRId64 "\n", device_id); 103 } 104 105 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) { 106 DP("Failed to get device %" PRId64 " ready\n", device_id); 107 HandleTargetOutcome(false); 108 return; 109 } 110 111 DeviceTy &Device = Devices[device_id]; 112 113 #ifdef OMPTARGET_DEBUG 114 for (int i = 0; i < arg_num; ++i) { 115 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 116 ", Type=0x%" PRIx64 "\n", 117 i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i]); 118 } 119 #endif 120 121 int rc = target_data_begin(Device, arg_num, args_base, args, arg_sizes, 122 arg_types, nullptr); 123 HandleTargetOutcome(rc == OFFLOAD_SUCCESS); 124 } 125 126 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num, 127 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types, 128 int32_t depNum, void *depList, int32_t noAliasDepNum, 129 void *noAliasDepList) { 130 if (depNum + noAliasDepNum > 0) 131 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 132 133 __tgt_target_data_begin(device_id, arg_num, args_base, args, arg_sizes, 134 arg_types); 135 } 136 137 /// passes data from the target, releases target memory and destroys 138 /// the host-target mapping (top entry from the stack of data maps) 139 /// created by the last __tgt_target_data_begin. 140 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num, 141 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) { 142 if (IsOffloadDisabled()) return; 143 DP("Entering data end region with %d mappings\n", arg_num); 144 145 // No devices available? 146 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 147 device_id = omp_get_default_device(); 148 } 149 150 RTLsMtx->lock(); 151 size_t Devices_size = Devices.size(); 152 RTLsMtx->unlock(); 153 if (Devices_size <= (size_t)device_id) { 154 DP("Device ID %" PRId64 " does not have a matching RTL.\n", device_id); 155 HandleTargetOutcome(false); 156 return; 157 } 158 159 DeviceTy &Device = Devices[device_id]; 160 if (!Device.IsInit) { 161 DP("Uninit device: ignore"); 162 HandleTargetOutcome(false); 163 return; 164 } 165 166 #ifdef OMPTARGET_DEBUG 167 for (int i=0; i<arg_num; ++i) { 168 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 169 ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]), 170 arg_sizes[i], arg_types[i]); 171 } 172 #endif 173 174 int rc = target_data_end(Device, arg_num, args_base, args, arg_sizes, 175 arg_types, nullptr); 176 HandleTargetOutcome(rc == OFFLOAD_SUCCESS); 177 } 178 179 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num, 180 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types, 181 int32_t depNum, void *depList, int32_t noAliasDepNum, 182 void *noAliasDepList) { 183 if (depNum + noAliasDepNum > 0) 184 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 185 186 __tgt_target_data_end(device_id, arg_num, args_base, args, arg_sizes, 187 arg_types); 188 } 189 190 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num, 191 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) { 192 if (IsOffloadDisabled()) return; 193 DP("Entering data update with %d mappings\n", arg_num); 194 195 // No devices available? 196 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 197 device_id = omp_get_default_device(); 198 } 199 200 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) { 201 DP("Failed to get device %" PRId64 " ready\n", device_id); 202 HandleTargetOutcome(false); 203 return; 204 } 205 206 DeviceTy& Device = Devices[device_id]; 207 int rc = target_data_update(Device, arg_num, args_base, 208 args, arg_sizes, arg_types); 209 HandleTargetOutcome(rc == OFFLOAD_SUCCESS); 210 } 211 212 EXTERN void __tgt_target_data_update_nowait( 213 int64_t device_id, int32_t arg_num, void **args_base, void **args, 214 int64_t *arg_sizes, int64_t *arg_types, int32_t depNum, void *depList, 215 int32_t noAliasDepNum, void *noAliasDepList) { 216 if (depNum + noAliasDepNum > 0) 217 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 218 219 __tgt_target_data_update(device_id, arg_num, args_base, args, arg_sizes, 220 arg_types); 221 } 222 223 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num, 224 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) { 225 if (IsOffloadDisabled()) return OFFLOAD_FAIL; 226 DP("Entering target region with entry point " DPxMOD " and device Id %" 227 PRId64 "\n", DPxPTR(host_ptr), device_id); 228 229 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 230 device_id = omp_get_default_device(); 231 } 232 233 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) { 234 DP("Failed to get device %" PRId64 " ready\n", device_id); 235 HandleTargetOutcome(false); 236 return OFFLOAD_FAIL; 237 } 238 239 #ifdef OMPTARGET_DEBUG 240 for (int i=0; i<arg_num; ++i) { 241 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 242 ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]), 243 arg_sizes[i], arg_types[i]); 244 } 245 #endif 246 247 int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes, 248 arg_types, 0, 0, false /*team*/); 249 HandleTargetOutcome(rc == OFFLOAD_SUCCESS); 250 return rc; 251 } 252 253 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr, 254 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 255 int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum, 256 void *noAliasDepList) { 257 if (depNum + noAliasDepNum > 0) 258 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 259 260 return __tgt_target(device_id, host_ptr, arg_num, args_base, args, arg_sizes, 261 arg_types); 262 } 263 264 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr, 265 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 266 int64_t *arg_types, int32_t team_num, int32_t thread_limit) { 267 if (IsOffloadDisabled()) return OFFLOAD_FAIL; 268 DP("Entering target region with entry point " DPxMOD " and device Id %" 269 PRId64 "\n", DPxPTR(host_ptr), device_id); 270 271 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 272 device_id = omp_get_default_device(); 273 } 274 275 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) { 276 DP("Failed to get device %" PRId64 " ready\n", device_id); 277 HandleTargetOutcome(false); 278 return OFFLOAD_FAIL; 279 } 280 281 #ifdef OMPTARGET_DEBUG 282 for (int i=0; i<arg_num; ++i) { 283 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 284 ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]), 285 arg_sizes[i], arg_types[i]); 286 } 287 #endif 288 289 int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes, 290 arg_types, team_num, thread_limit, true /*team*/); 291 HandleTargetOutcome(rc == OFFLOAD_SUCCESS); 292 293 return rc; 294 } 295 296 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr, 297 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 298 int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum, 299 void *depList, int32_t noAliasDepNum, void *noAliasDepList) { 300 if (depNum + noAliasDepNum > 0) 301 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 302 303 return __tgt_target_teams(device_id, host_ptr, arg_num, args_base, args, 304 arg_sizes, arg_types, team_num, thread_limit); 305 } 306 307 // Get the current number of components for a user-defined mapper. 308 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) { 309 auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle; 310 int64_t size = MapperComponentsPtr->Components.size(); 311 DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n", 312 DPxPTR(rt_mapper_handle), size); 313 return size; 314 } 315 316 // Push back one component for a user-defined mapper. 317 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base, 318 void *begin, int64_t size, 319 int64_t type) { 320 DP("__tgt_push_mapper_component(Handle=" DPxMOD 321 ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 322 ", Type=0x%" PRIx64 ").\n", 323 DPxPTR(rt_mapper_handle), DPxPTR(base), DPxPTR(begin), size, type); 324 auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle; 325 MapperComponentsPtr->Components.push_back( 326 MapComponentInfoTy(base, begin, size, type)); 327 } 328 329 EXTERN void __kmpc_push_target_tripcount(int64_t device_id, 330 uint64_t loop_tripcount) { 331 if (IsOffloadDisabled()) 332 return; 333 334 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 335 device_id = omp_get_default_device(); 336 } 337 338 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) { 339 DP("Failed to get device %" PRId64 " ready\n", device_id); 340 HandleTargetOutcome(false); 341 return; 342 } 343 344 DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id, 345 loop_tripcount); 346 TblMapMtx->lock(); 347 Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL), 348 loop_tripcount); 349 TblMapMtx->unlock(); 350 } 351