1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Implementation of the interface to be used by Clang during the codegen of a 10 // target region. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "device.h" 15 #include "private.h" 16 #include "rtl.h" 17 18 #include <cassert> 19 #include <cstdio> 20 #include <cstdlib> 21 #include <mutex> 22 23 // Store target policy (disabled, mandatory, default) 24 kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default; 25 std::mutex TargetOffloadMtx; 26 27 //////////////////////////////////////////////////////////////////////////////// 28 /// dump a table of all the host-target pointer pairs on failure 29 static void dumpTargetPointerMappings() { 30 for (const auto &Device : Devices) { 31 fprintf(stderr, "Device %d:\n", Device.DeviceID); 32 fprintf(stderr, "%-18s %-18s %s\n", "Host Ptr", "Target Ptr", "Size (B)"); 33 for (const auto &HostTargetMap : Device.HostDataToTargetMap) { 34 fprintf(stderr, DPxMOD " " DPxMOD " %lu\n", 35 DPxPTR(HostTargetMap.HstPtrBegin), 36 DPxPTR(HostTargetMap.TgtPtrBegin), 37 HostTargetMap.HstPtrEnd - HostTargetMap.HstPtrBegin); 38 } 39 } 40 } 41 42 //////////////////////////////////////////////////////////////////////////////// 43 /// manage the success or failure of a target construct 44 static void HandleDefaultTargetOffload() { 45 TargetOffloadMtx.lock(); 46 if (TargetOffloadPolicy == tgt_default) { 47 if (omp_get_num_devices() > 0) { 48 DP("Default TARGET OFFLOAD policy is now mandatory " 49 "(devices were found)\n"); 50 TargetOffloadPolicy = tgt_mandatory; 51 } else { 52 DP("Default TARGET OFFLOAD policy is now disabled " 53 "(no devices were found)\n"); 54 TargetOffloadPolicy = tgt_disabled; 55 } 56 } 57 TargetOffloadMtx.unlock(); 58 } 59 60 static int IsOffloadDisabled() { 61 if (TargetOffloadPolicy == tgt_default) HandleDefaultTargetOffload(); 62 return TargetOffloadPolicy == tgt_disabled; 63 } 64 65 static void HandleTargetOutcome(bool success) { 66 switch (TargetOffloadPolicy) { 67 case tgt_disabled: 68 if (success) { 69 FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled"); 70 } 71 break; 72 case tgt_default: 73 FATAL_MESSAGE0(1, "default offloading policy must be switched to " 74 "mandatory or disabled"); 75 break; 76 case tgt_mandatory: 77 if (!success) { 78 if (getInfoLevel() > 1) 79 dumpTargetPointerMappings(); 80 else 81 FAILURE_MESSAGE("run with env LIBOMPTARGET_INFO>1 to dump tables\n"); 82 83 FATAL_MESSAGE0(1, "failure of target construct while offloading is mandatory"); 84 } 85 break; 86 } 87 } 88 89 //////////////////////////////////////////////////////////////////////////////// 90 /// adds requires flags 91 EXTERN void __tgt_register_requires(int64_t flags) { 92 RTLs->RegisterRequires(flags); 93 } 94 95 //////////////////////////////////////////////////////////////////////////////// 96 /// adds a target shared library to the target execution image 97 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) { 98 RTLs->RegisterLib(desc); 99 } 100 101 //////////////////////////////////////////////////////////////////////////////// 102 /// unloads a target shared library 103 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) { 104 RTLs->UnregisterLib(desc); 105 } 106 107 /// creates host-to-target data mapping, stores it in the 108 /// libomptarget.so internal structure (an entry in a stack of data maps) 109 /// and passes the data to the device. 110 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num, 111 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) { 112 __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args, 113 arg_sizes, arg_types, nullptr); 114 } 115 116 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num, 117 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types, 118 int32_t depNum, void *depList, int32_t noAliasDepNum, 119 void *noAliasDepList) { 120 if (depNum + noAliasDepNum > 0) 121 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 122 123 __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args, 124 arg_sizes, arg_types, nullptr); 125 } 126 127 EXTERN void __tgt_target_data_begin_mapper(int64_t device_id, int32_t arg_num, 128 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types, 129 void **arg_mappers) { 130 if (IsOffloadDisabled()) return; 131 132 DP("Entering data begin region for device %" PRId64 " with %d mappings\n", 133 device_id, arg_num); 134 135 // No devices available? 136 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 137 device_id = omp_get_default_device(); 138 DP("Use default device id %" PRId64 "\n", device_id); 139 } 140 141 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) { 142 DP("Failed to get device %" PRId64 " ready\n", device_id); 143 HandleTargetOutcome(false); 144 return; 145 } 146 147 DeviceTy &Device = Devices[device_id]; 148 149 #ifdef OMPTARGET_DEBUG 150 for (int i = 0; i < arg_num; ++i) { 151 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 152 ", Type=0x%" PRIx64 "\n", 153 i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i]); 154 } 155 #endif 156 157 int rc = targetDataBegin(Device, arg_num, args_base, args, arg_sizes, 158 arg_types, arg_mappers, nullptr); 159 HandleTargetOutcome(rc == OFFLOAD_SUCCESS); 160 } 161 162 EXTERN void __tgt_target_data_begin_nowait_mapper(int64_t device_id, 163 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 164 int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList, 165 int32_t noAliasDepNum, void *noAliasDepList) { 166 if (depNum + noAliasDepNum > 0) 167 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 168 169 __tgt_target_data_begin_mapper(device_id, arg_num, args_base, args, 170 arg_sizes, arg_types, arg_mappers); 171 } 172 173 /// passes data from the target, releases target memory and destroys 174 /// the host-target mapping (top entry from the stack of data maps) 175 /// created by the last __tgt_target_data_begin. 176 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num, 177 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) { 178 __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes, 179 arg_types, nullptr); 180 } 181 182 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num, 183 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types, 184 int32_t depNum, void *depList, int32_t noAliasDepNum, 185 void *noAliasDepList) { 186 if (depNum + noAliasDepNum > 0) 187 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 188 189 __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes, 190 arg_types, nullptr); 191 } 192 193 EXTERN void __tgt_target_data_end_mapper(int64_t device_id, int32_t arg_num, 194 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types, 195 void **arg_mappers) { 196 if (IsOffloadDisabled()) return; 197 DP("Entering data end region with %d mappings\n", arg_num); 198 199 // No devices available? 200 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 201 device_id = omp_get_default_device(); 202 } 203 204 RTLsMtx->lock(); 205 size_t Devices_size = Devices.size(); 206 RTLsMtx->unlock(); 207 if (Devices_size <= (size_t)device_id) { 208 DP("Device ID %" PRId64 " does not have a matching RTL.\n", device_id); 209 HandleTargetOutcome(false); 210 return; 211 } 212 213 DeviceTy &Device = Devices[device_id]; 214 if (!Device.IsInit) { 215 DP("Uninit device: ignore"); 216 HandleTargetOutcome(false); 217 return; 218 } 219 220 #ifdef OMPTARGET_DEBUG 221 for (int i=0; i<arg_num; ++i) { 222 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 223 ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]), 224 arg_sizes[i], arg_types[i]); 225 } 226 #endif 227 228 int rc = targetDataEnd(Device, arg_num, args_base, args, arg_sizes, arg_types, 229 arg_mappers, nullptr); 230 HandleTargetOutcome(rc == OFFLOAD_SUCCESS); 231 } 232 233 EXTERN void __tgt_target_data_end_nowait_mapper(int64_t device_id, 234 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 235 int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList, 236 int32_t noAliasDepNum, void *noAliasDepList) { 237 if (depNum + noAliasDepNum > 0) 238 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 239 240 __tgt_target_data_end_mapper(device_id, arg_num, args_base, args, arg_sizes, 241 arg_types, arg_mappers); 242 } 243 244 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num, 245 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) { 246 __tgt_target_data_update_mapper(device_id, arg_num, args_base, args, 247 arg_sizes, arg_types, nullptr); 248 } 249 250 EXTERN void __tgt_target_data_update_nowait(int64_t device_id, int32_t arg_num, 251 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types, 252 int32_t depNum, void *depList, int32_t noAliasDepNum, 253 void *noAliasDepList) { 254 if (depNum + noAliasDepNum > 0) 255 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 256 257 __tgt_target_data_update_mapper(device_id, arg_num, args_base, args, 258 arg_sizes, arg_types, nullptr); 259 } 260 261 EXTERN void __tgt_target_data_update_mapper(int64_t device_id, int32_t arg_num, 262 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types, 263 void **arg_mappers) { 264 if (IsOffloadDisabled()) return; 265 DP("Entering data update with %d mappings\n", arg_num); 266 267 // No devices available? 268 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 269 device_id = omp_get_default_device(); 270 } 271 272 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) { 273 DP("Failed to get device %" PRId64 " ready\n", device_id); 274 HandleTargetOutcome(false); 275 return; 276 } 277 278 DeviceTy& Device = Devices[device_id]; 279 int rc = target_data_update(Device, arg_num, args_base, 280 args, arg_sizes, arg_types, arg_mappers); 281 HandleTargetOutcome(rc == OFFLOAD_SUCCESS); 282 } 283 284 EXTERN void __tgt_target_data_update_nowait_mapper(int64_t device_id, 285 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 286 int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList, 287 int32_t noAliasDepNum, void *noAliasDepList) { 288 if (depNum + noAliasDepNum > 0) 289 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 290 291 __tgt_target_data_update_mapper(device_id, arg_num, args_base, args, 292 arg_sizes, arg_types, arg_mappers); 293 } 294 295 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num, 296 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) { 297 return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args, 298 arg_sizes, arg_types, nullptr); 299 } 300 301 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr, 302 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 303 int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum, 304 void *noAliasDepList) { 305 if (depNum + noAliasDepNum > 0) 306 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 307 308 return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args, 309 arg_sizes, arg_types, nullptr); 310 } 311 312 EXTERN int __tgt_target_mapper(int64_t device_id, void *host_ptr, 313 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 314 int64_t *arg_types, void **arg_mappers) { 315 if (IsOffloadDisabled()) return OFFLOAD_FAIL; 316 DP("Entering target region with entry point " DPxMOD " and device Id %" 317 PRId64 "\n", DPxPTR(host_ptr), device_id); 318 319 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 320 device_id = omp_get_default_device(); 321 } 322 323 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) { 324 REPORT("Failed to get device %" PRId64 " ready\n", device_id); 325 HandleTargetOutcome(false); 326 return OFFLOAD_FAIL; 327 } 328 329 #ifdef OMPTARGET_DEBUG 330 for (int i=0; i<arg_num; ++i) { 331 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 332 ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]), 333 arg_sizes[i], arg_types[i]); 334 } 335 #endif 336 337 int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes, 338 arg_types, arg_mappers, 0, 0, false /*team*/); 339 HandleTargetOutcome(rc == OFFLOAD_SUCCESS); 340 return rc; 341 } 342 343 EXTERN int __tgt_target_nowait_mapper(int64_t device_id, void *host_ptr, 344 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 345 int64_t *arg_types, void **arg_mappers, int32_t depNum, void *depList, 346 int32_t noAliasDepNum, void *noAliasDepList) { 347 if (depNum + noAliasDepNum > 0) 348 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 349 350 return __tgt_target_mapper(device_id, host_ptr, arg_num, args_base, args, 351 arg_sizes, arg_types, arg_mappers); 352 } 353 354 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr, 355 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 356 int64_t *arg_types, int32_t team_num, int32_t thread_limit) { 357 return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base, 358 args, arg_sizes, arg_types, nullptr, team_num, thread_limit); 359 } 360 361 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr, 362 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 363 int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum, 364 void *depList, int32_t noAliasDepNum, void *noAliasDepList) { 365 if (depNum + noAliasDepNum > 0) 366 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 367 368 return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base, 369 args, arg_sizes, arg_types, nullptr, team_num, thread_limit); 370 } 371 372 EXTERN int __tgt_target_teams_mapper(int64_t device_id, void *host_ptr, 373 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 374 int64_t *arg_types, void **arg_mappers, int32_t team_num, int32_t thread_limit) { 375 if (IsOffloadDisabled()) return OFFLOAD_FAIL; 376 DP("Entering target region with entry point " DPxMOD " and device Id %" 377 PRId64 "\n", DPxPTR(host_ptr), device_id); 378 379 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 380 device_id = omp_get_default_device(); 381 } 382 383 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) { 384 REPORT("Failed to get device %" PRId64 " ready\n", device_id); 385 HandleTargetOutcome(false); 386 return OFFLOAD_FAIL; 387 } 388 389 #ifdef OMPTARGET_DEBUG 390 for (int i=0; i<arg_num; ++i) { 391 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 392 ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]), 393 arg_sizes[i], arg_types[i]); 394 } 395 #endif 396 397 int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes, 398 arg_types, arg_mappers, team_num, thread_limit, true /*team*/); 399 HandleTargetOutcome(rc == OFFLOAD_SUCCESS); 400 401 return rc; 402 } 403 404 EXTERN int __tgt_target_teams_nowait_mapper(int64_t device_id, void *host_ptr, 405 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes, 406 int64_t *arg_types, void **arg_mappers, int32_t team_num, 407 int32_t thread_limit, int32_t depNum, void *depList, int32_t noAliasDepNum, 408 void *noAliasDepList) { 409 if (depNum + noAliasDepNum > 0) 410 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL)); 411 412 return __tgt_target_teams_mapper(device_id, host_ptr, arg_num, args_base, 413 args, arg_sizes, arg_types, arg_mappers, team_num, thread_limit); 414 } 415 416 // Get the current number of components for a user-defined mapper. 417 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) { 418 auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle; 419 int64_t size = MapperComponentsPtr->Components.size(); 420 DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n", 421 DPxPTR(rt_mapper_handle), size); 422 return size; 423 } 424 425 // Push back one component for a user-defined mapper. 426 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base, 427 void *begin, int64_t size, 428 int64_t type) { 429 DP("__tgt_push_mapper_component(Handle=" DPxMOD 430 ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 431 ", Type=0x%" PRIx64 ").\n", 432 DPxPTR(rt_mapper_handle), DPxPTR(base), DPxPTR(begin), size, type); 433 auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle; 434 MapperComponentsPtr->Components.push_back( 435 MapComponentInfoTy(base, begin, size, type)); 436 } 437 438 EXTERN void __kmpc_push_target_tripcount(int64_t device_id, 439 uint64_t loop_tripcount) { 440 if (IsOffloadDisabled()) 441 return; 442 443 if (device_id == OFFLOAD_DEVICE_DEFAULT) { 444 device_id = omp_get_default_device(); 445 } 446 447 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) { 448 DP("Failed to get device %" PRId64 " ready\n", device_id); 449 HandleTargetOutcome(false); 450 return; 451 } 452 453 DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id, 454 loop_tripcount); 455 TblMapMtx->lock(); 456 Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL), 457 loop_tripcount); 458 TblMapMtx->unlock(); 459 } 460